2019-01-29 00:21:27 -08:00
|
|
|
//! The `entry_stream` module provides a method for streaming entries out via a
|
|
|
|
//! local unix socket, to provide client services such as a block explorer with
|
|
|
|
//! real-time access to entries.
|
|
|
|
|
|
|
|
use crate::entry::Entry;
|
2019-02-11 10:36:59 -08:00
|
|
|
use crate::leader_scheduler::LeaderScheduler;
|
2019-01-29 00:21:27 -08:00
|
|
|
use crate::result::Result;
|
2019-02-11 10:36:59 -08:00
|
|
|
use chrono::{SecondsFormat, Utc};
|
2019-02-12 15:59:02 -08:00
|
|
|
use solana_sdk::hash::Hash;
|
|
|
|
use std::cell::RefCell;
|
2019-01-29 00:21:27 -08:00
|
|
|
use std::io::prelude::*;
|
|
|
|
use std::net::Shutdown;
|
|
|
|
use std::os::unix::net::UnixStream;
|
|
|
|
use std::path::Path;
|
2019-02-11 10:36:59 -08:00
|
|
|
use std::sync::{Arc, RwLock};
|
2019-01-29 00:21:27 -08:00
|
|
|
|
2019-02-13 10:45:41 -08:00
|
|
|
pub trait EntryWriter: std::fmt::Debug {
|
2019-02-12 15:59:02 -08:00
|
|
|
fn write(&self, payload: String) -> Result<()>;
|
2019-01-29 00:21:27 -08:00
|
|
|
}
|
|
|
|
|
2019-02-12 15:59:02 -08:00
|
|
|
#[derive(Debug, Default)]
|
2019-02-13 10:45:41 -08:00
|
|
|
pub struct EntryVec {
|
2019-02-12 15:59:02 -08:00
|
|
|
values: RefCell<Vec<String>>,
|
2019-01-29 00:21:27 -08:00
|
|
|
}
|
|
|
|
|
2019-02-13 10:45:41 -08:00
|
|
|
impl EntryWriter for EntryVec {
|
2019-02-12 15:59:02 -08:00
|
|
|
fn write(&self, payload: String) -> Result<()> {
|
|
|
|
self.values.borrow_mut().push(payload);
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-13 10:45:41 -08:00
|
|
|
impl EntryVec {
|
2019-02-12 15:59:02 -08:00
|
|
|
pub fn new() -> Self {
|
2019-02-13 10:45:41 -08:00
|
|
|
EntryVec {
|
2019-02-12 15:59:02 -08:00
|
|
|
values: RefCell::new(Vec::new()),
|
2019-02-11 10:36:59 -08:00
|
|
|
}
|
2019-01-29 00:21:27 -08:00
|
|
|
}
|
2019-02-12 15:59:02 -08:00
|
|
|
|
|
|
|
pub fn entries(&self) -> Vec<String> {
|
|
|
|
self.values.borrow().clone()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Debug)]
|
2019-02-13 10:45:41 -08:00
|
|
|
pub struct EntrySocket {
|
2019-02-12 15:59:02 -08:00
|
|
|
socket: String,
|
2019-01-29 00:21:27 -08:00
|
|
|
}
|
|
|
|
|
2019-02-13 10:45:41 -08:00
|
|
|
impl EntryWriter for EntrySocket {
|
2019-02-12 15:59:02 -08:00
|
|
|
fn write(&self, payload: String) -> Result<()> {
|
2019-01-29 00:21:27 -08:00
|
|
|
let mut socket = UnixStream::connect(Path::new(&self.socket))?;
|
2019-02-12 15:59:02 -08:00
|
|
|
socket.write_all(payload.as_bytes())?;
|
|
|
|
socket.shutdown(Shutdown::Write)?;
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub trait EntryStreamHandler {
|
2019-02-13 10:42:56 -08:00
|
|
|
fn emit_entry_event(&self, slot: u64, leader_id: &str, entries: &Entry) -> Result<()>;
|
|
|
|
fn emit_block_event(
|
|
|
|
&self,
|
|
|
|
slot: u64,
|
|
|
|
leader_id: &str,
|
|
|
|
tick_height: u64,
|
|
|
|
last_id: Hash,
|
|
|
|
) -> Result<()>;
|
2019-02-12 15:59:02 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Debug)]
|
2019-02-13 10:45:41 -08:00
|
|
|
pub struct EntryStream<T: EntryWriter> {
|
2019-02-12 15:59:02 -08:00
|
|
|
pub output: T,
|
2019-02-12 22:45:00 -08:00
|
|
|
pub leader_scheduler: Arc<RwLock<LeaderScheduler>>,
|
|
|
|
pub queued_block: Option<EntryStreamBlock>,
|
2019-02-12 15:59:02 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
impl<T> EntryStreamHandler for EntryStream<T>
|
|
|
|
where
|
2019-02-13 10:45:41 -08:00
|
|
|
T: EntryWriter,
|
2019-02-12 15:59:02 -08:00
|
|
|
{
|
2019-02-13 10:42:56 -08:00
|
|
|
fn emit_entry_event(&self, slot: u64, leader_id: &str, entry: &Entry) -> Result<()> {
|
2019-02-12 22:45:00 -08:00
|
|
|
let json_entry = serde_json::to_string(&entry)?;
|
|
|
|
let payload = format!(
|
|
|
|
r#"{{"dt":"{}","t":"entry","s":{},"l":{:?},"entry":{}}}"#,
|
|
|
|
Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
|
|
|
slot,
|
|
|
|
leader_id,
|
|
|
|
json_entry,
|
|
|
|
);
|
|
|
|
self.output.write(payload)?;
|
2019-02-12 15:59:02 -08:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-02-13 10:42:56 -08:00
|
|
|
fn emit_block_event(
|
|
|
|
&self,
|
|
|
|
slot: u64,
|
|
|
|
leader_id: &str,
|
|
|
|
tick_height: u64,
|
|
|
|
last_id: Hash,
|
|
|
|
) -> Result<()> {
|
2019-02-12 15:59:02 -08:00
|
|
|
let payload = format!(
|
|
|
|
r#"{{"dt":"{}","t":"block","s":{},"h":{},"l":{:?},"id":"{:?}"}}"#,
|
|
|
|
Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
|
|
|
|
slot,
|
|
|
|
tick_height,
|
|
|
|
leader_id,
|
|
|
|
last_id,
|
|
|
|
);
|
|
|
|
self.output.write(payload)?;
|
2019-01-29 00:21:27 -08:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-13 10:45:41 -08:00
|
|
|
pub type SocketEntryStream = EntryStream<EntrySocket>;
|
2019-02-12 15:59:02 -08:00
|
|
|
|
|
|
|
impl SocketEntryStream {
|
|
|
|
pub fn new(socket: String, leader_scheduler: Arc<RwLock<LeaderScheduler>>) -> Self {
|
|
|
|
EntryStream {
|
2019-02-13 10:45:41 -08:00
|
|
|
output: EntrySocket { socket },
|
2019-02-12 15:59:02 -08:00
|
|
|
leader_scheduler,
|
2019-02-12 22:45:00 -08:00
|
|
|
queued_block: None,
|
2019-02-12 15:59:02 -08:00
|
|
|
}
|
|
|
|
}
|
2019-01-29 00:21:27 -08:00
|
|
|
}
|
|
|
|
|
2019-02-13 10:45:41 -08:00
|
|
|
pub type MockEntryStream = EntryStream<EntryVec>;
|
2019-02-12 15:59:02 -08:00
|
|
|
|
2019-01-29 00:21:27 -08:00
|
|
|
impl MockEntryStream {
|
2019-02-12 15:59:02 -08:00
|
|
|
pub fn new(_: String, leader_scheduler: Arc<RwLock<LeaderScheduler>>) -> Self {
|
|
|
|
EntryStream {
|
2019-02-13 10:45:41 -08:00
|
|
|
output: EntryVec::new(),
|
2019-02-11 10:36:59 -08:00
|
|
|
leader_scheduler,
|
2019-02-12 22:45:00 -08:00
|
|
|
queued_block: None,
|
2019-02-11 10:36:59 -08:00
|
|
|
}
|
2019-01-29 00:21:27 -08:00
|
|
|
}
|
2019-02-12 15:59:02 -08:00
|
|
|
|
|
|
|
pub fn entries(&self) -> Vec<String> {
|
|
|
|
self.output.entries()
|
|
|
|
}
|
2019-01-29 00:21:27 -08:00
|
|
|
}
|
|
|
|
|
2019-02-12 22:45:00 -08:00
|
|
|
#[derive(Debug)]
|
|
|
|
pub struct EntryStreamBlock {
|
|
|
|
pub tick_height: u64,
|
|
|
|
pub id: Hash,
|
|
|
|
}
|
|
|
|
|
2019-02-12 15:59:02 -08:00
|
|
|
#[cfg(test)]
|
|
|
|
mod test {
|
|
|
|
use super::*;
|
|
|
|
use crate::bank::Bank;
|
|
|
|
use crate::entry::Entry;
|
|
|
|
use crate::genesis_block::GenesisBlock;
|
|
|
|
use crate::leader_scheduler::LeaderSchedulerConfig;
|
|
|
|
use chrono::{DateTime, FixedOffset};
|
|
|
|
use serde_json::Value;
|
|
|
|
use solana_sdk::hash::Hash;
|
|
|
|
use std::collections::HashSet;
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_entry_stream() -> () {
|
|
|
|
// Set up bank and leader_scheduler
|
|
|
|
let leader_scheduler_config = LeaderSchedulerConfig::new(5, 2, 10);
|
|
|
|
let (genesis_block, _mint_keypair) = GenesisBlock::new(1_000_000);
|
|
|
|
let bank = Bank::new_with_leader_scheduler_config(&genesis_block, &leader_scheduler_config);
|
|
|
|
// Set up entry stream
|
|
|
|
let entry_stream =
|
|
|
|
MockEntryStream::new("test_stream".to_string(), bank.leader_scheduler.clone());
|
|
|
|
let ticks_per_slot = bank.leader_scheduler.read().unwrap().ticks_per_slot;
|
|
|
|
|
|
|
|
let mut last_id = Hash::default();
|
|
|
|
let mut entries = Vec::new();
|
|
|
|
let mut expected_entries = Vec::new();
|
|
|
|
|
|
|
|
let tick_height_initial = 0;
|
|
|
|
let tick_height_final = tick_height_initial + ticks_per_slot + 2;
|
|
|
|
let mut previous_slot = bank
|
|
|
|
.leader_scheduler
|
|
|
|
.read()
|
|
|
|
.unwrap()
|
|
|
|
.tick_height_to_slot(tick_height_initial);
|
2019-02-13 10:42:56 -08:00
|
|
|
let leader_id = bank
|
|
|
|
.leader_scheduler
|
|
|
|
.read()
|
|
|
|
.unwrap()
|
|
|
|
.get_leader_for_slot(previous_slot)
|
|
|
|
.map(|leader| leader.to_string())
|
|
|
|
.unwrap_or_else(|| "None".to_string());
|
2019-02-12 15:59:02 -08:00
|
|
|
|
|
|
|
for tick_height in tick_height_initial..=tick_height_final {
|
|
|
|
bank.leader_scheduler
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.update_tick_height(tick_height, &bank);
|
|
|
|
let curr_slot = bank
|
|
|
|
.leader_scheduler
|
|
|
|
.read()
|
|
|
|
.unwrap()
|
|
|
|
.tick_height_to_slot(tick_height);
|
|
|
|
if curr_slot != previous_slot {
|
|
|
|
entry_stream
|
2019-02-13 10:42:56 -08:00
|
|
|
.emit_block_event(previous_slot, &leader_id, tick_height - 1, last_id)
|
2019-02-12 15:59:02 -08:00
|
|
|
.unwrap();
|
|
|
|
}
|
2019-02-13 10:42:56 -08:00
|
|
|
let entry = Entry::new(&mut last_id, tick_height, 1, vec![]); // just ticks
|
2019-02-12 15:59:02 -08:00
|
|
|
last_id = entry.id;
|
|
|
|
previous_slot = curr_slot;
|
2019-02-13 10:42:56 -08:00
|
|
|
entry_stream
|
|
|
|
.emit_entry_event(curr_slot, &leader_id, &entry)
|
|
|
|
.unwrap();
|
2019-02-12 15:59:02 -08:00
|
|
|
expected_entries.push(entry.clone());
|
|
|
|
entries.push(entry);
|
2019-01-29 00:21:27 -08:00
|
|
|
}
|
2019-02-12 15:59:02 -08:00
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
entry_stream.entries().len() as u64,
|
|
|
|
// one entry per tick (0..=N+2) is +3, plus one block
|
|
|
|
ticks_per_slot + 3 + 1
|
|
|
|
);
|
|
|
|
|
|
|
|
let mut j = 0;
|
|
|
|
let mut matched_entries = 0;
|
|
|
|
let mut matched_slots = HashSet::new();
|
|
|
|
let mut matched_blocks = HashSet::new();
|
|
|
|
|
|
|
|
for item in entry_stream.entries() {
|
|
|
|
let json: Value = serde_json::from_str(&item).unwrap();
|
|
|
|
let dt_str = json["dt"].as_str().unwrap();
|
|
|
|
|
|
|
|
// Ensure `ts` field parses as valid DateTime
|
|
|
|
let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
|
|
|
|
|
|
|
|
let item_type = json["t"].as_str().unwrap();
|
|
|
|
match item_type {
|
|
|
|
"block" => {
|
|
|
|
let id = json["id"].to_string();
|
|
|
|
matched_blocks.insert(id);
|
|
|
|
}
|
|
|
|
|
|
|
|
"entry" => {
|
|
|
|
let slot = json["s"].as_u64().unwrap();
|
|
|
|
matched_slots.insert(slot);
|
|
|
|
let entry_obj = json["entry"].clone();
|
|
|
|
let entry: Entry = serde_json::from_value(entry_obj).unwrap();
|
|
|
|
|
|
|
|
assert_eq!(entry, expected_entries[j]);
|
|
|
|
matched_entries += 1;
|
|
|
|
j += 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
_ => {
|
|
|
|
assert!(false, "unknown item type {}", item);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
assert_eq!(matched_entries, expected_entries.len());
|
|
|
|
assert_eq!(matched_slots.len(), 2);
|
|
|
|
assert_eq!(matched_blocks.len(), 1);
|
2019-01-29 00:21:27 -08:00
|
|
|
}
|
|
|
|
}
|