2018-09-21 15:32:15 -07:00
|
|
|
use blob_fetch_stage::BlobFetchStage;
|
2018-10-08 19:55:54 -07:00
|
|
|
use cluster_info::{ClusterInfo, Node, NodeInfo};
|
2018-10-10 16:49:41 -07:00
|
|
|
use leader_scheduler::LeaderScheduler;
|
2018-09-21 15:32:15 -07:00
|
|
|
use ncp::Ncp;
|
|
|
|
use service::Service;
|
2018-11-16 08:04:46 -08:00
|
|
|
use solana_sdk::hash::{Hash, Hasher};
|
2018-10-02 11:47:51 -07:00
|
|
|
use std::fs::File;
|
|
|
|
use std::io;
|
|
|
|
use std::io::BufReader;
|
|
|
|
use std::io::Read;
|
|
|
|
use std::io::Seek;
|
|
|
|
use std::io::SeekFrom;
|
|
|
|
use std::io::{Error, ErrorKind};
|
|
|
|
use std::mem::size_of;
|
2018-09-21 15:32:15 -07:00
|
|
|
use std::net::SocketAddr;
|
|
|
|
use std::net::UdpSocket;
|
2018-10-02 11:47:51 -07:00
|
|
|
use std::path::Path;
|
2018-09-21 15:32:15 -07:00
|
|
|
use std::sync::atomic::AtomicBool;
|
|
|
|
use std::sync::mpsc::channel;
|
|
|
|
use std::sync::{Arc, RwLock};
|
|
|
|
use std::thread::JoinHandle;
|
2018-09-24 11:50:37 -07:00
|
|
|
use std::time::Duration;
|
2018-09-21 15:32:15 -07:00
|
|
|
use store_ledger_stage::StoreLedgerStage;
|
|
|
|
use streamer::BlobReceiver;
|
2018-10-02 16:10:56 -07:00
|
|
|
use thin_client::poll_gossip_for_leader;
|
2018-09-21 15:32:15 -07:00
|
|
|
use window;
|
2018-10-10 16:49:41 -07:00
|
|
|
use window_service::window_service;
|
2018-09-21 15:32:15 -07:00
|
|
|
|
|
|
|
pub struct Replicator {
|
|
|
|
ncp: Ncp,
|
|
|
|
fetch_stage: BlobFetchStage,
|
|
|
|
store_ledger_stage: StoreLedgerStage,
|
2018-10-10 16:49:41 -07:00
|
|
|
t_window: JoinHandle<()>,
|
2018-09-21 15:32:15 -07:00
|
|
|
pub retransmit_receiver: BlobReceiver,
|
|
|
|
}
|
|
|
|
|
2018-10-02 11:47:51 -07:00
|
|
|
pub fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
|
|
|
|
let in_file = File::open(in_path)?;
|
|
|
|
let metadata = in_file.metadata()?;
|
|
|
|
let mut buffer_file = BufReader::new(in_file);
|
|
|
|
|
|
|
|
let mut hasher = Hasher::default();
|
|
|
|
let sample_size = size_of::<Hash>();
|
|
|
|
let sample_size64 = sample_size as u64;
|
|
|
|
let mut buf = vec![0; sample_size];
|
|
|
|
|
|
|
|
let file_len = metadata.len();
|
2018-10-08 13:12:33 -07:00
|
|
|
if file_len < sample_size64 {
|
|
|
|
return Err(Error::new(ErrorKind::Other, "file too short!"));
|
|
|
|
}
|
2018-10-02 11:47:51 -07:00
|
|
|
for offset in sample_offsets {
|
|
|
|
if *offset > (file_len - sample_size64) / sample_size64 {
|
|
|
|
return Err(Error::new(ErrorKind::Other, "offset too large"));
|
|
|
|
}
|
|
|
|
buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?;
|
2018-10-02 16:10:56 -07:00
|
|
|
trace!("sampling @ {} ", *offset);
|
2018-10-02 11:47:51 -07:00
|
|
|
match buffer_file.read(&mut buf) {
|
|
|
|
Ok(size) => {
|
|
|
|
assert_eq!(size, buf.len());
|
|
|
|
hasher.hash(&buf);
|
|
|
|
}
|
|
|
|
Err(e) => {
|
|
|
|
warn!("Error sampling file");
|
|
|
|
return Err(e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(hasher.result())
|
|
|
|
}
|
|
|
|
|
2018-09-21 15:32:15 -07:00
|
|
|
impl Replicator {
|
|
|
|
pub fn new(
|
|
|
|
entry_height: u64,
|
2018-09-24 14:10:51 -07:00
|
|
|
max_entry_height: u64,
|
2018-09-21 15:32:15 -07:00
|
|
|
exit: &Arc<AtomicBool>,
|
|
|
|
ledger_path: Option<&str>,
|
|
|
|
node: Node,
|
|
|
|
network_addr: Option<SocketAddr>,
|
2018-09-24 14:10:51 -07:00
|
|
|
done: Arc<AtomicBool>,
|
2018-10-02 16:10:56 -07:00
|
|
|
) -> (Replicator, NodeInfo) {
|
2018-10-30 10:05:18 -07:00
|
|
|
const REPLICATOR_WINDOW_SIZE: usize = 32 * 1024;
|
|
|
|
let window = window::new_window(REPLICATOR_WINDOW_SIZE);
|
2018-09-21 15:32:15 -07:00
|
|
|
let shared_window = Arc::new(RwLock::new(window));
|
|
|
|
|
2018-11-19 11:25:14 -08:00
|
|
|
let cluster_info = Arc::new(RwLock::new(ClusterInfo::new(node.info)));
|
2018-09-21 15:32:15 -07:00
|
|
|
|
|
|
|
let leader_info = network_addr.map(|i| NodeInfo::new_entry_point(&i));
|
2018-10-10 16:49:41 -07:00
|
|
|
let leader_pubkey;
|
2018-11-15 13:23:26 -08:00
|
|
|
if let Some(leader_info) = leader_info {
|
2018-10-10 16:49:41 -07:00
|
|
|
leader_pubkey = leader_info.id;
|
2018-11-15 13:23:26 -08:00
|
|
|
cluster_info.write().unwrap().insert_info(leader_info);
|
2018-09-21 15:32:15 -07:00
|
|
|
} else {
|
|
|
|
panic!("No leader info!");
|
|
|
|
}
|
|
|
|
|
|
|
|
let repair_socket = Arc::new(node.sockets.repair);
|
|
|
|
let mut blob_sockets: Vec<Arc<UdpSocket>> =
|
|
|
|
node.sockets.replicate.into_iter().map(Arc::new).collect();
|
|
|
|
blob_sockets.push(repair_socket.clone());
|
|
|
|
let (fetch_stage, blob_fetch_receiver) =
|
|
|
|
BlobFetchStage::new_multi_socket(blob_sockets, exit.clone());
|
|
|
|
|
2018-09-21 16:50:58 -07:00
|
|
|
let (entry_window_sender, entry_window_receiver) = channel();
|
2018-09-21 15:32:15 -07:00
|
|
|
// todo: pull blobs off the retransmit_receiver and recycle them?
|
|
|
|
let (retransmit_sender, retransmit_receiver) = channel();
|
|
|
|
let t_window = window_service(
|
2018-10-08 19:55:54 -07:00
|
|
|
cluster_info.clone(),
|
2018-09-21 15:32:15 -07:00
|
|
|
shared_window.clone(),
|
2018-10-18 22:57:48 -07:00
|
|
|
0,
|
2018-09-21 15:32:15 -07:00
|
|
|
entry_height,
|
2018-09-24 14:10:51 -07:00
|
|
|
max_entry_height,
|
2018-09-21 15:32:15 -07:00
|
|
|
blob_fetch_receiver,
|
2018-09-21 16:50:58 -07:00
|
|
|
entry_window_sender,
|
2018-09-21 15:32:15 -07:00
|
|
|
retransmit_sender,
|
|
|
|
repair_socket,
|
2018-10-10 16:49:41 -07:00
|
|
|
Arc::new(RwLock::new(LeaderScheduler::from_bootstrap_leader(
|
|
|
|
leader_pubkey,
|
|
|
|
))),
|
2018-09-24 14:10:51 -07:00
|
|
|
done,
|
2018-09-21 15:32:15 -07:00
|
|
|
);
|
|
|
|
|
2018-09-21 16:50:58 -07:00
|
|
|
let store_ledger_stage = StoreLedgerStage::new(entry_window_receiver, ledger_path);
|
2018-09-21 15:32:15 -07:00
|
|
|
|
|
|
|
let ncp = Ncp::new(
|
2018-10-08 19:55:54 -07:00
|
|
|
&cluster_info,
|
2018-09-21 15:32:15 -07:00
|
|
|
shared_window.clone(),
|
|
|
|
ledger_path,
|
|
|
|
node.sockets.gossip,
|
|
|
|
exit.clone(),
|
|
|
|
);
|
|
|
|
|
2018-10-02 16:10:56 -07:00
|
|
|
let leader =
|
|
|
|
poll_gossip_for_leader(network_addr.unwrap(), Some(10)).expect("couldn't reach leader");
|
|
|
|
|
|
|
|
(
|
|
|
|
Replicator {
|
|
|
|
ncp,
|
|
|
|
fetch_stage,
|
|
|
|
store_ledger_stage,
|
|
|
|
t_window,
|
|
|
|
retransmit_receiver,
|
|
|
|
},
|
|
|
|
leader,
|
|
|
|
)
|
2018-09-21 15:32:15 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
pub fn join(self) {
|
|
|
|
self.ncp.join().unwrap();
|
|
|
|
self.fetch_stage.join().unwrap();
|
2018-09-21 18:56:20 -07:00
|
|
|
self.t_window.join().unwrap();
|
2018-09-21 15:32:15 -07:00
|
|
|
self.store_ledger_stage.join().unwrap();
|
2018-09-24 11:50:37 -07:00
|
|
|
|
|
|
|
// Drain the queue here to prevent self.retransmit_receiver from being dropped
|
|
|
|
// before the window_service thread is joined
|
|
|
|
let mut retransmit_queue_count = 0;
|
|
|
|
while let Ok(_blob) = self.retransmit_receiver.recv_timeout(Duration::new(1, 0)) {
|
|
|
|
retransmit_queue_count += 1;
|
|
|
|
}
|
|
|
|
debug!("retransmit channel count: {}", retransmit_queue_count);
|
2018-09-21 15:32:15 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
|
|
|
use client::mk_client;
|
2018-10-08 19:55:54 -07:00
|
|
|
use cluster_info::Node;
|
2018-09-21 15:32:15 -07:00
|
|
|
use fullnode::Fullnode;
|
2018-10-10 16:49:41 -07:00
|
|
|
use leader_scheduler::LeaderScheduler;
|
2018-10-17 13:42:54 -07:00
|
|
|
use ledger::{create_tmp_genesis, get_tmp_ledger_path, read_ledger};
|
2018-09-21 15:32:15 -07:00
|
|
|
use logger;
|
2018-10-02 11:47:51 -07:00
|
|
|
use replicator::sample_file;
|
2018-09-21 15:32:15 -07:00
|
|
|
use replicator::Replicator;
|
|
|
|
use signature::{Keypair, KeypairUtil};
|
2018-11-16 08:04:46 -08:00
|
|
|
use solana_sdk::hash::Hash;
|
2018-10-02 11:47:51 -07:00
|
|
|
use std::fs::File;
|
2018-10-08 16:15:17 -07:00
|
|
|
use std::fs::{create_dir_all, remove_dir_all, remove_file};
|
2018-10-02 11:47:51 -07:00
|
|
|
use std::io::Write;
|
|
|
|
use std::mem::size_of;
|
2018-10-08 16:15:17 -07:00
|
|
|
use std::path::PathBuf;
|
2018-09-21 15:32:15 -07:00
|
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
|
|
use std::sync::Arc;
|
|
|
|
use std::thread::sleep;
|
|
|
|
use std::time::Duration;
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_replicator_startup() {
|
|
|
|
logger::setup();
|
|
|
|
info!("starting replicator test");
|
|
|
|
let entry_height = 0;
|
2018-10-17 13:42:54 -07:00
|
|
|
let replicator_ledger_path = &get_tmp_ledger_path("replicator_test_replicator_ledger");
|
2018-09-21 15:32:15 -07:00
|
|
|
|
|
|
|
let exit = Arc::new(AtomicBool::new(false));
|
2018-09-24 14:10:51 -07:00
|
|
|
let done = Arc::new(AtomicBool::new(false));
|
2018-09-21 15:32:15 -07:00
|
|
|
|
|
|
|
info!("starting leader node");
|
2018-10-25 16:58:40 -07:00
|
|
|
let leader_keypair = Arc::new(Keypair::new());
|
2018-09-21 15:32:15 -07:00
|
|
|
let leader_node = Node::new_localhost_with_pubkey(leader_keypair.pubkey());
|
|
|
|
let network_addr = leader_node.sockets.gossip.local_addr().unwrap();
|
|
|
|
let leader_info = leader_node.info.clone();
|
2018-10-25 16:58:40 -07:00
|
|
|
let vote_account_keypair = Arc::new(Keypair::new());
|
2018-11-02 14:32:05 -07:00
|
|
|
|
|
|
|
let leader_ledger_path = "replicator_test_leader_ledger";
|
|
|
|
let (mint, leader_ledger_path) =
|
|
|
|
create_tmp_genesis(leader_ledger_path, 100, leader_info.id, 1);
|
|
|
|
|
2018-09-21 15:32:15 -07:00
|
|
|
let leader = Fullnode::new(
|
|
|
|
leader_node,
|
|
|
|
&leader_ledger_path,
|
|
|
|
leader_keypair,
|
2018-10-25 16:58:40 -07:00
|
|
|
vote_account_keypair,
|
2018-09-21 15:32:15 -07:00
|
|
|
None,
|
|
|
|
false,
|
2018-10-10 16:49:41 -07:00
|
|
|
LeaderScheduler::from_bootstrap_leader(leader_info.id),
|
2018-11-05 09:50:58 -08:00
|
|
|
None,
|
2018-09-21 15:32:15 -07:00
|
|
|
);
|
|
|
|
|
|
|
|
let mut leader_client = mk_client(&leader_info);
|
|
|
|
|
|
|
|
let bob = Keypair::new();
|
|
|
|
|
|
|
|
let last_id = leader_client.get_last_id();
|
|
|
|
leader_client
|
|
|
|
.transfer(1, &mint.keypair(), bob.pubkey(), &last_id)
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
let replicator_keypair = Keypair::new();
|
|
|
|
|
|
|
|
info!("starting replicator node");
|
|
|
|
let replicator_node = Node::new_localhost_with_pubkey(replicator_keypair.pubkey());
|
2018-10-02 16:10:56 -07:00
|
|
|
let (replicator, _leader_info) = Replicator::new(
|
2018-09-21 15:32:15 -07:00
|
|
|
entry_height,
|
2018-09-24 14:10:51 -07:00
|
|
|
1,
|
2018-09-21 15:32:15 -07:00
|
|
|
&exit,
|
|
|
|
Some(replicator_ledger_path),
|
|
|
|
replicator_node,
|
|
|
|
Some(network_addr),
|
2018-09-24 14:10:51 -07:00
|
|
|
done.clone(),
|
2018-09-21 15:32:15 -07:00
|
|
|
);
|
|
|
|
|
|
|
|
let mut num_entries = 0;
|
2018-09-24 11:50:37 -07:00
|
|
|
for _ in 0..60 {
|
2018-09-21 15:32:15 -07:00
|
|
|
match read_ledger(replicator_ledger_path, true) {
|
|
|
|
Ok(entries) => {
|
|
|
|
for _ in entries {
|
|
|
|
num_entries += 1;
|
|
|
|
}
|
|
|
|
info!("{} entries", num_entries);
|
|
|
|
if num_entries > 0 {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Err(e) => {
|
|
|
|
info!("error reading ledger: {:?}", e);
|
|
|
|
}
|
|
|
|
}
|
2018-09-24 11:50:37 -07:00
|
|
|
sleep(Duration::from_millis(300));
|
2018-09-21 15:32:15 -07:00
|
|
|
let last_id = leader_client.get_last_id();
|
|
|
|
leader_client
|
|
|
|
.transfer(1, &mint.keypair(), bob.pubkey(), &last_id)
|
|
|
|
.unwrap();
|
|
|
|
}
|
2018-09-24 14:10:51 -07:00
|
|
|
assert_eq!(done.load(Ordering::Relaxed), true);
|
2018-09-21 15:32:15 -07:00
|
|
|
assert!(num_entries > 0);
|
|
|
|
exit.store(true, Ordering::Relaxed);
|
|
|
|
replicator.join();
|
|
|
|
leader.exit();
|
2018-09-21 18:56:20 -07:00
|
|
|
let _ignored = remove_dir_all(&leader_ledger_path);
|
|
|
|
let _ignored = remove_dir_all(&replicator_ledger_path);
|
2018-09-21 15:32:15 -07:00
|
|
|
}
|
2018-10-02 11:47:51 -07:00
|
|
|
|
2018-10-08 16:15:17 -07:00
|
|
|
fn tmp_file_path(name: &str) -> PathBuf {
|
|
|
|
use std::env;
|
|
|
|
let out_dir = env::var("OUT_DIR").unwrap_or_else(|_| "target".to_string());
|
|
|
|
let keypair = Keypair::new();
|
|
|
|
|
|
|
|
let mut path = PathBuf::new();
|
|
|
|
path.push(out_dir);
|
|
|
|
path.push("tmp");
|
|
|
|
create_dir_all(&path).unwrap();
|
|
|
|
|
|
|
|
path.push(format!("{}-{}", name, keypair.pubkey()));
|
|
|
|
path
|
|
|
|
}
|
|
|
|
|
2018-10-02 11:47:51 -07:00
|
|
|
#[test]
|
|
|
|
fn test_sample_file() {
|
|
|
|
logger::setup();
|
2018-10-08 16:15:17 -07:00
|
|
|
let in_path = tmp_file_path("test_sample_file_input.txt");
|
2018-10-02 11:47:51 -07:00
|
|
|
let num_strings = 4096;
|
|
|
|
let string = "12foobar";
|
|
|
|
{
|
2018-10-08 16:15:17 -07:00
|
|
|
let mut in_file = File::create(&in_path).unwrap();
|
2018-10-02 11:47:51 -07:00
|
|
|
for _ in 0..num_strings {
|
|
|
|
in_file.write(string.as_bytes()).unwrap();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
let num_samples = (string.len() * num_strings / size_of::<Hash>()) as u64;
|
|
|
|
let samples: Vec<_> = (0..num_samples).collect();
|
2018-10-08 16:15:17 -07:00
|
|
|
let res = sample_file(&in_path, samples.as_slice());
|
2018-10-02 11:47:51 -07:00
|
|
|
assert!(res.is_ok());
|
|
|
|
let ref_hash: Hash = Hash::new(&[
|
|
|
|
173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144,
|
|
|
|
151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52,
|
|
|
|
]);
|
|
|
|
let res = res.unwrap();
|
|
|
|
assert_eq!(res, ref_hash);
|
|
|
|
|
|
|
|
// Sample just past the end
|
2018-10-08 16:15:17 -07:00
|
|
|
assert!(sample_file(&in_path, &[num_samples]).is_err());
|
|
|
|
remove_file(&in_path).unwrap();
|
2018-10-02 11:47:51 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_sample_file_invalid_offset() {
|
2018-10-08 16:15:17 -07:00
|
|
|
let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt");
|
2018-10-02 11:47:51 -07:00
|
|
|
{
|
2018-10-08 16:15:17 -07:00
|
|
|
let mut in_file = File::create(&in_path).unwrap();
|
2018-10-02 11:47:51 -07:00
|
|
|
for _ in 0..4096 {
|
|
|
|
in_file.write("123456foobar".as_bytes()).unwrap();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
let samples = [0, 200000];
|
2018-10-08 16:15:17 -07:00
|
|
|
let res = sample_file(&in_path, &samples);
|
2018-10-02 11:47:51 -07:00
|
|
|
assert!(res.is_err());
|
|
|
|
remove_file(in_path).unwrap();
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_sample_file_missing_file() {
|
2018-10-08 16:15:17 -07:00
|
|
|
let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt");
|
2018-10-02 11:47:51 -07:00
|
|
|
let samples = [0, 5];
|
2018-10-08 16:15:17 -07:00
|
|
|
let res = sample_file(&in_path, &samples);
|
2018-10-02 11:47:51 -07:00
|
|
|
assert!(res.is_err());
|
|
|
|
}
|
|
|
|
|
2018-09-21 15:32:15 -07:00
|
|
|
}
|