2018-06-13 18:42:30 -07:00
|
|
|
extern crate atty;
|
2018-05-25 10:53:57 -07:00
|
|
|
extern crate env_logger;
|
2018-04-09 11:28:54 -07:00
|
|
|
extern crate getopts;
|
2018-03-28 15:51:18 -07:00
|
|
|
extern crate rayon;
|
2018-03-05 14:34:15 -08:00
|
|
|
extern crate serde_json;
|
2018-03-27 15:24:05 -07:00
|
|
|
extern crate solana;
|
2018-02-28 09:07:54 -08:00
|
|
|
|
2018-06-13 15:07:36 -07:00
|
|
|
use atty::{is, Stream};
|
2018-06-13 18:42:30 -07:00
|
|
|
use getopts::Options;
|
2018-04-02 20:15:21 -07:00
|
|
|
use rayon::prelude::*;
|
2018-06-29 14:12:26 -07:00
|
|
|
use solana::crdt::{Crdt, ReplicatedData};
|
2018-06-14 16:42:27 -07:00
|
|
|
use solana::hash::Hash;
|
2018-06-27 12:53:01 -07:00
|
|
|
use solana::mint::Mint;
|
2018-06-28 11:28:28 -07:00
|
|
|
use solana::nat::{udp_public_bind, udp_random_bind};
|
2018-06-07 15:06:32 -07:00
|
|
|
use solana::ncp::Ncp;
|
2018-07-03 21:14:08 -07:00
|
|
|
use solana::service::Service;
|
2018-05-25 15:54:03 -07:00
|
|
|
use solana::signature::{GenKeys, KeyPair, KeyPairUtil};
|
|
|
|
use solana::streamer::default_window;
|
2018-05-08 17:59:01 -07:00
|
|
|
use solana::thin_client::ThinClient;
|
2018-06-14 16:42:27 -07:00
|
|
|
use solana::timing::{duration_as_ms, duration_as_s};
|
2018-03-27 15:24:05 -07:00
|
|
|
use solana::transaction::Transaction;
|
2018-04-11 19:24:14 -07:00
|
|
|
use std::env;
|
2018-05-25 15:54:03 -07:00
|
|
|
use std::fs::File;
|
2018-04-21 06:12:57 -07:00
|
|
|
use std::io::{stdin, Read};
|
2018-05-31 15:48:03 -07:00
|
|
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
|
2018-04-19 07:06:19 -07:00
|
|
|
use std::process::exit;
|
2018-05-25 15:54:03 -07:00
|
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
|
|
use std::sync::{Arc, RwLock};
|
2018-05-04 11:11:39 -07:00
|
|
|
use std::thread::sleep;
|
2018-06-14 16:42:27 -07:00
|
|
|
use std::thread::Builder;
|
2018-05-29 20:20:28 -07:00
|
|
|
use std::thread::JoinHandle;
|
2018-05-04 11:11:39 -07:00
|
|
|
use std::time::Duration;
|
2018-04-26 12:17:36 -07:00
|
|
|
use std::time::Instant;
|
2018-03-04 00:21:40 -08:00
|
|
|
|
2018-04-19 10:16:20 -07:00
|
|
|
fn print_usage(program: &str, opts: Options) {
|
|
|
|
let mut brief = format!("Usage: cat <mint.json> | {} [options]\n\n", program);
|
|
|
|
brief += " Solana client demo creates a number of transactions and\n";
|
|
|
|
brief += " sends them to a target node.";
|
|
|
|
brief += " Takes json formatted mint file to stdin.";
|
|
|
|
|
|
|
|
print!("{}", opts.usage(&brief));
|
|
|
|
}
|
|
|
|
|
2018-06-14 16:42:27 -07:00
|
|
|
fn sample_tx_count(
|
|
|
|
exit: Arc<AtomicBool>,
|
|
|
|
maxes: Arc<RwLock<Vec<(f64, u64)>>>,
|
|
|
|
first_count: u64,
|
|
|
|
v: ReplicatedData,
|
|
|
|
sample_period: u64,
|
|
|
|
) {
|
2018-06-29 14:12:26 -07:00
|
|
|
let mut client = mk_client(&v);
|
2018-06-14 16:42:27 -07:00
|
|
|
let mut now = Instant::now();
|
|
|
|
let mut initial_tx_count = client.transaction_count();
|
|
|
|
let mut max_tps = 0.0;
|
|
|
|
let mut total;
|
|
|
|
loop {
|
|
|
|
let tx_count = client.transaction_count();
|
|
|
|
let duration = now.elapsed();
|
|
|
|
now = Instant::now();
|
|
|
|
let sample = tx_count - initial_tx_count;
|
|
|
|
initial_tx_count = tx_count;
|
|
|
|
println!("{}: Transactions processed {}", v.transactions_addr, sample);
|
|
|
|
let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos());
|
|
|
|
let tps = (sample * 1_000_000_000) as f64 / ns as f64;
|
|
|
|
if tps > max_tps {
|
|
|
|
max_tps = tps;
|
|
|
|
}
|
2018-06-15 10:12:38 -07:00
|
|
|
println!("{}: {:.2} tps", v.transactions_addr, tps);
|
2018-06-14 16:42:27 -07:00
|
|
|
total = tx_count - first_count;
|
|
|
|
println!(
|
|
|
|
"{}: Total Transactions processed {}",
|
|
|
|
v.transactions_addr, total
|
|
|
|
);
|
|
|
|
sleep(Duration::new(sample_period, 0));
|
|
|
|
|
|
|
|
if exit.load(Ordering::Relaxed) {
|
|
|
|
println!("exiting validator thread");
|
|
|
|
maxes.write().unwrap().push((max_tps, total));
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn generate_and_send_txs(
|
|
|
|
client: &mut ThinClient,
|
2018-06-24 10:12:08 -07:00
|
|
|
tx_clients: &Vec<ThinClient>,
|
2018-06-27 12:53:01 -07:00
|
|
|
mint: &Mint,
|
|
|
|
keypairs: &Vec<KeyPair>,
|
2018-06-14 16:42:27 -07:00
|
|
|
leader: &ReplicatedData,
|
|
|
|
txs: i64,
|
|
|
|
last_id: &mut Hash,
|
|
|
|
threads: usize,
|
|
|
|
) {
|
2018-06-27 12:53:01 -07:00
|
|
|
println!("Signing transactions... {}", keypairs.len(),);
|
2018-06-14 16:42:27 -07:00
|
|
|
let signing_start = Instant::now();
|
2018-06-27 12:53:01 -07:00
|
|
|
let transactions: Vec<_> = keypairs
|
2018-06-14 16:42:27 -07:00
|
|
|
.par_iter()
|
2018-06-27 12:53:01 -07:00
|
|
|
.map(|keypair| Transaction::new(&mint.keypair(), keypair.pubkey(), 1, *last_id))
|
2018-06-14 16:42:27 -07:00
|
|
|
.collect();
|
|
|
|
|
|
|
|
let duration = signing_start.elapsed();
|
|
|
|
let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos());
|
|
|
|
let bsps = txs as f64 / ns as f64;
|
|
|
|
let nsps = ns as f64 / txs as f64;
|
|
|
|
println!(
|
2018-06-15 10:12:38 -07:00
|
|
|
"Done. {:.2} thousand signatures per second, {:.2} us per signature, {} ms total time",
|
2018-06-14 16:42:27 -07:00
|
|
|
bsps * 1_000_000_f64,
|
2018-06-15 10:12:38 -07:00
|
|
|
nsps / 1_000_f64,
|
|
|
|
duration_as_ms(&duration),
|
2018-06-14 16:42:27 -07:00
|
|
|
);
|
|
|
|
|
|
|
|
println!("Transfering {} transactions in {} batches", txs, threads);
|
|
|
|
let transfer_start = Instant::now();
|
|
|
|
let sz = transactions.len() / threads;
|
|
|
|
let chunks: Vec<_> = transactions.chunks(sz).collect();
|
2018-06-24 10:12:08 -07:00
|
|
|
chunks
|
|
|
|
.into_par_iter()
|
|
|
|
.zip(tx_clients)
|
|
|
|
.for_each(|(txs, client)| {
|
|
|
|
println!(
|
|
|
|
"Transferring 1 unit {} times... to {:?}",
|
|
|
|
txs.len(),
|
|
|
|
leader.transactions_addr
|
|
|
|
);
|
|
|
|
for tx in txs {
|
|
|
|
client.transfer_signed(tx.clone()).unwrap();
|
|
|
|
}
|
|
|
|
});
|
2018-06-14 16:42:27 -07:00
|
|
|
println!(
|
|
|
|
"Transfer done. {:?} ms {} tps",
|
|
|
|
duration_as_ms(&transfer_start.elapsed()),
|
|
|
|
txs as f32 / (duration_as_s(&transfer_start.elapsed()))
|
|
|
|
);
|
|
|
|
|
2018-06-24 10:12:08 -07:00
|
|
|
loop {
|
|
|
|
let new_id = client.get_last_id();
|
|
|
|
if *last_id != new_id {
|
|
|
|
*last_id = new_id;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
sleep(Duration::from_millis(100));
|
|
|
|
}
|
2018-06-14 16:42:27 -07:00
|
|
|
}
|
|
|
|
|
2018-02-28 09:07:54 -08:00
|
|
|
fn main() {
|
2018-06-14 15:45:14 -07:00
|
|
|
env_logger::init();
|
2018-04-09 11:28:54 -07:00
|
|
|
let mut threads = 4usize;
|
2018-05-31 16:48:05 -07:00
|
|
|
let mut num_nodes = 1usize;
|
2018-06-14 16:42:27 -07:00
|
|
|
let mut time_sec = 60;
|
2018-03-03 20:15:42 -08:00
|
|
|
|
2018-04-09 11:28:54 -07:00
|
|
|
let mut opts = Options::new();
|
2018-05-25 15:54:03 -07:00
|
|
|
opts.optopt("l", "", "leader", "leader.json");
|
2018-05-04 11:11:39 -07:00
|
|
|
opts.optopt("t", "", "number of threads", &format!("{}", threads));
|
2018-06-14 16:42:27 -07:00
|
|
|
opts.optopt(
|
|
|
|
"s",
|
|
|
|
"",
|
|
|
|
"send transactions for this many seconds",
|
|
|
|
&format!("{}", time_sec),
|
|
|
|
);
|
2018-05-25 15:54:03 -07:00
|
|
|
opts.optopt(
|
|
|
|
"n",
|
|
|
|
"",
|
|
|
|
"number of nodes to converge to",
|
|
|
|
&format!("{}", num_nodes),
|
|
|
|
);
|
2018-04-19 10:16:20 -07:00
|
|
|
opts.optflag("h", "help", "print help");
|
2018-04-09 11:28:54 -07:00
|
|
|
let args: Vec<String> = env::args().collect();
|
|
|
|
let matches = match opts.parse(&args[1..]) {
|
|
|
|
Ok(m) => m,
|
2018-04-19 07:06:19 -07:00
|
|
|
Err(e) => {
|
|
|
|
eprintln!("{}", e);
|
|
|
|
exit(1);
|
|
|
|
}
|
2018-04-09 11:28:54 -07:00
|
|
|
};
|
|
|
|
|
2018-04-19 10:16:20 -07:00
|
|
|
if matches.opt_present("h") {
|
|
|
|
let program = args[0].clone();
|
|
|
|
print_usage(&program, opts);
|
|
|
|
return;
|
|
|
|
}
|
2018-04-09 11:28:54 -07:00
|
|
|
if matches.opt_present("t") {
|
|
|
|
threads = matches.opt_str("t").unwrap().parse().expect("integer");
|
|
|
|
}
|
2018-05-25 15:54:03 -07:00
|
|
|
if matches.opt_present("n") {
|
|
|
|
num_nodes = matches.opt_str("n").unwrap().parse().expect("integer");
|
|
|
|
}
|
2018-06-14 16:42:27 -07:00
|
|
|
if matches.opt_present("s") {
|
|
|
|
time_sec = matches.opt_str("s").unwrap().parse().expect("integer");
|
|
|
|
}
|
2018-04-21 06:12:57 -07:00
|
|
|
|
2018-05-31 15:48:03 -07:00
|
|
|
let leader = if matches.opt_present("l") {
|
|
|
|
read_leader(matches.opt_str("l").unwrap())
|
|
|
|
} else {
|
|
|
|
let server_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8000);
|
|
|
|
ReplicatedData::new_leader(&server_addr)
|
|
|
|
};
|
|
|
|
|
2018-05-25 15:54:03 -07:00
|
|
|
let signal = Arc::new(AtomicBool::new(false));
|
|
|
|
let mut c_threads = vec![];
|
2018-06-29 14:12:26 -07:00
|
|
|
let validators = converge(&leader, signal.clone(), num_nodes, &mut c_threads);
|
2018-06-07 13:51:15 -07:00
|
|
|
assert_eq!(validators.len(), num_nodes);
|
2018-05-11 15:35:53 -07:00
|
|
|
|
2018-06-13 15:07:36 -07:00
|
|
|
if is(Stream::Stdin) {
|
2018-04-21 06:12:57 -07:00
|
|
|
eprintln!("nothing found on stdin, expected a json file");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut buffer = String::new();
|
|
|
|
let num_bytes = stdin().read_to_string(&mut buffer).unwrap();
|
|
|
|
if num_bytes == 0 {
|
|
|
|
eprintln!("empty file on stdin, expected a json file");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
2018-04-20 22:28:55 -07:00
|
|
|
println!("Parsing stdin...");
|
2018-06-27 12:53:01 -07:00
|
|
|
let mint: Mint = serde_json::from_str(&buffer).unwrap_or_else(|e| {
|
2018-04-19 07:55:47 -07:00
|
|
|
eprintln!("failed to parse json: {}", e);
|
|
|
|
exit(1);
|
|
|
|
});
|
2018-06-29 14:12:26 -07:00
|
|
|
let mut client = mk_client(&leader);
|
2018-04-20 22:28:55 -07:00
|
|
|
|
|
|
|
println!("Get last ID...");
|
2018-06-14 16:42:27 -07:00
|
|
|
let mut last_id = client.get_last_id();
|
2018-05-02 20:38:07 -07:00
|
|
|
println!("Got last ID {:?}", last_id);
|
2018-03-05 14:34:15 -08:00
|
|
|
|
2018-06-11 13:04:51 -07:00
|
|
|
let mut seed = [0u8; 32];
|
2018-06-27 12:53:01 -07:00
|
|
|
seed.copy_from_slice(&mint.keypair().public_key_bytes()[..32]);
|
2018-06-11 13:04:51 -07:00
|
|
|
let rnd = GenKeys::new(seed);
|
2018-05-08 21:03:05 -07:00
|
|
|
|
2018-04-20 22:28:55 -07:00
|
|
|
println!("Creating keypairs...");
|
2018-06-27 13:32:18 -07:00
|
|
|
let txs = 500_000;
|
|
|
|
let keypairs = rnd.gen_n_keypairs(txs);
|
2018-03-05 14:34:15 -08:00
|
|
|
|
2018-05-25 15:54:03 -07:00
|
|
|
let first_count = client.transaction_count();
|
|
|
|
println!("initial count {}", first_count);
|
2018-04-17 15:41:58 -07:00
|
|
|
|
2018-06-14 16:42:27 -07:00
|
|
|
println!("Sampling tps every second...",);
|
2018-04-16 13:51:06 -07:00
|
|
|
|
2018-06-15 10:12:38 -07:00
|
|
|
// Setup a thread per validator to sample every period
|
|
|
|
// collect the max transaction rate and total tx count seen
|
2018-06-14 16:42:27 -07:00
|
|
|
let maxes = Arc::new(RwLock::new(Vec::new()));
|
2018-06-02 09:59:39 -07:00
|
|
|
let sample_period = 1; // in seconds
|
2018-06-14 16:42:27 -07:00
|
|
|
let v_threads: Vec<_> = validators
|
|
|
|
.into_iter()
|
|
|
|
.map(|v| {
|
|
|
|
let exit = signal.clone();
|
|
|
|
let maxes = maxes.clone();
|
|
|
|
Builder::new()
|
|
|
|
.name("solana-client-sample".to_string())
|
|
|
|
.spawn(move || {
|
2018-06-29 14:12:26 -07:00
|
|
|
sample_tx_count(exit, maxes, first_count, v, sample_period);
|
2018-06-14 16:42:27 -07:00
|
|
|
})
|
|
|
|
.unwrap()
|
2018-06-02 09:59:39 -07:00
|
|
|
})
|
|
|
|
.collect();
|
2018-06-14 16:42:27 -07:00
|
|
|
|
2018-06-29 14:12:26 -07:00
|
|
|
let clients = (0..threads).map(|_| mk_client(&leader)).collect();
|
2018-06-24 10:12:08 -07:00
|
|
|
|
2018-06-15 10:12:38 -07:00
|
|
|
// generate and send transactions for the specified duration
|
2018-06-14 16:42:27 -07:00
|
|
|
let time = Duration::new(time_sec, 0);
|
|
|
|
let now = Instant::now();
|
|
|
|
while now.elapsed() < time {
|
|
|
|
generate_and_send_txs(
|
|
|
|
&mut client,
|
2018-06-24 10:12:08 -07:00
|
|
|
&clients,
|
2018-06-27 12:53:01 -07:00
|
|
|
&mint,
|
|
|
|
&keypairs,
|
2018-06-14 16:42:27 -07:00
|
|
|
&leader,
|
|
|
|
txs,
|
|
|
|
&mut last_id,
|
|
|
|
threads,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2018-06-15 10:12:38 -07:00
|
|
|
// Stop the sampling threads so it will collect the stats
|
2018-06-14 16:42:27 -07:00
|
|
|
signal.store(true, Ordering::Relaxed);
|
|
|
|
for t in v_threads {
|
|
|
|
t.join().unwrap();
|
|
|
|
}
|
|
|
|
|
2018-06-15 10:12:38 -07:00
|
|
|
// Compute/report stats
|
2018-06-02 09:59:39 -07:00
|
|
|
let mut max_of_maxes = 0.0;
|
|
|
|
let mut total_txs = 0;
|
2018-06-14 16:42:27 -07:00
|
|
|
for (max, txs) in maxes.read().unwrap().iter() {
|
2018-06-02 09:59:39 -07:00
|
|
|
if *max > max_of_maxes {
|
|
|
|
max_of_maxes = *max;
|
2018-05-25 15:54:03 -07:00
|
|
|
}
|
2018-06-02 09:59:39 -07:00
|
|
|
total_txs += *txs;
|
|
|
|
}
|
|
|
|
println!(
|
2018-06-15 10:12:38 -07:00
|
|
|
"\nHighest TPS: {:.2} sampling period {}s total transactions: {} clients: {}",
|
2018-06-02 09:59:39 -07:00
|
|
|
max_of_maxes,
|
|
|
|
sample_period,
|
|
|
|
total_txs,
|
2018-06-14 16:42:27 -07:00
|
|
|
maxes.read().unwrap().len()
|
2018-06-02 09:59:39 -07:00
|
|
|
);
|
2018-06-14 16:42:27 -07:00
|
|
|
|
2018-06-15 10:12:38 -07:00
|
|
|
// join the crdt client threads
|
2018-05-25 15:54:03 -07:00
|
|
|
for t in c_threads {
|
|
|
|
t.join().unwrap();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-06-29 14:12:26 -07:00
|
|
|
fn mk_client(r: &ReplicatedData) -> ThinClient {
|
2018-06-28 11:28:28 -07:00
|
|
|
let requests_socket = udp_random_bind(8000, 10000, 5).unwrap();
|
|
|
|
let transactions_socket = udp_random_bind(8000, 10000, 5).unwrap();
|
2018-06-29 14:12:26 -07:00
|
|
|
|
2018-07-02 13:43:33 -07:00
|
|
|
requests_socket
|
2018-06-14 16:42:27 -07:00
|
|
|
.set_read_timeout(Some(Duration::new(1, 0)))
|
|
|
|
.unwrap();
|
|
|
|
|
2018-05-25 15:54:03 -07:00
|
|
|
ThinClient::new(
|
|
|
|
r.requests_addr,
|
2018-07-02 13:43:33 -07:00
|
|
|
requests_socket,
|
2018-05-25 15:54:03 -07:00
|
|
|
r.transactions_addr,
|
2018-07-02 13:43:33 -07:00
|
|
|
transactions_socket,
|
2018-05-25 15:54:03 -07:00
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2018-06-29 14:12:26 -07:00
|
|
|
fn spy_node() -> (ReplicatedData, UdpSocket) {
|
2018-06-28 11:28:28 -07:00
|
|
|
let gossip_socket_pair = udp_public_bind("gossip", 8000, 10000);
|
2018-05-25 15:54:03 -07:00
|
|
|
let pubkey = KeyPair::new().pubkey();
|
2018-06-29 14:12:26 -07:00
|
|
|
let daddr = "0.0.0.0:0".parse().unwrap();
|
2018-06-02 08:32:51 -07:00
|
|
|
let node = ReplicatedData::new(
|
|
|
|
pubkey,
|
2018-06-29 14:12:26 -07:00
|
|
|
//gossip.local_addr().unwrap(),
|
|
|
|
gossip_socket_pair.addr,
|
2018-06-02 08:32:51 -07:00
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
daddr,
|
|
|
|
);
|
2018-06-29 14:12:26 -07:00
|
|
|
(node, gossip_socket_pair.receiver)
|
2018-05-25 15:54:03 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fn converge(
|
|
|
|
leader: &ReplicatedData,
|
|
|
|
exit: Arc<AtomicBool>,
|
|
|
|
num_nodes: usize,
|
|
|
|
threads: &mut Vec<JoinHandle<()>>,
|
|
|
|
) -> Vec<ReplicatedData> {
|
|
|
|
//lets spy on the network
|
|
|
|
let daddr = "0.0.0.0:0".parse().unwrap();
|
2018-06-29 14:12:26 -07:00
|
|
|
let (spy, spy_gossip) = spy_node();
|
2018-05-25 15:54:03 -07:00
|
|
|
let mut spy_crdt = Crdt::new(spy);
|
|
|
|
spy_crdt.insert(&leader);
|
|
|
|
spy_crdt.set_leader(leader.id);
|
|
|
|
let spy_ref = Arc::new(RwLock::new(spy_crdt));
|
2018-05-27 18:21:39 -07:00
|
|
|
let window = default_window();
|
2018-06-28 11:28:28 -07:00
|
|
|
let gossip_send_socket = udp_random_bind(8000, 10000, 5).unwrap();
|
2018-06-07 15:06:32 -07:00
|
|
|
let ncp = Ncp::new(
|
2018-05-27 18:21:39 -07:00
|
|
|
spy_ref.clone(),
|
|
|
|
window.clone(),
|
|
|
|
spy_gossip,
|
|
|
|
gossip_send_socket,
|
|
|
|
exit.clone(),
|
|
|
|
).expect("DataReplicator::new");
|
2018-06-07 13:51:15 -07:00
|
|
|
let mut rv = vec![];
|
|
|
|
//wait for the network to converge, 30 seconds should be plenty
|
2018-05-25 15:54:03 -07:00
|
|
|
for _ in 0..30 {
|
2018-06-07 13:51:15 -07:00
|
|
|
let v: Vec<ReplicatedData> = spy_ref
|
|
|
|
.read()
|
|
|
|
.unwrap()
|
|
|
|
.table
|
|
|
|
.values()
|
|
|
|
.into_iter()
|
|
|
|
.filter(|x| x.requests_addr != daddr)
|
|
|
|
.cloned()
|
|
|
|
.collect();
|
|
|
|
if v.len() >= num_nodes {
|
|
|
|
println!("CONVERGED!");
|
|
|
|
rv.extend(v.into_iter());
|
2018-05-25 15:54:03 -07:00
|
|
|
break;
|
|
|
|
}
|
2018-05-04 11:11:39 -07:00
|
|
|
sleep(Duration::new(1, 0));
|
2018-04-17 15:41:58 -07:00
|
|
|
}
|
2018-07-03 21:14:08 -07:00
|
|
|
threads.extend(ncp.thread_hdls().into_iter());
|
2018-06-07 13:51:15 -07:00
|
|
|
rv
|
2018-05-25 15:54:03 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
fn read_leader(path: String) -> ReplicatedData {
|
2018-06-15 15:29:43 -07:00
|
|
|
let file = File::open(path.clone()).expect(&format!("file not found: {}", path));
|
|
|
|
serde_json::from_reader(file).expect(&format!("failed to parse {}", path))
|
2018-02-28 09:07:54 -08:00
|
|
|
}
|