Validators now vote once a second regardless
This commit is contained in:
parent
80a02359f7
commit
7672506b45
|
@ -49,6 +49,7 @@ pub mod timing;
|
||||||
pub mod tpu;
|
pub mod tpu;
|
||||||
pub mod transaction;
|
pub mod transaction;
|
||||||
pub mod tvu;
|
pub mod tvu;
|
||||||
|
pub mod vote_stage;
|
||||||
pub mod voting;
|
pub mod voting;
|
||||||
pub mod window_stage;
|
pub mod window_stage;
|
||||||
pub mod write_stage;
|
pub mod write_stage;
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
//! The `replicate_stage` replicates transactions broadcast by the leader.
|
//! The `replicate_stage` replicates transactions broadcast by the leader.
|
||||||
|
|
||||||
use bank::Bank;
|
use bank::Bank;
|
||||||
use bincode::serialize;
|
|
||||||
use counter::Counter;
|
use counter::Counter;
|
||||||
use crdt::Crdt;
|
use crdt::Crdt;
|
||||||
use ledger;
|
use ledger;
|
||||||
|
@ -9,35 +8,29 @@ use packet::BlobRecycler;
|
||||||
use result::{Error, Result};
|
use result::{Error, Result};
|
||||||
use service::Service;
|
use service::Service;
|
||||||
use signature::KeyPair;
|
use signature::KeyPair;
|
||||||
use std::collections::VecDeque;
|
|
||||||
use std::net::UdpSocket;
|
use std::net::UdpSocket;
|
||||||
|
use std::sync::atomic::AtomicBool;
|
||||||
use std::sync::atomic::AtomicUsize;
|
use std::sync::atomic::AtomicUsize;
|
||||||
use std::sync::mpsc::channel;
|
use std::sync::mpsc::channel;
|
||||||
use std::sync::mpsc::RecvTimeoutError;
|
use std::sync::mpsc::RecvTimeoutError;
|
||||||
use std::sync::{Arc, RwLock};
|
use std::sync::{Arc, RwLock};
|
||||||
use std::thread::{self, Builder, JoinHandle};
|
use std::thread::{self, Builder, JoinHandle};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use streamer::{responder, BlobReceiver, BlobSender};
|
use streamer::{responder, BlobReceiver};
|
||||||
use timing;
|
use vote_stage::VoteStage;
|
||||||
use transaction::Transaction;
|
|
||||||
use voting::entries_to_votes;
|
use voting::entries_to_votes;
|
||||||
|
|
||||||
pub struct ReplicateStage {
|
pub struct ReplicateStage {
|
||||||
thread_hdls: Vec<JoinHandle<()>>,
|
thread_hdls: Vec<JoinHandle<()>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
const VOTE_TIMEOUT_MS: u64 = 1000;
|
|
||||||
|
|
||||||
impl ReplicateStage {
|
impl ReplicateStage {
|
||||||
/// Process entry blobs, already in order
|
/// Process entry blobs, already in order
|
||||||
fn replicate_requests(
|
fn replicate_requests(
|
||||||
keypair: &Arc<KeyPair>,
|
|
||||||
bank: &Arc<Bank>,
|
bank: &Arc<Bank>,
|
||||||
crdt: &Arc<RwLock<Crdt>>,
|
crdt: &Arc<RwLock<Crdt>>,
|
||||||
blob_recycler: &BlobRecycler,
|
blob_recycler: &BlobRecycler,
|
||||||
window_receiver: &BlobReceiver,
|
window_receiver: &BlobReceiver,
|
||||||
vote_blob_sender: &BlobSender,
|
|
||||||
last_vote: &mut u64,
|
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let timer = Duration::new(1, 0);
|
let timer = Duration::new(1, 0);
|
||||||
//coalesce all the available blobs into a single vote
|
//coalesce all the available blobs into a single vote
|
||||||
|
@ -61,30 +54,6 @@ impl ReplicateStage {
|
||||||
error!("process_entries {} {:?}", blobs_len, res);
|
error!("process_entries {} {:?}", blobs_len, res);
|
||||||
}
|
}
|
||||||
let _ = res?;
|
let _ = res?;
|
||||||
let now = timing::timestamp();
|
|
||||||
if now - *last_vote > VOTE_TIMEOUT_MS {
|
|
||||||
let last_id = bank.last_id();
|
|
||||||
let shared_blob = blob_recycler.allocate();
|
|
||||||
let (vote, addr) = {
|
|
||||||
let mut wcrdt = crdt.write().unwrap();
|
|
||||||
//TODO: doesn't seem like there is a synchronous call to get height and id
|
|
||||||
info!("replicate_stage {:?}", &last_id[..8]);
|
|
||||||
wcrdt.new_vote(last_id)
|
|
||||||
}?;
|
|
||||||
{
|
|
||||||
let mut blob = shared_blob.write().unwrap();
|
|
||||||
let tx = Transaction::new_vote(&keypair, vote, last_id, 0);
|
|
||||||
let bytes = serialize(&tx)?;
|
|
||||||
let len = bytes.len();
|
|
||||||
blob.data[..len].copy_from_slice(&bytes);
|
|
||||||
blob.meta.set_addr(&addr);
|
|
||||||
blob.meta.size = len;
|
|
||||||
}
|
|
||||||
inc_new_counter!("replicate-vote_sent", 1);
|
|
||||||
*last_vote = now;
|
|
||||||
|
|
||||||
vote_blob_sender.send(VecDeque::from(vec![shared_blob]))?;
|
|
||||||
}
|
|
||||||
while let Some(blob) = blobs.pop_front() {
|
while let Some(blob) = blobs.pop_front() {
|
||||||
blob_recycler.recycle(blob);
|
blob_recycler.recycle(blob);
|
||||||
}
|
}
|
||||||
|
@ -96,6 +65,7 @@ impl ReplicateStage {
|
||||||
crdt: Arc<RwLock<Crdt>>,
|
crdt: Arc<RwLock<Crdt>>,
|
||||||
blob_recycler: BlobRecycler,
|
blob_recycler: BlobRecycler,
|
||||||
window_receiver: BlobReceiver,
|
window_receiver: BlobReceiver,
|
||||||
|
exit: Arc<AtomicBool>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let (vote_blob_sender, vote_blob_receiver) = channel();
|
let (vote_blob_sender, vote_blob_receiver) = channel();
|
||||||
let send = UdpSocket::bind("0.0.0.0:0").expect("bind");
|
let send = UdpSocket::bind("0.0.0.0:0").expect("bind");
|
||||||
|
@ -105,34 +75,35 @@ impl ReplicateStage {
|
||||||
blob_recycler.clone(),
|
blob_recycler.clone(),
|
||||||
vote_blob_receiver,
|
vote_blob_receiver,
|
||||||
);
|
);
|
||||||
let skeypair = Arc::new(keypair);
|
|
||||||
|
let vote_stage = VoteStage::new(
|
||||||
|
Arc::new(keypair),
|
||||||
|
bank.clone(),
|
||||||
|
crdt.clone(),
|
||||||
|
blob_recycler.clone(),
|
||||||
|
vote_blob_sender,
|
||||||
|
exit,
|
||||||
|
);
|
||||||
|
|
||||||
let t_replicate = Builder::new()
|
let t_replicate = Builder::new()
|
||||||
.name("solana-replicate-stage".to_string())
|
.name("solana-replicate-stage".to_string())
|
||||||
.spawn(move || {
|
.spawn(move || loop {
|
||||||
let mut timestamp: u64 = 0;
|
if let Err(e) =
|
||||||
loop {
|
Self::replicate_requests(&bank, &crdt, &blob_recycler, &window_receiver)
|
||||||
if let Err(e) = Self::replicate_requests(
|
{
|
||||||
&skeypair,
|
match e {
|
||||||
&bank,
|
Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
|
||||||
&crdt,
|
Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
|
||||||
&blob_recycler,
|
_ => error!("{:?}", e),
|
||||||
&window_receiver,
|
|
||||||
&vote_blob_sender,
|
|
||||||
&mut timestamp,
|
|
||||||
) {
|
|
||||||
match e {
|
|
||||||
Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
|
|
||||||
Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
|
|
||||||
_ => error!("{:?}", e),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
ReplicateStage {
|
|
||||||
thread_hdls: vec![t_responder, t_replicate],
|
let mut thread_hdls = vec![t_responder, t_replicate];
|
||||||
}
|
thread_hdls.extend(vote_stage.thread_hdls());
|
||||||
|
|
||||||
|
ReplicateStage { thread_hdls }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
35
src/tvu.rs
35
src/tvu.rs
|
@ -4,22 +4,22 @@
|
||||||
//! ```text
|
//! ```text
|
||||||
//! .--------------------------------------------.
|
//! .--------------------------------------------.
|
||||||
//! | |
|
//! | |
|
||||||
//! | .--------------------------------+---------.
|
//! | .--------------------------------+------------.
|
||||||
//! | | TVU | |
|
//! | | TVU | |
|
||||||
//! | | | |
|
//! | | | |
|
||||||
//! | | | | .------------.
|
//! | | | | .------------.
|
||||||
//! | | .------------+----------->| Validators |
|
//! | | .------------+-------------->| Validators |
|
||||||
//! v | .-------. | | | `------------`
|
//! v | .-------. | | | `------------`
|
||||||
//! .----+---. | | | .----+---. .----+------. |
|
//! .----+---. | | | .----+---. .----+---------. |
|
||||||
//! | Leader |--------->| Blob | | Window | | Replicate | |
|
//! | Leader |--------->| Blob | | Window | | Replicate | |
|
||||||
//! `--------` | | Fetch |-->| Stage |-->| Stage | |
|
//! `--------` | | Fetch |-->| Stage |-->| Stage / | |
|
||||||
//! .------------. | | Stage | | | | | |
|
//! .------------. | | Stage | | | | Vote Stage | |
|
||||||
//! | Validators |----->| | `--------` `----+------` |
|
//! | Validators |----->| | `--------` `----+---------` |
|
||||||
//! `------------` | `-------` | |
|
//! `------------` | `-------` | |
|
||||||
//! | | |
|
//! | | |
|
||||||
//! | | |
|
//! | | |
|
||||||
//! | | |
|
//! | | |
|
||||||
//! `--------------------------------|---------`
|
//! `--------------------------------|------------`
|
||||||
//! |
|
//! |
|
||||||
//! v
|
//! v
|
||||||
//! .------.
|
//! .------.
|
||||||
|
@ -82,7 +82,7 @@ impl Tvu {
|
||||||
let blob_recycler = BlobRecycler::default();
|
let blob_recycler = BlobRecycler::default();
|
||||||
let (fetch_stage, blob_fetch_receiver) = BlobFetchStage::new_multi_socket(
|
let (fetch_stage, blob_fetch_receiver) = BlobFetchStage::new_multi_socket(
|
||||||
vec![replicate_socket, repair_socket],
|
vec![replicate_socket, repair_socket],
|
||||||
exit,
|
exit.clone(),
|
||||||
&blob_recycler,
|
&blob_recycler,
|
||||||
);
|
);
|
||||||
//TODO
|
//TODO
|
||||||
|
@ -103,6 +103,7 @@ impl Tvu {
|
||||||
crdt,
|
crdt,
|
||||||
blob_recycler,
|
blob_recycler,
|
||||||
blob_window_receiver,
|
blob_window_receiver,
|
||||||
|
exit,
|
||||||
);
|
);
|
||||||
|
|
||||||
Tvu {
|
Tvu {
|
||||||
|
|
|
@ -0,0 +1,153 @@
|
||||||
|
//! The `vote_stage` votes on the `last_id` of the bank at a regular cadence
|
||||||
|
|
||||||
|
use bank::Bank;
|
||||||
|
use bincode::serialize;
|
||||||
|
use counter::Counter;
|
||||||
|
use crdt::Crdt;
|
||||||
|
use hash::Hash;
|
||||||
|
use packet::BlobRecycler;
|
||||||
|
use result::Result;
|
||||||
|
use service::Service;
|
||||||
|
use signature::KeyPair;
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
|
use std::sync::{Arc, RwLock};
|
||||||
|
use std::thread::{self, sleep, spawn, JoinHandle};
|
||||||
|
use std::time::Duration;
|
||||||
|
use streamer::BlobSender;
|
||||||
|
use transaction::Transaction;
|
||||||
|
|
||||||
|
const VOTE_TIMEOUT_MS: u64 = 1000;
|
||||||
|
|
||||||
|
pub struct VoteStage {
|
||||||
|
thread_hdl: JoinHandle<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VoteStage {
|
||||||
|
pub fn new(
|
||||||
|
keypair: Arc<KeyPair>,
|
||||||
|
bank: Arc<Bank>,
|
||||||
|
crdt: Arc<RwLock<Crdt>>,
|
||||||
|
blob_recycler: BlobRecycler,
|
||||||
|
vote_blob_sender: BlobSender,
|
||||||
|
exit: Arc<AtomicBool>,
|
||||||
|
) -> Self {
|
||||||
|
let thread_hdl = spawn(move || {
|
||||||
|
Self::run(
|
||||||
|
&keypair,
|
||||||
|
&bank,
|
||||||
|
&crdt,
|
||||||
|
&blob_recycler,
|
||||||
|
&vote_blob_sender,
|
||||||
|
&exit,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
VoteStage { thread_hdl }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run(
|
||||||
|
keypair: &Arc<KeyPair>,
|
||||||
|
bank: &Arc<Bank>,
|
||||||
|
crdt: &Arc<RwLock<Crdt>>,
|
||||||
|
blob_recycler: &BlobRecycler,
|
||||||
|
vote_blob_sender: &BlobSender,
|
||||||
|
exit: &Arc<AtomicBool>,
|
||||||
|
) {
|
||||||
|
while !exit.load(Ordering::Relaxed) {
|
||||||
|
let last_id = bank.last_id();
|
||||||
|
|
||||||
|
if let Err(err) = Self::vote(&last_id, keypair, crdt, blob_recycler, vote_blob_sender) {
|
||||||
|
info!("Vote failed: {:?}", err);
|
||||||
|
}
|
||||||
|
sleep(Duration::from_millis(VOTE_TIMEOUT_MS));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn vote(
|
||||||
|
last_id: &Hash,
|
||||||
|
keypair: &Arc<KeyPair>,
|
||||||
|
crdt: &Arc<RwLock<Crdt>>,
|
||||||
|
blob_recycler: &BlobRecycler,
|
||||||
|
vote_blob_sender: &BlobSender,
|
||||||
|
) -> Result<()> {
|
||||||
|
let shared_blob = blob_recycler.allocate();
|
||||||
|
let (vote, addr) = {
|
||||||
|
let mut wcrdt = crdt.write().unwrap();
|
||||||
|
//TODO: doesn't seem like there is a synchronous call to get height and id
|
||||||
|
info!("replicate_stage {:?}", &last_id[..8]);
|
||||||
|
wcrdt.new_vote(*last_id)
|
||||||
|
}?;
|
||||||
|
{
|
||||||
|
let mut blob = shared_blob.write().unwrap();
|
||||||
|
let tx = Transaction::new_vote(&keypair, vote, *last_id, 0);
|
||||||
|
let bytes = serialize(&tx)?;
|
||||||
|
let len = bytes.len();
|
||||||
|
blob.data[..len].copy_from_slice(&bytes);
|
||||||
|
blob.meta.set_addr(&addr);
|
||||||
|
blob.meta.size = len;
|
||||||
|
}
|
||||||
|
inc_new_counter!("replicate-vote_sent", 1);
|
||||||
|
|
||||||
|
vote_blob_sender.send(VecDeque::from(vec![shared_blob]))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Service for VoteStage {
|
||||||
|
fn thread_hdls(self) -> Vec<JoinHandle<()>> {
|
||||||
|
vec![self.thread_hdl]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn join(self) -> thread::Result<()> {
|
||||||
|
self.thread_hdl.join()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub mod tests {
|
||||||
|
use super::*;
|
||||||
|
use bank::Bank;
|
||||||
|
use crdt::{Crdt, TestNode};
|
||||||
|
use mint::Mint;
|
||||||
|
use packet::BlobRecycler;
|
||||||
|
use service::Service;
|
||||||
|
use signature::{KeyPair, KeyPairUtil};
|
||||||
|
use std::sync::atomic::AtomicBool;
|
||||||
|
use std::sync::mpsc::channel;
|
||||||
|
use std::sync::{Arc, RwLock};
|
||||||
|
|
||||||
|
/// Ensure the VoteStage issues votes at the expected cadence
|
||||||
|
#[test]
|
||||||
|
fn test_vote_cadence() {
|
||||||
|
let keypair = KeyPair::new();
|
||||||
|
|
||||||
|
let mint = Mint::new(1234);
|
||||||
|
let bank = Arc::new(Bank::new(&mint));
|
||||||
|
|
||||||
|
let node = TestNode::new_localhost();
|
||||||
|
let mut crdt = Crdt::new(node.data.clone()).expect("Crdt::new");
|
||||||
|
crdt.set_leader(node.data.id);
|
||||||
|
let blob_recycler = BlobRecycler::default();
|
||||||
|
let (sender, receiver) = channel();
|
||||||
|
let exit = Arc::new(AtomicBool::new(false));
|
||||||
|
|
||||||
|
let vote_stage = VoteStage::new(
|
||||||
|
Arc::new(keypair),
|
||||||
|
bank.clone(),
|
||||||
|
Arc::new(RwLock::new(crdt)),
|
||||||
|
blob_recycler.clone(),
|
||||||
|
sender,
|
||||||
|
exit.clone(),
|
||||||
|
);
|
||||||
|
|
||||||
|
receiver.recv().unwrap();
|
||||||
|
|
||||||
|
let timeout = Duration::from_millis(VOTE_TIMEOUT_MS * 2);
|
||||||
|
receiver.recv_timeout(timeout).unwrap();
|
||||||
|
receiver.recv_timeout(timeout).unwrap();
|
||||||
|
|
||||||
|
exit.store(true, Ordering::Relaxed);
|
||||||
|
vote_stage.join().expect("join");
|
||||||
|
}
|
||||||
|
}
|
|
@ -360,8 +360,8 @@ fn test_leader_restart_validator_start_from_old_ledger() {
|
||||||
let getbal = retry_get_balance(&mut client, &bob_pubkey, Some(expected));
|
let getbal = retry_get_balance(&mut client, &bob_pubkey, Some(expected));
|
||||||
assert_eq!(getbal, Some(expected));
|
assert_eq!(getbal, Some(expected));
|
||||||
|
|
||||||
leader_fullnode.close().unwrap();
|
|
||||||
val_fullnode.close().unwrap();
|
val_fullnode.close().unwrap();
|
||||||
|
leader_fullnode.close().unwrap();
|
||||||
std::fs::remove_file(ledger_path).unwrap();
|
std::fs::remove_file(ledger_path).unwrap();
|
||||||
std::fs::remove_file(stale_ledger_path).unwrap();
|
std::fs::remove_file(stale_ledger_path).unwrap();
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue