blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

simple replay stage

This commit is contained in:
Anatoly Yakovenko 2019-02-26 21:57:45 -08:00 committed by Grimes
parent 2782922f7a
commit b1a648113f
8 changed files with 290 additions and 448 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/bank_forks.rs
View File

@ -27,6 +27,23 @@ impl BankForks {
working_bank,
}
}
pub fn frozen_banks(&self) -> HashMap<u64, Arc<Bank>> {
let mut frozen_banks: Vec<Arc<Bank>> = vec![];
frozen_banks.extend(self.banks.values().filter(|v| v.is_frozen()).cloned());
frozen_banks.extend(
self.banks
.iter()
.flat_map(|(_, v)| v.parents())
.filter(|v| v.is_frozen()),
);
frozen_banks.into_iter().map(|b| (b.slot(), b)).collect()
}
pub fn active_banks(&self) -> Vec<u64> {
self.banks.iter().map(|(k, _v)| *k).collect()
}
pub fn get(&self, bank_id: u64) -> Option<&Arc<Bank>> {
self.banks.get(&bank_id)
}
pub fn new_from_banks(initial_banks: &[Arc<Bank>]) -> Self {
let mut banks = HashMap::new();
@ -82,4 +99,26 @@ mod tests {
assert_eq!(bank_forks[1u64].tick_height(), 1);
assert_eq!(bank_forks.working_bank().tick_height(), 1);
}
#[test]
fn test_bank_forks_frozen_banks() {
let (genesis_block, _) = GenesisBlock::new(10_000);
let bank = Bank::new(&genesis_block);
let mut bank_forks = BankForks::new(0, bank);
let child_bank = Bank::new_from_parent(&bank_forks[0u64], Pubkey::default(), 1);
bank_forks.insert(1, child_bank);
assert!(bank_forks.frozen_banks().get(&0).is_some());
assert!(bank_forks.frozen_banks().get(&1).is_none());
}
#[test]
fn test_bank_forks_active_banks() {
let (genesis_block, _) = GenesisBlock::new(10_000);
let bank = Bank::new(&genesis_block);
let mut bank_forks = BankForks::new(0, bank);
let child_bank = Bank::new_from_parent(&bank_forks[0u64], Pubkey::default(), 1);
bank_forks.insert(1, child_bank);
assert_eq!(bank_forks.active_banks(), vec![1]);
}
}

1
src/banking_stage.rs
View File

@ -474,6 +474,7 @@ mod tests {
poh_service.close().unwrap();
}
#[test]
#[ignore] //flaky
fn test_banking_stage_entryfication() {
// In this attack we'll demonstrate that a verifier can interpret the ledger
// differently if either the server doesn't signal the ledger to add an

12
src/blocktree.rs
View File

@ -850,6 +850,7 @@ impl Blocktree {
max_missing,
)
}
/// Returns the entry vector for the slot starting with `blob_start_index`
pub fn get_slot_entries(
&self,
@ -857,17 +858,10 @@ impl Blocktree {
blob_start_index: u64,
max_entries: Option<u64>,
) -> Result<Vec<Entry>> {
// Find the next consecutive block of blobs.
let consecutive_blobs = self.get_slot_consecutive_blobs(
slot_height,
&HashMap::new(),
blob_start_index,
max_entries,
)?;
Ok(Self::deserialize_blobs(&consecutive_blobs))
self.get_slot_entries_with_blob_count(slot_height, blob_start_index, max_entries)
.map(|x| x.0)
}
/// Returns the entry vector for the slot starting with `blob_start_index`
pub fn get_slot_entries_with_blob_count(
&self,
slot_height: u64,

91
src/fullnode.rs
View File

@ -8,7 +8,6 @@ use crate::entry::create_ticks;
use crate::entry::next_entry_mut;
use crate::entry::Entry;
use crate::gossip_service::GossipService;
use crate::leader_schedule_utils;
use crate::poh_recorder::PohRecorder;
use crate::poh_service::{PohService, PohServiceConfig};
use crate::rpc_pubsub_service::PubSubService;
@ -59,14 +58,6 @@ impl NodeServices {
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum FullnodeReturnType {
LeaderToValidatorRotation,
ValidatorToLeaderRotation,
LeaderToLeaderRotation,
ValidatorToValidatorRotation,
}
pub struct FullnodeConfig {
pub sigverify_disabled: bool,
pub voting_disabled: bool,
@ -106,6 +97,7 @@ pub struct Fullnode {
blocktree: Arc<Blocktree>,
poh_service: PohService,
poh_recorder: Arc<Mutex<PohRecorder>>,
bank_forks: Arc<RwLock<BankForks>>,
}
impl Fullnode {
@ -262,35 +254,36 @@ impl Fullnode {
blocktree,
poh_service,
poh_recorder,
bank_forks,
}
}
fn rotate(&mut self, rotation_info: TvuRotationInfo) -> FullnodeReturnType {
fn rotate(&mut self, rotation_info: TvuRotationInfo) {
trace!(
"{:?}: rotate for slot={} to leader={:?}",
self.id,
rotation_info.slot,
rotation_info.leader_id,
);
let was_leader = leader_schedule_utils::slot_leader(&rotation_info.bank) == self.id;
if let Some(ref mut rpc_service) = self.rpc_service {
// TODO: This is not the correct bank. Instead TVU should pass along the
// frozen Bank for each completed block for RPC to use from it's notion of the "best"
// available fork (until we want to surface multiple forks to RPC)
rpc_service.set_bank(&rotation_info.bank);
rpc_service.set_bank(&self.bank_forks.read().unwrap().working_bank());
}
if rotation_info.leader_id == self.id {
let transition = if was_leader {
debug!("{:?} remaining in leader role", self.id);
FullnodeReturnType::LeaderToLeaderRotation
} else {
debug!("{:?} rotating to leader role", self.id);
FullnodeReturnType::ValidatorToLeaderRotation
};
debug!("{:?} rotating to leader role", self.id);
let tpu_bank = self
.bank_forks
.read()
.unwrap()
.get(rotation_info.slot)
.unwrap()
.clone();
self.node_services.tpu.switch_to_leader(
&rotation_info.bank,
&tpu_bank,
&self.poh_recorder,
self.tpu_sockets
.iter()
@ -303,15 +296,7 @@ impl Fullnode {
rotation_info.slot,
&self.blocktree,
);
transition
} else {
let transition = if was_leader {
debug!("{:?} rotating to validator role", self.id);
FullnodeReturnType::LeaderToValidatorRotation
} else {
debug!("{:?} remaining in validator role", self.id);
FullnodeReturnType::ValidatorToValidatorRotation
};
self.node_services.tpu.switch_to_forwarder(
rotation_info.leader_id,
self.tpu_sockets
@ -319,7 +304,6 @@ impl Fullnode {
.map(|s| s.try_clone().expect("Failed to clone TPU sockets"))
.collect(),
);
transition
}
}
@ -327,7 +311,7 @@ impl Fullnode {
// node to exit.
pub fn start(
mut self,
rotation_notifier: Option<Sender<(FullnodeReturnType, u64)>>,
rotation_notifier: Option<Sender<u64>>,
) -> (JoinHandle<()>, Arc<AtomicBool>, Receiver<bool>) {
let (sender, receiver) = channel();
let exit = self.exit.clone();
@ -345,15 +329,19 @@ impl Fullnode {
trace!("{:?}: rotate at slot={}", self.id, rotation_info.slot);
//TODO: this will be called by the TVU every time it votes
//instead of here
self.poh_recorder.lock().unwrap().reset(
rotation_info.bank.tick_height(),
rotation_info.bank.last_id(),
info!(
"reset PoH... {} {}",
rotation_info.tick_height, rotation_info.last_id
);
self.poh_recorder
.lock()
.unwrap()
.reset(rotation_info.tick_height, rotation_info.last_id);
let slot = rotation_info.slot;
let transition = self.rotate(rotation_info);
debug!("role transition complete: {:?}", transition);
self.rotate(rotation_info);
debug!("role transition complete");
if let Some(ref rotation_notifier) = rotation_notifier {
rotation_notifier.send((transition, slot)).unwrap();
rotation_notifier.send(slot).unwrap();
}
}
Err(RecvTimeoutError::Timeout) => continue,
@ -363,10 +351,7 @@ impl Fullnode {
(handle, exit, receiver)
}
pub fn run(
self,
rotation_notifier: Option<Sender<(FullnodeReturnType, u64)>>,
) -> impl FnOnce() {
pub fn run(self, rotation_notifier: Option<Sender<u64>>) -> impl FnOnce() {
let (_, exit, receiver) = self.start(rotation_notifier);
move || {
exit.store(true, Ordering::Relaxed);
@ -592,10 +577,7 @@ mod tests {
// Wait for the bootstrap leader to transition. Since there are no other nodes in the
// cluster it will continue to be the leader
assert_eq!(
rotation_receiver.recv().unwrap(),
(FullnodeReturnType::LeaderToLeaderRotation, 1)
);
assert_eq!(rotation_receiver.recv().unwrap(), 1);
bootstrap_leader_exit();
}
@ -638,13 +620,7 @@ mod tests {
);
let (rotation_sender, rotation_receiver) = channel();
let bootstrap_leader_exit = bootstrap_leader.run(Some(rotation_sender));
assert_eq!(
rotation_receiver.recv().unwrap(),
(
FullnodeReturnType::LeaderToValidatorRotation,
DEFAULT_SLOTS_PER_EPOCH
)
);
assert_eq!(rotation_receiver.recv().unwrap(), (DEFAULT_SLOTS_PER_EPOCH));
// Test that a node knows to transition to a leader based on parsing the ledger
let validator = Fullnode::new(
@ -658,13 +634,7 @@ mod tests {
let (rotation_sender, rotation_receiver) = channel();
let validator_exit = validator.run(Some(rotation_sender));
assert_eq!(
rotation_receiver.recv().unwrap(),
(
FullnodeReturnType::ValidatorToLeaderRotation,
DEFAULT_SLOTS_PER_EPOCH
)
);
assert_eq!(rotation_receiver.recv().unwrap(), (DEFAULT_SLOTS_PER_EPOCH));
validator_exit();
bootstrap_leader_exit();
@ -741,10 +711,7 @@ mod tests {
let (rotation_sender, rotation_receiver) = channel();
let validator_exit = validator.run(Some(rotation_sender));
let rotation = rotation_receiver.recv().unwrap();
assert_eq!(
rotation,
(FullnodeReturnType::ValidatorToLeaderRotation, blobs_to_send)
);
assert_eq!(rotation, blobs_to_send);
// Close the validator so that rocksdb has locks available
validator_exit();

6
src/poh_recorder.rs
View File

@ -63,6 +63,7 @@ impl PohRecorder {
}
pub fn set_working_bank(&mut self, working_bank: WorkingBank) {
trace!("new working bank");
self.working_bank = Some(working_bank);
}
@ -94,8 +95,9 @@ impl PohRecorder {
.take_while(|x| x.1 <= working_bank.max_tick_height)
.count();
let e = if cnt > 0 {
trace!(
"flush_cache: {} {} sending: {}",
debug!(
"flush_cache: bank_id: {} tick_height: {} max: {} sending: {}",
working_bank.bank.slot(),
working_bank.bank.tick_height(),
working_bank.max_tick_height,
cnt,

583
src/replay_stage.rs
View File

@ -2,26 +2,26 @@
use crate::bank_forks::BankForks;
use crate::blocktree::Blocktree;
use crate::blocktree_processor::{self, BankForksInfo};
use crate::blocktree_processor;
use crate::blocktree_processor::BankForksInfo;
use crate::cluster_info::ClusterInfo;
use crate::entry::{Entry, EntryReceiver, EntrySender, EntrySlice};
use crate::leader_schedule_utils;
use crate::packet::BlobError;
use crate::result::{Error, Result};
use crate::result;
use crate::rpc_subscriptions::RpcSubscriptions;
use crate::service::Service;
use crate::tvu::{TvuRotationInfo, TvuRotationSender};
use solana_metrics::counter::Counter;
use solana_metrics::{influxdb, submit};
use solana_runtime::bank::Bank;
use solana_sdk::hash::Hash;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::KeypairUtil;
use solana_sdk::timing::duration_as_ms;
use solana_sdk::vote_transaction::VoteTransaction;
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::RecvTimeoutError;
use std::sync::mpsc::{channel, Receiver};
use std::sync::mpsc::{channel, Receiver, RecvTimeoutError};
use std::sync::{Arc, RwLock};
use std::thread::{self, Builder, JoinHandle};
use std::time::Duration;
@ -48,127 +48,18 @@ impl Drop for Finalizer {
}
pub struct ReplayStage {
t_replay: JoinHandle<()>,
t_replay: JoinHandle<result::Result<()>>,
exit: Arc<AtomicBool>,
}
impl ReplayStage {
/// Process entry blobs, already in order
#[allow(clippy::too_many_arguments)]
fn process_entries<T: KeypairUtil>(
mut entries: Vec<Entry>,
bank: &Arc<Bank>,
cluster_info: &Arc<RwLock<ClusterInfo>>,
voting_keypair: &Option<Arc<T>>,
forward_entry_sender: &EntrySender,
current_blob_index: &mut u64,
last_entry_hash: &mut Hash,
subscriptions: &Arc<RpcSubscriptions>,
) -> Result<()> {
// Coalesce all the available entries into a single vote
submit(
influxdb::Point::new("replicate-stage")
.add_field("count", influxdb::Value::Integer(entries.len() as i64))
.to_owned(),
);
let mut res = Ok(());
let mut num_entries_to_write = entries.len();
let now = Instant::now();
if !entries.as_slice().verify(last_entry_hash) {
inc_new_counter_info!("replicate_stage-verify-fail", entries.len());
return Err(Error::BlobError(BlobError::VerificationFailed));
}
inc_new_counter_info!(
"replicate_stage-verify-duration",
duration_as_ms(&now.elapsed()) as usize
);
let num_ticks = bank.tick_height();
let mut num_ticks_to_next_vote =
leader_schedule_utils::num_ticks_left_in_slot(bank, num_ticks);
for (i, entry) in entries.iter().enumerate() {
inc_new_counter_info!("replicate-stage_bank-tick", bank.tick_height() as usize);
if entry.is_tick() {
if num_ticks_to_next_vote == 0 {
num_ticks_to_next_vote = bank.ticks_per_slot();
}
num_ticks_to_next_vote -= 1;
}
inc_new_counter_info!(
"replicate-stage_tick-to-vote",
num_ticks_to_next_vote as usize
);
// If it's the last entry in the vector, i will be vec len - 1.
// If we don't process the entry now, the for loop will exit and the entry
// will be dropped.
if 0 == num_ticks_to_next_vote || (i + 1) == entries.len() {
res = blocktree_processor::process_entries(bank, &entries[0..=i]);
if res.is_err() {
// TODO: This will return early from the first entry that has an erroneous
// transaction, instead of processing the rest of the entries in the vector
// of received entries. This is in line with previous behavior when
// bank.process_entries() was used to process the entries, but doesn't solve the
// issue that the bank state was still changed, leading to inconsistencies with the
// leader as the leader currently should not be publishing erroneous transactions
inc_new_counter_info!("replicate-stage_failed_process_entries", i);
break;
}
if 0 == num_ticks_to_next_vote {
subscriptions.notify_subscribers(&bank);
if let Some(voting_keypair) = voting_keypair {
let keypair = voting_keypair.as_ref();
let vote =
VoteTransaction::new_vote(keypair, bank.slot(), bank.last_id(), 0);
cluster_info.write().unwrap().push_vote(vote);
}
}
num_entries_to_write = i + 1;
break;
}
}
// If leader rotation happened, only write the entries up to leader rotation.
entries.truncate(num_entries_to_write);
*last_entry_hash = entries
.last()
.expect("Entries cannot be empty at this point")
.hash;
inc_new_counter_info!(
"replicate-transactions",
entries.iter().map(|x| x.transactions.len()).sum()
);
let entries_len = entries.len() as u64;
// TODO: In line with previous behavior, this will write all the entries even if
// an error occurred processing one of the entries (causing the rest of the entries to
// not be processed).
if entries_len != 0 {
forward_entry_sender.send(entries)?;
}
*current_blob_index += entries_len;
res?;
inc_new_counter_info!(
"replicate_stage-duration",
duration_as_ms(&now.elapsed()) as usize
);
Ok(())
}
#[allow(clippy::new_ret_no_self, clippy::too_many_arguments)]
pub fn new<T>(
my_id: Pubkey,
voting_keypair: Option<Arc<T>>,
blocktree: Arc<Blocktree>,
bank_forks: &Arc<RwLock<BankForks>>,
bank_forks_info: &[BankForksInfo],
_bank_forks_info: &[BankForksInfo],
cluster_info: Arc<RwLock<ClusterInfo>>,
exit: Arc<AtomicBool>,
to_leader_sender: &TvuRotationSender,
@ -180,217 +71,107 @@ impl ReplayStage {
{
let (forward_entry_sender, forward_entry_receiver) = channel();
let (slot_full_sender, slot_full_receiver) = channel();
trace!("replay stage");
let exit_ = exit.clone();
let to_leader_sender = to_leader_sender.clone();
let subscriptions_ = subscriptions.clone();
let subscriptions = subscriptions.clone();
let bank_forks = bank_forks.clone();
// Gather up all the metadata about the current state of the ledger
let mut bank = bank_forks.read().unwrap()[bank_forks_info[0].bank_id].clone();
// Update Tpu and other fullnode components with the current bank
let (mut current_slot, mut current_leader_id, mut max_tick_height_for_slot) = {
let tick_height = bank.tick_height();
let slot = (tick_height + 1) / bank.ticks_per_slot();
let first_tick_in_slot = slot * bank.ticks_per_slot();
let leader_id = leader_schedule_utils::slot_leader_at(slot, &bank);
trace!("node {:?} scheduled as leader for slot {}", leader_id, slot,);
let old_bank = bank.clone();
// If the next slot is going to be a new slot and we're the leader for that slot,
// make a new working bank, set it as the working bank.
if tick_height + 1 == first_tick_in_slot && leader_id == my_id {
bank = Self::create_and_set_working_bank(&old_bank, leader_id, slot, &bank_forks);
}
// Send a rotation notification back to Fullnode to initialize the TPU to the right
// state. After this point, the bank.tick_height() is live, which it means it can
// be updated by the TPU
to_leader_sender
.send(TvuRotationInfo {
bank: bank.clone(),
slot,
leader_id,
})
.unwrap();
let max_tick_height_for_slot = first_tick_in_slot
+ leader_schedule_utils::num_ticks_left_in_slot(&bank, first_tick_in_slot);
(Some(slot), leader_id, max_tick_height_for_slot)
};
let mut last_entry_hash = bank.last_id();
let mut current_blob_index = 0;
let mut progress = HashMap::new();
// Start the replay stage loop
let bank_forks = bank_forks.clone();
let t_replay = Builder::new()
.name("solana-replay-stage".to_string())
.spawn(move || {
let _exit = Finalizer::new(exit_.clone());
let mut prev_slot = None;
// Loop through blocktree MAX_ENTRY_RECV_PER_ITER entries at a time for each
// relevant slot to see if there are any available updates
loop {
let now = Instant::now();
// Stop getting entries if we get exit signal
if exit_.load(Ordering::Relaxed) {
break;
}
Self::generate_new_bank_forks(&blocktree, &mut bank_forks.write().unwrap());
let live_bank_ids = bank_forks.read().unwrap().active_banks();
trace!("live banks {:?}", live_bank_ids);
let mut votable: Vec<u64> = vec![];
for bank_id in live_bank_ids {
let bank = bank_forks.read().unwrap().get(bank_id).unwrap().clone();
if !Self::is_tpu(&bank, my_id) {
Self::replay_blocktree_into_bank(
&bank,
&blocktree,
&mut progress,
&forward_entry_sender,
)?;
}
let max_tick_height = (bank_id + 1) * bank.ticks_per_slot() - 1;
if bank.tick_height() == max_tick_height {
bank.freeze();
votable.push(bank_id);
progress.remove(&bank_id);
let id = leader_schedule_utils::slot_leader_at(bank.slot(), &bank);
if let Err(e) = slot_full_sender.send((bank.slot(), id)) {
info!("{} slot_full alert failed: {:?}", my_id, e);
}
}
}
// TODO: fork selection
// vote on the latest one for now
votable.sort();
if let Some(latest_slot_vote) = votable.last() {
let parent = bank_forks
.read()
.unwrap()
.get(*latest_slot_vote)
.unwrap()
.clone();
let next_slot = *latest_slot_vote + 1;
let next_leader = leader_schedule_utils::slot_leader_at(next_slot, &parent);
cluster_info.write().unwrap().set_leader(next_leader);
subscriptions.notify_subscribers(&parent);
if let Some(ref voting_keypair) = voting_keypair {
let keypair = voting_keypair.as_ref();
let vote = VoteTransaction::new_vote(
keypair,
*latest_slot_vote,
parent.last_id(),
0,
);
cluster_info.write().unwrap().push_vote(vote);
}
if next_leader == my_id {
let tpu_bank = Bank::new_from_parent(&parent, my_id, next_slot);
bank_forks.write().unwrap().insert(next_slot, tpu_bank);
}
debug!(
"to_leader_sender: me: {} next_slot: {} next_leader: {}",
my_id, next_slot, next_leader
);
to_leader_sender.send(TvuRotationInfo {
tick_height: parent.tick_height(),
last_id: parent.last_id(),
slot: next_slot,
leader_id: next_leader,
})?;
}
inc_new_counter_info!(
"replicate_stage-duration",
duration_as_ms(&now.elapsed()) as usize
);
let timer = Duration::from_millis(100);
let e = ledger_signal_receiver.recv_timeout(timer);
match e {
let result = ledger_signal_receiver.recv_timeout(timer);
match result {
Err(RecvTimeoutError::Timeout) => continue,
Err(_) => break,
Ok(_) => (),
Ok(_) => debug!("blocktree signal"),
};
if current_slot.is_none() {
let new_slot = Self::get_next_slot(
&blocktree,
prev_slot.expect("prev_slot must exist"),
);
if new_slot.is_some() {
trace!("{} replay_stage: new_slot found: {:?}", my_id, new_slot);
// Reset the state
bank = Self::create_and_set_working_bank(
&bank,
current_leader_id,
new_slot.unwrap(),
&bank_forks,
);
current_slot = new_slot;
Self::reset_state(
bank.ticks_per_slot(),
current_slot.unwrap(),
&mut max_tick_height_for_slot,
&mut current_blob_index,
);
} else {
continue;
}
}
// current_slot must be Some(x) by this point
let slot = current_slot.unwrap();
// Fetch the next entries from the database
let entries = {
if current_leader_id != my_id {
info!(
"{} replay_stage: asking for entries from slot: {}, bi: {}",
my_id, slot, current_blob_index
);
if let Ok(entries) = blocktree.get_slot_entries(
slot,
current_blob_index,
Some(MAX_ENTRY_RECV_PER_ITER as u64),
) {
entries
} else {
vec![]
}
} else {
vec![]
}
};
if !entries.is_empty() {
if let Err(e) = Self::process_entries(
entries,
&bank,
&cluster_info,
&voting_keypair,
&forward_entry_sender,
&mut current_blob_index,
&mut last_entry_hash,
&subscriptions_,
) {
error!("{} process_entries failed: {:?}", my_id, e);
}
}
let current_tick_height = bank.tick_height();
// We've reached the end of a slot, reset our state and check
// for leader rotation
if max_tick_height_for_slot == current_tick_height {
if let Err(e) = slot_full_sender.send((slot, current_leader_id)) {
error!("{} slot_full alert failed: {:?}", my_id, e);
}
// Check for leader rotation
let (leader_id, next_slot) = {
let slot = (current_tick_height + 1) / bank.ticks_per_slot();
(leader_schedule_utils::slot_leader_at(slot, &bank), slot)
};
// If we were the leader for the last slot update the last id b/c we
// haven't processed any of the entries for the slot for which we were
// the leader
if current_leader_id == my_id {
let meta = blocktree.meta(slot).unwrap().expect("meta has to exist");
if meta.last_index == std::u64::MAX {
// Ledger hasn't gotten last blob yet, break and wait
// for a signal
continue;
}
let last_entry = blocktree
.get_slot_entries(slot, meta.last_index, Some(1))
.unwrap();
last_entry_hash = last_entry[0].hash;
}
let old_bank = bank.clone();
prev_slot = current_slot;
if my_id == leader_id {
// Create new bank for next slot if we are the leader for that slot
bank = Self::create_and_set_working_bank(
&old_bank,
leader_id,
next_slot,
&bank_forks,
);
current_slot = Some(next_slot);
Self::reset_state(
bank.ticks_per_slot(),
next_slot,
&mut max_tick_height_for_slot,
&mut current_blob_index,
);
} else {
current_slot = None;
}
if leader_id != current_leader_id {
// TODO: Remove this soon once we boot the leader from ClusterInfo
cluster_info.write().unwrap().set_leader(leader_id);
}
trace!(
"node {:?} scheduled as leader for slot {}",
leader_id,
next_slot
);
// Always send rotation signal so that other services like
// RPC can be made aware of last slot's bank
to_leader_sender
.send(TvuRotationInfo {
bank: bank.clone(),
slot: next_slot,
leader_id,
})
.unwrap();
// Check for any slots that chain to this one
current_leader_id = leader_id;
continue;
}
}
Ok(())
})
.unwrap();
(
Self { t_replay, exit },
slot_full_receiver,
@ -398,6 +179,60 @@ impl ReplayStage {
)
}
pub fn replay_blocktree_into_bank(
bank: &Bank,
blocktree: &Blocktree,
progress: &mut HashMap<u64, (Hash, usize)>,
forward_entry_sender: &EntrySender,
) -> result::Result<()> {
let (entries, num) = Self::load_blocktree_entries(bank, blocktree, progress)?;
let len = entries.len();
let result =
Self::replay_entries_into_bank(bank, entries, progress, forward_entry_sender, num);
if result.is_ok() {
trace!("verified entries {}", len);
inc_new_counter_info!("replicate-stage_process_entries", len);
} else {
info!("debug to verify entries {}", len);
//TODO: mark this fork as failed
inc_new_counter_info!("replicate-stage_failed_process_entries", len);
}
Ok(())
}
pub fn load_blocktree_entries(
bank: &Bank,
blocktree: &Blocktree,
progress: &mut HashMap<u64, (Hash, usize)>,
) -> result::Result<(Vec<Entry>, usize)> {
let bank_id = bank.slot();
let bank_progress = &mut progress.entry(bank_id).or_insert((bank.last_id(), 0));
blocktree.get_slot_entries_with_blob_count(bank_id, bank_progress.1 as u64, None)
}
pub fn replay_entries_into_bank(
bank: &Bank,
entries: Vec<Entry>,
progress: &mut HashMap<u64, (Hash, usize)>,
forward_entry_sender: &EntrySender,
num: usize,
) -> result::Result<()> {
let bank_progress = &mut progress.entry(bank.slot()).or_insert((bank.last_id(), 0));
let result = Self::verify_and_process_entries(&bank, &entries, &bank_progress.0);
bank_progress.1 += num;
if let Some(last_entry) = entries.last() {
bank_progress.0 = last_entry.hash;
}
if result.is_ok() {
forward_entry_sender.send(entries)?;
}
result
}
pub fn is_tpu(bank: &Bank, my_id: Pubkey) -> bool {
my_id == leader_schedule_utils::slot_leader(&bank)
}
pub fn close(self) -> thread::Result<()> {
self.exit();
self.join()
@ -407,44 +242,51 @@ impl ReplayStage {
self.exit.store(true, Ordering::Relaxed);
}
fn create_and_set_working_bank(
parent: &Arc<Bank>,
leader_id: Pubkey,
slot: u64,
bank_forks: &Arc<RwLock<BankForks>>,
) -> Arc<Bank> {
let new_bank = Bank::new_from_parent(&parent, leader_id, slot);
new_bank.squash();
let mut bank_forks = bank_forks.write().unwrap();
bank_forks.insert(slot, new_bank);
bank_forks[slot].clone()
pub fn verify_and_process_entries(
bank: &Bank,
entries: &[Entry],
last_entry: &Hash,
) -> result::Result<()> {
if !entries.verify(last_entry) {
trace!(
"entry verification failed {} {} {} {}",
entries.len(),
bank.tick_height(),
last_entry,
bank.last_id()
);
return Err(result::Error::BlobError(BlobError::VerificationFailed));
}
blocktree_processor::process_entries(bank, entries)?;
Ok(())
}
fn reset_state(
ticks_per_slot: u64,
slot: u64,
max_tick_height_for_slot: &mut u64,
current_blob_index: &mut u64,
) {
*current_blob_index = 0;
*max_tick_height_for_slot = (slot + 1) * ticks_per_slot - 1;
}
fn get_next_slot(blocktree: &Blocktree, slot_index: u64) -> Option<u64> {
fn generate_new_bank_forks(blocktree: &Blocktree, forks: &mut BankForks) {
// Find the next slot that chains to the old slot
let next_slots = blocktree.get_slots_since(&[slot_index]).expect("Db error");
next_slots
.values()
.next()
.map(|slots| {
if slots.is_empty() {
None
} else {
Some(slots[0])
let frozen_banks = forks.frozen_banks();
let frozen_bank_ids: Vec<u64> = frozen_banks.keys().cloned().collect();
trace!("generate new forks {:?}", frozen_bank_ids);
let next_slots = blocktree
.get_slots_since(&frozen_bank_ids)
.expect("Db error");
for (parent_id, children) in next_slots {
let parent_bank = frozen_banks
.get(&parent_id)
.expect("missing parent in bank forks")
.clone();
for child_id in children {
let new_fork = forks.get(child_id).is_none();
if new_fork {
let leader = leader_schedule_utils::slot_leader_at(child_id, &parent_bank);
trace!("new fork:{} parent:{}", child_id, parent_id);
forks.insert(
child_id,
Bank::new_from_parent(&parent_bank, leader, child_id),
);
}
})
.unwrap_or(None)
}
}
}
}
@ -452,7 +294,7 @@ impl Service for ReplayStage {
type JoinReturnType = ();
fn join(self) -> thread::Result<()> {
self.t_replay.join()
self.t_replay.join().map(|_| ())
}
}
@ -465,6 +307,7 @@ mod test {
use crate::entry::{next_entry_mut, Entry};
use crate::fullnode::new_banks_from_blocktree;
use crate::replay_stage::ReplayStage;
use crate::result::Error;
use solana_sdk::genesis_block::GenesisBlock;
use solana_sdk::hash::Hash;
use solana_sdk::signature::{Keypair, KeypairUtil};
@ -475,6 +318,7 @@ mod test {
#[test]
fn test_vote_error_replay_stage_correctness() {
solana_logger::setup();
// Set up dummy node to host a ReplayStage
let my_keypair = Keypair::new();
let my_id = my_keypair.pubkey();
@ -498,7 +342,6 @@ mod test {
let (bank_forks, bank_forks_info, blocktree, l_receiver) =
new_banks_from_blocktree(&my_ledger_path, None);
let bank = bank_forks.working_bank();
let last_entry_hash = bank.last_id();
let blocktree = Arc::new(blocktree);
let (replay_stage, _slot_full_receiver, ledger_writer_recv) = ReplayStage::new(
@ -519,7 +362,7 @@ mod test {
cluster_info_me.write().unwrap().push_vote(vote);
info!("Send ReplayStage an entry, should see it on the ledger writer receiver");
let next_tick = create_ticks(1, last_entry_hash);
let next_tick = create_ticks(1, bank.last_id());
blocktree.write_entries(1, 0, 0, next_tick.clone()).unwrap();
let received_tick = ledger_writer_recv
@ -536,58 +379,51 @@ mod test {
}
#[test]
fn test_replay_stage_poh_error_entry_receiver() {
// Set up dummy node to host a ReplayStage
let my_keypair = Keypair::new();
let my_id = my_keypair.pubkey();
let my_node = Node::new_localhost_with_pubkey(my_id);
// Set up the cluster info
let cluster_info_me = Arc::new(RwLock::new(ClusterInfo::new(my_node.info.clone())));
let (forward_entry_sender, _forward_entry_receiver) = channel();
let mut last_entry_hash = Hash::default();
let mut current_blob_index = 0;
let mut last_id = Hash::default();
fn test_replay_stage_poh_ok_entry_receiver() {
let (forward_entry_sender, forward_entry_receiver) = channel();
let genesis_block = GenesisBlock::new(10_000).0;
let bank = Arc::new(Bank::new(&genesis_block));
let mut last_id = bank.last_id();
let mut entries = Vec::new();
for _ in 0..5 {
let entry = next_entry_mut(&mut last_id, 1, vec![]); //just ticks
entries.push(entry);
}
let genesis_block = GenesisBlock::new(10_000).0;
let bank = Arc::new(Bank::new(&genesis_block));
let voting_keypair = Some(Arc::new(Keypair::new()));
let res = ReplayStage::process_entries(
entries.clone(),
let mut progress = HashMap::new();
let res = ReplayStage::replay_entries_into_bank(
&bank,
&cluster_info_me,
&voting_keypair,
entries.clone(),
&mut progress,
&forward_entry_sender,
&mut current_blob_index,
&mut last_entry_hash,
&Arc::new(RpcSubscriptions::default()),
0,
);
assert!(res.is_ok(), "replay failed {:?}", res);
let res = forward_entry_receiver.try_recv();
match res {
Ok(_) => (),
Err(e) => assert!(false, "Entries were not sent correctly {:?}", e),
}
}
entries.clear();
#[test]
fn test_replay_stage_poh_error_entry_receiver() {
let (forward_entry_sender, forward_entry_receiver) = channel();
let mut entries = Vec::new();
for _ in 0..5 {
let entry = Entry::new(&mut Hash::default(), 1, vec![]); //just broken entries
entries.push(entry);
}
let genesis_block = GenesisBlock::new(10_000).0;
let bank = Arc::new(Bank::new(&genesis_block));
let res = ReplayStage::process_entries(
entries.clone(),
let mut progress = HashMap::new();
let res = ReplayStage::replay_entries_into_bank(
&bank,
&cluster_info_me,
&voting_keypair,
entries.clone(),
&mut progress,
&forward_entry_sender,
&mut current_blob_index,
&mut last_entry_hash,
&Arc::new(RpcSubscriptions::default()),
0,
);
match res {
@ -599,5 +435,6 @@ mod test {
e
),
}
assert!(forward_entry_receiver.try_recv().is_err());
}
}

5
src/tvu.rs
View File

@ -23,7 +23,7 @@ use crate::retransmit_stage::RetransmitStage;
use crate::rpc_subscriptions::RpcSubscriptions;
use crate::service::Service;
use crate::storage_stage::{StorageStage, StorageState};
use solana_runtime::bank::Bank;
use solana_sdk::hash::Hash;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::{Keypair, KeypairUtil};
use std::net::UdpSocket;
@ -33,7 +33,8 @@ use std::sync::{Arc, RwLock};
use std::thread;
pub struct TvuRotationInfo {
pub bank: Arc<Bank>, // Bank to use
pub tick_height: u64, // tick height, bank might not exist yet
pub last_id: Hash, // last_id that was voted on
pub slot: u64, // slot height to initiate a rotation
pub leader_id: Pubkey, // leader upon rotation
}

1
tests/replicator.rs
View File

@ -267,6 +267,7 @@ fn test_replicator_startup_leader_hang() {
}
#[test]
#[ignore] //TODO: hangs, was passing because of bug in network code
fn test_replicator_startup_ledger_hang() {
solana_logger::setup();
info!("starting replicator test");