blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Adopt heaviest subtree fork choice rule (#10441) 

* Add HeaviestSubtreeForkChoice

* Make replay stage switch between two fork choice rules

Co-authored-by: Carl <carl@solana.com>
This commit is contained in:
carllin 2020-06-11 12:16:04 -07:00 committed by GitHub
parent 0510b6e336
commit 2e1d59ff85
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 1700 additions and 635 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

152
core/src/bank_weight_fork_choice.rs Normal file
View File

@ -0,0 +1,152 @@
use crate::{
consensus::{ComputedBankState, Tower},
fork_choice::ForkChoice,
progress_map::{ForkStats, ProgressMap},
};
use solana_ledger::bank_forks::BankForks;
use solana_runtime::bank::Bank;
use solana_sdk::timing;
use std::time::Instant;
use std::{
collections::{HashMap, HashSet},
sync::{Arc, RwLock},
};
#[derive(Default)]
pub struct BankWeightForkChoice {}
impl ForkChoice for BankWeightForkChoice {
fn compute_bank_stats(
&mut self,
bank: &Bank,
_tower: &Tower,
progress: &mut ProgressMap,
computed_bank_stats: &ComputedBankState,
) {
let bank_slot = bank.slot();
// Only time progress map should be missing a bank slot
// is if this node was the leader for this slot as those banks
// are not replayed in replay_active_banks()
let parent_weight = bank
.parent()
.and_then(|b| progress.get(&b.slot()))
.map(|x| x.fork_stats.fork_weight)
.unwrap_or(0);
let stats = progress
.get_fork_stats_mut(bank_slot)
.expect("All frozen banks must exist in the Progress map");
let ComputedBankState { bank_weight, .. } = computed_bank_stats;
stats.weight = *bank_weight;
stats.fork_weight = stats.weight + parent_weight;
}
// Returns:
// 1) The heaviest overall bank
// 2) The heavest bank on the same fork as the last vote (doesn't require a
// switching proof to vote for)
fn select_forks(
&self,
frozen_banks: &[Arc<Bank>],
tower: &Tower,
progress: &ProgressMap,
ancestors: &HashMap<u64, HashSet<u64>>,
_bank_forks: &RwLock<BankForks>,
) -> (Arc<Bank>, Option<Arc<Bank>>) {
let tower_start = Instant::now();
assert!(!frozen_banks.is_empty());
let num_frozen_banks = frozen_banks.len();
trace!("frozen_banks {}", frozen_banks.len());
let num_old_banks = frozen_banks
.iter()
.filter(|b| b.slot() < tower.root().unwrap_or(0))
.count();
let last_vote = tower.last_vote().slots.last().cloned();
let mut heaviest_bank_on_same_fork = None;
let mut heaviest_same_fork_weight = 0;
let stats: Vec<&ForkStats> = frozen_banks
.iter()
.map(|bank| {
// Only time progress map should be missing a bank slot
// is if this node was the leader for this slot as those banks
// are not replayed in replay_active_banks()
let stats = progress
.get_fork_stats(bank.slot())
.expect("All frozen banks must exist in the Progress map");
if let Some(last_vote) = last_vote {
if ancestors
.get(&bank.slot())
.expect("Entry in frozen banks must exist in ancestors")
.contains(&last_vote)
{
// Descendant of last vote cannot be locked out
assert!(!stats.is_locked_out);
// ancestors(slot) should not contain the slot itself,
// so we should never get the same bank as the last vote
assert_ne!(bank.slot(), last_vote);
// highest weight, lowest slot first. frozen_banks is sorted
// from least slot to greatest slot, so if two banks have
// the same fork weight, the lower slot will be picked
if stats.fork_weight > heaviest_same_fork_weight {
heaviest_bank_on_same_fork = Some(bank.clone());
heaviest_same_fork_weight = stats.fork_weight;
}
}
}
stats
})
.collect();
let num_not_recent = stats.iter().filter(|s| !s.is_recent).count();
let num_has_voted = stats.iter().filter(|s| s.has_voted).count();
let num_empty = stats.iter().filter(|s| s.is_empty).count();
let num_threshold_failure = stats.iter().filter(|s| !s.vote_threshold).count();
let num_votable_threshold_failure = stats
.iter()
.filter(|s| s.is_recent && !s.has_voted && !s.vote_threshold)
.count();
let mut candidates: Vec<_> = frozen_banks.iter().zip(stats.iter()).collect();
//highest weight, lowest slot first
candidates.sort_by_key(|b| (b.1.fork_weight, 0i64 - b.0.slot() as i64));
let rv = candidates
.last()
.expect("frozen banks was nonempty so candidates must also be nonempty");
let ms = timing::duration_as_ms(&tower_start.elapsed());
let weights: Vec<(u128, u64, u64)> = candidates
.iter()
.map(|x| (x.1.weight, x.0.slot(), x.1.block_height))
.collect();
debug!(
"@{:?} tower duration: {:?} len: {}/{} weights: {:?}",
timing::timestamp(),
ms,
candidates.len(),
stats.iter().filter(|s| !s.has_voted).count(),
weights,
);
datapoint_debug!(
"replay_stage-select_forks",
("frozen_banks", num_frozen_banks as i64, i64),
("not_recent", num_not_recent as i64, i64),
("has_voted", num_has_voted as i64, i64),
("old_banks", num_old_banks as i64, i64),
("empty_banks", num_empty as i64, i64),
("threshold_failure", num_threshold_failure as i64, i64),
(
"votable_threshold_failure",
num_votable_threshold_failure as i64,
i64
),
("tower_duration", ms as i64, i64),
);
(rv.0.clone(), heaviest_bank_on_same_fork)
}
}

281
core/src/consensus.rs
View File

@ -21,7 +21,7 @@ use solana_vote_program::{
use std::{
collections::{BTreeMap, HashMap, HashSet},
ops::Bound::{Included, Unbounded},
sync::Arc,
sync::{Arc, RwLock},
};
#[derive(PartialEq, Clone, Debug)]
@ -79,6 +79,14 @@ impl StakeLockout {
}
}
pub(crate) struct ComputedBankState {
pub stake_lockouts: HashMap<Slot, StakeLockout>,
pub total_staked: u64,
pub bank_weight: u128,
pub lockout_intervals: LockoutIntervals,
pub pubkey_votes: Vec<(Pubkey, Slot)>,
}
pub struct Tower {
node_pubkey: Pubkey,
threshold_depth: usize,
@ -102,10 +110,14 @@ impl Default for Tower {
}
impl Tower {
pub fn new(node_pubkey: &Pubkey, vote_account_pubkey: &Pubkey, bank_forks: &BankForks) -> Self {
pub fn new(
node_pubkey: &Pubkey,
vote_account_pubkey: &Pubkey,
root: Slot,
heaviest_bank: &Bank,
) -> Self {
let mut tower = Self::new_with_key(node_pubkey);
tower.initialize_lockouts_from_bank_forks(&bank_forks, vote_account_pubkey);
tower.initialize_lockouts_from_bank_forks(vote_account_pubkey, root, heaviest_bank);
tower
}
@ -126,27 +138,28 @@ impl Tower {
}
}
pub fn collect_vote_lockouts<F>(
&self,
pub(crate) fn collect_vote_lockouts<F>(
node_pubkey: &Pubkey,
bank_slot: u64,
vote_accounts: F,
ancestors: &HashMap<Slot, HashSet<u64>>,
all_pubkeys: &mut PubkeyReferences,
) -> (HashMap<Slot, StakeLockout>, u64, u128, LockoutIntervals)
) -> ComputedBankState
where
F: Iterator<Item = (Pubkey, (u64, Account))>,
{
let mut stake_lockouts = HashMap::new();
let mut total_stake = 0;
let mut total_weight = 0;
let mut total_staked = 0;
let mut bank_weight = 0;
// Tree of intervals of lockouts of the form [slot, slot + slot.lockout],
// keyed by end of the range
let mut lockout_intervals = BTreeMap::new();
let mut pubkey_votes = vec![];
for (key, (lamports, account)) in vote_accounts {
if lamports == 0 {
continue;
}
trace!("{} {} with stake {}", self.node_pubkey, key, lamports);
trace!("{} {} with stake {}", node_pubkey, key, lamports);
let vote_state = VoteState::from(&account);
if vote_state.is_none() {
datapoint_warn!(
@ -169,7 +182,7 @@ impl Tower {
.push((vote.slot, key));
}
if key == self.node_pubkey || vote_state.node_pubkey == self.node_pubkey {
if key == *node_pubkey || vote_state.node_pubkey == *node_pubkey {
debug!("vote state {:?}", vote_state);
debug!(
"observed slot {}",
@ -188,10 +201,15 @@ impl Tower {
}
let start_root = vote_state.root_slot;
// Add the latest vote to update the `heaviest_subtree_fork_choice`
if let Some(latest_vote) = vote_state.votes.back() {
pubkey_votes.push((key, latest_vote.slot));
}
vote_state.process_slot_vote_unchecked(bank_slot);
for vote in &vote_state.votes {
total_weight += vote.lockout() as u128 * lamports as u128;
bank_weight += vote.lockout() as u128 * lamports as u128;
Self::update_ancestor_lockouts(&mut stake_lockouts, &vote, ancestors);
}
@ -202,7 +220,7 @@ impl Tower {
slot: root,
};
trace!("ROOT: {}", vote.slot);
total_weight += vote.lockout() as u128 * lamports as u128;
bank_weight += vote.lockout() as u128 * lamports as u128;
Self::update_ancestor_lockouts(&mut stake_lockouts, &vote, ancestors);
}
}
@ -211,7 +229,7 @@ impl Tower {
confirmation_count: MAX_LOCKOUT_HISTORY as u32,
slot: root,
};
total_weight += vote.lockout() as u128 * lamports as u128;
bank_weight += vote.lockout() as u128 * lamports as u128;
Self::update_ancestor_lockouts(&mut stake_lockouts, &vote, ancestors);
}
@ -232,9 +250,16 @@ impl Tower {
// Update all the parents of this last vote with the stake of this vote account
Self::update_ancestor_stakes(&mut stake_lockouts, vote.slot, lamports, ancestors);
}
total_stake += lamports;
total_staked += lamports;
}
ComputedBankState {
stake_lockouts,
total_staked,
bank_weight,
lockout_intervals,
pubkey_votes,
}
(stake_lockouts, total_stake, total_weight, lockout_intervals)
}
pub fn is_slot_confirmed(
@ -509,7 +534,7 @@ impl Tower {
}
/// Update lockouts for all the ancestors
fn update_ancestor_lockouts(
pub(crate) fn update_ancestor_lockouts(
stake_lockouts: &mut HashMap<Slot, StakeLockout>,
vote: &Lockout,
ancestors: &HashMap<Slot, HashSet<Slot>>,
@ -529,6 +554,28 @@ impl Tower {
}
}
pub(crate) fn find_heaviest_bank(
bank_forks: &RwLock<BankForks>,
node_pubkey: &Pubkey,
) -> Option<Arc<Bank>> {
let ancestors = bank_forks.read().unwrap().ancestors();
let mut bank_weights: Vec<_> = bank_forks
.read()
.unwrap()
.frozen_banks()
.values()
.map(|b| {
(
Self::bank_weight(node_pubkey, b, &ancestors),
b.parents().len(),
b.clone(),
)
})
.collect();
bank_weights.sort_by_key(|b| (b.0, b.1));
bank_weights.pop().map(|b| b.2)
}
/// Update stake for all the ancestors.
/// Note, stake is the same for all the ancestor.
fn update_ancestor_stakes(
@ -537,9 +584,8 @@ impl Tower {
lamports: u64,
ancestors: &HashMap<Slot, HashSet<Slot>>,
) {
// If there's no ancestors, that means this slot must be from before the current root,
// in which case the lockouts won't be calculated in bank_weight anyways, so ignore
// this slot
// If there's no ancestors, that means this slot must be from
// before the current root, so ignore this slot
let vote_slot_ancestors = ancestors.get(&slot);
if vote_slot_ancestors.is_none() {
return;
@ -552,8 +598,13 @@ impl Tower {
}
}
fn bank_weight(&self, bank: &Bank, ancestors: &HashMap<Slot, HashSet<Slot>>) -> u128 {
let (_, _, bank_weight, _) = self.collect_vote_lockouts(
fn bank_weight(
node_pubkey: &Pubkey,
bank: &Bank,
ancestors: &HashMap<Slot, HashSet<Slot>>,
) -> u128 {
let ComputedBankState { bank_weight, .. } = Self::collect_vote_lockouts(
node_pubkey,
bank.slot(),
bank.vote_accounts().into_iter(),
ancestors,
@ -562,31 +613,14 @@ impl Tower {
bank_weight
}
fn find_heaviest_bank(&self, bank_forks: &BankForks) -> Option<Arc<Bank>> {
let ancestors = bank_forks.ancestors();
let mut bank_weights: Vec<_> = bank_forks
.frozen_banks()
.values()
.map(|b| {
(
self.bank_weight(b, &ancestors),
b.parents().len(),
b.clone(),
)
})
.collect();
bank_weights.sort_by_key(|b| (b.0, b.1));
bank_weights.pop().map(|b| b.2)
}
fn initialize_lockouts_from_bank_forks(
&mut self,
bank_forks: &BankForks,
vote_account_pubkey: &Pubkey,
root: Slot,
heaviest_bank: &Bank,
) {
if let Some(bank) = self.find_heaviest_bank(bank_forks) {
let root = bank_forks.root();
if let Some((_stake, vote_account)) = bank.vote_accounts().get(vote_account_pubkey) {
if let Some((_stake, vote_account)) = heaviest_bank.vote_accounts().get(vote_account_pubkey)
{
let mut vote_state = VoteState::deserialize(&vote_account.data)
.expect("vote_account isn't a VoteState?");
vote_state.root_slot = Some(root);
@ -596,7 +630,6 @@ impl Tower {
self.node_pubkey,
vote_state
);
assert_eq!(
vote_state.node_pubkey, self.node_pubkey,
"vote account's node_pubkey doesn't match",
@ -604,7 +637,6 @@ impl Tower {
self.lockouts = vote_state;
}
}
}
fn maybe_timestamp(&mut self, current_slot: Slot) -> Option<UnixTimestamp> {
if self.last_timestamp.slot == 0
@ -626,10 +658,13 @@ impl Tower {
pub mod test {
use super::*;
use crate::{
bank_weight_fork_choice::BankWeightForkChoice,
cluster_info_vote_listener::VoteTracker,
cluster_slots::ClusterSlots,
fork_choice::SelectVoteAndResetForkResult,
heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice,
progress_map::ForkProgress,
replay_stage::{HeaviestForkFailures, ReplayStage, SelectVoteAndResetForkResult},
replay_stage::{HeaviestForkFailures, ReplayStage},
};
use solana_ledger::bank_forks::BankForks;
use solana_runtime::{
@ -645,7 +680,7 @@ pub mod test {
signature::{Keypair, Signer},
};
use solana_vote_program::{
vote_state::{Vote, VoteStateVersions},
vote_state::{Vote, VoteStateVersions, MAX_LOCKOUT_HISTORY},
vote_transaction,
};
use std::{
@ -662,18 +697,26 @@ pub mod test {
pub vote_pubkeys: Vec<Pubkey>,
pub bank_forks: RwLock<BankForks>,
pub progress: ProgressMap,
pub heaviest_subtree_fork_choice: HeaviestSubtreeForkChoice,
}
impl VoteSimulator {
pub(crate) fn new(num_keypairs: usize) -> Self {
let (validator_keypairs, node_pubkeys, vote_pubkeys, bank_forks, progress) =
Self::init_state(num_keypairs);
let (
validator_keypairs,
node_pubkeys,
vote_pubkeys,
bank_forks,
progress,
heaviest_subtree_fork_choice,
) = Self::init_state(num_keypairs);
Self {
validator_keypairs,
node_pubkeys,
vote_pubkeys,
bank_forks: RwLock::new(bank_forks),
progress,
heaviest_subtree_fork_choice,
}
}
pub(crate) fn fill_bank_forks(
@ -716,6 +759,8 @@ pub mod test {
}
}
new_bank.freeze();
self.heaviest_subtree_fork_choice
.add_new_leaf_slot(new_bank.slot(), Some(new_bank.parent_slot()));
self.bank_forks.write().unwrap().insert(new_bank);
walk.forward();
}
@ -750,6 +795,8 @@ pub mod test {
&ClusterSlots::default(),
&self.bank_forks,
&mut PubkeyReferences::default(),
&mut self.heaviest_subtree_fork_choice,
&mut BankWeightForkChoice::default(),
);
let vote_bank = self
@ -766,7 +813,7 @@ pub mod test {
heaviest_fork_failures,
..
} = ReplayStage::select_vote_and_reset_forks(
&Some(vote_bank.clone()),
&vote_bank.clone(),
&None,
&ancestors,
&descendants,
@ -795,6 +842,7 @@ pub mod test {
&None,
&mut PubkeyReferences::default(),
None,
&mut self.heaviest_subtree_fork_choice,
)
}
@ -880,6 +928,7 @@ pub mod test {
Vec<Pubkey>,
BankForks,
ProgressMap,
HeaviestSubtreeForkChoice,
) {
let keypairs: HashMap<_, _> = std::iter::repeat_with(|| {
let node_keypair = Keypair::new();
@ -902,8 +951,16 @@ pub mod test {
.map(|keys| keys.vote_keypair.pubkey())
.collect();
let (bank_forks, progress) = initialize_state(&keypairs, 10_000);
(keypairs, node_pubkeys, vote_pubkeys, bank_forks, progress)
let (bank_forks, progress, heaviest_subtree_fork_choice) =
initialize_state(&keypairs, 10_000);
(
keypairs,
node_pubkeys,
vote_pubkeys,
bank_forks,
progress,
heaviest_subtree_fork_choice,
)
}
}
@ -911,7 +968,7 @@ pub mod test {
pub(crate) fn initialize_state(
validator_keypairs_map: &HashMap<Pubkey, ValidatorVoteKeypairs>,
stake: u64,
) -> (BankForks, ProgressMap) {
) -> (BankForks, ProgressMap, HeaviestSubtreeForkChoice) {
let validator_keypairs: Vec<_> = validator_keypairs_map.values().collect();
let GenesisConfigInfo {
genesis_config,
@ -931,7 +988,10 @@ pub mod test {
0,
ForkProgress::new(bank0.last_blockhash(), None, None, 0, 0),
);
(BankForks::new(0, bank0), progress)
let bank_forks = BankForks::new(0, bank0);
let heaviest_subtree_fork_choice =
HeaviestSubtreeForkChoice::new_from_bank_forks(&bank_forks);
(bank_forks, progress, heaviest_subtree_fork_choice)
}
fn gen_stakes(stake_votes: &[(u64, &[u64])]) -> Vec<(Pubkey, (u64, Account))> {
@ -1291,20 +1351,32 @@ pub mod test {
#[test]
fn test_collect_vote_lockouts_sums() {
//two accounts voting for slot 0 with 1 token staked
let accounts = gen_stakes(&[(1, &[0]), (1, &[0])]);
let tower = Tower::new_for_tests(0, 0.67);
let mut accounts = gen_stakes(&[(1, &[0]), (1, &[0])]);
accounts.sort_by_key(|(pk, _)| *pk);
let account_latest_votes: Vec<(Pubkey, Slot)> =
accounts.iter().map(|(pubkey, _)| (*pubkey, 0)).collect();
let ancestors = vec![(1, vec![0].into_iter().collect()), (0, HashSet::new())]
.into_iter()
.collect();
let (staked_lockouts, total_staked, bank_weight, _) = tower.collect_vote_lockouts(
let ComputedBankState {
stake_lockouts,
total_staked,
bank_weight,
mut pubkey_votes,
..
} = Tower::collect_vote_lockouts(
&Pubkey::default(),
1,
accounts.into_iter(),
&ancestors,
&mut PubkeyReferences::default(),
);
assert_eq!(staked_lockouts[&0].stake, 2);
assert_eq!(staked_lockouts[&0].lockout, 2 + 2 + 4 + 4);
assert_eq!(stake_lockouts[&0].stake, 2);
assert_eq!(stake_lockouts[&0].lockout, 2 + 2 + 4 + 4);
assert_eq!(total_staked, 2);
pubkey_votes.sort();
assert_eq!(pubkey_votes, account_latest_votes);
// Each acccount has 1 vote in it. After simulating a vote in collect_vote_lockouts,
// the account will have 2 votes, with lockout 2 + 4 = 6. So expected weight for
@ -1316,7 +1388,12 @@ pub mod test {
fn test_collect_vote_lockouts_root() {
let votes: Vec<u64> = (0..MAX_LOCKOUT_HISTORY as u64).collect();
//two accounts voting for slots 0..MAX_LOCKOUT_HISTORY with 1 token staked
let accounts = gen_stakes(&[(1, &votes), (1, &votes)]);
let mut accounts = gen_stakes(&[(1, &votes), (1, &votes)]);
accounts.sort_by_key(|(pk, _)| *pk);
let account_latest_votes: Vec<(Pubkey, Slot)> = accounts
.iter()
.map(|(pubkey, _)| (*pubkey, (MAX_LOCKOUT_HISTORY - 1) as Slot))
.collect();
let mut tower = Tower::new_for_tests(0, 0.67);
let mut ancestors = HashMap::new();
for i in 0..(MAX_LOCKOUT_HISTORY + 1) {
@ -1337,18 +1414,27 @@ pub mod test {
+ root_weight;
let expected_bank_weight = 2 * vote_account_expected_weight;
assert_eq!(tower.lockouts.root_slot, Some(0));
let (staked_lockouts, _total_staked, bank_weight, _) = tower.collect_vote_lockouts(
let ComputedBankState {
stake_lockouts,
bank_weight,
mut pubkey_votes,
..
} = Tower::collect_vote_lockouts(
&Pubkey::default(),
MAX_LOCKOUT_HISTORY as u64,
accounts.into_iter(),
&ancestors,
&mut PubkeyReferences::default(),
);
for i in 0..MAX_LOCKOUT_HISTORY {
assert_eq!(staked_lockouts[&(i as u64)].stake, 2);
assert_eq!(stake_lockouts[&(i as u64)].stake, 2);
}
// should be the sum of all the weights for root
assert!(staked_lockouts[&0].lockout > (2 * (1 << MAX_LOCKOUT_HISTORY)));
assert!(stake_lockouts[&0].lockout > (2 * (1 << MAX_LOCKOUT_HISTORY)));
assert_eq!(bank_weight, expected_bank_weight);
pubkey_votes.sort();
assert_eq!(pubkey_votes, account_latest_votes);
}
#[test]
@ -1381,7 +1467,7 @@ pub mod test {
);
tower.record_vote(i, Hash::default());
}
assert!(!tower.check_vote_stake_threshold(MAX_LOCKOUT_HISTORY as u64 + 1, &stakes, 2));
assert!(!tower.check_vote_stake_threshold(MAX_LOCKOUT_HISTORY as u64 + 1, &stakes, 2,));
}
#[test]
@ -1609,48 +1695,7 @@ pub mod test {
tower.record_vote(0, Hash::default());
tower.record_vote(1, Hash::default());
tower.record_vote(2, Hash::default());
assert!(tower.check_vote_stake_threshold(6, &stakes, 2));
}
#[test]
fn test_lockout_is_updated_for_entire_branch() {
let mut stake_lockouts = HashMap::new();
let vote = Lockout {
slot: 2,
confirmation_count: 1,
};
let set: HashSet<u64> = vec![0u64, 1u64].into_iter().collect();
let mut ancestors = HashMap::new();
ancestors.insert(2, set);
let set: HashSet<u64> = vec![0u64].into_iter().collect();
ancestors.insert(1, set);
Tower::update_ancestor_lockouts(&mut stake_lockouts, &vote, &ancestors);
assert_eq!(stake_lockouts[&0].lockout, 2);
assert_eq!(stake_lockouts[&1].lockout, 2);
assert_eq!(stake_lockouts[&2].lockout, 2);
}
#[test]
fn test_lockout_is_updated_for_slot_or_lower() {
let mut stake_lockouts = HashMap::new();
let set: HashSet<u64> = vec![0u64, 1u64].into_iter().collect();
let mut ancestors = HashMap::new();
ancestors.insert(2, set);
let set: HashSet<u64> = vec![0u64].into_iter().collect();
ancestors.insert(1, set);
let vote = Lockout {
slot: 2,
confirmation_count: 1,
};
Tower::update_ancestor_lockouts(&mut stake_lockouts, &vote, &ancestors);
let vote = Lockout {
slot: 1,
confirmation_count: 2,
};
Tower::update_ancestor_lockouts(&mut stake_lockouts, &vote, &ancestors);
assert_eq!(stake_lockouts[&0].lockout, 2 + 4);
assert_eq!(stake_lockouts[&1].lockout, 2 + 4);
assert_eq!(stake_lockouts[&2].lockout, 2);
assert!(tower.check_vote_stake_threshold(6, &stakes, 2,));
}
#[test]
@ -1729,7 +1774,7 @@ pub mod test {
let total_stake = 4;
let threshold_size = 0.67;
let threshold_stake = (f64::ceil(total_stake as f64 * threshold_size)) as u64;
let tower_votes: Vec<u64> = (0..VOTE_THRESHOLD_DEPTH as u64).collect();
let tower_votes: Vec<Slot> = (0..VOTE_THRESHOLD_DEPTH as u64).collect();
let accounts = gen_stakes(&[
(threshold_stake, &[(VOTE_THRESHOLD_DEPTH - 2) as u64]),
(total_stake - threshold_stake, &tower_votes[..]),
@ -1746,29 +1791,35 @@ pub mod test {
for vote in &tower_votes {
tower.record_vote(*vote, Hash::default());
}
let (staked_lockouts, total_staked, _, _) = tower.collect_vote_lockouts(
let ComputedBankState {
stake_lockouts,
total_staked,
..
} = Tower::collect_vote_lockouts(
&Pubkey::default(),
vote_to_evaluate,
accounts.clone().into_iter(),
&ancestors,
&mut PubkeyReferences::default(),
);
assert!(tower.check_vote_stake_threshold(vote_to_evaluate, &staked_lockouts, total_staked));
assert!(tower.check_vote_stake_threshold(vote_to_evaluate, &stake_lockouts, total_staked,));
// CASE 2: Now we want to evaluate a vote for slot VOTE_THRESHOLD_DEPTH + 1. This slot
// will expire the vote in one of the vote accounts, so we should have insufficient
// stake to pass the threshold
let vote_to_evaluate = VOTE_THRESHOLD_DEPTH as u64 + 1;
let (staked_lockouts, total_staked, _, _) = tower.collect_vote_lockouts(
let ComputedBankState {
stake_lockouts,
total_staked,
..
} = Tower::collect_vote_lockouts(
&Pubkey::default(),
vote_to_evaluate,
accounts.into_iter(),
&ancestors,
&mut PubkeyReferences::default(),
);
assert!(!tower.check_vote_stake_threshold(
vote_to_evaluate,
&staked_lockouts,
total_staked
));
assert!(!tower.check_vote_stake_threshold(vote_to_evaluate, &stake_lockouts, total_staked,));
}
fn vote_and_check_recent(num_votes: usize) {

40
core/src/fork_choice.rs Normal file
View File

@ -0,0 +1,40 @@
use crate::{
consensus::{ComputedBankState, SwitchForkDecision, Tower},
progress_map::ProgressMap,
replay_stage::HeaviestForkFailures,
};
use solana_ledger::bank_forks::BankForks;
use solana_runtime::bank::Bank;
use std::{
collections::{HashMap, HashSet},
sync::{Arc, RwLock},
};
pub(crate) struct SelectVoteAndResetForkResult {
pub vote_bank: Option<(Arc<Bank>, SwitchForkDecision)>,
pub reset_bank: Option<Arc<Bank>>,
pub heaviest_fork_failures: Vec<HeaviestForkFailures>,
}
pub(crate) trait ForkChoice {
fn compute_bank_stats(
&mut self,
bank: &Bank,
tower: &Tower,
progress: &mut ProgressMap,
computed_bank_stats: &ComputedBankState,
);
// Returns:
// 1) The heaviest overall bbank
// 2) The heavest bank on the same fork as the last vote (doesn't require a
// switching proof to vote for)
fn select_forks(
&self,
frozen_banks: &[Arc<Bank>],
tower: &Tower,
progress: &ProgressMap,
ancestors: &HashMap<u64, HashSet<u64>>,
bank_forks: &RwLock<BankForks>,
) -> (Arc<Bank>, Option<Arc<Bank>>);
}

72
core/src/heaviest_subtree_fork_choice/fork_choice.rs Normal file
View File

@ -0,0 +1,72 @@
use crate::{
consensus::{ComputedBankState, Tower},
fork_choice::ForkChoice,
heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice,
progress_map::ProgressMap,
};
use solana_ledger::bank_forks::BankForks;
use solana_runtime::bank::Bank;
use std::{
collections::{HashMap, HashSet},
sync::{Arc, RwLock},
};
impl ForkChoice for HeaviestSubtreeForkChoice {
fn compute_bank_stats(
&mut self,
bank: &Bank,
_tower: &Tower,
_progress: &mut ProgressMap,
computed_bank_stats: &ComputedBankState,
) {
let ComputedBankState { pubkey_votes, .. } = computed_bank_stats;
// Update `heaviest_subtree_fork_choice` to find the best fork to build on
let best_overall_slot = self.add_votes(
&pubkey_votes,
bank.epoch_stakes_map(),
bank.epoch_schedule(),
);
datapoint_info!(
"best_slot",
("slot", bank.slot(), i64),
("best_slot", best_overall_slot, i64),
);
}
// Returns:
// 1) The heaviest overall bbank
// 2) The heavest bank on the same fork as the last vote (doesn't require a
// switching proof to vote for)
fn select_forks(
&self,
_frozen_banks: &[Arc<Bank>],
tower: &Tower,
_progress: &ProgressMap,
_ancestors: &HashMap<u64, HashSet<u64>>,
bank_forks: &RwLock<BankForks>,
) -> (Arc<Bank>, Option<Arc<Bank>>) {
let last_vote = tower.last_vote().slots.last().cloned();
let heaviest_slot_on_same_voted_fork = last_vote.map(|last_vote| {
let heaviest_slot_on_same_voted_fork =
self.best_slot(last_vote).expect("last_vote is a frozen bank so must have been added to heaviest_subtree_fork_choice at time of freezing");
if heaviest_slot_on_same_voted_fork == last_vote {
None
} else {
Some(heaviest_slot_on_same_voted_fork)
}
}).unwrap_or(None);
let heaviest_slot = self.best_overall_slot();
let r_bank_forks = bank_forks.read().unwrap();
(
r_bank_forks.get(heaviest_slot).unwrap().clone(),
heaviest_slot_on_same_voted_fork.map(|heaviest_slot_on_same_voted_fork| {
r_bank_forks
.get(heaviest_slot_on_same_voted_fork)
.unwrap()
.clone()
}),
)
}
}

1040
core/src/heaviest_subtree_fork_choice/mod.rs Normal file
View File

File diff suppressed because it is too large Load Diff

3
core/src/lib.rs
View File

@ -15,6 +15,7 @@ mod deprecated;
pub mod shred_fetch_stage;
#[macro_use]
pub mod contact_info;
pub mod bank_weight_fork_choice;
pub mod cluster_info;
pub mod cluster_slots;
pub mod consensus;
@ -26,8 +27,10 @@ pub mod crds_gossip_push;
pub mod crds_value;
pub mod epoch_slots;
pub mod fetch_stage;
pub mod fork_choice;
pub mod gen_keys;
pub mod gossip_service;
pub mod heaviest_subtree_fork_choice;
pub mod ledger_cleanup_service;
pub mod local_vote_signer_service;
pub mod non_circulating_supply;

648
core/src/replay_stage.rs
View File

@ -1,14 +1,17 @@
//! The `replay_stage` replays transactions broadcast by the leader.
use crate::{
bank_weight_fork_choice::BankWeightForkChoice,
broadcast_stage::RetransmitSlotsSender,
cluster_info::ClusterInfo,
cluster_info_vote_listener::VoteTracker,
cluster_slots::ClusterSlots,
commitment::{AggregateCommitmentService, BlockCommitmentCache, CommitmentAggregationData},
consensus::{StakeLockout, SwitchForkDecision, Tower},
consensus::{ComputedBankState, StakeLockout, SwitchForkDecision, Tower},
fork_choice::{ForkChoice, SelectVoteAndResetForkResult},
heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice,
poh_recorder::{PohRecorder, GRACE_TICKS_FACTOR, MAX_GRACE_SLOTS},
progress_map::{ForkProgress, ForkStats, ProgressMap, PropagatedStats},
progress_map::{ForkProgress, ProgressMap, PropagatedStats},
pubkey_references::PubkeyReferences,
repair_service::DuplicateSlotsResetReceiver,
result::Result,
@ -33,7 +36,7 @@ use solana_sdk::{
hash::Hash,
pubkey::Pubkey,
signature::{Keypair, Signer},
timing::{self, duration_as_ms},
timing::duration_as_ms,
transaction::Transaction,
};
use solana_vote_program::{
@ -140,12 +143,6 @@ impl ReplayTiming {
}
}
pub(crate) struct SelectVoteAndResetForkResult {
pub vote_bank: Option<(Arc<Bank>, SwitchForkDecision)>,
pub reset_bank: Option<Arc<Bank>>,
pub heaviest_fork_failures: Vec<HeaviestForkFailures>,
}
pub struct ReplayStage {
t_replay: JoinHandle<Result<()>>,
commitment_service: AggregateCommitmentService,
@ -180,8 +177,6 @@ impl ReplayStage {
} = config;
trace!("replay stage");
let mut tower = Tower::new(&my_pubkey, &vote_account, &bank_forks.read().unwrap());
// Start the replay stage loop
let (lockouts_sender, commitment_service) = AggregateCommitmentService::new(
&exit,
@ -208,12 +203,12 @@ impl ReplayStage {
frozen_banks.sort_by_key(|bank| bank.slot());
// Initialize progress map with any root banks
for bank in frozen_banks {
let prev_leader_slot = progress.get_bank_prev_leader_slot(&bank);
for bank in &frozen_banks {
let prev_leader_slot = progress.get_bank_prev_leader_slot(bank);
progress.insert(
bank.slot(),
ForkProgress::new_from_bank(
&bank,
bank,
&my_pubkey,
&vote_account,
prev_leader_slot,
@ -222,6 +217,27 @@ impl ReplayStage {
),
);
}
let root_bank = bank_forks.read().unwrap().root_bank().clone();
let root = root_bank.slot();
let unlock_heaviest_subtree_fork_choice_slot =
Self::get_unlock_heaviest_subtree_fork_choice(root_bank.operating_mode());
let mut heaviest_subtree_fork_choice =
HeaviestSubtreeForkChoice::new_from_frozen_banks(root, &frozen_banks);
let mut bank_weight_fork_choice = BankWeightForkChoice::default();
let heaviest_bank = if root > unlock_heaviest_subtree_fork_choice_slot {
bank_forks
.read()
.unwrap()
.get(heaviest_subtree_fork_choice.best_overall_slot())
.expect(
"The best overall slot must be one of `frozen_banks` which all
exist in bank_forks",
)
.clone()
} else {
Tower::find_heaviest_bank(&bank_forks, &my_pubkey).unwrap_or(root_bank)
};
let mut tower = Tower::new(&my_pubkey, &vote_account, root, &heaviest_bank);
let mut current_leader = None;
let mut last_reset = Hash::default();
let mut partition = false;
@ -259,6 +275,7 @@ impl ReplayStage {
&mut progress,
transaction_status_sender.clone(),
&verify_recyclers,
&mut heaviest_subtree_fork_choice,
);
Self::report_memory(&allocated, "replay_active_banks", start);
@ -296,6 +313,8 @@ impl ReplayStage {
&cluster_slots,
&bank_forks,
&mut all_pubkeys,
&mut heaviest_subtree_fork_choice,
&mut bank_weight_fork_choice,
);
let compute_bank_stats_elapsed = now.elapsed().as_micros();
for slot in newly_computed_slot_stats {
@ -317,8 +336,14 @@ impl ReplayStage {
}
}
let (heaviest_bank, heaviest_bank_on_same_fork) =
Self::select_forks(&frozen_banks, &tower, &progress, &ancestors);
let fork_choice: &mut dyn ForkChoice =
if forks_root > unlock_heaviest_subtree_fork_choice_slot {
&mut heaviest_subtree_fork_choice
} else {
&mut bank_weight_fork_choice
};
let (heaviest_bank, heaviest_bank_on_same_voted_fork) = fork_choice
.select_forks(&frozen_banks, &tower, &progress, &ancestors, &bank_forks);
Self::report_memory(&allocated, "select_fork", start);
@ -329,7 +354,7 @@ impl ReplayStage {
heaviest_fork_failures,
} = Self::select_vote_and_reset_forks(
&heaviest_bank,
&heaviest_bank_on_same_fork,
&heaviest_bank_on_same_voted_fork,
&ancestors,
&descendants,
&progress,
@ -341,13 +366,10 @@ impl ReplayStage {
select_vote_and_reset_forks_elapsed as u64,
);
if heaviest_bank.is_some()
&& tower.is_recent(heaviest_bank.as_ref().unwrap().slot())
&& !heaviest_fork_failures.is_empty()
{
if tower.is_recent(heaviest_bank.slot()) && !heaviest_fork_failures.is_empty() {
info!(
"Couldn't vote on heaviest fork: {:?}, heaviest_fork_failures: {:?}",
heaviest_bank.as_ref().map(|b| b.slot()),
heaviest_bank.slot(),
heaviest_fork_failures
);
@ -394,6 +416,7 @@ impl ReplayStage {
&mut all_pubkeys,
&subscriptions,
&block_commitment_cache,
&mut heaviest_subtree_fork_choice,
)?;
};
@ -872,6 +895,7 @@ impl ReplayStage {
all_pubkeys: &mut PubkeyReferences,
subscriptions: &Arc<RpcSubscriptions>,
block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
heaviest_subtree_fork_choice: &mut HeaviestSubtreeForkChoice,
) -> Result<()> {
if bank.is_empty() {
inc_new_counter_info!("replay_stage-voted_empty_bank", 1);
@ -910,6 +934,7 @@ impl ReplayStage {
accounts_hash_sender,
all_pubkeys,
largest_confirmed_root,
heaviest_subtree_fork_choice,
);
subscriptions.notify_roots(rooted_slots);
latest_root_senders.iter().for_each(|s| {
@ -1076,6 +1101,7 @@ impl ReplayStage {
progress: &mut ProgressMap,
transaction_status_sender: Option<TransactionStatusSender>,
verify_recyclers: &VerifyRecyclers,
heaviest_subtree_fork_choice: &mut HeaviestSubtreeForkChoice,
) -> bool {
let mut did_complete_bank = false;
let mut tx_count = 0;
@ -1143,6 +1169,8 @@ impl ReplayStage {
did_complete_bank = true;
info!("bank frozen: {}", bank.slot());
bank.freeze();
heaviest_subtree_fork_choice
.add_new_leaf_slot(bank.slot(), Some(bank.parent_slot()));
} else {
trace!(
"bank {} not completed tick_height: {}, max_tick_height: {}",
@ -1156,6 +1184,7 @@ impl ReplayStage {
did_complete_bank
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn compute_bank_stats(
my_pubkey: &Pubkey,
ancestors: &HashMap<u64, HashSet<u64>>,
@ -1166,37 +1195,56 @@ impl ReplayStage {
cluster_slots: &ClusterSlots,
bank_forks: &RwLock<BankForks>,
all_pubkeys: &mut PubkeyReferences,
heaviest_subtree_fork_choice: &mut dyn ForkChoice,
bank_weight_fork_choice: &mut dyn ForkChoice,
) -> Vec<Slot> {
frozen_banks.sort_by_key(|bank| bank.slot());
let mut new_stats = vec![];
for bank in frozen_banks {
let bank_slot = bank.slot();
// Only time progress map should be missing a bank slot
// is if this node was the leader for this slot as those banks
// are not replayed in replay_active_banks()
let parent_weight = bank
.parent()
.and_then(|b| progress.get(&b.slot()))
.map(|x| x.fork_stats.fork_weight)
.unwrap_or(0);
{
let stats = progress
let is_computed = progress
.get_fork_stats_mut(bank_slot)
.expect("All frozen banks must exist in the Progress map");
if !stats.computed {
let (stake_lockouts, total_staked, bank_weight, lockout_intervals) = tower
.collect_vote_lockouts(
.expect("All frozen banks must exist in the Progress map")
.computed;
if !is_computed {
let computed_bank_state = Tower::collect_vote_lockouts(
my_pubkey,
bank_slot,
bank.vote_accounts().into_iter(),
&ancestors,
all_pubkeys,
);
heaviest_subtree_fork_choice.compute_bank_stats(
&bank,
tower,
progress,
&computed_bank_state,
);
bank_weight_fork_choice.compute_bank_stats(
&bank,
tower,
progress,
&computed_bank_state,
);
let ComputedBankState {
stake_lockouts,
total_staked,
lockout_intervals,
..
} = computed_bank_state;
let stats = progress
.get_fork_stats_mut(bank_slot)
.expect("All frozen banks must exist in the Progress map");
stats.total_staked = total_staked;
stats.weight = bank_weight;
stats.fork_weight = stats.weight + parent_weight;
stats.stake_lockouts = stake_lockouts;
stats.lockout_intervals = lockout_intervals;
stats.block_height = bank.block_height();
stats.computed = true;
new_stats.push(bank_slot);
datapoint_info!(
"bank_weight",
("slot", bank_slot, i64),
@ -1211,11 +1259,6 @@ impl ReplayStage {
stats.fork_weight,
bank.parent().map(|b| b.slot()).unwrap_or(0)
);
stats.stake_lockouts = stake_lockouts;
stats.lockout_intervals = lockout_intervals;
stats.block_height = bank.block_height();
stats.computed = true;
new_stats.push(bank_slot);
}
}
@ -1289,11 +1332,11 @@ impl ReplayStage {
let newly_voted_pubkeys = slot_vote_tracker
.as_ref()
.and_then(|slot_vote_tracker| slot_vote_tracker.write().unwrap().get_updates())
.unwrap_or_else(Vec::new);
.unwrap_or_default();
let cluster_slot_pubkeys = cluster_slot_pubkeys
.map(|v| v.read().unwrap().keys().cloned().collect())
.unwrap_or_else(Vec::new);
.unwrap_or_default();
Self::update_fork_propagated_threshold_from_votes(
progress,
@ -1305,115 +1348,12 @@ impl ReplayStage {
);
}
// Returns:
// 1) The heaviest bank
// 2) The latest votable bank on the same fork as the last vote
pub(crate) fn select_forks(
frozen_banks: &[Arc<Bank>],
tower: &Tower,
progress: &ProgressMap,
ancestors: &HashMap<u64, HashSet<u64>>,
) -> (Option<Arc<Bank>>, Option<Arc<Bank>>) {
let tower_start = Instant::now();
let num_frozen_banks = frozen_banks.len();
trace!("frozen_banks {}", frozen_banks.len());
let num_old_banks = frozen_banks
.iter()
.filter(|b| b.slot() < tower.root().unwrap_or(0))
.count();
let last_vote = tower.last_vote().slots.last().cloned();
let mut heaviest_bank_on_same_fork = None;
let mut heaviest_same_fork_weight = 0;
let stats: Vec<&ForkStats> = frozen_banks
.iter()
.map(|bank| {
// Only time progress map should be missing a bank slot
// is if this node was the leader for this slot as those banks
// are not replayed in replay_active_banks()
let stats = progress
.get_fork_stats(bank.slot())
.expect("All frozen banks must exist in the Progress map");
if let Some(last_vote) = last_vote {
if ancestors
.get(&bank.slot())
.expect("Entry in frozen banks must exist in ancestors")
.contains(&last_vote)
{
// Descendant of last vote cannot be locked out
assert!(!stats.is_locked_out);
// ancestors(slot) should not contain the slot itself,
// so we should never get the same bank as the last vote
assert_ne!(bank.slot(), last_vote);
// highest weight, lowest slot first. frozen_banks is sorted
// from least slot to greatest slot, so if two banks have
// the same fork weight, the lower slot will be picked
if stats.fork_weight > heaviest_same_fork_weight {
heaviest_bank_on_same_fork = Some(bank.clone());
heaviest_same_fork_weight = stats.fork_weight;
}
}
}
stats
})
.collect();
let num_not_recent = stats.iter().filter(|s| !s.is_recent).count();
let num_has_voted = stats.iter().filter(|s| s.has_voted).count();
let num_empty = stats.iter().filter(|s| s.is_empty).count();
let num_threshold_failure = stats.iter().filter(|s| !s.vote_threshold).count();
let num_votable_threshold_failure = stats
.iter()
.filter(|s| s.is_recent && !s.has_voted && !s.vote_threshold)
.count();
let mut candidates: Vec<_> = frozen_banks.iter().zip(stats.iter()).collect();
//highest weight, lowest slot first
candidates.sort_by_key(|b| (b.1.fork_weight, 0i64 - b.0.slot() as i64));
let rv = candidates.last();
let ms = timing::duration_as_ms(&tower_start.elapsed());
let weights: Vec<(u128, u64, u64)> = candidates
.iter()
.map(|x| (x.1.weight, x.0.slot(), x.1.block_height))
.collect();
debug!(
"@{:?} tower duration: {:?} len: {}/{} weights: {:?} voting: {}",
timing::timestamp(),
ms,
candidates.len(),
stats.iter().filter(|s| !s.has_voted).count(),
weights,
rv.is_some()
);
datapoint_debug!(
"replay_stage-select_forks",
("frozen_banks", num_frozen_banks as i64, i64),
("not_recent", num_not_recent as i64, i64),
("has_voted", num_has_voted as i64, i64),
("old_banks", num_old_banks as i64, i64),
("empty_banks", num_empty as i64, i64),
("threshold_failure", num_threshold_failure as i64, i64),
(
"votable_threshold_failure",
num_votable_threshold_failure as i64,
i64
),
("tower_duration", ms as i64, i64),
);
(rv.map(|x| x.0.clone()), heaviest_bank_on_same_fork)
}
// Given a heaviest bank, `heaviest_bank` and the next votable bank
// `heaviest_bank_on_same_fork` as the validator's last vote, return
// `heaviest_bank_on_same_voted_fork` as the validator's last vote, return
// a bank to vote on, a bank to reset to,
pub(crate) fn select_vote_and_reset_forks(
heaviest_bank: &Option<Arc<Bank>>,
heaviest_bank_on_same_fork: &Option<Arc<Bank>>,
heaviest_bank: &Arc<Bank>,
heaviest_bank_on_same_voted_fork: &Option<Arc<Bank>>,
ancestors: &HashMap<u64, HashSet<u64>>,
descendants: &HashMap<u64, HashSet<u64>>,
progress: &ProgressMap,
@ -1434,37 +1374,35 @@ impl ReplayStage {
// switch_threshold succceeds
let mut failure_reasons = vec![];
let selected_fork = {
if let Some(bank) = heaviest_bank {
let switch_fork_decision = tower.check_switch_threshold(
bank.slot(),
heaviest_bank.slot(),
&ancestors,
&descendants,
&progress,
bank.total_epoch_stake(),
bank.epoch_vote_accounts(bank.epoch()).expect(
"Bank epoch vote accounts must contain entry for the bank's own epoch",
),
heaviest_bank.total_epoch_stake(),
heaviest_bank
.epoch_vote_accounts(heaviest_bank.epoch())
.expect("Bank epoch vote accounts must contain entry for the bank's own epoch"),
);
if switch_fork_decision == SwitchForkDecision::FailedSwitchThreshold {
// If we can't switch, then reset to the the next votable
// bank on the same fork as our last vote, but don't vote
info!(
"Waiting to switch vote to {}, resetting to slot {:?} on same fork for now",
bank.slot(),
heaviest_bank_on_same_fork.as_ref().map(|b| b.slot())
heaviest_bank.slot(),
heaviest_bank_on_same_voted_fork.as_ref().map(|b| b.slot())
);
failure_reasons.push(HeaviestForkFailures::FailedSwitchThreshold(bank.slot()));
heaviest_bank_on_same_fork
failure_reasons.push(HeaviestForkFailures::FailedSwitchThreshold(
heaviest_bank.slot(),
));
heaviest_bank_on_same_voted_fork
.as_ref()
.map(|b| (b, switch_fork_decision))
} else {
// If the switch threshold is observed, halt voting on
// the current fork and attempt to vote/reset Poh to
// the heaviest bank
heaviest_bank.as_ref().map(|b| (b, switch_fork_decision))
}
} else {
None
Some((heaviest_bank, switch_fork_decision))
}
};
@ -1690,6 +1628,7 @@ impl ReplayStage {
accounts_hash_sender: &Option<AccountsPackageSender>,
all_pubkeys: &mut PubkeyReferences,
largest_confirmed_root: Option<Slot>,
heaviest_subtree_fork_choice: &mut HeaviestSubtreeForkChoice,
) {
let old_epoch = bank_forks.read().unwrap().root_bank().epoch();
bank_forks.write().unwrap().set_root(
@ -1703,6 +1642,7 @@ impl ReplayStage {
all_pubkeys.purge();
}
progress.handle_new_root(&r_bank_forks);
heaviest_subtree_fork_choice.set_root(new_root);
}
fn generate_new_bank_forks(
@ -1812,6 +1752,14 @@ impl ReplayStage {
}
}
pub fn get_unlock_heaviest_subtree_fork_choice(operating_mode: OperatingMode) -> Slot {
match operating_mode {
OperatingMode::Development => std::u64::MAX / 2,
OperatingMode::Stable => std::u64::MAX / 2,
OperatingMode::Preview => std::u64::MAX / 2,
}
}
pub fn join(self) -> thread::Result<()> {
self.commitment_service.join()?;
self.t_replay.join().map(|_| ())
@ -1844,21 +1792,18 @@ pub(crate) mod tests {
};
use solana_runtime::genesis_utils::{self, GenesisConfigInfo, ValidatorVoteKeypairs};
use solana_sdk::{
account::Account,
clock::NUM_CONSECUTIVE_LEADER_SLOTS,
genesis_config,
hash::{hash, Hash},
instruction::InstructionError,
packet::PACKET_DATA_SIZE,
rent::Rent,
signature::{Keypair, Signature, Signer},
system_transaction,
transaction::TransactionError,
};
use solana_stake_program::stake_state;
use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta};
use solana_vote_program::{
vote_state::{self, Vote, VoteState, VoteStateVersions},
vote_state::{VoteState, VoteStateVersions},
vote_transaction,
};
use std::{
@ -1869,292 +1814,6 @@ pub(crate) mod tests {
};
use trees::tr;
struct ForkInfo {
leader: usize,
fork: Vec<Slot>,
voters: Vec<usize>,
}
struct ValidatorInfo {
stake: u64,
keypair: Keypair,
authorized_voter_keypair: Keypair,
staking_keypair: Keypair,
}
struct ForkSelectionResponse {
slot: u64,
is_locked_out: bool,
}
fn simulate_fork_selection(
neutral_fork: &ForkInfo,
forks: &[ForkInfo],
validators: &[ValidatorInfo],
) -> Vec<Option<ForkSelectionResponse>> {
fn vote(bank: &Arc<Bank>, pubkey: &Pubkey, slot: Slot) {
let mut vote_account = bank.get_account(&pubkey).unwrap();
let mut vote_state = VoteState::from(&vote_account).unwrap();
vote_state.process_slot_vote_unchecked(slot);
let versioned = VoteStateVersions::Current(Box::new(vote_state));
VoteState::to(&versioned, &mut vote_account).unwrap();
bank.store_account(&pubkey, &vote_account);
}
let vote_tracker = VoteTracker::default();
let cluster_slots = ClusterSlots::default();
let mut towers: Vec<Tower> = iter::repeat_with(|| Tower::new_for_tests(8, 0.67))
.take(validators.len())
.collect();
for slot in &neutral_fork.fork {
for tower in towers.iter_mut() {
tower.record_bank_vote(Vote {
hash: Hash::default(),
slots: vec![*slot],
timestamp: None,
});
}
}
for fork_info in forks.iter() {
for slot in fork_info.fork.iter() {
for voter_index in fork_info.voters.iter() {
towers[*voter_index].record_bank_vote(Vote {
hash: Hash::default(),
slots: vec![*slot],
timestamp: None,
});
}
}
}
let genesis_vote_accounts: Vec<Account> = validators
.iter()
.map(|validator| {
vote_state::create_account(
&validator.authorized_voter_keypair.pubkey(),
&validator.keypair.pubkey(),
0,
validator.stake,
)
})
.collect();
let genesis_stake_accounts: Vec<Account> = validators
.iter()
.enumerate()
.map(|(i, validator)| {
stake_state::create_account(
&validator.staking_keypair.pubkey(),
&validator.authorized_voter_keypair.pubkey(),
&genesis_vote_accounts[i],
&Rent::default(),
validator.stake,
)
})
.collect();
let mut genesis_config = create_genesis_config(10_000).genesis_config;
genesis_config.accounts.clear();
for i in 0..validators.len() {
genesis_config.accounts.insert(
validators[i].authorized_voter_keypair.pubkey(),
genesis_vote_accounts[i].clone(),
);
genesis_config.accounts.insert(
validators[i].staking_keypair.pubkey(),
genesis_stake_accounts[i].clone(),
);
}
let mut bank_forks = BankForks::new(neutral_fork.fork[0], Bank::new(&genesis_config));
let mut fork_progresses: Vec<ProgressMap> = iter::repeat_with(ProgressMap::default)
.take(validators.len())
.collect();
for fork_progress in fork_progresses.iter_mut() {
let bank = &bank_forks.banks[&0];
fork_progress
.entry(neutral_fork.fork[0])
.or_insert_with(|| ForkProgress::new(bank.last_blockhash(), None, None, 0, 0));
}
for index in 1..neutral_fork.fork.len() {
let bank = Bank::new_from_parent(
&bank_forks.banks[&neutral_fork.fork[index - 1]].clone(),
&validators[neutral_fork.leader].keypair.pubkey(),
neutral_fork.fork[index],
);
bank_forks.insert(bank);
for validator in validators.iter() {
vote(
&bank_forks.banks[&neutral_fork.fork[index]].clone(),
&validator.authorized_voter_keypair.pubkey(),
neutral_fork.fork[index - 1],
);
}
bank_forks.banks[&neutral_fork.fork[index]].freeze();
for fork_progress in fork_progresses.iter_mut() {
let bank = &bank_forks.banks[&neutral_fork.fork[index]];
fork_progress
.entry(bank_forks.banks[&neutral_fork.fork[index]].slot())
.or_insert_with(|| ForkProgress::new(bank.last_blockhash(), None, None, 0, 0));
}
}
let last_neutral_bank = &bank_forks.banks[neutral_fork.fork.last().unwrap()].clone();
for fork_info in forks.iter() {
for index in 0..fork_info.fork.len() {
let last_bank: &Arc<Bank>;
let last_bank_in_fork: Arc<Bank>;
if index == 0 {
last_bank = &last_neutral_bank;
} else {
last_bank_in_fork = bank_forks.banks[&fork_info.fork[index - 1]].clone();
last_bank = &last_bank_in_fork;
}
let bank = Bank::new_from_parent(
last_bank,
&validators[fork_info.leader].keypair.pubkey(),
fork_info.fork[index],
);
bank_forks.insert(bank);
for voter_index in fork_info.voters.iter() {
vote(
&bank_forks.banks[&fork_info.fork[index]].clone(),
&validators[*voter_index].authorized_voter_keypair.pubkey(),
last_bank.slot(),
);
}
bank_forks.banks[&fork_info.fork[index]].freeze();
for fork_progress in fork_progresses.iter_mut() {
let bank = &bank_forks.banks[&fork_info.fork[index]];
fork_progress
.entry(bank_forks.banks[&fork_info.fork[index]].slot())
.or_insert_with(|| {
ForkProgress::new(bank.last_blockhash(), None, None, 0, 0)
});
}
}
}
let bank_fork_ancestors = bank_forks.ancestors();
let wrapped_bank_fork = Arc::new(RwLock::new(bank_forks));
let mut all_pubkeys = PubkeyReferences::default();
(0..validators.len())
.map(|i| {
let mut frozen_banks: Vec<_> = wrapped_bank_fork
.read()
.unwrap()
.frozen_banks()
.values()
.cloned()
.collect();
ReplayStage::compute_bank_stats(
&validators[i].keypair.pubkey(),
&bank_fork_ancestors,
&mut frozen_banks,
&towers[i],
&mut fork_progresses[i],
&vote_tracker,
&cluster_slots,
&wrapped_bank_fork,
&mut all_pubkeys,
);
let (heaviest_bank, _) = ReplayStage::select_forks(
&frozen_banks,
&towers[i],
&fork_progresses[i],
&bank_fork_ancestors,
);
if let Some(bank) = heaviest_bank {
let stats = &fork_progresses[i].get_fork_stats(bank.slot()).unwrap();
Some(ForkSelectionResponse {
slot: bank.slot(),
is_locked_out: stats.is_locked_out,
})
} else {
None
}
})
.collect()
}
#[test]
fn test_minority_fork_overcommit_attack() {
let neutral_fork = ForkInfo {
leader: 0,
fork: vec![0, 1, 2],
voters: vec![],
};
let forks: Vec<ForkInfo> = vec![
// Minority fork
ForkInfo {
leader: 2,
fork: (3..=3 + 8).collect(),
voters: vec![2],
},
ForkInfo {
leader: 1,
fork: (12..12 + 8).collect(),
voters: vec![0, 1],
},
];
let validators: Vec<ValidatorInfo> = vec![
ValidatorInfo {
stake: 34_000_000,
keypair: Keypair::new(),
authorized_voter_keypair: Keypair::new(),
staking_keypair: Keypair::new(),
},
ValidatorInfo {
stake: 33_000_000,
keypair: Keypair::new(),
authorized_voter_keypair: Keypair::new(),
staking_keypair: Keypair::new(),
},
// Malicious Node
ValidatorInfo {
stake: 33_000_000,
keypair: Keypair::new(),
authorized_voter_keypair: Keypair::new(),
staking_keypair: Keypair::new(),
},
];
let resp = simulate_fork_selection(&neutral_fork, &forks, &validators);
// Both honest nodes are now want to switch to minority fork and are locked out
assert!(resp[0].is_some());
assert_eq!(resp[0].as_ref().unwrap().is_locked_out, true);
assert_eq!(
resp[0].as_ref().unwrap().slot,
forks[0].fork.last().unwrap().clone()
);
assert!(resp[1].is_some());
assert_eq!(resp[1].as_ref().unwrap().is_locked_out, true);
assert_eq!(
resp[1].as_ref().unwrap().slot,
forks[0].fork.last().unwrap().clone()
);
}
#[test]
fn test_child_slots_of_same_parent() {
let ledger_path = get_tmp_ledger_path!();
@ -2304,6 +1963,7 @@ pub(crate) mod tests {
let bank0 = Bank::new(&genesis_config);
let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank0)));
let root = 3;
let mut heaviest_subtree_fork_choice = HeaviestSubtreeForkChoice::new(root);
let root_bank = Bank::new_from_parent(
bank_forks.read().unwrap().get(0).unwrap(),
&Pubkey::default(),
@ -2321,6 +1981,7 @@ pub(crate) mod tests {
&None,
&mut PubkeyReferences::default(),
None,
&mut heaviest_subtree_fork_choice,
);
assert_eq!(bank_forks.read().unwrap().root(), root);
assert_eq!(progress.len(), 1);
@ -2353,6 +2014,7 @@ pub(crate) mod tests {
root,
);
bank_forks.write().unwrap().insert(root_bank);
let mut heaviest_subtree_fork_choice = HeaviestSubtreeForkChoice::new(root);
let mut progress = ProgressMap::default();
for i in 0..=root {
progress.insert(i, ForkProgress::new(Hash::default(), None, None, 0, 0));
@ -2364,6 +2026,7 @@ pub(crate) mod tests {
&None,
&mut PubkeyReferences::default(),
Some(confirmed_root),
&mut heaviest_subtree_fork_choice,
);
assert_eq!(bank_forks.read().unwrap().root(), root);
assert!(bank_forks.read().unwrap().get(confirmed_root).is_some());
@ -2880,7 +2543,8 @@ pub(crate) mod tests {
ValidatorVoteKeypairs::new(node_keypair, vote_keypair, stake_keypair),
);
let (bank_forks, mut progress) = initialize_state(&keypairs, 10_000);
let (bank_forks, mut progress, mut heaviest_subtree_fork_choice) =
initialize_state(&keypairs, 10_000);
let bank0 = bank_forks.get(0).unwrap().clone();
let my_keypairs = keypairs.get(&node_pubkey).unwrap();
let vote_tx = vote_transaction::new_vote_transaction(
@ -2917,6 +2581,8 @@ pub(crate) mod tests {
&ClusterSlots::default(),
&bank_forks,
&mut PubkeyReferences::default(),
&mut heaviest_subtree_fork_choice,
&mut BankWeightForkChoice::default(),
);
assert_eq!(newly_computed, vec![0]);
// The only vote is in bank 1, and bank_forks does not currently contain
@ -2958,6 +2624,8 @@ pub(crate) mod tests {
&ClusterSlots::default(),
&bank_forks,
&mut PubkeyReferences::default(),
&mut heaviest_subtree_fork_choice,
&mut BankWeightForkChoice::default(),
);
assert_eq!(newly_computed, vec![1]);
@ -2991,6 +2659,8 @@ pub(crate) mod tests {
&ClusterSlots::default(),
&bank_forks,
&mut PubkeyReferences::default(),
&mut heaviest_subtree_fork_choice,
&mut BankWeightForkChoice::default(),
);
// No new stats should have been computed
assert!(newly_computed.is_empty());
@ -3005,7 +2675,8 @@ pub(crate) mod tests {
// Create the tree of banks in a BankForks object
let forks = tr(0) / (tr(1)) / (tr(2));
vote_simulator.fill_bank_forks(forks, &HashMap::new());
vote_simulator.fill_bank_forks(forks.clone(), &HashMap::new());
let mut heaviest_subtree_fork_choice = HeaviestSubtreeForkChoice::new_from_tree(forks);
let mut frozen_banks: Vec<_> = vote_simulator
.bank_forks
.read()
@ -3017,7 +2688,7 @@ pub(crate) mod tests {
let ancestors = vote_simulator.bank_forks.read().unwrap().ancestors();
ReplayStage::compute_bank_stats(
&Pubkey::default(),
&node_pubkey,
&ancestors,
&mut frozen_banks,
&tower,
@ -3026,25 +2697,36 @@ pub(crate) mod tests {
&ClusterSlots::default(),
&vote_simulator.bank_forks,
&mut PubkeyReferences::default(),
&mut heaviest_subtree_fork_choice,
&mut BankWeightForkChoice::default(),
);
assert_eq!(
vote_simulator
.progress
.get_fork_stats(1)
.unwrap()
.fork_weight,
vote_simulator
.progress
.get_fork_stats(2)
.unwrap()
.fork_weight,
heaviest_subtree_fork_choice.stake_voted_subtree(1).unwrap(),
heaviest_subtree_fork_choice.stake_voted_subtree(2).unwrap()
);
let (heaviest_bank, _) = heaviest_subtree_fork_choice.select_forks(
&frozen_banks,
&tower,
&vote_simulator.progress,
&ancestors,
&vote_simulator.bank_forks,
);
let (heaviest_bank, _) =
ReplayStage::select_forks(&frozen_banks, &tower, &vote_simulator.progress, &ancestors);
// Should pick the lower of the two equally weighted banks
assert_eq!(heaviest_bank.unwrap().slot(), 1);
assert_eq!(heaviest_bank.slot(), 1);
let (heaviest_bank, _) = BankWeightForkChoice::default().select_forks(
&frozen_banks,
&tower,
&vote_simulator.progress,
&ancestors,
&vote_simulator.bank_forks,
);
// Should pick the lower of the two equally weighted banks
assert_eq!(heaviest_bank.slot(), 1);
}
#[test]
@ -3062,6 +2744,8 @@ pub(crate) mod tests {
let votes = vec![0, 2];
cluster_votes.insert(node_pubkey, votes.clone());
vote_simulator.fill_bank_forks(forks, &cluster_votes);
// Fill banks with votes
for vote in votes {
assert!(vote_simulator
.simulate_vote(vote, &node_pubkey, &mut tower,)
@ -3078,7 +2762,7 @@ pub(crate) mod tests {
.collect();
ReplayStage::compute_bank_stats(
&Pubkey::default(),
&node_pubkey,
&vote_simulator.bank_forks.read().unwrap().ancestors(),
&mut frozen_banks,
&tower,
@ -3087,6 +2771,8 @@ pub(crate) mod tests {
&ClusterSlots::default(),
&vote_simulator.bank_forks,
&mut PubkeyReferences::default(),
&mut vote_simulator.heaviest_subtree_fork_choice,
&mut BankWeightForkChoice::default(),
);
frozen_banks.sort_by_key(|bank| bank.slot());
@ -3103,6 +2789,16 @@ pub(crate) mod tests {
.fork_weight;
assert!(second >= first);
}
for bank in frozen_banks {
// The only leaf should always be chosen over parents
assert_eq!(
vote_simulator
.heaviest_subtree_fork_choice
.best_slot(bank.slot())
.unwrap(),
3
);
}
}
#[test]
@ -3200,7 +2896,7 @@ pub(crate) mod tests {
success_index: usize,
) {
let stake = 10_000;
let (bank_forks, _) = initialize_state(&all_keypairs, stake);
let (bank_forks, _, _) = initialize_state(&all_keypairs, stake);
let root_bank = bank_forks.root_bank().clone();
let mut propagated_stats = PropagatedStats {
total_epoch_stake: stake * all_keypairs.len() as u64,
@ -3337,7 +3033,7 @@ pub(crate) mod tests {
);
let stake = 10_000;
let (mut bank_forks, mut progress_map) = initialize_state(&keypairs, stake);
let (mut bank_forks, mut progress_map, _) = initialize_state(&keypairs, stake);
let bank0 = bank_forks.get(0).unwrap().clone();
bank_forks.insert(Bank::new_from_parent(&bank0, &Pubkey::default(), 9));
@ -3435,7 +3131,8 @@ pub(crate) mod tests {
.collect();
let stake_per_validator = 10_000;
let (mut bank_forks, mut progress_map) = initialize_state(&keypairs, stake_per_validator);
let (mut bank_forks, mut progress_map, _) =
initialize_state(&keypairs, stake_per_validator);
bank_forks.set_root(0, &None, None);
let total_epoch_stake = bank_forks.root_bank().total_epoch_stake();
@ -3517,7 +3214,8 @@ pub(crate) mod tests {
.collect();
let stake_per_validator = 10_000;
let (mut bank_forks, mut progress_map) = initialize_state(&keypairs, stake_per_validator);
let (mut bank_forks, mut progress_map, _) =
initialize_state(&keypairs, stake_per_validator);
bank_forks.set_root(0, &None, None);
let total_epoch_stake = num_validators as u64 * stake_per_validator;

16
local-cluster/tests/local_cluster.rs
View File

@ -357,20 +357,18 @@ fn test_cluster_partition_1_1_1() {
#[test]
#[serial]
fn test_kill_partition() {
fn test_kill_heaviest_partition() {
// This test:
// 1) Spins up three partitions
// 2) Forces more slots in the leader schedule for the first partition so
// that this partition will be the heaviiest
// 3) Schedules the other validators for sufficient slots in the schedule
// 1) Spins up four partitions, the heaviest being the first with more stake
// 2) Schedules the other validators for sufficient slots in the schedule
// so that they will still be locked out of voting for the major partitoin
// when the partition resolves
// 4) Kills the major partition. Validators are locked out, but should be
// able to reset to the major partition
// 5) Check for recovery
// 3) Kills the most staked partition. Validators are locked out, but should all
// eventually choose the major partition
// 4) Check for recovery
let mut leader_schedule = vec![];
let num_slots_per_validator = 8;
let partitions: [&[usize]; 3] = [&[9], &[10], &[10]];
let partitions: [&[usize]; 4] = [&[11], &[10], &[10], &[10]];
let validator_keys: Vec<_> = iter::repeat_with(|| Arc::new(Keypair::new()))
.take(partitions.len())
.collect();

8
runtime/src/bank.rs
View File

@ -2411,6 +2411,10 @@ impl Bank {
self.epoch_stakes.get(&epoch)
}
pub fn epoch_stakes_map(&self) -> &HashMap<Epoch, EpochStakes> {
&self.epoch_stakes
}
/// vote accounts for the specific epoch along with the stake
/// attributed to each account
pub fn epoch_vote_accounts(&self, epoch: Epoch) -> Option<&HashMap<Pubkey, (u64, Account)>> {
@ -2448,11 +2452,11 @@ impl Bank {
}
/// Get the fixed stake of the given vote account for the current epoch
pub fn epoch_vote_account_stake(&self, voting_pubkey: &Pubkey) -> u64 {
pub fn epoch_vote_account_stake(&self, vote_account: &Pubkey) -> u64 {
*self
.epoch_vote_accounts(self.epoch())
.expect("Bank epoch vote accounts must contain entry for the bank's own epoch")
.get(voting_pubkey)
.get(vote_account)
.map(|(stake, _)| stake)
.unwrap_or(&0)
}

7
runtime/src/epoch_stakes.rs
View File

@ -50,6 +50,13 @@ impl EpochStakes {
&self.epoch_authorized_voters
}
pub fn vote_account_stake(&self, vote_account: &Pubkey) -> u64 {
Stakes::vote_accounts(&self.stakes)
.get(vote_account)
.map(|(stake, _)| *stake)
.unwrap_or(0)
}
fn parse_epoch_vote_accounts(
epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
leader_schedule_epoch: Epoch,