From ce467bea20b29a16a0800b761ac0970b22228355 Mon Sep 17 00:00:00 2001 From: carllin Date: Sun, 18 Jul 2021 17:04:25 -0700 Subject: [PATCH] Add frozen hashes and marking DuplicateConfirmed in blockstore to state machine (#18648) --- core/src/cluster_slot_state_verifier.rs | 1412 ++++++++++++++-------- core/src/fork_choice.rs | 7 +- core/src/heaviest_subtree_fork_choice.rs | 13 +- core/src/replay_stage.rs | 224 ++-- core/src/vote_simulator.rs | 9 +- 5 files changed, 1075 insertions(+), 590 deletions(-) diff --git a/core/src/cluster_slot_state_verifier.rs b/core/src/cluster_slot_state_verifier.rs index 0717909404..3ba172bd04 100644 --- a/core/src/cluster_slot_state_verifier.rs +++ b/core/src/cluster_slot_state_verifier.rs @@ -1,25 +1,218 @@ -use crate::{ - fork_choice::ForkChoice, heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice, - progress_map::ProgressMap, -}; +use crate::{fork_choice::ForkChoice, heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice}; +use solana_ledger::blockstore::Blockstore; use solana_sdk::{clock::Slot, hash::Hash}; use std::collections::{BTreeMap, BTreeSet, HashSet}; pub(crate) type DuplicateSlotsTracker = BTreeSet; pub(crate) type DuplicateSlotsToRepair = HashSet<(Slot, Hash)>; pub(crate) type GossipDuplicateConfirmedSlots = BTreeMap; -type SlotStateHandler = fn(Slot, &Hash, Option<&Hash>, bool, bool) -> Vec; + +#[derive(PartialEq, Clone, Debug)] +pub enum BankStatus { + Frozen(Hash), + Dead, + Unprocessed, +} + +impl BankStatus { + pub fn new(is_dead: impl Fn() -> bool, get_hash: impl Fn() -> Option) -> Self { + if is_dead() { + Self::new_dead() + } else { + Self::new_from_hash(get_hash()) + } + } + + fn new_dead() -> Self { + BankStatus::Dead + } + + fn new_from_hash(hash: Option) -> Self { + if let Some(hash) = hash { + if hash == Hash::default() { + BankStatus::Unprocessed + } else { + BankStatus::Frozen(hash) + } + } else { + BankStatus::Unprocessed + } + } + + fn bank_hash(&self) -> Option { + match self { + BankStatus::Frozen(hash) => Some(*hash), + BankStatus::Dead => Some(Hash::default()), + BankStatus::Unprocessed => None, + } + } + + fn is_dead(&self) -> bool { + match self { + BankStatus::Frozen(_) => false, + BankStatus::Dead => true, + BankStatus::Unprocessed => false, + } + } +} + +#[derive(PartialEq, Debug)] +pub struct DeadState { + // Keep fields private, forces construction + // via constructor + duplicate_confirmed_hash: Option, + is_slot_duplicate: bool, +} + +impl DeadState { + pub fn new_from_state( + slot: Slot, + duplicate_slots_tracker: &mut DuplicateSlotsTracker, + gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots, + fork_choice: &mut HeaviestSubtreeForkChoice, + ) -> Self { + let duplicate_confirmed_hash = get_duplicate_confirmed_hash_from_state( + slot, + gossip_duplicate_confirmed_slots, + fork_choice, + Some(Hash::default()), + ); + let is_slot_duplicate = duplicate_slots_tracker.contains(&slot); + Self::new(duplicate_confirmed_hash, is_slot_duplicate) + } + + fn new(duplicate_confirmed_hash: Option, is_slot_duplicate: bool) -> Self { + Self { + duplicate_confirmed_hash, + is_slot_duplicate, + } + } +} + +#[derive(PartialEq, Debug)] +pub struct BankFrozenState { + // Keep fields private, forces construction + // via constructor + frozen_hash: Hash, + duplicate_confirmed_hash: Option, + is_slot_duplicate: bool, +} + +impl BankFrozenState { + pub fn new_from_state( + slot: Slot, + frozen_hash: Hash, + duplicate_slots_tracker: &mut DuplicateSlotsTracker, + gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots, + fork_choice: &mut HeaviestSubtreeForkChoice, + ) -> Self { + let duplicate_confirmed_hash = get_duplicate_confirmed_hash_from_state( + slot, + gossip_duplicate_confirmed_slots, + fork_choice, + Some(frozen_hash), + ); + let is_slot_duplicate = duplicate_slots_tracker.contains(&slot); + Self::new(frozen_hash, duplicate_confirmed_hash, is_slot_duplicate) + } + + fn new( + frozen_hash: Hash, + duplicate_confirmed_hash: Option, + is_slot_duplicate: bool, + ) -> Self { + assert!(frozen_hash != Hash::default()); + Self { + frozen_hash, + duplicate_confirmed_hash, + is_slot_duplicate, + } + } +} + +#[derive(PartialEq, Debug)] +pub struct DuplicateConfirmedState { + // Keep fields private, forces construction + // via constructor + duplicate_confirmed_hash: Hash, + bank_status: BankStatus, +} +impl DuplicateConfirmedState { + pub fn new_from_state( + duplicate_confirmed_hash: Hash, + is_dead: impl Fn() -> bool, + get_hash: impl Fn() -> Option, + ) -> Self { + let bank_status = BankStatus::new(is_dead, get_hash); + Self::new(duplicate_confirmed_hash, bank_status) + } + + fn new(duplicate_confirmed_hash: Hash, bank_status: BankStatus) -> Self { + Self { + duplicate_confirmed_hash, + bank_status, + } + } +} + +#[derive(PartialEq, Debug)] +pub struct DuplicateState { + // Keep fields private, forces construction + // via constructor + duplicate_confirmed_hash: Option, + bank_status: BankStatus, +} +impl DuplicateState { + pub fn new_from_state( + slot: Slot, + gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots, + fork_choice: &mut HeaviestSubtreeForkChoice, + is_dead: impl Fn() -> bool, + get_hash: impl Fn() -> Option, + ) -> Self { + let bank_status = BankStatus::new(is_dead, get_hash); + let duplicate_confirmed_hash = get_duplicate_confirmed_hash_from_state( + slot, + gossip_duplicate_confirmed_slots, + fork_choice, + bank_status.bank_hash(), + ); + Self::new(duplicate_confirmed_hash, bank_status) + } + + fn new(duplicate_confirmed_hash: Option, bank_status: BankStatus) -> Self { + Self { + duplicate_confirmed_hash, + bank_status, + } + } +} #[derive(PartialEq, Debug)] pub enum SlotStateUpdate { - Frozen, - DuplicateConfirmed, - Dead, - Duplicate, + BankFrozen(BankFrozenState), + DuplicateConfirmed(DuplicateConfirmedState), + Dead(DeadState), + Duplicate(DuplicateState), +} + +impl SlotStateUpdate { + fn bank_hash(&self) -> Option { + match self { + SlotStateUpdate::BankFrozen(bank_frozen_state) => Some(bank_frozen_state.frozen_hash), + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state) => { + duplicate_confirmed_state.bank_status.bank_hash() + } + SlotStateUpdate::Dead(_) => Some(Hash::default()), + SlotStateUpdate::Duplicate(duplicate_state) => duplicate_state.bank_status.bank_hash(), + } + } } #[derive(PartialEq, Debug)] pub enum ResultingStateChange { + // Bank was frozen + BankFrozen(Hash), // Hash of our current frozen version of the slot MarkSlotDuplicate(Hash), // Hash of the cluster confirmed slot that is not equivalent @@ -30,151 +223,215 @@ pub enum ResultingStateChange { } impl SlotStateUpdate { - fn to_handler(&self) -> SlotStateHandler { + fn into_state_changes(self, slot: Slot) -> Vec { + let bank_frozen_hash = self.bank_hash(); + if bank_frozen_hash == None { + // If the bank hasn't been frozen yet, then there's nothing to do + // since replay of the slot hasn't finished yet. + return vec![]; + } + match self { - SlotStateUpdate::Dead => on_dead_slot, - SlotStateUpdate::Frozen => on_frozen_slot, - SlotStateUpdate::DuplicateConfirmed => on_cluster_update, - SlotStateUpdate::Duplicate => on_cluster_update, + SlotStateUpdate::Dead(dead_state) => on_dead_slot(slot, dead_state), + SlotStateUpdate::BankFrozen(bank_frozen_state) => { + on_frozen_slot(slot, bank_frozen_state) + } + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state) => { + on_duplicate_confirmed(slot, duplicate_confirmed_state) + } + SlotStateUpdate::Duplicate(duplicate_state) => on_duplicate(duplicate_state), } } } -fn on_dead_slot( +fn check_duplicate_confirmed_hash_against_frozen_hash( + state_changes: &mut Vec, slot: Slot, - bank_frozen_hash: &Hash, - cluster_duplicate_confirmed_hash: Option<&Hash>, - _is_slot_duplicate: bool, + duplicate_confirmed_hash: Hash, + bank_frozen_hash: Hash, is_dead: bool, -) -> Vec { - assert!(is_dead); - // Bank should not have been frozen if the slot was marked dead - assert_eq!(*bank_frozen_hash, Hash::default()); - if let Some(cluster_duplicate_confirmed_hash) = cluster_duplicate_confirmed_hash { - // If the cluster duplicate_confirmed some version of this slot, then - // there's another version - warn!( - "Cluster duplicate_confirmed slot {} with hash {}, but we marked slot dead", - slot, cluster_duplicate_confirmed_hash - ); - // No need to check `is_slot_duplicate` and modify fork choice as dead slots - // are never frozen, and thus never added to fork choice. The state change for - // `MarkSlotDuplicate` will try to modify fork choice, but won't find the slot - // in the fork choice tree, so is equivalent to a no-op - return vec![ - ResultingStateChange::MarkSlotDuplicate(Hash::default()), - ResultingStateChange::RepairDuplicateConfirmedVersion( - *cluster_duplicate_confirmed_hash, - ), - ]; - } - - vec![] -} - -fn on_frozen_slot( - slot: Slot, - bank_frozen_hash: &Hash, - cluster_duplicate_confirmed_hash: Option<&Hash>, - is_slot_duplicate: bool, - is_dead: bool, -) -> Vec { - // If a slot is marked frozen, the bank hash should not be default, - // and the slot should not be dead - assert!(*bank_frozen_hash != Hash::default()); - assert!(!is_dead); - - if let Some(cluster_duplicate_confirmed_hash) = cluster_duplicate_confirmed_hash { - // If the cluster duplicate_confirmed some version of this slot, then - // confirm our version agrees with the cluster, - if cluster_duplicate_confirmed_hash != bank_frozen_hash { - // If the versions do not match, modify fork choice rule - // to exclude our version from being voted on and also - // repair correct version +) { + if duplicate_confirmed_hash != bank_frozen_hash { + if is_dead { + // If the cluster duplicate confirmed some version of this slot, then + // there's another version of our dead slot + warn!( + "Cluster duplicate_confirmed slot {} with hash {}, but we marked slot dead", + slot, duplicate_confirmed_hash + ); + } else { + // The duplicate confirmed slot hash does not match our frozen hash. + // Modify fork choice rule to exclude our version from being voted + // on and also repair the correct version warn!( "Cluster duplicate_confirmed slot {} with hash {}, but we froze slot with hash {}", - slot, cluster_duplicate_confirmed_hash, bank_frozen_hash + slot, duplicate_confirmed_hash, bank_frozen_hash ); - return vec![ - ResultingStateChange::MarkSlotDuplicate(*bank_frozen_hash), - ResultingStateChange::RepairDuplicateConfirmedVersion( - *cluster_duplicate_confirmed_hash, - ), - ]; - } else { - // If the versions match, then add the slot to the candidate - // set to account for the case where it was removed earlier - // by the `on_duplicate_slot()` handler - return vec![ResultingStateChange::DuplicateConfirmedSlotMatchesCluster( - *bank_frozen_hash, - )]; + } + state_changes.push(ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash)); + state_changes.push(ResultingStateChange::RepairDuplicateConfirmedVersion( + duplicate_confirmed_hash, + )); + } else { + // If the versions match, then add the slot to the candidate + // set to account for the case where it was removed earlier + // by the `on_duplicate_slot()` handler + state_changes.push(ResultingStateChange::DuplicateConfirmedSlotMatchesCluster( + bank_frozen_hash, + )); + } +} + +fn on_dead_slot(slot: Slot, dead_state: DeadState) -> Vec { + let DeadState { + duplicate_confirmed_hash, + is_slot_duplicate, + } = dead_state; + + let mut state_changes = vec![]; + if let Some(duplicate_confirmed_hash) = duplicate_confirmed_hash { + // If the cluster duplicate_confirmed some version of this slot, then + // check if our version agrees with the cluster, + let bank_hash = Hash::default(); + let is_dead = true; + check_duplicate_confirmed_hash_against_frozen_hash( + &mut state_changes, + slot, + duplicate_confirmed_hash, + bank_hash, + is_dead, + ); + } else if is_slot_duplicate { + state_changes.push(ResultingStateChange::MarkSlotDuplicate(Hash::default())); + } + + state_changes +} + +fn on_frozen_slot(slot: Slot, bank_frozen_state: BankFrozenState) -> Vec { + let BankFrozenState { + frozen_hash, + duplicate_confirmed_hash, + is_slot_duplicate, + } = bank_frozen_state; + let mut state_changes = vec![ResultingStateChange::BankFrozen(frozen_hash)]; + if let Some(duplicate_confirmed_hash) = duplicate_confirmed_hash { + // If the cluster duplicate_confirmed some version of this slot, then + // check if our version agrees with the cluster, + let is_dead = false; + check_duplicate_confirmed_hash_against_frozen_hash( + &mut state_changes, + slot, + duplicate_confirmed_hash, + frozen_hash, + is_dead, + ); + } else if is_slot_duplicate { + state_changes.push(ResultingStateChange::MarkSlotDuplicate(frozen_hash)); + } + + state_changes +} + +fn on_duplicate_confirmed( + slot: Slot, + duplicate_confirmed_state: DuplicateConfirmedState, +) -> Vec { + let DuplicateConfirmedState { + bank_status, + duplicate_confirmed_hash, + } = duplicate_confirmed_state; + + match bank_status { + BankStatus::Dead | BankStatus::Frozen(_) => (), + // No action to be taken yet + BankStatus::Unprocessed => { + return vec![]; } } - if is_slot_duplicate { - // If we detected a duplicate, but have not yet seen any version - // of the slot duplicate_confirmed (i.e. block above did not execute), then - // remove the slot from fork choice until we get confirmation. + let bank_hash = bank_status.bank_hash().expect("bank hash must exist"); + let is_dead = bank_status.is_dead(); - // If we get here, we either detected duplicate from - // 1) WindowService - // 2) A gossip duplicate_confirmed version that didn't match our frozen - // version. - // In both cases, mark the progress map for this slot as duplicate - return vec![ResultingStateChange::MarkSlotDuplicate(*bank_frozen_hash)]; + let mut state_changes = vec![]; + check_duplicate_confirmed_hash_against_frozen_hash( + &mut state_changes, + slot, + duplicate_confirmed_hash, + bank_hash, + is_dead, + ); + + state_changes +} + +fn on_duplicate(duplicate_state: DuplicateState) -> Vec { + let DuplicateState { + bank_status, + duplicate_confirmed_hash, + } = duplicate_state; + + match bank_status { + BankStatus::Dead | BankStatus::Frozen(_) => (), + // No action to be taken yet + BankStatus::Unprocessed => { + return vec![]; + } + } + + let bank_hash = bank_status.bank_hash().expect("bank hash must exist"); + + // If the cluster duplicate_confirmed some version of this slot + // then either the `SlotStateUpdate::DuplicateConfirmed`, `SlotStateUpdate::BankFrozen`, + // or `SlotStateUpdate::Dead` state transitions will take care of marking the fork as + // duplicate if there's a mismatch with our local version. + if duplicate_confirmed_hash.is_none() { + // If we have not yet seen any version of the slot duplicate confirmed, then mark + // the slot as duplicate + return vec![ResultingStateChange::MarkSlotDuplicate(bank_hash)]; } vec![] } -// Called when we receive either: -// 1) A duplicate slot signal from WindowStage, -// 2) Confirmation of a slot by observing votes from replay or gossip. -// -// This signals external information about this slot, which affects -// this validator's understanding of the validity of this slot -fn on_cluster_update( +fn get_duplicate_confirmed_hash_from_state( slot: Slot, - bank_frozen_hash: &Hash, - cluster_duplicate_confirmed_hash: Option<&Hash>, - is_slot_duplicate: bool, - is_dead: bool, -) -> Vec { - if is_dead { - on_dead_slot( - slot, - bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead, - ) - } else if *bank_frozen_hash != Hash::default() { - // This case is mutually exclusive with is_dead case above because if a slot is dead, - // it cannot have been frozen, and thus cannot have a non-default bank hash. - on_frozen_slot( - slot, - bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead, - ) + gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots, + fork_choice: &mut HeaviestSubtreeForkChoice, + bank_frozen_hash: Option, +) -> Option { + let gossip_duplicate_confirmed_hash = gossip_duplicate_confirmed_slots.get(&slot).cloned(); + // If the bank hasn't been frozen yet, then we haven't duplicate confirmed a local version + // this slot through replay yet. + let is_local_replay_duplicate_confirmed = if let Some(bank_frozen_hash) = bank_frozen_hash { + fork_choice + .is_duplicate_confirmed(&(slot, bank_frozen_hash)) + .unwrap_or(false) } else { - vec![] - } + false + }; + + get_duplicate_confirmed_hash( + slot, + gossip_duplicate_confirmed_hash, + bank_frozen_hash, + is_local_replay_duplicate_confirmed, + ) } -fn get_cluster_duplicate_confirmed_hash<'a>( +fn get_duplicate_confirmed_hash( slot: Slot, - gossip_duplicate_confirmed_hash: Option<&'a Hash>, - local_frozen_hash: &'a Hash, + gossip_duplicate_confirmed_hash: Option, + bank_frozen_hash: Option, is_local_replay_duplicate_confirmed: bool, -) -> Option<&'a Hash> { +) -> Option { let local_duplicate_confirmed_hash = if is_local_replay_duplicate_confirmed { // If local replay has duplicate_confirmed this slot, this slot must have // descendants with votes for this slot, hence this slot must be // frozen. - assert!(*local_frozen_hash != Hash::default()); - Some(local_frozen_hash) + let bank_frozen_hash = bank_frozen_hash.unwrap(); + assert!(bank_frozen_hash != Hash::default()); + Some(bank_frozen_hash) } else { None }; @@ -191,9 +448,9 @@ fn get_cluster_duplicate_confirmed_hash<'a>( slot, gossip_duplicate_confirmed_hash, local_duplicate_confirmed_hash ); } - Some(local_frozen_hash) + Some(local_duplicate_confirmed_hash) } - (Some(local_frozen_hash), None) => Some(local_frozen_hash), + (Some(bank_frozen_hash), None) => Some(bank_frozen_hash), _ => gossip_duplicate_confirmed_hash, } } @@ -202,33 +459,55 @@ fn apply_state_changes( slot: Slot, fork_choice: &mut HeaviestSubtreeForkChoice, duplicate_slots_to_repair: &mut DuplicateSlotsToRepair, + blockstore: &Blockstore, state_changes: Vec, ) { + // Handle cases where the bank is frozen, but not duplicate confirmed + // yet. + let mut not_duplicate_confirmed_frozen_hash = None; for state_change in state_changes { match state_change { + ResultingStateChange::BankFrozen(bank_frozen_hash) => { + if !fork_choice + .is_duplicate_confirmed(&(slot, bank_frozen_hash)) + .expect("frozen bank must exist in fork choice") + { + not_duplicate_confirmed_frozen_hash = Some(bank_frozen_hash); + } + } ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash) => { fork_choice.mark_fork_invalid_candidate(&(slot, bank_frozen_hash)); } - ResultingStateChange::RepairDuplicateConfirmedVersion( - cluster_duplicate_confirmed_hash, - ) => { - duplicate_slots_to_repair.insert((slot, cluster_duplicate_confirmed_hash)); + ResultingStateChange::RepairDuplicateConfirmedVersion(duplicate_confirmed_hash) => { + duplicate_slots_to_repair.insert((slot, duplicate_confirmed_hash)); } ResultingStateChange::DuplicateConfirmedSlotMatchesCluster(bank_frozen_hash) => { - fork_choice.mark_fork_valid_candidate(&(slot, bank_frozen_hash)); + not_duplicate_confirmed_frozen_hash = None; + // When we detect that our frozen slot matches the cluster version (note this + // will catch both bank frozen first -> confirmation, or confirmation first -> + // bank frozen), mark all the newly duplicate confirmed slots in blockstore + let new_duplicate_confirmed_slot_hashes = + fork_choice.mark_fork_valid_candidate(&(slot, bank_frozen_hash)); + blockstore + .set_duplicate_confirmed_slots_and_hashes( + new_duplicate_confirmed_slot_hashes.into_iter(), + ) + .unwrap(); } } } + + if let Some(frozen_hash) = not_duplicate_confirmed_frozen_hash { + blockstore.insert_bank_hash(slot, frozen_hash, false); + } } #[allow(clippy::too_many_arguments)] pub(crate) fn check_slot_agrees_with_cluster( slot: Slot, root: Slot, - frozen_hash: Option, + blockstore: &Blockstore, duplicate_slots_tracker: &mut DuplicateSlotsTracker, - gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots, - progress: &ProgressMap, fork_choice: &mut HeaviestSubtreeForkChoice, duplicate_slots_to_repair: &mut HashSet<(Slot, Hash)>, slot_state_update: SlotStateUpdate, @@ -237,75 +516,37 @@ pub(crate) fn check_slot_agrees_with_cluster( "check_slot_agrees_with_cluster() slot: {}, root: {}, - frozen_hash: {:?}, - update: {:?}", - slot, root, frozen_hash, slot_state_update + slot_state_update: {:?}", + slot, root, slot_state_update ); if slot <= root { return; } - // Needs to happen before the frozen_hash.is_none() check below to account for duplicate + // Needs to happen before the bank_frozen_hash.is_none() check below to account for duplicate // signals arriving before the bank is constructed in replay. - if matches!(slot_state_update, SlotStateUpdate::Duplicate) { + if matches!(slot_state_update, SlotStateUpdate::Duplicate(_)) { // If this slot has already been processed before, return if !duplicate_slots_tracker.insert(slot) { return; } } - if frozen_hash.is_none() { - // If the bank doesn't even exist in BankForks yet, - // then there's nothing to do as replay of the slot - // hasn't even started - return; - } - - let frozen_hash = frozen_hash.unwrap(); - let gossip_duplicate_confirmed_hash = gossip_duplicate_confirmed_slots.get(&slot); - - // If the bank hasn't been frozen yet, then we haven't duplicate confirmed a local version - // this slot through replay yet. - let is_local_replay_duplicate_confirmed = fork_choice - .is_duplicate_confirmed(&(slot, frozen_hash)) - .unwrap_or(false); - let cluster_duplicate_confirmed_hash = get_cluster_duplicate_confirmed_hash( + let state_changes = slot_state_update.into_state_changes(slot); + apply_state_changes( slot, - gossip_duplicate_confirmed_hash, - &frozen_hash, - is_local_replay_duplicate_confirmed, + fork_choice, + duplicate_slots_to_repair, + blockstore, + state_changes, ); - let is_slot_duplicate = duplicate_slots_tracker.contains(&slot); - let is_dead = progress.is_dead(slot).expect("If the frozen hash exists, then the slot must exist in bank forks and thus in progress map"); - - info!( - "check_slot_agrees_with_cluster() state - is_local_replay_duplicate_confirmed: {:?}, - cluster_duplicate_confirmed_hash: {:?}, - is_slot_duplicate: {:?}, - is_dead: {:?}", - is_local_replay_duplicate_confirmed, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead, - ); - - let state_handler = slot_state_update.to_handler(); - let state_changes = state_handler( - slot, - &frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead, - ); - apply_state_changes(slot, fork_choice, duplicate_slots_to_repair, state_changes); } #[cfg(test)] mod test { use super::*; - use crate::vote_simulator::VoteSimulator; + use crate::{progress_map::ProgressMap, replay_stage::tests::setup_forks_from_tree}; use solana_runtime::bank_forks::BankForks; use std::{ collections::{HashMap, HashSet}, @@ -313,18 +554,301 @@ mod test { }; use trees::tr; + macro_rules! state_update_tests { + ($($name:ident: $value:expr,)*) => { + $( + #[test] + fn $name() { + let slot = 10; + let (state_update, expected) = $value; + assert_eq!(expected, state_update.into_state_changes(slot)); + } + )* + } + } + + state_update_tests! { + bank_frozen_state_update_0: { + // frozen hash has to be non-default for frozen state transition + let frozen_hash = Hash::new_unique(); + let duplicate_confirmed_hash = None; + let is_slot_duplicate = false; + let bank_frozen_state = BankFrozenState::new( + frozen_hash, + duplicate_confirmed_hash, + is_slot_duplicate, + ); + ( + SlotStateUpdate::BankFrozen(bank_frozen_state), + vec![ResultingStateChange::BankFrozen(frozen_hash)] + ) + }, + bank_frozen_state_update_1: { + // frozen hash has to be non-default for frozen state transition + let frozen_hash = Hash::new_unique(); + let duplicate_confirmed_hash = None; + let is_slot_duplicate = true; + let bank_frozen_state = BankFrozenState::new( + frozen_hash, + duplicate_confirmed_hash, + is_slot_duplicate, + ); + ( + SlotStateUpdate::BankFrozen(bank_frozen_state), + vec![ResultingStateChange::BankFrozen(frozen_hash), ResultingStateChange::MarkSlotDuplicate(frozen_hash)] + ) + }, + bank_frozen_state_update_2: { + // frozen hash has to be non-default for frozen state transition + let frozen_hash = Hash::new_unique(); + let duplicate_confirmed_hash = Some(frozen_hash); + let is_slot_duplicate = false; + let bank_frozen_state = BankFrozenState::new( + frozen_hash, + duplicate_confirmed_hash, + is_slot_duplicate, + ); + ( + SlotStateUpdate::BankFrozen(bank_frozen_state), + vec![ResultingStateChange::BankFrozen(frozen_hash), + ResultingStateChange::DuplicateConfirmedSlotMatchesCluster(frozen_hash)] + ) + }, + bank_frozen_state_update_3: { + // frozen hash has to be non-default for frozen state transition + let frozen_hash = Hash::new_unique(); + let duplicate_confirmed_hash = Some(frozen_hash); + let is_slot_duplicate = true; + let bank_frozen_state = BankFrozenState::new( + frozen_hash, + duplicate_confirmed_hash, + is_slot_duplicate, + ); + ( + SlotStateUpdate::BankFrozen(bank_frozen_state), + vec![ResultingStateChange::BankFrozen(frozen_hash), + ResultingStateChange::DuplicateConfirmedSlotMatchesCluster(frozen_hash)] + ) + }, + bank_frozen_state_update_4: { + // frozen hash has to be non-default for frozen state transition + let frozen_hash = Hash::new_unique(); + let duplicate_confirmed_hash = Some(Hash::new_unique()); + let is_slot_duplicate = false; + let bank_frozen_state = BankFrozenState::new( + frozen_hash, + duplicate_confirmed_hash, + is_slot_duplicate, + ); + ( + SlotStateUpdate::BankFrozen(bank_frozen_state), + vec![ResultingStateChange::BankFrozen(frozen_hash), + ResultingStateChange::MarkSlotDuplicate(frozen_hash), + ResultingStateChange::RepairDuplicateConfirmedVersion(duplicate_confirmed_hash.unwrap())], + ) + }, + bank_frozen_state_update_5: { + // frozen hash has to be non-default for frozen state transition + let frozen_hash = Hash::new_unique(); + let duplicate_confirmed_hash = Some(Hash::new_unique()); + let is_slot_duplicate = true; + let bank_frozen_state = BankFrozenState::new( + frozen_hash, + duplicate_confirmed_hash, + is_slot_duplicate, + ); + ( + SlotStateUpdate::BankFrozen(bank_frozen_state), + vec![ResultingStateChange::BankFrozen(frozen_hash), + ResultingStateChange::MarkSlotDuplicate(frozen_hash), + ResultingStateChange::RepairDuplicateConfirmedVersion(duplicate_confirmed_hash.unwrap())], + ) + }, + duplicate_confirmed_state_update_0: { + let duplicate_confirmed_hash = Hash::new_unique(); + let bank_status = BankStatus::Unprocessed; + let duplicate_confirmed_state = DuplicateConfirmedState::new( + duplicate_confirmed_hash, + bank_status, + ); + ( + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), + Vec::::new() + ) + }, + duplicate_confirmed_state_update_1: { + let duplicate_confirmed_hash = Hash::new_unique(); + let bank_status = BankStatus::Dead; + let duplicate_confirmed_state = DuplicateConfirmedState::new( + duplicate_confirmed_hash, + bank_status, + ); + ( + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), + vec![ + ResultingStateChange::MarkSlotDuplicate(Hash::default()), + ResultingStateChange::RepairDuplicateConfirmedVersion(duplicate_confirmed_hash)], + ) + }, + duplicate_confirmed_state_update_2: { + let duplicate_confirmed_hash = Hash::new_unique(); + let bank_status = BankStatus::Frozen(duplicate_confirmed_hash); + let duplicate_confirmed_state = DuplicateConfirmedState::new( + duplicate_confirmed_hash, + bank_status, + ); + ( + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), + vec![ + ResultingStateChange::DuplicateConfirmedSlotMatchesCluster(duplicate_confirmed_hash)] + ) + }, + duplicate_confirmed_state_update_3: { + let duplicate_confirmed_hash = Hash::new_unique(); + let frozen_hash = Hash::new_unique(); + let bank_status = BankStatus::Frozen(frozen_hash); + let duplicate_confirmed_state = DuplicateConfirmedState::new( + duplicate_confirmed_hash, + bank_status, + ); + ( + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), + vec![ + ResultingStateChange::MarkSlotDuplicate(frozen_hash), + ResultingStateChange::RepairDuplicateConfirmedVersion(duplicate_confirmed_hash)], + ) + }, + dead_state_update_0: { + let duplicate_confirmed_hash = None; + let is_slot_duplicate = false; + let dead_state = DeadState::new( + duplicate_confirmed_hash, + is_slot_duplicate, + ); + ( + SlotStateUpdate::Dead(dead_state), + Vec::::new() + ) + }, + dead_state_update_1: { + let duplicate_confirmed_hash = None; + let is_slot_duplicate = true; + let dead_state = DeadState::new( + duplicate_confirmed_hash, + is_slot_duplicate, + ); + ( + SlotStateUpdate::Dead(dead_state), + vec![ResultingStateChange::MarkSlotDuplicate(Hash::default())], + ) + }, + dead_state_update_2: { + let duplicate_confirmed_hash = Some(Hash::new_unique()); + let is_slot_duplicate = false; + let dead_state = DeadState::new( + duplicate_confirmed_hash, + is_slot_duplicate, + ); + ( + SlotStateUpdate::Dead(dead_state), + vec![ + ResultingStateChange::MarkSlotDuplicate(Hash::default()), + ResultingStateChange::RepairDuplicateConfirmedVersion(duplicate_confirmed_hash.unwrap())], + ) + }, + dead_state_update_3: { + let duplicate_confirmed_hash = Some(Hash::new_unique()); + let is_slot_duplicate = true; + let dead_state = DeadState::new( + duplicate_confirmed_hash, + is_slot_duplicate, + ); + ( + SlotStateUpdate::Dead(dead_state), + vec![ + ResultingStateChange::MarkSlotDuplicate(Hash::default()), + ResultingStateChange::RepairDuplicateConfirmedVersion(duplicate_confirmed_hash.unwrap())], + ) + }, + duplicate_state_update_0: { + let duplicate_confirmed_hash = None; + let bank_status = BankStatus::Unprocessed; + let duplicate_state = DuplicateState::new(duplicate_confirmed_hash, bank_status); + ( + SlotStateUpdate::Duplicate(duplicate_state), + Vec::::new() + ) + }, + duplicate_state_update_1: { + let duplicate_confirmed_hash = None; + let bank_status = BankStatus::Dead; + let duplicate_state = DuplicateState::new(duplicate_confirmed_hash, bank_status); + ( + SlotStateUpdate::Duplicate(duplicate_state), + vec![ResultingStateChange::MarkSlotDuplicate(Hash::default())], + ) + }, + duplicate_state_update_2: { + let duplicate_confirmed_hash = None; + let bank_hash = Hash::new_unique(); + let bank_status = BankStatus::Frozen(bank_hash); + let duplicate_state = DuplicateState::new(duplicate_confirmed_hash, bank_status); + ( + SlotStateUpdate::Duplicate(duplicate_state), + vec![ResultingStateChange::MarkSlotDuplicate(bank_hash)], + ) + }, + duplicate_state_update_3: { + let duplicate_confirmed_hash = Some(Hash::new_unique()); + let bank_status = BankStatus::Unprocessed; + let duplicate_state = DuplicateState::new(duplicate_confirmed_hash, bank_status); + ( + SlotStateUpdate::Duplicate(duplicate_state), + Vec::::new(), + ) + }, + duplicate_state_update_4: { + let duplicate_confirmed_hash = Some(Hash::new_unique()); + let bank_status = BankStatus::Dead; + let duplicate_state = DuplicateState::new(duplicate_confirmed_hash, bank_status); + ( + SlotStateUpdate::Duplicate(duplicate_state), + Vec::::new() + ) + }, + duplicate_state_update_5: { + let duplicate_confirmed_hash = Some(Hash::new_unique()); + let bank_status = BankStatus::Frozen(duplicate_confirmed_hash.unwrap()); + let duplicate_state = DuplicateState::new(duplicate_confirmed_hash, bank_status); + ( + SlotStateUpdate::Duplicate(duplicate_state), + Vec::::new() + ) + }, + duplicate_state_update_6: { + let duplicate_confirmed_hash = Some(Hash::new_unique()); + let frozen_hash = Hash::new_unique(); + let bank_status = BankStatus::Frozen(frozen_hash); + let duplicate_state = DuplicateState::new(duplicate_confirmed_hash, bank_status); + ( + SlotStateUpdate::Duplicate(duplicate_state), + Vec::::new(), + ) + }, + } + struct InitialState { heaviest_subtree_fork_choice: HeaviestSubtreeForkChoice, progress: ProgressMap, descendants: HashMap>, bank_forks: Arc>, + blockstore: Blockstore, } fn setup() -> InitialState { // Create simple fork 0 -> 1 -> 2 -> 3 let forks = tr(0) / (tr(1) / (tr(2) / tr(3))); - let mut vote_simulator = VoteSimulator::new(1); - vote_simulator.fill_bank_forks(forks, &HashMap::new()); + let (vote_simulator, blockstore) = setup_forks_from_tree(forks, 1, None); let descendants = vote_simulator .bank_forks @@ -338,270 +862,10 @@ mod test { progress: vote_simulator.progress, descendants, bank_forks: vote_simulator.bank_forks, + blockstore, } } - #[test] - fn test_frozen_duplicate() { - // Common state - let slot = 0; - let cluster_duplicate_confirmed_hash = None; - let is_dead = false; - - // Slot is not detected as duplicate yet - let mut is_slot_duplicate = false; - - // Simulate freezing the bank, add a - // new non-default hash, should return - // no actionable state changes yet - let bank_frozen_hash = Hash::new_unique(); - assert!(on_frozen_slot( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ) - .is_empty()); - - // Now mark the slot as duplicate, should - // trigger marking the slot as a duplicate - is_slot_duplicate = true; - assert_eq!( - on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ), - vec![ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash)] - ); - } - - #[test] - fn test_frozen_duplicate_confirmed() { - // Common state - let slot = 0; - let is_slot_duplicate = false; - let is_dead = false; - - // No cluster duplicate_confirmed hash yet - let mut cluster_duplicate_confirmed_hash = None; - - // Simulate freezing the bank, add a - // new non-default hash, should return - // no actionable state changes - let bank_frozen_hash = Hash::new_unique(); - assert!(on_frozen_slot( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ) - .is_empty()); - - // Now mark the same frozen slot hash as duplicate_confirmed by the cluster, - // should just confirm the slot - cluster_duplicate_confirmed_hash = Some(&bank_frozen_hash); - assert_eq!( - on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ), - vec![ResultingStateChange::DuplicateConfirmedSlotMatchesCluster( - bank_frozen_hash - ),] - ); - - // If the cluster_duplicate_confirmed_hash does not match, then we - // should trigger marking the slot as a duplicate, and also - // try to repair correct version - let mismatched_hash = Hash::new_unique(); - cluster_duplicate_confirmed_hash = Some(&mismatched_hash); - assert_eq!( - on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ), - vec![ - ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash), - ResultingStateChange::RepairDuplicateConfirmedVersion(mismatched_hash), - ] - ); - } - - #[test] - fn test_duplicate_frozen_duplicate_confirmed() { - // Common state - let slot = 0; - let is_dead = false; - let is_slot_duplicate = true; - - // Bank is not frozen yet - let mut cluster_duplicate_confirmed_hash = None; - let mut bank_frozen_hash = Hash::default(); - - // Mark the slot as duplicate. Because our version of the slot is not - // frozen yet, we don't know which version we have, so no action is - // taken. - assert!(on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ) - .is_empty()); - - // Freeze the bank, should now mark the slot as duplicate since we have - // not seen confirmation yet. - bank_frozen_hash = Hash::new_unique(); - assert_eq!( - on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ), - vec![ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash),] - ); - - // If the cluster_duplicate_confirmed_hash matches, we just confirm - // the slot - cluster_duplicate_confirmed_hash = Some(&bank_frozen_hash); - assert_eq!( - on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ), - vec![ResultingStateChange::DuplicateConfirmedSlotMatchesCluster( - bank_frozen_hash - ),] - ); - - // If the cluster_duplicate_confirmed_hash does not match, then we - // should trigger marking the slot as a duplicate, and also - // try to repair correct version - let mismatched_hash = Hash::new_unique(); - cluster_duplicate_confirmed_hash = Some(&mismatched_hash); - assert_eq!( - on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ), - vec![ - ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash), - ResultingStateChange::RepairDuplicateConfirmedVersion(mismatched_hash), - ] - ); - } - - #[test] - fn test_duplicate_duplicate_confirmed() { - let slot = 0; - let correct_hash = Hash::new_unique(); - let cluster_duplicate_confirmed_hash = Some(&correct_hash); - let is_dead = false; - // Bank is not frozen yet - let bank_frozen_hash = Hash::default(); - - // Because our version of the slot is not frozen yet, then even though - // the cluster has duplicate_confirmed a hash, we don't know which version we - // have, so no action is taken. - let is_slot_duplicate = true; - assert!(on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ) - .is_empty()); - } - - #[test] - fn test_duplicate_dead() { - let slot = 0; - let cluster_duplicate_confirmed_hash = None; - let is_dead = true; - // Bank is not frozen yet - let bank_frozen_hash = Hash::default(); - - // Even though our version of the slot is dead, the cluster has not - // duplicate_confirmed a hash, we don't know which version we have, so no action - // is taken. - let is_slot_duplicate = true; - assert!(on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ) - .is_empty()); - } - - #[test] - fn test_duplicate_confirmed_dead_duplicate() { - let slot = 0; - let correct_hash = Hash::new_unique(); - // Cluster has duplicate_confirmed some version of the slot - let cluster_duplicate_confirmed_hash = Some(&correct_hash); - // Our version of the slot is dead - let is_dead = true; - let bank_frozen_hash = Hash::default(); - - // Even if the duplicate signal hasn't come in yet, - // we can deduce the slot is duplicate AND we have, - // the wrong version, so should mark the slot as duplicate, - // and repair the correct version - let mut is_slot_duplicate = false; - assert_eq!( - on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ), - vec![ - ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash), - ResultingStateChange::RepairDuplicateConfirmedVersion(correct_hash), - ] - ); - - // If the duplicate signal comes in, nothing should change - is_slot_duplicate = true; - assert_eq!( - on_cluster_update( - slot, - &bank_frozen_hash, - cluster_duplicate_confirmed_hash, - is_slot_duplicate, - is_dead - ), - vec![ - ResultingStateChange::MarkSlotDuplicate(bank_frozen_hash), - ResultingStateChange::RepairDuplicateConfirmedVersion(correct_hash), - ] - ); - } - #[test] fn test_apply_state_changes() { // Common state @@ -609,6 +873,7 @@ mod test { mut heaviest_subtree_fork_choice, descendants, bank_forks, + blockstore, .. } = setup(); @@ -627,6 +892,7 @@ mod test { duplicate_slot, &mut heaviest_subtree_fork_choice, &mut duplicate_slots_to_repair, + &blockstore, vec![ResultingStateChange::MarkSlotDuplicate(duplicate_slot_hash)], ); assert!(!heaviest_subtree_fork_choice @@ -650,15 +916,121 @@ mod test { } assert!(duplicate_slots_to_repair.is_empty()); - // DuplicateConfirmedSlotMatchesCluster should re-enable fork choice + // Simulate detecting another hash that is the correct version, + // RepairDuplicateConfirmedVersion should add the slot to repair + // to `duplicate_slots_to_repair` + assert!(duplicate_slots_to_repair.is_empty()); + let correct_hash = Hash::new_unique(); apply_state_changes( duplicate_slot, &mut heaviest_subtree_fork_choice, &mut duplicate_slots_to_repair, - vec![ResultingStateChange::DuplicateConfirmedSlotMatchesCluster( - duplicate_slot_hash, + &blockstore, + vec![ResultingStateChange::RepairDuplicateConfirmedVersion( + correct_hash, )], ); + assert_eq!(duplicate_slots_to_repair.len(), 1); + assert!(duplicate_slots_to_repair.contains(&(duplicate_slot, correct_hash))); + } + + #[test] + fn test_apply_state_changes_bank_frozen() { + // Common state + let InitialState { + mut heaviest_subtree_fork_choice, + bank_forks, + blockstore, + .. + } = setup(); + + let duplicate_slot = bank_forks.read().unwrap().root() + 1; + let duplicate_slot_hash = bank_forks + .read() + .unwrap() + .get(duplicate_slot) + .unwrap() + .hash(); + let mut duplicate_slots_to_repair = DuplicateSlotsToRepair::default(); + + // Simulate ReplayStage freezing a Bank with the given hash. + // BankFrozen should mark it down in Blockstore. + assert!(blockstore.get_bank_hash(duplicate_slot).is_none()); + apply_state_changes( + duplicate_slot, + &mut heaviest_subtree_fork_choice, + &mut duplicate_slots_to_repair, + &blockstore, + vec![ResultingStateChange::BankFrozen(duplicate_slot_hash)], + ); + assert_eq!( + blockstore.get_bank_hash(duplicate_slot).unwrap(), + duplicate_slot_hash + ); + assert!(!blockstore.is_duplicate_confirmed(duplicate_slot)); + + // If we freeze another version of the bank, it should overwrite the first + // version in blockstore. + let new_bank_hash = Hash::new_unique(); + let root_slot_hash = { + let root_bank = bank_forks.read().unwrap().root_bank(); + (root_bank.slot(), root_bank.hash()) + }; + heaviest_subtree_fork_choice + .add_new_leaf_slot((duplicate_slot, new_bank_hash), Some(root_slot_hash)); + apply_state_changes( + duplicate_slot, + &mut heaviest_subtree_fork_choice, + &mut duplicate_slots_to_repair, + &blockstore, + vec![ResultingStateChange::BankFrozen(new_bank_hash)], + ); + assert_eq!( + blockstore.get_bank_hash(duplicate_slot).unwrap(), + new_bank_hash + ); + assert!(!blockstore.is_duplicate_confirmed(duplicate_slot)); + } + + fn run_test_apply_state_changes_duplicate_confirmed_matches_frozen( + modify_state_changes: impl Fn(Hash, &mut Vec), + ) { + // Common state + let InitialState { + mut heaviest_subtree_fork_choice, + descendants, + bank_forks, + blockstore, + .. + } = setup(); + + let duplicate_slot = bank_forks.read().unwrap().root() + 1; + let our_duplicate_slot_hash = bank_forks + .read() + .unwrap() + .get(duplicate_slot) + .unwrap() + .hash(); + let mut duplicate_slots_to_repair = DuplicateSlotsToRepair::default(); + + // Setup and check the state that is about to change. + assert!(blockstore.get_bank_hash(duplicate_slot).is_none()); + assert!(!blockstore.is_duplicate_confirmed(duplicate_slot)); + + // DuplicateConfirmedSlotMatchesCluster should: + // 1) Re-enable fork choice + // 2) Set the status to duplicate confirmed in Blockstore + let mut state_changes = vec![ResultingStateChange::DuplicateConfirmedSlotMatchesCluster( + our_duplicate_slot_hash, + )]; + modify_state_changes(our_duplicate_slot_hash, &mut state_changes); + apply_state_changes( + duplicate_slot, + &mut heaviest_subtree_fork_choice, + &mut duplicate_slots_to_repair, + &blockstore, + state_changes, + ); for child_slot in descendants .get(&duplicate_slot) .unwrap() @@ -673,24 +1045,29 @@ mod test { .is_none()); } assert!(heaviest_subtree_fork_choice - .is_candidate(&(duplicate_slot, duplicate_slot_hash)) + .is_candidate(&(duplicate_slot, our_duplicate_slot_hash)) .unwrap()); - assert!(duplicate_slots_to_repair.is_empty()); - - // Simulate detecting another hash that is the correct version, - // RepairDuplicateConfirmedVersion should add the slot to repair - // to `duplicate_slots_to_repair` - let correct_hash = Hash::new_unique(); - apply_state_changes( - duplicate_slot, - &mut heaviest_subtree_fork_choice, - &mut duplicate_slots_to_repair, - vec![ResultingStateChange::RepairDuplicateConfirmedVersion( - correct_hash, - )], + assert_eq!( + blockstore.get_bank_hash(duplicate_slot).unwrap(), + our_duplicate_slot_hash + ); + assert!(blockstore.is_duplicate_confirmed(duplicate_slot)); + } + + #[test] + fn test_apply_state_changes_duplicate_confirmed_matches_frozen() { + run_test_apply_state_changes_duplicate_confirmed_matches_frozen( + |_our_duplicate_slot_hash, _state_changes: &mut Vec| {}, + ); + } + + #[test] + fn test_apply_state_changes_bank_frozen_and_duplicate_confirmed_matches_frozen() { + run_test_apply_state_changes_duplicate_confirmed_matches_frozen( + |our_duplicate_slot_hash, state_changes: &mut Vec| { + state_changes.push(ResultingStateChange::BankFrozen(our_duplicate_slot_hash)); + }, ); - assert_eq!(duplicate_slots_to_repair.len(), 1); - assert!(duplicate_slots_to_repair.contains(&(duplicate_slot, correct_hash))); } fn run_test_state_duplicate_then_bank_frozen(initial_bank_hash: Option) { @@ -699,6 +1076,7 @@ mod test { mut heaviest_subtree_fork_choice, progress, bank_forks, + blockstore, .. } = setup(); @@ -711,16 +1089,21 @@ mod test { let gossip_duplicate_confirmed_slots = GossipDuplicateConfirmedSlots::default(); let mut duplicate_slots_to_repair = DuplicateSlotsToRepair::default(); let duplicate_slot = 2; + let duplicate_state = DuplicateState::new_from_state( + duplicate_slot, + &gossip_duplicate_confirmed_slots, + &mut heaviest_subtree_fork_choice, + || progress.is_dead(duplicate_slot).unwrap_or(false), + || initial_bank_hash, + ); check_slot_agrees_with_cluster( duplicate_slot, root, - initial_bank_hash, + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut heaviest_subtree_fork_choice, &mut duplicate_slots_to_repair, - SlotStateUpdate::Duplicate, + SlotStateUpdate::Duplicate(duplicate_state), ); assert!(duplicate_slots_tracker.contains(&duplicate_slot)); // Nothing should be applied yet to fork choice, since bank was not yet frozen @@ -738,16 +1121,21 @@ mod test { .get(duplicate_slot) .unwrap() .hash(); + let bank_frozen_state = BankFrozenState::new_from_state( + duplicate_slot, + frozen_duplicate_slot_hash, + &mut duplicate_slots_tracker, + &gossip_duplicate_confirmed_slots, + &mut heaviest_subtree_fork_choice, + ); check_slot_agrees_with_cluster( duplicate_slot, root, - Some(frozen_duplicate_slot_hash), + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut heaviest_subtree_fork_choice, &mut duplicate_slots_to_repair, - SlotStateUpdate::Frozen, + SlotStateUpdate::BankFrozen(bank_frozen_state), ); // Progress map should have the correct updates, fork choice should mark duplicate @@ -785,6 +1173,7 @@ mod test { mut heaviest_subtree_fork_choice, progress, bank_forks, + blockstore, .. } = setup(); @@ -800,16 +1189,19 @@ mod test { // Mark slot 2 as duplicate confirmed let slot2_hash = bank_forks.read().unwrap().get(2).unwrap().hash(); gossip_duplicate_confirmed_slots.insert(2, slot2_hash); + let duplicate_confirmed_state = DuplicateConfirmedState::new_from_state( + slot2_hash, + || progress.is_dead(2).unwrap_or(false), + || Some(slot2_hash), + ); check_slot_agrees_with_cluster( 2, root, - Some(slot2_hash), + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut heaviest_subtree_fork_choice, &mut DuplicateSlotsToRepair::default(), - SlotStateUpdate::DuplicateConfirmed, + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), ); assert!(heaviest_subtree_fork_choice .is_duplicate_confirmed(&(2, slot2_hash)) @@ -830,16 +1222,21 @@ mod test { // Mark 3 as duplicate, should not remove the duplicate confirmed slot 2 from // fork choice + let duplicate_state = DuplicateState::new_from_state( + 3, + &gossip_duplicate_confirmed_slots, + &mut heaviest_subtree_fork_choice, + || progress.is_dead(3).unwrap_or(false), + || Some(slot3_hash), + ); check_slot_agrees_with_cluster( 3, root, - Some(slot3_hash), + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut heaviest_subtree_fork_choice, &mut DuplicateSlotsToRepair::default(), - SlotStateUpdate::Duplicate, + SlotStateUpdate::Duplicate(duplicate_state), ); assert!(duplicate_slots_tracker.contains(&3)); assert_eq!( @@ -876,6 +1273,7 @@ mod test { mut heaviest_subtree_fork_choice, progress, bank_forks, + blockstore, .. } = setup(); @@ -889,16 +1287,22 @@ mod test { let mut gossip_duplicate_confirmed_slots = GossipDuplicateConfirmedSlots::default(); // Mark 2 as duplicate + let slot2_hash = bank_forks.read().unwrap().get(2).unwrap().hash(); + let duplicate_state = DuplicateState::new_from_state( + 2, + &gossip_duplicate_confirmed_slots, + &mut heaviest_subtree_fork_choice, + || progress.is_dead(2).unwrap_or(false), + || Some(slot2_hash), + ); check_slot_agrees_with_cluster( 2, root, - Some(bank_forks.read().unwrap().get(2).unwrap().hash()), + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut heaviest_subtree_fork_choice, &mut DuplicateSlotsToRepair::default(), - SlotStateUpdate::Duplicate, + SlotStateUpdate::Duplicate(duplicate_state), ); assert!(duplicate_slots_tracker.contains(&2)); for slot in 2..=3 { @@ -919,16 +1323,19 @@ mod test { // Mark slot 3 as duplicate confirmed, should mark slot 2 as duplicate confirmed as well gossip_duplicate_confirmed_slots.insert(3, slot3_hash); + let duplicate_confirmed_state = DuplicateConfirmedState::new_from_state( + slot3_hash, + || progress.is_dead(3).unwrap_or(false), + || Some(slot3_hash), + ); check_slot_agrees_with_cluster( 3, root, - Some(slot3_hash), + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut heaviest_subtree_fork_choice, &mut DuplicateSlotsToRepair::default(), - SlotStateUpdate::DuplicateConfirmed, + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), ); for slot in 0..=3 { let slot_hash = bank_forks.read().unwrap().get(slot).unwrap().hash(); @@ -952,6 +1359,7 @@ mod test { mut heaviest_subtree_fork_choice, progress, bank_forks, + blockstore, .. } = setup(); @@ -967,16 +1375,19 @@ mod test { // Mark 3 as duplicate confirmed gossip_duplicate_confirmed_slots.insert(3, slot3_hash); + let duplicate_confirmed_state = DuplicateConfirmedState::new_from_state( + slot3_hash, + || progress.is_dead(3).unwrap_or(false), + || Some(slot3_hash), + ); check_slot_agrees_with_cluster( 3, root, - Some(slot3_hash), + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut heaviest_subtree_fork_choice, &mut duplicate_slots_to_repair, - SlotStateUpdate::DuplicateConfirmed, + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), ); let verify_all_slots_duplicate_confirmed = |bank_forks: &RwLock, @@ -1001,16 +1412,21 @@ mod test { // slot 1 was duplicate confirmed by the confirmation on its // descendant, 3. let slot1_hash = bank_forks.read().unwrap().get(1).unwrap().hash(); + let duplicate_state = DuplicateState::new_from_state( + 1, + &gossip_duplicate_confirmed_slots, + &mut heaviest_subtree_fork_choice, + || progress.is_dead(1).unwrap_or(false), + || Some(slot1_hash), + ); check_slot_agrees_with_cluster( 1, root, - Some(slot1_hash), + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut heaviest_subtree_fork_choice, &mut duplicate_slots_to_repair, - SlotStateUpdate::Duplicate, + SlotStateUpdate::Duplicate(duplicate_state), ); assert!(duplicate_slots_tracker.contains(&1)); verify_all_slots_duplicate_confirmed(&bank_forks, &heaviest_subtree_fork_choice); diff --git a/core/src/fork_choice.rs b/core/src/fork_choice.rs index e31b6e5b03..f358be303c 100644 --- a/core/src/fork_choice.rs +++ b/core/src/fork_choice.rs @@ -40,5 +40,10 @@ pub trait ForkChoice { fn mark_fork_invalid_candidate(&mut self, invalid_slot: &Self::ForkChoiceKey); - fn mark_fork_valid_candidate(&mut self, valid_slot: &Self::ForkChoiceKey); + /// Returns any newly duplicate confirmed ancestors of `valid_slot` up to and including + /// `valid_slot` itself + fn mark_fork_valid_candidate( + &mut self, + valid_slot: &Self::ForkChoiceKey, + ) -> Vec; } diff --git a/core/src/heaviest_subtree_fork_choice.rs b/core/src/heaviest_subtree_fork_choice.rs index b01630742c..02b8df40de 100644 --- a/core/src/heaviest_subtree_fork_choice.rs +++ b/core/src/heaviest_subtree_fork_choice.rs @@ -1007,11 +1007,21 @@ impl ForkChoice for HeaviestSubtreeForkChoice { } } - fn mark_fork_valid_candidate(&mut self, valid_slot_hash_key: &SlotHashKey) { + fn mark_fork_valid_candidate(&mut self, valid_slot_hash_key: &SlotHashKey) -> Vec { info!( "marking fork starting at: {:?} valid candidate", valid_slot_hash_key ); + let mut newly_duplicate_confirmed_ancestors = vec![]; + + for ancestor_key in std::iter::once(*valid_slot_hash_key) + .chain(self.ancestor_iterator(*valid_slot_hash_key)) + { + if !self.is_duplicate_confirmed(&ancestor_key).unwrap() { + newly_duplicate_confirmed_ancestors.push(ancestor_key); + } + } + let mut update_operations = UpdateOperations::default(); // Notify all the children of this node that a parent was marked as valid for child_hash_key in self.subtree_diff(*valid_slot_hash_key, SlotHashKey::default()) { @@ -1025,6 +1035,7 @@ impl ForkChoice for HeaviestSubtreeForkChoice { // Aggregate across all ancestors to find the new best slots including this fork self.insert_aggregate_operations(&mut update_operations, *valid_slot_hash_key); self.process_update_operations(update_operations); + newly_duplicate_confirmed_ancestors } } diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 527b627177..17ae19cb96 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -55,7 +55,7 @@ use solana_sdk::{ }; use solana_vote_program::vote_state::Vote; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{HashMap, HashSet}, result, sync::{ atomic::{AtomicBool, Ordering}, @@ -427,6 +427,7 @@ impl ReplayStage { let mut process_gossip_duplicate_confirmed_slots_time = Measure::start("process_gossip_duplicate_confirmed_slots"); Self::process_gossip_duplicate_confirmed_slots( &gossip_duplicate_confirmed_slots_receiver, + &blockstore, &mut duplicate_slots_tracker, &mut gossip_duplicate_confirmed_slots, &bank_forks, @@ -455,6 +456,7 @@ impl ReplayStage { let mut process_duplicate_slots_time = Measure::start("process_duplicate_slots"); if !tpu_has_bank { Self::process_duplicate_slots( + &blockstore, &duplicate_slots_receiver, &mut duplicate_slots_tracker, &gossip_duplicate_confirmed_slots, @@ -503,7 +505,7 @@ impl ReplayStage { &bank_forks, ); - Self::mark_slots_confirmed(&confirmed_forks, &bank_forks, &mut progress, &mut duplicate_slots_tracker, &mut heaviest_subtree_fork_choice, &mut duplicate_slots_to_repair); + Self::mark_slots_confirmed(&confirmed_forks, &blockstore, &bank_forks, &mut progress, &mut duplicate_slots_tracker, &mut heaviest_subtree_fork_choice, &mut duplicate_slots_to_repair); } compute_slot_stats_time.stop(); @@ -1048,6 +1050,7 @@ impl ReplayStage { // for duplicate slot recovery. fn process_gossip_duplicate_confirmed_slots( gossip_duplicate_confirmed_slots_receiver: &GossipDuplicateConfirmedSlotsReceiver, + blockstore: &Blockstore, duplicate_slots_tracker: &mut DuplicateSlotsTracker, gossip_duplicate_confirmed_slots: &mut GossipDuplicateConfirmedSlots, bank_forks: &RwLock, @@ -1057,27 +1060,30 @@ impl ReplayStage { ) { let root = bank_forks.read().unwrap().root(); for new_confirmed_slots in gossip_duplicate_confirmed_slots_receiver.try_iter() { - for (confirmed_slot, confirmed_hash) in new_confirmed_slots { + for (confirmed_slot, duplicate_confirmed_hash) in new_confirmed_slots { if confirmed_slot <= root { continue; - } else if let Some(prev_hash) = - gossip_duplicate_confirmed_slots.insert(confirmed_slot, confirmed_hash) + } else if let Some(prev_hash) = gossip_duplicate_confirmed_slots + .insert(confirmed_slot, duplicate_confirmed_hash) { - assert_eq!(prev_hash, confirmed_hash); + assert_eq!(prev_hash, duplicate_confirmed_hash); // Already processed this signal return; } + let duplicate_confirmed_state = DuplicateConfirmedState::new_from_state( + duplicate_confirmed_hash, + || progress.is_dead(confirmed_slot).unwrap_or(false), + || bank_forks.read().unwrap().bank_hash(confirmed_slot), + ); check_slot_agrees_with_cluster( confirmed_slot, root, - bank_forks.read().unwrap().bank_hash(confirmed_slot), + blockstore, duplicate_slots_tracker, - gossip_duplicate_confirmed_slots, - progress, fork_choice, duplicate_slots_to_repair, - SlotStateUpdate::DuplicateConfirmed, + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), ); } } @@ -1104,6 +1110,7 @@ impl ReplayStage { // Checks for and handle forks with duplicate slots. fn process_duplicate_slots( + blockstore: &Blockstore, duplicate_slots_receiver: &DuplicateSlotReceiver, duplicate_slots_tracker: &mut DuplicateSlotsTracker, gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots, @@ -1126,16 +1133,21 @@ impl ReplayStage { new_duplicate_slots.into_iter().zip(bank_hashes.into_iter()) { // WindowService should only send the signal once per slot + let duplicate_state = DuplicateState::new_from_state( + duplicate_slot, + gossip_duplicate_confirmed_slots, + fork_choice, + || progress.is_dead(duplicate_slot).unwrap_or(false), + || bank_hash, + ); check_slot_agrees_with_cluster( duplicate_slot, root_slot, - bank_hash, + blockstore, duplicate_slots_tracker, - gossip_duplicate_confirmed_slots, - progress, fork_choice, duplicate_slots_to_repair, - SlotStateUpdate::Duplicate, + SlotStateUpdate::Duplicate(duplicate_state), ); } } @@ -1417,16 +1429,20 @@ impl ReplayStage { err: format!("error: {:?}", err), timestamp: timestamp(), }); + let dead_state = DeadState::new_from_state( + slot, + duplicate_slots_tracker, + gossip_duplicate_confirmed_slots, + heaviest_subtree_fork_choice, + ); check_slot_agrees_with_cluster( slot, root, - Some(bank.hash()), + blockstore, duplicate_slots_tracker, - gossip_duplicate_confirmed_slots, - progress, heaviest_subtree_fork_choice, duplicate_slots_to_repair, - SlotStateUpdate::Dead, + SlotStateUpdate::Dead(dead_state), ); } @@ -1926,16 +1942,21 @@ impl ReplayStage { .get_fork_stats_mut(bank.slot()) .expect("All frozen banks must exist in the Progress map") .bank_hash = Some(bank.hash()); + let bank_frozen_state = BankFrozenState::new_from_state( + bank.slot(), + bank.hash(), + duplicate_slots_tracker, + gossip_duplicate_confirmed_slots, + heaviest_subtree_fork_choice, + ); check_slot_agrees_with_cluster( bank.slot(), bank_forks.read().unwrap().root(), - Some(bank.hash()), + blockstore, duplicate_slots_tracker, - gossip_duplicate_confirmed_slots, - progress, heaviest_subtree_fork_choice, duplicate_slots_to_repair, - SlotStateUpdate::Frozen, + SlotStateUpdate::BankFrozen(bank_frozen_state), ); if let Some(sender) = bank_notification_sender { sender @@ -2444,41 +2465,39 @@ impl ReplayStage { } fn mark_slots_confirmed( - confirmed_forks: &[Slot], + confirmed_forks: &[(Slot, Hash)], + blockstore: &Blockstore, bank_forks: &RwLock, progress: &mut ProgressMap, duplicate_slots_tracker: &mut DuplicateSlotsTracker, fork_choice: &mut HeaviestSubtreeForkChoice, duplicate_slots_to_repair: &mut DuplicateSlotsToRepair, ) { - let (root_slot, bank_hashes) = { - let r_bank_forks = bank_forks.read().unwrap(); - let bank_hashes: Vec> = confirmed_forks - .iter() - .map(|slot| r_bank_forks.bank_hash(*slot)) - .collect(); - - (r_bank_forks.root(), bank_hashes) - }; - for (slot, bank_hash) in confirmed_forks.iter().zip(bank_hashes.into_iter()) { + let root_slot = bank_forks.read().unwrap().root(); + for (slot, frozen_hash) in confirmed_forks.iter() { // This case should be guaranteed as false by confirm_forks() if let Some(false) = progress.is_supermajority_confirmed(*slot) { // Because supermajority confirmation will iterate through and update the // subtree in fork choice, only incur this cost if the slot wasn't already // confirmed progress.set_supermajority_confirmed_slot(*slot); + // If the slot was confirmed, then it must be frozen. Otherwise, we couldn't + // have replayed any of its descendants and figured out it was confirmed. + assert!(*frozen_hash != Hash::default()); + + let duplicate_confirmed_state = DuplicateConfirmedState::new_from_state( + *frozen_hash, + || false, + || Some(*frozen_hash), + ); check_slot_agrees_with_cluster( *slot, root_slot, - bank_hash, + blockstore, duplicate_slots_tracker, - // Don't need to pass the gossip confirmed slots since `slot` - // is already marked as confirmed in progress - &BTreeMap::new(), - progress, fork_choice, duplicate_slots_to_repair, - SlotStateUpdate::DuplicateConfirmed, + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), ); } } @@ -2490,7 +2509,7 @@ impl ReplayStage { total_stake: Stake, progress: &ProgressMap, bank_forks: &RwLock, - ) -> Vec { + ) -> Vec<(Slot, Hash)> { let mut confirmed_forks = vec![]; for (slot, prog) in progress.iter() { if !prog.fork_stats.is_supermajority_confirmed { @@ -2504,7 +2523,7 @@ impl ReplayStage { if bank.is_frozen() && tower.is_slot_confirmed(*slot, voted_stakes, total_stake) { info!("validator fork confirmed {} {}ms", *slot, duration); datapoint_info!("validator-confirmation", ("duration_ms", duration, i64)); - confirmed_forks.push(*slot); + confirmed_forks.push((*slot, bank.hash())); } else { debug!( "validator fork not confirmed {} {}ms {:?}", @@ -2673,7 +2692,7 @@ impl ReplayStage { } #[cfg(test)] -mod tests { +pub mod tests { use super::*; use crate::{ consensus::Tower, @@ -3247,14 +3266,14 @@ mod tests { #[test] fn test_dead_fork_trailing_entry() { let keypair = Keypair::new(); - let res = check_dead_fork(|genesis_keypair, bank| { + let res = check_dead_fork(|funded_keypair, bank| { let blockhash = bank.last_blockhash(); let slot = bank.slot(); let hashes_per_tick = bank.hashes_per_tick().unwrap_or(0); let mut entries = entry::create_ticks(bank.ticks_per_slot(), hashes_per_tick, blockhash); let last_entry_hash = entries.last().unwrap().hash; - let tx = system_transaction::transfer(genesis_keypair, &keypair.pubkey(), 2, blockhash); + let tx = system_transaction::transfer(funded_keypair, &keypair.pubkey(), 2, blockhash); let trailing_entry = entry::next_entry(&last_entry_hash, 1, vec![tx]); entries.push(trailing_entry); entries_to_test_shreds(entries, slot, slot.saturating_sub(1), true, 0) @@ -3270,14 +3289,19 @@ mod tests { #[test] fn test_dead_fork_entry_deserialize_failure() { // Insert entry that causes deserialization failure - let res = check_dead_fork(|_, _| { + let res = check_dead_fork(|_, bank| { let gibberish = [0xa5u8; PACKET_DATA_SIZE]; let mut data_header = DataShredHeader::default(); data_header.flags |= DATA_COMPLETE_SHRED; // Need to provide the right size for Shredder::deshred. data_header.size = SIZE_OF_DATA_SHRED_PAYLOAD as u16; + data_header.parent_offset = (bank.slot() - bank.parent_slot()) as u16; + let shred_common_header = ShredCommonHeader { + slot: bank.slot(), + ..ShredCommonHeader::default() + }; let mut shred = Shred::new_empty_from_header( - ShredCommonHeader::default(), + shred_common_header, data_header, CodingShredHeader::default(), ); @@ -3306,37 +3330,43 @@ mod tests { let ledger_path = get_tmp_ledger_path!(); let (replay_vote_sender, _replay_vote_receiver) = unbounded(); let res = { - let blockstore = Arc::new( - Blockstore::open(&ledger_path) - .expect("Expected to be able to open database ledger"), - ); - let GenesisConfigInfo { - mut genesis_config, - mint_keypair, + let ReplayBlockstoreComponents { + blockstore, + vote_simulator, .. - } = create_genesis_config(1000); - genesis_config.poh_config.hashes_per_tick = Some(2); - let bank_forks = BankForks::new(Bank::new(&genesis_config)); - let bank0 = bank_forks.working_bank(); - let mut progress = ProgressMap::default(); - let last_blockhash = bank0.last_blockhash(); - let mut bank0_progress = progress - .entry(bank0.slot()) - .or_insert_with(|| ForkProgress::new(last_blockhash, None, None, 0, 0)); - let shreds = shred_to_insert(&mint_keypair, bank0.clone()); + } = replay_blockstore_components(Some(tr(0)), 1, None); + let VoteSimulator { + mut progress, + bank_forks, + mut heaviest_subtree_fork_choice, + validator_keypairs, + .. + } = vote_simulator; + + let bank0 = bank_forks.read().unwrap().get(0).cloned().unwrap(); + assert!(bank0.is_frozen()); + assert_eq!(bank0.tick_height(), bank0.max_tick_height()); + let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1); + bank_forks.write().unwrap().insert(bank1); + let bank1 = bank_forks.read().unwrap().get(1).cloned().unwrap(); + let mut bank1_progress = progress + .entry(bank1.slot()) + .or_insert_with(|| ForkProgress::new(bank1.last_blockhash(), None, None, 0, 0)); + let shreds = shred_to_insert( + &validator_keypairs.values().next().unwrap().node_keypair, + bank1.clone(), + ); blockstore.insert_shreds(shreds, None, false).unwrap(); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); - let bank_forks = Arc::new(RwLock::new(bank_forks)); let exit = Arc::new(AtomicBool::new(false)); let res = ReplayStage::replay_blockstore_into_bank( - &bank0, + &bank1, &blockstore, - &mut bank0_progress, + &mut bank1_progress, None, &replay_vote_sender, &VerifyRecyclers::default(), ); - let rpc_subscriptions = Arc::new(RpcSubscriptions::new( &exit, bank_forks.clone(), @@ -3346,26 +3376,26 @@ mod tests { if let Err(err) = &res { ReplayStage::mark_dead_slot( &blockstore, - &bank0, + &bank1, 0, err, &rpc_subscriptions, &mut DuplicateSlotsTracker::default(), &GossipDuplicateConfirmedSlots::default(), &mut progress, - &mut HeaviestSubtreeForkChoice::new((0, Hash::default())), + &mut heaviest_subtree_fork_choice, &mut DuplicateSlotsToRepair::default(), ); } // Check that the erroring bank was marked as dead in the progress map assert!(progress - .get(&bank0.slot()) + .get(&bank1.slot()) .map(|b| b.is_dead) .unwrap_or(false)); // Check that the erroring bank was marked as dead in blockstore - assert!(blockstore.is_dead(bank0.slot())); + assert!(blockstore.is_dead(bank1.slot())); res.map(|_| ()) }; let _ignored = remove_dir_all(&ledger_path); @@ -3659,7 +3689,7 @@ mod tests { &bank_forks, ); // No new stats should have been computed - assert_eq!(confirmed_forks, vec![0]); + assert_eq!(confirmed_forks, vec![(0, bank0.hash())]); } let ancestors = bank_forks.read().unwrap().ancestors(); @@ -4785,16 +4815,21 @@ mod tests { let mut gossip_duplicate_confirmed_slots = GossipDuplicateConfirmedSlots::default(); let bank4_hash = bank_forks.read().unwrap().bank_hash(4).unwrap(); assert_ne!(bank4_hash, Hash::default()); + let duplicate_state = DuplicateState::new_from_state( + 4, + &gossip_duplicate_confirmed_slots, + &mut vote_simulator.heaviest_subtree_fork_choice, + || progress.is_dead(4).unwrap_or(false), + || Some(bank4_hash), + ); check_slot_agrees_with_cluster( 4, bank_forks.read().unwrap().root(), - Some(bank4_hash), + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut vote_simulator.heaviest_subtree_fork_choice, &mut DuplicateSlotsToRepair::default(), - SlotStateUpdate::Duplicate, + SlotStateUpdate::Duplicate(duplicate_state), ); let (vote_fork, reset_fork) = run_compute_and_select_forks( @@ -4811,16 +4846,21 @@ mod tests { blockstore.store_duplicate_slot(2, vec![], vec![]).unwrap(); let bank2_hash = bank_forks.read().unwrap().bank_hash(2).unwrap(); assert_ne!(bank2_hash, Hash::default()); + let duplicate_state = DuplicateState::new_from_state( + 2, + &gossip_duplicate_confirmed_slots, + &mut vote_simulator.heaviest_subtree_fork_choice, + || progress.is_dead(2).unwrap_or(false), + || Some(bank2_hash), + ); check_slot_agrees_with_cluster( 2, bank_forks.read().unwrap().root(), - Some(bank2_hash), + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut vote_simulator.heaviest_subtree_fork_choice, &mut DuplicateSlotsToRepair::default(), - SlotStateUpdate::Duplicate, + SlotStateUpdate::Duplicate(duplicate_state), ); let (vote_fork, reset_fork) = run_compute_and_select_forks( @@ -4840,16 +4880,19 @@ mod tests { // then slot 4 is now the heaviest bank again let mut duplicate_slots_to_repair = HashSet::new(); gossip_duplicate_confirmed_slots.insert(4, bank4_hash); + let duplicate_confirmed_state = DuplicateConfirmedState::new_from_state( + bank4_hash, + || progress.is_dead(4).unwrap_or(false), + || Some(bank4_hash), + ); check_slot_agrees_with_cluster( 4, bank_forks.read().unwrap().root(), - Some(bank4_hash), + &blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - &progress, &mut vote_simulator.heaviest_subtree_fork_choice, &mut duplicate_slots_to_repair, - SlotStateUpdate::DuplicateConfirmed, + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), ); // The confirmed hash is detected in `progress`, which means // it's confirmation on the replayed block. This means we have @@ -4980,16 +5023,19 @@ mod tests { // Mark fork choice branch as invalid so select forks below doesn't panic // on a nonexistent `heaviest_bank_on_same_fork` after we dump the duplciate fork. + let duplicate_confirmed_state = DuplicateConfirmedState::new_from_state( + duplicate_confirmed_bank2_hash, + || progress.is_dead(2).unwrap_or(false), + || Some(our_bank2_hash), + ); check_slot_agrees_with_cluster( 2, bank_forks.read().unwrap().root(), - Some(our_bank2_hash), + blockstore, &mut duplicate_slots_tracker, - &gossip_duplicate_confirmed_slots, - progress, heaviest_subtree_fork_choice, &mut duplicate_slots_to_repair, - SlotStateUpdate::DuplicateConfirmed, + SlotStateUpdate::DuplicateConfirmed(duplicate_confirmed_state), ); assert!(duplicate_slots_to_repair.contains(&(2, duplicate_confirmed_bank2_hash))); let mut ancestors = bank_forks.read().unwrap().ancestors(); @@ -5613,7 +5659,7 @@ mod tests { type GenerateVotes = Box) -> HashMap>>; - fn setup_forks_from_tree( + pub fn setup_forks_from_tree( tree: Tree, num_keys: usize, generate_votes: Option, diff --git a/core/src/vote_simulator.rs b/core/src/vote_simulator.rs index 4904c83a74..61d091592f 100644 --- a/core/src/vote_simulator.rs +++ b/core/src/vote_simulator.rs @@ -104,6 +104,9 @@ impl VoteSimulator { .any(|lockout| lockout.slot == parent)); } } + while new_bank.tick_height() < new_bank.max_tick_height() { + new_bank.register_tick(&Hash::new_unique()); + } new_bank.freeze(); self.progress .get_fork_stats_mut(new_bank.slot()) @@ -324,7 +327,7 @@ pub fn initialize_state( ) -> (BankForks, ProgressMap, HeaviestSubtreeForkChoice) { let validator_keypairs: Vec<_> = validator_keypairs_map.values().collect(); let GenesisConfigInfo { - genesis_config, + mut genesis_config, mint_keypair, voting_keypair: _, } = create_genesis_config_with_vote_accounts( @@ -333,12 +336,16 @@ pub fn initialize_state( vec![stake; validator_keypairs.len()], ); + genesis_config.poh_config.hashes_per_tick = Some(2); let bank0 = Bank::new(&genesis_config); for pubkey in validator_keypairs_map.keys() { bank0.transfer(10_000, &mint_keypair, pubkey).unwrap(); } + while bank0.tick_height() < bank0.max_tick_height() { + bank0.register_tick(&Hash::new_unique()); + } bank0.freeze(); let mut progress = ProgressMap::default(); progress.insert(