diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index da8dace2f0..30268565bd 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -1491,6 +1491,43 @@ pub struct AccountsDb { /// Some time later (to allow for slow calculation time), the bank hash at a slot calculated using 'M' includes the full accounts hash. /// Thus, the state of all accounts on a validator is known to be correct at least once per epoch. pub epoch_accounts_hash_manager: EpochAccountsHashManager, + + pub bank_progress: BankCreationFreezingProgress, +} + +impl BankCreationFreezingProgress { + fn report(&self) { + if self.last_report.should_update(60_000) { + datapoint_info!( + "bank_progress", + ( + "difference", + self.bank_creation_count + .load(Ordering::Acquire) + .wrapping_sub( + self.bank_freeze_or_destruction_count + .load(Ordering::Acquire) + ), + i64 + ) + ); + } + } +} + +#[derive(Debug, Default)] +/// Keeps track of when all banks that were started as of a known point in time have been frozen or otherwise destroyed. +/// When 'bank_freeze_or_destruction_count' exceeds a prior value of 'bank_creation_count', +/// this means that we can know all banks that began loading accounts have completed as of the prior value of 'bank_creation_count'. +pub struct BankCreationFreezingProgress { + /// Incremented each time a bank is created. + /// Starting now, this bank could be finding accounts in the index and loading them from accounts db. + pub bank_creation_count: AtomicU32, + /// Incremented each time a bank is frozen or destroyed. + /// At this point, this bank has completed all account loading. + pub bank_freeze_or_destruction_count: AtomicU32, + + last_report: AtomicInterval, } #[derive(Debug, Default)] @@ -2401,6 +2438,7 @@ impl AccountsDb { AccountsDb { assert_stakes_cache_consistency: false, + bank_progress: BankCreationFreezingProgress::default(), create_ancient_storage: CreateAncientStorage::Append, verify_accounts_hash_in_bg: VerifyAccountsHashInBackground::default(), filler_accounts_per_slot: AtomicU64::default(), @@ -4046,6 +4084,7 @@ impl AccountsDb { Self::update_shrink_stats(&self.shrink_stats, stats_sub); self.shrink_stats.report(); + self.bank_progress.report(); } pub(crate) fn update_shrink_stats(shrink_stats: &ShrinkStats, stats_sub: ShrinkStatsSub) { diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index af5a9ded13..8308ddc21d 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -171,7 +171,7 @@ use { sync::{ atomic::{ AtomicBool, AtomicI64, AtomicU64, AtomicUsize, - Ordering::{AcqRel, Acquire, Relaxed}, + Ordering::{AcqRel, Acquire, Relaxed, Release}, }, Arc, LockResult, RwLock, RwLockReadGuard, RwLockWriteGuard, }, @@ -800,6 +800,7 @@ impl PartialEq for Bank { return true; } let Self { + bank_freeze_or_destruction_incremented: _, rc: _, status_cache: _, blockhash_queue, @@ -1120,6 +1121,9 @@ pub struct Bank { pub incremental_snapshot_persistence: Option, pub loaded_programs_cache: Arc>, + + /// true when the bank's freezing or destruction has completed + bank_freeze_or_destruction_incremented: AtomicBool, } struct VoteWithStakeDelegations { @@ -1269,6 +1273,7 @@ impl Bank { fn default_with_accounts(accounts: Accounts) -> Self { let mut bank = Self { + bank_freeze_or_destruction_incremented: AtomicBool::default(), incremental_snapshot_persistence: None, rc: BankRc::new(accounts, Slot::default()), status_cache: Arc::>::default(), @@ -1332,6 +1337,7 @@ impl Bank { loaded_programs_cache: Arc::>::default(), }; + bank.bank_created(); let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64; bank.accounts_data_size_initial = accounts_data_size_initial; @@ -1543,7 +1549,9 @@ impl Bank { let (feature_set, feature_set_time_us) = measure_us!(parent.feature_set.clone()); let accounts_data_size_initial = parent.load_accounts_data_size(); - let mut new = Bank { + parent.bank_created(); + let mut new = Self { + bank_freeze_or_destruction_incremented: AtomicBool::default(), incremental_snapshot_persistence: None, rc, status_cache, @@ -1777,6 +1785,29 @@ impl Bank { self.vote_only_bank } + fn bank_created(&self) { + self.rc + .accounts + .accounts_db + .bank_progress + .bank_creation_count + .fetch_add(1, Release); + } + + fn bank_frozen_or_destroyed(&self) { + if !self + .bank_freeze_or_destruction_incremented + .swap(true, AcqRel) + { + self.rc + .accounts + .accounts_db + .bank_progress + .bank_freeze_or_destruction_count + .fetch_add(1, Release); + } + } + /// Like `new_from_parent` but additionally: /// * Doesn't assume that the parent is anywhere near `slot`, parent could be millions of slots /// in the past @@ -1860,6 +1891,7 @@ impl Bank { let feature_set = new(); let mut bank = Self { incremental_snapshot_persistence: fields.incremental_snapshot_persistence, + bank_freeze_or_destruction_incremented: AtomicBool::default(), rc: bank_rc, status_cache: new(), blockhash_queue: RwLock::new(fields.blockhash_queue), @@ -1921,6 +1953,8 @@ impl Bank { fee_structure: FeeStructure::default(), loaded_programs_cache: Arc::>::default(), }; + bank.bank_created(); + bank.finish_init( genesis_config, additional_builtins, @@ -3166,6 +3200,7 @@ impl Bank { self.freeze_started.store(true, Relaxed); *hash = self.hash_internal_state(); self.rc.accounts.accounts_db.mark_slot_frozen(self.slot()); + self.bank_frozen_or_destroyed(); } } @@ -8089,6 +8124,7 @@ impl TotalAccountsStats { impl Drop for Bank { fn drop(&mut self) { + self.bank_frozen_or_destroyed(); if let Some(drop_callback) = self.drop_callback.read().unwrap().0.as_ref() { drop_callback.callback(self); } else {