add bank progress stats (#30405)

* add bank progress stats

* handle more bank creation code paths

* use wrapping sub

runtime/src/accounts_db.rs

@@ -1491,6 +1491,43 @@ pub struct AccountsDb {
     /// Some time later (to allow for slow calculation time), the bank hash at a slot calculated using 'M' includes the full accounts hash.
     /// Thus, the state of all accounts on a validator is known to be correct at least once per epoch.
     pub epoch_accounts_hash_manager: EpochAccountsHashManager,
+
+    pub bank_progress: BankCreationFreezingProgress,
+}
+
+impl BankCreationFreezingProgress {
+    fn report(&self) {
+        if self.last_report.should_update(60_000) {
+            datapoint_info!(
+                "bank_progress",
+                (
+                    "difference",
+                    self.bank_creation_count
+                        .load(Ordering::Acquire)
+                        .wrapping_sub(
+                            self.bank_freeze_or_destruction_count
+                                .load(Ordering::Acquire)
+                        ),
+                    i64
+                )
+            );
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+/// Keeps track of when all banks that were started as of a known point in time have been frozen or otherwise destroyed.
+/// When 'bank_freeze_or_destruction_count' exceeds a prior value of 'bank_creation_count',
+/// this means that we can know all banks that began loading accounts have completed as of the prior value of 'bank_creation_count'.
+pub struct BankCreationFreezingProgress {
+    /// Incremented each time a bank is created.
+    /// Starting now, this bank could be finding accounts in the index and loading them from accounts db.
+    pub bank_creation_count: AtomicU32,
+    /// Incremented each time a bank is frozen or destroyed.
+    /// At this point, this bank has completed all account loading.
+    pub bank_freeze_or_destruction_count: AtomicU32,
+
+    last_report: AtomicInterval,
 }
 
 #[derive(Debug, Default)]
@@ -2401,6 +2438,7 @@ impl AccountsDb {
 
         AccountsDb {
             assert_stakes_cache_consistency: false,
+            bank_progress: BankCreationFreezingProgress::default(),
             create_ancient_storage: CreateAncientStorage::Append,
             verify_accounts_hash_in_bg: VerifyAccountsHashInBackground::default(),
             filler_accounts_per_slot: AtomicU64::default(),
@@ -4046,6 +4084,7 @@ impl AccountsDb {
 
         Self::update_shrink_stats(&self.shrink_stats, stats_sub);
         self.shrink_stats.report();
+        self.bank_progress.report();
     }
 
     pub(crate) fn update_shrink_stats(shrink_stats: &ShrinkStats, stats_sub: ShrinkStatsSub) {

runtime/src/bank.rs

@@ -171,7 +171,7 @@ use {
         sync::{
             atomic::{
                 AtomicBool, AtomicI64, AtomicU64, AtomicUsize,
-                Ordering::{AcqRel, Acquire, Relaxed},
+                Ordering::{AcqRel, Acquire, Relaxed, Release},
             },
             Arc, LockResult, RwLock, RwLockReadGuard, RwLockWriteGuard,
         },
@@ -800,6 +800,7 @@ impl PartialEq for Bank {
             return true;
         }
         let Self {
+            bank_freeze_or_destruction_incremented: _,
             rc: _,
             status_cache: _,
             blockhash_queue,
@@ -1120,6 +1121,9 @@ pub struct Bank {
     pub incremental_snapshot_persistence: Option<BankIncrementalSnapshotPersistence>,
 
     pub loaded_programs_cache: Arc<RwLock<LoadedPrograms>>,
+
+    /// true when the bank's freezing or destruction has completed
+    bank_freeze_or_destruction_incremented: AtomicBool,
 }
 
 struct VoteWithStakeDelegations {
@@ -1269,6 +1273,7 @@ impl Bank {
 
     fn default_with_accounts(accounts: Accounts) -> Self {
         let mut bank = Self {
+            bank_freeze_or_destruction_incremented: AtomicBool::default(),
             incremental_snapshot_persistence: None,
             rc: BankRc::new(accounts, Slot::default()),
             status_cache: Arc::<RwLock<BankStatusCache>>::default(),
@@ -1332,6 +1337,7 @@ impl Bank {
             loaded_programs_cache: Arc::<RwLock<LoadedPrograms>>::default(),
         };
 
+        bank.bank_created();
         let accounts_data_size_initial = bank.get_total_accounts_stats().unwrap().data_len as u64;
         bank.accounts_data_size_initial = accounts_data_size_initial;
 
@@ -1543,7 +1549,9 @@ impl Bank {
         let (feature_set, feature_set_time_us) = measure_us!(parent.feature_set.clone());
 
         let accounts_data_size_initial = parent.load_accounts_data_size();
-        let mut new = Bank {
+        parent.bank_created();
+        let mut new = Self {
+            bank_freeze_or_destruction_incremented: AtomicBool::default(),
             incremental_snapshot_persistence: None,
             rc,
             status_cache,
@@ -1777,6 +1785,29 @@ impl Bank {
         self.vote_only_bank
     }
 
+    fn bank_created(&self) {
+        self.rc
+            .accounts
+            .accounts_db
+            .bank_progress
+            .bank_creation_count
+            .fetch_add(1, Release);
+    }
+
+    fn bank_frozen_or_destroyed(&self) {
+        if !self
+            .bank_freeze_or_destruction_incremented
+            .swap(true, AcqRel)
+        {
+            self.rc
+                .accounts
+                .accounts_db
+                .bank_progress
+                .bank_freeze_or_destruction_count
+                .fetch_add(1, Release);
+        }
+    }
+
     /// Like `new_from_parent` but additionally:
     /// * Doesn't assume that the parent is anywhere near `slot`, parent could be millions of slots
     /// in the past
@@ -1860,6 +1891,7 @@ impl Bank {
         let feature_set = new();
         let mut bank = Self {
             incremental_snapshot_persistence: fields.incremental_snapshot_persistence,
+            bank_freeze_or_destruction_incremented: AtomicBool::default(),
             rc: bank_rc,
             status_cache: new(),
             blockhash_queue: RwLock::new(fields.blockhash_queue),
@@ -1921,6 +1953,8 @@ impl Bank {
             fee_structure: FeeStructure::default(),
             loaded_programs_cache: Arc::<RwLock<LoadedPrograms>>::default(),
         };
+        bank.bank_created();
+
         bank.finish_init(
             genesis_config,
             additional_builtins,
@@ -3166,6 +3200,7 @@ impl Bank {
             self.freeze_started.store(true, Relaxed);
             *hash = self.hash_internal_state();
             self.rc.accounts.accounts_db.mark_slot_frozen(self.slot());
+            self.bank_frozen_or_destroyed();
         }
     }
 
@@ -8089,6 +8124,7 @@ impl TotalAccountsStats {
 
 impl Drop for Bank {
     fn drop(&mut self) {
+        self.bank_frozen_or_destroyed();
         if let Some(drop_callback) = self.drop_callback.read().unwrap().0.as_ref() {
             drop_callback.callback(self);
         } else {