hash calc uses all threads during startup (#20735)

accounts-bench/src/main.rs

@@ -123,6 +123,7 @@ fn main() {
                 None,
                 false,
                 None,
+                false,
             );
             time_store.stop();
             if results != results_store {

ledger/src/blockstore_processor.rs

@@ -1204,6 +1204,7 @@ fn load_frozen_forks(
                             snapshot_config.accounts_hash_use_index,
                             snapshot_config.accounts_hash_debug_verify,
                             Some(new_root_bank.epoch_schedule().slots_per_epoch),
+                            false,
                         );
                         snapshot_utils::snapshot_bank(
                             new_root_bank,

runtime/src/accounts.rs

@@ -687,6 +687,7 @@ impl Accounts {
         debug_verify: bool,
     ) -> u64 {
         let use_index = false;
+        let is_startup = false; // there may be conditions where this is called at startup.
         self.accounts_db
             .update_accounts_hash_with_index_option(
                 use_index,
@@ -696,10 +697,12 @@ impl Accounts {
                 None,
                 can_cached_slot_be_unflushed,
                 None,
+                is_startup,
             )
             .1
     }
 
+    /// Only called from startup or test code.
     #[must_use]
     pub fn verify_bank_hash_and_lamports(
         &self,

runtime/src/accounts_background_service.rs

@@ -109,7 +109,7 @@ impl SnapshotRequestHandler {
                 let previous_hash = if test_hash_calculation {
                     // We have to use the index version here.
                     // We cannot calculate the non-index way because cache has not been flushed and stores don't match reality.
-                    snapshot_root_bank.update_accounts_hash_with_index_option(true, false, None)
+                    snapshot_root_bank.update_accounts_hash_with_index_option(true, false, None, false)
                 } else {
                     Hash::default()
                 };
@@ -148,6 +148,7 @@ impl SnapshotRequestHandler {
                     use_index_hash_calculation,
                     test_hash_calculation,
                     Some(snapshot_root_bank.epoch_schedule().slots_per_epoch),
+                    false,
                 );
                 let hash_for_testing = if test_hash_calculation {
                     assert_eq!(previous_hash, this_hash);

runtime/src/accounts_db.rs

@@ -5072,11 +5072,15 @@ impl AccountsDb {
     }
 
     pub fn update_accounts_hash(&self, slot: Slot, ancestors: &Ancestors) -> (Hash, u64) {
-        self.update_accounts_hash_with_index_option(true, false, slot, ancestors, None, false, None)
+        self.update_accounts_hash_with_index_option(
+            true, false, slot, ancestors, None, false, None, false,
+        )
     }
 
     pub fn update_accounts_hash_test(&self, slot: Slot, ancestors: &Ancestors) -> (Hash, u64) {
-        self.update_accounts_hash_with_index_option(true, true, slot, ancestors, None, false, None)
+        self.update_accounts_hash_with_index_option(
+            true, true, slot, ancestors, None, false, None, false,
+        )
     }
 
     fn scan_multiple_account_storages_one_slot<F, B>(
@@ -5323,6 +5327,7 @@ impl AccountsDb {
         check_hash: bool,
         can_cached_slot_be_unflushed: bool,
         slots_per_epoch: Option<Slot>,
+        is_startup: bool,
     ) -> Result<(Hash, u64), BankHashVerificationError> {
         if !use_index {
             let accounts_cache_and_ancestors = if can_cached_slot_be_unflushed {
@@ -5352,10 +5357,15 @@ impl AccountsDb {
                 ..HashStats::default()
             };
 
+            let thread_pool = if is_startup {
+                None
+            } else {
+                Some(&self.thread_pool_clean)
+            };
             Self::calculate_accounts_hash_without_index(
                 &self.accounts_hash_cache_path,
                 &storages,
-                Some(&self.thread_pool_clean),
+                thread_pool,
                 timings,
                 check_hash,
                 accounts_cache_and_ancestors,
@@ -5370,6 +5380,7 @@ impl AccountsDb {
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn calculate_accounts_hash_helper_with_verify(
         &self,
         use_index: bool,
@@ -5380,6 +5391,7 @@ impl AccountsDb {
         can_cached_slot_be_unflushed: bool,
         check_hash: bool,
         slots_per_epoch: Option<Slot>,
+        is_startup: bool,
     ) -> Result<(Hash, u64), BankHashVerificationError> {
         let (hash, total_lamports) = self.calculate_accounts_hash_helper(
             use_index,
@@ -5388,6 +5400,7 @@ impl AccountsDb {
             check_hash,
             can_cached_slot_be_unflushed,
             slots_per_epoch,
+            is_startup,
         )?;
         if debug_verify {
             // calculate the other way (store or non-store) and verify results match.
@@ -5398,6 +5411,7 @@ impl AccountsDb {
                 check_hash,
                 can_cached_slot_be_unflushed,
                 None,
+                is_startup,
             )?;
 
             let success = hash == hash_other
@@ -5417,6 +5431,7 @@ impl AccountsDb {
         expected_capitalization: Option<u64>,
         can_cached_slot_be_unflushed: bool,
         slots_per_epoch: Option<Slot>,
+        is_startup: bool,
     ) -> (Hash, u64) {
         let check_hash = false;
         let (hash, total_lamports) = self
@@ -5429,6 +5444,7 @@ impl AccountsDb {
                 can_cached_slot_be_unflushed,
                 check_hash,
                 slots_per_epoch,
+                is_startup,
             )
             .unwrap(); // unwrap here will never fail since check_hash = false
         let mut bank_hashes = self.bank_hashes.write().unwrap();
@@ -5609,6 +5625,7 @@ impl AccountsDb {
         }
     }
 
+    /// Only called from startup or test code.
     pub fn verify_bank_hash_and_lamports(
         &self,
         slot: Slot,
@@ -5620,6 +5637,7 @@ impl AccountsDb {
 
         let use_index = false;
         let check_hash = true;
+        let is_startup = true;
         let can_cached_slot_be_unflushed = false;
         let (calculated_hash, calculated_lamports) = self
             .calculate_accounts_hash_helper_with_verify(
@@ -5631,6 +5649,7 @@ impl AccountsDb {
                 can_cached_slot_be_unflushed,
                 check_hash,
                 None,
+                is_startup,
             )?;
 
         if calculated_lamports != total_lamports {
@@ -9779,7 +9798,7 @@ pub mod tests {
         for use_index in [true, false] {
             assert!(db
                 .calculate_accounts_hash_helper(
-                    use_index, some_slot, &ancestors, check_hash, false, None
+                    use_index, some_slot, &ancestors, check_hash, false, None, false,
                 )
                 .is_err());
         }
@@ -9802,11 +9821,13 @@ pub mod tests {
         let check_hash = true;
         assert_eq!(
             db.calculate_accounts_hash_helper(
-                false, some_slot, &ancestors, check_hash, false, None
+                false, some_slot, &ancestors, check_hash, false, None, false,
+            )
+            .unwrap(),
+            db.calculate_accounts_hash_helper(
+                true, some_slot, &ancestors, check_hash, false, None, false,
             )
             .unwrap(),
-            db.calculate_accounts_hash_helper(true, some_slot, &ancestors, check_hash, false, None)
-                .unwrap(),
         );
     }
 

runtime/src/bank.rs

@@ -5375,6 +5375,7 @@ impl Bank {
 
     /// Recalculate the hash_internal_state from the account stores. Would be used to verify a
     /// snapshot.
+    /// Only called from startup or test code.
     #[must_use]
     fn verify_bank_hash(&self, test_hash_calculation: bool) -> bool {
         self.rc.accounts.verify_bank_hash_and_lamports(
@@ -5492,6 +5493,7 @@ impl Bank {
         use_index: bool,
         mut debug_verify: bool,
         slots_per_epoch: Option<Slot>,
+        is_startup: bool,
     ) -> Hash {
         let (hash, total_lamports) = self
             .rc
@@ -5505,6 +5507,7 @@ impl Bank {
                 Some(self.capitalization()),
                 false,
                 slots_per_epoch,
+                is_startup,
             );
         if total_lamports != self.capitalization() {
             datapoint_info!(
@@ -5529,6 +5532,7 @@ impl Bank {
                         Some(self.capitalization()),
                         false,
                         slots_per_epoch,
+                        is_startup,
                     );
             }
 
@@ -5543,7 +5547,7 @@ impl Bank {
     }
 
     pub fn update_accounts_hash(&self) -> Hash {
-        self.update_accounts_hash_with_index_option(true, false, None)
+        self.update_accounts_hash_with_index_option(true, false, None, false)
     }
 
     /// A snapshot bank should be purged of 0 lamport accounts which are not part of the hash