bank.initial_blockstore_processing_complete to avoid concurrent hash calculations (#27776)

* bank.initial_blockstore_processing_complete to avoid concurrent hash calculations

* Update runtime/src/bank.rs

Co-authored-by: Brooks Prumo <brooks@prumo.org>

* Update runtime/src/bank.rs

Co-authored-by: Brooks Prumo <brooks@prumo.org>

* Rename TestValidator::set_startup_verification_complete()

* Initialize with `AtomicBool::new(false)` instead of `default()`

* snapshot tests: move where `initial_blockstore_processing_completed()` is called

* fixup bank_forks.rs calling `is_initial_blockstore_processing_complete()`

* only call initial_blockstore_processing_completed() in blockstore_processor

Co-authored-by: Brooks Prumo <brooks@prumo.org>
Co-authored-by: Brooks Prumo <brooks@solana.com>

core/tests/snapshots.rs

@@ -103,6 +103,8 @@ impl SnapshotTestConfig {
             accounts_db::AccountShrinkThreshold::default(),
         );
         bank0.freeze();
+        bank0.set_startup_verification_complete();
+        bank0.initial_blockstore_processing_completed();
         let mut bank_forks = BankForks::new(bank0);
         bank_forks.accounts_hash_interval_slots = accounts_hash_interval_slots;
 
@@ -208,7 +210,6 @@ fn run_bank_forks_snapshot_n<F>(
     );
 
     let bank_forks = &mut snapshot_test_config.bank_forks;
-    bank_forks.root_bank().set_startup_verification_complete();
     let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
 
     let (snapshot_request_sender, snapshot_request_receiver) = unbounded();
@@ -675,7 +676,6 @@ fn test_bank_forks_incremental_snapshot(
             snapshot_test_config.accounts_dir.path().display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.full_snapshot_archives_dir.path().display(), snapshot_test_config.incremental_snapshot_archives_dir.path().display());
 
     let bank_forks = &mut snapshot_test_config.bank_forks;
-    bank_forks.root_bank().set_startup_verification_complete();
     let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
 
     let (snapshot_request_sender, snapshot_request_receiver) = unbounded();

ledger/src/blockstore_processor.rs

@@ -874,6 +874,8 @@ pub fn process_blockstore_from_root(
     }
     time_cap.stop();
 
+    bank.initial_blockstore_processing_completed();
+
     datapoint_info!(
         "process_blockstore_from_root",
         ("total_time_us", processing_time.as_micros(), i64),

runtime/src/accounts_db.rs

@@ -1204,6 +1204,9 @@ pub struct AccountsDb {
     /// debug feature to scan every append vec and verify refcounts are equal
     exhaustively_verify_refcounts: bool,
 
+    /// true once all accounts hash calculations that may take place at startup have been requested
+    pub initial_blockstore_processing_complete: AtomicBool,
+
     /// the full accounts hash calculation as of a predetermined block height 'N'
     /// to be included in the bank hash at a predetermined block height 'M'
     /// The cadence is once per epoch, all nodes calculate a full accounts hash as of a known slot calculated using 'N'
@@ -2020,6 +2023,7 @@ impl AccountsDb {
             num_hash_scan_passes,
             log_dead_slots: AtomicBool::new(true),
             exhaustively_verify_refcounts: false,
+            initial_blockstore_processing_complete: AtomicBool::new(false),
             epoch_accounts_hash: Mutex::new(None),
         }
     }

runtime/src/bank.rs

@@ -163,7 +163,7 @@ use {
         sync::{
             atomic::{
                 AtomicBool, AtomicI64, AtomicU64, AtomicUsize,
-                Ordering::{AcqRel, Acquire, Relaxed},
+                Ordering::{AcqRel, Acquire, Relaxed, Release},
             },
             Arc, LockResult, RwLock, RwLockReadGuard, RwLockWriteGuard,
         },
@@ -7231,6 +7231,28 @@ impl Bank {
         self.update_accounts_hash_with_index_option(true, false, false)
     }
 
+    /// When the validator starts up, there is an initial hash calculation verification of the snapshot.
+    /// Then, in some conditions, there are additional slots replayed and a second hash calculation verification occurs.
+    /// This function is called when all initial hash calculations have been requested or completed.
+    pub fn initial_blockstore_processing_completed(&self) {
+        self.rc
+            .accounts
+            .accounts_db
+            .initial_blockstore_processing_complete
+            .store(true, Release);
+    }
+
+    /// until this occurs, no additional accounts hash calculations can be started
+    /// There will be 0..=2 of these requests.
+    pub fn is_initial_blockstore_processing_complete(&self) -> bool {
+        self.rc
+            .accounts
+            .accounts_db
+            .initial_blockstore_processing_complete
+            .load(Acquire)
+            && self.is_startup_verification_complete()
+    }
+
     /// A snapshot bank should be purged of 0 lamport accounts which are not part of the hash
     /// calculation and could shield other real accounts.
     pub fn verify_snapshot_bank(
@@ -8152,9 +8174,7 @@ pub(crate) mod tests {
                 MAX_LOCKOUT_HISTORY,
             },
         },
-        std::{
+        std::{result, str::FromStr, thread::Builder, time::Duration},
-            result, str::FromStr, sync::atomic::Ordering::Release, thread::Builder, time::Duration,
-        },
         test_utils::goto_end_of_slot,
     };
 

runtime/src/bank_forks.rs

@@ -276,7 +276,7 @@ impl BankForks {
             if self.snapshot_config.is_some()
                 && accounts_background_request_sender.is_snapshot_creation_enabled()
             {
-                if bank.is_startup_verification_complete() {
+                if bank.is_initial_blockstore_processing_complete() {
                     // Save off the status cache because these may get pruned if another
                     // `set_root()` is called before the snapshots package can be generated
                     let status_cache_slot_deltas =

test-validator/src/lib.rs

@@ -600,12 +600,17 @@ impl TestValidator {
     }
 
     /// allow tests to indicate that validator has completed initialization
-    pub fn set_startup_verification_complete(&self) {
+    pub fn set_startup_verification_complete_for_tests(&self) {
         self.bank_forks()
             .read()
             .unwrap()
             .root_bank()
             .set_startup_verification_complete();
+        self.bank_forks()
+            .read()
+            .unwrap()
+            .root_bank()
+            .initial_blockstore_processing_completed();
     }
 
     /// Initialize the ledger directory
@@ -1003,7 +1008,7 @@ mod test {
     #[test]
     fn get_health() {
         let (test_validator, _payer) = TestValidatorGenesis::default().start();
-        test_validator.set_startup_verification_complete();
+        test_validator.set_startup_verification_complete_for_tests();
         let rpc_client = test_validator.get_rpc_client();
         rpc_client.get_health().expect("health");
     }
@@ -1011,7 +1016,7 @@ mod test {
     #[tokio::test]
     async fn nonblocking_get_health() {
         let (test_validator, _payer) = TestValidatorGenesis::default().start_async().await;
-        test_validator.set_startup_verification_complete();
+        test_validator.set_startup_verification_complete_for_tests();
         let rpc_client = test_validator.get_async_rpc_client();
         rpc_client.get_health().await.expect("health");
     }

tokens/src/commands.rs

@@ -1279,7 +1279,7 @@ mod tests {
     fn simple_test_validator_no_fees(pubkey: Pubkey) -> TestValidator {
         let test_validator =
             TestValidator::with_no_fees(pubkey, None, SocketAddrSpace::Unspecified);
-        test_validator.set_startup_verification_complete();
+        test_validator.set_startup_verification_complete_for_tests();
         test_validator
     }
 
@@ -1826,7 +1826,7 @@ mod tests {
     fn simple_test_validator(alice: Pubkey) -> TestValidator {
         let test_validator =
             TestValidator::with_custom_fees(alice, 10_000, None, SocketAddrSpace::Unspecified);
-        test_validator.set_startup_verification_complete();
+        test_validator.set_startup_verification_complete_for_tests();
         test_validator
     }