Snapshots wait for EAH calculations to complete (#28777)

core/tests/epoch_accounts_hash.rs

@@ -466,7 +466,7 @@ fn test_snapshots_have_expected_epoch_accounts_hash() {
             assert_eq!(&deserialized_bank, bank.as_ref());
             assert_eq!(
                 deserialized_bank.epoch_accounts_hash(),
-                bank.epoch_accounts_hash(),
+                bank.get_epoch_accounts_hash_to_serialize(),
             );
         }
 

core/tests/snapshots.rs

@@ -37,6 +37,7 @@ use {
     },
     solana_sdk::{
         clock::Slot,
+        epoch_schedule::EpochSchedule,
         genesis_config::{
             ClusterType::{self, Development, Devnet, MainnetBeta, Testnet},
             GenesisConfig,
@@ -94,6 +95,8 @@ impl SnapshotTestConfig {
             &solana_sdk::pubkey::new_rand(), // validator_pubkey
             1,                               // validator_stake_lamports
         );
+        // NOTE: Must set `warmup == false` until EAH can handle short epochs
+        genesis_config_info.genesis_config.epoch_schedule = EpochSchedule::without_warmup();
         genesis_config_info.genesis_config.cluster_type = cluster_type;
         let bank0 = Bank::new_with_paths_for_tests(
             &genesis_config_info.genesis_config,

runtime/src/bank.rs

@@ -7715,10 +7715,28 @@ impl Bank {
         total_accounts_stats
     }
 
-    /// if we were to serialize THIS bank, what value should be saved for the prior accounts hash?
+    /// Get the EAH that will be used by snapshots
-    /// This depends on the proximity to the time to take the snapshot and the time to use the snapshot.
+    ///
-    pub(crate) fn get_epoch_accounts_hash_to_serialize(&self) -> Option<Hash> {
+    /// Since snapshots are taken on roots, if the bank is in the EAH calculation window then an
-        self.epoch_accounts_hash().map(|hash| *hash.as_ref())
+    /// EAH *must* be included.  This means if an EAH calculation is currently in-flight we will
+    /// wait for it to complete.
+    pub fn get_epoch_accounts_hash_to_serialize(&self) -> Option<EpochAccountsHash> {
+        let (epoch_accounts_hash, measure) = measure!(
+            epoch_accounts_hash::is_in_calculation_window(self).then(|| {
+                self.rc
+                    .accounts
+                    .accounts_db
+                    .epoch_accounts_hash_manager
+                    .wait_get_epoch_accounts_hash()
+            })
+        );
+
+        datapoint_info!(
+            "bank-get_epoch_accounts_hash_to_serialize",
+            ("slot", self.slot(), i64),
+            ("waiting-time-us", measure.as_us(), i64),
+        );
+        epoch_accounts_hash
     }
 
     /// Convenience fn to get the Epoch Accounts Hash

runtime/src/epoch_accounts_hash/utils.rs

@@ -38,6 +38,15 @@ pub fn calculation_stop(bank: &Bank) -> Slot {
     calculation_info(bank).calculation_stop
 }
 
+/// Is this bank in the calculation window?
+#[must_use]
+#[inline]
+pub fn is_in_calculation_window(bank: &Bank) -> bool {
+    let bank_slot = bank.slot();
+    let info = calculation_info(bank);
+    bank_slot >= info.calculation_start && bank_slot < info.calculation_stop
+}
+
 /// For the epoch that `bank` is in, get all the EAH calculation information
 pub fn calculation_info(bank: &Bank) -> CalculationInfo {
     let epoch = bank.epoch();

runtime/src/serde_snapshot/newer.rs

@@ -214,7 +214,8 @@ impl<'a> TypeContext<'a> for Context {
             None::<BankIncrementalSnapshotPersistence>,
             serializable_bank
                 .bank
-                .get_epoch_accounts_hash_to_serialize(),
+                .get_epoch_accounts_hash_to_serialize()
+                .map(|epoch_accounts_hash| *epoch_accounts_hash.as_ref()),
         )
             .serialize(serializer)
     }

runtime/src/serde_snapshot/tests.rs

@@ -7,6 +7,7 @@ use {
         accounts_db::{get_temp_accounts_paths, AccountShrinkThreshold, AccountStorageMap},
         append_vec::AppendVec,
         bank::{Bank, Rewrites},
+        epoch_accounts_hash,
         genesis_utils::{activate_all_features, activate_feature},
         snapshot_utils::ArchiveFormat,
         status_cache::StatusCache,
@@ -224,6 +225,7 @@ fn test_bank_serialize_style(
     solana_logger::setup();
     let (genesis_config, _) = create_genesis_config(500);
     let bank0 = Arc::new(Bank::new_for_tests(&genesis_config));
+    let eah_start_slot = epoch_accounts_hash::calculation_start(&bank0);
     let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
     bank0.squash();
 
@@ -231,7 +233,15 @@ fn test_bank_serialize_style(
     let key1 = Keypair::new();
     bank1.deposit(&key1.pubkey(), 5).unwrap();
 
-    let bank2 = Bank::new_from_parent(&bank0, &Pubkey::default(), 2);
+    // If setting an initial EAH, then the bank being snapshotted must be in the EAH calculation
+    // window.  Otherwise `bank_to_stream()` below will *not* include the EAH in the bank snapshot,
+    // and the later-deserialized bank's EAH will not match the expected EAH.
+    let bank2_slot = if initial_epoch_accounts_hash {
+        eah_start_slot
+    } else {
+        0
+    } + 2;
+    let bank2 = Bank::new_from_parent(&bank0, &Pubkey::default(), bank2_slot);
 
     // Test new account
     let key2 = Keypair::new();
@@ -260,7 +270,7 @@ fn test_bank_serialize_style(
             .epoch_accounts_hash_manager
             .set_valid(
                 EpochAccountsHash::new(expected_epoch_accounts_hash.unwrap()),
-                0,
+                eah_start_slot,
             );
     }
 
@@ -397,7 +407,7 @@ fn test_bank_serialize_style(
     assert_eq!(dbank.get_accounts_hash(), accounts_hash);
     assert!(bank2 == dbank);
     assert_eq!(dbank.incremental_snapshot_persistence, incremental);
-    assert_eq!(dbank.get_epoch_accounts_hash_to_serialize(), expected_epoch_accounts_hash,
+    assert_eq!(dbank.get_epoch_accounts_hash_to_serialize().map(|epoch_accounts_hash| *epoch_accounts_hash.as_ref()), expected_epoch_accounts_hash,
         "(reserialize_accounts_hash, incremental_snapshot_persistence, update_accounts_hash, initial_epoch_accounts_hash): {:?}",
         (
             reserialize_accounts_hash,