ledger tool verify can store debug info on hash calc (#26837)

ledger-tool/src/main.rs

@@ -1160,6 +1160,10 @@ fn main() {
         .long("accounts-db-ancient-append-vecs")
         .help("AppendVecs that are older than an epoch are squashed together.")
         .hidden(true);
+    let halt_at_slot_store_hash_raw_data = Arg::with_name("halt_at_slot_store_hash_raw_data")
+            .long("halt-at-slot-store-hash-raw-data")
+            .help("After halting at slot, run an accounts hash calculation and store the raw hash data for debugging.")
+            .hidden(true);
     let verify_index_arg = Arg::with_name("verify_accounts_index")
         .long("verify-accounts-index")
         .takes_value(false)
@@ -1512,6 +1516,7 @@ fn main() {
             .arg(&skip_rewrites_arg)
             .arg(&accounts_db_skip_initial_hash_calc_arg)
             .arg(&ancient_append_vecs)
+            .arg(&halt_at_slot_store_hash_raw_data)
             .arg(&hard_forks_arg)
             .arg(&no_accounts_db_caching_arg)
             .arg(&accounts_db_test_hash_calculation_arg)
@@ -2442,6 +2447,8 @@ fn main() {
                 let process_options = ProcessOptions {
                     new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
                     poh_verify: !arg_matches.is_present("skip_poh_verify"),
+                    on_halt_store_hash_raw_data_for_debug: arg_matches
+                        .is_present("halt_at_slot_store_hash_raw_data"),
                     halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(),
                     debug_keys,
                     accounts_db_caching_enabled: !arg_matches.is_present("no_accounts_db_caching"),

ledger/src/blockstore_processor.rs

@@ -708,6 +708,7 @@ pub struct ProcessOptions {
     pub verify_index: bool,
     pub shrink_ratio: AccountShrinkThreshold,
     pub runtime_config: RuntimeConfig,
+    pub on_halt_store_hash_raw_data_for_debug: bool,
 }
 
 pub fn test_process_blockstore(
@@ -1399,6 +1400,7 @@ fn load_frozen_forks(
     )?;
 
     let halt_at_slot = opts.halt_at_slot.unwrap_or(std::u64::MAX);
+    let on_halt_store_hash_raw_data_for_debug = opts.on_halt_store_hash_raw_data_for_debug;
     if bank_forks.read().unwrap().root() != halt_at_slot {
         while !pending_slots.is_empty() {
             timing.details.per_program_timings.clear();
@@ -1540,6 +1542,7 @@ fn load_frozen_forks(
                     ignore_mismatch: true,
                     require_rooted_bank: false,
                     run_in_background: false,
+                    store_hash_raw_data_for_debug: on_halt_store_hash_raw_data_for_debug,
                 });
                 break;
             }

runtime/benches/accounts.rs

@@ -126,6 +126,7 @@ fn test_accounts_hash_bank_hash(bencher: &mut Bencher) {
             &RentCollector::default(),
             false,
             false,
+            false,
         ))
     });
 }

runtime/src/accounts.rs

@@ -814,6 +814,7 @@ impl Accounts {
 
     /// Only called from startup or test code.
     #[must_use]
+    #[allow(clippy::too_many_arguments)]
     pub fn verify_bank_hash_and_lamports(
         &self,
         slot: Slot,
@@ -824,6 +825,7 @@ impl Accounts {
         rent_collector: &RentCollector,
         can_cached_slot_be_unflushed: bool,
         ignore_mismatch: bool,
+        store_detailed_debug_info: bool,
     ) -> bool {
         if let Err(err) = self.accounts_db.verify_bank_hash_and_lamports_new(
             slot,
@@ -834,6 +836,7 @@ impl Accounts {
             rent_collector,
             can_cached_slot_be_unflushed,
             ignore_mismatch,
+            store_detailed_debug_info,
         ) {
             warn!("verify_bank_hash failed: {:?}, slot: {}", err, slot);
             false

runtime/src/accounts_db.rs

@@ -6990,10 +6990,12 @@ impl AccountsDb {
             rent_collector,
             can_cached_slot_be_unflushed,
             false,
+            false,
         )
     }
 
     /// Only called from startup or test code.
+    #[allow(clippy::too_many_arguments)]
     pub fn verify_bank_hash_and_lamports_new(
         &self,
         slot: Slot,
@@ -7004,6 +7006,7 @@ impl AccountsDb {
         rent_collector: &RentCollector,
         can_cached_slot_be_unflushed: bool,
         ignore_mismatch: bool,
+        store_hash_raw_data_for_debug: bool,
     ) -> Result<(), BankHashVerificationError> {
         use BankHashVerificationError::*;
 
@@ -7023,7 +7026,7 @@ impl AccountsDb {
                     use_write_cache: can_cached_slot_be_unflushed,
                     epoch_schedule,
                     rent_collector,
-                    store_detailed_debug_info_on_failure: false,
+                    store_detailed_debug_info_on_failure: store_hash_raw_data_for_debug,
                     full_snapshot: None,
                 },
                 None,

runtime/src/bank.rs

@@ -178,6 +178,7 @@ pub struct VerifyBankHash {
     pub ignore_mismatch: bool,
     pub require_rooted_bank: bool,
     pub run_in_background: bool,
+    pub store_hash_raw_data_for_debug: bool,
 }
 
 #[derive(Debug, Default)]
@@ -6903,6 +6904,7 @@ impl Bank {
                             &rent_collector,
                             config.can_cached_slot_be_unflushed,
                             config.ignore_mismatch,
+                            config.store_hash_raw_data_for_debug,
                         );
                         accounts_
                             .accounts_db
@@ -6923,6 +6925,7 @@ impl Bank {
                 rent_collector,
                 config.can_cached_slot_be_unflushed,
                 config.ignore_mismatch,
+                config.store_hash_raw_data_for_debug,
             );
             self.set_initial_accounts_hash_verification_completed();
             result
@@ -7162,6 +7165,7 @@ impl Bank {
                 ignore_mismatch: false,
                 require_rooted_bank: false,
                 run_in_background: true,
+                store_hash_raw_data_for_debug: false,
             });
             verify_time.stop();
             (verify, verify_time.as_us())
@@ -10393,6 +10397,7 @@ pub(crate) mod tests {
                 ignore_mismatch: false,
                 require_rooted_bank: false,
                 run_in_background: false,
+                store_hash_raw_data_for_debug: false,
             }
         }
     }