From 115f376465a237695b4fec2a7fd33b237c6fb6b7 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Mon, 7 Mar 2022 11:23:22 +0100 Subject: [PATCH] Factor out bank_forks_utils::load_bank_forks() --- core/src/retransmit_stage.rs | 7 +-- core/src/validator.rs | 47 ++++++++++++-------- ledger-tool/src/main.rs | 29 ++++++------ ledger/src/bank_forks_utils.rs | 71 ++++++++++++++++++------------ ledger/src/blockstore_processor.rs | 4 +- 5 files changed, 89 insertions(+), 69 deletions(-) diff --git a/core/src/retransmit_stage.rs b/core/src/retransmit_stage.rs index d50c0badc..50c112fdf 100644 --- a/core/src/retransmit_stage.rs +++ b/core/src/retransmit_stage.rs @@ -550,11 +550,8 @@ mod tests { full_leader_cache: true, ..ProcessOptions::default() }; - let (bank_forks, cached_leader_schedule, _) = test_process_blockstore( - &genesis_config, - &blockstore, - opts, - ); + let (bank_forks, cached_leader_schedule, _) = + test_process_blockstore(&genesis_config, &blockstore, opts); let leader_schedule_cache = Arc::new(cached_leader_schedule); let bank_forks = Arc::new(RwLock::new(bank_forks)); diff --git a/core/src/validator.rs b/core/src/validator.rs index cfe7436a1..cb6bb7759 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -1322,31 +1322,40 @@ fn new_banks_from_ledger( TransactionHistoryServices::default() }; - let ( - mut bank_forks, - mut leader_schedule_cache, - last_full_snapshot_slot, - starting_snapshot_hashes, - ) = bank_forks_utils::load( + let cache_block_meta_sender = transaction_history_services + .cache_block_meta_sender + .as_ref(); + + let (bank_forks, starting_snapshot_hashes) = bank_forks_utils::load_bank_forks( &genesis_config, &blockstore, config.account_paths.clone(), config.account_shrink_paths.clone(), config.snapshot_config.as_ref(), - process_options, - transaction_history_services - .transaction_status_sender - .as_ref(), - transaction_history_services - .cache_block_meta_sender - .as_ref(), - accounts_package_sender, + &process_options, + cache_block_meta_sender, accounts_update_notifier, - ) - .unwrap_or_else(|err| { - error!("Failed to load ledger: {:?}", err); - abort() - }); + ); + + let (mut bank_forks, mut leader_schedule_cache, last_full_snapshot_slot) = + blockstore_processor::process_blockstore_from_root( + &blockstore, + bank_forks, + &process_options, + transaction_history_services + .transaction_status_sender + .as_ref(), + cache_block_meta_sender, + config.snapshot_config.as_ref(), + accounts_package_sender, + ) + .unwrap_or_else(|err| { + error!("Failed to load ledger: {:?}", err); + abort() + }); + + let last_full_snapshot_slot = + last_full_snapshot_slot.or_else(|| starting_snapshot_hashes.map(|x| x.full.hash.0)); if let Some(warp_slot) = config.warp_slot { let snapshot_config = config.snapshot_config.as_ref().unwrap_or_else(|| { diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 6467112d6..53074f58e 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -27,7 +27,7 @@ use { self, AccessType, BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRecoveryMode, Database, }, - blockstore_processor::ProcessOptions, + blockstore_processor::{BlockstoreProcessorError, ProcessOptions}, shred::Shred, }, solana_measure::measure::Measure, @@ -41,6 +41,7 @@ use { hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_config::SnapshotConfig, + snapshot_hash::StartingSnapshotHashes, snapshot_utils::{ self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, @@ -711,7 +712,7 @@ fn load_bank_forks( blockstore: &Blockstore, process_options: ProcessOptions, snapshot_archive_path: Option, -) -> bank_forks_utils::LoadResult { +) -> Result<(BankForks, Option), BlockstoreProcessorError> { let bank_snapshots_dir = blockstore .ledger_path() .join(if blockstore.is_primary_access() { @@ -764,6 +765,7 @@ fn load_bank_forks( accounts_package_sender, None, ) + .map(|(bank_forks, .., starting_snapshot_hashes)| (bank_forks, starting_snapshot_hashes)) } fn compute_slot_cost(blockstore: &Blockstore, slot: Slot) -> Result<(), String> { @@ -2221,7 +2223,7 @@ fn main() { }, snapshot_archive_path, ) { - Ok((bank_forks, .., starting_snapshot_hashes)) => { + Ok((bank_forks, starting_snapshot_hashes)) => { let mut bank = bank_forks .get(snapshot_slot) .unwrap_or_else(|| { @@ -2439,10 +2441,11 @@ fn main() { } let full_snapshot_slot = starting_snapshot_hashes.unwrap().full.hash.0; if bank.slot() <= full_snapshot_slot { - eprintln!("Unable to create incremental snapshot: Slot must be greater than full snapshot slot. slot: {}, full snapshot slot: {}", - bank.slot(), - full_snapshot_slot, - ); + eprintln!( + "Unable to create incremental snapshot: Slot must be greater than full snapshot slot. slot: {}, full snapshot slot: {}", + bank.slot(), + full_snapshot_slot, + ); exit(1); } @@ -2463,12 +2466,12 @@ fn main() { }); println!( - "Successfully created incremental snapshot for slot {}, hash {}, base slot: {}: {}", - bank.slot(), - bank.hash(), - full_snapshot_slot, - incremental_snapshot_archive_info.path().display(), - ); + "Successfully created incremental snapshot for slot {}, hash {}, base slot: {}: {}", + bank.slot(), + bank.hash(), + full_snapshot_slot, + incremental_snapshot_archive_info.path().display(), + ); } else { let full_snapshot_archive_info = snapshot_utils::bank_to_full_snapshot_archive( diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 0d40d4c44..25b43c0fe 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -17,7 +17,7 @@ use { snapshot_package::AccountsPackageSender, snapshot_utils, }, - solana_sdk::{clock::Slot, genesis_config::GenesisConfig}, + solana_sdk::genesis_config::GenesisConfig, std::{fs, path::PathBuf, process, result}, }; @@ -25,13 +25,12 @@ pub type LoadResult = result::Result< ( BankForks, LeaderScheduleCache, - Option, Option, ), BlockstoreProcessorError, >; -/// Load the banks and accounts +/// Load the banks via genesis or a snapshot then processes all full blocks in blockstore /// /// If a snapshot config is given, and a snapshot is found, it will be loaded. Otherwise, load /// from genesis. @@ -48,6 +47,42 @@ pub fn load( accounts_package_sender: AccountsPackageSender, accounts_update_notifier: Option, ) -> LoadResult { + let (bank_forks, starting_snapshot_hashes) = load_bank_forks( + genesis_config, + blockstore, + account_paths, + shrink_paths, + snapshot_config, + &process_options, + cache_block_meta_sender, + accounts_update_notifier, + ); + + blockstore_processor::process_blockstore_from_root( + blockstore, + bank_forks, + &process_options, + transaction_status_sender, + cache_block_meta_sender, + snapshot_config, + accounts_package_sender, + ) + .map(|(bank_forks, leader_schedule_cache, ..)| { + (bank_forks, leader_schedule_cache, starting_snapshot_hashes) + }) +} + +#[allow(clippy::too_many_arguments)] +pub fn load_bank_forks( + genesis_config: &GenesisConfig, + blockstore: &Blockstore, + account_paths: Vec, + shrink_paths: Option>, + snapshot_config: Option<&SnapshotConfig>, + process_options: &ProcessOptions, + cache_block_meta_sender: Option<&CacheBlockMetaSender>, + accounts_update_notifier: Option, +) -> (BankForks, Option) { let snapshot_present = if let Some(snapshot_config) = snapshot_config { info!( "Initializing bank snapshot path: {}", @@ -72,13 +107,13 @@ pub fn load( false }; - let (bank_forks, starting_snapshot_hashes) = if snapshot_present { + if snapshot_present { bank_forks_from_snapshot( genesis_config, account_paths, shrink_paths, snapshot_config.as_ref().unwrap(), - &process_options, + process_options, accounts_update_notifier, ) } else { @@ -98,35 +133,13 @@ pub fn load( genesis_config, blockstore, account_paths, - &process_options, + process_options, cache_block_meta_sender, accounts_update_notifier, ), None, ) - }; - - blockstore_processor::process_blockstore_from_root( - blockstore, - bank_forks, - &process_options, - transaction_status_sender, - cache_block_meta_sender, - snapshot_config, - accounts_package_sender, - ) - .map( - |(bank_forks, leader_schedule_cache, last_full_snapshot_slot)| { - let last_full_snapshot_slot = - last_full_snapshot_slot.or_else(|| starting_snapshot_hashes.map(|x| x.full.hash.0)); - ( - bank_forks, - leader_schedule_cache, - last_full_snapshot_slot, - starting_snapshot_hashes, - ) - }, - ) + } } #[allow(clippy::too_many_arguments)] diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 72c875613..268d896c6 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -630,7 +630,7 @@ pub(crate) fn process_blockstore_for_bank_0( /// Process blockstore from a known root bank #[allow(clippy::too_many_arguments)] -pub(crate) fn process_blockstore_from_root( +pub fn process_blockstore_from_root( blockstore: &Blockstore, mut bank_forks: BankForks, opts: &ProcessOptions, @@ -3170,7 +3170,6 @@ pub mod tests { None, None, accounts_package_sender, - None, ) .unwrap(); @@ -3278,7 +3277,6 @@ pub mod tests { None, Some(&snapshot_config), accounts_package_sender.clone(), - None, ) .unwrap();