From 5f1b5b877ad6892d5f472308aeefcbc018de6a76 Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 26 Jun 2023 12:44:25 -0400 Subject: [PATCH] Replace boot_from_local_state with use_snapshot_archives_at_startup (#32260) --- core/src/validator.rs | 7 +- ledger-tool/src/main.rs | 62 +++++----- ledger/src/bank_forks_utils.rs | 153 +++++++++++++------------ ledger/src/blockstore_processor.rs | 4 +- local-cluster/src/validator_configs.rs | 2 +- local-cluster/tests/local_cluster.rs | 3 +- validator/src/main.rs | 4 +- 7 files changed, 121 insertions(+), 114 deletions(-) diff --git a/core/src/validator.rs b/core/src/validator.rs index 619ecdf98..8a8c854e6 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -54,6 +54,7 @@ use { entry_notifier_service::{EntryNotifierSender, EntryNotifierService}, leader_schedule::FixedSchedule, leader_schedule_cache::LeaderScheduleCache, + use_snapshot_archives_at_startup::UseSnapshotArchivesAtStartup, }, solana_measure::measure::Measure, solana_metrics::{datapoint_info, poh_timing_point::PohTimingSender}, @@ -251,7 +252,7 @@ pub struct ValidatorConfig { pub block_verification_method: BlockVerificationMethod, pub block_production_method: BlockProductionMethod, pub generator_config: Option, - pub boot_from_local_state: bool, + pub use_snapshot_archives_at_startup: UseSnapshotArchivesAtStartup, } impl Default for ValidatorConfig { @@ -318,7 +319,7 @@ impl Default for ValidatorConfig { block_verification_method: BlockVerificationMethod::default(), block_production_method: BlockProductionMethod::default(), generator_config: None, - boot_from_local_state: false, + use_snapshot_archives_at_startup: UseSnapshotArchivesAtStartup::default(), } } } @@ -1622,7 +1623,7 @@ fn load_blockstore( accounts_db_test_hash_calculation: config.accounts_db_test_hash_calculation, accounts_db_skip_shrink: config.accounts_db_skip_shrink, runtime_config: config.runtime_config.clone(), - boot_from_local_state: config.boot_from_local_state, + use_snapshot_archives_at_startup: config.use_snapshot_archives_at_startup, ..blockstore_processor::ProcessOptions::default() }; diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 7fec357b8..f20accf66 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -980,10 +980,9 @@ fn assert_capitalization(bank: &Bank) { /// Get the AccessType required, based on `process_options` fn get_access_type(process_options: &ProcessOptions) -> AccessType { - if process_options.boot_from_local_state { - AccessType::PrimaryForMaintenance - } else { - AccessType::Secondary + match process_options.use_snapshot_archives_at_startup { + UseSnapshotArchivesAtStartup::Always => AccessType::Secondary, + UseSnapshotArchivesAtStartup::Never => AccessType::PrimaryForMaintenance, } } @@ -2513,11 +2512,6 @@ fn main() { ); } - let boot_from_local_state = value_t_or_exit!( - arg_matches, - use_snapshot_archives_at_startup::cli::NAME, - UseSnapshotArchivesAtStartup - ) == UseSnapshotArchivesAtStartup::Never; let process_options = ProcessOptions { new_hard_forks: hardforks_of(arg_matches, "hard_forks"), run_verification: !(arg_matches.is_present("skip_poh_verify") @@ -2540,7 +2534,11 @@ fn main() { .is_present("accounts_db_test_hash_calculation"), accounts_db_skip_shrink: arg_matches.is_present("accounts_db_skip_shrink"), runtime_config: RuntimeConfig::default(), - boot_from_local_state, + use_snapshot_archives_at_startup: value_t_or_exit!( + arg_matches, + use_snapshot_archives_at_startup::cli::NAME, + UseSnapshotArchivesAtStartup + ), ..ProcessOptions::default() }; let print_accounts_stats = arg_matches.is_present("print_accounts_stats"); @@ -2583,17 +2581,16 @@ fn main() { ), }; - let boot_from_local_state = value_t_or_exit!( - arg_matches, - use_snapshot_archives_at_startup::cli::NAME, - UseSnapshotArchivesAtStartup - ) == UseSnapshotArchivesAtStartup::Never; let process_options = ProcessOptions { new_hard_forks: hardforks_of(arg_matches, "hard_forks"), halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(), run_verification: false, accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), - boot_from_local_state, + use_snapshot_archives_at_startup: value_t_or_exit!( + arg_matches, + use_snapshot_archives_at_startup::cli::NAME, + UseSnapshotArchivesAtStartup + ), ..ProcessOptions::default() }; @@ -2715,17 +2712,16 @@ fn main() { NonZeroUsize ); let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); - let boot_from_local_state = value_t_or_exit!( - arg_matches, - use_snapshot_archives_at_startup::cli::NAME, - UseSnapshotArchivesAtStartup - ) == UseSnapshotArchivesAtStartup::Never; let mut process_options = ProcessOptions { new_hard_forks, run_verification: false, accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), accounts_db_skip_shrink: arg_matches.is_present("accounts_db_skip_shrink"), - boot_from_local_state, + use_snapshot_archives_at_startup: value_t_or_exit!( + arg_matches, + use_snapshot_archives_at_startup::cli::NAME, + UseSnapshotArchivesAtStartup + ), ..ProcessOptions::default() }; let blockstore = Arc::new(open_blockstore( @@ -3133,17 +3129,16 @@ fn main() { } ("accounts", Some(arg_matches)) => { let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok(); - let boot_from_local_state = value_t_or_exit!( - arg_matches, - use_snapshot_archives_at_startup::cli::NAME, - UseSnapshotArchivesAtStartup - ) == UseSnapshotArchivesAtStartup::Never; let process_options = ProcessOptions { new_hard_forks: hardforks_of(arg_matches, "hard_forks"), halt_at_slot, run_verification: false, accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), - boot_from_local_state, + use_snapshot_archives_at_startup: value_t_or_exit!( + arg_matches, + use_snapshot_archives_at_startup::cli::NAME, + UseSnapshotArchivesAtStartup + ), ..ProcessOptions::default() }; let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); @@ -3229,17 +3224,16 @@ fn main() { } ("capitalization", Some(arg_matches)) => { let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok(); - let boot_from_local_state = value_t_or_exit!( - arg_matches, - use_snapshot_archives_at_startup::cli::NAME, - UseSnapshotArchivesAtStartup - ) == UseSnapshotArchivesAtStartup::Never; let process_options = ProcessOptions { new_hard_forks: hardforks_of(arg_matches, "hard_forks"), halt_at_slot, run_verification: false, accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), - boot_from_local_state, + use_snapshot_archives_at_startup: value_t_or_exit!( + arg_matches, + use_snapshot_archives_at_startup::cli::NAME, + UseSnapshotArchivesAtStartup + ), ..ProcessOptions::default() }; let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 4193b5047..59e25fbeb 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -7,6 +7,7 @@ use { }, entry_notifier_service::EntryNotifierSender, leader_schedule_cache::LeaderScheduleCache, + use_snapshot_archives_at_startup::UseSnapshotArchivesAtStartup, }, log::*, solana_runtime::{ @@ -197,74 +198,23 @@ fn bank_forks_from_snapshot( } let (deserialized_bank, full_snapshot_archive_info, incremental_snapshot_archive_info) = - if process_options.boot_from_local_state { - let bank = snapshot_utils::bank_from_latest_snapshot_dir( - &snapshot_config.bank_snapshots_dir, - genesis_config, - &process_options.runtime_config, - &account_paths, - process_options.debug_keys.clone(), - None, - process_options.account_indexes.clone(), - process_options.limit_load_slot_count_from_snapshot, - process_options.shrink_ratio, - process_options.verify_index, - process_options.accounts_db_config.clone(), - accounts_update_notifier, - exit, - ) - .expect("load bank from local state"); - - // The highest snapshot *archives* are still needed, as they are - // the starting snapshot hashes, which are used by the background services - // when performing snapshot-related tasks. - let full_snapshot_archive_info = - snapshot_utils::get_highest_full_snapshot_archive_info( - &snapshot_config.full_snapshot_archives_dir, - ) - // SAFETY: Calling `bank_forks_from_snapshot` requires at least a full snapshot - .expect("get highest full snapshot"); - - let incremental_snapshot_archive_info = - snapshot_utils::get_highest_incremental_snapshot_archive_info( - &snapshot_config.incremental_snapshot_archives_dir, - full_snapshot_archive_info.slot(), + match process_options.use_snapshot_archives_at_startup { + UseSnapshotArchivesAtStartup::Always => { + // Given that we are going to boot from an archive, the append vecs held in the snapshot dirs for fast-boot should + // be released. They will be released by the account_background_service anyway. But in the case of the account_paths + // using memory-mounted file system, they are not released early enough to give space for the new append-vecs from + // the archives, causing the out-of-memory problem. So, purge the snapshot dirs upfront before loading from the archive. + snapshot_utils::purge_old_bank_snapshots( + &snapshot_config.bank_snapshots_dir, + 0, + None, ); - // If a newer snapshot archive was downloaded, it is possible that its slot is - // higher than the local bank we just loaded. Did the user intend for this? - let latest_snapshot_archive_slot = std::cmp::max( - full_snapshot_archive_info.slot(), - incremental_snapshot_archive_info - .as_ref() - .map(SnapshotArchiveInfoGetter::slot) - .unwrap_or(0), - ); - if bank.slot() < latest_snapshot_archive_slot { - warn!( - "Starting up from local state at slot {}, which is *older* \ - than the latest snapshot archive at slot {}. If this is not \ - desired, change the --use-snapshot-archives-at-startup \ - CLI option to \"always\" and restart.", - bank.slot(), - latest_snapshot_archive_slot, - ); - } - - ( - bank, - full_snapshot_archive_info, - incremental_snapshot_archive_info, - ) - } else { - // Given that we are going to boot from an archive, the accountvecs held in the snapshot dirs for fast-boot should - // be released. They will be released by the account_background_service anyway. But in the case of the account_paths - // using memory-mounted file system, they are not released early enough to give space for the new append-vecs from - // the archives, causing the out-of-memory problem. So, purge the snapshot dirs upfront before loading from the archive. - snapshot_utils::purge_old_bank_snapshots(&snapshot_config.bank_snapshots_dir, 0, None); - - let (deserialized_bank, full_snapshot_archive_info, incremental_snapshot_archive_info) = - snapshot_utils::bank_from_latest_snapshot_archives( + let ( + deserialized_bank, + full_snapshot_archive_info, + incremental_snapshot_archive_info, + ) = snapshot_utils::bank_from_latest_snapshot_archives( &snapshot_config.bank_snapshots_dir, &snapshot_config.full_snapshot_archives_dir, &snapshot_config.incremental_snapshot_archives_dir, @@ -285,11 +235,72 @@ fn bank_forks_from_snapshot( ) .expect("load bank from snapshot archives"); - ( - deserialized_bank, - full_snapshot_archive_info, - incremental_snapshot_archive_info, - ) + ( + deserialized_bank, + full_snapshot_archive_info, + incremental_snapshot_archive_info, + ) + } + UseSnapshotArchivesAtStartup::Never => { + let bank = snapshot_utils::bank_from_latest_snapshot_dir( + &snapshot_config.bank_snapshots_dir, + genesis_config, + &process_options.runtime_config, + &account_paths, + process_options.debug_keys.clone(), + None, + process_options.account_indexes.clone(), + process_options.limit_load_slot_count_from_snapshot, + process_options.shrink_ratio, + process_options.verify_index, + process_options.accounts_db_config.clone(), + accounts_update_notifier, + exit, + ) + .expect("load bank from local state"); + + // The highest snapshot *archives* are still needed, as they are + // the starting snapshot hashes, which are used by the background services + // when performing snapshot-related tasks. + let full_snapshot_archive_info = + snapshot_utils::get_highest_full_snapshot_archive_info( + &snapshot_config.full_snapshot_archives_dir, + ) + // SAFETY: Calling `bank_forks_from_snapshot` requires at least a full snapshot + .expect("get highest full snapshot"); + + let incremental_snapshot_archive_info = + snapshot_utils::get_highest_incremental_snapshot_archive_info( + &snapshot_config.incremental_snapshot_archives_dir, + full_snapshot_archive_info.slot(), + ); + + // If a newer snapshot archive was downloaded, it is possible that its slot is + // higher than the local bank we just loaded. Did the user intend for this? + let latest_snapshot_archive_slot = std::cmp::max( + full_snapshot_archive_info.slot(), + incremental_snapshot_archive_info + .as_ref() + .map(SnapshotArchiveInfoGetter::slot) + .unwrap_or(0), + ); + if bank.slot() < latest_snapshot_archive_slot { + warn!( + "Starting up from local state at slot {}, which is *older* \ + than the latest snapshot archive at slot {}. If this is not \ + desired, change the --use-snapshot-archives-at-startup \ + CLI option to \"always\" and restart.", + bank.slot(), + latest_snapshot_archive_slot, + ); + } + + ( + bank, + full_snapshot_archive_info, + incremental_snapshot_archive_info, + ) + } }; if let Some(shrink_paths) = shrink_paths { diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 99e1da377..8a8c328d4 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -7,6 +7,7 @@ use { entry_notifier_service::{EntryNotification, EntryNotifierSender}, leader_schedule_cache::LeaderScheduleCache, token_balances::collect_token_balances, + use_snapshot_archives_at_startup::UseSnapshotArchivesAtStartup, }, chrono_humanize::{Accuracy, HumanTime, Tense}, crossbeam_channel::Sender, @@ -632,8 +633,7 @@ pub struct ProcessOptions { /// true if after processing the contents of the blockstore at startup, we should run an accounts hash calc /// This is useful for debugging. pub run_final_accounts_hash_calc: bool, - /// Enable booting from state already on disk, instead of extracting if from a snapshot archive. - pub boot_from_local_state: bool, + pub use_snapshot_archives_at_startup: UseSnapshotArchivesAtStartup, } pub fn test_process_blockstore( diff --git a/local-cluster/src/validator_configs.rs b/local-cluster/src/validator_configs.rs index 4c3de6439..70211b5da 100644 --- a/local-cluster/src/validator_configs.rs +++ b/local-cluster/src/validator_configs.rs @@ -67,7 +67,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig { block_verification_method: config.block_verification_method.clone(), block_production_method: config.block_production_method.clone(), generator_config: config.generator_config.clone(), - boot_from_local_state: config.boot_from_local_state, + use_snapshot_archives_at_startup: config.use_snapshot_archives_at_startup, } } diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 13aa39d1b..d40c4d389 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -19,6 +19,7 @@ use { solana_ledger::{ ancestor_iterator::AncestorIterator, bank_forks_utils, blockstore::Blockstore, blockstore_processor::ProcessOptions, leader_schedule::FixedSchedule, + use_snapshot_archives_at_startup::UseSnapshotArchivesAtStartup, }, solana_local_cluster::{ cluster::{Cluster, ClusterValidatorInfo}, @@ -4876,7 +4877,7 @@ fn test_boot_from_local_state() { // restart WITH fastboot info!("Restarting validator2 from local state..."); let mut validator2_info = cluster.exit_node(&validator2_identity.pubkey()); - validator2_info.config.boot_from_local_state = true; + validator2_info.config.use_snapshot_archives_at_startup = UseSnapshotArchivesAtStartup::Never; cluster.restart_node( &validator2_identity.pubkey(), validator2_info, diff --git a/validator/src/main.rs b/validator/src/main.rs index 1c4180665..9ceced461 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1378,11 +1378,11 @@ pub fn main() { }, staked_nodes_overrides: staked_nodes_overrides.clone(), replay_slots_concurrently: matches.is_present("replay_slots_concurrently"), - boot_from_local_state: value_t_or_exit!( + use_snapshot_archives_at_startup: value_t_or_exit!( matches, use_snapshot_archives_at_startup::cli::NAME, UseSnapshotArchivesAtStartup - ) == UseSnapshotArchivesAtStartup::Never, + ), ..ValidatorConfig::default() };