diff --git a/ledger-tool/src/args.rs b/ledger-tool/src/args.rs index 93f842ed3..db39f6062 100644 --- a/ledger-tool/src/args.rs +++ b/ledger-tool/src/args.rs @@ -1,16 +1,76 @@ use { crate::LEDGER_TOOL_DIRECTORY, - clap::{value_t, values_t, values_t_or_exit, ArgMatches}, + clap::{value_t, value_t_or_exit, values_t, values_t_or_exit, ArgMatches}, solana_accounts_db::{ accounts_db::{AccountsDb, AccountsDbConfig}, accounts_index::{AccountsIndexConfig, IndexLimitMb}, partitioned_rewards::TestPartitionedEpochRewards, }, - solana_runtime::snapshot_utils, + solana_clap_utils::input_parsers::pubkeys_of, + solana_ledger::{ + blockstore_processor::ProcessOptions, + use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup}, + }, + solana_runtime::{runtime_config::RuntimeConfig, snapshot_utils}, solana_sdk::clock::Slot, - std::path::{Path, PathBuf}, + std::{ + collections::HashSet, + path::{Path, PathBuf}, + sync::Arc, + }, }; +/// Parse a `ProcessOptions` from subcommand arguments. This function attempts +/// to parse all flags related to `ProcessOptions`; however, subcommands that +/// use this function may not support all flags. +pub fn parse_process_options(ledger_path: &Path, arg_matches: &ArgMatches<'_>) -> ProcessOptions { + let new_hard_forks = hardforks_of(arg_matches, "hard_forks"); + let accounts_db_config = Some(get_accounts_db_config(ledger_path, arg_matches)); + let runtime_config = RuntimeConfig::default(); + + if arg_matches.is_present("skip_poh_verify") { + eprintln!("--skip-poh-verify is deprecated. Replace with --skip-verification."); + } + let run_verification = + !(arg_matches.is_present("skip_poh_verify") || arg_matches.is_present("skip_verification")); + let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok(); + let use_snapshot_archives_at_startup = value_t_or_exit!( + arg_matches, + use_snapshot_archives_at_startup::cli::NAME, + UseSnapshotArchivesAtStartup + ); + let accounts_db_skip_shrink = arg_matches.is_present("accounts_db_skip_shrink"); + let accounts_db_test_hash_calculation = + arg_matches.is_present("accounts_db_test_hash_calculation"); + let verify_index = arg_matches.is_present("verify_accounts_index"); + let limit_load_slot_count_from_snapshot = + value_t!(arg_matches, "limit_load_slot_count_from_snapshot", usize).ok(); + let on_halt_store_hash_raw_data_for_debug = + arg_matches.is_present("halt_at_slot_store_hash_raw_data"); + let run_final_accounts_hash_calc = arg_matches.is_present("run_final_hash_calc"); + let debug_keys = pubkeys_of(arg_matches, "debug_key") + .map(|pubkeys| Arc::new(pubkeys.into_iter().collect::>())); + let allow_dead_slots = arg_matches.is_present("allow_dead_slots"); + + ProcessOptions { + new_hard_forks, + runtime_config, + accounts_db_config, + accounts_db_skip_shrink, + accounts_db_test_hash_calculation, + verify_index, + limit_load_slot_count_from_snapshot, + on_halt_store_hash_raw_data_for_debug, + run_final_accounts_hash_calc, + debug_keys, + run_verification, + allow_dead_slots, + halt_at_slot, + use_snapshot_archives_at_startup, + ..ProcessOptions::default() + } +} + // Build an `AccountsDbConfig` from subcommand arguments. All of the arguments // matched by this functional are either optional or have a default value. // Thus, a subcommand need not support all of the arguments that are matched diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index 1961225ce..5c2713ef1 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -21,6 +21,7 @@ use { blockstore_processor::{ self, BlockstoreProcessorError, ProcessOptions, TransactionStatusSender, }, + use_snapshot_archives_at_startup::UseSnapshotArchivesAtStartup, }, solana_measure::measure, solana_rpc::transaction_status_service::TransactionStatusService, @@ -564,3 +565,12 @@ pub fn get_program_ids(tx: &VersionedTransaction) -> impl Iterator AccessType { + match process_options.use_snapshot_archives_at_startup { + UseSnapshotArchivesAtStartup::Always => AccessType::Secondary, + UseSnapshotArchivesAtStartup::Never => AccessType::PrimaryForMaintenance, + UseSnapshotArchivesAtStartup::WhenNewest => AccessType::PrimaryForMaintenance, + } +} diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 41e034877..d81d9c212 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -34,14 +34,12 @@ use { solana_ledger::{ blockstore::{create_new_ledger, Blockstore}, blockstore_options::{AccessType, LedgerColumnOptions}, - blockstore_processor::ProcessOptions, - use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup}, + use_snapshot_archives_at_startup, }, solana_measure::{measure, measure::Measure}, solana_runtime::{ bank::{bank_hash_details, Bank, RewardCalculationEvent, TotalAccountsStats}, bank_forks::BankForks, - runtime_config::RuntimeConfig, snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_bank_utils, snapshot_minimizer::SnapshotMinimizer, @@ -561,15 +559,6 @@ fn assert_capitalization(bank: &Bank) { assert!(bank.calculate_and_verify_capitalization(debug_verify)); } -/// Get the AccessType required, based on `process_options` -fn get_access_type(process_options: &ProcessOptions) -> AccessType { - match process_options.use_snapshot_archives_at_startup { - UseSnapshotArchivesAtStartup::Always => AccessType::Secondary, - UseSnapshotArchivesAtStartup::Never => AccessType::PrimaryForMaintenance, - UseSnapshotArchivesAtStartup::WhenNewest => AccessType::PrimaryForMaintenance, - } -} - #[cfg(not(target_env = "msvc"))] use jemallocator::Jemalloc; @@ -969,7 +958,8 @@ fn main() { .arg(&disable_disk_index) .arg(&accountsdb_verify_refcounts) .arg(&accounts_db_skip_initial_hash_calc_arg) - .arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash), + .arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash) + .arg(&use_snapshot_archives_at_startup), ) .subcommand( SubCommand::with_name("bank-hash") @@ -981,7 +971,8 @@ fn main() { .arg(&disable_disk_index) .arg(&accountsdb_verify_refcounts) .arg(&accounts_db_skip_initial_hash_calc_arg) - .arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash), + .arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash) + .arg(&use_snapshot_archives_at_startup), ) .subcommand( SubCommand::with_name("verify") @@ -1555,13 +1546,12 @@ fn main() { println!("{}", open_genesis_config_by(&output_directory, arg_matches)); } ("shred-version", Some(arg_matches)) => { - let process_options = ProcessOptions { - new_hard_forks: hardforks_of(arg_matches, "hard_forks"), - halt_at_slot: Some(0), - run_verification: false, - accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), - ..ProcessOptions::default() - }; + let mut process_options = parse_process_options(&ledger_path, arg_matches); + // Respect a user-set --halt-at-slot; otherwise, set Some(0) to avoid + // processing any additional banks and just use the snapshot bank + if process_options.halt_at_slot.is_none() { + process_options.halt_at_slot = Some(0); + } let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let blockstore = open_blockstore( &ledger_path, @@ -1586,13 +1576,7 @@ fn main() { ); } ("bank-hash", Some(arg_matches)) => { - let process_options = ProcessOptions { - new_hard_forks: hardforks_of(arg_matches, "hard_forks"), - halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(), - run_verification: false, - accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), - ..ProcessOptions::default() - }; + let process_options = parse_process_options(&ledger_path, arg_matches); let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let blockstore = open_blockstore( &ledger_path, @@ -1623,44 +1607,7 @@ fn main() { }, ); - let debug_keys = pubkeys_of(arg_matches, "debug_key") - .map(|pubkeys| Arc::new(pubkeys.into_iter().collect::>())); - - if arg_matches.is_present("skip_poh_verify") { - eprintln!( - "--skip-poh-verify is deprecated. Replace with --skip-verification." - ); - } - - let process_options = ProcessOptions { - new_hard_forks: hardforks_of(arg_matches, "hard_forks"), - run_verification: !(arg_matches.is_present("skip_poh_verify") - || arg_matches.is_present("skip_verification")), - on_halt_store_hash_raw_data_for_debug: arg_matches - .is_present("halt_at_slot_store_hash_raw_data"), - run_final_accounts_hash_calc: arg_matches.is_present("run_final_hash_calc"), - halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(), - debug_keys, - limit_load_slot_count_from_snapshot: value_t!( - arg_matches, - "limit_load_slot_count_from_snapshot", - usize - ) - .ok(), - accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), - verify_index: arg_matches.is_present("verify_accounts_index"), - allow_dead_slots: arg_matches.is_present("allow_dead_slots"), - accounts_db_test_hash_calculation: arg_matches - .is_present("accounts_db_test_hash_calculation"), - accounts_db_skip_shrink: arg_matches.is_present("accounts_db_skip_shrink"), - runtime_config: RuntimeConfig::default(), - use_snapshot_archives_at_startup: value_t_or_exit!( - arg_matches, - use_snapshot_archives_at_startup::cli::NAME, - UseSnapshotArchivesAtStartup - ), - ..ProcessOptions::default() - }; + let process_options = parse_process_options(&ledger_path, arg_matches); let print_accounts_stats = arg_matches.is_present("print_accounts_stats"); let write_bank_file = arg_matches.is_present("write_bank_file"); let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); @@ -1706,19 +1653,7 @@ fn main() { ), }; - let process_options = ProcessOptions { - new_hard_forks: hardforks_of(arg_matches, "hard_forks"), - halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(), - run_verification: false, - accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), - use_snapshot_archives_at_startup: value_t_or_exit!( - arg_matches, - use_snapshot_archives_at_startup::cli::NAME, - UseSnapshotArchivesAtStartup - ), - ..ProcessOptions::default() - }; - + let process_options = parse_process_options(&ledger_path, arg_matches); let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let blockstore = open_blockstore( &ledger_path, @@ -1771,7 +1706,6 @@ fn main() { }); let mut warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok(); let remove_stake_accounts = arg_matches.is_present("remove_stake_accounts"); - let new_hard_forks = hardforks_of(arg_matches, "hard_forks"); let faucet_pubkey = pubkey_of(arg_matches, "faucet_pubkey"); let faucet_lamports = @@ -1834,18 +1768,8 @@ fn main() { NonZeroUsize ); let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); - let mut process_options = ProcessOptions { - new_hard_forks, - run_verification: false, - accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), - accounts_db_skip_shrink: arg_matches.is_present("accounts_db_skip_shrink"), - use_snapshot_archives_at_startup: value_t_or_exit!( - arg_matches, - use_snapshot_archives_at_startup::cli::NAME, - UseSnapshotArchivesAtStartup - ), - ..ProcessOptions::default() - }; + let mut process_options = parse_process_options(&ledger_path, arg_matches); + let blockstore = Arc::new(open_blockstore( &ledger_path, arg_matches, @@ -2266,19 +2190,7 @@ fn main() { ); } ("accounts", Some(arg_matches)) => { - let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok(); - let process_options = ProcessOptions { - new_hard_forks: hardforks_of(arg_matches, "hard_forks"), - halt_at_slot, - run_verification: false, - accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), - use_snapshot_archives_at_startup: value_t_or_exit!( - arg_matches, - use_snapshot_archives_at_startup::cli::NAME, - UseSnapshotArchivesAtStartup - ), - ..ProcessOptions::default() - }; + let process_options = parse_process_options(&ledger_path, arg_matches); let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let include_sysvars = arg_matches.is_present("include_sysvars"); let blockstore = open_blockstore( @@ -2360,19 +2272,7 @@ fn main() { } } ("capitalization", Some(arg_matches)) => { - let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok(); - let process_options = ProcessOptions { - new_hard_forks: hardforks_of(arg_matches, "hard_forks"), - halt_at_slot, - run_verification: false, - accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)), - use_snapshot_archives_at_startup: value_t_or_exit!( - arg_matches, - use_snapshot_archives_at_startup::cli::NAME, - UseSnapshotArchivesAtStartup - ), - ..ProcessOptions::default() - }; + let process_options = parse_process_options(&ledger_path, arg_matches); let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let blockstore = open_blockstore( &ledger_path, diff --git a/ledger-tool/src/program.rs b/ledger-tool/src/program.rs index 97d676aff..732c2e8fe 100644 --- a/ledger-tool/src/program.rs +++ b/ledger-tool/src/program.rs @@ -8,9 +8,8 @@ use { create_vm, load_program_from_bytes, serialization::serialize_parameters, syscalls::create_program_runtime_environment_v1, }, - solana_clap_utils::input_parsers::pubkeys_of, solana_cli_output::{OutputFormat, QuietDisplay, VerboseDisplay}, - solana_ledger::{blockstore_options::AccessType, blockstore_processor::ProcessOptions}, + solana_ledger::{blockstore_options::AccessType, use_snapshot_archives_at_startup}, solana_program_runtime::{ invoke_context::InvokeContext, loaded_programs::{LoadProgramMetrics, LoadedProgramType, DELAY_VISIBILITY_SLOT_OFFSET}, @@ -20,7 +19,7 @@ use { assembler::assemble, elf::Executable, static_analysis::Analysis, verifier::RequisiteVerifier, }, - solana_runtime::{bank::Bank, runtime_config::RuntimeConfig}, + solana_runtime::bank::Bank, solana_sdk::{ account::AccountSharedData, account_utils::StateMut, @@ -30,7 +29,6 @@ use { transaction_context::{IndexOfAccount, InstructionAccount}, }, std::{ - collections::HashSet, fmt::{self, Debug, Formatter}, fs::File, io::{Read, Seek, Write}, @@ -72,29 +70,7 @@ fn load_accounts(path: &Path) -> Result { } fn load_blockstore(ledger_path: &Path, arg_matches: &ArgMatches<'_>) -> Arc { - let debug_keys = pubkeys_of(arg_matches, "debug_key") - .map(|pubkeys| Arc::new(pubkeys.into_iter().collect::>())); - let process_options = ProcessOptions { - new_hard_forks: hardforks_of(arg_matches, "hard_forks"), - run_verification: false, - on_halt_store_hash_raw_data_for_debug: false, - run_final_accounts_hash_calc: false, - halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(), - debug_keys, - limit_load_slot_count_from_snapshot: value_t!( - arg_matches, - "limit_load_slot_count_from_snapshot", - usize - ) - .ok(), - accounts_db_config: Some(get_accounts_db_config(ledger_path, arg_matches)), - verify_index: false, - allow_dead_slots: arg_matches.is_present("allow_dead_slots"), - accounts_db_test_hash_calculation: false, - accounts_db_skip_shrink: arg_matches.is_present("accounts_db_skip_shrink"), - runtime_config: RuntimeConfig::default(), - ..ProcessOptions::default() - }; + let process_options = parse_process_options(ledger_path, arg_matches); let snapshot_archive_path = value_t!(arg_matches, "snapshot_archive_path", String) .ok() .map(PathBuf::from); @@ -137,6 +113,17 @@ impl ProgramSubCommand for App<'_, '_> { .takes_value(true) .default_value("10485760") .help("maximum total uncompressed size of unpacked genesis archive"); + let use_snapshot_archives_at_startup = + Arg::with_name(use_snapshot_archives_at_startup::cli::NAME) + .long(use_snapshot_archives_at_startup::cli::LONG_ARG) + .takes_value(true) + .possible_values(use_snapshot_archives_at_startup::cli::POSSIBLE_VALUES) + .default_value( + use_snapshot_archives_at_startup::cli::default_value_for_ledger_tool(), + ) + .help(use_snapshot_archives_at_startup::cli::HELP) + .long_help(use_snapshot_archives_at_startup::cli::LONG_HELP); + self.subcommand( SubCommand::with_name("program") .about("Run to test, debug, and analyze on-chain programs.") @@ -188,6 +175,7 @@ and the following fields are required .default_value("0"), ) .arg(&max_genesis_arg) + .arg(&use_snapshot_archives_at_startup) .arg( Arg::with_name("memory") .help("Heap memory for the program to run on")