blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ledger-tool: Condense ProcessOptions parsing logic (#34694) 

The code to parse process options was repeated in several (8) commands
that replay block processing. So, move the argument parsing into a
common helper that can be used by all of those commands.
This commit is contained in:
steviez 2024-01-11 13:24:21 -06:00 committed by GitHub
parent 8e2943604b
commit a203f1489e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 106 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
ledger-tool/src/args.rs
View File

@ -1,16 +1,76 @@
use {
crate::LEDGER_TOOL_DIRECTORY,
clap::{value_t, values_t, values_t_or_exit, ArgMatches},
clap::{value_t, value_t_or_exit, values_t, values_t_or_exit, ArgMatches},
solana_accounts_db::{
accounts_db::{AccountsDb, AccountsDbConfig},
accounts_index::{AccountsIndexConfig, IndexLimitMb},
partitioned_rewards::TestPartitionedEpochRewards,
},
solana_runtime::snapshot_utils,
solana_clap_utils::input_parsers::pubkeys_of,
solana_ledger::{
blockstore_processor::ProcessOptions,
use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup},
},
solana_runtime::{runtime_config::RuntimeConfig, snapshot_utils},
solana_sdk::clock::Slot,
std::path::{Path, PathBuf},
std::{
collections::HashSet,
path::{Path, PathBuf},
sync::Arc,
},
};
/// Parse a `ProcessOptions` from subcommand arguments. This function attempts
/// to parse all flags related to `ProcessOptions`; however, subcommands that
/// use this function may not support all flags.
pub fn parse_process_options(ledger_path: &Path, arg_matches: &ArgMatches<'_>) -> ProcessOptions {
let new_hard_forks = hardforks_of(arg_matches, "hard_forks");
let accounts_db_config = Some(get_accounts_db_config(ledger_path, arg_matches));
let runtime_config = RuntimeConfig::default();
if arg_matches.is_present("skip_poh_verify") {
eprintln!("--skip-poh-verify is deprecated. Replace with --skip-verification.");
}
let run_verification =
!(arg_matches.is_present("skip_poh_verify") || arg_matches.is_present("skip_verification"));
let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok();
let use_snapshot_archives_at_startup = value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
);
let accounts_db_skip_shrink = arg_matches.is_present("accounts_db_skip_shrink");
let accounts_db_test_hash_calculation =
arg_matches.is_present("accounts_db_test_hash_calculation");
let verify_index = arg_matches.is_present("verify_accounts_index");
let limit_load_slot_count_from_snapshot =
value_t!(arg_matches, "limit_load_slot_count_from_snapshot", usize).ok();
let on_halt_store_hash_raw_data_for_debug =
arg_matches.is_present("halt_at_slot_store_hash_raw_data");
let run_final_accounts_hash_calc = arg_matches.is_present("run_final_hash_calc");
let debug_keys = pubkeys_of(arg_matches, "debug_key")
.map(|pubkeys| Arc::new(pubkeys.into_iter().collect::<HashSet<_>>()));
let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
ProcessOptions {
new_hard_forks,
runtime_config,
accounts_db_config,
accounts_db_skip_shrink,
accounts_db_test_hash_calculation,
verify_index,
limit_load_slot_count_from_snapshot,
on_halt_store_hash_raw_data_for_debug,
run_final_accounts_hash_calc,
debug_keys,
run_verification,
allow_dead_slots,
halt_at_slot,
use_snapshot_archives_at_startup,
..ProcessOptions::default()
}
}
// Build an `AccountsDbConfig` from subcommand arguments. All of the arguments
// matched by this functional are either optional or have a default value.
// Thus, a subcommand need not support all of the arguments that are matched

10
ledger-tool/src/ledger_utils.rs
View File

@ -21,6 +21,7 @@ use {
blockstore_processor::{
self, BlockstoreProcessorError, ProcessOptions, TransactionStatusSender,
},
use_snapshot_archives_at_startup::UseSnapshotArchivesAtStartup,
},
solana_measure::measure,
solana_rpc::transaction_status_service::TransactionStatusService,
@ -564,3 +565,12 @@ pub fn get_program_ids(tx: &VersionedTransaction) -> impl Iterator<Item = &Pubke
.iter()
.map(|ix| ix.program_id(account_keys))
}
/// Get the AccessType required, based on `process_options`
pub(crate) fn get_access_type(process_options: &ProcessOptions) -> AccessType {
match process_options.use_snapshot_archives_at_startup {
UseSnapshotArchivesAtStartup::Always => AccessType::Secondary,
UseSnapshotArchivesAtStartup::Never => AccessType::PrimaryForMaintenance,
UseSnapshotArchivesAtStartup::WhenNewest => AccessType::PrimaryForMaintenance,
}
}

136
ledger-tool/src/main.rs
View File

@ -34,14 +34,12 @@ use {
solana_ledger::{
blockstore::{create_new_ledger, Blockstore},
blockstore_options::{AccessType, LedgerColumnOptions},
blockstore_processor::ProcessOptions,
use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup},
use_snapshot_archives_at_startup,
},
solana_measure::{measure, measure::Measure},
solana_runtime::{
bank::{bank_hash_details, Bank, RewardCalculationEvent, TotalAccountsStats},
bank_forks::BankForks,
runtime_config::RuntimeConfig,
snapshot_archive_info::SnapshotArchiveInfoGetter,
snapshot_bank_utils,
snapshot_minimizer::SnapshotMinimizer,
@ -561,15 +559,6 @@ fn assert_capitalization(bank: &Bank) {
assert!(bank.calculate_and_verify_capitalization(debug_verify));
}
/// Get the AccessType required, based on `process_options`
fn get_access_type(process_options: &ProcessOptions) -> AccessType {
match process_options.use_snapshot_archives_at_startup {
UseSnapshotArchivesAtStartup::Always => AccessType::Secondary,
UseSnapshotArchivesAtStartup::Never => AccessType::PrimaryForMaintenance,
UseSnapshotArchivesAtStartup::WhenNewest => AccessType::PrimaryForMaintenance,
}
}
#[cfg(not(target_env = "msvc"))]
use jemallocator::Jemalloc;
@ -969,7 +958,8 @@ fn main() {
.arg(&disable_disk_index)
.arg(&accountsdb_verify_refcounts)
.arg(&accounts_db_skip_initial_hash_calc_arg)
.arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash),
.arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash)
.arg(&use_snapshot_archives_at_startup),
)
.subcommand(
SubCommand::with_name("bank-hash")
@ -981,7 +971,8 @@ fn main() {
.arg(&disable_disk_index)
.arg(&accountsdb_verify_refcounts)
.arg(&accounts_db_skip_initial_hash_calc_arg)
.arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash),
.arg(&accounts_db_test_skip_rewrites_but_include_in_bank_hash)
.arg(&use_snapshot_archives_at_startup),
)
.subcommand(
SubCommand::with_name("verify")
@ -1555,13 +1546,12 @@ fn main() {
println!("{}", open_genesis_config_by(&output_directory, arg_matches));
}
("shred-version", Some(arg_matches)) => {
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
halt_at_slot: Some(0),
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
..ProcessOptions::default()
};
let mut process_options = parse_process_options(&ledger_path, arg_matches);
// Respect a user-set --halt-at-slot; otherwise, set Some(0) to avoid
// processing any additional banks and just use the snapshot bank
if process_options.halt_at_slot.is_none() {
process_options.halt_at_slot = Some(0);
}
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
@ -1586,13 +1576,7 @@ fn main() {
);
}
("bank-hash", Some(arg_matches)) => {
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(),
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
..ProcessOptions::default()
};
let process_options = parse_process_options(&ledger_path, arg_matches);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
@ -1623,44 +1607,7 @@ fn main() {
},
);
let debug_keys = pubkeys_of(arg_matches, "debug_key")
.map(|pubkeys| Arc::new(pubkeys.into_iter().collect::<HashSet<_>>()));
if arg_matches.is_present("skip_poh_verify") {
eprintln!(
"--skip-poh-verify is deprecated. Replace with --skip-verification."
);
}
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
run_verification: !(arg_matches.is_present("skip_poh_verify")
|| arg_matches.is_present("skip_verification")),
on_halt_store_hash_raw_data_for_debug: arg_matches
.is_present("halt_at_slot_store_hash_raw_data"),
run_final_accounts_hash_calc: arg_matches.is_present("run_final_hash_calc"),
halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(),
debug_keys,
limit_load_slot_count_from_snapshot: value_t!(
arg_matches,
"limit_load_slot_count_from_snapshot",
usize
)
.ok(),
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
verify_index: arg_matches.is_present("verify_accounts_index"),
allow_dead_slots: arg_matches.is_present("allow_dead_slots"),
accounts_db_test_hash_calculation: arg_matches
.is_present("accounts_db_test_hash_calculation"),
accounts_db_skip_shrink: arg_matches.is_present("accounts_db_skip_shrink"),
runtime_config: RuntimeConfig::default(),
use_snapshot_archives_at_startup: value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
),
..ProcessOptions::default()
};
let process_options = parse_process_options(&ledger_path, arg_matches);
let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
let write_bank_file = arg_matches.is_present("write_bank_file");
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
@ -1706,19 +1653,7 @@ fn main() {
),
};
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(),
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
use_snapshot_archives_at_startup: value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
),
..ProcessOptions::default()
};
let process_options = parse_process_options(&ledger_path, arg_matches);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
@ -1771,7 +1706,6 @@ fn main() {
});
let mut warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok();
let remove_stake_accounts = arg_matches.is_present("remove_stake_accounts");
let new_hard_forks = hardforks_of(arg_matches, "hard_forks");
let faucet_pubkey = pubkey_of(arg_matches, "faucet_pubkey");
let faucet_lamports =
@ -1834,18 +1768,8 @@ fn main() {
NonZeroUsize
);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let mut process_options = ProcessOptions {
new_hard_forks,
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
accounts_db_skip_shrink: arg_matches.is_present("accounts_db_skip_shrink"),
use_snapshot_archives_at_startup: value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
),
..ProcessOptions::default()
};
let mut process_options = parse_process_options(&ledger_path, arg_matches);
let blockstore = Arc::new(open_blockstore(
&ledger_path,
arg_matches,
@ -2266,19 +2190,7 @@ fn main() {
);
}
("accounts", Some(arg_matches)) => {
let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok();
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
halt_at_slot,
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
use_snapshot_archives_at_startup: value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
),
..ProcessOptions::default()
};
let process_options = parse_process_options(&ledger_path, arg_matches);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let include_sysvars = arg_matches.is_present("include_sysvars");
let blockstore = open_blockstore(
@ -2360,19 +2272,7 @@ fn main() {
}
}
("capitalization", Some(arg_matches)) => {
let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok();
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
halt_at_slot,
run_verification: false,
accounts_db_config: Some(get_accounts_db_config(&ledger_path, arg_matches)),
use_snapshot_archives_at_startup: value_t_or_exit!(
arg_matches,
use_snapshot_archives_at_startup::cli::NAME,
UseSnapshotArchivesAtStartup
),
..ProcessOptions::default()
};
let process_options = parse_process_options(&ledger_path, arg_matches);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,

42
ledger-tool/src/program.rs
View File

@ -8,9 +8,8 @@ use {
create_vm, load_program_from_bytes, serialization::serialize_parameters,
syscalls::create_program_runtime_environment_v1,
},
solana_clap_utils::input_parsers::pubkeys_of,
solana_cli_output::{OutputFormat, QuietDisplay, VerboseDisplay},
solana_ledger::{blockstore_options::AccessType, blockstore_processor::ProcessOptions},
solana_ledger::{blockstore_options::AccessType, use_snapshot_archives_at_startup},
solana_program_runtime::{
invoke_context::InvokeContext,
loaded_programs::{LoadProgramMetrics, LoadedProgramType, DELAY_VISIBILITY_SLOT_OFFSET},
@ -20,7 +19,7 @@ use {
assembler::assemble, elf::Executable, static_analysis::Analysis,
verifier::RequisiteVerifier,
},
solana_runtime::{bank::Bank, runtime_config::RuntimeConfig},
solana_runtime::bank::Bank,
solana_sdk::{
account::AccountSharedData,
account_utils::StateMut,
@ -30,7 +29,6 @@ use {
transaction_context::{IndexOfAccount, InstructionAccount},
},
std::{
collections::HashSet,
fmt::{self, Debug, Formatter},
fs::File,
io::{Read, Seek, Write},
@ -72,29 +70,7 @@ fn load_accounts(path: &Path) -> Result<Input> {
}
fn load_blockstore(ledger_path: &Path, arg_matches: &ArgMatches<'_>) -> Arc<Bank> {
let debug_keys = pubkeys_of(arg_matches, "debug_key")
.map(|pubkeys| Arc::new(pubkeys.into_iter().collect::<HashSet<_>>()));
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
run_verification: false,
on_halt_store_hash_raw_data_for_debug: false,
run_final_accounts_hash_calc: false,
halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(),
debug_keys,
limit_load_slot_count_from_snapshot: value_t!(
arg_matches,
"limit_load_slot_count_from_snapshot",
usize
)
.ok(),
accounts_db_config: Some(get_accounts_db_config(ledger_path, arg_matches)),
verify_index: false,
allow_dead_slots: arg_matches.is_present("allow_dead_slots"),
accounts_db_test_hash_calculation: false,
accounts_db_skip_shrink: arg_matches.is_present("accounts_db_skip_shrink"),
runtime_config: RuntimeConfig::default(),
..ProcessOptions::default()
};
let process_options = parse_process_options(ledger_path, arg_matches);
let snapshot_archive_path = value_t!(arg_matches, "snapshot_archive_path", String)
.ok()
.map(PathBuf::from);
@ -137,6 +113,17 @@ impl ProgramSubCommand for App<'_, '_> {
.takes_value(true)
.default_value("10485760")
.help("maximum total uncompressed size of unpacked genesis archive");
let use_snapshot_archives_at_startup =
Arg::with_name(use_snapshot_archives_at_startup::cli::NAME)
.long(use_snapshot_archives_at_startup::cli::LONG_ARG)
.takes_value(true)
.possible_values(use_snapshot_archives_at_startup::cli::POSSIBLE_VALUES)
.default_value(
use_snapshot_archives_at_startup::cli::default_value_for_ledger_tool(),
)
.help(use_snapshot_archives_at_startup::cli::HELP)
.long_help(use_snapshot_archives_at_startup::cli::LONG_HELP);
self.subcommand(
SubCommand::with_name("program")
.about("Run to test, debug, and analyze on-chain programs.")
@ -188,6 +175,7 @@ and the following fields are required
.default_value("0"),
)
.arg(&max_genesis_arg)
.arg(&use_snapshot_archives_at_startup)
.arg(
Arg::with_name("memory")
.help("Heap memory for the program to run on")