blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

ledger-tool: Move blockstore arg parsing to open_blockstore() (#34596) 

The open_blockstore() helper currently takes multiple configurable
options. While the arguments are parsed at a high enough scope in
main.rs to avoid repeated calls, this parsing is duplicated in the
separate subcommand files (program.rs and bigtable.rs).

The repeated parsing is redundant, and also prone to having to missing
an arg (as was the case with bigtable not having wal_recovery_mode).

So, this PR consolidates the parsing to a single function and uses that
function across the previous callers.
This commit is contained in:
steviez 2024-01-03 16:04:51 -06:00 committed by GitHub
parent 5a3a10e847
commit 0b49d82d3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 59 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ledger-tool/src/bigtable.rs
View File

@ -1075,9 +1075,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
let runtime = tokio::runtime::Runtime::new().unwrap();
let verbose = matches.is_present("verbose");
let force_update_to_open = matches.is_present("force_update_to_open");
let output_format = OutputFormat::from_matches(matches, "output_format", verbose);
let enforce_ulimit_nofile = !matches.is_present("ignore_ulimit_nofile_error");
let (subcommand, sub_matches) = matches.subcommand();
let instance_name = get_global_subcommand_arg(
@ -1100,10 +1098,8 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
let force_reupload = arg_matches.is_present("force_reupload");
let blockstore = crate::open_blockstore(
&canonicalize_ledger_path(ledger_path),
arg_matches,
AccessType::Secondary,
None,
force_update_to_open,
enforce_ulimit_nofile,
);
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,

41
ledger-tool/src/ledger_utils.rs
View File

@ -94,20 +94,6 @@ pub(crate) enum LoadAndProcessLedgerError {
ProcessBlockstoreFromRoot(#[source] BlockstoreProcessorError),
}
pub fn get_shred_storage_type(ledger_path: &Path, message: &str) -> ShredStorageType {
// TODO: the following shred_storage_type inference must be updated once
// the rocksdb options can be constructed via load_options_file() as the
// value picked by passing None for `max_shred_storage_size` could affect
// the persisted rocksdb options file.
match ShredStorageType::from_ledger_path(ledger_path, None) {
Some(s) => s,
None => {
info!("{}", message);
ShredStorageType::RocksLevel
}
}
}
pub fn load_and_process_ledger_or_exit(
arg_matches: &ArgMatches,
genesis_config: &GenesisConfig,
@ -376,10 +362,8 @@ pub fn load_and_process_ledger(
let tss_blockstore = if enable_rpc_transaction_history {
Arc::new(open_blockstore(
blockstore.ledger_path(),
arg_matches,
AccessType::PrimaryForMaintenance,
None,
false,
false,
))
} else {
blockstore.clone()
@ -430,11 +414,14 @@ pub fn load_and_process_ledger(
pub fn open_blockstore(
ledger_path: &Path,
matches: &ArgMatches,
access_type: AccessType,
wal_recovery_mode: Option<BlockstoreRecoveryMode>,
force_update_to_open: bool,
enforce_ulimit_nofile: bool,
) -> Blockstore {
let wal_recovery_mode = matches
.value_of("wal_recovery_mode")
.map(BlockstoreRecoveryMode::from);
let force_update_to_open = matches.is_present("force_update_to_open");
let enforce_ulimit_nofile = !matches.is_present("ignore_ulimit_nofile_error");
let shred_storage_type = get_shred_storage_type(
ledger_path,
&format!(
@ -508,6 +495,20 @@ pub fn open_blockstore(
}
}
pub fn get_shred_storage_type(ledger_path: &Path, message: &str) -> ShredStorageType {
// TODO: the following shred_storage_type inference must be updated once
// the rocksdb options can be constructed via load_options_file() as the
// value picked by passing None for `max_shred_storage_size` could affect
// the persisted rocksdb options file.
match ShredStorageType::from_ledger_path(ledger_path, None) {
Some(s) => s,
None => {
info!("{}", message);
ShredStorageType::RocksLevel
}
}
}
/// Open blockstore with temporary primary access to allow necessary,
/// persistent changes to be made to the blockstore (such as creation of new
/// column family(s)). Then, continue opening with `original_access_type`

224
ledger-tool/src/main.rs
View File

@ -39,10 +39,7 @@ use {
ancestor_iterator::AncestorIterator,
blockstore::{create_new_ledger, Blockstore, PurgeType},
blockstore_db::{self, columns as cf, Column, ColumnName, Database},
blockstore_options::{
AccessType, BlockstoreRecoveryMode, LedgerColumnOptions,
BLOCKSTORE_DIRECTORY_ROCKS_FIFO,
},
blockstore_options::{AccessType, LedgerColumnOptions, BLOCKSTORE_DIRECTORY_ROCKS_FIFO},
blockstore_processor::ProcessOptions,
shred::Shred,
use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup},
@ -2019,11 +2016,6 @@ fn main() {
.ok()
.map(PathBuf::from);
let wal_recovery_mode = matches
.value_of("wal_recovery_mode")
.map(BlockstoreRecoveryMode::from);
let force_update_to_open = matches.is_present("force_update_to_open");
let enforce_ulimit_nofile = !matches.is_present("ignore_ulimit_nofile_error");
let verbose_level = matches.occurrences_of("verbose");
if let ("bigtable", Some(arg_matches)) = matches.subcommand() {
@ -2041,13 +2033,7 @@ fn main() {
let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
let only_rooted = arg_matches.is_present("only_rooted");
output_ledger(
open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
),
open_blockstore(&ledger_path, arg_matches, AccessType::Secondary),
starting_slot,
ending_slot,
allow_dead_slots,
@ -2062,13 +2048,7 @@ fn main() {
let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String));
let source = open_blockstore(
&ledger_path,
AccessType::Secondary,
None,
force_update_to_open,
enforce_ulimit_nofile,
);
let source = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
// Check if shred storage type can be inferred; if not, a new
// ledger is being created. open_blockstore() will attempt to
@ -2084,13 +2064,7 @@ fn main() {
&target_db
),
);
let target = open_blockstore(
&target_db,
AccessType::Primary,
None,
force_update_to_open,
enforce_ulimit_nofile,
);
let target = open_blockstore(&target_db, arg_matches, AccessType::Primary);
for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() {
if slot > ending_slot {
break;
@ -2166,13 +2140,8 @@ fn main() {
..ProcessOptions::default()
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
get_access_type(&process_options),
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore =
open_blockstore(&ledger_path, arg_matches, get_access_type(&process_options));
let (bank_forks, _) = load_and_process_ledger_or_exit(
arg_matches,
&genesis_config,
@ -2205,13 +2174,7 @@ fn main() {
}
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let ending_slot = value_t!(arg_matches, "ending_slot", Slot).unwrap_or(Slot::MAX);
let ledger = open_blockstore(
&ledger_path,
AccessType::Secondary,
None,
force_update_to_open,
enforce_ulimit_nofile,
);
let ledger = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
for (slot, _meta) in ledger
.slot_meta_iterator(starting_slot)
.unwrap()
@ -2246,13 +2209,8 @@ fn main() {
..ProcessOptions::default()
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
get_access_type(&process_options),
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore =
open_blockstore(&ledger_path, arg_matches, get_access_type(&process_options));
let (bank_forks, _) = load_and_process_ledger_or_exit(
arg_matches,
&genesis_config,
@ -2266,13 +2224,7 @@ fn main() {
("slot", Some(arg_matches)) => {
let slots = values_t_or_exit!(arg_matches, "slots", Slot);
let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
for slot in slots {
println!("Slot {slot}");
if let Err(err) = output_slot(
@ -2291,13 +2243,7 @@ fn main() {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
output_ledger(
open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
),
open_blockstore(&ledger_path, arg_matches, AccessType::Secondary),
starting_slot,
Slot::MAX,
allow_dead_slots,
@ -2308,26 +2254,14 @@ fn main() {
);
}
("dead-slots", Some(arg_matches)) => {
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
for slot in blockstore.dead_slots_iterator(starting_slot).unwrap() {
println!("{slot}");
}
}
("duplicate-slots", Some(arg_matches)) => {
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
for slot in blockstore.duplicate_slots_iterator(starting_slot).unwrap() {
println!("{slot}");
@ -2335,13 +2269,7 @@ fn main() {
}
("set-dead-slot", Some(arg_matches)) => {
let slots = values_t_or_exit!(arg_matches, "slots", Slot);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Primary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Primary);
for slot in slots {
match blockstore.set_dead_slot(slot) {
Ok(_) => println!("Slot {slot} dead"),
@ -2351,13 +2279,7 @@ fn main() {
}
("remove-dead-slot", Some(arg_matches)) => {
let slots = values_t_or_exit!(arg_matches, "slots", Slot);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Primary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Primary);
for slot in slots {
match blockstore.remove_dead_slot(slot) {
Ok(_) => println!("Slot {slot} not longer marked dead"),
@ -2370,13 +2292,7 @@ fn main() {
("parse_full_frozen", Some(arg_matches)) => {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
let mut ancestors = BTreeSet::new();
assert!(
blockstore.meta(ending_slot).unwrap().is_some(),
@ -2490,13 +2406,8 @@ fn main() {
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
info!("genesis hash: {}", genesis_config.hash());
let blockstore = open_blockstore(
&ledger_path,
get_access_type(&process_options),
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore =
open_blockstore(&ledger_path, arg_matches, get_access_type(&process_options));
let (bank_forks, _) = load_and_process_ledger_or_exit(
arg_matches,
&genesis_config,
@ -2546,13 +2457,8 @@ fn main() {
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
get_access_type(&process_options),
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore =
open_blockstore(&ledger_path, arg_matches, get_access_type(&process_options));
let (bank_forks, _) = load_and_process_ledger_or_exit(
arg_matches,
&genesis_config,
@ -2672,10 +2578,8 @@ fn main() {
};
let blockstore = Arc::new(open_blockstore(
&ledger_path,
arg_matches,
get_access_type(&process_options),
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
));
let snapshot_slot = if Some("ROOT") == arg_matches.value_of("snapshot_slot") {
@ -3092,13 +2996,8 @@ fn main() {
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let include_sysvars = arg_matches.is_present("include_sysvars");
let blockstore = open_blockstore(
&ledger_path,
get_access_type(&process_options),
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore =
open_blockstore(&ledger_path, arg_matches, get_access_type(&process_options));
let (bank_forks, _) = load_and_process_ledger_or_exit(
arg_matches,
&genesis_config,
@ -3182,13 +3081,8 @@ fn main() {
..ProcessOptions::default()
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = open_blockstore(
&ledger_path,
get_access_type(&process_options),
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore =
open_blockstore(&ledger_path, arg_matches, get_access_type(&process_options));
let (bank_forks, _) = load_and_process_ledger_or_exit(
arg_matches,
&genesis_config,
@ -3678,13 +3572,8 @@ fn main() {
let dead_slots_only = arg_matches.is_present("dead_slots_only");
let batch_size = value_t_or_exit!(arg_matches, "batch_size", usize);
let blockstore = open_blockstore(
&ledger_path,
AccessType::PrimaryForMaintenance,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore =
open_blockstore(&ledger_path, arg_matches, AccessType::PrimaryForMaintenance);
let end_slot = match end_slot {
Some(end_slot) => end_slot,
@ -3753,13 +3642,7 @@ fn main() {
}
}
("list-roots", Some(arg_matches)) => {
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
let max_height = value_t!(arg_matches, "max_height", usize).unwrap_or(usize::MAX);
let start_root = value_t!(arg_matches, "start_root", Slot).unwrap_or(0);
@ -3796,13 +3679,7 @@ fn main() {
});
}
("latest-optimistic-slots", Some(arg_matches)) => {
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
let num_slots = value_t_or_exit!(arg_matches, "num_slots", usize);
let exclude_vote_only_slots = arg_matches.is_present("exclude_vote_only_slots");
let slots =
@ -3832,13 +3709,7 @@ fn main() {
}
}
("repair-roots", Some(arg_matches)) => {
let blockstore = open_blockstore(
&ledger_path,
AccessType::Primary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Primary);
let start_root = value_t!(arg_matches, "start_root", Slot)
.unwrap_or_else(|_| blockstore.max_root());
@ -3864,13 +3735,7 @@ fn main() {
println!("Successfully repaired {num_repaired_roots} roots");
}
("bounds", Some(arg_matches)) => {
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
match blockstore.slot_meta_iterator(0) {
Ok(metas) => {
@ -3935,26 +3800,13 @@ fn main() {
}
};
}
("analyze-storage", _) => {
("analyze-storage", Some(arg_matches)) => {
analyze_storage(
&open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
)
.db(),
&open_blockstore(&ledger_path, arg_matches, AccessType::Secondary).db(),
);
}
("compute-slot-cost", Some(arg_matches)) => {
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
let mut slots: Vec<u64> = vec![];
if !arg_matches.is_present("slots") {
@ -3972,13 +3824,7 @@ fn main() {
}
}
("print-file-metadata", Some(arg_matches)) => {
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
false,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(&ledger_path, arg_matches, AccessType::Secondary);
let sst_file_name = arg_matches.value_of("file_name");
if let Err(err) = print_blockstore_file_metadata(&blockstore, &sst_file_name) {
eprintln!("{err}");

18
ledger-tool/src/program.rs
View File

@ -10,10 +10,7 @@ use {
},
solana_clap_utils::input_parsers::pubkeys_of,
solana_cli_output::{OutputFormat, QuietDisplay, VerboseDisplay},
solana_ledger::{
blockstore_options::{AccessType, BlockstoreRecoveryMode},
blockstore_processor::ProcessOptions,
},
solana_ledger::{blockstore_options::AccessType, blockstore_processor::ProcessOptions},
solana_program_runtime::{
invoke_context::InvokeContext,
loaded_programs::{LoadProgramMetrics, LoadedProgramType, DELAY_VISIBILITY_SLOT_OFFSET},
@ -77,8 +74,6 @@ fn load_accounts(path: &Path) -> Result<Input> {
fn load_blockstore(ledger_path: &Path, arg_matches: &ArgMatches<'_>) -> Arc<Bank> {
let debug_keys = pubkeys_of(arg_matches, "debug_key")
.map(|pubkeys| Arc::new(pubkeys.into_iter().collect::<HashSet<_>>()));
let force_update_to_open = arg_matches.is_present("force_update_to_open");
let enforce_ulimit_nofile = !arg_matches.is_present("ignore_ulimit_nofile_error");
let process_options = ProcessOptions {
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
run_verification: false,
@ -108,18 +103,9 @@ fn load_blockstore(ledger_path: &Path, arg_matches: &ArgMatches<'_>) -> Arc<Bank
.ok()
.map(PathBuf::from);
let wal_recovery_mode = arg_matches
.value_of("wal_recovery_mode")
.map(BlockstoreRecoveryMode::from);
let genesis_config = open_genesis_config_by(ledger_path, arg_matches);
info!("genesis hash: {}", genesis_config.hash());
let blockstore = open_blockstore(
ledger_path,
AccessType::Secondary,
wal_recovery_mode,
force_update_to_open,
enforce_ulimit_nofile,
);
let blockstore = open_blockstore(ledger_path, arg_matches, AccessType::Secondary);
let (bank_forks, ..) = load_and_process_ledger_or_exit(
arg_matches,
&genesis_config,