blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

ledger-tool: Move shred storage type inference next to Blockstore::open (#30084)

This commit is contained in:
steviez 2023-02-02 16:55:18 -06:00 committed by GitHub
parent 3249e4a123
commit 37461976e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 32 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ledger-tool/src/bigtable.rs
View File

@ -17,9 +17,8 @@ use {
OutputFormat,
},
solana_ledger::{
bigtable_upload::ConfirmedBlockUploadConfig,
blockstore::Blockstore,
blockstore_options::{AccessType, ShredStorageType},
bigtable_upload::ConfirmedBlockUploadConfig, blockstore::Blockstore,
blockstore_options::AccessType,
},
solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature},
solana_storage_bigtable::CredentialType,
@ -985,11 +984,7 @@ fn get_global_subcommand_arg<T: FromStr>(
}
}
pub fn bigtable_process_command(
ledger_path: &Path,
matches: &ArgMatches<'_>,
shred_storage_type: &ShredStorageType,
) {
pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
let runtime = tokio::runtime::Runtime::new().unwrap();
let verbose = matches.is_present("verbose");
@ -1019,7 +1014,6 @@ pub fn bigtable_process_command(
&canonicalize_ledger_path(ledger_path),
AccessType::Secondary,
None,
shred_storage_type,
force_update_to_open,
);
let config = solana_storage_bigtable::LedgerStorageConfig {

77
ledger-tool/src/main.rs
View File

@ -875,7 +875,7 @@ fn open_blockstore_with_temporary_primary_access(
)
}
fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorageType {
fn get_shred_storage_type(ledger_path: &Path, message: &str) -> ShredStorageType {
// TODO: the following shred_storage_type inference must be updated once
// the rocksdb options can be constructed via load_options_file() as the
// value picked by passing None for `max_shred_storage_size` could affect
@ -883,7 +883,7 @@ fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorag
match ShredStorageType::from_ledger_path(ledger_path, None) {
Some(s) => s,
None => {
warn!("{}", warn_message);
info!("{}", message);
ShredStorageType::RocksLevel
}
}
@ -893,9 +893,17 @@ fn open_blockstore(
ledger_path: &Path,
access_type: AccessType,
wal_recovery_mode: Option<BlockstoreRecoveryMode>,
shred_storage_type: &ShredStorageType,
force_update_to_open: bool,
) -> Blockstore {
let shred_storage_type = get_shred_storage_type(
ledger_path,
&format!(
"Shred stroage type cannot be inferred for ledger at {:?}, \
using default RocksLevel",
ledger_path
),
);
match Blockstore::open_with_options(
ledger_path,
BlockstoreOptions {
@ -903,7 +911,7 @@ fn open_blockstore(
recovery_mode: wal_recovery_mode.clone(),
enforce_ulimit_nofile: true,
column_options: LedgerColumnOptions {
shred_storage_type: shred_storage_type.clone(),
shred_storage_type,
..LedgerColumnOptions::default()
},
},
@ -2326,13 +2334,9 @@ fn main() {
.map(BlockstoreRecoveryMode::from);
let force_update_to_open = matches.is_present("force_update_to_open");
let verbose_level = matches.occurrences_of("verbose");
let shred_storage_type = get_shred_storage_type(
&ledger_path,
"Shred storage type cannot be inferred, the default RocksLevel will be used",
);
if let ("bigtable", Some(arg_matches)) = matches.subcommand() {
bigtable_process_command(&ledger_path, arg_matches, &shred_storage_type)
bigtable_process_command(&ledger_path, arg_matches)
} else {
let ledger_path = canonicalize_ledger_path(&ledger_path);
@ -2348,7 +2352,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
),
starting_slot,
@ -2364,30 +2367,31 @@ fn main() {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String));
let target_shred_storage_type = get_shred_storage_type(
&target_db,
&format!(
"Shred storage type of target_db cannot be inferred, \
the default RocksLevel will be used. \
If you want to use FIFO shred_storage_type on an empty target_db, \
create {BLOCKSTORE_DIRECTORY_ROCKS_FIFO} foldar the specified target_db directory.",
),
);
let source = open_blockstore(
&ledger_path,
AccessType::Secondary,
None,
&shred_storage_type,
force_update_to_open,
);
let target = open_blockstore(
// Check if shred storage type can be inferred; if not, a new
// ledger is being created. open_blockstore() will attempt to
// to infer shred storage type as well, but this check provides
// extra insight to user on how to create a FIFO ledger.
let _ = get_shred_storage_type(
&target_db,
AccessType::Primary,
None,
&target_shred_storage_type,
force_update_to_open,
&format!(
"No --target-db ledger at {:?} was detected, default \
compaction (RocksLevel) will be used. Fifo compaction \
can be enabled for a new ledger by manually creating \
{BLOCKSTORE_DIRECTORY_ROCKS_FIFO} directory within \
the specified --target_db directory.",
&target_db
),
);
let target =
open_blockstore(&target_db, AccessType::Primary, None, force_update_to_open);
for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() {
if slot > ending_slot {
break;
@ -2466,7 +2470,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
match load_bank_forks(
@ -2519,7 +2522,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
None,
&shred_storage_type,
force_update_to_open,
);
for (slot, _meta) in ledger
@ -2559,7 +2561,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
match load_bank_forks(
@ -2586,7 +2587,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
for slot in slots {
@ -2611,7 +2611,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
),
starting_slot,
@ -2628,7 +2627,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
@ -2641,7 +2639,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
@ -2655,7 +2652,6 @@ fn main() {
&ledger_path,
AccessType::Primary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
for slot in slots {
@ -2671,7 +2667,6 @@ fn main() {
&ledger_path,
AccessType::Primary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
for slot in slots {
@ -2690,7 +2685,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
let mut ancestors = BTreeSet::new();
@ -2802,7 +2796,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
let (bank_forks, ..) = load_bank_forks(
@ -2846,7 +2839,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
match load_bank_forks(
@ -2960,7 +2952,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
@ -3355,7 +3346,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
let (bank_forks, ..) = load_bank_forks(
@ -3444,7 +3434,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
match load_bank_forks(
@ -3976,7 +3965,6 @@ fn main() {
&ledger_path,
access_type,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
@ -4050,7 +4038,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
let max_height = if let Some(height) = arg_matches.value_of("max_height") {
@ -4104,7 +4091,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
let num_slots = value_t_or_exit!(arg_matches, "num_slots", usize);
@ -4129,7 +4115,6 @@ fn main() {
&ledger_path,
AccessType::Primary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
let start_root = if let Some(root) = arg_matches.value_of("start_root") {
@ -4179,7 +4164,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
@ -4252,7 +4236,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
)
.db(),
@ -4263,7 +4246,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
force_update_to_open,
);
@ -4287,7 +4269,6 @@ fn main() {
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
false,
);
let sst_file_name = arg_matches.value_of("file_name");