diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index 9dc87dc6e8..0ab5ba5649 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -17,9 +17,8 @@ use { OutputFormat, }, solana_ledger::{ - bigtable_upload::ConfirmedBlockUploadConfig, - blockstore::Blockstore, - blockstore_options::{AccessType, ShredStorageType}, + bigtable_upload::ConfirmedBlockUploadConfig, blockstore::Blockstore, + blockstore_options::AccessType, }, solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature}, solana_storage_bigtable::CredentialType, @@ -985,11 +984,7 @@ fn get_global_subcommand_arg( } } -pub fn bigtable_process_command( - ledger_path: &Path, - matches: &ArgMatches<'_>, - shred_storage_type: &ShredStorageType, -) { +pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { let runtime = tokio::runtime::Runtime::new().unwrap(); let verbose = matches.is_present("verbose"); @@ -1019,7 +1014,6 @@ pub fn bigtable_process_command( &canonicalize_ledger_path(ledger_path), AccessType::Secondary, None, - shred_storage_type, force_update_to_open, ); let config = solana_storage_bigtable::LedgerStorageConfig { diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 5b9a1cd0db..b1ce07b489 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -875,7 +875,7 @@ fn open_blockstore_with_temporary_primary_access( ) } -fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorageType { +fn get_shred_storage_type(ledger_path: &Path, message: &str) -> ShredStorageType { // TODO: the following shred_storage_type inference must be updated once // the rocksdb options can be constructed via load_options_file() as the // value picked by passing None for `max_shred_storage_size` could affect @@ -883,7 +883,7 @@ fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorag match ShredStorageType::from_ledger_path(ledger_path, None) { Some(s) => s, None => { - warn!("{}", warn_message); + info!("{}", message); ShredStorageType::RocksLevel } } @@ -893,9 +893,17 @@ fn open_blockstore( ledger_path: &Path, access_type: AccessType, wal_recovery_mode: Option, - shred_storage_type: &ShredStorageType, force_update_to_open: bool, ) -> Blockstore { + let shred_storage_type = get_shred_storage_type( + ledger_path, + &format!( + "Shred stroage type cannot be inferred for ledger at {:?}, \ + using default RocksLevel", + ledger_path + ), + ); + match Blockstore::open_with_options( ledger_path, BlockstoreOptions { @@ -903,7 +911,7 @@ fn open_blockstore( recovery_mode: wal_recovery_mode.clone(), enforce_ulimit_nofile: true, column_options: LedgerColumnOptions { - shred_storage_type: shred_storage_type.clone(), + shred_storage_type, ..LedgerColumnOptions::default() }, }, @@ -2326,13 +2334,9 @@ fn main() { .map(BlockstoreRecoveryMode::from); let force_update_to_open = matches.is_present("force_update_to_open"); let verbose_level = matches.occurrences_of("verbose"); - let shred_storage_type = get_shred_storage_type( - &ledger_path, - "Shred storage type cannot be inferred, the default RocksLevel will be used", - ); if let ("bigtable", Some(arg_matches)) = matches.subcommand() { - bigtable_process_command(&ledger_path, arg_matches, &shred_storage_type) + bigtable_process_command(&ledger_path, arg_matches) } else { let ledger_path = canonicalize_ledger_path(&ledger_path); @@ -2348,7 +2352,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ), starting_slot, @@ -2364,30 +2367,31 @@ fn main() { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot); let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String)); - let target_shred_storage_type = get_shred_storage_type( - &target_db, - &format!( - "Shred storage type of target_db cannot be inferred, \ - the default RocksLevel will be used. \ - If you want to use FIFO shred_storage_type on an empty target_db, \ - create {BLOCKSTORE_DIRECTORY_ROCKS_FIFO} foldar the specified target_db directory.", - ), - ); let source = open_blockstore( &ledger_path, AccessType::Secondary, None, - &shred_storage_type, force_update_to_open, ); - let target = open_blockstore( + + // Check if shred storage type can be inferred; if not, a new + // ledger is being created. open_blockstore() will attempt to + // to infer shred storage type as well, but this check provides + // extra insight to user on how to create a FIFO ledger. + let _ = get_shred_storage_type( &target_db, - AccessType::Primary, - None, - &target_shred_storage_type, - force_update_to_open, + &format!( + "No --target-db ledger at {:?} was detected, default \ + compaction (RocksLevel) will be used. Fifo compaction \ + can be enabled for a new ledger by manually creating \ + {BLOCKSTORE_DIRECTORY_ROCKS_FIFO} directory within \ + the specified --target_db directory.", + &target_db + ), ); + let target = + open_blockstore(&target_db, AccessType::Primary, None, force_update_to_open); for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() { if slot > ending_slot { break; @@ -2466,7 +2470,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); match load_bank_forks( @@ -2519,7 +2522,6 @@ fn main() { &ledger_path, AccessType::Secondary, None, - &shred_storage_type, force_update_to_open, ); for (slot, _meta) in ledger @@ -2559,7 +2561,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); match load_bank_forks( @@ -2586,7 +2587,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); for slot in slots { @@ -2611,7 +2611,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ), starting_slot, @@ -2628,7 +2627,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); @@ -2641,7 +2639,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); @@ -2655,7 +2652,6 @@ fn main() { &ledger_path, AccessType::Primary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); for slot in slots { @@ -2671,7 +2667,6 @@ fn main() { &ledger_path, AccessType::Primary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); for slot in slots { @@ -2690,7 +2685,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); let mut ancestors = BTreeSet::new(); @@ -2802,7 +2796,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); let (bank_forks, ..) = load_bank_forks( @@ -2846,7 +2839,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); match load_bank_forks( @@ -2960,7 +2952,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); @@ -3355,7 +3346,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); let (bank_forks, ..) = load_bank_forks( @@ -3444,7 +3434,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); match load_bank_forks( @@ -3976,7 +3965,6 @@ fn main() { &ledger_path, access_type, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); @@ -4050,7 +4038,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); let max_height = if let Some(height) = arg_matches.value_of("max_height") { @@ -4104,7 +4091,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); let num_slots = value_t_or_exit!(arg_matches, "num_slots", usize); @@ -4129,7 +4115,6 @@ fn main() { &ledger_path, AccessType::Primary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); let start_root = if let Some(root) = arg_matches.value_of("start_root") { @@ -4179,7 +4164,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); @@ -4252,7 +4236,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ) .db(), @@ -4263,7 +4246,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, force_update_to_open, ); @@ -4287,7 +4269,6 @@ fn main() { &ledger_path, AccessType::Secondary, wal_recovery_mode, - &shred_storage_type, false, ); let sst_file_name = arg_matches.value_of("file_name");