diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index 51c3ff4563..8dcce92534 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -15,8 +15,9 @@ use { OutputFormat, }, solana_ledger::{ - bigtable_upload::ConfirmedBlockUploadConfig, blockstore::Blockstore, - blockstore_options::AccessType, + bigtable_upload::ConfirmedBlockUploadConfig, + blockstore::Blockstore, + blockstore_options::{AccessType, ShredStorageType}, }, solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature}, solana_storage_bigtable::CredentialType, @@ -615,7 +616,11 @@ fn get_global_subcommand_arg( } } -pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { +pub fn bigtable_process_command( + ledger_path: &Path, + matches: &ArgMatches<'_>, + shred_storage_type: &ShredStorageType, +) { let runtime = tokio::runtime::Runtime::new().unwrap(); let verbose = matches.is_present("verbose"); @@ -644,6 +649,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { &canonicalize_ledger_path(ledger_path), AccessType::Secondary, None, + shred_storage_type, ); let config = solana_storage_bigtable::LedgerStorageConfig { read_only: false, diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 2ea26d1b73..463b1f8528 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -27,6 +27,7 @@ use { blockstore_db::{self, Database}, blockstore_options::{ AccessType, BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions, + ShredStorageType, }, blockstore_processor::{BlockstoreProcessorError, ProcessOptions}, shred::Shred, @@ -724,6 +725,7 @@ fn open_blockstore( ledger_path: &Path, access_type: AccessType, wal_recovery_mode: Option, + shred_storage_type: &ShredStorageType, ) -> Blockstore { match Blockstore::open_with_options( ledger_path, @@ -731,7 +733,10 @@ fn open_blockstore( access_type, recovery_mode: wal_recovery_mode, enforce_ulimit_nofile: true, - ..BlockstoreOptions::default() + column_options: LedgerColumnOptions { + shred_storage_type: shred_storage_type.clone(), + ..LedgerColumnOptions::default() + }, }, ) { Ok(blockstore) => blockstore, @@ -953,6 +958,10 @@ use jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; +/// The default size for data and coding shred column families in FIFO compaction. +/// u64::MAX as the default value means it won't delete any files by default. +const DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = std::u64::MAX; + #[allow(clippy::cognitive_complexity)] fn main() { // Ignore SIGUSR1 to prevent long-running calls being killed by logrotate @@ -1872,8 +1881,23 @@ fn main() { .map(BlockstoreRecoveryMode::from); let verbose_level = matches.occurrences_of("verbose"); + // TODO: the following shred_storage_type inference must be updated once the + // rocksdb options can be constructed via load_options_file() as the + // temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES + // could affect the persisted rocksdb options file. + let shred_storage_type = match ShredStorageType::from_ledger_path( + &ledger_path, + DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES, + ) { + Some(s) => s, + None => { + error!("Shred storage type cannot be inferred, the default RocksLevel will be used"); + ShredStorageType::RocksLevel + } + }; + if let ("bigtable", Some(arg_matches)) = matches.subcommand() { - bigtable_process_command(&ledger_path, arg_matches) + bigtable_process_command(&ledger_path, arg_matches, &shred_storage_type) } else { let ledger_path = canonicalize_ledger_path(&ledger_path); @@ -1885,7 +1909,12 @@ fn main() { let allow_dead_slots = arg_matches.is_present("allow_dead_slots"); let only_rooted = arg_matches.is_present("only_rooted"); output_ledger( - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode), + open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ), starting_slot, ending_slot, allow_dead_slots, @@ -1899,8 +1928,14 @@ fn main() { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot); let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String)); - let source = open_blockstore(&ledger_path, AccessType::Secondary, None); - let target = open_blockstore(&target_db, AccessType::Primary, None); + let source = open_blockstore( + &ledger_path, + AccessType::Secondary, + None, + &shred_storage_type, + ); + let target = + open_blockstore(&target_db, AccessType::Primary, None, &shred_storage_type); for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() { if slot > ending_slot { break; @@ -1973,8 +2008,12 @@ fn main() { ..ProcessOptions::default() }; let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); match load_bank_forks( arg_matches, &genesis_config, @@ -2021,7 +2060,12 @@ fn main() { } let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let ending_slot = value_t!(arg_matches, "ending_slot", Slot).unwrap_or(Slot::MAX); - let ledger = open_blockstore(&ledger_path, AccessType::Secondary, None); + let ledger = open_blockstore( + &ledger_path, + AccessType::Secondary, + None, + &shred_storage_type, + ); for (slot, _meta) in ledger .slot_meta_iterator(starting_slot) .unwrap() @@ -2055,8 +2099,12 @@ fn main() { ..ProcessOptions::default() }; let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); match load_bank_forks( arg_matches, &genesis_config, @@ -2077,8 +2125,12 @@ fn main() { ("slot", Some(arg_matches)) => { let slots = values_t_or_exit!(arg_matches, "slots", Slot); let allow_dead_slots = arg_matches.is_present("allow_dead_slots"); - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); for slot in slots { println!("Slot {}", slot); if let Err(err) = output_slot( @@ -2097,7 +2149,12 @@ fn main() { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let allow_dead_slots = arg_matches.is_present("allow_dead_slots"); output_ledger( - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode), + open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ), starting_slot, Slot::MAX, allow_dead_slots, @@ -2108,16 +2165,24 @@ fn main() { ); } ("dead-slots", Some(arg_matches)) => { - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); for slot in blockstore.dead_slots_iterator(starting_slot).unwrap() { println!("{}", slot); } } ("duplicate-slots", Some(arg_matches)) => { - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); for slot in blockstore.duplicate_slots_iterator(starting_slot).unwrap() { println!("{}", slot); @@ -2125,8 +2190,12 @@ fn main() { } ("set-dead-slot", Some(arg_matches)) => { let slots = values_t_or_exit!(arg_matches, "slots", Slot); - let blockstore = - open_blockstore(&ledger_path, AccessType::Primary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Primary, + wal_recovery_mode, + &shred_storage_type, + ); for slot in slots { match blockstore.set_dead_slot(slot) { Ok(_) => println!("Slot {} dead", slot), @@ -2136,8 +2205,12 @@ fn main() { } ("remove-dead-slot", Some(arg_matches)) => { let slots = values_t_or_exit!(arg_matches, "slots", Slot); - let blockstore = - open_blockstore(&ledger_path, AccessType::Primary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Primary, + wal_recovery_mode, + &shred_storage_type, + ); for slot in slots { match blockstore.remove_dead_slot(slot) { Ok(_) => println!("Slot {} not longer marked dead", slot), @@ -2150,8 +2223,12 @@ fn main() { ("parse_full_frozen", Some(arg_matches)) => { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot); - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); let mut ancestors = BTreeSet::new(); assert!( blockstore.meta(ending_slot).unwrap().is_some(), @@ -2304,8 +2381,12 @@ fn main() { open_genesis_config_by(&ledger_path, arg_matches).hash() ); - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); let (bank_forks, ..) = load_bank_forks( arg_matches, &open_genesis_config_by(&ledger_path, arg_matches), @@ -2336,8 +2417,12 @@ fn main() { ..ProcessOptions::default() }; - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); match load_bank_forks( arg_matches, &open_genesis_config_by(&ledger_path, arg_matches), @@ -2449,8 +2534,12 @@ fn main() { usize ); let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); let snapshot_slot = if Some("ROOT") == arg_matches.value_of("snapshot_slot") { blockstore @@ -2833,8 +2922,12 @@ fn main() { }; let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let include_sysvars = arg_matches.is_present("include_sysvars"); - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); let (bank_forks, ..) = load_bank_forks( arg_matches, &genesis_config, @@ -2893,8 +2986,12 @@ fn main() { ..ProcessOptions::default() }; let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); match load_bank_forks( arg_matches, &genesis_config, @@ -3424,7 +3521,12 @@ fn main() { } else { AccessType::PrimaryForMaintenance }; - let blockstore = open_blockstore(&ledger_path, access_type, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + access_type, + wal_recovery_mode, + &shred_storage_type, + ); let end_slot = match end_slot { Some(end_slot) => end_slot, @@ -3495,8 +3597,12 @@ fn main() { } } ("list-roots", Some(arg_matches)) => { - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); let max_height = if let Some(height) = arg_matches.value_of("max_height") { usize::from_str(height).expect("Maximum height must be a number") } else { @@ -3558,8 +3664,12 @@ fn main() { }); } ("latest-optimistic-slots", Some(arg_matches)) => { - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); let num_slots = value_t_or_exit!(arg_matches, "num_slots", usize); let slots = blockstore .get_latest_optimistic_slots(num_slots) @@ -3578,8 +3688,12 @@ fn main() { } } ("repair-roots", Some(arg_matches)) => { - let blockstore = - open_blockstore(&ledger_path, AccessType::Primary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Primary, + wal_recovery_mode, + &shred_storage_type, + ); let start_root = if let Some(root) = arg_matches.value_of("start_root") { Slot::from_str(root).expect("Before root must be a number") } else { @@ -3627,8 +3741,12 @@ fn main() { } } ("bounds", Some(arg_matches)) => { - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); match blockstore.slot_meta_iterator(0) { Ok(metas) => { let all = arg_matches.is_present("all"); @@ -3689,13 +3807,23 @@ fn main() { } ("analyze-storage", _) => { analyze_storage( - &open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode).db(), + &open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ) + .db(), ); println!("Ok."); } ("compute-slot-cost", Some(arg_matches)) => { - let blockstore = - open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); + let blockstore = open_blockstore( + &ledger_path, + AccessType::Secondary, + wal_recovery_mode, + &shred_storage_type, + ); let mut slots: Vec = vec![]; if !arg_matches.is_present("slots") { diff --git a/ledger-tool/tests/basic.rs b/ledger-tool/tests/basic.rs index 050b242d54..d3d70f80b7 100644 --- a/ledger-tool/tests/basic.rs +++ b/ledger-tool/tests/basic.rs @@ -1,6 +1,8 @@ use { assert_cmd::prelude::*, - solana_ledger::{create_new_tmp_ledger, genesis_utils::create_genesis_config}, + solana_ledger::{ + create_new_tmp_ledger, create_new_tmp_ledger_fifo, genesis_utils::create_genesis_config, + }, std::process::{Command, Output}, }; @@ -27,19 +29,10 @@ fn bad_arguments() { .success()); } -#[test] -fn nominal() { - let genesis_config = create_genesis_config(100).genesis_config; - let ticks_per_slot = genesis_config.ticks_per_slot; +fn nominal_test_helper(ledger_path: &str, ticks: usize) { let meta_lines = 2; let summary_lines = 1; - let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config); - let ticks = ticks_per_slot as usize; - - let ledger_path = ledger_path.to_str().unwrap(); - - // Basic validation let output = run_ledger_tool(&["-l", ledger_path, "verify"]); assert!(output.status.success()); @@ -47,7 +40,29 @@ fn nominal() { let output = run_ledger_tool(&["-l", ledger_path, "print", "-vvv"]); assert!(output.status.success()); assert_eq!( - count_newlines(&output.stdout), - ticks + meta_lines + summary_lines + count_newlines(&output.stdout) + .saturating_sub(meta_lines) + .saturating_sub(summary_lines), + ticks + ); +} + +#[test] +fn nominal_default() { + let genesis_config = create_genesis_config(100).genesis_config; + let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config); + nominal_test_helper( + ledger_path.to_str().unwrap(), + genesis_config.ticks_per_slot as usize, + ); +} + +#[test] +fn nominal_fifo() { + let genesis_config = create_genesis_config(100).genesis_config; + let (ledger_path, _blockhash) = create_new_tmp_ledger_fifo!(&genesis_config); + nominal_test_helper( + ledger_path.to_str().unwrap(), + genesis_config.ticks_per_slot as usize, ); } diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 19985d1eef..93423fd0f5 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -4010,6 +4010,22 @@ macro_rules! create_new_tmp_ledger { }; } +#[macro_export] +macro_rules! create_new_tmp_ledger_fifo { + ($genesis_config:expr) => { + $crate::blockstore::create_new_ledger_from_name( + $crate::tmp_ledger_name!(), + $genesis_config, + $crate::blockstore_options::LedgerColumnOptions { + shred_storage_type: $crate::blockstore_options::ShredStorageType::RocksFifo( + $crate::blockstore_options::BlockstoreRocksFifoOptions::default(), + ), + ..$crate::blockstore_options::LedgerColumnOptions::default() + }, + ) + }; +} + #[macro_export] macro_rules! create_new_tmp_ledger_auto_delete { ($genesis_config:expr) => { diff --git a/ledger/src/blockstore_options.rs b/ledger/src/blockstore_options.rs index 2609cec057..54ab1f9865 100644 --- a/ledger/src/blockstore_options.rs +++ b/ledger/src/blockstore_options.rs @@ -1,4 +1,7 @@ -use rocksdb::{DBCompressionType as RocksCompressionType, DBRecoveryMode}; +use { + rocksdb::{DBCompressionType as RocksCompressionType, DBRecoveryMode}, + std::path::Path, +}; pub struct BlockstoreOptions { // The access type of blockstore. Default: Primary @@ -153,6 +156,38 @@ impl ShredStorageType { ShredStorageType::RocksFifo(_) => BLOCKSTORE_DIRECTORY_ROCKS_FIFO, } } + + /// Returns the ShredStorageType that is used under the specified + /// ledger_path. + /// + /// None will be returned if the ShredStorageType cannot be inferred. + pub fn from_ledger_path( + ledger_path: &Path, + fifo_shred_storage_size: u64, + ) -> Option { + let mut result: Option = None; + + if Path::new(ledger_path) + .join(BLOCKSTORE_DIRECTORY_ROCKS_LEVEL) + .exists() + { + result = Some(ShredStorageType::RocksLevel); + } + + if Path::new(ledger_path) + .join(BLOCKSTORE_DIRECTORY_ROCKS_FIFO) + .exists() + { + if result.is_none() { + result = Some(ShredStorageType::RocksFifo( + BlockstoreRocksFifoOptions::new(fifo_shred_storage_size), + )); + } else { + result = None; + } + } + result + } } #[derive(Debug, Clone)]