From 70c4f6ea96681c13805db8be5fb5330b3b0a0411 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang <93241502+yhchiang-sol@users.noreply.github.com> Date: Tue, 13 Sep 2022 10:27:58 -0700 Subject: [PATCH] Enable ledger-tool copy command to infer target_db shred storage type. (#27438) #### Problem The ledger-tool copy command currently assumes target_db uses the same shred storage type as the source ledger. #### Summary of Changes This PR enables ledger-tool copy command to infer the target_db shred storage type based on whether rocksdb or rocksdb_fifo exists under the target_db directory (same way as how ledger-tool infers the shred storage type for the main db.) If the ledger-tool is not able to infer the shred storage type of the target_db, then the default level compaction will be used. --- ledger-tool/src/main.rs | 54 +++++++++++++++++++---------- ledger-tool/tests/basic.rs | 71 +++++++++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 19 deletions(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index e376585a94..17ef713998 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -30,8 +30,8 @@ use { blockstore::{create_new_ledger, Blockstore, BlockstoreError, PurgeType}, blockstore_db::{self, columns as cf, Column, ColumnName, Database}, blockstore_options::{ - AccessType, BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions, - ShredStorageType, + AccessType, BlockstoreOptions, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions, + LedgerColumnOptions, ShredStorageType, }, blockstore_processor::{self, BlockstoreProcessorError, ProcessOptions}, shred::Shred, @@ -847,6 +847,23 @@ fn open_blockstore_with_temporary_primary_access( ) } +fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorageType { + // TODO: the following shred_storage_type inference must be updated once the + // rocksdb options can be constructed via load_options_file() as the + // temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES + // could affect the persisted rocksdb options file. + match ShredStorageType::from_ledger_path( + ledger_path, + DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES, + ) { + Some(s) => s, + None => { + warn!("{}", warn_message); + ShredStorageType::RocksLevel + } + } +} + fn open_blockstore( ledger_path: &Path, access_type: AccessType, @@ -1466,6 +1483,7 @@ fn main() { let default_graph_vote_account_mode = GraphVoteAccountMode::default(); let mut measure_total_execution_time = Measure::start("ledger tool"); + let matches = App::new(crate_name!()) .about(crate_description!()) .version(solana_version::version!()) @@ -2215,21 +2233,10 @@ fn main() { .map(BlockstoreRecoveryMode::from); let force_update_to_open = matches.is_present("force_update_to_open"); let verbose_level = matches.occurrences_of("verbose"); - - // TODO: the following shred_storage_type inference must be updated once the - // rocksdb options can be constructed via load_options_file() as the - // temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES - // could affect the persisted rocksdb options file. - let shred_storage_type = match ShredStorageType::from_ledger_path( + let shred_storage_type = get_shred_storage_type( &ledger_path, - DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES, - ) { - Some(s) => s, - None => { - error!("Shred storage type cannot be inferred, the default RocksLevel will be used"); - ShredStorageType::RocksLevel - } - }; + "Shred storage type cannot be inferred, the default RocksLevel will be used", + ); if let ("bigtable", Some(arg_matches)) = matches.subcommand() { bigtable_process_command(&ledger_path, arg_matches, &shred_storage_type) @@ -2264,6 +2271,18 @@ fn main() { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot); let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String)); + let target_shred_storage_type = get_shred_storage_type( + &target_db, + &format!( + "Shred storage type of target_db cannot be inferred, \ + the default RocksLevel will be used. \ + If you want to use FIFO shred_storage_type on an empty target_db, \ + create {} foldar the specified target_db directory.", + ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions::default()) + .blockstore_directory() + ), + ); + let source = open_blockstore( &ledger_path, AccessType::Secondary, @@ -2275,10 +2294,9 @@ fn main() { &target_db, AccessType::Primary, None, - &shred_storage_type, + &target_shred_storage_type, force_update_to_open, ); - for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() { if slot > ending_slot { break; diff --git a/ledger-tool/tests/basic.rs b/ledger-tool/tests/basic.rs index d3d70f80b7..df5606e3e7 100644 --- a/ledger-tool/tests/basic.rs +++ b/ledger-tool/tests/basic.rs @@ -1,9 +1,17 @@ use { assert_cmd::prelude::*, + solana_entry::entry, solana_ledger::{ + blockstore, blockstore::Blockstore, blockstore_options::ShredStorageType, create_new_tmp_ledger, create_new_tmp_ledger_fifo, genesis_utils::create_genesis_config, + get_tmp_ledger_path_auto_delete, + }, + solana_sdk::hash::Hash, + std::{ + fs, + path::Path, + process::{Command, Output}, }, - std::process::{Command, Output}, }; fn run_ledger_tool(args: &[&str]) -> Output { @@ -66,3 +74,64 @@ fn nominal_fifo() { genesis_config.ticks_per_slot as usize, ); } + +fn insert_test_shreds(ledger_path: &Path, ending_slot: u64) { + let blockstore = Blockstore::open(ledger_path).unwrap(); + for i in 1..ending_slot { + let entries = entry::create_ticks(1, 0, Hash::default()); + let shreds = blockstore::entries_to_test_shreds(&entries, i, 0, false, 0); + blockstore.insert_shreds(shreds, None, false).unwrap(); + } +} + +fn ledger_tool_copy_test(src_shred_compaction: &str, dst_shred_compaction: &str) { + const TEST_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = std::u64::MAX; + let genesis_config = create_genesis_config(100).genesis_config; + + let (ledger_path, _blockhash) = match src_shred_compaction { + "fifo" => create_new_tmp_ledger_fifo!(&genesis_config), + _ => create_new_tmp_ledger!(&genesis_config), + }; + const LEDGER_TOOL_COPY_TEST_SHRED_COUNT: u64 = 25; + const LEDGER_TOOL_COPY_TEST_ENDING_SLOT: u64 = LEDGER_TOOL_COPY_TEST_SHRED_COUNT + 1; + insert_test_shreds(&ledger_path, LEDGER_TOOL_COPY_TEST_ENDING_SLOT); + let ledger_path = ledger_path.to_str().unwrap(); + + let target_ledger_path = get_tmp_ledger_path_auto_delete!(); + if dst_shred_compaction == "fifo" { + let rocksdb_fifo_path = target_ledger_path.path().join( + ShredStorageType::rocks_fifo(TEST_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES) + .blockstore_directory(), + ); + fs::create_dir_all(rocksdb_fifo_path).unwrap(); + } + let target_ledger_path = target_ledger_path.path().to_str().unwrap(); + let output = run_ledger_tool(&[ + "-l", + ledger_path, + "copy", + "--target-db", + target_ledger_path, + "--ending-slot", + &(LEDGER_TOOL_COPY_TEST_ENDING_SLOT).to_string(), + ]); + assert!(output.status.success()); + for slot_id in 0..LEDGER_TOOL_COPY_TEST_ENDING_SLOT { + let src_slot_output = run_ledger_tool(&["-l", ledger_path, "slot", &slot_id.to_string()]); + + let dst_slot_output = + run_ledger_tool(&["-l", target_ledger_path, "slot", &slot_id.to_string()]); + assert!(src_slot_output.status.success()); + assert!(dst_slot_output.status.success()); + assert!(!src_slot_output.stdout.is_empty()); + assert_eq!(src_slot_output.stdout, dst_slot_output.stdout); + } +} + +#[test] +fn copy_test() { + ledger_tool_copy_test("level", "level"); + ledger_tool_copy_test("level", "fifo"); + ledger_tool_copy_test("fifo", "level"); + ledger_tool_copy_test("fifo", "fifo"); +}