Enable ledger-tool copy command to infer target_db shred storage type. (#27438)
#### Problem The ledger-tool copy command currently assumes target_db uses the same shred storage type as the source ledger. #### Summary of Changes This PR enables ledger-tool copy command to infer the target_db shred storage type based on whether rocksdb or rocksdb_fifo exists under the target_db directory (same way as how ledger-tool infers the shred storage type for the main db.) If the ledger-tool is not able to infer the shred storage type of the target_db, then the default level compaction will be used.
This commit is contained in:
parent
c0e4379f43
commit
70c4f6ea96
|
@ -30,8 +30,8 @@ use {
|
||||||
blockstore::{create_new_ledger, Blockstore, BlockstoreError, PurgeType},
|
blockstore::{create_new_ledger, Blockstore, BlockstoreError, PurgeType},
|
||||||
blockstore_db::{self, columns as cf, Column, ColumnName, Database},
|
blockstore_db::{self, columns as cf, Column, ColumnName, Database},
|
||||||
blockstore_options::{
|
blockstore_options::{
|
||||||
AccessType, BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions,
|
AccessType, BlockstoreOptions, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions,
|
||||||
ShredStorageType,
|
LedgerColumnOptions, ShredStorageType,
|
||||||
},
|
},
|
||||||
blockstore_processor::{self, BlockstoreProcessorError, ProcessOptions},
|
blockstore_processor::{self, BlockstoreProcessorError, ProcessOptions},
|
||||||
shred::Shred,
|
shred::Shred,
|
||||||
|
@ -847,6 +847,23 @@ fn open_blockstore_with_temporary_primary_access(
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorageType {
|
||||||
|
// TODO: the following shred_storage_type inference must be updated once the
|
||||||
|
// rocksdb options can be constructed via load_options_file() as the
|
||||||
|
// temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES
|
||||||
|
// could affect the persisted rocksdb options file.
|
||||||
|
match ShredStorageType::from_ledger_path(
|
||||||
|
ledger_path,
|
||||||
|
DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
|
||||||
|
) {
|
||||||
|
Some(s) => s,
|
||||||
|
None => {
|
||||||
|
warn!("{}", warn_message);
|
||||||
|
ShredStorageType::RocksLevel
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn open_blockstore(
|
fn open_blockstore(
|
||||||
ledger_path: &Path,
|
ledger_path: &Path,
|
||||||
access_type: AccessType,
|
access_type: AccessType,
|
||||||
|
@ -1466,6 +1483,7 @@ fn main() {
|
||||||
let default_graph_vote_account_mode = GraphVoteAccountMode::default();
|
let default_graph_vote_account_mode = GraphVoteAccountMode::default();
|
||||||
|
|
||||||
let mut measure_total_execution_time = Measure::start("ledger tool");
|
let mut measure_total_execution_time = Measure::start("ledger tool");
|
||||||
|
|
||||||
let matches = App::new(crate_name!())
|
let matches = App::new(crate_name!())
|
||||||
.about(crate_description!())
|
.about(crate_description!())
|
||||||
.version(solana_version::version!())
|
.version(solana_version::version!())
|
||||||
|
@ -2215,21 +2233,10 @@ fn main() {
|
||||||
.map(BlockstoreRecoveryMode::from);
|
.map(BlockstoreRecoveryMode::from);
|
||||||
let force_update_to_open = matches.is_present("force_update_to_open");
|
let force_update_to_open = matches.is_present("force_update_to_open");
|
||||||
let verbose_level = matches.occurrences_of("verbose");
|
let verbose_level = matches.occurrences_of("verbose");
|
||||||
|
let shred_storage_type = get_shred_storage_type(
|
||||||
// TODO: the following shred_storage_type inference must be updated once the
|
|
||||||
// rocksdb options can be constructed via load_options_file() as the
|
|
||||||
// temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES
|
|
||||||
// could affect the persisted rocksdb options file.
|
|
||||||
let shred_storage_type = match ShredStorageType::from_ledger_path(
|
|
||||||
&ledger_path,
|
&ledger_path,
|
||||||
DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
|
"Shred storage type cannot be inferred, the default RocksLevel will be used",
|
||||||
) {
|
);
|
||||||
Some(s) => s,
|
|
||||||
None => {
|
|
||||||
error!("Shred storage type cannot be inferred, the default RocksLevel will be used");
|
|
||||||
ShredStorageType::RocksLevel
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if let ("bigtable", Some(arg_matches)) = matches.subcommand() {
|
if let ("bigtable", Some(arg_matches)) = matches.subcommand() {
|
||||||
bigtable_process_command(&ledger_path, arg_matches, &shred_storage_type)
|
bigtable_process_command(&ledger_path, arg_matches, &shred_storage_type)
|
||||||
|
@ -2264,6 +2271,18 @@ fn main() {
|
||||||
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
|
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
|
||||||
let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
|
let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
|
||||||
let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String));
|
let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String));
|
||||||
|
let target_shred_storage_type = get_shred_storage_type(
|
||||||
|
&target_db,
|
||||||
|
&format!(
|
||||||
|
"Shred storage type of target_db cannot be inferred, \
|
||||||
|
the default RocksLevel will be used. \
|
||||||
|
If you want to use FIFO shred_storage_type on an empty target_db, \
|
||||||
|
create {} foldar the specified target_db directory.",
|
||||||
|
ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions::default())
|
||||||
|
.blockstore_directory()
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
let source = open_blockstore(
|
let source = open_blockstore(
|
||||||
&ledger_path,
|
&ledger_path,
|
||||||
AccessType::Secondary,
|
AccessType::Secondary,
|
||||||
|
@ -2275,10 +2294,9 @@ fn main() {
|
||||||
&target_db,
|
&target_db,
|
||||||
AccessType::Primary,
|
AccessType::Primary,
|
||||||
None,
|
None,
|
||||||
&shred_storage_type,
|
&target_shred_storage_type,
|
||||||
force_update_to_open,
|
force_update_to_open,
|
||||||
);
|
);
|
||||||
|
|
||||||
for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() {
|
for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() {
|
||||||
if slot > ending_slot {
|
if slot > ending_slot {
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -1,9 +1,17 @@
|
||||||
use {
|
use {
|
||||||
assert_cmd::prelude::*,
|
assert_cmd::prelude::*,
|
||||||
|
solana_entry::entry,
|
||||||
solana_ledger::{
|
solana_ledger::{
|
||||||
|
blockstore, blockstore::Blockstore, blockstore_options::ShredStorageType,
|
||||||
create_new_tmp_ledger, create_new_tmp_ledger_fifo, genesis_utils::create_genesis_config,
|
create_new_tmp_ledger, create_new_tmp_ledger_fifo, genesis_utils::create_genesis_config,
|
||||||
|
get_tmp_ledger_path_auto_delete,
|
||||||
|
},
|
||||||
|
solana_sdk::hash::Hash,
|
||||||
|
std::{
|
||||||
|
fs,
|
||||||
|
path::Path,
|
||||||
|
process::{Command, Output},
|
||||||
},
|
},
|
||||||
std::process::{Command, Output},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
fn run_ledger_tool(args: &[&str]) -> Output {
|
fn run_ledger_tool(args: &[&str]) -> Output {
|
||||||
|
@ -66,3 +74,64 @@ fn nominal_fifo() {
|
||||||
genesis_config.ticks_per_slot as usize,
|
genesis_config.ticks_per_slot as usize,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn insert_test_shreds(ledger_path: &Path, ending_slot: u64) {
|
||||||
|
let blockstore = Blockstore::open(ledger_path).unwrap();
|
||||||
|
for i in 1..ending_slot {
|
||||||
|
let entries = entry::create_ticks(1, 0, Hash::default());
|
||||||
|
let shreds = blockstore::entries_to_test_shreds(&entries, i, 0, false, 0);
|
||||||
|
blockstore.insert_shreds(shreds, None, false).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ledger_tool_copy_test(src_shred_compaction: &str, dst_shred_compaction: &str) {
|
||||||
|
const TEST_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = std::u64::MAX;
|
||||||
|
let genesis_config = create_genesis_config(100).genesis_config;
|
||||||
|
|
||||||
|
let (ledger_path, _blockhash) = match src_shred_compaction {
|
||||||
|
"fifo" => create_new_tmp_ledger_fifo!(&genesis_config),
|
||||||
|
_ => create_new_tmp_ledger!(&genesis_config),
|
||||||
|
};
|
||||||
|
const LEDGER_TOOL_COPY_TEST_SHRED_COUNT: u64 = 25;
|
||||||
|
const LEDGER_TOOL_COPY_TEST_ENDING_SLOT: u64 = LEDGER_TOOL_COPY_TEST_SHRED_COUNT + 1;
|
||||||
|
insert_test_shreds(&ledger_path, LEDGER_TOOL_COPY_TEST_ENDING_SLOT);
|
||||||
|
let ledger_path = ledger_path.to_str().unwrap();
|
||||||
|
|
||||||
|
let target_ledger_path = get_tmp_ledger_path_auto_delete!();
|
||||||
|
if dst_shred_compaction == "fifo" {
|
||||||
|
let rocksdb_fifo_path = target_ledger_path.path().join(
|
||||||
|
ShredStorageType::rocks_fifo(TEST_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES)
|
||||||
|
.blockstore_directory(),
|
||||||
|
);
|
||||||
|
fs::create_dir_all(rocksdb_fifo_path).unwrap();
|
||||||
|
}
|
||||||
|
let target_ledger_path = target_ledger_path.path().to_str().unwrap();
|
||||||
|
let output = run_ledger_tool(&[
|
||||||
|
"-l",
|
||||||
|
ledger_path,
|
||||||
|
"copy",
|
||||||
|
"--target-db",
|
||||||
|
target_ledger_path,
|
||||||
|
"--ending-slot",
|
||||||
|
&(LEDGER_TOOL_COPY_TEST_ENDING_SLOT).to_string(),
|
||||||
|
]);
|
||||||
|
assert!(output.status.success());
|
||||||
|
for slot_id in 0..LEDGER_TOOL_COPY_TEST_ENDING_SLOT {
|
||||||
|
let src_slot_output = run_ledger_tool(&["-l", ledger_path, "slot", &slot_id.to_string()]);
|
||||||
|
|
||||||
|
let dst_slot_output =
|
||||||
|
run_ledger_tool(&["-l", target_ledger_path, "slot", &slot_id.to_string()]);
|
||||||
|
assert!(src_slot_output.status.success());
|
||||||
|
assert!(dst_slot_output.status.success());
|
||||||
|
assert!(!src_slot_output.stdout.is_empty());
|
||||||
|
assert_eq!(src_slot_output.stdout, dst_slot_output.stdout);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn copy_test() {
|
||||||
|
ledger_tool_copy_test("level", "level");
|
||||||
|
ledger_tool_copy_test("level", "fifo");
|
||||||
|
ledger_tool_copy_test("fifo", "level");
|
||||||
|
ledger_tool_copy_test("fifo", "fifo");
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue