blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Enable ledger-tool copy command to infer target_db shred storage type. (#27438) 

#### Problem
The ledger-tool copy command currently assumes target_db uses the same
shred storage type as the source ledger.

#### Summary of Changes
This PR enables ledger-tool copy command to infer the target_db shred storage
type based on whether rocksdb or rocksdb_fifo exists under the target_db directory
(same way as how ledger-tool infers the shred storage type for the main db.)
If the ledger-tool is not able to infer the shred storage type of the target_db, then
the default level compaction will be used.
This commit is contained in:
Yueh-Hsuan Chiang 2022-09-13 10:27:58 -07:00 committed by GitHub
parent c0e4379f43
commit 70c4f6ea96
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 106 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
ledger-tool/src/main.rs
View File

@ -30,8 +30,8 @@ use {
blockstore::{create_new_ledger, Blockstore, BlockstoreError, PurgeType},
blockstore_db::{self, columns as cf, Column, ColumnName, Database},
blockstore_options::{
AccessType, BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions,
ShredStorageType,
AccessType, BlockstoreOptions, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions,
LedgerColumnOptions, ShredStorageType,
},
blockstore_processor::{self, BlockstoreProcessorError, ProcessOptions},
shred::Shred,
@ -847,6 +847,23 @@ fn open_blockstore_with_temporary_primary_access(
)
}
fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorageType {
// TODO: the following shred_storage_type inference must be updated once the
// rocksdb options can be constructed via load_options_file() as the
// temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES
// could affect the persisted rocksdb options file.
match ShredStorageType::from_ledger_path(
ledger_path,
DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
) {
Some(s) => s,
None => {
warn!("{}", warn_message);
ShredStorageType::RocksLevel
}
}
}
fn open_blockstore(
ledger_path: &Path,
access_type: AccessType,
@ -1466,6 +1483,7 @@ fn main() {
let default_graph_vote_account_mode = GraphVoteAccountMode::default();
let mut measure_total_execution_time = Measure::start("ledger tool");
let matches = App::new(crate_name!())
.about(crate_description!())
.version(solana_version::version!())
@ -2215,21 +2233,10 @@ fn main() {
.map(BlockstoreRecoveryMode::from);
let force_update_to_open = matches.is_present("force_update_to_open");
let verbose_level = matches.occurrences_of("verbose");
// TODO: the following shred_storage_type inference must be updated once the
// rocksdb options can be constructed via load_options_file() as the
// temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES
// could affect the persisted rocksdb options file.
let shred_storage_type = match ShredStorageType::from_ledger_path(
let shred_storage_type = get_shred_storage_type(
&ledger_path,
DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
) {
Some(s) => s,
None => {
error!("Shred storage type cannot be inferred, the default RocksLevel will be used");
ShredStorageType::RocksLevel
}
};
"Shred storage type cannot be inferred, the default RocksLevel will be used",
);
if let ("bigtable", Some(arg_matches)) = matches.subcommand() {
bigtable_process_command(&ledger_path, arg_matches, &shred_storage_type)
@ -2264,6 +2271,18 @@ fn main() {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String));
let target_shred_storage_type = get_shred_storage_type(
&target_db,
&format!(
"Shred storage type of target_db cannot be inferred, \
the default RocksLevel will be used. \
If you want to use FIFO shred_storage_type on an empty target_db, \
create {} foldar the specified target_db directory.",
ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions::default())
.blockstore_directory()
),
);
let source = open_blockstore(
&ledger_path,
AccessType::Secondary,
@ -2275,10 +2294,9 @@ fn main() {
&target_db,
AccessType::Primary,
None,
&shred_storage_type,
&target_shred_storage_type,
force_update_to_open,
);
for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() {
if slot > ending_slot {
break;

71
ledger-tool/tests/basic.rs
View File

@ -1,9 +1,17 @@
use {
assert_cmd::prelude::*,
solana_entry::entry,
solana_ledger::{
blockstore, blockstore::Blockstore, blockstore_options::ShredStorageType,
create_new_tmp_ledger, create_new_tmp_ledger_fifo, genesis_utils::create_genesis_config,
get_tmp_ledger_path_auto_delete,
},
solana_sdk::hash::Hash,
std::{
fs,
path::Path,
process::{Command, Output},
},
std::process::{Command, Output},
};
fn run_ledger_tool(args: &[&str]) -> Output {
@ -66,3 +74,64 @@ fn nominal_fifo() {
genesis_config.ticks_per_slot as usize,
);
}
fn insert_test_shreds(ledger_path: &Path, ending_slot: u64) {
let blockstore = Blockstore::open(ledger_path).unwrap();
for i in 1..ending_slot {
let entries = entry::create_ticks(1, 0, Hash::default());
let shreds = blockstore::entries_to_test_shreds(&entries, i, 0, false, 0);
blockstore.insert_shreds(shreds, None, false).unwrap();
}
}
fn ledger_tool_copy_test(src_shred_compaction: &str, dst_shred_compaction: &str) {
const TEST_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = std::u64::MAX;
let genesis_config = create_genesis_config(100).genesis_config;
let (ledger_path, _blockhash) = match src_shred_compaction {
"fifo" => create_new_tmp_ledger_fifo!(&genesis_config),
_ => create_new_tmp_ledger!(&genesis_config),
};
const LEDGER_TOOL_COPY_TEST_SHRED_COUNT: u64 = 25;
const LEDGER_TOOL_COPY_TEST_ENDING_SLOT: u64 = LEDGER_TOOL_COPY_TEST_SHRED_COUNT + 1;
insert_test_shreds(&ledger_path, LEDGER_TOOL_COPY_TEST_ENDING_SLOT);
let ledger_path = ledger_path.to_str().unwrap();
let target_ledger_path = get_tmp_ledger_path_auto_delete!();
if dst_shred_compaction == "fifo" {
let rocksdb_fifo_path = target_ledger_path.path().join(
ShredStorageType::rocks_fifo(TEST_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES)
.blockstore_directory(),
);
fs::create_dir_all(rocksdb_fifo_path).unwrap();
}
let target_ledger_path = target_ledger_path.path().to_str().unwrap();
let output = run_ledger_tool(&[
"-l",
ledger_path,
"copy",
"--target-db",
target_ledger_path,
"--ending-slot",
&(LEDGER_TOOL_COPY_TEST_ENDING_SLOT).to_string(),
]);
assert!(output.status.success());
for slot_id in 0..LEDGER_TOOL_COPY_TEST_ENDING_SLOT {
let src_slot_output = run_ledger_tool(&["-l", ledger_path, "slot", &slot_id.to_string()]);
let dst_slot_output =
run_ledger_tool(&["-l", target_ledger_path, "slot", &slot_id.to_string()]);
assert!(src_slot_output.status.success());
assert!(dst_slot_output.status.success());
assert!(!src_slot_output.stdout.is_empty());
assert_eq!(src_slot_output.stdout, dst_slot_output.stdout);
}
}
#[test]
fn copy_test() {
ledger_tool_copy_test("level", "level");
ledger_tool_copy_test("level", "fifo");
ledger_tool_copy_test("fifo", "level");
ledger_tool_copy_test("fifo", "fifo");
}