blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

ledger-tool: Move shred storage type inference next to Blockstore::open (#30084)

This commit is contained in:
steviez 2023-02-02 16:55:18 -06:00 committed by GitHub
parent 3249e4a123
commit 37461976e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 32 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ledger-tool/src/bigtable.rs
View File

@ -17,9 +17,8 @@ use {
OutputFormat, OutputFormat,
}, },
solana_ledger::{ solana_ledger::{
bigtable_upload::ConfirmedBlockUploadConfig, bigtable_upload::ConfirmedBlockUploadConfig, blockstore::Blockstore,
blockstore::Blockstore, blockstore_options::AccessType,
blockstore_options::{AccessType, ShredStorageType},
}, },
solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature}, solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature},
solana_storage_bigtable::CredentialType, solana_storage_bigtable::CredentialType,
@ -985,11 +984,7 @@ fn get_global_subcommand_arg<T: FromStr>(
} }
} }
pub fn bigtable_process_command( pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
ledger_path: &Path,
matches: &ArgMatches<'_>,
shred_storage_type: &ShredStorageType,
) {
let runtime = tokio::runtime::Runtime::new().unwrap(); let runtime = tokio::runtime::Runtime::new().unwrap();
let verbose = matches.is_present("verbose"); let verbose = matches.is_present("verbose");
@ -1019,7 +1014,6 @@ pub fn bigtable_process_command(
&canonicalize_ledger_path(ledger_path), &canonicalize_ledger_path(ledger_path),
AccessType::Secondary, AccessType::Secondary,
None, None,
shred_storage_type,
force_update_to_open, force_update_to_open,
); );
let config = solana_storage_bigtable::LedgerStorageConfig { let config = solana_storage_bigtable::LedgerStorageConfig {

77
ledger-tool/src/main.rs
View File

@ -875,7 +875,7 @@ fn open_blockstore_with_temporary_primary_access(
) )
} }
fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorageType { fn get_shred_storage_type(ledger_path: &Path, message: &str) -> ShredStorageType {
// TODO: the following shred_storage_type inference must be updated once // TODO: the following shred_storage_type inference must be updated once
// the rocksdb options can be constructed via load_options_file() as the // the rocksdb options can be constructed via load_options_file() as the
// value picked by passing None for `max_shred_storage_size` could affect // value picked by passing None for `max_shred_storage_size` could affect
@ -883,7 +883,7 @@ fn get_shred_storage_type(ledger_path: &Path, warn_message: &str) -> ShredStorag
match ShredStorageType::from_ledger_path(ledger_path, None) { match ShredStorageType::from_ledger_path(ledger_path, None) {
Some(s) => s, Some(s) => s,
None => { None => {
warn!("{}", warn_message); info!("{}", message);
ShredStorageType::RocksLevel ShredStorageType::RocksLevel
} }
} }
@ -893,9 +893,17 @@ fn open_blockstore(
ledger_path: &Path, ledger_path: &Path,
access_type: AccessType, access_type: AccessType,
wal_recovery_mode: Option<BlockstoreRecoveryMode>, wal_recovery_mode: Option<BlockstoreRecoveryMode>,
shred_storage_type: &ShredStorageType,
force_update_to_open: bool, force_update_to_open: bool,
) -> Blockstore { ) -> Blockstore {
let shred_storage_type = get_shred_storage_type(
ledger_path,
&format!(
"Shred stroage type cannot be inferred for ledger at {:?}, \
using default RocksLevel",
ledger_path
),
);
match Blockstore::open_with_options( match Blockstore::open_with_options(
ledger_path, ledger_path,
BlockstoreOptions { BlockstoreOptions {
@ -903,7 +911,7 @@ fn open_blockstore(
recovery_mode: wal_recovery_mode.clone(), recovery_mode: wal_recovery_mode.clone(),
enforce_ulimit_nofile: true, enforce_ulimit_nofile: true,
column_options: LedgerColumnOptions { column_options: LedgerColumnOptions {
shred_storage_type: shred_storage_type.clone(), shred_storage_type,
..LedgerColumnOptions::default() ..LedgerColumnOptions::default()
}, },
}, },
@ -2326,13 +2334,9 @@ fn main() {
.map(BlockstoreRecoveryMode::from); .map(BlockstoreRecoveryMode::from);
let force_update_to_open = matches.is_present("force_update_to_open"); let force_update_to_open = matches.is_present("force_update_to_open");
let verbose_level = matches.occurrences_of("verbose"); let verbose_level = matches.occurrences_of("verbose");
let shred_storage_type = get_shred_storage_type(
&ledger_path,
"Shred storage type cannot be inferred, the default RocksLevel will be used",
);
if let ("bigtable", Some(arg_matches)) = matches.subcommand() { if let ("bigtable", Some(arg_matches)) = matches.subcommand() {
bigtable_process_command(&ledger_path, arg_matches, &shred_storage_type) bigtable_process_command(&ledger_path, arg_matches)
} else { } else {
let ledger_path = canonicalize_ledger_path(&ledger_path); let ledger_path = canonicalize_ledger_path(&ledger_path);
@ -2348,7 +2352,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
), ),
starting_slot, starting_slot,
@ -2364,30 +2367,31 @@ fn main() {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot); let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String)); let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String));
let target_shred_storage_type = get_shred_storage_type(
&target_db,
&format!(
"Shred storage type of target_db cannot be inferred, \
the default RocksLevel will be used. \
If you want to use FIFO shred_storage_type on an empty target_db, \
create {BLOCKSTORE_DIRECTORY_ROCKS_FIFO} foldar the specified target_db directory.",
),
);
let source = open_blockstore( let source = open_blockstore(
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
None, None,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
let target = open_blockstore(
// Check if shred storage type can be inferred; if not, a new
// ledger is being created. open_blockstore() will attempt to
// to infer shred storage type as well, but this check provides
// extra insight to user on how to create a FIFO ledger.
let _ = get_shred_storage_type(
&target_db, &target_db,
AccessType::Primary, &format!(
None, "No --target-db ledger at {:?} was detected, default \
&target_shred_storage_type, compaction (RocksLevel) will be used. Fifo compaction \
force_update_to_open, can be enabled for a new ledger by manually creating \
{BLOCKSTORE_DIRECTORY_ROCKS_FIFO} directory within \
the specified --target_db directory.",
&target_db
),
); );
let target =
open_blockstore(&target_db, AccessType::Primary, None, force_update_to_open);
for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() { for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() {
if slot > ending_slot { if slot > ending_slot {
break; break;
@ -2466,7 +2470,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
match load_bank_forks( match load_bank_forks(
@ -2519,7 +2522,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
None, None,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
for (slot, _meta) in ledger for (slot, _meta) in ledger
@ -2559,7 +2561,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
match load_bank_forks( match load_bank_forks(
@ -2586,7 +2587,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
for slot in slots { for slot in slots {
@ -2611,7 +2611,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
), ),
starting_slot, starting_slot,
@ -2628,7 +2627,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
@ -2641,7 +2639,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
@ -2655,7 +2652,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Primary, AccessType::Primary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
for slot in slots { for slot in slots {
@ -2671,7 +2667,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Primary, AccessType::Primary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
for slot in slots { for slot in slots {
@ -2690,7 +2685,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
let mut ancestors = BTreeSet::new(); let mut ancestors = BTreeSet::new();
@ -2802,7 +2796,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
let (bank_forks, ..) = load_bank_forks( let (bank_forks, ..) = load_bank_forks(
@ -2846,7 +2839,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
match load_bank_forks( match load_bank_forks(
@ -2960,7 +2952,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
@ -3355,7 +3346,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
let (bank_forks, ..) = load_bank_forks( let (bank_forks, ..) = load_bank_forks(
@ -3444,7 +3434,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
match load_bank_forks( match load_bank_forks(
@ -3976,7 +3965,6 @@ fn main() {
&ledger_path, &ledger_path,
access_type, access_type,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
@ -4050,7 +4038,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
let max_height = if let Some(height) = arg_matches.value_of("max_height") { let max_height = if let Some(height) = arg_matches.value_of("max_height") {
@ -4104,7 +4091,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
let num_slots = value_t_or_exit!(arg_matches, "num_slots", usize); let num_slots = value_t_or_exit!(arg_matches, "num_slots", usize);
@ -4129,7 +4115,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Primary, AccessType::Primary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
let start_root = if let Some(root) = arg_matches.value_of("start_root") { let start_root = if let Some(root) = arg_matches.value_of("start_root") {
@ -4179,7 +4164,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
@ -4252,7 +4236,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
) )
.db(), .db(),
@ -4263,7 +4246,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
force_update_to_open, force_update_to_open,
); );
@ -4287,7 +4269,6 @@ fn main() {
&ledger_path, &ledger_path,
AccessType::Secondary, AccessType::Secondary,
wal_recovery_mode, wal_recovery_mode,
&shred_storage_type,
false, false,
); );
let sst_file_name = arg_matches.value_of("file_name"); let sst_file_name = arg_matches.value_of("file_name");