blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Allow ledger tool to automatically detect the shred compaction style (#26182) 

#### Problem
Ledger-tool doesn't support shred-compaction-type other than the default rocksdb level compaction.

#### Summary of Changes
This PR enables ledger-tool to automatically detect the shred-compaction-type of the specified ledger.

#### Test Plan
New ledger-tool tests are added for both level and fifo compactions.
This commit is contained in:
Yueh-Hsuan Chiang 2022-07-19 01:19:17 +08:00 committed by GitHub
parent 2dd8573287
commit dcbda2c6c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 262 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ledger-tool/src/bigtable.rs
View File

@ -15,8 +15,9 @@ use {
OutputFormat,
},
solana_ledger::{
bigtable_upload::ConfirmedBlockUploadConfig, blockstore::Blockstore,
blockstore_options::AccessType,
bigtable_upload::ConfirmedBlockUploadConfig,
blockstore::Blockstore,
blockstore_options::{AccessType, ShredStorageType},
},
solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature},
solana_storage_bigtable::CredentialType,
@ -615,7 +616,11 @@ fn get_global_subcommand_arg<T: FromStr>(
}
}
pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
pub fn bigtable_process_command(
ledger_path: &Path,
matches: &ArgMatches<'_>,
shred_storage_type: &ShredStorageType,
) {
let runtime = tokio::runtime::Runtime::new().unwrap();
let verbose = matches.is_present("verbose");
@ -644,6 +649,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
&canonicalize_ledger_path(ledger_path),
AccessType::Secondary,
None,
shred_storage_type,
);
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,

218
ledger-tool/src/main.rs
View File

@ -27,6 +27,7 @@ use {
blockstore_db::{self, Database},
blockstore_options::{
AccessType, BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions,
ShredStorageType,
},
blockstore_processor::{BlockstoreProcessorError, ProcessOptions},
shred::Shred,
@ -724,6 +725,7 @@ fn open_blockstore(
ledger_path: &Path,
access_type: AccessType,
wal_recovery_mode: Option<BlockstoreRecoveryMode>,
shred_storage_type: &ShredStorageType,
) -> Blockstore {
match Blockstore::open_with_options(
ledger_path,
@ -731,7 +733,10 @@ fn open_blockstore(
access_type,
recovery_mode: wal_recovery_mode,
enforce_ulimit_nofile: true,
..BlockstoreOptions::default()
column_options: LedgerColumnOptions {
shred_storage_type: shred_storage_type.clone(),
..LedgerColumnOptions::default()
},
},
) {
Ok(blockstore) => blockstore,
@ -953,6 +958,10 @@ use jemallocator::Jemalloc;
#[global_allocator]
static GLOBAL: Jemalloc = Jemalloc;
/// The default size for data and coding shred column families in FIFO compaction.
/// u64::MAX as the default value means it won't delete any files by default.
const DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES: u64 = std::u64::MAX;
#[allow(clippy::cognitive_complexity)]
fn main() {
// Ignore SIGUSR1 to prevent long-running calls being killed by logrotate
@ -1872,8 +1881,23 @@ fn main() {
.map(BlockstoreRecoveryMode::from);
let verbose_level = matches.occurrences_of("verbose");
// TODO: the following shred_storage_type inference must be updated once the
// rocksdb options can be constructed via load_options_file() as the
// temporary use of DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES
// could affect the persisted rocksdb options file.
let shred_storage_type = match ShredStorageType::from_ledger_path(
&ledger_path,
DEFAULT_LEDGER_TOOL_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
) {
Some(s) => s,
None => {
error!("Shred storage type cannot be inferred, the default RocksLevel will be used");
ShredStorageType::RocksLevel
}
};
if let ("bigtable", Some(arg_matches)) = matches.subcommand() {
bigtable_process_command(&ledger_path, arg_matches)
bigtable_process_command(&ledger_path, arg_matches, &shred_storage_type)
} else {
let ledger_path = canonicalize_ledger_path(&ledger_path);
@ -1885,7 +1909,12 @@ fn main() {
let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
let only_rooted = arg_matches.is_present("only_rooted");
output_ledger(
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode),
open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
),
starting_slot,
ending_slot,
allow_dead_slots,
@ -1899,8 +1928,14 @@ fn main() {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
let target_db = PathBuf::from(value_t_or_exit!(arg_matches, "target_db", String));
let source = open_blockstore(&ledger_path, AccessType::Secondary, None);
let target = open_blockstore(&target_db, AccessType::Primary, None);
let source = open_blockstore(
&ledger_path,
AccessType::Secondary,
None,
&shred_storage_type,
);
let target =
open_blockstore(&target_db, AccessType::Primary, None, &shred_storage_type);
for (slot, _meta) in source.slot_meta_iterator(starting_slot).unwrap() {
if slot > ending_slot {
break;
@ -1973,8 +2008,12 @@ fn main() {
..ProcessOptions::default()
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
match load_bank_forks(
arg_matches,
&genesis_config,
@ -2021,7 +2060,12 @@ fn main() {
}
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let ending_slot = value_t!(arg_matches, "ending_slot", Slot).unwrap_or(Slot::MAX);
let ledger = open_blockstore(&ledger_path, AccessType::Secondary, None);
let ledger = open_blockstore(
&ledger_path,
AccessType::Secondary,
None,
&shred_storage_type,
);
for (slot, _meta) in ledger
.slot_meta_iterator(starting_slot)
.unwrap()
@ -2055,8 +2099,12 @@ fn main() {
..ProcessOptions::default()
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
match load_bank_forks(
arg_matches,
&genesis_config,
@ -2077,8 +2125,12 @@ fn main() {
("slot", Some(arg_matches)) => {
let slots = values_t_or_exit!(arg_matches, "slots", Slot);
let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
for slot in slots {
println!("Slot {}", slot);
if let Err(err) = output_slot(
@ -2097,7 +2149,12 @@ fn main() {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
output_ledger(
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode),
open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
),
starting_slot,
Slot::MAX,
allow_dead_slots,
@ -2108,16 +2165,24 @@ fn main() {
);
}
("dead-slots", Some(arg_matches)) => {
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
for slot in blockstore.dead_slots_iterator(starting_slot).unwrap() {
println!("{}", slot);
}
}
("duplicate-slots", Some(arg_matches)) => {
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
for slot in blockstore.duplicate_slots_iterator(starting_slot).unwrap() {
println!("{}", slot);
@ -2125,8 +2190,12 @@ fn main() {
}
("set-dead-slot", Some(arg_matches)) => {
let slots = values_t_or_exit!(arg_matches, "slots", Slot);
let blockstore =
open_blockstore(&ledger_path, AccessType::Primary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Primary,
wal_recovery_mode,
&shred_storage_type,
);
for slot in slots {
match blockstore.set_dead_slot(slot) {
Ok(_) => println!("Slot {} dead", slot),
@ -2136,8 +2205,12 @@ fn main() {
}
("remove-dead-slot", Some(arg_matches)) => {
let slots = values_t_or_exit!(arg_matches, "slots", Slot);
let blockstore =
open_blockstore(&ledger_path, AccessType::Primary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Primary,
wal_recovery_mode,
&shred_storage_type,
);
for slot in slots {
match blockstore.remove_dead_slot(slot) {
Ok(_) => println!("Slot {} not longer marked dead", slot),
@ -2150,8 +2223,12 @@ fn main() {
("parse_full_frozen", Some(arg_matches)) => {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
let mut ancestors = BTreeSet::new();
assert!(
blockstore.meta(ending_slot).unwrap().is_some(),
@ -2304,8 +2381,12 @@ fn main() {
open_genesis_config_by(&ledger_path, arg_matches).hash()
);
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
let (bank_forks, ..) = load_bank_forks(
arg_matches,
&open_genesis_config_by(&ledger_path, arg_matches),
@ -2336,8 +2417,12 @@ fn main() {
..ProcessOptions::default()
};
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
match load_bank_forks(
arg_matches,
&open_genesis_config_by(&ledger_path, arg_matches),
@ -2449,8 +2534,12 @@ fn main() {
usize
);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
let snapshot_slot = if Some("ROOT") == arg_matches.value_of("snapshot_slot") {
blockstore
@ -2833,8 +2922,12 @@ fn main() {
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let include_sysvars = arg_matches.is_present("include_sysvars");
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
let (bank_forks, ..) = load_bank_forks(
arg_matches,
&genesis_config,
@ -2893,8 +2986,12 @@ fn main() {
..ProcessOptions::default()
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
match load_bank_forks(
arg_matches,
&genesis_config,
@ -3424,7 +3521,12 @@ fn main() {
} else {
AccessType::PrimaryForMaintenance
};
let blockstore = open_blockstore(&ledger_path, access_type, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
access_type,
wal_recovery_mode,
&shred_storage_type,
);
let end_slot = match end_slot {
Some(end_slot) => end_slot,
@ -3495,8 +3597,12 @@ fn main() {
}
}
("list-roots", Some(arg_matches)) => {
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
let max_height = if let Some(height) = arg_matches.value_of("max_height") {
usize::from_str(height).expect("Maximum height must be a number")
} else {
@ -3558,8 +3664,12 @@ fn main() {
});
}
("latest-optimistic-slots", Some(arg_matches)) => {
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
let num_slots = value_t_or_exit!(arg_matches, "num_slots", usize);
let slots = blockstore
.get_latest_optimistic_slots(num_slots)
@ -3578,8 +3688,12 @@ fn main() {
}
}
("repair-roots", Some(arg_matches)) => {
let blockstore =
open_blockstore(&ledger_path, AccessType::Primary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Primary,
wal_recovery_mode,
&shred_storage_type,
);
let start_root = if let Some(root) = arg_matches.value_of("start_root") {
Slot::from_str(root).expect("Before root must be a number")
} else {
@ -3627,8 +3741,12 @@ fn main() {
}
}
("bounds", Some(arg_matches)) => {
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
match blockstore.slot_meta_iterator(0) {
Ok(metas) => {
let all = arg_matches.is_present("all");
@ -3689,13 +3807,23 @@ fn main() {
}
("analyze-storage", _) => {
analyze_storage(
&open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode).db(),
&open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
)
.db(),
);
println!("Ok.");
}
("compute-slot-cost", Some(arg_matches)) => {
let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let blockstore = open_blockstore(
&ledger_path,
AccessType::Secondary,
wal_recovery_mode,
&shred_storage_type,
);
let mut slots: Vec<u64> = vec![];
if !arg_matches.is_present("slots") {

41
ledger-tool/tests/basic.rs
View File

@ -1,6 +1,8 @@
use {
assert_cmd::prelude::*,
solana_ledger::{create_new_tmp_ledger, genesis_utils::create_genesis_config},
solana_ledger::{
create_new_tmp_ledger, create_new_tmp_ledger_fifo, genesis_utils::create_genesis_config,
},
std::process::{Command, Output},
};
@ -27,19 +29,10 @@ fn bad_arguments() {
.success());
}
#[test]
fn nominal() {
let genesis_config = create_genesis_config(100).genesis_config;
let ticks_per_slot = genesis_config.ticks_per_slot;
fn nominal_test_helper(ledger_path: &str, ticks: usize) {
let meta_lines = 2;
let summary_lines = 1;
let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
let ticks = ticks_per_slot as usize;
let ledger_path = ledger_path.to_str().unwrap();
// Basic validation
let output = run_ledger_tool(&["-l", ledger_path, "verify"]);
assert!(output.status.success());
@ -47,7 +40,29 @@ fn nominal() {
let output = run_ledger_tool(&["-l", ledger_path, "print", "-vvv"]);
assert!(output.status.success());
assert_eq!(
count_newlines(&output.stdout),
ticks + meta_lines + summary_lines
count_newlines(&output.stdout)
.saturating_sub(meta_lines)
.saturating_sub(summary_lines),
ticks
);
}
#[test]
fn nominal_default() {
let genesis_config = create_genesis_config(100).genesis_config;
let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
nominal_test_helper(
ledger_path.to_str().unwrap(),
genesis_config.ticks_per_slot as usize,
);
}
#[test]
fn nominal_fifo() {
let genesis_config = create_genesis_config(100).genesis_config;
let (ledger_path, _blockhash) = create_new_tmp_ledger_fifo!(&genesis_config);
nominal_test_helper(
ledger_path.to_str().unwrap(),
genesis_config.ticks_per_slot as usize,
);
}

16
ledger/src/blockstore.rs
View File

@ -4010,6 +4010,22 @@ macro_rules! create_new_tmp_ledger {
};
}
#[macro_export]
macro_rules! create_new_tmp_ledger_fifo {
($genesis_config:expr) => {
$crate::blockstore::create_new_ledger_from_name(
$crate::tmp_ledger_name!(),
$genesis_config,
$crate::blockstore_options::LedgerColumnOptions {
shred_storage_type: $crate::blockstore_options::ShredStorageType::RocksFifo(
$crate::blockstore_options::BlockstoreRocksFifoOptions::default(),
),
..$crate::blockstore_options::LedgerColumnOptions::default()
},
)
};
}
#[macro_export]
macro_rules! create_new_tmp_ledger_auto_delete {
($genesis_config:expr) => {

37
ledger/src/blockstore_options.rs
View File

@ -1,4 +1,7 @@
use rocksdb::{DBCompressionType as RocksCompressionType, DBRecoveryMode};
use {
rocksdb::{DBCompressionType as RocksCompressionType, DBRecoveryMode},
std::path::Path,
};
pub struct BlockstoreOptions {
// The access type of blockstore. Default: Primary
@ -153,6 +156,38 @@ impl ShredStorageType {
ShredStorageType::RocksFifo(_) => BLOCKSTORE_DIRECTORY_ROCKS_FIFO,
}
}
/// Returns the ShredStorageType that is used under the specified
/// ledger_path.
///
/// None will be returned if the ShredStorageType cannot be inferred.
pub fn from_ledger_path(
ledger_path: &Path,
fifo_shred_storage_size: u64,
) -> Option<ShredStorageType> {
let mut result: Option<ShredStorageType> = None;
if Path::new(ledger_path)
.join(BLOCKSTORE_DIRECTORY_ROCKS_LEVEL)
.exists()
{
result = Some(ShredStorageType::RocksLevel);
}
if Path::new(ledger_path)
.join(BLOCKSTORE_DIRECTORY_ROCKS_FIFO)
.exists()
{
if result.is_none() {
result = Some(ShredStorageType::RocksFifo(
BlockstoreRocksFifoOptions::new(fifo_shred_storage_size),
));
} else {
result = None;
}
}
result
}
}
#[derive(Debug, Clone)]