blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

ledger-tool cleanup and additions (#15179) 

* Plumb allow-dead-slots to ledger-tool verify

* ledger-tool cleanup and add some useful missing args

Print root slots and how many unrooted past last root.
This commit is contained in:
sakridge 2021-02-06 17:26:42 -08:00 committed by GitHub
parent 774416a546
commit bbae23358c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 76 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/src/replay_stage.rs
View File

@ -1010,6 +1010,7 @@ impl ReplayStage {
Some(replay_vote_sender), Some(replay_vote_sender),
None, None,
verify_recyclers, verify_recyclers,
false,
); );
let tx_count_after = bank_progress.replay_progress.num_txs; let tx_count_after = bank_progress.replay_progress.num_txs;
let tx_count = tx_count_after - tx_count_before; let tx_count = tx_count_after - tx_count_before;

104
ledger-tool/src/main.rs
View File

@ -20,7 +20,6 @@ use solana_ledger::{
blockstore::{create_new_ledger, Blockstore, PurgeType}, blockstore::{create_new_ledger, Blockstore, PurgeType},
blockstore_db::{self, AccessType, BlockstoreRecoveryMode, Column, Database}, blockstore_db::{self, AccessType, BlockstoreRecoveryMode, Column, Database},
blockstore_processor::ProcessOptions, blockstore_processor::ProcessOptions,
rooted_slot_iterator::RootedSlotIterator,
shred::Shred, shred::Shred,
}; };
use solana_runtime::{ use solana_runtime::{
@ -209,6 +208,7 @@ fn output_slot(
fn output_ledger( fn output_ledger(
blockstore: Blockstore, blockstore: Blockstore,
starting_slot: Slot, starting_slot: Slot,
ending_slot: Slot,
allow_dead_slots: bool, allow_dead_slots: bool,
method: LedgerOutputMethod, method: LedgerOutputMethod,
num_slots: Option<Slot>, num_slots: Option<Slot>,
@ -229,12 +229,15 @@ fn output_ledger(
stdout().write_all(b"{\"ledger\":[\n").expect("open array"); stdout().write_all(b"{\"ledger\":[\n").expect("open array");
} }
let num_slots = num_slots.unwrap_or(std::u64::MAX); let num_slots = num_slots.unwrap_or(Slot::MAX);
let mut num_printed = 0; let mut num_printed = 0;
for (slot, slot_meta) in slot_iterator { for (slot, slot_meta) in slot_iterator {
if only_rooted && !blockstore.is_root(slot) { if only_rooted && !blockstore.is_root(slot) {
continue; continue;
} }
if slot > ending_slot {
break;
}
match method { match method {
LedgerOutputMethod::Print => { LedgerOutputMethod::Print => {
@ -845,6 +848,7 @@ fn main() {
.about("Print the ledger") .about("Print the ledger")
.arg(&starting_slot_arg) .arg(&starting_slot_arg)
.arg(&allow_dead_slots_arg) .arg(&allow_dead_slots_arg)
.arg(&ending_slot_arg)
.arg( .arg(
Arg::with_name("num_slots") Arg::with_name("num_slots")
.long("num-slots") .long("num-slots")
@ -872,14 +876,7 @@ fn main() {
SubCommand::with_name("copy") SubCommand::with_name("copy")
.about("Copy the ledger") .about("Copy the ledger")
.arg(&starting_slot_arg) .arg(&starting_slot_arg)
.arg( .arg(&ending_slot_arg)
Arg::with_name("ending_slot")
.long("ending-slot")
.value_name("SLOT")
.validator(is_slot)
.takes_value(true)
.help("Slot to stop copy"),
)
.arg( .arg(
Arg::with_name("target_db") Arg::with_name("target_db")
.long("target-db") .long("target-db")
@ -1009,6 +1006,7 @@ fn main() {
.arg(&hard_forks_arg) .arg(&hard_forks_arg)
.arg(&no_accounts_db_caching_arg) .arg(&no_accounts_db_caching_arg)
.arg(&bpf_jit_arg) .arg(&bpf_jit_arg)
.arg(&allow_dead_slots_arg)
.arg(&max_genesis_archive_unpacked_size_arg) .arg(&max_genesis_archive_unpacked_size_arg)
.arg( .arg(
Arg::with_name("skip_poh_verify") Arg::with_name("skip_poh_verify")
@ -1281,7 +1279,6 @@ fn main() {
.long("max-height") .long("max-height")
.value_name("NUM") .value_name("NUM")
.takes_value(true) .takes_value(true)
.required(true)
.help("Maximum block height") .help("Maximum block height")
) )
.arg( .arg(
@ -1338,6 +1335,7 @@ fn main() {
("bigtable", Some(arg_matches)) => bigtable_process_command(&ledger_path, arg_matches), ("bigtable", Some(arg_matches)) => bigtable_process_command(&ledger_path, arg_matches),
("print", Some(arg_matches)) => { ("print", Some(arg_matches)) => {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let ending_slot = value_t!(arg_matches, "ending_slot", Slot).unwrap_or(Slot::MAX);
let num_slots = value_t!(arg_matches, "num_slots", Slot).ok(); let num_slots = value_t!(arg_matches, "num_slots", Slot).ok();
let allow_dead_slots = arg_matches.is_present("allow_dead_slots"); let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
let only_rooted = arg_matches.is_present("only_rooted"); let only_rooted = arg_matches.is_present("only_rooted");
@ -1349,6 +1347,7 @@ fn main() {
wal_recovery_mode, wal_recovery_mode,
), ),
starting_slot, starting_slot,
ending_slot,
allow_dead_slots, allow_dead_slots,
LedgerOutputMethod::Print, LedgerOutputMethod::Print,
num_slots, num_slots,
@ -1540,6 +1539,7 @@ fn main() {
wal_recovery_mode, wal_recovery_mode,
), ),
starting_slot, starting_slot,
Slot::MAX,
allow_dead_slots, allow_dead_slots,
LedgerOutputMethod::Json, LedgerOutputMethod::Json,
None, None,
@ -1644,6 +1644,7 @@ fn main() {
poh_verify: !arg_matches.is_present("skip_poh_verify"), poh_verify: !arg_matches.is_present("skip_poh_verify"),
bpf_jit: arg_matches.is_present("bpf_jit"), bpf_jit: arg_matches.is_present("bpf_jit"),
accounts_db_caching_enabled: !arg_matches.is_present("no_accounts_db_caching"), accounts_db_caching_enabled: !arg_matches.is_present("no_accounts_db_caching"),
allow_dead_slots: arg_matches.is_present("allow_dead_slots"),
..ProcessOptions::default() ..ProcessOptions::default()
}; };
let print_accounts_stats = arg_matches.is_present("print_accounts_stats"); let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
@ -2675,7 +2676,7 @@ fn main() {
let max_height = if let Some(height) = arg_matches.value_of("max_height") { let max_height = if let Some(height) = arg_matches.value_of("max_height") {
usize::from_str(height).expect("Maximum height must be a number") usize::from_str(height).expect("Maximum height must be a number")
} else { } else {
panic!("Maximum height must be provided"); usize::MAX
}; };
let num_roots = if let Some(roots) = arg_matches.value_of("num_roots") { let num_roots = if let Some(roots) = arg_matches.value_of("num_roots") {
usize::from_str(roots).expect("Number of roots must be a number") usize::from_str(roots).expect("Number of roots must be a number")
@ -2683,23 +2684,27 @@ fn main() {
usize::from_str(DEFAULT_ROOT_COUNT).unwrap() usize::from_str(DEFAULT_ROOT_COUNT).unwrap()
}; };
let iter = RootedSlotIterator::new(0, &blockstore).expect("Failed to get rooted slot"); let iter = blockstore
.rooted_slot_iterator(0)
.expect("Failed to get rooted slot");
let slot_hash: Vec<_> = iter let mut slot_hash = Vec::new();
.filter_map(|(slot, _meta)| { for (i, slot) in iter.into_iter().enumerate() {
if slot <= max_height as u64 { if i > num_roots {
let blockhash = blockstore break;
.get_slot_entries(slot, 0) }
.unwrap() if slot <= max_height as u64 {
.last() let blockhash = blockstore
.unwrap() .get_slot_entries(slot, 0)
.hash; .unwrap()
Some((slot, blockhash)) .last()
} else { .unwrap()
None .hash;
} slot_hash.push((slot, blockhash));
}) } else {
.collect(); break;
}
}
let mut output_file: Box<dyn Write> = let mut output_file: Box<dyn Write> =
if let Some(path) = arg_matches.value_of("slot_list") { if let Some(path) = arg_matches.value_of("slot_list") {
@ -2724,13 +2729,12 @@ fn main() {
}); });
} }
("bounds", Some(arg_matches)) => { ("bounds", Some(arg_matches)) => {
match open_blockstore( let blockstore = open_blockstore(
&ledger_path, &ledger_path,
AccessType::TryPrimaryThenSecondary, AccessType::TryPrimaryThenSecondary,
wal_recovery_mode, wal_recovery_mode,
) );
.slot_meta_iterator(0) match blockstore.slot_meta_iterator(0) {
{
Ok(metas) => { Ok(metas) => {
let all = arg_matches.is_present("all"); let all = arg_matches.is_present("all");
@ -2741,7 +2745,12 @@ fn main() {
let first = slots.first().unwrap(); let first = slots.first().unwrap();
let last = slots.last().unwrap_or(first); let last = slots.last().unwrap_or(first);
if first != last { if first != last {
println!("Ledger has data for slots {:?} to {:?}", first, last); println!(
"Ledger has data for {} slots {:?} to {:?}",
slots.len(),
first,
last
);
if all { if all {
println!("Non-empty slots: {:?}", slots); println!("Non-empty slots: {:?}", slots);
} }
@ -2749,12 +2758,39 @@ fn main() {
println!("Ledger has data for slot {:?}", first); println!("Ledger has data for slot {:?}", first);
} }
} }
if let Ok(rooted) = blockstore.rooted_slot_iterator(0) {
let mut first_rooted = 0;
let mut last_rooted = 0;
let mut total_rooted = 0;
for (i, slot) in rooted.into_iter().enumerate() {
if i == 0 {
first_rooted = slot;
}
last_rooted = slot;
total_rooted += 1;
}
let mut count_past_root = 0;
for slot in slots.iter().rev() {
if *slot > last_rooted {
count_past_root += 1;
} else {
break;
}
}
println!(
" with {} rooted slots from {:?} to {:?}",
total_rooted, first_rooted, last_rooted
);
println!(" and {} slots past the last root", count_past_root);
} else {
println!(" with no rooted slots");
}
} }
Err(err) => { Err(err) => {
eprintln!("Unable to read the Ledger: {:?}", err); eprintln!("Unable to read the Ledger: {:?}", err);
exit(1); exit(1);
} }
} };
} }
("analyze-storage", _) => { ("analyze-storage", _) => {
analyze_storage(&open_database( analyze_storage(&open_database(

6
ledger/src/blockstore_processor.rs
View File

@ -367,6 +367,7 @@ pub struct ProcessOptions {
pub debug_keys: Option<Arc<HashSet<Pubkey>>>, pub debug_keys: Option<Arc<HashSet<Pubkey>>>,
pub account_indexes: HashSet<AccountIndex>, pub account_indexes: HashSet<AccountIndex>,
pub accounts_db_caching_enabled: bool, pub accounts_db_caching_enabled: bool,
pub allow_dead_slots: bool,
} }
pub fn process_blockstore( pub fn process_blockstore(
@ -596,6 +597,7 @@ fn confirm_full_slot(
replay_vote_sender, replay_vote_sender,
opts.entry_callback.as_ref(), opts.entry_callback.as_ref(),
recyclers, recyclers,
opts.allow_dead_slots,
)?; )?;
if !bank.is_complete() { if !bank.is_complete() {
@ -649,6 +651,7 @@ impl ConfirmationProgress {
} }
} }
#[allow(clippy::too_many_arguments)]
pub fn confirm_slot( pub fn confirm_slot(
blockstore: &Blockstore, blockstore: &Blockstore,
bank: &Arc<Bank>, bank: &Arc<Bank>,
@ -659,13 +662,14 @@ pub fn confirm_slot(
replay_vote_sender: Option<&ReplayVoteSender>, replay_vote_sender: Option<&ReplayVoteSender>,
entry_callback: Option<&ProcessCallback>, entry_callback: Option<&ProcessCallback>,
recyclers: &VerifyRecyclers, recyclers: &VerifyRecyclers,
allow_dead_slots: bool,
) -> result::Result<(), BlockstoreProcessorError> { ) -> result::Result<(), BlockstoreProcessorError> {
let slot = bank.slot(); let slot = bank.slot();
let (entries, num_shreds, slot_full) = { let (entries, num_shreds, slot_full) = {
let mut load_elapsed = Measure::start("load_elapsed"); let mut load_elapsed = Measure::start("load_elapsed");
let load_result = blockstore let load_result = blockstore
.get_slot_entries_with_shred_info(slot, progress.num_shreds, false) .get_slot_entries_with_shred_info(slot, progress.num_shreds, allow_dead_slots)
.map_err(BlockstoreProcessorError::FailedToLoadEntries); .map_err(BlockstoreProcessorError::FailedToLoadEntries);
load_elapsed.stop(); load_elapsed.stop();
if load_result.is_err() { if load_result.is_err() {