blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add --print-account-stats/--snapshot-archive-path arguments to ledger-tool (#10945) 

Allows for seeing how the accounts are being stored
and specifying a different snapshot source directory.
This commit is contained in:
sakridge 2020-07-08 09:32:11 -07:00 committed by GitHub
parent c06505934f
commit 6429042b6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 102 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
ledger-tool/src/main.rs
View File

@ -634,6 +634,7 @@ fn load_bank_forks(
process_options: ProcessOptions,
access_type: AccessType,
wal_recovery_mode: Option<BlockstoreRecoveryMode>,
snapshot_archive_path: Option<PathBuf>,
) -> bank_forks_utils::LoadResult {
let blockstore = open_blockstore(&ledger_path, access_type, wal_recovery_mode);
let snapshot_path = ledger_path.clone().join(if blockstore.is_primary_access() {
@ -644,9 +645,11 @@ fn load_bank_forks(
let snapshot_config = if arg_matches.is_present("no_snapshot") {
None
} else {
let snapshot_package_output_path =
snapshot_archive_path.unwrap_or_else(|| ledger_path.clone());
Some(SnapshotConfig {
snapshot_interval_slots: 0, // Value doesn't matter
snapshot_package_output_path: ledger_path.clone(),
snapshot_package_output_path,
snapshot_path,
compression: CompressionType::Bzip2,
snapshot_version: SnapshotVersion::default(),
@ -654,7 +657,7 @@ fn load_bank_forks(
};
let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
if !blockstore.is_primary_access() {
// Be defenstive, when default account dir is explicitly specified, it's still possible
// Be defensive, when default account dir is explicitly specified, it's still possible
// to wipe the dir possibly shared by the running validator!
eprintln!("Error: custom accounts path is not supported under secondary access");
exit(1);
@ -774,6 +777,14 @@ fn main() {
"Mode to recovery the ledger db write ahead log."
),
)
.arg(
Arg::with_name("snapshot_archive_path")
.long("snapshot-archive-path")
.value_name("DIR")
.takes_value(true)
.global(true)
.help("Use DIR for ledger location"),
)
.subcommand(
SubCommand::with_name("print")
.about("Print the ledger")
@ -916,6 +927,12 @@ fn main() {
.takes_value(false)
.help("Skip ledger PoH verification"),
)
.arg(
Arg::with_name("print_accounts_stats")
.long("print-accounts-stats")
.takes_value(false)
.help("After verifying the ledger, print some information about the account stores."),
)
).subcommand(
SubCommand::with_name("graph")
.about("Create a Graphviz rendering of the ledger")
@ -1068,6 +1085,10 @@ fn main() {
exit(1);
});
let snapshot_archive_path = value_t!(matches, "snapshot_archive_path", String)
.ok()
.map(PathBuf::from);
let wal_recovery_mode = matches
.value_of("wal_recovery_mode")
.map(BlockstoreRecoveryMode::from);
@ -1134,6 +1155,7 @@ fn main() {
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
println!(
@ -1274,23 +1296,29 @@ fn main() {
poh_verify: !arg_matches.is_present("skip_poh_verify"),
..ProcessOptions::default()
};
let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
println!(
"genesis hash: {}",
open_genesis_config_by(&ledger_path, arg_matches).hash()
);
load_bank_forks(
let (bank_forks, _, _) = load_bank_forks(
arg_matches,
&ledger_path,
&open_genesis_config_by(&ledger_path, arg_matches),
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
)
.unwrap_or_else(|err| {
eprintln!("Ledger verification failed: {:?}", err);
exit(1);
});
if print_accounts_stats {
let working_bank = bank_forks.working_bank();
working_bank.print_accounts_stats();
}
println!("Ok");
}
("graph", Some(arg_matches)) => {
@ -1310,6 +1338,7 @@ fn main() {
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
let dot = graph_forks(&bank_forks, arg_matches.is_present("include_all_votes"));
@ -1363,6 +1392,7 @@ fn main() {
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
let bank = bank_forks
@ -1461,6 +1491,7 @@ fn main() {
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
let slot = bank_forks.working_bank().slot();
@ -1510,6 +1541,7 @@ fn main() {
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
let slot = bank_forks.working_bank().slot();

123
runtime/src/accounts_db.rs
View File

@ -1975,6 +1975,53 @@ impl AccountsDB {
}
}
}
pub fn print_accounts_stats(&self, label: &'static str) {
self.print_index(label);
self.print_count_and_status(label);
}
fn print_index(&self, label: &'static str) {
let mut roots: Vec<_> = self
.accounts_index
.read()
.unwrap()
.roots
.iter()
.cloned()
.collect();
roots.sort();
info!("{}: accounts_index roots: {:?}", label, roots,);
for (pubkey, list) in &self.accounts_index.read().unwrap().account_maps {
info!(" key: {}", pubkey);
info!(" slots: {:?}", *list.1.read().unwrap());
}
}
fn print_count_and_status(&self, label: &'static str) {
let storage = self.storage.read().unwrap();
let mut slots: Vec<_> = storage.0.keys().cloned().collect();
slots.sort();
info!("{}: count_and status for {} slots:", label, slots.len());
for slot in &slots {
let slot_stores = storage.0.get(slot).unwrap();
let mut ids: Vec<_> = slot_stores.keys().cloned().collect();
ids.sort();
for id in &ids {
let entry = slot_stores.get(id).unwrap();
info!(
" slot: {} id: {} count_and_status: {:?} approx_store_count: {} len: {} capacity: {}",
slot,
id,
*entry.count_and_status.read().unwrap(),
entry.approx_store_count.load(Ordering::Relaxed),
entry.accounts.len(),
entry.accounts.capacity(),
);
}
}
}
}
#[cfg(test)]
@ -2557,7 +2604,7 @@ pub mod tests {
accounts.store(1, &[(&pubkey, &account)]);
//slot is gone
print_accounts("pre-clean", &accounts);
accounts.print_accounts_stats("pre-clean");
accounts.clean_accounts();
accounts.process_dead_slots(None);
assert!(accounts.storage.read().unwrap().0.get(&0).is_none());
@ -2737,50 +2784,6 @@ pub mod tests {
assert_eq!(accounts.uncleaned_root_count(), 0);
}
fn print_accounts(label: &'static str, accounts: &AccountsDB) {
print_index(label, accounts);
print_count_and_status(label, accounts);
}
fn print_index(label: &'static str, accounts: &AccountsDB) {
let mut roots: Vec<_> = accounts
.accounts_index
.read()
.unwrap()
.roots
.iter()
.cloned()
.collect();
roots.sort();
info!("{}: accounts.accounts_index roots: {:?}", label, roots,);
for (pubkey, list) in &accounts.accounts_index.read().unwrap().account_maps {
info!(" key: {}", pubkey);
info!(" slots: {:?}", *list.1.read().unwrap());
}
}
fn print_count_and_status(label: &'static str, accounts: &AccountsDB) {
let storage = accounts.storage.read().unwrap();
let mut slots: Vec<_> = storage.0.keys().cloned().collect();
slots.sort();
info!("{}: count_and status for {} slots:", label, slots.len());
for slot in &slots {
let slot_stores = storage.0.get(slot).unwrap();
let mut ids: Vec<_> = slot_stores.keys().cloned().collect();
ids.sort();
for id in &ids {
let entry = slot_stores.get(id).unwrap();
info!(
" slot: {} id: {} count_and_status: {:?}",
slot,
id,
*entry.count_and_status.read().unwrap()
);
}
}
}
#[test]
fn test_accounts_db_serialize1() {
solana_logger::setup();
@ -2849,7 +2852,7 @@ pub mod tests {
accounts.bank_hashes.read().unwrap().get(&latest_slot)
);
print_count_and_status("daccounts", &daccounts);
daccounts.print_count_and_status("daccounts");
// Don't check the first 35 accounts which have not been modified on slot 0
check_accounts(&daccounts, &pubkeys[35..], 0, 65, 37);
@ -2884,7 +2887,7 @@ pub mod tests {
fn reconstruct_accounts_db_via_serialization(accounts: &AccountsDB, slot: Slot) -> AccountsDB {
let daccounts =
crate::serde_snapshot::reconstruct_accounts_db_via_serialization(accounts, slot);
print_count_and_status("daccounts", &daccounts);
daccounts.print_count_and_status("daccounts");
daccounts
}
@ -2930,11 +2933,11 @@ pub mod tests {
current_slot += 1;
accounts.add_root(current_slot);
print_accounts("pre_purge", &accounts);
accounts.print_accounts_stats("pre_purge");
accounts.clean_accounts();
print_accounts("post_purge", &accounts);
accounts.print_accounts_stats("post_purge");
// Make sure the index is not touched
assert_eq!(
@ -2990,7 +2993,7 @@ pub mod tests {
accounts.set_hash(current_slot, current_slot - 1);
accounts.add_root(current_slot);
print_accounts("pre_purge", &accounts);
accounts.print_accounts_stats("pre_purge");
let ancestors = linear_ancestors(current_slot);
info!("ancestors: {:?}", ancestors);
@ -3004,7 +3007,7 @@ pub mod tests {
hash
);
print_accounts("post_purge", &accounts);
accounts.print_accounts_stats("post_purge");
// Make sure the index is for pubkey cleared
assert!(accounts
@ -3066,14 +3069,14 @@ pub mod tests {
assert_load_account(&accounts, current_slot, pubkey, zero_lamport);
print_accounts("accounts", &accounts);
accounts.print_accounts_stats("accounts");
accounts.clean_accounts();
print_accounts("accounts_post_purge", &accounts);
accounts.print_accounts_stats("accounts_post_purge");
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
print_accounts("reconstructed", &accounts);
accounts.print_accounts_stats("reconstructed");
assert_load_account(&accounts, current_slot, pubkey, zero_lamport);
}
@ -3120,12 +3123,12 @@ pub mod tests {
accounts.store(current_slot, &[(&dummy_pubkey, &dummy_account)]);
accounts.add_root(current_slot);
print_accounts("pre_f", &accounts);
accounts.print_accounts_stats("pre_f");
accounts.update_accounts_hash(4, &HashMap::default());
let accounts = f(accounts, current_slot);
print_accounts("post_f", &accounts);
accounts.print_accounts_stats("post_f");
assert_load_account(&accounts, current_slot, pubkey, some_lamport);
assert_load_account(&accounts, current_slot, purged_pubkey1, 0);
@ -3149,7 +3152,7 @@ pub mod tests {
solana_logger::setup();
with_chained_zero_lamport_accounts(|accounts, current_slot| {
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
print_accounts("after_reconstruct", &accounts);
accounts.print_accounts_stats("after_reconstruct");
accounts.clean_accounts();
reconstruct_accounts_db_via_serialization(&accounts, current_slot)
});
@ -3739,11 +3742,11 @@ pub mod tests {
accounts.store(current_slot, &[(&dummy_pubkey, &dummy_account)]);
accounts.add_root(current_slot);
print_count_and_status("before reconstruct", &accounts);
accounts.print_count_and_status("before reconstruct");
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
print_count_and_status("before purge zero", &accounts);
accounts.print_count_and_status("before purge zero");
accounts.clean_accounts();
print_count_and_status("after purge zero", &accounts);
accounts.print_count_and_status("after purge zero");
assert_load_account(&accounts, current_slot, pubkey, old_lamport);
assert_load_account(&accounts, current_slot, purged_pubkey1, 0);

4
runtime/src/bank.rs
View File

@ -2692,6 +2692,10 @@ impl Bank {
self.rc.accounts.accounts_db.shrink_all_slots();
}
pub fn print_accounts_stats(&self) {
self.rc.accounts.accounts_db.print_accounts_stats("");
}
pub fn process_stale_slot_with_budget(
&self,
mut consumed_budget: usize,