From 53a579bed1139a5c69e9ab79b680c7800d6a0288 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Tue, 18 Oct 2022 14:03:13 -0600 Subject: [PATCH] ledger-tool: stream output of `accounts` subcommand --- ledger-tool/src/main.rs | 114 ++++++++++++++++++++++++---------------- runtime/src/accounts.rs | 15 +++++- runtime/src/bank.rs | 9 ++++ 3 files changed, 91 insertions(+), 47 deletions(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 50c375a25c..ffad9a5a30 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -11,7 +11,10 @@ use { itertools::Itertools, log::*, regex::Regex, - serde::Serialize, + serde::{ + ser::{SerializeSeq, Serializer}, + Serialize, + }, serde_json::json, solana_account_decoder::{UiAccount, UiAccountData, UiAccountEncoding}, solana_clap_utils::{ @@ -20,6 +23,7 @@ use { is_parsable, is_pow2, is_pubkey, is_pubkey_or_keypair, is_slot, is_valid_percentage, }, }, + solana_cli_output::{CliAccount, CliAccountNewConfig, OutputFormat}, solana_core::{ system_monitor_service::SystemMonitorService, validator::move_and_async_delete_path, }, @@ -39,6 +43,7 @@ use { }, solana_measure::{measure, measure::Measure}, solana_runtime::{ + accounts::Accounts, accounts_background_service::{ AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService, PrunedBanksRequestHandler, SnapshotRequestHandler, @@ -46,7 +51,7 @@ use { accounts_db::{AccountsDbConfig, FillerAccountsConfig}, accounts_index::{AccountsIndexConfig, IndexLimitMb, ScanConfig}, accounts_update_notifier_interface::AccountsUpdateNotifier, - bank::{Bank, RewardCalculationEvent}, + bank::{Bank, RewardCalculationEvent, TotalAccountsStats}, bank_forks::BankForks, cost_model::CostModel, cost_tracker::CostTracker, @@ -3377,52 +3382,69 @@ fn main() { }); let bank = bank_forks.read().unwrap().working_bank(); - let mut measure = Measure::start("getting accounts"); - let accounts: BTreeMap<_, _> = bank - .get_all_accounts_with_modified_slots() - .unwrap() - .into_iter() - .filter(|(pubkey, _account, _slot)| { - include_sysvars || !solana_sdk::sysvar::is_sysvar_id(pubkey) - }) - .map(|(pubkey, account, slot)| (pubkey, (account, slot))) - .collect(); - measure.stop(); - info!("{}", measure); - - let mut measure = Measure::start("calculating total accounts stats"); - let total_accounts_stats = bank.calculate_total_accounts_stats( - accounts - .iter() - .map(|(pubkey, (account, _slot))| (pubkey, account)), - ); - measure.stop(); - info!("{}", measure); - - let print_account_contents = !arg_matches.is_present("no_account_contents"); - if print_account_contents { - let print_account_data = !arg_matches.is_present("no_account_data"); - let print_encoding_format = match arg_matches.value_of("encoding") { - Some("jsonParsed") => UiAccountEncoding::JsonParsed, - Some("base64") => UiAccountEncoding::Base64, - Some("base64+zstd") => UiAccountEncoding::Base64Zstd, - _ => UiAccountEncoding::Base64, + let mut serializer = serde_json::Serializer::new(stdout()); + let (summarize, mut json_serializer) = + match OutputFormat::from_matches(arg_matches, "output_format", false) { + OutputFormat::Json | OutputFormat::JsonCompact => { + (false, Some(serializer.serialize_seq(None).unwrap())) + } + _ => (true, None), }; - let mut measure = Measure::start("printing account contents"); - for (pubkey, (account, slot)) in accounts.into_iter() { - output_account( - &pubkey, - &account, - Some(slot), - print_account_data, - print_encoding_format, - ); - } - measure.stop(); - info!("{}", measure); - } + let mut total_accounts_stats = TotalAccountsStats::default(); + let rent_collector = bank.rent_collector(); + let print_account_contents = !arg_matches.is_present("no_account_contents"); + let print_account_data = !arg_matches.is_present("no_account_data"); + let data_encoding = match arg_matches.value_of("encoding") { + Some("jsonParsed") => UiAccountEncoding::JsonParsed, + Some("base64") => UiAccountEncoding::Base64, + Some("base64+zstd") => UiAccountEncoding::Base64Zstd, + _ => UiAccountEncoding::Base64, + }; + let cli_account_new_config = CliAccountNewConfig { + data_encoding, + ..CliAccountNewConfig::default() + }; + let scan_func = |some_account_tuple: Option<(&Pubkey, AccountSharedData, Slot)>| { + if let Some((pubkey, account, slot)) = some_account_tuple + .filter(|(_, account, _)| Accounts::is_loadable(account.lamports())) + { + if !include_sysvars && solana_sdk::sysvar::is_sysvar_id(pubkey) { + return; + } - println!("{:#?}", total_accounts_stats); + total_accounts_stats.accumulate_account(pubkey, &account, rent_collector); + + if print_account_contents { + if let Some(json_serializer) = json_serializer.as_mut() { + let cli_account = CliAccount::new_with_config( + pubkey, + &account, + &cli_account_new_config, + ); + json_serializer.serialize_element(&cli_account).unwrap(); + } else { + output_account( + pubkey, + &account, + Some(slot), + print_account_data, + data_encoding, + ); + } + } + } + }; + let mut measure = Measure::start("scanning accounts"); + bank.scan_all_accounts_with_modified_slots(scan_func) + .unwrap(); + measure.stop(); + info!("{}", measure); + if let Some(json_serializer) = json_serializer { + json_serializer.end().unwrap(); + } + if summarize { + println!("\n{:#?}", total_accounts_stats); + } } ("capitalization", Some(arg_matches)) => { let halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok(); diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index f48604ccdb..b73af9b52e 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -935,7 +935,7 @@ impl Accounts { } } - fn is_loadable(lamports: u64) -> bool { + pub fn is_loadable(lamports: u64) -> bool { // Don't ever load zero lamport accounts into runtime because // the existence of zero-lamport accounts are never deterministic!! lamports > 0 @@ -1113,6 +1113,19 @@ impl Accounts { .map(|_| collector) } + pub fn scan_all( + &self, + ancestors: &Ancestors, + bank_id: BankId, + scan_func: F, + ) -> ScanResult<()> + where + F: FnMut(Option<(&Pubkey, AccountSharedData, Slot)>), + { + self.accounts_db + .scan_accounts(ancestors, bank_id, scan_func, &ScanConfig::default()) + } + pub fn hold_range_in_memory( &self, range: &R, diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 9939f61855..0d953c3e9f 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -6589,6 +6589,15 @@ impl Bank { self.rc.accounts.load_all(&self.ancestors, self.bank_id) } + pub fn scan_all_accounts_with_modified_slots(&self, scan_func: F) -> ScanResult<()> + where + F: FnMut(Option<(&Pubkey, AccountSharedData, Slot)>), + { + self.rc + .accounts + .scan_all(&self.ancestors, self.bank_id, scan_func) + } + pub fn get_program_accounts_modified_since_parent( &self, program_id: &Pubkey,