blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ledger-tool: Refactor accounts subcommand output code (#34915) 

The accounts command currently dumps every single account in the
AccountsDb. This is obviously a lot of output, so a previous change
streamed the accounts instead of collecting and dumping at the end.

The streaming approach is much more performant, but the implementation
is non-trivial. This change
- Moves the accounts output code to output.rs
- Refactor the logic to several objects that implment the functionality
- Adjust the json output to also include the summary

This change lays the groundwork for cleanly adding several more flags
that will allow for querying different subsets of accounts.
This commit is contained in:
steviez 2024-01-26 00:55:05 -06:00 committed by GitHub
parent 89fd6acb8f
commit 3add40fc07
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 217 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

146
ledger-tool/src/main.rs
View File

@ -1,19 +1,24 @@
#![allow(clippy::arithmetic_side_effects)]
use {
crate::{args::*, bigtable::*, blockstore::*, ledger_path::*, ledger_utils::*, program::*},
crate::{
args::*,
bigtable::*,
blockstore::*,
ledger_path::*,
ledger_utils::*,
output::{output_account, AccountsOutputConfig, AccountsOutputStreamer},
program::*,
},
clap::{
crate_description, crate_name, value_t, value_t_or_exit, values_t_or_exit, App,
AppSettings, Arg, ArgMatches, SubCommand,
},
dashmap::DashMap,
log::*,
serde::{
ser::{SerializeSeq, Serializer},
Serialize,
},
solana_account_decoder::{UiAccount, UiAccountData, UiAccountEncoding},
serde::Serialize,
solana_account_decoder::UiAccountEncoding,
solana_accounts_db::{
accounts::Accounts, accounts_db::CalcAccountsHashDataSource, accounts_index::ScanConfig,
accounts_db::CalcAccountsHashDataSource, accounts_index::ScanConfig,
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
},
solana_clap_utils::{
@ -25,7 +30,7 @@ use {
validate_maximum_incremental_snapshot_archives_to_retain,
},
},
solana_cli_output::{CliAccount, CliAccountNewConfig, OutputFormat},
solana_cli_output::OutputFormat,
solana_core::{
system_monitor_service::{SystemMonitorService, SystemMonitorStatsReportConfig},
validator::BlockVerificationMethod,
@ -38,7 +43,7 @@ use {
},
solana_measure::{measure, measure::Measure},
solana_runtime::{
bank::{bank_hash_details, Bank, RewardCalculationEvent, TotalAccountsStats},
bank::{bank_hash_details, Bank, RewardCalculationEvent},
bank_forks::BankForks,
snapshot_archive_info::SnapshotArchiveInfoGetter,
snapshot_bank_utils,
@ -73,7 +78,7 @@ use {
collections::{HashMap, HashSet},
ffi::OsStr,
fs::File,
io::{self, stdout, Write},
io::{self, Write},
num::NonZeroUsize,
path::{Path, PathBuf},
process::{exit, Command, Stdio},
@ -102,44 +107,6 @@ fn parse_encoding_format(matches: &ArgMatches<'_>) -> UiAccountEncoding {
}
}
fn output_account(
pubkey: &Pubkey,
account: &AccountSharedData,
modified_slot: Option<Slot>,
print_account_data: bool,
encoding: UiAccountEncoding,
) {
println!("{pubkey}:");
println!(" balance: {} SOL", lamports_to_sol(account.lamports()));
println!(" owner: '{}'", account.owner());
println!(" executable: {}", account.executable());
if let Some(slot) = modified_slot {
println!(" slot: {slot}");
}
println!(" rent_epoch: {}", account.rent_epoch());
println!(" data_len: {}", account.data().len());
if print_account_data {
let account_data = UiAccount::encode(pubkey, account, encoding, None, None).data;
match account_data {
UiAccountData::Binary(data, data_encoding) => {
println!(" data: '{data}'");
println!(
" encoding: {}",
serde_json::to_string(&data_encoding).unwrap()
);
}
UiAccountData::Json(account_data) => {
println!(
" data: '{}'",
serde_json::to_string(&account_data).unwrap()
);
println!(" encoding: \"jsonParsed\"");
}
UiAccountData::LegacyBinary(_) => {}
};
}
}
fn render_dot(dot: String, output_file: &str, output_format: &str) -> io::Result<()> {
let mut child = Command::new("dot")
.arg(format!("-T{output_format}"))
@ -2192,7 +2159,6 @@ fn main() {
("accounts", Some(arg_matches)) => {
let process_options = parse_process_options(&ledger_path, arg_matches);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let include_sysvars = arg_matches.is_present("include_sysvars");
let blockstore = open_blockstore(
&ledger_path,
arg_matches,
@ -2206,70 +2172,30 @@ fn main() {
snapshot_archive_path,
incremental_snapshot_archive_path,
);
let bank = bank_forks.read().unwrap().working_bank();
let mut serializer = serde_json::Serializer::new(stdout());
let (summarize, mut json_serializer) =
match OutputFormat::from_matches(arg_matches, "output_format", false) {
OutputFormat::Json | OutputFormat::JsonCompact => {
(false, Some(serializer.serialize_seq(None).unwrap()))
}
_ => (true, None),
};
let mut total_accounts_stats = TotalAccountsStats::default();
let rent_collector = bank.rent_collector();
let print_account_contents = !arg_matches.is_present("no_account_contents");
let print_account_data = !arg_matches.is_present("no_account_data");
let data_encoding = parse_encoding_format(arg_matches);
let cli_account_new_config = CliAccountNewConfig {
data_encoding,
..CliAccountNewConfig::default()
let include_sysvars = arg_matches.is_present("include_sysvars");
let include_account_contents = !arg_matches.is_present("no_account_contents");
let include_account_data = !arg_matches.is_present("no_account_data");
let account_data_encoding = parse_encoding_format(arg_matches);
let config = AccountsOutputConfig {
include_sysvars,
include_account_contents,
include_account_data,
account_data_encoding,
};
let scan_func =
|some_account_tuple: Option<(&Pubkey, AccountSharedData, Slot)>| {
if let Some((pubkey, account, slot)) = some_account_tuple
.filter(|(_, account, _)| Accounts::is_loadable(account.lamports()))
{
if !include_sysvars && solana_sdk::sysvar::is_sysvar_id(pubkey) {
return;
}
let output_format =
OutputFormat::from_matches(arg_matches, "output_format", false);
total_accounts_stats.accumulate_account(
pubkey,
&account,
rent_collector,
);
if print_account_contents {
if let Some(json_serializer) = json_serializer.as_mut() {
let cli_account = CliAccount::new_with_config(
pubkey,
&account,
&cli_account_new_config,
);
json_serializer.serialize_element(&cli_account).unwrap();
} else {
output_account(
pubkey,
&account,
Some(slot),
print_account_data,
data_encoding,
);
}
}
}
};
let mut measure = Measure::start("scanning accounts");
bank.scan_all_accounts(scan_func).unwrap();
measure.stop();
info!("{}", measure);
if let Some(json_serializer) = json_serializer {
json_serializer.end().unwrap();
}
if summarize {
println!("\n{total_accounts_stats:#?}");
}
let accounts_streamer =
AccountsOutputStreamer::new(bank, output_format, config);
let (_, scan_time) = measure!(
accounts_streamer
.output()
.map_err(|err| error!("Error while outputting accounts: {err}")),
"accounts scan"
);
info!("{scan_time}");
}
("capitalization", Some(arg_matches)) => {
let process_options = parse_process_options(&ledger_path, arg_matches);

181
ledger-tool/src/output.rs
View File

@ -1,11 +1,20 @@
use {
crate::ledger_utils::get_program_ids,
chrono::{Local, TimeZone},
serde::{Deserialize, Serialize},
solana_cli_output::{display::writeln_transaction, OutputFormat, QuietDisplay, VerboseDisplay},
serde::{
ser::{Impossible, SerializeSeq, SerializeStruct, Serializer},
Deserialize, Serialize,
},
solana_account_decoder::{UiAccount, UiAccountData, UiAccountEncoding},
solana_cli_output::{
display::writeln_transaction, CliAccount, CliAccountNewConfig, OutputFormat, QuietDisplay,
VerboseDisplay,
},
solana_entry::entry::Entry,
solana_ledger::blockstore::Blockstore,
solana_runtime::bank::{Bank, TotalAccountsStats},
solana_sdk::{
account::{AccountSharedData, ReadableAccount},
clock::{Slot, UnixTimestamp},
hash::Hash,
native_token::lamports_to_sol,
@ -15,10 +24,13 @@ use {
EncodedConfirmedBlock, EncodedTransactionWithStatusMeta, EntrySummary, Rewards,
},
std::{
cell::RefCell,
collections::HashMap,
fmt::{self, Display, Formatter},
io::{stdout, Write},
rc::Rc,
result::Result,
sync::Arc,
},
};
@ -548,3 +560,168 @@ pub fn output_sorted_program_ids(program_ids: HashMap<Pubkey, u64>) {
println!("{:<44}: {}", program_id.to_string(), count);
}
}
/// A type to facilitate streaming account information to an output destination
///
/// This type scans every account, so streaming is preferred over the simpler
/// approach of accumulating all the accounts into a Vec and printing or
/// serializing the Vec directly.
pub struct AccountsOutputStreamer {
account_scanner: AccountsScanner,
total_accounts_stats: Rc<RefCell<TotalAccountsStats>>,
output_format: OutputFormat,
}
pub struct AccountsOutputConfig {
pub include_sysvars: bool,
pub include_account_contents: bool,
pub include_account_data: bool,
pub account_data_encoding: UiAccountEncoding,
}
impl AccountsOutputStreamer {
pub fn new(bank: Arc<Bank>, output_format: OutputFormat, config: AccountsOutputConfig) -> Self {
let total_accounts_stats = Rc::new(RefCell::new(TotalAccountsStats::default()));
let account_scanner = AccountsScanner {
bank,
total_accounts_stats: total_accounts_stats.clone(),
config,
};
Self {
account_scanner,
total_accounts_stats,
output_format,
}
}
pub fn output(&self) -> Result<(), String> {
match self.output_format {
OutputFormat::Json | OutputFormat::JsonCompact => {
let mut serializer = serde_json::Serializer::new(stdout());
let mut struct_serializer = serializer
.serialize_struct("accountInfo", 2)
.map_err(|err| format!("unable to start serialization: {err}"))?;
struct_serializer
.serialize_field("accounts", &self.account_scanner)
.map_err(|err| format!("unable to serialize accounts scanner: {err}"))?;
struct_serializer
.serialize_field("summary", &*self.total_accounts_stats.borrow())
.map_err(|err| format!("unable to serialize accounts summary: {err}"))?;
SerializeStruct::end(struct_serializer)
.map_err(|err| format!("unable to end serialization: {err}"))
}
_ => {
// The compiler needs a placeholder type to satisfy the generic
// SerializeSeq trait on AccountScanner::output(). The type
// doesn't really matter since we're passing None, so just use
// serde::ser::Impossible as it already implements SerializeSeq
self.account_scanner
.output::<Impossible<(), serde_json::Error>>(&mut None);
println!("\n{:#?}", self.total_accounts_stats.borrow());
Ok(())
}
}
}
}
struct AccountsScanner {
bank: Arc<Bank>,
total_accounts_stats: Rc<RefCell<TotalAccountsStats>>,
config: AccountsOutputConfig,
}
impl AccountsScanner {
/// Returns true if this account should be included in the output
fn should_process_account(&self, account: &AccountSharedData, pubkey: &Pubkey) -> bool {
solana_accounts_db::accounts::Accounts::is_loadable(account.lamports())
&& (self.config.include_sysvars || !solana_sdk::sysvar::is_sysvar_id(pubkey))
}
pub fn output<S>(&self, seq_serializer: &mut Option<S>)
where
S: SerializeSeq,
{
let mut total_accounts_stats = self.total_accounts_stats.borrow_mut();
let rent_collector = self.bank.rent_collector();
let cli_account_new_config = CliAccountNewConfig {
data_encoding: self.config.account_data_encoding,
..CliAccountNewConfig::default()
};
let scan_func = |account_tuple: Option<(&Pubkey, AccountSharedData, Slot)>| {
if let Some((pubkey, account, slot)) = account_tuple
.filter(|(pubkey, account, _)| self.should_process_account(account, pubkey))
{
total_accounts_stats.accumulate_account(pubkey, &account, rent_collector);
if self.config.include_account_contents {
if let Some(serializer) = seq_serializer {
let cli_account =
CliAccount::new_with_config(pubkey, &account, &cli_account_new_config);
serializer.serialize_element(&cli_account).unwrap();
} else {
output_account(
pubkey,
&account,
Some(slot),
self.config.include_account_data,
self.config.account_data_encoding,
);
}
}
}
};
self.bank.scan_all_accounts(scan_func).unwrap();
}
}
impl Serialize for AccountsScanner {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut seq_serializer = Some(serializer.serialize_seq(None)?);
self.output(&mut seq_serializer);
seq_serializer.unwrap().end()
}
}
pub fn output_account(
pubkey: &Pubkey,
account: &AccountSharedData,
modified_slot: Option<Slot>,
print_account_data: bool,
encoding: UiAccountEncoding,
) {
println!("{pubkey}:");
println!(" balance: {} SOL", lamports_to_sol(account.lamports()));
println!(" owner: '{}'", account.owner());
println!(" executable: {}", account.executable());
if let Some(slot) = modified_slot {
println!(" slot: {slot}");
}
println!(" rent_epoch: {}", account.rent_epoch());
println!(" data_len: {}", account.data().len());
if print_account_data {
let account_data = UiAccount::encode(pubkey, account, encoding, None, None).data;
match account_data {
UiAccountData::Binary(data, data_encoding) => {
println!(" data: '{data}'");
println!(
" encoding: {}",
serde_json::to_string(&data_encoding).unwrap()
);
}
UiAccountData::Json(account_data) => {
println!(
" data: '{}'",
serde_json::to_string(&account_data).unwrap()
);
println!(" encoding: \"jsonParsed\"");
}
UiAccountData::LegacyBinary(_) => {}
};
}
}

3
runtime/src/bank.rs
View File

@ -72,6 +72,7 @@ use {
slice::ParallelSlice,
ThreadPool, ThreadPoolBuilder,
},
serde::Serialize,
solana_accounts_db::{
account_overrides::AccountOverrides,
accounts::{
@ -8485,7 +8486,7 @@ impl CollectRentInPartitionInfo {
}
/// Struct to collect stats when scanning all accounts in `get_total_accounts_stats()`
#[derive(Debug, Default, Copy, Clone)]
#[derive(Debug, Default, Copy, Clone, Serialize)]
pub struct TotalAccountsStats {
/// Total number of accounts
pub num_accounts: usize,