From 115bf2613dfa7f81b3e8be05c3a61937baab9659 Mon Sep 17 00:00:00 2001 From: Sunny Gleason Date: Thu, 12 Dec 2019 18:54:50 -0500 Subject: [PATCH] feat: add analyze-storage command to ledger-tool (#7165) --- Cargo.lock | 7 +++ ledger-tool/Cargo.toml | 1 + ledger-tool/src/main.rs | 106 +++++++++++++++++++++++++++++++++++++ ledger/src/blocktree.rs | 4 ++ ledger/src/blocktree_db.rs | 4 ++ ledger/src/lib.rs | 2 +- 6 files changed, 123 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index ee2d48703..5597022cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1309,6 +1309,11 @@ name = "hex_fmt" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "histogram" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "hmac" version = "0.7.1" @@ -3617,6 +3622,7 @@ dependencies = [ "assert_cmd 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "histogram 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5624,6 +5630,7 @@ dependencies = [ "checksum hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "961de220ec9a91af2e1e5bd80d02109155695e516771762381ef8581317066e0" "checksum hex-literal-impl 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "06095d08c7c05760f11a071b3e1d4c5b723761c01bd8d7201c30a9536668a612" "checksum hex_fmt 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" +"checksum histogram 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)" = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669" "checksum hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" "checksum http 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "372bcb56f939e449117fb0869c2e8fd8753a8223d92a172c6e808cf123a5b6e4" "checksum http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index 36b869d13..f9a72ffc6 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -11,6 +11,7 @@ homepage = "https://solana.com/" [dependencies] bincode = "1.2.1" clap = "2.33.0" +histogram = "*" serde = "1.0.103" serde_derive = "1.0.103" serde_json = "1.0.44" diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index c42450b22..6d4aa503f 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1,10 +1,15 @@ use clap::{ crate_description, crate_name, value_t, value_t_or_exit, values_t_or_exit, App, Arg, SubCommand, }; +use histogram; +use serde_json::json; +use solana_ledger::blocktree_db::Database; use solana_ledger::{ bank_forks::{BankForks, SnapshotConfig}, bank_forks_utils, blocktree::Blocktree, + blocktree_db, + blocktree_db::Column, blocktree_processor, rooted_slot_iterator::RootedSlotIterator, }; @@ -389,6 +394,94 @@ fn graph_forks( dot.join("\n") } +fn analyze_column( + db: &Database, + name: &str, + key_size: usize, +) -> Result<(), String> { + let mut key_tot: u64 = 0; + let mut val_hist = histogram::Histogram::new(); + let mut val_tot: u64 = 0; + let mut row_hist = histogram::Histogram::new(); + let a = key_size as u64; + for (_x, y) in db.iter::(blocktree_db::IteratorMode::Start).unwrap() { + let b = y.len() as u64; + key_tot += a; + val_hist.increment(b).unwrap(); + val_tot += b; + row_hist.increment(a + b).unwrap(); + } + + let json_result = if val_hist.entries() > 0 { + json!({ + "column":name, + "entries":val_hist.entries(), + "key_stats":{ + "max":a, + "total_bytes":key_tot, + }, + "val_stats":{ + "p50":val_hist.percentile(50.0).unwrap(), + "p90":val_hist.percentile(90.0).unwrap(), + "p99":val_hist.percentile(99.0).unwrap(), + "p999":val_hist.percentile(99.9).unwrap(), + "min":val_hist.minimum().unwrap(), + "max":val_hist.maximum().unwrap(), + "stddev":val_hist.stddev().unwrap(), + "total_bytes":val_tot, + }, + "row_stats":{ + "p50":row_hist.percentile(50.0).unwrap(), + "p90":row_hist.percentile(90.0).unwrap(), + "p99":row_hist.percentile(99.0).unwrap(), + "p999":row_hist.percentile(99.9).unwrap(), + "min":row_hist.minimum().unwrap(), + "max":row_hist.maximum().unwrap(), + "stddev":row_hist.stddev().unwrap(), + "total_bytes":key_tot + val_tot, + }, + }) + } else { + json!({ + "column":name, + "entries":val_hist.entries(), + "key_stats":{ + "max":a, + "total_bytes":0, + }, + "val_stats":{ + "total_bytes":0, + }, + "row_stats":{ + "total_bytes":0, + }, + }) + }; + + println!("{}", serde_json::to_string_pretty(&json_result).unwrap()); + + Ok(()) +} + +fn analyze_storage(blocktree: &Database) -> Result<(), String> { + use blocktree_db::columns::*; + analyze_column::(blocktree, "SlotMeta", SlotMeta::key_size())?; + analyze_column::(blocktree, "Orphans", Orphans::key_size())?; + analyze_column::(blocktree, "DeadSlots", DeadSlots::key_size())?; + analyze_column::(blocktree, "ErasureMeta", ErasureMeta::key_size())?; + analyze_column::(blocktree, "Root", Root::key_size())?; + analyze_column::(blocktree, "Index", Index::key_size())?; + analyze_column::(blocktree, "ShredData", ShredData::key_size())?; + analyze_column::(blocktree, "ShredCode", ShredCode::key_size())?; + analyze_column::( + blocktree, + "TransactionStatus", + TransactionStatus::key_size(), + )?; + + Ok(()) +} + #[allow(clippy::cognitive_complexity)] fn main() { const DEFAULT_ROOT_COUNT: &str = "1"; @@ -527,6 +620,10 @@ fn main() { .help("Number of roots in the output"), ) ) + .subcommand( + SubCommand::with_name("analyze-storage") + .about("Output statistics in JSON format about all column families in the ledger rocksDB") + ) .get_matches(); let ledger_path = PathBuf::from(value_t_or_exit!(matches, "ledger", String)); @@ -744,6 +841,15 @@ fn main() { exit(1); } }, + ("analyze-storage", _) => match analyze_storage(&blocktree.db()) { + Ok(()) => { + println!("Ok."); + } + Err(err) => { + eprintln!("Unable to read the Ledger: {:?}", err); + exit(1); + } + }, ("", _) => { eprintln!("{}", matches.usage()); exit(1); diff --git a/ledger/src/blocktree.rs b/ledger/src/blocktree.rs index f4ed93bb3..60dc00858 100644 --- a/ledger/src/blocktree.rs +++ b/ledger/src/blocktree.rs @@ -148,6 +148,10 @@ impl BlocktreeInsertionMetrics { } impl Blocktree { + pub fn db(self) -> Arc { + self.db + } + /// Opens a Ledger in directory, provides "infinite" window of shreds pub fn open(ledger_path: &Path) -> Result { fs::create_dir_all(&ledger_path)?; diff --git a/ledger/src/blocktree_db.rs b/ledger/src/blocktree_db.rs index 6c1a0ff3b..61c5ad22c 100644 --- a/ledger/src/blocktree_db.rs +++ b/ledger/src/blocktree_db.rs @@ -228,6 +228,10 @@ pub trait Column { const NAME: &'static str; type Index; + fn key_size() -> usize { + std::mem::size_of::() + } + fn key(index: Self::Index) -> Vec; fn index(key: &[u8]) -> Self::Index; fn slot(index: Self::Index) -> Slot; diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index 85af3d9b5..731d15633 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -3,7 +3,7 @@ pub mod bank_forks_utils; pub mod block_error; #[macro_use] pub mod blocktree; -mod blocktree_db; +pub mod blocktree_db; mod blocktree_meta; pub mod blocktree_processor; pub mod entry;