feat: add analyze-storage command to ledger-tool (#7165)

Cargo.lock

@@ -1309,6 +1309,11 @@ name = "hex_fmt"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "histogram"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "hmac"
 version = "0.7.1"
@@ -3617,6 +3622,7 @@ dependencies = [
  "assert_cmd 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "histogram 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -5624,6 +5630,7 @@ dependencies = [
 "checksum hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "961de220ec9a91af2e1e5bd80d02109155695e516771762381ef8581317066e0"
 "checksum hex-literal-impl 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "06095d08c7c05760f11a071b3e1d4c5b723761c01bd8d7201c30a9536668a612"
 "checksum hex_fmt 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f"
+"checksum histogram 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)" = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669"
 "checksum hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695"
 "checksum http 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "372bcb56f939e449117fb0869c2e8fd8753a8223d92a172c6e808cf123a5b6e4"
 "checksum http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"

ledger-tool/Cargo.toml

@@ -11,6 +11,7 @@ homepage = "https://solana.com/"
 [dependencies]
 bincode = "1.2.1"
 clap = "2.33.0"
+histogram = "*"
 serde = "1.0.103"
 serde_derive = "1.0.103"
 serde_json = "1.0.44"

ledger-tool/src/main.rs

@@ -1,10 +1,15 @@
 use clap::{
     crate_description, crate_name, value_t, value_t_or_exit, values_t_or_exit, App, Arg, SubCommand,
 };
+use histogram;
+use serde_json::json;
+use solana_ledger::blocktree_db::Database;
 use solana_ledger::{
     bank_forks::{BankForks, SnapshotConfig},
     bank_forks_utils,
     blocktree::Blocktree,
+    blocktree_db,
+    blocktree_db::Column,
     blocktree_processor,
     rooted_slot_iterator::RootedSlotIterator,
 };
@@ -389,6 +394,94 @@ fn graph_forks(
     dot.join("\n")
 }
 
+fn analyze_column<T: solana_ledger::blocktree_db::Column>(
+    db: &Database,
+    name: &str,
+    key_size: usize,
+) -> Result<(), String> {
+    let mut key_tot: u64 = 0;
+    let mut val_hist = histogram::Histogram::new();
+    let mut val_tot: u64 = 0;
+    let mut row_hist = histogram::Histogram::new();
+    let a = key_size as u64;
+    for (_x, y) in db.iter::<T>(blocktree_db::IteratorMode::Start).unwrap() {
+        let b = y.len() as u64;
+        key_tot += a;
+        val_hist.increment(b).unwrap();
+        val_tot += b;
+        row_hist.increment(a + b).unwrap();
+    }
+
+    let json_result = if val_hist.entries() > 0 {
+        json!({
+            "column":name,
+            "entries":val_hist.entries(),
+            "key_stats":{
+                "max":a,
+                "total_bytes":key_tot,
+            },
+            "val_stats":{
+                "p50":val_hist.percentile(50.0).unwrap(),
+                "p90":val_hist.percentile(90.0).unwrap(),
+                "p99":val_hist.percentile(99.0).unwrap(),
+                "p999":val_hist.percentile(99.9).unwrap(),
+                "min":val_hist.minimum().unwrap(),
+                "max":val_hist.maximum().unwrap(),
+                "stddev":val_hist.stddev().unwrap(),
+                "total_bytes":val_tot,
+            },
+            "row_stats":{
+                "p50":row_hist.percentile(50.0).unwrap(),
+                "p90":row_hist.percentile(90.0).unwrap(),
+                "p99":row_hist.percentile(99.0).unwrap(),
+                "p999":row_hist.percentile(99.9).unwrap(),
+                "min":row_hist.minimum().unwrap(),
+                "max":row_hist.maximum().unwrap(),
+                "stddev":row_hist.stddev().unwrap(),
+                "total_bytes":key_tot + val_tot,
+            },
+        })
+    } else {
+        json!({
+        "column":name,
+        "entries":val_hist.entries(),
+        "key_stats":{
+            "max":a,
+            "total_bytes":0,
+        },
+        "val_stats":{
+            "total_bytes":0,
+        },
+        "row_stats":{
+            "total_bytes":0,
+        },
+        })
+    };
+
+    println!("{}", serde_json::to_string_pretty(&json_result).unwrap());
+
+    Ok(())
+}
+
+fn analyze_storage(blocktree: &Database) -> Result<(), String> {
+    use blocktree_db::columns::*;
+    analyze_column::<SlotMeta>(blocktree, "SlotMeta", SlotMeta::key_size())?;
+    analyze_column::<Orphans>(blocktree, "Orphans", Orphans::key_size())?;
+    analyze_column::<DeadSlots>(blocktree, "DeadSlots", DeadSlots::key_size())?;
+    analyze_column::<ErasureMeta>(blocktree, "ErasureMeta", ErasureMeta::key_size())?;
+    analyze_column::<Root>(blocktree, "Root", Root::key_size())?;
+    analyze_column::<Index>(blocktree, "Index", Index::key_size())?;
+    analyze_column::<ShredData>(blocktree, "ShredData", ShredData::key_size())?;
+    analyze_column::<ShredCode>(blocktree, "ShredCode", ShredCode::key_size())?;
+    analyze_column::<TransactionStatus>(
+        blocktree,
+        "TransactionStatus",
+        TransactionStatus::key_size(),
+    )?;
+
+    Ok(())
+}
+
 #[allow(clippy::cognitive_complexity)]
 fn main() {
     const DEFAULT_ROOT_COUNT: &str = "1";
@@ -527,6 +620,10 @@ fn main() {
                     .help("Number of roots in the output"),
             )
         )
+        .subcommand(
+            SubCommand::with_name("analyze-storage")
+                .about("Output statistics in JSON format about all column families in the ledger rocksDB")
+        )
         .get_matches();
 
     let ledger_path = PathBuf::from(value_t_or_exit!(matches, "ledger", String));
@@ -744,6 +841,15 @@ fn main() {
                 exit(1);
             }
         },
+        ("analyze-storage", _) => match analyze_storage(&blocktree.db()) {
+            Ok(()) => {
+                println!("Ok.");
+            }
+            Err(err) => {
+                eprintln!("Unable to read the Ledger: {:?}", err);
+                exit(1);
+            }
+        },
         ("", _) => {
             eprintln!("{}", matches.usage());
             exit(1);

ledger/src/blocktree.rs

@@ -148,6 +148,10 @@ impl BlocktreeInsertionMetrics {
 }
 
 impl Blocktree {
+    pub fn db(self) -> Arc<Database> {
+        self.db
+    }
+
     /// Opens a Ledger in directory, provides "infinite" window of shreds
     pub fn open(ledger_path: &Path) -> Result<Blocktree> {
         fs::create_dir_all(&ledger_path)?;

ledger/src/blocktree_db.rs

@@ -228,6 +228,10 @@ pub trait Column {
     const NAME: &'static str;
     type Index;
 
+    fn key_size() -> usize {
+        std::mem::size_of::<Self::Index>()
+    }
+
     fn key(index: Self::Index) -> Vec<u8>;
     fn index(key: &[u8]) -> Self::Index;
     fn slot(index: Self::Index) -> Slot;

ledger/src/lib.rs

@@ -3,7 +3,7 @@ pub mod bank_forks_utils;
 pub mod block_error;
 #[macro_use]
 pub mod blocktree;
-mod blocktree_db;
+pub mod blocktree_db;
 mod blocktree_meta;
 pub mod blocktree_processor;
 pub mod entry;