From e3eb002f663814b717b1f626875c4e9b0615423f Mon Sep 17 00:00:00 2001
From: "Jeff Washington (jwash)" <wash678@gmail.com>
Date: Thu, 24 Mar 2022 09:40:35 -0500
Subject: [PATCH] Log storage size stats at hash calc (#23843)

---
 runtime/src/accounts_db.rs   |  3 +-
 runtime/src/accounts_hash.rs | 70 +++++++++++++++++++++++++++++++++++-
 2 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs
index e66f7ac3d7..e623fb59bf 100644
--- a/runtime/src/accounts_db.rs
+++ b/runtime/src/accounts_db.rs
@@ -5508,11 +5508,12 @@ impl AccountsDb {
             self.mark_old_slots_as_dirty(&storages, slots_per_epoch);
             sort_time.stop();
 
-            let timings = HashStats {
+            let mut timings = HashStats {
                 collect_snapshots_us: collect_time.as_us(),
                 storage_sort_us: sort_time.as_us(),
                 ..HashStats::default()
             };
+            timings.calc_storage_size_quartiles(&combined_maps);
 
             self.calculate_accounts_hash_without_index(&mut CalcAccountsHashConfig {
                 storages: &storages,
diff --git a/runtime/src/accounts_hash.rs b/runtime/src/accounts_hash.rs
index e82fb1f6eb..9d3569f5ea 100644
--- a/runtime/src/accounts_hash.rs
+++ b/runtime/src/accounts_hash.rs
@@ -1,6 +1,8 @@
 use {
     crate::{
-        accounts_cache::AccountsCache, accounts_db::AccountInfoAccountsIndex, ancestors::Ancestors,
+        accounts_cache::AccountsCache,
+        accounts_db::{AccountInfoAccountsIndex, SnapshotStorages},
+        ancestors::Ancestors,
         sorted_storages::SortedStorages,
     },
     log::*,
@@ -43,6 +45,9 @@ pub struct CalcAccountsHashConfig<'a> {
     */
 }
 
+// smallest, 3 quartiles, largest, average
+pub type StorageSizeQuartileStats = [usize; 6];
+
 #[derive(Debug, Default)]
 pub struct HashStats {
     pub scan_time_total_us: u64,
@@ -58,8 +63,41 @@ pub struct HashStats {
     pub storage_sort_us: u64,
     pub min_bin_size: usize,
     pub max_bin_size: usize,
+    pub storage_size_quartiles: StorageSizeQuartileStats,
 }
 impl HashStats {
+    pub fn calc_storage_size_quartiles(&mut self, storages: &SnapshotStorages) {
+        let mut sum = 0;
+        let mut sizes = storages
+            .iter()
+            .flat_map(|storages| {
+                let result = storages
+                    .iter()
+                    .map(|storage| {
+                        let cap = storage.accounts.capacity() as usize;
+                        sum += cap;
+                        cap
+                    })
+                    .collect::<Vec<_>>();
+                result
+            })
+            .collect::<Vec<_>>();
+        sizes.sort_unstable();
+        let len = sizes.len();
+        self.storage_size_quartiles = if len == 0 {
+            StorageSizeQuartileStats::default()
+        } else {
+            [
+                *sizes.first().unwrap(),
+                sizes[len / 4],
+                sizes[len * 2 / 4],
+                sizes[len * 3 / 4],
+                *sizes.last().unwrap(),
+                sum / len,
+            ]
+        };
+    }
+
     fn log(&mut self) {
         let total_time_us = self.scan_time_total_us
             + self.zeros_time_total_us
@@ -89,6 +127,36 @@ impl HashStats {
             ("num_slots", self.num_slots as i64, i64),
             ("min_bin_size", self.min_bin_size as i64, i64),
             ("max_bin_size", self.max_bin_size as i64, i64),
+            (
+                "storage_size_min",
+                self.storage_size_quartiles[0] as i64,
+                i64
+            ),
+            (
+                "storage_size_quartile_1",
+                self.storage_size_quartiles[1] as i64,
+                i64
+            ),
+            (
+                "storage_size_quartile_2",
+                self.storage_size_quartiles[2] as i64,
+                i64
+            ),
+            (
+                "storage_size_quartile_3",
+                self.storage_size_quartiles[3] as i64,
+                i64
+            ),
+            (
+                "storage_size_max",
+                self.storage_size_quartiles[4] as i64,
+                i64
+            ),
+            (
+                "storage_size_avg",
+                self.storage_size_quartiles[5] as i64,
+                i64
+            ),
             ("total", total_time_us as i64, i64),
         );
     }