From a21f536b3af1c7368b6266a8169622e88c4f4bba Mon Sep 17 00:00:00 2001
From: "Jeff Washington (jwash)" <jeff.washington@solana.com>
Date: Tue, 15 Nov 2022 09:31:24 -0800
Subject: [PATCH] add scan statistics in hash calc for ancient append vecs
 (#28816)

---
 runtime/src/accounts_db.rs   | 24 +++++++++++++++++++++---
 runtime/src/accounts_hash.rs | 18 ++++++++++++++++++
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs
index 4f9e45a76a..996a9ce5f9 100644
--- a/runtime/src/accounts_db.rs
+++ b/runtime/src/accounts_db.rs
@@ -4443,8 +4443,9 @@ impl AccountsDb {
             // randomly shrink ancient slots
             // this exercises the ancient shrink code more often
             let written_bytes = storage.written_bytes();
+            let mut alive_ratio = 0;
             let is_candidate = if written_bytes > 0 {
-                let alive_ratio = (storage.alive_bytes() as u64) * 100 / written_bytes;
+                alive_ratio = (storage.alive_bytes() as u64) * 100 / written_bytes;
                 alive_ratio < 90
             } else {
                 false
@@ -4452,7 +4453,10 @@ impl AccountsDb {
             if is_candidate || (can_randomly_shrink && thread_rng().gen_range(0, 100) == 0) {
                 // we are a candidate for shrink, so either append us to the previous append vec
                 // or recreate us as a new append vec and eliminate the dead accounts
-                info!("ancient_append_vec: shrinking full ancient: {}", slot);
+                info!(
+                    "ancient_append_vec: shrinking full ancient: {}, random: {}, alive_ratio: {}",
+                    slot, !is_candidate, alive_ratio
+                );
                 if !is_candidate {
                     self.shrink_ancient_stats
                         .random_shrink
@@ -7298,8 +7302,22 @@ impl AccountsDb {
 
                 for (slot, sub_storages) in snapshot_storages.iter_range(&range_this_chunk) {
                     scanner.set_slot(slot);
-                    if let Some(sub_storages) = sub_storages {
+
+                    let mut ancient = false;
+                    let (_, scan) = measure!(if let Some(sub_storages) = sub_storages {
+                        if let Some(storage) = sub_storages.first() {
+                            ancient = is_ancient(&storage.accounts);
+                        }
                         Self::scan_multiple_account_storages_one_slot(sub_storages, &mut scanner);
+                    });
+                    if ancient {
+                        stats
+                            .sum_ancient_scans_us
+                            .fetch_add(scan.as_us(), Ordering::Relaxed);
+                        stats.count_ancient_scans.fetch_add(1, Ordering::Relaxed);
+                        stats
+                            .longest_ancient_scan_us
+                            .fetch_max(scan.as_us(), Ordering::Relaxed);
                     }
                 }
                 let r = scanner.scanning_complete();
diff --git a/runtime/src/accounts_hash.rs b/runtime/src/accounts_hash.rs
index 9adf96b215..301f7a8a3f 100644
--- a/runtime/src/accounts_hash.rs
+++ b/runtime/src/accounts_hash.rs
@@ -104,6 +104,9 @@ pub struct HashStats {
     pub append_vec_sizes_older_than_epoch: AtomicUsize,
     /// # ancient append vecs encountered
     pub ancient_append_vecs: AtomicUsize,
+    pub longest_ancient_scan_us: AtomicU64,
+    pub sum_ancient_scans_us: AtomicU64,
+    pub count_ancient_scans: AtomicU64,
 }
 impl HashStats {
     pub fn calc_storage_size_quartiles(&mut self, storages: &SnapshotStorages) {
@@ -231,6 +234,21 @@ impl HashStats {
                 self.ancient_append_vecs.load(Ordering::Relaxed) as i64,
                 i64
             ),
+            (
+                "longest_ancient_scan_us",
+                self.longest_ancient_scan_us.load(Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "sum_ancient_scans_us",
+                self.sum_ancient_scans_us.load(Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "count_ancient_scans",
+                self.count_ancient_scans.load(Ordering::Relaxed) as i64,
+                i64
+            ),
             (
                 "append_vec_sizes_older_than_epoch",
                 self.append_vec_sizes_older_than_epoch