add metrics for # scans active and scan distance (#26395)

runtime/src/accounts_db.rs

@@ -2837,6 +2837,18 @@ impl AccountsDb {
                 self.accounts_index.roots_removed.swap(0, Ordering::Relaxed) as i64,
                 i64
             ),
+            (
+                "active_scans",
+                self.accounts_index.active_scans.swap(0, Ordering::Relaxed) as i64,
+                i64
+            ),
+            (
+                "max_distance_to_min_scan_slot",
+                self.accounts_index
+                    .max_distance_to_min_scan_slot
+                    .swap(0, Ordering::Relaxed),
+                i64
+            ),
             ("next_store_id", self.next_id.load(Ordering::Relaxed), i64),
         );
     }

runtime/src/accounts_index.rs

@@ -688,6 +688,10 @@ pub struct AccountsIndex<T: IndexValue> {
     pub roots_added: AtomicUsize,
     /// # roots removed since last check
     pub roots_removed: AtomicUsize,
+    /// # scans active currently
+    pub active_scans: AtomicUsize,
+    /// # of slots between latest max and latest scan
+    pub max_distance_to_min_scan_slot: AtomicU64,
 }
 
 impl<T: IndexValue> AccountsIndex<T> {
@@ -719,6 +723,8 @@ impl<T: IndexValue> AccountsIndex<T> {
             scan_results_limit_bytes,
             roots_added: AtomicUsize::default(),
             roots_removed: AtomicUsize::default(),
+            active_scans: AtomicUsize::default(),
+            max_distance_to_min_scan_slot: AtomicU64::default(),
         }
     }
 
@@ -755,6 +761,10 @@ impl<T: IndexValue> AccountsIndex<T> {
         self.storage.storage.is_disk_index_enabled()
     }
 
+    fn min_ongoing_scan_root_from_btree(ongoing_scan_roots: &BTreeMap<Slot, u64>) -> Option<Slot> {
+        ongoing_scan_roots.keys().next().cloned()
+    }
+
     fn do_checked_scan_accounts<F, R>(
         &self,
         metric_name: &'static str,
@@ -778,6 +788,7 @@ impl<T: IndexValue> AccountsIndex<T> {
             }
         }
 
+        self.active_scans.fetch_add(1, Ordering::Relaxed);
         let max_root = {
             let mut w_ongoing_scan_roots = self
                 // This lock is also grabbed by clean_accounts(), so clean
@@ -792,6 +803,15 @@ impl<T: IndexValue> AccountsIndex<T> {
             // make sure inverse doesn't happen to avoid
             // deadlock
             let max_root_inclusive = self.max_root_inclusive();
+            if let Some(min_ongoing_scan_root) =
+                Self::min_ongoing_scan_root_from_btree(&w_ongoing_scan_roots)
+            {
+                if min_ongoing_scan_root < max_root_inclusive {
+                    let current = max_root_inclusive - min_ongoing_scan_root;
+                    self.max_distance_to_min_scan_slot
+                        .fetch_max(current, Ordering::Relaxed);
+                }
+            }
             *w_ongoing_scan_roots.entry(max_root_inclusive).or_default() += 1;
 
             max_root_inclusive
@@ -950,6 +970,7 @@ impl<T: IndexValue> AccountsIndex<T> {
         }
 
         {
+            self.active_scans.fetch_sub(1, Ordering::Relaxed);
             let mut ongoing_scan_roots = self.ongoing_scan_roots.write().unwrap();
             let count = ongoing_scan_roots.get_mut(&max_root).unwrap();
             *count -= 1;
@@ -1252,12 +1273,7 @@ impl<T: IndexValue> AccountsIndex<T> {
     }
 
     pub fn min_ongoing_scan_root(&self) -> Option<Slot> {
-        self.ongoing_scan_roots
-            .read()
-            .unwrap()
-            .keys()
-            .next()
-            .cloned()
+        Self::min_ongoing_scan_root_from_btree(&self.ongoing_scan_roots.read().unwrap())
     }
 
     // Given a SlotSlice `L`, a list of ancestors and a maximum slot, find the latest element