From 742155c21421ee016d7b45a19d7dfec11df5f5da Mon Sep 17 00:00:00 2001
From: "Jeff Washington (jwash)"
 <75863576+jeffwashington@users.noreply.github.com>
Date: Sat, 18 Sep 2021 22:08:58 -0500
Subject: [PATCH] call set_startup and add metrics on generate_index (#20006)

---
 runtime/src/accounts_db.rs            | 15 +++++++++++++++
 runtime/src/accounts_index.rs         |  5 +++++
 runtime/src/accounts_index_storage.rs |  2 +-
 runtime/src/bucket_map_holder.rs      |  7 +++++++
 4 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs
index 71415bdc2e..b7150f5018 100644
--- a/runtime/src/accounts_db.rs
+++ b/runtime/src/accounts_db.rs
@@ -219,6 +219,7 @@ struct GenerateIndexTimings {
     pub storage_size_accounts_map_us: u64,
     pub storage_size_storages_us: u64,
     pub storage_size_accounts_map_flatten_us: u64,
+    pub index_flush_us: u64,
 }
 
 #[derive(Default, Debug, PartialEq)]
@@ -253,6 +254,7 @@ impl GenerateIndexTimings {
                 self.storage_size_accounts_map_flatten_us as i64,
                 i64
             ),
+            ("index_flush_us", self.index_flush_us as i64, i64),
         );
     }
 }
@@ -6408,6 +6410,9 @@ impl AccountsDb {
         // verify checks that all the expected items are in the accounts index and measures how long it takes to look them all up
         let passes = if verify { 2 } else { 1 };
         for pass in 0..passes {
+            if pass == 0 {
+                self.accounts_index.set_startup(true);
+            }
             let storage_info = StorageSizeAndCountMap::default();
             let total_processed_slots_across_all_threads = AtomicU64::new(0);
             let outer_slots_len = slots.len();
@@ -6496,7 +6501,17 @@ impl AccountsDb {
 
             let storage_info_timings = storage_info_timings.into_inner().unwrap();
 
+            let mut index_flush_us = 0;
+            if pass == 0 {
+                // tell accounts index we are done adding the initial accounts at startup
+                let mut m = Measure::start("accounts_index_idle_us");
+                self.accounts_index.set_startup(false);
+                m.stop();
+                index_flush_us = m.as_us();
+            }
+
             let mut timings = GenerateIndexTimings {
+                index_flush_us,
                 scan_time,
                 index_time: index_time.as_us(),
                 insertion_time_us: insertion_time_us.load(Ordering::Relaxed),
diff --git a/runtime/src/accounts_index.rs b/runtime/src/accounts_index.rs
index f20cc061a8..efe060eb88 100644
--- a/runtime/src/accounts_index.rs
+++ b/runtime/src/accounts_index.rs
@@ -1376,6 +1376,11 @@ impl<T: IndexValue> AccountsIndex<T> {
         let iter = self.iter(Some(range), true);
         iter.hold_range_in_memory(range, start_holding);
     }
+
+    pub fn set_startup(&self, value: bool) {
+        self.storage.storage.set_startup(value);
+    }
+
     /// Get an account
     /// The latest account that appears in `ancestors` or `roots` is returned.
     pub(crate) fn get(
diff --git a/runtime/src/accounts_index_storage.rs b/runtime/src/accounts_index_storage.rs
index 31de2ab55d..aeafffe56e 100644
--- a/runtime/src/accounts_index_storage.rs
+++ b/runtime/src/accounts_index_storage.rs
@@ -23,7 +23,7 @@ pub struct AccountsIndexStorage<T: IndexValue> {
     handles: Option<Vec<JoinHandle<()>>>,
 
     // eventually the backing storage
-    storage: Arc<BucketMapHolder<T>>,
+    pub storage: Arc<BucketMapHolder<T>>,
     pub in_mem: Vec<Arc<InMemAccountsIndex<T>>>,
 }
 
diff --git a/runtime/src/bucket_map_holder.rs b/runtime/src/bucket_map_holder.rs
index 791ed80bf8..c532f99e2c 100644
--- a/runtime/src/bucket_map_holder.rs
+++ b/runtime/src/bucket_map_holder.rs
@@ -48,9 +48,16 @@ impl<T: IndexValue> BucketMapHolder<T> {
     }
 
     pub fn set_startup(&self, value: bool) {
+        if !value {
+            self.wait_for_idle();
+        }
         self.startup.store(value, Ordering::Relaxed)
     }
 
+    pub(crate) fn wait_for_idle(&self) {
+        assert!(self.get_startup());
+    }
+
     pub fn current_age(&self) -> Age {
         self.age.load(Ordering::Relaxed)
     }