stat for time spent copying generate index contents (#33187)

* stat for time spent copying generate index contents

* rework to move stat to generate_index

* fix fmt

accounts-db/src/accounts_db.rs

@@ -60,6 +60,7 @@ use {
         cache_hash_data::{CacheHashData, CacheHashDataFileReference},
         contains::Contains,
         epoch_accounts_hash::EpochAccountsHashManager,
+        in_mem_accounts_index::StartupStats,
         partitioned_rewards::{PartitionedEpochRewardsConfig, TestPartitionedEpochRewards},
         pubkey_bins::PubkeyBinCalculator24,
         read_only_accounts_cache::ReadOnlyAccountsCache,
@@ -636,7 +637,7 @@ struct StorageSizeAndCount {
 type StorageSizeAndCountMap = DashMap<AppendVecId, StorageSizeAndCount>;
 
 impl GenerateIndexTimings {
-    pub fn report(&self) {
+    pub fn report(&self, startup_stats: &StartupStats) {
         datapoint_info!(
             "generate_index",
             ("overall_us", self.total_time_us, i64),
@@ -695,6 +696,11 @@ impl GenerateIndexTimings {
             ),
             ("total_slots", self.total_slots, i64),
             ("slots_to_clean", self.slots_to_clean, i64),
+            (
+                "copy_data_us",
+                startup_stats.copy_data_us.swap(0, Ordering::Relaxed),
+                i64
+            ),
         );
     }
 }
@@ -9369,7 +9375,7 @@ impl AccountsDb {
             }
             total_time.stop();
             timings.total_time_us = total_time.as_us();
-            timings.report();
+            timings.report(self.accounts_index.get_startup_stats());
         }
 
         self.accounts_index.log_secondary_indexes();

accounts-db/src/accounts_index.rs

@@ -5,7 +5,7 @@ use {
         ancestors::Ancestors,
         bucket_map_holder::{Age, BucketMapHolder},
         contains::Contains,
-        in_mem_accounts_index::{InMemAccountsIndex, InsertNewEntryResults},
+        in_mem_accounts_index::{InMemAccountsIndex, InsertNewEntryResults, StartupStats},
         inline_spl_token::{self, GenericTokenAccount},
         inline_spl_token_2022,
         pubkey_bins::PubkeyBinCalculator24,
@@ -1336,6 +1336,11 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndex<T, U> {
         iter.hold_range_in_memory(range, start_holding, thread_pool);
     }
 
+    /// get stats related to startup
+    pub(crate) fn get_startup_stats(&self) -> &StartupStats {
+        &self.storage.storage.startup_stats
+    }
+
     pub fn set_startup(&self, value: Startup) {
         self.storage.set_startup(value);
     }

accounts-db/src/bucket_map_holder.rs

@@ -2,7 +2,7 @@ use {
     crate::{
         accounts_index::{AccountsIndexConfig, DiskIndexValue, IndexLimitMb, IndexValue},
         bucket_map_holder_stats::BucketMapHolderStats,
-        in_mem_accounts_index::InMemAccountsIndex,
+        in_mem_accounts_index::{InMemAccountsIndex, StartupStats},
         waitable_condvar::WaitableCondvar,
     },
     solana_bucket_map::bucket_map::{BucketMap, BucketMapConfig},
@@ -68,6 +68,8 @@ pub struct BucketMapHolder<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>>
     /// Note startup is an optimization and is not required for correctness.
     startup: AtomicBool,
     _phantom: PhantomData<T>,
+
+    pub(crate) startup_stats: Arc<StartupStats>,
 }
 
 impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> Debug for BucketMapHolder<T, U> {
@@ -259,6 +261,7 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> BucketMapHolder<T, U>
             mem_budget_mb,
             threads,
             _phantom: PhantomData,
+            startup_stats: Arc::default(),
         }
     }
 

accounts-db/src/in_mem_accounts_index.rs

@@ -27,6 +27,11 @@ type CacheRangesHeld = RwLock<Vec<RangeInclusive<Pubkey>>>;
 
 type InMemMap<T> = HashMap<Pubkey, AccountMapEntry<T>>;
 
+#[derive(Debug, Default)]
+pub struct StartupStats {
+    pub copy_data_us: AtomicU64,
+}
+
 #[derive(Debug)]
 pub struct PossibleEvictions<T: IndexValue> {
     /// vec per age in the future, up to size 'ages_to_stay_in_cache'
@@ -116,6 +121,9 @@ pub struct InMemAccountsIndex<T: IndexValue, U: DiskIndexValue + From<T> + Into<
     /// Higher numbers mean we flush less buckets/s
     /// Lower numbers mean we flush more buckets/s
     num_ages_to_distribute_flushes: Age,
+
+    /// stats related to starting up
+    pub(crate) startup_stats: Arc<StartupStats>,
 }
 
 impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> Debug for InMemAccountsIndex<T, U> {
@@ -182,6 +190,7 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> InMemAccountsIndex<T,
                 thread_rng().gen_range(0..num_ages_to_distribute_flushes),
             ),
             num_ages_to_distribute_flushes,
+            startup_stats: Arc::clone(&storage.startup_stats),
         }
     }
 
@@ -677,11 +686,13 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> InMemAccountsIndex<T,
         assert!(self.bucket.is_some());
 
         let mut insert = self.startup_info.insert.lock().unwrap();
-        // todo: memcpy the new slice into our vector already
+        let m = Measure::start("copy");
-        // todo: avoid reallocs and just allocate another vec instead of likely resizing this one over and over
         items
             .into_iter()
             .for_each(|(k, (slot, v))| insert.push((k, (slot, v.into()))));
+        self.startup_stats
+            .copy_data_us
+            .fetch_add(m.end_as_us(), Ordering::Relaxed);
     }
 
     pub fn insert_new_entry_if_missing_with_lock(