add metrics to handle_snapshot_requests (#17937)

core/tests/snapshots.rs

@@ -222,7 +222,7 @@ mod tests {
                 // set_root should send a snapshot request
                 bank_forks.set_root(bank.slot(), &request_sender, None);
                 bank.update_accounts_hash();
-                snapshot_request_handler.handle_snapshot_requests(false, false, false);
+                snapshot_request_handler.handle_snapshot_requests(false, false, false, 0);
             }
         }
 

runtime/src/accounts_background_service.rs

@@ -89,11 +89,13 @@ impl SnapshotRequestHandler {
         accounts_db_caching_enabled: bool,
         test_hash_calculation: bool,
         use_index_hash_calculation: bool,
+        non_snapshot_time_us: u128,
     ) -> Option<u64> {
         self.snapshot_request_receiver
             .try_iter()
             .last()
             .map(|snapshot_request| {
+                let mut total_time = Measure::start("wallclock time elapsed");
                 let SnapshotRequest {
                     snapshot_root_bank,
                     status_cache_slot_deltas,
@@ -188,6 +190,7 @@ impl SnapshotRequestHandler {
                 let mut purge_old_snapshots_time = Measure::start("purge_old_snapshots_time");
                 snapshot_utils::purge_old_snapshots(&self.snapshot_config.snapshot_path);
                 purge_old_snapshots_time.stop();
+                total_time.stop();
 
                 datapoint_info!(
                     "handle_snapshot_requests-timing",
@@ -205,6 +208,8 @@ impl SnapshotRequestHandler {
                         purge_old_snapshots_time.as_us(),
                         i64
                     ),
+                    ("total_us", total_time.as_us(), i64),
+                    ("non_snapshot_time_us", non_snapshot_time_us, i64),
                 );
                 snapshot_root_bank.block_height()
             })
@@ -251,6 +256,7 @@ impl AbsRequestHandler {
         accounts_db_caching_enabled: bool,
         test_hash_calculation: bool,
         use_index_hash_calculation: bool,
+        non_snapshot_time_us: u128,
     ) -> Option<u64> {
         self.snapshot_request_handler
             .as_ref()
@@ -259,6 +265,7 @@ impl AbsRequestHandler {
                     accounts_db_caching_enabled,
                     test_hash_calculation,
                     use_index_hash_calculation,
+                    non_snapshot_time_us,
                 )
             })
     }
@@ -297,7 +304,9 @@ impl AccountsBackgroundService {
         let mut last_expiration_check_time = Instant::now();
         let t_background = Builder::new()
             .name("solana-bg-accounts".to_string())
-            .spawn(move || loop {
+            .spawn(move || {
+                let mut last_snapshot_end_time = None;
+                loop {
                     if exit.load(Ordering::Relaxed) {
                         break;
                     }
@@ -315,6 +324,12 @@ impl AccountsBackgroundService {
 
                     Self::expire_old_recycle_stores(&bank, &mut last_expiration_check_time);
 
+                    let non_snapshot_time = last_snapshot_end_time
+                        .map(|last_snapshot_end_time: Instant| {
+                            last_snapshot_end_time.elapsed().as_micros()
+                        })
+                        .unwrap_or_default();
+
                     // Check to see if there were any requests for snapshotting banks
                     // < the current root bank `bank` above.
 
@@ -336,7 +351,12 @@ impl AccountsBackgroundService {
                         accounts_db_caching_enabled,
                         test_hash_calculation,
                         use_index_hash_calculation,
+                        non_snapshot_time,
                     );
+                    if snapshot_block_height.is_some() {
+                        last_snapshot_end_time = Some(Instant::now());
+                    }
+
                     if accounts_db_caching_enabled {
                         // Note that the flush will do an internal clean of the
                         // cache up to bank.slot(), so should be safe as long
@@ -378,6 +398,7 @@ impl AccountsBackgroundService {
                         }
                     }
                     sleep(Duration::from_millis(INTERVAL_MS));
+                }
             })
             .unwrap();
         Self { t_background }