blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

metrics for generating index time (#17192) 

* metrics for generating index time

* update metrics to include scan time
This commit is contained in:
Jeff Washington (jwash) 2021-05-13 14:32:19 -05:00 committed by GitHub
parent 3e0c0abb53
commit 3dbc7744ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 99 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
runtime/src/accounts_db.rs
View File

@ -4962,11 +4962,16 @@ impl AccountsDb {
let total_processed_slots_across_all_threads = AtomicU64::new(0); let total_processed_slots_across_all_threads = AtomicU64::new(0);
let outer_slots_len = slots.len(); let outer_slots_len = slots.len();
let chunk_size = (outer_slots_len / 7) + 1; // approximately 400k slots in a snapshot let chunk_size = (outer_slots_len / 7) + 1; // approximately 400k slots in a snapshot
slots.par_chunks(chunk_size).for_each(|slots| { let mut index_time = Measure::start("index");
let scan_time: u64 = slots
.par_chunks(chunk_size)
.map(|slots| {
let mut last_log_update = Instant::now(); let mut last_log_update = Instant::now();
let mut my_last_reported_number_of_processed_slots = 0; let mut my_last_reported_number_of_processed_slots = 0;
let mut was_first = false; let mut was_first = false;
let mut scan_time_sum = 0;
for (index, slot) in slots.iter().enumerate() { for (index, slot) in slots.iter().enumerate() {
let mut scan_time = Measure::start("scan");
let now = Instant::now(); let now = Instant::now();
if now.duration_since(last_log_update).as_secs() >= 2 { if now.duration_since(last_log_update).as_secs() >= 2 {
let my_total_newly_processed_slots_since_last_report = let my_total_newly_processed_slots_since_last_report =
@ -4977,7 +4982,8 @@ impl AccountsDb {
my_total_newly_processed_slots_since_last_report, my_total_newly_processed_slots_since_last_report,
Ordering::Relaxed, Ordering::Relaxed,
); );
was_first = was_first || 0 == previous_total_processed_slots_across_all_threads; was_first =
was_first || 0 == previous_total_processed_slots_across_all_threads;
if was_first { if was_first {
info!( info!(
"generating index: {}/{} slots...", "generating index: {}/{} slots...",
@ -5015,13 +5021,17 @@ impl AccountsDb {
); );
}) })
}); });
scan_time.stop();
scan_time_sum += scan_time.as_us();
// Need to restore indexes even with older write versions which may // Need to restore indexes even with older write versions which may
// be shielding other accounts. When they are then purged, the // be shielding other accounts. When they are then purged, the
// original non-shielded account value will be visible when the account // original non-shielded account value will be visible when the account
// is restored from the append-vec // is restored from the append-vec
if !accounts_map.is_empty() { if !accounts_map.is_empty() {
let mut _reclaims: Vec<(u64, AccountInfo)> = vec![]; let mut _reclaims: Vec<(u64, AccountInfo)> = vec![];
let dirty_keys = accounts_map.iter().map(|(pubkey, _info)| *pubkey).collect(); let dirty_keys =
accounts_map.iter().map(|(pubkey, _info)| *pubkey).collect();
self.uncleaned_pubkeys.insert(*slot, dirty_keys); self.uncleaned_pubkeys.insert(*slot, dirty_keys);
for (pubkey, account_infos) in accounts_map.into_iter() { for (pubkey, account_infos) in accounts_map.into_iter() {
for (_, (store_id, stored_account)) in account_infos.into_iter() { for (_, (store_id, stored_account)) in account_infos.into_iter() {
@ -5044,7 +5054,17 @@ impl AccountsDb {
} }
} }
} }
}); scan_time_sum
})
.sum();
index_time.stop();
datapoint_info!(
"generate_index",
// we cannot accurately measure index insertion time because of many threads and lock contention
("total_us", index_time.as_us(), i64),
("scan_stores_us", scan_time, i64),
);
// Need to add these last, otherwise older updates will be cleaned // Need to add these last, otherwise older updates will be cleaned
for slot in slots { for slot in slots {