add stats of dirty_slots for cleaning during hash calc
This commit is contained in:
parent
407717eb2a
commit
43e3545c26
|
@ -6783,7 +6783,14 @@ impl AccountsDb {
|
||||||
|
|
||||||
/// storages are sorted by slot and have range info.
|
/// storages are sorted by slot and have range info.
|
||||||
/// add all stores older than slots_per_epoch to dirty_stores so clean visits these slots
|
/// add all stores older than slots_per_epoch to dirty_stores so clean visits these slots
|
||||||
fn mark_old_slots_as_dirty(&self, storages: &SortedStorages, slots_per_epoch: Slot) {
|
fn mark_old_slots_as_dirty(
|
||||||
|
&self,
|
||||||
|
storages: &SortedStorages,
|
||||||
|
slots_per_epoch: Slot,
|
||||||
|
mut stats: &mut crate::accounts_hash::HashStats,
|
||||||
|
) {
|
||||||
|
let mut mark_time = Measure::start("mark_time");
|
||||||
|
let mut num_dirty_slots: usize = 0;
|
||||||
let max = storages.max_slot_inclusive();
|
let max = storages.max_slot_inclusive();
|
||||||
let acceptable_straggler_slot_count = 100; // do nothing special for these old stores which will likely get cleaned up shortly
|
let acceptable_straggler_slot_count = 100; // do nothing special for these old stores which will likely get cleaned up shortly
|
||||||
let sub = slots_per_epoch + acceptable_straggler_slot_count;
|
let sub = slots_per_epoch + acceptable_straggler_slot_count;
|
||||||
|
@ -6797,10 +6804,14 @@ impl AccountsDb {
|
||||||
// If we included them here then ALL accounts in ALL ancient append vecs will be visited by clean each time.
|
// If we included them here then ALL accounts in ALL ancient append vecs will be visited by clean each time.
|
||||||
self.dirty_stores
|
self.dirty_stores
|
||||||
.insert((slot, store.append_vec_id()), store.clone());
|
.insert((slot, store.append_vec_id()), store.clone());
|
||||||
|
num_dirty_slots += 1;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
mark_time.stop();
|
||||||
|
stats.mark_time_us = mark_time.as_us();
|
||||||
|
stats.num_dirty_slots = num_dirty_slots;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn calculate_accounts_hash_helper(
|
pub(crate) fn calculate_accounts_hash_helper(
|
||||||
|
@ -7029,7 +7040,6 @@ impl AccountsDb {
|
||||||
mut stats: HashStats,
|
mut stats: HashStats,
|
||||||
) -> Result<(Hash, u64), BankHashVerificationError> {
|
) -> Result<(Hash, u64), BankHashVerificationError> {
|
||||||
let _guard = self.active_stats.activate(ActiveStatItem::Hash);
|
let _guard = self.active_stats.activate(ActiveStatItem::Hash);
|
||||||
let mut total_time = Measure::start("total_time");
|
|
||||||
stats.oldest_root = storages.range().start;
|
stats.oldest_root = storages.range().start;
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
|
@ -7037,13 +7047,11 @@ impl AccountsDb {
|
||||||
"cannot accurately capture all data for debugging if accounts cache is being used"
|
"cannot accurately capture all data for debugging if accounts cache is being used"
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut mark_time = Measure::start("mark_time");
|
self.mark_old_slots_as_dirty(storages, config.epoch_schedule.slots_per_epoch, &mut stats);
|
||||||
self.mark_old_slots_as_dirty(storages, config.epoch_schedule.slots_per_epoch);
|
|
||||||
mark_time.stop();
|
|
||||||
|
|
||||||
let (num_hash_scan_passes, bins_per_pass) = Self::bins_per_pass(self.num_hash_scan_passes);
|
let (num_hash_scan_passes, bins_per_pass) = Self::bins_per_pass(self.num_hash_scan_passes);
|
||||||
let use_bg_thread_pool = config.use_bg_thread_pool;
|
let use_bg_thread_pool = config.use_bg_thread_pool;
|
||||||
let mut scan_and_hash = move || {
|
let mut scan_and_hash = || {
|
||||||
let mut previous_pass = PreviousPass::default();
|
let mut previous_pass = PreviousPass::default();
|
||||||
let mut final_result = (Hash::default(), 0);
|
let mut final_result = (Hash::default(), 0);
|
||||||
|
|
||||||
|
@ -7092,25 +7100,16 @@ impl AccountsDb {
|
||||||
Ok(final_result)
|
Ok(final_result)
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut calc_time = Measure::start("calc_time");
|
|
||||||
let result = if use_bg_thread_pool {
|
let result = if use_bg_thread_pool {
|
||||||
self.thread_pool_clean.install(scan_and_hash)
|
self.thread_pool_clean.install(scan_and_hash)
|
||||||
} else {
|
} else {
|
||||||
scan_and_hash()
|
scan_and_hash()
|
||||||
};
|
};
|
||||||
calc_time.stop();
|
|
||||||
self.assert_safe_squashing_accounts_hash(
|
self.assert_safe_squashing_accounts_hash(
|
||||||
storages.max_slot_inclusive(),
|
storages.max_slot_inclusive(),
|
||||||
config.epoch_schedule,
|
config.epoch_schedule,
|
||||||
);
|
);
|
||||||
total_time.stop();
|
stats.log();
|
||||||
|
|
||||||
datapoint_info!(
|
|
||||||
"accounts_hash_with_cache",
|
|
||||||
("total_time_ms", total_time.as_ms(), i64),
|
|
||||||
("mark_time_ms", mark_time.as_ms(), i64),
|
|
||||||
("calc_time_ms", calc_time.as_ms(), i64),
|
|
||||||
);
|
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -71,6 +71,7 @@ pub type StorageSizeQuartileStats = [usize; 6];
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
pub struct HashStats {
|
pub struct HashStats {
|
||||||
|
pub mark_time_us: u64,
|
||||||
pub scan_time_total_us: u64,
|
pub scan_time_total_us: u64,
|
||||||
pub zeros_time_total_us: u64,
|
pub zeros_time_total_us: u64,
|
||||||
pub hash_time_total_us: u64,
|
pub hash_time_total_us: u64,
|
||||||
|
@ -80,6 +81,7 @@ pub struct HashStats {
|
||||||
pub unreduced_entries: usize,
|
pub unreduced_entries: usize,
|
||||||
pub num_snapshot_storage: usize,
|
pub num_snapshot_storage: usize,
|
||||||
pub num_slots: usize,
|
pub num_slots: usize,
|
||||||
|
pub num_dirty_slots: usize,
|
||||||
pub collect_snapshots_us: u64,
|
pub collect_snapshots_us: u64,
|
||||||
pub storage_sort_us: u64,
|
pub storage_sort_us: u64,
|
||||||
pub min_bin_size: usize,
|
pub min_bin_size: usize,
|
||||||
|
@ -133,7 +135,7 @@ impl HashStats {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
fn log(&mut self) {
|
pub fn log(&mut self) {
|
||||||
let total_time_us = self.scan_time_total_us
|
let total_time_us = self.scan_time_total_us
|
||||||
+ self.zeros_time_total_us
|
+ self.zeros_time_total_us
|
||||||
+ self.hash_time_total_us
|
+ self.hash_time_total_us
|
||||||
|
@ -141,6 +143,7 @@ impl HashStats {
|
||||||
+ self.storage_sort_us;
|
+ self.storage_sort_us;
|
||||||
datapoint_info!(
|
datapoint_info!(
|
||||||
"calculate_accounts_hash_without_index",
|
"calculate_accounts_hash_without_index",
|
||||||
|
("mark_time_us", self.mark_time_us, i64),
|
||||||
("accounts_scan", self.scan_time_total_us, i64),
|
("accounts_scan", self.scan_time_total_us, i64),
|
||||||
("eliminate_zeros", self.zeros_time_total_us, i64),
|
("eliminate_zeros", self.zeros_time_total_us, i64),
|
||||||
("hash", self.hash_time_total_us, i64),
|
("hash", self.hash_time_total_us, i64),
|
||||||
|
@ -160,6 +163,7 @@ impl HashStats {
|
||||||
i64
|
i64
|
||||||
),
|
),
|
||||||
("num_slots", self.num_slots as i64, i64),
|
("num_slots", self.num_slots as i64, i64),
|
||||||
|
("num_dirty_slots", self.num_dirty_slots as i64, i64),
|
||||||
("min_bin_size", self.min_bin_size as i64, i64),
|
("min_bin_size", self.min_bin_size as i64, i64),
|
||||||
("max_bin_size", self.max_bin_size as i64, i64),
|
("max_bin_size", self.max_bin_size as i64, i64),
|
||||||
(
|
(
|
||||||
|
@ -994,10 +998,6 @@ impl AccountsHash {
|
||||||
} else {
|
} else {
|
||||||
Hash::default()
|
Hash::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
if is_last_pass {
|
|
||||||
stats.log();
|
|
||||||
}
|
|
||||||
(hash, total_lamports, next_pass)
|
(hash, total_lamports, next_pass)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue