speedup populate_and_retrieve_duplicate_keys_from_startup (#33013)
This commit is contained in:
parent
e036a0375d
commit
35bd2df0a6
|
@ -9261,7 +9261,7 @@ impl AccountsDb {
|
||||||
.sum();
|
.sum();
|
||||||
|
|
||||||
let mut index_flush_us = 0;
|
let mut index_flush_us = 0;
|
||||||
let mut total_duplicate_slot_keys = 0;
|
let total_duplicate_slot_keys = AtomicU64::default();
|
||||||
let mut populate_duplicate_keys_us = 0;
|
let mut populate_duplicate_keys_us = 0;
|
||||||
if pass == 0 {
|
if pass == 0 {
|
||||||
// tell accounts index we are done adding the initial accounts at startup
|
// tell accounts index we are done adding the initial accounts at startup
|
||||||
|
@ -9273,13 +9273,11 @@ impl AccountsDb {
|
||||||
populate_duplicate_keys_us = measure_us!({
|
populate_duplicate_keys_us = measure_us!({
|
||||||
// this has to happen before visit_duplicate_pubkeys_during_startup below
|
// this has to happen before visit_duplicate_pubkeys_during_startup below
|
||||||
// get duplicate keys from acct idx. We have to wait until we've finished flushing.
|
// get duplicate keys from acct idx. We have to wait until we've finished flushing.
|
||||||
for (slot, key) in self
|
self.accounts_index
|
||||||
.accounts_index
|
.populate_and_retrieve_duplicate_keys_from_startup(|slot_keys| {
|
||||||
.populate_and_retrieve_duplicate_keys_from_startup()
|
total_duplicate_slot_keys
|
||||||
.into_iter()
|
.fetch_add(slot_keys.len() as u64, Ordering::Relaxed);
|
||||||
.flatten()
|
for (slot, key) in slot_keys {
|
||||||
{
|
|
||||||
total_duplicate_slot_keys += 1;
|
|
||||||
match self.uncleaned_pubkeys.entry(slot) {
|
match self.uncleaned_pubkeys.entry(slot) {
|
||||||
Occupied(mut occupied) => occupied.get_mut().push(key),
|
Occupied(mut occupied) => occupied.get_mut().push(key),
|
||||||
Vacant(vacant) => {
|
Vacant(vacant) => {
|
||||||
|
@ -9287,6 +9285,7 @@ impl AccountsDb {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
});
|
||||||
})
|
})
|
||||||
.1;
|
.1;
|
||||||
}
|
}
|
||||||
|
@ -9302,7 +9301,7 @@ impl AccountsDb {
|
||||||
total_items,
|
total_items,
|
||||||
rent_paying,
|
rent_paying,
|
||||||
amount_to_top_off_rent,
|
amount_to_top_off_rent,
|
||||||
total_duplicate_slot_keys,
|
total_duplicate_slot_keys: total_duplicate_slot_keys.load(Ordering::Relaxed),
|
||||||
populate_duplicate_keys_us,
|
populate_duplicate_keys_us,
|
||||||
total_including_duplicates: total_including_duplicates.load(Ordering::Relaxed),
|
total_including_duplicates: total_including_duplicates.load(Ordering::Relaxed),
|
||||||
storage_size_accounts_map_us: storage_info_timings.storage_size_accounts_map_us,
|
storage_size_accounts_map_us: storage_info_timings.storage_size_accounts_map_us,
|
||||||
|
|
|
@ -1679,17 +1679,18 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndex<T, U> {
|
||||||
(dirty_pubkeys, insertion_time.load(Ordering::Relaxed))
|
(dirty_pubkeys, insertion_time.load(Ordering::Relaxed))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// return Vec<Vec<>> because the internal vecs are already allocated per bin
|
/// use Vec<> because the internal vecs are already allocated per bin
|
||||||
pub(crate) fn populate_and_retrieve_duplicate_keys_from_startup(
|
pub(crate) fn populate_and_retrieve_duplicate_keys_from_startup(
|
||||||
&self,
|
&self,
|
||||||
) -> Vec<Vec<(Slot, Pubkey)>> {
|
f: impl Fn(Vec<(Slot, Pubkey)>) + Sync + Send,
|
||||||
|
) {
|
||||||
(0..self.bins())
|
(0..self.bins())
|
||||||
.into_par_iter()
|
.into_par_iter()
|
||||||
.map(|pubkey_bin| {
|
.map(|pubkey_bin| {
|
||||||
let r_account_maps = &self.account_maps[pubkey_bin];
|
let r_account_maps = &self.account_maps[pubkey_bin];
|
||||||
r_account_maps.populate_and_retrieve_duplicate_keys_from_startup()
|
r_account_maps.populate_and_retrieve_duplicate_keys_from_startup()
|
||||||
})
|
})
|
||||||
.collect()
|
.for_each(f);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Updates the given pubkey at the given slot with the new account information.
|
/// Updates the given pubkey at the given slot with the new account information.
|
||||||
|
@ -2497,7 +2498,7 @@ pub mod tests {
|
||||||
index.set_startup(Startup::Normal);
|
index.set_startup(Startup::Normal);
|
||||||
}
|
}
|
||||||
assert!(gc.is_empty());
|
assert!(gc.is_empty());
|
||||||
index.populate_and_retrieve_duplicate_keys_from_startup();
|
index.populate_and_retrieve_duplicate_keys_from_startup(|_slot_keys| {});
|
||||||
|
|
||||||
for lock in &[false, true] {
|
for lock in &[false, true] {
|
||||||
let read_lock = if *lock {
|
let read_lock = if *lock {
|
||||||
|
|
Loading…
Reference in New Issue