blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

speedup populate_and_retrieve_duplicate_keys_from_startup (#33013)

This commit is contained in:
Jeff Washington (jwash) 2023-08-28 10:20:29 -07:00 committed by GitHub
parent e036a0375d
commit 35bd2df0a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
accounts-db/src/accounts_db.rs
View File

@ -9261,7 +9261,7 @@ impl AccountsDb {
.sum(); .sum();
let mut index_flush_us = 0; let mut index_flush_us = 0;
let mut total_duplicate_slot_keys = 0; let total_duplicate_slot_keys = AtomicU64::default();
let mut populate_duplicate_keys_us = 0; let mut populate_duplicate_keys_us = 0;
if pass == 0 { if pass == 0 {
// tell accounts index we are done adding the initial accounts at startup // tell accounts index we are done adding the initial accounts at startup
@ -9273,20 +9273,19 @@ impl AccountsDb {
populate_duplicate_keys_us = measure_us!({ populate_duplicate_keys_us = measure_us!({
// this has to happen before visit_duplicate_pubkeys_during_startup below // this has to happen before visit_duplicate_pubkeys_during_startup below
// get duplicate keys from acct idx. We have to wait until we've finished flushing. // get duplicate keys from acct idx. We have to wait until we've finished flushing.
for (slot, key) in self self.accounts_index
.accounts_index .populate_and_retrieve_duplicate_keys_from_startup(|slot_keys| {
.populate_and_retrieve_duplicate_keys_from_startup() total_duplicate_slot_keys
.into_iter() .fetch_add(slot_keys.len() as u64, Ordering::Relaxed);
.flatten() for (slot, key) in slot_keys {
{ match self.uncleaned_pubkeys.entry(slot) {
total_duplicate_slot_keys += 1; Occupied(mut occupied) => occupied.get_mut().push(key),
match self.uncleaned_pubkeys.entry(slot) { Vacant(vacant) => {
Occupied(mut occupied) => occupied.get_mut().push(key), vacant.insert(vec![key]);
Vacant(vacant) => { }
vacant.insert(vec![key]); }
} }
} });
}
}) })
.1; .1;
} }
@ -9302,7 +9301,7 @@ impl AccountsDb {
total_items, total_items,
rent_paying, rent_paying,
amount_to_top_off_rent, amount_to_top_off_rent,
total_duplicate_slot_keys, total_duplicate_slot_keys: total_duplicate_slot_keys.load(Ordering::Relaxed),
populate_duplicate_keys_us, populate_duplicate_keys_us,
total_including_duplicates: total_including_duplicates.load(Ordering::Relaxed), total_including_duplicates: total_including_duplicates.load(Ordering::Relaxed),
storage_size_accounts_map_us: storage_info_timings.storage_size_accounts_map_us, storage_size_accounts_map_us: storage_info_timings.storage_size_accounts_map_us,

9
accounts-db/src/accounts_index.rs
View File

@ -1679,17 +1679,18 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> AccountsIndex<T, U> {
(dirty_pubkeys, insertion_time.load(Ordering::Relaxed)) (dirty_pubkeys, insertion_time.load(Ordering::Relaxed))
} }
/// return Vec<Vec<>> because the internal vecs are already allocated per bin /// use Vec<> because the internal vecs are already allocated per bin
pub(crate) fn populate_and_retrieve_duplicate_keys_from_startup( pub(crate) fn populate_and_retrieve_duplicate_keys_from_startup(
&self, &self,
) -> Vec<Vec<(Slot, Pubkey)>> { f: impl Fn(Vec<(Slot, Pubkey)>) + Sync + Send,
) {
(0..self.bins()) (0..self.bins())
.into_par_iter() .into_par_iter()
.map(|pubkey_bin| { .map(|pubkey_bin| {
let r_account_maps = &self.account_maps[pubkey_bin]; let r_account_maps = &self.account_maps[pubkey_bin];
r_account_maps.populate_and_retrieve_duplicate_keys_from_startup() r_account_maps.populate_and_retrieve_duplicate_keys_from_startup()
}) })
.collect() .for_each(f);
} }
/// Updates the given pubkey at the given slot with the new account information. /// Updates the given pubkey at the given slot with the new account information.
@ -2497,7 +2498,7 @@ pub mod tests {
index.set_startup(Startup::Normal); index.set_startup(Startup::Normal);
} }
assert!(gc.is_empty()); assert!(gc.is_empty());
index.populate_and_retrieve_duplicate_keys_from_startup(); index.populate_and_retrieve_duplicate_keys_from_startup(|_slot_keys| {});
for lock in &[false, true] { for lock in &[false, true] {
let read_lock = if *lock { let read_lock = if *lock {