get_unique_accounts_from_storages takes 1 append vec (#29588)

This commit is contained in:
Jeff Washington (jwash) 2023-01-09 16:53:50 -06:00 committed by GitHub
parent 3234af41a7
commit dda34b208c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 25 additions and 34 deletions

View File

@ -3666,36 +3666,27 @@ impl AccountsDb {
/// get all accounts in all the storages passed in /// get all accounts in all the storages passed in
/// for duplicate pubkeys, the account with the highest write_value is returned /// for duplicate pubkeys, the account with the highest write_value is returned
pub(crate) fn get_unique_accounts_from_storages<'a, I>( pub(crate) fn get_unique_accounts_from_storages<'a>(
&'a self, &self,
stores: I, store: &'a Arc<AccountStorageEntry>,
) -> GetUniqueAccountsResult<'a> ) -> GetUniqueAccountsResult<'a> {
where
I: Iterator<Item = &'a Arc<AccountStorageEntry>>,
{
let mut stored_accounts: HashMap<Pubkey, FoundStoredAccount> = HashMap::new(); let mut stored_accounts: HashMap<Pubkey, FoundStoredAccount> = HashMap::new();
let mut original_bytes = 0; let original_bytes = store.total_bytes();
let mut count = 0; let store_id = store.append_vec_id();
stores.into_iter().for_each(|store| { store.accounts.account_iter().for_each(|account| {
count += 1; let new_entry = FoundStoredAccount { account, store_id };
assert!(count < 2, "there should be a max of 1 append vec per slot"); match stored_accounts.entry(*new_entry.account.pubkey()) {
original_bytes += store.total_bytes(); Entry::Occupied(mut occupied_entry) => {
let store_id = store.append_vec_id(); assert!(
store.accounts.account_iter().for_each(|account| { new_entry.account.meta.write_version_obsolete
let new_entry = FoundStoredAccount { account, store_id }; > occupied_entry.get().account.meta.write_version_obsolete
match stored_accounts.entry(*new_entry.account.pubkey()) { );
Entry::Occupied(mut occupied_entry) => { occupied_entry.insert(new_entry);
assert!(
new_entry.account.meta.write_version_obsolete
> occupied_entry.get().account.meta.write_version_obsolete
);
occupied_entry.insert(new_entry);
}
Entry::Vacant(vacant_entry) => {
vacant_entry.insert(new_entry);
}
} }
}); Entry::Vacant(vacant_entry) => {
vacant_entry.insert(new_entry);
}
}
}); });
// sort by pubkey to keep account index lookups close // sort by pubkey to keep account index lookups close
@ -3722,7 +3713,7 @@ impl AccountsDb {
original_bytes, original_bytes,
}, },
storage_read_elapsed, storage_read_elapsed,
) = measure!(self.get_unique_accounts_from_storages(std::iter::once(store))); ) = measure!(self.get_unique_accounts_from_storages(store));
stats stats
.storage_read_elapsed .storage_read_elapsed
.fetch_add(storage_read_elapsed.as_us(), Ordering::Relaxed); .fetch_add(storage_read_elapsed.as_us(), Ordering::Relaxed);
@ -17399,7 +17390,7 @@ pub mod tests {
let GetUniqueAccountsResult { let GetUniqueAccountsResult {
stored_accounts: mut after_stored_accounts, stored_accounts: mut after_stored_accounts,
.. ..
} = db.get_unique_accounts_from_storages(std::iter::once(&ancient)); } = db.get_unique_accounts_from_storages(&ancient);
assert_eq!( assert_eq!(
after_stored_accounts.len(), after_stored_accounts.len(),
num_normal_slots + 1 - dead_accounts, num_normal_slots + 1 - dead_accounts,
@ -17479,7 +17470,7 @@ pub mod tests {
} }
let storage = db.get_storage_for_slot(starting_slot).unwrap(); let storage = db.get_storage_for_slot(starting_slot).unwrap();
let created_accounts = db.get_unique_accounts_from_storages(std::iter::once(&storage)); let created_accounts = db.get_unique_accounts_from_storages(&storage);
assert_eq!(created_accounts.stored_accounts.len(), 1); assert_eq!(created_accounts.stored_accounts.len(), 1);
if alive { if alive {
@ -17513,7 +17504,7 @@ pub mod tests {
) -> (AccountsDb, Slot) { ) -> (AccountsDb, Slot) {
let (db, slot1) = create_db_with_storages_and_index(alive, num_normal_slots + 1); let (db, slot1) = create_db_with_storages_and_index(alive, num_normal_slots + 1);
let storage = db.get_storage_for_slot(slot1).unwrap(); let storage = db.get_storage_for_slot(slot1).unwrap();
let created_accounts = db.get_unique_accounts_from_storages(std::iter::once(&storage)); let created_accounts = db.get_unique_accounts_from_storages(&storage);
db.combine_ancient_slots(vec![slot1], CAN_RANDOMLY_SHRINK_FALSE); db.combine_ancient_slots(vec![slot1], CAN_RANDOMLY_SHRINK_FALSE);
assert_eq!(1, db.get_storages_for_slot(slot1).unwrap().len()); assert_eq!(1, db.get_storages_for_slot(slot1).unwrap().len());
@ -17523,7 +17514,7 @@ pub mod tests {
let GetUniqueAccountsResult { let GetUniqueAccountsResult {
stored_accounts: after_stored_accounts, stored_accounts: after_stored_accounts,
original_bytes: after_original_bytes, original_bytes: after_original_bytes,
} = db.get_unique_accounts_from_storages(std::iter::once(&after_store)); } = db.get_unique_accounts_from_storages(&after_store);
assert_ne!(created_accounts.original_bytes, after_original_bytes); assert_ne!(created_accounts.original_bytes, after_original_bytes);
assert_eq!(created_accounts.stored_accounts.len(), 1); assert_eq!(created_accounts.stored_accounts.len(), 1);
assert_eq!(after_stored_accounts.len(), usize::from(alive)); assert_eq!(after_stored_accounts.len(), usize::from(alive));

View File

@ -305,7 +305,7 @@ impl<'a> SnapshotMinimizer<'a> {
stored_accounts, .. stored_accounts, ..
} = self } = self
.accounts_db() .accounts_db()
.get_unique_accounts_from_storages(storages.iter()); .get_unique_accounts_from_storages(storages.first().unwrap());
let keep_accounts_collect = Mutex::new(Vec::with_capacity(stored_accounts.len())); let keep_accounts_collect = Mutex::new(Vec::with_capacity(stored_accounts.len()));
let purge_pubkeys_collect = Mutex::new(Vec::with_capacity(stored_accounts.len())); let purge_pubkeys_collect = Mutex::new(Vec::with_capacity(stored_accounts.len()));