add ScanAccountStorageData (#694)
This commit is contained in:
parent
72ee270943
commit
9778e489b0
|
@ -1,8 +1,8 @@
|
|||
use {
|
||||
crate::{
|
||||
accounts_db::{
|
||||
AccountsAddRootTiming, AccountsDb, LoadHint, LoadedAccount, ScanStorageResult,
|
||||
VerifyAccountsHashAndLamportsConfig,
|
||||
AccountsAddRootTiming, AccountsDb, LoadHint, LoadedAccount, ScanAccountStorageData,
|
||||
ScanStorageResult, VerifyAccountsHashAndLamportsConfig,
|
||||
},
|
||||
accounts_index::{IndexKey, ScanConfig, ScanError, ScanResult, ZeroLamport},
|
||||
ancestors::Ancestors,
|
||||
|
@ -210,12 +210,13 @@ impl Accounts {
|
|||
// Cache only has one version per key, don't need to worry about versioning
|
||||
func(loaded_account)
|
||||
},
|
||||
|accum: &DashMap<Pubkey, B>, loaded_account: &LoadedAccount| {
|
||||
|accum: &DashMap<Pubkey, B>, loaded_account: &LoadedAccount, _data| {
|
||||
let loaded_account_pubkey = *loaded_account.pubkey();
|
||||
if let Some(val) = func(loaded_account) {
|
||||
accum.insert(loaded_account_pubkey, val);
|
||||
}
|
||||
},
|
||||
ScanAccountStorageData::NoData,
|
||||
);
|
||||
|
||||
match scan_result {
|
||||
|
|
|
@ -170,6 +170,15 @@ enum ScanAccountStorageResult {
|
|||
CacheFileNeedsToBeCreated((String, Range<Slot>)),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum ScanAccountStorageData {
|
||||
/// callback for accounts in storage will not include `data`
|
||||
NoData,
|
||||
/// return data (&[u8]) for each account.
|
||||
/// This can be expensive to get and is not necessary for many scan operations.
|
||||
DataRefForStorage,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
/// hold alive accounts
|
||||
/// alive means in the accounts index
|
||||
|
@ -4836,11 +4845,12 @@ impl AccountsDb {
|
|||
}
|
||||
|
||||
/// Scan a specific slot through all the account storage
|
||||
pub fn scan_account_storage<R, B>(
|
||||
pub(crate) fn scan_account_storage<R, B>(
|
||||
&self,
|
||||
slot: Slot,
|
||||
cache_map_func: impl Fn(&LoadedAccount) -> Option<R> + Sync,
|
||||
storage_scan_func: impl Fn(&B, &LoadedAccount) + Sync,
|
||||
storage_scan_func: impl Fn(&B, &LoadedAccount, Option<&[u8]>) + Sync,
|
||||
scan_account_storage_data: ScanAccountStorageData,
|
||||
) -> ScanStorageResult<R, B>
|
||||
where
|
||||
R: Send,
|
||||
|
@ -4891,7 +4901,11 @@ impl AccountsDb {
|
|||
.get_slot_storage_entry_shrinking_in_progress_ok(slot)
|
||||
{
|
||||
storage.accounts.account_iter().for_each(|account| {
|
||||
storage_scan_func(&retval, &LoadedAccount::Stored(account))
|
||||
let loaded_account = LoadedAccount::Stored(account);
|
||||
let data = (scan_account_storage_data
|
||||
== ScanAccountStorageData::DataRefForStorage)
|
||||
.then_some(loaded_account.data());
|
||||
storage_scan_func(&retval, &loaded_account, data)
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -7571,7 +7585,7 @@ impl AccountsDb {
|
|||
// Cache only has one version per key, don't need to worry about versioning
|
||||
Some((*loaded_account.pubkey(), loaded_account.loaded_hash()))
|
||||
},
|
||||
|accum: &DashMap<Pubkey, AccountHash>, loaded_account: &LoadedAccount| {
|
||||
|accum: &DashMap<Pubkey, AccountHash>, loaded_account: &LoadedAccount, _data| {
|
||||
let mut loaded_hash = loaded_account.loaded_hash();
|
||||
if loaded_hash == AccountHash(Hash::default()) {
|
||||
loaded_hash = Self::hash_account_data(
|
||||
|
@ -7585,6 +7599,7 @@ impl AccountsDb {
|
|||
}
|
||||
accum.insert(*loaded_account.pubkey(), loaded_hash);
|
||||
},
|
||||
ScanAccountStorageData::NoData,
|
||||
);
|
||||
scan.stop();
|
||||
|
||||
|
@ -7612,7 +7627,8 @@ impl AccountsDb {
|
|||
})
|
||||
},
|
||||
|accum: &DashMap<Pubkey, (AccountHash, AccountSharedData)>,
|
||||
loaded_account: &LoadedAccount| {
|
||||
loaded_account: &LoadedAccount,
|
||||
_data| {
|
||||
// Storage may have duplicates so only keep the latest version for each key
|
||||
let mut loaded_hash = loaded_account.loaded_hash();
|
||||
let key = *loaded_account.pubkey();
|
||||
|
@ -7622,6 +7638,7 @@ impl AccountsDb {
|
|||
}
|
||||
accum.insert(key, (loaded_hash, account));
|
||||
},
|
||||
ScanAccountStorageData::NoData,
|
||||
);
|
||||
|
||||
match scan_result {
|
||||
|
@ -14184,9 +14201,10 @@ pub mod tests {
|
|||
if let ScanStorageResult::Stored(slot_accounts) = accounts_db.scan_account_storage(
|
||||
*slot as Slot,
|
||||
|_| Some(0),
|
||||
|slot_accounts: &DashSet<Pubkey>, loaded_account: &LoadedAccount| {
|
||||
|slot_accounts: &DashSet<Pubkey>, loaded_account: &LoadedAccount, _data| {
|
||||
slot_accounts.insert(*loaded_account.pubkey());
|
||||
},
|
||||
ScanAccountStorageData::NoData,
|
||||
) {
|
||||
if *slot == alive_slot {
|
||||
assert_eq!(slot_accounts.len(), keys.len());
|
||||
|
@ -14224,9 +14242,10 @@ pub mod tests {
|
|||
if let ScanStorageResult::Stored(slot_account) = accounts_db.scan_account_storage(
|
||||
*slot as Slot,
|
||||
|_| Some(0),
|
||||
|slot_account: &Arc<RwLock<Pubkey>>, loaded_account: &LoadedAccount| {
|
||||
|slot_account: &Arc<RwLock<Pubkey>>, loaded_account: &LoadedAccount, _data| {
|
||||
*slot_account.write().unwrap() = *loaded_account.pubkey();
|
||||
},
|
||||
ScanAccountStorageData::NoData,
|
||||
) {
|
||||
assert_eq!(*slot_account.read().unwrap(), keys[*slot as usize]);
|
||||
} else {
|
||||
|
@ -14299,7 +14318,7 @@ pub mod tests {
|
|||
assert!(*slot > requested_flush_root);
|
||||
Some(*loaded_account.pubkey())
|
||||
},
|
||||
|slot_accounts: &DashSet<Pubkey>, loaded_account: &LoadedAccount| {
|
||||
|slot_accounts: &DashSet<Pubkey>, loaded_account: &LoadedAccount, _data| {
|
||||
slot_accounts.insert(*loaded_account.pubkey());
|
||||
if !is_cache_at_limit {
|
||||
// Only true when the limit hasn't been reached and there are still
|
||||
|
@ -14307,6 +14326,7 @@ pub mod tests {
|
|||
assert!(*slot <= requested_flush_root);
|
||||
}
|
||||
},
|
||||
ScanAccountStorageData::NoData,
|
||||
);
|
||||
|
||||
let slot_accounts = match slot_accounts {
|
||||
|
@ -14417,9 +14437,10 @@ pub mod tests {
|
|||
.scan_account_storage(
|
||||
*slot as Slot,
|
||||
|_| Some(0),
|
||||
|slot_account: &DashSet<Pubkey>, loaded_account: &LoadedAccount| {
|
||||
|slot_account: &DashSet<Pubkey>, loaded_account: &LoadedAccount, _data| {
|
||||
slot_account.insert(*loaded_account.pubkey());
|
||||
},
|
||||
ScanAccountStorageData::NoData,
|
||||
) {
|
||||
slot_accounts.into_iter().collect::<HashSet<Pubkey>>()
|
||||
} else {
|
||||
|
|
Loading…
Reference in New Issue