blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

remove IndexAccountMapEntry.store_id (#31600) 

* remove IndexAccountMapEntry.store_id

* fix empty case that test hit

* store_id checks

* cleanup generate index when slot contains no storage

* remove unwrap
This commit is contained in:
Jeff Washington (jwash) 2023-05-11 14:12:41 -05:00 committed by GitHub
parent 43c0f05ca0
commit ea95173b27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 71 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
runtime/src/accounts_db.rs
View File

@ -2266,7 +2266,6 @@ impl<'a> ZeroLamport for StoredAccountMeta<'a> {
} }
struct IndexAccountMapEntry<'a> { struct IndexAccountMapEntry<'a> {
pub store_id: AppendVecId,
pub stored_account: StoredAccountMeta<'a>, pub stored_account: StoredAccountMeta<'a>,
} }
@ -8741,13 +8740,7 @@ impl AccountsDb {
storage.accounts.account_iter().for_each(|stored_account| { storage.accounts.account_iter().for_each(|stored_account| {
let pubkey = stored_account.pubkey(); let pubkey = stored_account.pubkey();
assert!(!self.is_filler_account(pubkey)); assert!(!self.is_filler_account(pubkey));
accounts_map.insert( accounts_map.insert(*pubkey, IndexAccountMapEntry { stored_account });
*pubkey,
IndexAccountMapEntry {
store_id: storage.append_vec_id(),
stored_account,
},
);
}); });
accounts_map accounts_map
} }
@ -8774,6 +8767,7 @@ impl AccountsDb {
&self, &self,
accounts_map: GenerateIndexAccountsMap<'_>, accounts_map: GenerateIndexAccountsMap<'_>,
slot: &Slot, slot: &Slot,
store_id: AppendVecId,
rent_collector: &RentCollector, rent_collector: &RentCollector,
) -> SlotIndexGenerationInfo { ) -> SlotIndexGenerationInfo {
if accounts_map.is_empty() { if accounts_map.is_empty() {
@ -8787,43 +8781,38 @@ impl AccountsDb {
let mut num_accounts_rent_paying = 0; let mut num_accounts_rent_paying = 0;
let num_accounts = accounts_map.len(); let num_accounts = accounts_map.len();
let mut amount_to_top_off_rent = 0; let mut amount_to_top_off_rent = 0;
let items = accounts_map.into_iter().map( let items =
|( accounts_map
pubkey, .into_iter()
IndexAccountMapEntry { .map(|(pubkey, IndexAccountMapEntry { stored_account })| {
store_id, if secondary {
stored_account, self.accounts_index.update_secondary_indexes(
}, &pubkey,
)| { &stored_account,
if secondary { &self.account_indexes,
self.accounts_index.update_secondary_indexes( );
&pubkey, }
&stored_account, if !stored_account.is_zero_lamport() {
&self.account_indexes, accounts_data_len += stored_account.data().len() as u64;
); }
}
if !stored_account.is_zero_lamport() {
accounts_data_len += stored_account.data().len() as u64;
}
if let Some(amount_to_top_off_rent_this_account) = if let Some(amount_to_top_off_rent_this_account) =
Self::stats_for_rent_payers(&pubkey, &stored_account, rent_collector) Self::stats_for_rent_payers(&pubkey, &stored_account, rent_collector)
{ {
amount_to_top_off_rent += amount_to_top_off_rent_this_account; amount_to_top_off_rent += amount_to_top_off_rent_this_account;
num_accounts_rent_paying += 1; num_accounts_rent_paying += 1;
// remember this rent-paying account pubkey // remember this rent-paying account pubkey
rent_paying_accounts_by_partition.push(pubkey); rent_paying_accounts_by_partition.push(pubkey);
} }
( (
pubkey, pubkey,
AccountInfo::new( AccountInfo::new(
StorageLocation::AppendVec(store_id, stored_account.offset()), // will never be cached StorageLocation::AppendVec(store_id, stored_account.offset()), // will never be cached
stored_account.lamports(), stored_account.lamports(),
), ),
) )
}, });
);
let (dirty_pubkeys, insert_time_us) = self let (dirty_pubkeys, insert_time_us) = self
.accounts_index .accounts_index
@ -8994,11 +8983,12 @@ impl AccountsDb {
for (index, slot) in slots.iter().enumerate() { for (index, slot) in slots.iter().enumerate() {
let mut scan_time = Measure::start("scan"); let mut scan_time = Measure::start("scan");
log_status.report(index as u64); log_status.report(index as u64);
let storage = self.storage.get_slot_storage_entry(*slot); let Some(storage) = self.storage.get_slot_storage_entry(*slot) else {
let accounts_map = storage // no storage at this slot, no information to pull out
.as_ref() continue;
.map(|storage| self.process_storage_slot(storage)) };
.unwrap_or_default(); let accounts_map = self.process_storage_slot(&storage);
let store_id = storage.append_vec_id();
scan_time.stop(); scan_time.stop();
scan_time_sum += scan_time.as_us(); scan_time_sum += scan_time.as_us();
@ -9006,6 +8996,7 @@ impl AccountsDb {
&storage_info, &storage_info,
&accounts_map, &accounts_map,
&storage_info_timings, &storage_info_timings,
store_id,
); );
let insert_us = if pass == 0 { let insert_us = if pass == 0 {
@ -9019,7 +9010,12 @@ impl AccountsDb {
amount_to_top_off_rent: amount_to_top_off_rent_this_slot, amount_to_top_off_rent: amount_to_top_off_rent_this_slot,
rent_paying_accounts_by_partition: rent_paying_accounts_by_partition:
rent_paying_accounts_by_partition_this_slot, rent_paying_accounts_by_partition_this_slot,
} = self.generate_index_for_slot(accounts_map, slot, &rent_collector); } = self.generate_index_for_slot(
accounts_map,
slot,
store_id,
&rent_collector,
);
rent_paying.fetch_add(rent_paying_this_slot, Ordering::Relaxed); rent_paying.fetch_add(rent_paying_this_slot, Ordering::Relaxed);
amount_to_top_off_rent amount_to_top_off_rent
.fetch_add(amount_to_top_off_rent_this_slot, Ordering::Relaxed); .fetch_add(amount_to_top_off_rent_this_slot, Ordering::Relaxed);
@ -9051,7 +9047,7 @@ impl AccountsDb {
count += 1; count += 1;
let ai = AccountInfo::new( let ai = AccountInfo::new(
StorageLocation::AppendVec( StorageLocation::AppendVec(
account_info.store_id, store_id,
account_info.stored_account.offset(), account_info.stored_account.offset(),
), // will never be cached ), // will never be cached
account_info.stored_account.lamports(), account_info.stored_account.lamports(),
@ -9285,28 +9281,26 @@ impl AccountsDb {
storage_info: &StorageSizeAndCountMap, storage_info: &StorageSizeAndCountMap,
accounts_map: &GenerateIndexAccountsMap<'_>, accounts_map: &GenerateIndexAccountsMap<'_>,
timings: &Mutex<GenerateIndexTimings>, timings: &Mutex<GenerateIndexTimings>,
store_id: AppendVecId,
) { ) {
let mut storage_size_accounts_map_time = Measure::start("storage_size_accounts_map"); let mut storage_size_accounts_map_time = Measure::start("storage_size_accounts_map");
let mut storage_info_local = HashMap::<AppendVecId, StorageSizeAndCount>::default();
// first collect into a local HashMap with no lock contention // first collect into a local HashMap with no lock contention
let mut storage_info_local = StorageSizeAndCount::default();
for (_, v) in accounts_map.iter() { for (_, v) in accounts_map.iter() {
let mut info = storage_info_local storage_info_local.stored_size += v.stored_account.stored_size();
.entry(v.store_id) storage_info_local.count += 1;
.or_insert_with(StorageSizeAndCount::default);
info.stored_size += v.stored_account.stored_size();
info.count += 1;
} }
storage_size_accounts_map_time.stop(); storage_size_accounts_map_time.stop();
// second, collect into the shared DashMap once we've figured out all the info per store_id // second, collect into the shared DashMap once we've figured out all the info per store_id
let mut storage_size_accounts_map_flatten_time = let mut storage_size_accounts_map_flatten_time =
Measure::start("storage_size_accounts_map_flatten_time"); Measure::start("storage_size_accounts_map_flatten_time");
for (store_id, v) in storage_info_local.into_iter() { if !accounts_map.is_empty() {
let mut info = storage_info let mut info = storage_info
.entry(store_id) .entry(store_id)
.or_insert_with(StorageSizeAndCount::default); .or_insert_with(StorageSizeAndCount::default);
info.stored_size += v.stored_size; info.stored_size += storage_info_local.stored_size;
info.count += v.count; info.count += storage_info_local.count;
} }
storage_size_accounts_map_flatten_time.stop(); storage_size_accounts_map_flatten_time.stop();
@ -15982,7 +15976,12 @@ pub mod tests {
let storage = accounts.storage.get_slot_storage_entry(slot0).unwrap(); let storage = accounts.storage.get_slot_storage_entry(slot0).unwrap();
let storage_info = StorageSizeAndCountMap::default(); let storage_info = StorageSizeAndCountMap::default();
let accounts_map = accounts.process_storage_slot(&storage); let accounts_map = accounts.process_storage_slot(&storage);
AccountsDb::update_storage_info(&storage_info, &accounts_map, &Mutex::default()); AccountsDb::update_storage_info(
&storage_info,
&accounts_map,
&Mutex::default(),
storage.append_vec_id(),
);
assert_eq!(storage_info.len(), 1); assert_eq!(storage_info.len(), 1);
for entry in storage_info.iter() { for entry in storage_info.iter() {
assert_eq!( assert_eq!(
@ -15999,7 +15998,12 @@ pub mod tests {
let storage = accounts.create_and_insert_store(0, 1, "test"); let storage = accounts.create_and_insert_store(0, 1, "test");
let storage_info = StorageSizeAndCountMap::default(); let storage_info = StorageSizeAndCountMap::default();
let accounts_map = accounts.process_storage_slot(&storage); let accounts_map = accounts.process_storage_slot(&storage);
AccountsDb::update_storage_info(&storage_info, &accounts_map, &Mutex::default()); AccountsDb::update_storage_info(
&storage_info,
&accounts_map,
&Mutex::default(),
storage.append_vec_id(),
);
assert!(storage_info.is_empty()); assert!(storage_info.is_empty());
} }
@ -16032,7 +16036,12 @@ pub mod tests {
let storage = accounts.storage.get_slot_storage_entry(slot0).unwrap(); let storage = accounts.storage.get_slot_storage_entry(slot0).unwrap();
let storage_info = StorageSizeAndCountMap::default(); let storage_info = StorageSizeAndCountMap::default();
let accounts_map = accounts.process_storage_slot(&storage); let accounts_map = accounts.process_storage_slot(&storage);
AccountsDb::update_storage_info(&storage_info, &accounts_map, &Mutex::default()); AccountsDb::update_storage_info(
&storage_info,
&accounts_map,
&Mutex::default(),
storage.append_vec_id(),
);
assert_eq!(storage_info.len(), 1); assert_eq!(storage_info.len(), 1);
for entry in storage_info.iter() { for entry in storage_info.iter() {
assert_eq!( assert_eq!(