blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

single pass through each storage to populate info (#33227)

This commit is contained in:
Jeff Washington (jwash) 2023-09-13 12:37:01 -07:00 committed by GitHub
parent 525e59f01a
commit 093ae61e85
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 58 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
accounts-db/src/accounts_db.rs
View File

@ -8964,6 +8964,7 @@ impl AccountsDb {
slot: &Slot,
store_id: AppendVecId,
rent_collector: &RentCollector,
storage_info: &StorageSizeAndCountMap,
) -> SlotIndexGenerationInfo {
if accounts_map.is_empty() {
return SlotIndexGenerationInfo::default();
@ -8976,7 +8977,11 @@ impl AccountsDb {
let mut num_accounts_rent_paying = 0;
let num_accounts = accounts_map.len();
let mut amount_to_top_off_rent = 0;
// first collect into a local HashMap with no lock contention
let mut storage_info_local = StorageSizeAndCount::default();
let items = accounts_map.into_iter().map(|(pubkey, stored_account)| {
storage_info_local.stored_size += stored_account.stored_size();
if secondary {
self.accounts_index.update_secondary_indexes(
&pubkey,
@ -9010,6 +9015,13 @@ impl AccountsDb {
.accounts_index
.insert_new_if_missing_into_primary_index(*slot, num_accounts, items);
{
// second, collect into the shared DashMap once we've figured out all the info per store_id
let mut info = storage_info.entry(store_id).or_default();
info.stored_size += storage_info_local.stored_size;
info.count += num_accounts;
}
// dirty_pubkeys will contain a pubkey if an item has multiple rooted entries for
// a given pubkey. If there is just a single item, there is no cleaning to
// be done on that pubkey. Use only those pubkeys with multiple updates.
@ -9183,12 +9195,6 @@ impl AccountsDb {
scan_time.stop();
scan_time_sum += scan_time.as_us();
Self::update_storage_info(
&storage_info,
&accounts_map,
&storage_info_timings,
store_id,
);
let insert_us = if pass == 0 {
// generate index
@ -9206,6 +9212,7 @@ impl AccountsDb {
slot,
store_id,
&rent_collector,
&storage_info,
);
rent_paying.fetch_add(rent_paying_this_slot, Ordering::Relaxed);
amount_to_top_off_rent
@ -9481,36 +9488,6 @@ impl AccountsDb {
(accounts_data_len_from_duplicates as u64, uncleaned_slots)
}
fn update_storage_info(
storage_info: &StorageSizeAndCountMap,
accounts_map: &GenerateIndexAccountsMap<'_>,
timings: &Mutex<GenerateIndexTimings>,
store_id: AppendVecId,
) {
let mut storage_size_accounts_map_time = Measure::start("storage_size_accounts_map");
// first collect into a local HashMap with no lock contention
let mut storage_info_local = StorageSizeAndCount::default();
for (_, v) in accounts_map.iter() {
storage_info_local.stored_size += v.stored_size();
storage_info_local.count += 1;
}
storage_size_accounts_map_time.stop();
// second, collect into the shared DashMap once we've figured out all the info per store_id
let mut storage_size_accounts_map_flatten_time =
Measure::start("storage_size_accounts_map_flatten_time");
if !accounts_map.is_empty() {
let mut info = storage_info.entry(store_id).or_default();
info.stored_size += storage_info_local.stored_size;
info.count += storage_info_local.count;
}
storage_size_accounts_map_flatten_time.stop();
let mut timings = timings.lock().unwrap();
timings.storage_size_accounts_map_us += storage_size_accounts_map_time.as_us();
timings.storage_size_accounts_map_flatten_us +=
storage_size_accounts_map_flatten_time.as_us();
}
fn set_storage_count_and_alive_bytes(
&self,
stored_sizes_and_counts: StorageSizeAndCountMap,
@ -15765,20 +15742,34 @@ pub mod tests {
#[test]
fn test_calculate_storage_count_and_alive_bytes() {
let accounts = AccountsDb::new_single_for_tests();
accounts.accounts_index.set_startup(Startup::Startup);
let shared_key = solana_sdk::pubkey::new_rand();
let account = AccountSharedData::new(1, 1, AccountSharedData::default().owner());
let slot0 = 0;
accounts.store_for_tests(slot0, &[(&shared_key, &account)]);
accounts.add_root_and_flush_write_cache(slot0);
accounts.accounts_index.set_startup(Startup::Startup);
let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache");
let hashes = vec![Hash::default(); 1];
let write_version = vec![0; 1];
storage.accounts.append_accounts(
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&(slot0, &[(&shared_key, &account)][..]),
hashes,
write_version,
),
0,
);
let storage = accounts.storage.get_slot_storage_entry(slot0).unwrap();
let storage_info = StorageSizeAndCountMap::default();
let accounts_map = accounts.process_storage_slot(&storage);
AccountsDb::update_storage_info(
accounts.generate_index_for_slot(
accounts_map,
&slot0,
0,
&RentCollector::default(),
&storage_info,
&accounts_map,
&Mutex::default(),
storage.append_vec_id(),
);
assert_eq!(storage_info.len(), 1);
for entry in storage_info.iter() {
@ -15787,6 +15778,7 @@ pub mod tests {
(&0, 1, 144)
);
}
accounts.accounts_index.set_startup(Startup::Normal);
}
#[test]
@ -15796,11 +15788,12 @@ pub mod tests {
let storage = accounts.create_and_insert_store(0, 1, "test");
let storage_info = StorageSizeAndCountMap::default();
let accounts_map = accounts.process_storage_slot(&storage);
AccountsDb::update_storage_info(
accounts.generate_index_for_slot(
accounts_map,
&0,
0,
&RentCollector::default(),
&storage_info,
&accounts_map,
&Mutex::default(),
storage.append_vec_id(),
);
assert!(storage_info.is_empty());
}
@ -15812,6 +15805,8 @@ pub mod tests {
solana_sdk::pubkey::Pubkey::from([0; 32]),
solana_sdk::pubkey::Pubkey::from([255; 32]),
];
accounts.accounts_index.set_startup(Startup::Startup);
// make sure accounts are in 2 different bins
assert!(
(accounts.accounts_index.bins() == 1)
@ -15827,18 +15822,26 @@ pub mod tests {
let account = AccountSharedData::new(1, 1, AccountSharedData::default().owner());
let account_big = AccountSharedData::new(1, 1000, AccountSharedData::default().owner());
let slot0 = 0;
accounts.store_for_tests(slot0, &[(&keys[0], &account)]);
accounts.store_for_tests(slot0, &[(&keys[1], &account_big)]);
accounts.add_root_and_flush_write_cache(slot0);
let storage = accounts.create_and_insert_store(slot0, 4_000, "flush_slot_cache");
let hashes = vec![Hash::default(); 2];
let write_version = vec![0; 2];
storage.accounts.append_accounts(
&StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&(slot0, &[(&keys[0], &account), (&keys[1], &account_big)][..]),
hashes,
write_version,
),
0,
);
let storage = accounts.storage.get_slot_storage_entry(slot0).unwrap();
let storage_info = StorageSizeAndCountMap::default();
let accounts_map = accounts.process_storage_slot(&storage);
AccountsDb::update_storage_info(
accounts.generate_index_for_slot(
accounts_map,
&0,
0,
&RentCollector::default(),
&storage_info,
&accounts_map,
&Mutex::default(),
storage.append_vec_id(),
);
assert_eq!(storage_info.len(), 1);
for entry in storage_info.iter() {
@ -15847,6 +15850,7 @@ pub mod tests {
(&0, 2, 1280)
);
}
accounts.accounts_index.set_startup(Startup::Normal);
}
#[test]