blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

AcctIdx: disk generate index and filler accounts use more threads (#21566)

This commit is contained in:
Jeff Washington (jwash) 2021-12-09 11:54:14 -06:00 committed by GitHub
parent 8063273d09
commit a2df1eb502
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 48 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
runtime/src/accounts_db.rs
View File

@ -6798,7 +6798,6 @@ impl AccountsDb {
.skip(pass * per_pass) .skip(pass * per_pass)
.take(per_pass) .take(per_pass)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
self.thread_pool.install(|| {
roots_in_this_pass.into_par_iter().for_each(|slot| { roots_in_this_pass.into_par_iter().for_each(|slot| {
let storage_maps: Vec<Arc<AccountStorageEntry>> = self let storage_maps: Vec<Arc<AccountStorageEntry>> = self
.storage .storage
@ -6842,7 +6841,6 @@ impl AccountsDb {
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let hashes = (0..filler_entries).map(|_| hash).collect::<Vec<_>>(); let hashes = (0..filler_entries).map(|_| hash).collect::<Vec<_>>();
self.store_accounts_frozen(*slot, &add[..], Some(&hashes[..]), None, None); self.store_accounts_frozen(*slot, &add[..], Some(&hashes[..]), None, None);
})
}); });
self.accounts_index.set_startup(false); self.accounts_index.set_startup(false);
} }
@ -6882,7 +6880,14 @@ impl AccountsDb {
let storage_info = StorageSizeAndCountMap::default(); let storage_info = StorageSizeAndCountMap::default();
let total_processed_slots_across_all_threads = AtomicU64::new(0); let total_processed_slots_across_all_threads = AtomicU64::new(0);
let outer_slots_len = slots.len(); let outer_slots_len = slots.len();
let chunk_size = (outer_slots_len / 7) + 1; // approximately 400k slots in a snapshot let threads = if self.accounts_index.is_disk_index_enabled() {
// these write directly to disk, so the more threads, the better
num_cpus::get()
} else {
// seems to be a good hueristic given varying # cpus for in-mem disk index
8
};
let chunk_size = (outer_slots_len / (std::cmp::max(1, threads.saturating_sub(1)))) + 1; // approximately 400k slots in a snapshot
let mut index_time = Measure::start("index"); let mut index_time = Measure::start("index");
let insertion_time_us = AtomicU64::new(0); let insertion_time_us = AtomicU64::new(0);
let rent_exempt = AtomicU64::new(0); let rent_exempt = AtomicU64::new(0);