From 592013eaf45051581151e5175d0083f13e2a8163 Mon Sep 17 00:00:00 2001 From: sakridge Date: Thu, 5 Aug 2021 23:26:38 -0700 Subject: [PATCH] Clean within shrink_all_slots (#19042) Co-authored-by: Carl Lin --- runtime/src/accounts_db.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index 9b399e5758..5d1f1fc187 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -2480,12 +2480,20 @@ impl AccountsDb { } pub fn shrink_all_slots(&self, is_startup: bool) { + const DIRTY_STORES_CLEANING_THRESHOLD: usize = 10_000; + const OUTER_CHUNK_SIZE: usize = 2000; + const INNER_CHUNK_SIZE: usize = OUTER_CHUNK_SIZE / 8; if is_startup && self.caching_enabled { let slots = self.all_slots_in_storage(); - let chunk_size = std::cmp::max(slots.len() / 8, 1); // approximately 400k slots in a snapshot - slots.par_chunks(chunk_size).for_each(|slots| { - for slot in slots { - self.shrink_slot_forced(*slot, is_startup); + let inner_chunk_size = std::cmp::max(INNER_CHUNK_SIZE, 1); + slots.chunks(OUTER_CHUNK_SIZE).for_each(|chunk| { + chunk.par_chunks(inner_chunk_size).for_each(|slots| { + for slot in slots { + self.shrink_slot_forced(*slot, is_startup); + } + }); + if self.dirty_stores.len() > DIRTY_STORES_CLEANING_THRESHOLD { + self.clean_accounts(None, is_startup); } }); } else { @@ -2495,6 +2503,9 @@ impl AccountsDb { } else { self.do_shrink_slot_forced_v1(slot); } + if self.dirty_stores.len() > DIRTY_STORES_CLEANING_THRESHOLD { + self.clean_accounts(None, is_startup); + } } } }