blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Track RecycleStore basic stats with needed refactor (#15291) 

* Track RecycleStore basic stats with needed refactor

* Fix another wrong metrics def
This commit is contained in:
Ryo Onodera 2021-02-15 16:09:17 +09:00 committed by GitHub
parent e403aeaf05
commit 30f18319f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 63 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
runtime/src/accounts_db.rs
View File

@ -744,6 +744,43 @@ pub struct StoreAccountsTiming {
update_index_elapsed: u64,
handle_reclaims_elapsed: u64,
}
#[derive(Debug, Default)]
struct RecycleStores {
entries: Vec<Arc<AccountStorageEntry>>,
total_bytes: u64,
}
impl RecycleStores {
fn add_entry(&mut self, new_entry: Arc<AccountStorageEntry>) {
self.total_bytes += new_entry.total_bytes();
self.entries.push(new_entry)
}
fn iter(&self) -> std::slice::Iter<Arc<AccountStorageEntry>> {
self.entries.iter()
}
fn add_entries(&mut self, new_entries: Vec<Arc<AccountStorageEntry>>) {
self.total_bytes += new_entries.iter().map(|e| e.total_bytes()).sum::<u64>();
self.entries.extend(new_entries);
}
fn remove_entry(&mut self, index: usize) -> Arc<AccountStorageEntry> {
let removed_entry = self.entries.swap_remove(index);
self.total_bytes -= removed_entry.total_bytes();
removed_entry
}
fn entry_count(&self) -> usize {
self.entries.len()
}
fn total_bytes(&self) -> u64 {
self.total_bytes
}
}
// This structure handles the load/store of the accounts
#[derive(Debug)]
pub struct AccountsDB {
@ -754,7 +791,7 @@ pub struct AccountsDB {
pub accounts_cache: AccountsCache,
recycle_stores: RwLock<Vec<Arc<AccountStorageEntry>>>,
recycle_stores: RwLock<RecycleStores>,
/// distribute the accounts across storage lists
pub next_id: AtomicUsize,
@ -1122,7 +1159,7 @@ impl Default for AccountsDB {
accounts_index: AccountsIndex::default(),
storage: AccountStorage::default(),
accounts_cache: AccountsCache::default(),
recycle_stores: RwLock::new(Vec::new()),
recycle_stores: RwLock::new(RecycleStores::default()),
uncleaned_pubkeys: DashMap::new(),
next_id: AtomicUsize::new(0),
shrink_candidate_slots_v1: Mutex::new(Vec::new()),
@ -1941,13 +1978,13 @@ impl AccountsDB {
recycle_stores_write_elapsed.stop();
let mut drop_storage_entries_elapsed = Measure::start("drop_storage_entries_elapsed");
if recycle_stores.len() < MAX_RECYCLE_STORES {
recycle_stores.extend(dead_storages);
if recycle_stores.entry_count() < MAX_RECYCLE_STORES {
recycle_stores.add_entries(dead_storages);
drop(recycle_stores);
} else {
self.stats
.dropped_stores
.fetch_add(recycle_stores.len() as u64, Ordering::Relaxed);
.fetch_add(dead_storages.len() as u64, Ordering::Relaxed);
drop(recycle_stores);
drop(dead_storages);
}
@ -2362,7 +2399,7 @@ impl AccountsDB {
avail += 1;
if store.accounts.capacity() >= min_size && store.accounts.capacity() < max_size {
let ret = recycle_stores.swap_remove(i);
let ret = recycle_stores.remove_entry(i);
drop(recycle_stores);
let old_id = ret.append_vec_id();
ret.recycle(slot, self.next_id.fetch_add(1, Ordering::Relaxed));
@ -2380,7 +2417,7 @@ impl AccountsDB {
"no recycle stores max: {} min: {} len: {} looking: {}, {} avail: {}",
max,
min,
recycle_stores.len(),
recycle_stores.entry_count(),
min_size,
max_size,
avail,
@ -2583,14 +2620,14 @@ impl AccountsDB {
for slot_entries in slot_stores {
let entry = slot_entries.read().unwrap();
for (_store_id, stores) in entry.iter() {
if recycle_stores.len() > MAX_RECYCLE_STORES {
if recycle_stores.entry_count() > MAX_RECYCLE_STORES {
let dropped_count = total_removed_storage_entries - recycled_count;
self.stats
.dropped_stores
.fetch_add(dropped_count as u64, Ordering::Relaxed);
return recycle_stores_write_elapsed.as_us();
}
recycle_stores.push(stores.clone());
recycle_stores.add_entry(stores.clone());
recycled_count += 1;
}
}
@ -3454,7 +3491,7 @@ impl AccountsDB {
("total_count", total_count, i64),
(
"recycle_count",
self.recycle_stores.read().unwrap().len() as u64,
self.recycle_stores.read().unwrap().entry_count() as u64,
i64
),
);
@ -4578,6 +4615,7 @@ impl AccountsDB {
),
);
let recycle_stores = self.recycle_stores.read().unwrap();
datapoint_info!(
"accounts_db_store_timings2",
(
@ -4585,6 +4623,16 @@ impl AccountsDB {
self.stats.recycle_store_count.swap(0, Ordering::Relaxed),
i64
),
(
"current_recycle_store_count",
recycle_stores.entry_count(),
i64
),
(
"current_recycle_store_bytes",
recycle_stores.total_bytes(),
i64
),
(
"create_store_count",
self.stats.create_store_count.swap(0, Ordering::Relaxed),
@ -5173,13 +5221,13 @@ impl AccountsDB {
recycle_stores_write_elapsed.stop();
let mut drop_storage_entries_elapsed = Measure::start("drop_storage_entries_elapsed");
if recycle_stores.len() < MAX_RECYCLE_STORES {
recycle_stores.extend(dead_storages);
if recycle_stores.entry_count() < MAX_RECYCLE_STORES {
recycle_stores.add_entries(dead_storages);
drop(recycle_stores);
} else {
self.stats
.dropped_stores
.fetch_add(recycle_stores.len() as u64, Ordering::Relaxed);
.fetch_add(dead_storages.len() as u64, Ordering::Relaxed);
drop(recycle_stores);
drop(dead_storages);
}
@ -8784,7 +8832,7 @@ pub mod tests {
accounts.clean_accounts(None);
accounts.shrink_all_slots();
accounts.print_accounts_stats("post-shrink");
let num_stores = accounts.recycle_stores.read().unwrap().len();
let num_stores = accounts.recycle_stores.read().unwrap().entry_count();
assert!(num_stores > 0);
let mut account_refs = Vec::new();
@ -8798,7 +8846,7 @@ pub mod tests {
accounts.store_uncached(2, &[(key, &account)]);
account_refs.push(account);
}
assert!(accounts.recycle_stores.read().unwrap().len() < num_stores);
assert!(accounts.recycle_stores.read().unwrap().entry_count() < num_stores);
accounts.print_accounts_stats("post-store");