blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ancient pack: add low water mark (#33785)

This commit is contained in:
Jeff Washington (jwash) 2023-10-23 10:56:18 -07:00 committed by GitHub
parent 0fc2c9f214
commit 54b796f5a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 48 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
accounts-db/src/ancient_append_vecs.rs
View File

@ -182,15 +182,17 @@ impl AncientSlotInfos {
self.shrink_indexes.clear(); self.shrink_indexes.clear();
let total_storages = self.all_infos.len(); let total_storages = self.all_infos.len();
let mut cumulative_bytes = 0u64; let mut cumulative_bytes = 0u64;
let low_threshold = max_storages * 50 / 100;
for (i, info) in self.all_infos.iter().enumerate() { for (i, info) in self.all_infos.iter().enumerate() {
saturating_add_assign!(cumulative_bytes, info.alive_bytes); saturating_add_assign!(cumulative_bytes, info.alive_bytes);
let ancient_storages_required = (cumulative_bytes / ideal_storage_size + 1) as usize; let ancient_storages_required = (cumulative_bytes / ideal_storage_size + 1) as usize;
let storages_remaining = total_storages - i - 1; let storages_remaining = total_storages - i - 1;
// if the remaining uncombined storages and the # of resulting // if the remaining uncombined storages and the # of resulting
// combined ancient storages is less than the threshold, then // combined ancient storages is less than the threshold, then
// we've gone too far, so get rid of this entry and all after it. // we've gone too far, so get rid of this entry and all after it.
// Every storage after this one is larger. // Every storage after this one is larger.
if storages_remaining + ancient_storages_required < max_storages { if storages_remaining + ancient_storages_required < low_threshold {
self.all_infos.truncate(i); self.all_infos.truncate(i);
break; break;
} }
@ -2387,25 +2389,28 @@ pub mod tests {
#[test] #[test]
fn test_filter_by_smaller_capacity_sort() { fn test_filter_by_smaller_capacity_sort() {
// max is 3 // max is 6
// 4 storages // 7 storages
// storage[3] is big enough to cause us to need another storage // storage[last] is big enough to cause us to need another storage
// so, storage[0] and [1] can be combined into 1, resulting in 3 remaining storages, which is // so, storage[0..=4] can be combined into 1, resulting in 3 remaining storages, which is
// the goal, so we only have to combine the first 2 to hit the goal // the goal, so we only have to combine the first 5 to hit the goal
for method in TestSmallestCapacity::iter() { for method in TestSmallestCapacity::iter() {
let ideal_storage_size_large = get_ancient_append_vec_capacity(); let ideal_storage_size_large = get_ancient_append_vec_capacity();
for reorder in [false, true] { for reorder in [false, true] {
let mut infos = create_test_infos(4); let mut infos = create_test_infos(7);
infos infos
.all_infos .all_infos
.iter_mut() .iter_mut()
.enumerate() .enumerate()
.for_each(|(i, info)| info.capacity = 1 + i as u64); .for_each(|(i, info)| info.capacity = 1 + i as u64);
if reorder { if reorder {
infos.all_infos[3].capacity = 0; // sort to beginning infos.all_infos.last_mut().unwrap().capacity = 0; // sort to beginning
} }
infos.all_infos[3].alive_bytes = ideal_storage_size_large; infos.all_infos.last_mut().unwrap().alive_bytes = ideal_storage_size_large;
let max_storages = 3; // if we use max_storages = 3 or 4, then the low limit is 1 or 2. To get below 2 requires a result of 1, which packs everyone.
// This isn't what we want for this test. So, we make max_storages big enough that we can get to something reasonable (like 3)
// for a low mark.
let max_storages = 6;
match method { match method {
TestSmallestCapacity::FilterBySmallestCapacity => { TestSmallestCapacity::FilterBySmallestCapacity => {
infos.filter_by_smallest_capacity( infos.filter_by_smallest_capacity(
@ -2431,8 +2436,12 @@ pub mod tests {
.iter() .iter()
.map(|info| info.slot) .map(|info| info.slot)
.collect::<Vec<_>>(), .collect::<Vec<_>>(),
if reorder { vec![3, 0, 1] } else { vec![0, 1] }, if reorder {
"reorder: {reorder}" vec![6, 0, 1, 2, 3, 4]
} else {
vec![0, 1, 2, 3, 4]
},
"reorder: {reorder}, method: {method:?}"
); );
} }
} }
@ -2478,7 +2487,20 @@ pub mod tests {
max_storages, max_storages,
NonZeroU64::new(ideal_storage_size_large).unwrap(), NonZeroU64::new(ideal_storage_size_large).unwrap(),
); );
assert!(infos.all_infos.is_empty());
if filter {
assert!(infos.all_infos.is_empty());
} else {
// no short circuit, so truncate to shrink below low water
assert_eq!(
infos
.all_infos
.iter()
.map(|info| info.slot)
.collect::<Vec<_>>(),
vec![0]
);
}
let mut infos = create_test_infos(1); let mut infos = create_test_infos(1);
infos.all_infos[0].alive_bytes = ideal_storage_size_large + 1; infos.all_infos[0].alive_bytes = ideal_storage_size_large + 1;
@ -2511,14 +2533,14 @@ pub mod tests {
assert_eq!(infos.all_infos.len(), 2); assert_eq!(infos.all_infos.len(), 2);
} }
// max is 3 // max is 4
// 4 storages // 5 storages
// storage[3] is big enough to cause us to need another storage // storage[4] is big enough to cause us to need another storage
// so, storage[0] and [1] can be combined into 1, resulting in 3 remaining storages, which is // so, storage[0..=2] can be combined into 1, resulting in 3 remaining storages, which is
// the goal, so we only have to combine the first 2 to hit the goal // the goal, so we only have to combine the first 3 to hit the goal
let mut infos = create_test_infos(4); let mut infos = create_test_infos(5);
infos.all_infos[3].alive_bytes = ideal_storage_size_large; infos.all_infos[4].alive_bytes = ideal_storage_size_large;
let max_storages = 3; let max_storages = 4;
test( test(
&mut infos, &mut infos,
max_storages, max_storages,
@ -2530,7 +2552,7 @@ pub mod tests {
.iter() .iter()
.map(|info| info.slot) .map(|info| info.slot)
.collect::<Vec<_>>(), .collect::<Vec<_>>(),
vec![0, 1] vec![0, 1, 2, 3, 4]
); );
} }
} }
@ -2910,8 +2932,10 @@ pub mod tests {
let active_slots = (0..num_slots) let active_slots = (0..num_slots)
.filter_map(|slot| db.storage.get_slot_storage_entry((slot as Slot) + slot1)) .filter_map(|slot| db.storage.get_slot_storage_entry((slot as Slot) + slot1))
.count(); .count();
let mut expected_slots = max_ancient_slots.min(num_slots); let mut expected_slots = (max_ancient_slots / 2).min(num_slots);
if max_ancient_slots == 0 { if max_ancient_slots >= num_slots {
expected_slots = num_slots;
} else if max_ancient_slots == 0 || num_slots > 0 && expected_slots == 0 {
expected_slots = 1; expected_slots = 1;
} }
assert_eq!( assert_eq!(