add ancient ancient filter_by_smallest_capacity and truncate_to_max_storages (#30129)
This commit is contained in:
parent
f3457a8485
commit
01f0dcdad4
|
@ -88,6 +88,60 @@ impl AncientSlotInfos {
|
||||||
amount_shrunk(r).cmp(&amount_shrunk(l))
|
amount_shrunk(r).cmp(&amount_shrunk(l))
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// truncate 'all_infos' such that when the remaining entries in
|
||||||
|
/// 'all_infos' are combined, the total number of storages <= 'max_storages'
|
||||||
|
/// The idea is that 'all_infos' is sorted from smallest capacity to largest,
|
||||||
|
/// but that isn't required for this function to be 'correct'.
|
||||||
|
#[allow(dead_code)]
|
||||||
|
fn truncate_to_max_storages(&mut self, max_storages: usize, ideal_storage_size: u64) {
|
||||||
|
// these indexes into 'all_infos' are useless once we truncate 'all_infos', so make sure they're cleared out to avoid any issues
|
||||||
|
self.shrink_indexes.clear();
|
||||||
|
let total_storages = self.all_infos.len();
|
||||||
|
let mut cumulative_bytes = 0u64;
|
||||||
|
for (i, info) in self.all_infos.iter().enumerate() {
|
||||||
|
saturating_add_assign!(cumulative_bytes, info.alive_bytes);
|
||||||
|
let ancient_storages_required = (cumulative_bytes / ideal_storage_size + 1) as usize;
|
||||||
|
let storages_remaining = total_storages - i - 1;
|
||||||
|
// if the remaining uncombined storages and the # of resulting
|
||||||
|
// combined ancient storages is less than the threshold, then
|
||||||
|
// we've gone too far, so get rid of this entry and all after it.
|
||||||
|
// Every storage after this one is larger.
|
||||||
|
if storages_remaining + ancient_storages_required < max_storages {
|
||||||
|
self.all_infos.truncate(i);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// remove entries from 'all_infos' such that combining
|
||||||
|
/// the remaining entries into storages of 'ideal_storage_size'
|
||||||
|
/// will get us below 'max_storages'
|
||||||
|
/// The entires that are removed will be reconsidered the next time around.
|
||||||
|
/// Combining too many storages costs i/o and cpu so the goal is to find the sweet spot so
|
||||||
|
/// that we make progress in cleaning/shrinking/combining but that we don't cause unnecessary
|
||||||
|
/// churn.
|
||||||
|
#[allow(dead_code)]
|
||||||
|
fn filter_by_smallest_capacity(&mut self, max_storages: usize, ideal_storage_size: u64) {
|
||||||
|
let total_storages = self.all_infos.len();
|
||||||
|
if total_storages <= max_storages {
|
||||||
|
// currently fewer storages than max, so nothing to shrink
|
||||||
|
self.shrink_indexes.clear();
|
||||||
|
self.all_infos.clear();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// sort by 'should_shrink' then smallest capacity to largest
|
||||||
|
self.all_infos.sort_unstable_by(|l, r| {
|
||||||
|
r.should_shrink
|
||||||
|
.cmp(&l.should_shrink)
|
||||||
|
.then_with(|| l.capacity.cmp(&r.capacity))
|
||||||
|
});
|
||||||
|
|
||||||
|
// remove any storages we don't need to combine this pass to achieve
|
||||||
|
// # resulting storages <= 'max_storages'
|
||||||
|
self.truncate_to_max_storages(max_storages, ideal_storage_size);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AccountsDb {
|
impl AccountsDb {
|
||||||
|
@ -490,6 +544,141 @@ pub mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn create_test_infos(count: usize) -> AncientSlotInfos {
|
||||||
|
let (db, slot1) = create_db_with_storages_and_index(true /*alive*/, 1, None);
|
||||||
|
let storage = db.storage.get_slot_storage_entry(slot1).unwrap();
|
||||||
|
AncientSlotInfos {
|
||||||
|
all_infos: (0..count)
|
||||||
|
.map(|index| SlotInfo {
|
||||||
|
storage: Arc::clone(&storage),
|
||||||
|
slot: index as Slot,
|
||||||
|
capacity: 1,
|
||||||
|
alive_bytes: 1,
|
||||||
|
should_shrink: false,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
shrink_indexes: (0..count).collect(),
|
||||||
|
..AncientSlotInfos::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_filter_by_smallest_capacity_empty() {
|
||||||
|
for max_storages in 1..3 {
|
||||||
|
// requesting N max storage, has 1 storage, N >= 1 so nothing to do
|
||||||
|
let ideal_storage_size_large = get_ancient_append_vec_capacity();
|
||||||
|
let mut infos = create_test_infos(1);
|
||||||
|
infos.filter_by_smallest_capacity(max_storages, ideal_storage_size_large);
|
||||||
|
assert!(infos.all_infos.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_filter_by_smaller_capacity_sort() {
|
||||||
|
// max is 3
|
||||||
|
// 4 storages
|
||||||
|
// storage[3] is big enough to cause us to need another storage
|
||||||
|
// so, storage[0] and [1] can be combined into 1, resulting in 3 remaining storages, which is
|
||||||
|
// the goal, so we only have to combine the first 2 to hit the goal
|
||||||
|
let ideal_storage_size_large = get_ancient_append_vec_capacity();
|
||||||
|
for reorder in [false, true] {
|
||||||
|
let mut infos = create_test_infos(4);
|
||||||
|
infos
|
||||||
|
.all_infos
|
||||||
|
.iter_mut()
|
||||||
|
.enumerate()
|
||||||
|
.for_each(|(i, info)| info.capacity = 1 + i as u64);
|
||||||
|
if reorder {
|
||||||
|
infos.all_infos[3].capacity = 0; // sort to beginning
|
||||||
|
}
|
||||||
|
infos.all_infos[3].alive_bytes = ideal_storage_size_large;
|
||||||
|
let max_storages = 3;
|
||||||
|
infos.filter_by_smallest_capacity(max_storages, ideal_storage_size_large);
|
||||||
|
assert_eq!(
|
||||||
|
infos
|
||||||
|
.all_infos
|
||||||
|
.iter()
|
||||||
|
.map(|info| info.slot)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
if reorder { vec![3, 0, 1] } else { vec![0, 1] },
|
||||||
|
"reorder: {reorder}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_to_max_storages() {
|
||||||
|
for filter in [false, true] {
|
||||||
|
let test = |infos: &mut AncientSlotInfos, max_storages, ideal_storage_size| {
|
||||||
|
if filter {
|
||||||
|
infos.filter_by_smallest_capacity(max_storages, ideal_storage_size);
|
||||||
|
} else {
|
||||||
|
infos.truncate_to_max_storages(max_storages, ideal_storage_size);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let ideal_storage_size_large = get_ancient_append_vec_capacity();
|
||||||
|
let mut infos = create_test_infos(1);
|
||||||
|
let max_storages = 1;
|
||||||
|
// 1 storage, 1 max, but 1 storage does not fill the entire new combined storage, so truncate nothing
|
||||||
|
test(&mut infos, max_storages, ideal_storage_size_large);
|
||||||
|
assert_eq!(infos.all_infos.len(), usize::from(!filter));
|
||||||
|
|
||||||
|
let mut infos = create_test_infos(1);
|
||||||
|
let max_storages = 1;
|
||||||
|
infos.all_infos[0].alive_bytes = ideal_storage_size_large + 1; // too big for 1 ideal storage
|
||||||
|
// 1 storage, 1 max, but 1 overflows the entire new combined storage, so truncate nothing
|
||||||
|
test(&mut infos, max_storages, ideal_storage_size_large);
|
||||||
|
assert_eq!(infos.all_infos.len(), usize::from(!filter));
|
||||||
|
|
||||||
|
let mut infos = create_test_infos(1);
|
||||||
|
let max_storages = 2;
|
||||||
|
// all truncated because these infos will fit into the # storages
|
||||||
|
test(&mut infos, max_storages, ideal_storage_size_large);
|
||||||
|
assert!(infos.all_infos.is_empty());
|
||||||
|
|
||||||
|
let mut infos = create_test_infos(1);
|
||||||
|
infos.all_infos[0].alive_bytes = ideal_storage_size_large + 1;
|
||||||
|
let max_storages = 2;
|
||||||
|
// none truncated because the one storage calculates to be larger than 1 ideal storage, so we need to
|
||||||
|
// combine
|
||||||
|
test(&mut infos, max_storages, ideal_storage_size_large);
|
||||||
|
assert_eq!(
|
||||||
|
infos
|
||||||
|
.all_infos
|
||||||
|
.iter()
|
||||||
|
.map(|info| info.slot)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
if filter { Vec::default() } else { vec![0] }
|
||||||
|
);
|
||||||
|
|
||||||
|
// both need to be combined to reach '1'
|
||||||
|
let max_storages = 1;
|
||||||
|
for ideal_storage_size in [1, 2] {
|
||||||
|
let mut infos = create_test_infos(2);
|
||||||
|
test(&mut infos, max_storages, ideal_storage_size);
|
||||||
|
assert_eq!(infos.all_infos.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
// max is 3
|
||||||
|
// 4 storages
|
||||||
|
// storage[3] is big enough to cause us to need another storage
|
||||||
|
// so, storage[0] and [1] can be combined into 1, resulting in 3 remaining storages, which is
|
||||||
|
// the goal, so we only have to combine the first 2 to hit the goal
|
||||||
|
let mut infos = create_test_infos(4);
|
||||||
|
infos.all_infos[3].alive_bytes = ideal_storage_size_large;
|
||||||
|
let max_storages = 3;
|
||||||
|
test(&mut infos, max_storages, ideal_storage_size_large);
|
||||||
|
assert_eq!(
|
||||||
|
infos
|
||||||
|
.all_infos
|
||||||
|
.iter()
|
||||||
|
.map(|info| info.slot)
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec![0, 1]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_calc_ancient_slot_info_one_shrink_one_not() {
|
fn test_calc_ancient_slot_info_one_shrink_one_not() {
|
||||||
let can_randomly_shrink = false;
|
let can_randomly_shrink = false;
|
||||||
|
|
Loading…
Reference in New Issue