diff --git a/runtime/src/ancient_append_vecs.rs b/runtime/src/ancient_append_vecs.rs index b20d2998f7..74d86dc4e0 100644 --- a/runtime/src/ancient_append_vecs.rs +++ b/runtime/src/ancient_append_vecs.rs @@ -25,6 +25,8 @@ struct PackedAncientStorageTuning { max_ancient_slots: usize, /// # of bytes in an ideal ancient storage size ideal_storage_size: NonZeroU64, + /// true if storages can be randomly shrunk even if they aren't eligible + can_randomly_shrink: bool, } /// info about a storage eligible to be combined into an ancient append vec. @@ -203,6 +205,20 @@ impl AncientSlotInfos { } impl AccountsDb { + /// calculate all storage info for the storages in slots + /// prune the ones to ignore given tuning parameters + #[allow(dead_code)] + fn collect_sort_filter_info( + &self, + slots: Vec, + tuning: PackedAncientStorageTuning, + ) -> AncientSlotInfos { + let mut ancient_slot_infos = self.calc_ancient_slot_info(slots, tuning.can_randomly_shrink); + + ancient_slot_infos.filter_ancient_slots(&tuning); + ancient_slot_infos + } + /// go through all slots and populate 'SlotInfo', per slot /// This provides the list of possible ancient slots to sort, filter, and then combine. #[allow(dead_code)] @@ -556,30 +572,57 @@ pub mod tests { assert_eq!(should_shrink, info.should_shrink); } + #[derive(EnumIter, Debug, PartialEq, Eq)] + enum TestCollectInfo { + CollectSortFilterInfo, + CalcAncientSlotInfo, + Add, + } + #[test] - fn test_calc_ancient_slot_info_one_alive() { + fn test_calc_ancient_slot_info_one_alive_only() { let can_randomly_shrink = false; let alive = true; let slots = 1; - for call_add in [false, true] { + for method in TestCollectInfo::iter() { // 1_040_000 is big enough relative to page size to cause shrink ratio to be triggered for data_size in [None, Some(1_040_000)] { let (db, slot1) = create_db_with_storages_and_index(alive, slots, data_size); let mut infos = AncientSlotInfos::default(); let storage = db.storage.get_slot_storage_entry(slot1).unwrap(); let alive_bytes_expected = storage.alive_bytes(); - if call_add { - // test lower level 'add' - infos.add(slot1, Arc::clone(&storage), can_randomly_shrink); - } else { - infos = db.calc_ancient_slot_info(vec![slot1], can_randomly_shrink); + match method { + TestCollectInfo::Add => { + // test lower level 'add' + infos.add(slot1, Arc::clone(&storage), can_randomly_shrink); + } + TestCollectInfo::CalcAncientSlotInfo => { + infos = db.calc_ancient_slot_info(vec![slot1], can_randomly_shrink); + } + TestCollectInfo::CollectSortFilterInfo => { + let tuning = PackedAncientStorageTuning { + percent_of_alive_shrunk_data: 100, + max_ancient_slots: 0, + // irrelevant + ideal_storage_size: NonZeroU64::new(1).unwrap(), + can_randomly_shrink, + }; + infos = db.collect_sort_filter_info(vec![slot1], tuning); + } } assert_eq!(infos.all_infos.len(), 1); let should_shrink = data_size.is_none(); assert_storage_info(infos.all_infos.first().unwrap(), &storage, should_shrink); if should_shrink { // data size is so small compared to min aligned file size that the storage is marked as should_shrink - assert_eq!(infos.shrink_indexes, vec![0]); + assert_eq!( + infos.shrink_indexes, + if !matches!(method, TestCollectInfo::CollectSortFilterInfo) { + vec![0] + } else { + Vec::default() + } + ); assert_eq!(infos.total_alive_bytes, alive_bytes_expected as u64); assert_eq!(infos.total_alive_bytes_shrink, alive_bytes_expected as u64); } else { @@ -670,54 +713,84 @@ pub mod tests { #[test] fn test_calc_ancient_slot_info_one_alive_one_dead() { let can_randomly_shrink = false; - for slot1_is_alive in [false, true] { - let alives = vec![false /*dummy*/, slot1_is_alive, !slot1_is_alive]; - let slots = 2; - // 1_040_000 is big enough relative to page size to cause shrink ratio to be triggered - for data_size in [None, Some(1_040_000)] { - let (db, slot1) = - create_db_with_storages_and_index(true /*alive*/, slots, data_size); - assert_eq!(slot1, 1); // make sure index into alives will be correct - assert_eq!(alives[slot1 as usize], slot1_is_alive); - let slot_vec = (slot1..(slot1 + slots as Slot)).collect::>(); - let storages = slot_vec - .iter() - .map(|slot| db.storage.get_slot_storage_entry(*slot).unwrap()) - .collect::>(); - storages.iter().for_each(|storage| { - let slot = storage.slot(); - let alive = alives[slot as usize]; - if !alive { - // make this storage not alive - remove_account_for_tests(storage, storage.written_bytes() as usize, false); - } - }); - let alive_storages = storages - .iter() - .filter_map(|storage| alives[storage.slot() as usize].then_some(storage)) - .collect::>(); - let alive_bytes_expected = alive_storages - .iter() - .map(|storage| storage.alive_bytes() as u64) - .sum::(); - let infos = db.calc_ancient_slot_info(slot_vec.clone(), can_randomly_shrink); - assert_eq!(infos.all_infos.len(), 1); - let should_shrink = data_size.is_none(); - alive_storages - .iter() - .zip(infos.all_infos.iter()) - .for_each(|(storage, info)| { - assert_storage_info(info, storage, should_shrink); + for method in TestCollectInfo::iter() { + for slot1_is_alive in [false, true] { + let alives = vec![false /*dummy*/, slot1_is_alive, !slot1_is_alive]; + let slots = 2; + // 1_040_000 is big enough relative to page size to cause shrink ratio to be triggered + for data_size in [None, Some(1_040_000)] { + let (db, slot1) = + create_db_with_storages_and_index(true /*alive*/, slots, data_size); + assert_eq!(slot1, 1); // make sure index into alives will be correct + assert_eq!(alives[slot1 as usize], slot1_is_alive); + let slot_vec = (slot1..(slot1 + slots as Slot)).collect::>(); + let storages = slot_vec + .iter() + .map(|slot| db.storage.get_slot_storage_entry(*slot).unwrap()) + .collect::>(); + storages.iter().for_each(|storage| { + let slot = storage.slot(); + let alive = alives[slot as usize]; + if !alive { + // make this storage not alive + remove_account_for_tests( + storage, + storage.written_bytes() as usize, + false, + ); + } }); - if should_shrink { - // data size is so small compared to min aligned file size that the storage is marked as should_shrink - assert_eq!(infos.shrink_indexes, vec![0]); - assert_eq!(infos.total_alive_bytes, alive_bytes_expected); - assert_eq!(infos.total_alive_bytes_shrink, alive_bytes_expected); - } else { - assert!(infos.shrink_indexes.is_empty()); - assert_eq!(infos.total_alive_bytes, alive_bytes_expected); - assert_eq!(infos.total_alive_bytes_shrink, 0); + let alive_storages = storages + .iter() + .filter_map(|storage| alives[storage.slot() as usize].then_some(storage)) + .collect::>(); + let alive_bytes_expected = alive_storages + .iter() + .map(|storage| storage.alive_bytes() as u64) + .sum::(); + + let infos = match method { + TestCollectInfo::CalcAncientSlotInfo => { + db.calc_ancient_slot_info(slot_vec.clone(), can_randomly_shrink) + } + TestCollectInfo::Add => { + continue; // unsupportable + } + TestCollectInfo::CollectSortFilterInfo => { + let tuning = PackedAncientStorageTuning { + percent_of_alive_shrunk_data: 100, + max_ancient_slots: 0, + // irrelevant + ideal_storage_size: NonZeroU64::new(1).unwrap(), + can_randomly_shrink, + }; + db.collect_sort_filter_info(slot_vec.clone(), tuning) + } + }; + assert_eq!(infos.all_infos.len(), 1, "method: {method:?}"); + let should_shrink = data_size.is_none(); + alive_storages.iter().zip(infos.all_infos.iter()).for_each( + |(storage, info)| { + assert_storage_info(info, storage, should_shrink); + }, + ); + if should_shrink { + // data size is so small compared to min aligned file size that the storage is marked as should_shrink + assert_eq!( + infos.shrink_indexes, + if !matches!(method, TestCollectInfo::CollectSortFilterInfo) { + vec![0] + } else { + Vec::default() + } + ); + assert_eq!(infos.total_alive_bytes, alive_bytes_expected); + assert_eq!(infos.total_alive_bytes_shrink, alive_bytes_expected); + } else { + assert!(infos.shrink_indexes.is_empty()); + assert_eq!(infos.total_alive_bytes, alive_bytes_expected); + assert_eq!(infos.total_alive_bytes_shrink, 0); + } } } } @@ -761,6 +834,7 @@ pub mod tests { ideal_storage_size: NonZeroU64::new(ideal_storage_size_large).unwrap(), // irrelevant since we clear 'shrink_indexes' percent_of_alive_shrunk_data: 0, + can_randomly_shrink: false, }; infos.shrink_indexes.clear(); infos.filter_ancient_slots(&tuning); @@ -811,6 +885,7 @@ pub mod tests { ideal_storage_size: NonZeroU64::new(ideal_storage_size_large).unwrap(), // irrelevant since we clear 'shrink_indexes' percent_of_alive_shrunk_data: 0, + can_randomly_shrink: false, }; infos.shrink_indexes.clear(); infos.filter_ancient_slots(&tuning); @@ -929,61 +1004,86 @@ pub mod tests { #[test] fn test_calc_ancient_slot_info_one_shrink_one_not() { let can_randomly_shrink = false; - for slot1_shrink in [false, true] { - let shrinks = vec![false /*dummy*/, slot1_shrink, !slot1_shrink]; - let slots = 2; - // 1_040_000 is big enough relative to page size to cause shrink ratio to be triggered - let data_sizes = shrinks - .iter() - .map(|shrink| (!shrink).then_some(1_040_000)) - .collect::>(); - let (db, slot1) = - create_db_with_storages_and_index(true /*alive*/, 1, data_sizes[1]); - let dead_bytes = 184; // constant based on None data size - create_storages_and_update_index( - &db, - None, - slot1 + 1, - 1, - true, - data_sizes[(slot1 + 1) as usize], - ); - - assert_eq!(slot1, 1); // make sure index into shrinks will be correct - assert_eq!(shrinks[slot1 as usize], slot1_shrink); - let slot_vec = (slot1..(slot1 + slots as Slot)).collect::>(); - let storages = slot_vec - .iter() - .map(|slot| { - let storage = db.storage.get_slot_storage_entry(*slot).unwrap(); - assert_eq!(*slot, storage.slot()); - storage - }) - .collect::>(); - let alive_bytes_expected = storages - .iter() - .map(|storage| storage.alive_bytes() as u64) - .sum::(); - let infos = db.calc_ancient_slot_info(slot_vec.clone(), can_randomly_shrink); - assert_eq!(infos.all_infos.len(), 2); - storages - .iter() - .zip(infos.all_infos.iter()) - .for_each(|(storage, info)| { - assert_storage_info(info, storage, shrinks[storage.slot() as usize]); - }); - // data size is so small compared to min aligned file size that the storage is marked as should_shrink - assert_eq!( - infos.shrink_indexes, - shrinks + for method in TestCollectInfo::iter() { + for slot1_shrink in [false, true] { + let shrinks = vec![false /*dummy*/, slot1_shrink, !slot1_shrink]; + let slots = 2; + // 1_040_000 is big enough relative to page size to cause shrink ratio to be triggered + let data_sizes = shrinks .iter() - .skip(1) - .enumerate() - .filter_map(|(i, shrink)| shrink.then_some(i)) - .collect::>() - ); - assert_eq!(infos.total_alive_bytes, alive_bytes_expected); - assert_eq!(infos.total_alive_bytes_shrink, dead_bytes); + .map(|shrink| (!shrink).then_some(1_040_000)) + .collect::>(); + let (db, slot1) = + create_db_with_storages_and_index(true /*alive*/, 1, data_sizes[1]); + let dead_bytes = 184; // constant based on None data size + create_storages_and_update_index( + &db, + None, + slot1 + 1, + 1, + true, + data_sizes[(slot1 + 1) as usize], + ); + + assert_eq!(slot1, 1); // make sure index into shrinks will be correct + assert_eq!(shrinks[slot1 as usize], slot1_shrink); + let slot_vec = (slot1..(slot1 + slots as Slot)).collect::>(); + let storages = slot_vec + .iter() + .map(|slot| { + let storage = db.storage.get_slot_storage_entry(*slot).unwrap(); + assert_eq!(*slot, storage.slot()); + storage + }) + .collect::>(); + let alive_bytes_expected = storages + .iter() + .map(|storage| storage.alive_bytes() as u64) + .sum::(); + let infos = match method { + TestCollectInfo::CalcAncientSlotInfo => { + db.calc_ancient_slot_info(slot_vec.clone(), can_randomly_shrink) + } + TestCollectInfo::Add => { + continue; // unsupportable + } + TestCollectInfo::CollectSortFilterInfo => { + let tuning = PackedAncientStorageTuning { + percent_of_alive_shrunk_data: 100, + max_ancient_slots: 0, + // irrelevant + ideal_storage_size: NonZeroU64::new(1).unwrap(), + can_randomly_shrink, + }; + // note this can sort infos.all_infos + db.collect_sort_filter_info(slot_vec.clone(), tuning) + } + }; + + assert_eq!(infos.all_infos.len(), 2); + storages.iter().for_each(|storage| { + assert!(infos + .all_infos + .iter() + .any(|info| info.slot == storage.slot())); + }); + // data size is so small compared to min aligned file size that the storage is marked as should_shrink + assert_eq!( + infos.shrink_indexes, + if matches!(method, TestCollectInfo::CollectSortFilterInfo) { + Vec::default() + } else { + shrinks + .iter() + .skip(1) + .enumerate() + .filter_map(|(i, shrink)| shrink.then_some(i)) + .collect::>() + } + ); + assert_eq!(infos.total_alive_bytes, alive_bytes_expected); + assert_eq!(infos.total_alive_bytes_shrink, dead_bytes); + } } } @@ -1047,6 +1147,7 @@ pub mod tests { max_ancient_slots: 0, // this is irrelevant since the limit is artificially 0 ideal_storage_size: NonZeroU64::new(1).unwrap(), + can_randomly_shrink: false, }; infos.filter_ancient_slots(&tuning); }