add ancient write_packed_storages (#30260)

runtime/src/ancient_append_vecs.rs

@@ -291,6 +291,37 @@ impl AccountsDb {
         infos
     }
 
+    /// write packed storages as described in 'accounts_to_combine'
+    /// and 'packed_contents'
+    #[allow(dead_code)]
+    fn write_packed_storages<'a, 'b>(
+        &'a self,
+        accounts_to_combine: &'b AccountsToCombine<'b>,
+        packed_contents: Vec<PackedAncientStorage<'b>>,
+    ) -> WriteAncientAccounts<'a> {
+        let mut write_ancient_accounts = WriteAncientAccounts::default();
+
+        // ok if we have more slots, but NOT ok if we have fewer slots than we have contents
+        assert!(accounts_to_combine.target_slots_sorted.len() >= packed_contents.len());
+        // write packed storages containing contents from many original slots
+        // iterate slots in highest to lowest
+        accounts_to_combine
+            .target_slots_sorted
+            .iter()
+            .rev()
+            .zip(packed_contents)
+            .for_each(|(target_slot, pack)| {
+                self.write_one_packed_storage(&pack, *target_slot, &mut write_ancient_accounts);
+            });
+
+        // write new storages where contents were unable to move because ref_count > 1
+        self.write_ancient_accounts_to_same_slot_multiple_refs(
+            accounts_to_combine.accounts_keep_slots.values(),
+            &mut write_ancient_accounts,
+        );
+        write_ancient_accounts
+    }
+
     /// for each slot in 'ancient_slots', collect all accounts in that slot
     /// return the collection of accounts by slot
     #[allow(dead_code)]
@@ -683,6 +714,36 @@ pub mod tests {
         compare_all_accounts(&unique_to_accounts(one), &unique_to_accounts(two));
     }
 
+    #[test]
+    fn test_write_packed_storages_empty() {
+        let (db, _storages, _slots, _infos) = get_sample_storages(0, None);
+        let write_ancient_accounts =
+            db.write_packed_storages(&AccountsToCombine::default(), Vec::default());
+        assert!(write_ancient_accounts.shrinks_in_progress.is_empty());
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "assertion failed: accounts_to_combine.target_slots_sorted.len() >= packed_contents.len()"
+    )]
+    fn test_write_packed_storages_too_few_slots() {
+        let (db, storages, slots, _infos) = get_sample_storages(1, None);
+        let accounts_to_combine = AccountsToCombine::default();
+        let accounts = [storages
+            .first()
+            .unwrap()
+            .accounts
+            .accounts(0)
+            .pop()
+            .unwrap()];
+        let accounts = accounts.iter().collect::<Vec<_>>();
+        let packed_contents = vec![PackedAncientStorage {
+            bytes: 0,
+            accounts: vec![(slots.start, &accounts[..])],
+        }];
+        db.write_packed_storages(&accounts_to_combine, packed_contents);
+    }
+
     #[test]
     fn test_write_ancient_accounts_to_same_slot_multiple_refs_empty() {
         let (db, _storages, _slots, _infos) = get_sample_storages(0, None);
@@ -982,11 +1043,18 @@ pub mod tests {
             .collect::<Vec<_>>()
     }
 
+    #[derive(EnumIter, Debug, PartialEq, Eq)]
+    enum TestWriteMultipleRefs {
+        MultipleRefs,
+        PackedStorages,
+    }
+
     #[test]
     fn test_calc_accounts_to_combine_simple() {
         // n storages
         // 1 account each
         // all accounts have 1 ref or all accounts have 2 refs
+        for method in TestWriteMultipleRefs::iter() {
             for num_slots in 0..3 {
                 for two_refs in [false, true] {
                     let (db, storages, slots, infos) = get_sample_storages(num_slots, None);
@@ -1024,50 +1092,55 @@ pub mod tests {
                         assert_eq!(accounts_keep, slots_vec);
                         assert!(accounts_to_combine.target_slots_sorted.is_empty());
                         assert_eq!(accounts_to_combine.accounts_keep_slots.len(), num_slots);
-                    assert!(accounts_to_combine
+                        assert!(accounts_to_combine.accounts_to_combine.iter().all(
-                        .accounts_to_combine
+                            |shrink_collect| shrink_collect
-                        .iter()
-                        .all(|shrink_collect| shrink_collect
                                 .alive_accounts
                                 .one_ref
                                 .accounts
-                            .is_empty()));
+                                .is_empty()
-                    assert!(accounts_to_combine
+                        ));
-                        .accounts_to_combine
+                        assert!(accounts_to_combine.accounts_to_combine.iter().all(
-                        .iter()
+                            |shrink_collect| shrink_collect
-                        .all(|shrink_collect| shrink_collect
                                 .alive_accounts
                                 .many_refs
                                 .accounts
-                            .is_empty()));
+                                .is_empty()
+                        ));
                     } else {
                         // all accounts should be in one_ref and all slots are available as target slots
                         assert_eq!(accounts_to_combine.target_slots_sorted, slots_vec);
                         assert!(accounts_to_combine.accounts_keep_slots.is_empty());
-                    assert!(accounts_to_combine
+                        assert!(accounts_to_combine.accounts_to_combine.iter().all(
-                        .accounts_to_combine
+                            |shrink_collect| !shrink_collect
-                        .iter()
-                        .all(|shrink_collect| !shrink_collect
                                 .alive_accounts
                                 .one_ref
                                 .accounts
-                            .is_empty()));
+                                .is_empty()
-                    assert!(accounts_to_combine
+                        ));
-                        .accounts_to_combine
+                        assert!(accounts_to_combine.accounts_to_combine.iter().all(
-                        .iter()
+                            |shrink_collect| shrink_collect
-                        .all(|shrink_collect| shrink_collect
                                 .alive_accounts
                                 .many_refs
                                 .accounts
-                            .is_empty()));
+                                .is_empty()
+                        ));
                     }
 
                     // test write_ancient_accounts_to_same_slot_multiple_refs since we built interesting 'AccountsToCombine'
+                    let write_ancient_accounts = match method {
+                        TestWriteMultipleRefs::MultipleRefs => {
                             let mut write_ancient_accounts = WriteAncientAccounts::default();
                             db.write_ancient_accounts_to_same_slot_multiple_refs(
                                 accounts_to_combine.accounts_keep_slots.values(),
                                 &mut write_ancient_accounts,
                             );
+                            write_ancient_accounts
+                        }
+                        TestWriteMultipleRefs::PackedStorages => {
+                            let packed_contents = Vec::default();
+                            db.write_packed_storages(&accounts_to_combine, packed_contents)
+                        }
+                    };
                     if two_refs {
                         assert_eq!(write_ancient_accounts.shrinks_in_progress.len(), num_slots);
                         let mut shrinks_in_progress = write_ancient_accounts
@@ -1100,6 +1173,7 @@ pub mod tests {
                 }
             }
         }
+    }
 
     #[test]
     fn test_calc_accounts_to_combine_opposite() {
@@ -1107,6 +1181,7 @@ pub mod tests {
         // 2 accounts
         // 1 with 1 ref
         // 1 with 2 refs
+        for method in TestWriteMultipleRefs::iter() {
             let num_slots = 1;
             let (db, storages, slots, infos) = get_sample_storages(num_slots, None);
             let original_results = storages
@@ -1206,11 +1281,20 @@ pub mod tests {
                 .all(|shrink_collect| shrink_collect.alive_accounts.many_refs.accounts.is_empty()));
 
             // test write_ancient_accounts_to_same_slot_multiple_refs since we built interesting 'AccountsToCombine'
+            let write_ancient_accounts = match method {
+                TestWriteMultipleRefs::MultipleRefs => {
                     let mut write_ancient_accounts = WriteAncientAccounts::default();
                     db.write_ancient_accounts_to_same_slot_multiple_refs(
                         accounts_to_combine.accounts_keep_slots.values(),
                         &mut write_ancient_accounts,
                     );
+                    write_ancient_accounts
+                }
+                TestWriteMultipleRefs::PackedStorages => {
+                    let packed_contents = Vec::default();
+                    db.write_packed_storages(&accounts_to_combine, packed_contents)
+                }
+            };
             assert_eq!(write_ancient_accounts.shrinks_in_progress.len(), num_slots);
             let mut shrinks_in_progress = write_ancient_accounts
                 .shrinks_in_progress
@@ -1250,6 +1334,7 @@ pub mod tests {
                 account_with_2_refs.to_account_shared_data()
             );
         }
+    }
 
     #[test]
     fn test_get_unique_accounts_from_storage_for_combining_ancient_slots() {
@@ -1941,6 +2026,7 @@ pub mod tests {
     enum TestWriteAncient {
         OnePackedStorage,
         AncientAccounts,
+        PackedStorages,
     }
 
     #[test]
@@ -2009,6 +2095,22 @@ pub mod tests {
                                         &mut write_ancient_accounts,
                                     );
                                 }
+                                TestWriteAncient::PackedStorages => {
+                                    let packed = PackedAncientStorage { accounts, bytes };
+
+                                    let accounts_to_combine = AccountsToCombine {
+                                        // target slots are supposed to be read in reverse order, so test that
+                                        target_slots_sorted: vec![
+                                            Slot::MAX, // this asserts if it gets used
+                                            Slot::MAX,
+                                            target_slot,
+                                        ],
+                                        ..AccountsToCombine::default()
+                                    };
+
+                                    write_ancient_accounts = db
+                                        .write_packed_storages(&accounts_to_combine, vec![packed]);
+                                }
                             };
                             let mut result = write_ancient_accounts.shrinks_in_progress;
                             let one = result.drain().collect::<Vec<_>>();