add shrink ancient test (#28858)

* add shrink ancient test

* improvements

runtime/src/accounts_db.rs

@@ -17707,6 +17707,142 @@ pub mod tests {
         }
     }
 
+    fn get_all_accounts(db: &AccountsDb, slots: Range<Slot>) -> Vec<(Pubkey, AccountSharedData)> {
+        slots
+            .clone()
+            .into_iter()
+            .filter_map(|slot| {
+                let storages = db.get_storages_for_slot(slot);
+                storages.map(|storages| {
+                    assert_eq!(storages.len(), 1, "slot: {}, slots: {:?}", slot, slots);
+                    let storage = storages.first().unwrap();
+                    storage
+                        .accounts
+                        .account_iter()
+                        .map(|account| (*account.pubkey(), account.to_account_shared_data()))
+                        .collect::<Vec<_>>()
+                })
+            })
+            .flatten()
+            .collect::<Vec<_>>()
+    }
+
+    fn compare_all_accounts(
+        one: &[(Pubkey, AccountSharedData)],
+        two: &[(Pubkey, AccountSharedData)],
+    ) {
+        let mut two_indexes = (0..two.len()).collect::<Vec<_>>();
+        one.iter().for_each(|(pubkey, account)| {
+            for i in 0..two_indexes.len() {
+                let pubkey2 = two[two_indexes[i]].0;
+                if pubkey2 == *pubkey {
+                    assert!(accounts_equal(account, &two[i].1));
+                    two_indexes.remove(i);
+                    break;
+                }
+            }
+        });
+        assert!(
+            two_indexes.is_empty(),
+            "one: {one:?}, two: {two:?}, two_indexes: {two_indexes:?}"
+        );
+    }
+
+    #[test]
+    fn test_shrink_ancient() {
+        solana_logger::setup();
+
+        let num_normal_slots = 1;
+        // build an ancient append vec at slot 'ancient_slot'
+        let (db, ancient_slot) = get_one_ancient_append_vec_and_others(true, num_normal_slots);
+
+        let max_slot_inclusive = ancient_slot + (num_normal_slots as Slot);
+        let initial_accounts = get_all_accounts(&db, ancient_slot..(max_slot_inclusive + 1));
+        compare_all_accounts(
+            &initial_accounts,
+            &get_all_accounts(&db, ancient_slot..(max_slot_inclusive + 1)),
+        );
+
+        // combine normal append vec(s) into existing ancient append vec
+        db.combine_ancient_slots(
+            (ancient_slot..=max_slot_inclusive).collect(),
+            CAN_RANDOMLY_SHRINK_FALSE,
+        );
+
+        compare_all_accounts(
+            &initial_accounts,
+            &get_all_accounts(&db, ancient_slot..(max_slot_inclusive + 1)),
+        );
+
+        // create a 2nd ancient append vec at 'next_slot'
+        let tf = crate::append_vec::test_utils::get_append_vec_path("test_shrink_ancient");
+        let next_slot = max_slot_inclusive + 1;
+        create_storages_and_update_index(&db, &tf, next_slot, num_normal_slots, true);
+        let max_slot_inclusive = next_slot + (num_normal_slots as Slot);
+
+        let initial_accounts = get_all_accounts(&db, ancient_slot..(max_slot_inclusive + 1));
+        compare_all_accounts(
+            &initial_accounts,
+            &get_all_accounts(&db, ancient_slot..(max_slot_inclusive + 1)),
+        );
+
+        db.combine_ancient_slots(
+            (next_slot..=max_slot_inclusive).collect(),
+            CAN_RANDOMLY_SHRINK_FALSE,
+        );
+
+        compare_all_accounts(
+            &initial_accounts,
+            &get_all_accounts(&db, ancient_slot..(max_slot_inclusive + 1)),
+        );
+
+        // now, shrink the second ancient append vec into the first one
+        let mut current_ancient = CurrentAncientAppendVec::new(
+            ancient_slot,
+            Arc::clone(
+                db.get_storages_for_slot(ancient_slot)
+                    .unwrap()
+                    .first()
+                    .unwrap(),
+            ),
+        );
+        let mut dropped_roots = Vec::default();
+        db.combine_one_store_into_ancient(
+            next_slot,
+            &db.get_storages_for_slot(next_slot).unwrap(),
+            &mut current_ancient,
+            &mut AncientSlotPubkeys::default(),
+            &mut dropped_roots,
+        );
+        assert!(db.get_storages_for_slot(next_slot).unwrap().is_empty());
+        // this removes the storages entry completely from the hashmap for 'next_slot'.
+        // Otherwise, we have a zero length vec in that hashmap
+        db.handle_dropped_roots_for_ancient(dropped_roots);
+        assert!(db.get_storages_for_slot(next_slot).is_none());
+
+        // include all the slots we put into the ancient append vec - they should contain nothing
+        compare_all_accounts(
+            &initial_accounts,
+            &get_all_accounts(&db, ancient_slot..(max_slot_inclusive + 1)),
+        );
+        // look at just the ancient append vec
+        compare_all_accounts(
+            &initial_accounts,
+            &get_all_accounts(&db, ancient_slot..(ancient_slot + 1)),
+        );
+        // make sure there is only 1 ancient append vec at the ancient slot
+        assert_eq!(db.get_storages_for_slot(ancient_slot).unwrap().len(), 1);
+        assert!(is_ancient(
+            &db.get_storages_for_slot(ancient_slot)
+                .unwrap()
+                .first()
+                .unwrap()
+                .accounts
+        ));
+        ((ancient_slot + 1)..=max_slot_inclusive)
+            .for_each(|slot| assert!(db.get_storages_for_slot(slot).is_none()));
+    }
+
     #[test]
     fn test_combine_ancient_slots_append() {
         solana_logger::setup();
@@ -17804,13 +17940,38 @@ pub mod tests {
         }
     }
 
+    fn populate_index(db: &AccountsDb, slots: Range<Slot>) {
+        slots.into_iter().for_each(|slot| {
+            if let Some(storages) = db.get_storages_for_slot(slot) {
+                storages.iter().for_each(|storage| {
+                    storage.accounts.account_iter().for_each(|account| {
+                        let info = AccountInfo::new(
+                            StorageLocation::AppendVec(storage.append_vec_id(), account.offset),
+                            account.stored_size as u32,
+                            account.lamports(),
+                        );
+                        db.accounts_index.upsert(
+                            slot,
+                            slot,
+                            account.pubkey(),
+                            &account,
+                            &AccountSecondaryIndexes::default(),
+                            info,
+                            &mut Vec::default(),
+                            UpsertReclaim::IgnoreReclaims,
+                        );
+                    })
+                })
+            }
+        })
+    }
+
     fn create_storages_and_update_index(
         db: &AccountsDb,
         tf: &TempFile,
-        starting_slot: usize,
+        starting_slot: Slot,
         num_slots: usize,
         alive: bool,
-        genesis_config: &GenesisConfig,
     ) {
         let write_version1 = 0;
         let starting_id = 999;
@@ -17820,7 +17981,7 @@ pub mod tests {
             let storages = sample_storage_with_entries_id(
                 tf,
                 write_version1,
-                (starting_slot + i) as Slot,
+                starting_slot + (i as Slot),
                 &pubkey1,
                 id,
             )
@@ -17829,13 +17990,12 @@ pub mod tests {
             insert_store(db, Arc::clone(&storages[0]));
         }
 
-        let starting_slot = starting_slot as Slot;
         let storages = db.get_storages_for_slot(starting_slot).unwrap();
         let created_accounts = db.get_unique_accounts_from_storages(storages.iter());
         assert_eq!(created_accounts.store_ids[0], starting_id);
 
         if alive {
-            db.generate_index(None, false, genesis_config);
+            populate_index(db, starting_slot..(starting_slot + (num_slots as Slot) + 1));
         }
     }
 
@@ -17848,13 +18008,12 @@ pub mod tests {
         // add the pubkey to index if alive
         // call combine_ancient_slots with the slot
         // verify we create an ancient appendvec that has alive accounts and does not have dead accounts
-        let genesis_config = solana_sdk::genesis_config::create_genesis_config(100).0;
 
         let slot1 = 1;
         let tf = crate::append_vec::test_utils::get_append_vec_path(
             "get_one_ancient_append_vec_and_others",
         );
-        create_storages_and_update_index(&db, &tf, slot1, num_slots, alive, &genesis_config);
+        create_storages_and_update_index(&db, &tf, slot1, num_slots, alive);
 
         let slot1 = slot1 as Slot;
         (db, slot1)