add StorableAccountsBySlot to move accounts from multiple slots to 1 (#30023)

runtime/src/storable_accounts.rs

@@ -162,6 +162,110 @@ impl<'a> StorableAccounts<'a, StoredAccountMeta<'a>>
     }
 }
 
+/// holds slices of accounts being moved FROM a common source slot to 'target_slot'
+pub struct StorableAccountsBySlot<'a> {
+    target_slot: Slot,
+    /// each element is (source slot, accounts moving FROM source slot)
+    slots_and_accounts: &'a [(Slot, &'a [&'a StoredAccountMeta<'a>])],
+    include_slot_in_hash: IncludeSlotInHash,
+
+    /// This is calculated based off slots_and_accounts.
+    /// cumulative offset of all account slices prior to this one
+    /// starting_offsets[0] is the starting offset of slots_and_accounts[1]
+    /// The starting offset of slots_and_accounts[0] is always 0
+    starting_offsets: Vec<usize>,
+    /// true if there is more than 1 slot represented in slots_and_accounts
+    contains_multiple_slots: bool,
+    /// total len of all accounts, across all slots_and_accounts
+    len: usize,
+}
+
+impl<'a> StorableAccountsBySlot<'a> {
+    #[allow(dead_code)]
+    /// each element of slots_and_accounts is (source slot, accounts moving FROM source slot)
+    pub(crate) fn new(
+        target_slot: Slot,
+        slots_and_accounts: &'a [(Slot, &'a [&'a StoredAccountMeta<'a>])],
+        include_slot_in_hash: IncludeSlotInHash,
+    ) -> Self {
+        let mut cumulative_len = 0usize;
+        let mut starting_offsets = Vec::with_capacity(slots_and_accounts.len());
+        let first_slot = slots_and_accounts
+            .first()
+            .map(|(slot, _)| *slot)
+            .unwrap_or_default();
+        let mut contains_multiple_slots = false;
+        for (slot, accounts) in slots_and_accounts {
+            cumulative_len = cumulative_len.saturating_add(accounts.len());
+            starting_offsets.push(cumulative_len);
+            contains_multiple_slots |= &first_slot != slot;
+        }
+        Self {
+            target_slot,
+            slots_and_accounts,
+            starting_offsets,
+            include_slot_in_hash,
+            contains_multiple_slots,
+            len: cumulative_len,
+        }
+    }
+    /// given an overall index for all accounts in self:
+    /// return (slots_and_accounts index, index within those accounts)
+    fn find_internal_index(&self, index: usize) -> (usize, usize) {
+        // search offsets for the accounts slice that contains 'index'.
+        // This could be a binary search.
+        for (offset_index, next_offset) in self.starting_offsets.iter().enumerate() {
+            if next_offset > &index {
+                // offset of prior entry
+                let prior_offset = if offset_index > 0 {
+                    self.starting_offsets[offset_index.saturating_sub(1)]
+                } else {
+                    0
+                };
+                return (offset_index, index - prior_offset);
+            }
+        }
+        panic!("failed");
+    }
+}
+
+/// The last parameter exists until this feature is activated:
+///  ignore slot when calculating an account hash #28420
+impl<'a> StorableAccounts<'a, StoredAccountMeta<'a>> for StorableAccountsBySlot<'a> {
+    fn pubkey(&self, index: usize) -> &Pubkey {
+        self.account(index).pubkey()
+    }
+    fn account(&self, index: usize) -> &StoredAccountMeta<'a> {
+        let indexes = self.find_internal_index(index);
+        self.slots_and_accounts[indexes.0].1[indexes.1]
+    }
+    fn slot(&self, index: usize) -> Slot {
+        let indexes = self.find_internal_index(index);
+        self.slots_and_accounts[indexes.0].0
+    }
+    fn target_slot(&self) -> Slot {
+        self.target_slot
+    }
+    fn len(&self) -> usize {
+        self.len
+    }
+    fn contains_multiple_slots(&self) -> bool {
+        self.contains_multiple_slots
+    }
+    fn include_slot_in_hash(&self) -> IncludeSlotInHash {
+        self.include_slot_in_hash
+    }
+    fn has_hash_and_write_version(&self) -> bool {
+        true
+    }
+    fn hash(&self, index: usize) -> &Hash {
+        self.account(index).hash
+    }
+    fn write_version(&self, index: usize) -> u64 {
+        self.account(index).meta.write_version_obsolete
+    }
+}
+
 /// this tuple contains a single different source slot that applies to all accounts
 /// accounts are StoredAccountMeta
 impl<'a> StorableAccounts<'a, StoredAccountMeta<'a>>
@@ -349,16 +453,25 @@ pub mod tests {
                         old_slot,
                         include_slot_in_hash: INCLUDE_SLOT_IN_HASH_TESTS,
                     };
+                    let for_slice = [(old_slot, &three[..])];
+                    let test_moving_slots2 = StorableAccountsBySlot::new(
+                        target_slot,
+                        &for_slice,
+                        INCLUDE_SLOT_IN_HASH_TESTS,
+                    );
                     compare(&test2, &test3);
                     compare(&test2, &test_moving_slots);
+                    compare(&test2, &test_moving_slots2);
                     for (i, raw) in raw.iter().enumerate() {
                         assert_eq!(raw.0, *test3.pubkey(i));
                         assert!(accounts_equal(&raw.1, test3.account(i)));
                         assert_eq!(raw.2, test3.slot(i));
                         assert_eq!(target_slot, test2.slot(i));
                         assert_eq!(old_slot, test_moving_slots.slot(i));
+                        assert_eq!(old_slot, test_moving_slots2.slot(i));
                     }
                     assert_eq!(target_slot, test3.target_slot());
+                    assert_eq!(target_slot, test_moving_slots2.target_slot());
                     assert!(!test2.contains_multiple_slots());
                     assert!(!test_moving_slots.contains_multiple_slots());
                     assert_eq!(test3.contains_multiple_slots(), entries > 1);
@@ -366,4 +479,103 @@ pub mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_storable_accounts_by_slot() {
+        solana_logger::setup();
+        // slots 0..4
+        // each one containing a subset of the overall # of entries (0..4)
+        for entries in 0..6 {
+            let data = Vec::default();
+            let hashes = (0..entries).map(|_| Hash::new_unique()).collect::<Vec<_>>();
+            let mut raw = Vec::new();
+            let mut raw2 = Vec::new();
+            for entry in 0..entries {
+                let pk = Pubkey::from([entry; 32]);
+                let account = AccountSharedData::create(
+                    entry as u64,
+                    Vec::default(),
+                    Pubkey::default(),
+                    false,
+                    0,
+                );
+                raw.push((
+                    pk,
+                    account.clone(),
+                    StoredMeta {
+                        write_version_obsolete: 500 + (entry * 3) as u64, // just something
+                        pubkey: pk,
+                        data_len: (entry * 2) as u64, // just something
+                    },
+                    AccountMeta {
+                        lamports: account.lamports(),
+                        owner: *account.owner(),
+                        executable: account.executable(),
+                        rent_epoch: account.rent_epoch(),
+                    },
+                ));
+            }
+            for entry in 0..entries {
+                let offset = 99;
+                let stored_size = 101;
+                raw2.push(StoredAccountMeta {
+                    meta: &raw[entry as usize].2,
+                    account_meta: &raw[entry as usize].3,
+                    data: &data,
+                    offset,
+                    stored_size,
+                    hash: &hashes[entry as usize],
+                });
+            }
+            let raw2_refs = raw2.iter().collect::<Vec<_>>();
+
+            // enumerate through permutations of # entries (ie. accounts) in each slot. Each one is 0..=entries.
+            for entries0 in 0..=entries {
+                let remaining1 = entries.saturating_sub(entries0);
+                for entries1 in 0..=remaining1 {
+                    let remaining2 = entries.saturating_sub(entries0 + entries1);
+                    for entries2 in 0..=remaining2 {
+                        let remaining3 = entries.saturating_sub(entries0 + entries1 + entries2);
+                        let entries_by_level = vec![entries0, entries1, entries2, remaining3];
+                        let mut overall_index = 0;
+                        let mut expected_slots = Vec::default();
+                        let slots_and_accounts = entries_by_level
+                            .iter()
+                            .enumerate()
+                            .filter_map(|(slot, count)| {
+                                let slot = slot as Slot;
+                                let count = *count as usize;
+                                (overall_index < raw2.len()).then(|| {
+                                    let range = overall_index..(overall_index + count);
+                                    let result = &raw2_refs[range.clone()];
+                                    range.for_each(|_| expected_slots.push(slot));
+                                    overall_index += count;
+                                    (slot, result)
+                                })
+                            })
+                            .collect::<Vec<_>>();
+                        let storable = StorableAccountsBySlot::new(
+                            99,
+                            &slots_and_accounts[..],
+                            INCLUDE_SLOT_IN_HASH_TESTS,
+                        );
+                        assert!(storable.has_hash_and_write_version());
+                        assert_eq!(99, storable.target_slot());
+                        assert_eq!(entries0 != entries, storable.contains_multiple_slots());
+                        (0..entries).for_each(|index| {
+                            let index = index as usize;
+                            assert_eq!(storable.account(index), &raw2[index]);
+                            assert_eq!(storable.pubkey(index), raw2[index].pubkey());
+                            assert_eq!(storable.hash(index), raw2[index].hash);
+                            assert_eq!(storable.slot(index), expected_slots[index]);
+                            assert_eq!(
+                                storable.write_version(index),
+                                raw2[index].meta.write_version_obsolete
+                            );
+                        })
+                    }
+                }
+            }
+        }
+    }
 }