More serde snapshot cleanup (#22449)

runtime/src/serde_snapshot.rs

@@ -3,7 +3,7 @@ use {
         accounts::Accounts,
         accounts_db::{
             AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId,
-            AtomicAppendVecId, BankHashInfo, IndexGenerationInfo,
+            AtomicAppendVecId, BankHashInfo, IndexGenerationInfo, SnapshotStorage,
         },
         accounts_index::AccountSecondaryIndexes,
         accounts_update_notifier_interface::AccountsUpdateNotifier,
@@ -14,7 +14,6 @@ use {
         epoch_stakes::EpochStakes,
         hardened_unpack::UnpackedAppendVecMap,
         rent_collector::RentCollector,
-        serde_snapshot::newer::{AppendVecIdSerialized, SerializableStorage},
         stakes::Stakes,
     },
     bincode::{self, config::Options, Error},
@@ -47,15 +46,15 @@ use {
 
 mod common;
 mod newer;
+mod storage;
 mod tests;
 mod utils;
 
+use storage::{SerializableStorage, SerializedAppendVecId};
+
 // a number of test cases in accounts_db use this
 #[cfg(test)]
-pub(crate) use self::tests::reconstruct_accounts_db_via_serialization;
-pub(crate) use crate::accounts_db::{SnapshotStorage, SnapshotStorages};
-#[allow(unused_imports)]
-use utils::{serialize_iter_as_map, serialize_iter_as_seq, serialize_iter_as_tuple};
+pub(crate) use tests::reconstruct_accounts_db_via_serialization;
 
 #[derive(Copy, Clone, Eq, PartialEq)]
 pub(crate) enum SerdeStyle {
@@ -465,7 +464,7 @@ where
                     //    rename the file to this new path.
                     //    **DEVELOPER NOTE:**  Keep this check last so that it can short-circuit if
                     //    possible.
-                    if storage_entry.id() == remapped_append_vec_id as AppendVecIdSerialized
+                    if storage_entry.id() == remapped_append_vec_id as SerializedAppendVecId
                         || std::fs::metadata(&remapped_append_vec_path).is_err()
                     {
                         break (remapped_append_vec_id, remapped_append_vec_path);
@@ -476,7 +475,7 @@ where
                     num_collisions.fetch_add(1, Ordering::Relaxed);
                 };
                 // Only rename the file if the new ID is actually different from the original.
-                if storage_entry.id() != remapped_append_vec_id as AppendVecIdSerialized {
+                if storage_entry.id() != remapped_append_vec_id as SerializedAppendVecId {
                     std::fs::rename(append_vec_path, &remapped_append_vec_path)?;
                 }
 

runtime/src/serde_snapshot/newer.rs

@@ -1,5 +1,10 @@
 use {
-    super::{common::UnusedAccounts, *},
+    super::{
+        common::UnusedAccounts,
+        storage::SerializableAccountStorageEntry,
+        utils::{serialize_iter_as_map, serialize_iter_as_seq},
+        *,
+    },
     crate::{ancestors::AncestorsForSerialization, stakes::StakesCache},
     solana_measure::measure::Measure,
     std::{cell::RefCell, sync::RwLock},
@@ -7,80 +12,44 @@ use {
 
 type AccountsDbFields = super::AccountsDbFields<SerializableAccountStorageEntry>;
 
-/// the serialized type is fixed as usize
-pub type AppendVecIdSerialized = usize;
-
-// Serializable version of AccountStorageEntry for snapshot format
-#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
-pub(super) struct SerializableAccountStorageEntry {
-    id: AppendVecIdSerialized,
-    accounts_current_len: usize,
-}
-
-pub trait SerializableStorage {
-    fn id(&self) -> AppendVecIdSerialized;
-    fn current_len(&self) -> usize;
-}
-
-impl SerializableStorage for SerializableAccountStorageEntry {
-    fn id(&self) -> AppendVecIdSerialized {
-        self.id
-    }
-    fn current_len(&self) -> usize {
-        self.accounts_current_len
-    }
-}
-
-#[cfg(RUSTC_WITH_SPECIALIZATION)]
-impl solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableAccountStorageEntry {}
-
-impl From<&AccountStorageEntry> for SerializableAccountStorageEntry {
-    fn from(rhs: &AccountStorageEntry) -> Self {
-        Self {
-            id: rhs.append_vec_id() as AppendVecIdSerialized,
-            accounts_current_len: rhs.accounts.len(),
-        }
-    }
-}
-
 // Deserializable version of Bank which need not be serializable,
 // because it's handled by SerializableVersionedBank.
 // So, sync fields with it!
 #[derive(Clone, Deserialize)]
-pub(crate) struct DeserializableVersionedBank {
-    pub(crate) blockhash_queue: BlockhashQueue,
-    pub(crate) ancestors: AncestorsForSerialization,
-    pub(crate) hash: Hash,
-    pub(crate) parent_hash: Hash,
-    pub(crate) parent_slot: Slot,
-    pub(crate) hard_forks: HardForks,
-    pub(crate) transaction_count: u64,
-    pub(crate) tick_height: u64,
-    pub(crate) signature_count: u64,
-    pub(crate) capitalization: u64,
-    pub(crate) max_tick_height: u64,
-    pub(crate) hashes_per_tick: Option<u64>,
-    pub(crate) ticks_per_slot: u64,
-    pub(crate) ns_per_slot: u128,
-    pub(crate) genesis_creation_time: UnixTimestamp,
-    pub(crate) slots_per_year: f64,
-    pub(crate) unused: u64,
-    pub(crate) slot: Slot,
-    pub(crate) epoch: Epoch,
-    pub(crate) block_height: u64,
-    pub(crate) collector_id: Pubkey,
-    pub(crate) collector_fees: u64,
-    pub(crate) fee_calculator: FeeCalculator,
-    pub(crate) fee_rate_governor: FeeRateGovernor,
-    pub(crate) collected_rent: u64,
-    pub(crate) rent_collector: RentCollector,
-    pub(crate) epoch_schedule: EpochSchedule,
-    pub(crate) inflation: Inflation,
-    pub(crate) stakes: Stakes,
+struct DeserializableVersionedBank {
+    blockhash_queue: BlockhashQueue,
+    ancestors: AncestorsForSerialization,
+    hash: Hash,
+    parent_hash: Hash,
+    parent_slot: Slot,
+    hard_forks: HardForks,
+    transaction_count: u64,
+    tick_height: u64,
+    signature_count: u64,
+    capitalization: u64,
+    max_tick_height: u64,
+    hashes_per_tick: Option<u64>,
+    ticks_per_slot: u64,
+    ns_per_slot: u128,
+    genesis_creation_time: UnixTimestamp,
+    slots_per_year: f64,
+    unused: u64,
+    slot: Slot,
+    epoch: Epoch,
+    block_height: u64,
+    collector_id: Pubkey,
+    collector_fees: u64,
+    fee_calculator: FeeCalculator,
+    fee_rate_governor: FeeRateGovernor,
+    collected_rent: u64,
+    rent_collector: RentCollector,
+    epoch_schedule: EpochSchedule,
+    inflation: Inflation,
+    stakes: Stakes,
     #[allow(dead_code)]
-    pub(crate) unused_accounts: UnusedAccounts,
-    pub(crate) epoch_stakes: HashMap<Epoch, EpochStakes>,
-    pub(crate) is_delta: bool,
+    unused_accounts: UnusedAccounts,
+    epoch_stakes: HashMap<Epoch, EpochStakes>,
+    is_delta: bool,
 }
 
 impl From<DeserializableVersionedBank> for BankFieldsToDeserialize {
@@ -124,39 +93,39 @@ impl From<DeserializableVersionedBank> for BankFieldsToDeserialize {
 // Serializable version of Bank, not Deserializable to avoid cloning by using refs.
 // Sync fields with DeserializableVersionedBank!
 #[derive(Serialize)]
-pub(crate) struct SerializableVersionedBank<'a> {
-    pub(crate) blockhash_queue: &'a RwLock<BlockhashQueue>,
-    pub(crate) ancestors: &'a AncestorsForSerialization,
-    pub(crate) hash: Hash,
-    pub(crate) parent_hash: Hash,
-    pub(crate) parent_slot: Slot,
-    pub(crate) hard_forks: &'a RwLock<HardForks>,
-    pub(crate) transaction_count: u64,
-    pub(crate) tick_height: u64,
-    pub(crate) signature_count: u64,
-    pub(crate) capitalization: u64,
-    pub(crate) max_tick_height: u64,
-    pub(crate) hashes_per_tick: Option<u64>,
-    pub(crate) ticks_per_slot: u64,
-    pub(crate) ns_per_slot: u128,
-    pub(crate) genesis_creation_time: UnixTimestamp,
-    pub(crate) slots_per_year: f64,
-    pub(crate) unused: u64,
-    pub(crate) slot: Slot,
-    pub(crate) epoch: Epoch,
-    pub(crate) block_height: u64,
-    pub(crate) collector_id: Pubkey,
-    pub(crate) collector_fees: u64,
-    pub(crate) fee_calculator: FeeCalculator,
-    pub(crate) fee_rate_governor: FeeRateGovernor,
-    pub(crate) collected_rent: u64,
-    pub(crate) rent_collector: RentCollector,
-    pub(crate) epoch_schedule: EpochSchedule,
-    pub(crate) inflation: Inflation,
-    pub(crate) stakes: &'a StakesCache,
-    pub(crate) unused_accounts: UnusedAccounts,
-    pub(crate) epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
-    pub(crate) is_delta: bool,
+struct SerializableVersionedBank<'a> {
+    blockhash_queue: &'a RwLock<BlockhashQueue>,
+    ancestors: &'a AncestorsForSerialization,
+    hash: Hash,
+    parent_hash: Hash,
+    parent_slot: Slot,
+    hard_forks: &'a RwLock<HardForks>,
+    transaction_count: u64,
+    tick_height: u64,
+    signature_count: u64,
+    capitalization: u64,
+    max_tick_height: u64,
+    hashes_per_tick: Option<u64>,
+    ticks_per_slot: u64,
+    ns_per_slot: u128,
+    genesis_creation_time: UnixTimestamp,
+    slots_per_year: f64,
+    unused: u64,
+    slot: Slot,
+    epoch: Epoch,
+    block_height: u64,
+    collector_id: Pubkey,
+    collector_fees: u64,
+    fee_calculator: FeeCalculator,
+    fee_rate_governor: FeeRateGovernor,
+    collected_rent: u64,
+    rent_collector: RentCollector,
+    epoch_schedule: EpochSchedule,
+    inflation: Inflation,
+    stakes: &'a StakesCache,
+    unused_accounts: UnusedAccounts,
+    epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
+    is_delta: bool,
 }
 
 impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedBank<'a> {
@@ -205,6 +174,7 @@ impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedB
 impl<'a> solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableVersionedBank<'a> {}
 
 pub(super) struct Context {}
+
 impl<'a> TypeContext<'a> for Context {
     type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
 

runtime/src/serde_snapshot/storage.rs

@@ -0,0 +1,40 @@
+use {
+    crate::accounts_db::AccountStorageEntry,
+    serde::{Deserialize, Serialize},
+};
+
+/// The serialized AppendVecId type is fixed as usize
+pub(super) type SerializedAppendVecId = usize;
+
+// Serializable version of AccountStorageEntry for snapshot format
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+pub(super) struct SerializableAccountStorageEntry {
+    id: SerializedAppendVecId,
+    accounts_current_len: usize,
+}
+
+pub(super) trait SerializableStorage {
+    fn id(&self) -> SerializedAppendVecId;
+    fn current_len(&self) -> usize;
+}
+
+impl SerializableStorage for SerializableAccountStorageEntry {
+    fn id(&self) -> SerializedAppendVecId {
+        self.id
+    }
+    fn current_len(&self) -> usize {
+        self.accounts_current_len
+    }
+}
+
+impl From<&AccountStorageEntry> for SerializableAccountStorageEntry {
+    fn from(rhs: &AccountStorageEntry) -> Self {
+        Self {
+            id: rhs.append_vec_id() as SerializedAppendVecId,
+            accounts_current_len: rhs.accounts.len(),
+        }
+    }
+}
+
+#[cfg(RUSTC_WITH_SPECIALIZATION)]
+impl solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableAccountStorageEntry {}

runtime/src/serde_snapshot/tests.rs

@@ -305,7 +305,7 @@ mod test_bank_serialize {
 
     // This some what long test harness is required to freeze the ABI of
     // Bank's serialization due to versioned nature
-    #[frozen_abi(digest = "4xi75P1M48JwDjxf5k8y43r2w57AjYmgjMB1BmX6hXKK")]
+    #[frozen_abi(digest = "7PcarCw6gpw9Yw8xypdxQP24TFjLiaHyuDkq95cgwtte")]
     #[derive(Serialize, AbiExample)]
     pub struct BankAbiTestWrapperNewer {
         #[serde(serialize_with = "wrapper_newer")]

runtime/src/snapshot_utils.rs

@@ -1,15 +1,14 @@
 use {
     crate::{
-        accounts_db::{AccountShrinkThreshold, AccountsDbConfig},
+        accounts_db::{
+            AccountShrinkThreshold, AccountsDbConfig, SnapshotStorage, SnapshotStorages,
+        },
         accounts_index::AccountSecondaryIndexes,
         accounts_update_notifier_interface::AccountsUpdateNotifier,
         bank::{Bank, BankSlotDelta},
         builtins::Builtins,
         hardened_unpack::{unpack_snapshot, ParallelSelector, UnpackError, UnpackedAppendVecMap},
-        serde_snapshot::{
-            bank_from_streams, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages,
-            SnapshotStreams,
-        },
+        serde_snapshot::{bank_from_streams, bank_to_stream, SerdeStyle, SnapshotStreams},
         shared_buffer_reader::{SharedBuffer, SharedBufferReader},
         snapshot_archive_info::{
             FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter,