From 2756abce3922349a60004e322f2a9a66ca4381cc Mon Sep 17 00:00:00 2001 From: Brooks Prumo Date: Thu, 13 Jan 2022 09:20:20 -0600 Subject: [PATCH] More serde snapshot cleanup (#22449) --- runtime/src/serde_snapshot.rs | 15 +-- runtime/src/serde_snapshot/newer.rs | 176 +++++++++++--------------- runtime/src/serde_snapshot/storage.rs | 40 ++++++ runtime/src/serde_snapshot/tests.rs | 2 +- runtime/src/snapshot_utils.rs | 9 +- 5 files changed, 125 insertions(+), 117 deletions(-) create mode 100644 runtime/src/serde_snapshot/storage.rs diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 21f0eeca67..ded689318f 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -3,7 +3,7 @@ use { accounts::Accounts, accounts_db::{ AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId, - AtomicAppendVecId, BankHashInfo, IndexGenerationInfo, + AtomicAppendVecId, BankHashInfo, IndexGenerationInfo, SnapshotStorage, }, accounts_index::AccountSecondaryIndexes, accounts_update_notifier_interface::AccountsUpdateNotifier, @@ -14,7 +14,6 @@ use { epoch_stakes::EpochStakes, hardened_unpack::UnpackedAppendVecMap, rent_collector::RentCollector, - serde_snapshot::newer::{AppendVecIdSerialized, SerializableStorage}, stakes::Stakes, }, bincode::{self, config::Options, Error}, @@ -47,15 +46,15 @@ use { mod common; mod newer; +mod storage; mod tests; mod utils; +use storage::{SerializableStorage, SerializedAppendVecId}; + // a number of test cases in accounts_db use this #[cfg(test)] -pub(crate) use self::tests::reconstruct_accounts_db_via_serialization; -pub(crate) use crate::accounts_db::{SnapshotStorage, SnapshotStorages}; -#[allow(unused_imports)] -use utils::{serialize_iter_as_map, serialize_iter_as_seq, serialize_iter_as_tuple}; +pub(crate) use tests::reconstruct_accounts_db_via_serialization; #[derive(Copy, Clone, Eq, PartialEq)] pub(crate) enum SerdeStyle { @@ -465,7 +464,7 @@ where // rename the file to this new path. // **DEVELOPER NOTE:** Keep this check last so that it can short-circuit if // possible. - if storage_entry.id() == remapped_append_vec_id as AppendVecIdSerialized + if storage_entry.id() == remapped_append_vec_id as SerializedAppendVecId || std::fs::metadata(&remapped_append_vec_path).is_err() { break (remapped_append_vec_id, remapped_append_vec_path); @@ -476,7 +475,7 @@ where num_collisions.fetch_add(1, Ordering::Relaxed); }; // Only rename the file if the new ID is actually different from the original. - if storage_entry.id() != remapped_append_vec_id as AppendVecIdSerialized { + if storage_entry.id() != remapped_append_vec_id as SerializedAppendVecId { std::fs::rename(append_vec_path, &remapped_append_vec_path)?; } diff --git a/runtime/src/serde_snapshot/newer.rs b/runtime/src/serde_snapshot/newer.rs index 996b3dfc01..6f182bc444 100644 --- a/runtime/src/serde_snapshot/newer.rs +++ b/runtime/src/serde_snapshot/newer.rs @@ -1,5 +1,10 @@ use { - super::{common::UnusedAccounts, *}, + super::{ + common::UnusedAccounts, + storage::SerializableAccountStorageEntry, + utils::{serialize_iter_as_map, serialize_iter_as_seq}, + *, + }, crate::{ancestors::AncestorsForSerialization, stakes::StakesCache}, solana_measure::measure::Measure, std::{cell::RefCell, sync::RwLock}, @@ -7,80 +12,44 @@ use { type AccountsDbFields = super::AccountsDbFields; -/// the serialized type is fixed as usize -pub type AppendVecIdSerialized = usize; - -// Serializable version of AccountStorageEntry for snapshot format -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -pub(super) struct SerializableAccountStorageEntry { - id: AppendVecIdSerialized, - accounts_current_len: usize, -} - -pub trait SerializableStorage { - fn id(&self) -> AppendVecIdSerialized; - fn current_len(&self) -> usize; -} - -impl SerializableStorage for SerializableAccountStorageEntry { - fn id(&self) -> AppendVecIdSerialized { - self.id - } - fn current_len(&self) -> usize { - self.accounts_current_len - } -} - -#[cfg(RUSTC_WITH_SPECIALIZATION)] -impl solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableAccountStorageEntry {} - -impl From<&AccountStorageEntry> for SerializableAccountStorageEntry { - fn from(rhs: &AccountStorageEntry) -> Self { - Self { - id: rhs.append_vec_id() as AppendVecIdSerialized, - accounts_current_len: rhs.accounts.len(), - } - } -} - // Deserializable version of Bank which need not be serializable, // because it's handled by SerializableVersionedBank. // So, sync fields with it! #[derive(Clone, Deserialize)] -pub(crate) struct DeserializableVersionedBank { - pub(crate) blockhash_queue: BlockhashQueue, - pub(crate) ancestors: AncestorsForSerialization, - pub(crate) hash: Hash, - pub(crate) parent_hash: Hash, - pub(crate) parent_slot: Slot, - pub(crate) hard_forks: HardForks, - pub(crate) transaction_count: u64, - pub(crate) tick_height: u64, - pub(crate) signature_count: u64, - pub(crate) capitalization: u64, - pub(crate) max_tick_height: u64, - pub(crate) hashes_per_tick: Option, - pub(crate) ticks_per_slot: u64, - pub(crate) ns_per_slot: u128, - pub(crate) genesis_creation_time: UnixTimestamp, - pub(crate) slots_per_year: f64, - pub(crate) unused: u64, - pub(crate) slot: Slot, - pub(crate) epoch: Epoch, - pub(crate) block_height: u64, - pub(crate) collector_id: Pubkey, - pub(crate) collector_fees: u64, - pub(crate) fee_calculator: FeeCalculator, - pub(crate) fee_rate_governor: FeeRateGovernor, - pub(crate) collected_rent: u64, - pub(crate) rent_collector: RentCollector, - pub(crate) epoch_schedule: EpochSchedule, - pub(crate) inflation: Inflation, - pub(crate) stakes: Stakes, +struct DeserializableVersionedBank { + blockhash_queue: BlockhashQueue, + ancestors: AncestorsForSerialization, + hash: Hash, + parent_hash: Hash, + parent_slot: Slot, + hard_forks: HardForks, + transaction_count: u64, + tick_height: u64, + signature_count: u64, + capitalization: u64, + max_tick_height: u64, + hashes_per_tick: Option, + ticks_per_slot: u64, + ns_per_slot: u128, + genesis_creation_time: UnixTimestamp, + slots_per_year: f64, + unused: u64, + slot: Slot, + epoch: Epoch, + block_height: u64, + collector_id: Pubkey, + collector_fees: u64, + fee_calculator: FeeCalculator, + fee_rate_governor: FeeRateGovernor, + collected_rent: u64, + rent_collector: RentCollector, + epoch_schedule: EpochSchedule, + inflation: Inflation, + stakes: Stakes, #[allow(dead_code)] - pub(crate) unused_accounts: UnusedAccounts, - pub(crate) epoch_stakes: HashMap, - pub(crate) is_delta: bool, + unused_accounts: UnusedAccounts, + epoch_stakes: HashMap, + is_delta: bool, } impl From for BankFieldsToDeserialize { @@ -124,39 +93,39 @@ impl From for BankFieldsToDeserialize { // Serializable version of Bank, not Deserializable to avoid cloning by using refs. // Sync fields with DeserializableVersionedBank! #[derive(Serialize)] -pub(crate) struct SerializableVersionedBank<'a> { - pub(crate) blockhash_queue: &'a RwLock, - pub(crate) ancestors: &'a AncestorsForSerialization, - pub(crate) hash: Hash, - pub(crate) parent_hash: Hash, - pub(crate) parent_slot: Slot, - pub(crate) hard_forks: &'a RwLock, - pub(crate) transaction_count: u64, - pub(crate) tick_height: u64, - pub(crate) signature_count: u64, - pub(crate) capitalization: u64, - pub(crate) max_tick_height: u64, - pub(crate) hashes_per_tick: Option, - pub(crate) ticks_per_slot: u64, - pub(crate) ns_per_slot: u128, - pub(crate) genesis_creation_time: UnixTimestamp, - pub(crate) slots_per_year: f64, - pub(crate) unused: u64, - pub(crate) slot: Slot, - pub(crate) epoch: Epoch, - pub(crate) block_height: u64, - pub(crate) collector_id: Pubkey, - pub(crate) collector_fees: u64, - pub(crate) fee_calculator: FeeCalculator, - pub(crate) fee_rate_governor: FeeRateGovernor, - pub(crate) collected_rent: u64, - pub(crate) rent_collector: RentCollector, - pub(crate) epoch_schedule: EpochSchedule, - pub(crate) inflation: Inflation, - pub(crate) stakes: &'a StakesCache, - pub(crate) unused_accounts: UnusedAccounts, - pub(crate) epoch_stakes: &'a HashMap, - pub(crate) is_delta: bool, +struct SerializableVersionedBank<'a> { + blockhash_queue: &'a RwLock, + ancestors: &'a AncestorsForSerialization, + hash: Hash, + parent_hash: Hash, + parent_slot: Slot, + hard_forks: &'a RwLock, + transaction_count: u64, + tick_height: u64, + signature_count: u64, + capitalization: u64, + max_tick_height: u64, + hashes_per_tick: Option, + ticks_per_slot: u64, + ns_per_slot: u128, + genesis_creation_time: UnixTimestamp, + slots_per_year: f64, + unused: u64, + slot: Slot, + epoch: Epoch, + block_height: u64, + collector_id: Pubkey, + collector_fees: u64, + fee_calculator: FeeCalculator, + fee_rate_governor: FeeRateGovernor, + collected_rent: u64, + rent_collector: RentCollector, + epoch_schedule: EpochSchedule, + inflation: Inflation, + stakes: &'a StakesCache, + unused_accounts: UnusedAccounts, + epoch_stakes: &'a HashMap, + is_delta: bool, } impl<'a> From> for SerializableVersionedBank<'a> { @@ -205,6 +174,7 @@ impl<'a> From> for SerializableVersionedB impl<'a> solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableVersionedBank<'a> {} pub(super) struct Context {} + impl<'a> TypeContext<'a> for Context { type SerializableAccountStorageEntry = SerializableAccountStorageEntry; diff --git a/runtime/src/serde_snapshot/storage.rs b/runtime/src/serde_snapshot/storage.rs new file mode 100644 index 0000000000..2de2987d31 --- /dev/null +++ b/runtime/src/serde_snapshot/storage.rs @@ -0,0 +1,40 @@ +use { + crate::accounts_db::AccountStorageEntry, + serde::{Deserialize, Serialize}, +}; + +/// The serialized AppendVecId type is fixed as usize +pub(super) type SerializedAppendVecId = usize; + +// Serializable version of AccountStorageEntry for snapshot format +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub(super) struct SerializableAccountStorageEntry { + id: SerializedAppendVecId, + accounts_current_len: usize, +} + +pub(super) trait SerializableStorage { + fn id(&self) -> SerializedAppendVecId; + fn current_len(&self) -> usize; +} + +impl SerializableStorage for SerializableAccountStorageEntry { + fn id(&self) -> SerializedAppendVecId { + self.id + } + fn current_len(&self) -> usize { + self.accounts_current_len + } +} + +impl From<&AccountStorageEntry> for SerializableAccountStorageEntry { + fn from(rhs: &AccountStorageEntry) -> Self { + Self { + id: rhs.append_vec_id() as SerializedAppendVecId, + accounts_current_len: rhs.accounts.len(), + } + } +} + +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableAccountStorageEntry {} diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index 0c11b3c86f..212c830bd7 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -305,7 +305,7 @@ mod test_bank_serialize { // This some what long test harness is required to freeze the ABI of // Bank's serialization due to versioned nature - #[frozen_abi(digest = "4xi75P1M48JwDjxf5k8y43r2w57AjYmgjMB1BmX6hXKK")] + #[frozen_abi(digest = "7PcarCw6gpw9Yw8xypdxQP24TFjLiaHyuDkq95cgwtte")] #[derive(Serialize, AbiExample)] pub struct BankAbiTestWrapperNewer { #[serde(serialize_with = "wrapper_newer")] diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index 84f2aefe4c..b0254154b5 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -1,15 +1,14 @@ use { crate::{ - accounts_db::{AccountShrinkThreshold, AccountsDbConfig}, + accounts_db::{ + AccountShrinkThreshold, AccountsDbConfig, SnapshotStorage, SnapshotStorages, + }, accounts_index::AccountSecondaryIndexes, accounts_update_notifier_interface::AccountsUpdateNotifier, bank::{Bank, BankSlotDelta}, builtins::Builtins, hardened_unpack::{unpack_snapshot, ParallelSelector, UnpackError, UnpackedAppendVecMap}, - serde_snapshot::{ - bank_from_streams, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages, - SnapshotStreams, - }, + serde_snapshot::{bank_from_streams, bank_to_stream, SerdeStyle, SnapshotStreams}, shared_buffer_reader::{SharedBuffer, SharedBufferReader}, snapshot_archive_info::{ FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter,