More serde snapshot cleanup (#22449)
This commit is contained in:
parent
9c3144e286
commit
2756abce39
|
@ -3,7 +3,7 @@ use {
|
||||||
accounts::Accounts,
|
accounts::Accounts,
|
||||||
accounts_db::{
|
accounts_db::{
|
||||||
AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId,
|
AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId,
|
||||||
AtomicAppendVecId, BankHashInfo, IndexGenerationInfo,
|
AtomicAppendVecId, BankHashInfo, IndexGenerationInfo, SnapshotStorage,
|
||||||
},
|
},
|
||||||
accounts_index::AccountSecondaryIndexes,
|
accounts_index::AccountSecondaryIndexes,
|
||||||
accounts_update_notifier_interface::AccountsUpdateNotifier,
|
accounts_update_notifier_interface::AccountsUpdateNotifier,
|
||||||
|
@ -14,7 +14,6 @@ use {
|
||||||
epoch_stakes::EpochStakes,
|
epoch_stakes::EpochStakes,
|
||||||
hardened_unpack::UnpackedAppendVecMap,
|
hardened_unpack::UnpackedAppendVecMap,
|
||||||
rent_collector::RentCollector,
|
rent_collector::RentCollector,
|
||||||
serde_snapshot::newer::{AppendVecIdSerialized, SerializableStorage},
|
|
||||||
stakes::Stakes,
|
stakes::Stakes,
|
||||||
},
|
},
|
||||||
bincode::{self, config::Options, Error},
|
bincode::{self, config::Options, Error},
|
||||||
|
@ -47,15 +46,15 @@ use {
|
||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
mod newer;
|
mod newer;
|
||||||
|
mod storage;
|
||||||
mod tests;
|
mod tests;
|
||||||
mod utils;
|
mod utils;
|
||||||
|
|
||||||
|
use storage::{SerializableStorage, SerializedAppendVecId};
|
||||||
|
|
||||||
// a number of test cases in accounts_db use this
|
// a number of test cases in accounts_db use this
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use self::tests::reconstruct_accounts_db_via_serialization;
|
pub(crate) use tests::reconstruct_accounts_db_via_serialization;
|
||||||
pub(crate) use crate::accounts_db::{SnapshotStorage, SnapshotStorages};
|
|
||||||
#[allow(unused_imports)]
|
|
||||||
use utils::{serialize_iter_as_map, serialize_iter_as_seq, serialize_iter_as_tuple};
|
|
||||||
|
|
||||||
#[derive(Copy, Clone, Eq, PartialEq)]
|
#[derive(Copy, Clone, Eq, PartialEq)]
|
||||||
pub(crate) enum SerdeStyle {
|
pub(crate) enum SerdeStyle {
|
||||||
|
@ -465,7 +464,7 @@ where
|
||||||
// rename the file to this new path.
|
// rename the file to this new path.
|
||||||
// **DEVELOPER NOTE:** Keep this check last so that it can short-circuit if
|
// **DEVELOPER NOTE:** Keep this check last so that it can short-circuit if
|
||||||
// possible.
|
// possible.
|
||||||
if storage_entry.id() == remapped_append_vec_id as AppendVecIdSerialized
|
if storage_entry.id() == remapped_append_vec_id as SerializedAppendVecId
|
||||||
|| std::fs::metadata(&remapped_append_vec_path).is_err()
|
|| std::fs::metadata(&remapped_append_vec_path).is_err()
|
||||||
{
|
{
|
||||||
break (remapped_append_vec_id, remapped_append_vec_path);
|
break (remapped_append_vec_id, remapped_append_vec_path);
|
||||||
|
@ -476,7 +475,7 @@ where
|
||||||
num_collisions.fetch_add(1, Ordering::Relaxed);
|
num_collisions.fetch_add(1, Ordering::Relaxed);
|
||||||
};
|
};
|
||||||
// Only rename the file if the new ID is actually different from the original.
|
// Only rename the file if the new ID is actually different from the original.
|
||||||
if storage_entry.id() != remapped_append_vec_id as AppendVecIdSerialized {
|
if storage_entry.id() != remapped_append_vec_id as SerializedAppendVecId {
|
||||||
std::fs::rename(append_vec_path, &remapped_append_vec_path)?;
|
std::fs::rename(append_vec_path, &remapped_append_vec_path)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,10 @@
|
||||||
use {
|
use {
|
||||||
super::{common::UnusedAccounts, *},
|
super::{
|
||||||
|
common::UnusedAccounts,
|
||||||
|
storage::SerializableAccountStorageEntry,
|
||||||
|
utils::{serialize_iter_as_map, serialize_iter_as_seq},
|
||||||
|
*,
|
||||||
|
},
|
||||||
crate::{ancestors::AncestorsForSerialization, stakes::StakesCache},
|
crate::{ancestors::AncestorsForSerialization, stakes::StakesCache},
|
||||||
solana_measure::measure::Measure,
|
solana_measure::measure::Measure,
|
||||||
std::{cell::RefCell, sync::RwLock},
|
std::{cell::RefCell, sync::RwLock},
|
||||||
|
@ -7,80 +12,44 @@ use {
|
||||||
|
|
||||||
type AccountsDbFields = super::AccountsDbFields<SerializableAccountStorageEntry>;
|
type AccountsDbFields = super::AccountsDbFields<SerializableAccountStorageEntry>;
|
||||||
|
|
||||||
/// the serialized type is fixed as usize
|
|
||||||
pub type AppendVecIdSerialized = usize;
|
|
||||||
|
|
||||||
// Serializable version of AccountStorageEntry for snapshot format
|
|
||||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
|
||||||
pub(super) struct SerializableAccountStorageEntry {
|
|
||||||
id: AppendVecIdSerialized,
|
|
||||||
accounts_current_len: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait SerializableStorage {
|
|
||||||
fn id(&self) -> AppendVecIdSerialized;
|
|
||||||
fn current_len(&self) -> usize;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SerializableStorage for SerializableAccountStorageEntry {
|
|
||||||
fn id(&self) -> AppendVecIdSerialized {
|
|
||||||
self.id
|
|
||||||
}
|
|
||||||
fn current_len(&self) -> usize {
|
|
||||||
self.accounts_current_len
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
|
||||||
impl solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableAccountStorageEntry {}
|
|
||||||
|
|
||||||
impl From<&AccountStorageEntry> for SerializableAccountStorageEntry {
|
|
||||||
fn from(rhs: &AccountStorageEntry) -> Self {
|
|
||||||
Self {
|
|
||||||
id: rhs.append_vec_id() as AppendVecIdSerialized,
|
|
||||||
accounts_current_len: rhs.accounts.len(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deserializable version of Bank which need not be serializable,
|
// Deserializable version of Bank which need not be serializable,
|
||||||
// because it's handled by SerializableVersionedBank.
|
// because it's handled by SerializableVersionedBank.
|
||||||
// So, sync fields with it!
|
// So, sync fields with it!
|
||||||
#[derive(Clone, Deserialize)]
|
#[derive(Clone, Deserialize)]
|
||||||
pub(crate) struct DeserializableVersionedBank {
|
struct DeserializableVersionedBank {
|
||||||
pub(crate) blockhash_queue: BlockhashQueue,
|
blockhash_queue: BlockhashQueue,
|
||||||
pub(crate) ancestors: AncestorsForSerialization,
|
ancestors: AncestorsForSerialization,
|
||||||
pub(crate) hash: Hash,
|
hash: Hash,
|
||||||
pub(crate) parent_hash: Hash,
|
parent_hash: Hash,
|
||||||
pub(crate) parent_slot: Slot,
|
parent_slot: Slot,
|
||||||
pub(crate) hard_forks: HardForks,
|
hard_forks: HardForks,
|
||||||
pub(crate) transaction_count: u64,
|
transaction_count: u64,
|
||||||
pub(crate) tick_height: u64,
|
tick_height: u64,
|
||||||
pub(crate) signature_count: u64,
|
signature_count: u64,
|
||||||
pub(crate) capitalization: u64,
|
capitalization: u64,
|
||||||
pub(crate) max_tick_height: u64,
|
max_tick_height: u64,
|
||||||
pub(crate) hashes_per_tick: Option<u64>,
|
hashes_per_tick: Option<u64>,
|
||||||
pub(crate) ticks_per_slot: u64,
|
ticks_per_slot: u64,
|
||||||
pub(crate) ns_per_slot: u128,
|
ns_per_slot: u128,
|
||||||
pub(crate) genesis_creation_time: UnixTimestamp,
|
genesis_creation_time: UnixTimestamp,
|
||||||
pub(crate) slots_per_year: f64,
|
slots_per_year: f64,
|
||||||
pub(crate) unused: u64,
|
unused: u64,
|
||||||
pub(crate) slot: Slot,
|
slot: Slot,
|
||||||
pub(crate) epoch: Epoch,
|
epoch: Epoch,
|
||||||
pub(crate) block_height: u64,
|
block_height: u64,
|
||||||
pub(crate) collector_id: Pubkey,
|
collector_id: Pubkey,
|
||||||
pub(crate) collector_fees: u64,
|
collector_fees: u64,
|
||||||
pub(crate) fee_calculator: FeeCalculator,
|
fee_calculator: FeeCalculator,
|
||||||
pub(crate) fee_rate_governor: FeeRateGovernor,
|
fee_rate_governor: FeeRateGovernor,
|
||||||
pub(crate) collected_rent: u64,
|
collected_rent: u64,
|
||||||
pub(crate) rent_collector: RentCollector,
|
rent_collector: RentCollector,
|
||||||
pub(crate) epoch_schedule: EpochSchedule,
|
epoch_schedule: EpochSchedule,
|
||||||
pub(crate) inflation: Inflation,
|
inflation: Inflation,
|
||||||
pub(crate) stakes: Stakes,
|
stakes: Stakes,
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub(crate) unused_accounts: UnusedAccounts,
|
unused_accounts: UnusedAccounts,
|
||||||
pub(crate) epoch_stakes: HashMap<Epoch, EpochStakes>,
|
epoch_stakes: HashMap<Epoch, EpochStakes>,
|
||||||
pub(crate) is_delta: bool,
|
is_delta: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<DeserializableVersionedBank> for BankFieldsToDeserialize {
|
impl From<DeserializableVersionedBank> for BankFieldsToDeserialize {
|
||||||
|
@ -124,39 +93,39 @@ impl From<DeserializableVersionedBank> for BankFieldsToDeserialize {
|
||||||
// Serializable version of Bank, not Deserializable to avoid cloning by using refs.
|
// Serializable version of Bank, not Deserializable to avoid cloning by using refs.
|
||||||
// Sync fields with DeserializableVersionedBank!
|
// Sync fields with DeserializableVersionedBank!
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub(crate) struct SerializableVersionedBank<'a> {
|
struct SerializableVersionedBank<'a> {
|
||||||
pub(crate) blockhash_queue: &'a RwLock<BlockhashQueue>,
|
blockhash_queue: &'a RwLock<BlockhashQueue>,
|
||||||
pub(crate) ancestors: &'a AncestorsForSerialization,
|
ancestors: &'a AncestorsForSerialization,
|
||||||
pub(crate) hash: Hash,
|
hash: Hash,
|
||||||
pub(crate) parent_hash: Hash,
|
parent_hash: Hash,
|
||||||
pub(crate) parent_slot: Slot,
|
parent_slot: Slot,
|
||||||
pub(crate) hard_forks: &'a RwLock<HardForks>,
|
hard_forks: &'a RwLock<HardForks>,
|
||||||
pub(crate) transaction_count: u64,
|
transaction_count: u64,
|
||||||
pub(crate) tick_height: u64,
|
tick_height: u64,
|
||||||
pub(crate) signature_count: u64,
|
signature_count: u64,
|
||||||
pub(crate) capitalization: u64,
|
capitalization: u64,
|
||||||
pub(crate) max_tick_height: u64,
|
max_tick_height: u64,
|
||||||
pub(crate) hashes_per_tick: Option<u64>,
|
hashes_per_tick: Option<u64>,
|
||||||
pub(crate) ticks_per_slot: u64,
|
ticks_per_slot: u64,
|
||||||
pub(crate) ns_per_slot: u128,
|
ns_per_slot: u128,
|
||||||
pub(crate) genesis_creation_time: UnixTimestamp,
|
genesis_creation_time: UnixTimestamp,
|
||||||
pub(crate) slots_per_year: f64,
|
slots_per_year: f64,
|
||||||
pub(crate) unused: u64,
|
unused: u64,
|
||||||
pub(crate) slot: Slot,
|
slot: Slot,
|
||||||
pub(crate) epoch: Epoch,
|
epoch: Epoch,
|
||||||
pub(crate) block_height: u64,
|
block_height: u64,
|
||||||
pub(crate) collector_id: Pubkey,
|
collector_id: Pubkey,
|
||||||
pub(crate) collector_fees: u64,
|
collector_fees: u64,
|
||||||
pub(crate) fee_calculator: FeeCalculator,
|
fee_calculator: FeeCalculator,
|
||||||
pub(crate) fee_rate_governor: FeeRateGovernor,
|
fee_rate_governor: FeeRateGovernor,
|
||||||
pub(crate) collected_rent: u64,
|
collected_rent: u64,
|
||||||
pub(crate) rent_collector: RentCollector,
|
rent_collector: RentCollector,
|
||||||
pub(crate) epoch_schedule: EpochSchedule,
|
epoch_schedule: EpochSchedule,
|
||||||
pub(crate) inflation: Inflation,
|
inflation: Inflation,
|
||||||
pub(crate) stakes: &'a StakesCache,
|
stakes: &'a StakesCache,
|
||||||
pub(crate) unused_accounts: UnusedAccounts,
|
unused_accounts: UnusedAccounts,
|
||||||
pub(crate) epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
|
epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
|
||||||
pub(crate) is_delta: bool,
|
is_delta: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedBank<'a> {
|
impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedBank<'a> {
|
||||||
|
@ -205,6 +174,7 @@ impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedB
|
||||||
impl<'a> solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableVersionedBank<'a> {}
|
impl<'a> solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableVersionedBank<'a> {}
|
||||||
|
|
||||||
pub(super) struct Context {}
|
pub(super) struct Context {}
|
||||||
|
|
||||||
impl<'a> TypeContext<'a> for Context {
|
impl<'a> TypeContext<'a> for Context {
|
||||||
type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
|
type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,40 @@
|
||||||
|
use {
|
||||||
|
crate::accounts_db::AccountStorageEntry,
|
||||||
|
serde::{Deserialize, Serialize},
|
||||||
|
};
|
||||||
|
|
||||||
|
/// The serialized AppendVecId type is fixed as usize
|
||||||
|
pub(super) type SerializedAppendVecId = usize;
|
||||||
|
|
||||||
|
// Serializable version of AccountStorageEntry for snapshot format
|
||||||
|
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub(super) struct SerializableAccountStorageEntry {
|
||||||
|
id: SerializedAppendVecId,
|
||||||
|
accounts_current_len: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) trait SerializableStorage {
|
||||||
|
fn id(&self) -> SerializedAppendVecId;
|
||||||
|
fn current_len(&self) -> usize;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SerializableStorage for SerializableAccountStorageEntry {
|
||||||
|
fn id(&self) -> SerializedAppendVecId {
|
||||||
|
self.id
|
||||||
|
}
|
||||||
|
fn current_len(&self) -> usize {
|
||||||
|
self.accounts_current_len
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&AccountStorageEntry> for SerializableAccountStorageEntry {
|
||||||
|
fn from(rhs: &AccountStorageEntry) -> Self {
|
||||||
|
Self {
|
||||||
|
id: rhs.append_vec_id() as SerializedAppendVecId,
|
||||||
|
accounts_current_len: rhs.accounts.len(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
||||||
|
impl solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableAccountStorageEntry {}
|
|
@ -305,7 +305,7 @@ mod test_bank_serialize {
|
||||||
|
|
||||||
// This some what long test harness is required to freeze the ABI of
|
// This some what long test harness is required to freeze the ABI of
|
||||||
// Bank's serialization due to versioned nature
|
// Bank's serialization due to versioned nature
|
||||||
#[frozen_abi(digest = "4xi75P1M48JwDjxf5k8y43r2w57AjYmgjMB1BmX6hXKK")]
|
#[frozen_abi(digest = "7PcarCw6gpw9Yw8xypdxQP24TFjLiaHyuDkq95cgwtte")]
|
||||||
#[derive(Serialize, AbiExample)]
|
#[derive(Serialize, AbiExample)]
|
||||||
pub struct BankAbiTestWrapperNewer {
|
pub struct BankAbiTestWrapperNewer {
|
||||||
#[serde(serialize_with = "wrapper_newer")]
|
#[serde(serialize_with = "wrapper_newer")]
|
||||||
|
|
|
@ -1,15 +1,14 @@
|
||||||
use {
|
use {
|
||||||
crate::{
|
crate::{
|
||||||
accounts_db::{AccountShrinkThreshold, AccountsDbConfig},
|
accounts_db::{
|
||||||
|
AccountShrinkThreshold, AccountsDbConfig, SnapshotStorage, SnapshotStorages,
|
||||||
|
},
|
||||||
accounts_index::AccountSecondaryIndexes,
|
accounts_index::AccountSecondaryIndexes,
|
||||||
accounts_update_notifier_interface::AccountsUpdateNotifier,
|
accounts_update_notifier_interface::AccountsUpdateNotifier,
|
||||||
bank::{Bank, BankSlotDelta},
|
bank::{Bank, BankSlotDelta},
|
||||||
builtins::Builtins,
|
builtins::Builtins,
|
||||||
hardened_unpack::{unpack_snapshot, ParallelSelector, UnpackError, UnpackedAppendVecMap},
|
hardened_unpack::{unpack_snapshot, ParallelSelector, UnpackError, UnpackedAppendVecMap},
|
||||||
serde_snapshot::{
|
serde_snapshot::{bank_from_streams, bank_to_stream, SerdeStyle, SnapshotStreams},
|
||||||
bank_from_streams, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages,
|
|
||||||
SnapshotStreams,
|
|
||||||
},
|
|
||||||
shared_buffer_reader::{SharedBuffer, SharedBufferReader},
|
shared_buffer_reader::{SharedBuffer, SharedBufferReader},
|
||||||
snapshot_archive_info::{
|
snapshot_archive_info::{
|
||||||
FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter,
|
FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter,
|
||||||
|
|
Loading…
Reference in New Issue