428 lines
16 KiB
Rust
428 lines
16 KiB
Rust
use {
|
|
super::{
|
|
storage::SerializableAccountStorageEntry,
|
|
utils::{serialize_iter_as_map, serialize_iter_as_seq},
|
|
*,
|
|
},
|
|
crate::{
|
|
accounts_hash::AccountsHash,
|
|
ancestors::AncestorsForSerialization,
|
|
stakes::{serde_stakes_enum_compat, StakesEnum},
|
|
},
|
|
solana_measure::measure::Measure,
|
|
solana_sdk::{deserialize_utils::ignore_eof_error, stake::state::Delegation},
|
|
std::{cell::RefCell, collections::HashSet, sync::RwLock},
|
|
};
|
|
|
|
pub(super) type AccountsDbFields = super::AccountsDbFields<SerializableAccountStorageEntry>;
|
|
|
|
#[derive(Default, Clone, PartialEq, Eq, Debug, Deserialize, Serialize, AbiExample)]
|
|
struct UnusedAccounts {
|
|
unused1: HashSet<Pubkey>,
|
|
unused2: HashSet<Pubkey>,
|
|
unused3: HashMap<Pubkey, u64>,
|
|
}
|
|
|
|
// Deserializable version of Bank which need not be serializable,
|
|
// because it's handled by SerializableVersionedBank.
|
|
// So, sync fields with it!
|
|
#[derive(Clone, Deserialize)]
|
|
struct DeserializableVersionedBank {
|
|
blockhash_queue: BlockhashQueue,
|
|
ancestors: AncestorsForSerialization,
|
|
hash: Hash,
|
|
parent_hash: Hash,
|
|
parent_slot: Slot,
|
|
hard_forks: HardForks,
|
|
transaction_count: u64,
|
|
tick_height: u64,
|
|
signature_count: u64,
|
|
capitalization: u64,
|
|
max_tick_height: u64,
|
|
hashes_per_tick: Option<u64>,
|
|
ticks_per_slot: u64,
|
|
ns_per_slot: u128,
|
|
genesis_creation_time: UnixTimestamp,
|
|
slots_per_year: f64,
|
|
accounts_data_len: u64,
|
|
slot: Slot,
|
|
epoch: Epoch,
|
|
block_height: u64,
|
|
collector_id: Pubkey,
|
|
collector_fees: u64,
|
|
fee_calculator: FeeCalculator,
|
|
fee_rate_governor: FeeRateGovernor,
|
|
collected_rent: u64,
|
|
rent_collector: RentCollector,
|
|
epoch_schedule: EpochSchedule,
|
|
inflation: Inflation,
|
|
stakes: Stakes<Delegation>,
|
|
#[allow(dead_code)]
|
|
unused_accounts: UnusedAccounts,
|
|
epoch_stakes: HashMap<Epoch, EpochStakes>,
|
|
is_delta: bool,
|
|
}
|
|
|
|
impl From<DeserializableVersionedBank> for BankFieldsToDeserialize {
|
|
fn from(dvb: DeserializableVersionedBank) -> Self {
|
|
BankFieldsToDeserialize {
|
|
blockhash_queue: dvb.blockhash_queue,
|
|
ancestors: dvb.ancestors,
|
|
hash: dvb.hash,
|
|
parent_hash: dvb.parent_hash,
|
|
parent_slot: dvb.parent_slot,
|
|
hard_forks: dvb.hard_forks,
|
|
transaction_count: dvb.transaction_count,
|
|
tick_height: dvb.tick_height,
|
|
signature_count: dvb.signature_count,
|
|
capitalization: dvb.capitalization,
|
|
max_tick_height: dvb.max_tick_height,
|
|
hashes_per_tick: dvb.hashes_per_tick,
|
|
ticks_per_slot: dvb.ticks_per_slot,
|
|
ns_per_slot: dvb.ns_per_slot,
|
|
genesis_creation_time: dvb.genesis_creation_time,
|
|
slots_per_year: dvb.slots_per_year,
|
|
accounts_data_len: dvb.accounts_data_len,
|
|
slot: dvb.slot,
|
|
epoch: dvb.epoch,
|
|
block_height: dvb.block_height,
|
|
collector_id: dvb.collector_id,
|
|
collector_fees: dvb.collector_fees,
|
|
fee_calculator: dvb.fee_calculator,
|
|
fee_rate_governor: dvb.fee_rate_governor,
|
|
collected_rent: dvb.collected_rent,
|
|
rent_collector: dvb.rent_collector,
|
|
epoch_schedule: dvb.epoch_schedule,
|
|
inflation: dvb.inflation,
|
|
stakes: dvb.stakes,
|
|
epoch_stakes: dvb.epoch_stakes,
|
|
is_delta: dvb.is_delta,
|
|
incremental_snapshot_persistence: None,
|
|
epoch_accounts_hash: None,
|
|
}
|
|
}
|
|
}
|
|
|
|
// Serializable version of Bank, not Deserializable to avoid cloning by using refs.
|
|
// Sync fields with DeserializableVersionedBank!
|
|
#[derive(Serialize)]
|
|
struct SerializableVersionedBank<'a> {
|
|
blockhash_queue: &'a RwLock<BlockhashQueue>,
|
|
ancestors: &'a AncestorsForSerialization,
|
|
hash: Hash,
|
|
parent_hash: Hash,
|
|
parent_slot: Slot,
|
|
hard_forks: &'a RwLock<HardForks>,
|
|
transaction_count: u64,
|
|
tick_height: u64,
|
|
signature_count: u64,
|
|
capitalization: u64,
|
|
max_tick_height: u64,
|
|
hashes_per_tick: Option<u64>,
|
|
ticks_per_slot: u64,
|
|
ns_per_slot: u128,
|
|
genesis_creation_time: UnixTimestamp,
|
|
slots_per_year: f64,
|
|
accounts_data_len: u64,
|
|
slot: Slot,
|
|
epoch: Epoch,
|
|
block_height: u64,
|
|
collector_id: Pubkey,
|
|
collector_fees: u64,
|
|
fee_calculator: FeeCalculator,
|
|
fee_rate_governor: FeeRateGovernor,
|
|
collected_rent: u64,
|
|
rent_collector: RentCollector,
|
|
epoch_schedule: EpochSchedule,
|
|
inflation: Inflation,
|
|
#[serde(serialize_with = "serde_stakes_enum_compat::serialize")]
|
|
stakes: StakesEnum,
|
|
unused_accounts: UnusedAccounts,
|
|
epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
|
|
is_delta: bool,
|
|
}
|
|
|
|
impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedBank<'a> {
|
|
fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self {
|
|
Self {
|
|
blockhash_queue: rhs.blockhash_queue,
|
|
ancestors: rhs.ancestors,
|
|
hash: rhs.hash,
|
|
parent_hash: rhs.parent_hash,
|
|
parent_slot: rhs.parent_slot,
|
|
hard_forks: rhs.hard_forks,
|
|
transaction_count: rhs.transaction_count,
|
|
tick_height: rhs.tick_height,
|
|
signature_count: rhs.signature_count,
|
|
capitalization: rhs.capitalization,
|
|
max_tick_height: rhs.max_tick_height,
|
|
hashes_per_tick: rhs.hashes_per_tick,
|
|
ticks_per_slot: rhs.ticks_per_slot,
|
|
ns_per_slot: rhs.ns_per_slot,
|
|
genesis_creation_time: rhs.genesis_creation_time,
|
|
slots_per_year: rhs.slots_per_year,
|
|
accounts_data_len: rhs.accounts_data_len,
|
|
slot: rhs.slot,
|
|
epoch: rhs.epoch,
|
|
block_height: rhs.block_height,
|
|
collector_id: rhs.collector_id,
|
|
collector_fees: rhs.collector_fees,
|
|
fee_calculator: rhs.fee_calculator,
|
|
fee_rate_governor: rhs.fee_rate_governor,
|
|
collected_rent: rhs.collected_rent,
|
|
rent_collector: rhs.rent_collector,
|
|
epoch_schedule: rhs.epoch_schedule,
|
|
inflation: rhs.inflation,
|
|
stakes: StakesEnum::from(rhs.stakes.stakes().clone()),
|
|
unused_accounts: UnusedAccounts::default(),
|
|
epoch_stakes: rhs.epoch_stakes,
|
|
is_delta: rhs.is_delta,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
|
impl<'a> solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableVersionedBank<'a> {}
|
|
|
|
#[derive(PartialEq, Eq)]
|
|
pub(super) struct Context {}
|
|
|
|
impl<'a> TypeContext<'a> for Context {
|
|
type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
|
|
|
|
fn serialize_bank_and_storage<S: serde::ser::Serializer>(
|
|
serializer: S,
|
|
serializable_bank: &SerializableBankAndStorage<'a, Self>,
|
|
) -> std::result::Result<S::Ok, S::Error>
|
|
where
|
|
Self: std::marker::Sized,
|
|
{
|
|
let ancestors = HashMap::from(&serializable_bank.bank.ancestors);
|
|
let fields = serializable_bank.bank.get_fields_to_serialize(&ancestors);
|
|
let lamports_per_signature = fields.fee_rate_governor.lamports_per_signature;
|
|
(
|
|
SerializableVersionedBank::from(fields),
|
|
SerializableAccountsDb::<'a, Self> {
|
|
accounts_db: &serializable_bank.bank.rc.accounts.accounts_db,
|
|
slot: serializable_bank.bank.rc.slot,
|
|
account_storage_entries: serializable_bank.snapshot_storages,
|
|
phantom: std::marker::PhantomData::default(),
|
|
},
|
|
// Additional fields, we manually store the lamps per signature here so that
|
|
// we can grab it on restart.
|
|
// TODO: if we do a snapshot version bump, consider moving this out.
|
|
lamports_per_signature,
|
|
None::<BankIncrementalSnapshotPersistence>,
|
|
serializable_bank
|
|
.bank
|
|
.get_epoch_accounts_hash_to_serialize()
|
|
.map(|epoch_accounts_hash| *epoch_accounts_hash.as_ref()),
|
|
)
|
|
.serialize(serializer)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn serialize_bank_and_storage_without_extra_fields<S: serde::ser::Serializer>(
|
|
serializer: S,
|
|
serializable_bank: &SerializableBankAndStorageNoExtra<'a, Self>,
|
|
) -> std::result::Result<S::Ok, S::Error>
|
|
where
|
|
Self: std::marker::Sized,
|
|
{
|
|
let ancestors = HashMap::from(&serializable_bank.bank.ancestors);
|
|
let fields = serializable_bank.bank.get_fields_to_serialize(&ancestors);
|
|
(
|
|
SerializableVersionedBank::from(fields),
|
|
SerializableAccountsDb::<'a, Self> {
|
|
accounts_db: &serializable_bank.bank.rc.accounts.accounts_db,
|
|
slot: serializable_bank.bank.rc.slot,
|
|
account_storage_entries: serializable_bank.snapshot_storages,
|
|
phantom: std::marker::PhantomData::default(),
|
|
},
|
|
)
|
|
.serialize(serializer)
|
|
}
|
|
|
|
fn serialize_accounts_db_fields<S: serde::ser::Serializer>(
|
|
serializer: S,
|
|
serializable_db: &SerializableAccountsDb<'a, Self>,
|
|
) -> std::result::Result<S::Ok, S::Error>
|
|
where
|
|
Self: std::marker::Sized,
|
|
{
|
|
// sample write version before serializing storage entries
|
|
let version = serializable_db
|
|
.accounts_db
|
|
.write_version
|
|
.load(Ordering::Acquire);
|
|
|
|
// (1st of 3 elements) write the list of account storage entry lists out as a map
|
|
let entry_count = RefCell::<usize>::new(0);
|
|
let entries =
|
|
serialize_iter_as_map(serializable_db.account_storage_entries.iter().map(|x| {
|
|
*entry_count.borrow_mut() += x.len();
|
|
(
|
|
x.first().unwrap().slot(),
|
|
serialize_iter_as_seq(
|
|
x.iter()
|
|
.map(|x| Self::SerializableAccountStorageEntry::from(x.as_ref())),
|
|
),
|
|
)
|
|
}));
|
|
let slot = serializable_db.slot;
|
|
let accounts_delta_hash = serializable_db
|
|
.accounts_db
|
|
.get_accounts_delta_hash(slot)
|
|
.map(Into::into)
|
|
.unwrap_or_else(|| panic!("Missing accounts delta hash entry for slot {slot}"));
|
|
// NOTE: The accounts hash is calculated in AHV, which is *after* a bank snapshot is taken
|
|
// (and serialized here). Thus it is expected that an accounts hash is *not* found for
|
|
// this slot, and a placeholder value will be used instead. The real accounts hash will be
|
|
// set by `reserialize_bank_with_new_accounts_hash` from AHV.
|
|
let accounts_hash = serializable_db
|
|
.accounts_db
|
|
.get_accounts_hash(slot)
|
|
.map(|(accounts_hash, _)| accounts_hash.into())
|
|
.unwrap_or_default();
|
|
let stats = serializable_db
|
|
.accounts_db
|
|
.get_bank_hash_stats(slot)
|
|
.unwrap_or_else(|| panic!("Missing bank hash stats entry for slot {slot}"));
|
|
let bank_hash_info = BankHashInfo {
|
|
accounts_delta_hash,
|
|
accounts_hash,
|
|
stats,
|
|
};
|
|
|
|
let historical_roots = serializable_db
|
|
.accounts_db
|
|
.accounts_index
|
|
.roots_tracker
|
|
.read()
|
|
.unwrap()
|
|
.historical_roots
|
|
.get_all();
|
|
let historical_roots_with_hash = Vec::<(Slot, Hash)>::default();
|
|
|
|
let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms");
|
|
let result = (
|
|
entries,
|
|
version,
|
|
slot,
|
|
bank_hash_info,
|
|
historical_roots,
|
|
historical_roots_with_hash,
|
|
)
|
|
.serialize(serializer);
|
|
serialize_account_storage_timer.stop();
|
|
datapoint_info!(
|
|
"serialize_account_storage_ms",
|
|
("duration", serialize_account_storage_timer.as_ms(), i64),
|
|
("num_entries", *entry_count.borrow(), i64),
|
|
);
|
|
result
|
|
}
|
|
|
|
fn deserialize_bank_fields<R>(
|
|
mut stream: &mut BufReader<R>,
|
|
) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error>
|
|
where
|
|
R: Read,
|
|
{
|
|
let mut bank_fields: BankFieldsToDeserialize =
|
|
deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into();
|
|
let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?;
|
|
// Process extra fields
|
|
let lamports_per_signature = ignore_eof_error(deserialize_from(&mut stream))?;
|
|
bank_fields.fee_rate_governor = bank_fields
|
|
.fee_rate_governor
|
|
.clone_with_lamports_per_signature(lamports_per_signature);
|
|
|
|
let incremental_snapshot_persistence = ignore_eof_error(deserialize_from(&mut stream))?;
|
|
bank_fields.incremental_snapshot_persistence = incremental_snapshot_persistence;
|
|
|
|
let epoch_accounts_hash = ignore_eof_error(deserialize_from(stream))?;
|
|
bank_fields.epoch_accounts_hash = epoch_accounts_hash;
|
|
|
|
Ok((bank_fields, accounts_db_fields))
|
|
}
|
|
|
|
fn deserialize_accounts_db_fields<R>(
|
|
stream: &mut BufReader<R>,
|
|
) -> Result<AccountsDbFields, Error>
|
|
where
|
|
R: Read,
|
|
{
|
|
deserialize_from(stream)
|
|
}
|
|
|
|
/// deserialize the bank from 'stream_reader'
|
|
/// modify the accounts_hash and incremental_snapshot_persistence
|
|
/// reserialize the bank to 'stream_writer'
|
|
fn reserialize_bank_fields_with_hash<R, W>(
|
|
stream_reader: &mut BufReader<R>,
|
|
stream_writer: &mut BufWriter<W>,
|
|
accounts_hash: &AccountsHash,
|
|
incremental_snapshot_persistence: Option<&BankIncrementalSnapshotPersistence>,
|
|
) -> std::result::Result<(), Box<bincode::ErrorKind>>
|
|
where
|
|
R: Read,
|
|
W: Write,
|
|
{
|
|
let (bank_fields, mut accounts_db_fields) =
|
|
Self::deserialize_bank_fields(stream_reader).unwrap();
|
|
accounts_db_fields.3.accounts_hash = (*accounts_hash).into();
|
|
let mut rhs = bank_fields;
|
|
let blockhash_queue = RwLock::new(std::mem::take(&mut rhs.blockhash_queue));
|
|
let hard_forks = RwLock::new(std::mem::take(&mut rhs.hard_forks));
|
|
let lamports_per_signature = rhs.fee_rate_governor.lamports_per_signature;
|
|
let epoch_accounts_hash = rhs.epoch_accounts_hash.as_ref();
|
|
|
|
let bank = SerializableVersionedBank {
|
|
blockhash_queue: &blockhash_queue,
|
|
ancestors: &rhs.ancestors,
|
|
hash: rhs.hash,
|
|
parent_hash: rhs.parent_hash,
|
|
parent_slot: rhs.parent_slot,
|
|
hard_forks: &hard_forks,
|
|
transaction_count: rhs.transaction_count,
|
|
tick_height: rhs.tick_height,
|
|
signature_count: rhs.signature_count,
|
|
capitalization: rhs.capitalization,
|
|
max_tick_height: rhs.max_tick_height,
|
|
hashes_per_tick: rhs.hashes_per_tick,
|
|
ticks_per_slot: rhs.ticks_per_slot,
|
|
ns_per_slot: rhs.ns_per_slot,
|
|
genesis_creation_time: rhs.genesis_creation_time,
|
|
slots_per_year: rhs.slots_per_year,
|
|
accounts_data_len: rhs.accounts_data_len,
|
|
slot: rhs.slot,
|
|
epoch: rhs.epoch,
|
|
block_height: rhs.block_height,
|
|
collector_id: rhs.collector_id,
|
|
collector_fees: rhs.collector_fees,
|
|
fee_calculator: rhs.fee_calculator,
|
|
fee_rate_governor: rhs.fee_rate_governor,
|
|
collected_rent: rhs.collected_rent,
|
|
rent_collector: rhs.rent_collector,
|
|
epoch_schedule: rhs.epoch_schedule,
|
|
inflation: rhs.inflation,
|
|
stakes: StakesEnum::from(rhs.stakes),
|
|
unused_accounts: UnusedAccounts::default(),
|
|
epoch_stakes: &rhs.epoch_stakes,
|
|
is_delta: rhs.is_delta,
|
|
};
|
|
|
|
bincode::serialize_into(
|
|
stream_writer,
|
|
&(
|
|
bank,
|
|
accounts_db_fields,
|
|
lamports_per_signature,
|
|
incremental_snapshot_persistence,
|
|
epoch_accounts_hash,
|
|
),
|
|
)
|
|
}
|
|
}
|