use { super::{ storage::SerializableAccountStorageEntry, utils::{serialize_iter_as_map, serialize_iter_as_seq}, *, }, crate::{ accounts_hash::AccountsHash, ancestors::AncestorsForSerialization, stakes::{serde_stakes_enum_compat, StakesEnum}, }, solana_measure::measure::Measure, solana_sdk::{deserialize_utils::ignore_eof_error, stake::state::Delegation}, std::{cell::RefCell, collections::HashSet, sync::RwLock}, }; pub(super) type AccountsDbFields = super::AccountsDbFields; #[derive(Default, Clone, PartialEq, Eq, Debug, Deserialize, Serialize, AbiExample)] struct UnusedAccounts { unused1: HashSet, unused2: HashSet, unused3: HashMap, } // Deserializable version of Bank which need not be serializable, // because it's handled by SerializableVersionedBank. // So, sync fields with it! #[derive(Clone, Deserialize)] struct DeserializableVersionedBank { blockhash_queue: BlockhashQueue, ancestors: AncestorsForSerialization, hash: Hash, parent_hash: Hash, parent_slot: Slot, hard_forks: HardForks, transaction_count: u64, tick_height: u64, signature_count: u64, capitalization: u64, max_tick_height: u64, hashes_per_tick: Option, ticks_per_slot: u64, ns_per_slot: u128, genesis_creation_time: UnixTimestamp, slots_per_year: f64, accounts_data_len: u64, slot: Slot, epoch: Epoch, block_height: u64, collector_id: Pubkey, collector_fees: u64, fee_calculator: FeeCalculator, fee_rate_governor: FeeRateGovernor, collected_rent: u64, rent_collector: RentCollector, epoch_schedule: EpochSchedule, inflation: Inflation, stakes: Stakes, #[allow(dead_code)] unused_accounts: UnusedAccounts, epoch_stakes: HashMap, is_delta: bool, } impl From for BankFieldsToDeserialize { fn from(dvb: DeserializableVersionedBank) -> Self { BankFieldsToDeserialize { blockhash_queue: dvb.blockhash_queue, ancestors: dvb.ancestors, hash: dvb.hash, parent_hash: dvb.parent_hash, parent_slot: dvb.parent_slot, hard_forks: dvb.hard_forks, transaction_count: dvb.transaction_count, tick_height: dvb.tick_height, signature_count: dvb.signature_count, capitalization: dvb.capitalization, max_tick_height: dvb.max_tick_height, hashes_per_tick: dvb.hashes_per_tick, ticks_per_slot: dvb.ticks_per_slot, ns_per_slot: dvb.ns_per_slot, genesis_creation_time: dvb.genesis_creation_time, slots_per_year: dvb.slots_per_year, accounts_data_len: dvb.accounts_data_len, slot: dvb.slot, epoch: dvb.epoch, block_height: dvb.block_height, collector_id: dvb.collector_id, collector_fees: dvb.collector_fees, fee_calculator: dvb.fee_calculator, fee_rate_governor: dvb.fee_rate_governor, collected_rent: dvb.collected_rent, rent_collector: dvb.rent_collector, epoch_schedule: dvb.epoch_schedule, inflation: dvb.inflation, stakes: dvb.stakes, epoch_stakes: dvb.epoch_stakes, is_delta: dvb.is_delta, incremental_snapshot_persistence: None, epoch_accounts_hash: None, } } } // Serializable version of Bank, not Deserializable to avoid cloning by using refs. // Sync fields with DeserializableVersionedBank! #[derive(Serialize)] struct SerializableVersionedBank<'a> { blockhash_queue: &'a RwLock, ancestors: &'a AncestorsForSerialization, hash: Hash, parent_hash: Hash, parent_slot: Slot, hard_forks: &'a RwLock, transaction_count: u64, tick_height: u64, signature_count: u64, capitalization: u64, max_tick_height: u64, hashes_per_tick: Option, ticks_per_slot: u64, ns_per_slot: u128, genesis_creation_time: UnixTimestamp, slots_per_year: f64, accounts_data_len: u64, slot: Slot, epoch: Epoch, block_height: u64, collector_id: Pubkey, collector_fees: u64, fee_calculator: FeeCalculator, fee_rate_governor: FeeRateGovernor, collected_rent: u64, rent_collector: RentCollector, epoch_schedule: EpochSchedule, inflation: Inflation, #[serde(serialize_with = "serde_stakes_enum_compat::serialize")] stakes: StakesEnum, unused_accounts: UnusedAccounts, epoch_stakes: &'a HashMap, is_delta: bool, } impl<'a> From> for SerializableVersionedBank<'a> { fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self { Self { blockhash_queue: rhs.blockhash_queue, ancestors: rhs.ancestors, hash: rhs.hash, parent_hash: rhs.parent_hash, parent_slot: rhs.parent_slot, hard_forks: rhs.hard_forks, transaction_count: rhs.transaction_count, tick_height: rhs.tick_height, signature_count: rhs.signature_count, capitalization: rhs.capitalization, max_tick_height: rhs.max_tick_height, hashes_per_tick: rhs.hashes_per_tick, ticks_per_slot: rhs.ticks_per_slot, ns_per_slot: rhs.ns_per_slot, genesis_creation_time: rhs.genesis_creation_time, slots_per_year: rhs.slots_per_year, accounts_data_len: rhs.accounts_data_len, slot: rhs.slot, epoch: rhs.epoch, block_height: rhs.block_height, collector_id: rhs.collector_id, collector_fees: rhs.collector_fees, fee_calculator: rhs.fee_calculator, fee_rate_governor: rhs.fee_rate_governor, collected_rent: rhs.collected_rent, rent_collector: rhs.rent_collector, epoch_schedule: rhs.epoch_schedule, inflation: rhs.inflation, stakes: StakesEnum::from(rhs.stakes.stakes().clone()), unused_accounts: UnusedAccounts::default(), epoch_stakes: rhs.epoch_stakes, is_delta: rhs.is_delta, } } } #[cfg(RUSTC_WITH_SPECIALIZATION)] impl<'a> solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableVersionedBank<'a> {} #[derive(PartialEq, Eq)] pub(super) struct Context {} impl<'a> TypeContext<'a> for Context { type SerializableAccountStorageEntry = SerializableAccountStorageEntry; fn serialize_bank_and_storage( serializer: S, serializable_bank: &SerializableBankAndStorage<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized, { let ancestors = HashMap::from(&serializable_bank.bank.ancestors); let fields = serializable_bank.bank.get_fields_to_serialize(&ancestors); let lamports_per_signature = fields.fee_rate_governor.lamports_per_signature; ( SerializableVersionedBank::from(fields), SerializableAccountsDb::<'a, Self> { accounts_db: &serializable_bank.bank.rc.accounts.accounts_db, slot: serializable_bank.bank.rc.slot, account_storage_entries: serializable_bank.snapshot_storages, phantom: std::marker::PhantomData::default(), }, // Additional fields, we manually store the lamps per signature here so that // we can grab it on restart. // TODO: if we do a snapshot version bump, consider moving this out. lamports_per_signature, None::, serializable_bank .bank .get_epoch_accounts_hash_to_serialize() .map(|epoch_accounts_hash| *epoch_accounts_hash.as_ref()), ) .serialize(serializer) } #[cfg(test)] fn serialize_bank_and_storage_without_extra_fields( serializer: S, serializable_bank: &SerializableBankAndStorageNoExtra<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized, { let ancestors = HashMap::from(&serializable_bank.bank.ancestors); let fields = serializable_bank.bank.get_fields_to_serialize(&ancestors); ( SerializableVersionedBank::from(fields), SerializableAccountsDb::<'a, Self> { accounts_db: &serializable_bank.bank.rc.accounts.accounts_db, slot: serializable_bank.bank.rc.slot, account_storage_entries: serializable_bank.snapshot_storages, phantom: std::marker::PhantomData::default(), }, ) .serialize(serializer) } fn serialize_accounts_db_fields( serializer: S, serializable_db: &SerializableAccountsDb<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized, { // sample write version before serializing storage entries let version = serializable_db .accounts_db .write_version .load(Ordering::Acquire); // (1st of 3 elements) write the list of account storage entry lists out as a map let entry_count = RefCell::::new(0); let entries = serialize_iter_as_map(serializable_db.account_storage_entries.iter().map(|x| { *entry_count.borrow_mut() += x.len(); ( x.first().unwrap().slot(), serialize_iter_as_seq( x.iter() .map(|x| Self::SerializableAccountStorageEntry::from(x.as_ref())), ), ) })); let slot = serializable_db.slot; let accounts_delta_hash = serializable_db .accounts_db .get_accounts_delta_hash(slot) .map(Into::into) .unwrap_or_else(|| panic!("Missing accounts delta hash entry for slot {slot}")); // NOTE: The accounts hash is calculated in AHV, which is *after* a bank snapshot is taken // (and serialized here). Thus it is expected that an accounts hash is *not* found for // this slot, and a placeholder value will be used instead. The real accounts hash will be // set by `reserialize_bank_with_new_accounts_hash` from AHV. let accounts_hash = serializable_db .accounts_db .get_accounts_hash(slot) .map(|(accounts_hash, _)| accounts_hash.into()) .unwrap_or_default(); let stats = serializable_db .accounts_db .get_bank_hash_stats(slot) .unwrap_or_else(|| panic!("Missing bank hash stats entry for slot {slot}")); let bank_hash_info = BankHashInfo { accounts_delta_hash, accounts_hash, stats, }; let historical_roots = serializable_db .accounts_db .accounts_index .roots_tracker .read() .unwrap() .historical_roots .get_all(); let historical_roots_with_hash = Vec::<(Slot, Hash)>::default(); let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms"); let result = ( entries, version, slot, bank_hash_info, historical_roots, historical_roots_with_hash, ) .serialize(serializer); serialize_account_storage_timer.stop(); datapoint_info!( "serialize_account_storage_ms", ("duration", serialize_account_storage_timer.as_ms(), i64), ("num_entries", *entry_count.borrow(), i64), ); result } fn deserialize_bank_fields( mut stream: &mut BufReader, ) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error> where R: Read, { let mut bank_fields: BankFieldsToDeserialize = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?; // Process extra fields let lamports_per_signature = ignore_eof_error(deserialize_from(&mut stream))?; bank_fields.fee_rate_governor = bank_fields .fee_rate_governor .clone_with_lamports_per_signature(lamports_per_signature); let incremental_snapshot_persistence = ignore_eof_error(deserialize_from(&mut stream))?; bank_fields.incremental_snapshot_persistence = incremental_snapshot_persistence; let epoch_accounts_hash = ignore_eof_error(deserialize_from(stream))?; bank_fields.epoch_accounts_hash = epoch_accounts_hash; Ok((bank_fields, accounts_db_fields)) } fn deserialize_accounts_db_fields( stream: &mut BufReader, ) -> Result where R: Read, { deserialize_from(stream) } /// deserialize the bank from 'stream_reader' /// modify the accounts_hash and incremental_snapshot_persistence /// reserialize the bank to 'stream_writer' fn reserialize_bank_fields_with_hash( stream_reader: &mut BufReader, stream_writer: &mut BufWriter, accounts_hash: &AccountsHash, incremental_snapshot_persistence: Option<&BankIncrementalSnapshotPersistence>, ) -> std::result::Result<(), Box> where R: Read, W: Write, { let (bank_fields, mut accounts_db_fields) = Self::deserialize_bank_fields(stream_reader).unwrap(); accounts_db_fields.3.accounts_hash = (*accounts_hash).into(); let mut rhs = bank_fields; let blockhash_queue = RwLock::new(std::mem::take(&mut rhs.blockhash_queue)); let hard_forks = RwLock::new(std::mem::take(&mut rhs.hard_forks)); let lamports_per_signature = rhs.fee_rate_governor.lamports_per_signature; let epoch_accounts_hash = rhs.epoch_accounts_hash.as_ref(); let bank = SerializableVersionedBank { blockhash_queue: &blockhash_queue, ancestors: &rhs.ancestors, hash: rhs.hash, parent_hash: rhs.parent_hash, parent_slot: rhs.parent_slot, hard_forks: &hard_forks, transaction_count: rhs.transaction_count, tick_height: rhs.tick_height, signature_count: rhs.signature_count, capitalization: rhs.capitalization, max_tick_height: rhs.max_tick_height, hashes_per_tick: rhs.hashes_per_tick, ticks_per_slot: rhs.ticks_per_slot, ns_per_slot: rhs.ns_per_slot, genesis_creation_time: rhs.genesis_creation_time, slots_per_year: rhs.slots_per_year, accounts_data_len: rhs.accounts_data_len, slot: rhs.slot, epoch: rhs.epoch, block_height: rhs.block_height, collector_id: rhs.collector_id, collector_fees: rhs.collector_fees, fee_calculator: rhs.fee_calculator, fee_rate_governor: rhs.fee_rate_governor, collected_rent: rhs.collected_rent, rent_collector: rhs.rent_collector, epoch_schedule: rhs.epoch_schedule, inflation: rhs.inflation, stakes: StakesEnum::from(rhs.stakes), unused_accounts: UnusedAccounts::default(), epoch_stakes: &rhs.epoch_stakes, is_delta: rhs.is_delta, }; bincode::serialize_into( stream_writer, &( bank, accounts_db_fields, lamports_per_signature, incremental_snapshot_persistence, epoch_accounts_hash, ), ) } }