diff --git a/core/tests/bank_forks.rs b/core/tests/bank_forks.rs index 1ef9914b5f..4ffb56d382 100644 --- a/core/tests/bank_forks.rs +++ b/core/tests/bank_forks.rs @@ -110,6 +110,7 @@ mod tests { fn restore_from_snapshot( old_bank_forks: &BankForks, old_last_slot: Slot, + old_genesis_config: &GenesisConfig, account_paths: &[PathBuf], ) { let (snapshot_path, snapshot_package_output_path) = old_bank_forks @@ -134,7 +135,7 @@ mod tests { &CompressionType::Bzip2, ), CompressionType::Bzip2, - &GenesisConfig::default(), + old_genesis_config, ) .unwrap(); @@ -169,7 +170,6 @@ mod tests { let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1); let bank_forks = &mut snapshot_test_config.bank_forks; - let accounts_dir = &snapshot_test_config.accounts_dir; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let (s, _r) = channel(); @@ -185,6 +185,7 @@ mod tests { bank_forks.set_root(bank.slot(), &sender, None); } } + // Generate a snapshot package for last bank let last_bank = bank_forks.get(last_slot).unwrap(); let snapshot_config = &snapshot_test_config.snapshot_config; @@ -203,10 +204,12 @@ mod tests { snapshot_version, ) .unwrap(); - snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap(); - restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]); + // Restore bank from snapshot + let account_paths = &[snapshot_test_config.accounts_dir.path().to_path_buf()]; + let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config; + restore_from_snapshot(bank_forks, last_slot, genesis_config, account_paths); } fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) { diff --git a/programs/librapay/src/lib.rs b/programs/librapay/src/lib.rs index e24a199f70..68efbfa6cb 100644 --- a/programs/librapay/src/lib.rs +++ b/programs/librapay/src/lib.rs @@ -38,7 +38,9 @@ pub fn create_genesis(from: &Keypair, client: &T, amount: u64) -> Key let instruction = librapay_instruction::genesis(&genesis.pubkey(), amount); let message = Message::new(&[instruction], Some(&from.pubkey())); - client.send_and_confirm_message(&[from, &genesis], message).unwrap(); + client + .send_and_confirm_message(&[from, &genesis], message) + .unwrap(); genesis } diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index 76bd9b093a..2458af54d4 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -66,23 +66,19 @@ pub enum AccountAddressFilter { } impl Accounts { - pub(crate) fn new_empty(accounts_db: AccountsDB) -> Self { + pub fn new(paths: Vec) -> Self { Self { - accounts_db: Arc::new(accounts_db), + slot: 0, + accounts_db: Arc::new(AccountsDB::new(paths)), account_locks: Mutex::new(HashSet::new()), readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))), - ..Self::default() } } - pub fn new(paths: Vec) -> Self { - Self::new_with_frozen_accounts(paths, &HashMap::default(), &[]) - } - pub fn new_from_parent(parent: &Accounts, slot: Slot, parent_slot: Slot) -> Self { let accounts_db = parent.accounts_db.clone(); accounts_db.set_hash(slot, parent_slot); - Accounts { + Self { slot, accounts_db, account_locks: Mutex::new(HashSet::new()), @@ -90,25 +86,13 @@ impl Accounts { } } - pub fn new_with_frozen_accounts( - paths: Vec, - ancestors: &Ancestors, - frozen_account_pubkeys: &[Pubkey], - ) -> Self { - let mut accounts = Accounts { + pub(crate) fn new_empty(accounts_db: AccountsDB) -> Self { + Self { slot: 0, - accounts_db: Arc::new(AccountsDB::new(paths)), + accounts_db: Arc::new(accounts_db), account_locks: Mutex::new(HashSet::new()), readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))), - }; - accounts.freeze_accounts(ancestors, frozen_account_pubkeys); - accounts - } - - pub fn freeze_accounts(&mut self, ancestors: &Ancestors, frozen_account_pubkeys: &[Pubkey]) { - Arc::get_mut(&mut self.accounts_db) - .unwrap() - .freeze_accounts(ancestors, frozen_account_pubkeys); + } } /// Return true if the slice has any duplicate elements diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index 08534acc28..f0d7aeced2 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -1800,7 +1800,7 @@ impl AccountsDB { hashes } - pub fn freeze_accounts(&mut self, ancestors: &Ancestors, account_pubkeys: &[Pubkey]) { + pub(crate) fn freeze_accounts(&mut self, ancestors: &Ancestors, account_pubkeys: &[Pubkey]) { for account_pubkey in account_pubkeys { if let Some((account, _slot)) = self.load_slow(ancestors, &account_pubkey) { let frozen_account_info = FrozenAccountInfo { @@ -1994,7 +1994,7 @@ impl AccountsDB { } } - pub fn print_accounts_stats(&self, label: &'static str) { + pub(crate) fn print_accounts_stats(&self, label: &'static str) { self.print_index(label); self.print_count_and_status(label); } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index ce98b84467..6e6b4b94c6 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -25,7 +25,6 @@ use crate::{ use byteorder::{ByteOrder, LittleEndian}; use itertools::Itertools; use log::*; -use serde::{Deserialize, Serialize}; use solana_measure::measure::Measure; use solana_metrics::{ datapoint_debug, inc_new_counter_debug, inc_new_counter_error, inc_new_counter_info, @@ -108,20 +107,6 @@ pub struct BankRc { pub(crate) slot: Slot, } -#[cfg(RUSTC_WITH_SPECIALIZATION)] -use solana_sdk::abi_example::AbiExample; -#[cfg(RUSTC_WITH_SPECIALIZATION)] -impl AbiExample for BankRc { - fn example() -> Self { - BankRc { - // Set parent to None to cut the recursion into another Bank - parent: RwLock::new(None), - accounts: AbiExample::example(), - slot: AbiExample::example(), - } - } -} - impl BankRc { pub(crate) fn new(accounts: Accounts, slot: Slot) -> Self { Self { @@ -203,22 +188,88 @@ impl HashAgeKind { } } -#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)] -struct UnusedAccounts { - unused1: HashSet, - unused2: HashSet, - unused3: HashMap, +// Bank's common fields shared by all supported snapshot versions for deserialization. +// Sync fields with BankFieldsToSerialize! This is paired with it. +// All members are made public to remain Bank's members private and to make versioned deserializer workable on this +#[derive(Clone, Default)] +pub(crate) struct BankFieldsToDeserialize { + pub(crate) blockhash_queue: BlockhashQueue, + pub(crate) ancestors: Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: HardForks, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: Stakes, + pub(crate) epoch_stakes: HashMap, + pub(crate) is_delta: bool, +} + +// Bank's common fields shared by all supported snapshot versions for serialization. +// This is separated from BankFieldsToDeserialize to avoid cloning by using refs. +// So, sync fields with BankFieldsToDeserialize! +// all members are made public to remain Bank private and to make versioned serializer workable on this +pub(crate) struct BankFieldsToSerialize<'a> { + pub(crate) blockhash_queue: &'a RwLock, + pub(crate) ancestors: &'a Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: &'a RwLock, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: &'a RwLock, + pub(crate) epoch_stakes: &'a HashMap, + pub(crate) is_delta: bool, } /// Manager for the state of all accounts and programs after processing its entries. -#[frozen_abi(digest = "GsvisJfTaHxmTX4s6gQs2bZtxVggB7F325XDdXTA573p")] -#[derive(Default, Deserialize, Serialize, AbiExample)] +#[derive(Default)] pub struct Bank { /// References to accounts, parent and signature status - #[serde(skip)] pub rc: BankRc, - #[serde(skip)] pub src: StatusCacheRc, /// FIFO queue of `recent_blockhash` items @@ -308,9 +359,6 @@ pub struct Bank { /// cache of vote_account and stake_account state for this fork stakes: RwLock, - /// unused - unused_accounts: RwLock, - /// staked nodes on epoch boundaries, saved off when a bank.slot() is at /// a leader schedule calculation boundary epoch_stakes: HashMap, @@ -324,24 +372,18 @@ pub struct Bank { /// Callback to be notified when a bank enters a new Epoch /// (used to adjust cluster features over time) - #[serde(skip)] entered_epoch_callback: Arc>>, /// Last time when the cluster info vote listener has synced with this bank - #[serde(skip)] pub last_vote_sync: AtomicU64, /// Rewards that were paid out immediately after this bank was created - #[serde(skip)] pub rewards: Option>, - #[serde(skip)] pub skip_drop: AtomicBool, - #[serde(skip)] pub operating_mode: Option, - #[serde(skip)] pub lazy_rent_collection: AtomicBool, } @@ -370,7 +412,7 @@ impl Bank { bank.finish_init(); // Freeze accounts after process_genesis_config creates the initial append vecs - Arc::get_mut(&mut bank.rc.accounts) + Arc::get_mut(&mut Arc::get_mut(&mut bank.rc.accounts).unwrap().accounts_db) .unwrap() .freeze_accounts(&bank.ancestors, frozen_account_pubkeys); @@ -440,7 +482,6 @@ impl Bank { transaction_count: AtomicU64::new(parent.transaction_count()), stakes: RwLock::new(parent.stakes.read().unwrap().clone_with_epoch(epoch)), epoch_stakes: parent.epoch_stakes.clone(), - unused_accounts: RwLock::new(parent.unused_accounts.read().unwrap().clone()), parent_hash: parent.hash(), parent_slot: parent.slot(), collector_id: *collector_id, @@ -514,6 +555,136 @@ impl Bank { new } + /// Create a bank from explicit arguments and deserialized fields from snapshot + #[allow(clippy::float_cmp)] + pub(crate) fn new_from_fields( + bank_rc: BankRc, + genesis_config: &GenesisConfig, + fields: BankFieldsToDeserialize, + ) -> Self { + fn new() -> T { + T::default() + } + let mut bank = Self { + rc: bank_rc, + src: new(), + blockhash_queue: RwLock::new(fields.blockhash_queue), + ancestors: fields.ancestors, + hash: RwLock::new(fields.hash), + parent_hash: fields.parent_hash, + parent_slot: fields.parent_slot, + hard_forks: Arc::new(RwLock::new(fields.hard_forks)), + transaction_count: AtomicU64::new(fields.transaction_count), + tick_height: AtomicU64::new(fields.tick_height), + signature_count: AtomicU64::new(fields.signature_count), + capitalization: AtomicU64::new(fields.capitalization), + max_tick_height: fields.max_tick_height, + hashes_per_tick: fields.hashes_per_tick, + ticks_per_slot: fields.ticks_per_slot, + ns_per_slot: fields.ns_per_slot, + genesis_creation_time: fields.genesis_creation_time, + slots_per_year: fields.slots_per_year, + unused: genesis_config.unused, + slot: fields.slot, + epoch: fields.epoch, + block_height: fields.block_height, + collector_id: fields.collector_id, + collector_fees: AtomicU64::new(fields.collector_fees), + fee_calculator: fields.fee_calculator, + fee_rate_governor: fields.fee_rate_governor, + collected_rent: AtomicU64::new(fields.collected_rent), + rent_collector: fields.rent_collector, + epoch_schedule: fields.epoch_schedule, + inflation: Arc::new(RwLock::new(fields.inflation)), + stakes: RwLock::new(fields.stakes), + epoch_stakes: fields.epoch_stakes, + is_delta: AtomicBool::new(fields.is_delta), + message_processor: new(), + entered_epoch_callback: new(), + last_vote_sync: new(), + rewards: new(), + skip_drop: new(), + operating_mode: Some(genesis_config.operating_mode), + lazy_rent_collection: new(), + }; + bank.finish_init(); + + // Sanity assertions between bank snapshot and genesis config + // Consider removing from serializable bank state ([Ref]BankFields) and initializing + // from the passed in genesis_config instead (as new()/new_with_paths() already do) + assert_eq!( + bank.hashes_per_tick, + genesis_config.poh_config.hashes_per_tick + ); + assert_eq!(bank.ticks_per_slot, genesis_config.ticks_per_slot); + assert_eq!( + bank.ns_per_slot, + genesis_config.poh_config.target_tick_duration.as_nanos() + * genesis_config.ticks_per_slot as u128 + ); + assert_eq!(bank.genesis_creation_time, genesis_config.creation_time); + assert_eq!(bank.unused, genesis_config.unused); + assert_eq!(bank.max_tick_height, (bank.slot + 1) * bank.ticks_per_slot); + assert_eq!( + bank.slots_per_year, + years_as_slots( + 1.0, + &genesis_config.poh_config.target_tick_duration, + bank.ticks_per_slot, + ) + ); + assert_eq!(bank.epoch_schedule, genesis_config.epoch_schedule); + assert_eq!(bank.epoch, bank.epoch_schedule.get_epoch(bank.slot)); + assert_eq!( + bank.rent_collector, + RentCollector::new( + bank.epoch, + &bank.epoch_schedule, + bank.slots_per_year, + &genesis_config.rent, + ) + ); + + bank + } + + /// Return subset of bank fields representing serializable state + pub(crate) fn get_fields_to_serialize(&self) -> BankFieldsToSerialize { + BankFieldsToSerialize { + blockhash_queue: &self.blockhash_queue, + ancestors: &self.ancestors, + hash: *self.hash.read().unwrap(), + parent_hash: self.parent_hash, + parent_slot: self.parent_slot, + hard_forks: &*self.hard_forks, + transaction_count: self.transaction_count.load(Ordering::Relaxed), + tick_height: self.tick_height.load(Ordering::Relaxed), + signature_count: self.signature_count.load(Ordering::Relaxed), + capitalization: self.capitalization.load(Ordering::Relaxed), + max_tick_height: self.max_tick_height, + hashes_per_tick: self.hashes_per_tick, + ticks_per_slot: self.ticks_per_slot, + ns_per_slot: self.ns_per_slot, + genesis_creation_time: self.genesis_creation_time, + slots_per_year: self.slots_per_year, + unused: self.unused, + slot: self.slot, + epoch: self.epoch, + block_height: self.block_height, + collector_id: self.collector_id, + collector_fees: self.collector_fees.load(Ordering::Relaxed), + fee_calculator: self.fee_calculator.clone(), + fee_rate_governor: self.fee_rate_governor.clone(), + collected_rent: self.collected_rent.load(Ordering::Relaxed), + rent_collector: self.rent_collector.clone(), + epoch_schedule: self.epoch_schedule, + inflation: *self.inflation.read().unwrap(), + stakes: &self.stakes, + epoch_stakes: &self.epoch_stakes, + is_delta: self.is_delta.load(Ordering::Relaxed), + } + } + pub fn collector_id(&self) -> &Pubkey { &self.collector_id } diff --git a/runtime/src/rent_collector.rs b/runtime/src/rent_collector.rs index 375b61f8c1..0b42d617a4 100644 --- a/runtime/src/rent_collector.rs +++ b/runtime/src/rent_collector.rs @@ -4,7 +4,7 @@ use solana_sdk::{ rent::Rent, sysvar, }; -#[derive(Default, Serialize, Deserialize, Clone, AbiExample)] +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Debug, AbiExample)] pub struct RentCollector { pub epoch: Epoch, pub epoch_schedule: EpochSchedule, diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index a09021c410..fac72d12d1 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -4,10 +4,17 @@ use { accounts_db::{ AccountStorageEntry, AccountStorageStatus, AccountsDB, AppendVecId, BankHashInfo, }, + accounts_index::Ancestors, append_vec::AppendVec, - bank::BankRc, + bank::{Bank, BankFieldsToDeserialize, BankRc}, + blockhash_queue::BlockhashQueue, + epoch_stakes::EpochStakes, + message_processor::MessageProcessor, + rent_collector::RentCollector, + stakes::Stakes, }, - bincode::{deserialize_from, serialize_into, Error}, + bincode, + bincode::{config::Options, serialize_into, Error}, fs_extra::dir::CopyOptions, log::{info, warn}, rand::{thread_rng, Rng}, @@ -15,7 +22,16 @@ use { de::{DeserializeOwned, Visitor}, Deserialize, Deserializer, Serialize, Serializer, }, - solana_sdk::clock::Slot, + solana_sdk::{ + clock::{Epoch, Slot, UnixTimestamp}, + epoch_schedule::EpochSchedule, + fee_calculator::{FeeCalculator, FeeRateGovernor}, + genesis_config::GenesisConfig, + hard_forks::HardForks, + hash::Hash, + inflation::Inflation, + pubkey::Pubkey, + }, std::{ cmp::min, collections::HashMap, @@ -28,6 +44,10 @@ use { }, }; +#[cfg(RUSTC_WITH_SPECIALIZATION)] +use solana_sdk::abi_example::IgnoreAsHelper; + +mod common; mod future; mod legacy; mod tests; @@ -42,28 +62,28 @@ use utils::{serialize_iter_as_map, serialize_iter_as_seq, serialize_iter_as_tupl #[cfg(test)] pub(crate) use self::tests::reconstruct_accounts_db_via_serialization; -pub use crate::accounts_db::{SnapshotStorage, SnapshotStorages}; +pub(crate) use crate::accounts_db::{SnapshotStorage, SnapshotStorages}; #[derive(Copy, Clone, Eq, PartialEq)] -pub enum SerdeStyle { +pub(crate) enum SerdeStyle { NEWER, OLDER, } -const MAX_ACCOUNTS_DB_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024; +const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024; -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -pub struct AccountDBFields(HashMap>, u64, Slot, BankHashInfo); +#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)] +struct AccountsDbFields(HashMap>, u64, Slot, BankHashInfo); -pub trait TypeContext<'a> { +trait TypeContext<'a> { type SerializableAccountStorageEntry: Serialize + DeserializeOwned + From<&'a AccountStorageEntry> + Into; - fn serialize_bank_rc_fields( + fn serialize_bank_and_storage( serializer: S, - serializable_bank: &SerializableBankRc<'a, Self>, + serializable_bank: &SerializableBankAndStorage<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized; @@ -75,37 +95,63 @@ pub trait TypeContext<'a> { where Self: std::marker::Sized; - fn deserialize_accounts_db_fields( + fn deserialize_bank_fields( stream: &mut BufReader, - ) -> Result, Error> + ) -> Result< + ( + BankFieldsToDeserialize, + AccountsDbFields, + ), + Error, + > where R: Read; - // we might define fn (de)serialize_bank(...) -> Result for versionized bank serialization in the future + fn deserialize_accounts_db_fields( + stream: &mut BufReader, + ) -> Result, Error> + where + R: Read; } -pub fn bankrc_from_stream( +fn deserialize_from(reader: R) -> bincode::Result +where + R: Read, + T: DeserializeOwned, +{ + bincode::options() + .with_limit(MAX_STREAM_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() + .deserialize_from::(reader) +} + +pub(crate) fn bank_from_stream( serde_style: SerdeStyle, - account_paths: &[PathBuf], - slot: Slot, stream: &mut BufReader, - stream_append_vecs_path: P, -) -> std::result::Result + append_vecs_path: P, + account_paths: &[PathBuf], + genesis_config: &GenesisConfig, + frozen_account_pubkeys: &[Pubkey], +) -> std::result::Result where R: Read, P: AsRef, { macro_rules! INTO { - ($x:ident) => { - Ok(BankRc::new( - Accounts::new_empty(context_accountsdb_from_fields::<$x, P>( - $x::deserialize_accounts_db_fields(stream)?, - account_paths, - stream_append_vecs_path, - )?), - slot, - )) - }; + ($x:ident) => {{ + let (bank_fields, accounts_db_fields) = $x::deserialize_bank_fields(stream)?; + + let bank = reconstruct_bank_from_fields( + bank_fields, + accounts_db_fields, + genesis_config, + frozen_account_pubkeys, + account_paths, + append_vecs_path, + )?; + Ok(bank) + }}; } match serde_style { SerdeStyle::NEWER => INTO!(TypeContextFuture), @@ -117,10 +163,10 @@ where }) } -pub fn bankrc_to_stream( +pub(crate) fn bank_to_stream( serde_style: SerdeStyle, stream: &mut BufWriter, - bank_rc: &BankRc, + bank: &Bank, snapshot_storages: &[SnapshotStorage], ) -> Result<(), Error> where @@ -128,10 +174,10 @@ where { macro_rules! INTO { ($x:ident) => { - serialize_into( + bincode::serialize_into( stream, - &SerializableBankRc::<$x> { - bank_rc, + &SerializableBankAndStorage::<$x> { + bank, snapshot_storages, phantom: std::marker::PhantomData::default(), }, @@ -148,22 +194,22 @@ where }) } -pub struct SerializableBankRc<'a, C> { - bank_rc: &'a BankRc, +struct SerializableBankAndStorage<'a, C> { + bank: &'a Bank, snapshot_storages: &'a [SnapshotStorage], phantom: std::marker::PhantomData, } -impl<'a, C: TypeContext<'a>> Serialize for SerializableBankRc<'a, C> { +impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorage<'a, C> { fn serialize(&self, serializer: S) -> std::result::Result where S: serde::ser::Serializer, { - C::serialize_bank_rc_fields(serializer, self) + C::serialize_bank_and_storage(serializer, self) } } -pub struct SerializableAccountsDB<'a, C> { +struct SerializableAccountsDB<'a, C> { accounts_db: &'a AccountsDB, slot: Slot, account_storage_entries: &'a [SnapshotStorage], @@ -179,18 +225,43 @@ impl<'a, C: TypeContext<'a>> Serialize for SerializableAccountsDB<'a, C> { } } -fn context_accountsdb_from_fields<'a, C, P>( - account_db_fields: AccountDBFields, +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl<'a, C> IgnoreAsHelper for SerializableAccountsDB<'a, C> {} + +fn reconstruct_bank_from_fields( + bank_fields: BankFieldsToDeserialize, + accounts_db_fields: AccountsDbFields, + genesis_config: &GenesisConfig, + frozen_account_pubkeys: &[Pubkey], + account_paths: &[PathBuf], + append_vecs_path: P, +) -> Result +where + E: Into, + P: AsRef, +{ + let mut accounts_db = + reconstruct_accountsdb_from_fields(accounts_db_fields, account_paths, append_vecs_path)?; + accounts_db.freeze_accounts(&bank_fields.ancestors, frozen_account_pubkeys); + + let bank_rc = BankRc::new(Accounts::new_empty(accounts_db), bank_fields.slot); + let bank = Bank::new_from_fields(bank_rc, genesis_config, bank_fields); + + Ok(bank) +} + +fn reconstruct_accountsdb_from_fields( + accounts_db_fields: AccountsDbFields, account_paths: &[PathBuf], stream_append_vecs_path: P, ) -> Result where - C: TypeContext<'a>, + E: Into, P: AsRef, { let accounts_db = AccountsDB::new(account_paths.to_vec()); - let AccountDBFields(storage, version, slot, bank_hash_info) = account_db_fields; + let AccountsDbFields(storage, version, slot, bank_hash_info) = accounts_db_fields; // convert to two level map of slot -> id -> account storage entry let storage = { diff --git a/runtime/src/serde_snapshot/common.rs b/runtime/src/serde_snapshot/common.rs new file mode 100644 index 0000000000..23cdf45eec --- /dev/null +++ b/runtime/src/serde_snapshot/common.rs @@ -0,0 +1,9 @@ +use super::*; +use std::collections::HashSet; + +#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)] +pub(crate) struct UnusedAccounts { + unused1: HashSet, + unused2: HashSet, + unused3: HashMap, +} diff --git a/runtime/src/serde_snapshot/future.rs b/runtime/src/serde_snapshot/future.rs index ce12527e32..da1c6fdd36 100644 --- a/runtime/src/serde_snapshot/future.rs +++ b/runtime/src/serde_snapshot/future.rs @@ -1,5 +1,10 @@ +use super::common::UnusedAccounts; +#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] +use solana_sdk::abi_example::IgnoreAsHelper; use {super::*, solana_measure::measure::Measure, std::cell::RefCell}; +type AccountsDbFields = super::AccountsDbFields; + // Serializable version of AccountStorageEntry for snapshot format #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub(super) struct SerializableAccountStorageEntry { @@ -25,25 +30,190 @@ impl Into for SerializableAccountStorageEntry { } } +use std::sync::RwLock; +// Deserializable version of Bank which need not be serializable, +// because it's handled by SerializableVersionedBank. +// So, sync fields with it! +#[derive(Clone, Deserialize)] +pub(crate) struct DeserializableVersionedBank { + pub(crate) blockhash_queue: BlockhashQueue, + pub(crate) ancestors: Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: HardForks, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: Stakes, + pub(crate) unused_accounts: UnusedAccounts, + pub(crate) epoch_stakes: HashMap, + pub(crate) is_delta: bool, + pub(crate) message_processor: MessageProcessor, +} + +impl Into for DeserializableVersionedBank { + fn into(self) -> BankFieldsToDeserialize { + BankFieldsToDeserialize { + blockhash_queue: self.blockhash_queue, + ancestors: self.ancestors, + hash: self.hash, + parent_hash: self.parent_hash, + parent_slot: self.parent_slot, + hard_forks: self.hard_forks, + transaction_count: self.transaction_count, + tick_height: self.tick_height, + signature_count: self.signature_count, + capitalization: self.capitalization, + max_tick_height: self.max_tick_height, + hashes_per_tick: self.hashes_per_tick, + ticks_per_slot: self.ticks_per_slot, + ns_per_slot: self.ns_per_slot, + genesis_creation_time: self.genesis_creation_time, + slots_per_year: self.slots_per_year, + unused: self.unused, + slot: self.slot, + epoch: self.epoch, + block_height: self.block_height, + collector_id: self.collector_id, + collector_fees: self.collector_fees, + fee_calculator: self.fee_calculator, + fee_rate_governor: self.fee_rate_governor, + collected_rent: self.collected_rent, + rent_collector: self.rent_collector, + epoch_schedule: self.epoch_schedule, + inflation: self.inflation, + stakes: self.stakes, + epoch_stakes: self.epoch_stakes, + is_delta: self.is_delta, + } + } +} +// Serializable version of Bank, not Deserializable to avoid cloning by using refs. +// Sync fields with DeserializableVersionedBank! +#[derive(Serialize)] +pub(crate) struct SerializableVersionedBank<'a> { + pub(crate) blockhash_queue: &'a RwLock, + pub(crate) ancestors: &'a Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: &'a RwLock, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: &'a RwLock, + pub(crate) unused_accounts: UnusedAccounts, + pub(crate) epoch_stakes: &'a HashMap, + pub(crate) is_delta: bool, + pub(crate) message_processor: MessageProcessor, +} + +impl<'a> From> for SerializableVersionedBank<'a> { + fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self { + fn new() -> T { + T::default() + } + Self { + blockhash_queue: rhs.blockhash_queue, + ancestors: rhs.ancestors, + hash: rhs.hash, + parent_hash: rhs.parent_hash, + parent_slot: rhs.parent_slot, + hard_forks: rhs.hard_forks, + transaction_count: rhs.transaction_count, + tick_height: rhs.tick_height, + signature_count: rhs.signature_count, + capitalization: rhs.capitalization, + max_tick_height: rhs.max_tick_height, + hashes_per_tick: rhs.hashes_per_tick, + ticks_per_slot: rhs.ticks_per_slot, + ns_per_slot: rhs.ns_per_slot, + genesis_creation_time: rhs.genesis_creation_time, + slots_per_year: rhs.slots_per_year, + unused: rhs.unused, + slot: rhs.slot, + epoch: rhs.epoch, + block_height: rhs.block_height, + collector_id: rhs.collector_id, + collector_fees: rhs.collector_fees, + fee_calculator: rhs.fee_calculator, + fee_rate_governor: rhs.fee_rate_governor, + collected_rent: rhs.collected_rent, + rent_collector: rhs.rent_collector, + epoch_schedule: rhs.epoch_schedule, + inflation: rhs.inflation, + stakes: rhs.stakes, + unused_accounts: new(), + epoch_stakes: rhs.epoch_stakes, + is_delta: rhs.is_delta, + message_processor: new(), + } + } +} + +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl<'a> IgnoreAsHelper for SerializableVersionedBank<'a> {} + pub(super) struct Context {} impl<'a> TypeContext<'a> for Context { type SerializableAccountStorageEntry = SerializableAccountStorageEntry; - fn serialize_bank_rc_fields( + fn serialize_bank_and_storage( serializer: S, - serializable_bank: &SerializableBankRc<'a, Self>, + serializable_bank: &SerializableBankAndStorage<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized, { - let accounts_db_serialize = SerializableAccountsDB::<'a, Self> { - accounts_db: &*serializable_bank.bank_rc.accounts.accounts_db, - slot: serializable_bank.bank_rc.slot, - account_storage_entries: serializable_bank.snapshot_storages, - phantom: std::marker::PhantomData::default(), - }; - - accounts_db_serialize.serialize(serializer) + ( + SerializableVersionedBank::from(serializable_bank.bank.get_fields_to_serialize()), + SerializableAccountsDB::<'a, Self> { + accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db, + slot: serializable_bank.bank.rc.slot, + account_storage_entries: serializable_bank.snapshot_storages, + phantom: std::marker::PhantomData::default(), + }, + ) + .serialize(serializer) } fn serialize_accounts_db_fields( @@ -93,12 +263,23 @@ impl<'a> TypeContext<'a> for Context { result } - fn deserialize_accounts_db_fields( + fn deserialize_bank_fields( mut stream: &mut BufReader, - ) -> Result, Error> + ) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error> where R: Read, { - deserialize_from(&mut stream) + let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); + let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?; + Ok((bank_fields, accounts_db_fields)) + } + + fn deserialize_accounts_db_fields( + stream: &mut BufReader, + ) -> Result + where + R: Read, + { + deserialize_from(stream) } } diff --git a/runtime/src/serde_snapshot/legacy.rs b/runtime/src/serde_snapshot/legacy.rs index 62f477355d..226ec44119 100644 --- a/runtime/src/serde_snapshot/legacy.rs +++ b/runtime/src/serde_snapshot/legacy.rs @@ -1,7 +1,12 @@ +use super::common::UnusedAccounts; #[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] use solana_sdk::abi_example::IgnoreAsHelper; use {super::*, bincode::config::Options, solana_measure::measure::Measure, std::cell::RefCell}; +const MAX_ACCOUNTS_DB_STREAM_SIZE: u64 = MAX_STREAM_SIZE; + +type AccountsDbFields = super::AccountsDbFields; + // Serializable version of AccountStorageEntry for snapshot format #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub(super) struct SerializableAccountStorageEntry { @@ -52,7 +57,7 @@ impl Into for SerializableAppendVec { } } -// Serialization of AppendVec requires serialization of u64 to +// Serialization of AppendVec requires serialization of u64 to // eight byte vector which is then itself serialized to the stream impl Serialize for SerializableAppendVec { fn serialize(&self, serializer: S) -> Result @@ -67,7 +72,7 @@ impl Serialize for SerializableAppendVec { } } -// Deserialization of AppendVec requires deserialization +// Deserialization of AppendVec requires deserialization // of eight byte vector from which u64 is then deserialized impl<'de> Deserialize<'de> for SerializableAppendVec { fn deserialize(deserializer: D) -> Result @@ -99,27 +104,193 @@ impl<'de> Deserialize<'de> for SerializableAppendVec { } } +// Deserializable version of Bank which need not be serializable, +// because it's handled by SerializableVersionedBank. +// So, sync fields with it! +#[derive(Clone, Deserialize)] +pub(crate) struct DeserializableVersionedBank { + pub(crate) blockhash_queue: BlockhashQueue, + pub(crate) ancestors: Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: HardForks, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: Stakes, + pub(crate) unused_accounts: UnusedAccounts, + pub(crate) epoch_stakes: HashMap, + pub(crate) is_delta: bool, + pub(crate) message_processor: MessageProcessor, +} + +impl Into for DeserializableVersionedBank { + fn into(self) -> BankFieldsToDeserialize { + BankFieldsToDeserialize { + blockhash_queue: self.blockhash_queue, + ancestors: self.ancestors, + hash: self.hash, + parent_hash: self.parent_hash, + parent_slot: self.parent_slot, + hard_forks: self.hard_forks, + transaction_count: self.transaction_count, + tick_height: self.tick_height, + signature_count: self.signature_count, + capitalization: self.capitalization, + max_tick_height: self.max_tick_height, + hashes_per_tick: self.hashes_per_tick, + ticks_per_slot: self.ticks_per_slot, + ns_per_slot: self.ns_per_slot, + genesis_creation_time: self.genesis_creation_time, + slots_per_year: self.slots_per_year, + unused: self.unused, + slot: self.slot, + epoch: self.epoch, + block_height: self.block_height, + collector_id: self.collector_id, + collector_fees: self.collector_fees, + fee_calculator: self.fee_calculator, + fee_rate_governor: self.fee_rate_governor, + collected_rent: self.collected_rent, + rent_collector: self.rent_collector, + epoch_schedule: self.epoch_schedule, + inflation: self.inflation, + stakes: self.stakes, + epoch_stakes: self.epoch_stakes, + is_delta: self.is_delta, + } + } +} +// Serializable version of Bank, not Deserializable to avoid cloning by using refs. +// Sync fields with DeserializableVersionedBank! +#[derive(Serialize)] +pub(crate) struct SerializableVersionedBank<'a> { + pub(crate) blockhash_queue: &'a RwLock, + pub(crate) ancestors: &'a Ancestors, + pub(crate) hash: Hash, + pub(crate) parent_hash: Hash, + pub(crate) parent_slot: Slot, + pub(crate) hard_forks: &'a RwLock, + pub(crate) transaction_count: u64, + pub(crate) tick_height: u64, + pub(crate) signature_count: u64, + pub(crate) capitalization: u64, + pub(crate) max_tick_height: u64, + pub(crate) hashes_per_tick: Option, + pub(crate) ticks_per_slot: u64, + pub(crate) ns_per_slot: u128, + pub(crate) genesis_creation_time: UnixTimestamp, + pub(crate) slots_per_year: f64, + pub(crate) unused: u64, + pub(crate) slot: Slot, + pub(crate) epoch: Epoch, + pub(crate) block_height: u64, + pub(crate) collector_id: Pubkey, + pub(crate) collector_fees: u64, + pub(crate) fee_calculator: FeeCalculator, + pub(crate) fee_rate_governor: FeeRateGovernor, + pub(crate) collected_rent: u64, + pub(crate) rent_collector: RentCollector, + pub(crate) epoch_schedule: EpochSchedule, + pub(crate) inflation: Inflation, + pub(crate) stakes: &'a RwLock, + pub(crate) unused_accounts: UnusedAccounts, + pub(crate) epoch_stakes: &'a HashMap, + pub(crate) is_delta: bool, + pub(crate) message_processor: MessageProcessor, +} + +use std::sync::RwLock; + +impl<'a> From> for SerializableVersionedBank<'a> { + fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self { + fn new() -> T { + T::default() + } + Self { + blockhash_queue: rhs.blockhash_queue, + ancestors: rhs.ancestors, + hash: rhs.hash, + parent_hash: rhs.parent_hash, + parent_slot: rhs.parent_slot, + hard_forks: rhs.hard_forks, + transaction_count: rhs.transaction_count, + tick_height: rhs.tick_height, + signature_count: rhs.signature_count, + capitalization: rhs.capitalization, + max_tick_height: rhs.max_tick_height, + hashes_per_tick: rhs.hashes_per_tick, + ticks_per_slot: rhs.ticks_per_slot, + ns_per_slot: rhs.ns_per_slot, + genesis_creation_time: rhs.genesis_creation_time, + slots_per_year: rhs.slots_per_year, + unused: rhs.unused, + slot: rhs.slot, + epoch: rhs.epoch, + block_height: rhs.block_height, + collector_id: rhs.collector_id, + collector_fees: rhs.collector_fees, + fee_calculator: rhs.fee_calculator, + fee_rate_governor: rhs.fee_rate_governor, + collected_rent: rhs.collected_rent, + rent_collector: rhs.rent_collector, + epoch_schedule: rhs.epoch_schedule, + inflation: rhs.inflation, + stakes: rhs.stakes, + unused_accounts: new(), + epoch_stakes: rhs.epoch_stakes, + is_delta: rhs.is_delta, + message_processor: new(), + } + } +} + +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl<'a> IgnoreAsHelper for SerializableVersionedBank<'a> {} + pub(super) struct Context {} impl<'a> TypeContext<'a> for Context { type SerializableAccountStorageEntry = SerializableAccountStorageEntry; - fn serialize_bank_rc_fields( + fn serialize_bank_and_storage( serializer: S, - serializable_bank: &SerializableBankRc<'a, Self>, + serializable_bank_and_storage: &SerializableBankAndStorage<'a, Self>, ) -> std::result::Result where Self: std::marker::Sized, { - // as there is no deserialize_bank_rc_fields(), do not emit the u64 - // size field here and have serialize_accounts_db_fields() emit two - // u64 size fields instead - SerializableAccountsDB::<'a, Self> { - accounts_db: &*serializable_bank.bank_rc.accounts.accounts_db, - slot: serializable_bank.bank_rc.slot, - account_storage_entries: serializable_bank.snapshot_storages, - phantom: std::marker::PhantomData::default(), - } - .serialize(serializer) + ( + SerializableVersionedBank::from( + serializable_bank_and_storage.bank.get_fields_to_serialize(), + ), + SerializableAccountsDB::<'a, Self> { + accounts_db: &*serializable_bank_and_storage.bank.rc.accounts.accounts_db, + slot: serializable_bank_and_storage.bank.rc.slot, + account_storage_entries: serializable_bank_and_storage.snapshot_storages, + phantom: std::marker::PhantomData::default(), + }, + ) + .serialize(serializer) } fn serialize_accounts_db_fields( @@ -161,6 +332,7 @@ impl<'a> TypeContext<'a> for Context { .clone(), ); + // as there is no deserialize_bank_rc_fields(), emit two u64 size fields here instead let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms"); let result = ( &MAX_ACCOUNTS_DB_STREAM_SIZE, @@ -179,9 +351,20 @@ impl<'a> TypeContext<'a> for Context { result } + fn deserialize_bank_fields( + mut stream: &mut BufReader, + ) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error> + where + R: Read, + { + let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); + let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?; + Ok((bank_fields, accounts_db_fields)) + } + fn deserialize_accounts_db_fields( mut stream: &mut BufReader, - ) -> Result, Error> + ) -> Result where R: Read, { @@ -203,6 +386,6 @@ impl<'a> TypeContext<'a> for Context { // (3rd of 3 elements) read in (slot, bank hashes) pair let (slot, bank_hash_info): (Slot, BankHashInfo) = deserialize_from(&mut stream)?; - Ok(AccountDBFields(storage, version, slot, bank_hash_info)) + Ok(AccountsDbFields(storage, version, slot, bank_hash_info)) } } diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index 14a9d53a78..32d677148b 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -64,7 +64,7 @@ where P: AsRef, { // read and deserialise the accounts database directly from the stream - context_accountsdb_from_fields::( + reconstruct_accountsdb_from_fields( C::deserialize_accounts_db_fields(stream)?, account_paths, stream_append_vecs_path, @@ -198,17 +198,15 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) { let snapshot_storages = bank2.get_snapshot_storages(); let mut buf = vec![]; let mut writer = Cursor::new(&mut buf); - serialize_into(&mut writer, &bank2).unwrap(); - crate::serde_snapshot::bankrc_to_stream( + crate::serde_snapshot::bank_to_stream( serde_style, &mut std::io::BufWriter::new(&mut writer), - &bank2.rc, + &bank2, &snapshot_storages, ) .unwrap(); - let mut rdr = Cursor::new(&buf[..]); - let mut dbank: Bank = bincode::deserialize_from(&mut rdr).unwrap(); + let rdr = Cursor::new(&buf[..]); let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]); // Create a new set of directories for this bank's accounts @@ -218,17 +216,16 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) { // Create a directory to simulate AppendVecs unpackaged from a snapshot tar let copied_accounts = TempDir::new().unwrap(); copy_append_vecs(&bank2.rc.accounts.accounts_db, copied_accounts.path()).unwrap(); - dbank.set_bank_rc( - crate::serde_snapshot::bankrc_from_stream( - serde_style, - &dbank_paths, - dbank.slot(), - &mut reader, - copied_accounts.path(), - ) - .unwrap(), - ref_sc, - ); + let mut dbank = crate::serde_snapshot::bank_from_stream( + serde_style, + &mut reader, + copied_accounts.path(), + &dbank_paths, + &genesis_config, + &[], + ) + .unwrap(); + dbank.src = ref_sc; assert_eq!(dbank.get_balance(&key1.pubkey()), 0); assert_eq!(dbank.get_balance(&key2.pubkey()), 10); assert_eq!(dbank.get_balance(&key3.pubkey()), 0); @@ -280,45 +277,58 @@ fn test_bank_serialize_older() { } #[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] -mod test_bank_rc_serialize { +mod test_bank_serialize { use super::*; + use solana_sdk::abi_example::AbiExample; // These some what long test harness is required to freeze the ABI of - // BankRc's serialization due to versioned nature - #[frozen_abi(digest = "HfCP74JKqPdeAccNJEj7KEoNxtsmX3zRqc2rpTy1NC7H")] - #[derive(Serialize, AbiExample)] - pub struct BandRcAbiTestWrapperFuture { + // Bank's serialization due to versioned nature + #[frozen_abi(digest = "9BGkhttaVsELn1zoHMKXLvi3Qty51nY1yz584Fao2Ev9")] + #[derive(Default, Serialize)] + pub struct BankAbiTestWrapperFuture { #[serde(serialize_with = "wrapper_future")] - bank_rc: BankRc, + bank: Bank, } - pub fn wrapper_future(bank_rc: &BankRc, s: S) -> std::result::Result + impl AbiExample for BankAbiTestWrapperFuture { + fn example() -> Self { + Self::default() + } + } + + pub fn wrapper_future(bank: &Bank, s: S) -> std::result::Result where S: serde::Serializer, { - let snapshot_storages = bank_rc.accounts.accounts_db.get_snapshot_storages(0); - (SerializableBankRc:: { - bank_rc, + let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0); + (SerializableBankAndStorage:: { + bank, snapshot_storages: &snapshot_storages, phantom: std::marker::PhantomData::default(), }) .serialize(s) } - #[frozen_abi(digest = "43niyekyWwreLALcdEeFFpd7h8U6pgSXGqfKBRw8H7Vy")] - #[derive(Serialize, AbiExample)] - pub struct BandRcAbiTestWrapperLegacy { + #[frozen_abi(digest = "HYmXta1fhiHe6hZLGJriMqKx9N2U8YRJY51AGZmxzM5m")] + #[derive(Default, Serialize)] + pub struct BankAbiTestWrapperLegacy { #[serde(serialize_with = "wrapper_legacy")] - bank_rc: BankRc, + bank: Bank, } - pub fn wrapper_legacy(bank_rc: &BankRc, s: S) -> std::result::Result + impl AbiExample for BankAbiTestWrapperLegacy { + fn example() -> Self { + Self::default() + } + } + + pub fn wrapper_legacy(bank: &Bank, s: S) -> std::result::Result where S: serde::Serializer, { - let snapshot_storages = bank_rc.accounts.accounts_db.get_snapshot_storages(0); - (SerializableBankRc:: { - bank_rc, + let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0); + (SerializableBankAndStorage:: { + bank, snapshot_storages: &snapshot_storages, phantom: std::marker::PhantomData::default(), }) diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index d417fa9c7e..e552aeca73 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -3,7 +3,7 @@ use crate::{ bank_forks::CompressionType, hardened_unpack::{unpack_snapshot, UnpackError}, serde_snapshot::{ - bankrc_from_stream, bankrc_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages, + bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages, }, snapshot_package::AccountsPackage, }; @@ -23,7 +23,6 @@ use std::{ path::{Path, PathBuf}, process::ExitStatus, str::FromStr, - sync::Arc, }; use tar::Archive; use tempfile::TempDir; @@ -462,8 +461,7 @@ pub fn add_snapshot>( SnapshotVersion::V1_1_0 => SerdeStyle::OLDER, SnapshotVersion::V1_2_0 => SerdeStyle::NEWER, }; - serialize_into(stream.by_ref(), bank)?; - bankrc_to_stream(serde_style, stream.by_ref(), &bank.rc, snapshot_storages)?; + bank_to_stream(serde_style, stream.by_ref(), bank, snapshot_storages)?; Ok(()) }; let consumed_size = @@ -723,38 +721,24 @@ where info!("Loading bank from {:?}", &root_paths.snapshot_file_path); let bank = deserialize_snapshot_data_file(&root_paths.snapshot_file_path, |mut stream| { - let mut bank: Bank = bincode::options() - .with_limit(MAX_SNAPSHOT_DATA_FILE_SIZE) - .with_fixint_encoding() - .allow_trailing_bytes() - .deserialize_from(&mut stream)?; - - info!("Rebuilding accounts..."); - - let mut bankrc = match snapshot_version_enum { - SnapshotVersion::V1_1_0 => bankrc_from_stream( + Ok(match snapshot_version_enum { + SnapshotVersion::V1_1_0 => bank_from_stream( SerdeStyle::OLDER, - account_paths, - bank.slot(), &mut stream, &append_vecs_path, + account_paths, + genesis_config, + frozen_account_pubkeys, ), - SnapshotVersion::V1_2_0 => bankrc_from_stream( + SnapshotVersion::V1_2_0 => bank_from_stream( SerdeStyle::NEWER, - account_paths, - bank.slot(), &mut stream, &append_vecs_path, + account_paths, + genesis_config, + frozen_account_pubkeys, ), - }?; - Arc::get_mut(&mut Arc::get_mut(&mut bankrc.accounts).unwrap().accounts_db) - .unwrap() - .freeze_accounts(&bank.ancestors, frozen_account_pubkeys); - - bank.rc = bankrc; - bank.operating_mode = Some(genesis_config.operating_mode); - bank.finish_init(); - Ok(bank) + }?) })?; let status_cache_path = unpacked_snapshots_dir.join(SNAPSHOT_STATUS_CACHE_FILE_NAME); diff --git a/sdk/macro-frozen-abi/src/lib.rs b/sdk/macro-frozen-abi/src/lib.rs index e53171dc1d..876400c9b3 100644 --- a/sdk/macro-frozen-abi/src/lib.rs +++ b/sdk/macro-frozen-abi/src/lib.rs @@ -78,8 +78,11 @@ fn quote_for_specialization_detection() -> TokenStream2 { std::sync::atomic::AtomicBool::new(false); } - if !SPECIALIZATION_DETECTOR_INJECTED.load(std::sync::atomic::Ordering::Relaxed) { - SPECIALIZATION_DETECTOR_INJECTED.store(true, std::sync::atomic::Ordering::Relaxed); + if !SPECIALIZATION_DETECTOR_INJECTED.compare_and_swap( + false, + true, + std::sync::atomic::Ordering::AcqRel, + ) { quote! { mod specialization_detector { trait SpecializedTrait {