blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fold bank serialisation into serde snapshot (#10581) 

* Move bank (de)serialisation logic from bank and snapshot_utils to serde_snapshot.
Add sanity assertions between genesis config and bank fields on deserialisation.
Atomically update atomic bool in quote_for_specialization_detection().
Use same genesis config when restoring snapshots in test cases.

* Tidy up namings and duplicate structs to version

* Apply struct renames to tests

* Update abi hashes

Co-authored-by: Ryo Onodera <ryoqun@gmail.com>
This commit is contained in:
Kristofer Peterson 2020-07-13 15:00:59 +01:00 committed by GitHub
parent 5c86766fb2
commit ed5a2f2a90
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 805 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
core/tests/bank_forks.rs
View File

@ -110,6 +110,7 @@ mod tests {
fn restore_from_snapshot(
old_bank_forks: &BankForks,
old_last_slot: Slot,
old_genesis_config: &GenesisConfig,
account_paths: &[PathBuf],
) {
let (snapshot_path, snapshot_package_output_path) = old_bank_forks
@ -134,7 +135,7 @@ mod tests {
&CompressionType::Bzip2,
),
CompressionType::Bzip2,
&GenesisConfig::default(),
old_genesis_config,
)
.unwrap();
@ -169,7 +170,6 @@ mod tests {
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
let bank_forks = &mut snapshot_test_config.bank_forks;
let accounts_dir = &snapshot_test_config.accounts_dir;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
let (s, _r) = channel();
@ -185,6 +185,7 @@ mod tests {
bank_forks.set_root(bank.slot(), &sender, None);
}
}
// Generate a snapshot package for last bank
let last_bank = bank_forks.get(last_slot).unwrap();
let snapshot_config = &snapshot_test_config.snapshot_config;
@ -203,10 +204,12 @@ mod tests {
snapshot_version,
)
.unwrap();
snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]);
// Restore bank from snapshot
let account_paths = &[snapshot_test_config.accounts_dir.path().to_path_buf()];
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
restore_from_snapshot(bank_forks, last_slot, genesis_config, account_paths);
}
fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) {

4
programs/librapay/src/lib.rs
View File

@ -38,7 +38,9 @@ pub fn create_genesis<T: Client>(from: &Keypair, client: &T, amount: u64) -> Key
let instruction = librapay_instruction::genesis(&genesis.pubkey(), amount);
let message = Message::new(&[instruction], Some(&from.pubkey()));
client.send_and_confirm_message(&[from, &genesis], message).unwrap();
client
.send_and_confirm_message(&[from, &genesis], message)
.unwrap();
genesis
}

32
runtime/src/accounts.rs
View File

@ -66,23 +66,19 @@ pub enum AccountAddressFilter {
}
impl Accounts {
pub(crate) fn new_empty(accounts_db: AccountsDB) -> Self {
pub fn new(paths: Vec<PathBuf>) -> Self {
Self {
accounts_db: Arc::new(accounts_db),
slot: 0,
accounts_db: Arc::new(AccountsDB::new(paths)),
account_locks: Mutex::new(HashSet::new()),
readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
..Self::default()
}
}
pub fn new(paths: Vec<PathBuf>) -> Self {
Self::new_with_frozen_accounts(paths, &HashMap::default(), &[])
}
pub fn new_from_parent(parent: &Accounts, slot: Slot, parent_slot: Slot) -> Self {
let accounts_db = parent.accounts_db.clone();
accounts_db.set_hash(slot, parent_slot);
Accounts {
Self {
slot,
accounts_db,
account_locks: Mutex::new(HashSet::new()),
@ -90,25 +86,13 @@ impl Accounts {
}
}
pub fn new_with_frozen_accounts(
paths: Vec<PathBuf>,
ancestors: &Ancestors,
frozen_account_pubkeys: &[Pubkey],
) -> Self {
let mut accounts = Accounts {
pub(crate) fn new_empty(accounts_db: AccountsDB) -> Self {
Self {
slot: 0,
accounts_db: Arc::new(AccountsDB::new(paths)),
accounts_db: Arc::new(accounts_db),
account_locks: Mutex::new(HashSet::new()),
readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
};
accounts.freeze_accounts(ancestors, frozen_account_pubkeys);
accounts
}
pub fn freeze_accounts(&mut self, ancestors: &Ancestors, frozen_account_pubkeys: &[Pubkey]) {
Arc::get_mut(&mut self.accounts_db)
.unwrap()
.freeze_accounts(ancestors, frozen_account_pubkeys);
}
}
/// Return true if the slice has any duplicate elements

4
runtime/src/accounts_db.rs
View File

@ -1800,7 +1800,7 @@ impl AccountsDB {
hashes
}
pub fn freeze_accounts(&mut self, ancestors: &Ancestors, account_pubkeys: &[Pubkey]) {
pub(crate) fn freeze_accounts(&mut self, ancestors: &Ancestors, account_pubkeys: &[Pubkey]) {
for account_pubkey in account_pubkeys {
if let Some((account, _slot)) = self.load_slow(ancestors, &account_pubkey) {
let frozen_account_info = FrozenAccountInfo {
@ -1994,7 +1994,7 @@ impl AccountsDB {
}
}
pub fn print_accounts_stats(&self, label: &'static str) {
pub(crate) fn print_accounts_stats(&self, label: &'static str) {
self.print_index(label);
self.print_count_and_status(label);
}

241
runtime/src/bank.rs
View File

@ -25,7 +25,6 @@ use crate::{
use byteorder::{ByteOrder, LittleEndian};
use itertools::Itertools;
use log::*;
use serde::{Deserialize, Serialize};
use solana_measure::measure::Measure;
use solana_metrics::{
datapoint_debug, inc_new_counter_debug, inc_new_counter_error, inc_new_counter_info,
@ -108,20 +107,6 @@ pub struct BankRc {
pub(crate) slot: Slot,
}
#[cfg(RUSTC_WITH_SPECIALIZATION)]
use solana_sdk::abi_example::AbiExample;
#[cfg(RUSTC_WITH_SPECIALIZATION)]
impl AbiExample for BankRc {
fn example() -> Self {
BankRc {
// Set parent to None to cut the recursion into another Bank
parent: RwLock::new(None),
accounts: AbiExample::example(),
slot: AbiExample::example(),
}
}
}
impl BankRc {
pub(crate) fn new(accounts: Accounts, slot: Slot) -> Self {
Self {
@ -203,22 +188,88 @@ impl HashAgeKind {
}
}
#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)]
struct UnusedAccounts {
unused1: HashSet<Pubkey>,
unused2: HashSet<Pubkey>,
unused3: HashMap<Pubkey, u64>,
// Bank's common fields shared by all supported snapshot versions for deserialization.
// Sync fields with BankFieldsToSerialize! This is paired with it.
// All members are made public to remain Bank's members private and to make versioned deserializer workable on this
#[derive(Clone, Default)]
pub(crate) struct BankFieldsToDeserialize {
pub(crate) blockhash_queue: BlockhashQueue,
pub(crate) ancestors: Ancestors,
pub(crate) hash: Hash,
pub(crate) parent_hash: Hash,
pub(crate) parent_slot: Slot,
pub(crate) hard_forks: HardForks,
pub(crate) transaction_count: u64,
pub(crate) tick_height: u64,
pub(crate) signature_count: u64,
pub(crate) capitalization: u64,
pub(crate) max_tick_height: u64,
pub(crate) hashes_per_tick: Option<u64>,
pub(crate) ticks_per_slot: u64,
pub(crate) ns_per_slot: u128,
pub(crate) genesis_creation_time: UnixTimestamp,
pub(crate) slots_per_year: f64,
pub(crate) unused: u64,
pub(crate) slot: Slot,
pub(crate) epoch: Epoch,
pub(crate) block_height: u64,
pub(crate) collector_id: Pubkey,
pub(crate) collector_fees: u64,
pub(crate) fee_calculator: FeeCalculator,
pub(crate) fee_rate_governor: FeeRateGovernor,
pub(crate) collected_rent: u64,
pub(crate) rent_collector: RentCollector,
pub(crate) epoch_schedule: EpochSchedule,
pub(crate) inflation: Inflation,
pub(crate) stakes: Stakes,
pub(crate) epoch_stakes: HashMap<Epoch, EpochStakes>,
pub(crate) is_delta: bool,
}
// Bank's common fields shared by all supported snapshot versions for serialization.
// This is separated from BankFieldsToDeserialize to avoid cloning by using refs.
// So, sync fields with BankFieldsToDeserialize!
// all members are made public to remain Bank private and to make versioned serializer workable on this
pub(crate) struct BankFieldsToSerialize<'a> {
pub(crate) blockhash_queue: &'a RwLock<BlockhashQueue>,
pub(crate) ancestors: &'a Ancestors,
pub(crate) hash: Hash,
pub(crate) parent_hash: Hash,
pub(crate) parent_slot: Slot,
pub(crate) hard_forks: &'a RwLock<HardForks>,
pub(crate) transaction_count: u64,
pub(crate) tick_height: u64,
pub(crate) signature_count: u64,
pub(crate) capitalization: u64,
pub(crate) max_tick_height: u64,
pub(crate) hashes_per_tick: Option<u64>,
pub(crate) ticks_per_slot: u64,
pub(crate) ns_per_slot: u128,
pub(crate) genesis_creation_time: UnixTimestamp,
pub(crate) slots_per_year: f64,
pub(crate) unused: u64,
pub(crate) slot: Slot,
pub(crate) epoch: Epoch,
pub(crate) block_height: u64,
pub(crate) collector_id: Pubkey,
pub(crate) collector_fees: u64,
pub(crate) fee_calculator: FeeCalculator,
pub(crate) fee_rate_governor: FeeRateGovernor,
pub(crate) collected_rent: u64,
pub(crate) rent_collector: RentCollector,
pub(crate) epoch_schedule: EpochSchedule,
pub(crate) inflation: Inflation,
pub(crate) stakes: &'a RwLock<Stakes>,
pub(crate) epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
pub(crate) is_delta: bool,
}
/// Manager for the state of all accounts and programs after processing its entries.
#[frozen_abi(digest = "GsvisJfTaHxmTX4s6gQs2bZtxVggB7F325XDdXTA573p")]
#[derive(Default, Deserialize, Serialize, AbiExample)]
#[derive(Default)]
pub struct Bank {
/// References to accounts, parent and signature status
#[serde(skip)]
pub rc: BankRc,
#[serde(skip)]
pub src: StatusCacheRc,
/// FIFO queue of `recent_blockhash` items
@ -308,9 +359,6 @@ pub struct Bank {
/// cache of vote_account and stake_account state for this fork
stakes: RwLock<Stakes>,
/// unused
unused_accounts: RwLock<UnusedAccounts>,
/// staked nodes on epoch boundaries, saved off when a bank.slot() is at
/// a leader schedule calculation boundary
epoch_stakes: HashMap<Epoch, EpochStakes>,
@ -324,24 +372,18 @@ pub struct Bank {
/// Callback to be notified when a bank enters a new Epoch
/// (used to adjust cluster features over time)
#[serde(skip)]
entered_epoch_callback: Arc<RwLock<Option<EnteredEpochCallback>>>,
/// Last time when the cluster info vote listener has synced with this bank
#[serde(skip)]
pub last_vote_sync: AtomicU64,
/// Rewards that were paid out immediately after this bank was created
#[serde(skip)]
pub rewards: Option<Vec<(Pubkey, i64)>>,
#[serde(skip)]
pub skip_drop: AtomicBool,
#[serde(skip)]
pub operating_mode: Option<OperatingMode>,
#[serde(skip)]
pub lazy_rent_collection: AtomicBool,
}
@ -370,7 +412,7 @@ impl Bank {
bank.finish_init();
// Freeze accounts after process_genesis_config creates the initial append vecs
Arc::get_mut(&mut bank.rc.accounts)
Arc::get_mut(&mut Arc::get_mut(&mut bank.rc.accounts).unwrap().accounts_db)
.unwrap()
.freeze_accounts(&bank.ancestors, frozen_account_pubkeys);
@ -440,7 +482,6 @@ impl Bank {
transaction_count: AtomicU64::new(parent.transaction_count()),
stakes: RwLock::new(parent.stakes.read().unwrap().clone_with_epoch(epoch)),
epoch_stakes: parent.epoch_stakes.clone(),
unused_accounts: RwLock::new(parent.unused_accounts.read().unwrap().clone()),
parent_hash: parent.hash(),
parent_slot: parent.slot(),
collector_id: *collector_id,
@ -514,6 +555,136 @@ impl Bank {
new
}
/// Create a bank from explicit arguments and deserialized fields from snapshot
#[allow(clippy::float_cmp)]
pub(crate) fn new_from_fields(
bank_rc: BankRc,
genesis_config: &GenesisConfig,
fields: BankFieldsToDeserialize,
) -> Self {
fn new<T: Default>() -> T {
T::default()
}
let mut bank = Self {
rc: bank_rc,
src: new(),
blockhash_queue: RwLock::new(fields.blockhash_queue),
ancestors: fields.ancestors,
hash: RwLock::new(fields.hash),
parent_hash: fields.parent_hash,
parent_slot: fields.parent_slot,
hard_forks: Arc::new(RwLock::new(fields.hard_forks)),
transaction_count: AtomicU64::new(fields.transaction_count),
tick_height: AtomicU64::new(fields.tick_height),
signature_count: AtomicU64::new(fields.signature_count),
capitalization: AtomicU64::new(fields.capitalization),
max_tick_height: fields.max_tick_height,
hashes_per_tick: fields.hashes_per_tick,
ticks_per_slot: fields.ticks_per_slot,
ns_per_slot: fields.ns_per_slot,
genesis_creation_time: fields.genesis_creation_time,
slots_per_year: fields.slots_per_year,
unused: genesis_config.unused,
slot: fields.slot,
epoch: fields.epoch,
block_height: fields.block_height,
collector_id: fields.collector_id,
collector_fees: AtomicU64::new(fields.collector_fees),
fee_calculator: fields.fee_calculator,
fee_rate_governor: fields.fee_rate_governor,
collected_rent: AtomicU64::new(fields.collected_rent),
rent_collector: fields.rent_collector,
epoch_schedule: fields.epoch_schedule,
inflation: Arc::new(RwLock::new(fields.inflation)),
stakes: RwLock::new(fields.stakes),
epoch_stakes: fields.epoch_stakes,
is_delta: AtomicBool::new(fields.is_delta),
message_processor: new(),
entered_epoch_callback: new(),
last_vote_sync: new(),
rewards: new(),
skip_drop: new(),
operating_mode: Some(genesis_config.operating_mode),
lazy_rent_collection: new(),
};
bank.finish_init();
// Sanity assertions between bank snapshot and genesis config
// Consider removing from serializable bank state ([Ref]BankFields) and initializing
// from the passed in genesis_config instead (as new()/new_with_paths() already do)
assert_eq!(
bank.hashes_per_tick,
genesis_config.poh_config.hashes_per_tick
);
assert_eq!(bank.ticks_per_slot, genesis_config.ticks_per_slot);
assert_eq!(
bank.ns_per_slot,
genesis_config.poh_config.target_tick_duration.as_nanos()
* genesis_config.ticks_per_slot as u128
);
assert_eq!(bank.genesis_creation_time, genesis_config.creation_time);
assert_eq!(bank.unused, genesis_config.unused);
assert_eq!(bank.max_tick_height, (bank.slot + 1) * bank.ticks_per_slot);
assert_eq!(
bank.slots_per_year,
years_as_slots(
1.0,
&genesis_config.poh_config.target_tick_duration,
bank.ticks_per_slot,
)
);
assert_eq!(bank.epoch_schedule, genesis_config.epoch_schedule);
assert_eq!(bank.epoch, bank.epoch_schedule.get_epoch(bank.slot));
assert_eq!(
bank.rent_collector,
RentCollector::new(
bank.epoch,
&bank.epoch_schedule,
bank.slots_per_year,
&genesis_config.rent,
)
);
bank
}
/// Return subset of bank fields representing serializable state
pub(crate) fn get_fields_to_serialize(&self) -> BankFieldsToSerialize {
BankFieldsToSerialize {
blockhash_queue: &self.blockhash_queue,
ancestors: &self.ancestors,
hash: *self.hash.read().unwrap(),
parent_hash: self.parent_hash,
parent_slot: self.parent_slot,
hard_forks: &*self.hard_forks,
transaction_count: self.transaction_count.load(Ordering::Relaxed),
tick_height: self.tick_height.load(Ordering::Relaxed),
signature_count: self.signature_count.load(Ordering::Relaxed),
capitalization: self.capitalization.load(Ordering::Relaxed),
max_tick_height: self.max_tick_height,
hashes_per_tick: self.hashes_per_tick,
ticks_per_slot: self.ticks_per_slot,
ns_per_slot: self.ns_per_slot,
genesis_creation_time: self.genesis_creation_time,
slots_per_year: self.slots_per_year,
unused: self.unused,
slot: self.slot,
epoch: self.epoch,
block_height: self.block_height,
collector_id: self.collector_id,
collector_fees: self.collector_fees.load(Ordering::Relaxed),
fee_calculator: self.fee_calculator.clone(),
fee_rate_governor: self.fee_rate_governor.clone(),
collected_rent: self.collected_rent.load(Ordering::Relaxed),
rent_collector: self.rent_collector.clone(),
epoch_schedule: self.epoch_schedule,
inflation: *self.inflation.read().unwrap(),
stakes: &self.stakes,
epoch_stakes: &self.epoch_stakes,
is_delta: self.is_delta.load(Ordering::Relaxed),
}
}
pub fn collector_id(&self) -> &Pubkey {
&self.collector_id
}

2
runtime/src/rent_collector.rs
View File

@ -4,7 +4,7 @@ use solana_sdk::{
rent::Rent, sysvar,
};
#[derive(Default, Serialize, Deserialize, Clone, AbiExample)]
#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Debug, AbiExample)]
pub struct RentCollector {
pub epoch: Epoch,
pub epoch_schedule: EpochSchedule,

157
runtime/src/serde_snapshot.rs
View File

@ -4,10 +4,17 @@ use {
accounts_db::{
AccountStorageEntry, AccountStorageStatus, AccountsDB, AppendVecId, BankHashInfo,
},
accounts_index::Ancestors,
append_vec::AppendVec,
bank::BankRc,
bank::{Bank, BankFieldsToDeserialize, BankRc},
blockhash_queue::BlockhashQueue,
epoch_stakes::EpochStakes,
message_processor::MessageProcessor,
rent_collector::RentCollector,
stakes::Stakes,
},
bincode::{deserialize_from, serialize_into, Error},
bincode,
bincode::{config::Options, serialize_into, Error},
fs_extra::dir::CopyOptions,
log::{info, warn},
rand::{thread_rng, Rng},
@ -15,7 +22,16 @@ use {
de::{DeserializeOwned, Visitor},
Deserialize, Deserializer, Serialize, Serializer,
},
solana_sdk::clock::Slot,
solana_sdk::{
clock::{Epoch, Slot, UnixTimestamp},
epoch_schedule::EpochSchedule,
fee_calculator::{FeeCalculator, FeeRateGovernor},
genesis_config::GenesisConfig,
hard_forks::HardForks,
hash::Hash,
inflation::Inflation,
pubkey::Pubkey,
},
std::{
cmp::min,
collections::HashMap,
@ -28,6 +44,10 @@ use {
},
};
#[cfg(RUSTC_WITH_SPECIALIZATION)]
use solana_sdk::abi_example::IgnoreAsHelper;
mod common;
mod future;
mod legacy;
mod tests;
@ -42,28 +62,28 @@ use utils::{serialize_iter_as_map, serialize_iter_as_seq, serialize_iter_as_tupl
#[cfg(test)]
pub(crate) use self::tests::reconstruct_accounts_db_via_serialization;
pub use crate::accounts_db::{SnapshotStorage, SnapshotStorages};
pub(crate) use crate::accounts_db::{SnapshotStorage, SnapshotStorages};
#[derive(Copy, Clone, Eq, PartialEq)]
pub enum SerdeStyle {
pub(crate) enum SerdeStyle {
NEWER,
OLDER,
}
const MAX_ACCOUNTS_DB_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024;
const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024;
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct AccountDBFields<T>(HashMap<Slot, Vec<T>>, u64, Slot, BankHashInfo);
#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)]
struct AccountsDbFields<T>(HashMap<Slot, Vec<T>>, u64, Slot, BankHashInfo);
pub trait TypeContext<'a> {
trait TypeContext<'a> {
type SerializableAccountStorageEntry: Serialize
+ DeserializeOwned
+ From<&'a AccountStorageEntry>
+ Into<AccountStorageEntry>;
fn serialize_bank_rc_fields<S: serde::ser::Serializer>(
fn serialize_bank_and_storage<S: serde::ser::Serializer>(
serializer: S,
serializable_bank: &SerializableBankRc<'a, Self>,
serializable_bank: &SerializableBankAndStorage<'a, Self>,
) -> std::result::Result<S::Ok, S::Error>
where
Self: std::marker::Sized;
@ -75,37 +95,63 @@ pub trait TypeContext<'a> {
where
Self: std::marker::Sized;
fn deserialize_accounts_db_fields<R>(
fn deserialize_bank_fields<R>(
stream: &mut BufReader<R>,
) -> Result<AccountDBFields<Self::SerializableAccountStorageEntry>, Error>
) -> Result<
(
BankFieldsToDeserialize,
AccountsDbFields<Self::SerializableAccountStorageEntry>,
),
Error,
>
where
R: Read;
// we might define fn (de)serialize_bank(...) -> Result<Bank,...> for versionized bank serialization in the future
fn deserialize_accounts_db_fields<R>(
stream: &mut BufReader<R>,
) -> Result<AccountsDbFields<Self::SerializableAccountStorageEntry>, Error>
where
R: Read;
}
pub fn bankrc_from_stream<R, P>(
fn deserialize_from<R, T>(reader: R) -> bincode::Result<T>
where
R: Read,
T: DeserializeOwned,
{
bincode::options()
.with_limit(MAX_STREAM_SIZE)
.with_fixint_encoding()
.allow_trailing_bytes()
.deserialize_from::<R, T>(reader)
}
pub(crate) fn bank_from_stream<R, P>(
serde_style: SerdeStyle,
account_paths: &[PathBuf],
slot: Slot,
stream: &mut BufReader<R>,
stream_append_vecs_path: P,
) -> std::result::Result<BankRc, Error>
append_vecs_path: P,
account_paths: &[PathBuf],
genesis_config: &GenesisConfig,
frozen_account_pubkeys: &[Pubkey],
) -> std::result::Result<Bank, Error>
where
R: Read,
P: AsRef<Path>,
{
macro_rules! INTO {
($x:ident) => {
Ok(BankRc::new(
Accounts::new_empty(context_accountsdb_from_fields::<$x, P>(
$x::deserialize_accounts_db_fields(stream)?,
account_paths,
stream_append_vecs_path,
)?),
slot,
))
};
($x:ident) => {{
let (bank_fields, accounts_db_fields) = $x::deserialize_bank_fields(stream)?;
let bank = reconstruct_bank_from_fields(
bank_fields,
accounts_db_fields,
genesis_config,
frozen_account_pubkeys,
account_paths,
append_vecs_path,
)?;
Ok(bank)
}};
}
match serde_style {
SerdeStyle::NEWER => INTO!(TypeContextFuture),
@ -117,10 +163,10 @@ where
})
}
pub fn bankrc_to_stream<W>(
pub(crate) fn bank_to_stream<W>(
serde_style: SerdeStyle,
stream: &mut BufWriter<W>,
bank_rc: &BankRc,
bank: &Bank,
snapshot_storages: &[SnapshotStorage],
) -> Result<(), Error>
where
@ -128,10 +174,10 @@ where
{
macro_rules! INTO {
($x:ident) => {
serialize_into(
bincode::serialize_into(
stream,
&SerializableBankRc::<$x> {
bank_rc,
&SerializableBankAndStorage::<$x> {
bank,
snapshot_storages,
phantom: std::marker::PhantomData::default(),
},
@ -148,22 +194,22 @@ where
})
}
pub struct SerializableBankRc<'a, C> {
bank_rc: &'a BankRc,
struct SerializableBankAndStorage<'a, C> {
bank: &'a Bank,
snapshot_storages: &'a [SnapshotStorage],
phantom: std::marker::PhantomData<C>,
}
impl<'a, C: TypeContext<'a>> Serialize for SerializableBankRc<'a, C> {
impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorage<'a, C> {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::ser::Serializer,
{
C::serialize_bank_rc_fields(serializer, self)
C::serialize_bank_and_storage(serializer, self)
}
}
pub struct SerializableAccountsDB<'a, C> {
struct SerializableAccountsDB<'a, C> {
accounts_db: &'a AccountsDB,
slot: Slot,
account_storage_entries: &'a [SnapshotStorage],
@ -179,18 +225,43 @@ impl<'a, C: TypeContext<'a>> Serialize for SerializableAccountsDB<'a, C> {
}
}
fn context_accountsdb_from_fields<'a, C, P>(
account_db_fields: AccountDBFields<C::SerializableAccountStorageEntry>,
#[cfg(RUSTC_WITH_SPECIALIZATION)]
impl<'a, C> IgnoreAsHelper for SerializableAccountsDB<'a, C> {}
fn reconstruct_bank_from_fields<E, P>(
bank_fields: BankFieldsToDeserialize,
accounts_db_fields: AccountsDbFields<E>,
genesis_config: &GenesisConfig,
frozen_account_pubkeys: &[Pubkey],
account_paths: &[PathBuf],
append_vecs_path: P,
) -> Result<Bank, Error>
where
E: Into<AccountStorageEntry>,
P: AsRef<Path>,
{
let mut accounts_db =
reconstruct_accountsdb_from_fields(accounts_db_fields, account_paths, append_vecs_path)?;
accounts_db.freeze_accounts(&bank_fields.ancestors, frozen_account_pubkeys);
let bank_rc = BankRc::new(Accounts::new_empty(accounts_db), bank_fields.slot);
let bank = Bank::new_from_fields(bank_rc, genesis_config, bank_fields);
Ok(bank)
}
fn reconstruct_accountsdb_from_fields<E, P>(
accounts_db_fields: AccountsDbFields<E>,
account_paths: &[PathBuf],
stream_append_vecs_path: P,
) -> Result<AccountsDB, Error>
where
C: TypeContext<'a>,
E: Into<AccountStorageEntry>,
P: AsRef<Path>,
{
let accounts_db = AccountsDB::new(account_paths.to_vec());
let AccountDBFields(storage, version, slot, bank_hash_info) = account_db_fields;
let AccountsDbFields(storage, version, slot, bank_hash_info) = accounts_db_fields;
// convert to two level map of slot -> id -> account storage entry
let storage = {

9
runtime/src/serde_snapshot/common.rs Normal file
View File

@ -0,0 +1,9 @@
use super::*;
use std::collections::HashSet;
#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)]
pub(crate) struct UnusedAccounts {
unused1: HashSet<Pubkey>,
unused2: HashSet<Pubkey>,
unused3: HashMap<Pubkey, u64>,
}

207
runtime/src/serde_snapshot/future.rs
View File

@ -1,5 +1,10 @@
use super::common::UnusedAccounts;
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
use solana_sdk::abi_example::IgnoreAsHelper;
use {super::*, solana_measure::measure::Measure, std::cell::RefCell};
type AccountsDbFields = super::AccountsDbFields<SerializableAccountStorageEntry>;
// Serializable version of AccountStorageEntry for snapshot format
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
pub(super) struct SerializableAccountStorageEntry {
@ -25,25 +30,190 @@ impl Into<AccountStorageEntry> for SerializableAccountStorageEntry {
}
}
use std::sync::RwLock;
// Deserializable version of Bank which need not be serializable,
// because it's handled by SerializableVersionedBank.
// So, sync fields with it!
#[derive(Clone, Deserialize)]
pub(crate) struct DeserializableVersionedBank {
pub(crate) blockhash_queue: BlockhashQueue,
pub(crate) ancestors: Ancestors,
pub(crate) hash: Hash,
pub(crate) parent_hash: Hash,
pub(crate) parent_slot: Slot,
pub(crate) hard_forks: HardForks,
pub(crate) transaction_count: u64,
pub(crate) tick_height: u64,
pub(crate) signature_count: u64,
pub(crate) capitalization: u64,
pub(crate) max_tick_height: u64,
pub(crate) hashes_per_tick: Option<u64>,
pub(crate) ticks_per_slot: u64,
pub(crate) ns_per_slot: u128,
pub(crate) genesis_creation_time: UnixTimestamp,
pub(crate) slots_per_year: f64,
pub(crate) unused: u64,
pub(crate) slot: Slot,
pub(crate) epoch: Epoch,
pub(crate) block_height: u64,
pub(crate) collector_id: Pubkey,
pub(crate) collector_fees: u64,
pub(crate) fee_calculator: FeeCalculator,
pub(crate) fee_rate_governor: FeeRateGovernor,
pub(crate) collected_rent: u64,
pub(crate) rent_collector: RentCollector,
pub(crate) epoch_schedule: EpochSchedule,
pub(crate) inflation: Inflation,
pub(crate) stakes: Stakes,
pub(crate) unused_accounts: UnusedAccounts,
pub(crate) epoch_stakes: HashMap<Epoch, EpochStakes>,
pub(crate) is_delta: bool,
pub(crate) message_processor: MessageProcessor,
}
impl Into<BankFieldsToDeserialize> for DeserializableVersionedBank {
fn into(self) -> BankFieldsToDeserialize {
BankFieldsToDeserialize {
blockhash_queue: self.blockhash_queue,
ancestors: self.ancestors,
hash: self.hash,
parent_hash: self.parent_hash,
parent_slot: self.parent_slot,
hard_forks: self.hard_forks,
transaction_count: self.transaction_count,
tick_height: self.tick_height,
signature_count: self.signature_count,
capitalization: self.capitalization,
max_tick_height: self.max_tick_height,
hashes_per_tick: self.hashes_per_tick,
ticks_per_slot: self.ticks_per_slot,
ns_per_slot: self.ns_per_slot,
genesis_creation_time: self.genesis_creation_time,
slots_per_year: self.slots_per_year,
unused: self.unused,
slot: self.slot,
epoch: self.epoch,
block_height: self.block_height,
collector_id: self.collector_id,
collector_fees: self.collector_fees,
fee_calculator: self.fee_calculator,
fee_rate_governor: self.fee_rate_governor,
collected_rent: self.collected_rent,
rent_collector: self.rent_collector,
epoch_schedule: self.epoch_schedule,
inflation: self.inflation,
stakes: self.stakes,
epoch_stakes: self.epoch_stakes,
is_delta: self.is_delta,
}
}
}
// Serializable version of Bank, not Deserializable to avoid cloning by using refs.
// Sync fields with DeserializableVersionedBank!
#[derive(Serialize)]
pub(crate) struct SerializableVersionedBank<'a> {
pub(crate) blockhash_queue: &'a RwLock<BlockhashQueue>,
pub(crate) ancestors: &'a Ancestors,
pub(crate) hash: Hash,
pub(crate) parent_hash: Hash,
pub(crate) parent_slot: Slot,
pub(crate) hard_forks: &'a RwLock<HardForks>,
pub(crate) transaction_count: u64,
pub(crate) tick_height: u64,
pub(crate) signature_count: u64,
pub(crate) capitalization: u64,
pub(crate) max_tick_height: u64,
pub(crate) hashes_per_tick: Option<u64>,
pub(crate) ticks_per_slot: u64,
pub(crate) ns_per_slot: u128,
pub(crate) genesis_creation_time: UnixTimestamp,
pub(crate) slots_per_year: f64,
pub(crate) unused: u64,
pub(crate) slot: Slot,
pub(crate) epoch: Epoch,
pub(crate) block_height: u64,
pub(crate) collector_id: Pubkey,
pub(crate) collector_fees: u64,
pub(crate) fee_calculator: FeeCalculator,
pub(crate) fee_rate_governor: FeeRateGovernor,
pub(crate) collected_rent: u64,
pub(crate) rent_collector: RentCollector,
pub(crate) epoch_schedule: EpochSchedule,
pub(crate) inflation: Inflation,
pub(crate) stakes: &'a RwLock<Stakes>,
pub(crate) unused_accounts: UnusedAccounts,
pub(crate) epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
pub(crate) is_delta: bool,
pub(crate) message_processor: MessageProcessor,
}
impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedBank<'a> {
fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self {
fn new<T: Default>() -> T {
T::default()
}
Self {
blockhash_queue: rhs.blockhash_queue,
ancestors: rhs.ancestors,
hash: rhs.hash,
parent_hash: rhs.parent_hash,
parent_slot: rhs.parent_slot,
hard_forks: rhs.hard_forks,
transaction_count: rhs.transaction_count,
tick_height: rhs.tick_height,
signature_count: rhs.signature_count,
capitalization: rhs.capitalization,
max_tick_height: rhs.max_tick_height,
hashes_per_tick: rhs.hashes_per_tick,
ticks_per_slot: rhs.ticks_per_slot,
ns_per_slot: rhs.ns_per_slot,
genesis_creation_time: rhs.genesis_creation_time,
slots_per_year: rhs.slots_per_year,
unused: rhs.unused,
slot: rhs.slot,
epoch: rhs.epoch,
block_height: rhs.block_height,
collector_id: rhs.collector_id,
collector_fees: rhs.collector_fees,
fee_calculator: rhs.fee_calculator,
fee_rate_governor: rhs.fee_rate_governor,
collected_rent: rhs.collected_rent,
rent_collector: rhs.rent_collector,
epoch_schedule: rhs.epoch_schedule,
inflation: rhs.inflation,
stakes: rhs.stakes,
unused_accounts: new(),
epoch_stakes: rhs.epoch_stakes,
is_delta: rhs.is_delta,
message_processor: new(),
}
}
}
#[cfg(RUSTC_WITH_SPECIALIZATION)]
impl<'a> IgnoreAsHelper for SerializableVersionedBank<'a> {}
pub(super) struct Context {}
impl<'a> TypeContext<'a> for Context {
type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
fn serialize_bank_rc_fields<S: serde::ser::Serializer>(
fn serialize_bank_and_storage<S: serde::ser::Serializer>(
serializer: S,
serializable_bank: &SerializableBankRc<'a, Self>,
serializable_bank: &SerializableBankAndStorage<'a, Self>,
) -> std::result::Result<S::Ok, S::Error>
where
Self: std::marker::Sized,
{
let accounts_db_serialize = SerializableAccountsDB::<'a, Self> {
accounts_db: &*serializable_bank.bank_rc.accounts.accounts_db,
slot: serializable_bank.bank_rc.slot,
account_storage_entries: serializable_bank.snapshot_storages,
phantom: std::marker::PhantomData::default(),
};
accounts_db_serialize.serialize(serializer)
(
SerializableVersionedBank::from(serializable_bank.bank.get_fields_to_serialize()),
SerializableAccountsDB::<'a, Self> {
accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db,
slot: serializable_bank.bank.rc.slot,
account_storage_entries: serializable_bank.snapshot_storages,
phantom: std::marker::PhantomData::default(),
},
)
.serialize(serializer)
}
fn serialize_accounts_db_fields<S: serde::ser::Serializer>(
@ -93,12 +263,23 @@ impl<'a> TypeContext<'a> for Context {
result
}
fn deserialize_accounts_db_fields<R>(
fn deserialize_bank_fields<R>(
mut stream: &mut BufReader<R>,
) -> Result<AccountDBFields<Self::SerializableAccountStorageEntry>, Error>
) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error>
where
R: Read,
{
deserialize_from(&mut stream)
let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into();
let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?;
Ok((bank_fields, accounts_db_fields))
}
fn deserialize_accounts_db_fields<R>(
stream: &mut BufReader<R>,
) -> Result<AccountsDbFields, Error>
where
R: Read,
{
deserialize_from(stream)
}
}

215
runtime/src/serde_snapshot/legacy.rs
View File

@ -1,7 +1,12 @@
use super::common::UnusedAccounts;
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
use solana_sdk::abi_example::IgnoreAsHelper;
use {super::*, bincode::config::Options, solana_measure::measure::Measure, std::cell::RefCell};
const MAX_ACCOUNTS_DB_STREAM_SIZE: u64 = MAX_STREAM_SIZE;
type AccountsDbFields = super::AccountsDbFields<SerializableAccountStorageEntry>;
// Serializable version of AccountStorageEntry for snapshot format
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
pub(super) struct SerializableAccountStorageEntry {
@ -52,7 +57,7 @@ impl Into<AppendVec> for SerializableAppendVec {
}
}
// Serialization of AppendVec requires serialization of u64 to
// Serialization of AppendVec requires serialization of u64 to
// eight byte vector which is then itself serialized to the stream
impl Serialize for SerializableAppendVec {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
@ -67,7 +72,7 @@ impl Serialize for SerializableAppendVec {
}
}
// Deserialization of AppendVec requires deserialization
// Deserialization of AppendVec requires deserialization
// of eight byte vector from which u64 is then deserialized
impl<'de> Deserialize<'de> for SerializableAppendVec {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
@ -99,27 +104,193 @@ impl<'de> Deserialize<'de> for SerializableAppendVec {
}
}
// Deserializable version of Bank which need not be serializable,
// because it's handled by SerializableVersionedBank.
// So, sync fields with it!
#[derive(Clone, Deserialize)]
pub(crate) struct DeserializableVersionedBank {
pub(crate) blockhash_queue: BlockhashQueue,
pub(crate) ancestors: Ancestors,
pub(crate) hash: Hash,
pub(crate) parent_hash: Hash,
pub(crate) parent_slot: Slot,
pub(crate) hard_forks: HardForks,
pub(crate) transaction_count: u64,
pub(crate) tick_height: u64,
pub(crate) signature_count: u64,
pub(crate) capitalization: u64,
pub(crate) max_tick_height: u64,
pub(crate) hashes_per_tick: Option<u64>,
pub(crate) ticks_per_slot: u64,
pub(crate) ns_per_slot: u128,
pub(crate) genesis_creation_time: UnixTimestamp,
pub(crate) slots_per_year: f64,
pub(crate) unused: u64,
pub(crate) slot: Slot,
pub(crate) epoch: Epoch,
pub(crate) block_height: u64,
pub(crate) collector_id: Pubkey,
pub(crate) collector_fees: u64,
pub(crate) fee_calculator: FeeCalculator,
pub(crate) fee_rate_governor: FeeRateGovernor,
pub(crate) collected_rent: u64,
pub(crate) rent_collector: RentCollector,
pub(crate) epoch_schedule: EpochSchedule,
pub(crate) inflation: Inflation,
pub(crate) stakes: Stakes,
pub(crate) unused_accounts: UnusedAccounts,
pub(crate) epoch_stakes: HashMap<Epoch, EpochStakes>,
pub(crate) is_delta: bool,
pub(crate) message_processor: MessageProcessor,
}
impl Into<BankFieldsToDeserialize> for DeserializableVersionedBank {
fn into(self) -> BankFieldsToDeserialize {
BankFieldsToDeserialize {
blockhash_queue: self.blockhash_queue,
ancestors: self.ancestors,
hash: self.hash,
parent_hash: self.parent_hash,
parent_slot: self.parent_slot,
hard_forks: self.hard_forks,
transaction_count: self.transaction_count,
tick_height: self.tick_height,
signature_count: self.signature_count,
capitalization: self.capitalization,
max_tick_height: self.max_tick_height,
hashes_per_tick: self.hashes_per_tick,
ticks_per_slot: self.ticks_per_slot,
ns_per_slot: self.ns_per_slot,
genesis_creation_time: self.genesis_creation_time,
slots_per_year: self.slots_per_year,
unused: self.unused,
slot: self.slot,
epoch: self.epoch,
block_height: self.block_height,
collector_id: self.collector_id,
collector_fees: self.collector_fees,
fee_calculator: self.fee_calculator,
fee_rate_governor: self.fee_rate_governor,
collected_rent: self.collected_rent,
rent_collector: self.rent_collector,
epoch_schedule: self.epoch_schedule,
inflation: self.inflation,
stakes: self.stakes,
epoch_stakes: self.epoch_stakes,
is_delta: self.is_delta,
}
}
}
// Serializable version of Bank, not Deserializable to avoid cloning by using refs.
// Sync fields with DeserializableVersionedBank!
#[derive(Serialize)]
pub(crate) struct SerializableVersionedBank<'a> {
pub(crate) blockhash_queue: &'a RwLock<BlockhashQueue>,
pub(crate) ancestors: &'a Ancestors,
pub(crate) hash: Hash,
pub(crate) parent_hash: Hash,
pub(crate) parent_slot: Slot,
pub(crate) hard_forks: &'a RwLock<HardForks>,
pub(crate) transaction_count: u64,
pub(crate) tick_height: u64,
pub(crate) signature_count: u64,
pub(crate) capitalization: u64,
pub(crate) max_tick_height: u64,
pub(crate) hashes_per_tick: Option<u64>,
pub(crate) ticks_per_slot: u64,
pub(crate) ns_per_slot: u128,
pub(crate) genesis_creation_time: UnixTimestamp,
pub(crate) slots_per_year: f64,
pub(crate) unused: u64,
pub(crate) slot: Slot,
pub(crate) epoch: Epoch,
pub(crate) block_height: u64,
pub(crate) collector_id: Pubkey,
pub(crate) collector_fees: u64,
pub(crate) fee_calculator: FeeCalculator,
pub(crate) fee_rate_governor: FeeRateGovernor,
pub(crate) collected_rent: u64,
pub(crate) rent_collector: RentCollector,
pub(crate) epoch_schedule: EpochSchedule,
pub(crate) inflation: Inflation,
pub(crate) stakes: &'a RwLock<Stakes>,
pub(crate) unused_accounts: UnusedAccounts,
pub(crate) epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
pub(crate) is_delta: bool,
pub(crate) message_processor: MessageProcessor,
}
use std::sync::RwLock;
impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedBank<'a> {
fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self {
fn new<T: Default>() -> T {
T::default()
}
Self {
blockhash_queue: rhs.blockhash_queue,
ancestors: rhs.ancestors,
hash: rhs.hash,
parent_hash: rhs.parent_hash,
parent_slot: rhs.parent_slot,
hard_forks: rhs.hard_forks,
transaction_count: rhs.transaction_count,
tick_height: rhs.tick_height,
signature_count: rhs.signature_count,
capitalization: rhs.capitalization,
max_tick_height: rhs.max_tick_height,
hashes_per_tick: rhs.hashes_per_tick,
ticks_per_slot: rhs.ticks_per_slot,
ns_per_slot: rhs.ns_per_slot,
genesis_creation_time: rhs.genesis_creation_time,
slots_per_year: rhs.slots_per_year,
unused: rhs.unused,
slot: rhs.slot,
epoch: rhs.epoch,
block_height: rhs.block_height,
collector_id: rhs.collector_id,
collector_fees: rhs.collector_fees,
fee_calculator: rhs.fee_calculator,
fee_rate_governor: rhs.fee_rate_governor,
collected_rent: rhs.collected_rent,
rent_collector: rhs.rent_collector,
epoch_schedule: rhs.epoch_schedule,
inflation: rhs.inflation,
stakes: rhs.stakes,
unused_accounts: new(),
epoch_stakes: rhs.epoch_stakes,
is_delta: rhs.is_delta,
message_processor: new(),
}
}
}
#[cfg(RUSTC_WITH_SPECIALIZATION)]
impl<'a> IgnoreAsHelper for SerializableVersionedBank<'a> {}
pub(super) struct Context {}
impl<'a> TypeContext<'a> for Context {
type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
fn serialize_bank_rc_fields<S: serde::ser::Serializer>(
fn serialize_bank_and_storage<S: serde::ser::Serializer>(
serializer: S,
serializable_bank: &SerializableBankRc<'a, Self>,
serializable_bank_and_storage: &SerializableBankAndStorage<'a, Self>,
) -> std::result::Result<S::Ok, S::Error>
where
Self: std::marker::Sized,
{
// as there is no deserialize_bank_rc_fields(), do not emit the u64
// size field here and have serialize_accounts_db_fields() emit two
// u64 size fields instead
SerializableAccountsDB::<'a, Self> {
accounts_db: &*serializable_bank.bank_rc.accounts.accounts_db,
slot: serializable_bank.bank_rc.slot,
account_storage_entries: serializable_bank.snapshot_storages,
phantom: std::marker::PhantomData::default(),
}
.serialize(serializer)
(
SerializableVersionedBank::from(
serializable_bank_and_storage.bank.get_fields_to_serialize(),
),
SerializableAccountsDB::<'a, Self> {
accounts_db: &*serializable_bank_and_storage.bank.rc.accounts.accounts_db,
slot: serializable_bank_and_storage.bank.rc.slot,
account_storage_entries: serializable_bank_and_storage.snapshot_storages,
phantom: std::marker::PhantomData::default(),
},
)
.serialize(serializer)
}
fn serialize_accounts_db_fields<S: serde::ser::Serializer>(
@ -161,6 +332,7 @@ impl<'a> TypeContext<'a> for Context {
.clone(),
);
// as there is no deserialize_bank_rc_fields(), emit two u64 size fields here instead
let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms");
let result = (
&MAX_ACCOUNTS_DB_STREAM_SIZE,
@ -179,9 +351,20 @@ impl<'a> TypeContext<'a> for Context {
result
}
fn deserialize_bank_fields<R>(
mut stream: &mut BufReader<R>,
) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error>
where
R: Read,
{
let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into();
let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?;
Ok((bank_fields, accounts_db_fields))
}
fn deserialize_accounts_db_fields<R>(
mut stream: &mut BufReader<R>,
) -> Result<AccountDBFields<Self::SerializableAccountStorageEntry>, Error>
) -> Result<AccountsDbFields, Error>
where
R: Read,
{
@ -203,6 +386,6 @@ impl<'a> TypeContext<'a> for Context {
// (3rd of 3 elements) read in (slot, bank hashes) pair
let (slot, bank_hash_info): (Slot, BankHashInfo) = deserialize_from(&mut stream)?;
Ok(AccountDBFields(storage, version, slot, bank_hash_info))
Ok(AccountsDbFields(storage, version, slot, bank_hash_info))
}
}

80
runtime/src/serde_snapshot/tests.rs
View File

@ -64,7 +64,7 @@ where
P: AsRef<Path>,
{
// read and deserialise the accounts database directly from the stream
context_accountsdb_from_fields::<C, P>(
reconstruct_accountsdb_from_fields(
C::deserialize_accounts_db_fields(stream)?,
account_paths,
stream_append_vecs_path,
@ -198,17 +198,15 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) {
let snapshot_storages = bank2.get_snapshot_storages();
let mut buf = vec![];
let mut writer = Cursor::new(&mut buf);
serialize_into(&mut writer, &bank2).unwrap();
crate::serde_snapshot::bankrc_to_stream(
crate::serde_snapshot::bank_to_stream(
serde_style,
&mut std::io::BufWriter::new(&mut writer),
&bank2.rc,
&bank2,
&snapshot_storages,
)
.unwrap();
let mut rdr = Cursor::new(&buf[..]);
let mut dbank: Bank = bincode::deserialize_from(&mut rdr).unwrap();
let rdr = Cursor::new(&buf[..]);
let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]);
// Create a new set of directories for this bank's accounts
@ -218,17 +216,16 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) {
// Create a directory to simulate AppendVecs unpackaged from a snapshot tar
let copied_accounts = TempDir::new().unwrap();
copy_append_vecs(&bank2.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
dbank.set_bank_rc(
crate::serde_snapshot::bankrc_from_stream(
serde_style,
&dbank_paths,
dbank.slot(),
&mut reader,
copied_accounts.path(),
)
.unwrap(),
ref_sc,
);
let mut dbank = crate::serde_snapshot::bank_from_stream(
serde_style,
&mut reader,
copied_accounts.path(),
&dbank_paths,
&genesis_config,
&[],
)
.unwrap();
dbank.src = ref_sc;
assert_eq!(dbank.get_balance(&key1.pubkey()), 0);
assert_eq!(dbank.get_balance(&key2.pubkey()), 10);
assert_eq!(dbank.get_balance(&key3.pubkey()), 0);
@ -280,45 +277,58 @@ fn test_bank_serialize_older() {
}
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
mod test_bank_rc_serialize {
mod test_bank_serialize {
use super::*;
use solana_sdk::abi_example::AbiExample;
// These some what long test harness is required to freeze the ABI of
// BankRc's serialization due to versioned nature
#[frozen_abi(digest = "HfCP74JKqPdeAccNJEj7KEoNxtsmX3zRqc2rpTy1NC7H")]
#[derive(Serialize, AbiExample)]
pub struct BandRcAbiTestWrapperFuture {
// Bank's serialization due to versioned nature
#[frozen_abi(digest = "9BGkhttaVsELn1zoHMKXLvi3Qty51nY1yz584Fao2Ev9")]
#[derive(Default, Serialize)]
pub struct BankAbiTestWrapperFuture {
#[serde(serialize_with = "wrapper_future")]
bank_rc: BankRc,
bank: Bank,
}
pub fn wrapper_future<S>(bank_rc: &BankRc, s: S) -> std::result::Result<S::Ok, S::Error>
impl AbiExample for BankAbiTestWrapperFuture {
fn example() -> Self {
Self::default()
}
}
pub fn wrapper_future<S>(bank: &Bank, s: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let snapshot_storages = bank_rc.accounts.accounts_db.get_snapshot_storages(0);
(SerializableBankRc::<future::Context> {
bank_rc,
let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0);
(SerializableBankAndStorage::<future::Context> {
bank,
snapshot_storages: &snapshot_storages,
phantom: std::marker::PhantomData::default(),
})
.serialize(s)
}
#[frozen_abi(digest = "43niyekyWwreLALcdEeFFpd7h8U6pgSXGqfKBRw8H7Vy")]
#[derive(Serialize, AbiExample)]
pub struct BandRcAbiTestWrapperLegacy {
#[frozen_abi(digest = "HYmXta1fhiHe6hZLGJriMqKx9N2U8YRJY51AGZmxzM5m")]
#[derive(Default, Serialize)]
pub struct BankAbiTestWrapperLegacy {
#[serde(serialize_with = "wrapper_legacy")]
bank_rc: BankRc,
bank: Bank,
}
pub fn wrapper_legacy<S>(bank_rc: &BankRc, s: S) -> std::result::Result<S::Ok, S::Error>
impl AbiExample for BankAbiTestWrapperLegacy {
fn example() -> Self {
Self::default()
}
}
pub fn wrapper_legacy<S>(bank: &Bank, s: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let snapshot_storages = bank_rc.accounts.accounts_db.get_snapshot_storages(0);
(SerializableBankRc::<legacy::Context> {
bank_rc,
let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0);
(SerializableBankAndStorage::<legacy::Context> {
bank,
snapshot_storages: &snapshot_storages,
phantom: std::marker::PhantomData::default(),
})

40
runtime/src/snapshot_utils.rs
View File

@ -3,7 +3,7 @@ use crate::{
bank_forks::CompressionType,
hardened_unpack::{unpack_snapshot, UnpackError},
serde_snapshot::{
bankrc_from_stream, bankrc_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages,
bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages,
},
snapshot_package::AccountsPackage,
};
@ -23,7 +23,6 @@ use std::{
path::{Path, PathBuf},
process::ExitStatus,
str::FromStr,
sync::Arc,
};
use tar::Archive;
use tempfile::TempDir;
@ -462,8 +461,7 @@ pub fn add_snapshot<P: AsRef<Path>>(
SnapshotVersion::V1_1_0 => SerdeStyle::OLDER,
SnapshotVersion::V1_2_0 => SerdeStyle::NEWER,
};
serialize_into(stream.by_ref(), bank)?;
bankrc_to_stream(serde_style, stream.by_ref(), &bank.rc, snapshot_storages)?;
bank_to_stream(serde_style, stream.by_ref(), bank, snapshot_storages)?;
Ok(())
};
let consumed_size =
@ -723,38 +721,24 @@ where
info!("Loading bank from {:?}", &root_paths.snapshot_file_path);
let bank = deserialize_snapshot_data_file(&root_paths.snapshot_file_path, |mut stream| {
let mut bank: Bank = bincode::options()
.with_limit(MAX_SNAPSHOT_DATA_FILE_SIZE)
.with_fixint_encoding()
.allow_trailing_bytes()
.deserialize_from(&mut stream)?;
info!("Rebuilding accounts...");
let mut bankrc = match snapshot_version_enum {
SnapshotVersion::V1_1_0 => bankrc_from_stream(
Ok(match snapshot_version_enum {
SnapshotVersion::V1_1_0 => bank_from_stream(
SerdeStyle::OLDER,
account_paths,
bank.slot(),
&mut stream,
&append_vecs_path,
account_paths,
genesis_config,
frozen_account_pubkeys,
),
SnapshotVersion::V1_2_0 => bankrc_from_stream(
SnapshotVersion::V1_2_0 => bank_from_stream(
SerdeStyle::NEWER,
account_paths,
bank.slot(),
&mut stream,
&append_vecs_path,
account_paths,
genesis_config,
frozen_account_pubkeys,
),
}?;
Arc::get_mut(&mut Arc::get_mut(&mut bankrc.accounts).unwrap().accounts_db)
.unwrap()
.freeze_accounts(&bank.ancestors, frozen_account_pubkeys);
bank.rc = bankrc;
bank.operating_mode = Some(genesis_config.operating_mode);
bank.finish_init();
Ok(bank)
}?)
})?;
let status_cache_path = unpacked_snapshots_dir.join(SNAPSHOT_STATUS_CACHE_FILE_NAME);

7
sdk/macro-frozen-abi/src/lib.rs
View File

@ -78,8 +78,11 @@ fn quote_for_specialization_detection() -> TokenStream2 {
std::sync::atomic::AtomicBool::new(false);
}
if !SPECIALIZATION_DETECTOR_INJECTED.load(std::sync::atomic::Ordering::Relaxed) {
SPECIALIZATION_DETECTOR_INJECTED.store(true, std::sync::atomic::Ordering::Relaxed);
if !SPECIALIZATION_DETECTOR_INJECTED.compare_and_swap(
false,
true,
std::sync::atomic::Ordering::AcqRel,
) {
quote! {
mod specialization_detector {
trait SpecializedTrait {