Fold bank serialisation into serde snapshot (#10581)
* Move bank (de)serialisation logic from bank and snapshot_utils to serde_snapshot. Add sanity assertions between genesis config and bank fields on deserialisation. Atomically update atomic bool in quote_for_specialization_detection(). Use same genesis config when restoring snapshots in test cases. * Tidy up namings and duplicate structs to version * Apply struct renames to tests * Update abi hashes Co-authored-by: Ryo Onodera <ryoqun@gmail.com>
This commit is contained in:
parent
5c86766fb2
commit
ed5a2f2a90
|
@ -110,6 +110,7 @@ mod tests {
|
||||||
fn restore_from_snapshot(
|
fn restore_from_snapshot(
|
||||||
old_bank_forks: &BankForks,
|
old_bank_forks: &BankForks,
|
||||||
old_last_slot: Slot,
|
old_last_slot: Slot,
|
||||||
|
old_genesis_config: &GenesisConfig,
|
||||||
account_paths: &[PathBuf],
|
account_paths: &[PathBuf],
|
||||||
) {
|
) {
|
||||||
let (snapshot_path, snapshot_package_output_path) = old_bank_forks
|
let (snapshot_path, snapshot_package_output_path) = old_bank_forks
|
||||||
|
@ -134,7 +135,7 @@ mod tests {
|
||||||
&CompressionType::Bzip2,
|
&CompressionType::Bzip2,
|
||||||
),
|
),
|
||||||
CompressionType::Bzip2,
|
CompressionType::Bzip2,
|
||||||
&GenesisConfig::default(),
|
old_genesis_config,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
@ -169,7 +170,6 @@ mod tests {
|
||||||
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
|
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
|
||||||
|
|
||||||
let bank_forks = &mut snapshot_test_config.bank_forks;
|
let bank_forks = &mut snapshot_test_config.bank_forks;
|
||||||
let accounts_dir = &snapshot_test_config.accounts_dir;
|
|
||||||
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
||||||
|
|
||||||
let (s, _r) = channel();
|
let (s, _r) = channel();
|
||||||
|
@ -185,6 +185,7 @@ mod tests {
|
||||||
bank_forks.set_root(bank.slot(), &sender, None);
|
bank_forks.set_root(bank.slot(), &sender, None);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate a snapshot package for last bank
|
// Generate a snapshot package for last bank
|
||||||
let last_bank = bank_forks.get(last_slot).unwrap();
|
let last_bank = bank_forks.get(last_slot).unwrap();
|
||||||
let snapshot_config = &snapshot_test_config.snapshot_config;
|
let snapshot_config = &snapshot_test_config.snapshot_config;
|
||||||
|
@ -203,10 +204,12 @@ mod tests {
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
|
snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
|
||||||
|
|
||||||
restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]);
|
// Restore bank from snapshot
|
||||||
|
let account_paths = &[snapshot_test_config.accounts_dir.path().to_path_buf()];
|
||||||
|
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
|
||||||
|
restore_from_snapshot(bank_forks, last_slot, genesis_config, account_paths);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) {
|
fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) {
|
||||||
|
|
|
@ -38,7 +38,9 @@ pub fn create_genesis<T: Client>(from: &Keypair, client: &T, amount: u64) -> Key
|
||||||
|
|
||||||
let instruction = librapay_instruction::genesis(&genesis.pubkey(), amount);
|
let instruction = librapay_instruction::genesis(&genesis.pubkey(), amount);
|
||||||
let message = Message::new(&[instruction], Some(&from.pubkey()));
|
let message = Message::new(&[instruction], Some(&from.pubkey()));
|
||||||
client.send_and_confirm_message(&[from, &genesis], message).unwrap();
|
client
|
||||||
|
.send_and_confirm_message(&[from, &genesis], message)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
genesis
|
genesis
|
||||||
}
|
}
|
||||||
|
|
|
@ -66,23 +66,19 @@ pub enum AccountAddressFilter {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Accounts {
|
impl Accounts {
|
||||||
pub(crate) fn new_empty(accounts_db: AccountsDB) -> Self {
|
pub fn new(paths: Vec<PathBuf>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
accounts_db: Arc::new(accounts_db),
|
slot: 0,
|
||||||
|
accounts_db: Arc::new(AccountsDB::new(paths)),
|
||||||
account_locks: Mutex::new(HashSet::new()),
|
account_locks: Mutex::new(HashSet::new()),
|
||||||
readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
|
readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
|
||||||
..Self::default()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new(paths: Vec<PathBuf>) -> Self {
|
|
||||||
Self::new_with_frozen_accounts(paths, &HashMap::default(), &[])
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new_from_parent(parent: &Accounts, slot: Slot, parent_slot: Slot) -> Self {
|
pub fn new_from_parent(parent: &Accounts, slot: Slot, parent_slot: Slot) -> Self {
|
||||||
let accounts_db = parent.accounts_db.clone();
|
let accounts_db = parent.accounts_db.clone();
|
||||||
accounts_db.set_hash(slot, parent_slot);
|
accounts_db.set_hash(slot, parent_slot);
|
||||||
Accounts {
|
Self {
|
||||||
slot,
|
slot,
|
||||||
accounts_db,
|
accounts_db,
|
||||||
account_locks: Mutex::new(HashSet::new()),
|
account_locks: Mutex::new(HashSet::new()),
|
||||||
|
@ -90,25 +86,13 @@ impl Accounts {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new_with_frozen_accounts(
|
pub(crate) fn new_empty(accounts_db: AccountsDB) -> Self {
|
||||||
paths: Vec<PathBuf>,
|
Self {
|
||||||
ancestors: &Ancestors,
|
|
||||||
frozen_account_pubkeys: &[Pubkey],
|
|
||||||
) -> Self {
|
|
||||||
let mut accounts = Accounts {
|
|
||||||
slot: 0,
|
slot: 0,
|
||||||
accounts_db: Arc::new(AccountsDB::new(paths)),
|
accounts_db: Arc::new(accounts_db),
|
||||||
account_locks: Mutex::new(HashSet::new()),
|
account_locks: Mutex::new(HashSet::new()),
|
||||||
readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
|
readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
|
||||||
};
|
}
|
||||||
accounts.freeze_accounts(ancestors, frozen_account_pubkeys);
|
|
||||||
accounts
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn freeze_accounts(&mut self, ancestors: &Ancestors, frozen_account_pubkeys: &[Pubkey]) {
|
|
||||||
Arc::get_mut(&mut self.accounts_db)
|
|
||||||
.unwrap()
|
|
||||||
.freeze_accounts(ancestors, frozen_account_pubkeys);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return true if the slice has any duplicate elements
|
/// Return true if the slice has any duplicate elements
|
||||||
|
|
|
@ -1800,7 +1800,7 @@ impl AccountsDB {
|
||||||
hashes
|
hashes
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn freeze_accounts(&mut self, ancestors: &Ancestors, account_pubkeys: &[Pubkey]) {
|
pub(crate) fn freeze_accounts(&mut self, ancestors: &Ancestors, account_pubkeys: &[Pubkey]) {
|
||||||
for account_pubkey in account_pubkeys {
|
for account_pubkey in account_pubkeys {
|
||||||
if let Some((account, _slot)) = self.load_slow(ancestors, &account_pubkey) {
|
if let Some((account, _slot)) = self.load_slow(ancestors, &account_pubkey) {
|
||||||
let frozen_account_info = FrozenAccountInfo {
|
let frozen_account_info = FrozenAccountInfo {
|
||||||
|
@ -1994,7 +1994,7 @@ impl AccountsDB {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn print_accounts_stats(&self, label: &'static str) {
|
pub(crate) fn print_accounts_stats(&self, label: &'static str) {
|
||||||
self.print_index(label);
|
self.print_index(label);
|
||||||
self.print_count_and_status(label);
|
self.print_count_and_status(label);
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,7 +25,6 @@ use crate::{
|
||||||
use byteorder::{ByteOrder, LittleEndian};
|
use byteorder::{ByteOrder, LittleEndian};
|
||||||
use itertools::Itertools;
|
use itertools::Itertools;
|
||||||
use log::*;
|
use log::*;
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use solana_measure::measure::Measure;
|
use solana_measure::measure::Measure;
|
||||||
use solana_metrics::{
|
use solana_metrics::{
|
||||||
datapoint_debug, inc_new_counter_debug, inc_new_counter_error, inc_new_counter_info,
|
datapoint_debug, inc_new_counter_debug, inc_new_counter_error, inc_new_counter_info,
|
||||||
|
@ -108,20 +107,6 @@ pub struct BankRc {
|
||||||
pub(crate) slot: Slot,
|
pub(crate) slot: Slot,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
|
||||||
use solana_sdk::abi_example::AbiExample;
|
|
||||||
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
|
||||||
impl AbiExample for BankRc {
|
|
||||||
fn example() -> Self {
|
|
||||||
BankRc {
|
|
||||||
// Set parent to None to cut the recursion into another Bank
|
|
||||||
parent: RwLock::new(None),
|
|
||||||
accounts: AbiExample::example(),
|
|
||||||
slot: AbiExample::example(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl BankRc {
|
impl BankRc {
|
||||||
pub(crate) fn new(accounts: Accounts, slot: Slot) -> Self {
|
pub(crate) fn new(accounts: Accounts, slot: Slot) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
@ -203,22 +188,88 @@ impl HashAgeKind {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)]
|
// Bank's common fields shared by all supported snapshot versions for deserialization.
|
||||||
struct UnusedAccounts {
|
// Sync fields with BankFieldsToSerialize! This is paired with it.
|
||||||
unused1: HashSet<Pubkey>,
|
// All members are made public to remain Bank's members private and to make versioned deserializer workable on this
|
||||||
unused2: HashSet<Pubkey>,
|
#[derive(Clone, Default)]
|
||||||
unused3: HashMap<Pubkey, u64>,
|
pub(crate) struct BankFieldsToDeserialize {
|
||||||
|
pub(crate) blockhash_queue: BlockhashQueue,
|
||||||
|
pub(crate) ancestors: Ancestors,
|
||||||
|
pub(crate) hash: Hash,
|
||||||
|
pub(crate) parent_hash: Hash,
|
||||||
|
pub(crate) parent_slot: Slot,
|
||||||
|
pub(crate) hard_forks: HardForks,
|
||||||
|
pub(crate) transaction_count: u64,
|
||||||
|
pub(crate) tick_height: u64,
|
||||||
|
pub(crate) signature_count: u64,
|
||||||
|
pub(crate) capitalization: u64,
|
||||||
|
pub(crate) max_tick_height: u64,
|
||||||
|
pub(crate) hashes_per_tick: Option<u64>,
|
||||||
|
pub(crate) ticks_per_slot: u64,
|
||||||
|
pub(crate) ns_per_slot: u128,
|
||||||
|
pub(crate) genesis_creation_time: UnixTimestamp,
|
||||||
|
pub(crate) slots_per_year: f64,
|
||||||
|
pub(crate) unused: u64,
|
||||||
|
pub(crate) slot: Slot,
|
||||||
|
pub(crate) epoch: Epoch,
|
||||||
|
pub(crate) block_height: u64,
|
||||||
|
pub(crate) collector_id: Pubkey,
|
||||||
|
pub(crate) collector_fees: u64,
|
||||||
|
pub(crate) fee_calculator: FeeCalculator,
|
||||||
|
pub(crate) fee_rate_governor: FeeRateGovernor,
|
||||||
|
pub(crate) collected_rent: u64,
|
||||||
|
pub(crate) rent_collector: RentCollector,
|
||||||
|
pub(crate) epoch_schedule: EpochSchedule,
|
||||||
|
pub(crate) inflation: Inflation,
|
||||||
|
pub(crate) stakes: Stakes,
|
||||||
|
pub(crate) epoch_stakes: HashMap<Epoch, EpochStakes>,
|
||||||
|
pub(crate) is_delta: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bank's common fields shared by all supported snapshot versions for serialization.
|
||||||
|
// This is separated from BankFieldsToDeserialize to avoid cloning by using refs.
|
||||||
|
// So, sync fields with BankFieldsToDeserialize!
|
||||||
|
// all members are made public to remain Bank private and to make versioned serializer workable on this
|
||||||
|
pub(crate) struct BankFieldsToSerialize<'a> {
|
||||||
|
pub(crate) blockhash_queue: &'a RwLock<BlockhashQueue>,
|
||||||
|
pub(crate) ancestors: &'a Ancestors,
|
||||||
|
pub(crate) hash: Hash,
|
||||||
|
pub(crate) parent_hash: Hash,
|
||||||
|
pub(crate) parent_slot: Slot,
|
||||||
|
pub(crate) hard_forks: &'a RwLock<HardForks>,
|
||||||
|
pub(crate) transaction_count: u64,
|
||||||
|
pub(crate) tick_height: u64,
|
||||||
|
pub(crate) signature_count: u64,
|
||||||
|
pub(crate) capitalization: u64,
|
||||||
|
pub(crate) max_tick_height: u64,
|
||||||
|
pub(crate) hashes_per_tick: Option<u64>,
|
||||||
|
pub(crate) ticks_per_slot: u64,
|
||||||
|
pub(crate) ns_per_slot: u128,
|
||||||
|
pub(crate) genesis_creation_time: UnixTimestamp,
|
||||||
|
pub(crate) slots_per_year: f64,
|
||||||
|
pub(crate) unused: u64,
|
||||||
|
pub(crate) slot: Slot,
|
||||||
|
pub(crate) epoch: Epoch,
|
||||||
|
pub(crate) block_height: u64,
|
||||||
|
pub(crate) collector_id: Pubkey,
|
||||||
|
pub(crate) collector_fees: u64,
|
||||||
|
pub(crate) fee_calculator: FeeCalculator,
|
||||||
|
pub(crate) fee_rate_governor: FeeRateGovernor,
|
||||||
|
pub(crate) collected_rent: u64,
|
||||||
|
pub(crate) rent_collector: RentCollector,
|
||||||
|
pub(crate) epoch_schedule: EpochSchedule,
|
||||||
|
pub(crate) inflation: Inflation,
|
||||||
|
pub(crate) stakes: &'a RwLock<Stakes>,
|
||||||
|
pub(crate) epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
|
||||||
|
pub(crate) is_delta: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Manager for the state of all accounts and programs after processing its entries.
|
/// Manager for the state of all accounts and programs after processing its entries.
|
||||||
#[frozen_abi(digest = "GsvisJfTaHxmTX4s6gQs2bZtxVggB7F325XDdXTA573p")]
|
#[derive(Default)]
|
||||||
#[derive(Default, Deserialize, Serialize, AbiExample)]
|
|
||||||
pub struct Bank {
|
pub struct Bank {
|
||||||
/// References to accounts, parent and signature status
|
/// References to accounts, parent and signature status
|
||||||
#[serde(skip)]
|
|
||||||
pub rc: BankRc,
|
pub rc: BankRc,
|
||||||
|
|
||||||
#[serde(skip)]
|
|
||||||
pub src: StatusCacheRc,
|
pub src: StatusCacheRc,
|
||||||
|
|
||||||
/// FIFO queue of `recent_blockhash` items
|
/// FIFO queue of `recent_blockhash` items
|
||||||
|
@ -308,9 +359,6 @@ pub struct Bank {
|
||||||
/// cache of vote_account and stake_account state for this fork
|
/// cache of vote_account and stake_account state for this fork
|
||||||
stakes: RwLock<Stakes>,
|
stakes: RwLock<Stakes>,
|
||||||
|
|
||||||
/// unused
|
|
||||||
unused_accounts: RwLock<UnusedAccounts>,
|
|
||||||
|
|
||||||
/// staked nodes on epoch boundaries, saved off when a bank.slot() is at
|
/// staked nodes on epoch boundaries, saved off when a bank.slot() is at
|
||||||
/// a leader schedule calculation boundary
|
/// a leader schedule calculation boundary
|
||||||
epoch_stakes: HashMap<Epoch, EpochStakes>,
|
epoch_stakes: HashMap<Epoch, EpochStakes>,
|
||||||
|
@ -324,24 +372,18 @@ pub struct Bank {
|
||||||
|
|
||||||
/// Callback to be notified when a bank enters a new Epoch
|
/// Callback to be notified when a bank enters a new Epoch
|
||||||
/// (used to adjust cluster features over time)
|
/// (used to adjust cluster features over time)
|
||||||
#[serde(skip)]
|
|
||||||
entered_epoch_callback: Arc<RwLock<Option<EnteredEpochCallback>>>,
|
entered_epoch_callback: Arc<RwLock<Option<EnteredEpochCallback>>>,
|
||||||
|
|
||||||
/// Last time when the cluster info vote listener has synced with this bank
|
/// Last time when the cluster info vote listener has synced with this bank
|
||||||
#[serde(skip)]
|
|
||||||
pub last_vote_sync: AtomicU64,
|
pub last_vote_sync: AtomicU64,
|
||||||
|
|
||||||
/// Rewards that were paid out immediately after this bank was created
|
/// Rewards that were paid out immediately after this bank was created
|
||||||
#[serde(skip)]
|
|
||||||
pub rewards: Option<Vec<(Pubkey, i64)>>,
|
pub rewards: Option<Vec<(Pubkey, i64)>>,
|
||||||
|
|
||||||
#[serde(skip)]
|
|
||||||
pub skip_drop: AtomicBool,
|
pub skip_drop: AtomicBool,
|
||||||
|
|
||||||
#[serde(skip)]
|
|
||||||
pub operating_mode: Option<OperatingMode>,
|
pub operating_mode: Option<OperatingMode>,
|
||||||
|
|
||||||
#[serde(skip)]
|
|
||||||
pub lazy_rent_collection: AtomicBool,
|
pub lazy_rent_collection: AtomicBool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -370,7 +412,7 @@ impl Bank {
|
||||||
bank.finish_init();
|
bank.finish_init();
|
||||||
|
|
||||||
// Freeze accounts after process_genesis_config creates the initial append vecs
|
// Freeze accounts after process_genesis_config creates the initial append vecs
|
||||||
Arc::get_mut(&mut bank.rc.accounts)
|
Arc::get_mut(&mut Arc::get_mut(&mut bank.rc.accounts).unwrap().accounts_db)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.freeze_accounts(&bank.ancestors, frozen_account_pubkeys);
|
.freeze_accounts(&bank.ancestors, frozen_account_pubkeys);
|
||||||
|
|
||||||
|
@ -440,7 +482,6 @@ impl Bank {
|
||||||
transaction_count: AtomicU64::new(parent.transaction_count()),
|
transaction_count: AtomicU64::new(parent.transaction_count()),
|
||||||
stakes: RwLock::new(parent.stakes.read().unwrap().clone_with_epoch(epoch)),
|
stakes: RwLock::new(parent.stakes.read().unwrap().clone_with_epoch(epoch)),
|
||||||
epoch_stakes: parent.epoch_stakes.clone(),
|
epoch_stakes: parent.epoch_stakes.clone(),
|
||||||
unused_accounts: RwLock::new(parent.unused_accounts.read().unwrap().clone()),
|
|
||||||
parent_hash: parent.hash(),
|
parent_hash: parent.hash(),
|
||||||
parent_slot: parent.slot(),
|
parent_slot: parent.slot(),
|
||||||
collector_id: *collector_id,
|
collector_id: *collector_id,
|
||||||
|
@ -514,6 +555,136 @@ impl Bank {
|
||||||
new
|
new
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Create a bank from explicit arguments and deserialized fields from snapshot
|
||||||
|
#[allow(clippy::float_cmp)]
|
||||||
|
pub(crate) fn new_from_fields(
|
||||||
|
bank_rc: BankRc,
|
||||||
|
genesis_config: &GenesisConfig,
|
||||||
|
fields: BankFieldsToDeserialize,
|
||||||
|
) -> Self {
|
||||||
|
fn new<T: Default>() -> T {
|
||||||
|
T::default()
|
||||||
|
}
|
||||||
|
let mut bank = Self {
|
||||||
|
rc: bank_rc,
|
||||||
|
src: new(),
|
||||||
|
blockhash_queue: RwLock::new(fields.blockhash_queue),
|
||||||
|
ancestors: fields.ancestors,
|
||||||
|
hash: RwLock::new(fields.hash),
|
||||||
|
parent_hash: fields.parent_hash,
|
||||||
|
parent_slot: fields.parent_slot,
|
||||||
|
hard_forks: Arc::new(RwLock::new(fields.hard_forks)),
|
||||||
|
transaction_count: AtomicU64::new(fields.transaction_count),
|
||||||
|
tick_height: AtomicU64::new(fields.tick_height),
|
||||||
|
signature_count: AtomicU64::new(fields.signature_count),
|
||||||
|
capitalization: AtomicU64::new(fields.capitalization),
|
||||||
|
max_tick_height: fields.max_tick_height,
|
||||||
|
hashes_per_tick: fields.hashes_per_tick,
|
||||||
|
ticks_per_slot: fields.ticks_per_slot,
|
||||||
|
ns_per_slot: fields.ns_per_slot,
|
||||||
|
genesis_creation_time: fields.genesis_creation_time,
|
||||||
|
slots_per_year: fields.slots_per_year,
|
||||||
|
unused: genesis_config.unused,
|
||||||
|
slot: fields.slot,
|
||||||
|
epoch: fields.epoch,
|
||||||
|
block_height: fields.block_height,
|
||||||
|
collector_id: fields.collector_id,
|
||||||
|
collector_fees: AtomicU64::new(fields.collector_fees),
|
||||||
|
fee_calculator: fields.fee_calculator,
|
||||||
|
fee_rate_governor: fields.fee_rate_governor,
|
||||||
|
collected_rent: AtomicU64::new(fields.collected_rent),
|
||||||
|
rent_collector: fields.rent_collector,
|
||||||
|
epoch_schedule: fields.epoch_schedule,
|
||||||
|
inflation: Arc::new(RwLock::new(fields.inflation)),
|
||||||
|
stakes: RwLock::new(fields.stakes),
|
||||||
|
epoch_stakes: fields.epoch_stakes,
|
||||||
|
is_delta: AtomicBool::new(fields.is_delta),
|
||||||
|
message_processor: new(),
|
||||||
|
entered_epoch_callback: new(),
|
||||||
|
last_vote_sync: new(),
|
||||||
|
rewards: new(),
|
||||||
|
skip_drop: new(),
|
||||||
|
operating_mode: Some(genesis_config.operating_mode),
|
||||||
|
lazy_rent_collection: new(),
|
||||||
|
};
|
||||||
|
bank.finish_init();
|
||||||
|
|
||||||
|
// Sanity assertions between bank snapshot and genesis config
|
||||||
|
// Consider removing from serializable bank state ([Ref]BankFields) and initializing
|
||||||
|
// from the passed in genesis_config instead (as new()/new_with_paths() already do)
|
||||||
|
assert_eq!(
|
||||||
|
bank.hashes_per_tick,
|
||||||
|
genesis_config.poh_config.hashes_per_tick
|
||||||
|
);
|
||||||
|
assert_eq!(bank.ticks_per_slot, genesis_config.ticks_per_slot);
|
||||||
|
assert_eq!(
|
||||||
|
bank.ns_per_slot,
|
||||||
|
genesis_config.poh_config.target_tick_duration.as_nanos()
|
||||||
|
* genesis_config.ticks_per_slot as u128
|
||||||
|
);
|
||||||
|
assert_eq!(bank.genesis_creation_time, genesis_config.creation_time);
|
||||||
|
assert_eq!(bank.unused, genesis_config.unused);
|
||||||
|
assert_eq!(bank.max_tick_height, (bank.slot + 1) * bank.ticks_per_slot);
|
||||||
|
assert_eq!(
|
||||||
|
bank.slots_per_year,
|
||||||
|
years_as_slots(
|
||||||
|
1.0,
|
||||||
|
&genesis_config.poh_config.target_tick_duration,
|
||||||
|
bank.ticks_per_slot,
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert_eq!(bank.epoch_schedule, genesis_config.epoch_schedule);
|
||||||
|
assert_eq!(bank.epoch, bank.epoch_schedule.get_epoch(bank.slot));
|
||||||
|
assert_eq!(
|
||||||
|
bank.rent_collector,
|
||||||
|
RentCollector::new(
|
||||||
|
bank.epoch,
|
||||||
|
&bank.epoch_schedule,
|
||||||
|
bank.slots_per_year,
|
||||||
|
&genesis_config.rent,
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
bank
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return subset of bank fields representing serializable state
|
||||||
|
pub(crate) fn get_fields_to_serialize(&self) -> BankFieldsToSerialize {
|
||||||
|
BankFieldsToSerialize {
|
||||||
|
blockhash_queue: &self.blockhash_queue,
|
||||||
|
ancestors: &self.ancestors,
|
||||||
|
hash: *self.hash.read().unwrap(),
|
||||||
|
parent_hash: self.parent_hash,
|
||||||
|
parent_slot: self.parent_slot,
|
||||||
|
hard_forks: &*self.hard_forks,
|
||||||
|
transaction_count: self.transaction_count.load(Ordering::Relaxed),
|
||||||
|
tick_height: self.tick_height.load(Ordering::Relaxed),
|
||||||
|
signature_count: self.signature_count.load(Ordering::Relaxed),
|
||||||
|
capitalization: self.capitalization.load(Ordering::Relaxed),
|
||||||
|
max_tick_height: self.max_tick_height,
|
||||||
|
hashes_per_tick: self.hashes_per_tick,
|
||||||
|
ticks_per_slot: self.ticks_per_slot,
|
||||||
|
ns_per_slot: self.ns_per_slot,
|
||||||
|
genesis_creation_time: self.genesis_creation_time,
|
||||||
|
slots_per_year: self.slots_per_year,
|
||||||
|
unused: self.unused,
|
||||||
|
slot: self.slot,
|
||||||
|
epoch: self.epoch,
|
||||||
|
block_height: self.block_height,
|
||||||
|
collector_id: self.collector_id,
|
||||||
|
collector_fees: self.collector_fees.load(Ordering::Relaxed),
|
||||||
|
fee_calculator: self.fee_calculator.clone(),
|
||||||
|
fee_rate_governor: self.fee_rate_governor.clone(),
|
||||||
|
collected_rent: self.collected_rent.load(Ordering::Relaxed),
|
||||||
|
rent_collector: self.rent_collector.clone(),
|
||||||
|
epoch_schedule: self.epoch_schedule,
|
||||||
|
inflation: *self.inflation.read().unwrap(),
|
||||||
|
stakes: &self.stakes,
|
||||||
|
epoch_stakes: &self.epoch_stakes,
|
||||||
|
is_delta: self.is_delta.load(Ordering::Relaxed),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn collector_id(&self) -> &Pubkey {
|
pub fn collector_id(&self) -> &Pubkey {
|
||||||
&self.collector_id
|
&self.collector_id
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,7 @@ use solana_sdk::{
|
||||||
rent::Rent, sysvar,
|
rent::Rent, sysvar,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Default, Serialize, Deserialize, Clone, AbiExample)]
|
#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Debug, AbiExample)]
|
||||||
pub struct RentCollector {
|
pub struct RentCollector {
|
||||||
pub epoch: Epoch,
|
pub epoch: Epoch,
|
||||||
pub epoch_schedule: EpochSchedule,
|
pub epoch_schedule: EpochSchedule,
|
||||||
|
|
|
@ -4,10 +4,17 @@ use {
|
||||||
accounts_db::{
|
accounts_db::{
|
||||||
AccountStorageEntry, AccountStorageStatus, AccountsDB, AppendVecId, BankHashInfo,
|
AccountStorageEntry, AccountStorageStatus, AccountsDB, AppendVecId, BankHashInfo,
|
||||||
},
|
},
|
||||||
|
accounts_index::Ancestors,
|
||||||
append_vec::AppendVec,
|
append_vec::AppendVec,
|
||||||
bank::BankRc,
|
bank::{Bank, BankFieldsToDeserialize, BankRc},
|
||||||
|
blockhash_queue::BlockhashQueue,
|
||||||
|
epoch_stakes::EpochStakes,
|
||||||
|
message_processor::MessageProcessor,
|
||||||
|
rent_collector::RentCollector,
|
||||||
|
stakes::Stakes,
|
||||||
},
|
},
|
||||||
bincode::{deserialize_from, serialize_into, Error},
|
bincode,
|
||||||
|
bincode::{config::Options, serialize_into, Error},
|
||||||
fs_extra::dir::CopyOptions,
|
fs_extra::dir::CopyOptions,
|
||||||
log::{info, warn},
|
log::{info, warn},
|
||||||
rand::{thread_rng, Rng},
|
rand::{thread_rng, Rng},
|
||||||
|
@ -15,7 +22,16 @@ use {
|
||||||
de::{DeserializeOwned, Visitor},
|
de::{DeserializeOwned, Visitor},
|
||||||
Deserialize, Deserializer, Serialize, Serializer,
|
Deserialize, Deserializer, Serialize, Serializer,
|
||||||
},
|
},
|
||||||
solana_sdk::clock::Slot,
|
solana_sdk::{
|
||||||
|
clock::{Epoch, Slot, UnixTimestamp},
|
||||||
|
epoch_schedule::EpochSchedule,
|
||||||
|
fee_calculator::{FeeCalculator, FeeRateGovernor},
|
||||||
|
genesis_config::GenesisConfig,
|
||||||
|
hard_forks::HardForks,
|
||||||
|
hash::Hash,
|
||||||
|
inflation::Inflation,
|
||||||
|
pubkey::Pubkey,
|
||||||
|
},
|
||||||
std::{
|
std::{
|
||||||
cmp::min,
|
cmp::min,
|
||||||
collections::HashMap,
|
collections::HashMap,
|
||||||
|
@ -28,6 +44,10 @@ use {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
||||||
|
use solana_sdk::abi_example::IgnoreAsHelper;
|
||||||
|
|
||||||
|
mod common;
|
||||||
mod future;
|
mod future;
|
||||||
mod legacy;
|
mod legacy;
|
||||||
mod tests;
|
mod tests;
|
||||||
|
@ -42,28 +62,28 @@ use utils::{serialize_iter_as_map, serialize_iter_as_seq, serialize_iter_as_tupl
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use self::tests::reconstruct_accounts_db_via_serialization;
|
pub(crate) use self::tests::reconstruct_accounts_db_via_serialization;
|
||||||
|
|
||||||
pub use crate::accounts_db::{SnapshotStorage, SnapshotStorages};
|
pub(crate) use crate::accounts_db::{SnapshotStorage, SnapshotStorages};
|
||||||
|
|
||||||
#[derive(Copy, Clone, Eq, PartialEq)]
|
#[derive(Copy, Clone, Eq, PartialEq)]
|
||||||
pub enum SerdeStyle {
|
pub(crate) enum SerdeStyle {
|
||||||
NEWER,
|
NEWER,
|
||||||
OLDER,
|
OLDER,
|
||||||
}
|
}
|
||||||
|
|
||||||
const MAX_ACCOUNTS_DB_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024;
|
const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024;
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
|
#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)]
|
||||||
pub struct AccountDBFields<T>(HashMap<Slot, Vec<T>>, u64, Slot, BankHashInfo);
|
struct AccountsDbFields<T>(HashMap<Slot, Vec<T>>, u64, Slot, BankHashInfo);
|
||||||
|
|
||||||
pub trait TypeContext<'a> {
|
trait TypeContext<'a> {
|
||||||
type SerializableAccountStorageEntry: Serialize
|
type SerializableAccountStorageEntry: Serialize
|
||||||
+ DeserializeOwned
|
+ DeserializeOwned
|
||||||
+ From<&'a AccountStorageEntry>
|
+ From<&'a AccountStorageEntry>
|
||||||
+ Into<AccountStorageEntry>;
|
+ Into<AccountStorageEntry>;
|
||||||
|
|
||||||
fn serialize_bank_rc_fields<S: serde::ser::Serializer>(
|
fn serialize_bank_and_storage<S: serde::ser::Serializer>(
|
||||||
serializer: S,
|
serializer: S,
|
||||||
serializable_bank: &SerializableBankRc<'a, Self>,
|
serializable_bank: &SerializableBankAndStorage<'a, Self>,
|
||||||
) -> std::result::Result<S::Ok, S::Error>
|
) -> std::result::Result<S::Ok, S::Error>
|
||||||
where
|
where
|
||||||
Self: std::marker::Sized;
|
Self: std::marker::Sized;
|
||||||
|
@ -75,37 +95,63 @@ pub trait TypeContext<'a> {
|
||||||
where
|
where
|
||||||
Self: std::marker::Sized;
|
Self: std::marker::Sized;
|
||||||
|
|
||||||
fn deserialize_accounts_db_fields<R>(
|
fn deserialize_bank_fields<R>(
|
||||||
stream: &mut BufReader<R>,
|
stream: &mut BufReader<R>,
|
||||||
) -> Result<AccountDBFields<Self::SerializableAccountStorageEntry>, Error>
|
) -> Result<
|
||||||
|
(
|
||||||
|
BankFieldsToDeserialize,
|
||||||
|
AccountsDbFields<Self::SerializableAccountStorageEntry>,
|
||||||
|
),
|
||||||
|
Error,
|
||||||
|
>
|
||||||
where
|
where
|
||||||
R: Read;
|
R: Read;
|
||||||
|
|
||||||
// we might define fn (de)serialize_bank(...) -> Result<Bank,...> for versionized bank serialization in the future
|
fn deserialize_accounts_db_fields<R>(
|
||||||
|
stream: &mut BufReader<R>,
|
||||||
|
) -> Result<AccountsDbFields<Self::SerializableAccountStorageEntry>, Error>
|
||||||
|
where
|
||||||
|
R: Read;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn bankrc_from_stream<R, P>(
|
fn deserialize_from<R, T>(reader: R) -> bincode::Result<T>
|
||||||
|
where
|
||||||
|
R: Read,
|
||||||
|
T: DeserializeOwned,
|
||||||
|
{
|
||||||
|
bincode::options()
|
||||||
|
.with_limit(MAX_STREAM_SIZE)
|
||||||
|
.with_fixint_encoding()
|
||||||
|
.allow_trailing_bytes()
|
||||||
|
.deserialize_from::<R, T>(reader)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn bank_from_stream<R, P>(
|
||||||
serde_style: SerdeStyle,
|
serde_style: SerdeStyle,
|
||||||
account_paths: &[PathBuf],
|
|
||||||
slot: Slot,
|
|
||||||
stream: &mut BufReader<R>,
|
stream: &mut BufReader<R>,
|
||||||
stream_append_vecs_path: P,
|
append_vecs_path: P,
|
||||||
) -> std::result::Result<BankRc, Error>
|
account_paths: &[PathBuf],
|
||||||
|
genesis_config: &GenesisConfig,
|
||||||
|
frozen_account_pubkeys: &[Pubkey],
|
||||||
|
) -> std::result::Result<Bank, Error>
|
||||||
where
|
where
|
||||||
R: Read,
|
R: Read,
|
||||||
P: AsRef<Path>,
|
P: AsRef<Path>,
|
||||||
{
|
{
|
||||||
macro_rules! INTO {
|
macro_rules! INTO {
|
||||||
($x:ident) => {
|
($x:ident) => {{
|
||||||
Ok(BankRc::new(
|
let (bank_fields, accounts_db_fields) = $x::deserialize_bank_fields(stream)?;
|
||||||
Accounts::new_empty(context_accountsdb_from_fields::<$x, P>(
|
|
||||||
$x::deserialize_accounts_db_fields(stream)?,
|
let bank = reconstruct_bank_from_fields(
|
||||||
account_paths,
|
bank_fields,
|
||||||
stream_append_vecs_path,
|
accounts_db_fields,
|
||||||
)?),
|
genesis_config,
|
||||||
slot,
|
frozen_account_pubkeys,
|
||||||
))
|
account_paths,
|
||||||
};
|
append_vecs_path,
|
||||||
|
)?;
|
||||||
|
Ok(bank)
|
||||||
|
}};
|
||||||
}
|
}
|
||||||
match serde_style {
|
match serde_style {
|
||||||
SerdeStyle::NEWER => INTO!(TypeContextFuture),
|
SerdeStyle::NEWER => INTO!(TypeContextFuture),
|
||||||
|
@ -117,10 +163,10 @@ where
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn bankrc_to_stream<W>(
|
pub(crate) fn bank_to_stream<W>(
|
||||||
serde_style: SerdeStyle,
|
serde_style: SerdeStyle,
|
||||||
stream: &mut BufWriter<W>,
|
stream: &mut BufWriter<W>,
|
||||||
bank_rc: &BankRc,
|
bank: &Bank,
|
||||||
snapshot_storages: &[SnapshotStorage],
|
snapshot_storages: &[SnapshotStorage],
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error>
|
||||||
where
|
where
|
||||||
|
@ -128,10 +174,10 @@ where
|
||||||
{
|
{
|
||||||
macro_rules! INTO {
|
macro_rules! INTO {
|
||||||
($x:ident) => {
|
($x:ident) => {
|
||||||
serialize_into(
|
bincode::serialize_into(
|
||||||
stream,
|
stream,
|
||||||
&SerializableBankRc::<$x> {
|
&SerializableBankAndStorage::<$x> {
|
||||||
bank_rc,
|
bank,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
phantom: std::marker::PhantomData::default(),
|
phantom: std::marker::PhantomData::default(),
|
||||||
},
|
},
|
||||||
|
@ -148,22 +194,22 @@ where
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct SerializableBankRc<'a, C> {
|
struct SerializableBankAndStorage<'a, C> {
|
||||||
bank_rc: &'a BankRc,
|
bank: &'a Bank,
|
||||||
snapshot_storages: &'a [SnapshotStorage],
|
snapshot_storages: &'a [SnapshotStorage],
|
||||||
phantom: std::marker::PhantomData<C>,
|
phantom: std::marker::PhantomData<C>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a, C: TypeContext<'a>> Serialize for SerializableBankRc<'a, C> {
|
impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorage<'a, C> {
|
||||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||||
where
|
where
|
||||||
S: serde::ser::Serializer,
|
S: serde::ser::Serializer,
|
||||||
{
|
{
|
||||||
C::serialize_bank_rc_fields(serializer, self)
|
C::serialize_bank_and_storage(serializer, self)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct SerializableAccountsDB<'a, C> {
|
struct SerializableAccountsDB<'a, C> {
|
||||||
accounts_db: &'a AccountsDB,
|
accounts_db: &'a AccountsDB,
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
account_storage_entries: &'a [SnapshotStorage],
|
account_storage_entries: &'a [SnapshotStorage],
|
||||||
|
@ -179,18 +225,43 @@ impl<'a, C: TypeContext<'a>> Serialize for SerializableAccountsDB<'a, C> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn context_accountsdb_from_fields<'a, C, P>(
|
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
||||||
account_db_fields: AccountDBFields<C::SerializableAccountStorageEntry>,
|
impl<'a, C> IgnoreAsHelper for SerializableAccountsDB<'a, C> {}
|
||||||
|
|
||||||
|
fn reconstruct_bank_from_fields<E, P>(
|
||||||
|
bank_fields: BankFieldsToDeserialize,
|
||||||
|
accounts_db_fields: AccountsDbFields<E>,
|
||||||
|
genesis_config: &GenesisConfig,
|
||||||
|
frozen_account_pubkeys: &[Pubkey],
|
||||||
|
account_paths: &[PathBuf],
|
||||||
|
append_vecs_path: P,
|
||||||
|
) -> Result<Bank, Error>
|
||||||
|
where
|
||||||
|
E: Into<AccountStorageEntry>,
|
||||||
|
P: AsRef<Path>,
|
||||||
|
{
|
||||||
|
let mut accounts_db =
|
||||||
|
reconstruct_accountsdb_from_fields(accounts_db_fields, account_paths, append_vecs_path)?;
|
||||||
|
accounts_db.freeze_accounts(&bank_fields.ancestors, frozen_account_pubkeys);
|
||||||
|
|
||||||
|
let bank_rc = BankRc::new(Accounts::new_empty(accounts_db), bank_fields.slot);
|
||||||
|
let bank = Bank::new_from_fields(bank_rc, genesis_config, bank_fields);
|
||||||
|
|
||||||
|
Ok(bank)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reconstruct_accountsdb_from_fields<E, P>(
|
||||||
|
accounts_db_fields: AccountsDbFields<E>,
|
||||||
account_paths: &[PathBuf],
|
account_paths: &[PathBuf],
|
||||||
stream_append_vecs_path: P,
|
stream_append_vecs_path: P,
|
||||||
) -> Result<AccountsDB, Error>
|
) -> Result<AccountsDB, Error>
|
||||||
where
|
where
|
||||||
C: TypeContext<'a>,
|
E: Into<AccountStorageEntry>,
|
||||||
P: AsRef<Path>,
|
P: AsRef<Path>,
|
||||||
{
|
{
|
||||||
let accounts_db = AccountsDB::new(account_paths.to_vec());
|
let accounts_db = AccountsDB::new(account_paths.to_vec());
|
||||||
|
|
||||||
let AccountDBFields(storage, version, slot, bank_hash_info) = account_db_fields;
|
let AccountsDbFields(storage, version, slot, bank_hash_info) = accounts_db_fields;
|
||||||
|
|
||||||
// convert to two level map of slot -> id -> account storage entry
|
// convert to two level map of slot -> id -> account storage entry
|
||||||
let storage = {
|
let storage = {
|
||||||
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
use super::*;
|
||||||
|
use std::collections::HashSet;
|
||||||
|
|
||||||
|
#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize, AbiExample)]
|
||||||
|
pub(crate) struct UnusedAccounts {
|
||||||
|
unused1: HashSet<Pubkey>,
|
||||||
|
unused2: HashSet<Pubkey>,
|
||||||
|
unused3: HashMap<Pubkey, u64>,
|
||||||
|
}
|
|
@ -1,5 +1,10 @@
|
||||||
|
use super::common::UnusedAccounts;
|
||||||
|
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
|
||||||
|
use solana_sdk::abi_example::IgnoreAsHelper;
|
||||||
use {super::*, solana_measure::measure::Measure, std::cell::RefCell};
|
use {super::*, solana_measure::measure::Measure, std::cell::RefCell};
|
||||||
|
|
||||||
|
type AccountsDbFields = super::AccountsDbFields<SerializableAccountStorageEntry>;
|
||||||
|
|
||||||
// Serializable version of AccountStorageEntry for snapshot format
|
// Serializable version of AccountStorageEntry for snapshot format
|
||||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||||
pub(super) struct SerializableAccountStorageEntry {
|
pub(super) struct SerializableAccountStorageEntry {
|
||||||
|
@ -25,25 +30,190 @@ impl Into<AccountStorageEntry> for SerializableAccountStorageEntry {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
use std::sync::RwLock;
|
||||||
|
// Deserializable version of Bank which need not be serializable,
|
||||||
|
// because it's handled by SerializableVersionedBank.
|
||||||
|
// So, sync fields with it!
|
||||||
|
#[derive(Clone, Deserialize)]
|
||||||
|
pub(crate) struct DeserializableVersionedBank {
|
||||||
|
pub(crate) blockhash_queue: BlockhashQueue,
|
||||||
|
pub(crate) ancestors: Ancestors,
|
||||||
|
pub(crate) hash: Hash,
|
||||||
|
pub(crate) parent_hash: Hash,
|
||||||
|
pub(crate) parent_slot: Slot,
|
||||||
|
pub(crate) hard_forks: HardForks,
|
||||||
|
pub(crate) transaction_count: u64,
|
||||||
|
pub(crate) tick_height: u64,
|
||||||
|
pub(crate) signature_count: u64,
|
||||||
|
pub(crate) capitalization: u64,
|
||||||
|
pub(crate) max_tick_height: u64,
|
||||||
|
pub(crate) hashes_per_tick: Option<u64>,
|
||||||
|
pub(crate) ticks_per_slot: u64,
|
||||||
|
pub(crate) ns_per_slot: u128,
|
||||||
|
pub(crate) genesis_creation_time: UnixTimestamp,
|
||||||
|
pub(crate) slots_per_year: f64,
|
||||||
|
pub(crate) unused: u64,
|
||||||
|
pub(crate) slot: Slot,
|
||||||
|
pub(crate) epoch: Epoch,
|
||||||
|
pub(crate) block_height: u64,
|
||||||
|
pub(crate) collector_id: Pubkey,
|
||||||
|
pub(crate) collector_fees: u64,
|
||||||
|
pub(crate) fee_calculator: FeeCalculator,
|
||||||
|
pub(crate) fee_rate_governor: FeeRateGovernor,
|
||||||
|
pub(crate) collected_rent: u64,
|
||||||
|
pub(crate) rent_collector: RentCollector,
|
||||||
|
pub(crate) epoch_schedule: EpochSchedule,
|
||||||
|
pub(crate) inflation: Inflation,
|
||||||
|
pub(crate) stakes: Stakes,
|
||||||
|
pub(crate) unused_accounts: UnusedAccounts,
|
||||||
|
pub(crate) epoch_stakes: HashMap<Epoch, EpochStakes>,
|
||||||
|
pub(crate) is_delta: bool,
|
||||||
|
pub(crate) message_processor: MessageProcessor,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Into<BankFieldsToDeserialize> for DeserializableVersionedBank {
|
||||||
|
fn into(self) -> BankFieldsToDeserialize {
|
||||||
|
BankFieldsToDeserialize {
|
||||||
|
blockhash_queue: self.blockhash_queue,
|
||||||
|
ancestors: self.ancestors,
|
||||||
|
hash: self.hash,
|
||||||
|
parent_hash: self.parent_hash,
|
||||||
|
parent_slot: self.parent_slot,
|
||||||
|
hard_forks: self.hard_forks,
|
||||||
|
transaction_count: self.transaction_count,
|
||||||
|
tick_height: self.tick_height,
|
||||||
|
signature_count: self.signature_count,
|
||||||
|
capitalization: self.capitalization,
|
||||||
|
max_tick_height: self.max_tick_height,
|
||||||
|
hashes_per_tick: self.hashes_per_tick,
|
||||||
|
ticks_per_slot: self.ticks_per_slot,
|
||||||
|
ns_per_slot: self.ns_per_slot,
|
||||||
|
genesis_creation_time: self.genesis_creation_time,
|
||||||
|
slots_per_year: self.slots_per_year,
|
||||||
|
unused: self.unused,
|
||||||
|
slot: self.slot,
|
||||||
|
epoch: self.epoch,
|
||||||
|
block_height: self.block_height,
|
||||||
|
collector_id: self.collector_id,
|
||||||
|
collector_fees: self.collector_fees,
|
||||||
|
fee_calculator: self.fee_calculator,
|
||||||
|
fee_rate_governor: self.fee_rate_governor,
|
||||||
|
collected_rent: self.collected_rent,
|
||||||
|
rent_collector: self.rent_collector,
|
||||||
|
epoch_schedule: self.epoch_schedule,
|
||||||
|
inflation: self.inflation,
|
||||||
|
stakes: self.stakes,
|
||||||
|
epoch_stakes: self.epoch_stakes,
|
||||||
|
is_delta: self.is_delta,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Serializable version of Bank, not Deserializable to avoid cloning by using refs.
|
||||||
|
// Sync fields with DeserializableVersionedBank!
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(crate) struct SerializableVersionedBank<'a> {
|
||||||
|
pub(crate) blockhash_queue: &'a RwLock<BlockhashQueue>,
|
||||||
|
pub(crate) ancestors: &'a Ancestors,
|
||||||
|
pub(crate) hash: Hash,
|
||||||
|
pub(crate) parent_hash: Hash,
|
||||||
|
pub(crate) parent_slot: Slot,
|
||||||
|
pub(crate) hard_forks: &'a RwLock<HardForks>,
|
||||||
|
pub(crate) transaction_count: u64,
|
||||||
|
pub(crate) tick_height: u64,
|
||||||
|
pub(crate) signature_count: u64,
|
||||||
|
pub(crate) capitalization: u64,
|
||||||
|
pub(crate) max_tick_height: u64,
|
||||||
|
pub(crate) hashes_per_tick: Option<u64>,
|
||||||
|
pub(crate) ticks_per_slot: u64,
|
||||||
|
pub(crate) ns_per_slot: u128,
|
||||||
|
pub(crate) genesis_creation_time: UnixTimestamp,
|
||||||
|
pub(crate) slots_per_year: f64,
|
||||||
|
pub(crate) unused: u64,
|
||||||
|
pub(crate) slot: Slot,
|
||||||
|
pub(crate) epoch: Epoch,
|
||||||
|
pub(crate) block_height: u64,
|
||||||
|
pub(crate) collector_id: Pubkey,
|
||||||
|
pub(crate) collector_fees: u64,
|
||||||
|
pub(crate) fee_calculator: FeeCalculator,
|
||||||
|
pub(crate) fee_rate_governor: FeeRateGovernor,
|
||||||
|
pub(crate) collected_rent: u64,
|
||||||
|
pub(crate) rent_collector: RentCollector,
|
||||||
|
pub(crate) epoch_schedule: EpochSchedule,
|
||||||
|
pub(crate) inflation: Inflation,
|
||||||
|
pub(crate) stakes: &'a RwLock<Stakes>,
|
||||||
|
pub(crate) unused_accounts: UnusedAccounts,
|
||||||
|
pub(crate) epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
|
||||||
|
pub(crate) is_delta: bool,
|
||||||
|
pub(crate) message_processor: MessageProcessor,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedBank<'a> {
|
||||||
|
fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self {
|
||||||
|
fn new<T: Default>() -> T {
|
||||||
|
T::default()
|
||||||
|
}
|
||||||
|
Self {
|
||||||
|
blockhash_queue: rhs.blockhash_queue,
|
||||||
|
ancestors: rhs.ancestors,
|
||||||
|
hash: rhs.hash,
|
||||||
|
parent_hash: rhs.parent_hash,
|
||||||
|
parent_slot: rhs.parent_slot,
|
||||||
|
hard_forks: rhs.hard_forks,
|
||||||
|
transaction_count: rhs.transaction_count,
|
||||||
|
tick_height: rhs.tick_height,
|
||||||
|
signature_count: rhs.signature_count,
|
||||||
|
capitalization: rhs.capitalization,
|
||||||
|
max_tick_height: rhs.max_tick_height,
|
||||||
|
hashes_per_tick: rhs.hashes_per_tick,
|
||||||
|
ticks_per_slot: rhs.ticks_per_slot,
|
||||||
|
ns_per_slot: rhs.ns_per_slot,
|
||||||
|
genesis_creation_time: rhs.genesis_creation_time,
|
||||||
|
slots_per_year: rhs.slots_per_year,
|
||||||
|
unused: rhs.unused,
|
||||||
|
slot: rhs.slot,
|
||||||
|
epoch: rhs.epoch,
|
||||||
|
block_height: rhs.block_height,
|
||||||
|
collector_id: rhs.collector_id,
|
||||||
|
collector_fees: rhs.collector_fees,
|
||||||
|
fee_calculator: rhs.fee_calculator,
|
||||||
|
fee_rate_governor: rhs.fee_rate_governor,
|
||||||
|
collected_rent: rhs.collected_rent,
|
||||||
|
rent_collector: rhs.rent_collector,
|
||||||
|
epoch_schedule: rhs.epoch_schedule,
|
||||||
|
inflation: rhs.inflation,
|
||||||
|
stakes: rhs.stakes,
|
||||||
|
unused_accounts: new(),
|
||||||
|
epoch_stakes: rhs.epoch_stakes,
|
||||||
|
is_delta: rhs.is_delta,
|
||||||
|
message_processor: new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
||||||
|
impl<'a> IgnoreAsHelper for SerializableVersionedBank<'a> {}
|
||||||
|
|
||||||
pub(super) struct Context {}
|
pub(super) struct Context {}
|
||||||
impl<'a> TypeContext<'a> for Context {
|
impl<'a> TypeContext<'a> for Context {
|
||||||
type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
|
type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
|
||||||
|
|
||||||
fn serialize_bank_rc_fields<S: serde::ser::Serializer>(
|
fn serialize_bank_and_storage<S: serde::ser::Serializer>(
|
||||||
serializer: S,
|
serializer: S,
|
||||||
serializable_bank: &SerializableBankRc<'a, Self>,
|
serializable_bank: &SerializableBankAndStorage<'a, Self>,
|
||||||
) -> std::result::Result<S::Ok, S::Error>
|
) -> std::result::Result<S::Ok, S::Error>
|
||||||
where
|
where
|
||||||
Self: std::marker::Sized,
|
Self: std::marker::Sized,
|
||||||
{
|
{
|
||||||
let accounts_db_serialize = SerializableAccountsDB::<'a, Self> {
|
(
|
||||||
accounts_db: &*serializable_bank.bank_rc.accounts.accounts_db,
|
SerializableVersionedBank::from(serializable_bank.bank.get_fields_to_serialize()),
|
||||||
slot: serializable_bank.bank_rc.slot,
|
SerializableAccountsDB::<'a, Self> {
|
||||||
account_storage_entries: serializable_bank.snapshot_storages,
|
accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db,
|
||||||
phantom: std::marker::PhantomData::default(),
|
slot: serializable_bank.bank.rc.slot,
|
||||||
};
|
account_storage_entries: serializable_bank.snapshot_storages,
|
||||||
|
phantom: std::marker::PhantomData::default(),
|
||||||
accounts_db_serialize.serialize(serializer)
|
},
|
||||||
|
)
|
||||||
|
.serialize(serializer)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn serialize_accounts_db_fields<S: serde::ser::Serializer>(
|
fn serialize_accounts_db_fields<S: serde::ser::Serializer>(
|
||||||
|
@ -93,12 +263,23 @@ impl<'a> TypeContext<'a> for Context {
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
fn deserialize_accounts_db_fields<R>(
|
fn deserialize_bank_fields<R>(
|
||||||
mut stream: &mut BufReader<R>,
|
mut stream: &mut BufReader<R>,
|
||||||
) -> Result<AccountDBFields<Self::SerializableAccountStorageEntry>, Error>
|
) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error>
|
||||||
where
|
where
|
||||||
R: Read,
|
R: Read,
|
||||||
{
|
{
|
||||||
deserialize_from(&mut stream)
|
let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into();
|
||||||
|
let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?;
|
||||||
|
Ok((bank_fields, accounts_db_fields))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn deserialize_accounts_db_fields<R>(
|
||||||
|
stream: &mut BufReader<R>,
|
||||||
|
) -> Result<AccountsDbFields, Error>
|
||||||
|
where
|
||||||
|
R: Read,
|
||||||
|
{
|
||||||
|
deserialize_from(stream)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,12 @@
|
||||||
|
use super::common::UnusedAccounts;
|
||||||
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
|
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
|
||||||
use solana_sdk::abi_example::IgnoreAsHelper;
|
use solana_sdk::abi_example::IgnoreAsHelper;
|
||||||
use {super::*, bincode::config::Options, solana_measure::measure::Measure, std::cell::RefCell};
|
use {super::*, bincode::config::Options, solana_measure::measure::Measure, std::cell::RefCell};
|
||||||
|
|
||||||
|
const MAX_ACCOUNTS_DB_STREAM_SIZE: u64 = MAX_STREAM_SIZE;
|
||||||
|
|
||||||
|
type AccountsDbFields = super::AccountsDbFields<SerializableAccountStorageEntry>;
|
||||||
|
|
||||||
// Serializable version of AccountStorageEntry for snapshot format
|
// Serializable version of AccountStorageEntry for snapshot format
|
||||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||||
pub(super) struct SerializableAccountStorageEntry {
|
pub(super) struct SerializableAccountStorageEntry {
|
||||||
|
@ -52,7 +57,7 @@ impl Into<AppendVec> for SerializableAppendVec {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Serialization of AppendVec requires serialization of u64 to
|
// Serialization of AppendVec requires serialization of u64 to
|
||||||
// eight byte vector which is then itself serialized to the stream
|
// eight byte vector which is then itself serialized to the stream
|
||||||
impl Serialize for SerializableAppendVec {
|
impl Serialize for SerializableAppendVec {
|
||||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||||
|
@ -67,7 +72,7 @@ impl Serialize for SerializableAppendVec {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Deserialization of AppendVec requires deserialization
|
// Deserialization of AppendVec requires deserialization
|
||||||
// of eight byte vector from which u64 is then deserialized
|
// of eight byte vector from which u64 is then deserialized
|
||||||
impl<'de> Deserialize<'de> for SerializableAppendVec {
|
impl<'de> Deserialize<'de> for SerializableAppendVec {
|
||||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||||
|
@ -99,27 +104,193 @@ impl<'de> Deserialize<'de> for SerializableAppendVec {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Deserializable version of Bank which need not be serializable,
|
||||||
|
// because it's handled by SerializableVersionedBank.
|
||||||
|
// So, sync fields with it!
|
||||||
|
#[derive(Clone, Deserialize)]
|
||||||
|
pub(crate) struct DeserializableVersionedBank {
|
||||||
|
pub(crate) blockhash_queue: BlockhashQueue,
|
||||||
|
pub(crate) ancestors: Ancestors,
|
||||||
|
pub(crate) hash: Hash,
|
||||||
|
pub(crate) parent_hash: Hash,
|
||||||
|
pub(crate) parent_slot: Slot,
|
||||||
|
pub(crate) hard_forks: HardForks,
|
||||||
|
pub(crate) transaction_count: u64,
|
||||||
|
pub(crate) tick_height: u64,
|
||||||
|
pub(crate) signature_count: u64,
|
||||||
|
pub(crate) capitalization: u64,
|
||||||
|
pub(crate) max_tick_height: u64,
|
||||||
|
pub(crate) hashes_per_tick: Option<u64>,
|
||||||
|
pub(crate) ticks_per_slot: u64,
|
||||||
|
pub(crate) ns_per_slot: u128,
|
||||||
|
pub(crate) genesis_creation_time: UnixTimestamp,
|
||||||
|
pub(crate) slots_per_year: f64,
|
||||||
|
pub(crate) unused: u64,
|
||||||
|
pub(crate) slot: Slot,
|
||||||
|
pub(crate) epoch: Epoch,
|
||||||
|
pub(crate) block_height: u64,
|
||||||
|
pub(crate) collector_id: Pubkey,
|
||||||
|
pub(crate) collector_fees: u64,
|
||||||
|
pub(crate) fee_calculator: FeeCalculator,
|
||||||
|
pub(crate) fee_rate_governor: FeeRateGovernor,
|
||||||
|
pub(crate) collected_rent: u64,
|
||||||
|
pub(crate) rent_collector: RentCollector,
|
||||||
|
pub(crate) epoch_schedule: EpochSchedule,
|
||||||
|
pub(crate) inflation: Inflation,
|
||||||
|
pub(crate) stakes: Stakes,
|
||||||
|
pub(crate) unused_accounts: UnusedAccounts,
|
||||||
|
pub(crate) epoch_stakes: HashMap<Epoch, EpochStakes>,
|
||||||
|
pub(crate) is_delta: bool,
|
||||||
|
pub(crate) message_processor: MessageProcessor,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Into<BankFieldsToDeserialize> for DeserializableVersionedBank {
|
||||||
|
fn into(self) -> BankFieldsToDeserialize {
|
||||||
|
BankFieldsToDeserialize {
|
||||||
|
blockhash_queue: self.blockhash_queue,
|
||||||
|
ancestors: self.ancestors,
|
||||||
|
hash: self.hash,
|
||||||
|
parent_hash: self.parent_hash,
|
||||||
|
parent_slot: self.parent_slot,
|
||||||
|
hard_forks: self.hard_forks,
|
||||||
|
transaction_count: self.transaction_count,
|
||||||
|
tick_height: self.tick_height,
|
||||||
|
signature_count: self.signature_count,
|
||||||
|
capitalization: self.capitalization,
|
||||||
|
max_tick_height: self.max_tick_height,
|
||||||
|
hashes_per_tick: self.hashes_per_tick,
|
||||||
|
ticks_per_slot: self.ticks_per_slot,
|
||||||
|
ns_per_slot: self.ns_per_slot,
|
||||||
|
genesis_creation_time: self.genesis_creation_time,
|
||||||
|
slots_per_year: self.slots_per_year,
|
||||||
|
unused: self.unused,
|
||||||
|
slot: self.slot,
|
||||||
|
epoch: self.epoch,
|
||||||
|
block_height: self.block_height,
|
||||||
|
collector_id: self.collector_id,
|
||||||
|
collector_fees: self.collector_fees,
|
||||||
|
fee_calculator: self.fee_calculator,
|
||||||
|
fee_rate_governor: self.fee_rate_governor,
|
||||||
|
collected_rent: self.collected_rent,
|
||||||
|
rent_collector: self.rent_collector,
|
||||||
|
epoch_schedule: self.epoch_schedule,
|
||||||
|
inflation: self.inflation,
|
||||||
|
stakes: self.stakes,
|
||||||
|
epoch_stakes: self.epoch_stakes,
|
||||||
|
is_delta: self.is_delta,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Serializable version of Bank, not Deserializable to avoid cloning by using refs.
|
||||||
|
// Sync fields with DeserializableVersionedBank!
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(crate) struct SerializableVersionedBank<'a> {
|
||||||
|
pub(crate) blockhash_queue: &'a RwLock<BlockhashQueue>,
|
||||||
|
pub(crate) ancestors: &'a Ancestors,
|
||||||
|
pub(crate) hash: Hash,
|
||||||
|
pub(crate) parent_hash: Hash,
|
||||||
|
pub(crate) parent_slot: Slot,
|
||||||
|
pub(crate) hard_forks: &'a RwLock<HardForks>,
|
||||||
|
pub(crate) transaction_count: u64,
|
||||||
|
pub(crate) tick_height: u64,
|
||||||
|
pub(crate) signature_count: u64,
|
||||||
|
pub(crate) capitalization: u64,
|
||||||
|
pub(crate) max_tick_height: u64,
|
||||||
|
pub(crate) hashes_per_tick: Option<u64>,
|
||||||
|
pub(crate) ticks_per_slot: u64,
|
||||||
|
pub(crate) ns_per_slot: u128,
|
||||||
|
pub(crate) genesis_creation_time: UnixTimestamp,
|
||||||
|
pub(crate) slots_per_year: f64,
|
||||||
|
pub(crate) unused: u64,
|
||||||
|
pub(crate) slot: Slot,
|
||||||
|
pub(crate) epoch: Epoch,
|
||||||
|
pub(crate) block_height: u64,
|
||||||
|
pub(crate) collector_id: Pubkey,
|
||||||
|
pub(crate) collector_fees: u64,
|
||||||
|
pub(crate) fee_calculator: FeeCalculator,
|
||||||
|
pub(crate) fee_rate_governor: FeeRateGovernor,
|
||||||
|
pub(crate) collected_rent: u64,
|
||||||
|
pub(crate) rent_collector: RentCollector,
|
||||||
|
pub(crate) epoch_schedule: EpochSchedule,
|
||||||
|
pub(crate) inflation: Inflation,
|
||||||
|
pub(crate) stakes: &'a RwLock<Stakes>,
|
||||||
|
pub(crate) unused_accounts: UnusedAccounts,
|
||||||
|
pub(crate) epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
|
||||||
|
pub(crate) is_delta: bool,
|
||||||
|
pub(crate) message_processor: MessageProcessor,
|
||||||
|
}
|
||||||
|
|
||||||
|
use std::sync::RwLock;
|
||||||
|
|
||||||
|
impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedBank<'a> {
|
||||||
|
fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self {
|
||||||
|
fn new<T: Default>() -> T {
|
||||||
|
T::default()
|
||||||
|
}
|
||||||
|
Self {
|
||||||
|
blockhash_queue: rhs.blockhash_queue,
|
||||||
|
ancestors: rhs.ancestors,
|
||||||
|
hash: rhs.hash,
|
||||||
|
parent_hash: rhs.parent_hash,
|
||||||
|
parent_slot: rhs.parent_slot,
|
||||||
|
hard_forks: rhs.hard_forks,
|
||||||
|
transaction_count: rhs.transaction_count,
|
||||||
|
tick_height: rhs.tick_height,
|
||||||
|
signature_count: rhs.signature_count,
|
||||||
|
capitalization: rhs.capitalization,
|
||||||
|
max_tick_height: rhs.max_tick_height,
|
||||||
|
hashes_per_tick: rhs.hashes_per_tick,
|
||||||
|
ticks_per_slot: rhs.ticks_per_slot,
|
||||||
|
ns_per_slot: rhs.ns_per_slot,
|
||||||
|
genesis_creation_time: rhs.genesis_creation_time,
|
||||||
|
slots_per_year: rhs.slots_per_year,
|
||||||
|
unused: rhs.unused,
|
||||||
|
slot: rhs.slot,
|
||||||
|
epoch: rhs.epoch,
|
||||||
|
block_height: rhs.block_height,
|
||||||
|
collector_id: rhs.collector_id,
|
||||||
|
collector_fees: rhs.collector_fees,
|
||||||
|
fee_calculator: rhs.fee_calculator,
|
||||||
|
fee_rate_governor: rhs.fee_rate_governor,
|
||||||
|
collected_rent: rhs.collected_rent,
|
||||||
|
rent_collector: rhs.rent_collector,
|
||||||
|
epoch_schedule: rhs.epoch_schedule,
|
||||||
|
inflation: rhs.inflation,
|
||||||
|
stakes: rhs.stakes,
|
||||||
|
unused_accounts: new(),
|
||||||
|
epoch_stakes: rhs.epoch_stakes,
|
||||||
|
is_delta: rhs.is_delta,
|
||||||
|
message_processor: new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
||||||
|
impl<'a> IgnoreAsHelper for SerializableVersionedBank<'a> {}
|
||||||
|
|
||||||
pub(super) struct Context {}
|
pub(super) struct Context {}
|
||||||
impl<'a> TypeContext<'a> for Context {
|
impl<'a> TypeContext<'a> for Context {
|
||||||
type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
|
type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
|
||||||
|
|
||||||
fn serialize_bank_rc_fields<S: serde::ser::Serializer>(
|
fn serialize_bank_and_storage<S: serde::ser::Serializer>(
|
||||||
serializer: S,
|
serializer: S,
|
||||||
serializable_bank: &SerializableBankRc<'a, Self>,
|
serializable_bank_and_storage: &SerializableBankAndStorage<'a, Self>,
|
||||||
) -> std::result::Result<S::Ok, S::Error>
|
) -> std::result::Result<S::Ok, S::Error>
|
||||||
where
|
where
|
||||||
Self: std::marker::Sized,
|
Self: std::marker::Sized,
|
||||||
{
|
{
|
||||||
// as there is no deserialize_bank_rc_fields(), do not emit the u64
|
(
|
||||||
// size field here and have serialize_accounts_db_fields() emit two
|
SerializableVersionedBank::from(
|
||||||
// u64 size fields instead
|
serializable_bank_and_storage.bank.get_fields_to_serialize(),
|
||||||
SerializableAccountsDB::<'a, Self> {
|
),
|
||||||
accounts_db: &*serializable_bank.bank_rc.accounts.accounts_db,
|
SerializableAccountsDB::<'a, Self> {
|
||||||
slot: serializable_bank.bank_rc.slot,
|
accounts_db: &*serializable_bank_and_storage.bank.rc.accounts.accounts_db,
|
||||||
account_storage_entries: serializable_bank.snapshot_storages,
|
slot: serializable_bank_and_storage.bank.rc.slot,
|
||||||
phantom: std::marker::PhantomData::default(),
|
account_storage_entries: serializable_bank_and_storage.snapshot_storages,
|
||||||
}
|
phantom: std::marker::PhantomData::default(),
|
||||||
.serialize(serializer)
|
},
|
||||||
|
)
|
||||||
|
.serialize(serializer)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn serialize_accounts_db_fields<S: serde::ser::Serializer>(
|
fn serialize_accounts_db_fields<S: serde::ser::Serializer>(
|
||||||
|
@ -161,6 +332,7 @@ impl<'a> TypeContext<'a> for Context {
|
||||||
.clone(),
|
.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// as there is no deserialize_bank_rc_fields(), emit two u64 size fields here instead
|
||||||
let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms");
|
let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms");
|
||||||
let result = (
|
let result = (
|
||||||
&MAX_ACCOUNTS_DB_STREAM_SIZE,
|
&MAX_ACCOUNTS_DB_STREAM_SIZE,
|
||||||
|
@ -179,9 +351,20 @@ impl<'a> TypeContext<'a> for Context {
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn deserialize_bank_fields<R>(
|
||||||
|
mut stream: &mut BufReader<R>,
|
||||||
|
) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error>
|
||||||
|
where
|
||||||
|
R: Read,
|
||||||
|
{
|
||||||
|
let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into();
|
||||||
|
let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?;
|
||||||
|
Ok((bank_fields, accounts_db_fields))
|
||||||
|
}
|
||||||
|
|
||||||
fn deserialize_accounts_db_fields<R>(
|
fn deserialize_accounts_db_fields<R>(
|
||||||
mut stream: &mut BufReader<R>,
|
mut stream: &mut BufReader<R>,
|
||||||
) -> Result<AccountDBFields<Self::SerializableAccountStorageEntry>, Error>
|
) -> Result<AccountsDbFields, Error>
|
||||||
where
|
where
|
||||||
R: Read,
|
R: Read,
|
||||||
{
|
{
|
||||||
|
@ -203,6 +386,6 @@ impl<'a> TypeContext<'a> for Context {
|
||||||
// (3rd of 3 elements) read in (slot, bank hashes) pair
|
// (3rd of 3 elements) read in (slot, bank hashes) pair
|
||||||
let (slot, bank_hash_info): (Slot, BankHashInfo) = deserialize_from(&mut stream)?;
|
let (slot, bank_hash_info): (Slot, BankHashInfo) = deserialize_from(&mut stream)?;
|
||||||
|
|
||||||
Ok(AccountDBFields(storage, version, slot, bank_hash_info))
|
Ok(AccountsDbFields(storage, version, slot, bank_hash_info))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -64,7 +64,7 @@ where
|
||||||
P: AsRef<Path>,
|
P: AsRef<Path>,
|
||||||
{
|
{
|
||||||
// read and deserialise the accounts database directly from the stream
|
// read and deserialise the accounts database directly from the stream
|
||||||
context_accountsdb_from_fields::<C, P>(
|
reconstruct_accountsdb_from_fields(
|
||||||
C::deserialize_accounts_db_fields(stream)?,
|
C::deserialize_accounts_db_fields(stream)?,
|
||||||
account_paths,
|
account_paths,
|
||||||
stream_append_vecs_path,
|
stream_append_vecs_path,
|
||||||
|
@ -198,17 +198,15 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) {
|
||||||
let snapshot_storages = bank2.get_snapshot_storages();
|
let snapshot_storages = bank2.get_snapshot_storages();
|
||||||
let mut buf = vec![];
|
let mut buf = vec![];
|
||||||
let mut writer = Cursor::new(&mut buf);
|
let mut writer = Cursor::new(&mut buf);
|
||||||
serialize_into(&mut writer, &bank2).unwrap();
|
crate::serde_snapshot::bank_to_stream(
|
||||||
crate::serde_snapshot::bankrc_to_stream(
|
|
||||||
serde_style,
|
serde_style,
|
||||||
&mut std::io::BufWriter::new(&mut writer),
|
&mut std::io::BufWriter::new(&mut writer),
|
||||||
&bank2.rc,
|
&bank2,
|
||||||
&snapshot_storages,
|
&snapshot_storages,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let mut rdr = Cursor::new(&buf[..]);
|
let rdr = Cursor::new(&buf[..]);
|
||||||
let mut dbank: Bank = bincode::deserialize_from(&mut rdr).unwrap();
|
|
||||||
let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]);
|
let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]);
|
||||||
|
|
||||||
// Create a new set of directories for this bank's accounts
|
// Create a new set of directories for this bank's accounts
|
||||||
|
@ -218,17 +216,16 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) {
|
||||||
// Create a directory to simulate AppendVecs unpackaged from a snapshot tar
|
// Create a directory to simulate AppendVecs unpackaged from a snapshot tar
|
||||||
let copied_accounts = TempDir::new().unwrap();
|
let copied_accounts = TempDir::new().unwrap();
|
||||||
copy_append_vecs(&bank2.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
|
copy_append_vecs(&bank2.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
|
||||||
dbank.set_bank_rc(
|
let mut dbank = crate::serde_snapshot::bank_from_stream(
|
||||||
crate::serde_snapshot::bankrc_from_stream(
|
serde_style,
|
||||||
serde_style,
|
&mut reader,
|
||||||
&dbank_paths,
|
copied_accounts.path(),
|
||||||
dbank.slot(),
|
&dbank_paths,
|
||||||
&mut reader,
|
&genesis_config,
|
||||||
copied_accounts.path(),
|
&[],
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap();
|
||||||
ref_sc,
|
dbank.src = ref_sc;
|
||||||
);
|
|
||||||
assert_eq!(dbank.get_balance(&key1.pubkey()), 0);
|
assert_eq!(dbank.get_balance(&key1.pubkey()), 0);
|
||||||
assert_eq!(dbank.get_balance(&key2.pubkey()), 10);
|
assert_eq!(dbank.get_balance(&key2.pubkey()), 10);
|
||||||
assert_eq!(dbank.get_balance(&key3.pubkey()), 0);
|
assert_eq!(dbank.get_balance(&key3.pubkey()), 0);
|
||||||
|
@ -280,45 +277,58 @@ fn test_bank_serialize_older() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
|
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
|
||||||
mod test_bank_rc_serialize {
|
mod test_bank_serialize {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use solana_sdk::abi_example::AbiExample;
|
||||||
|
|
||||||
// These some what long test harness is required to freeze the ABI of
|
// These some what long test harness is required to freeze the ABI of
|
||||||
// BankRc's serialization due to versioned nature
|
// Bank's serialization due to versioned nature
|
||||||
#[frozen_abi(digest = "HfCP74JKqPdeAccNJEj7KEoNxtsmX3zRqc2rpTy1NC7H")]
|
#[frozen_abi(digest = "9BGkhttaVsELn1zoHMKXLvi3Qty51nY1yz584Fao2Ev9")]
|
||||||
#[derive(Serialize, AbiExample)]
|
#[derive(Default, Serialize)]
|
||||||
pub struct BandRcAbiTestWrapperFuture {
|
pub struct BankAbiTestWrapperFuture {
|
||||||
#[serde(serialize_with = "wrapper_future")]
|
#[serde(serialize_with = "wrapper_future")]
|
||||||
bank_rc: BankRc,
|
bank: Bank,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn wrapper_future<S>(bank_rc: &BankRc, s: S) -> std::result::Result<S::Ok, S::Error>
|
impl AbiExample for BankAbiTestWrapperFuture {
|
||||||
|
fn example() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn wrapper_future<S>(bank: &Bank, s: S) -> std::result::Result<S::Ok, S::Error>
|
||||||
where
|
where
|
||||||
S: serde::Serializer,
|
S: serde::Serializer,
|
||||||
{
|
{
|
||||||
let snapshot_storages = bank_rc.accounts.accounts_db.get_snapshot_storages(0);
|
let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0);
|
||||||
(SerializableBankRc::<future::Context> {
|
(SerializableBankAndStorage::<future::Context> {
|
||||||
bank_rc,
|
bank,
|
||||||
snapshot_storages: &snapshot_storages,
|
snapshot_storages: &snapshot_storages,
|
||||||
phantom: std::marker::PhantomData::default(),
|
phantom: std::marker::PhantomData::default(),
|
||||||
})
|
})
|
||||||
.serialize(s)
|
.serialize(s)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[frozen_abi(digest = "43niyekyWwreLALcdEeFFpd7h8U6pgSXGqfKBRw8H7Vy")]
|
#[frozen_abi(digest = "HYmXta1fhiHe6hZLGJriMqKx9N2U8YRJY51AGZmxzM5m")]
|
||||||
#[derive(Serialize, AbiExample)]
|
#[derive(Default, Serialize)]
|
||||||
pub struct BandRcAbiTestWrapperLegacy {
|
pub struct BankAbiTestWrapperLegacy {
|
||||||
#[serde(serialize_with = "wrapper_legacy")]
|
#[serde(serialize_with = "wrapper_legacy")]
|
||||||
bank_rc: BankRc,
|
bank: Bank,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn wrapper_legacy<S>(bank_rc: &BankRc, s: S) -> std::result::Result<S::Ok, S::Error>
|
impl AbiExample for BankAbiTestWrapperLegacy {
|
||||||
|
fn example() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn wrapper_legacy<S>(bank: &Bank, s: S) -> std::result::Result<S::Ok, S::Error>
|
||||||
where
|
where
|
||||||
S: serde::Serializer,
|
S: serde::Serializer,
|
||||||
{
|
{
|
||||||
let snapshot_storages = bank_rc.accounts.accounts_db.get_snapshot_storages(0);
|
let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0);
|
||||||
(SerializableBankRc::<legacy::Context> {
|
(SerializableBankAndStorage::<legacy::Context> {
|
||||||
bank_rc,
|
bank,
|
||||||
snapshot_storages: &snapshot_storages,
|
snapshot_storages: &snapshot_storages,
|
||||||
phantom: std::marker::PhantomData::default(),
|
phantom: std::marker::PhantomData::default(),
|
||||||
})
|
})
|
||||||
|
|
|
@ -3,7 +3,7 @@ use crate::{
|
||||||
bank_forks::CompressionType,
|
bank_forks::CompressionType,
|
||||||
hardened_unpack::{unpack_snapshot, UnpackError},
|
hardened_unpack::{unpack_snapshot, UnpackError},
|
||||||
serde_snapshot::{
|
serde_snapshot::{
|
||||||
bankrc_from_stream, bankrc_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages,
|
bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages,
|
||||||
},
|
},
|
||||||
snapshot_package::AccountsPackage,
|
snapshot_package::AccountsPackage,
|
||||||
};
|
};
|
||||||
|
@ -23,7 +23,6 @@ use std::{
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
process::ExitStatus,
|
process::ExitStatus,
|
||||||
str::FromStr,
|
str::FromStr,
|
||||||
sync::Arc,
|
|
||||||
};
|
};
|
||||||
use tar::Archive;
|
use tar::Archive;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
@ -462,8 +461,7 @@ pub fn add_snapshot<P: AsRef<Path>>(
|
||||||
SnapshotVersion::V1_1_0 => SerdeStyle::OLDER,
|
SnapshotVersion::V1_1_0 => SerdeStyle::OLDER,
|
||||||
SnapshotVersion::V1_2_0 => SerdeStyle::NEWER,
|
SnapshotVersion::V1_2_0 => SerdeStyle::NEWER,
|
||||||
};
|
};
|
||||||
serialize_into(stream.by_ref(), bank)?;
|
bank_to_stream(serde_style, stream.by_ref(), bank, snapshot_storages)?;
|
||||||
bankrc_to_stream(serde_style, stream.by_ref(), &bank.rc, snapshot_storages)?;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
};
|
};
|
||||||
let consumed_size =
|
let consumed_size =
|
||||||
|
@ -723,38 +721,24 @@ where
|
||||||
|
|
||||||
info!("Loading bank from {:?}", &root_paths.snapshot_file_path);
|
info!("Loading bank from {:?}", &root_paths.snapshot_file_path);
|
||||||
let bank = deserialize_snapshot_data_file(&root_paths.snapshot_file_path, |mut stream| {
|
let bank = deserialize_snapshot_data_file(&root_paths.snapshot_file_path, |mut stream| {
|
||||||
let mut bank: Bank = bincode::options()
|
Ok(match snapshot_version_enum {
|
||||||
.with_limit(MAX_SNAPSHOT_DATA_FILE_SIZE)
|
SnapshotVersion::V1_1_0 => bank_from_stream(
|
||||||
.with_fixint_encoding()
|
|
||||||
.allow_trailing_bytes()
|
|
||||||
.deserialize_from(&mut stream)?;
|
|
||||||
|
|
||||||
info!("Rebuilding accounts...");
|
|
||||||
|
|
||||||
let mut bankrc = match snapshot_version_enum {
|
|
||||||
SnapshotVersion::V1_1_0 => bankrc_from_stream(
|
|
||||||
SerdeStyle::OLDER,
|
SerdeStyle::OLDER,
|
||||||
account_paths,
|
|
||||||
bank.slot(),
|
|
||||||
&mut stream,
|
&mut stream,
|
||||||
&append_vecs_path,
|
&append_vecs_path,
|
||||||
|
account_paths,
|
||||||
|
genesis_config,
|
||||||
|
frozen_account_pubkeys,
|
||||||
),
|
),
|
||||||
SnapshotVersion::V1_2_0 => bankrc_from_stream(
|
SnapshotVersion::V1_2_0 => bank_from_stream(
|
||||||
SerdeStyle::NEWER,
|
SerdeStyle::NEWER,
|
||||||
account_paths,
|
|
||||||
bank.slot(),
|
|
||||||
&mut stream,
|
&mut stream,
|
||||||
&append_vecs_path,
|
&append_vecs_path,
|
||||||
|
account_paths,
|
||||||
|
genesis_config,
|
||||||
|
frozen_account_pubkeys,
|
||||||
),
|
),
|
||||||
}?;
|
}?)
|
||||||
Arc::get_mut(&mut Arc::get_mut(&mut bankrc.accounts).unwrap().accounts_db)
|
|
||||||
.unwrap()
|
|
||||||
.freeze_accounts(&bank.ancestors, frozen_account_pubkeys);
|
|
||||||
|
|
||||||
bank.rc = bankrc;
|
|
||||||
bank.operating_mode = Some(genesis_config.operating_mode);
|
|
||||||
bank.finish_init();
|
|
||||||
Ok(bank)
|
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let status_cache_path = unpacked_snapshots_dir.join(SNAPSHOT_STATUS_CACHE_FILE_NAME);
|
let status_cache_path = unpacked_snapshots_dir.join(SNAPSHOT_STATUS_CACHE_FILE_NAME);
|
||||||
|
|
|
@ -78,8 +78,11 @@ fn quote_for_specialization_detection() -> TokenStream2 {
|
||||||
std::sync::atomic::AtomicBool::new(false);
|
std::sync::atomic::AtomicBool::new(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
if !SPECIALIZATION_DETECTOR_INJECTED.load(std::sync::atomic::Ordering::Relaxed) {
|
if !SPECIALIZATION_DETECTOR_INJECTED.compare_and_swap(
|
||||||
SPECIALIZATION_DETECTOR_INJECTED.store(true, std::sync::atomic::Ordering::Relaxed);
|
false,
|
||||||
|
true,
|
||||||
|
std::sync::atomic::Ordering::AcqRel,
|
||||||
|
) {
|
||||||
quote! {
|
quote! {
|
||||||
mod specialization_detector {
|
mod specialization_detector {
|
||||||
trait SpecializedTrait {
|
trait SpecializedTrait {
|
||||||
|
|
Loading…
Reference in New Issue