From 1d40685b8e794cc601199c4389c96abaa023e6f4 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Mon, 3 Aug 2020 16:27:17 -0700 Subject: [PATCH] Remove support for 1.1.0 version snapshots --- core/tests/bank_forks.rs | 1 - runtime/src/serde_snapshot.rs | 20 +- runtime/src/serde_snapshot/legacy.rs | 391 --------------------------- runtime/src/serde_snapshot/tests.rs | 48 +--- runtime/src/snapshot_utils.rs | 13 - validator/src/main.rs | 8 +- 6 files changed, 9 insertions(+), 472 deletions(-) delete mode 100644 runtime/src/serde_snapshot/legacy.rs diff --git a/core/tests/bank_forks.rs b/core/tests/bank_forks.rs index 4ffb56d382..3332d7f5c0 100644 --- a/core/tests/bank_forks.rs +++ b/core/tests/bank_forks.rs @@ -58,7 +58,6 @@ mod tests { use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc}; use tempfile::TempDir; - DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0); DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0); struct SnapshotTestConfig { diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index fac72d12d1..5a40685a8a 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -1,9 +1,7 @@ use { crate::{ accounts::Accounts, - accounts_db::{ - AccountStorageEntry, AccountStorageStatus, AccountsDB, AppendVecId, BankHashInfo, - }, + accounts_db::{AccountStorageEntry, AccountsDB, AppendVecId, BankHashInfo}, accounts_index::Ancestors, append_vec::AppendVec, bank::{Bank, BankFieldsToDeserialize, BankRc}, @@ -14,14 +12,11 @@ use { stakes::Stakes, }, bincode, - bincode::{config::Options, serialize_into, Error}, + bincode::{config::Options, Error}, fs_extra::dir::CopyOptions, log::{info, warn}, rand::{thread_rng, Rng}, - serde::{ - de::{DeserializeOwned, Visitor}, - Deserialize, Deserializer, Serialize, Serializer, - }, + serde::{de::DeserializeOwned, Deserialize, Serialize}, solana_sdk::{ clock::{Epoch, Slot, UnixTimestamp}, epoch_schedule::EpochSchedule, @@ -33,10 +28,8 @@ use { pubkey::Pubkey, }, std::{ - cmp::min, collections::HashMap, - fmt::{Formatter, Result as FormatResult}, - io::{BufReader, BufWriter, Cursor, Read, Write}, + io::{BufReader, BufWriter, Read, Write}, path::{Path, PathBuf}, result::Result, sync::{atomic::Ordering, Arc}, @@ -49,12 +42,10 @@ use solana_sdk::abi_example::IgnoreAsHelper; mod common; mod future; -mod legacy; mod tests; mod utils; use future::Context as TypeContextFuture; -use legacy::Context as TypeContextLegacy; #[allow(unused_imports)] use utils::{serialize_iter_as_map, serialize_iter_as_seq, serialize_iter_as_tuple}; @@ -67,7 +58,6 @@ pub(crate) use crate::accounts_db::{SnapshotStorage, SnapshotStorages}; #[derive(Copy, Clone, Eq, PartialEq)] pub(crate) enum SerdeStyle { NEWER, - OLDER, } const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024; @@ -155,7 +145,6 @@ where } match serde_style { SerdeStyle::NEWER => INTO!(TypeContextFuture), - SerdeStyle::OLDER => INTO!(TypeContextLegacy), } .map_err(|err| { warn!("bankrc_from_stream error: {:?}", err); @@ -186,7 +175,6 @@ where } match serde_style { SerdeStyle::NEWER => INTO!(TypeContextFuture), - SerdeStyle::OLDER => INTO!(TypeContextLegacy), } .map_err(|err| { warn!("bankrc_to_stream error: {:?}", err); diff --git a/runtime/src/serde_snapshot/legacy.rs b/runtime/src/serde_snapshot/legacy.rs deleted file mode 100644 index 226ec44119..0000000000 --- a/runtime/src/serde_snapshot/legacy.rs +++ /dev/null @@ -1,391 +0,0 @@ -use super::common::UnusedAccounts; -#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] -use solana_sdk::abi_example::IgnoreAsHelper; -use {super::*, bincode::config::Options, solana_measure::measure::Measure, std::cell::RefCell}; - -const MAX_ACCOUNTS_DB_STREAM_SIZE: u64 = MAX_STREAM_SIZE; - -type AccountsDbFields = super::AccountsDbFields; - -// Serializable version of AccountStorageEntry for snapshot format -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -pub(super) struct SerializableAccountStorageEntry { - id: AppendVecId, - accounts: SerializableAppendVec, - count_and_status: (usize, AccountStorageStatus), -} - -#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] -impl IgnoreAsHelper for SerializableAccountStorageEntry {} - -impl From<&AccountStorageEntry> for SerializableAccountStorageEntry { - fn from(rhs: &AccountStorageEntry) -> Self { - Self { - id: rhs.id, - accounts: SerializableAppendVec::from(&rhs.accounts), - ..Self::default() - } - } -} - -impl Into for SerializableAccountStorageEntry { - fn into(self) -> AccountStorageEntry { - AccountStorageEntry::new_empty_map(self.id, self.accounts.current_len) - } -} - -// Serializable version of AppendVec for snapshot format -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -struct SerializableAppendVec { - current_len: usize, -} - -#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] -impl IgnoreAsHelper for SerializableAppendVec {} - -impl From<&AppendVec> for SerializableAppendVec { - fn from(rhs: &AppendVec) -> SerializableAppendVec { - SerializableAppendVec { - current_len: rhs.len(), - } - } -} - -impl Into for SerializableAppendVec { - fn into(self) -> AppendVec { - AppendVec::new_empty_map(self.current_len) - } -} - -// Serialization of AppendVec requires serialization of u64 to -// eight byte vector which is then itself serialized to the stream -impl Serialize for SerializableAppendVec { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - const LEN: usize = std::mem::size_of::(); - let mut buf = [0u8; LEN]; - serialize_into(Cursor::new(&mut buf[..]), &(self.current_len as u64)) - .map_err(serde::ser::Error::custom)?; - serializer.serialize_bytes(&buf) - } -} - -// Deserialization of AppendVec requires deserialization -// of eight byte vector from which u64 is then deserialized -impl<'de> Deserialize<'de> for SerializableAppendVec { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - use serde::de::Error; - struct SerializableAppendVecVisitor; - impl<'a> Visitor<'a> for SerializableAppendVecVisitor { - type Value = SerializableAppendVec; - fn expecting(&self, formatter: &mut Formatter) -> FormatResult { - formatter.write_str("Expecting SerializableAppendVec") - } - fn visit_bytes(self, data: &[u8]) -> std::result::Result - where - E: Error, - { - const LEN: u64 = std::mem::size_of::() as u64; - let mut rd = Cursor::new(&data[..]); - let current_len: usize = deserialize_from(&mut rd).map_err(Error::custom)?; - if rd.position() != LEN { - Err(Error::custom("SerializableAppendVec: unexpected length")) - } else { - Ok(SerializableAppendVec { current_len }) - } - } - } - deserializer.deserialize_bytes(SerializableAppendVecVisitor) - } -} - -// Deserializable version of Bank which need not be serializable, -// because it's handled by SerializableVersionedBank. -// So, sync fields with it! -#[derive(Clone, Deserialize)] -pub(crate) struct DeserializableVersionedBank { - pub(crate) blockhash_queue: BlockhashQueue, - pub(crate) ancestors: Ancestors, - pub(crate) hash: Hash, - pub(crate) parent_hash: Hash, - pub(crate) parent_slot: Slot, - pub(crate) hard_forks: HardForks, - pub(crate) transaction_count: u64, - pub(crate) tick_height: u64, - pub(crate) signature_count: u64, - pub(crate) capitalization: u64, - pub(crate) max_tick_height: u64, - pub(crate) hashes_per_tick: Option, - pub(crate) ticks_per_slot: u64, - pub(crate) ns_per_slot: u128, - pub(crate) genesis_creation_time: UnixTimestamp, - pub(crate) slots_per_year: f64, - pub(crate) unused: u64, - pub(crate) slot: Slot, - pub(crate) epoch: Epoch, - pub(crate) block_height: u64, - pub(crate) collector_id: Pubkey, - pub(crate) collector_fees: u64, - pub(crate) fee_calculator: FeeCalculator, - pub(crate) fee_rate_governor: FeeRateGovernor, - pub(crate) collected_rent: u64, - pub(crate) rent_collector: RentCollector, - pub(crate) epoch_schedule: EpochSchedule, - pub(crate) inflation: Inflation, - pub(crate) stakes: Stakes, - pub(crate) unused_accounts: UnusedAccounts, - pub(crate) epoch_stakes: HashMap, - pub(crate) is_delta: bool, - pub(crate) message_processor: MessageProcessor, -} - -impl Into for DeserializableVersionedBank { - fn into(self) -> BankFieldsToDeserialize { - BankFieldsToDeserialize { - blockhash_queue: self.blockhash_queue, - ancestors: self.ancestors, - hash: self.hash, - parent_hash: self.parent_hash, - parent_slot: self.parent_slot, - hard_forks: self.hard_forks, - transaction_count: self.transaction_count, - tick_height: self.tick_height, - signature_count: self.signature_count, - capitalization: self.capitalization, - max_tick_height: self.max_tick_height, - hashes_per_tick: self.hashes_per_tick, - ticks_per_slot: self.ticks_per_slot, - ns_per_slot: self.ns_per_slot, - genesis_creation_time: self.genesis_creation_time, - slots_per_year: self.slots_per_year, - unused: self.unused, - slot: self.slot, - epoch: self.epoch, - block_height: self.block_height, - collector_id: self.collector_id, - collector_fees: self.collector_fees, - fee_calculator: self.fee_calculator, - fee_rate_governor: self.fee_rate_governor, - collected_rent: self.collected_rent, - rent_collector: self.rent_collector, - epoch_schedule: self.epoch_schedule, - inflation: self.inflation, - stakes: self.stakes, - epoch_stakes: self.epoch_stakes, - is_delta: self.is_delta, - } - } -} -// Serializable version of Bank, not Deserializable to avoid cloning by using refs. -// Sync fields with DeserializableVersionedBank! -#[derive(Serialize)] -pub(crate) struct SerializableVersionedBank<'a> { - pub(crate) blockhash_queue: &'a RwLock, - pub(crate) ancestors: &'a Ancestors, - pub(crate) hash: Hash, - pub(crate) parent_hash: Hash, - pub(crate) parent_slot: Slot, - pub(crate) hard_forks: &'a RwLock, - pub(crate) transaction_count: u64, - pub(crate) tick_height: u64, - pub(crate) signature_count: u64, - pub(crate) capitalization: u64, - pub(crate) max_tick_height: u64, - pub(crate) hashes_per_tick: Option, - pub(crate) ticks_per_slot: u64, - pub(crate) ns_per_slot: u128, - pub(crate) genesis_creation_time: UnixTimestamp, - pub(crate) slots_per_year: f64, - pub(crate) unused: u64, - pub(crate) slot: Slot, - pub(crate) epoch: Epoch, - pub(crate) block_height: u64, - pub(crate) collector_id: Pubkey, - pub(crate) collector_fees: u64, - pub(crate) fee_calculator: FeeCalculator, - pub(crate) fee_rate_governor: FeeRateGovernor, - pub(crate) collected_rent: u64, - pub(crate) rent_collector: RentCollector, - pub(crate) epoch_schedule: EpochSchedule, - pub(crate) inflation: Inflation, - pub(crate) stakes: &'a RwLock, - pub(crate) unused_accounts: UnusedAccounts, - pub(crate) epoch_stakes: &'a HashMap, - pub(crate) is_delta: bool, - pub(crate) message_processor: MessageProcessor, -} - -use std::sync::RwLock; - -impl<'a> From> for SerializableVersionedBank<'a> { - fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self { - fn new() -> T { - T::default() - } - Self { - blockhash_queue: rhs.blockhash_queue, - ancestors: rhs.ancestors, - hash: rhs.hash, - parent_hash: rhs.parent_hash, - parent_slot: rhs.parent_slot, - hard_forks: rhs.hard_forks, - transaction_count: rhs.transaction_count, - tick_height: rhs.tick_height, - signature_count: rhs.signature_count, - capitalization: rhs.capitalization, - max_tick_height: rhs.max_tick_height, - hashes_per_tick: rhs.hashes_per_tick, - ticks_per_slot: rhs.ticks_per_slot, - ns_per_slot: rhs.ns_per_slot, - genesis_creation_time: rhs.genesis_creation_time, - slots_per_year: rhs.slots_per_year, - unused: rhs.unused, - slot: rhs.slot, - epoch: rhs.epoch, - block_height: rhs.block_height, - collector_id: rhs.collector_id, - collector_fees: rhs.collector_fees, - fee_calculator: rhs.fee_calculator, - fee_rate_governor: rhs.fee_rate_governor, - collected_rent: rhs.collected_rent, - rent_collector: rhs.rent_collector, - epoch_schedule: rhs.epoch_schedule, - inflation: rhs.inflation, - stakes: rhs.stakes, - unused_accounts: new(), - epoch_stakes: rhs.epoch_stakes, - is_delta: rhs.is_delta, - message_processor: new(), - } - } -} - -#[cfg(RUSTC_WITH_SPECIALIZATION)] -impl<'a> IgnoreAsHelper for SerializableVersionedBank<'a> {} - -pub(super) struct Context {} -impl<'a> TypeContext<'a> for Context { - type SerializableAccountStorageEntry = SerializableAccountStorageEntry; - - fn serialize_bank_and_storage( - serializer: S, - serializable_bank_and_storage: &SerializableBankAndStorage<'a, Self>, - ) -> std::result::Result - where - Self: std::marker::Sized, - { - ( - SerializableVersionedBank::from( - serializable_bank_and_storage.bank.get_fields_to_serialize(), - ), - SerializableAccountsDB::<'a, Self> { - accounts_db: &*serializable_bank_and_storage.bank.rc.accounts.accounts_db, - slot: serializable_bank_and_storage.bank.rc.slot, - account_storage_entries: serializable_bank_and_storage.snapshot_storages, - phantom: std::marker::PhantomData::default(), - }, - ) - .serialize(serializer) - } - - fn serialize_accounts_db_fields( - serializer: S, - serializable_db: &SerializableAccountsDB<'a, Self>, - ) -> std::result::Result - where - Self: std::marker::Sized, - { - // sample write version before serializing storage entries - let version = serializable_db - .accounts_db - .write_version - .load(Ordering::Relaxed); - - // (1st of 3 elements) write the list of account storage entry lists out as a map - let entry_count = RefCell::::new(0); - let entries = - serialize_iter_as_map(serializable_db.account_storage_entries.iter().map(|x| { - *entry_count.borrow_mut() += x.len(); - ( - x.first().unwrap().slot, - serialize_iter_as_seq( - x.iter() - .map(|x| Self::SerializableAccountStorageEntry::from(x.as_ref())), - ), - ) - })); - - let slot_hash = ( - serializable_db.slot, - serializable_db - .accounts_db - .bank_hashes - .read() - .unwrap() - .get(&serializable_db.slot) - .unwrap_or_else(|| panic!("No bank_hashes entry for slot {}", serializable_db.slot)) - .clone(), - ); - - // as there is no deserialize_bank_rc_fields(), emit two u64 size fields here instead - let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms"); - let result = ( - &MAX_ACCOUNTS_DB_STREAM_SIZE, - &MAX_ACCOUNTS_DB_STREAM_SIZE, - &entries, - &version, - &slot_hash, - ) - .serialize(serializer); - serialize_account_storage_timer.stop(); - datapoint_info!( - "serialize_account_storage_ms", - ("duration", serialize_account_storage_timer.as_ms(), i64), - ("num_entries", *entry_count.borrow(), i64), - ); - result - } - - fn deserialize_bank_fields( - mut stream: &mut BufReader, - ) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error> - where - R: Read, - { - let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); - let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?; - Ok((bank_fields, accounts_db_fields)) - } - - fn deserialize_accounts_db_fields( - mut stream: &mut BufReader, - ) -> Result - where - R: Read, - { - // read and discard two u64 byte vector lengths - let serialized_len = MAX_ACCOUNTS_DB_STREAM_SIZE; - let serialized_len = min(serialized_len, deserialize_from(&mut stream)?); - let serialized_len = min(serialized_len, deserialize_from(&mut stream)?); - - // (1st of 3 elements) read in map of slots to account storage entries - let storage: HashMap> = bincode::options() - .with_limit(serialized_len) - .with_fixint_encoding() - .allow_trailing_bytes() - .deserialize_from(&mut stream)?; - - // (2nd of 3 elements) read in write version - let version: u64 = deserialize_from(&mut stream)?; - - // (3rd of 3 elements) read in (slot, bank hashes) pair - let (slot, bank_hash_info): (Slot, BankHashInfo) = deserialize_from(&mut stream)?; - - Ok(AccountsDbFields(storage, version, slot, bank_hash_info)) - } -} diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index 09b14a0a14..85ca577214 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -6,6 +6,7 @@ use { accounts_db::get_temp_accounts_paths, bank::{Bank, StatusCacheRc}, }, + bincode::serialize_into, rand::{thread_rng, Rng}, solana_sdk::{ account::Account, @@ -88,11 +89,6 @@ where account_paths, stream_append_vecs_path, ), - SerdeStyle::OLDER => context_accountsdb_from_stream::( - stream, - account_paths, - stream_append_vecs_path, - ), } } @@ -117,15 +113,6 @@ where phantom: std::marker::PhantomData::default(), }, ), - SerdeStyle::OLDER => serialize_into( - stream, - &SerializableAccountsDB:: { - accounts_db, - slot, - account_storage_entries, - phantom: std::marker::PhantomData::default(), - }, - ), } } @@ -261,21 +248,11 @@ fn test_accounts_serialize_newer() { test_accounts_serialize_style(SerdeStyle::NEWER) } -#[test] -fn test_accounts_serialize_older() { - test_accounts_serialize_style(SerdeStyle::OLDER) -} - #[test] fn test_bank_serialize_newer() { test_bank_serialize_style(SerdeStyle::NEWER) } -#[test] -fn test_bank_serialize_older() { - test_bank_serialize_style(SerdeStyle::OLDER) -} - #[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] mod test_bank_serialize { use super::*; @@ -304,27 +281,4 @@ mod test_bank_serialize { }) .serialize(s) } - - #[frozen_abi(digest = "9g4bYykzsC86fULgu9iUh4kpvb1pxvAmipvyZPChLhws")] - #[derive(Serialize, AbiExample)] - pub struct BankAbiTestWrapperLegacy { - #[serde(serialize_with = "wrapper_legacy")] - bank: Bank, - } - - pub fn wrapper_legacy(bank: &Bank, s: S) -> std::result::Result - where - S: serde::Serializer, - { - let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0); - // ensure there is a single snapshot storage example for ABI digesting - assert_eq!(snapshot_storages.len(), 1); - - (SerializableBankAndStorage:: { - bank, - snapshot_storages: &snapshot_storages, - phantom: std::marker::PhantomData::default(), - }) - .serialize(s) - } } diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index e552aeca73..dbdf95a3ad 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -34,13 +34,11 @@ pub const TAR_ACCOUNTS_DIR: &str = "accounts"; pub const TAR_VERSION_FILE: &str = "version"; const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB -const VERSION_STRING_V1_1_0: &str = "1.1.0"; const VERSION_STRING_V1_2_0: &str = "1.2.0"; const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0; #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum SnapshotVersion { - V1_1_0, V1_2_0, } @@ -59,7 +57,6 @@ impl fmt::Display for SnapshotVersion { impl From for &'static str { fn from(snapshot_version: SnapshotVersion) -> &'static str { match snapshot_version { - SnapshotVersion::V1_1_0 => VERSION_STRING_V1_1_0, SnapshotVersion::V1_2_0 => VERSION_STRING_V1_2_0, } } @@ -79,7 +76,6 @@ impl FromStr for SnapshotVersion { version_string }; match version_string { - VERSION_STRING_V1_1_0 => Ok(SnapshotVersion::V1_1_0), VERSION_STRING_V1_2_0 => Ok(SnapshotVersion::V1_2_0), _ => Err("unsupported snapshot version"), } @@ -458,7 +454,6 @@ pub fn add_snapshot>( let mut bank_serialize = Measure::start("bank-serialize-ms"); let bank_snapshot_serializer = move |stream: &mut BufWriter| -> Result<()> { let serde_style = match snapshot_version { - SnapshotVersion::V1_1_0 => SerdeStyle::OLDER, SnapshotVersion::V1_2_0 => SerdeStyle::NEWER, }; bank_to_stream(serde_style, stream.by_ref(), bank, snapshot_storages)?; @@ -722,14 +717,6 @@ where info!("Loading bank from {:?}", &root_paths.snapshot_file_path); let bank = deserialize_snapshot_data_file(&root_paths.snapshot_file_path, |mut stream| { Ok(match snapshot_version_enum { - SnapshotVersion::V1_1_0 => bank_from_stream( - SerdeStyle::OLDER, - &mut stream, - &append_vecs_path, - account_paths, - genesis_config, - frozen_account_pubkeys, - ), SnapshotVersion::V1_2_0 => bank_from_stream( SerdeStyle::NEWER, &mut stream, diff --git a/validator/src/main.rs b/validator/src/main.rs index 5606a8e165..aa1d9d33de 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -712,10 +712,10 @@ pub fn main() { Arg::with_name("snapshot_version") .long("snapshot-version") .value_name("SNAPSHOT_VERSION") - .validator(is_parsable::) - .takes_value(true) - .default_value(SnapshotVersion::default().into()) - .help("Output snapshot version"), + .validator(is_parsable::) + .takes_value(true) + .default_value(SnapshotVersion::default().into()) + .help("Output snapshot version"), ) .arg( Arg::with_name("limit_ledger_size")