blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Remove support for 1.1.0 version snapshots

This commit is contained in:
Michael Vines 2020-08-03 16:27:17 -07:00 committed by mergify[bot]
parent 7db144c5da
commit 1d40685b8e
6 changed files with 9 additions and 472 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/tests/bank_forks.rs
View File

@ -58,7 +58,6 @@ mod tests {
use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc};
use tempfile::TempDir;
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0);
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0);
struct SnapshotTestConfig {

20
runtime/src/serde_snapshot.rs
View File

@ -1,9 +1,7 @@
use {
crate::{
accounts::Accounts,
accounts_db::{
AccountStorageEntry, AccountStorageStatus, AccountsDB, AppendVecId, BankHashInfo,
},
accounts_db::{AccountStorageEntry, AccountsDB, AppendVecId, BankHashInfo},
accounts_index::Ancestors,
append_vec::AppendVec,
bank::{Bank, BankFieldsToDeserialize, BankRc},
@ -14,14 +12,11 @@ use {
stakes::Stakes,
},
bincode,
bincode::{config::Options, serialize_into, Error},
bincode::{config::Options, Error},
fs_extra::dir::CopyOptions,
log::{info, warn},
rand::{thread_rng, Rng},
serde::{
de::{DeserializeOwned, Visitor},
Deserialize, Deserializer, Serialize, Serializer,
},
serde::{de::DeserializeOwned, Deserialize, Serialize},
solana_sdk::{
clock::{Epoch, Slot, UnixTimestamp},
epoch_schedule::EpochSchedule,
@ -33,10 +28,8 @@ use {
pubkey::Pubkey,
},
std::{
cmp::min,
collections::HashMap,
fmt::{Formatter, Result as FormatResult},
io::{BufReader, BufWriter, Cursor, Read, Write},
io::{BufReader, BufWriter, Read, Write},
path::{Path, PathBuf},
result::Result,
sync::{atomic::Ordering, Arc},
@ -49,12 +42,10 @@ use solana_sdk::abi_example::IgnoreAsHelper;
mod common;
mod future;
mod legacy;
mod tests;
mod utils;
use future::Context as TypeContextFuture;
use legacy::Context as TypeContextLegacy;
#[allow(unused_imports)]
use utils::{serialize_iter_as_map, serialize_iter_as_seq, serialize_iter_as_tuple};
@ -67,7 +58,6 @@ pub(crate) use crate::accounts_db::{SnapshotStorage, SnapshotStorages};
#[derive(Copy, Clone, Eq, PartialEq)]
pub(crate) enum SerdeStyle {
NEWER,
OLDER,
}
const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024;
@ -155,7 +145,6 @@ where
}
match serde_style {
SerdeStyle::NEWER => INTO!(TypeContextFuture),
SerdeStyle::OLDER => INTO!(TypeContextLegacy),
}
.map_err(|err| {
warn!("bankrc_from_stream error: {:?}", err);
@ -186,7 +175,6 @@ where
}
match serde_style {
SerdeStyle::NEWER => INTO!(TypeContextFuture),
SerdeStyle::OLDER => INTO!(TypeContextLegacy),
}
.map_err(|err| {
warn!("bankrc_to_stream error: {:?}", err);

391
runtime/src/serde_snapshot/legacy.rs
View File

@ -1,391 +0,0 @@
use super::common::UnusedAccounts;
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
use solana_sdk::abi_example::IgnoreAsHelper;
use {super::*, bincode::config::Options, solana_measure::measure::Measure, std::cell::RefCell};
const MAX_ACCOUNTS_DB_STREAM_SIZE: u64 = MAX_STREAM_SIZE;
type AccountsDbFields = super::AccountsDbFields<SerializableAccountStorageEntry>;
// Serializable version of AccountStorageEntry for snapshot format
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
pub(super) struct SerializableAccountStorageEntry {
id: AppendVecId,
accounts: SerializableAppendVec,
count_and_status: (usize, AccountStorageStatus),
}
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
impl IgnoreAsHelper for SerializableAccountStorageEntry {}
impl From<&AccountStorageEntry> for SerializableAccountStorageEntry {
fn from(rhs: &AccountStorageEntry) -> Self {
Self {
id: rhs.id,
accounts: SerializableAppendVec::from(&rhs.accounts),
..Self::default()
}
}
}
impl Into<AccountStorageEntry> for SerializableAccountStorageEntry {
fn into(self) -> AccountStorageEntry {
AccountStorageEntry::new_empty_map(self.id, self.accounts.current_len)
}
}
// Serializable version of AppendVec for snapshot format
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
struct SerializableAppendVec {
current_len: usize,
}
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
impl IgnoreAsHelper for SerializableAppendVec {}
impl From<&AppendVec> for SerializableAppendVec {
fn from(rhs: &AppendVec) -> SerializableAppendVec {
SerializableAppendVec {
current_len: rhs.len(),
}
}
}
impl Into<AppendVec> for SerializableAppendVec {
fn into(self) -> AppendVec {
AppendVec::new_empty_map(self.current_len)
}
}
// Serialization of AppendVec requires serialization of u64 to
// eight byte vector which is then itself serialized to the stream
impl Serialize for SerializableAppendVec {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
const LEN: usize = std::mem::size_of::<usize>();
let mut buf = [0u8; LEN];
serialize_into(Cursor::new(&mut buf[..]), &(self.current_len as u64))
.map_err(serde::ser::Error::custom)?;
serializer.serialize_bytes(&buf)
}
}
// Deserialization of AppendVec requires deserialization
// of eight byte vector from which u64 is then deserialized
impl<'de> Deserialize<'de> for SerializableAppendVec {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
use serde::de::Error;
struct SerializableAppendVecVisitor;
impl<'a> Visitor<'a> for SerializableAppendVecVisitor {
type Value = SerializableAppendVec;
fn expecting(&self, formatter: &mut Formatter) -> FormatResult {
formatter.write_str("Expecting SerializableAppendVec")
}
fn visit_bytes<E>(self, data: &[u8]) -> std::result::Result<Self::Value, E>
where
E: Error,
{
const LEN: u64 = std::mem::size_of::<usize>() as u64;
let mut rd = Cursor::new(&data[..]);
let current_len: usize = deserialize_from(&mut rd).map_err(Error::custom)?;
if rd.position() != LEN {
Err(Error::custom("SerializableAppendVec: unexpected length"))
} else {
Ok(SerializableAppendVec { current_len })
}
}
}
deserializer.deserialize_bytes(SerializableAppendVecVisitor)
}
}
// Deserializable version of Bank which need not be serializable,
// because it's handled by SerializableVersionedBank.
// So, sync fields with it!
#[derive(Clone, Deserialize)]
pub(crate) struct DeserializableVersionedBank {
pub(crate) blockhash_queue: BlockhashQueue,
pub(crate) ancestors: Ancestors,
pub(crate) hash: Hash,
pub(crate) parent_hash: Hash,
pub(crate) parent_slot: Slot,
pub(crate) hard_forks: HardForks,
pub(crate) transaction_count: u64,
pub(crate) tick_height: u64,
pub(crate) signature_count: u64,
pub(crate) capitalization: u64,
pub(crate) max_tick_height: u64,
pub(crate) hashes_per_tick: Option<u64>,
pub(crate) ticks_per_slot: u64,
pub(crate) ns_per_slot: u128,
pub(crate) genesis_creation_time: UnixTimestamp,
pub(crate) slots_per_year: f64,
pub(crate) unused: u64,
pub(crate) slot: Slot,
pub(crate) epoch: Epoch,
pub(crate) block_height: u64,
pub(crate) collector_id: Pubkey,
pub(crate) collector_fees: u64,
pub(crate) fee_calculator: FeeCalculator,
pub(crate) fee_rate_governor: FeeRateGovernor,
pub(crate) collected_rent: u64,
pub(crate) rent_collector: RentCollector,
pub(crate) epoch_schedule: EpochSchedule,
pub(crate) inflation: Inflation,
pub(crate) stakes: Stakes,
pub(crate) unused_accounts: UnusedAccounts,
pub(crate) epoch_stakes: HashMap<Epoch, EpochStakes>,
pub(crate) is_delta: bool,
pub(crate) message_processor: MessageProcessor,
}
impl Into<BankFieldsToDeserialize> for DeserializableVersionedBank {
fn into(self) -> BankFieldsToDeserialize {
BankFieldsToDeserialize {
blockhash_queue: self.blockhash_queue,
ancestors: self.ancestors,
hash: self.hash,
parent_hash: self.parent_hash,
parent_slot: self.parent_slot,
hard_forks: self.hard_forks,
transaction_count: self.transaction_count,
tick_height: self.tick_height,
signature_count: self.signature_count,
capitalization: self.capitalization,
max_tick_height: self.max_tick_height,
hashes_per_tick: self.hashes_per_tick,
ticks_per_slot: self.ticks_per_slot,
ns_per_slot: self.ns_per_slot,
genesis_creation_time: self.genesis_creation_time,
slots_per_year: self.slots_per_year,
unused: self.unused,
slot: self.slot,
epoch: self.epoch,
block_height: self.block_height,
collector_id: self.collector_id,
collector_fees: self.collector_fees,
fee_calculator: self.fee_calculator,
fee_rate_governor: self.fee_rate_governor,
collected_rent: self.collected_rent,
rent_collector: self.rent_collector,
epoch_schedule: self.epoch_schedule,
inflation: self.inflation,
stakes: self.stakes,
epoch_stakes: self.epoch_stakes,
is_delta: self.is_delta,
}
}
}
// Serializable version of Bank, not Deserializable to avoid cloning by using refs.
// Sync fields with DeserializableVersionedBank!
#[derive(Serialize)]
pub(crate) struct SerializableVersionedBank<'a> {
pub(crate) blockhash_queue: &'a RwLock<BlockhashQueue>,
pub(crate) ancestors: &'a Ancestors,
pub(crate) hash: Hash,
pub(crate) parent_hash: Hash,
pub(crate) parent_slot: Slot,
pub(crate) hard_forks: &'a RwLock<HardForks>,
pub(crate) transaction_count: u64,
pub(crate) tick_height: u64,
pub(crate) signature_count: u64,
pub(crate) capitalization: u64,
pub(crate) max_tick_height: u64,
pub(crate) hashes_per_tick: Option<u64>,
pub(crate) ticks_per_slot: u64,
pub(crate) ns_per_slot: u128,
pub(crate) genesis_creation_time: UnixTimestamp,
pub(crate) slots_per_year: f64,
pub(crate) unused: u64,
pub(crate) slot: Slot,
pub(crate) epoch: Epoch,
pub(crate) block_height: u64,
pub(crate) collector_id: Pubkey,
pub(crate) collector_fees: u64,
pub(crate) fee_calculator: FeeCalculator,
pub(crate) fee_rate_governor: FeeRateGovernor,
pub(crate) collected_rent: u64,
pub(crate) rent_collector: RentCollector,
pub(crate) epoch_schedule: EpochSchedule,
pub(crate) inflation: Inflation,
pub(crate) stakes: &'a RwLock<Stakes>,
pub(crate) unused_accounts: UnusedAccounts,
pub(crate) epoch_stakes: &'a HashMap<Epoch, EpochStakes>,
pub(crate) is_delta: bool,
pub(crate) message_processor: MessageProcessor,
}
use std::sync::RwLock;
impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedBank<'a> {
fn from(rhs: crate::bank::BankFieldsToSerialize<'a>) -> Self {
fn new<T: Default>() -> T {
T::default()
}
Self {
blockhash_queue: rhs.blockhash_queue,
ancestors: rhs.ancestors,
hash: rhs.hash,
parent_hash: rhs.parent_hash,
parent_slot: rhs.parent_slot,
hard_forks: rhs.hard_forks,
transaction_count: rhs.transaction_count,
tick_height: rhs.tick_height,
signature_count: rhs.signature_count,
capitalization: rhs.capitalization,
max_tick_height: rhs.max_tick_height,
hashes_per_tick: rhs.hashes_per_tick,
ticks_per_slot: rhs.ticks_per_slot,
ns_per_slot: rhs.ns_per_slot,
genesis_creation_time: rhs.genesis_creation_time,
slots_per_year: rhs.slots_per_year,
unused: rhs.unused,
slot: rhs.slot,
epoch: rhs.epoch,
block_height: rhs.block_height,
collector_id: rhs.collector_id,
collector_fees: rhs.collector_fees,
fee_calculator: rhs.fee_calculator,
fee_rate_governor: rhs.fee_rate_governor,
collected_rent: rhs.collected_rent,
rent_collector: rhs.rent_collector,
epoch_schedule: rhs.epoch_schedule,
inflation: rhs.inflation,
stakes: rhs.stakes,
unused_accounts: new(),
epoch_stakes: rhs.epoch_stakes,
is_delta: rhs.is_delta,
message_processor: new(),
}
}
}
#[cfg(RUSTC_WITH_SPECIALIZATION)]
impl<'a> IgnoreAsHelper for SerializableVersionedBank<'a> {}
pub(super) struct Context {}
impl<'a> TypeContext<'a> for Context {
type SerializableAccountStorageEntry = SerializableAccountStorageEntry;
fn serialize_bank_and_storage<S: serde::ser::Serializer>(
serializer: S,
serializable_bank_and_storage: &SerializableBankAndStorage<'a, Self>,
) -> std::result::Result<S::Ok, S::Error>
where
Self: std::marker::Sized,
{
(
SerializableVersionedBank::from(
serializable_bank_and_storage.bank.get_fields_to_serialize(),
),
SerializableAccountsDB::<'a, Self> {
accounts_db: &*serializable_bank_and_storage.bank.rc.accounts.accounts_db,
slot: serializable_bank_and_storage.bank.rc.slot,
account_storage_entries: serializable_bank_and_storage.snapshot_storages,
phantom: std::marker::PhantomData::default(),
},
)
.serialize(serializer)
}
fn serialize_accounts_db_fields<S: serde::ser::Serializer>(
serializer: S,
serializable_db: &SerializableAccountsDB<'a, Self>,
) -> std::result::Result<S::Ok, S::Error>
where
Self: std::marker::Sized,
{
// sample write version before serializing storage entries
let version = serializable_db
.accounts_db
.write_version
.load(Ordering::Relaxed);
// (1st of 3 elements) write the list of account storage entry lists out as a map
let entry_count = RefCell::<usize>::new(0);
let entries =
serialize_iter_as_map(serializable_db.account_storage_entries.iter().map(|x| {
*entry_count.borrow_mut() += x.len();
(
x.first().unwrap().slot,
serialize_iter_as_seq(
x.iter()
.map(|x| Self::SerializableAccountStorageEntry::from(x.as_ref())),
),
)
}));
let slot_hash = (
serializable_db.slot,
serializable_db
.accounts_db
.bank_hashes
.read()
.unwrap()
.get(&serializable_db.slot)
.unwrap_or_else(|| panic!("No bank_hashes entry for slot {}", serializable_db.slot))
.clone(),
);
// as there is no deserialize_bank_rc_fields(), emit two u64 size fields here instead
let mut serialize_account_storage_timer = Measure::start("serialize_account_storage_ms");
let result = (
&MAX_ACCOUNTS_DB_STREAM_SIZE,
&MAX_ACCOUNTS_DB_STREAM_SIZE,
&entries,
&version,
&slot_hash,
)
.serialize(serializer);
serialize_account_storage_timer.stop();
datapoint_info!(
"serialize_account_storage_ms",
("duration", serialize_account_storage_timer.as_ms(), i64),
("num_entries", *entry_count.borrow(), i64),
);
result
}
fn deserialize_bank_fields<R>(
mut stream: &mut BufReader<R>,
) -> Result<(BankFieldsToDeserialize, AccountsDbFields), Error>
where
R: Read,
{
let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into();
let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?;
Ok((bank_fields, accounts_db_fields))
}
fn deserialize_accounts_db_fields<R>(
mut stream: &mut BufReader<R>,
) -> Result<AccountsDbFields, Error>
where
R: Read,
{
// read and discard two u64 byte vector lengths
let serialized_len = MAX_ACCOUNTS_DB_STREAM_SIZE;
let serialized_len = min(serialized_len, deserialize_from(&mut stream)?);
let serialized_len = min(serialized_len, deserialize_from(&mut stream)?);
// (1st of 3 elements) read in map of slots to account storage entries
let storage: HashMap<Slot, Vec<Self::SerializableAccountStorageEntry>> = bincode::options()
.with_limit(serialized_len)
.with_fixint_encoding()
.allow_trailing_bytes()
.deserialize_from(&mut stream)?;
// (2nd of 3 elements) read in write version
let version: u64 = deserialize_from(&mut stream)?;
// (3rd of 3 elements) read in (slot, bank hashes) pair
let (slot, bank_hash_info): (Slot, BankHashInfo) = deserialize_from(&mut stream)?;
Ok(AccountsDbFields(storage, version, slot, bank_hash_info))
}
}

48
runtime/src/serde_snapshot/tests.rs
View File

@ -6,6 +6,7 @@ use {
accounts_db::get_temp_accounts_paths,
bank::{Bank, StatusCacheRc},
},
bincode::serialize_into,
rand::{thread_rng, Rng},
solana_sdk::{
account::Account,
@ -88,11 +89,6 @@ where
account_paths,
stream_append_vecs_path,
),
SerdeStyle::OLDER => context_accountsdb_from_stream::<TypeContextLegacy, R, P>(
stream,
account_paths,
stream_append_vecs_path,
),
}
}
@ -117,15 +113,6 @@ where
phantom: std::marker::PhantomData::default(),
},
),
SerdeStyle::OLDER => serialize_into(
stream,
&SerializableAccountsDB::<TypeContextLegacy> {
accounts_db,
slot,
account_storage_entries,
phantom: std::marker::PhantomData::default(),
},
),
}
}
@ -261,21 +248,11 @@ fn test_accounts_serialize_newer() {
test_accounts_serialize_style(SerdeStyle::NEWER)
}
#[test]
fn test_accounts_serialize_older() {
test_accounts_serialize_style(SerdeStyle::OLDER)
}
#[test]
fn test_bank_serialize_newer() {
test_bank_serialize_style(SerdeStyle::NEWER)
}
#[test]
fn test_bank_serialize_older() {
test_bank_serialize_style(SerdeStyle::OLDER)
}
#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))]
mod test_bank_serialize {
use super::*;
@ -304,27 +281,4 @@ mod test_bank_serialize {
})
.serialize(s)
}
#[frozen_abi(digest = "9g4bYykzsC86fULgu9iUh4kpvb1pxvAmipvyZPChLhws")]
#[derive(Serialize, AbiExample)]
pub struct BankAbiTestWrapperLegacy {
#[serde(serialize_with = "wrapper_legacy")]
bank: Bank,
}
pub fn wrapper_legacy<S>(bank: &Bank, s: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let snapshot_storages = bank.rc.accounts.accounts_db.get_snapshot_storages(0);
// ensure there is a single snapshot storage example for ABI digesting
assert_eq!(snapshot_storages.len(), 1);
(SerializableBankAndStorage::<legacy::Context> {
bank,
snapshot_storages: &snapshot_storages,
phantom: std::marker::PhantomData::default(),
})
.serialize(s)
}
}

13
runtime/src/snapshot_utils.rs
View File

@ -34,13 +34,11 @@ pub const TAR_ACCOUNTS_DIR: &str = "accounts";
pub const TAR_VERSION_FILE: &str = "version";
const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
const VERSION_STRING_V1_1_0: &str = "1.1.0";
const VERSION_STRING_V1_2_0: &str = "1.2.0";
const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0;
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
pub enum SnapshotVersion {
V1_1_0,
V1_2_0,
}
@ -59,7 +57,6 @@ impl fmt::Display for SnapshotVersion {
impl From<SnapshotVersion> for &'static str {
fn from(snapshot_version: SnapshotVersion) -> &'static str {
match snapshot_version {
SnapshotVersion::V1_1_0 => VERSION_STRING_V1_1_0,
SnapshotVersion::V1_2_0 => VERSION_STRING_V1_2_0,
}
}
@ -79,7 +76,6 @@ impl FromStr for SnapshotVersion {
version_string
};
match version_string {
VERSION_STRING_V1_1_0 => Ok(SnapshotVersion::V1_1_0),
VERSION_STRING_V1_2_0 => Ok(SnapshotVersion::V1_2_0),
_ => Err("unsupported snapshot version"),
}
@ -458,7 +454,6 @@ pub fn add_snapshot<P: AsRef<Path>>(
let mut bank_serialize = Measure::start("bank-serialize-ms");
let bank_snapshot_serializer = move |stream: &mut BufWriter<File>| -> Result<()> {
let serde_style = match snapshot_version {
SnapshotVersion::V1_1_0 => SerdeStyle::OLDER,
SnapshotVersion::V1_2_0 => SerdeStyle::NEWER,
};
bank_to_stream(serde_style, stream.by_ref(), bank, snapshot_storages)?;
@ -722,14 +717,6 @@ where
info!("Loading bank from {:?}", &root_paths.snapshot_file_path);
let bank = deserialize_snapshot_data_file(&root_paths.snapshot_file_path, |mut stream| {
Ok(match snapshot_version_enum {
SnapshotVersion::V1_1_0 => bank_from_stream(
SerdeStyle::OLDER,
&mut stream,
&append_vecs_path,
account_paths,
genesis_config,
frozen_account_pubkeys,
),
SnapshotVersion::V1_2_0 => bank_from_stream(
SerdeStyle::NEWER,
&mut stream,

8
validator/src/main.rs
View File

@ -712,10 +712,10 @@ pub fn main() {
Arg::with_name("snapshot_version")
.long("snapshot-version")
.value_name("SNAPSHOT_VERSION")
.validator(is_parsable::<SnapshotVersion>)
.takes_value(true)
.default_value(SnapshotVersion::default().into())
.help("Output snapshot version"),
.validator(is_parsable::<SnapshotVersion>)
.takes_value(true)
.default_value(SnapshotVersion::default().into())
.help("Output snapshot version"),
)
.arg(
Arg::with_name("limit_ledger_size")