From 934c32cbc687d88ad6e4f937a89dd039f0a833c8 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Tue, 21 Jan 2020 22:06:21 -0700 Subject: [PATCH] Add mechanism to load v0.22.3 snapshots on newer Solana versions --- core/src/snapshot_packager_service.rs | 13 ++- ledger/src/snapshot_utils.rs | 37 ++++++++- runtime/src/bank.rs | 115 ++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 5 deletions(-) diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index 2f5e18f76..aa5f5a76a 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -1,6 +1,6 @@ use solana_ledger::snapshot_package::{SnapshotPackage, SnapshotPackageReceiver}; use solana_ledger::snapshot_utils::{ - serialize_status_cache, SnapshotError, TAR_ACCOUNTS_DIR, TAR_SNAPSHOTS_DIR, + serialize_status_cache, SnapshotError, TAR_ACCOUNTS_DIR, TAR_SNAPSHOTS_DIR, TAR_VERSION_FILE, }; use solana_measure::measure::Measure; use solana_metrics::datapoint_info; @@ -91,6 +91,7 @@ impl SnapshotPackagerService { let staging_dir = TempDir::new()?; let staging_accounts_dir = staging_dir.path().join(TAR_ACCOUNTS_DIR); let staging_snapshots_dir = staging_dir.path().join(TAR_SNAPSHOTS_DIR); + let staging_version_file = staging_dir.path().join(TAR_VERSION_FILE); fs::create_dir_all(&staging_accounts_dir)?; // Add the snapshots to the staging directory @@ -119,6 +120,15 @@ impl SnapshotPackagerService { } } + // Write version file + { + use std::io::Write; + let snapshot_version = format!("{}\n", env!("CARGO_PKG_VERSION")); + let mut f = std::fs::File::create(staging_version_file)?; + //f.write_all(&snapshot_version.to_string().into_bytes())?; + f.write_all(&snapshot_version.into_bytes())?; + } + // Tar the staging directory into the archive at `archive_path` let archive_path = tar_dir.join("new_state.tar.bz2"); let args = vec![ @@ -128,6 +138,7 @@ impl SnapshotPackagerService { staging_dir.path().to_str().unwrap(), TAR_ACCOUNTS_DIR, TAR_SNAPSHOTS_DIR, + TAR_VERSION_FILE, ]; let output = std::process::Command::new("tar").args(&args).output()?; diff --git a/ledger/src/snapshot_utils.rs b/ledger/src/snapshot_utils.rs index 0bdf8c6cc..7bef3e2e7 100644 --- a/ledger/src/snapshot_utils.rs +++ b/ledger/src/snapshot_utils.rs @@ -24,6 +24,7 @@ use thiserror::Error; pub const SNAPSHOT_STATUS_CACHE_FILE_NAME: &str = "status_cache"; pub const TAR_SNAPSHOTS_DIR: &str = "snapshots"; pub const TAR_ACCOUNTS_DIR: &str = "accounts"; +pub const TAR_VERSION_FILE: &str = "version"; #[derive(PartialEq, Ord, Eq, Debug)] pub struct SlotSnapshotPaths { @@ -328,7 +329,20 @@ pub fn bank_from_archive>( let mut measure = Measure::start("bank rebuild from snapshot"); let unpacked_accounts_dir = unpack_dir.as_ref().join(TAR_ACCOUNTS_DIR); let unpacked_snapshots_dir = unpack_dir.as_ref().join(TAR_SNAPSHOTS_DIR); + let unpacked_version_file = unpack_dir.as_ref().join(TAR_VERSION_FILE); + + let snapshot_version = if let Ok(mut f) = File::open(unpacked_version_file) { + let mut snapshot_version = String::new(); + f.read_to_string(&mut snapshot_version)?; + snapshot_version + } else { + // Once v0.23.x is deployed, this default can be removed and snapshots without a version + // file can be rejected + String::from("v0.22.3") + }; + let bank = rebuild_bank_from_snapshots( + snapshot_version.trim(), account_paths, &unpacked_snapshots_dir, unpacked_accounts_dir, @@ -376,6 +390,7 @@ pub fn untar_snapshot_in, Q: AsRef>( } fn rebuild_bank_from_snapshots

( + snapshot_version: &str, local_account_paths: &[PathBuf], unpacked_snapshots_dir: &PathBuf, append_vecs_path: P, @@ -383,6 +398,8 @@ fn rebuild_bank_from_snapshots

( where P: AsRef, { + info!("snapshot version: {}", snapshot_version); + let mut snapshot_paths = get_snapshot_paths(&unpacked_snapshots_dir); if snapshot_paths.len() > 1 { return Err(get_io_error("invalid snapshot format")); @@ -391,13 +408,25 @@ where .pop() .ok_or_else(|| get_io_error("No snapshots found in snapshots directory"))?; - info!("Loading from {:?}", &root_paths.snapshot_file_path); + info!("Loading bank from {:?}", &root_paths.snapshot_file_path); let bank = deserialize_snapshot_data_file( &root_paths.snapshot_file_path, MAX_SNAPSHOT_DATA_FILE_SIZE, |stream| { - // Rebuild the root bank - let bank: Bank = deserialize_from_snapshot(stream.by_ref())?; + let bank: Bank = match snapshot_version { + env!("CARGO_PKG_VERSION") => deserialize_from_snapshot(stream.by_ref())?, + "v0.22.3" => { + let bank0223: solana_runtime::bank::LegacyBank0223 = + deserialize_from_snapshot(stream.by_ref())?; + bank0223.into() + } + _ => { + return Err(get_io_error(&format!( + "unsupported snapshot version: {}", + snapshot_version + ))); + } + }; // Rebuild accounts bank.rc.accounts_from_stream( stream.by_ref(), @@ -415,7 +444,7 @@ where |stream| { // Rebuild status cache let slot_deltas: Vec>> = - deserialize_from_snapshot(stream).unwrap_or_default(); + deserialize_from_snapshot(stream)?; Ok(slot_deltas) }, diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 4335b5953..d5a6e0078 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -2043,6 +2043,121 @@ where .deserialize_from(reader) } +// The `bank` struct as defined in Solana v0.22.3 +#[derive(Default, Deserialize, Serialize)] +pub struct LegacyBank0223 { + #[serde(skip)] + pub rc: BankRc, + + #[serde(skip)] + pub src: StatusCacheRc, + + pub blockhash_queue: RwLock, + pub ancestors: HashMap, + pub hash: RwLock, + pub parent_hash: Hash, + pub parent_slot: Slot, + + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub transaction_count: AtomicU64, + + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub tick_height: AtomicU64, + + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub signature_count: AtomicU64, + + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub capitalization: AtomicU64, + + pub max_tick_height: u64, + pub hashes_per_tick: Option, + pub ticks_per_slot: u64, + pub ns_per_slot: u128, + pub genesis_creation_time: UnixTimestamp, + pub slots_per_year: f64, + pub slots_per_segment: u64, + pub slot: Slot, + pub epoch: Epoch, + pub block_height: u64, + pub collector_id: Pubkey, + + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub collector_fees: AtomicU64, + + pub fee_calculator: FeeCalculator, + + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub collected_rent: AtomicU64, + + pub rent_collector: RentCollector, + pub epoch_schedule: EpochSchedule, + pub inflation: Arc>, + pub stakes: RwLock, + pub storage_accounts: RwLock, + pub epoch_stakes: HashMap, + + #[serde(serialize_with = "serialize_atomicbool")] + #[serde(deserialize_with = "deserialize_atomicbool")] + pub is_delta: AtomicBool, + + pub message_processor: MessageProcessor, + + #[serde(skip)] + pub entered_epoch_callback: Arc>>, + + #[serde(skip)] + pub last_vote_sync: AtomicU64, +} + +impl From for Bank { + fn from(bank: LegacyBank0223) -> Self { + Bank { + rc: bank.rc, + src: bank.src, + slot: bank.slot, + epoch: bank.epoch, + blockhash_queue: bank.blockhash_queue, + hashes_per_tick: bank.hashes_per_tick, + ticks_per_slot: bank.ticks_per_slot, + ns_per_slot: bank.ns_per_slot, + genesis_creation_time: bank.genesis_creation_time, + slots_per_segment: bank.slots_per_segment, + slots_per_year: bank.slots_per_year, + epoch_schedule: bank.epoch_schedule, + collected_rent: bank.collected_rent, + rent_collector: bank.rent_collector, + max_tick_height: bank.max_tick_height, + block_height: bank.block_height, + fee_calculator: bank.fee_calculator, + capitalization: bank.capitalization, + inflation: bank.inflation, + transaction_count: bank.transaction_count, + stakes: bank.stakes, + epoch_stakes: bank.epoch_stakes, + storage_accounts: bank.storage_accounts, + parent_hash: bank.parent_hash, + parent_slot: bank.parent_slot, + collector_id: bank.collector_id, + collector_fees: bank.collector_fees, + ancestors: bank.ancestors, + hash: bank.hash, + is_delta: bank.is_delta, + tick_height: bank.tick_height, + signature_count: bank.signature_count, + message_processor: bank.message_processor, + entered_epoch_callback: bank.entered_epoch_callback, + last_vote_sync: bank.last_vote_sync, + } + } +} + #[cfg(test)] mod tests { use super::*;