From 6d81eede93de8f6005021460c0c038203cd188ad Mon Sep 17 00:00:00 2001 From: Kristofer Peterson Date: Fri, 19 Jun 2020 06:38:37 +0100 Subject: [PATCH] Add CLI options and runtime support for selection of output snapshot version. (#10536) --- core/src/accounts_hash_verifier.rs | 2 + core/src/rpc_service.rs | 5 +- core/src/snapshot_packager_service.rs | 3 +- core/tests/bank_forks.rs | 202 ++++++++++++++++---------- ledger-tool/src/main.rs | 32 +++- local-cluster/tests/local_cluster.rs | 1 + runtime/src/bank_forks.rs | 13 +- runtime/src/snapshot_package.rs | 4 + runtime/src/snapshot_utils.rs | 39 ++++- validator/src/main.rs | 25 +++- 10 files changed, 228 insertions(+), 98 deletions(-) diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index 96f06d699..88b3d2136 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -176,6 +176,7 @@ mod tests { use crate::cluster_info::make_accounts_hashes_message; use crate::contact_info::ContactInfo; use solana_runtime::bank_forks::CompressionType; + use solana_runtime::snapshot_utils::SnapshotVersion; use solana_sdk::{ hash::hash, signature::{Keypair, Signer}, @@ -238,6 +239,7 @@ mod tests { tar_output_file: PathBuf::from("."), storages: vec![], compression: CompressionType::Bzip2, + snapshot_version: SnapshotVersion::default(), }; AccountsHashVerifier::process_accounts_package( diff --git a/core/src/rpc_service.rs b/core/src/rpc_service.rs index f1043ef31..dfafcfc27 100644 --- a/core/src/rpc_service.rs +++ b/core/src/rpc_service.rs @@ -353,7 +353,9 @@ mod tests { genesis_utils::{create_genesis_config, GenesisConfigInfo}, get_tmp_ledger_path, }; - use solana_runtime::{bank::Bank, bank_forks::CompressionType}; + use solana_runtime::{ + bank::Bank, bank_forks::CompressionType, snapshot_utils::SnapshotVersion, + }; use solana_sdk::signature::Signer; use std::net::{IpAddr, Ipv4Addr}; @@ -445,6 +447,7 @@ mod tests { snapshot_package_output_path: PathBuf::from("/"), snapshot_path: PathBuf::from("/"), compression: CompressionType::Bzip2, + snapshot_version: SnapshotVersion::default(), }), bank_forks, RpcHealth::stub(), diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index 7f6740734..1fc903865 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -83,7 +83,7 @@ mod tests { bank::BankSlotDelta, bank_forks::CompressionType, snapshot_package::AccountsPackage, - snapshot_utils::{self, SNAPSHOT_STATUS_CACHE_FILE_NAME}, + snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME}, }; use solana_sdk::hash::Hash; use std::{ @@ -174,6 +174,7 @@ mod tests { output_tar_path.clone(), Hash::default(), CompressionType::Bzip2, + SnapshotVersion::default(), ); // Make tarball from packageable snapshot diff --git a/core/tests/bank_forks.rs b/core/tests/bank_forks.rs index 807766480..0783952d1 100644 --- a/core/tests/bank_forks.rs +++ b/core/tests/bank_forks.rs @@ -1,5 +1,36 @@ // Long-running bank_forks tests +macro_rules! DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS { + ($x:ident) => { + #[allow(non_snake_case)] + mod $x { + use super::*; + + const SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::$x; + + #[test] + fn test_bank_forks_status_cache_snapshot_n() { + run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION) + } + + #[test] + fn test_bank_forks_snapshot_n() { + run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION) + } + + #[test] + fn test_concurrent_snapshot_packaging() { + run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION) + } + + #[test] + fn test_slots_to_snapshot() { + run_test_slots_to_snapshot(SNAPSHOT_VERSION) + } + } + }; +} + #[cfg(test)] mod tests { use bincode::serialize_into; @@ -13,6 +44,7 @@ mod tests { bank_forks::{BankForks, CompressionType, SnapshotConfig}, genesis_utils::{create_genesis_config, GenesisConfigInfo}, snapshot_utils, + snapshot_utils::SnapshotVersion, status_cache::MAX_CACHE_ENTRIES, }; use solana_sdk::{ @@ -26,6 +58,9 @@ mod tests { use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc}; use tempfile::TempDir; + DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0); + DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0); + struct SnapshotTestConfig { accounts_dir: TempDir, snapshot_dir: TempDir, @@ -35,41 +70,47 @@ mod tests { genesis_config_info: GenesisConfigInfo, } - fn setup_snapshot_test(snapshot_interval_slots: u64) -> SnapshotTestConfig { - let accounts_dir = TempDir::new().unwrap(); - let snapshot_dir = TempDir::new().unwrap(); - let snapshot_output_path = TempDir::new().unwrap(); - let genesis_config_info = create_genesis_config(10_000); - let bank0 = Bank::new_with_paths( - &genesis_config_info.genesis_config, - vec![accounts_dir.path().to_path_buf()], - &[], - ); - bank0.freeze(); - let mut bank_forks = BankForks::new(bank0); - bank_forks.accounts_hash_interval_slots = snapshot_interval_slots; + impl SnapshotTestConfig { + fn new( + snapshot_version: SnapshotVersion, + snapshot_interval_slots: u64, + ) -> SnapshotTestConfig { + let accounts_dir = TempDir::new().unwrap(); + let snapshot_dir = TempDir::new().unwrap(); + let snapshot_output_path = TempDir::new().unwrap(); + let genesis_config_info = create_genesis_config(10_000); + let bank0 = Bank::new_with_paths( + &genesis_config_info.genesis_config, + vec![accounts_dir.path().to_path_buf()], + &[], + ); + bank0.freeze(); + let mut bank_forks = BankForks::new(bank0); + bank_forks.accounts_hash_interval_slots = snapshot_interval_slots; - let snapshot_config = SnapshotConfig { - snapshot_interval_slots, - snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), - snapshot_path: PathBuf::from(snapshot_dir.path()), - compression: CompressionType::Bzip2, - }; - bank_forks.set_snapshot_config(Some(snapshot_config.clone())); - SnapshotTestConfig { - accounts_dir, - snapshot_dir, - _snapshot_output_path: snapshot_output_path, - snapshot_config, - bank_forks, - genesis_config_info, + let snapshot_config = SnapshotConfig { + snapshot_interval_slots, + snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), + snapshot_path: PathBuf::from(snapshot_dir.path()), + compression: CompressionType::Bzip2, + snapshot_version, + }; + bank_forks.set_snapshot_config(Some(snapshot_config.clone())); + SnapshotTestConfig { + accounts_dir, + snapshot_dir, + _snapshot_output_path: snapshot_output_path, + snapshot_config, + bank_forks, + genesis_config_info, + } } } fn restore_from_snapshot( old_bank_forks: &BankForks, old_last_slot: Slot, - account_paths: Vec, + account_paths: &[PathBuf], ) { let (snapshot_path, snapshot_package_output_path) = old_bank_forks .snapshot_config @@ -115,17 +156,20 @@ mod tests { // also marks each bank as root and generates snapshots // finally tries to restore from the last bank's snapshot and compares the restored bank to the // `last_slot` bank - fn run_bank_forks_snapshot_n(last_slot: Slot, f: F, set_root_interval: u64) - where + fn run_bank_forks_snapshot_n( + snapshot_version: SnapshotVersion, + last_slot: Slot, + f: F, + set_root_interval: u64, + ) where F: Fn(&mut Bank, &Keypair), { solana_logger::setup(); // Set up snapshotting config - let mut snapshot_test_config = setup_snapshot_test(1); + let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1); let bank_forks = &mut snapshot_test_config.bank_forks; let accounts_dir = &snapshot_test_config.accounts_dir; - let snapshot_config = &snapshot_test_config.snapshot_config; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let (s, _r) = channel(); @@ -143,36 +187,33 @@ mod tests { } // Generate a snapshot package for last bank let last_bank = bank_forks.get(last_slot).unwrap(); - let storages: Vec<_> = last_bank.get_snapshot_storages(); - let slot_snapshot_paths = - snapshot_utils::get_snapshot_paths(&snapshot_config.snapshot_path); + let snapshot_config = &snapshot_test_config.snapshot_config; + let snapshot_path = &snapshot_config.snapshot_path; + let last_slot_snapshot_path = snapshot_utils::get_snapshot_paths(snapshot_path) + .pop() + .expect("no snapshots found in path"); let snapshot_package = snapshot_utils::package_snapshot( last_bank, - slot_snapshot_paths - .last() - .expect("no snapshots found in path"), - &snapshot_config.snapshot_path, + &last_slot_snapshot_path, + snapshot_path, &last_bank.src.roots(), &snapshot_config.snapshot_package_output_path, - storages, + last_bank.get_snapshot_storages(), CompressionType::Bzip2, + snapshot_version, ) .unwrap(); snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap(); - restore_from_snapshot( - bank_forks, - last_slot, - vec![accounts_dir.path().to_path_buf()], - ); + restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]); } - #[test] - fn test_bank_forks_snapshot_n() { + fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) { // create banks up to slot 4 and create 1 new account in each bank. test that bank 4 snapshots // and restores correctly run_bank_forks_snapshot_n( + snapshot_version, 4, |bank, mint_keypair| { let key1 = Keypair::new().pubkey(); @@ -203,24 +244,25 @@ mod tests { } } - #[test] - fn test_concurrent_snapshot_packaging() { + fn run_test_concurrent_snapshot_packaging(snapshot_version: SnapshotVersion) { solana_logger::setup(); // Set up snapshotting config - let mut snapshot_test_config = setup_snapshot_test(1); + let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1); let bank_forks = &mut snapshot_test_config.bank_forks; let accounts_dir = &snapshot_test_config.accounts_dir; let snapshots_dir = &snapshot_test_config.snapshot_dir; let snapshot_config = &snapshot_test_config.snapshot_config; + let snapshot_path = &snapshot_config.snapshot_path; + let snapshot_package_output_path = &snapshot_config.snapshot_package_output_path; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config; // Take snapshot of zeroth bank let bank0 = bank_forks.get(0).unwrap(); - let storages: Vec<_> = bank0.get_snapshot_storages(); - snapshot_utils::add_snapshot(&snapshot_config.snapshot_path, bank0, &storages).unwrap(); + let storages = bank0.get_snapshot_storages(); + snapshot_utils::add_snapshot(snapshot_path, bank0, &storages, snapshot_version).unwrap(); // Set up snapshotting channels let (sender, receiver) = channel(); @@ -270,7 +312,7 @@ mod tests { if slot == saved_slot as u64 { let options = CopyOptions::new(); fs_extra::dir::copy(accounts_dir, &saved_accounts_dir, &options).unwrap(); - let snapshot_paths: Vec<_> = fs::read_dir(&snapshot_config.snapshot_path) + let snapshot_paths = fs::read_dir(snapshot_path) .unwrap() .filter_map(|entry| { let e = entry.unwrap(); @@ -282,17 +324,13 @@ mod tests { .unwrap_or(None) }) .sorted() - .collect(); + .last() + .unwrap(); // only save off the snapshot of this slot, we don't need the others. - fs_extra::dir::copy( - &snapshot_paths.last().unwrap(), - &saved_snapshots_dir, - &options, - ) - .unwrap(); + fs_extra::dir::copy(&snapshot_paths, &saved_snapshots_dir, &options).unwrap(); saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path( - &snapshot_config.snapshot_package_output_path, + snapshot_package_output_path, &(slot, accounts_hash), &CompressionType::Bzip2, )); @@ -302,11 +340,12 @@ mod tests { // Purge all the outdated snapshots, including the ones needed to generate the package // currently sitting in the channel bank_forks.purge_old_snapshots(); - let mut snapshot_paths = snapshot_utils::get_snapshot_paths(&snapshots_dir); - snapshot_paths.sort(); assert_eq!( - snapshot_paths.iter().map(|path| path.slot).collect_vec(), - (3..=MAX_CACHE_ENTRIES as u64 + 2).collect_vec() + snapshot_utils::get_snapshot_paths(&snapshots_dir) + .into_iter() + .map(|path| path.slot) + .eq(3..=MAX_CACHE_ENTRIES as u64 + 2), + true ); // Create a SnapshotPackagerService to create tarballs from all the pending @@ -336,13 +375,12 @@ mod tests { // before we compare, stick an empty status_cache in this dir so that the package comparison works // This is needed since the status_cache is added by the packager and is not collected from // the source dir for snapshots - let dummy_slot_deltas: Vec = vec![]; snapshot_utils::serialize_snapshot_data_file( &saved_snapshots_dir .path() .join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILE_NAME), |stream| { - serialize_into(stream, &dummy_slot_deltas)?; + serialize_into(stream, &[] as &[BankSlotDelta])?; Ok(()) }, ) @@ -358,16 +396,17 @@ mod tests { ); } - #[test] - fn test_slots_to_snapshot() { + fn run_test_slots_to_snapshot(snapshot_version: SnapshotVersion) { solana_logger::setup(); let num_set_roots = MAX_CACHE_ENTRIES * 2; for add_root_interval in &[1, 3, 9] { let (snapshot_sender, _snapshot_receiver) = channel(); // Make sure this test never clears bank.slots_since_snapshot - let mut snapshot_test_config = - setup_snapshot_test((add_root_interval * num_set_roots * 2) as u64); + let mut snapshot_test_config = SnapshotTestConfig::new( + snapshot_version, + (*add_root_interval * num_set_roots * 2) as u64, + ); let mut current_bank = snapshot_test_config.bank_forks[0].clone(); let snapshot_sender = Some(snapshot_sender); for _ in 0..num_set_roots { @@ -386,21 +425,23 @@ mod tests { } let num_old_slots = num_set_roots * *add_root_interval - MAX_CACHE_ENTRIES + 1; - let expected_slots_to_snapshot = (num_old_slots as u64 - ..=num_set_roots as u64 * *add_root_interval as u64) - .collect_vec(); + let expected_slots_to_snapshot = + num_old_slots as u64..=num_set_roots as u64 * *add_root_interval as u64; - let rooted_bank = snapshot_test_config + let slots_to_snapshot = snapshot_test_config .bank_forks .get(snapshot_test_config.bank_forks.root()) - .unwrap(); - let slots_to_snapshot = rooted_bank.src.roots(); - assert_eq!(slots_to_snapshot, expected_slots_to_snapshot); + .unwrap() + .src + .roots(); + assert_eq!( + slots_to_snapshot.into_iter().eq(expected_slots_to_snapshot), + true + ); } } - #[test] - fn test_bank_forks_status_cache_snapshot_n() { + fn run_test_bank_forks_status_cache_snapshot_n(snapshot_version: SnapshotVersion) { // create banks up to slot (MAX_CACHE_ENTRIES * 2) + 1 while transferring 1 lamport into 2 different accounts each time // this is done to ensure the AccountStorageEntries keep getting cleaned up as the root moves // ahead. Also tests the status_cache purge and status cache snapshotting. @@ -409,6 +450,7 @@ mod tests { let key2 = Keypair::new().pubkey(); for set_root_interval in &[1, 4] { run_bank_forks_snapshot_n( + snapshot_version, (MAX_CACHE_ENTRIES * 2 + 1) as u64, |bank, mint_keypair| { let tx = system_transaction::transfer( diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index a929aab4f..0bec900fb 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -3,7 +3,7 @@ use clap::{ ArgMatches, SubCommand, }; use serde_json::json; -use solana_clap_utils::input_validators::is_slot; +use solana_clap_utils::input_validators::{is_parsable, is_slot}; use solana_ledger::{ bank_forks_utils, blockstore::Blockstore, @@ -16,6 +16,7 @@ use solana_runtime::{ bank_forks::{BankForks, CompressionType, SnapshotConfig}, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_utils, + snapshot_utils::SnapshotVersion, }; use solana_sdk::{ clock::Slot, genesis_config::GenesisConfig, native_token::lamports_to_sol, pubkey::Pubkey, @@ -574,6 +575,7 @@ fn load_bank_forks( snapshot_package_output_path: ledger_path.clone(), snapshot_path, compression: CompressionType::Bzip2, + snapshot_version: SnapshotVersion::default(), }) }; let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") { @@ -654,7 +656,13 @@ fn main() { .takes_value(true) .default_value(&default_genesis_archive_unpacked_size) .help("maximum total uncompressed size of unpacked genesis archive"); - + let snapshot_version_arg = Arg::with_name("snapshot_version") + .long("snapshot-version") + .value_name("SNAPSHOT_VERSION") + .validator(is_parsable::) + .takes_value(true) + .default_value(SnapshotVersion::default().into()) + .help("Output snapshot version"); let matches = App::new(crate_name!()) .about(crate_description!()) .version(solana_version::version!()) @@ -775,6 +783,7 @@ fn main() { .arg(&account_paths_arg) .arg(&hard_forks_arg) .arg(&max_genesis_archive_unpacked_size_arg) + .arg(&snapshot_version_arg) .arg( Arg::with_name("snapshot_slot") .index(1) @@ -1049,7 +1058,15 @@ fn main() { let snapshot_slot = value_t_or_exit!(arg_matches, "snapshot_slot", Slot); let output_directory = value_t_or_exit!(arg_matches, "output_directory", String); let warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok(); - + let snapshot_version = + arg_matches + .value_of("snapshot_version") + .map_or(SnapshotVersion::default(), |s| { + s.parse::().unwrap_or_else(|e| { + eprintln!("Error: {}", e); + exit(1) + }) + }); let process_options = ProcessOptions { dev_halt_at_slot: Some(snapshot_slot), new_hard_forks: hardforks_of(arg_matches, "hard_forks"), @@ -1083,7 +1100,11 @@ fn main() { bank }; - println!("Creating a snapshot of slot {}", bank.slot()); + println!( + "Creating a version {} snapshot of slot {}", + snapshot_version, + bank.slot(), + ); assert!(bank.is_complete()); bank.squash(); bank.clean_accounts(); @@ -1095,7 +1116,7 @@ fn main() { }); let storages: Vec<_> = bank.get_snapshot_storages(); - snapshot_utils::add_snapshot(&temp_dir, &bank, &storages) + snapshot_utils::add_snapshot(&temp_dir, &bank, &storages, snapshot_version) .and_then(|slot_snapshot_paths| { snapshot_utils::package_snapshot( &bank, @@ -1105,6 +1126,7 @@ fn main() { output_directory, storages, CompressionType::Bzip2, + snapshot_version, ) }) .and_then(|package| { diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index a72d15399..736ebffa5 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -1210,6 +1210,7 @@ fn setup_snapshot_validator_config( snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()), snapshot_path: PathBuf::from(snapshot_dir.path()), compression: CompressionType::Bzip2, + snapshot_version: snapshot_utils::SnapshotVersion::default(), }; // Create the account paths diff --git a/runtime/src/bank_forks.rs b/runtime/src/bank_forks.rs index 13e779693..dd723004c 100644 --- a/runtime/src/bank_forks.rs +++ b/runtime/src/bank_forks.rs @@ -16,6 +16,8 @@ use std::{ }; use thiserror::Error; +pub use crate::snapshot_utils::SnapshotVersion; + #[derive(Clone, Debug, Eq, PartialEq)] pub enum CompressionType { Bzip2, @@ -36,6 +38,9 @@ pub struct SnapshotConfig { pub snapshot_path: PathBuf, pub compression: CompressionType, + + // Snapshot version to generate + pub snapshot_version: SnapshotVersion, } #[derive(Error, Debug)] @@ -302,7 +307,12 @@ impl BankForks { let storages: Vec<_> = bank.get_snapshot_storages(); let mut add_snapshot_time = Measure::start("add-snapshot-ms"); - snapshot_utils::add_snapshot(&config.snapshot_path, &bank, &storages)?; + snapshot_utils::add_snapshot( + &config.snapshot_path, + &bank, + &storages, + config.snapshot_version, + )?; add_snapshot_time.stop(); inc_new_counter_info!("add-snapshot-ms", add_snapshot_time.as_ms() as usize); @@ -321,6 +331,7 @@ impl BankForks { &config.snapshot_package_output_path, storages, config.compression.clone(), + config.snapshot_version, )?; accounts_package_sender.send(package)?; diff --git a/runtime/src/snapshot_package.rs b/runtime/src/snapshot_package.rs index 0a787be5a..31a8cda03 100644 --- a/runtime/src/snapshot_package.rs +++ b/runtime/src/snapshot_package.rs @@ -1,4 +1,5 @@ use crate::bank_forks::CompressionType; +use crate::snapshot_utils::SnapshotVersion; use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta}; use solana_sdk::clock::Slot; use solana_sdk::hash::Hash; @@ -22,6 +23,7 @@ pub struct AccountsPackage { pub tar_output_file: PathBuf, pub hash: Hash, pub compression: CompressionType, + pub snapshot_version: SnapshotVersion, } impl AccountsPackage { @@ -34,6 +36,7 @@ impl AccountsPackage { tar_output_file: PathBuf, hash: Hash, compression: CompressionType, + snapshot_version: SnapshotVersion, ) -> Self { Self { root, @@ -44,6 +47,7 @@ impl AccountsPackage { tar_output_file, hash, compression, + snapshot_version, } } } diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index f5d53fe8f..e14721c66 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -37,7 +37,7 @@ pub const TAR_VERSION_FILE: &str = "version"; const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB const VERSION_STRING_V1_1_0: &str = "1.1.0"; const VERSION_STRING_V1_2_0: &str = "1.2.0"; -const OUTPUT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0; +const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0; #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum SnapshotVersion { @@ -45,6 +45,18 @@ pub enum SnapshotVersion { V1_2_0, } +impl Default for SnapshotVersion { + fn default() -> Self { + DEFAULT_SNAPSHOT_VERSION + } +} + +impl fmt::Display for SnapshotVersion { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(From::from(*self)) + } +} + impl From for &'static str { fn from(snapshot_version: SnapshotVersion) -> &'static str { match snapshot_version { @@ -58,6 +70,15 @@ impl FromStr for SnapshotVersion { type Err = &'static str; fn from_str(version_string: &str) -> std::result::Result { + // Remove leading 'v' or 'V' from slice + let version_string = if version_string + .get(..1) + .map_or(false, |s| s.eq_ignore_ascii_case("v")) + { + &version_string[1..] + } else { + version_string + }; match version_string { VERSION_STRING_V1_1_0 => Ok(SnapshotVersion::V1_1_0), VERSION_STRING_V1_2_0 => Ok(SnapshotVersion::V1_2_0), @@ -134,6 +155,7 @@ pub fn package_snapshot, Q: AsRef>( snapshot_package_output_path: P, snapshot_storages: SnapshotStorages, compression: CompressionType, + snapshot_version: SnapshotVersion, ) -> Result { // Hard link all the snapshots we need for this package let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?; @@ -164,6 +186,7 @@ pub fn package_snapshot, Q: AsRef>( snapshot_package_output_file, bank.get_accounts_hash(), compression, + snapshot_version, ); Ok(package) @@ -234,7 +257,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<() // Write version file { let mut f = std::fs::File::create(staging_version_file)?; - f.write_all(OUTPUT_SNAPSHOT_VERSION.as_str().as_bytes())?; + f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?; } let (compression_option, file_ext) = get_compression_ext(&snapshot_package.compression); @@ -417,6 +440,7 @@ pub fn add_snapshot>( snapshot_path: P, bank: &Bank, snapshot_storages: &[SnapshotStorage], + snapshot_version: SnapshotVersion, ) -> Result { let slot = bank.slot(); // snapshot_path/slot @@ -432,13 +456,12 @@ pub fn add_snapshot>( let mut bank_serialize = Measure::start("bank-serialize-ms"); let bank_snapshot_serializer = move |stream: &mut BufWriter| -> Result<()> { + let serde_style = match snapshot_version { + SnapshotVersion::V1_1_0 => SerdeStyle::OLDER, + SnapshotVersion::V1_2_0 => SerdeStyle::NEWER, + }; serialize_into(stream.by_ref(), bank)?; - bankrc_to_stream( - SerdeStyle::NEWER, - stream.by_ref(), - &bank.rc, - snapshot_storages, - )?; + bankrc_to_stream(serde_style, stream.by_ref(), &bank.rc, snapshot_storages)?; Ok(()) }; let consumed_size = diff --git a/validator/src/main.rs b/validator/src/main.rs index 29f9347fc..0e493c4af 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -6,7 +6,9 @@ use log::*; use rand::{thread_rng, Rng}; use solana_clap_utils::{ input_parsers::{keypair_of, keypairs_of, pubkey_of}, - input_validators::{is_keypair_or_ask_keyword, is_pubkey, is_pubkey_or_keypair, is_slot}, + input_validators::{ + is_keypair_or_ask_keyword, is_parsable, is_pubkey, is_pubkey_or_keypair, is_slot, + }, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, }; use solana_client::rpc_client::RpcClient; @@ -23,7 +25,7 @@ use solana_core::{ use solana_download_utils::{download_genesis_if_missing, download_snapshot}; use solana_perf::recycler::enable_recycler_warming; use solana_runtime::{ - bank_forks::{CompressionType, SnapshotConfig}, + bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion}, hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, }; use solana_sdk::{ @@ -694,6 +696,15 @@ pub fn main() { .default_value("100") .help("Number of slots between generating accounts hash."), ) + .arg( + Arg::with_name("snapshot_version") + .long("snapshot-version") + .value_name("SNAPSHOT_VERSION") + .validator(is_parsable::) + .takes_value(true) + .default_value(SnapshotVersion::default().into()) + .help("Output snapshot version"), + ) .arg( Arg::with_name("limit_ledger_size") .long("limit-ledger-size") @@ -969,6 +980,15 @@ pub fn main() { _ => panic!("Compression type not recognized: {}", compression_str), } } + let snapshot_version = + matches + .value_of("snapshot_version") + .map_or(SnapshotVersion::default(), |s| { + s.parse::().unwrap_or_else(|err| { + eprintln!("Error: {}", err); + exit(1) + }) + }); validator_config.snapshot_config = Some(SnapshotConfig { snapshot_interval_slots: if snapshot_interval_slots > 0 { snapshot_interval_slots @@ -978,6 +998,7 @@ pub fn main() { snapshot_path, snapshot_package_output_path: ledger_path.clone(), compression: snapshot_compression, + snapshot_version, }); validator_config.accounts_hash_interval_slots =