Add CLI options and runtime support for selection of output snapshot version. (#10536)
This commit is contained in:
parent
b172b3369e
commit
6d81eede93
|
@ -176,6 +176,7 @@ mod tests {
|
|||
use crate::cluster_info::make_accounts_hashes_message;
|
||||
use crate::contact_info::ContactInfo;
|
||||
use solana_runtime::bank_forks::CompressionType;
|
||||
use solana_runtime::snapshot_utils::SnapshotVersion;
|
||||
use solana_sdk::{
|
||||
hash::hash,
|
||||
signature::{Keypair, Signer},
|
||||
|
@ -238,6 +239,7 @@ mod tests {
|
|||
tar_output_file: PathBuf::from("."),
|
||||
storages: vec![],
|
||||
compression: CompressionType::Bzip2,
|
||||
snapshot_version: SnapshotVersion::default(),
|
||||
};
|
||||
|
||||
AccountsHashVerifier::process_accounts_package(
|
||||
|
|
|
@ -353,7 +353,9 @@ mod tests {
|
|||
genesis_utils::{create_genesis_config, GenesisConfigInfo},
|
||||
get_tmp_ledger_path,
|
||||
};
|
||||
use solana_runtime::{bank::Bank, bank_forks::CompressionType};
|
||||
use solana_runtime::{
|
||||
bank::Bank, bank_forks::CompressionType, snapshot_utils::SnapshotVersion,
|
||||
};
|
||||
use solana_sdk::signature::Signer;
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
|
||||
|
@ -445,6 +447,7 @@ mod tests {
|
|||
snapshot_package_output_path: PathBuf::from("/"),
|
||||
snapshot_path: PathBuf::from("/"),
|
||||
compression: CompressionType::Bzip2,
|
||||
snapshot_version: SnapshotVersion::default(),
|
||||
}),
|
||||
bank_forks,
|
||||
RpcHealth::stub(),
|
||||
|
|
|
@ -83,7 +83,7 @@ mod tests {
|
|||
bank::BankSlotDelta,
|
||||
bank_forks::CompressionType,
|
||||
snapshot_package::AccountsPackage,
|
||||
snapshot_utils::{self, SNAPSHOT_STATUS_CACHE_FILE_NAME},
|
||||
snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME},
|
||||
};
|
||||
use solana_sdk::hash::Hash;
|
||||
use std::{
|
||||
|
@ -174,6 +174,7 @@ mod tests {
|
|||
output_tar_path.clone(),
|
||||
Hash::default(),
|
||||
CompressionType::Bzip2,
|
||||
SnapshotVersion::default(),
|
||||
);
|
||||
|
||||
// Make tarball from packageable snapshot
|
||||
|
|
|
@ -1,5 +1,36 @@
|
|||
// Long-running bank_forks tests
|
||||
|
||||
macro_rules! DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS {
|
||||
($x:ident) => {
|
||||
#[allow(non_snake_case)]
|
||||
mod $x {
|
||||
use super::*;
|
||||
|
||||
const SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::$x;
|
||||
|
||||
#[test]
|
||||
fn test_bank_forks_status_cache_snapshot_n() {
|
||||
run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bank_forks_snapshot_n() {
|
||||
run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_snapshot_packaging() {
|
||||
run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_slots_to_snapshot() {
|
||||
run_test_slots_to_snapshot(SNAPSHOT_VERSION)
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use bincode::serialize_into;
|
||||
|
@ -13,6 +44,7 @@ mod tests {
|
|||
bank_forks::{BankForks, CompressionType, SnapshotConfig},
|
||||
genesis_utils::{create_genesis_config, GenesisConfigInfo},
|
||||
snapshot_utils,
|
||||
snapshot_utils::SnapshotVersion,
|
||||
status_cache::MAX_CACHE_ENTRIES,
|
||||
};
|
||||
use solana_sdk::{
|
||||
|
@ -26,6 +58,9 @@ mod tests {
|
|||
use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc};
|
||||
use tempfile::TempDir;
|
||||
|
||||
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0);
|
||||
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0);
|
||||
|
||||
struct SnapshotTestConfig {
|
||||
accounts_dir: TempDir,
|
||||
snapshot_dir: TempDir,
|
||||
|
@ -35,7 +70,11 @@ mod tests {
|
|||
genesis_config_info: GenesisConfigInfo,
|
||||
}
|
||||
|
||||
fn setup_snapshot_test(snapshot_interval_slots: u64) -> SnapshotTestConfig {
|
||||
impl SnapshotTestConfig {
|
||||
fn new(
|
||||
snapshot_version: SnapshotVersion,
|
||||
snapshot_interval_slots: u64,
|
||||
) -> SnapshotTestConfig {
|
||||
let accounts_dir = TempDir::new().unwrap();
|
||||
let snapshot_dir = TempDir::new().unwrap();
|
||||
let snapshot_output_path = TempDir::new().unwrap();
|
||||
|
@ -54,6 +93,7 @@ mod tests {
|
|||
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
|
||||
snapshot_path: PathBuf::from(snapshot_dir.path()),
|
||||
compression: CompressionType::Bzip2,
|
||||
snapshot_version,
|
||||
};
|
||||
bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
|
||||
SnapshotTestConfig {
|
||||
|
@ -65,11 +105,12 @@ mod tests {
|
|||
genesis_config_info,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_from_snapshot(
|
||||
old_bank_forks: &BankForks,
|
||||
old_last_slot: Slot,
|
||||
account_paths: Vec<PathBuf>,
|
||||
account_paths: &[PathBuf],
|
||||
) {
|
||||
let (snapshot_path, snapshot_package_output_path) = old_bank_forks
|
||||
.snapshot_config
|
||||
|
@ -115,17 +156,20 @@ mod tests {
|
|||
// also marks each bank as root and generates snapshots
|
||||
// finally tries to restore from the last bank's snapshot and compares the restored bank to the
|
||||
// `last_slot` bank
|
||||
fn run_bank_forks_snapshot_n<F>(last_slot: Slot, f: F, set_root_interval: u64)
|
||||
where
|
||||
fn run_bank_forks_snapshot_n<F>(
|
||||
snapshot_version: SnapshotVersion,
|
||||
last_slot: Slot,
|
||||
f: F,
|
||||
set_root_interval: u64,
|
||||
) where
|
||||
F: Fn(&mut Bank, &Keypair),
|
||||
{
|
||||
solana_logger::setup();
|
||||
// Set up snapshotting config
|
||||
let mut snapshot_test_config = setup_snapshot_test(1);
|
||||
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
|
||||
|
||||
let bank_forks = &mut snapshot_test_config.bank_forks;
|
||||
let accounts_dir = &snapshot_test_config.accounts_dir;
|
||||
let snapshot_config = &snapshot_test_config.snapshot_config;
|
||||
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
||||
|
||||
let (s, _r) = channel();
|
||||
|
@ -143,36 +187,33 @@ mod tests {
|
|||
}
|
||||
// Generate a snapshot package for last bank
|
||||
let last_bank = bank_forks.get(last_slot).unwrap();
|
||||
let storages: Vec<_> = last_bank.get_snapshot_storages();
|
||||
let slot_snapshot_paths =
|
||||
snapshot_utils::get_snapshot_paths(&snapshot_config.snapshot_path);
|
||||
let snapshot_config = &snapshot_test_config.snapshot_config;
|
||||
let snapshot_path = &snapshot_config.snapshot_path;
|
||||
let last_slot_snapshot_path = snapshot_utils::get_snapshot_paths(snapshot_path)
|
||||
.pop()
|
||||
.expect("no snapshots found in path");
|
||||
let snapshot_package = snapshot_utils::package_snapshot(
|
||||
last_bank,
|
||||
slot_snapshot_paths
|
||||
.last()
|
||||
.expect("no snapshots found in path"),
|
||||
&snapshot_config.snapshot_path,
|
||||
&last_slot_snapshot_path,
|
||||
snapshot_path,
|
||||
&last_bank.src.roots(),
|
||||
&snapshot_config.snapshot_package_output_path,
|
||||
storages,
|
||||
last_bank.get_snapshot_storages(),
|
||||
CompressionType::Bzip2,
|
||||
snapshot_version,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
|
||||
|
||||
restore_from_snapshot(
|
||||
bank_forks,
|
||||
last_slot,
|
||||
vec![accounts_dir.path().to_path_buf()],
|
||||
);
|
||||
restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bank_forks_snapshot_n() {
|
||||
fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) {
|
||||
// create banks up to slot 4 and create 1 new account in each bank. test that bank 4 snapshots
|
||||
// and restores correctly
|
||||
run_bank_forks_snapshot_n(
|
||||
snapshot_version,
|
||||
4,
|
||||
|bank, mint_keypair| {
|
||||
let key1 = Keypair::new().pubkey();
|
||||
|
@ -203,24 +244,25 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_snapshot_packaging() {
|
||||
fn run_test_concurrent_snapshot_packaging(snapshot_version: SnapshotVersion) {
|
||||
solana_logger::setup();
|
||||
|
||||
// Set up snapshotting config
|
||||
let mut snapshot_test_config = setup_snapshot_test(1);
|
||||
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
|
||||
|
||||
let bank_forks = &mut snapshot_test_config.bank_forks;
|
||||
let accounts_dir = &snapshot_test_config.accounts_dir;
|
||||
let snapshots_dir = &snapshot_test_config.snapshot_dir;
|
||||
let snapshot_config = &snapshot_test_config.snapshot_config;
|
||||
let snapshot_path = &snapshot_config.snapshot_path;
|
||||
let snapshot_package_output_path = &snapshot_config.snapshot_package_output_path;
|
||||
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
||||
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
|
||||
|
||||
// Take snapshot of zeroth bank
|
||||
let bank0 = bank_forks.get(0).unwrap();
|
||||
let storages: Vec<_> = bank0.get_snapshot_storages();
|
||||
snapshot_utils::add_snapshot(&snapshot_config.snapshot_path, bank0, &storages).unwrap();
|
||||
let storages = bank0.get_snapshot_storages();
|
||||
snapshot_utils::add_snapshot(snapshot_path, bank0, &storages, snapshot_version).unwrap();
|
||||
|
||||
// Set up snapshotting channels
|
||||
let (sender, receiver) = channel();
|
||||
|
@ -270,7 +312,7 @@ mod tests {
|
|||
if slot == saved_slot as u64 {
|
||||
let options = CopyOptions::new();
|
||||
fs_extra::dir::copy(accounts_dir, &saved_accounts_dir, &options).unwrap();
|
||||
let snapshot_paths: Vec<_> = fs::read_dir(&snapshot_config.snapshot_path)
|
||||
let snapshot_paths = fs::read_dir(snapshot_path)
|
||||
.unwrap()
|
||||
.filter_map(|entry| {
|
||||
let e = entry.unwrap();
|
||||
|
@ -282,17 +324,13 @@ mod tests {
|
|||
.unwrap_or(None)
|
||||
})
|
||||
.sorted()
|
||||
.collect();
|
||||
// only save off the snapshot of this slot, we don't need the others.
|
||||
fs_extra::dir::copy(
|
||||
&snapshot_paths.last().unwrap(),
|
||||
&saved_snapshots_dir,
|
||||
&options,
|
||||
)
|
||||
.last()
|
||||
.unwrap();
|
||||
// only save off the snapshot of this slot, we don't need the others.
|
||||
fs_extra::dir::copy(&snapshot_paths, &saved_snapshots_dir, &options).unwrap();
|
||||
|
||||
saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
|
||||
&snapshot_config.snapshot_package_output_path,
|
||||
snapshot_package_output_path,
|
||||
&(slot, accounts_hash),
|
||||
&CompressionType::Bzip2,
|
||||
));
|
||||
|
@ -302,11 +340,12 @@ mod tests {
|
|||
// Purge all the outdated snapshots, including the ones needed to generate the package
|
||||
// currently sitting in the channel
|
||||
bank_forks.purge_old_snapshots();
|
||||
let mut snapshot_paths = snapshot_utils::get_snapshot_paths(&snapshots_dir);
|
||||
snapshot_paths.sort();
|
||||
assert_eq!(
|
||||
snapshot_paths.iter().map(|path| path.slot).collect_vec(),
|
||||
(3..=MAX_CACHE_ENTRIES as u64 + 2).collect_vec()
|
||||
snapshot_utils::get_snapshot_paths(&snapshots_dir)
|
||||
.into_iter()
|
||||
.map(|path| path.slot)
|
||||
.eq(3..=MAX_CACHE_ENTRIES as u64 + 2),
|
||||
true
|
||||
);
|
||||
|
||||
// Create a SnapshotPackagerService to create tarballs from all the pending
|
||||
|
@ -336,13 +375,12 @@ mod tests {
|
|||
// before we compare, stick an empty status_cache in this dir so that the package comparison works
|
||||
// This is needed since the status_cache is added by the packager and is not collected from
|
||||
// the source dir for snapshots
|
||||
let dummy_slot_deltas: Vec<BankSlotDelta> = vec![];
|
||||
snapshot_utils::serialize_snapshot_data_file(
|
||||
&saved_snapshots_dir
|
||||
.path()
|
||||
.join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILE_NAME),
|
||||
|stream| {
|
||||
serialize_into(stream, &dummy_slot_deltas)?;
|
||||
serialize_into(stream, &[] as &[BankSlotDelta])?;
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
|
@ -358,16 +396,17 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_slots_to_snapshot() {
|
||||
fn run_test_slots_to_snapshot(snapshot_version: SnapshotVersion) {
|
||||
solana_logger::setup();
|
||||
let num_set_roots = MAX_CACHE_ENTRIES * 2;
|
||||
|
||||
for add_root_interval in &[1, 3, 9] {
|
||||
let (snapshot_sender, _snapshot_receiver) = channel();
|
||||
// Make sure this test never clears bank.slots_since_snapshot
|
||||
let mut snapshot_test_config =
|
||||
setup_snapshot_test((add_root_interval * num_set_roots * 2) as u64);
|
||||
let mut snapshot_test_config = SnapshotTestConfig::new(
|
||||
snapshot_version,
|
||||
(*add_root_interval * num_set_roots * 2) as u64,
|
||||
);
|
||||
let mut current_bank = snapshot_test_config.bank_forks[0].clone();
|
||||
let snapshot_sender = Some(snapshot_sender);
|
||||
for _ in 0..num_set_roots {
|
||||
|
@ -386,21 +425,23 @@ mod tests {
|
|||
}
|
||||
|
||||
let num_old_slots = num_set_roots * *add_root_interval - MAX_CACHE_ENTRIES + 1;
|
||||
let expected_slots_to_snapshot = (num_old_slots as u64
|
||||
..=num_set_roots as u64 * *add_root_interval as u64)
|
||||
.collect_vec();
|
||||
let expected_slots_to_snapshot =
|
||||
num_old_slots as u64..=num_set_roots as u64 * *add_root_interval as u64;
|
||||
|
||||
let rooted_bank = snapshot_test_config
|
||||
let slots_to_snapshot = snapshot_test_config
|
||||
.bank_forks
|
||||
.get(snapshot_test_config.bank_forks.root())
|
||||
.unwrap();
|
||||
let slots_to_snapshot = rooted_bank.src.roots();
|
||||
assert_eq!(slots_to_snapshot, expected_slots_to_snapshot);
|
||||
.unwrap()
|
||||
.src
|
||||
.roots();
|
||||
assert_eq!(
|
||||
slots_to_snapshot.into_iter().eq(expected_slots_to_snapshot),
|
||||
true
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bank_forks_status_cache_snapshot_n() {
|
||||
fn run_test_bank_forks_status_cache_snapshot_n(snapshot_version: SnapshotVersion) {
|
||||
// create banks up to slot (MAX_CACHE_ENTRIES * 2) + 1 while transferring 1 lamport into 2 different accounts each time
|
||||
// this is done to ensure the AccountStorageEntries keep getting cleaned up as the root moves
|
||||
// ahead. Also tests the status_cache purge and status cache snapshotting.
|
||||
|
@ -409,6 +450,7 @@ mod tests {
|
|||
let key2 = Keypair::new().pubkey();
|
||||
for set_root_interval in &[1, 4] {
|
||||
run_bank_forks_snapshot_n(
|
||||
snapshot_version,
|
||||
(MAX_CACHE_ENTRIES * 2 + 1) as u64,
|
||||
|bank, mint_keypair| {
|
||||
let tx = system_transaction::transfer(
|
||||
|
|
|
@ -3,7 +3,7 @@ use clap::{
|
|||
ArgMatches, SubCommand,
|
||||
};
|
||||
use serde_json::json;
|
||||
use solana_clap_utils::input_validators::is_slot;
|
||||
use solana_clap_utils::input_validators::{is_parsable, is_slot};
|
||||
use solana_ledger::{
|
||||
bank_forks_utils,
|
||||
blockstore::Blockstore,
|
||||
|
@ -16,6 +16,7 @@ use solana_runtime::{
|
|||
bank_forks::{BankForks, CompressionType, SnapshotConfig},
|
||||
hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
|
||||
snapshot_utils,
|
||||
snapshot_utils::SnapshotVersion,
|
||||
};
|
||||
use solana_sdk::{
|
||||
clock::Slot, genesis_config::GenesisConfig, native_token::lamports_to_sol, pubkey::Pubkey,
|
||||
|
@ -574,6 +575,7 @@ fn load_bank_forks(
|
|||
snapshot_package_output_path: ledger_path.clone(),
|
||||
snapshot_path,
|
||||
compression: CompressionType::Bzip2,
|
||||
snapshot_version: SnapshotVersion::default(),
|
||||
})
|
||||
};
|
||||
let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
|
||||
|
@ -654,7 +656,13 @@ fn main() {
|
|||
.takes_value(true)
|
||||
.default_value(&default_genesis_archive_unpacked_size)
|
||||
.help("maximum total uncompressed size of unpacked genesis archive");
|
||||
|
||||
let snapshot_version_arg = Arg::with_name("snapshot_version")
|
||||
.long("snapshot-version")
|
||||
.value_name("SNAPSHOT_VERSION")
|
||||
.validator(is_parsable::<SnapshotVersion>)
|
||||
.takes_value(true)
|
||||
.default_value(SnapshotVersion::default().into())
|
||||
.help("Output snapshot version");
|
||||
let matches = App::new(crate_name!())
|
||||
.about(crate_description!())
|
||||
.version(solana_version::version!())
|
||||
|
@ -775,6 +783,7 @@ fn main() {
|
|||
.arg(&account_paths_arg)
|
||||
.arg(&hard_forks_arg)
|
||||
.arg(&max_genesis_archive_unpacked_size_arg)
|
||||
.arg(&snapshot_version_arg)
|
||||
.arg(
|
||||
Arg::with_name("snapshot_slot")
|
||||
.index(1)
|
||||
|
@ -1049,7 +1058,15 @@ fn main() {
|
|||
let snapshot_slot = value_t_or_exit!(arg_matches, "snapshot_slot", Slot);
|
||||
let output_directory = value_t_or_exit!(arg_matches, "output_directory", String);
|
||||
let warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok();
|
||||
|
||||
let snapshot_version =
|
||||
arg_matches
|
||||
.value_of("snapshot_version")
|
||||
.map_or(SnapshotVersion::default(), |s| {
|
||||
s.parse::<SnapshotVersion>().unwrap_or_else(|e| {
|
||||
eprintln!("Error: {}", e);
|
||||
exit(1)
|
||||
})
|
||||
});
|
||||
let process_options = ProcessOptions {
|
||||
dev_halt_at_slot: Some(snapshot_slot),
|
||||
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
|
||||
|
@ -1083,7 +1100,11 @@ fn main() {
|
|||
bank
|
||||
};
|
||||
|
||||
println!("Creating a snapshot of slot {}", bank.slot());
|
||||
println!(
|
||||
"Creating a version {} snapshot of slot {}",
|
||||
snapshot_version,
|
||||
bank.slot(),
|
||||
);
|
||||
assert!(bank.is_complete());
|
||||
bank.squash();
|
||||
bank.clean_accounts();
|
||||
|
@ -1095,7 +1116,7 @@ fn main() {
|
|||
});
|
||||
|
||||
let storages: Vec<_> = bank.get_snapshot_storages();
|
||||
snapshot_utils::add_snapshot(&temp_dir, &bank, &storages)
|
||||
snapshot_utils::add_snapshot(&temp_dir, &bank, &storages, snapshot_version)
|
||||
.and_then(|slot_snapshot_paths| {
|
||||
snapshot_utils::package_snapshot(
|
||||
&bank,
|
||||
|
@ -1105,6 +1126,7 @@ fn main() {
|
|||
output_directory,
|
||||
storages,
|
||||
CompressionType::Bzip2,
|
||||
snapshot_version,
|
||||
)
|
||||
})
|
||||
.and_then(|package| {
|
||||
|
|
|
@ -1210,6 +1210,7 @@ fn setup_snapshot_validator_config(
|
|||
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
|
||||
snapshot_path: PathBuf::from(snapshot_dir.path()),
|
||||
compression: CompressionType::Bzip2,
|
||||
snapshot_version: snapshot_utils::SnapshotVersion::default(),
|
||||
};
|
||||
|
||||
// Create the account paths
|
||||
|
|
|
@ -16,6 +16,8 @@ use std::{
|
|||
};
|
||||
use thiserror::Error;
|
||||
|
||||
pub use crate::snapshot_utils::SnapshotVersion;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum CompressionType {
|
||||
Bzip2,
|
||||
|
@ -36,6 +38,9 @@ pub struct SnapshotConfig {
|
|||
pub snapshot_path: PathBuf,
|
||||
|
||||
pub compression: CompressionType,
|
||||
|
||||
// Snapshot version to generate
|
||||
pub snapshot_version: SnapshotVersion,
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
|
@ -302,7 +307,12 @@ impl BankForks {
|
|||
|
||||
let storages: Vec<_> = bank.get_snapshot_storages();
|
||||
let mut add_snapshot_time = Measure::start("add-snapshot-ms");
|
||||
snapshot_utils::add_snapshot(&config.snapshot_path, &bank, &storages)?;
|
||||
snapshot_utils::add_snapshot(
|
||||
&config.snapshot_path,
|
||||
&bank,
|
||||
&storages,
|
||||
config.snapshot_version,
|
||||
)?;
|
||||
add_snapshot_time.stop();
|
||||
inc_new_counter_info!("add-snapshot-ms", add_snapshot_time.as_ms() as usize);
|
||||
|
||||
|
@ -321,6 +331,7 @@ impl BankForks {
|
|||
&config.snapshot_package_output_path,
|
||||
storages,
|
||||
config.compression.clone(),
|
||||
config.snapshot_version,
|
||||
)?;
|
||||
|
||||
accounts_package_sender.send(package)?;
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
use crate::bank_forks::CompressionType;
|
||||
use crate::snapshot_utils::SnapshotVersion;
|
||||
use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta};
|
||||
use solana_sdk::clock::Slot;
|
||||
use solana_sdk::hash::Hash;
|
||||
|
@ -22,6 +23,7 @@ pub struct AccountsPackage {
|
|||
pub tar_output_file: PathBuf,
|
||||
pub hash: Hash,
|
||||
pub compression: CompressionType,
|
||||
pub snapshot_version: SnapshotVersion,
|
||||
}
|
||||
|
||||
impl AccountsPackage {
|
||||
|
@ -34,6 +36,7 @@ impl AccountsPackage {
|
|||
tar_output_file: PathBuf,
|
||||
hash: Hash,
|
||||
compression: CompressionType,
|
||||
snapshot_version: SnapshotVersion,
|
||||
) -> Self {
|
||||
Self {
|
||||
root,
|
||||
|
@ -44,6 +47,7 @@ impl AccountsPackage {
|
|||
tar_output_file,
|
||||
hash,
|
||||
compression,
|
||||
snapshot_version,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -37,7 +37,7 @@ pub const TAR_VERSION_FILE: &str = "version";
|
|||
const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
|
||||
const VERSION_STRING_V1_1_0: &str = "1.1.0";
|
||||
const VERSION_STRING_V1_2_0: &str = "1.2.0";
|
||||
const OUTPUT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0;
|
||||
const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0;
|
||||
|
||||
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
|
||||
pub enum SnapshotVersion {
|
||||
|
@ -45,6 +45,18 @@ pub enum SnapshotVersion {
|
|||
V1_2_0,
|
||||
}
|
||||
|
||||
impl Default for SnapshotVersion {
|
||||
fn default() -> Self {
|
||||
DEFAULT_SNAPSHOT_VERSION
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for SnapshotVersion {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.write_str(From::from(*self))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SnapshotVersion> for &'static str {
|
||||
fn from(snapshot_version: SnapshotVersion) -> &'static str {
|
||||
match snapshot_version {
|
||||
|
@ -58,6 +70,15 @@ impl FromStr for SnapshotVersion {
|
|||
type Err = &'static str;
|
||||
|
||||
fn from_str(version_string: &str) -> std::result::Result<Self, Self::Err> {
|
||||
// Remove leading 'v' or 'V' from slice
|
||||
let version_string = if version_string
|
||||
.get(..1)
|
||||
.map_or(false, |s| s.eq_ignore_ascii_case("v"))
|
||||
{
|
||||
&version_string[1..]
|
||||
} else {
|
||||
version_string
|
||||
};
|
||||
match version_string {
|
||||
VERSION_STRING_V1_1_0 => Ok(SnapshotVersion::V1_1_0),
|
||||
VERSION_STRING_V1_2_0 => Ok(SnapshotVersion::V1_2_0),
|
||||
|
@ -134,6 +155,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||
snapshot_package_output_path: P,
|
||||
snapshot_storages: SnapshotStorages,
|
||||
compression: CompressionType,
|
||||
snapshot_version: SnapshotVersion,
|
||||
) -> Result<AccountsPackage> {
|
||||
// Hard link all the snapshots we need for this package
|
||||
let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?;
|
||||
|
@ -164,6 +186,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||
snapshot_package_output_file,
|
||||
bank.get_accounts_hash(),
|
||||
compression,
|
||||
snapshot_version,
|
||||
);
|
||||
|
||||
Ok(package)
|
||||
|
@ -234,7 +257,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
|
|||
// Write version file
|
||||
{
|
||||
let mut f = std::fs::File::create(staging_version_file)?;
|
||||
f.write_all(OUTPUT_SNAPSHOT_VERSION.as_str().as_bytes())?;
|
||||
f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?;
|
||||
}
|
||||
|
||||
let (compression_option, file_ext) = get_compression_ext(&snapshot_package.compression);
|
||||
|
@ -417,6 +440,7 @@ pub fn add_snapshot<P: AsRef<Path>>(
|
|||
snapshot_path: P,
|
||||
bank: &Bank,
|
||||
snapshot_storages: &[SnapshotStorage],
|
||||
snapshot_version: SnapshotVersion,
|
||||
) -> Result<SlotSnapshotPaths> {
|
||||
let slot = bank.slot();
|
||||
// snapshot_path/slot
|
||||
|
@ -432,13 +456,12 @@ pub fn add_snapshot<P: AsRef<Path>>(
|
|||
|
||||
let mut bank_serialize = Measure::start("bank-serialize-ms");
|
||||
let bank_snapshot_serializer = move |stream: &mut BufWriter<File>| -> Result<()> {
|
||||
let serde_style = match snapshot_version {
|
||||
SnapshotVersion::V1_1_0 => SerdeStyle::OLDER,
|
||||
SnapshotVersion::V1_2_0 => SerdeStyle::NEWER,
|
||||
};
|
||||
serialize_into(stream.by_ref(), bank)?;
|
||||
bankrc_to_stream(
|
||||
SerdeStyle::NEWER,
|
||||
stream.by_ref(),
|
||||
&bank.rc,
|
||||
snapshot_storages,
|
||||
)?;
|
||||
bankrc_to_stream(serde_style, stream.by_ref(), &bank.rc, snapshot_storages)?;
|
||||
Ok(())
|
||||
};
|
||||
let consumed_size =
|
||||
|
|
|
@ -6,7 +6,9 @@ use log::*;
|
|||
use rand::{thread_rng, Rng};
|
||||
use solana_clap_utils::{
|
||||
input_parsers::{keypair_of, keypairs_of, pubkey_of},
|
||||
input_validators::{is_keypair_or_ask_keyword, is_pubkey, is_pubkey_or_keypair, is_slot},
|
||||
input_validators::{
|
||||
is_keypair_or_ask_keyword, is_parsable, is_pubkey, is_pubkey_or_keypair, is_slot,
|
||||
},
|
||||
keypair::SKIP_SEED_PHRASE_VALIDATION_ARG,
|
||||
};
|
||||
use solana_client::rpc_client::RpcClient;
|
||||
|
@ -23,7 +25,7 @@ use solana_core::{
|
|||
use solana_download_utils::{download_genesis_if_missing, download_snapshot};
|
||||
use solana_perf::recycler::enable_recycler_warming;
|
||||
use solana_runtime::{
|
||||
bank_forks::{CompressionType, SnapshotConfig},
|
||||
bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion},
|
||||
hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
|
||||
};
|
||||
use solana_sdk::{
|
||||
|
@ -694,6 +696,15 @@ pub fn main() {
|
|||
.default_value("100")
|
||||
.help("Number of slots between generating accounts hash."),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("snapshot_version")
|
||||
.long("snapshot-version")
|
||||
.value_name("SNAPSHOT_VERSION")
|
||||
.validator(is_parsable::<SnapshotVersion>)
|
||||
.takes_value(true)
|
||||
.default_value(SnapshotVersion::default().into())
|
||||
.help("Output snapshot version"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("limit_ledger_size")
|
||||
.long("limit-ledger-size")
|
||||
|
@ -969,6 +980,15 @@ pub fn main() {
|
|||
_ => panic!("Compression type not recognized: {}", compression_str),
|
||||
}
|
||||
}
|
||||
let snapshot_version =
|
||||
matches
|
||||
.value_of("snapshot_version")
|
||||
.map_or(SnapshotVersion::default(), |s| {
|
||||
s.parse::<SnapshotVersion>().unwrap_or_else(|err| {
|
||||
eprintln!("Error: {}", err);
|
||||
exit(1)
|
||||
})
|
||||
});
|
||||
validator_config.snapshot_config = Some(SnapshotConfig {
|
||||
snapshot_interval_slots: if snapshot_interval_slots > 0 {
|
||||
snapshot_interval_slots
|
||||
|
@ -978,6 +998,7 @@ pub fn main() {
|
|||
snapshot_path,
|
||||
snapshot_package_output_path: ledger_path.clone(),
|
||||
compression: snapshot_compression,
|
||||
snapshot_version,
|
||||
});
|
||||
|
||||
validator_config.accounts_hash_interval_slots =
|
||||
|
|
Loading…
Reference in New Issue