blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add CLI options and runtime support for selection of output snapshot version. (#10536)

This commit is contained in:
Kristofer Peterson 2020-06-19 06:38:37 +01:00 committed by GitHub
parent b172b3369e
commit 6d81eede93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 228 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/accounts_hash_verifier.rs
View File

@ -176,6 +176,7 @@ mod tests {
use crate::cluster_info::make_accounts_hashes_message;
use crate::contact_info::ContactInfo;
use solana_runtime::bank_forks::CompressionType;
use solana_runtime::snapshot_utils::SnapshotVersion;
use solana_sdk::{
hash::hash,
signature::{Keypair, Signer},
@ -238,6 +239,7 @@ mod tests {
tar_output_file: PathBuf::from("."),
storages: vec![],
compression: CompressionType::Bzip2,
snapshot_version: SnapshotVersion::default(),
};
AccountsHashVerifier::process_accounts_package(

5
core/src/rpc_service.rs
View File

@ -353,7 +353,9 @@ mod tests {
genesis_utils::{create_genesis_config, GenesisConfigInfo},
get_tmp_ledger_path,
};
use solana_runtime::{bank::Bank, bank_forks::CompressionType};
use solana_runtime::{
bank::Bank, bank_forks::CompressionType, snapshot_utils::SnapshotVersion,
};
use solana_sdk::signature::Signer;
use std::net::{IpAddr, Ipv4Addr};
@ -445,6 +447,7 @@ mod tests {
snapshot_package_output_path: PathBuf::from("/"),
snapshot_path: PathBuf::from("/"),
compression: CompressionType::Bzip2,
snapshot_version: SnapshotVersion::default(),
}),
bank_forks,
RpcHealth::stub(),

3
core/src/snapshot_packager_service.rs
View File

@ -83,7 +83,7 @@ mod tests {
bank::BankSlotDelta,
bank_forks::CompressionType,
snapshot_package::AccountsPackage,
snapshot_utils::{self, SNAPSHOT_STATUS_CACHE_FILE_NAME},
snapshot_utils::{self, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME},
};
use solana_sdk::hash::Hash;
use std::{
@ -174,6 +174,7 @@ mod tests {
output_tar_path.clone(),
Hash::default(),
CompressionType::Bzip2,
SnapshotVersion::default(),
);
// Make tarball from packageable snapshot

202
core/tests/bank_forks.rs
View File

@ -1,5 +1,36 @@
// Long-running bank_forks tests
macro_rules! DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS {
($x:ident) => {
#[allow(non_snake_case)]
mod $x {
use super::*;
const SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::$x;
#[test]
fn test_bank_forks_status_cache_snapshot_n() {
run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION)
}
#[test]
fn test_bank_forks_snapshot_n() {
run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION)
}
#[test]
fn test_concurrent_snapshot_packaging() {
run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION)
}
#[test]
fn test_slots_to_snapshot() {
run_test_slots_to_snapshot(SNAPSHOT_VERSION)
}
}
};
}
#[cfg(test)]
mod tests {
use bincode::serialize_into;
@ -13,6 +44,7 @@ mod tests {
bank_forks::{BankForks, CompressionType, SnapshotConfig},
genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_utils,
snapshot_utils::SnapshotVersion,
status_cache::MAX_CACHE_ENTRIES,
};
use solana_sdk::{
@ -26,6 +58,9 @@ mod tests {
use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc};
use tempfile::TempDir;
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0);
DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0);
struct SnapshotTestConfig {
accounts_dir: TempDir,
snapshot_dir: TempDir,
@ -35,41 +70,47 @@ mod tests {
genesis_config_info: GenesisConfigInfo,
}
fn setup_snapshot_test(snapshot_interval_slots: u64) -> SnapshotTestConfig {
let accounts_dir = TempDir::new().unwrap();
let snapshot_dir = TempDir::new().unwrap();
let snapshot_output_path = TempDir::new().unwrap();
let genesis_config_info = create_genesis_config(10_000);
let bank0 = Bank::new_with_paths(
&genesis_config_info.genesis_config,
vec![accounts_dir.path().to_path_buf()],
&[],
);
bank0.freeze();
let mut bank_forks = BankForks::new(bank0);
bank_forks.accounts_hash_interval_slots = snapshot_interval_slots;
impl SnapshotTestConfig {
fn new(
snapshot_version: SnapshotVersion,
snapshot_interval_slots: u64,
) -> SnapshotTestConfig {
let accounts_dir = TempDir::new().unwrap();
let snapshot_dir = TempDir::new().unwrap();
let snapshot_output_path = TempDir::new().unwrap();
let genesis_config_info = create_genesis_config(10_000);
let bank0 = Bank::new_with_paths(
&genesis_config_info.genesis_config,
vec![accounts_dir.path().to_path_buf()],
&[],
);
bank0.freeze();
let mut bank_forks = BankForks::new(bank0);
bank_forks.accounts_hash_interval_slots = snapshot_interval_slots;
let snapshot_config = SnapshotConfig {
snapshot_interval_slots,
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()),
compression: CompressionType::Bzip2,
};
bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
SnapshotTestConfig {
accounts_dir,
snapshot_dir,
_snapshot_output_path: snapshot_output_path,
snapshot_config,
bank_forks,
genesis_config_info,
let snapshot_config = SnapshotConfig {
snapshot_interval_slots,
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()),
compression: CompressionType::Bzip2,
snapshot_version,
};
bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
SnapshotTestConfig {
accounts_dir,
snapshot_dir,
_snapshot_output_path: snapshot_output_path,
snapshot_config,
bank_forks,
genesis_config_info,
}
}
}
fn restore_from_snapshot(
old_bank_forks: &BankForks,
old_last_slot: Slot,
account_paths: Vec<PathBuf>,
account_paths: &[PathBuf],
) {
let (snapshot_path, snapshot_package_output_path) = old_bank_forks
.snapshot_config
@ -115,17 +156,20 @@ mod tests {
// also marks each bank as root and generates snapshots
// finally tries to restore from the last bank's snapshot and compares the restored bank to the
// `last_slot` bank
fn run_bank_forks_snapshot_n<F>(last_slot: Slot, f: F, set_root_interval: u64)
where
fn run_bank_forks_snapshot_n<F>(
snapshot_version: SnapshotVersion,
last_slot: Slot,
f: F,
set_root_interval: u64,
) where
F: Fn(&mut Bank, &Keypair),
{
solana_logger::setup();
// Set up snapshotting config
let mut snapshot_test_config = setup_snapshot_test(1);
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
let bank_forks = &mut snapshot_test_config.bank_forks;
let accounts_dir = &snapshot_test_config.accounts_dir;
let snapshot_config = &snapshot_test_config.snapshot_config;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
let (s, _r) = channel();
@ -143,36 +187,33 @@ mod tests {
}
// Generate a snapshot package for last bank
let last_bank = bank_forks.get(last_slot).unwrap();
let storages: Vec<_> = last_bank.get_snapshot_storages();
let slot_snapshot_paths =
snapshot_utils::get_snapshot_paths(&snapshot_config.snapshot_path);
let snapshot_config = &snapshot_test_config.snapshot_config;
let snapshot_path = &snapshot_config.snapshot_path;
let last_slot_snapshot_path = snapshot_utils::get_snapshot_paths(snapshot_path)
.pop()
.expect("no snapshots found in path");
let snapshot_package = snapshot_utils::package_snapshot(
last_bank,
slot_snapshot_paths
.last()
.expect("no snapshots found in path"),
&snapshot_config.snapshot_path,
&last_slot_snapshot_path,
snapshot_path,
&last_bank.src.roots(),
&snapshot_config.snapshot_package_output_path,
storages,
last_bank.get_snapshot_storages(),
CompressionType::Bzip2,
snapshot_version,
)
.unwrap();
snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
restore_from_snapshot(
bank_forks,
last_slot,
vec![accounts_dir.path().to_path_buf()],
);
restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]);
}
#[test]
fn test_bank_forks_snapshot_n() {
fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) {
// create banks up to slot 4 and create 1 new account in each bank. test that bank 4 snapshots
// and restores correctly
run_bank_forks_snapshot_n(
snapshot_version,
4,
|bank, mint_keypair| {
let key1 = Keypair::new().pubkey();
@ -203,24 +244,25 @@ mod tests {
}
}
#[test]
fn test_concurrent_snapshot_packaging() {
fn run_test_concurrent_snapshot_packaging(snapshot_version: SnapshotVersion) {
solana_logger::setup();
// Set up snapshotting config
let mut snapshot_test_config = setup_snapshot_test(1);
let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
let bank_forks = &mut snapshot_test_config.bank_forks;
let accounts_dir = &snapshot_test_config.accounts_dir;
let snapshots_dir = &snapshot_test_config.snapshot_dir;
let snapshot_config = &snapshot_test_config.snapshot_config;
let snapshot_path = &snapshot_config.snapshot_path;
let snapshot_package_output_path = &snapshot_config.snapshot_package_output_path;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
// Take snapshot of zeroth bank
let bank0 = bank_forks.get(0).unwrap();
let storages: Vec<_> = bank0.get_snapshot_storages();
snapshot_utils::add_snapshot(&snapshot_config.snapshot_path, bank0, &storages).unwrap();
let storages = bank0.get_snapshot_storages();
snapshot_utils::add_snapshot(snapshot_path, bank0, &storages, snapshot_version).unwrap();
// Set up snapshotting channels
let (sender, receiver) = channel();
@ -270,7 +312,7 @@ mod tests {
if slot == saved_slot as u64 {
let options = CopyOptions::new();
fs_extra::dir::copy(accounts_dir, &saved_accounts_dir, &options).unwrap();
let snapshot_paths: Vec<_> = fs::read_dir(&snapshot_config.snapshot_path)
let snapshot_paths = fs::read_dir(snapshot_path)
.unwrap()
.filter_map(|entry| {
let e = entry.unwrap();
@ -282,17 +324,13 @@ mod tests {
.unwrap_or(None)
})
.sorted()
.collect();
.last()
.unwrap();
// only save off the snapshot of this slot, we don't need the others.
fs_extra::dir::copy(
&snapshot_paths.last().unwrap(),
&saved_snapshots_dir,
&options,
)
.unwrap();
fs_extra::dir::copy(&snapshot_paths, &saved_snapshots_dir, &options).unwrap();
saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
&snapshot_config.snapshot_package_output_path,
snapshot_package_output_path,
&(slot, accounts_hash),
&CompressionType::Bzip2,
));
@ -302,11 +340,12 @@ mod tests {
// Purge all the outdated snapshots, including the ones needed to generate the package
// currently sitting in the channel
bank_forks.purge_old_snapshots();
let mut snapshot_paths = snapshot_utils::get_snapshot_paths(&snapshots_dir);
snapshot_paths.sort();
assert_eq!(
snapshot_paths.iter().map(|path| path.slot).collect_vec(),
(3..=MAX_CACHE_ENTRIES as u64 + 2).collect_vec()
snapshot_utils::get_snapshot_paths(&snapshots_dir)
.into_iter()
.map(|path| path.slot)
.eq(3..=MAX_CACHE_ENTRIES as u64 + 2),
true
);
// Create a SnapshotPackagerService to create tarballs from all the pending
@ -336,13 +375,12 @@ mod tests {
// before we compare, stick an empty status_cache in this dir so that the package comparison works
// This is needed since the status_cache is added by the packager and is not collected from
// the source dir for snapshots
let dummy_slot_deltas: Vec<BankSlotDelta> = vec![];
snapshot_utils::serialize_snapshot_data_file(
&saved_snapshots_dir
.path()
.join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILE_NAME),
|stream| {
serialize_into(stream, &dummy_slot_deltas)?;
serialize_into(stream, &[] as &[BankSlotDelta])?;
Ok(())
},
)
@ -358,16 +396,17 @@ mod tests {
);
}
#[test]
fn test_slots_to_snapshot() {
fn run_test_slots_to_snapshot(snapshot_version: SnapshotVersion) {
solana_logger::setup();
let num_set_roots = MAX_CACHE_ENTRIES * 2;
for add_root_interval in &[1, 3, 9] {
let (snapshot_sender, _snapshot_receiver) = channel();
// Make sure this test never clears bank.slots_since_snapshot
let mut snapshot_test_config =
setup_snapshot_test((add_root_interval * num_set_roots * 2) as u64);
let mut snapshot_test_config = SnapshotTestConfig::new(
snapshot_version,
(*add_root_interval * num_set_roots * 2) as u64,
);
let mut current_bank = snapshot_test_config.bank_forks[0].clone();
let snapshot_sender = Some(snapshot_sender);
for _ in 0..num_set_roots {
@ -386,21 +425,23 @@ mod tests {
}
let num_old_slots = num_set_roots * *add_root_interval - MAX_CACHE_ENTRIES + 1;
let expected_slots_to_snapshot = (num_old_slots as u64
..=num_set_roots as u64 * *add_root_interval as u64)
.collect_vec();
let expected_slots_to_snapshot =
num_old_slots as u64..=num_set_roots as u64 * *add_root_interval as u64;
let rooted_bank = snapshot_test_config
let slots_to_snapshot = snapshot_test_config
.bank_forks
.get(snapshot_test_config.bank_forks.root())
.unwrap();
let slots_to_snapshot = rooted_bank.src.roots();
assert_eq!(slots_to_snapshot, expected_slots_to_snapshot);
.unwrap()
.src
.roots();
assert_eq!(
slots_to_snapshot.into_iter().eq(expected_slots_to_snapshot),
true
);
}
}
#[test]
fn test_bank_forks_status_cache_snapshot_n() {
fn run_test_bank_forks_status_cache_snapshot_n(snapshot_version: SnapshotVersion) {
// create banks up to slot (MAX_CACHE_ENTRIES * 2) + 1 while transferring 1 lamport into 2 different accounts each time
// this is done to ensure the AccountStorageEntries keep getting cleaned up as the root moves
// ahead. Also tests the status_cache purge and status cache snapshotting.
@ -409,6 +450,7 @@ mod tests {
let key2 = Keypair::new().pubkey();
for set_root_interval in &[1, 4] {
run_bank_forks_snapshot_n(
snapshot_version,
(MAX_CACHE_ENTRIES * 2 + 1) as u64,
|bank, mint_keypair| {
let tx = system_transaction::transfer(

32
ledger-tool/src/main.rs
View File

@ -3,7 +3,7 @@ use clap::{
ArgMatches, SubCommand,
};
use serde_json::json;
use solana_clap_utils::input_validators::is_slot;
use solana_clap_utils::input_validators::{is_parsable, is_slot};
use solana_ledger::{
bank_forks_utils,
blockstore::Blockstore,
@ -16,6 +16,7 @@ use solana_runtime::{
bank_forks::{BankForks, CompressionType, SnapshotConfig},
hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
snapshot_utils,
snapshot_utils::SnapshotVersion,
};
use solana_sdk::{
clock::Slot, genesis_config::GenesisConfig, native_token::lamports_to_sol, pubkey::Pubkey,
@ -574,6 +575,7 @@ fn load_bank_forks(
snapshot_package_output_path: ledger_path.clone(),
snapshot_path,
compression: CompressionType::Bzip2,
snapshot_version: SnapshotVersion::default(),
})
};
let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
@ -654,7 +656,13 @@ fn main() {
.takes_value(true)
.default_value(&default_genesis_archive_unpacked_size)
.help("maximum total uncompressed size of unpacked genesis archive");
let snapshot_version_arg = Arg::with_name("snapshot_version")
.long("snapshot-version")
.value_name("SNAPSHOT_VERSION")
.validator(is_parsable::<SnapshotVersion>)
.takes_value(true)
.default_value(SnapshotVersion::default().into())
.help("Output snapshot version");
let matches = App::new(crate_name!())
.about(crate_description!())
.version(solana_version::version!())
@ -775,6 +783,7 @@ fn main() {
.arg(&account_paths_arg)
.arg(&hard_forks_arg)
.arg(&max_genesis_archive_unpacked_size_arg)
.arg(&snapshot_version_arg)
.arg(
Arg::with_name("snapshot_slot")
.index(1)
@ -1049,7 +1058,15 @@ fn main() {
let snapshot_slot = value_t_or_exit!(arg_matches, "snapshot_slot", Slot);
let output_directory = value_t_or_exit!(arg_matches, "output_directory", String);
let warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok();
let snapshot_version =
arg_matches
.value_of("snapshot_version")
.map_or(SnapshotVersion::default(), |s| {
s.parse::<SnapshotVersion>().unwrap_or_else(|e| {
eprintln!("Error: {}", e);
exit(1)
})
});
let process_options = ProcessOptions {
dev_halt_at_slot: Some(snapshot_slot),
new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
@ -1083,7 +1100,11 @@ fn main() {
bank
};
println!("Creating a snapshot of slot {}", bank.slot());
println!(
"Creating a version {} snapshot of slot {}",
snapshot_version,
bank.slot(),
);
assert!(bank.is_complete());
bank.squash();
bank.clean_accounts();
@ -1095,7 +1116,7 @@ fn main() {
});
let storages: Vec<_> = bank.get_snapshot_storages();
snapshot_utils::add_snapshot(&temp_dir, &bank, &storages)
snapshot_utils::add_snapshot(&temp_dir, &bank, &storages, snapshot_version)
.and_then(|slot_snapshot_paths| {
snapshot_utils::package_snapshot(
&bank,
@ -1105,6 +1126,7 @@ fn main() {
output_directory,
storages,
CompressionType::Bzip2,
snapshot_version,
)
})
.and_then(|package| {

1
local-cluster/tests/local_cluster.rs
View File

@ -1210,6 +1210,7 @@ fn setup_snapshot_validator_config(
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()),
compression: CompressionType::Bzip2,
snapshot_version: snapshot_utils::SnapshotVersion::default(),
};
// Create the account paths

13
runtime/src/bank_forks.rs
View File

@ -16,6 +16,8 @@ use std::{
};
use thiserror::Error;
pub use crate::snapshot_utils::SnapshotVersion;
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum CompressionType {
Bzip2,
@ -36,6 +38,9 @@ pub struct SnapshotConfig {
pub snapshot_path: PathBuf,
pub compression: CompressionType,
// Snapshot version to generate
pub snapshot_version: SnapshotVersion,
}
#[derive(Error, Debug)]
@ -302,7 +307,12 @@ impl BankForks {
let storages: Vec<_> = bank.get_snapshot_storages();
let mut add_snapshot_time = Measure::start("add-snapshot-ms");
snapshot_utils::add_snapshot(&config.snapshot_path, &bank, &storages)?;
snapshot_utils::add_snapshot(
&config.snapshot_path,
&bank,
&storages,
config.snapshot_version,
)?;
add_snapshot_time.stop();
inc_new_counter_info!("add-snapshot-ms", add_snapshot_time.as_ms() as usize);
@ -321,6 +331,7 @@ impl BankForks {
&config.snapshot_package_output_path,
storages,
config.compression.clone(),
config.snapshot_version,
)?;
accounts_package_sender.send(package)?;

4
runtime/src/snapshot_package.rs
View File

@ -1,4 +1,5 @@
use crate::bank_forks::CompressionType;
use crate::snapshot_utils::SnapshotVersion;
use crate::{accounts_db::SnapshotStorages, bank::BankSlotDelta};
use solana_sdk::clock::Slot;
use solana_sdk::hash::Hash;
@ -22,6 +23,7 @@ pub struct AccountsPackage {
pub tar_output_file: PathBuf,
pub hash: Hash,
pub compression: CompressionType,
pub snapshot_version: SnapshotVersion,
}
impl AccountsPackage {
@ -34,6 +36,7 @@ impl AccountsPackage {
tar_output_file: PathBuf,
hash: Hash,
compression: CompressionType,
snapshot_version: SnapshotVersion,
) -> Self {
Self {
root,
@ -44,6 +47,7 @@ impl AccountsPackage {
tar_output_file,
hash,
compression,
snapshot_version,
}
}
}

39
runtime/src/snapshot_utils.rs
View File

@ -37,7 +37,7 @@ pub const TAR_VERSION_FILE: &str = "version";
const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
const VERSION_STRING_V1_1_0: &str = "1.1.0";
const VERSION_STRING_V1_2_0: &str = "1.2.0";
const OUTPUT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0;
const DEFAULT_SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::V1_2_0;
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
pub enum SnapshotVersion {
@ -45,6 +45,18 @@ pub enum SnapshotVersion {
V1_2_0,
}
impl Default for SnapshotVersion {
fn default() -> Self {
DEFAULT_SNAPSHOT_VERSION
}
}
impl fmt::Display for SnapshotVersion {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str(From::from(*self))
}
}
impl From<SnapshotVersion> for &'static str {
fn from(snapshot_version: SnapshotVersion) -> &'static str {
match snapshot_version {
@ -58,6 +70,15 @@ impl FromStr for SnapshotVersion {
type Err = &'static str;
fn from_str(version_string: &str) -> std::result::Result<Self, Self::Err> {
// Remove leading 'v' or 'V' from slice
let version_string = if version_string
.get(..1)
.map_or(false, |s| s.eq_ignore_ascii_case("v"))
{
&version_string[1..]
} else {
version_string
};
match version_string {
VERSION_STRING_V1_1_0 => Ok(SnapshotVersion::V1_1_0),
VERSION_STRING_V1_2_0 => Ok(SnapshotVersion::V1_2_0),
@ -134,6 +155,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
snapshot_package_output_path: P,
snapshot_storages: SnapshotStorages,
compression: CompressionType,
snapshot_version: SnapshotVersion,
) -> Result<AccountsPackage> {
// Hard link all the snapshots we need for this package
let snapshot_hard_links_dir = tempfile::tempdir_in(snapshot_path)?;
@ -164,6 +186,7 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
snapshot_package_output_file,
bank.get_accounts_hash(),
compression,
snapshot_version,
);
Ok(package)
@ -234,7 +257,7 @@ pub fn archive_snapshot_package(snapshot_package: &AccountsPackage) -> Result<()
// Write version file
{
let mut f = std::fs::File::create(staging_version_file)?;
f.write_all(OUTPUT_SNAPSHOT_VERSION.as_str().as_bytes())?;
f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())?;
}
let (compression_option, file_ext) = get_compression_ext(&snapshot_package.compression);
@ -417,6 +440,7 @@ pub fn add_snapshot<P: AsRef<Path>>(
snapshot_path: P,
bank: &Bank,
snapshot_storages: &[SnapshotStorage],
snapshot_version: SnapshotVersion,
) -> Result<SlotSnapshotPaths> {
let slot = bank.slot();
// snapshot_path/slot
@ -432,13 +456,12 @@ pub fn add_snapshot<P: AsRef<Path>>(
let mut bank_serialize = Measure::start("bank-serialize-ms");
let bank_snapshot_serializer = move |stream: &mut BufWriter<File>| -> Result<()> {
let serde_style = match snapshot_version {
SnapshotVersion::V1_1_0 => SerdeStyle::OLDER,
SnapshotVersion::V1_2_0 => SerdeStyle::NEWER,
};
serialize_into(stream.by_ref(), bank)?;
bankrc_to_stream(
SerdeStyle::NEWER,
stream.by_ref(),
&bank.rc,
snapshot_storages,
)?;
bankrc_to_stream(serde_style, stream.by_ref(), &bank.rc, snapshot_storages)?;
Ok(())
};
let consumed_size =

25
validator/src/main.rs
View File

@ -6,7 +6,9 @@ use log::*;
use rand::{thread_rng, Rng};
use solana_clap_utils::{
input_parsers::{keypair_of, keypairs_of, pubkey_of},
input_validators::{is_keypair_or_ask_keyword, is_pubkey, is_pubkey_or_keypair, is_slot},
input_validators::{
is_keypair_or_ask_keyword, is_parsable, is_pubkey, is_pubkey_or_keypair, is_slot,
},
keypair::SKIP_SEED_PHRASE_VALIDATION_ARG,
};
use solana_client::rpc_client::RpcClient;
@ -23,7 +25,7 @@ use solana_core::{
use solana_download_utils::{download_genesis_if_missing, download_snapshot};
use solana_perf::recycler::enable_recycler_warming;
use solana_runtime::{
bank_forks::{CompressionType, SnapshotConfig},
bank_forks::{CompressionType, SnapshotConfig, SnapshotVersion},
hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
};
use solana_sdk::{
@ -694,6 +696,15 @@ pub fn main() {
.default_value("100")
.help("Number of slots between generating accounts hash."),
)
.arg(
Arg::with_name("snapshot_version")
.long("snapshot-version")
.value_name("SNAPSHOT_VERSION")
.validator(is_parsable::<SnapshotVersion>)
.takes_value(true)
.default_value(SnapshotVersion::default().into())
.help("Output snapshot version"),
)
.arg(
Arg::with_name("limit_ledger_size")
.long("limit-ledger-size")
@ -969,6 +980,15 @@ pub fn main() {
_ => panic!("Compression type not recognized: {}", compression_str),
}
}
let snapshot_version =
matches
.value_of("snapshot_version")
.map_or(SnapshotVersion::default(), |s| {
s.parse::<SnapshotVersion>().unwrap_or_else(|err| {
eprintln!("Error: {}", err);
exit(1)
})
});
validator_config.snapshot_config = Some(SnapshotConfig {
snapshot_interval_slots: if snapshot_interval_slots > 0 {
snapshot_interval_slots
@ -978,6 +998,7 @@ pub fn main() {
snapshot_path,
snapshot_package_output_path: ledger_path.clone(),
compression: snapshot_compression,
snapshot_version,
});
validator_config.accounts_hash_interval_slots =