blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add incremental snapshot utils (#18504) 

This commit adds high-level functions for creating and loading-from
incremental snapshots, plus all low-level functions required to perform
those tasks.  This commit **does not** add taking incremental snapshots
as part of a running validator, nor starting up a node with an
incremental snapshot; just laying ground work.

Additionally, `snapshot_utils` and `serde_snapshot` have been
refactored to use a common code paths for the different snapshots.

Also of note, some renaming has happened:
  1. Snapshots are now either `full_` or `incremental_` throughout the
     codebase.  If not specified, the code applies to both.
  2. Bank snapshots now are called "bank snapshots"
     (before they were called "slot snapshots", "bank snapshots", or
      just "snapshots").  The one exception is within `Bank`, where they
     are still just "snapshots", because they are already "bank
     snapshots".
  3. Snapshot archives now have `_archive` in the code.  This
     should clear up an ambiguity between bank snapshots and snapshot
     archives.
This commit is contained in:
Brooks Prumo 2021-07-22 14:40:37 -05:00 committed by GitHub
parent 7fc4cfebc8
commit d1debcd971
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 1773 additions and 507 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/src/accounts_hash_verifier.rs
View File

@ -99,6 +99,7 @@ impl AccountsHashVerifier {
let accounts_package = solana_runtime::snapshot_utils::process_accounts_package_pre(
accounts_package,
thread_pool,
None,
);
Self::process_accounts_package(
accounts_package,

4
core/src/snapshot_packager_service.rs
View File

@ -156,7 +156,7 @@ mod tests {
}
// Create a packageable snapshot
let output_tar_path = snapshot_utils::build_snapshot_archive_path(
let output_tar_path = snapshot_utils::build_full_snapshot_archive_path(
snapshot_package_output_path,
42,
&Hash::default(),
@ -177,7 +177,7 @@ mod tests {
// Make tarball from packageable snapshot
snapshot_utils::archive_snapshot_package(
&snapshot_package,
snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();

6
core/src/test_validator.rs
View File

@ -13,7 +13,9 @@ use {
genesis_utils::create_genesis_config_with_leader_ex,
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
snapshot_config::SnapshotConfig,
snapshot_utils::{ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
snapshot_utils::{
ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
},
},
solana_sdk::{
account::{Account, AccountSharedData},
@ -492,7 +494,7 @@ impl TestValidator {
snapshot_package_output_path: ledger_path.to_path_buf(),
archive_format: ArchiveFormat::Tar,
snapshot_version: SnapshotVersion::default(),
maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
}),
enforce_ulimit_nofile: false,
warp_slot: config.warp_slot,

2
core/src/validator.rs
View File

@ -1187,7 +1187,7 @@ fn new_banks_from_ledger(
);
leader_schedule_cache.set_root(&bank_forks.root_bank());
let archive_file = solana_runtime::snapshot_utils::bank_to_snapshot_archive(
let archive_file = solana_runtime::snapshot_utils::bank_to_full_snapshot_archive(
ledger_path,
&bank_forks.root_bank(),
None,

46
core/tests/snapshots.rs
View File

@ -49,7 +49,9 @@ mod tests {
bank_forks::BankForks,
genesis_utils::{create_genesis_config, GenesisConfigInfo},
snapshot_config::SnapshotConfig,
snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
snapshot_utils::{
self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
},
status_cache::MAX_CACHE_ENTRIES,
};
use solana_sdk::{
@ -119,7 +121,7 @@ mod tests {
snapshot_path: PathBuf::from(snapshot_dir.path()),
archive_format: ArchiveFormat::TarBzip2,
snapshot_version,
maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
};
bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
SnapshotTestConfig {
@ -148,7 +150,7 @@ mod tests {
let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
let check_hash_calculation = false;
let (deserialized_bank, _timing) = snapshot_utils::bank_from_snapshot_archive(
let (deserialized_bank, _timing) = snapshot_utils::bank_from_snapshot_archives(
account_paths,
&[],
&old_bank_forks
@ -156,12 +158,13 @@ mod tests {
.as_ref()
.unwrap()
.snapshot_path,
snapshot_utils::build_snapshot_archive_path(
snapshot_utils::build_full_snapshot_archive_path(
snapshot_package_output_path.to_path_buf(),
old_last_bank.slot(),
&old_last_bank.get_accounts_hash(),
ArchiveFormat::TarBzip2,
),
None,
ArchiveFormat::TarBzip2,
old_genesis_config,
None,
@ -181,10 +184,10 @@ mod tests {
.clone();
assert_eq!(*bank, deserialized_bank);
let slot_snapshot_paths = snapshot_utils::get_snapshot_paths(&snapshot_path);
let bank_snapshot_infos = snapshot_utils::get_bank_snapshots(&snapshot_path);
for p in slot_snapshot_paths {
snapshot_utils::remove_snapshot(p.slot, &snapshot_path).unwrap();
for p in bank_snapshot_infos {
snapshot_utils::remove_bank_snapshot(p.slot, &snapshot_path).unwrap();
}
}
@ -235,12 +238,11 @@ mod tests {
let last_bank = bank_forks.get(last_slot).unwrap();
let snapshot_config = &snapshot_test_config.snapshot_config;
let snapshot_path = &snapshot_config.snapshot_path;
let last_slot_snapshot_path = snapshot_utils::get_snapshot_paths(snapshot_path)
.pop()
let last_bank_snapshot_info = snapshot_utils::get_highest_bank_snapshot_info(snapshot_path)
.expect("no snapshots found in path");
let snapshot_package = snapshot_utils::package_snapshot(
let snapshot_package = snapshot_utils::package_full_snapshot(
last_bank,
&last_slot_snapshot_path,
&last_bank_snapshot_info,
snapshot_path,
last_bank.src.slot_deltas(&last_bank.src.roots()),
&snapshot_config.snapshot_package_output_path,
@ -253,10 +255,11 @@ mod tests {
let snapshot_package = snapshot_utils::process_accounts_package_pre(
snapshot_package,
Some(last_bank.get_thread_pool()),
None,
);
snapshot_utils::archive_snapshot_package(
&snapshot_package,
DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
)
.unwrap();
@ -325,7 +328,8 @@ mod tests {
// Take snapshot of zeroth bank
let bank0 = bank_forks.get(0).unwrap();
let storages = bank0.get_snapshot_storages();
snapshot_utils::add_snapshot(snapshot_path, bank0, &storages, snapshot_version).unwrap();
snapshot_utils::add_bank_snapshot(snapshot_path, bank0, &storages, snapshot_version)
.unwrap();
// Set up snapshotting channels
let (sender, receiver) = channel();
@ -343,7 +347,7 @@ mod tests {
let saved_slot = 4;
let mut saved_archive_path = None;
for forks in 0..snapshot_utils::MAX_SNAPSHOTS + 2 {
for forks in 0..snapshot_utils::MAX_BANK_SNAPSHOTS + 2 {
let bank = Bank::new_from_parent(
&bank_forks[forks as u64],
&Pubkey::default(),
@ -420,7 +424,7 @@ mod tests {
let options = CopyOptions::new();
fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
saved_archive_path = Some(snapshot_utils::build_snapshot_archive_path(
saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path(
snapshot_package_output_path.to_path_buf(),
slot,
&accounts_hash,
@ -431,11 +435,14 @@ mod tests {
// Purge all the outdated snapshots, including the ones needed to generate the package
// currently sitting in the channel
snapshot_utils::purge_old_snapshots(snapshot_path);
assert!(snapshot_utils::get_snapshot_paths(&snapshots_dir)
snapshot_utils::purge_old_bank_snapshots(snapshot_path);
let mut bank_snapshot_infos = snapshot_utils::get_bank_snapshots(&snapshots_dir);
bank_snapshot_infos.sort_unstable();
assert!(bank_snapshot_infos
.into_iter()
.map(|path| path.slot)
.eq(3..=snapshot_utils::MAX_SNAPSHOTS as u64 + 2));
.eq(3..=snapshot_utils::MAX_BANK_SNAPSHOTS as u64 + 2));
// Create a SnapshotPackagerService to create tarballs from all the pending
// SnapshotPackage's on the channel. By the time this service starts, we have already
@ -453,7 +460,7 @@ mod tests {
None,
&exit,
&cluster_info,
DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
);
let thread_pool = accounts_db::make_min_priority_thread_pool();
@ -471,6 +478,7 @@ mod tests {
solana_runtime::snapshot_utils::process_accounts_package_pre(
snapshot_package,
Some(&thread_pool),
None,
);
*pending_snapshot_package.lock().unwrap() = Some(snapshot_package);
}

8
download-utils/src/lib.rs
View File

@ -246,13 +246,13 @@ pub fn download_genesis_if_missing(
pub fn download_snapshot<'a, 'b>(
rpc_addr: &SocketAddr,
snapshot_output_dir: &Path,
snapshot_archives_dir: &Path,
desired_snapshot_hash: (Slot, Hash),
use_progress_bar: bool,
maximum_snapshots_to_retain: usize,
progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>,
) -> Result<(), String> {
snapshot_utils::purge_old_snapshot_archives(snapshot_output_dir, maximum_snapshots_to_retain);
snapshot_utils::purge_old_snapshot_archives(snapshot_archives_dir, maximum_snapshots_to_retain);
for compression in &[
ArchiveFormat::TarZstd,
@ -260,8 +260,8 @@ pub fn download_snapshot<'a, 'b>(
ArchiveFormat::TarBzip2,
ArchiveFormat::Tar, // `solana-test-validator` creates uncompressed snapshots
] {
let desired_snapshot_package = snapshot_utils::build_snapshot_archive_path(
snapshot_output_dir.to_path_buf(),
let desired_snapshot_package = snapshot_utils::build_full_snapshot_archive_path(
snapshot_archives_dir.to_path_buf(),
desired_snapshot_hash.0,
&desired_snapshot_hash.1,
*compression,

10
ledger-tool/src/main.rs
View File

@ -30,7 +30,9 @@ use solana_runtime::{
bank_forks::BankForks,
hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
snapshot_config::SnapshotConfig,
snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
snapshot_utils::{
self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
},
};
use solana_sdk::{
account::{AccountSharedData, ReadableAccount, WritableAccount},
@ -697,7 +699,7 @@ fn load_bank_forks(
snapshot_path,
archive_format: ArchiveFormat::TarBzip2,
snapshot_version: SnapshotVersion::default(),
maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
})
};
let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
@ -904,7 +906,7 @@ fn main() {
.default_value(SnapshotVersion::default().into())
.help("Output snapshot version");
let default_max_snapshot_to_retain = &DEFAULT_MAX_SNAPSHOTS_TO_RETAIN.to_string();
let default_max_snapshot_to_retain = &DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string();
let maximum_snapshots_to_retain_arg = Arg::with_name("maximum_snapshots_to_retain")
.long("maximum-snapshots-to-retain")
.value_name("NUMBER")
@ -2196,7 +2198,7 @@ fn main() {
bank.slot(),
);
let archive_file = snapshot_utils::bank_to_snapshot_archive(
let archive_file = snapshot_utils::bank_to_full_snapshot_archive(
ledger_path,
&bank,
Some(snapshot_version),

35
ledger/src/bank_forks_utils.rs
View File

@ -11,7 +11,7 @@ use solana_entry::entry::VerifyRecyclers;
use solana_runtime::{
bank_forks::BankForks,
snapshot_config::SnapshotConfig,
snapshot_utils::{self, SnapshotArchiveInfo},
snapshot_utils::{self, FullSnapshotArchiveInfo},
};
use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash};
use std::{fs, path::PathBuf, process, result};
@ -53,9 +53,11 @@ pub fn load(
fs::create_dir_all(&snapshot_config.snapshot_path)
.expect("Couldn't create snapshot directory");
if let Some(snapshot_archive_info) = snapshot_utils::get_highest_snapshot_archive_info(
&snapshot_config.snapshot_package_output_path,
) {
if let Some(full_snapshot_archive_info) =
snapshot_utils::get_highest_full_snapshot_archive_info(
&snapshot_config.snapshot_package_output_path,
)
{
return load_from_snapshot(
genesis_config,
blockstore,
@ -65,7 +67,7 @@ pub fn load(
process_options,
transaction_status_sender,
cache_block_meta_sender,
&snapshot_archive_info,
&full_snapshot_archive_info,
);
} else {
info!("No snapshot package available; will load from genesis");
@ -113,11 +115,11 @@ fn load_from_snapshot(
process_options: ProcessOptions,
transaction_status_sender: Option<&TransactionStatusSender>,
cache_block_meta_sender: Option<&CacheBlockMetaSender>,
snapshot_archive_info: &SnapshotArchiveInfo,
full_snapshot_archive_info: &FullSnapshotArchiveInfo,
) -> LoadResult {
info!(
"Loading snapshot package: {:?}",
&snapshot_archive_info.path
full_snapshot_archive_info.path()
);
// Fail hard here if snapshot fails to load, don't silently continue
@ -126,12 +128,13 @@ fn load_from_snapshot(
process::exit(1);
}
let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archive(
let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archives(
&account_paths,
&process_options.frozen_accounts,
&snapshot_config.snapshot_path,
&snapshot_archive_info.path,
snapshot_archive_info.archive_format,
full_snapshot_archive_info.path(),
None,
*full_snapshot_archive_info.archive_format(),
genesis_config,
process_options.debug_keys.clone(),
Some(&crate::builtins::get(process_options.bpf_jit)),
@ -152,10 +155,18 @@ fn load_from_snapshot(
deserialized_bank.get_accounts_hash(),
);
if deserialized_bank_slot_and_hash != (snapshot_archive_info.slot, snapshot_archive_info.hash) {
if deserialized_bank_slot_and_hash
!= (
*full_snapshot_archive_info.slot(),
*full_snapshot_archive_info.hash(),
)
{
error!(
"Snapshot has mismatch:\narchive: {:?}\ndeserialized: {:?}",
(snapshot_archive_info.slot, snapshot_archive_info.hash),
(
full_snapshot_archive_info.slot(),
full_snapshot_archive_info.hash()
),
deserialized_bank_slot_and_hash
);
process::exit(1);

11
ledger/src/blockstore_processor.rs
View File

@ -567,7 +567,16 @@ fn do_process_blockstore_from_root(
("slot", bank_forks.root(), i64),
("forks", initial_forks.len(), i64),
("calculate_capitalization_us", time_cap.as_us(), i64),
("untar_us", timings.untar_us, i64),
(
"full_snapshot_untar_us",
timings.full_snapshot_untar_us,
i64
),
(
"incremental_snapshot_untar_us",
timings.incremental_snapshot_untar_us,
i64
),
(
"rebuild_bank_from_snapshots_us",
timings.rebuild_bank_from_snapshots_us,

66
local-cluster/tests/local_cluster.rs
View File

@ -1642,25 +1642,15 @@ fn test_snapshot_download() {
trace!("Waiting for snapshot");
let (archive_filename, archive_snapshot_hash) =
wait_for_next_snapshot(&cluster, snapshot_package_output_path);
trace!("found: {:?}", archive_filename);
let validator_archive_path = snapshot_utils::build_snapshot_archive_path(
validator_snapshot_test_config
.snapshot_output_path
.path()
.to_path_buf(),
archive_snapshot_hash.0,
&archive_snapshot_hash.1,
ArchiveFormat::TarBzip2,
);
// Download the snapshot, then boot a validator from it.
download_snapshot(
&cluster.entry_point_info.rpc,
&validator_archive_path,
validator_snapshot_test_config.snapshot_archives_dir.path(),
archive_snapshot_hash,
false,
snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
&mut None,
)
.unwrap();
@ -1720,9 +1710,9 @@ fn test_snapshot_restart_tower() {
wait_for_next_snapshot(&cluster, snapshot_package_output_path);
// Copy archive to validator's snapshot output directory
let validator_archive_path = snapshot_utils::build_snapshot_archive_path(
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
validator_snapshot_test_config
.snapshot_output_path
.snapshot_archives_dir
.path()
.to_path_buf(),
archive_snapshot_hash.0,
@ -1783,7 +1773,7 @@ fn test_snapshots_blockstore_floor() {
let archive_info = loop {
let archive =
snapshot_utils::get_highest_snapshot_archive_info(&snapshot_package_output_path);
snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_package_output_path);
if archive.is_some() {
trace!("snapshot exists");
break archive.unwrap();
@ -1792,17 +1782,17 @@ fn test_snapshots_blockstore_floor() {
};
// Copy archive to validator's snapshot output directory
let validator_archive_path = snapshot_utils::build_snapshot_archive_path(
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
validator_snapshot_test_config
.snapshot_output_path
.snapshot_archives_dir
.path()
.to_path_buf(),
archive_info.slot,
&archive_info.hash,
*archive_info.slot(),
archive_info.hash(),
ArchiveFormat::TarBzip2,
);
fs::hard_link(archive_info.path, &validator_archive_path).unwrap();
let slot_floor = archive_info.slot;
fs::hard_link(archive_info.path(), &validator_archive_path).unwrap();
let slot_floor = *archive_info.slot();
// Start up a new node from a snapshot
let validator_stake = 5;
@ -3284,19 +3274,25 @@ fn wait_for_next_snapshot(
last_slot
);
loop {
if let Some(snapshot_archive_info) =
snapshot_utils::get_highest_snapshot_archive_info(snapshot_package_output_path)
if let Some(full_snapshot_archive_info) =
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_package_output_path)
{
trace!("snapshot for slot {} exists", snapshot_archive_info.slot);
if snapshot_archive_info.slot >= last_slot {
trace!(
"full snapshot for slot {} exists",
full_snapshot_archive_info.slot()
);
if *full_snapshot_archive_info.slot() >= last_slot {
return (
snapshot_archive_info.path,
(snapshot_archive_info.slot, snapshot_archive_info.hash),
full_snapshot_archive_info.path().clone(),
(
*full_snapshot_archive_info.slot(),
*full_snapshot_archive_info.hash(),
),
);
}
trace!(
"snapshot slot {} < last_slot {}",
snapshot_archive_info.slot,
"full snapshot slot {} < last_slot {}",
full_snapshot_archive_info.slot(),
last_slot
);
}
@ -3323,7 +3319,7 @@ fn generate_account_paths(num_account_paths: usize) -> (Vec<TempDir>, Vec<PathBu
struct SnapshotValidatorConfig {
_snapshot_dir: TempDir,
snapshot_output_path: TempDir,
snapshot_archives_dir: TempDir,
account_storage_dirs: Vec<TempDir>,
validator_config: ValidatorConfig,
}
@ -3334,14 +3330,14 @@ fn setup_snapshot_validator_config(
) -> SnapshotValidatorConfig {
// Create the snapshot config
let snapshot_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let snapshot_output_path = tempfile::tempdir_in(farf_dir()).unwrap();
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let snapshot_config = SnapshotConfig {
snapshot_interval_slots,
snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
snapshot_path: PathBuf::from(snapshot_dir.path()),
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
snapshot_path: snapshot_dir.path().to_path_buf(),
archive_format: ArchiveFormat::TarBzip2,
snapshot_version: snapshot_utils::SnapshotVersion::default(),
maximum_snapshots_to_retain: snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
maximum_snapshots_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
};
// Create the account paths
@ -3357,7 +3353,7 @@ fn setup_snapshot_validator_config(
SnapshotValidatorConfig {
_snapshot_dir: snapshot_dir,
snapshot_output_path,
snapshot_archives_dir,
account_storage_dirs,
validator_config,
}

1
measure/src/measure.rs
View File

@ -1,6 +1,7 @@
use solana_sdk::timing::duration_as_ns;
use std::{fmt, time::Instant};
#[derive(Debug)]
pub struct Measure {
name: &'static str,
start: Instant,

2
rpc/src/rpc.rs
View File

@ -2271,7 +2271,7 @@ pub mod rpc_minimal {
meta.snapshot_config
.and_then(|snapshot_config| {
snapshot_utils::get_highest_snapshot_archive_slot(
snapshot_utils::get_highest_full_snapshot_archive_slot(
&snapshot_config.snapshot_package_output_path,
)
})

14
rpc/src/rpc_service.rs
View File

@ -205,14 +205,14 @@ impl RequestMiddleware for RpcRequestMiddleware {
if let Some(ref snapshot_config) = self.snapshot_config {
if request.uri().path() == "/snapshot.tar.bz2" {
// Convenience redirect to the latest snapshot
return if let Some(snapshot_archive_info) =
snapshot_utils::get_highest_snapshot_archive_info(
return if let Some(full_snapshot_archive_info) =
snapshot_utils::get_highest_full_snapshot_archive_info(
&snapshot_config.snapshot_package_output_path,
) {
RpcRequestMiddleware::redirect(&format!(
"/{}",
snapshot_archive_info
.path
full_snapshot_archive_info
.path()
.file_name()
.unwrap_or_else(|| std::ffi::OsStr::new(""))
.to_str()
@ -500,7 +500,9 @@ mod tests {
},
solana_runtime::{
bank::Bank,
snapshot_utils::{ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
snapshot_utils::{
ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
},
},
solana_sdk::{
genesis_config::{ClusterType, DEFAULT_GENESIS_ARCHIVE},
@ -606,7 +608,7 @@ mod tests {
snapshot_path: PathBuf::from("/"),
archive_format: ArchiveFormat::TarBzip2,
snapshot_version: SnapshotVersion::default(),
maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
}),
bank_forks,
RpcHealth::stub(),

2
runtime/src/accounts_background_service.rs
View File

@ -192,7 +192,7 @@ impl SnapshotRequestHandler {
// Cleanup outdated snapshots
let mut purge_old_snapshots_time = Measure::start("purge_old_snapshots_time");
snapshot_utils::purge_old_snapshots(&self.snapshot_config.snapshot_path);
snapshot_utils::purge_old_bank_snapshots(&self.snapshot_config.snapshot_path);
purge_old_snapshots_time.stop();
total_time.stop();

17
runtime/src/bank.rs
View File

@ -38,7 +38,7 @@ use crate::{
AccountAddressFilter, Accounts, TransactionAccounts, TransactionLoadResult,
TransactionLoaders,
},
accounts_db::{AccountShrinkThreshold, ErrorCounters, SnapshotStorages},
accounts_db::{AccountShrinkThreshold, ErrorCounters, SnapshotStorage, SnapshotStorages},
accounts_index::{AccountSecondaryIndexes, IndexKey, ScanResult},
ancestors::{Ancestors, AncestorsForSerialization},
blockhash_queue::BlockhashQueue,
@ -4737,6 +4737,21 @@ impl Bank {
self.rc.get_snapshot_storages(self.slot())
}
/// Get the snapshot storages _higher than_ the `full_snapshot_slot`. This is used when making an
/// incremental snapshot.
pub fn get_incremental_snapshot_storages(&self, full_snapshot_slot: Slot) -> SnapshotStorages {
self.get_snapshot_storages()
.into_iter()
.map(|storage| {
storage
.into_iter()
.filter(|entry| entry.slot() > full_snapshot_slot)
.collect::<SnapshotStorage>()
})
.filter(|storage| !storage.is_empty())
.collect()
}
#[must_use]
fn verify_hash(&self) -> bool {
assert!(self.is_frozen());

119
runtime/src/serde_snapshot.rs
View File

@ -74,6 +74,62 @@ struct AccountsDbFields<T>(
BankHashInfo,
);
/// Helper type to wrap BufReader streams when deserializing and reconstructing from either just a
/// full snapshot, or both a full and incremental snapshot
pub struct SnapshotStreams<'a, R> {
pub full_snapshot_stream: &'a mut BufReader<R>,
pub incremental_snapshot_stream: Option<&'a mut BufReader<R>>,
}
/// Helper type to wrap AccountsDbFields when reconstructing AccountsDb from either just a full
/// snapshot, or both a full and incremental snapshot
#[derive(Debug)]
struct SnapshotAccountsDbFields<T> {
full_snapshot_accounts_db_fields: AccountsDbFields<T>,
incremental_snapshot_accounts_db_fields: Option<AccountsDbFields<T>>,
}
impl<T> SnapshotAccountsDbFields<T> {
/// Collapse the SnapshotAccountsDbFields into a single AccountsDbFields. If there is no
/// incremental snapshot, this returns the AccountsDbFields from the full snapshot. Otherwise
/// this uses the version, slot, and bank hash info from the incremental snapshot, then the
/// combination of the storages from both the full and incremental snapshots.
fn collapse_into(self) -> Result<AccountsDbFields<T>, Error> {
match self.incremental_snapshot_accounts_db_fields {
None => Ok(self.full_snapshot_accounts_db_fields),
Some(AccountsDbFields(
mut incremental_snapshot_storages,
incremental_snapshot_version,
incremental_snapshot_slot,
incremental_snapshot_bank_hash_info,
)) => {
let full_snapshot_storages = self.full_snapshot_accounts_db_fields.0;
let full_snapshot_slot = self.full_snapshot_accounts_db_fields.2;
// filter out incremental snapshot storages with slot <= full snapshot slot
incremental_snapshot_storages.retain(|slot, _| *slot > full_snapshot_slot);
// There must not be any overlap in the slots of storages between the full snapshot and the incremental snapshot
incremental_snapshot_storages
.iter()
.all(|storage_entry| !full_snapshot_storages.contains_key(storage_entry.0)).then(|| ()).ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "Snapshots are incompatible: There are storages for the same slot in both the full snapshot and the incremental snapshot!")
})?;
let mut combined_storages = full_snapshot_storages;
combined_storages.extend(incremental_snapshot_storages.into_iter());
Ok(AccountsDbFields(
combined_storages,
incremental_snapshot_version,
incremental_snapshot_slot,
incremental_snapshot_bank_hash_info,
))
}
}
}
}
trait TypeContext<'a> {
type SerializableAccountStorageEntry: Serialize
+ DeserializeOwned
@ -127,16 +183,16 @@ where
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn bank_from_stream<R>(
pub(crate) fn bank_from_streams<R>(
serde_style: SerdeStyle,
stream: &mut BufReader<R>,
snapshot_streams: &mut SnapshotStreams<R>,
account_paths: &[PathBuf],
unpacked_append_vec_map: UnpackedAppendVecMap,
genesis_config: &GenesisConfig,
frozen_account_pubkeys: &[Pubkey],
debug_keys: Option<Arc<HashSet<Pubkey>>>,
additional_builtins: Option<&Builtins>,
account_indexes: AccountSecondaryIndexes,
account_secondary_indexes: AccountSecondaryIndexes,
caching_enabled: bool,
limit_load_slot_count_from_snapshot: Option<usize>,
shrink_ratio: AccountShrinkThreshold,
@ -147,18 +203,33 @@ where
{
macro_rules! INTO {
($x:ident) => {{
let (bank_fields, accounts_db_fields) = $x::deserialize_bank_fields(stream)?;
let (full_snapshot_bank_fields, full_snapshot_accounts_db_fields) =
$x::deserialize_bank_fields(snapshot_streams.full_snapshot_stream)?;
let (incremental_snapshot_bank_fields, incremental_snapshot_accounts_db_fields) =
if let Some(ref mut incremental_snapshot_stream) =
snapshot_streams.incremental_snapshot_stream
{
let (bank_fields, accounts_db_fields) =
$x::deserialize_bank_fields(incremental_snapshot_stream)?;
(Some(bank_fields), Some(accounts_db_fields))
} else {
(None, None)
};
let snapshot_accounts_db_fields = SnapshotAccountsDbFields {
full_snapshot_accounts_db_fields,
incremental_snapshot_accounts_db_fields,
};
let bank = reconstruct_bank_from_fields(
bank_fields,
accounts_db_fields,
incremental_snapshot_bank_fields.unwrap_or(full_snapshot_bank_fields),
snapshot_accounts_db_fields,
genesis_config,
frozen_account_pubkeys,
account_paths,
unpacked_append_vec_map,
debug_keys,
additional_builtins,
account_indexes,
account_secondary_indexes,
caching_enabled,
limit_load_slot_count_from_snapshot,
shrink_ratio,
@ -243,14 +314,14 @@ impl<'a, C> IgnoreAsHelper for SerializableAccountsDb<'a, C> {}
#[allow(clippy::too_many_arguments)]
fn reconstruct_bank_from_fields<E>(
bank_fields: BankFieldsToDeserialize,
accounts_db_fields: AccountsDbFields<E>,
snapshot_accounts_db_fields: SnapshotAccountsDbFields<E>,
genesis_config: &GenesisConfig,
frozen_account_pubkeys: &[Pubkey],
account_paths: &[PathBuf],
unpacked_append_vec_map: UnpackedAppendVecMap,
debug_keys: Option<Arc<HashSet<Pubkey>>>,
additional_builtins: Option<&Builtins>,
account_indexes: AccountSecondaryIndexes,
account_secondary_indexes: AccountSecondaryIndexes,
caching_enabled: bool,
limit_load_slot_count_from_snapshot: Option<usize>,
shrink_ratio: AccountShrinkThreshold,
@ -260,11 +331,11 @@ where
E: SerializableStorage + std::marker::Sync,
{
let mut accounts_db = reconstruct_accountsdb_from_fields(
accounts_db_fields,
snapshot_accounts_db_fields,
account_paths,
unpacked_append_vec_map,
&genesis_config.cluster_type,
account_indexes,
account_secondary_indexes,
caching_enabled,
limit_load_slot_count_from_snapshot,
shrink_ratio,
@ -310,11 +381,11 @@ where
}
fn reconstruct_accountsdb_from_fields<E>(
accounts_db_fields: AccountsDbFields<E>,
snapshot_accounts_db_fields: SnapshotAccountsDbFields<E>,
account_paths: &[PathBuf],
unpacked_append_vec_map: UnpackedAppendVecMap,
cluster_type: &ClusterType,
account_indexes: AccountSecondaryIndexes,
account_secondary_indexes: AccountSecondaryIndexes,
caching_enabled: bool,
limit_load_slot_count_from_snapshot: Option<usize>,
shrink_ratio: AccountShrinkThreshold,
@ -326,11 +397,19 @@ where
let mut accounts_db = AccountsDb::new_with_config(
account_paths.to_vec(),
cluster_type,
account_indexes,
account_secondary_indexes,
caching_enabled,
shrink_ratio,
);
let AccountsDbFields(storage, version, slot, bank_hash_info) = accounts_db_fields;
let AccountsDbFields(
snapshot_storages,
snapshot_version,
snapshot_slot,
snapshot_bank_hash_info,
) = snapshot_accounts_db_fields.collapse_into()?;
let snapshot_storages = snapshot_storages.into_iter().collect::<Vec<_>>();
// Ensure all account paths exist
for path in &accounts_db.paths {
@ -338,13 +417,11 @@ where
.unwrap_or_else(|err| panic!("Failed to create directory {}: {}", path.display(), err));
}
let storage = storage.into_iter().collect::<Vec<_>>();
// Remap the deserialized AppendVec paths to point to correct local paths
let mut storage = (0..storage.len())
let mut storage = (0..snapshot_storages.len())
.into_par_iter()
.map(|i| {
let (slot, slot_storage) = &storage[i];
let (slot, slot_storage) = &snapshot_storages[i];
let mut new_slot_storage = HashMap::new();
for storage_entry in slot_storage {
let file_name = AppendVec::file_name(*slot, storage_entry.id());
@ -376,7 +453,7 @@ where
.bank_hashes
.write()
.unwrap()
.insert(slot, bank_hash_info);
.insert(snapshot_slot, snapshot_bank_hash_info);
// Process deserialized data, set necessary fields in self
let max_id: usize = *storage
@ -400,7 +477,7 @@ where
accounts_db.next_id.store(max_id + 1, Ordering::Relaxed);
accounts_db
.write_version
.fetch_add(version, Ordering::Relaxed);
.fetch_add(snapshot_version, Ordering::Relaxed);
accounts_db.generate_index(limit_load_slot_count_from_snapshot, verify_index);
Ok(accounts_db)
}

15
runtime/src/serde_snapshot/tests.rs
View File

@ -66,8 +66,13 @@ where
R: Read,
{
// read and deserialise the accounts database directly from the stream
let accounts_db_fields = C::deserialize_accounts_db_fields(stream)?;
let snapshot_accounts_db_fields = SnapshotAccountsDbFields {
full_snapshot_accounts_db_fields: accounts_db_fields,
incremental_snapshot_accounts_db_fields: None,
};
reconstruct_accountsdb_from_fields(
C::deserialize_accounts_db_fields(stream)?,
snapshot_accounts_db_fields,
account_paths,
unpacked_append_vec_map,
&ClusterType::Development,
@ -219,9 +224,13 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) {
let copied_accounts = TempDir::new().unwrap();
let unpacked_append_vec_map =
copy_append_vecs(&bank2.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
let mut dbank = crate::serde_snapshot::bank_from_stream(
let mut snapshot_streams = SnapshotStreams {
full_snapshot_stream: &mut reader,
incremental_snapshot_stream: None,
};
let mut dbank = crate::serde_snapshot::bank_from_streams(
serde_style,
&mut reader,
&mut snapshot_streams,
&dbank_paths,
unpacked_append_vec_map,
&genesis_config,

1905
runtime/src/snapshot_utils.rs
View File

File diff suppressed because it is too large Load Diff

16
validator/src/main.rs
View File

@ -47,7 +47,9 @@ use {
},
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
snapshot_config::SnapshotConfig,
snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
snapshot_utils::{
self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
},
},
solana_sdk::{
clock::{Slot, DEFAULT_S_PER_SLOT},
@ -475,8 +477,10 @@ fn get_rpc_node(
blacklist_timeout = Instant::now();
let mut highest_snapshot_hash: Option<(Slot, Hash)> =
snapshot_utils::get_highest_snapshot_archive_info(snapshot_output_dir).map(
|snapshot_archive_info| (snapshot_archive_info.slot, snapshot_archive_info.hash),
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_output_dir).map(
|snapshot_archive_info| {
(*snapshot_archive_info.slot(), *snapshot_archive_info.hash())
},
);
let eligible_rpc_peers = if snapshot_not_required {
rpc_peers
@ -858,7 +862,7 @@ fn rpc_bootstrap(
let mut use_local_snapshot = false;
if let Some(highest_local_snapshot_slot) =
snapshot_utils::get_highest_snapshot_archive_slot(snapshot_output_dir)
snapshot_utils::get_highest_full_snapshot_archive_slot(snapshot_output_dir)
{
if highest_local_snapshot_slot
> snapshot_hash.0.saturating_sub(maximum_local_snapshot_age)
@ -900,7 +904,7 @@ fn rpc_bootstrap(
{
snapshot_config.maximum_snapshots_to_retain
} else {
DEFAULT_MAX_SNAPSHOTS_TO_RETAIN
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN
};
let ret = download_snapshot(
&rpc_contact_info.rpc,
@ -1041,7 +1045,7 @@ pub fn main() {
.send_transaction_leader_forward_count
.to_string();
let default_rpc_threads = num_cpus::get().to_string();
let default_max_snapshot_to_retain = &DEFAULT_MAX_SNAPSHOTS_TO_RETAIN.to_string();
let default_max_snapshot_to_retain = &DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string();
let default_min_snapshot_download_speed = &DEFAULT_MIN_SNAPSHOT_DOWNLOAD_SPEED.to_string();
let default_max_snapshot_download_abort = &MAX_SNAPSHOT_DOWNLOAD_ABORT.to_string();
let default_accounts_shrink_optimize_total_space =