blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add options to store full and/or incremental snapshots in separate locations (#24247)

This commit is contained in:
DimAn 2022-05-10 23:37:41 +03:00 committed by GitHub
parent dc6e28099b
commit 2fa9bc3e70
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 645 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/accounts_hash_verifier.rs
View File

@ -384,7 +384,8 @@ mod tests {
snapshot_storages: vec![], snapshot_storages: vec![],
archive_format: ArchiveFormat::TarBzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),
snapshot_archives_dir: PathBuf::default(), full_snapshot_archives_dir: PathBuf::default(),
incremental_snapshot_archives_dir: PathBuf::default(),
expected_capitalization: 0, expected_capitalization: 0,
accounts_hash_for_testing: None, accounts_hash_for_testing: None,
cluster_type: ClusterType::MainnetBeta, cluster_type: ClusterType::MainnetBeta,

12
core/src/snapshot_packager_service.rs
View File

@ -84,6 +84,8 @@ impl SnapshotPackagerService {
// last_full_snapshot_slot that requires this archive call to succeed. // last_full_snapshot_slot that requires this archive call to succeed.
snapshot_utils::archive_snapshot_package( snapshot_utils::archive_snapshot_package(
&snapshot_package, &snapshot_package,
&snapshot_config.full_snapshot_archives_dir,
&snapshot_config.incremental_snapshot_archives_dir,
snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_full_snapshot_archives_to_retain,
snapshot_config.maximum_incremental_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain,
) )
@ -259,8 +261,10 @@ mod tests {
fn create_and_verify_snapshot(temp_dir: &Path) { fn create_and_verify_snapshot(temp_dir: &Path) {
let accounts_dir = temp_dir.join("accounts"); let accounts_dir = temp_dir.join("accounts");
let snapshots_dir = temp_dir.join("snapshots"); let snapshots_dir = temp_dir.join("snapshots");
let snapshot_archives_dir = temp_dir.join("snapshots_output"); let full_snapshot_archives_dir = temp_dir.join("full_snapshot_archives");
fs::create_dir_all(&snapshot_archives_dir).unwrap(); let incremental_snapshot_archives_dir = temp_dir.join("incremental_snapshot_archives");
fs::create_dir_all(&full_snapshot_archives_dir).unwrap();
fs::create_dir_all(&incremental_snapshot_archives_dir).unwrap();
fs::create_dir_all(&accounts_dir).unwrap(); fs::create_dir_all(&accounts_dir).unwrap();
// Create some storage entries // Create some storage entries
@ -302,7 +306,7 @@ mod tests {
let hash = Hash::default(); let hash = Hash::default();
let archive_format = ArchiveFormat::TarBzip2; let archive_format = ArchiveFormat::TarBzip2;
let output_tar_path = snapshot_utils::build_full_snapshot_archive_path( let output_tar_path = snapshot_utils::build_full_snapshot_archive_path(
snapshot_archives_dir, &full_snapshot_archives_dir,
slot, slot,
&hash, &hash,
archive_format, archive_format,
@ -325,6 +329,8 @@ mod tests {
// Make tarball from packageable snapshot // Make tarball from packageable snapshot
snapshot_utils::archive_snapshot_package( snapshot_utils::archive_snapshot_package(
&snapshot_package, &snapshot_package,
full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
) )

3
core/src/validator.rs
View File

@ -1676,7 +1676,8 @@ fn maybe_warp_slot(
ledger_path, ledger_path,
&bank_forks.root_bank(), &bank_forks.root_bank(),
None, None,
&snapshot_config.snapshot_archives_dir, &snapshot_config.full_snapshot_archives_dir,
&snapshot_config.incremental_snapshot_archives_dir,
snapshot_config.archive_format, snapshot_config.archive_format,
snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_full_snapshot_archives_to_retain,
snapshot_config.maximum_incremental_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain,

52
core/tests/snapshots.rs
View File

@ -107,7 +107,8 @@ mod tests {
struct SnapshotTestConfig { struct SnapshotTestConfig {
accounts_dir: TempDir, accounts_dir: TempDir,
bank_snapshots_dir: TempDir, bank_snapshots_dir: TempDir,
snapshot_archives_dir: TempDir, full_snapshot_archives_dir: TempDir,
incremental_snapshot_archives_dir: TempDir,
snapshot_config: SnapshotConfig, snapshot_config: SnapshotConfig,
bank_forks: BankForks, bank_forks: BankForks,
genesis_config_info: GenesisConfigInfo, genesis_config_info: GenesisConfigInfo,
@ -123,7 +124,8 @@ mod tests {
) -> SnapshotTestConfig { ) -> SnapshotTestConfig {
let accounts_dir = TempDir::new().unwrap(); let accounts_dir = TempDir::new().unwrap();
let bank_snapshots_dir = TempDir::new().unwrap(); let bank_snapshots_dir = TempDir::new().unwrap();
let snapshot_archives_dir = TempDir::new().unwrap(); let full_snapshot_archives_dir = TempDir::new().unwrap();
let incremental_snapshot_archives_dir = TempDir::new().unwrap();
// validator_stake_lamports should be non-zero otherwise stake // validator_stake_lamports should be non-zero otherwise stake
// account will not be stored in accounts-db but still cached in // account will not be stored in accounts-db but still cached in
// bank stakes which results in mismatch when banks are loaded from // bank stakes which results in mismatch when banks are loaded from
@ -151,7 +153,10 @@ mod tests {
let snapshot_config = SnapshotConfig { let snapshot_config = SnapshotConfig {
full_snapshot_archive_interval_slots, full_snapshot_archive_interval_slots,
incremental_snapshot_archive_interval_slots, incremental_snapshot_archive_interval_slots,
snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(), full_snapshot_archives_dir: full_snapshot_archives_dir.path().to_path_buf(),
incremental_snapshot_archives_dir: incremental_snapshot_archives_dir
.path()
.to_path_buf(),
bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(), bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
snapshot_version, snapshot_version,
..SnapshotConfig::default() ..SnapshotConfig::default()
@ -160,7 +165,8 @@ mod tests {
SnapshotTestConfig { SnapshotTestConfig {
accounts_dir, accounts_dir,
bank_snapshots_dir, bank_snapshots_dir,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
snapshot_config, snapshot_config,
bank_forks, bank_forks,
genesis_config_info, genesis_config_info,
@ -174,17 +180,17 @@ mod tests {
old_genesis_config: &GenesisConfig, old_genesis_config: &GenesisConfig,
account_paths: &[PathBuf], account_paths: &[PathBuf],
) { ) {
let snapshot_archives_dir = old_bank_forks let full_snapshot_archives_dir = old_bank_forks
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.map(|c| &c.snapshot_archives_dir) .map(|c| &c.full_snapshot_archives_dir)
.unwrap(); .unwrap();
let old_last_bank = old_bank_forks.get(old_last_slot).unwrap(); let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
let check_hash_calculation = false; let check_hash_calculation = false;
let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path( let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path(
snapshot_archives_dir, full_snapshot_archives_dir,
old_last_bank.slot(), old_last_bank.slot(),
&old_last_bank.get_accounts_hash(), &old_last_bank.get_accounts_hash(),
ArchiveFormat::TarBzip2, ArchiveFormat::TarBzip2,
@ -280,7 +286,8 @@ mod tests {
&last_bank_snapshot_info, &last_bank_snapshot_info,
bank_snapshots_dir, bank_snapshots_dir,
last_bank.src.slot_deltas(&last_bank.src.roots()), last_bank.src.slot_deltas(&last_bank.src.roots()),
&snapshot_config.snapshot_archives_dir, &snapshot_config.full_snapshot_archives_dir,
&snapshot_config.incremental_snapshot_archives_dir,
last_bank.get_snapshot_storages(None), last_bank.get_snapshot_storages(None),
ArchiveFormat::TarBzip2, ArchiveFormat::TarBzip2,
snapshot_version, snapshot_version,
@ -297,6 +304,8 @@ mod tests {
SnapshotPackage::new(accounts_package, last_bank.get_accounts_hash()); SnapshotPackage::new(accounts_package, last_bank.get_accounts_hash());
snapshot_utils::archive_snapshot_package( snapshot_utils::archive_snapshot_package(
&snapshot_package, &snapshot_package,
&snapshot_config.full_snapshot_archives_dir,
&snapshot_config.incremental_snapshot_archives_dir,
snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_full_snapshot_archives_to_retain,
snapshot_config.maximum_incremental_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain,
) )
@ -360,7 +369,8 @@ mod tests {
let bank_forks = &mut snapshot_test_config.bank_forks; let bank_forks = &mut snapshot_test_config.bank_forks;
let snapshot_config = &snapshot_test_config.snapshot_config; let snapshot_config = &snapshot_test_config.snapshot_config;
let bank_snapshots_dir = &snapshot_config.bank_snapshots_dir; let bank_snapshots_dir = &snapshot_config.bank_snapshots_dir;
let snapshot_archives_dir = &snapshot_config.snapshot_archives_dir; let full_snapshot_archives_dir = &snapshot_config.full_snapshot_archives_dir;
let incremental_snapshot_archives_dir = &snapshot_config.incremental_snapshot_archives_dir;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config; let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
@ -414,7 +424,8 @@ mod tests {
vec![], vec![],
pending_accounts_package, pending_accounts_package,
bank_snapshots_dir, bank_snapshots_dir,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
snapshot_config.snapshot_version, snapshot_config.snapshot_version,
snapshot_config.archive_format, snapshot_config.archive_format,
None, None,
@ -464,7 +475,7 @@ mod tests {
fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap(); fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path( saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path(
snapshot_archives_dir, full_snapshot_archives_dir,
slot, slot,
// this needs to match the hash value that we reserialize with later. It is complicated, so just use default. // this needs to match the hash value that we reserialize with later. It is complicated, so just use default.
// This hash value is just used to build the file name. Since this is mocked up test code, it is sufficient to pass default here. // This hash value is just used to build the file name. Since this is mocked up test code, it is sufficient to pass default here.
@ -679,7 +690,8 @@ mod tests {
FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
); );
trace!("SnapshotTestConfig:\naccounts_dir: {}\nbank_snapshots_dir: {}\nsnapshot_archives_dir: {}", snapshot_test_config.accounts_dir.path().display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.snapshot_archives_dir.path().display()); trace!("SnapshotTestConfig:\naccounts_dir: {}\nbank_snapshots_dir: {}\nfull_snapshot_archives_dir: {}\nincremental_snapshot_archives_dir: {}",
snapshot_test_config.accounts_dir.path().display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.full_snapshot_archives_dir.path().display(), snapshot_test_config.incremental_snapshot_archives_dir.path().display());
let bank_forks = &mut snapshot_test_config.bank_forks; let bank_forks = &mut snapshot_test_config.bank_forks;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
@ -783,7 +795,8 @@ mod tests {
bank, bank,
&bank_snapshot_info, &bank_snapshot_info,
&snapshot_config.bank_snapshots_dir, &snapshot_config.bank_snapshots_dir,
&snapshot_config.snapshot_archives_dir, &snapshot_config.full_snapshot_archives_dir,
&snapshot_config.incremental_snapshot_archives_dir,
bank.get_snapshot_storages(None), bank.get_snapshot_storages(None),
snapshot_config.archive_format, snapshot_config.archive_format,
snapshot_config.snapshot_version, snapshot_config.snapshot_version,
@ -820,7 +833,8 @@ mod tests {
incremental_snapshot_base_slot, incremental_snapshot_base_slot,
&bank_snapshot_info, &bank_snapshot_info,
&snapshot_config.bank_snapshots_dir, &snapshot_config.bank_snapshots_dir,
&snapshot_config.snapshot_archives_dir, &snapshot_config.full_snapshot_archives_dir,
&snapshot_config.incremental_snapshot_archives_dir,
storages, storages,
snapshot_config.archive_format, snapshot_config.archive_format,
snapshot_config.snapshot_version, snapshot_config.snapshot_version,
@ -839,7 +853,8 @@ mod tests {
) -> snapshot_utils::Result<()> { ) -> snapshot_utils::Result<()> {
let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives( let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives(
&snapshot_config.bank_snapshots_dir, &snapshot_config.bank_snapshots_dir,
&snapshot_config.snapshot_archives_dir, &snapshot_config.full_snapshot_archives_dir,
&snapshot_config.incremental_snapshot_archives_dir,
&[accounts_dir], &[accounts_dir],
genesis_config, genesis_config,
None, None,
@ -1015,7 +1030,12 @@ mod tests {
let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives( let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives(
&snapshot_test_config.snapshot_config.bank_snapshots_dir, &snapshot_test_config.snapshot_config.bank_snapshots_dir,
&snapshot_test_config.snapshot_config.snapshot_archives_dir, &snapshot_test_config
.snapshot_config
.full_snapshot_archives_dir,
&snapshot_test_config
.snapshot_config
.incremental_snapshot_archives_dir,
&[snapshot_test_config.accounts_dir.as_ref().to_path_buf()], &[snapshot_test_config.accounts_dir.as_ref().to_path_buf()],
&snapshot_test_config.genesis_config_info.genesis_config, &snapshot_test_config.genesis_config_info.genesis_config,
None, None,

11
download-utils/src/lib.rs
View File

@ -254,7 +254,8 @@ pub fn download_genesis_if_missing(
/// a full snapshot or an incremental snapshot. /// a full snapshot or an incremental snapshot.
pub fn download_snapshot_archive<'a, 'b>( pub fn download_snapshot_archive<'a, 'b>(
rpc_addr: &SocketAddr, rpc_addr: &SocketAddr,
snapshot_archives_dir: &Path, full_snapshot_archives_dir: &Path,
incremental_snapshot_archives_dir: &Path,
desired_snapshot_hash: (Slot, Hash), desired_snapshot_hash: (Slot, Hash),
snapshot_type: SnapshotType, snapshot_type: SnapshotType,
maximum_full_snapshot_archives_to_retain: usize, maximum_full_snapshot_archives_to_retain: usize,
@ -263,13 +264,17 @@ pub fn download_snapshot_archive<'a, 'b>(
progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>, progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>,
) -> Result<(), String> { ) -> Result<(), String> {
snapshot_utils::purge_old_snapshot_archives( snapshot_utils::purge_old_snapshot_archives(
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain,
); );
let snapshot_archives_remote_dir = let snapshot_archives_remote_dir =
snapshot_utils::build_snapshot_archives_remote_dir(snapshot_archives_dir); snapshot_utils::build_snapshot_archives_remote_dir(match snapshot_type {
SnapshotType::FullSnapshot => full_snapshot_archives_dir,
SnapshotType::IncrementalSnapshot(_) => incremental_snapshot_archives_dir,
});
fs::create_dir_all(&snapshot_archives_remote_dir).unwrap(); fs::create_dir_all(&snapshot_archives_remote_dir).unwrap();
for archive_format in [ for archive_format in [

48
ledger-tool/src/main.rs
View File

@ -731,6 +731,7 @@ fn load_bank_forks(
blockstore: &Blockstore, blockstore: &Blockstore,
process_options: ProcessOptions, process_options: ProcessOptions,
snapshot_archive_path: Option<PathBuf>, snapshot_archive_path: Option<PathBuf>,
incremental_snapshot_archive_path: Option<PathBuf>,
) -> Result<(Arc<RwLock<BankForks>>, Option<StartingSnapshotHashes>), BlockstoreProcessorError> { ) -> Result<(Arc<RwLock<BankForks>>, Option<StartingSnapshotHashes>), BlockstoreProcessorError> {
let bank_snapshots_dir = blockstore let bank_snapshots_dir = blockstore
.ledger_path() .ledger_path()
@ -742,12 +743,15 @@ fn load_bank_forks(
let snapshot_config = if arg_matches.is_present("no_snapshot") { let snapshot_config = if arg_matches.is_present("no_snapshot") {
None None
} else { } else {
let snapshot_archives_dir = let full_snapshot_archives_dir =
snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf()); snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf());
let incremental_snapshot_archives_dir =
incremental_snapshot_archive_path.unwrap_or_else(|| full_snapshot_archives_dir.clone());
Some(SnapshotConfig { Some(SnapshotConfig {
full_snapshot_archive_interval_slots: Slot::MAX, full_snapshot_archive_interval_slots: Slot::MAX,
incremental_snapshot_archive_interval_slots: Slot::MAX, incremental_snapshot_archive_interval_slots: Slot::MAX,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
bank_snapshots_dir, bank_snapshots_dir,
..SnapshotConfig::default() ..SnapshotConfig::default()
}) })
@ -1102,7 +1106,15 @@ fn main() {
.value_name("DIR") .value_name("DIR")
.takes_value(true) .takes_value(true)
.global(true) .global(true)
.help("Use DIR for ledger location"), .help("Use DIR for snapshot location"),
)
.arg(
Arg::with_name("incremental_snapshot_archive_path")
.long("incremental-snapshot-archive-path")
.value_name("DIR")
.takes_value(true)
.global(true)
.help("Use DIR for separate incremental snapshot location"),
) )
.arg( .arg(
Arg::with_name("output_format") Arg::with_name("output_format")
@ -1713,6 +1725,10 @@ fn main() {
let snapshot_archive_path = value_t!(matches, "snapshot_archive_path", String) let snapshot_archive_path = value_t!(matches, "snapshot_archive_path", String)
.ok() .ok()
.map(PathBuf::from); .map(PathBuf::from);
let incremental_snapshot_archive_path =
value_t!(matches, "incremental_snapshot_archive_path", String)
.ok()
.map(PathBuf::from);
let wal_recovery_mode = matches let wal_recovery_mode = matches
.value_of("wal_recovery_mode") .value_of("wal_recovery_mode")
@ -1828,6 +1844,7 @@ fn main() {
&blockstore, &blockstore,
process_options, process_options,
snapshot_archive_path, snapshot_archive_path,
incremental_snapshot_archive_path,
) { ) {
Ok((bank_forks, ..)) => { Ok((bank_forks, ..)) => {
println!( println!(
@ -1909,6 +1926,7 @@ fn main() {
&blockstore, &blockstore,
process_options, process_options,
snapshot_archive_path, snapshot_archive_path,
incremental_snapshot_archive_path,
) { ) {
Ok((bank_forks, ..)) => { Ok((bank_forks, ..)) => {
println!("{}", &bank_forks.read().unwrap().working_bank().hash()); println!("{}", &bank_forks.read().unwrap().working_bank().hash());
@ -2148,6 +2166,7 @@ fn main() {
&blockstore, &blockstore,
process_options, process_options,
snapshot_archive_path, snapshot_archive_path,
incremental_snapshot_archive_path,
) )
.unwrap_or_else(|err| { .unwrap_or_else(|err| {
eprintln!("Ledger verification failed: {:?}", err); eprintln!("Ledger verification failed: {:?}", err);
@ -2179,6 +2198,7 @@ fn main() {
&blockstore, &blockstore,
process_options, process_options,
snapshot_archive_path, snapshot_archive_path,
incremental_snapshot_archive_path,
) { ) {
Ok((bank_forks, ..)) => { Ok((bank_forks, ..)) => {
let dot = graph_forks( let dot = graph_forks(
@ -2208,10 +2228,20 @@ fn main() {
} }
} }
("create-snapshot", Some(arg_matches)) => { ("create-snapshot", Some(arg_matches)) => {
let is_incremental = arg_matches.is_present("incremental");
let output_directory = value_t!(arg_matches, "output_directory", PathBuf) let output_directory = value_t!(arg_matches, "output_directory", PathBuf)
.unwrap_or_else(|_| match &snapshot_archive_path { .unwrap_or_else(|_| {
Some(snapshot_archive_path) => snapshot_archive_path.clone(), match (
None => ledger_path.clone(), is_incremental,
&snapshot_archive_path,
&incremental_snapshot_archive_path,
) {
(true, _, Some(incremental_snapshot_archive_path)) => {
incremental_snapshot_archive_path.clone()
}
(_, Some(snapshot_archive_path), _) => snapshot_archive_path.clone(),
(_, _, _) => ledger_path.clone(),
}
}); });
let mut warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok(); let mut warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok();
let remove_stake_accounts = arg_matches.is_present("remove_stake_accounts"); let remove_stake_accounts = arg_matches.is_present("remove_stake_accounts");
@ -2266,7 +2296,6 @@ fn main() {
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let blockstore = let blockstore =
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
let is_incremental = arg_matches.is_present("incremental");
let snapshot_slot = if Some("ROOT") == arg_matches.value_of("snapshot_slot") { let snapshot_slot = if Some("ROOT") == arg_matches.value_of("snapshot_slot") {
blockstore blockstore
@ -2296,6 +2325,7 @@ fn main() {
..ProcessOptions::default() ..ProcessOptions::default()
}, },
snapshot_archive_path, snapshot_archive_path,
incremental_snapshot_archive_path,
) { ) {
Ok((bank_forks, starting_snapshot_hashes)) => { Ok((bank_forks, starting_snapshot_hashes)) => {
let mut bank = bank_forks let mut bank = bank_forks
@ -2530,6 +2560,7 @@ fn main() {
&bank, &bank,
full_snapshot_slot, full_snapshot_slot,
Some(snapshot_version), Some(snapshot_version),
output_directory.clone(),
output_directory, output_directory,
ArchiveFormat::TarZstd, ArchiveFormat::TarZstd,
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
@ -2553,6 +2584,7 @@ fn main() {
ledger_path, ledger_path,
&bank, &bank,
Some(snapshot_version), Some(snapshot_version),
output_directory.clone(),
output_directory, output_directory,
ArchiveFormat::TarZstd, ArchiveFormat::TarZstd,
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
@ -2603,6 +2635,7 @@ fn main() {
&blockstore, &blockstore,
process_options, process_options,
snapshot_archive_path, snapshot_archive_path,
incremental_snapshot_archive_path,
) )
.unwrap_or_else(|err| { .unwrap_or_else(|err| {
eprintln!("Failed to load ledger: {:?}", err); eprintln!("Failed to load ledger: {:?}", err);
@ -2662,6 +2695,7 @@ fn main() {
&blockstore, &blockstore,
process_options, process_options,
snapshot_archive_path, snapshot_archive_path,
incremental_snapshot_archive_path,
) { ) {
Ok((bank_forks, ..)) => { Ok((bank_forks, ..)) => {
let bank_forks = bank_forks.read().unwrap(); let bank_forks = bank_forks.read().unwrap();

5
ledger/src/bank_forks_utils.rs
View File

@ -98,7 +98,7 @@ pub fn load_bank_forks(
.expect("Couldn't create snapshot directory"); .expect("Couldn't create snapshot directory");
if snapshot_utils::get_highest_full_snapshot_archive_info( if snapshot_utils::get_highest_full_snapshot_archive_info(
&snapshot_config.snapshot_archives_dir, &snapshot_config.full_snapshot_archives_dir,
) )
.is_some() .is_some()
{ {
@ -188,7 +188,8 @@ fn bank_forks_from_snapshot(
let (mut deserialized_bank, full_snapshot_archive_info, incremental_snapshot_archive_info) = let (mut deserialized_bank, full_snapshot_archive_info, incremental_snapshot_archive_info) =
snapshot_utils::bank_from_latest_snapshot_archives( snapshot_utils::bank_from_latest_snapshot_archives(
&snapshot_config.bank_snapshots_dir, &snapshot_config.bank_snapshots_dir,
&snapshot_config.snapshot_archives_dir, &snapshot_config.full_snapshot_archives_dir,
&snapshot_config.incremental_snapshot_archives_dir,
&account_paths, &account_paths,
genesis_config, genesis_config,
process_options.debug_keys.clone(), process_options.debug_keys.clone(),

30
local-cluster/src/local_cluster_snapshot_utils.rs
View File

@ -13,11 +13,18 @@ use {
impl LocalCluster { impl LocalCluster {
/// Return the next full snapshot archive info after the cluster's last processed slot /// Return the next full snapshot archive info after the cluster's last processed slot
pub fn wait_for_next_full_snapshot( pub fn wait_for_next_full_snapshot<T>(
&self, &self,
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: T,
) -> FullSnapshotArchiveInfo { ) -> FullSnapshotArchiveInfo
match self.wait_for_next_snapshot(snapshot_archives_dir, NextSnapshotType::FullSnapshot) { where
T: AsRef<Path>,
{
match self.wait_for_next_snapshot(
full_snapshot_archives_dir,
None::<T>,
NextSnapshotType::FullSnapshot,
) {
NextSnapshotResult::FullSnapshot(full_snapshot_archive_info) => { NextSnapshotResult::FullSnapshot(full_snapshot_archive_info) => {
full_snapshot_archive_info full_snapshot_archive_info
} }
@ -29,10 +36,12 @@ impl LocalCluster {
/// after the cluster's last processed slot /// after the cluster's last processed slot
pub fn wait_for_next_incremental_snapshot( pub fn wait_for_next_incremental_snapshot(
&self, &self,
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
) -> (IncrementalSnapshotArchiveInfo, FullSnapshotArchiveInfo) { ) -> (IncrementalSnapshotArchiveInfo, FullSnapshotArchiveInfo) {
match self.wait_for_next_snapshot( match self.wait_for_next_snapshot(
snapshot_archives_dir, full_snapshot_archives_dir,
Some(incremental_snapshot_archives_dir),
NextSnapshotType::IncrementalAndFullSnapshot, NextSnapshotType::IncrementalAndFullSnapshot,
) { ) {
NextSnapshotResult::IncrementalAndFullSnapshot( NextSnapshotResult::IncrementalAndFullSnapshot(
@ -47,9 +56,10 @@ impl LocalCluster {
} }
/// Return the next snapshot archive infos after the cluster's last processed slot /// Return the next snapshot archive infos after the cluster's last processed slot
pub fn wait_for_next_snapshot( fn wait_for_next_snapshot(
&self, &self,
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: Option<impl AsRef<Path>>,
next_snapshot_type: NextSnapshotType, next_snapshot_type: NextSnapshotType,
) -> NextSnapshotResult { ) -> NextSnapshotResult {
// Get slot after which this was generated // Get slot after which this was generated
@ -69,7 +79,7 @@ impl LocalCluster {
); );
loop { loop {
if let Some(full_snapshot_archive_info) = if let Some(full_snapshot_archive_info) =
snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir) snapshot_utils::get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir)
{ {
match next_snapshot_type { match next_snapshot_type {
NextSnapshotType::FullSnapshot => { NextSnapshotType::FullSnapshot => {
@ -80,7 +90,7 @@ impl LocalCluster {
NextSnapshotType::IncrementalAndFullSnapshot => { NextSnapshotType::IncrementalAndFullSnapshot => {
if let Some(incremental_snapshot_archive_info) = if let Some(incremental_snapshot_archive_info) =
snapshot_utils::get_highest_incremental_snapshot_archive_info( snapshot_utils::get_highest_incremental_snapshot_archive_info(
&snapshot_archives_dir, incremental_snapshot_archives_dir.as_ref().unwrap(),
full_snapshot_archive_info.slot(), full_snapshot_archive_info.slot(),
) )
{ {

14
local-cluster/tests/common.rs
View File

@ -441,7 +441,8 @@ pub fn generate_account_paths(num_account_paths: usize) -> (Vec<TempDir>, Vec<Pa
pub struct SnapshotValidatorConfig { pub struct SnapshotValidatorConfig {
pub bank_snapshots_dir: TempDir, pub bank_snapshots_dir: TempDir,
pub snapshot_archives_dir: TempDir, pub full_snapshot_archives_dir: TempDir,
pub incremental_snapshot_archives_dir: TempDir,
pub account_storage_dirs: Vec<TempDir>, pub account_storage_dirs: Vec<TempDir>,
pub validator_config: ValidatorConfig, pub validator_config: ValidatorConfig,
} }
@ -470,11 +471,15 @@ impl SnapshotValidatorConfig {
// Create the snapshot config // Create the snapshot config
let _ = fs::create_dir_all(farf_dir()); let _ = fs::create_dir_all(farf_dir());
let bank_snapshots_dir = tempfile::tempdir_in(farf_dir()).unwrap(); let bank_snapshots_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap(); let full_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let incremental_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let snapshot_config = SnapshotConfig { let snapshot_config = SnapshotConfig {
full_snapshot_archive_interval_slots, full_snapshot_archive_interval_slots,
incremental_snapshot_archive_interval_slots, incremental_snapshot_archive_interval_slots,
snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(), full_snapshot_archives_dir: full_snapshot_archives_dir.path().to_path_buf(),
incremental_snapshot_archives_dir: incremental_snapshot_archives_dir
.path()
.to_path_buf(),
bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(), bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
maximum_full_snapshot_archives_to_retain: usize::MAX, maximum_full_snapshot_archives_to_retain: usize::MAX,
maximum_incremental_snapshot_archives_to_retain: usize::MAX, maximum_incremental_snapshot_archives_to_retain: usize::MAX,
@ -495,7 +500,8 @@ impl SnapshotValidatorConfig {
SnapshotValidatorConfig { SnapshotValidatorConfig {
bank_snapshots_dir, bank_snapshots_dir,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
account_storage_dirs, account_storage_dirs,
validator_config, validator_config,
} }

219
local-cluster/tests/local_cluster.rs
View File

@ -467,21 +467,29 @@ fn test_snapshot_download() {
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified); let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
let snapshot_archives_dir = &leader_snapshot_test_config let full_snapshot_archives_dir = &leader_snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_archives_dir; .full_snapshot_archives_dir;
let incremental_snapshot_archives_dir = &leader_snapshot_test_config
.validator_config
.snapshot_config
.as_ref()
.unwrap()
.incremental_snapshot_archives_dir;
trace!("Waiting for snapshot"); trace!("Waiting for snapshot");
let full_snapshot_archive_info = cluster.wait_for_next_full_snapshot(snapshot_archives_dir); let full_snapshot_archive_info =
cluster.wait_for_next_full_snapshot(full_snapshot_archives_dir);
trace!("found: {}", full_snapshot_archive_info.path().display()); trace!("found: {}", full_snapshot_archive_info.path().display());
// Download the snapshot, then boot a validator from it. // Download the snapshot, then boot a validator from it.
download_snapshot_archive( download_snapshot_archive(
&cluster.entry_point_info.rpc, &cluster.entry_point_info.rpc,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
( (
full_snapshot_archive_info.slot(), full_snapshot_archive_info.slot(),
*full_snapshot_archive_info.hash(), *full_snapshot_archive_info.hash(),
@ -549,43 +557,60 @@ fn test_incremental_snapshot_download() {
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified); let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
let snapshot_archives_dir = &leader_snapshot_test_config let full_snapshot_archives_dir = &leader_snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_archives_dir; .full_snapshot_archives_dir;
let incremental_snapshot_archives_dir = &leader_snapshot_test_config
.validator_config
.snapshot_config
.as_ref()
.unwrap()
.incremental_snapshot_archives_dir;
debug!("snapshot config:\n\tfull snapshot interval: {}\n\tincremental snapshot interval: {}\n\taccounts hash interval: {}", debug!("snapshot config:\n\tfull snapshot interval: {}\n\tincremental snapshot interval: {}\n\taccounts hash interval: {}",
full_snapshot_interval, full_snapshot_interval,
incremental_snapshot_interval, incremental_snapshot_interval,
accounts_hash_interval); accounts_hash_interval);
debug!( debug!(
"leader config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}", "leader config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}",
leader_snapshot_test_config leader_snapshot_test_config
.bank_snapshots_dir .bank_snapshots_dir
.path() .path()
.display(), .display(),
leader_snapshot_test_config leader_snapshot_test_config
.snapshot_archives_dir .full_snapshot_archives_dir
.path()
.display(),
leader_snapshot_test_config
.incremental_snapshot_archives_dir
.path() .path()
.display(), .display(),
); );
debug!( debug!(
"validator config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}", "validator config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}",
validator_snapshot_test_config validator_snapshot_test_config
.bank_snapshots_dir .bank_snapshots_dir
.path() .path()
.display(), .display(),
validator_snapshot_test_config validator_snapshot_test_config
.snapshot_archives_dir .full_snapshot_archives_dir
.path()
.display(),
validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path() .path()
.display(), .display(),
); );
trace!("Waiting for snapshots"); trace!("Waiting for snapshots");
let (incremental_snapshot_archive_info, full_snapshot_archive_info) = let (incremental_snapshot_archive_info, full_snapshot_archive_info) = cluster
cluster.wait_for_next_incremental_snapshot(snapshot_archives_dir); .wait_for_next_incremental_snapshot(
full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
);
trace!( trace!(
"found: {} and {}", "found: {} and {}",
full_snapshot_archive_info.path().display(), full_snapshot_archive_info.path().display(),
@ -595,7 +620,8 @@ fn test_incremental_snapshot_download() {
// Download the snapshots, then boot a validator from them. // Download the snapshots, then boot a validator from them.
download_snapshot_archive( download_snapshot_archive(
&cluster.entry_point_info.rpc, &cluster.entry_point_info.rpc,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
( (
full_snapshot_archive_info.slot(), full_snapshot_archive_info.slot(),
*full_snapshot_archive_info.hash(), *full_snapshot_archive_info.hash(),
@ -620,7 +646,8 @@ fn test_incremental_snapshot_download() {
download_snapshot_archive( download_snapshot_archive(
&cluster.entry_point_info.rpc, &cluster.entry_point_info.rpc,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
( (
incremental_snapshot_archive_info.slot(), incremental_snapshot_archive_info.slot(),
*incremental_snapshot_archive_info.hash(), *incremental_snapshot_archive_info.hash(),
@ -708,24 +735,32 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
incremental_snapshot_interval, incremental_snapshot_interval,
accounts_hash_interval); accounts_hash_interval);
debug!( debug!(
"leader config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}", "leader config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}",
leader_snapshot_test_config leader_snapshot_test_config
.bank_snapshots_dir .bank_snapshots_dir
.path() .path()
.display(), .display(),
leader_snapshot_test_config leader_snapshot_test_config
.snapshot_archives_dir .full_snapshot_archives_dir
.path()
.display(),
leader_snapshot_test_config
.incremental_snapshot_archives_dir
.path() .path()
.display(), .display(),
); );
debug!( debug!(
"validator config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}", "validator config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}",
validator_snapshot_test_config validator_snapshot_test_config
.bank_snapshots_dir .bank_snapshots_dir
.path() .path()
.display(), .display(),
validator_snapshot_test_config validator_snapshot_test_config
.snapshot_archives_dir .full_snapshot_archives_dir
.path()
.display(),
validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path() .path()
.display(), .display(),
); );
@ -734,7 +769,12 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
let (incremental_snapshot_archive, full_snapshot_archive) = let (incremental_snapshot_archive, full_snapshot_archive) =
LocalCluster::wait_for_next_incremental_snapshot( LocalCluster::wait_for_next_incremental_snapshot(
&cluster, &cluster,
leader_snapshot_test_config.snapshot_archives_dir.path(), leader_snapshot_test_config
.full_snapshot_archives_dir
.path(),
leader_snapshot_test_config
.incremental_snapshot_archives_dir
.path(),
); );
info!( info!(
"Found snapshots:\n\tfull snapshot: {}\n\tincremental snapshot: {}", "Found snapshots:\n\tfull snapshot: {}\n\tincremental snapshot: {}",
@ -751,7 +791,12 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
info!("Downloading full snapshot to validator..."); info!("Downloading full snapshot to validator...");
download_snapshot_archive( download_snapshot_archive(
&cluster.entry_point_info.rpc, &cluster.entry_point_info.rpc,
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path(),
(full_snapshot_archive.slot(), *full_snapshot_archive.hash()), (full_snapshot_archive.slot(), *full_snapshot_archive.hash()),
SnapshotType::FullSnapshot, SnapshotType::FullSnapshot,
validator_snapshot_test_config validator_snapshot_test_config
@ -771,7 +816,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
) )
.unwrap(); .unwrap();
let downloaded_full_snapshot_archive = snapshot_utils::get_highest_full_snapshot_archive_info( let downloaded_full_snapshot_archive = snapshot_utils::get_highest_full_snapshot_archive_info(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
) )
.unwrap(); .unwrap();
info!( info!(
@ -782,7 +829,12 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
info!("Downloading incremental snapshot to validator..."); info!("Downloading incremental snapshot to validator...");
download_snapshot_archive( download_snapshot_archive(
&cluster.entry_point_info.rpc, &cluster.entry_point_info.rpc,
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path(),
( (
incremental_snapshot_archive.slot(), incremental_snapshot_archive.slot(),
*incremental_snapshot_archive.hash(), *incremental_snapshot_archive.hash(),
@ -806,7 +858,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
.unwrap(); .unwrap();
let downloaded_incremental_snapshot_archive = let downloaded_incremental_snapshot_archive =
snapshot_utils::get_highest_incremental_snapshot_archive_info( snapshot_utils::get_highest_incremental_snapshot_archive_info(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path(),
full_snapshot_archive.slot(), full_snapshot_archive.slot(),
) )
.unwrap(); .unwrap();
@ -876,14 +930,30 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
// restart the node and guarantee that the only snapshots present are these initial ones. So, // restart the node and guarantee that the only snapshots present are these initial ones. So,
// the easiest way to do that is create a backup now, delete the ones on the node before // the easiest way to do that is create a backup now, delete the ones on the node before
// restart, then copy the backup ones over again. // restart, then copy the backup ones over again.
let backup_validator_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap(); let backup_validator_full_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
trace!( trace!(
"Backing up validator snapshots to dir: {}...", "Backing up validator full snapshots to dir: {}...",
backup_validator_snapshot_archives_dir.path().display() backup_validator_full_snapshot_archives_dir.path().display()
); );
copy_files_with_remote( copy_files_with_remote(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
backup_validator_snapshot_archives_dir.path(), .full_snapshot_archives_dir
.path(),
backup_validator_full_snapshot_archives_dir.path(),
);
let backup_validator_incremental_snapshot_archives_dir =
tempfile::tempdir_in(farf_dir()).unwrap();
trace!(
"Backing up validator incremental snapshots to dir: {}...",
backup_validator_incremental_snapshot_archives_dir
.path()
.display()
);
copy_files_with_remote(
validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path(),
backup_validator_incremental_snapshot_archives_dir.path(),
); );
info!("Starting the validator..."); info!("Starting the validator...");
@ -931,13 +1001,17 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
// Putting this all in its own block so its clear we're only intended to keep the leader's info // Putting this all in its own block so its clear we're only intended to keep the leader's info
let leader_full_snapshot_archive_for_comparison = { let leader_full_snapshot_archive_for_comparison = {
let validator_full_snapshot = snapshot_utils::get_highest_full_snapshot_archive_info( let validator_full_snapshot = snapshot_utils::get_highest_full_snapshot_archive_info(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
) )
.unwrap(); .unwrap();
// Now get the same full snapshot on the LEADER that we just got from the validator // Now get the same full snapshot on the LEADER that we just got from the validator
let mut leader_full_snapshots = snapshot_utils::get_full_snapshot_archives( let mut leader_full_snapshots = snapshot_utils::get_full_snapshot_archives(
leader_snapshot_test_config.snapshot_archives_dir.path(), leader_snapshot_test_config
.full_snapshot_archives_dir
.path(),
); );
leader_full_snapshots.retain(|full_snapshot| { leader_full_snapshots.retain(|full_snapshot| {
full_snapshot.slot() == validator_full_snapshot.slot() full_snapshot.slot() == validator_full_snapshot.slot()
@ -959,10 +1033,27 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
info!("leader full snapshot archive for comparison: {leader_full_snapshot_archive_for_comparison:#?}"); info!("leader full snapshot archive for comparison: {leader_full_snapshot_archive_for_comparison:#?}");
info!("Delete all the snapshots on the validator and restore the originals from the backup..."); info!("Delete all the snapshots on the validator and restore the originals from the backup...");
delete_files_with_remote(validator_snapshot_test_config.snapshot_archives_dir.path()); delete_files_with_remote(
validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
);
delete_files_with_remote(
validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path(),
);
copy_files_with_remote( copy_files_with_remote(
backup_validator_snapshot_archives_dir.path(), backup_validator_full_snapshot_archives_dir.path(),
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
);
copy_files_with_remote(
backup_validator_incremental_snapshot_archives_dir.path(),
validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path(),
); );
info!( info!(
"Delete all the snapshots on the validator and restore the originals from the backup... DONE" "Delete all the snapshots on the validator and restore the originals from the backup... DONE"
@ -971,7 +1062,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
// Get the highest full snapshot slot *before* restarting, as a comparison // Get the highest full snapshot slot *before* restarting, as a comparison
let validator_full_snapshot_slot_at_startup = let validator_full_snapshot_slot_at_startup =
snapshot_utils::get_highest_full_snapshot_archive_slot( snapshot_utils::get_highest_full_snapshot_archive_slot(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
) )
.unwrap(); .unwrap();
@ -998,12 +1091,16 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
let timer = Instant::now(); let timer = Instant::now();
loop { loop {
if let Some(full_snapshot_slot) = snapshot_utils::get_highest_full_snapshot_archive_slot( if let Some(full_snapshot_slot) = snapshot_utils::get_highest_full_snapshot_archive_slot(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
) { ) {
if full_snapshot_slot >= validator_next_full_snapshot_slot { if full_snapshot_slot >= validator_next_full_snapshot_slot {
if let Some(incremental_snapshot_slot) = if let Some(incremental_snapshot_slot) =
snapshot_utils::get_highest_incremental_snapshot_archive_slot( snapshot_utils::get_highest_incremental_snapshot_archive_slot(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path(),
full_snapshot_slot, full_snapshot_slot,
) )
{ {
@ -1034,7 +1131,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
// Check to make sure that the full snapshot the validator created during startup is the same // Check to make sure that the full snapshot the validator created during startup is the same
// or one greater than the snapshot the leader created. // or one greater than the snapshot the leader created.
let validator_full_snapshot_archives = snapshot_utils::get_full_snapshot_archives( let validator_full_snapshot_archives = snapshot_utils::get_full_snapshot_archives(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
); );
info!("validator full snapshot archives: {validator_full_snapshot_archives:#?}"); info!("validator full snapshot archives: {validator_full_snapshot_archives:#?}");
let validator_full_snapshot_archive_for_comparison = validator_full_snapshot_archives let validator_full_snapshot_archive_for_comparison = validator_full_snapshot_archives
@ -1071,12 +1170,29 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
// Copy over the snapshots to the new node, but need to remove the tmp snapshot dir so it // Copy over the snapshots to the new node, but need to remove the tmp snapshot dir so it
// doesn't break the simple copy_files closure. // doesn't break the simple copy_files closure.
snapshot_utils::remove_tmp_snapshot_archives( snapshot_utils::remove_tmp_snapshot_archives(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
);
snapshot_utils::remove_tmp_snapshot_archives(
validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path(),
); );
copy_files( copy_files(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config
.full_snapshot_archives_dir
.path(),
final_validator_snapshot_test_config final_validator_snapshot_test_config
.snapshot_archives_dir .full_snapshot_archives_dir
.path(),
);
copy_files(
validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path(),
final_validator_snapshot_test_config
.incremental_snapshot_archives_dir
.path(), .path(),
); );
@ -1128,19 +1244,20 @@ fn test_snapshot_restart_tower() {
let validator_info = cluster.exit_node(&validator_id); let validator_info = cluster.exit_node(&validator_id);
// Get slot after which this was generated // Get slot after which this was generated
let snapshot_archives_dir = &leader_snapshot_test_config let full_snapshot_archives_dir = &leader_snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_archives_dir; .full_snapshot_archives_dir;
let full_snapshot_archive_info = cluster.wait_for_next_full_snapshot(snapshot_archives_dir); let full_snapshot_archive_info =
cluster.wait_for_next_full_snapshot(full_snapshot_archives_dir);
// Copy archive to validator's snapshot output directory // Copy archive to validator's snapshot output directory
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path( let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
validator_snapshot_test_config validator_snapshot_test_config
.snapshot_archives_dir .full_snapshot_archives_dir
.into_path(), .into_path(),
full_snapshot_archive_info.slot(), full_snapshot_archive_info.slot(),
full_snapshot_archive_info.hash(), full_snapshot_archive_info.hash(),
@ -1178,12 +1295,12 @@ fn test_snapshots_blockstore_floor() {
let mut validator_snapshot_test_config = let mut validator_snapshot_test_config =
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths); setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
let snapshot_archives_dir = &leader_snapshot_test_config let full_snapshot_archives_dir = &leader_snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_archives_dir; .full_snapshot_archives_dir;
let mut config = ClusterConfig { let mut config = ClusterConfig {
node_stakes: vec![DEFAULT_NODE_STAKE], node_stakes: vec![DEFAULT_NODE_STAKE],
@ -1201,7 +1318,7 @@ fn test_snapshots_blockstore_floor() {
let archive_info = loop { let archive_info = loop {
let archive = let archive =
snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir); snapshot_utils::get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir);
if archive.is_some() { if archive.is_some() {
trace!("snapshot exists"); trace!("snapshot exists");
break archive.unwrap(); break archive.unwrap();
@ -1212,7 +1329,7 @@ fn test_snapshots_blockstore_floor() {
// Copy archive to validator's snapshot output directory // Copy archive to validator's snapshot output directory
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path( let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
validator_snapshot_test_config validator_snapshot_test_config
.snapshot_archives_dir .full_snapshot_archives_dir
.into_path(), .into_path(),
archive_info.slot(), archive_info.slot(),
archive_info.hash(), archive_info.hash(),
@ -1280,12 +1397,12 @@ fn test_snapshots_restart_validity() {
let num_account_paths = 1; let num_account_paths = 1;
let mut snapshot_test_config = let mut snapshot_test_config =
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths); setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
let snapshot_archives_dir = &snapshot_test_config let full_snapshot_archives_dir = &snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_archives_dir; .full_snapshot_archives_dir;
// Set up the cluster with 1 snapshotting validator // Set up the cluster with 1 snapshotting validator
let mut all_account_storage_dirs = vec![vec![]]; let mut all_account_storage_dirs = vec![vec![]];
@ -1323,7 +1440,7 @@ fn test_snapshots_restart_validity() {
expected_balances.extend(new_balances); expected_balances.extend(new_balances);
cluster.wait_for_next_full_snapshot(snapshot_archives_dir); cluster.wait_for_next_full_snapshot(full_snapshot_archives_dir);
// Create new account paths since validator exit is not guaranteed to cleanup RPC threads, // Create new account paths since validator exit is not guaranteed to cleanup RPC threads,
// which may delete the old accounts on exit at any point // which may delete the old accounts on exit at any point

15
rpc/src/rpc.rs
View File

@ -2546,17 +2546,22 @@ pub mod rpc_minimal {
return Err(RpcCustomError::NoSnapshot.into()); return Err(RpcCustomError::NoSnapshot.into());
} }
let snapshot_archives_dir = meta let (full_snapshot_archives_dir, incremental_snapshot_archives_dir) = meta
.snapshot_config .snapshot_config
.map(|snapshot_config| snapshot_config.snapshot_archives_dir) .map(|snapshot_config| {
(
snapshot_config.full_snapshot_archives_dir,
snapshot_config.incremental_snapshot_archives_dir,
)
})
.unwrap(); .unwrap();
let full_snapshot_slot = let full_snapshot_slot =
snapshot_utils::get_highest_full_snapshot_archive_slot(&snapshot_archives_dir) snapshot_utils::get_highest_full_snapshot_archive_slot(&full_snapshot_archives_dir)
.ok_or(RpcCustomError::NoSnapshot)?; .ok_or(RpcCustomError::NoSnapshot)?;
let incremental_snapshot_slot = let incremental_snapshot_slot =
snapshot_utils::get_highest_incremental_snapshot_archive_slot( snapshot_utils::get_highest_incremental_snapshot_archive_slot(
&snapshot_archives_dir, &incremental_snapshot_archives_dir,
full_snapshot_slot, full_snapshot_slot,
); );
@ -3925,7 +3930,7 @@ pub mod rpc_deprecated_v1_9 {
meta.snapshot_config meta.snapshot_config
.and_then(|snapshot_config| { .and_then(|snapshot_config| {
snapshot_utils::get_highest_full_snapshot_archive_slot( snapshot_utils::get_highest_full_snapshot_archive_slot(
&snapshot_config.snapshot_archives_dir, &snapshot_config.full_snapshot_archives_dir,
) )
}) })
.ok_or_else(|| RpcCustomError::NoSnapshot.into()) .ok_or_else(|| RpcCustomError::NoSnapshot.into())

21
rpc/src/rpc_service.rs
View File

@ -160,7 +160,22 @@ impl RpcRequestMiddleware {
where where
P: AsRef<Path>, P: AsRef<Path>,
{ {
let root = &self.snapshot_config.as_ref().unwrap().snapshot_archives_dir; let root = if self
.full_snapshot_archive_path_regex
.is_match(Path::new("").join(&stem).to_str().unwrap())
{
&self
.snapshot_config
.as_ref()
.unwrap()
.full_snapshot_archives_dir
} else {
&self
.snapshot_config
.as_ref()
.unwrap()
.incremental_snapshot_archives_dir
};
let local_path = root.join(&stem); let local_path = root.join(&stem);
if local_path.exists() { if local_path.exists() {
local_path local_path
@ -236,7 +251,7 @@ impl RequestMiddleware for RpcRequestMiddleware {
// Convenience redirect to the latest snapshot // Convenience redirect to the latest snapshot
let full_snapshot_archive_info = let full_snapshot_archive_info =
snapshot_utils::get_highest_full_snapshot_archive_info( snapshot_utils::get_highest_full_snapshot_archive_info(
&snapshot_config.snapshot_archives_dir, &snapshot_config.full_snapshot_archives_dir,
); );
let snapshot_archive_info = let snapshot_archive_info =
if let Some(full_snapshot_archive_info) = full_snapshot_archive_info { if let Some(full_snapshot_archive_info) = full_snapshot_archive_info {
@ -244,7 +259,7 @@ impl RequestMiddleware for RpcRequestMiddleware {
Some(full_snapshot_archive_info.snapshot_archive_info().clone()) Some(full_snapshot_archive_info.snapshot_archive_info().clone())
} else { } else {
snapshot_utils::get_highest_incremental_snapshot_archive_info( snapshot_utils::get_highest_incremental_snapshot_archive_info(
&snapshot_config.snapshot_archives_dir, &snapshot_config.incremental_snapshot_archives_dir,
full_snapshot_archive_info.slot(), full_snapshot_archive_info.slot(),
) )
.map(|incremental_snapshot_archive_info| { .map(|incremental_snapshot_archive_info| {

3
runtime/src/accounts_background_service.rs
View File

@ -285,7 +285,8 @@ impl SnapshotRequestHandler {
status_cache_slot_deltas, status_cache_slot_deltas,
&self.pending_accounts_package, &self.pending_accounts_package,
&self.snapshot_config.bank_snapshots_dir, &self.snapshot_config.bank_snapshots_dir,
&self.snapshot_config.snapshot_archives_dir, &self.snapshot_config.full_snapshot_archives_dir,
&self.snapshot_config.incremental_snapshot_archives_dir,
self.snapshot_config.snapshot_version, self.snapshot_config.snapshot_version,
self.snapshot_config.archive_format, self.snapshot_config.archive_format,
hash_for_testing, hash_for_testing,

10
runtime/src/snapshot_config.rs
View File

@ -13,8 +13,11 @@ pub struct SnapshotConfig {
/// Generate a new incremental snapshot archive every this many slots /// Generate a new incremental snapshot archive every this many slots
pub incremental_snapshot_archive_interval_slots: Slot, pub incremental_snapshot_archive_interval_slots: Slot,
/// Path to the directory where snapshot archives are stored /// Path to the directory where full snapshot archives are stored
pub snapshot_archives_dir: PathBuf, pub full_snapshot_archives_dir: PathBuf,
/// Path to the directory where incremental snapshot archives are stored
pub incremental_snapshot_archives_dir: PathBuf,
/// Path to the directory where bank snapshots are stored /// Path to the directory where bank snapshots are stored
pub bank_snapshots_dir: PathBuf, pub bank_snapshots_dir: PathBuf,
@ -46,7 +49,8 @@ impl Default for SnapshotConfig {
snapshot_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, snapshot_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
incremental_snapshot_archive_interval_slots: incremental_snapshot_archive_interval_slots:
snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
snapshot_archives_dir: PathBuf::default(), full_snapshot_archives_dir: PathBuf::default(),
incremental_snapshot_archives_dir: PathBuf::default(),
bank_snapshots_dir: PathBuf::default(), bank_snapshots_dir: PathBuf::default(),
archive_format: ArchiveFormat::TarBzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),

15
runtime/src/snapshot_package.rs
View File

@ -39,7 +39,8 @@ pub struct AccountsPackage {
pub snapshot_storages: SnapshotStorages, pub snapshot_storages: SnapshotStorages,
pub archive_format: ArchiveFormat, pub archive_format: ArchiveFormat,
pub snapshot_version: SnapshotVersion, pub snapshot_version: SnapshotVersion,
pub snapshot_archives_dir: PathBuf, pub full_snapshot_archives_dir: PathBuf,
pub incremental_snapshot_archives_dir: PathBuf,
pub expected_capitalization: u64, pub expected_capitalization: u64,
pub accounts_hash_for_testing: Option<Hash>, pub accounts_hash_for_testing: Option<Hash>,
pub cluster_type: ClusterType, pub cluster_type: ClusterType,
@ -57,7 +58,8 @@ impl AccountsPackage {
bank_snapshot_info: &BankSnapshotInfo, bank_snapshot_info: &BankSnapshotInfo,
bank_snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
slot_deltas: Vec<BankSlotDelta>, slot_deltas: Vec<BankSlotDelta>,
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
snapshot_storages: SnapshotStorages, snapshot_storages: SnapshotStorages,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
@ -105,7 +107,10 @@ impl AccountsPackage {
snapshot_storages, snapshot_storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
snapshot_archives_dir: snapshot_archives_dir.as_ref().to_path_buf(), full_snapshot_archives_dir: full_snapshot_archives_dir.as_ref().to_path_buf(),
incremental_snapshot_archives_dir: incremental_snapshot_archives_dir
.as_ref()
.to_path_buf(),
expected_capitalization: bank.capitalization(), expected_capitalization: bank.capitalization(),
accounts_hash_for_testing, accounts_hash_for_testing,
cluster_type: bank.cluster_type(), cluster_type: bank.cluster_type(),
@ -137,7 +142,7 @@ impl SnapshotPackage {
let mut snapshot_storages = accounts_package.snapshot_storages; let mut snapshot_storages = accounts_package.snapshot_storages;
let snapshot_archive_path = match accounts_package.snapshot_type.unwrap() { let snapshot_archive_path = match accounts_package.snapshot_type.unwrap() {
SnapshotType::FullSnapshot => snapshot_utils::build_full_snapshot_archive_path( SnapshotType::FullSnapshot => snapshot_utils::build_full_snapshot_archive_path(
accounts_package.snapshot_archives_dir, accounts_package.full_snapshot_archives_dir,
accounts_package.slot, accounts_package.slot,
&accounts_hash, &accounts_hash,
accounts_package.archive_format, accounts_package.archive_format,
@ -156,7 +161,7 @@ impl SnapshotPackage {
"Incremental snapshot package must only contain storage entries where slot > incremental snapshot base slot (i.e. full snapshot slot)!" "Incremental snapshot package must only contain storage entries where slot > incremental snapshot base slot (i.e. full snapshot slot)!"
); );
snapshot_utils::build_incremental_snapshot_archive_path( snapshot_utils::build_incremental_snapshot_archive_path(
accounts_package.snapshot_archives_dir, accounts_package.incremental_snapshot_archives_dir,
incremental_snapshot_base_slot, incremental_snapshot_base_slot,
accounts_package.slot, accounts_package.slot,
&accounts_hash, &accounts_hash,

306
runtime/src/snapshot_utils.rs
View File

@ -249,6 +249,8 @@ pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: impl AsRef<Path>) {
/// Make a snapshot archive out of the snapshot package /// Make a snapshot archive out of the snapshot package
pub fn archive_snapshot_package( pub fn archive_snapshot_package(
snapshot_package: &SnapshotPackage, snapshot_package: &SnapshotPackage,
full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
maximum_full_snapshot_archives_to_retain: usize, maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize, maximum_incremental_snapshot_archives_to_retain: usize,
) -> Result<()> { ) -> Result<()> {
@ -379,7 +381,8 @@ pub fn archive_snapshot_package(
.map_err(|e| SnapshotError::IoWithSource(e, "archive path rename"))?; .map_err(|e| SnapshotError::IoWithSource(e, "archive path rename"))?;
purge_old_snapshot_archives( purge_old_snapshot_archives(
tar_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain,
); );
@ -892,12 +895,13 @@ pub fn bank_from_snapshot_archives(
Ok((bank, timings)) Ok((bank, timings))
} }
/// Rebuild bank from snapshot archives. This function searches `snapshot_archives_dir` for the /// Rebuild bank from snapshot archives. This function searches `full_snapshot_archives_dir` and `incremental_snapshot_archives_dir` for the
/// highest full snapshot and highest corresponding incremental snapshot, then rebuilds the bank. /// highest full snapshot and highest corresponding incremental snapshot, then rebuilds the bank.
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn bank_from_latest_snapshot_archives( pub fn bank_from_latest_snapshot_archives(
bank_snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
account_paths: &[PathBuf], account_paths: &[PathBuf],
genesis_config: &GenesisConfig, genesis_config: &GenesisConfig,
debug_keys: Option<Arc<HashSet<Pubkey>>>, debug_keys: Option<Arc<HashSet<Pubkey>>>,
@ -916,11 +920,12 @@ pub fn bank_from_latest_snapshot_archives(
FullSnapshotArchiveInfo, FullSnapshotArchiveInfo,
Option<IncrementalSnapshotArchiveInfo>, Option<IncrementalSnapshotArchiveInfo>,
)> { )> {
let full_snapshot_archive_info = get_highest_full_snapshot_archive_info(&snapshot_archives_dir) let full_snapshot_archive_info =
.ok_or(SnapshotError::NoSnapshotArchives)?; get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir)
.ok_or(SnapshotError::NoSnapshotArchives)?;
let incremental_snapshot_archive_info = get_highest_incremental_snapshot_archive_info( let incremental_snapshot_archive_info = get_highest_incremental_snapshot_archive_info(
&snapshot_archives_dir, &incremental_snapshot_archives_dir,
full_snapshot_archive_info.slot(), full_snapshot_archive_info.slot(),
); );
@ -1125,12 +1130,12 @@ pub fn build_snapshot_archives_remote_dir(snapshot_archives_dir: impl AsRef<Path
/// Build the full snapshot archive path from its components: the snapshot archives directory, the /// Build the full snapshot archive path from its components: the snapshot archives directory, the
/// snapshot slot, the accounts hash, and the archive format. /// snapshot slot, the accounts hash, and the archive format.
pub fn build_full_snapshot_archive_path( pub fn build_full_snapshot_archive_path(
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: impl AsRef<Path>,
slot: Slot, slot: Slot,
hash: &Hash, hash: &Hash,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
) -> PathBuf { ) -> PathBuf {
snapshot_archives_dir.as_ref().join(format!( full_snapshot_archives_dir.as_ref().join(format!(
"snapshot-{}-{}.{}", "snapshot-{}-{}.{}",
slot, slot,
hash, hash,
@ -1142,13 +1147,13 @@ pub fn build_full_snapshot_archive_path(
/// directory, the snapshot base slot, the snapshot slot, the accounts hash, and the archive /// directory, the snapshot base slot, the snapshot slot, the accounts hash, and the archive
/// format. /// format.
pub fn build_incremental_snapshot_archive_path( pub fn build_incremental_snapshot_archive_path(
snapshot_archives_dir: impl AsRef<Path>, incremental_snapshot_archives_dir: impl AsRef<Path>,
base_slot: Slot, base_slot: Slot,
slot: Slot, slot: Slot,
hash: &Hash, hash: &Hash,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
) -> PathBuf { ) -> PathBuf {
snapshot_archives_dir.as_ref().join(format!( incremental_snapshot_archives_dir.as_ref().join(format!(
"incremental-snapshot-{}-{}-{}.{}", "incremental-snapshot-{}-{}-{}.{}",
base_slot, base_slot,
slot, slot,
@ -1256,74 +1261,66 @@ where
} }
/// Get a list of the full snapshot archives from a directory /// Get a list of the full snapshot archives from a directory
pub fn get_full_snapshot_archives<P>(snapshot_archives_dir: P) -> Vec<FullSnapshotArchiveInfo> pub fn get_full_snapshot_archives(
where full_snapshot_archives_dir: impl AsRef<Path>,
P: AsRef<Path>, ) -> Vec<FullSnapshotArchiveInfo> {
{
get_snapshot_archives( get_snapshot_archives(
snapshot_archives_dir.as_ref(), full_snapshot_archives_dir.as_ref(),
FullSnapshotArchiveInfo::new_from_path, FullSnapshotArchiveInfo::new_from_path,
) )
} }
/// Get a list of the incremental snapshot archives from a directory /// Get a list of the incremental snapshot archives from a directory
pub fn get_incremental_snapshot_archives<P>( pub fn get_incremental_snapshot_archives(
snapshot_archives_dir: P, incremental_snapshot_archives_dir: impl AsRef<Path>,
) -> Vec<IncrementalSnapshotArchiveInfo> ) -> Vec<IncrementalSnapshotArchiveInfo> {
where
P: AsRef<Path>,
{
get_snapshot_archives( get_snapshot_archives(
snapshot_archives_dir.as_ref(), incremental_snapshot_archives_dir.as_ref(),
IncrementalSnapshotArchiveInfo::new_from_path, IncrementalSnapshotArchiveInfo::new_from_path,
) )
} }
/// Get the highest slot of the full snapshot archives in a directory /// Get the highest slot of the full snapshot archives in a directory
pub fn get_highest_full_snapshot_archive_slot<P>(snapshot_archives_dir: P) -> Option<Slot> pub fn get_highest_full_snapshot_archive_slot(
where full_snapshot_archives_dir: impl AsRef<Path>,
P: AsRef<Path>, ) -> Option<Slot> {
{ get_highest_full_snapshot_archive_info(full_snapshot_archives_dir)
get_highest_full_snapshot_archive_info(snapshot_archives_dir)
.map(|full_snapshot_archive_info| full_snapshot_archive_info.slot()) .map(|full_snapshot_archive_info| full_snapshot_archive_info.slot())
} }
/// Get the highest slot of the incremental snapshot archives in a directory, for a given full /// Get the highest slot of the incremental snapshot archives in a directory, for a given full
/// snapshot slot /// snapshot slot
pub fn get_highest_incremental_snapshot_archive_slot<P: AsRef<Path>>( pub fn get_highest_incremental_snapshot_archive_slot(
snapshot_archives_dir: P, incremental_snapshot_archives_dir: impl AsRef<Path>,
full_snapshot_slot: Slot, full_snapshot_slot: Slot,
) -> Option<Slot> { ) -> Option<Slot> {
get_highest_incremental_snapshot_archive_info(snapshot_archives_dir, full_snapshot_slot) get_highest_incremental_snapshot_archive_info(
.map(|incremental_snapshot_archive_info| incremental_snapshot_archive_info.slot()) incremental_snapshot_archives_dir,
full_snapshot_slot,
)
.map(|incremental_snapshot_archive_info| incremental_snapshot_archive_info.slot())
} }
/// Get the path (and metadata) for the full snapshot archive with the highest slot in a directory /// Get the path (and metadata) for the full snapshot archive with the highest slot in a directory
pub fn get_highest_full_snapshot_archive_info<P>( pub fn get_highest_full_snapshot_archive_info(
snapshot_archives_dir: P, full_snapshot_archives_dir: impl AsRef<Path>,
) -> Option<FullSnapshotArchiveInfo> ) -> Option<FullSnapshotArchiveInfo> {
where let mut full_snapshot_archives = get_full_snapshot_archives(full_snapshot_archives_dir);
P: AsRef<Path>,
{
let mut full_snapshot_archives = get_full_snapshot_archives(snapshot_archives_dir);
full_snapshot_archives.sort_unstable(); full_snapshot_archives.sort_unstable();
full_snapshot_archives.into_iter().rev().next() full_snapshot_archives.into_iter().rev().next()
} }
/// Get the path for the incremental snapshot archive with the highest slot, for a given full /// Get the path for the incremental snapshot archive with the highest slot, for a given full
/// snapshot slot, in a directory /// snapshot slot, in a directory
pub fn get_highest_incremental_snapshot_archive_info<P>( pub fn get_highest_incremental_snapshot_archive_info(
snapshot_archives_dir: P, incremental_snapshot_archives_dir: impl AsRef<Path>,
full_snapshot_slot: Slot, full_snapshot_slot: Slot,
) -> Option<IncrementalSnapshotArchiveInfo> ) -> Option<IncrementalSnapshotArchiveInfo> {
where
P: AsRef<Path>,
{
// Since we want to filter down to only the incremental snapshot archives that have the same // Since we want to filter down to only the incremental snapshot archives that have the same
// full snapshot slot as the value passed in, perform the filtering before sorting to avoid // full snapshot slot as the value passed in, perform the filtering before sorting to avoid
// doing unnecessary work. // doing unnecessary work.
let mut incremental_snapshot_archives = let mut incremental_snapshot_archives =
get_incremental_snapshot_archives(snapshot_archives_dir) get_incremental_snapshot_archives(incremental_snapshot_archives_dir)
.into_iter() .into_iter()
.filter(|incremental_snapshot_archive_info| { .filter(|incremental_snapshot_archive_info| {
incremental_snapshot_archive_info.base_slot() == full_snapshot_slot incremental_snapshot_archive_info.base_slot() == full_snapshot_slot
@ -1333,21 +1330,19 @@ where
incremental_snapshot_archives.into_iter().rev().next() incremental_snapshot_archives.into_iter().rev().next()
} }
pub fn purge_old_snapshot_archives<P>( pub fn purge_old_snapshot_archives(
snapshot_archives_dir: P, full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
maximum_full_snapshot_archives_to_retain: usize, maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize, maximum_incremental_snapshot_archives_to_retain: usize,
) where ) {
P: AsRef<Path>,
{
info!( info!(
"Purging old snapshot archives in {}, retaining up to {} full snapshots and up to {} incremental snapshots", "Purging old full snapshot archives in {}, retaining up to {} full snapshots",
snapshot_archives_dir.as_ref().display(), full_snapshot_archives_dir.as_ref().display(),
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain
maximum_incremental_snapshot_archives_to_retain
); );
let mut full_snapshot_archives = get_full_snapshot_archives(&snapshot_archives_dir); let mut full_snapshot_archives = get_full_snapshot_archives(&full_snapshot_archives_dir);
full_snapshot_archives.sort_unstable(); full_snapshot_archives.sort_unstable();
full_snapshot_archives.reverse(); full_snapshot_archives.reverse();
@ -1383,8 +1378,15 @@ pub fn purge_old_snapshot_archives<P>(
} }
remove_archives(full_snapshot_archives_to_remove); remove_archives(full_snapshot_archives_to_remove);
info!(
"Purging old incremental snapshot archives in {}, retaining up to {} incremental snapshots",
incremental_snapshot_archives_dir.as_ref().display(),
maximum_incremental_snapshot_archives_to_retain
);
let mut incremental_snapshot_archives_by_base_slot = HashMap::<Slot, Vec<_>>::new(); let mut incremental_snapshot_archives_by_base_slot = HashMap::<Slot, Vec<_>>::new();
for incremental_snapshot_archive in get_incremental_snapshot_archives(&snapshot_archives_dir) { for incremental_snapshot_archive in
get_incremental_snapshot_archives(&incremental_snapshot_archives_dir)
{
incremental_snapshot_archives_by_base_slot incremental_snapshot_archives_by_base_slot
.entry(incremental_snapshot_archive.base_slot()) .entry(incremental_snapshot_archive.base_slot())
.or_default() .or_default()
@ -1724,12 +1726,14 @@ pub fn purge_old_bank_snapshots(bank_snapshots_dir: impl AsRef<Path>) {
/// function is called from AccountsBackgroundService to handle snapshot requests. Since taking a /// function is called from AccountsBackgroundService to handle snapshot requests. Since taking a
/// snapshot is not permitted to fail, any errors returned here will trigger the node to shutdown. /// snapshot is not permitted to fail, any errors returned here will trigger the node to shutdown.
/// So, be careful whenever adding new code that may return errors. /// So, be careful whenever adding new code that may return errors.
#[allow(clippy::too_many_arguments)]
pub fn snapshot_bank( pub fn snapshot_bank(
root_bank: &Bank, root_bank: &Bank,
status_cache_slot_deltas: Vec<BankSlotDelta>, status_cache_slot_deltas: Vec<BankSlotDelta>,
pending_accounts_package: &PendingAccountsPackage, pending_accounts_package: &PendingAccountsPackage,
bank_snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
hash_for_testing: Option<Hash>, hash_for_testing: Option<Hash>,
@ -1752,7 +1756,8 @@ pub fn snapshot_bank(
&bank_snapshot_info, &bank_snapshot_info,
bank_snapshots_dir, bank_snapshots_dir,
status_cache_slot_deltas, status_cache_slot_deltas,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
snapshot_storages, snapshot_storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
@ -1810,7 +1815,8 @@ pub fn bank_to_full_snapshot_archive(
bank_snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
bank: &Bank, bank: &Bank,
snapshot_version: Option<SnapshotVersion>, snapshot_version: Option<SnapshotVersion>,
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
maximum_full_snapshot_archives_to_retain: usize, maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize, maximum_incremental_snapshot_archives_to_retain: usize,
@ -1833,7 +1839,8 @@ pub fn bank_to_full_snapshot_archive(
bank, bank,
&bank_snapshot_info, &bank_snapshot_info,
&temp_dir, &temp_dir,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
snapshot_storages, snapshot_storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
@ -1853,7 +1860,8 @@ pub fn bank_to_incremental_snapshot_archive(
bank: &Bank, bank: &Bank,
full_snapshot_slot: Slot, full_snapshot_slot: Slot,
snapshot_version: Option<SnapshotVersion>, snapshot_version: Option<SnapshotVersion>,
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
maximum_full_snapshot_archives_to_retain: usize, maximum_full_snapshot_archives_to_retain: usize,
maximum_incremental_snapshot_archives_to_retain: usize, maximum_incremental_snapshot_archives_to_retain: usize,
@ -1878,7 +1886,8 @@ pub fn bank_to_incremental_snapshot_archive(
full_snapshot_slot, full_snapshot_slot,
&bank_snapshot_info, &bank_snapshot_info,
&temp_dir, &temp_dir,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
snapshot_storages, snapshot_storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
@ -1888,11 +1897,13 @@ pub fn bank_to_incremental_snapshot_archive(
} }
/// Helper function to hold shared code to package, process, and archive full snapshots /// Helper function to hold shared code to package, process, and archive full snapshots
#[allow(clippy::too_many_arguments)]
pub fn package_and_archive_full_snapshot( pub fn package_and_archive_full_snapshot(
bank: &Bank, bank: &Bank,
bank_snapshot_info: &BankSnapshotInfo, bank_snapshot_info: &BankSnapshotInfo,
bank_snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
snapshot_storages: SnapshotStorages, snapshot_storages: SnapshotStorages,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
@ -1904,7 +1915,8 @@ pub fn package_and_archive_full_snapshot(
bank_snapshot_info, bank_snapshot_info,
bank_snapshots_dir, bank_snapshots_dir,
bank.src.slot_deltas(&bank.src.roots()), bank.src.slot_deltas(&bank.src.roots()),
snapshot_archives_dir, &full_snapshot_archives_dir,
&incremental_snapshot_archives_dir,
snapshot_storages, snapshot_storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
@ -1921,6 +1933,8 @@ pub fn package_and_archive_full_snapshot(
let snapshot_package = SnapshotPackage::new(accounts_package, bank.get_accounts_hash()); let snapshot_package = SnapshotPackage::new(accounts_package, bank.get_accounts_hash());
archive_snapshot_package( archive_snapshot_package(
&snapshot_package, &snapshot_package,
full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain,
)?; )?;
@ -1937,7 +1951,8 @@ pub fn package_and_archive_incremental_snapshot(
incremental_snapshot_base_slot: Slot, incremental_snapshot_base_slot: Slot,
bank_snapshot_info: &BankSnapshotInfo, bank_snapshot_info: &BankSnapshotInfo,
bank_snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
snapshot_archives_dir: impl AsRef<Path>, full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
snapshot_storages: SnapshotStorages, snapshot_storages: SnapshotStorages,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
@ -1949,7 +1964,8 @@ pub fn package_and_archive_incremental_snapshot(
bank_snapshot_info, bank_snapshot_info,
bank_snapshots_dir, bank_snapshots_dir,
bank.src.slot_deltas(&bank.src.roots()), bank.src.slot_deltas(&bank.src.roots()),
snapshot_archives_dir, &full_snapshot_archives_dir,
&incremental_snapshot_archives_dir,
snapshot_storages, snapshot_storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
@ -1968,6 +1984,8 @@ pub fn package_and_archive_incremental_snapshot(
let snapshot_package = SnapshotPackage::new(accounts_package, bank.get_accounts_hash()); let snapshot_package = SnapshotPackage::new(accounts_package, bank.get_accounts_hash());
archive_snapshot_package( archive_snapshot_package(
&snapshot_package, &snapshot_package,
full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain,
)?; )?;
@ -2414,13 +2432,15 @@ mod tests {
/// `max_incremental_snapshot_slot`]. Additionally, "bad" files are created for both full and /// `max_incremental_snapshot_slot`]. Additionally, "bad" files are created for both full and
/// incremental snapshots to ensure the tests properly filter them out. /// incremental snapshots to ensure the tests properly filter them out.
fn common_create_snapshot_archive_files( fn common_create_snapshot_archive_files(
snapshot_archives_dir: &Path, full_snapshot_archives_dir: &Path,
incremental_snapshot_archives_dir: &Path,
min_full_snapshot_slot: Slot, min_full_snapshot_slot: Slot,
max_full_snapshot_slot: Slot, max_full_snapshot_slot: Slot,
min_incremental_snapshot_slot: Slot, min_incremental_snapshot_slot: Slot,
max_incremental_snapshot_slot: Slot, max_incremental_snapshot_slot: Slot,
) { ) {
fs::create_dir_all(snapshot_archives_dir).unwrap(); fs::create_dir_all(full_snapshot_archives_dir).unwrap();
fs::create_dir_all(incremental_snapshot_archives_dir).unwrap();
for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot { for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot {
for incremental_snapshot_slot in for incremental_snapshot_slot in
min_incremental_snapshot_slot..max_incremental_snapshot_slot min_incremental_snapshot_slot..max_incremental_snapshot_slot
@ -2431,13 +2451,13 @@ mod tests {
incremental_snapshot_slot, incremental_snapshot_slot,
Hash::default() Hash::default()
); );
let snapshot_filepath = snapshot_archives_dir.join(snapshot_filename); let snapshot_filepath = incremental_snapshot_archives_dir.join(snapshot_filename);
File::create(snapshot_filepath).unwrap(); File::create(snapshot_filepath).unwrap();
} }
let snapshot_filename = let snapshot_filename =
format!("snapshot-{}-{}.tar", full_snapshot_slot, Hash::default()); format!("snapshot-{}-{}.tar", full_snapshot_slot, Hash::default());
let snapshot_filepath = snapshot_archives_dir.join(snapshot_filename); let snapshot_filepath = full_snapshot_archives_dir.join(snapshot_filename);
File::create(snapshot_filepath).unwrap(); File::create(snapshot_filepath).unwrap();
// Add in an incremental snapshot with a bad filename and high slot to ensure filename are filtered and sorted correctly // Add in an incremental snapshot with a bad filename and high slot to ensure filename are filtered and sorted correctly
@ -2446,50 +2466,54 @@ mod tests {
full_snapshot_slot, full_snapshot_slot,
max_incremental_snapshot_slot + 1, max_incremental_snapshot_slot + 1,
); );
let bad_filepath = snapshot_archives_dir.join(bad_filename); let bad_filepath = incremental_snapshot_archives_dir.join(bad_filename);
File::create(bad_filepath).unwrap(); File::create(bad_filepath).unwrap();
} }
// Add in a snapshot with a bad filename and high slot to ensure filename are filtered and // Add in a snapshot with a bad filename and high slot to ensure filename are filtered and
// sorted correctly // sorted correctly
let bad_filename = format!("snapshot-{}-bad!hash.tar", max_full_snapshot_slot + 1); let bad_filename = format!("snapshot-{}-bad!hash.tar", max_full_snapshot_slot + 1);
let bad_filepath = snapshot_archives_dir.join(bad_filename); let bad_filepath = full_snapshot_archives_dir.join(bad_filename);
File::create(bad_filepath).unwrap(); File::create(bad_filepath).unwrap();
} }
#[test] #[test]
fn test_get_full_snapshot_archives() { fn test_get_full_snapshot_archives() {
solana_logger::setup(); solana_logger::setup();
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let min_slot = 123; let min_slot = 123;
let max_slot = 456; let max_slot = 456;
common_create_snapshot_archive_files( common_create_snapshot_archive_files(
temp_snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
min_slot, min_slot,
max_slot, max_slot,
0, 0,
0, 0,
); );
let snapshot_archives = get_full_snapshot_archives(temp_snapshot_archives_dir); let snapshot_archives = get_full_snapshot_archives(full_snapshot_archives_dir);
assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot); assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot);
} }
#[test] #[test]
fn test_get_full_snapshot_archives_remote() { fn test_get_full_snapshot_archives_remote() {
solana_logger::setup(); solana_logger::setup();
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let min_slot = 123; let min_slot = 123;
let max_slot = 456; let max_slot = 456;
common_create_snapshot_archive_files( common_create_snapshot_archive_files(
&temp_snapshot_archives_dir.path().join("remote"), &full_snapshot_archives_dir.path().join("remote"),
&incremental_snapshot_archives_dir.path().join("remote"),
min_slot, min_slot,
max_slot, max_slot,
0, 0,
0, 0,
); );
let snapshot_archives = get_full_snapshot_archives(temp_snapshot_archives_dir); let snapshot_archives = get_full_snapshot_archives(full_snapshot_archives_dir);
assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot); assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot);
assert!(snapshot_archives.iter().all(|info| info.is_remote())); assert!(snapshot_archives.iter().all(|info| info.is_remote()));
} }
@ -2497,13 +2521,15 @@ mod tests {
#[test] #[test]
fn test_get_incremental_snapshot_archives() { fn test_get_incremental_snapshot_archives() {
solana_logger::setup(); solana_logger::setup();
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let min_full_snapshot_slot = 12; let min_full_snapshot_slot = 12;
let max_full_snapshot_slot = 23; let max_full_snapshot_slot = 23;
let min_incremental_snapshot_slot = 34; let min_incremental_snapshot_slot = 34;
let max_incremental_snapshot_slot = 45; let max_incremental_snapshot_slot = 45;
common_create_snapshot_archive_files( common_create_snapshot_archive_files(
temp_snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
min_full_snapshot_slot, min_full_snapshot_slot,
max_full_snapshot_slot, max_full_snapshot_slot,
min_incremental_snapshot_slot, min_incremental_snapshot_slot,
@ -2511,7 +2537,7 @@ mod tests {
); );
let incremental_snapshot_archives = let incremental_snapshot_archives =
get_incremental_snapshot_archives(temp_snapshot_archives_dir); get_incremental_snapshot_archives(incremental_snapshot_archives_dir);
assert_eq!( assert_eq!(
incremental_snapshot_archives.len() as Slot, incremental_snapshot_archives.len() as Slot,
(max_full_snapshot_slot - min_full_snapshot_slot) (max_full_snapshot_slot - min_full_snapshot_slot)
@ -2522,13 +2548,15 @@ mod tests {
#[test] #[test]
fn test_get_incremental_snapshot_archives_remote() { fn test_get_incremental_snapshot_archives_remote() {
solana_logger::setup(); solana_logger::setup();
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let min_full_snapshot_slot = 12; let min_full_snapshot_slot = 12;
let max_full_snapshot_slot = 23; let max_full_snapshot_slot = 23;
let min_incremental_snapshot_slot = 34; let min_incremental_snapshot_slot = 34;
let max_incremental_snapshot_slot = 45; let max_incremental_snapshot_slot = 45;
common_create_snapshot_archive_files( common_create_snapshot_archive_files(
&temp_snapshot_archives_dir.path().join("remote"), &full_snapshot_archives_dir.path().join("remote"),
&incremental_snapshot_archives_dir.path().join("remote"),
min_full_snapshot_slot, min_full_snapshot_slot,
max_full_snapshot_slot, max_full_snapshot_slot,
min_incremental_snapshot_slot, min_incremental_snapshot_slot,
@ -2536,7 +2564,7 @@ mod tests {
); );
let incremental_snapshot_archives = let incremental_snapshot_archives =
get_incremental_snapshot_archives(temp_snapshot_archives_dir); get_incremental_snapshot_archives(incremental_snapshot_archives_dir);
assert_eq!( assert_eq!(
incremental_snapshot_archives.len() as Slot, incremental_snapshot_archives.len() as Slot,
(max_full_snapshot_slot - min_full_snapshot_slot) (max_full_snapshot_slot - min_full_snapshot_slot)
@ -2550,11 +2578,13 @@ mod tests {
#[test] #[test]
fn test_get_highest_full_snapshot_archive_slot() { fn test_get_highest_full_snapshot_archive_slot() {
solana_logger::setup(); solana_logger::setup();
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let min_slot = 123; let min_slot = 123;
let max_slot = 456; let max_slot = 456;
common_create_snapshot_archive_files( common_create_snapshot_archive_files(
temp_snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
min_slot, min_slot,
max_slot, max_slot,
0, 0,
@ -2562,7 +2592,7 @@ mod tests {
); );
assert_eq!( assert_eq!(
get_highest_full_snapshot_archive_slot(temp_snapshot_archives_dir.path()), get_highest_full_snapshot_archive_slot(full_snapshot_archives_dir.path()),
Some(max_slot - 1) Some(max_slot - 1)
); );
} }
@ -2570,13 +2600,15 @@ mod tests {
#[test] #[test]
fn test_get_highest_incremental_snapshot_slot() { fn test_get_highest_incremental_snapshot_slot() {
solana_logger::setup(); solana_logger::setup();
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let min_full_snapshot_slot = 12; let min_full_snapshot_slot = 12;
let max_full_snapshot_slot = 23; let max_full_snapshot_slot = 23;
let min_incremental_snapshot_slot = 34; let min_incremental_snapshot_slot = 34;
let max_incremental_snapshot_slot = 45; let max_incremental_snapshot_slot = 45;
common_create_snapshot_archive_files( common_create_snapshot_archive_files(
temp_snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
min_full_snapshot_slot, min_full_snapshot_slot,
max_full_snapshot_slot, max_full_snapshot_slot,
min_incremental_snapshot_slot, min_incremental_snapshot_slot,
@ -2586,7 +2618,7 @@ mod tests {
for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot { for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot {
assert_eq!( assert_eq!(
get_highest_incremental_snapshot_archive_slot( get_highest_incremental_snapshot_archive_slot(
temp_snapshot_archives_dir.path(), incremental_snapshot_archives_dir.path(),
full_snapshot_slot full_snapshot_slot
), ),
Some(max_incremental_snapshot_slot - 1) Some(max_incremental_snapshot_slot - 1)
@ -2595,7 +2627,7 @@ mod tests {
assert_eq!( assert_eq!(
get_highest_incremental_snapshot_archive_slot( get_highest_incremental_snapshot_archive_slot(
temp_snapshot_archives_dir.path(), incremental_snapshot_archives_dir.path(),
max_full_snapshot_slot max_full_snapshot_slot
), ),
None None
@ -2615,6 +2647,7 @@ mod tests {
let mut _snap_file = File::create(snap_path); let mut _snap_file = File::create(snap_path);
} }
purge_old_snapshot_archives( purge_old_snapshot_archives(
temp_snap_dir.path(),
temp_snap_dir.path(), temp_snap_dir.path(),
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain,
@ -2691,14 +2724,15 @@ mod tests {
/// snapshot archives on disk are correct. /// snapshot archives on disk are correct.
#[test] #[test]
fn test_purge_old_full_snapshot_archives_in_the_loop() { fn test_purge_old_full_snapshot_archives_in_the_loop() {
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let maximum_snapshots_to_retain = 5; let maximum_snapshots_to_retain = 5;
let starting_slot: Slot = 42; let starting_slot: Slot = 42;
for slot in (starting_slot..).take(100) { for slot in (starting_slot..).take(100) {
let full_snapshot_archive_file_name = let full_snapshot_archive_file_name =
format!("snapshot-{}-{}.tar", slot, Hash::default()); format!("snapshot-{}-{}.tar", slot, Hash::default());
let full_snapshot_archive_path = snapshot_archives_dir let full_snapshot_archive_path = full_snapshot_archives_dir
.as_ref() .as_ref()
.join(full_snapshot_archive_file_name); .join(full_snapshot_archive_file_name);
File::create(full_snapshot_archive_path).unwrap(); File::create(full_snapshot_archive_path).unwrap();
@ -2714,11 +2748,13 @@ mod tests {
} }
purge_old_snapshot_archives( purge_old_snapshot_archives(
&snapshot_archives_dir, &full_snapshot_archives_dir,
&incremental_snapshot_archives_dir,
maximum_snapshots_to_retain, maximum_snapshots_to_retain,
usize::MAX, usize::MAX,
); );
let mut full_snapshot_archives = get_full_snapshot_archives(&snapshot_archives_dir); let mut full_snapshot_archives =
get_full_snapshot_archives(&full_snapshot_archives_dir);
full_snapshot_archives.sort_unstable(); full_snapshot_archives.sort_unstable();
assert_eq!(full_snapshot_archives.len(), maximum_snapshots_to_retain); assert_eq!(full_snapshot_archives.len(), maximum_snapshots_to_retain);
assert_eq!(full_snapshot_archives.last().unwrap().slot(), slot); assert_eq!(full_snapshot_archives.last().unwrap().slot(), slot);
@ -2731,7 +2767,8 @@ mod tests {
#[test] #[test]
fn test_purge_old_incremental_snapshot_archives() { fn test_purge_old_incremental_snapshot_archives() {
solana_logger::setup(); solana_logger::setup();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let starting_slot = 100_000; let starting_slot = 100_000;
let maximum_incremental_snapshot_archives_to_retain = let maximum_incremental_snapshot_archives_to_retain =
@ -2751,7 +2788,7 @@ mod tests {
.for_each(|full_snapshot_slot| { .for_each(|full_snapshot_slot| {
let snapshot_filename = let snapshot_filename =
format!("snapshot-{}-{}.tar", full_snapshot_slot, Hash::default()); format!("snapshot-{}-{}.tar", full_snapshot_slot, Hash::default());
let snapshot_path = snapshot_archives_dir.path().join(&snapshot_filename); let snapshot_path = full_snapshot_archives_dir.path().join(&snapshot_filename);
File::create(snapshot_path).unwrap(); File::create(snapshot_path).unwrap();
snapshot_filenames.push(snapshot_filename); snapshot_filenames.push(snapshot_filename);
@ -2766,14 +2803,17 @@ mod tests {
incremental_snapshot_slot, incremental_snapshot_slot,
Hash::default() Hash::default()
); );
let snapshot_path = snapshot_archives_dir.path().join(&snapshot_filename); let snapshot_path = incremental_snapshot_archives_dir
.path()
.join(&snapshot_filename);
File::create(snapshot_path).unwrap(); File::create(snapshot_path).unwrap();
snapshot_filenames.push(snapshot_filename); snapshot_filenames.push(snapshot_filename);
}); });
}); });
purge_old_snapshot_archives( purge_old_snapshot_archives(
snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
maximum_incremental_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain,
); );
@ -2781,7 +2821,7 @@ mod tests {
// Ensure correct number of full snapshot archives are purged/retained // Ensure correct number of full snapshot archives are purged/retained
// NOTE: One extra full snapshot is always kept (the oldest), hence the `+1` // NOTE: One extra full snapshot is always kept (the oldest), hence the `+1`
let mut remaining_full_snapshot_archives = let mut remaining_full_snapshot_archives =
get_full_snapshot_archives(snapshot_archives_dir.path()); get_full_snapshot_archives(full_snapshot_archives_dir.path());
assert_eq!( assert_eq!(
remaining_full_snapshot_archives.len(), remaining_full_snapshot_archives.len(),
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
@ -2792,7 +2832,7 @@ mod tests {
// Ensure correct number of incremental snapshot archives are purged/retained // Ensure correct number of incremental snapshot archives are purged/retained
let mut remaining_incremental_snapshot_archives = let mut remaining_incremental_snapshot_archives =
get_incremental_snapshot_archives(snapshot_archives_dir.path()); get_incremental_snapshot_archives(incremental_snapshot_archives_dir.path());
assert_eq!( assert_eq!(
remaining_incremental_snapshot_archives.len(), remaining_incremental_snapshot_archives.len(),
maximum_incremental_snapshot_archives_to_retain maximum_incremental_snapshot_archives_to_retain
@ -2846,7 +2886,8 @@ mod tests {
#[test] #[test]
fn test_purge_all_incremental_snapshot_archives_when_no_full_snapshot_archives() { fn test_purge_all_incremental_snapshot_archives_when_no_full_snapshot_archives() {
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
for snapshot_filenames in [ for snapshot_filenames in [
format!("incremental-snapshot-100-120-{}.tar", Hash::default()), format!("incremental-snapshot-100-120-{}.tar", Hash::default()),
@ -2858,14 +2899,21 @@ mod tests {
format!("incremental-snapshot-200-260-{}.tar", Hash::default()), format!("incremental-snapshot-200-260-{}.tar", Hash::default()),
format!("incremental-snapshot-200-280-{}.tar", Hash::default()), format!("incremental-snapshot-200-280-{}.tar", Hash::default()),
] { ] {
let snapshot_path = snapshot_archives_dir.path().join(&snapshot_filenames); let snapshot_path = incremental_snapshot_archives_dir
.path()
.join(&snapshot_filenames);
File::create(snapshot_path).unwrap(); File::create(snapshot_path).unwrap();
} }
purge_old_snapshot_archives(snapshot_archives_dir.path(), usize::MAX, usize::MAX); purge_old_snapshot_archives(
full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
usize::MAX,
usize::MAX,
);
let remaining_incremental_snapshot_archives = let remaining_incremental_snapshot_archives =
get_incremental_snapshot_archives(snapshot_archives_dir.path()); get_incremental_snapshot_archives(incremental_snapshot_archives_dir.path());
assert!(remaining_incremental_snapshot_archives.is_empty()); assert!(remaining_incremental_snapshot_archives.is_empty());
} }
@ -2883,14 +2931,16 @@ mod tests {
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::Tar; let snapshot_archive_format = ArchiveFormat::Tar;
let snapshot_archive_info = bank_to_full_snapshot_archive( let snapshot_archive_info = bank_to_full_snapshot_archive(
&bank_snapshots_dir, &bank_snapshots_dir,
&original_bank, &original_bank,
None, None,
snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
snapshot_archive_format, snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -2974,14 +3024,16 @@ mod tests {
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::TarGzip; let snapshot_archive_format = ArchiveFormat::TarGzip;
let full_snapshot_archive_info = bank_to_full_snapshot_archive( let full_snapshot_archive_info = bank_to_full_snapshot_archive(
bank_snapshots_dir.path(), bank_snapshots_dir.path(),
&bank4, &bank4,
None, None,
snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
snapshot_archive_format, snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -3050,7 +3102,8 @@ mod tests {
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::TarZstd; let snapshot_archive_format = ArchiveFormat::TarZstd;
let full_snapshot_slot = slot; let full_snapshot_slot = slot;
@ -3058,7 +3111,8 @@ mod tests {
bank_snapshots_dir.path(), bank_snapshots_dir.path(),
&bank1, &bank1,
None, None,
snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
snapshot_archive_format, snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -3091,7 +3145,8 @@ mod tests {
&bank4, &bank4,
full_snapshot_slot, full_snapshot_slot,
None, None,
snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
snapshot_archive_format, snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -3150,7 +3205,8 @@ mod tests {
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::Tar; let snapshot_archive_format = ArchiveFormat::Tar;
let full_snapshot_slot = slot; let full_snapshot_slot = slot;
@ -3158,7 +3214,8 @@ mod tests {
&bank_snapshots_dir, &bank_snapshots_dir,
&bank1, &bank1,
None, None,
&snapshot_archives_dir, &full_snapshot_archives_dir,
&incremental_snapshot_archives_dir,
snapshot_archive_format, snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -3191,7 +3248,8 @@ mod tests {
&bank4, &bank4,
full_snapshot_slot, full_snapshot_slot,
None, None,
&snapshot_archives_dir, &full_snapshot_archives_dir,
&incremental_snapshot_archives_dir,
snapshot_archive_format, snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -3200,7 +3258,8 @@ mod tests {
let (deserialized_bank, ..) = bank_from_latest_snapshot_archives( let (deserialized_bank, ..) = bank_from_latest_snapshot_archives(
&bank_snapshots_dir, &bank_snapshots_dir,
&snapshot_archives_dir, &full_snapshot_archives_dir,
&incremental_snapshot_archives_dir,
&[accounts_dir.as_ref().to_path_buf()], &[accounts_dir.as_ref().to_path_buf()],
&genesis_config, &genesis_config,
None, None,
@ -3252,7 +3311,8 @@ mod tests {
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::Tar; let snapshot_archive_format = ArchiveFormat::Tar;
let (genesis_config, mint_keypair) = create_genesis_config(1_000_000); let (genesis_config, mint_keypair) = create_genesis_config(1_000_000);
@ -3289,7 +3349,8 @@ mod tests {
bank_snapshots_dir.path(), bank_snapshots_dir.path(),
&bank1, &bank1,
None, None,
snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
snapshot_archive_format, snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -3329,7 +3390,8 @@ mod tests {
&bank2, &bank2,
full_snapshot_slot, full_snapshot_slot,
None, None,
snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
snapshot_archive_format, snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -3390,7 +3452,8 @@ mod tests {
&bank4, &bank4,
full_snapshot_slot, full_snapshot_slot,
None, None,
snapshot_archives_dir.path(), full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
snapshot_archive_format, snapshot_archive_format,
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -3455,7 +3518,8 @@ mod tests {
snapshot_storages: SnapshotStorages::default(), snapshot_storages: SnapshotStorages::default(),
archive_format: ArchiveFormat::Tar, archive_format: ArchiveFormat::Tar,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),
snapshot_archives_dir: PathBuf::default(), full_snapshot_archives_dir: PathBuf::default(),
incremental_snapshot_archives_dir: PathBuf::default(),
expected_capitalization: u64::default(), expected_capitalization: u64::default(),
accounts_hash_for_testing: None, accounts_hash_for_testing: None,
cluster_type: solana_sdk::genesis_config::ClusterType::Development, cluster_type: solana_sdk::genesis_config::ClusterType::Development,

3
test-validator/src/lib.rs
View File

@ -711,7 +711,8 @@ impl TestValidator {
full_snapshot_archive_interval_slots: 100, full_snapshot_archive_interval_slots: 100,
incremental_snapshot_archive_interval_slots: Slot::MAX, incremental_snapshot_archive_interval_slots: Slot::MAX,
bank_snapshots_dir: ledger_path.join("snapshot"), bank_snapshots_dir: ledger_path.join("snapshot"),
snapshot_archives_dir: ledger_path.to_path_buf(), full_snapshot_archives_dir: ledger_path.to_path_buf(),
incremental_snapshot_archives_dir: ledger_path.to_path_buf(),
..SnapshotConfig::default() ..SnapshotConfig::default()
}), }),
enforce_ulimit_nofile: false, enforce_ulimit_nofile: false,

125
validator/src/bootstrap.rs
View File

@ -56,7 +56,8 @@ pub fn rpc_bootstrap(
node: &Node, node: &Node,
identity_keypair: &Arc<Keypair>, identity_keypair: &Arc<Keypair>,
ledger_path: &Path, ledger_path: &Path,
snapshot_archives_dir: &Path, full_snapshot_archives_dir: &Path,
incremental_snapshot_archives_dir: &Path,
vote_account: &Pubkey, vote_account: &Pubkey,
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>, authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
cluster_entrypoints: &[ContactInfo], cluster_entrypoints: &[ContactInfo],
@ -96,7 +97,8 @@ pub fn rpc_bootstrap(
node, node,
identity_keypair, identity_keypair,
ledger_path, ledger_path,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
vote_account, vote_account,
authorized_voter_keypairs, authorized_voter_keypairs,
cluster_entrypoints, cluster_entrypoints,
@ -116,7 +118,7 @@ pub fn rpc_bootstrap(
node, node,
identity_keypair, identity_keypair,
ledger_path, ledger_path,
snapshot_archives_dir, full_snapshot_archives_dir,
vote_account, vote_account,
authorized_voter_keypairs, authorized_voter_keypairs,
cluster_entrypoints, cluster_entrypoints,
@ -384,32 +386,6 @@ fn check_vote_account(
Ok(()) Ok(())
} }
/// Get the Slot and Hash of the local snapshot with the highest slot. Can be either a full
/// snapshot or an incremental snapshot.
fn get_highest_local_snapshot_hash(
snapshot_archives_dir: impl AsRef<Path>,
) -> Option<(Slot, Hash)> {
if let Some(full_snapshot_info) =
snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir)
{
if let Some(incremental_snapshot_info) =
snapshot_utils::get_highest_incremental_snapshot_archive_info(
&snapshot_archives_dir,
full_snapshot_info.slot(),
)
{
Some((
incremental_snapshot_info.slot(),
*incremental_snapshot_info.hash(),
))
} else {
Some((full_snapshot_info.slot(), *full_snapshot_info.hash()))
}
} else {
None
}
}
mod without_incremental_snapshots { mod without_incremental_snapshots {
use super::*; use super::*;
@ -418,7 +394,7 @@ mod without_incremental_snapshots {
node: &Node, node: &Node,
identity_keypair: &Arc<Keypair>, identity_keypair: &Arc<Keypair>,
ledger_path: &Path, ledger_path: &Path,
snapshot_archives_dir: &Path, full_snapshot_archives_dir: &Path,
vote_account: &Pubkey, vote_account: &Pubkey,
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>, authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
cluster_entrypoints: &[ContactInfo], cluster_entrypoints: &[ContactInfo],
@ -458,7 +434,7 @@ mod without_incremental_snapshots {
validator_config, validator_config,
&mut blacklisted_rpc_nodes, &mut blacklisted_rpc_nodes,
&bootstrap_config, &bootstrap_config,
snapshot_archives_dir, full_snapshot_archives_dir,
); );
if rpc_node_details.is_none() { if rpc_node_details.is_none() {
return; return;
@ -512,7 +488,7 @@ mod without_incremental_snapshots {
} }
if let Some(snapshot_hash) = snapshot_hash { if let Some(snapshot_hash) = snapshot_hash {
let use_local_snapshot = match get_highest_local_snapshot_hash(snapshot_archives_dir) { let use_local_snapshot = match get_highest_local_snapshot_hash(full_snapshot_archives_dir) {
None => { None => {
info!("Downloading snapshot for slot {} since there is not a local snapshot", snapshot_hash.0); info!("Downloading snapshot for slot {} since there is not a local snapshot", snapshot_hash.0);
false false
@ -555,16 +531,17 @@ mod without_incremental_snapshots {
gossip.take().unwrap(); gossip.take().unwrap();
cluster_info.save_contact_info(); cluster_info.save_contact_info();
gossip_exit_flag.store(true, Ordering::Relaxed); gossip_exit_flag.store(true, Ordering::Relaxed);
let (maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain) = if let Some(snapshot_config) = let (maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain, incremental_snapshot_archives_dir) = if let Some(snapshot_config) =
validator_config.snapshot_config.as_ref() validator_config.snapshot_config.as_ref()
{ {
(snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain) (snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain, snapshot_config.incremental_snapshot_archives_dir.as_path())
} else { } else {
(DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN) (DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, full_snapshot_archives_dir)
}; };
let ret = download_snapshot_archive( let ret = download_snapshot_archive(
&rpc_contact_info.rpc, &rpc_contact_info.rpc,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
snapshot_hash, snapshot_hash,
SnapshotType::FullSnapshot, SnapshotType::FullSnapshot,
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
@ -668,7 +645,7 @@ mod without_incremental_snapshots {
validator_config: &ValidatorConfig, validator_config: &ValidatorConfig,
blacklisted_rpc_nodes: &mut HashSet<Pubkey>, blacklisted_rpc_nodes: &mut HashSet<Pubkey>,
bootstrap_config: &RpcBootstrapConfig, bootstrap_config: &RpcBootstrapConfig,
snapshot_archives_dir: &Path, full_snapshot_archives_dir: &Path,
) -> Option<(ContactInfo, Option<(Slot, Hash)>)> { ) -> Option<(ContactInfo, Option<(Slot, Hash)>)> {
let mut blacklist_timeout = Instant::now(); let mut blacklist_timeout = Instant::now();
let mut newer_cluster_snapshot_timeout = None; let mut newer_cluster_snapshot_timeout = None;
@ -692,7 +669,8 @@ mod without_incremental_snapshots {
let rpc_peers = rpc_peers.unwrap(); let rpc_peers = rpc_peers.unwrap();
blacklist_timeout = Instant::now(); blacklist_timeout = Instant::now();
let mut highest_snapshot_hash = get_highest_local_snapshot_hash(snapshot_archives_dir); let mut highest_snapshot_hash =
get_highest_local_snapshot_hash(full_snapshot_archives_dir);
let eligible_rpc_peers = if bootstrap_config.no_snapshot_fetch { let eligible_rpc_peers = if bootstrap_config.no_snapshot_fetch {
rpc_peers rpc_peers
} else { } else {
@ -795,6 +773,14 @@ mod without_incremental_snapshots {
None None
} }
} }
/// Get the Slot and Hash of the local snapshot with the highest slot.
fn get_highest_local_snapshot_hash(
full_snapshot_archives_dir: impl AsRef<Path>,
) -> Option<(Slot, Hash)> {
snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir)
.map(|full_snapshot_info| (full_snapshot_info.slot(), *full_snapshot_info.hash()))
}
} }
mod with_incremental_snapshots { mod with_incremental_snapshots {
@ -829,7 +815,8 @@ mod with_incremental_snapshots {
node: &Node, node: &Node,
identity_keypair: &Arc<Keypair>, identity_keypair: &Arc<Keypair>,
ledger_path: &Path, ledger_path: &Path,
snapshot_archives_dir: &Path, full_snapshot_archives_dir: &Path,
incremental_snapshot_archives_dir: &Path,
vote_account: &Pubkey, vote_account: &Pubkey,
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>, authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
cluster_entrypoints: &[ContactInfo], cluster_entrypoints: &[ContactInfo],
@ -935,7 +922,8 @@ mod with_incremental_snapshots {
info!("RPC node root slot: {}", rpc_client_slot); info!("RPC node root slot: {}", rpc_client_slot);
download_snapshots( download_snapshots(
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
validator_config, validator_config,
&bootstrap_config, &bootstrap_config,
use_progress_bar, use_progress_bar,
@ -1088,6 +1076,33 @@ mod with_incremental_snapshots {
} }
} }
/// Get the Slot and Hash of the local snapshot with the highest slot. Can be either a full
/// snapshot or an incremental snapshot.
fn get_highest_local_snapshot_hash(
full_snapshot_archives_dir: impl AsRef<Path>,
incremental_snapshot_archives_dir: impl AsRef<Path>,
) -> Option<(Slot, Hash)> {
if let Some(full_snapshot_info) =
snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir)
{
if let Some(incremental_snapshot_info) =
snapshot_utils::get_highest_incremental_snapshot_archive_info(
incremental_snapshot_archives_dir,
full_snapshot_info.slot(),
)
{
Some((
incremental_snapshot_info.slot(),
*incremental_snapshot_info.hash(),
))
} else {
Some((full_snapshot_info.slot(), *full_snapshot_info.hash()))
}
} else {
None
}
}
/// Get peer snapshot hashes /// Get peer snapshot hashes
/// ///
/// The result is a vector of peers with snapshot hashes that: /// The result is a vector of peers with snapshot hashes that:
@ -1420,7 +1435,8 @@ mod with_incremental_snapshots {
/// Check to see if we can use our local snapshots, otherwise download newer ones. /// Check to see if we can use our local snapshots, otherwise download newer ones.
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
fn download_snapshots( fn download_snapshots(
snapshot_archives_dir: &Path, full_snapshot_archives_dir: &Path,
incremental_snapshot_archives_dir: &Path,
validator_config: &ValidatorConfig, validator_config: &ValidatorConfig,
bootstrap_config: &RpcBootstrapConfig, bootstrap_config: &RpcBootstrapConfig,
use_progress_bar: bool, use_progress_bar: bool,
@ -1442,7 +1458,8 @@ mod with_incremental_snapshots {
// If the local snapshots are new enough, then use 'em; no need to download new snapshots // If the local snapshots are new enough, then use 'em; no need to download new snapshots
if should_use_local_snapshot( if should_use_local_snapshot(
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
maximum_local_snapshot_age, maximum_local_snapshot_age,
full_snapshot_hash, full_snapshot_hash,
incremental_snapshot_hash, incremental_snapshot_hash,
@ -1451,7 +1468,7 @@ mod with_incremental_snapshots {
} }
// Check and see if we've already got the full snapshot; if not, download it // Check and see if we've already got the full snapshot; if not, download it
if snapshot_utils::get_full_snapshot_archives(snapshot_archives_dir) if snapshot_utils::get_full_snapshot_archives(full_snapshot_archives_dir)
.into_iter() .into_iter()
.any(|snapshot_archive| { .any(|snapshot_archive| {
snapshot_archive.slot() == full_snapshot_hash.0 snapshot_archive.slot() == full_snapshot_hash.0
@ -1464,7 +1481,8 @@ mod with_incremental_snapshots {
); );
} else { } else {
download_snapshot( download_snapshot(
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
validator_config, validator_config,
bootstrap_config, bootstrap_config,
use_progress_bar, use_progress_bar,
@ -1480,7 +1498,7 @@ mod with_incremental_snapshots {
// Check and see if we've already got the incremental snapshot; if not, download it // Check and see if we've already got the incremental snapshot; if not, download it
if let Some(incremental_snapshot_hash) = incremental_snapshot_hash { if let Some(incremental_snapshot_hash) = incremental_snapshot_hash {
if snapshot_utils::get_incremental_snapshot_archives(snapshot_archives_dir) if snapshot_utils::get_incremental_snapshot_archives(incremental_snapshot_archives_dir)
.into_iter() .into_iter()
.any(|snapshot_archive| { .any(|snapshot_archive| {
snapshot_archive.slot() == incremental_snapshot_hash.0 snapshot_archive.slot() == incremental_snapshot_hash.0
@ -1494,7 +1512,8 @@ mod with_incremental_snapshots {
); );
} else { } else {
download_snapshot( download_snapshot(
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
validator_config, validator_config,
bootstrap_config, bootstrap_config,
use_progress_bar, use_progress_bar,
@ -1515,7 +1534,8 @@ mod with_incremental_snapshots {
/// Download a snapshot /// Download a snapshot
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
fn download_snapshot( fn download_snapshot(
snapshot_archives_dir: &Path, full_snapshot_archives_dir: &Path,
incremental_snapshot_archives_dir: &Path,
validator_config: &ValidatorConfig, validator_config: &ValidatorConfig,
bootstrap_config: &RpcBootstrapConfig, bootstrap_config: &RpcBootstrapConfig,
use_progress_bar: bool, use_progress_bar: bool,
@ -1547,7 +1567,8 @@ mod with_incremental_snapshots {
}; };
download_snapshot_archive( download_snapshot_archive(
&rpc_contact_info.rpc, &rpc_contact_info.rpc,
snapshot_archives_dir, full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
desired_snapshot_hash, desired_snapshot_hash,
snapshot_type, snapshot_type,
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
@ -1591,7 +1612,8 @@ mod with_incremental_snapshots {
/// Check to see if bootstrap should load from its local snapshots or not. If not, then snapshots /// Check to see if bootstrap should load from its local snapshots or not. If not, then snapshots
/// will be downloaded. /// will be downloaded.
fn should_use_local_snapshot( fn should_use_local_snapshot(
snapshot_archives_dir: &Path, full_snapshot_archives_dir: &Path,
incremental_snapshot_archives_dir: &Path,
maximum_local_snapshot_age: Slot, maximum_local_snapshot_age: Slot,
full_snapshot_hash: (Slot, Hash), full_snapshot_hash: (Slot, Hash),
incremental_snapshot_hash: Option<(Slot, Hash)>, incremental_snapshot_hash: Option<(Slot, Hash)>,
@ -1600,7 +1622,10 @@ mod with_incremental_snapshots {
.map(|(slot, _)| slot) .map(|(slot, _)| slot)
.unwrap_or(full_snapshot_hash.0); .unwrap_or(full_snapshot_hash.0);
match get_highest_local_snapshot_hash(snapshot_archives_dir) { match get_highest_local_snapshot_hash(
full_snapshot_archives_dir,
incremental_snapshot_archives_dir,
) {
None => { None => {
info!( info!(
"Downloading a snapshot for slot {} since there is not a local snapshot.", "Downloading a snapshot for slot {} since there is not a local snapshot.",

43
validator/src/main.rs
View File

@ -727,6 +727,14 @@ pub fn main() {
.takes_value(true) .takes_value(true)
.help("Use DIR as snapshot location [default: --ledger value]"), .help("Use DIR as snapshot location [default: --ledger value]"),
) )
.arg(
Arg::with_name("incremental_snapshot_archive_path")
.long("incremental-snapshot-archive-path")
.conflicts_with("no-incremental-snapshots")
.value_name("DIR")
.takes_value(true)
.help("Use DIR as separate location for incremental snapshot archives [default: --snapshots value]"),
)
.arg( .arg(
Arg::with_name("tower") Arg::with_name("tower")
.long("tower") .long("tower")
@ -2602,16 +2610,36 @@ pub fn main() {
let maximum_snapshot_download_abort = let maximum_snapshot_download_abort =
value_t_or_exit!(matches, "maximum_snapshot_download_abort", u64); value_t_or_exit!(matches, "maximum_snapshot_download_abort", u64);
let snapshot_archives_dir = if matches.is_present("snapshots") { let full_snapshot_archives_dir = if matches.is_present("snapshots") {
PathBuf::from(matches.value_of("snapshots").unwrap()) PathBuf::from(matches.value_of("snapshots").unwrap())
} else { } else {
ledger_path.clone() ledger_path.clone()
}; };
let bank_snapshots_dir = snapshot_archives_dir.join("snapshot"); let incremental_snapshot_archives_dir =
if matches.is_present("incremental_snapshot_archive_path") {
let incremental_snapshot_archives_dir = PathBuf::from(
matches
.value_of("incremental_snapshot_archive_path")
.unwrap(),
);
fs::create_dir_all(&incremental_snapshot_archives_dir).unwrap_or_else(|err| {
eprintln!(
"Failed to create incremental snapshot archives directory {:?}: {}",
incremental_snapshot_archives_dir.display(),
err
);
exit(1);
});
incremental_snapshot_archives_dir
} else {
full_snapshot_archives_dir.clone()
};
let bank_snapshots_dir = incremental_snapshot_archives_dir.join("snapshot");
fs::create_dir_all(&bank_snapshots_dir).unwrap_or_else(|err| { fs::create_dir_all(&bank_snapshots_dir).unwrap_or_else(|err| {
eprintln!( eprintln!(
"Failed to create snapshots directory {:?}: {}", "Failed to create snapshots directory {:?}: {}",
bank_snapshots_dir, err bank_snapshots_dir.display(),
err
); );
exit(1); exit(1);
}); });
@ -2657,7 +2685,8 @@ pub fn main() {
full_snapshot_archive_interval_slots, full_snapshot_archive_interval_slots,
incremental_snapshot_archive_interval_slots, incremental_snapshot_archive_interval_slots,
bank_snapshots_dir, bank_snapshots_dir,
snapshot_archives_dir: snapshot_archives_dir.clone(), full_snapshot_archives_dir: full_snapshot_archives_dir.clone(),
incremental_snapshot_archives_dir: incremental_snapshot_archives_dir.clone(),
archive_format, archive_format,
snapshot_version, snapshot_version,
maximum_full_snapshot_archives_to_retain, maximum_full_snapshot_archives_to_retain,
@ -2887,7 +2916,8 @@ pub fn main() {
Some(version) Some(version)
}); });
solana_entry::entry::init_poh(); solana_entry::entry::init_poh();
snapshot_utils::remove_tmp_snapshot_archives(&snapshot_archives_dir); snapshot_utils::remove_tmp_snapshot_archives(&full_snapshot_archives_dir);
snapshot_utils::remove_tmp_snapshot_archives(&incremental_snapshot_archives_dir);
let identity_keypair = Arc::new(identity_keypair); let identity_keypair = Arc::new(identity_keypair);
@ -2897,7 +2927,8 @@ pub fn main() {
&node, &node,
&identity_keypair, &identity_keypair,
&ledger_path, &ledger_path,
&snapshot_archives_dir, &full_snapshot_archives_dir,
&incremental_snapshot_archives_dir,
&vote_account, &vote_account,
authorized_voter_keypairs.clone(), authorized_voter_keypairs.clone(),
&cluster_entrypoints, &cluster_entrypoints,