diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index 6a2f769e31..1c31bd0142 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -384,7 +384,8 @@ mod tests { snapshot_storages: vec![], archive_format: ArchiveFormat::TarBzip2, snapshot_version: SnapshotVersion::default(), - snapshot_archives_dir: PathBuf::default(), + full_snapshot_archives_dir: PathBuf::default(), + incremental_snapshot_archives_dir: PathBuf::default(), expected_capitalization: 0, accounts_hash_for_testing: None, cluster_type: ClusterType::MainnetBeta, diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index 475e0c5fa8..cce9632ed7 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -84,6 +84,8 @@ impl SnapshotPackagerService { // last_full_snapshot_slot that requires this archive call to succeed. snapshot_utils::archive_snapshot_package( &snapshot_package, + &snapshot_config.full_snapshot_archives_dir, + &snapshot_config.incremental_snapshot_archives_dir, snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain, ) @@ -259,8 +261,10 @@ mod tests { fn create_and_verify_snapshot(temp_dir: &Path) { let accounts_dir = temp_dir.join("accounts"); let snapshots_dir = temp_dir.join("snapshots"); - let snapshot_archives_dir = temp_dir.join("snapshots_output"); - fs::create_dir_all(&snapshot_archives_dir).unwrap(); + let full_snapshot_archives_dir = temp_dir.join("full_snapshot_archives"); + let incremental_snapshot_archives_dir = temp_dir.join("incremental_snapshot_archives"); + fs::create_dir_all(&full_snapshot_archives_dir).unwrap(); + fs::create_dir_all(&incremental_snapshot_archives_dir).unwrap(); fs::create_dir_all(&accounts_dir).unwrap(); // Create some storage entries @@ -302,7 +306,7 @@ mod tests { let hash = Hash::default(); let archive_format = ArchiveFormat::TarBzip2; let output_tar_path = snapshot_utils::build_full_snapshot_archive_path( - snapshot_archives_dir, + &full_snapshot_archives_dir, slot, &hash, archive_format, @@ -325,6 +329,8 @@ mod tests { // Make tarball from packageable snapshot snapshot_utils::archive_snapshot_package( &snapshot_package, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, ) diff --git a/core/src/validator.rs b/core/src/validator.rs index 88c4380fd9..bd90dfab23 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -1676,7 +1676,8 @@ fn maybe_warp_slot( ledger_path, &bank_forks.root_bank(), None, - &snapshot_config.snapshot_archives_dir, + &snapshot_config.full_snapshot_archives_dir, + &snapshot_config.incremental_snapshot_archives_dir, snapshot_config.archive_format, snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain, diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 42c1414d55..b0499d0721 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -107,7 +107,8 @@ mod tests { struct SnapshotTestConfig { accounts_dir: TempDir, bank_snapshots_dir: TempDir, - snapshot_archives_dir: TempDir, + full_snapshot_archives_dir: TempDir, + incremental_snapshot_archives_dir: TempDir, snapshot_config: SnapshotConfig, bank_forks: BankForks, genesis_config_info: GenesisConfigInfo, @@ -123,7 +124,8 @@ mod tests { ) -> SnapshotTestConfig { let accounts_dir = TempDir::new().unwrap(); let bank_snapshots_dir = TempDir::new().unwrap(); - let snapshot_archives_dir = TempDir::new().unwrap(); + let full_snapshot_archives_dir = TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = TempDir::new().unwrap(); // validator_stake_lamports should be non-zero otherwise stake // account will not be stored in accounts-db but still cached in // bank stakes which results in mismatch when banks are loaded from @@ -151,7 +153,10 @@ mod tests { let snapshot_config = SnapshotConfig { full_snapshot_archive_interval_slots, incremental_snapshot_archive_interval_slots, - snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(), + full_snapshot_archives_dir: full_snapshot_archives_dir.path().to_path_buf(), + incremental_snapshot_archives_dir: incremental_snapshot_archives_dir + .path() + .to_path_buf(), bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(), snapshot_version, ..SnapshotConfig::default() @@ -160,7 +165,8 @@ mod tests { SnapshotTestConfig { accounts_dir, bank_snapshots_dir, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, snapshot_config, bank_forks, genesis_config_info, @@ -174,17 +180,17 @@ mod tests { old_genesis_config: &GenesisConfig, account_paths: &[PathBuf], ) { - let snapshot_archives_dir = old_bank_forks + let full_snapshot_archives_dir = old_bank_forks .snapshot_config .as_ref() - .map(|c| &c.snapshot_archives_dir) + .map(|c| &c.full_snapshot_archives_dir) .unwrap(); let old_last_bank = old_bank_forks.get(old_last_slot).unwrap(); let check_hash_calculation = false; let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path( - snapshot_archives_dir, + full_snapshot_archives_dir, old_last_bank.slot(), &old_last_bank.get_accounts_hash(), ArchiveFormat::TarBzip2, @@ -280,7 +286,8 @@ mod tests { &last_bank_snapshot_info, bank_snapshots_dir, last_bank.src.slot_deltas(&last_bank.src.roots()), - &snapshot_config.snapshot_archives_dir, + &snapshot_config.full_snapshot_archives_dir, + &snapshot_config.incremental_snapshot_archives_dir, last_bank.get_snapshot_storages(None), ArchiveFormat::TarBzip2, snapshot_version, @@ -297,6 +304,8 @@ mod tests { SnapshotPackage::new(accounts_package, last_bank.get_accounts_hash()); snapshot_utils::archive_snapshot_package( &snapshot_package, + &snapshot_config.full_snapshot_archives_dir, + &snapshot_config.incremental_snapshot_archives_dir, snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain, ) @@ -360,7 +369,8 @@ mod tests { let bank_forks = &mut snapshot_test_config.bank_forks; let snapshot_config = &snapshot_test_config.snapshot_config; let bank_snapshots_dir = &snapshot_config.bank_snapshots_dir; - let snapshot_archives_dir = &snapshot_config.snapshot_archives_dir; + let full_snapshot_archives_dir = &snapshot_config.full_snapshot_archives_dir; + let incremental_snapshot_archives_dir = &snapshot_config.incremental_snapshot_archives_dir; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config; @@ -414,7 +424,8 @@ mod tests { vec![], pending_accounts_package, bank_snapshots_dir, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, snapshot_config.snapshot_version, snapshot_config.archive_format, None, @@ -464,7 +475,7 @@ mod tests { fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap(); saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path( - snapshot_archives_dir, + full_snapshot_archives_dir, slot, // this needs to match the hash value that we reserialize with later. It is complicated, so just use default. // This hash value is just used to build the file name. Since this is mocked up test code, it is sufficient to pass default here. @@ -679,7 +690,8 @@ mod tests { FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, ); - trace!("SnapshotTestConfig:\naccounts_dir: {}\nbank_snapshots_dir: {}\nsnapshot_archives_dir: {}", snapshot_test_config.accounts_dir.path().display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.snapshot_archives_dir.path().display()); + trace!("SnapshotTestConfig:\naccounts_dir: {}\nbank_snapshots_dir: {}\nfull_snapshot_archives_dir: {}\nincremental_snapshot_archives_dir: {}", + snapshot_test_config.accounts_dir.path().display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.full_snapshot_archives_dir.path().display(), snapshot_test_config.incremental_snapshot_archives_dir.path().display()); let bank_forks = &mut snapshot_test_config.bank_forks; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; @@ -783,7 +795,8 @@ mod tests { bank, &bank_snapshot_info, &snapshot_config.bank_snapshots_dir, - &snapshot_config.snapshot_archives_dir, + &snapshot_config.full_snapshot_archives_dir, + &snapshot_config.incremental_snapshot_archives_dir, bank.get_snapshot_storages(None), snapshot_config.archive_format, snapshot_config.snapshot_version, @@ -820,7 +833,8 @@ mod tests { incremental_snapshot_base_slot, &bank_snapshot_info, &snapshot_config.bank_snapshots_dir, - &snapshot_config.snapshot_archives_dir, + &snapshot_config.full_snapshot_archives_dir, + &snapshot_config.incremental_snapshot_archives_dir, storages, snapshot_config.archive_format, snapshot_config.snapshot_version, @@ -839,7 +853,8 @@ mod tests { ) -> snapshot_utils::Result<()> { let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives( &snapshot_config.bank_snapshots_dir, - &snapshot_config.snapshot_archives_dir, + &snapshot_config.full_snapshot_archives_dir, + &snapshot_config.incremental_snapshot_archives_dir, &[accounts_dir], genesis_config, None, @@ -1015,7 +1030,12 @@ mod tests { let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives( &snapshot_test_config.snapshot_config.bank_snapshots_dir, - &snapshot_test_config.snapshot_config.snapshot_archives_dir, + &snapshot_test_config + .snapshot_config + .full_snapshot_archives_dir, + &snapshot_test_config + .snapshot_config + .incremental_snapshot_archives_dir, &[snapshot_test_config.accounts_dir.as_ref().to_path_buf()], &snapshot_test_config.genesis_config_info.genesis_config, None, diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs index f191b5e7d6..f05f7cf583 100644 --- a/download-utils/src/lib.rs +++ b/download-utils/src/lib.rs @@ -254,7 +254,8 @@ pub fn download_genesis_if_missing( /// a full snapshot or an incremental snapshot. pub fn download_snapshot_archive<'a, 'b>( rpc_addr: &SocketAddr, - snapshot_archives_dir: &Path, + full_snapshot_archives_dir: &Path, + incremental_snapshot_archives_dir: &Path, desired_snapshot_hash: (Slot, Hash), snapshot_type: SnapshotType, maximum_full_snapshot_archives_to_retain: usize, @@ -263,13 +264,17 @@ pub fn download_snapshot_archive<'a, 'b>( progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>, ) -> Result<(), String> { snapshot_utils::purge_old_snapshot_archives( - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain, ); let snapshot_archives_remote_dir = - snapshot_utils::build_snapshot_archives_remote_dir(snapshot_archives_dir); + snapshot_utils::build_snapshot_archives_remote_dir(match snapshot_type { + SnapshotType::FullSnapshot => full_snapshot_archives_dir, + SnapshotType::IncrementalSnapshot(_) => incremental_snapshot_archives_dir, + }); fs::create_dir_all(&snapshot_archives_remote_dir).unwrap(); for archive_format in [ diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 56c18be79e..fda15fb631 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -731,6 +731,7 @@ fn load_bank_forks( blockstore: &Blockstore, process_options: ProcessOptions, snapshot_archive_path: Option, + incremental_snapshot_archive_path: Option, ) -> Result<(Arc>, Option), BlockstoreProcessorError> { let bank_snapshots_dir = blockstore .ledger_path() @@ -742,12 +743,15 @@ fn load_bank_forks( let snapshot_config = if arg_matches.is_present("no_snapshot") { None } else { - let snapshot_archives_dir = + let full_snapshot_archives_dir = snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf()); + let incremental_snapshot_archives_dir = + incremental_snapshot_archive_path.unwrap_or_else(|| full_snapshot_archives_dir.clone()); Some(SnapshotConfig { full_snapshot_archive_interval_slots: Slot::MAX, incremental_snapshot_archive_interval_slots: Slot::MAX, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, bank_snapshots_dir, ..SnapshotConfig::default() }) @@ -1102,7 +1106,15 @@ fn main() { .value_name("DIR") .takes_value(true) .global(true) - .help("Use DIR for ledger location"), + .help("Use DIR for snapshot location"), + ) + .arg( + Arg::with_name("incremental_snapshot_archive_path") + .long("incremental-snapshot-archive-path") + .value_name("DIR") + .takes_value(true) + .global(true) + .help("Use DIR for separate incremental snapshot location"), ) .arg( Arg::with_name("output_format") @@ -1713,6 +1725,10 @@ fn main() { let snapshot_archive_path = value_t!(matches, "snapshot_archive_path", String) .ok() .map(PathBuf::from); + let incremental_snapshot_archive_path = + value_t!(matches, "incremental_snapshot_archive_path", String) + .ok() + .map(PathBuf::from); let wal_recovery_mode = matches .value_of("wal_recovery_mode") @@ -1828,6 +1844,7 @@ fn main() { &blockstore, process_options, snapshot_archive_path, + incremental_snapshot_archive_path, ) { Ok((bank_forks, ..)) => { println!( @@ -1909,6 +1926,7 @@ fn main() { &blockstore, process_options, snapshot_archive_path, + incremental_snapshot_archive_path, ) { Ok((bank_forks, ..)) => { println!("{}", &bank_forks.read().unwrap().working_bank().hash()); @@ -2148,6 +2166,7 @@ fn main() { &blockstore, process_options, snapshot_archive_path, + incremental_snapshot_archive_path, ) .unwrap_or_else(|err| { eprintln!("Ledger verification failed: {:?}", err); @@ -2179,6 +2198,7 @@ fn main() { &blockstore, process_options, snapshot_archive_path, + incremental_snapshot_archive_path, ) { Ok((bank_forks, ..)) => { let dot = graph_forks( @@ -2208,10 +2228,20 @@ fn main() { } } ("create-snapshot", Some(arg_matches)) => { + let is_incremental = arg_matches.is_present("incremental"); let output_directory = value_t!(arg_matches, "output_directory", PathBuf) - .unwrap_or_else(|_| match &snapshot_archive_path { - Some(snapshot_archive_path) => snapshot_archive_path.clone(), - None => ledger_path.clone(), + .unwrap_or_else(|_| { + match ( + is_incremental, + &snapshot_archive_path, + &incremental_snapshot_archive_path, + ) { + (true, _, Some(incremental_snapshot_archive_path)) => { + incremental_snapshot_archive_path.clone() + } + (_, Some(snapshot_archive_path), _) => snapshot_archive_path.clone(), + (_, _, _) => ledger_path.clone(), + } }); let mut warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok(); let remove_stake_accounts = arg_matches.is_present("remove_stake_accounts"); @@ -2266,7 +2296,6 @@ fn main() { let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); let blockstore = open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode); - let is_incremental = arg_matches.is_present("incremental"); let snapshot_slot = if Some("ROOT") == arg_matches.value_of("snapshot_slot") { blockstore @@ -2296,6 +2325,7 @@ fn main() { ..ProcessOptions::default() }, snapshot_archive_path, + incremental_snapshot_archive_path, ) { Ok((bank_forks, starting_snapshot_hashes)) => { let mut bank = bank_forks @@ -2530,6 +2560,7 @@ fn main() { &bank, full_snapshot_slot, Some(snapshot_version), + output_directory.clone(), output_directory, ArchiveFormat::TarZstd, maximum_full_snapshot_archives_to_retain, @@ -2553,6 +2584,7 @@ fn main() { ledger_path, &bank, Some(snapshot_version), + output_directory.clone(), output_directory, ArchiveFormat::TarZstd, maximum_full_snapshot_archives_to_retain, @@ -2603,6 +2635,7 @@ fn main() { &blockstore, process_options, snapshot_archive_path, + incremental_snapshot_archive_path, ) .unwrap_or_else(|err| { eprintln!("Failed to load ledger: {:?}", err); @@ -2662,6 +2695,7 @@ fn main() { &blockstore, process_options, snapshot_archive_path, + incremental_snapshot_archive_path, ) { Ok((bank_forks, ..)) => { let bank_forks = bank_forks.read().unwrap(); diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index c7c29f8da1..27a8b0611a 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -98,7 +98,7 @@ pub fn load_bank_forks( .expect("Couldn't create snapshot directory"); if snapshot_utils::get_highest_full_snapshot_archive_info( - &snapshot_config.snapshot_archives_dir, + &snapshot_config.full_snapshot_archives_dir, ) .is_some() { @@ -188,7 +188,8 @@ fn bank_forks_from_snapshot( let (mut deserialized_bank, full_snapshot_archive_info, incremental_snapshot_archive_info) = snapshot_utils::bank_from_latest_snapshot_archives( &snapshot_config.bank_snapshots_dir, - &snapshot_config.snapshot_archives_dir, + &snapshot_config.full_snapshot_archives_dir, + &snapshot_config.incremental_snapshot_archives_dir, &account_paths, genesis_config, process_options.debug_keys.clone(), diff --git a/local-cluster/src/local_cluster_snapshot_utils.rs b/local-cluster/src/local_cluster_snapshot_utils.rs index 1cdf9ef938..852aa41286 100644 --- a/local-cluster/src/local_cluster_snapshot_utils.rs +++ b/local-cluster/src/local_cluster_snapshot_utils.rs @@ -13,11 +13,18 @@ use { impl LocalCluster { /// Return the next full snapshot archive info after the cluster's last processed slot - pub fn wait_for_next_full_snapshot( + pub fn wait_for_next_full_snapshot( &self, - snapshot_archives_dir: impl AsRef, - ) -> FullSnapshotArchiveInfo { - match self.wait_for_next_snapshot(snapshot_archives_dir, NextSnapshotType::FullSnapshot) { + full_snapshot_archives_dir: T, + ) -> FullSnapshotArchiveInfo + where + T: AsRef, + { + match self.wait_for_next_snapshot( + full_snapshot_archives_dir, + None::, + NextSnapshotType::FullSnapshot, + ) { NextSnapshotResult::FullSnapshot(full_snapshot_archive_info) => { full_snapshot_archive_info } @@ -29,10 +36,12 @@ impl LocalCluster { /// after the cluster's last processed slot pub fn wait_for_next_incremental_snapshot( &self, - snapshot_archives_dir: impl AsRef, + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, ) -> (IncrementalSnapshotArchiveInfo, FullSnapshotArchiveInfo) { match self.wait_for_next_snapshot( - snapshot_archives_dir, + full_snapshot_archives_dir, + Some(incremental_snapshot_archives_dir), NextSnapshotType::IncrementalAndFullSnapshot, ) { NextSnapshotResult::IncrementalAndFullSnapshot( @@ -47,9 +56,10 @@ impl LocalCluster { } /// Return the next snapshot archive infos after the cluster's last processed slot - pub fn wait_for_next_snapshot( + fn wait_for_next_snapshot( &self, - snapshot_archives_dir: impl AsRef, + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: Option>, next_snapshot_type: NextSnapshotType, ) -> NextSnapshotResult { // Get slot after which this was generated @@ -69,7 +79,7 @@ impl LocalCluster { ); loop { if let Some(full_snapshot_archive_info) = - snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir) + snapshot_utils::get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir) { match next_snapshot_type { NextSnapshotType::FullSnapshot => { @@ -80,7 +90,7 @@ impl LocalCluster { NextSnapshotType::IncrementalAndFullSnapshot => { if let Some(incremental_snapshot_archive_info) = snapshot_utils::get_highest_incremental_snapshot_archive_info( - &snapshot_archives_dir, + incremental_snapshot_archives_dir.as_ref().unwrap(), full_snapshot_archive_info.slot(), ) { diff --git a/local-cluster/tests/common.rs b/local-cluster/tests/common.rs index b243def93c..a3758f17c3 100644 --- a/local-cluster/tests/common.rs +++ b/local-cluster/tests/common.rs @@ -441,7 +441,8 @@ pub fn generate_account_paths(num_account_paths: usize) -> (Vec, Vec, pub validator_config: ValidatorConfig, } @@ -470,11 +471,15 @@ impl SnapshotValidatorConfig { // Create the snapshot config let _ = fs::create_dir_all(farf_dir()); let bank_snapshots_dir = tempfile::tempdir_in(farf_dir()).unwrap(); - let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap(); + let full_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap(); + let incremental_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap(); let snapshot_config = SnapshotConfig { full_snapshot_archive_interval_slots, incremental_snapshot_archive_interval_slots, - snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(), + full_snapshot_archives_dir: full_snapshot_archives_dir.path().to_path_buf(), + incremental_snapshot_archives_dir: incremental_snapshot_archives_dir + .path() + .to_path_buf(), bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(), maximum_full_snapshot_archives_to_retain: usize::MAX, maximum_incremental_snapshot_archives_to_retain: usize::MAX, @@ -495,7 +500,8 @@ impl SnapshotValidatorConfig { SnapshotValidatorConfig { bank_snapshots_dir, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, account_storage_dirs, validator_config, } diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 838d253973..8be6790b63 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -467,21 +467,29 @@ fn test_snapshot_download() { let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified); - let snapshot_archives_dir = &leader_snapshot_test_config + let full_snapshot_archives_dir = &leader_snapshot_test_config .validator_config .snapshot_config .as_ref() .unwrap() - .snapshot_archives_dir; + .full_snapshot_archives_dir; + let incremental_snapshot_archives_dir = &leader_snapshot_test_config + .validator_config + .snapshot_config + .as_ref() + .unwrap() + .incremental_snapshot_archives_dir; trace!("Waiting for snapshot"); - let full_snapshot_archive_info = cluster.wait_for_next_full_snapshot(snapshot_archives_dir); + let full_snapshot_archive_info = + cluster.wait_for_next_full_snapshot(full_snapshot_archives_dir); trace!("found: {}", full_snapshot_archive_info.path().display()); // Download the snapshot, then boot a validator from it. download_snapshot_archive( &cluster.entry_point_info.rpc, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, ( full_snapshot_archive_info.slot(), *full_snapshot_archive_info.hash(), @@ -549,43 +557,60 @@ fn test_incremental_snapshot_download() { let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified); - let snapshot_archives_dir = &leader_snapshot_test_config + let full_snapshot_archives_dir = &leader_snapshot_test_config .validator_config .snapshot_config .as_ref() .unwrap() - .snapshot_archives_dir; + .full_snapshot_archives_dir; + let incremental_snapshot_archives_dir = &leader_snapshot_test_config + .validator_config + .snapshot_config + .as_ref() + .unwrap() + .incremental_snapshot_archives_dir; debug!("snapshot config:\n\tfull snapshot interval: {}\n\tincremental snapshot interval: {}\n\taccounts hash interval: {}", full_snapshot_interval, incremental_snapshot_interval, accounts_hash_interval); debug!( - "leader config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}", + "leader config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}", leader_snapshot_test_config .bank_snapshots_dir .path() .display(), leader_snapshot_test_config - .snapshot_archives_dir + .full_snapshot_archives_dir + .path() + .display(), + leader_snapshot_test_config + .incremental_snapshot_archives_dir .path() .display(), ); debug!( - "validator config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}", + "validator config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}", validator_snapshot_test_config .bank_snapshots_dir .path() .display(), validator_snapshot_test_config - .snapshot_archives_dir + .full_snapshot_archives_dir + .path() + .display(), + validator_snapshot_test_config + .incremental_snapshot_archives_dir .path() .display(), ); trace!("Waiting for snapshots"); - let (incremental_snapshot_archive_info, full_snapshot_archive_info) = - cluster.wait_for_next_incremental_snapshot(snapshot_archives_dir); + let (incremental_snapshot_archive_info, full_snapshot_archive_info) = cluster + .wait_for_next_incremental_snapshot( + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, + ); trace!( "found: {} and {}", full_snapshot_archive_info.path().display(), @@ -595,7 +620,8 @@ fn test_incremental_snapshot_download() { // Download the snapshots, then boot a validator from them. download_snapshot_archive( &cluster.entry_point_info.rpc, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, ( full_snapshot_archive_info.slot(), *full_snapshot_archive_info.hash(), @@ -620,7 +646,8 @@ fn test_incremental_snapshot_download() { download_snapshot_archive( &cluster.entry_point_info.rpc, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, ( incremental_snapshot_archive_info.slot(), *incremental_snapshot_archive_info.hash(), @@ -708,24 +735,32 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st incremental_snapshot_interval, accounts_hash_interval); debug!( - "leader config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}", + "leader config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}", leader_snapshot_test_config .bank_snapshots_dir .path() .display(), leader_snapshot_test_config - .snapshot_archives_dir + .full_snapshot_archives_dir + .path() + .display(), + leader_snapshot_test_config + .incremental_snapshot_archives_dir .path() .display(), ); debug!( - "validator config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}", + "validator config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}", validator_snapshot_test_config .bank_snapshots_dir .path() .display(), validator_snapshot_test_config - .snapshot_archives_dir + .full_snapshot_archives_dir + .path() + .display(), + validator_snapshot_test_config + .incremental_snapshot_archives_dir .path() .display(), ); @@ -734,7 +769,12 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st let (incremental_snapshot_archive, full_snapshot_archive) = LocalCluster::wait_for_next_incremental_snapshot( &cluster, - leader_snapshot_test_config.snapshot_archives_dir.path(), + leader_snapshot_test_config + .full_snapshot_archives_dir + .path(), + leader_snapshot_test_config + .incremental_snapshot_archives_dir + .path(), ); info!( "Found snapshots:\n\tfull snapshot: {}\n\tincremental snapshot: {}", @@ -751,7 +791,12 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st info!("Downloading full snapshot to validator..."); download_snapshot_archive( &cluster.entry_point_info.rpc, - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), + validator_snapshot_test_config + .incremental_snapshot_archives_dir + .path(), (full_snapshot_archive.slot(), *full_snapshot_archive.hash()), SnapshotType::FullSnapshot, validator_snapshot_test_config @@ -771,7 +816,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st ) .unwrap(); let downloaded_full_snapshot_archive = snapshot_utils::get_highest_full_snapshot_archive_info( - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), ) .unwrap(); info!( @@ -782,7 +829,12 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st info!("Downloading incremental snapshot to validator..."); download_snapshot_archive( &cluster.entry_point_info.rpc, - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), + validator_snapshot_test_config + .incremental_snapshot_archives_dir + .path(), ( incremental_snapshot_archive.slot(), *incremental_snapshot_archive.hash(), @@ -806,7 +858,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st .unwrap(); let downloaded_incremental_snapshot_archive = snapshot_utils::get_highest_incremental_snapshot_archive_info( - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .incremental_snapshot_archives_dir + .path(), full_snapshot_archive.slot(), ) .unwrap(); @@ -876,14 +930,30 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st // restart the node and guarantee that the only snapshots present are these initial ones. So, // the easiest way to do that is create a backup now, delete the ones on the node before // restart, then copy the backup ones over again. - let backup_validator_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap(); + let backup_validator_full_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap(); trace!( - "Backing up validator snapshots to dir: {}...", - backup_validator_snapshot_archives_dir.path().display() + "Backing up validator full snapshots to dir: {}...", + backup_validator_full_snapshot_archives_dir.path().display() ); copy_files_with_remote( - validator_snapshot_test_config.snapshot_archives_dir.path(), - backup_validator_snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), + backup_validator_full_snapshot_archives_dir.path(), + ); + let backup_validator_incremental_snapshot_archives_dir = + tempfile::tempdir_in(farf_dir()).unwrap(); + trace!( + "Backing up validator incremental snapshots to dir: {}...", + backup_validator_incremental_snapshot_archives_dir + .path() + .display() + ); + copy_files_with_remote( + validator_snapshot_test_config + .incremental_snapshot_archives_dir + .path(), + backup_validator_incremental_snapshot_archives_dir.path(), ); info!("Starting the validator..."); @@ -931,13 +1001,17 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st // Putting this all in its own block so its clear we're only intended to keep the leader's info let leader_full_snapshot_archive_for_comparison = { let validator_full_snapshot = snapshot_utils::get_highest_full_snapshot_archive_info( - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), ) .unwrap(); // Now get the same full snapshot on the LEADER that we just got from the validator let mut leader_full_snapshots = snapshot_utils::get_full_snapshot_archives( - leader_snapshot_test_config.snapshot_archives_dir.path(), + leader_snapshot_test_config + .full_snapshot_archives_dir + .path(), ); leader_full_snapshots.retain(|full_snapshot| { full_snapshot.slot() == validator_full_snapshot.slot() @@ -959,10 +1033,27 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st info!("leader full snapshot archive for comparison: {leader_full_snapshot_archive_for_comparison:#?}"); info!("Delete all the snapshots on the validator and restore the originals from the backup..."); - delete_files_with_remote(validator_snapshot_test_config.snapshot_archives_dir.path()); + delete_files_with_remote( + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), + ); + delete_files_with_remote( + validator_snapshot_test_config + .incremental_snapshot_archives_dir + .path(), + ); copy_files_with_remote( - backup_validator_snapshot_archives_dir.path(), - validator_snapshot_test_config.snapshot_archives_dir.path(), + backup_validator_full_snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), + ); + copy_files_with_remote( + backup_validator_incremental_snapshot_archives_dir.path(), + validator_snapshot_test_config + .incremental_snapshot_archives_dir + .path(), ); info!( "Delete all the snapshots on the validator and restore the originals from the backup... DONE" @@ -971,7 +1062,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st // Get the highest full snapshot slot *before* restarting, as a comparison let validator_full_snapshot_slot_at_startup = snapshot_utils::get_highest_full_snapshot_archive_slot( - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), ) .unwrap(); @@ -998,12 +1091,16 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st let timer = Instant::now(); loop { if let Some(full_snapshot_slot) = snapshot_utils::get_highest_full_snapshot_archive_slot( - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), ) { if full_snapshot_slot >= validator_next_full_snapshot_slot { if let Some(incremental_snapshot_slot) = snapshot_utils::get_highest_incremental_snapshot_archive_slot( - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .incremental_snapshot_archives_dir + .path(), full_snapshot_slot, ) { @@ -1034,7 +1131,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st // Check to make sure that the full snapshot the validator created during startup is the same // or one greater than the snapshot the leader created. let validator_full_snapshot_archives = snapshot_utils::get_full_snapshot_archives( - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), ); info!("validator full snapshot archives: {validator_full_snapshot_archives:#?}"); let validator_full_snapshot_archive_for_comparison = validator_full_snapshot_archives @@ -1071,12 +1170,29 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st // Copy over the snapshots to the new node, but need to remove the tmp snapshot dir so it // doesn't break the simple copy_files closure. snapshot_utils::remove_tmp_snapshot_archives( - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), + ); + snapshot_utils::remove_tmp_snapshot_archives( + validator_snapshot_test_config + .incremental_snapshot_archives_dir + .path(), ); copy_files( - validator_snapshot_test_config.snapshot_archives_dir.path(), + validator_snapshot_test_config + .full_snapshot_archives_dir + .path(), final_validator_snapshot_test_config - .snapshot_archives_dir + .full_snapshot_archives_dir + .path(), + ); + copy_files( + validator_snapshot_test_config + .incremental_snapshot_archives_dir + .path(), + final_validator_snapshot_test_config + .incremental_snapshot_archives_dir .path(), ); @@ -1128,19 +1244,20 @@ fn test_snapshot_restart_tower() { let validator_info = cluster.exit_node(&validator_id); // Get slot after which this was generated - let snapshot_archives_dir = &leader_snapshot_test_config + let full_snapshot_archives_dir = &leader_snapshot_test_config .validator_config .snapshot_config .as_ref() .unwrap() - .snapshot_archives_dir; + .full_snapshot_archives_dir; - let full_snapshot_archive_info = cluster.wait_for_next_full_snapshot(snapshot_archives_dir); + let full_snapshot_archive_info = + cluster.wait_for_next_full_snapshot(full_snapshot_archives_dir); // Copy archive to validator's snapshot output directory let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path( validator_snapshot_test_config - .snapshot_archives_dir + .full_snapshot_archives_dir .into_path(), full_snapshot_archive_info.slot(), full_snapshot_archive_info.hash(), @@ -1178,12 +1295,12 @@ fn test_snapshots_blockstore_floor() { let mut validator_snapshot_test_config = setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths); - let snapshot_archives_dir = &leader_snapshot_test_config + let full_snapshot_archives_dir = &leader_snapshot_test_config .validator_config .snapshot_config .as_ref() .unwrap() - .snapshot_archives_dir; + .full_snapshot_archives_dir; let mut config = ClusterConfig { node_stakes: vec![DEFAULT_NODE_STAKE], @@ -1201,7 +1318,7 @@ fn test_snapshots_blockstore_floor() { let archive_info = loop { let archive = - snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir); + snapshot_utils::get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir); if archive.is_some() { trace!("snapshot exists"); break archive.unwrap(); @@ -1212,7 +1329,7 @@ fn test_snapshots_blockstore_floor() { // Copy archive to validator's snapshot output directory let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path( validator_snapshot_test_config - .snapshot_archives_dir + .full_snapshot_archives_dir .into_path(), archive_info.slot(), archive_info.hash(), @@ -1280,12 +1397,12 @@ fn test_snapshots_restart_validity() { let num_account_paths = 1; let mut snapshot_test_config = setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths); - let snapshot_archives_dir = &snapshot_test_config + let full_snapshot_archives_dir = &snapshot_test_config .validator_config .snapshot_config .as_ref() .unwrap() - .snapshot_archives_dir; + .full_snapshot_archives_dir; // Set up the cluster with 1 snapshotting validator let mut all_account_storage_dirs = vec![vec![]]; @@ -1323,7 +1440,7 @@ fn test_snapshots_restart_validity() { expected_balances.extend(new_balances); - cluster.wait_for_next_full_snapshot(snapshot_archives_dir); + cluster.wait_for_next_full_snapshot(full_snapshot_archives_dir); // Create new account paths since validator exit is not guaranteed to cleanup RPC threads, // which may delete the old accounts on exit at any point diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index e72c3551b6..fcd8934120 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -2546,17 +2546,22 @@ pub mod rpc_minimal { return Err(RpcCustomError::NoSnapshot.into()); } - let snapshot_archives_dir = meta + let (full_snapshot_archives_dir, incremental_snapshot_archives_dir) = meta .snapshot_config - .map(|snapshot_config| snapshot_config.snapshot_archives_dir) + .map(|snapshot_config| { + ( + snapshot_config.full_snapshot_archives_dir, + snapshot_config.incremental_snapshot_archives_dir, + ) + }) .unwrap(); let full_snapshot_slot = - snapshot_utils::get_highest_full_snapshot_archive_slot(&snapshot_archives_dir) + snapshot_utils::get_highest_full_snapshot_archive_slot(&full_snapshot_archives_dir) .ok_or(RpcCustomError::NoSnapshot)?; let incremental_snapshot_slot = snapshot_utils::get_highest_incremental_snapshot_archive_slot( - &snapshot_archives_dir, + &incremental_snapshot_archives_dir, full_snapshot_slot, ); @@ -3925,7 +3930,7 @@ pub mod rpc_deprecated_v1_9 { meta.snapshot_config .and_then(|snapshot_config| { snapshot_utils::get_highest_full_snapshot_archive_slot( - &snapshot_config.snapshot_archives_dir, + &snapshot_config.full_snapshot_archives_dir, ) }) .ok_or_else(|| RpcCustomError::NoSnapshot.into()) diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index dd1659593b..97eb71b955 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -160,7 +160,22 @@ impl RpcRequestMiddleware { where P: AsRef, { - let root = &self.snapshot_config.as_ref().unwrap().snapshot_archives_dir; + let root = if self + .full_snapshot_archive_path_regex + .is_match(Path::new("").join(&stem).to_str().unwrap()) + { + &self + .snapshot_config + .as_ref() + .unwrap() + .full_snapshot_archives_dir + } else { + &self + .snapshot_config + .as_ref() + .unwrap() + .incremental_snapshot_archives_dir + }; let local_path = root.join(&stem); if local_path.exists() { local_path @@ -236,7 +251,7 @@ impl RequestMiddleware for RpcRequestMiddleware { // Convenience redirect to the latest snapshot let full_snapshot_archive_info = snapshot_utils::get_highest_full_snapshot_archive_info( - &snapshot_config.snapshot_archives_dir, + &snapshot_config.full_snapshot_archives_dir, ); let snapshot_archive_info = if let Some(full_snapshot_archive_info) = full_snapshot_archive_info { @@ -244,7 +259,7 @@ impl RequestMiddleware for RpcRequestMiddleware { Some(full_snapshot_archive_info.snapshot_archive_info().clone()) } else { snapshot_utils::get_highest_incremental_snapshot_archive_info( - &snapshot_config.snapshot_archives_dir, + &snapshot_config.incremental_snapshot_archives_dir, full_snapshot_archive_info.slot(), ) .map(|incremental_snapshot_archive_info| { diff --git a/runtime/src/accounts_background_service.rs b/runtime/src/accounts_background_service.rs index 46bc614427..d158427e70 100644 --- a/runtime/src/accounts_background_service.rs +++ b/runtime/src/accounts_background_service.rs @@ -285,7 +285,8 @@ impl SnapshotRequestHandler { status_cache_slot_deltas, &self.pending_accounts_package, &self.snapshot_config.bank_snapshots_dir, - &self.snapshot_config.snapshot_archives_dir, + &self.snapshot_config.full_snapshot_archives_dir, + &self.snapshot_config.incremental_snapshot_archives_dir, self.snapshot_config.snapshot_version, self.snapshot_config.archive_format, hash_for_testing, diff --git a/runtime/src/snapshot_config.rs b/runtime/src/snapshot_config.rs index 40190b7443..07c1def31f 100644 --- a/runtime/src/snapshot_config.rs +++ b/runtime/src/snapshot_config.rs @@ -13,8 +13,11 @@ pub struct SnapshotConfig { /// Generate a new incremental snapshot archive every this many slots pub incremental_snapshot_archive_interval_slots: Slot, - /// Path to the directory where snapshot archives are stored - pub snapshot_archives_dir: PathBuf, + /// Path to the directory where full snapshot archives are stored + pub full_snapshot_archives_dir: PathBuf, + + /// Path to the directory where incremental snapshot archives are stored + pub incremental_snapshot_archives_dir: PathBuf, /// Path to the directory where bank snapshots are stored pub bank_snapshots_dir: PathBuf, @@ -46,7 +49,8 @@ impl Default for SnapshotConfig { snapshot_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, incremental_snapshot_archive_interval_slots: snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, - snapshot_archives_dir: PathBuf::default(), + full_snapshot_archives_dir: PathBuf::default(), + incremental_snapshot_archives_dir: PathBuf::default(), bank_snapshots_dir: PathBuf::default(), archive_format: ArchiveFormat::TarBzip2, snapshot_version: SnapshotVersion::default(), diff --git a/runtime/src/snapshot_package.rs b/runtime/src/snapshot_package.rs index 87bf2038b6..cb050bc7f8 100644 --- a/runtime/src/snapshot_package.rs +++ b/runtime/src/snapshot_package.rs @@ -39,7 +39,8 @@ pub struct AccountsPackage { pub snapshot_storages: SnapshotStorages, pub archive_format: ArchiveFormat, pub snapshot_version: SnapshotVersion, - pub snapshot_archives_dir: PathBuf, + pub full_snapshot_archives_dir: PathBuf, + pub incremental_snapshot_archives_dir: PathBuf, pub expected_capitalization: u64, pub accounts_hash_for_testing: Option, pub cluster_type: ClusterType, @@ -57,7 +58,8 @@ impl AccountsPackage { bank_snapshot_info: &BankSnapshotInfo, bank_snapshots_dir: impl AsRef, slot_deltas: Vec, - snapshot_archives_dir: impl AsRef, + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, snapshot_storages: SnapshotStorages, archive_format: ArchiveFormat, snapshot_version: SnapshotVersion, @@ -105,7 +107,10 @@ impl AccountsPackage { snapshot_storages, archive_format, snapshot_version, - snapshot_archives_dir: snapshot_archives_dir.as_ref().to_path_buf(), + full_snapshot_archives_dir: full_snapshot_archives_dir.as_ref().to_path_buf(), + incremental_snapshot_archives_dir: incremental_snapshot_archives_dir + .as_ref() + .to_path_buf(), expected_capitalization: bank.capitalization(), accounts_hash_for_testing, cluster_type: bank.cluster_type(), @@ -137,7 +142,7 @@ impl SnapshotPackage { let mut snapshot_storages = accounts_package.snapshot_storages; let snapshot_archive_path = match accounts_package.snapshot_type.unwrap() { SnapshotType::FullSnapshot => snapshot_utils::build_full_snapshot_archive_path( - accounts_package.snapshot_archives_dir, + accounts_package.full_snapshot_archives_dir, accounts_package.slot, &accounts_hash, accounts_package.archive_format, @@ -156,7 +161,7 @@ impl SnapshotPackage { "Incremental snapshot package must only contain storage entries where slot > incremental snapshot base slot (i.e. full snapshot slot)!" ); snapshot_utils::build_incremental_snapshot_archive_path( - accounts_package.snapshot_archives_dir, + accounts_package.incremental_snapshot_archives_dir, incremental_snapshot_base_slot, accounts_package.slot, &accounts_hash, diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index 359b283d03..534c47742d 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -249,6 +249,8 @@ pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: impl AsRef) { /// Make a snapshot archive out of the snapshot package pub fn archive_snapshot_package( snapshot_package: &SnapshotPackage, + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, maximum_full_snapshot_archives_to_retain: usize, maximum_incremental_snapshot_archives_to_retain: usize, ) -> Result<()> { @@ -379,7 +381,8 @@ pub fn archive_snapshot_package( .map_err(|e| SnapshotError::IoWithSource(e, "archive path rename"))?; purge_old_snapshot_archives( - tar_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain, ); @@ -892,12 +895,13 @@ pub fn bank_from_snapshot_archives( Ok((bank, timings)) } -/// Rebuild bank from snapshot archives. This function searches `snapshot_archives_dir` for the +/// Rebuild bank from snapshot archives. This function searches `full_snapshot_archives_dir` and `incremental_snapshot_archives_dir` for the /// highest full snapshot and highest corresponding incremental snapshot, then rebuilds the bank. #[allow(clippy::too_many_arguments)] pub fn bank_from_latest_snapshot_archives( bank_snapshots_dir: impl AsRef, - snapshot_archives_dir: impl AsRef, + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, account_paths: &[PathBuf], genesis_config: &GenesisConfig, debug_keys: Option>>, @@ -916,11 +920,12 @@ pub fn bank_from_latest_snapshot_archives( FullSnapshotArchiveInfo, Option, )> { - let full_snapshot_archive_info = get_highest_full_snapshot_archive_info(&snapshot_archives_dir) - .ok_or(SnapshotError::NoSnapshotArchives)?; + let full_snapshot_archive_info = + get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir) + .ok_or(SnapshotError::NoSnapshotArchives)?; let incremental_snapshot_archive_info = get_highest_incremental_snapshot_archive_info( - &snapshot_archives_dir, + &incremental_snapshot_archives_dir, full_snapshot_archive_info.slot(), ); @@ -1125,12 +1130,12 @@ pub fn build_snapshot_archives_remote_dir(snapshot_archives_dir: impl AsRef, + full_snapshot_archives_dir: impl AsRef, slot: Slot, hash: &Hash, archive_format: ArchiveFormat, ) -> PathBuf { - snapshot_archives_dir.as_ref().join(format!( + full_snapshot_archives_dir.as_ref().join(format!( "snapshot-{}-{}.{}", slot, hash, @@ -1142,13 +1147,13 @@ pub fn build_full_snapshot_archive_path( /// directory, the snapshot base slot, the snapshot slot, the accounts hash, and the archive /// format. pub fn build_incremental_snapshot_archive_path( - snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, base_slot: Slot, slot: Slot, hash: &Hash, archive_format: ArchiveFormat, ) -> PathBuf { - snapshot_archives_dir.as_ref().join(format!( + incremental_snapshot_archives_dir.as_ref().join(format!( "incremental-snapshot-{}-{}-{}.{}", base_slot, slot, @@ -1256,74 +1261,66 @@ where } /// Get a list of the full snapshot archives from a directory -pub fn get_full_snapshot_archives

(snapshot_archives_dir: P) -> Vec -where - P: AsRef, -{ +pub fn get_full_snapshot_archives( + full_snapshot_archives_dir: impl AsRef, +) -> Vec { get_snapshot_archives( - snapshot_archives_dir.as_ref(), + full_snapshot_archives_dir.as_ref(), FullSnapshotArchiveInfo::new_from_path, ) } /// Get a list of the incremental snapshot archives from a directory -pub fn get_incremental_snapshot_archives

( - snapshot_archives_dir: P, -) -> Vec -where - P: AsRef, -{ +pub fn get_incremental_snapshot_archives( + incremental_snapshot_archives_dir: impl AsRef, +) -> Vec { get_snapshot_archives( - snapshot_archives_dir.as_ref(), + incremental_snapshot_archives_dir.as_ref(), IncrementalSnapshotArchiveInfo::new_from_path, ) } /// Get the highest slot of the full snapshot archives in a directory -pub fn get_highest_full_snapshot_archive_slot

(snapshot_archives_dir: P) -> Option -where - P: AsRef, -{ - get_highest_full_snapshot_archive_info(snapshot_archives_dir) +pub fn get_highest_full_snapshot_archive_slot( + full_snapshot_archives_dir: impl AsRef, +) -> Option { + get_highest_full_snapshot_archive_info(full_snapshot_archives_dir) .map(|full_snapshot_archive_info| full_snapshot_archive_info.slot()) } /// Get the highest slot of the incremental snapshot archives in a directory, for a given full /// snapshot slot -pub fn get_highest_incremental_snapshot_archive_slot>( - snapshot_archives_dir: P, +pub fn get_highest_incremental_snapshot_archive_slot( + incremental_snapshot_archives_dir: impl AsRef, full_snapshot_slot: Slot, ) -> Option { - get_highest_incremental_snapshot_archive_info(snapshot_archives_dir, full_snapshot_slot) - .map(|incremental_snapshot_archive_info| incremental_snapshot_archive_info.slot()) + get_highest_incremental_snapshot_archive_info( + incremental_snapshot_archives_dir, + full_snapshot_slot, + ) + .map(|incremental_snapshot_archive_info| incremental_snapshot_archive_info.slot()) } /// Get the path (and metadata) for the full snapshot archive with the highest slot in a directory -pub fn get_highest_full_snapshot_archive_info

( - snapshot_archives_dir: P, -) -> Option -where - P: AsRef, -{ - let mut full_snapshot_archives = get_full_snapshot_archives(snapshot_archives_dir); +pub fn get_highest_full_snapshot_archive_info( + full_snapshot_archives_dir: impl AsRef, +) -> Option { + let mut full_snapshot_archives = get_full_snapshot_archives(full_snapshot_archives_dir); full_snapshot_archives.sort_unstable(); full_snapshot_archives.into_iter().rev().next() } /// Get the path for the incremental snapshot archive with the highest slot, for a given full /// snapshot slot, in a directory -pub fn get_highest_incremental_snapshot_archive_info

( - snapshot_archives_dir: P, +pub fn get_highest_incremental_snapshot_archive_info( + incremental_snapshot_archives_dir: impl AsRef, full_snapshot_slot: Slot, -) -> Option -where - P: AsRef, -{ +) -> Option { // Since we want to filter down to only the incremental snapshot archives that have the same // full snapshot slot as the value passed in, perform the filtering before sorting to avoid // doing unnecessary work. let mut incremental_snapshot_archives = - get_incremental_snapshot_archives(snapshot_archives_dir) + get_incremental_snapshot_archives(incremental_snapshot_archives_dir) .into_iter() .filter(|incremental_snapshot_archive_info| { incremental_snapshot_archive_info.base_slot() == full_snapshot_slot @@ -1333,21 +1330,19 @@ where incremental_snapshot_archives.into_iter().rev().next() } -pub fn purge_old_snapshot_archives

( - snapshot_archives_dir: P, +pub fn purge_old_snapshot_archives( + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, maximum_full_snapshot_archives_to_retain: usize, maximum_incremental_snapshot_archives_to_retain: usize, -) where - P: AsRef, -{ +) { info!( - "Purging old snapshot archives in {}, retaining up to {} full snapshots and up to {} incremental snapshots", - snapshot_archives_dir.as_ref().display(), - maximum_full_snapshot_archives_to_retain, - maximum_incremental_snapshot_archives_to_retain + "Purging old full snapshot archives in {}, retaining up to {} full snapshots", + full_snapshot_archives_dir.as_ref().display(), + maximum_full_snapshot_archives_to_retain ); - let mut full_snapshot_archives = get_full_snapshot_archives(&snapshot_archives_dir); + let mut full_snapshot_archives = get_full_snapshot_archives(&full_snapshot_archives_dir); full_snapshot_archives.sort_unstable(); full_snapshot_archives.reverse(); @@ -1383,8 +1378,15 @@ pub fn purge_old_snapshot_archives

( } remove_archives(full_snapshot_archives_to_remove); + info!( + "Purging old incremental snapshot archives in {}, retaining up to {} incremental snapshots", + incremental_snapshot_archives_dir.as_ref().display(), + maximum_incremental_snapshot_archives_to_retain + ); let mut incremental_snapshot_archives_by_base_slot = HashMap::>::new(); - for incremental_snapshot_archive in get_incremental_snapshot_archives(&snapshot_archives_dir) { + for incremental_snapshot_archive in + get_incremental_snapshot_archives(&incremental_snapshot_archives_dir) + { incremental_snapshot_archives_by_base_slot .entry(incremental_snapshot_archive.base_slot()) .or_default() @@ -1724,12 +1726,14 @@ pub fn purge_old_bank_snapshots(bank_snapshots_dir: impl AsRef) { /// function is called from AccountsBackgroundService to handle snapshot requests. Since taking a /// snapshot is not permitted to fail, any errors returned here will trigger the node to shutdown. /// So, be careful whenever adding new code that may return errors. +#[allow(clippy::too_many_arguments)] pub fn snapshot_bank( root_bank: &Bank, status_cache_slot_deltas: Vec, pending_accounts_package: &PendingAccountsPackage, bank_snapshots_dir: impl AsRef, - snapshot_archives_dir: impl AsRef, + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, snapshot_version: SnapshotVersion, archive_format: ArchiveFormat, hash_for_testing: Option, @@ -1752,7 +1756,8 @@ pub fn snapshot_bank( &bank_snapshot_info, bank_snapshots_dir, status_cache_slot_deltas, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, snapshot_storages, archive_format, snapshot_version, @@ -1810,7 +1815,8 @@ pub fn bank_to_full_snapshot_archive( bank_snapshots_dir: impl AsRef, bank: &Bank, snapshot_version: Option, - snapshot_archives_dir: impl AsRef, + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, archive_format: ArchiveFormat, maximum_full_snapshot_archives_to_retain: usize, maximum_incremental_snapshot_archives_to_retain: usize, @@ -1833,7 +1839,8 @@ pub fn bank_to_full_snapshot_archive( bank, &bank_snapshot_info, &temp_dir, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, snapshot_storages, archive_format, snapshot_version, @@ -1853,7 +1860,8 @@ pub fn bank_to_incremental_snapshot_archive( bank: &Bank, full_snapshot_slot: Slot, snapshot_version: Option, - snapshot_archives_dir: impl AsRef, + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, archive_format: ArchiveFormat, maximum_full_snapshot_archives_to_retain: usize, maximum_incremental_snapshot_archives_to_retain: usize, @@ -1878,7 +1886,8 @@ pub fn bank_to_incremental_snapshot_archive( full_snapshot_slot, &bank_snapshot_info, &temp_dir, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, snapshot_storages, archive_format, snapshot_version, @@ -1888,11 +1897,13 @@ pub fn bank_to_incremental_snapshot_archive( } /// Helper function to hold shared code to package, process, and archive full snapshots +#[allow(clippy::too_many_arguments)] pub fn package_and_archive_full_snapshot( bank: &Bank, bank_snapshot_info: &BankSnapshotInfo, bank_snapshots_dir: impl AsRef, - snapshot_archives_dir: impl AsRef, + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, snapshot_storages: SnapshotStorages, archive_format: ArchiveFormat, snapshot_version: SnapshotVersion, @@ -1904,7 +1915,8 @@ pub fn package_and_archive_full_snapshot( bank_snapshot_info, bank_snapshots_dir, bank.src.slot_deltas(&bank.src.roots()), - snapshot_archives_dir, + &full_snapshot_archives_dir, + &incremental_snapshot_archives_dir, snapshot_storages, archive_format, snapshot_version, @@ -1921,6 +1933,8 @@ pub fn package_and_archive_full_snapshot( let snapshot_package = SnapshotPackage::new(accounts_package, bank.get_accounts_hash()); archive_snapshot_package( &snapshot_package, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain, )?; @@ -1937,7 +1951,8 @@ pub fn package_and_archive_incremental_snapshot( incremental_snapshot_base_slot: Slot, bank_snapshot_info: &BankSnapshotInfo, bank_snapshots_dir: impl AsRef, - snapshot_archives_dir: impl AsRef, + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, snapshot_storages: SnapshotStorages, archive_format: ArchiveFormat, snapshot_version: SnapshotVersion, @@ -1949,7 +1964,8 @@ pub fn package_and_archive_incremental_snapshot( bank_snapshot_info, bank_snapshots_dir, bank.src.slot_deltas(&bank.src.roots()), - snapshot_archives_dir, + &full_snapshot_archives_dir, + &incremental_snapshot_archives_dir, snapshot_storages, archive_format, snapshot_version, @@ -1968,6 +1984,8 @@ pub fn package_and_archive_incremental_snapshot( let snapshot_package = SnapshotPackage::new(accounts_package, bank.get_accounts_hash()); archive_snapshot_package( &snapshot_package, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain, )?; @@ -2414,13 +2432,15 @@ mod tests { /// `max_incremental_snapshot_slot`]. Additionally, "bad" files are created for both full and /// incremental snapshots to ensure the tests properly filter them out. fn common_create_snapshot_archive_files( - snapshot_archives_dir: &Path, + full_snapshot_archives_dir: &Path, + incremental_snapshot_archives_dir: &Path, min_full_snapshot_slot: Slot, max_full_snapshot_slot: Slot, min_incremental_snapshot_slot: Slot, max_incremental_snapshot_slot: Slot, ) { - fs::create_dir_all(snapshot_archives_dir).unwrap(); + fs::create_dir_all(full_snapshot_archives_dir).unwrap(); + fs::create_dir_all(incremental_snapshot_archives_dir).unwrap(); for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot { for incremental_snapshot_slot in min_incremental_snapshot_slot..max_incremental_snapshot_slot @@ -2431,13 +2451,13 @@ mod tests { incremental_snapshot_slot, Hash::default() ); - let snapshot_filepath = snapshot_archives_dir.join(snapshot_filename); + let snapshot_filepath = incremental_snapshot_archives_dir.join(snapshot_filename); File::create(snapshot_filepath).unwrap(); } let snapshot_filename = format!("snapshot-{}-{}.tar", full_snapshot_slot, Hash::default()); - let snapshot_filepath = snapshot_archives_dir.join(snapshot_filename); + let snapshot_filepath = full_snapshot_archives_dir.join(snapshot_filename); File::create(snapshot_filepath).unwrap(); // Add in an incremental snapshot with a bad filename and high slot to ensure filename are filtered and sorted correctly @@ -2446,50 +2466,54 @@ mod tests { full_snapshot_slot, max_incremental_snapshot_slot + 1, ); - let bad_filepath = snapshot_archives_dir.join(bad_filename); + let bad_filepath = incremental_snapshot_archives_dir.join(bad_filename); File::create(bad_filepath).unwrap(); } // Add in a snapshot with a bad filename and high slot to ensure filename are filtered and // sorted correctly let bad_filename = format!("snapshot-{}-bad!hash.tar", max_full_snapshot_slot + 1); - let bad_filepath = snapshot_archives_dir.join(bad_filename); + let bad_filepath = full_snapshot_archives_dir.join(bad_filename); File::create(bad_filepath).unwrap(); } #[test] fn test_get_full_snapshot_archives() { solana_logger::setup(); - let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let min_slot = 123; let max_slot = 456; common_create_snapshot_archive_files( - temp_snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), min_slot, max_slot, 0, 0, ); - let snapshot_archives = get_full_snapshot_archives(temp_snapshot_archives_dir); + let snapshot_archives = get_full_snapshot_archives(full_snapshot_archives_dir); assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot); } #[test] fn test_get_full_snapshot_archives_remote() { solana_logger::setup(); - let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let min_slot = 123; let max_slot = 456; common_create_snapshot_archive_files( - &temp_snapshot_archives_dir.path().join("remote"), + &full_snapshot_archives_dir.path().join("remote"), + &incremental_snapshot_archives_dir.path().join("remote"), min_slot, max_slot, 0, 0, ); - let snapshot_archives = get_full_snapshot_archives(temp_snapshot_archives_dir); + let snapshot_archives = get_full_snapshot_archives(full_snapshot_archives_dir); assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot); assert!(snapshot_archives.iter().all(|info| info.is_remote())); } @@ -2497,13 +2521,15 @@ mod tests { #[test] fn test_get_incremental_snapshot_archives() { solana_logger::setup(); - let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let min_full_snapshot_slot = 12; let max_full_snapshot_slot = 23; let min_incremental_snapshot_slot = 34; let max_incremental_snapshot_slot = 45; common_create_snapshot_archive_files( - temp_snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), min_full_snapshot_slot, max_full_snapshot_slot, min_incremental_snapshot_slot, @@ -2511,7 +2537,7 @@ mod tests { ); let incremental_snapshot_archives = - get_incremental_snapshot_archives(temp_snapshot_archives_dir); + get_incremental_snapshot_archives(incremental_snapshot_archives_dir); assert_eq!( incremental_snapshot_archives.len() as Slot, (max_full_snapshot_slot - min_full_snapshot_slot) @@ -2522,13 +2548,15 @@ mod tests { #[test] fn test_get_incremental_snapshot_archives_remote() { solana_logger::setup(); - let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let min_full_snapshot_slot = 12; let max_full_snapshot_slot = 23; let min_incremental_snapshot_slot = 34; let max_incremental_snapshot_slot = 45; common_create_snapshot_archive_files( - &temp_snapshot_archives_dir.path().join("remote"), + &full_snapshot_archives_dir.path().join("remote"), + &incremental_snapshot_archives_dir.path().join("remote"), min_full_snapshot_slot, max_full_snapshot_slot, min_incremental_snapshot_slot, @@ -2536,7 +2564,7 @@ mod tests { ); let incremental_snapshot_archives = - get_incremental_snapshot_archives(temp_snapshot_archives_dir); + get_incremental_snapshot_archives(incremental_snapshot_archives_dir); assert_eq!( incremental_snapshot_archives.len() as Slot, (max_full_snapshot_slot - min_full_snapshot_slot) @@ -2550,11 +2578,13 @@ mod tests { #[test] fn test_get_highest_full_snapshot_archive_slot() { solana_logger::setup(); - let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let min_slot = 123; let max_slot = 456; common_create_snapshot_archive_files( - temp_snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), min_slot, max_slot, 0, @@ -2562,7 +2592,7 @@ mod tests { ); assert_eq!( - get_highest_full_snapshot_archive_slot(temp_snapshot_archives_dir.path()), + get_highest_full_snapshot_archive_slot(full_snapshot_archives_dir.path()), Some(max_slot - 1) ); } @@ -2570,13 +2600,15 @@ mod tests { #[test] fn test_get_highest_incremental_snapshot_slot() { solana_logger::setup(); - let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let min_full_snapshot_slot = 12; let max_full_snapshot_slot = 23; let min_incremental_snapshot_slot = 34; let max_incremental_snapshot_slot = 45; common_create_snapshot_archive_files( - temp_snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), min_full_snapshot_slot, max_full_snapshot_slot, min_incremental_snapshot_slot, @@ -2586,7 +2618,7 @@ mod tests { for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot { assert_eq!( get_highest_incremental_snapshot_archive_slot( - temp_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), full_snapshot_slot ), Some(max_incremental_snapshot_slot - 1) @@ -2595,7 +2627,7 @@ mod tests { assert_eq!( get_highest_incremental_snapshot_archive_slot( - temp_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), max_full_snapshot_slot ), None @@ -2615,6 +2647,7 @@ mod tests { let mut _snap_file = File::create(snap_path); } purge_old_snapshot_archives( + temp_snap_dir.path(), temp_snap_dir.path(), maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain, @@ -2691,14 +2724,15 @@ mod tests { /// snapshot archives on disk are correct. #[test] fn test_purge_old_full_snapshot_archives_in_the_loop() { - let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let maximum_snapshots_to_retain = 5; let starting_slot: Slot = 42; for slot in (starting_slot..).take(100) { let full_snapshot_archive_file_name = format!("snapshot-{}-{}.tar", slot, Hash::default()); - let full_snapshot_archive_path = snapshot_archives_dir + let full_snapshot_archive_path = full_snapshot_archives_dir .as_ref() .join(full_snapshot_archive_file_name); File::create(full_snapshot_archive_path).unwrap(); @@ -2714,11 +2748,13 @@ mod tests { } purge_old_snapshot_archives( - &snapshot_archives_dir, + &full_snapshot_archives_dir, + &incremental_snapshot_archives_dir, maximum_snapshots_to_retain, usize::MAX, ); - let mut full_snapshot_archives = get_full_snapshot_archives(&snapshot_archives_dir); + let mut full_snapshot_archives = + get_full_snapshot_archives(&full_snapshot_archives_dir); full_snapshot_archives.sort_unstable(); assert_eq!(full_snapshot_archives.len(), maximum_snapshots_to_retain); assert_eq!(full_snapshot_archives.last().unwrap().slot(), slot); @@ -2731,7 +2767,8 @@ mod tests { #[test] fn test_purge_old_incremental_snapshot_archives() { solana_logger::setup(); - let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let starting_slot = 100_000; let maximum_incremental_snapshot_archives_to_retain = @@ -2751,7 +2788,7 @@ mod tests { .for_each(|full_snapshot_slot| { let snapshot_filename = format!("snapshot-{}-{}.tar", full_snapshot_slot, Hash::default()); - let snapshot_path = snapshot_archives_dir.path().join(&snapshot_filename); + let snapshot_path = full_snapshot_archives_dir.path().join(&snapshot_filename); File::create(snapshot_path).unwrap(); snapshot_filenames.push(snapshot_filename); @@ -2766,14 +2803,17 @@ mod tests { incremental_snapshot_slot, Hash::default() ); - let snapshot_path = snapshot_archives_dir.path().join(&snapshot_filename); + let snapshot_path = incremental_snapshot_archives_dir + .path() + .join(&snapshot_filename); File::create(snapshot_path).unwrap(); snapshot_filenames.push(snapshot_filename); }); }); purge_old_snapshot_archives( - snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain, ); @@ -2781,7 +2821,7 @@ mod tests { // Ensure correct number of full snapshot archives are purged/retained // NOTE: One extra full snapshot is always kept (the oldest), hence the `+1` let mut remaining_full_snapshot_archives = - get_full_snapshot_archives(snapshot_archives_dir.path()); + get_full_snapshot_archives(full_snapshot_archives_dir.path()); assert_eq!( remaining_full_snapshot_archives.len(), maximum_full_snapshot_archives_to_retain, @@ -2792,7 +2832,7 @@ mod tests { // Ensure correct number of incremental snapshot archives are purged/retained let mut remaining_incremental_snapshot_archives = - get_incremental_snapshot_archives(snapshot_archives_dir.path()); + get_incremental_snapshot_archives(incremental_snapshot_archives_dir.path()); assert_eq!( remaining_incremental_snapshot_archives.len(), maximum_incremental_snapshot_archives_to_retain @@ -2846,7 +2886,8 @@ mod tests { #[test] fn test_purge_all_incremental_snapshot_archives_when_no_full_snapshot_archives() { - let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); for snapshot_filenames in [ format!("incremental-snapshot-100-120-{}.tar", Hash::default()), @@ -2858,14 +2899,21 @@ mod tests { format!("incremental-snapshot-200-260-{}.tar", Hash::default()), format!("incremental-snapshot-200-280-{}.tar", Hash::default()), ] { - let snapshot_path = snapshot_archives_dir.path().join(&snapshot_filenames); + let snapshot_path = incremental_snapshot_archives_dir + .path() + .join(&snapshot_filenames); File::create(snapshot_path).unwrap(); } - purge_old_snapshot_archives(snapshot_archives_dir.path(), usize::MAX, usize::MAX); + purge_old_snapshot_archives( + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), + usize::MAX, + usize::MAX, + ); let remaining_incremental_snapshot_archives = - get_incremental_snapshot_archives(snapshot_archives_dir.path()); + get_incremental_snapshot_archives(incremental_snapshot_archives_dir.path()); assert!(remaining_incremental_snapshot_archives.is_empty()); } @@ -2883,14 +2931,16 @@ mod tests { let accounts_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); - let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archive_format = ArchiveFormat::Tar; let snapshot_archive_info = bank_to_full_snapshot_archive( &bank_snapshots_dir, &original_bank, None, - snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), snapshot_archive_format, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, @@ -2974,14 +3024,16 @@ mod tests { let accounts_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); - let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archive_format = ArchiveFormat::TarGzip; let full_snapshot_archive_info = bank_to_full_snapshot_archive( bank_snapshots_dir.path(), &bank4, None, - snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), snapshot_archive_format, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, @@ -3050,7 +3102,8 @@ mod tests { let accounts_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); - let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archive_format = ArchiveFormat::TarZstd; let full_snapshot_slot = slot; @@ -3058,7 +3111,8 @@ mod tests { bank_snapshots_dir.path(), &bank1, None, - snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), snapshot_archive_format, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, @@ -3091,7 +3145,8 @@ mod tests { &bank4, full_snapshot_slot, None, - snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), snapshot_archive_format, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, @@ -3150,7 +3205,8 @@ mod tests { let accounts_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); - let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archive_format = ArchiveFormat::Tar; let full_snapshot_slot = slot; @@ -3158,7 +3214,8 @@ mod tests { &bank_snapshots_dir, &bank1, None, - &snapshot_archives_dir, + &full_snapshot_archives_dir, + &incremental_snapshot_archives_dir, snapshot_archive_format, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, @@ -3191,7 +3248,8 @@ mod tests { &bank4, full_snapshot_slot, None, - &snapshot_archives_dir, + &full_snapshot_archives_dir, + &incremental_snapshot_archives_dir, snapshot_archive_format, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, @@ -3200,7 +3258,8 @@ mod tests { let (deserialized_bank, ..) = bank_from_latest_snapshot_archives( &bank_snapshots_dir, - &snapshot_archives_dir, + &full_snapshot_archives_dir, + &incremental_snapshot_archives_dir, &[accounts_dir.as_ref().to_path_buf()], &genesis_config, None, @@ -3252,7 +3311,8 @@ mod tests { let accounts_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap(); - let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archive_format = ArchiveFormat::Tar; let (genesis_config, mint_keypair) = create_genesis_config(1_000_000); @@ -3289,7 +3349,8 @@ mod tests { bank_snapshots_dir.path(), &bank1, None, - snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), snapshot_archive_format, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, @@ -3329,7 +3390,8 @@ mod tests { &bank2, full_snapshot_slot, None, - snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), snapshot_archive_format, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, @@ -3390,7 +3452,8 @@ mod tests { &bank4, full_snapshot_slot, None, - snapshot_archives_dir.path(), + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), snapshot_archive_format, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, @@ -3455,7 +3518,8 @@ mod tests { snapshot_storages: SnapshotStorages::default(), archive_format: ArchiveFormat::Tar, snapshot_version: SnapshotVersion::default(), - snapshot_archives_dir: PathBuf::default(), + full_snapshot_archives_dir: PathBuf::default(), + incremental_snapshot_archives_dir: PathBuf::default(), expected_capitalization: u64::default(), accounts_hash_for_testing: None, cluster_type: solana_sdk::genesis_config::ClusterType::Development, diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index 0b95727213..2c697fcb06 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -711,7 +711,8 @@ impl TestValidator { full_snapshot_archive_interval_slots: 100, incremental_snapshot_archive_interval_slots: Slot::MAX, bank_snapshots_dir: ledger_path.join("snapshot"), - snapshot_archives_dir: ledger_path.to_path_buf(), + full_snapshot_archives_dir: ledger_path.to_path_buf(), + incremental_snapshot_archives_dir: ledger_path.to_path_buf(), ..SnapshotConfig::default() }), enforce_ulimit_nofile: false, diff --git a/validator/src/bootstrap.rs b/validator/src/bootstrap.rs index e53a1c278d..274035cd69 100644 --- a/validator/src/bootstrap.rs +++ b/validator/src/bootstrap.rs @@ -56,7 +56,8 @@ pub fn rpc_bootstrap( node: &Node, identity_keypair: &Arc, ledger_path: &Path, - snapshot_archives_dir: &Path, + full_snapshot_archives_dir: &Path, + incremental_snapshot_archives_dir: &Path, vote_account: &Pubkey, authorized_voter_keypairs: Arc>>>, cluster_entrypoints: &[ContactInfo], @@ -96,7 +97,8 @@ pub fn rpc_bootstrap( node, identity_keypair, ledger_path, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, vote_account, authorized_voter_keypairs, cluster_entrypoints, @@ -116,7 +118,7 @@ pub fn rpc_bootstrap( node, identity_keypair, ledger_path, - snapshot_archives_dir, + full_snapshot_archives_dir, vote_account, authorized_voter_keypairs, cluster_entrypoints, @@ -384,32 +386,6 @@ fn check_vote_account( Ok(()) } -/// Get the Slot and Hash of the local snapshot with the highest slot. Can be either a full -/// snapshot or an incremental snapshot. -fn get_highest_local_snapshot_hash( - snapshot_archives_dir: impl AsRef, -) -> Option<(Slot, Hash)> { - if let Some(full_snapshot_info) = - snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir) - { - if let Some(incremental_snapshot_info) = - snapshot_utils::get_highest_incremental_snapshot_archive_info( - &snapshot_archives_dir, - full_snapshot_info.slot(), - ) - { - Some(( - incremental_snapshot_info.slot(), - *incremental_snapshot_info.hash(), - )) - } else { - Some((full_snapshot_info.slot(), *full_snapshot_info.hash())) - } - } else { - None - } -} - mod without_incremental_snapshots { use super::*; @@ -418,7 +394,7 @@ mod without_incremental_snapshots { node: &Node, identity_keypair: &Arc, ledger_path: &Path, - snapshot_archives_dir: &Path, + full_snapshot_archives_dir: &Path, vote_account: &Pubkey, authorized_voter_keypairs: Arc>>>, cluster_entrypoints: &[ContactInfo], @@ -458,7 +434,7 @@ mod without_incremental_snapshots { validator_config, &mut blacklisted_rpc_nodes, &bootstrap_config, - snapshot_archives_dir, + full_snapshot_archives_dir, ); if rpc_node_details.is_none() { return; @@ -512,7 +488,7 @@ mod without_incremental_snapshots { } if let Some(snapshot_hash) = snapshot_hash { - let use_local_snapshot = match get_highest_local_snapshot_hash(snapshot_archives_dir) { + let use_local_snapshot = match get_highest_local_snapshot_hash(full_snapshot_archives_dir) { None => { info!("Downloading snapshot for slot {} since there is not a local snapshot", snapshot_hash.0); false @@ -555,16 +531,17 @@ mod without_incremental_snapshots { gossip.take().unwrap(); cluster_info.save_contact_info(); gossip_exit_flag.store(true, Ordering::Relaxed); - let (maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain) = if let Some(snapshot_config) = + let (maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain, incremental_snapshot_archives_dir) = if let Some(snapshot_config) = validator_config.snapshot_config.as_ref() { - (snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain) + (snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain, snapshot_config.incremental_snapshot_archives_dir.as_path()) } else { - (DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN) + (DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, full_snapshot_archives_dir) }; let ret = download_snapshot_archive( &rpc_contact_info.rpc, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, snapshot_hash, SnapshotType::FullSnapshot, maximum_full_snapshot_archives_to_retain, @@ -668,7 +645,7 @@ mod without_incremental_snapshots { validator_config: &ValidatorConfig, blacklisted_rpc_nodes: &mut HashSet, bootstrap_config: &RpcBootstrapConfig, - snapshot_archives_dir: &Path, + full_snapshot_archives_dir: &Path, ) -> Option<(ContactInfo, Option<(Slot, Hash)>)> { let mut blacklist_timeout = Instant::now(); let mut newer_cluster_snapshot_timeout = None; @@ -692,7 +669,8 @@ mod without_incremental_snapshots { let rpc_peers = rpc_peers.unwrap(); blacklist_timeout = Instant::now(); - let mut highest_snapshot_hash = get_highest_local_snapshot_hash(snapshot_archives_dir); + let mut highest_snapshot_hash = + get_highest_local_snapshot_hash(full_snapshot_archives_dir); let eligible_rpc_peers = if bootstrap_config.no_snapshot_fetch { rpc_peers } else { @@ -795,6 +773,14 @@ mod without_incremental_snapshots { None } } + + /// Get the Slot and Hash of the local snapshot with the highest slot. + fn get_highest_local_snapshot_hash( + full_snapshot_archives_dir: impl AsRef, + ) -> Option<(Slot, Hash)> { + snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir) + .map(|full_snapshot_info| (full_snapshot_info.slot(), *full_snapshot_info.hash())) + } } mod with_incremental_snapshots { @@ -829,7 +815,8 @@ mod with_incremental_snapshots { node: &Node, identity_keypair: &Arc, ledger_path: &Path, - snapshot_archives_dir: &Path, + full_snapshot_archives_dir: &Path, + incremental_snapshot_archives_dir: &Path, vote_account: &Pubkey, authorized_voter_keypairs: Arc>>>, cluster_entrypoints: &[ContactInfo], @@ -935,7 +922,8 @@ mod with_incremental_snapshots { info!("RPC node root slot: {}", rpc_client_slot); download_snapshots( - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, validator_config, &bootstrap_config, use_progress_bar, @@ -1088,6 +1076,33 @@ mod with_incremental_snapshots { } } + /// Get the Slot and Hash of the local snapshot with the highest slot. Can be either a full + /// snapshot or an incremental snapshot. + fn get_highest_local_snapshot_hash( + full_snapshot_archives_dir: impl AsRef, + incremental_snapshot_archives_dir: impl AsRef, + ) -> Option<(Slot, Hash)> { + if let Some(full_snapshot_info) = + snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir) + { + if let Some(incremental_snapshot_info) = + snapshot_utils::get_highest_incremental_snapshot_archive_info( + incremental_snapshot_archives_dir, + full_snapshot_info.slot(), + ) + { + Some(( + incremental_snapshot_info.slot(), + *incremental_snapshot_info.hash(), + )) + } else { + Some((full_snapshot_info.slot(), *full_snapshot_info.hash())) + } + } else { + None + } + } + /// Get peer snapshot hashes /// /// The result is a vector of peers with snapshot hashes that: @@ -1420,7 +1435,8 @@ mod with_incremental_snapshots { /// Check to see if we can use our local snapshots, otherwise download newer ones. #[allow(clippy::too_many_arguments)] fn download_snapshots( - snapshot_archives_dir: &Path, + full_snapshot_archives_dir: &Path, + incremental_snapshot_archives_dir: &Path, validator_config: &ValidatorConfig, bootstrap_config: &RpcBootstrapConfig, use_progress_bar: bool, @@ -1442,7 +1458,8 @@ mod with_incremental_snapshots { // If the local snapshots are new enough, then use 'em; no need to download new snapshots if should_use_local_snapshot( - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, maximum_local_snapshot_age, full_snapshot_hash, incremental_snapshot_hash, @@ -1451,7 +1468,7 @@ mod with_incremental_snapshots { } // Check and see if we've already got the full snapshot; if not, download it - if snapshot_utils::get_full_snapshot_archives(snapshot_archives_dir) + if snapshot_utils::get_full_snapshot_archives(full_snapshot_archives_dir) .into_iter() .any(|snapshot_archive| { snapshot_archive.slot() == full_snapshot_hash.0 @@ -1464,7 +1481,8 @@ mod with_incremental_snapshots { ); } else { download_snapshot( - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, validator_config, bootstrap_config, use_progress_bar, @@ -1480,7 +1498,7 @@ mod with_incremental_snapshots { // Check and see if we've already got the incremental snapshot; if not, download it if let Some(incremental_snapshot_hash) = incremental_snapshot_hash { - if snapshot_utils::get_incremental_snapshot_archives(snapshot_archives_dir) + if snapshot_utils::get_incremental_snapshot_archives(incremental_snapshot_archives_dir) .into_iter() .any(|snapshot_archive| { snapshot_archive.slot() == incremental_snapshot_hash.0 @@ -1494,7 +1512,8 @@ mod with_incremental_snapshots { ); } else { download_snapshot( - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, validator_config, bootstrap_config, use_progress_bar, @@ -1515,7 +1534,8 @@ mod with_incremental_snapshots { /// Download a snapshot #[allow(clippy::too_many_arguments)] fn download_snapshot( - snapshot_archives_dir: &Path, + full_snapshot_archives_dir: &Path, + incremental_snapshot_archives_dir: &Path, validator_config: &ValidatorConfig, bootstrap_config: &RpcBootstrapConfig, use_progress_bar: bool, @@ -1547,7 +1567,8 @@ mod with_incremental_snapshots { }; download_snapshot_archive( &rpc_contact_info.rpc, - snapshot_archives_dir, + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, desired_snapshot_hash, snapshot_type, maximum_full_snapshot_archives_to_retain, @@ -1591,7 +1612,8 @@ mod with_incremental_snapshots { /// Check to see if bootstrap should load from its local snapshots or not. If not, then snapshots /// will be downloaded. fn should_use_local_snapshot( - snapshot_archives_dir: &Path, + full_snapshot_archives_dir: &Path, + incremental_snapshot_archives_dir: &Path, maximum_local_snapshot_age: Slot, full_snapshot_hash: (Slot, Hash), incremental_snapshot_hash: Option<(Slot, Hash)>, @@ -1600,7 +1622,10 @@ mod with_incremental_snapshots { .map(|(slot, _)| slot) .unwrap_or(full_snapshot_hash.0); - match get_highest_local_snapshot_hash(snapshot_archives_dir) { + match get_highest_local_snapshot_hash( + full_snapshot_archives_dir, + incremental_snapshot_archives_dir, + ) { None => { info!( "Downloading a snapshot for slot {} since there is not a local snapshot.", diff --git a/validator/src/main.rs b/validator/src/main.rs index 98f0e47117..2a60790b5a 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -727,6 +727,14 @@ pub fn main() { .takes_value(true) .help("Use DIR as snapshot location [default: --ledger value]"), ) + .arg( + Arg::with_name("incremental_snapshot_archive_path") + .long("incremental-snapshot-archive-path") + .conflicts_with("no-incremental-snapshots") + .value_name("DIR") + .takes_value(true) + .help("Use DIR as separate location for incremental snapshot archives [default: --snapshots value]"), + ) .arg( Arg::with_name("tower") .long("tower") @@ -2602,16 +2610,36 @@ pub fn main() { let maximum_snapshot_download_abort = value_t_or_exit!(matches, "maximum_snapshot_download_abort", u64); - let snapshot_archives_dir = if matches.is_present("snapshots") { + let full_snapshot_archives_dir = if matches.is_present("snapshots") { PathBuf::from(matches.value_of("snapshots").unwrap()) } else { ledger_path.clone() }; - let bank_snapshots_dir = snapshot_archives_dir.join("snapshot"); + let incremental_snapshot_archives_dir = + if matches.is_present("incremental_snapshot_archive_path") { + let incremental_snapshot_archives_dir = PathBuf::from( + matches + .value_of("incremental_snapshot_archive_path") + .unwrap(), + ); + fs::create_dir_all(&incremental_snapshot_archives_dir).unwrap_or_else(|err| { + eprintln!( + "Failed to create incremental snapshot archives directory {:?}: {}", + incremental_snapshot_archives_dir.display(), + err + ); + exit(1); + }); + incremental_snapshot_archives_dir + } else { + full_snapshot_archives_dir.clone() + }; + let bank_snapshots_dir = incremental_snapshot_archives_dir.join("snapshot"); fs::create_dir_all(&bank_snapshots_dir).unwrap_or_else(|err| { eprintln!( "Failed to create snapshots directory {:?}: {}", - bank_snapshots_dir, err + bank_snapshots_dir.display(), + err ); exit(1); }); @@ -2657,7 +2685,8 @@ pub fn main() { full_snapshot_archive_interval_slots, incremental_snapshot_archive_interval_slots, bank_snapshots_dir, - snapshot_archives_dir: snapshot_archives_dir.clone(), + full_snapshot_archives_dir: full_snapshot_archives_dir.clone(), + incremental_snapshot_archives_dir: incremental_snapshot_archives_dir.clone(), archive_format, snapshot_version, maximum_full_snapshot_archives_to_retain, @@ -2887,7 +2916,8 @@ pub fn main() { Some(version) }); solana_entry::entry::init_poh(); - snapshot_utils::remove_tmp_snapshot_archives(&snapshot_archives_dir); + snapshot_utils::remove_tmp_snapshot_archives(&full_snapshot_archives_dir); + snapshot_utils::remove_tmp_snapshot_archives(&incremental_snapshot_archives_dir); let identity_keypair = Arc::new(identity_keypair); @@ -2897,7 +2927,8 @@ pub fn main() { &node, &identity_keypair, &ledger_path, - &snapshot_archives_dir, + &full_snapshot_archives_dir, + &incremental_snapshot_archives_dir, &vote_account, authorized_voter_keypairs.clone(), &cluster_entrypoints,