Name snapshots consistently (#19346)

#### Problem

Snapshot names are overloaded, and there are multiple terms that mean the same thing. This is confusing. Here's a list of ones in the codebase that I've found:

```
- snapshot_dir
- snapshots_dir
- snapshot_path
- snapshot_output_dir
- snapshot_package_output_path
- snapshot_archives_dir
```

#### Summary of Changes

For all the ones that are about the directory where snapshot archives are stored, ensure they are `snapshot_archives_dir`. For the ones about the (bank) snapshots directory, set to `bank_snapshots_dir`.


Co-authored-by: Michael Vines <mvines@gmail.com>
This commit is contained in:
Brooks Prumo 2021-08-21 15:41:03 -05:00 committed by GitHub
parent 9483866e0b
commit 6d939811e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 237 additions and 221 deletions

View File

@ -293,8 +293,8 @@ mod tests {
let snapshot_config = SnapshotConfig { let snapshot_config = SnapshotConfig {
full_snapshot_archive_interval_slots, full_snapshot_archive_interval_slots,
incremental_snapshot_archive_interval_slots: Slot::MAX, incremental_snapshot_archive_interval_slots: Slot::MAX,
snapshot_package_output_path: PathBuf::default(), snapshot_archives_dir: PathBuf::default(),
snapshot_path: PathBuf::default(), bank_snapshots_dir: PathBuf::default(),
archive_format: ArchiveFormat::Tar, archive_format: ArchiveFormat::Tar,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),
maximum_snapshots_to_retain: usize::MAX, maximum_snapshots_to_retain: usize::MAX,

View File

@ -121,8 +121,8 @@ mod tests {
fn create_and_verify_snapshot(temp_dir: &Path) { fn create_and_verify_snapshot(temp_dir: &Path) {
let accounts_dir = temp_dir.join("accounts"); let accounts_dir = temp_dir.join("accounts");
let snapshots_dir = temp_dir.join("snapshots"); let snapshots_dir = temp_dir.join("snapshots");
let snapshot_package_output_path = temp_dir.join("snapshots_output"); let snapshot_archives_dir = temp_dir.join("snapshots_output");
fs::create_dir_all(&snapshot_package_output_path).unwrap(); fs::create_dir_all(&snapshot_archives_dir).unwrap();
fs::create_dir_all(&accounts_dir).unwrap(); fs::create_dir_all(&accounts_dir).unwrap();
// Create some storage entries // Create some storage entries
@ -161,7 +161,7 @@ mod tests {
// Create a packageable snapshot // Create a packageable snapshot
let output_tar_path = snapshot_utils::build_full_snapshot_archive_path( let output_tar_path = snapshot_utils::build_full_snapshot_archive_path(
snapshot_package_output_path, snapshot_archives_dir,
42, 42,
&Hash::default(), &Hash::default(),
ArchiveFormat::TarBzip2, ArchiveFormat::TarBzip2,

View File

@ -523,8 +523,8 @@ impl TestValidator {
snapshot_config: Some(SnapshotConfig { snapshot_config: Some(SnapshotConfig {
full_snapshot_archive_interval_slots: 100, full_snapshot_archive_interval_slots: 100,
incremental_snapshot_archive_interval_slots: Slot::MAX, incremental_snapshot_archive_interval_slots: Slot::MAX,
snapshot_path: ledger_path.join("snapshot"), bank_snapshots_dir: ledger_path.join("snapshot"),
snapshot_package_output_path: ledger_path.to_path_buf(), snapshot_archives_dir: ledger_path.to_path_buf(),
archive_format: ArchiveFormat::Tar, archive_format: ArchiveFormat::Tar,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,

View File

@ -1210,7 +1210,7 @@ fn new_banks_from_ledger(
ledger_path, ledger_path,
&bank_forks.root_bank(), &bank_forks.root_bank(),
None, None,
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_archives_dir,
snapshot_config.archive_format, snapshot_config.archive_format,
Some(bank_forks.root_bank().get_thread_pool()), Some(bank_forks.root_bank().get_thread_pool()),
snapshot_config.maximum_snapshots_to_retain, snapshot_config.maximum_snapshots_to_retain,

View File

@ -141,8 +141,8 @@ mod tests {
let snapshot_config = SnapshotConfig { let snapshot_config = SnapshotConfig {
full_snapshot_archive_interval_slots, full_snapshot_archive_interval_slots,
incremental_snapshot_archive_interval_slots, incremental_snapshot_archive_interval_slots,
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(), snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(),
snapshot_path: bank_snapshots_dir.path().to_path_buf(), bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
archive_format: ArchiveFormat::TarBzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version, snapshot_version,
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -166,17 +166,17 @@ mod tests {
old_genesis_config: &GenesisConfig, old_genesis_config: &GenesisConfig,
account_paths: &[PathBuf], account_paths: &[PathBuf],
) { ) {
let (snapshot_path, snapshot_package_output_path) = old_bank_forks let (snapshot_path, snapshot_archives_dir) = old_bank_forks
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.map(|c| (&c.snapshot_path, &c.snapshot_package_output_path)) .map(|c| (&c.bank_snapshots_dir, &c.snapshot_archives_dir))
.unwrap(); .unwrap();
let old_last_bank = old_bank_forks.get(old_last_slot).unwrap(); let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
let check_hash_calculation = false; let check_hash_calculation = false;
let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path( let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path(
snapshot_package_output_path.to_path_buf(), snapshot_archives_dir.to_path_buf(),
old_last_bank.slot(), old_last_bank.slot(),
&old_last_bank.get_accounts_hash(), &old_last_bank.get_accounts_hash(),
ArchiveFormat::TarBzip2, ArchiveFormat::TarBzip2,
@ -191,7 +191,7 @@ mod tests {
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_path, .bank_snapshots_dir,
&full_snapshot_archive_info, &full_snapshot_archive_info,
None, None,
old_genesis_config, old_genesis_config,
@ -273,15 +273,16 @@ mod tests {
// Generate a snapshot package for last bank // Generate a snapshot package for last bank
let last_bank = bank_forks.get(last_slot).unwrap(); let last_bank = bank_forks.get(last_slot).unwrap();
let snapshot_config = &snapshot_test_config.snapshot_config; let snapshot_config = &snapshot_test_config.snapshot_config;
let snapshot_path = &snapshot_config.snapshot_path; let bank_snapshots_dir = &snapshot_config.bank_snapshots_dir;
let last_bank_snapshot_info = snapshot_utils::get_highest_bank_snapshot_info(snapshot_path) let last_bank_snapshot_info =
.expect("no snapshots found in path"); snapshot_utils::get_highest_bank_snapshot_info(bank_snapshots_dir)
.expect("no bank snapshots found in path");
let accounts_package = AccountsPackage::new_for_full_snapshot( let accounts_package = AccountsPackage::new_for_full_snapshot(
last_bank, last_bank,
&last_bank_snapshot_info, &last_bank_snapshot_info,
snapshot_path, bank_snapshots_dir,
last_bank.src.slot_deltas(&last_bank.src.roots()), last_bank.src.slot_deltas(&last_bank.src.roots()),
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_archives_dir,
last_bank.get_snapshot_storages(None), last_bank.get_snapshot_storages(None),
ArchiveFormat::TarBzip2, ArchiveFormat::TarBzip2,
snapshot_version, snapshot_version,
@ -355,17 +356,16 @@ mod tests {
SnapshotTestConfig::new(snapshot_version, cluster_type, 1, 1, Slot::MAX); SnapshotTestConfig::new(snapshot_version, cluster_type, 1, 1, Slot::MAX);
let bank_forks = &mut snapshot_test_config.bank_forks; let bank_forks = &mut snapshot_test_config.bank_forks;
let bank_snapshots_dir = &snapshot_test_config.bank_snapshots_dir;
let snapshot_config = &snapshot_test_config.snapshot_config; let snapshot_config = &snapshot_test_config.snapshot_config;
let snapshot_path = &snapshot_config.snapshot_path; let bank_snapshots_dir = &snapshot_config.bank_snapshots_dir;
let snapshot_package_output_path = &snapshot_config.snapshot_package_output_path; let snapshot_archives_dir = &snapshot_config.snapshot_archives_dir;
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config; let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
// Take snapshot of zeroth bank // Take snapshot of zeroth bank
let bank0 = bank_forks.get(0).unwrap(); let bank0 = bank_forks.get(0).unwrap();
let storages = bank0.get_snapshot_storages(None); let storages = bank0.get_snapshot_storages(None);
snapshot_utils::add_bank_snapshot(snapshot_path, bank0, &storages, snapshot_version) snapshot_utils::add_bank_snapshot(bank_snapshots_dir, bank0, &storages, snapshot_version)
.unwrap(); .unwrap();
// Set up snapshotting channels // Set up snapshotting channels
@ -412,8 +412,8 @@ mod tests {
&bank, &bank,
vec![], vec![],
package_sender, package_sender,
snapshot_path, bank_snapshots_dir,
snapshot_package_output_path, snapshot_archives_dir,
snapshot_config.snapshot_version, snapshot_config.snapshot_version,
&snapshot_config.archive_format, &snapshot_config.archive_format,
None, None,
@ -443,7 +443,7 @@ mod tests {
) )
.unwrap(); .unwrap();
} }
let last_snapshot_path = fs::read_dir(snapshot_path) let last_snapshot_path = fs::read_dir(bank_snapshots_dir)
.unwrap() .unwrap()
.filter_map(|entry| { .filter_map(|entry| {
let e = entry.unwrap(); let e = entry.unwrap();
@ -462,7 +462,7 @@ mod tests {
fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap(); fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path( saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path(
snapshot_package_output_path.to_path_buf(), snapshot_archives_dir.to_path_buf(),
slot, slot,
&accounts_hash, &accounts_hash,
ArchiveFormat::TarBzip2, ArchiveFormat::TarBzip2,
@ -472,7 +472,7 @@ mod tests {
// Purge all the outdated snapshots, including the ones needed to generate the package // Purge all the outdated snapshots, including the ones needed to generate the package
// currently sitting in the channel // currently sitting in the channel
snapshot_utils::purge_old_bank_snapshots(snapshot_path); snapshot_utils::purge_old_bank_snapshots(bank_snapshots_dir);
let mut bank_snapshots = snapshot_utils::get_bank_snapshots(&bank_snapshots_dir); let mut bank_snapshots = snapshot_utils::get_bank_snapshots(&bank_snapshots_dir);
bank_snapshots.sort_unstable(); bank_snapshots.sort_unstable();
@ -754,15 +754,21 @@ mod tests {
) -> snapshot_utils::Result<()> { ) -> snapshot_utils::Result<()> {
let slot = bank.slot(); let slot = bank.slot();
info!("Making full snapshot archive from bank at slot: {}", slot); info!("Making full snapshot archive from bank at slot: {}", slot);
let bank_snapshot_info = snapshot_utils::get_bank_snapshots(&snapshot_config.snapshot_path) let bank_snapshot_info =
.into_iter() snapshot_utils::get_bank_snapshots(&snapshot_config.bank_snapshots_dir)
.find(|elem| elem.slot == slot) .into_iter()
.ok_or_else(|| Error::new(ErrorKind::Other, "did not find snapshot with this path"))?; .find(|elem| elem.slot == slot)
.ok_or_else(|| {
Error::new(
ErrorKind::Other,
"did not find bank snapshot with this path",
)
})?;
snapshot_utils::package_process_and_archive_full_snapshot( snapshot_utils::package_process_and_archive_full_snapshot(
bank, bank,
&bank_snapshot_info, &bank_snapshot_info,
&snapshot_config.snapshot_path, &snapshot_config.bank_snapshots_dir,
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_archives_dir,
bank.get_snapshot_storages(None), bank.get_snapshot_storages(None),
snapshot_config.archive_format, snapshot_config.archive_format,
snapshot_config.snapshot_version, snapshot_config.snapshot_version,
@ -783,10 +789,16 @@ mod tests {
"Making incremental snapshot archive from bank at slot: {}, and base slot: {}", "Making incremental snapshot archive from bank at slot: {}, and base slot: {}",
slot, incremental_snapshot_base_slot, slot, incremental_snapshot_base_slot,
); );
let bank_snapshot_info = snapshot_utils::get_bank_snapshots(&snapshot_config.snapshot_path) let bank_snapshot_info =
.into_iter() snapshot_utils::get_bank_snapshots(&snapshot_config.bank_snapshots_dir)
.find(|elem| elem.slot == slot) .into_iter()
.ok_or_else(|| Error::new(ErrorKind::Other, "did not find snapshot with this path"))?; .find(|elem| elem.slot == slot)
.ok_or_else(|| {
Error::new(
ErrorKind::Other,
"did not find bank snapshot with this path",
)
})?;
let storages = { let storages = {
let mut storages = bank.get_snapshot_storages(Some(incremental_snapshot_base_slot)); let mut storages = bank.get_snapshot_storages(Some(incremental_snapshot_base_slot));
snapshot_utils::filter_snapshot_storages_for_incremental_snapshot( snapshot_utils::filter_snapshot_storages_for_incremental_snapshot(
@ -799,8 +811,8 @@ mod tests {
bank, bank,
incremental_snapshot_base_slot, incremental_snapshot_base_slot,
&bank_snapshot_info, &bank_snapshot_info,
&snapshot_config.snapshot_path, &snapshot_config.bank_snapshots_dir,
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_archives_dir,
storages, storages,
snapshot_config.archive_format, snapshot_config.archive_format,
snapshot_config.snapshot_version, snapshot_config.snapshot_version,
@ -818,8 +830,8 @@ mod tests {
genesis_config: &GenesisConfig, genesis_config: &GenesisConfig,
) -> snapshot_utils::Result<()> { ) -> snapshot_utils::Result<()> {
let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives( let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives(
&snapshot_config.snapshot_path, &snapshot_config.bank_snapshots_dir,
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_archives_dir,
&[accounts_dir], &[accounts_dir],
&[], &[],
genesis_config, genesis_config,
@ -996,10 +1008,8 @@ mod tests {
info!("Awake! Rebuilding bank from latest snapshot archives..."); info!("Awake! Rebuilding bank from latest snapshot archives...");
let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives( let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives(
&snapshot_test_config.snapshot_config.snapshot_path, &snapshot_test_config.snapshot_config.bank_snapshots_dir,
&snapshot_test_config &snapshot_test_config.snapshot_config.snapshot_archives_dir,
.snapshot_config
.snapshot_package_output_path,
&[snapshot_test_config.accounts_dir.as_ref().to_path_buf()], &[snapshot_test_config.accounts_dir.as_ref().to_path_buf()],
&[], &[],
&snapshot_test_config.genesis_config_info.genesis_config, &snapshot_test_config.genesis_config_info.genesis_config,

View File

@ -699,7 +699,7 @@ fn load_bank_forks(
process_options: ProcessOptions, process_options: ProcessOptions,
snapshot_archive_path: Option<PathBuf>, snapshot_archive_path: Option<PathBuf>,
) -> bank_forks_utils::LoadResult { ) -> bank_forks_utils::LoadResult {
let snapshot_path = blockstore let bank_snapshots_dir = blockstore
.ledger_path() .ledger_path()
.join(if blockstore.is_primary_access() { .join(if blockstore.is_primary_access() {
"snapshot" "snapshot"
@ -709,13 +709,13 @@ fn load_bank_forks(
let snapshot_config = if arg_matches.is_present("no_snapshot") { let snapshot_config = if arg_matches.is_present("no_snapshot") {
None None
} else { } else {
let snapshot_package_output_path = let snapshot_archives_dir =
snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf()); snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf());
Some(SnapshotConfig { Some(SnapshotConfig {
full_snapshot_archive_interval_slots: 0, // Value doesn't matter full_snapshot_archive_interval_slots: 0, // Value doesn't matter
incremental_snapshot_archive_interval_slots: Slot::MAX, incremental_snapshot_archive_interval_slots: Slot::MAX,
snapshot_package_output_path, snapshot_archives_dir,
snapshot_path, bank_snapshots_dir,
archive_format: ArchiveFormat::TarBzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,

View File

@ -42,15 +42,15 @@ pub fn load(
) -> LoadResult { ) -> LoadResult {
if let Some(snapshot_config) = snapshot_config { if let Some(snapshot_config) = snapshot_config {
info!( info!(
"Initializing snapshot path: {}", "Initializing bank snapshot path: {}",
snapshot_config.snapshot_path.display() snapshot_config.bank_snapshots_dir.display()
); );
let _ = fs::remove_dir_all(&snapshot_config.snapshot_path); let _ = fs::remove_dir_all(&snapshot_config.bank_snapshots_dir);
fs::create_dir_all(&snapshot_config.snapshot_path) fs::create_dir_all(&snapshot_config.bank_snapshots_dir)
.expect("Couldn't create snapshot directory"); .expect("Couldn't create snapshot directory");
if snapshot_utils::get_highest_full_snapshot_archive_info( if snapshot_utils::get_highest_full_snapshot_archive_info(
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_archives_dir,
) )
.is_some() .is_some()
{ {
@ -118,8 +118,8 @@ fn load_from_snapshot(
} }
let (deserialized_bank, timings) = snapshot_utils::bank_from_latest_snapshot_archives( let (deserialized_bank, timings) = snapshot_utils::bank_from_latest_snapshot_archives(
&snapshot_config.snapshot_path, &snapshot_config.bank_snapshots_dir,
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_archives_dir,
&account_paths, &account_paths,
&process_options.frozen_accounts, &process_options.frozen_accounts,
genesis_config, genesis_config,

View File

@ -1555,16 +1555,16 @@ fn test_frozen_account_from_snapshot() {
}; };
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified); let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
let snapshot_package_output_path = &snapshot_test_config let snapshot_archives_dir = &snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_package_output_path; .snapshot_archives_dir;
trace!("Waiting for snapshot at {:?}", snapshot_package_output_path); trace!("Waiting for snapshot at {:?}", snapshot_archives_dir);
let (archive_filename, _archive_snapshot_hash) = let (archive_filename, _archive_snapshot_hash) =
wait_for_next_snapshot(&cluster, snapshot_package_output_path); wait_for_next_snapshot(&cluster, snapshot_archives_dir);
trace!("Found snapshot: {:?}", archive_filename); trace!("Found snapshot: {:?}", archive_filename);
@ -1709,16 +1709,16 @@ fn test_snapshot_download() {
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified); let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
// Get slot after which this was generated // Get slot after which this was generated
let snapshot_package_output_path = &leader_snapshot_test_config let snapshot_archives_dir = &leader_snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_package_output_path; .snapshot_archives_dir;
trace!("Waiting for snapshot"); trace!("Waiting for snapshot");
let (archive_filename, archive_snapshot_hash) = let (archive_filename, archive_snapshot_hash) =
wait_for_next_snapshot(&cluster, snapshot_package_output_path); wait_for_next_snapshot(&cluster, snapshot_archives_dir);
trace!("found: {:?}", archive_filename); trace!("found: {:?}", archive_filename);
// Download the snapshot, then boot a validator from it. // Download the snapshot, then boot a validator from it.
@ -1777,15 +1777,15 @@ fn test_snapshot_restart_tower() {
let validator_info = cluster.exit_node(&validator_id); let validator_info = cluster.exit_node(&validator_id);
// Get slot after which this was generated // Get slot after which this was generated
let snapshot_package_output_path = &leader_snapshot_test_config let snapshot_archives_dir = &leader_snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_package_output_path; .snapshot_archives_dir;
let (archive_filename, archive_snapshot_hash) = let (archive_filename, archive_snapshot_hash) =
wait_for_next_snapshot(&cluster, snapshot_package_output_path); wait_for_next_snapshot(&cluster, snapshot_archives_dir);
// Copy archive to validator's snapshot output directory // Copy archive to validator's snapshot output directory
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path( let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
@ -1829,12 +1829,12 @@ fn test_snapshots_blockstore_floor() {
let mut validator_snapshot_test_config = let mut validator_snapshot_test_config =
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths); setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
let snapshot_package_output_path = &leader_snapshot_test_config let snapshot_archives_dir = &leader_snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_package_output_path; .snapshot_archives_dir;
let mut config = ClusterConfig { let mut config = ClusterConfig {
node_stakes: vec![10000], node_stakes: vec![10000],
@ -1852,7 +1852,7 @@ fn test_snapshots_blockstore_floor() {
let archive_info = loop { let archive_info = loop {
let archive = let archive =
snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_package_output_path); snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir);
if archive.is_some() { if archive.is_some() {
trace!("snapshot exists"); trace!("snapshot exists");
break archive.unwrap(); break archive.unwrap();
@ -1934,12 +1934,12 @@ fn test_snapshots_restart_validity() {
let num_account_paths = 1; let num_account_paths = 1;
let mut snapshot_test_config = let mut snapshot_test_config =
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths); setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
let snapshot_package_output_path = &snapshot_test_config let snapshot_archives_dir = &snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_package_output_path; .snapshot_archives_dir;
// Set up the cluster with 1 snapshotting validator // Set up the cluster with 1 snapshotting validator
let mut all_account_storage_dirs = vec![vec![]]; let mut all_account_storage_dirs = vec![vec![]];
@ -1977,7 +1977,7 @@ fn test_snapshots_restart_validity() {
expected_balances.extend(new_balances); expected_balances.extend(new_balances);
wait_for_next_snapshot(&cluster, snapshot_package_output_path); wait_for_next_snapshot(&cluster, snapshot_archives_dir);
// Create new account paths since validator exit is not guaranteed to cleanup RPC threads, // Create new account paths since validator exit is not guaranteed to cleanup RPC threads,
// which may delete the old accounts on exit at any point // which may delete the old accounts on exit at any point
@ -3403,7 +3403,7 @@ fn run_test_load_program_accounts(scan_commitment: CommitmentConfig) {
fn wait_for_next_snapshot( fn wait_for_next_snapshot(
cluster: &LocalCluster, cluster: &LocalCluster,
snapshot_package_output_path: &Path, snapshot_archives_dir: &Path,
) -> (PathBuf, (Slot, Hash)) { ) -> (PathBuf, (Slot, Hash)) {
// Get slot after which this was generated // Get slot after which this was generated
let client = cluster let client = cluster
@ -3421,7 +3421,7 @@ fn wait_for_next_snapshot(
); );
loop { loop {
if let Some(full_snapshot_archive_info) = if let Some(full_snapshot_archive_info) =
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_package_output_path) snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_archives_dir)
{ {
trace!( trace!(
"full snapshot for slot {} exists", "full snapshot for slot {} exists",
@ -3475,13 +3475,13 @@ fn setup_snapshot_validator_config(
num_account_paths: usize, num_account_paths: usize,
) -> SnapshotValidatorConfig { ) -> SnapshotValidatorConfig {
// Create the snapshot config // Create the snapshot config
let snapshot_dir = tempfile::tempdir_in(farf_dir()).unwrap(); let bank_snapshots_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap(); let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let snapshot_config = SnapshotConfig { let snapshot_config = SnapshotConfig {
full_snapshot_archive_interval_slots: snapshot_interval_slots, full_snapshot_archive_interval_slots: snapshot_interval_slots,
incremental_snapshot_archive_interval_slots: Slot::MAX, incremental_snapshot_archive_interval_slots: Slot::MAX,
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(), snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(),
snapshot_path: snapshot_dir.path().to_path_buf(), bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
archive_format: ArchiveFormat::TarBzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: snapshot_utils::SnapshotVersion::default(), snapshot_version: snapshot_utils::SnapshotVersion::default(),
maximum_snapshots_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, maximum_snapshots_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -3500,7 +3500,7 @@ fn setup_snapshot_validator_config(
}; };
SnapshotValidatorConfig { SnapshotValidatorConfig {
_snapshot_dir: snapshot_dir, _snapshot_dir: bank_snapshots_dir,
snapshot_archives_dir, snapshot_archives_dir,
account_storage_dirs, account_storage_dirs,
validator_config, validator_config,

View File

@ -278,12 +278,12 @@ pub fn main() {
); );
let ledger_path = PathBuf::from(matches.value_of("ledger_path").unwrap()); let ledger_path = PathBuf::from(matches.value_of("ledger_path").unwrap());
let snapshot_output_dir = if let Some(snapshots) = matches.value_of("snapshots") { let snapshot_archives_dir = if let Some(snapshots) = matches.value_of("snapshots") {
PathBuf::from(snapshots) PathBuf::from(snapshots)
} else { } else {
ledger_path.clone() ledger_path.clone()
}; };
let snapshot_path = snapshot_output_dir.join("snapshot"); let bank_snapshots_dir = snapshot_archives_dir.join("snapshot");
let account_paths: Vec<PathBuf> = let account_paths: Vec<PathBuf> =
if let Ok(account_paths) = values_t!(matches, "account_paths", String) { if let Ok(account_paths) = values_t!(matches, "account_paths", String) {
@ -348,7 +348,7 @@ pub fn main() {
&node, &node,
expected_shred_version, expected_shred_version,
&peer_pubkey, &peer_pubkey,
&snapshot_output_dir, &snapshot_archives_dir,
socket_addr_space, socket_addr_space,
); );
@ -362,8 +362,8 @@ pub fn main() {
rpc_addr: rpc_addrs.0, rpc_addr: rpc_addrs.0,
rpc_pubsub_addr: rpc_addrs.1, rpc_pubsub_addr: rpc_addrs.1,
ledger_path, ledger_path,
snapshot_output_dir, snapshot_archives_dir,
snapshot_path, bank_snapshots_dir,
account_paths, account_paths,
snapshot_info: snapshot_info.unwrap(), snapshot_info: snapshot_info.unwrap(),
cluster_info, cluster_info,

View File

@ -44,8 +44,8 @@ pub struct ReplicaNodeConfig {
pub rpc_addr: SocketAddr, pub rpc_addr: SocketAddr,
pub rpc_pubsub_addr: SocketAddr, pub rpc_pubsub_addr: SocketAddr,
pub ledger_path: PathBuf, pub ledger_path: PathBuf,
pub snapshot_output_dir: PathBuf, pub snapshot_archives_dir: PathBuf,
pub snapshot_path: PathBuf, pub bank_snapshots_dir: PathBuf,
pub account_paths: Vec<PathBuf>, pub account_paths: Vec<PathBuf>,
pub snapshot_info: (Slot, Hash), pub snapshot_info: (Slot, Hash),
pub cluster_info: Arc<ClusterInfo>, pub cluster_info: Arc<ClusterInfo>,
@ -82,12 +82,12 @@ fn initialize_from_snapshot(
) -> ReplicaBankInfo { ) -> ReplicaBankInfo {
info!( info!(
"Downloading snapshot from the peer into {:?}", "Downloading snapshot from the peer into {:?}",
replica_config.snapshot_output_dir replica_config.snapshot_archives_dir
); );
download_snapshot( download_snapshot(
&replica_config.rpc_source_addr, &replica_config.rpc_source_addr,
&replica_config.snapshot_output_dir, &replica_config.snapshot_archives_dir,
replica_config.snapshot_info, replica_config.snapshot_info,
false, false,
snapshot_config.maximum_snapshots_to_retain, snapshot_config.maximum_snapshots_to_retain,
@ -95,11 +95,13 @@ fn initialize_from_snapshot(
) )
.unwrap(); .unwrap();
fs::create_dir_all(&snapshot_config.snapshot_path).expect("Couldn't create snapshot directory"); fs::create_dir_all(&snapshot_config.bank_snapshots_dir)
.expect("Couldn't create bank snapshot directory");
let archive_info = let archive_info = snapshot_utils::get_highest_full_snapshot_archive_info(
snapshot_utils::get_highest_full_snapshot_archive_info(&replica_config.snapshot_output_dir) &replica_config.snapshot_archives_dir,
.unwrap(); )
.unwrap();
let process_options = blockstore_processor::ProcessOptions { let process_options = blockstore_processor::ProcessOptions {
account_indexes: replica_config.account_indexes.clone(), account_indexes: replica_config.account_indexes.clone(),
@ -109,12 +111,12 @@ fn initialize_from_snapshot(
info!( info!(
"Build bank from snapshot archive: {:?}", "Build bank from snapshot archive: {:?}",
&snapshot_config.snapshot_path &snapshot_config.bank_snapshots_dir
); );
let (bank0, _) = snapshot_utils::bank_from_snapshot_archives( let (bank0, _) = snapshot_utils::bank_from_snapshot_archives(
&replica_config.account_paths, &replica_config.account_paths,
&[], &[],
&snapshot_config.snapshot_path, &snapshot_config.bank_snapshots_dir,
&archive_info, &archive_info,
None, None,
genesis_config, genesis_config,
@ -257,8 +259,8 @@ impl ReplicaNode {
let snapshot_config = SnapshotConfig { let snapshot_config = SnapshotConfig {
full_snapshot_archive_interval_slots: std::u64::MAX, full_snapshot_archive_interval_slots: std::u64::MAX,
incremental_snapshot_archive_interval_slots: std::u64::MAX, incremental_snapshot_archive_interval_slots: std::u64::MAX,
snapshot_package_output_path: replica_config.snapshot_output_dir.clone(), snapshot_archives_dir: replica_config.snapshot_archives_dir.clone(),
snapshot_path: replica_config.snapshot_path.clone(), bank_snapshots_dir: replica_config.bank_snapshots_dir.clone(),
archive_format: ArchiveFormat::TarBzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: snapshot_utils::SnapshotVersion::default(), snapshot_version: snapshot_utils::SnapshotVersion::default(),
maximum_snapshots_to_retain: maximum_snapshots_to_retain:

View File

@ -55,7 +55,7 @@ fn get_rpc_peer_node(
cluster_entrypoints: &[ContactInfo], cluster_entrypoints: &[ContactInfo],
expected_shred_version: Option<u16>, expected_shred_version: Option<u16>,
peer_pubkey: &Pubkey, peer_pubkey: &Pubkey,
snapshot_output_dir: &Path, snapshot_archives_dir: &Path,
) -> Option<(ContactInfo, Option<(Slot, Hash)>)> { ) -> Option<(ContactInfo, Option<(Slot, Hash)>)> {
let mut newer_cluster_snapshot_timeout = None; let mut newer_cluster_snapshot_timeout = None;
let mut retry_reason = None; let mut retry_reason = None;
@ -110,7 +110,7 @@ fn get_rpc_peer_node(
); );
let mut highest_snapshot_info: Option<(Slot, Hash)> = let mut highest_snapshot_info: Option<(Slot, Hash)> =
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_output_dir).map( snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_archives_dir).map(
|snapshot_archive_info| { |snapshot_archive_info| {
(snapshot_archive_info.slot(), *snapshot_archive_info.hash()) (snapshot_archive_info.slot(), *snapshot_archive_info.hash())
}, },
@ -238,7 +238,7 @@ pub fn get_rpc_peer_info(
node: &Node, node: &Node,
expected_shred_version: Option<u16>, expected_shred_version: Option<u16>,
peer_pubkey: &Pubkey, peer_pubkey: &Pubkey,
snapshot_output_dir: &Path, snapshot_archives_dir: &Path,
socket_addr_space: SocketAddrSpace, socket_addr_space: SocketAddrSpace,
) -> (Arc<ClusterInfo>, ContactInfo, Option<(Slot, Hash)>) { ) -> (Arc<ClusterInfo>, ContactInfo, Option<(Slot, Hash)>) {
let identity_keypair = Arc::new(identity_keypair); let identity_keypair = Arc::new(identity_keypair);
@ -260,7 +260,7 @@ pub fn get_rpc_peer_info(
cluster_entrypoints, cluster_entrypoints,
expected_shred_version, expected_shred_version,
peer_pubkey, peer_pubkey,
snapshot_output_dir, snapshot_archives_dir,
); );
let rpc_node_details = rpc_node_details.unwrap(); let rpc_node_details = rpc_node_details.unwrap();

View File

@ -45,7 +45,7 @@ const RUST_LOG_FILTER: &str =
fn wait_for_next_snapshot( fn wait_for_next_snapshot(
cluster: &LocalCluster, cluster: &LocalCluster,
snapshot_package_output_path: &Path, snapshot_archives_dir: &Path,
) -> (PathBuf, (Slot, Hash)) { ) -> (PathBuf, (Slot, Hash)) {
// Get slot after which this was generated // Get slot after which this was generated
let client = cluster let client = cluster
@ -63,7 +63,7 @@ fn wait_for_next_snapshot(
); );
loop { loop {
if let Some(full_snapshot_archive_info) = if let Some(full_snapshot_archive_info) =
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_package_output_path) snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_archives_dir)
{ {
trace!( trace!(
"full snapshot for slot {} exists", "full snapshot for slot {} exists",
@ -117,13 +117,13 @@ fn setup_snapshot_validator_config(
num_account_paths: usize, num_account_paths: usize,
) -> SnapshotValidatorConfig { ) -> SnapshotValidatorConfig {
// Create the snapshot config // Create the snapshot config
let snapshot_dir = tempfile::tempdir_in(farf_dir()).unwrap(); let bank_snapshots_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap(); let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let snapshot_config = SnapshotConfig { let snapshot_config = SnapshotConfig {
full_snapshot_archive_interval_slots: snapshot_interval_slots, full_snapshot_archive_interval_slots: snapshot_interval_slots,
incremental_snapshot_archive_interval_slots: Slot::MAX, incremental_snapshot_archive_interval_slots: Slot::MAX,
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(), snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(),
snapshot_path: snapshot_dir.path().to_path_buf(), bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
archive_format: ArchiveFormat::TarBzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: snapshot_utils::SnapshotVersion::default(), snapshot_version: snapshot_utils::SnapshotVersion::default(),
maximum_snapshots_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, maximum_snapshots_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
@ -142,7 +142,7 @@ fn setup_snapshot_validator_config(
}; };
SnapshotValidatorConfig { SnapshotValidatorConfig {
_snapshot_dir: snapshot_dir, _snapshot_dir: bank_snapshots_dir,
snapshot_archives_dir, snapshot_archives_dir,
account_storage_dirs, account_storage_dirs,
validator_config, validator_config,
@ -204,15 +204,15 @@ fn test_replica_bootstrap() {
info!("Contact info: {:?}", contact_info); info!("Contact info: {:?}", contact_info);
// Get slot after which this was generated // Get slot after which this was generated
let snapshot_package_output_path = &leader_snapshot_test_config let snapshot_archives_dir = &leader_snapshot_test_config
.validator_config .validator_config
.snapshot_config .snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_package_output_path; .snapshot_archives_dir;
info!("Waiting for snapshot"); info!("Waiting for snapshot");
let (archive_filename, archive_snapshot_hash) = let (archive_filename, archive_snapshot_hash) =
wait_for_next_snapshot(&cluster, snapshot_package_output_path); wait_for_next_snapshot(&cluster, snapshot_archives_dir);
info!("found: {:?}", archive_filename); info!("found: {:?}", archive_filename);
let identity_keypair = Keypair::new(); let identity_keypair = Keypair::new();
@ -228,8 +228,8 @@ fn test_replica_bootstrap() {
let ledger_dir = tempfile::tempdir_in(farf_dir()).unwrap(); let ledger_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let ledger_path = ledger_dir.path(); let ledger_path = ledger_dir.path();
let snapshot_output_dir = tempfile::tempdir_in(farf_dir()).unwrap(); let snapshot_output_dir = tempfile::tempdir_in(farf_dir()).unwrap();
let snapshot_output_path = snapshot_output_dir.path(); let snapshot_archives_dir = snapshot_output_dir.path();
let snapshot_path = snapshot_output_path.join("snapshot"); let bank_snapshots_dir = snapshot_archives_dir.join("snapshot");
let account_paths: Vec<PathBuf> = vec![ledger_path.join("accounts")]; let account_paths: Vec<PathBuf> = vec![ledger_path.join("accounts")];
let port = solana_net_utils::find_available_port_in_range(ip_addr, (8301, 8400)).unwrap(); let port = solana_net_utils::find_available_port_in_range(ip_addr, (8301, 8400)).unwrap();
@ -253,7 +253,7 @@ fn test_replica_bootstrap() {
&node, &node,
None, None,
&contact_info.id, &contact_info.id,
snapshot_output_path, snapshot_archives_dir,
socket_addr_space, socket_addr_space,
); );
@ -264,8 +264,8 @@ fn test_replica_bootstrap() {
rpc_addr, rpc_addr,
rpc_pubsub_addr, rpc_pubsub_addr,
ledger_path: ledger_path.to_path_buf(), ledger_path: ledger_path.to_path_buf(),
snapshot_output_dir: snapshot_output_path.to_path_buf(), snapshot_archives_dir: snapshot_archives_dir.to_path_buf(),
snapshot_path, bank_snapshots_dir,
account_paths, account_paths,
snapshot_info: archive_snapshot_hash, snapshot_info: archive_snapshot_hash,
cluster_info, cluster_info,

View File

@ -2369,7 +2369,7 @@ pub mod rpc_minimal {
meta.snapshot_config meta.snapshot_config
.and_then(|snapshot_config| { .and_then(|snapshot_config| {
snapshot_utils::get_highest_full_snapshot_archive_slot( snapshot_utils::get_highest_full_snapshot_archive_slot(
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_archives_dir,
) )
}) })
.ok_or_else(|| RpcCustomError::NoSnapshot.into()) .ok_or_else(|| RpcCustomError::NoSnapshot.into())

View File

@ -149,7 +149,7 @@ impl RpcRequestMiddleware {
self.snapshot_config self.snapshot_config
.as_ref() .as_ref()
.unwrap() .unwrap()
.snapshot_package_output_path .snapshot_archives_dir
.join(stem) .join(stem)
} }
} }
@ -204,7 +204,7 @@ impl RequestMiddleware for RpcRequestMiddleware {
// Convenience redirect to the latest snapshot // Convenience redirect to the latest snapshot
return if let Some(full_snapshot_archive_info) = return if let Some(full_snapshot_archive_info) =
snapshot_utils::get_highest_full_snapshot_archive_info( snapshot_utils::get_highest_full_snapshot_archive_info(
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_archives_dir,
) { ) {
RpcRequestMiddleware::redirect(&format!( RpcRequestMiddleware::redirect(&format!(
"/{}", "/{}",
@ -604,8 +604,8 @@ mod tests {
Some(SnapshotConfig { Some(SnapshotConfig {
full_snapshot_archive_interval_slots: 0, full_snapshot_archive_interval_slots: 0,
incremental_snapshot_archive_interval_slots: u64::MAX, incremental_snapshot_archive_interval_slots: u64::MAX,
snapshot_package_output_path: PathBuf::from("/"), snapshot_archives_dir: PathBuf::from("/"),
snapshot_path: PathBuf::from("/"), bank_snapshots_dir: PathBuf::from("/"),
archive_format: ArchiveFormat::TarBzip2, archive_format: ArchiveFormat::TarBzip2,
snapshot_version: SnapshotVersion::default(), snapshot_version: SnapshotVersion::default(),
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,

View File

@ -175,8 +175,8 @@ impl SnapshotRequestHandler {
&snapshot_root_bank, &snapshot_root_bank,
status_cache_slot_deltas, status_cache_slot_deltas,
&self.accounts_package_sender, &self.accounts_package_sender,
&self.snapshot_config.snapshot_path, &self.snapshot_config.bank_snapshots_dir,
&self.snapshot_config.snapshot_package_output_path, &self.snapshot_config.snapshot_archives_dir,
self.snapshot_config.snapshot_version, self.snapshot_config.snapshot_version,
&self.snapshot_config.archive_format, &self.snapshot_config.archive_format,
hash_for_testing, hash_for_testing,
@ -192,7 +192,7 @@ impl SnapshotRequestHandler {
// Cleanup outdated snapshots // Cleanup outdated snapshots
let mut purge_old_snapshots_time = Measure::start("purge_old_snapshots_time"); let mut purge_old_snapshots_time = Measure::start("purge_old_snapshots_time");
snapshot_utils::purge_old_bank_snapshots(&self.snapshot_config.snapshot_path); snapshot_utils::purge_old_bank_snapshots(&self.snapshot_config.bank_snapshots_dir);
purge_old_snapshots_time.stop(); purge_old_snapshots_time.stop();
total_time.stop(); total_time.stop();

View File

@ -15,11 +15,11 @@ pub struct SnapshotConfig {
/// Generate a new incremental snapshot archive every this many slots /// Generate a new incremental snapshot archive every this many slots
pub incremental_snapshot_archive_interval_slots: Slot, pub incremental_snapshot_archive_interval_slots: Slot,
/// Where to store the latest packaged snapshot archives /// Path to the directory where snapshot archives are stored
pub snapshot_package_output_path: PathBuf, pub snapshot_archives_dir: PathBuf,
/// Where to place the bank snapshots for recent slots /// Path to the directory where bank snapshots are stored
pub snapshot_path: PathBuf, pub bank_snapshots_dir: PathBuf,
/// The archive format to use for snapshots /// The archive format to use for snapshots
pub archive_format: ArchiveFormat, pub archive_format: ArchiveFormat,

View File

@ -46,7 +46,7 @@ pub struct AccountsPackage {
pub hash: Hash, // temporarily here while we still have to calculate hash before serializing bank pub hash: Hash, // temporarily here while we still have to calculate hash before serializing bank
pub archive_format: ArchiveFormat, pub archive_format: ArchiveFormat,
pub snapshot_version: SnapshotVersion, pub snapshot_version: SnapshotVersion,
pub snapshot_output_dir: PathBuf, pub snapshot_archives_dir: PathBuf,
pub expected_capitalization: u64, pub expected_capitalization: u64,
pub hash_for_testing: Option<Hash>, pub hash_for_testing: Option<Hash>,
pub cluster_type: ClusterType, pub cluster_type: ClusterType,
@ -59,7 +59,7 @@ impl AccountsPackage {
bank: &Bank, bank: &Bank,
bank_snapshot_info: &BankSnapshotInfo, bank_snapshot_info: &BankSnapshotInfo,
status_cache_slot_deltas: Vec<BankSlotDelta>, status_cache_slot_deltas: Vec<BankSlotDelta>,
snapshot_package_output_path: impl AsRef<Path>, snapshot_archives_dir: impl AsRef<Path>,
snapshot_storages: SnapshotStorages, snapshot_storages: SnapshotStorages,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
@ -87,7 +87,7 @@ impl AccountsPackage {
hash: bank.get_accounts_hash(), hash: bank.get_accounts_hash(),
archive_format, archive_format,
snapshot_version, snapshot_version,
snapshot_output_dir: snapshot_package_output_path.as_ref().to_path_buf(), snapshot_archives_dir: snapshot_archives_dir.as_ref().to_path_buf(),
expected_capitalization: bank.capitalization(), expected_capitalization: bank.capitalization(),
hash_for_testing, hash_for_testing,
cluster_type: bank.cluster_type(), cluster_type: bank.cluster_type(),
@ -99,9 +99,9 @@ impl AccountsPackage {
pub fn new_for_full_snapshot( pub fn new_for_full_snapshot(
bank: &Bank, bank: &Bank,
bank_snapshot_info: &BankSnapshotInfo, bank_snapshot_info: &BankSnapshotInfo,
snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
status_cache_slot_deltas: Vec<BankSlotDelta>, status_cache_slot_deltas: Vec<BankSlotDelta>,
snapshot_package_output_path: impl AsRef<Path>, snapshot_archives_dir: impl AsRef<Path>,
snapshot_storages: SnapshotStorages, snapshot_storages: SnapshotStorages,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
@ -115,13 +115,13 @@ impl AccountsPackage {
let snapshot_tmpdir = tempfile::Builder::new() let snapshot_tmpdir = tempfile::Builder::new()
.prefix(&format!("{}{}-", TMP_FULL_SNAPSHOT_PREFIX, bank.slot())) .prefix(&format!("{}{}-", TMP_FULL_SNAPSHOT_PREFIX, bank.slot()))
.tempdir_in(snapshots_dir)?; .tempdir_in(bank_snapshots_dir)?;
Self::new( Self::new(
bank, bank,
bank_snapshot_info, bank_snapshot_info,
status_cache_slot_deltas, status_cache_slot_deltas,
snapshot_package_output_path, snapshot_archives_dir,
snapshot_storages, snapshot_storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
@ -136,9 +136,9 @@ impl AccountsPackage {
bank: &Bank, bank: &Bank,
incremental_snapshot_base_slot: Slot, incremental_snapshot_base_slot: Slot,
bank_snapshot_info: &BankSnapshotInfo, bank_snapshot_info: &BankSnapshotInfo,
snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
status_cache_slot_deltas: Vec<BankSlotDelta>, status_cache_slot_deltas: Vec<BankSlotDelta>,
snapshot_package_output_path: impl AsRef<Path>, snapshot_archives_dir: impl AsRef<Path>,
snapshot_storages: SnapshotStorages, snapshot_storages: SnapshotStorages,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
@ -165,13 +165,13 @@ impl AccountsPackage {
incremental_snapshot_base_slot, incremental_snapshot_base_slot,
bank.slot() bank.slot()
)) ))
.tempdir_in(snapshots_dir)?; .tempdir_in(bank_snapshots_dir)?;
Self::new( Self::new(
bank, bank,
bank_snapshot_info, bank_snapshot_info,
status_cache_slot_deltas, status_cache_slot_deltas,
snapshot_package_output_path, snapshot_archives_dir,
snapshot_storages, snapshot_storages,
archive_format, archive_format,
snapshot_version, snapshot_version,

View File

@ -217,7 +217,7 @@ fn get_archive_ext(archive_format: ArchiveFormat) -> &'static str {
/// If the validator halts in the middle of `archive_snapshot_package()`, the temporary staging /// If the validator halts in the middle of `archive_snapshot_package()`, the temporary staging
/// directory won't be cleaned up. Call this function to clean them up. /// directory won't be cleaned up. Call this function to clean them up.
pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: &Path) { pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: impl AsRef<Path>) {
if let Ok(entries) = fs::read_dir(snapshot_archives_dir) { if let Ok(entries) = fs::read_dir(snapshot_archives_dir) {
for entry in entries.filter_map(|entry| entry.ok()) { for entry in entries.filter_map(|entry| entry.ok()) {
let file_name = entry let file_name = entry
@ -393,11 +393,11 @@ pub fn archive_snapshot_package(
} }
/// Get a list of bank snapshots in a directory /// Get a list of bank snapshots in a directory
pub fn get_bank_snapshots<P>(snapshots_dir: P) -> Vec<BankSnapshotInfo> pub fn get_bank_snapshots<P>(bank_snapshots_dir: P) -> Vec<BankSnapshotInfo>
where where
P: AsRef<Path>, P: AsRef<Path>,
{ {
match fs::read_dir(&snapshots_dir) { match fs::read_dir(&bank_snapshots_dir) {
Ok(paths) => paths Ok(paths) => paths
.filter_map(|entry| { .filter_map(|entry| {
entry.ok().and_then(|e| { entry.ok().and_then(|e| {
@ -411,7 +411,7 @@ where
let snapshot_file_name = get_snapshot_file_name(slot); let snapshot_file_name = get_snapshot_file_name(slot);
// So nice I join-ed it twice! The redundant `snapshot_file_name` is unintentional // So nice I join-ed it twice! The redundant `snapshot_file_name` is unintentional
// and should be simplified. Kept for compatibility. // and should be simplified. Kept for compatibility.
let snapshot_path = snapshots_dir let snapshot_path = bank_snapshots_dir
.as_ref() .as_ref()
.join(&snapshot_file_name) .join(&snapshot_file_name)
.join(snapshot_file_name); .join(snapshot_file_name);
@ -424,7 +424,7 @@ where
Err(err) => { Err(err) => {
info!( info!(
"Unable to read snapshots directory {}: {}", "Unable to read snapshots directory {}: {}",
snapshots_dir.as_ref().display(), bank_snapshots_dir.as_ref().display(),
err err
); );
vec![] vec![]
@ -433,11 +433,11 @@ where
} }
/// Get the bank snapshot with the highest slot in a directory /// Get the bank snapshot with the highest slot in a directory
pub fn get_highest_bank_snapshot_info<P>(snapshots_dir: P) -> Option<BankSnapshotInfo> pub fn get_highest_bank_snapshot_info<P>(bank_snapshots_dir: P) -> Option<BankSnapshotInfo>
where where
P: AsRef<Path>, P: AsRef<Path>,
{ {
let mut bank_snapshot_infos = get_bank_snapshots(snapshots_dir); let mut bank_snapshot_infos = get_bank_snapshots(bank_snapshots_dir);
bank_snapshot_infos.sort_unstable(); bank_snapshot_infos.sort_unstable();
bank_snapshot_infos.into_iter().rev().next() bank_snapshot_infos.into_iter().rev().next()
} }
@ -613,17 +613,17 @@ where
/// Serialize a bank to a snapshot /// Serialize a bank to a snapshot
pub fn add_bank_snapshot<P: AsRef<Path>>( pub fn add_bank_snapshot<P: AsRef<Path>>(
snapshots_dir: P, bank_snapshots_dir: P,
bank: &Bank, bank: &Bank,
snapshot_storages: &[SnapshotStorage], snapshot_storages: &[SnapshotStorage],
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
) -> Result<BankSnapshotInfo> { ) -> Result<BankSnapshotInfo> {
let slot = bank.slot(); let slot = bank.slot();
// snapshots_dir/slot // bank_snapshots_dir/slot
let bank_snapshots_dir = get_bank_snapshots_dir(snapshots_dir, slot); let bank_snapshots_dir = get_bank_snapshots_dir(bank_snapshots_dir, slot);
fs::create_dir_all(&bank_snapshots_dir)?; fs::create_dir_all(&bank_snapshots_dir)?;
// the bank snapshot is stored as snapshots_dir/slot/slot // the bank snapshot is stored as bank_snapshots_dir/slot/slot
let snapshot_bank_file_path = bank_snapshots_dir.join(get_snapshot_file_name(slot)); let snapshot_bank_file_path = bank_snapshots_dir.join(get_snapshot_file_name(slot));
info!( info!(
"Creating snapshot for slot {}, path: {:?}", "Creating snapshot for slot {}, path: {:?}",
@ -689,11 +689,11 @@ fn serialize_status_cache(
} }
/// Remove the snapshot directory for this slot /// Remove the snapshot directory for this slot
pub fn remove_bank_snapshot<P>(slot: Slot, snapshots_dir: P) -> Result<()> pub fn remove_bank_snapshot<P>(slot: Slot, bank_snapshots_dir: P) -> Result<()>
where where
P: AsRef<Path>, P: AsRef<Path>,
{ {
let bank_snapshot_dir = get_bank_snapshots_dir(&snapshots_dir, slot); let bank_snapshot_dir = get_bank_snapshots_dir(&bank_snapshots_dir, slot);
fs::remove_dir_all(bank_snapshot_dir)?; fs::remove_dir_all(bank_snapshot_dir)?;
Ok(()) Ok(())
} }
@ -715,7 +715,7 @@ const PARALLEL_UNTAR_READERS_DEFAULT: usize = 4;
pub fn bank_from_snapshot_archives( pub fn bank_from_snapshot_archives(
account_paths: &[PathBuf], account_paths: &[PathBuf],
frozen_account_pubkeys: &[Pubkey], frozen_account_pubkeys: &[Pubkey],
snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
full_snapshot_archive_info: &FullSnapshotArchiveInfo, full_snapshot_archive_info: &FullSnapshotArchiveInfo,
incremental_snapshot_archive_info: Option<&IncrementalSnapshotArchiveInfo>, incremental_snapshot_archive_info: Option<&IncrementalSnapshotArchiveInfo>,
genesis_config: &GenesisConfig, genesis_config: &GenesisConfig,
@ -741,7 +741,7 @@ pub fn bank_from_snapshot_archives(
); );
let unarchived_full_snapshot = unarchive_snapshot( let unarchived_full_snapshot = unarchive_snapshot(
&snapshots_dir, &bank_snapshots_dir,
TMP_FULL_SNAPSHOT_PREFIX, TMP_FULL_SNAPSHOT_PREFIX,
full_snapshot_archive_info.path(), full_snapshot_archive_info.path(),
"snapshot untar", "snapshot untar",
@ -753,7 +753,7 @@ pub fn bank_from_snapshot_archives(
let mut unarchived_incremental_snapshot = let mut unarchived_incremental_snapshot =
if let Some(incremental_snapshot_archive_info) = incremental_snapshot_archive_info { if let Some(incremental_snapshot_archive_info) = incremental_snapshot_archive_info {
let unarchived_incremental_snapshot = unarchive_snapshot( let unarchived_incremental_snapshot = unarchive_snapshot(
&snapshots_dir, &bank_snapshots_dir,
TMP_INCREMENTAL_SNAPSHOT_PREFIX, TMP_INCREMENTAL_SNAPSHOT_PREFIX,
incremental_snapshot_archive_info.path(), incremental_snapshot_archive_info.path(),
"incremental snapshot untar", "incremental snapshot untar",
@ -824,7 +824,7 @@ pub fn bank_from_snapshot_archives(
/// highest full snapshot and highest corresponding incremental snapshot, then rebuilds the bank. /// highest full snapshot and highest corresponding incremental snapshot, then rebuilds the bank.
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn bank_from_latest_snapshot_archives( pub fn bank_from_latest_snapshot_archives(
snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
snapshot_archives_dir: impl AsRef<Path>, snapshot_archives_dir: impl AsRef<Path>,
account_paths: &[PathBuf], account_paths: &[PathBuf],
frozen_account_pubkeys: &[Pubkey], frozen_account_pubkeys: &[Pubkey],
@ -863,7 +863,7 @@ pub fn bank_from_latest_snapshot_archives(
let (bank, timings) = bank_from_snapshot_archives( let (bank, timings) = bank_from_snapshot_archives(
account_paths, account_paths,
frozen_account_pubkeys, frozen_account_pubkeys,
snapshots_dir.as_ref(), bank_snapshots_dir.as_ref(),
&full_snapshot_archive_info, &full_snapshot_archive_info,
incremental_snapshot_archive_info.as_ref(), incremental_snapshot_archive_info.as_ref(),
genesis_config, genesis_config,
@ -918,7 +918,7 @@ fn verify_bank_against_expected_slot_hash(
/// directories, untaring, reading the version file, and then returning those fields plus the /// directories, untaring, reading the version file, and then returning those fields plus the
/// unpacked append vec map. /// unpacked append vec map.
fn unarchive_snapshot<P, Q>( fn unarchive_snapshot<P, Q>(
snapshots_dir: P, bank_snapshots_dir: P,
unpacked_snapshots_dir_prefix: &'static str, unpacked_snapshots_dir_prefix: &'static str,
snapshot_archive_path: Q, snapshot_archive_path: Q,
measure_name: &'static str, measure_name: &'static str,
@ -932,7 +932,7 @@ where
{ {
let unpack_dir = tempfile::Builder::new() let unpack_dir = tempfile::Builder::new()
.prefix(unpacked_snapshots_dir_prefix) .prefix(unpacked_snapshots_dir_prefix)
.tempdir_in(snapshots_dir)?; .tempdir_in(bank_snapshots_dir)?;
let unpacked_snapshots_dir = unpack_dir.path().join("snapshots"); let unpacked_snapshots_dir = unpack_dir.path().join("snapshots");
let mut measure_untar = Measure::start(measure_name); let mut measure_untar = Measure::start(measure_name);
@ -1509,18 +1509,18 @@ pub fn verify_snapshot_archive<P, Q, R>(
} }
/// Remove outdated bank snapshots /// Remove outdated bank snapshots
pub fn purge_old_bank_snapshots<P>(snapshots_dir: P) pub fn purge_old_bank_snapshots<P>(bank_snapshots_dir: P)
where where
P: AsRef<Path>, P: AsRef<Path>,
{ {
let mut bank_snapshot_infos = get_bank_snapshots(&snapshots_dir); let mut bank_snapshot_infos = get_bank_snapshots(&bank_snapshots_dir);
bank_snapshot_infos.sort_unstable(); bank_snapshot_infos.sort_unstable();
bank_snapshot_infos bank_snapshot_infos
.into_iter() .into_iter()
.rev() .rev()
.skip(MAX_BANK_SNAPSHOTS) .skip(MAX_BANK_SNAPSHOTS)
.for_each(|bank_snapshot_info| { .for_each(|bank_snapshot_info| {
let r = remove_bank_snapshot(bank_snapshot_info.slot, &snapshots_dir); let r = remove_bank_snapshot(bank_snapshot_info.slot, &bank_snapshots_dir);
if r.is_err() { if r.is_err() {
warn!( warn!(
"Couldn't remove snapshot at: {}", "Couldn't remove snapshot at: {}",
@ -1535,28 +1535,28 @@ pub fn snapshot_bank(
root_bank: &Bank, root_bank: &Bank,
status_cache_slot_deltas: Vec<BankSlotDelta>, status_cache_slot_deltas: Vec<BankSlotDelta>,
accounts_package_sender: &AccountsPackageSender, accounts_package_sender: &AccountsPackageSender,
snapshots_dir: &Path, bank_snapshots_dir: impl AsRef<Path>,
snapshot_package_output_path: &Path, snapshot_archives_dir: impl AsRef<Path>,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
archive_format: &ArchiveFormat, archive_format: &ArchiveFormat,
hash_for_testing: Option<Hash>, hash_for_testing: Option<Hash>,
) -> Result<()> { ) -> Result<()> {
let storages = root_bank.get_snapshot_storages(None); let storages = root_bank.get_snapshot_storages(None);
let mut add_snapshot_time = Measure::start("add-snapshot-ms"); let mut add_snapshot_time = Measure::start("add-snapshot-ms");
add_bank_snapshot(snapshots_dir, root_bank, &storages, snapshot_version)?; add_bank_snapshot(&bank_snapshots_dir, root_bank, &storages, snapshot_version)?;
add_snapshot_time.stop(); add_snapshot_time.stop();
inc_new_counter_info!("add-snapshot-ms", add_snapshot_time.as_ms() as usize); inc_new_counter_info!("add-snapshot-ms", add_snapshot_time.as_ms() as usize);
// Package the relevant snapshots // Package the relevant snapshots
let highest_bank_snapshot_info = get_highest_bank_snapshot_info(snapshots_dir) let highest_bank_snapshot_info = get_highest_bank_snapshot_info(&bank_snapshots_dir)
.expect("no snapshots found in config snapshots_dir"); .expect("no snapshots found in config bank_snapshots_dir");
let accounts_package = AccountsPackage::new_for_full_snapshot( let accounts_package = AccountsPackage::new_for_full_snapshot(
root_bank, root_bank,
&highest_bank_snapshot_info, &highest_bank_snapshot_info,
snapshots_dir, bank_snapshots_dir,
status_cache_slot_deltas, status_cache_slot_deltas,
snapshot_package_output_path, snapshot_archives_dir,
storages, storages,
*archive_format, *archive_format,
snapshot_version, snapshot_version,
@ -1574,10 +1574,10 @@ pub fn snapshot_bank(
/// Requires: /// Requires:
/// - `bank` is complete /// - `bank` is complete
pub fn bank_to_full_snapshot_archive( pub fn bank_to_full_snapshot_archive(
snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
bank: &Bank, bank: &Bank,
snapshot_version: Option<SnapshotVersion>, snapshot_version: Option<SnapshotVersion>,
snapshot_package_output_path: impl AsRef<Path>, snapshot_archives_dir: impl AsRef<Path>,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
thread_pool: Option<&ThreadPool>, thread_pool: Option<&ThreadPool>,
maximum_snapshots_to_retain: usize, maximum_snapshots_to_retain: usize,
@ -1591,7 +1591,7 @@ pub fn bank_to_full_snapshot_archive(
bank.update_accounts_hash(); bank.update_accounts_hash();
bank.rehash(); // Bank accounts may have been manually modified by the caller bank.rehash(); // Bank accounts may have been manually modified by the caller
let temp_dir = tempfile::tempdir_in(snapshots_dir)?; let temp_dir = tempfile::tempdir_in(bank_snapshots_dir)?;
let storages = bank.get_snapshot_storages(None); let storages = bank.get_snapshot_storages(None);
let bank_snapshot_info = add_bank_snapshot(&temp_dir, bank, &storages, snapshot_version)?; let bank_snapshot_info = add_bank_snapshot(&temp_dir, bank, &storages, snapshot_version)?;
@ -1599,7 +1599,7 @@ pub fn bank_to_full_snapshot_archive(
bank, bank,
&bank_snapshot_info, &bank_snapshot_info,
&temp_dir, &temp_dir,
snapshot_package_output_path, snapshot_archives_dir,
storages, storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
@ -1615,11 +1615,11 @@ pub fn bank_to_full_snapshot_archive(
/// - `bank` is complete /// - `bank` is complete
/// - `bank`'s slot is greater than `full_snapshot_slot` /// - `bank`'s slot is greater than `full_snapshot_slot`
pub fn bank_to_incremental_snapshot_archive( pub fn bank_to_incremental_snapshot_archive(
snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
bank: &Bank, bank: &Bank,
full_snapshot_slot: Slot, full_snapshot_slot: Slot,
snapshot_version: Option<SnapshotVersion>, snapshot_version: Option<SnapshotVersion>,
snapshot_package_output_path: impl AsRef<Path>, snapshot_archives_dir: impl AsRef<Path>,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
thread_pool: Option<&ThreadPool>, thread_pool: Option<&ThreadPool>,
maximum_snapshots_to_retain: usize, maximum_snapshots_to_retain: usize,
@ -1634,7 +1634,7 @@ pub fn bank_to_incremental_snapshot_archive(
bank.update_accounts_hash(); bank.update_accounts_hash();
bank.rehash(); // Bank accounts may have been manually modified by the caller bank.rehash(); // Bank accounts may have been manually modified by the caller
let temp_dir = tempfile::tempdir_in(snapshots_dir)?; let temp_dir = tempfile::tempdir_in(bank_snapshots_dir)?;
let storages = { let storages = {
let mut storages = bank.get_snapshot_storages(Some(full_snapshot_slot)); let mut storages = bank.get_snapshot_storages(Some(full_snapshot_slot));
filter_snapshot_storages_for_incremental_snapshot(&mut storages, full_snapshot_slot); filter_snapshot_storages_for_incremental_snapshot(&mut storages, full_snapshot_slot);
@ -1647,7 +1647,7 @@ pub fn bank_to_incremental_snapshot_archive(
full_snapshot_slot, full_snapshot_slot,
&bank_snapshot_info, &bank_snapshot_info,
&temp_dir, &temp_dir,
snapshot_package_output_path, snapshot_archives_dir,
storages, storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
@ -1660,8 +1660,8 @@ pub fn bank_to_incremental_snapshot_archive(
pub fn package_process_and_archive_full_snapshot( pub fn package_process_and_archive_full_snapshot(
bank: &Bank, bank: &Bank,
bank_snapshot_info: &BankSnapshotInfo, bank_snapshot_info: &BankSnapshotInfo,
snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
snapshot_package_output_path: impl AsRef<Path>, snapshot_archives_dir: impl AsRef<Path>,
snapshot_storages: SnapshotStorages, snapshot_storages: SnapshotStorages,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
@ -1671,9 +1671,9 @@ pub fn package_process_and_archive_full_snapshot(
let accounts_package = AccountsPackage::new_for_full_snapshot( let accounts_package = AccountsPackage::new_for_full_snapshot(
bank, bank,
bank_snapshot_info, bank_snapshot_info,
snapshots_dir, bank_snapshots_dir,
bank.src.slot_deltas(&bank.src.roots()), bank.src.slot_deltas(&bank.src.roots()),
snapshot_package_output_path, snapshot_archives_dir,
snapshot_storages, snapshot_storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
@ -1698,8 +1698,8 @@ pub fn package_process_and_archive_incremental_snapshot(
bank: &Bank, bank: &Bank,
incremental_snapshot_base_slot: Slot, incremental_snapshot_base_slot: Slot,
bank_snapshot_info: &BankSnapshotInfo, bank_snapshot_info: &BankSnapshotInfo,
snapshots_dir: impl AsRef<Path>, bank_snapshots_dir: impl AsRef<Path>,
snapshot_package_output_path: impl AsRef<Path>, snapshot_archives_dir: impl AsRef<Path>,
snapshot_storages: SnapshotStorages, snapshot_storages: SnapshotStorages,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
snapshot_version: SnapshotVersion, snapshot_version: SnapshotVersion,
@ -1710,9 +1710,9 @@ pub fn package_process_and_archive_incremental_snapshot(
bank, bank,
incremental_snapshot_base_slot, incremental_snapshot_base_slot,
bank_snapshot_info, bank_snapshot_info,
snapshots_dir, bank_snapshots_dir,
bank.src.slot_deltas(&bank.src.roots()), bank.src.slot_deltas(&bank.src.roots()),
snapshot_package_output_path, snapshot_archives_dir,
snapshot_storages, snapshot_storages,
archive_format, archive_format,
snapshot_version, snapshot_version,
@ -1782,13 +1782,13 @@ pub fn process_accounts_package(
let snapshot_archive_path = match incremental_snapshot_base_slot { let snapshot_archive_path = match incremental_snapshot_base_slot {
None => build_full_snapshot_archive_path( None => build_full_snapshot_archive_path(
accounts_package.snapshot_output_dir, accounts_package.snapshot_archives_dir,
accounts_package.slot, accounts_package.slot,
&hash, &hash,
accounts_package.archive_format, accounts_package.archive_format,
), ),
Some(incremental_snapshot_base_slot) => build_incremental_snapshot_archive_path( Some(incremental_snapshot_base_slot) => build_incremental_snapshot_archive_path(
accounts_package.snapshot_output_dir, accounts_package.snapshot_archives_dir,
incremental_snapshot_base_slot, incremental_snapshot_base_slot,
accounts_package.slot, accounts_package.slot,
&hash, &hash,
@ -2146,9 +2146,13 @@ mod tests {
} }
/// A test heler function that creates bank snapshot files /// A test heler function that creates bank snapshot files
fn common_create_bank_snapshot_files(snapshots_dir: &Path, min_slot: Slot, max_slot: Slot) { fn common_create_bank_snapshot_files(
bank_snapshots_dir: &Path,
min_slot: Slot,
max_slot: Slot,
) {
for slot in min_slot..max_slot { for slot in min_slot..max_slot {
let snapshot_dir = get_bank_snapshots_dir(snapshots_dir, slot); let snapshot_dir = get_bank_snapshots_dir(bank_snapshots_dir, slot);
fs::create_dir_all(&snapshot_dir).unwrap(); fs::create_dir_all(&snapshot_dir).unwrap();
let snapshot_filename = get_snapshot_file_name(slot); let snapshot_filename = get_snapshot_file_name(slot);
@ -2494,12 +2498,12 @@ mod tests {
} }
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::Tar; let snapshot_archive_format = ArchiveFormat::Tar;
let snapshot_archive_info = bank_to_full_snapshot_archive( let snapshot_archive_info = bank_to_full_snapshot_archive(
&snapshots_dir, &bank_snapshots_dir,
&original_bank, &original_bank,
None, None,
snapshot_archives_dir.path(), snapshot_archives_dir.path(),
@ -2512,7 +2516,7 @@ mod tests {
let (roundtrip_bank, _) = bank_from_snapshot_archives( let (roundtrip_bank, _) = bank_from_snapshot_archives(
&[PathBuf::from(accounts_dir.path())], &[PathBuf::from(accounts_dir.path())],
&[], &[],
snapshots_dir.path(), bank_snapshots_dir.path(),
&snapshot_archive_info, &snapshot_archive_info,
None, None,
&genesis_config, &genesis_config,
@ -2585,12 +2589,12 @@ mod tests {
} }
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::TarGzip; let snapshot_archive_format = ArchiveFormat::TarGzip;
let full_snapshot_archive_info = bank_to_full_snapshot_archive( let full_snapshot_archive_info = bank_to_full_snapshot_archive(
snapshots_dir.path(), bank_snapshots_dir.path(),
&bank4, &bank4,
None, None,
snapshot_archives_dir.path(), snapshot_archives_dir.path(),
@ -2603,7 +2607,7 @@ mod tests {
let (roundtrip_bank, _) = bank_from_snapshot_archives( let (roundtrip_bank, _) = bank_from_snapshot_archives(
&[PathBuf::from(accounts_dir.path())], &[PathBuf::from(accounts_dir.path())],
&[], &[],
snapshots_dir.path(), bank_snapshots_dir.path(),
&full_snapshot_archive_info, &full_snapshot_archive_info,
None, None,
&genesis_config, &genesis_config,
@ -2661,13 +2665,13 @@ mod tests {
} }
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::TarZstd; let snapshot_archive_format = ArchiveFormat::TarZstd;
let full_snapshot_slot = slot; let full_snapshot_slot = slot;
let full_snapshot_archive_info = bank_to_full_snapshot_archive( let full_snapshot_archive_info = bank_to_full_snapshot_archive(
snapshots_dir.path(), bank_snapshots_dir.path(),
&bank1, &bank1,
None, None,
snapshot_archives_dir.path(), snapshot_archives_dir.path(),
@ -2699,7 +2703,7 @@ mod tests {
} }
let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive( let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive(
snapshots_dir.path(), bank_snapshots_dir.path(),
&bank4, &bank4,
full_snapshot_slot, full_snapshot_slot,
None, None,
@ -2713,7 +2717,7 @@ mod tests {
let (roundtrip_bank, _) = bank_from_snapshot_archives( let (roundtrip_bank, _) = bank_from_snapshot_archives(
&[PathBuf::from(accounts_dir.path())], &[PathBuf::from(accounts_dir.path())],
&[], &[],
snapshots_dir.path(), bank_snapshots_dir.path(),
&full_snapshot_archive_info, &full_snapshot_archive_info,
Some(&incremental_snapshot_archive_info), Some(&incremental_snapshot_archive_info),
&genesis_config, &genesis_config,
@ -2761,13 +2765,13 @@ mod tests {
} }
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::Tar; let snapshot_archive_format = ArchiveFormat::Tar;
let full_snapshot_slot = slot; let full_snapshot_slot = slot;
bank_to_full_snapshot_archive( bank_to_full_snapshot_archive(
&snapshots_dir, &bank_snapshots_dir,
&bank1, &bank1,
None, None,
&snapshot_archives_dir, &snapshot_archives_dir,
@ -2799,7 +2803,7 @@ mod tests {
} }
bank_to_incremental_snapshot_archive( bank_to_incremental_snapshot_archive(
&snapshots_dir, &bank_snapshots_dir,
&bank4, &bank4,
full_snapshot_slot, full_snapshot_slot,
None, None,
@ -2811,7 +2815,7 @@ mod tests {
.unwrap(); .unwrap();
let (deserialized_bank, _) = bank_from_latest_snapshot_archives( let (deserialized_bank, _) = bank_from_latest_snapshot_archives(
&snapshots_dir, &bank_snapshots_dir,
&snapshot_archives_dir, &snapshot_archives_dir,
&[accounts_dir.as_ref().to_path_buf()], &[accounts_dir.as_ref().to_path_buf()],
&[], &[],
@ -2863,7 +2867,7 @@ mod tests {
let key2 = Keypair::new(); let key2 = Keypair::new();
let accounts_dir = tempfile::TempDir::new().unwrap(); let accounts_dir = tempfile::TempDir::new().unwrap();
let snapshots_dir = tempfile::TempDir::new().unwrap(); let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
let snapshot_archives_dir = tempfile::TempDir::new().unwrap(); let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
let snapshot_archive_format = ArchiveFormat::Tar; let snapshot_archive_format = ArchiveFormat::Tar;
@ -2899,7 +2903,7 @@ mod tests {
let full_snapshot_slot = slot; let full_snapshot_slot = slot;
let full_snapshot_archive_info = bank_to_full_snapshot_archive( let full_snapshot_archive_info = bank_to_full_snapshot_archive(
snapshots_dir.path(), bank_snapshots_dir.path(),
&bank1, &bank1,
None, None,
snapshot_archives_dir.path(), snapshot_archives_dir.path(),
@ -2940,7 +2944,7 @@ mod tests {
// Take an incremental snapshot and then do a roundtrip on the bank and ensure it // Take an incremental snapshot and then do a roundtrip on the bank and ensure it
// deserializes correctly. // deserializes correctly.
let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive( let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive(
snapshots_dir.path(), bank_snapshots_dir.path(),
&bank2, &bank2,
full_snapshot_slot, full_snapshot_slot,
None, None,
@ -2953,7 +2957,7 @@ mod tests {
let (deserialized_bank, _) = bank_from_snapshot_archives( let (deserialized_bank, _) = bank_from_snapshot_archives(
&[accounts_dir.path().to_path_buf()], &[accounts_dir.path().to_path_buf()],
&[], &[],
snapshots_dir.path(), bank_snapshots_dir.path(),
&full_snapshot_archive_info, &full_snapshot_archive_info,
Some(&incremental_snapshot_archive_info), Some(&incremental_snapshot_archive_info),
&genesis_config, &genesis_config,
@ -3001,7 +3005,7 @@ mod tests {
// Take an incremental snapshot and then do a roundtrip on the bank and ensure it // Take an incremental snapshot and then do a roundtrip on the bank and ensure it
// deserializes correctly // deserializes correctly
let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive( let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive(
snapshots_dir.path(), bank_snapshots_dir.path(),
&bank4, &bank4,
full_snapshot_slot, full_snapshot_slot,
None, None,
@ -3015,7 +3019,7 @@ mod tests {
let (deserialized_bank, _) = bank_from_snapshot_archives( let (deserialized_bank, _) = bank_from_snapshot_archives(
&[accounts_dir.path().to_path_buf()], &[accounts_dir.path().to_path_buf()],
&[], &[],
snapshots_dir.path(), bank_snapshots_dir.path(),
&full_snapshot_archive_info, &full_snapshot_archive_info,
Some(&incremental_snapshot_archive_info), Some(&incremental_snapshot_archive_info),
&genesis_config, &genesis_config,

View File

@ -403,7 +403,7 @@ fn get_rpc_node(
blacklisted_rpc_nodes: &mut HashSet<Pubkey>, blacklisted_rpc_nodes: &mut HashSet<Pubkey>,
snapshot_not_required: bool, snapshot_not_required: bool,
no_untrusted_rpc: bool, no_untrusted_rpc: bool,
snapshot_output_dir: &Path, snapshot_archives_dir: &Path,
) -> Option<(ContactInfo, Option<(Slot, Hash)>)> { ) -> Option<(ContactInfo, Option<(Slot, Hash)>)> {
let mut blacklist_timeout = Instant::now(); let mut blacklist_timeout = Instant::now();
let mut newer_cluster_snapshot_timeout = None; let mut newer_cluster_snapshot_timeout = None;
@ -482,7 +482,7 @@ fn get_rpc_node(
blacklist_timeout = Instant::now(); blacklist_timeout = Instant::now();
let mut highest_snapshot_hash: Option<(Slot, Hash)> = let mut highest_snapshot_hash: Option<(Slot, Hash)> =
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_output_dir).map( snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_archives_dir).map(
|snapshot_archive_info| { |snapshot_archive_info| {
(snapshot_archive_info.slot(), *snapshot_archive_info.hash()) (snapshot_archive_info.slot(), *snapshot_archive_info.hash())
}, },
@ -756,7 +756,7 @@ fn rpc_bootstrap(
node: &Node, node: &Node,
identity_keypair: &Arc<Keypair>, identity_keypair: &Arc<Keypair>,
ledger_path: &Path, ledger_path: &Path,
snapshot_output_dir: &Path, snapshot_archives_dir: &Path,
vote_account: &Pubkey, vote_account: &Pubkey,
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>, authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
cluster_entrypoints: &[ContactInfo], cluster_entrypoints: &[ContactInfo],
@ -817,7 +817,7 @@ fn rpc_bootstrap(
&mut blacklisted_rpc_nodes, &mut blacklisted_rpc_nodes,
bootstrap_config.no_snapshot_fetch, bootstrap_config.no_snapshot_fetch,
bootstrap_config.no_untrusted_rpc, bootstrap_config.no_untrusted_rpc,
snapshot_output_dir, snapshot_archives_dir,
); );
if rpc_node_details.is_none() { if rpc_node_details.is_none() {
return; return;
@ -874,7 +874,7 @@ fn rpc_bootstrap(
let mut use_local_snapshot = false; let mut use_local_snapshot = false;
if let Some(highest_local_snapshot_slot) = if let Some(highest_local_snapshot_slot) =
snapshot_utils::get_highest_full_snapshot_archive_slot(snapshot_output_dir) snapshot_utils::get_highest_full_snapshot_archive_slot(snapshot_archives_dir)
{ {
if highest_local_snapshot_slot if highest_local_snapshot_slot
> snapshot_hash.0.saturating_sub(maximum_local_snapshot_age) > snapshot_hash.0.saturating_sub(maximum_local_snapshot_age)
@ -920,7 +920,7 @@ fn rpc_bootstrap(
}; };
let ret = download_snapshot( let ret = download_snapshot(
&rpc_contact_info.rpc, &rpc_contact_info.rpc,
snapshot_output_dir, snapshot_archives_dir,
snapshot_hash, snapshot_hash,
use_progress_bar, use_progress_bar,
maximum_snapshots_to_retain, maximum_snapshots_to_retain,
@ -2575,16 +2575,16 @@ pub fn main() {
let maximum_snapshot_download_abort = let maximum_snapshot_download_abort =
value_t_or_exit!(matches, "maximum_snapshot_download_abort", u64); value_t_or_exit!(matches, "maximum_snapshot_download_abort", u64);
let snapshot_output_dir = if matches.is_present("snapshots") { let snapshot_archives_dir = if matches.is_present("snapshots") {
PathBuf::from(matches.value_of("snapshots").unwrap()) PathBuf::from(matches.value_of("snapshots").unwrap())
} else { } else {
ledger_path.clone() ledger_path.clone()
}; };
let snapshot_path = snapshot_output_dir.join("snapshot"); let bank_snapshots_dir = snapshot_archives_dir.join("snapshot");
fs::create_dir_all(&snapshot_path).unwrap_or_else(|err| { fs::create_dir_all(&bank_snapshots_dir).unwrap_or_else(|err| {
eprintln!( eprintln!(
"Failed to create snapshots directory {:?}: {}", "Failed to create snapshots directory {:?}: {}",
snapshot_path, err bank_snapshots_dir, err
); );
exit(1); exit(1);
}); });
@ -2616,8 +2616,8 @@ pub fn main() {
std::u64::MAX std::u64::MAX
}, },
incremental_snapshot_archive_interval_slots: Slot::MAX, incremental_snapshot_archive_interval_slots: Slot::MAX,
snapshot_path, bank_snapshots_dir,
snapshot_package_output_path: snapshot_output_dir.clone(), snapshot_archives_dir: snapshot_archives_dir.clone(),
archive_format, archive_format,
snapshot_version, snapshot_version,
maximum_snapshots_to_retain, maximum_snapshots_to_retain,
@ -2784,7 +2784,7 @@ pub fn main() {
solana_metrics::set_panic_hook("validator"); solana_metrics::set_panic_hook("validator");
solana_entry::entry::init_poh(); solana_entry::entry::init_poh();
solana_runtime::snapshot_utils::remove_tmp_snapshot_archives(&snapshot_output_dir); solana_runtime::snapshot_utils::remove_tmp_snapshot_archives(&snapshot_archives_dir);
let identity_keypair = Arc::new(identity_keypair); let identity_keypair = Arc::new(identity_keypair);
@ -2794,7 +2794,7 @@ pub fn main() {
&node, &node,
&identity_keypair, &identity_keypair,
&ledger_path, &ledger_path,
&snapshot_output_dir, &snapshot_archives_dir,
&vote_account, &vote_account,
authorized_voter_keypairs.clone(), authorized_voter_keypairs.clone(),
&cluster_entrypoints, &cluster_entrypoints,