Name snapshots consistently (#19346)
#### Problem Snapshot names are overloaded, and there are multiple terms that mean the same thing. This is confusing. Here's a list of ones in the codebase that I've found: ``` - snapshot_dir - snapshots_dir - snapshot_path - snapshot_output_dir - snapshot_package_output_path - snapshot_archives_dir ``` #### Summary of Changes For all the ones that are about the directory where snapshot archives are stored, ensure they are `snapshot_archives_dir`. For the ones about the (bank) snapshots directory, set to `bank_snapshots_dir`. Co-authored-by: Michael Vines <mvines@gmail.com>
This commit is contained in:
parent
9483866e0b
commit
6d939811e9
|
@ -293,8 +293,8 @@ mod tests {
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots,
|
full_snapshot_archive_interval_slots,
|
||||||
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_package_output_path: PathBuf::default(),
|
snapshot_archives_dir: PathBuf::default(),
|
||||||
snapshot_path: PathBuf::default(),
|
bank_snapshots_dir: PathBuf::default(),
|
||||||
archive_format: ArchiveFormat::Tar,
|
archive_format: ArchiveFormat::Tar,
|
||||||
snapshot_version: SnapshotVersion::default(),
|
snapshot_version: SnapshotVersion::default(),
|
||||||
maximum_snapshots_to_retain: usize::MAX,
|
maximum_snapshots_to_retain: usize::MAX,
|
||||||
|
|
|
@ -121,8 +121,8 @@ mod tests {
|
||||||
fn create_and_verify_snapshot(temp_dir: &Path) {
|
fn create_and_verify_snapshot(temp_dir: &Path) {
|
||||||
let accounts_dir = temp_dir.join("accounts");
|
let accounts_dir = temp_dir.join("accounts");
|
||||||
let snapshots_dir = temp_dir.join("snapshots");
|
let snapshots_dir = temp_dir.join("snapshots");
|
||||||
let snapshot_package_output_path = temp_dir.join("snapshots_output");
|
let snapshot_archives_dir = temp_dir.join("snapshots_output");
|
||||||
fs::create_dir_all(&snapshot_package_output_path).unwrap();
|
fs::create_dir_all(&snapshot_archives_dir).unwrap();
|
||||||
|
|
||||||
fs::create_dir_all(&accounts_dir).unwrap();
|
fs::create_dir_all(&accounts_dir).unwrap();
|
||||||
// Create some storage entries
|
// Create some storage entries
|
||||||
|
@ -161,7 +161,7 @@ mod tests {
|
||||||
|
|
||||||
// Create a packageable snapshot
|
// Create a packageable snapshot
|
||||||
let output_tar_path = snapshot_utils::build_full_snapshot_archive_path(
|
let output_tar_path = snapshot_utils::build_full_snapshot_archive_path(
|
||||||
snapshot_package_output_path,
|
snapshot_archives_dir,
|
||||||
42,
|
42,
|
||||||
&Hash::default(),
|
&Hash::default(),
|
||||||
ArchiveFormat::TarBzip2,
|
ArchiveFormat::TarBzip2,
|
||||||
|
|
|
@ -523,8 +523,8 @@ impl TestValidator {
|
||||||
snapshot_config: Some(SnapshotConfig {
|
snapshot_config: Some(SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots: 100,
|
full_snapshot_archive_interval_slots: 100,
|
||||||
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_path: ledger_path.join("snapshot"),
|
bank_snapshots_dir: ledger_path.join("snapshot"),
|
||||||
snapshot_package_output_path: ledger_path.to_path_buf(),
|
snapshot_archives_dir: ledger_path.to_path_buf(),
|
||||||
archive_format: ArchiveFormat::Tar,
|
archive_format: ArchiveFormat::Tar,
|
||||||
snapshot_version: SnapshotVersion::default(),
|
snapshot_version: SnapshotVersion::default(),
|
||||||
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
|
|
@ -1210,7 +1210,7 @@ fn new_banks_from_ledger(
|
||||||
ledger_path,
|
ledger_path,
|
||||||
&bank_forks.root_bank(),
|
&bank_forks.root_bank(),
|
||||||
None,
|
None,
|
||||||
&snapshot_config.snapshot_package_output_path,
|
&snapshot_config.snapshot_archives_dir,
|
||||||
snapshot_config.archive_format,
|
snapshot_config.archive_format,
|
||||||
Some(bank_forks.root_bank().get_thread_pool()),
|
Some(bank_forks.root_bank().get_thread_pool()),
|
||||||
snapshot_config.maximum_snapshots_to_retain,
|
snapshot_config.maximum_snapshots_to_retain,
|
||||||
|
|
|
@ -141,8 +141,8 @@ mod tests {
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots,
|
full_snapshot_archive_interval_slots,
|
||||||
incremental_snapshot_archive_interval_slots,
|
incremental_snapshot_archive_interval_slots,
|
||||||
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
|
snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(),
|
||||||
snapshot_path: bank_snapshots_dir.path().to_path_buf(),
|
bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -166,17 +166,17 @@ mod tests {
|
||||||
old_genesis_config: &GenesisConfig,
|
old_genesis_config: &GenesisConfig,
|
||||||
account_paths: &[PathBuf],
|
account_paths: &[PathBuf],
|
||||||
) {
|
) {
|
||||||
let (snapshot_path, snapshot_package_output_path) = old_bank_forks
|
let (snapshot_path, snapshot_archives_dir) = old_bank_forks
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|c| (&c.snapshot_path, &c.snapshot_package_output_path))
|
.map(|c| (&c.bank_snapshots_dir, &c.snapshot_archives_dir))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
|
let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
|
||||||
|
|
||||||
let check_hash_calculation = false;
|
let check_hash_calculation = false;
|
||||||
let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path(
|
let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path(
|
||||||
snapshot_package_output_path.to_path_buf(),
|
snapshot_archives_dir.to_path_buf(),
|
||||||
old_last_bank.slot(),
|
old_last_bank.slot(),
|
||||||
&old_last_bank.get_accounts_hash(),
|
&old_last_bank.get_accounts_hash(),
|
||||||
ArchiveFormat::TarBzip2,
|
ArchiveFormat::TarBzip2,
|
||||||
|
@ -191,7 +191,7 @@ mod tests {
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_path,
|
.bank_snapshots_dir,
|
||||||
&full_snapshot_archive_info,
|
&full_snapshot_archive_info,
|
||||||
None,
|
None,
|
||||||
old_genesis_config,
|
old_genesis_config,
|
||||||
|
@ -273,15 +273,16 @@ mod tests {
|
||||||
// Generate a snapshot package for last bank
|
// Generate a snapshot package for last bank
|
||||||
let last_bank = bank_forks.get(last_slot).unwrap();
|
let last_bank = bank_forks.get(last_slot).unwrap();
|
||||||
let snapshot_config = &snapshot_test_config.snapshot_config;
|
let snapshot_config = &snapshot_test_config.snapshot_config;
|
||||||
let snapshot_path = &snapshot_config.snapshot_path;
|
let bank_snapshots_dir = &snapshot_config.bank_snapshots_dir;
|
||||||
let last_bank_snapshot_info = snapshot_utils::get_highest_bank_snapshot_info(snapshot_path)
|
let last_bank_snapshot_info =
|
||||||
.expect("no snapshots found in path");
|
snapshot_utils::get_highest_bank_snapshot_info(bank_snapshots_dir)
|
||||||
|
.expect("no bank snapshots found in path");
|
||||||
let accounts_package = AccountsPackage::new_for_full_snapshot(
|
let accounts_package = AccountsPackage::new_for_full_snapshot(
|
||||||
last_bank,
|
last_bank,
|
||||||
&last_bank_snapshot_info,
|
&last_bank_snapshot_info,
|
||||||
snapshot_path,
|
bank_snapshots_dir,
|
||||||
last_bank.src.slot_deltas(&last_bank.src.roots()),
|
last_bank.src.slot_deltas(&last_bank.src.roots()),
|
||||||
&snapshot_config.snapshot_package_output_path,
|
&snapshot_config.snapshot_archives_dir,
|
||||||
last_bank.get_snapshot_storages(None),
|
last_bank.get_snapshot_storages(None),
|
||||||
ArchiveFormat::TarBzip2,
|
ArchiveFormat::TarBzip2,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -355,17 +356,16 @@ mod tests {
|
||||||
SnapshotTestConfig::new(snapshot_version, cluster_type, 1, 1, Slot::MAX);
|
SnapshotTestConfig::new(snapshot_version, cluster_type, 1, 1, Slot::MAX);
|
||||||
|
|
||||||
let bank_forks = &mut snapshot_test_config.bank_forks;
|
let bank_forks = &mut snapshot_test_config.bank_forks;
|
||||||
let bank_snapshots_dir = &snapshot_test_config.bank_snapshots_dir;
|
|
||||||
let snapshot_config = &snapshot_test_config.snapshot_config;
|
let snapshot_config = &snapshot_test_config.snapshot_config;
|
||||||
let snapshot_path = &snapshot_config.snapshot_path;
|
let bank_snapshots_dir = &snapshot_config.bank_snapshots_dir;
|
||||||
let snapshot_package_output_path = &snapshot_config.snapshot_package_output_path;
|
let snapshot_archives_dir = &snapshot_config.snapshot_archives_dir;
|
||||||
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
||||||
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
|
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
|
||||||
|
|
||||||
// Take snapshot of zeroth bank
|
// Take snapshot of zeroth bank
|
||||||
let bank0 = bank_forks.get(0).unwrap();
|
let bank0 = bank_forks.get(0).unwrap();
|
||||||
let storages = bank0.get_snapshot_storages(None);
|
let storages = bank0.get_snapshot_storages(None);
|
||||||
snapshot_utils::add_bank_snapshot(snapshot_path, bank0, &storages, snapshot_version)
|
snapshot_utils::add_bank_snapshot(bank_snapshots_dir, bank0, &storages, snapshot_version)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
// Set up snapshotting channels
|
// Set up snapshotting channels
|
||||||
|
@ -412,8 +412,8 @@ mod tests {
|
||||||
&bank,
|
&bank,
|
||||||
vec![],
|
vec![],
|
||||||
package_sender,
|
package_sender,
|
||||||
snapshot_path,
|
bank_snapshots_dir,
|
||||||
snapshot_package_output_path,
|
snapshot_archives_dir,
|
||||||
snapshot_config.snapshot_version,
|
snapshot_config.snapshot_version,
|
||||||
&snapshot_config.archive_format,
|
&snapshot_config.archive_format,
|
||||||
None,
|
None,
|
||||||
|
@ -443,7 +443,7 @@ mod tests {
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
let last_snapshot_path = fs::read_dir(snapshot_path)
|
let last_snapshot_path = fs::read_dir(bank_snapshots_dir)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.filter_map(|entry| {
|
.filter_map(|entry| {
|
||||||
let e = entry.unwrap();
|
let e = entry.unwrap();
|
||||||
|
@ -462,7 +462,7 @@ mod tests {
|
||||||
fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
|
fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
|
||||||
|
|
||||||
saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path(
|
saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path(
|
||||||
snapshot_package_output_path.to_path_buf(),
|
snapshot_archives_dir.to_path_buf(),
|
||||||
slot,
|
slot,
|
||||||
&accounts_hash,
|
&accounts_hash,
|
||||||
ArchiveFormat::TarBzip2,
|
ArchiveFormat::TarBzip2,
|
||||||
|
@ -472,7 +472,7 @@ mod tests {
|
||||||
|
|
||||||
// Purge all the outdated snapshots, including the ones needed to generate the package
|
// Purge all the outdated snapshots, including the ones needed to generate the package
|
||||||
// currently sitting in the channel
|
// currently sitting in the channel
|
||||||
snapshot_utils::purge_old_bank_snapshots(snapshot_path);
|
snapshot_utils::purge_old_bank_snapshots(bank_snapshots_dir);
|
||||||
|
|
||||||
let mut bank_snapshots = snapshot_utils::get_bank_snapshots(&bank_snapshots_dir);
|
let mut bank_snapshots = snapshot_utils::get_bank_snapshots(&bank_snapshots_dir);
|
||||||
bank_snapshots.sort_unstable();
|
bank_snapshots.sort_unstable();
|
||||||
|
@ -754,15 +754,21 @@ mod tests {
|
||||||
) -> snapshot_utils::Result<()> {
|
) -> snapshot_utils::Result<()> {
|
||||||
let slot = bank.slot();
|
let slot = bank.slot();
|
||||||
info!("Making full snapshot archive from bank at slot: {}", slot);
|
info!("Making full snapshot archive from bank at slot: {}", slot);
|
||||||
let bank_snapshot_info = snapshot_utils::get_bank_snapshots(&snapshot_config.snapshot_path)
|
let bank_snapshot_info =
|
||||||
|
snapshot_utils::get_bank_snapshots(&snapshot_config.bank_snapshots_dir)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.find(|elem| elem.slot == slot)
|
.find(|elem| elem.slot == slot)
|
||||||
.ok_or_else(|| Error::new(ErrorKind::Other, "did not find snapshot with this path"))?;
|
.ok_or_else(|| {
|
||||||
|
Error::new(
|
||||||
|
ErrorKind::Other,
|
||||||
|
"did not find bank snapshot with this path",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
snapshot_utils::package_process_and_archive_full_snapshot(
|
snapshot_utils::package_process_and_archive_full_snapshot(
|
||||||
bank,
|
bank,
|
||||||
&bank_snapshot_info,
|
&bank_snapshot_info,
|
||||||
&snapshot_config.snapshot_path,
|
&snapshot_config.bank_snapshots_dir,
|
||||||
&snapshot_config.snapshot_package_output_path,
|
&snapshot_config.snapshot_archives_dir,
|
||||||
bank.get_snapshot_storages(None),
|
bank.get_snapshot_storages(None),
|
||||||
snapshot_config.archive_format,
|
snapshot_config.archive_format,
|
||||||
snapshot_config.snapshot_version,
|
snapshot_config.snapshot_version,
|
||||||
|
@ -783,10 +789,16 @@ mod tests {
|
||||||
"Making incremental snapshot archive from bank at slot: {}, and base slot: {}",
|
"Making incremental snapshot archive from bank at slot: {}, and base slot: {}",
|
||||||
slot, incremental_snapshot_base_slot,
|
slot, incremental_snapshot_base_slot,
|
||||||
);
|
);
|
||||||
let bank_snapshot_info = snapshot_utils::get_bank_snapshots(&snapshot_config.snapshot_path)
|
let bank_snapshot_info =
|
||||||
|
snapshot_utils::get_bank_snapshots(&snapshot_config.bank_snapshots_dir)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.find(|elem| elem.slot == slot)
|
.find(|elem| elem.slot == slot)
|
||||||
.ok_or_else(|| Error::new(ErrorKind::Other, "did not find snapshot with this path"))?;
|
.ok_or_else(|| {
|
||||||
|
Error::new(
|
||||||
|
ErrorKind::Other,
|
||||||
|
"did not find bank snapshot with this path",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
let storages = {
|
let storages = {
|
||||||
let mut storages = bank.get_snapshot_storages(Some(incremental_snapshot_base_slot));
|
let mut storages = bank.get_snapshot_storages(Some(incremental_snapshot_base_slot));
|
||||||
snapshot_utils::filter_snapshot_storages_for_incremental_snapshot(
|
snapshot_utils::filter_snapshot_storages_for_incremental_snapshot(
|
||||||
|
@ -799,8 +811,8 @@ mod tests {
|
||||||
bank,
|
bank,
|
||||||
incremental_snapshot_base_slot,
|
incremental_snapshot_base_slot,
|
||||||
&bank_snapshot_info,
|
&bank_snapshot_info,
|
||||||
&snapshot_config.snapshot_path,
|
&snapshot_config.bank_snapshots_dir,
|
||||||
&snapshot_config.snapshot_package_output_path,
|
&snapshot_config.snapshot_archives_dir,
|
||||||
storages,
|
storages,
|
||||||
snapshot_config.archive_format,
|
snapshot_config.archive_format,
|
||||||
snapshot_config.snapshot_version,
|
snapshot_config.snapshot_version,
|
||||||
|
@ -818,8 +830,8 @@ mod tests {
|
||||||
genesis_config: &GenesisConfig,
|
genesis_config: &GenesisConfig,
|
||||||
) -> snapshot_utils::Result<()> {
|
) -> snapshot_utils::Result<()> {
|
||||||
let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives(
|
let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives(
|
||||||
&snapshot_config.snapshot_path,
|
&snapshot_config.bank_snapshots_dir,
|
||||||
&snapshot_config.snapshot_package_output_path,
|
&snapshot_config.snapshot_archives_dir,
|
||||||
&[accounts_dir],
|
&[accounts_dir],
|
||||||
&[],
|
&[],
|
||||||
genesis_config,
|
genesis_config,
|
||||||
|
@ -996,10 +1008,8 @@ mod tests {
|
||||||
info!("Awake! Rebuilding bank from latest snapshot archives...");
|
info!("Awake! Rebuilding bank from latest snapshot archives...");
|
||||||
|
|
||||||
let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives(
|
let (deserialized_bank, _) = snapshot_utils::bank_from_latest_snapshot_archives(
|
||||||
&snapshot_test_config.snapshot_config.snapshot_path,
|
&snapshot_test_config.snapshot_config.bank_snapshots_dir,
|
||||||
&snapshot_test_config
|
&snapshot_test_config.snapshot_config.snapshot_archives_dir,
|
||||||
.snapshot_config
|
|
||||||
.snapshot_package_output_path,
|
|
||||||
&[snapshot_test_config.accounts_dir.as_ref().to_path_buf()],
|
&[snapshot_test_config.accounts_dir.as_ref().to_path_buf()],
|
||||||
&[],
|
&[],
|
||||||
&snapshot_test_config.genesis_config_info.genesis_config,
|
&snapshot_test_config.genesis_config_info.genesis_config,
|
||||||
|
|
|
@ -699,7 +699,7 @@ fn load_bank_forks(
|
||||||
process_options: ProcessOptions,
|
process_options: ProcessOptions,
|
||||||
snapshot_archive_path: Option<PathBuf>,
|
snapshot_archive_path: Option<PathBuf>,
|
||||||
) -> bank_forks_utils::LoadResult {
|
) -> bank_forks_utils::LoadResult {
|
||||||
let snapshot_path = blockstore
|
let bank_snapshots_dir = blockstore
|
||||||
.ledger_path()
|
.ledger_path()
|
||||||
.join(if blockstore.is_primary_access() {
|
.join(if blockstore.is_primary_access() {
|
||||||
"snapshot"
|
"snapshot"
|
||||||
|
@ -709,13 +709,13 @@ fn load_bank_forks(
|
||||||
let snapshot_config = if arg_matches.is_present("no_snapshot") {
|
let snapshot_config = if arg_matches.is_present("no_snapshot") {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
let snapshot_package_output_path =
|
let snapshot_archives_dir =
|
||||||
snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf());
|
snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf());
|
||||||
Some(SnapshotConfig {
|
Some(SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots: 0, // Value doesn't matter
|
full_snapshot_archive_interval_slots: 0, // Value doesn't matter
|
||||||
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_package_output_path,
|
snapshot_archives_dir,
|
||||||
snapshot_path,
|
bank_snapshots_dir,
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
snapshot_version: SnapshotVersion::default(),
|
snapshot_version: SnapshotVersion::default(),
|
||||||
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
|
|
@ -42,15 +42,15 @@ pub fn load(
|
||||||
) -> LoadResult {
|
) -> LoadResult {
|
||||||
if let Some(snapshot_config) = snapshot_config {
|
if let Some(snapshot_config) = snapshot_config {
|
||||||
info!(
|
info!(
|
||||||
"Initializing snapshot path: {}",
|
"Initializing bank snapshot path: {}",
|
||||||
snapshot_config.snapshot_path.display()
|
snapshot_config.bank_snapshots_dir.display()
|
||||||
);
|
);
|
||||||
let _ = fs::remove_dir_all(&snapshot_config.snapshot_path);
|
let _ = fs::remove_dir_all(&snapshot_config.bank_snapshots_dir);
|
||||||
fs::create_dir_all(&snapshot_config.snapshot_path)
|
fs::create_dir_all(&snapshot_config.bank_snapshots_dir)
|
||||||
.expect("Couldn't create snapshot directory");
|
.expect("Couldn't create snapshot directory");
|
||||||
|
|
||||||
if snapshot_utils::get_highest_full_snapshot_archive_info(
|
if snapshot_utils::get_highest_full_snapshot_archive_info(
|
||||||
&snapshot_config.snapshot_package_output_path,
|
&snapshot_config.snapshot_archives_dir,
|
||||||
)
|
)
|
||||||
.is_some()
|
.is_some()
|
||||||
{
|
{
|
||||||
|
@ -118,8 +118,8 @@ fn load_from_snapshot(
|
||||||
}
|
}
|
||||||
|
|
||||||
let (deserialized_bank, timings) = snapshot_utils::bank_from_latest_snapshot_archives(
|
let (deserialized_bank, timings) = snapshot_utils::bank_from_latest_snapshot_archives(
|
||||||
&snapshot_config.snapshot_path,
|
&snapshot_config.bank_snapshots_dir,
|
||||||
&snapshot_config.snapshot_package_output_path,
|
&snapshot_config.snapshot_archives_dir,
|
||||||
&account_paths,
|
&account_paths,
|
||||||
&process_options.frozen_accounts,
|
&process_options.frozen_accounts,
|
||||||
genesis_config,
|
genesis_config,
|
||||||
|
|
|
@ -1555,16 +1555,16 @@ fn test_frozen_account_from_snapshot() {
|
||||||
};
|
};
|
||||||
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
|
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
|
||||||
|
|
||||||
let snapshot_package_output_path = &snapshot_test_config
|
let snapshot_archives_dir = &snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_package_output_path;
|
.snapshot_archives_dir;
|
||||||
|
|
||||||
trace!("Waiting for snapshot at {:?}", snapshot_package_output_path);
|
trace!("Waiting for snapshot at {:?}", snapshot_archives_dir);
|
||||||
let (archive_filename, _archive_snapshot_hash) =
|
let (archive_filename, _archive_snapshot_hash) =
|
||||||
wait_for_next_snapshot(&cluster, snapshot_package_output_path);
|
wait_for_next_snapshot(&cluster, snapshot_archives_dir);
|
||||||
|
|
||||||
trace!("Found snapshot: {:?}", archive_filename);
|
trace!("Found snapshot: {:?}", archive_filename);
|
||||||
|
|
||||||
|
@ -1709,16 +1709,16 @@ fn test_snapshot_download() {
|
||||||
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
|
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
|
||||||
|
|
||||||
// Get slot after which this was generated
|
// Get slot after which this was generated
|
||||||
let snapshot_package_output_path = &leader_snapshot_test_config
|
let snapshot_archives_dir = &leader_snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_package_output_path;
|
.snapshot_archives_dir;
|
||||||
|
|
||||||
trace!("Waiting for snapshot");
|
trace!("Waiting for snapshot");
|
||||||
let (archive_filename, archive_snapshot_hash) =
|
let (archive_filename, archive_snapshot_hash) =
|
||||||
wait_for_next_snapshot(&cluster, snapshot_package_output_path);
|
wait_for_next_snapshot(&cluster, snapshot_archives_dir);
|
||||||
trace!("found: {:?}", archive_filename);
|
trace!("found: {:?}", archive_filename);
|
||||||
|
|
||||||
// Download the snapshot, then boot a validator from it.
|
// Download the snapshot, then boot a validator from it.
|
||||||
|
@ -1777,15 +1777,15 @@ fn test_snapshot_restart_tower() {
|
||||||
let validator_info = cluster.exit_node(&validator_id);
|
let validator_info = cluster.exit_node(&validator_id);
|
||||||
|
|
||||||
// Get slot after which this was generated
|
// Get slot after which this was generated
|
||||||
let snapshot_package_output_path = &leader_snapshot_test_config
|
let snapshot_archives_dir = &leader_snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_package_output_path;
|
.snapshot_archives_dir;
|
||||||
|
|
||||||
let (archive_filename, archive_snapshot_hash) =
|
let (archive_filename, archive_snapshot_hash) =
|
||||||
wait_for_next_snapshot(&cluster, snapshot_package_output_path);
|
wait_for_next_snapshot(&cluster, snapshot_archives_dir);
|
||||||
|
|
||||||
// Copy archive to validator's snapshot output directory
|
// Copy archive to validator's snapshot output directory
|
||||||
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
|
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
|
||||||
|
@ -1829,12 +1829,12 @@ fn test_snapshots_blockstore_floor() {
|
||||||
let mut validator_snapshot_test_config =
|
let mut validator_snapshot_test_config =
|
||||||
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
||||||
|
|
||||||
let snapshot_package_output_path = &leader_snapshot_test_config
|
let snapshot_archives_dir = &leader_snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_package_output_path;
|
.snapshot_archives_dir;
|
||||||
|
|
||||||
let mut config = ClusterConfig {
|
let mut config = ClusterConfig {
|
||||||
node_stakes: vec![10000],
|
node_stakes: vec![10000],
|
||||||
|
@ -1852,7 +1852,7 @@ fn test_snapshots_blockstore_floor() {
|
||||||
|
|
||||||
let archive_info = loop {
|
let archive_info = loop {
|
||||||
let archive =
|
let archive =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_package_output_path);
|
snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir);
|
||||||
if archive.is_some() {
|
if archive.is_some() {
|
||||||
trace!("snapshot exists");
|
trace!("snapshot exists");
|
||||||
break archive.unwrap();
|
break archive.unwrap();
|
||||||
|
@ -1934,12 +1934,12 @@ fn test_snapshots_restart_validity() {
|
||||||
let num_account_paths = 1;
|
let num_account_paths = 1;
|
||||||
let mut snapshot_test_config =
|
let mut snapshot_test_config =
|
||||||
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
||||||
let snapshot_package_output_path = &snapshot_test_config
|
let snapshot_archives_dir = &snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_package_output_path;
|
.snapshot_archives_dir;
|
||||||
|
|
||||||
// Set up the cluster with 1 snapshotting validator
|
// Set up the cluster with 1 snapshotting validator
|
||||||
let mut all_account_storage_dirs = vec![vec![]];
|
let mut all_account_storage_dirs = vec![vec![]];
|
||||||
|
@ -1977,7 +1977,7 @@ fn test_snapshots_restart_validity() {
|
||||||
|
|
||||||
expected_balances.extend(new_balances);
|
expected_balances.extend(new_balances);
|
||||||
|
|
||||||
wait_for_next_snapshot(&cluster, snapshot_package_output_path);
|
wait_for_next_snapshot(&cluster, snapshot_archives_dir);
|
||||||
|
|
||||||
// Create new account paths since validator exit is not guaranteed to cleanup RPC threads,
|
// Create new account paths since validator exit is not guaranteed to cleanup RPC threads,
|
||||||
// which may delete the old accounts on exit at any point
|
// which may delete the old accounts on exit at any point
|
||||||
|
@ -3403,7 +3403,7 @@ fn run_test_load_program_accounts(scan_commitment: CommitmentConfig) {
|
||||||
|
|
||||||
fn wait_for_next_snapshot(
|
fn wait_for_next_snapshot(
|
||||||
cluster: &LocalCluster,
|
cluster: &LocalCluster,
|
||||||
snapshot_package_output_path: &Path,
|
snapshot_archives_dir: &Path,
|
||||||
) -> (PathBuf, (Slot, Hash)) {
|
) -> (PathBuf, (Slot, Hash)) {
|
||||||
// Get slot after which this was generated
|
// Get slot after which this was generated
|
||||||
let client = cluster
|
let client = cluster
|
||||||
|
@ -3421,7 +3421,7 @@ fn wait_for_next_snapshot(
|
||||||
);
|
);
|
||||||
loop {
|
loop {
|
||||||
if let Some(full_snapshot_archive_info) =
|
if let Some(full_snapshot_archive_info) =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_package_output_path)
|
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_archives_dir)
|
||||||
{
|
{
|
||||||
trace!(
|
trace!(
|
||||||
"full snapshot for slot {} exists",
|
"full snapshot for slot {} exists",
|
||||||
|
@ -3475,13 +3475,13 @@ fn setup_snapshot_validator_config(
|
||||||
num_account_paths: usize,
|
num_account_paths: usize,
|
||||||
) -> SnapshotValidatorConfig {
|
) -> SnapshotValidatorConfig {
|
||||||
// Create the snapshot config
|
// Create the snapshot config
|
||||||
let snapshot_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let bank_snapshots_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots: snapshot_interval_slots,
|
full_snapshot_archive_interval_slots: snapshot_interval_slots,
|
||||||
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
|
snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(),
|
||||||
snapshot_path: snapshot_dir.path().to_path_buf(),
|
bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
snapshot_version: snapshot_utils::SnapshotVersion::default(),
|
snapshot_version: snapshot_utils::SnapshotVersion::default(),
|
||||||
maximum_snapshots_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
maximum_snapshots_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -3500,7 +3500,7 @@ fn setup_snapshot_validator_config(
|
||||||
};
|
};
|
||||||
|
|
||||||
SnapshotValidatorConfig {
|
SnapshotValidatorConfig {
|
||||||
_snapshot_dir: snapshot_dir,
|
_snapshot_dir: bank_snapshots_dir,
|
||||||
snapshot_archives_dir,
|
snapshot_archives_dir,
|
||||||
account_storage_dirs,
|
account_storage_dirs,
|
||||||
validator_config,
|
validator_config,
|
||||||
|
|
|
@ -278,12 +278,12 @@ pub fn main() {
|
||||||
);
|
);
|
||||||
|
|
||||||
let ledger_path = PathBuf::from(matches.value_of("ledger_path").unwrap());
|
let ledger_path = PathBuf::from(matches.value_of("ledger_path").unwrap());
|
||||||
let snapshot_output_dir = if let Some(snapshots) = matches.value_of("snapshots") {
|
let snapshot_archives_dir = if let Some(snapshots) = matches.value_of("snapshots") {
|
||||||
PathBuf::from(snapshots)
|
PathBuf::from(snapshots)
|
||||||
} else {
|
} else {
|
||||||
ledger_path.clone()
|
ledger_path.clone()
|
||||||
};
|
};
|
||||||
let snapshot_path = snapshot_output_dir.join("snapshot");
|
let bank_snapshots_dir = snapshot_archives_dir.join("snapshot");
|
||||||
|
|
||||||
let account_paths: Vec<PathBuf> =
|
let account_paths: Vec<PathBuf> =
|
||||||
if let Ok(account_paths) = values_t!(matches, "account_paths", String) {
|
if let Ok(account_paths) = values_t!(matches, "account_paths", String) {
|
||||||
|
@ -348,7 +348,7 @@ pub fn main() {
|
||||||
&node,
|
&node,
|
||||||
expected_shred_version,
|
expected_shred_version,
|
||||||
&peer_pubkey,
|
&peer_pubkey,
|
||||||
&snapshot_output_dir,
|
&snapshot_archives_dir,
|
||||||
socket_addr_space,
|
socket_addr_space,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -362,8 +362,8 @@ pub fn main() {
|
||||||
rpc_addr: rpc_addrs.0,
|
rpc_addr: rpc_addrs.0,
|
||||||
rpc_pubsub_addr: rpc_addrs.1,
|
rpc_pubsub_addr: rpc_addrs.1,
|
||||||
ledger_path,
|
ledger_path,
|
||||||
snapshot_output_dir,
|
snapshot_archives_dir,
|
||||||
snapshot_path,
|
bank_snapshots_dir,
|
||||||
account_paths,
|
account_paths,
|
||||||
snapshot_info: snapshot_info.unwrap(),
|
snapshot_info: snapshot_info.unwrap(),
|
||||||
cluster_info,
|
cluster_info,
|
||||||
|
|
|
@ -44,8 +44,8 @@ pub struct ReplicaNodeConfig {
|
||||||
pub rpc_addr: SocketAddr,
|
pub rpc_addr: SocketAddr,
|
||||||
pub rpc_pubsub_addr: SocketAddr,
|
pub rpc_pubsub_addr: SocketAddr,
|
||||||
pub ledger_path: PathBuf,
|
pub ledger_path: PathBuf,
|
||||||
pub snapshot_output_dir: PathBuf,
|
pub snapshot_archives_dir: PathBuf,
|
||||||
pub snapshot_path: PathBuf,
|
pub bank_snapshots_dir: PathBuf,
|
||||||
pub account_paths: Vec<PathBuf>,
|
pub account_paths: Vec<PathBuf>,
|
||||||
pub snapshot_info: (Slot, Hash),
|
pub snapshot_info: (Slot, Hash),
|
||||||
pub cluster_info: Arc<ClusterInfo>,
|
pub cluster_info: Arc<ClusterInfo>,
|
||||||
|
@ -82,12 +82,12 @@ fn initialize_from_snapshot(
|
||||||
) -> ReplicaBankInfo {
|
) -> ReplicaBankInfo {
|
||||||
info!(
|
info!(
|
||||||
"Downloading snapshot from the peer into {:?}",
|
"Downloading snapshot from the peer into {:?}",
|
||||||
replica_config.snapshot_output_dir
|
replica_config.snapshot_archives_dir
|
||||||
);
|
);
|
||||||
|
|
||||||
download_snapshot(
|
download_snapshot(
|
||||||
&replica_config.rpc_source_addr,
|
&replica_config.rpc_source_addr,
|
||||||
&replica_config.snapshot_output_dir,
|
&replica_config.snapshot_archives_dir,
|
||||||
replica_config.snapshot_info,
|
replica_config.snapshot_info,
|
||||||
false,
|
false,
|
||||||
snapshot_config.maximum_snapshots_to_retain,
|
snapshot_config.maximum_snapshots_to_retain,
|
||||||
|
@ -95,10 +95,12 @@ fn initialize_from_snapshot(
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
fs::create_dir_all(&snapshot_config.snapshot_path).expect("Couldn't create snapshot directory");
|
fs::create_dir_all(&snapshot_config.bank_snapshots_dir)
|
||||||
|
.expect("Couldn't create bank snapshot directory");
|
||||||
|
|
||||||
let archive_info =
|
let archive_info = snapshot_utils::get_highest_full_snapshot_archive_info(
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(&replica_config.snapshot_output_dir)
|
&replica_config.snapshot_archives_dir,
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let process_options = blockstore_processor::ProcessOptions {
|
let process_options = blockstore_processor::ProcessOptions {
|
||||||
|
@ -109,12 +111,12 @@ fn initialize_from_snapshot(
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
"Build bank from snapshot archive: {:?}",
|
"Build bank from snapshot archive: {:?}",
|
||||||
&snapshot_config.snapshot_path
|
&snapshot_config.bank_snapshots_dir
|
||||||
);
|
);
|
||||||
let (bank0, _) = snapshot_utils::bank_from_snapshot_archives(
|
let (bank0, _) = snapshot_utils::bank_from_snapshot_archives(
|
||||||
&replica_config.account_paths,
|
&replica_config.account_paths,
|
||||||
&[],
|
&[],
|
||||||
&snapshot_config.snapshot_path,
|
&snapshot_config.bank_snapshots_dir,
|
||||||
&archive_info,
|
&archive_info,
|
||||||
None,
|
None,
|
||||||
genesis_config,
|
genesis_config,
|
||||||
|
@ -257,8 +259,8 @@ impl ReplicaNode {
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots: std::u64::MAX,
|
full_snapshot_archive_interval_slots: std::u64::MAX,
|
||||||
incremental_snapshot_archive_interval_slots: std::u64::MAX,
|
incremental_snapshot_archive_interval_slots: std::u64::MAX,
|
||||||
snapshot_package_output_path: replica_config.snapshot_output_dir.clone(),
|
snapshot_archives_dir: replica_config.snapshot_archives_dir.clone(),
|
||||||
snapshot_path: replica_config.snapshot_path.clone(),
|
bank_snapshots_dir: replica_config.bank_snapshots_dir.clone(),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
snapshot_version: snapshot_utils::SnapshotVersion::default(),
|
snapshot_version: snapshot_utils::SnapshotVersion::default(),
|
||||||
maximum_snapshots_to_retain:
|
maximum_snapshots_to_retain:
|
||||||
|
|
|
@ -55,7 +55,7 @@ fn get_rpc_peer_node(
|
||||||
cluster_entrypoints: &[ContactInfo],
|
cluster_entrypoints: &[ContactInfo],
|
||||||
expected_shred_version: Option<u16>,
|
expected_shred_version: Option<u16>,
|
||||||
peer_pubkey: &Pubkey,
|
peer_pubkey: &Pubkey,
|
||||||
snapshot_output_dir: &Path,
|
snapshot_archives_dir: &Path,
|
||||||
) -> Option<(ContactInfo, Option<(Slot, Hash)>)> {
|
) -> Option<(ContactInfo, Option<(Slot, Hash)>)> {
|
||||||
let mut newer_cluster_snapshot_timeout = None;
|
let mut newer_cluster_snapshot_timeout = None;
|
||||||
let mut retry_reason = None;
|
let mut retry_reason = None;
|
||||||
|
@ -110,7 +110,7 @@ fn get_rpc_peer_node(
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut highest_snapshot_info: Option<(Slot, Hash)> =
|
let mut highest_snapshot_info: Option<(Slot, Hash)> =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_output_dir).map(
|
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_archives_dir).map(
|
||||||
|snapshot_archive_info| {
|
|snapshot_archive_info| {
|
||||||
(snapshot_archive_info.slot(), *snapshot_archive_info.hash())
|
(snapshot_archive_info.slot(), *snapshot_archive_info.hash())
|
||||||
},
|
},
|
||||||
|
@ -238,7 +238,7 @@ pub fn get_rpc_peer_info(
|
||||||
node: &Node,
|
node: &Node,
|
||||||
expected_shred_version: Option<u16>,
|
expected_shred_version: Option<u16>,
|
||||||
peer_pubkey: &Pubkey,
|
peer_pubkey: &Pubkey,
|
||||||
snapshot_output_dir: &Path,
|
snapshot_archives_dir: &Path,
|
||||||
socket_addr_space: SocketAddrSpace,
|
socket_addr_space: SocketAddrSpace,
|
||||||
) -> (Arc<ClusterInfo>, ContactInfo, Option<(Slot, Hash)>) {
|
) -> (Arc<ClusterInfo>, ContactInfo, Option<(Slot, Hash)>) {
|
||||||
let identity_keypair = Arc::new(identity_keypair);
|
let identity_keypair = Arc::new(identity_keypair);
|
||||||
|
@ -260,7 +260,7 @@ pub fn get_rpc_peer_info(
|
||||||
cluster_entrypoints,
|
cluster_entrypoints,
|
||||||
expected_shred_version,
|
expected_shred_version,
|
||||||
peer_pubkey,
|
peer_pubkey,
|
||||||
snapshot_output_dir,
|
snapshot_archives_dir,
|
||||||
);
|
);
|
||||||
let rpc_node_details = rpc_node_details.unwrap();
|
let rpc_node_details = rpc_node_details.unwrap();
|
||||||
|
|
||||||
|
|
|
@ -45,7 +45,7 @@ const RUST_LOG_FILTER: &str =
|
||||||
|
|
||||||
fn wait_for_next_snapshot(
|
fn wait_for_next_snapshot(
|
||||||
cluster: &LocalCluster,
|
cluster: &LocalCluster,
|
||||||
snapshot_package_output_path: &Path,
|
snapshot_archives_dir: &Path,
|
||||||
) -> (PathBuf, (Slot, Hash)) {
|
) -> (PathBuf, (Slot, Hash)) {
|
||||||
// Get slot after which this was generated
|
// Get slot after which this was generated
|
||||||
let client = cluster
|
let client = cluster
|
||||||
|
@ -63,7 +63,7 @@ fn wait_for_next_snapshot(
|
||||||
);
|
);
|
||||||
loop {
|
loop {
|
||||||
if let Some(full_snapshot_archive_info) =
|
if let Some(full_snapshot_archive_info) =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_package_output_path)
|
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_archives_dir)
|
||||||
{
|
{
|
||||||
trace!(
|
trace!(
|
||||||
"full snapshot for slot {} exists",
|
"full snapshot for slot {} exists",
|
||||||
|
@ -117,13 +117,13 @@ fn setup_snapshot_validator_config(
|
||||||
num_account_paths: usize,
|
num_account_paths: usize,
|
||||||
) -> SnapshotValidatorConfig {
|
) -> SnapshotValidatorConfig {
|
||||||
// Create the snapshot config
|
// Create the snapshot config
|
||||||
let snapshot_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let bank_snapshots_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots: snapshot_interval_slots,
|
full_snapshot_archive_interval_slots: snapshot_interval_slots,
|
||||||
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
|
snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(),
|
||||||
snapshot_path: snapshot_dir.path().to_path_buf(),
|
bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
snapshot_version: snapshot_utils::SnapshotVersion::default(),
|
snapshot_version: snapshot_utils::SnapshotVersion::default(),
|
||||||
maximum_snapshots_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
maximum_snapshots_to_retain: snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -142,7 +142,7 @@ fn setup_snapshot_validator_config(
|
||||||
};
|
};
|
||||||
|
|
||||||
SnapshotValidatorConfig {
|
SnapshotValidatorConfig {
|
||||||
_snapshot_dir: snapshot_dir,
|
_snapshot_dir: bank_snapshots_dir,
|
||||||
snapshot_archives_dir,
|
snapshot_archives_dir,
|
||||||
account_storage_dirs,
|
account_storage_dirs,
|
||||||
validator_config,
|
validator_config,
|
||||||
|
@ -204,15 +204,15 @@ fn test_replica_bootstrap() {
|
||||||
info!("Contact info: {:?}", contact_info);
|
info!("Contact info: {:?}", contact_info);
|
||||||
|
|
||||||
// Get slot after which this was generated
|
// Get slot after which this was generated
|
||||||
let snapshot_package_output_path = &leader_snapshot_test_config
|
let snapshot_archives_dir = &leader_snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_package_output_path;
|
.snapshot_archives_dir;
|
||||||
info!("Waiting for snapshot");
|
info!("Waiting for snapshot");
|
||||||
let (archive_filename, archive_snapshot_hash) =
|
let (archive_filename, archive_snapshot_hash) =
|
||||||
wait_for_next_snapshot(&cluster, snapshot_package_output_path);
|
wait_for_next_snapshot(&cluster, snapshot_archives_dir);
|
||||||
info!("found: {:?}", archive_filename);
|
info!("found: {:?}", archive_filename);
|
||||||
|
|
||||||
let identity_keypair = Keypair::new();
|
let identity_keypair = Keypair::new();
|
||||||
|
@ -228,8 +228,8 @@ fn test_replica_bootstrap() {
|
||||||
let ledger_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let ledger_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let ledger_path = ledger_dir.path();
|
let ledger_path = ledger_dir.path();
|
||||||
let snapshot_output_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let snapshot_output_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_output_path = snapshot_output_dir.path();
|
let snapshot_archives_dir = snapshot_output_dir.path();
|
||||||
let snapshot_path = snapshot_output_path.join("snapshot");
|
let bank_snapshots_dir = snapshot_archives_dir.join("snapshot");
|
||||||
let account_paths: Vec<PathBuf> = vec![ledger_path.join("accounts")];
|
let account_paths: Vec<PathBuf> = vec![ledger_path.join("accounts")];
|
||||||
|
|
||||||
let port = solana_net_utils::find_available_port_in_range(ip_addr, (8301, 8400)).unwrap();
|
let port = solana_net_utils::find_available_port_in_range(ip_addr, (8301, 8400)).unwrap();
|
||||||
|
@ -253,7 +253,7 @@ fn test_replica_bootstrap() {
|
||||||
&node,
|
&node,
|
||||||
None,
|
None,
|
||||||
&contact_info.id,
|
&contact_info.id,
|
||||||
snapshot_output_path,
|
snapshot_archives_dir,
|
||||||
socket_addr_space,
|
socket_addr_space,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -264,8 +264,8 @@ fn test_replica_bootstrap() {
|
||||||
rpc_addr,
|
rpc_addr,
|
||||||
rpc_pubsub_addr,
|
rpc_pubsub_addr,
|
||||||
ledger_path: ledger_path.to_path_buf(),
|
ledger_path: ledger_path.to_path_buf(),
|
||||||
snapshot_output_dir: snapshot_output_path.to_path_buf(),
|
snapshot_archives_dir: snapshot_archives_dir.to_path_buf(),
|
||||||
snapshot_path,
|
bank_snapshots_dir,
|
||||||
account_paths,
|
account_paths,
|
||||||
snapshot_info: archive_snapshot_hash,
|
snapshot_info: archive_snapshot_hash,
|
||||||
cluster_info,
|
cluster_info,
|
||||||
|
|
|
@ -2369,7 +2369,7 @@ pub mod rpc_minimal {
|
||||||
meta.snapshot_config
|
meta.snapshot_config
|
||||||
.and_then(|snapshot_config| {
|
.and_then(|snapshot_config| {
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_slot(
|
snapshot_utils::get_highest_full_snapshot_archive_slot(
|
||||||
&snapshot_config.snapshot_package_output_path,
|
&snapshot_config.snapshot_archives_dir,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.ok_or_else(|| RpcCustomError::NoSnapshot.into())
|
.ok_or_else(|| RpcCustomError::NoSnapshot.into())
|
||||||
|
|
|
@ -149,7 +149,7 @@ impl RpcRequestMiddleware {
|
||||||
self.snapshot_config
|
self.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_package_output_path
|
.snapshot_archives_dir
|
||||||
.join(stem)
|
.join(stem)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -204,7 +204,7 @@ impl RequestMiddleware for RpcRequestMiddleware {
|
||||||
// Convenience redirect to the latest snapshot
|
// Convenience redirect to the latest snapshot
|
||||||
return if let Some(full_snapshot_archive_info) =
|
return if let Some(full_snapshot_archive_info) =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(
|
snapshot_utils::get_highest_full_snapshot_archive_info(
|
||||||
&snapshot_config.snapshot_package_output_path,
|
&snapshot_config.snapshot_archives_dir,
|
||||||
) {
|
) {
|
||||||
RpcRequestMiddleware::redirect(&format!(
|
RpcRequestMiddleware::redirect(&format!(
|
||||||
"/{}",
|
"/{}",
|
||||||
|
@ -604,8 +604,8 @@ mod tests {
|
||||||
Some(SnapshotConfig {
|
Some(SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots: 0,
|
full_snapshot_archive_interval_slots: 0,
|
||||||
incremental_snapshot_archive_interval_slots: u64::MAX,
|
incremental_snapshot_archive_interval_slots: u64::MAX,
|
||||||
snapshot_package_output_path: PathBuf::from("/"),
|
snapshot_archives_dir: PathBuf::from("/"),
|
||||||
snapshot_path: PathBuf::from("/"),
|
bank_snapshots_dir: PathBuf::from("/"),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
snapshot_version: SnapshotVersion::default(),
|
snapshot_version: SnapshotVersion::default(),
|
||||||
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
|
|
@ -175,8 +175,8 @@ impl SnapshotRequestHandler {
|
||||||
&snapshot_root_bank,
|
&snapshot_root_bank,
|
||||||
status_cache_slot_deltas,
|
status_cache_slot_deltas,
|
||||||
&self.accounts_package_sender,
|
&self.accounts_package_sender,
|
||||||
&self.snapshot_config.snapshot_path,
|
&self.snapshot_config.bank_snapshots_dir,
|
||||||
&self.snapshot_config.snapshot_package_output_path,
|
&self.snapshot_config.snapshot_archives_dir,
|
||||||
self.snapshot_config.snapshot_version,
|
self.snapshot_config.snapshot_version,
|
||||||
&self.snapshot_config.archive_format,
|
&self.snapshot_config.archive_format,
|
||||||
hash_for_testing,
|
hash_for_testing,
|
||||||
|
@ -192,7 +192,7 @@ impl SnapshotRequestHandler {
|
||||||
|
|
||||||
// Cleanup outdated snapshots
|
// Cleanup outdated snapshots
|
||||||
let mut purge_old_snapshots_time = Measure::start("purge_old_snapshots_time");
|
let mut purge_old_snapshots_time = Measure::start("purge_old_snapshots_time");
|
||||||
snapshot_utils::purge_old_bank_snapshots(&self.snapshot_config.snapshot_path);
|
snapshot_utils::purge_old_bank_snapshots(&self.snapshot_config.bank_snapshots_dir);
|
||||||
purge_old_snapshots_time.stop();
|
purge_old_snapshots_time.stop();
|
||||||
total_time.stop();
|
total_time.stop();
|
||||||
|
|
||||||
|
|
|
@ -15,11 +15,11 @@ pub struct SnapshotConfig {
|
||||||
/// Generate a new incremental snapshot archive every this many slots
|
/// Generate a new incremental snapshot archive every this many slots
|
||||||
pub incremental_snapshot_archive_interval_slots: Slot,
|
pub incremental_snapshot_archive_interval_slots: Slot,
|
||||||
|
|
||||||
/// Where to store the latest packaged snapshot archives
|
/// Path to the directory where snapshot archives are stored
|
||||||
pub snapshot_package_output_path: PathBuf,
|
pub snapshot_archives_dir: PathBuf,
|
||||||
|
|
||||||
/// Where to place the bank snapshots for recent slots
|
/// Path to the directory where bank snapshots are stored
|
||||||
pub snapshot_path: PathBuf,
|
pub bank_snapshots_dir: PathBuf,
|
||||||
|
|
||||||
/// The archive format to use for snapshots
|
/// The archive format to use for snapshots
|
||||||
pub archive_format: ArchiveFormat,
|
pub archive_format: ArchiveFormat,
|
||||||
|
|
|
@ -46,7 +46,7 @@ pub struct AccountsPackage {
|
||||||
pub hash: Hash, // temporarily here while we still have to calculate hash before serializing bank
|
pub hash: Hash, // temporarily here while we still have to calculate hash before serializing bank
|
||||||
pub archive_format: ArchiveFormat,
|
pub archive_format: ArchiveFormat,
|
||||||
pub snapshot_version: SnapshotVersion,
|
pub snapshot_version: SnapshotVersion,
|
||||||
pub snapshot_output_dir: PathBuf,
|
pub snapshot_archives_dir: PathBuf,
|
||||||
pub expected_capitalization: u64,
|
pub expected_capitalization: u64,
|
||||||
pub hash_for_testing: Option<Hash>,
|
pub hash_for_testing: Option<Hash>,
|
||||||
pub cluster_type: ClusterType,
|
pub cluster_type: ClusterType,
|
||||||
|
@ -59,7 +59,7 @@ impl AccountsPackage {
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
bank_snapshot_info: &BankSnapshotInfo,
|
bank_snapshot_info: &BankSnapshotInfo,
|
||||||
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
||||||
snapshot_package_output_path: impl AsRef<Path>,
|
snapshot_archives_dir: impl AsRef<Path>,
|
||||||
snapshot_storages: SnapshotStorages,
|
snapshot_storages: SnapshotStorages,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
|
@ -87,7 +87,7 @@ impl AccountsPackage {
|
||||||
hash: bank.get_accounts_hash(),
|
hash: bank.get_accounts_hash(),
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
snapshot_output_dir: snapshot_package_output_path.as_ref().to_path_buf(),
|
snapshot_archives_dir: snapshot_archives_dir.as_ref().to_path_buf(),
|
||||||
expected_capitalization: bank.capitalization(),
|
expected_capitalization: bank.capitalization(),
|
||||||
hash_for_testing,
|
hash_for_testing,
|
||||||
cluster_type: bank.cluster_type(),
|
cluster_type: bank.cluster_type(),
|
||||||
|
@ -99,9 +99,9 @@ impl AccountsPackage {
|
||||||
pub fn new_for_full_snapshot(
|
pub fn new_for_full_snapshot(
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
bank_snapshot_info: &BankSnapshotInfo,
|
bank_snapshot_info: &BankSnapshotInfo,
|
||||||
snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
||||||
snapshot_package_output_path: impl AsRef<Path>,
|
snapshot_archives_dir: impl AsRef<Path>,
|
||||||
snapshot_storages: SnapshotStorages,
|
snapshot_storages: SnapshotStorages,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
|
@ -115,13 +115,13 @@ impl AccountsPackage {
|
||||||
|
|
||||||
let snapshot_tmpdir = tempfile::Builder::new()
|
let snapshot_tmpdir = tempfile::Builder::new()
|
||||||
.prefix(&format!("{}{}-", TMP_FULL_SNAPSHOT_PREFIX, bank.slot()))
|
.prefix(&format!("{}{}-", TMP_FULL_SNAPSHOT_PREFIX, bank.slot()))
|
||||||
.tempdir_in(snapshots_dir)?;
|
.tempdir_in(bank_snapshots_dir)?;
|
||||||
|
|
||||||
Self::new(
|
Self::new(
|
||||||
bank,
|
bank,
|
||||||
bank_snapshot_info,
|
bank_snapshot_info,
|
||||||
status_cache_slot_deltas,
|
status_cache_slot_deltas,
|
||||||
snapshot_package_output_path,
|
snapshot_archives_dir,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -136,9 +136,9 @@ impl AccountsPackage {
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
incremental_snapshot_base_slot: Slot,
|
incremental_snapshot_base_slot: Slot,
|
||||||
bank_snapshot_info: &BankSnapshotInfo,
|
bank_snapshot_info: &BankSnapshotInfo,
|
||||||
snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
||||||
snapshot_package_output_path: impl AsRef<Path>,
|
snapshot_archives_dir: impl AsRef<Path>,
|
||||||
snapshot_storages: SnapshotStorages,
|
snapshot_storages: SnapshotStorages,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
|
@ -165,13 +165,13 @@ impl AccountsPackage {
|
||||||
incremental_snapshot_base_slot,
|
incremental_snapshot_base_slot,
|
||||||
bank.slot()
|
bank.slot()
|
||||||
))
|
))
|
||||||
.tempdir_in(snapshots_dir)?;
|
.tempdir_in(bank_snapshots_dir)?;
|
||||||
|
|
||||||
Self::new(
|
Self::new(
|
||||||
bank,
|
bank,
|
||||||
bank_snapshot_info,
|
bank_snapshot_info,
|
||||||
status_cache_slot_deltas,
|
status_cache_slot_deltas,
|
||||||
snapshot_package_output_path,
|
snapshot_archives_dir,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
|
|
@ -217,7 +217,7 @@ fn get_archive_ext(archive_format: ArchiveFormat) -> &'static str {
|
||||||
|
|
||||||
/// If the validator halts in the middle of `archive_snapshot_package()`, the temporary staging
|
/// If the validator halts in the middle of `archive_snapshot_package()`, the temporary staging
|
||||||
/// directory won't be cleaned up. Call this function to clean them up.
|
/// directory won't be cleaned up. Call this function to clean them up.
|
||||||
pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: &Path) {
|
pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: impl AsRef<Path>) {
|
||||||
if let Ok(entries) = fs::read_dir(snapshot_archives_dir) {
|
if let Ok(entries) = fs::read_dir(snapshot_archives_dir) {
|
||||||
for entry in entries.filter_map(|entry| entry.ok()) {
|
for entry in entries.filter_map(|entry| entry.ok()) {
|
||||||
let file_name = entry
|
let file_name = entry
|
||||||
|
@ -393,11 +393,11 @@ pub fn archive_snapshot_package(
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get a list of bank snapshots in a directory
|
/// Get a list of bank snapshots in a directory
|
||||||
pub fn get_bank_snapshots<P>(snapshots_dir: P) -> Vec<BankSnapshotInfo>
|
pub fn get_bank_snapshots<P>(bank_snapshots_dir: P) -> Vec<BankSnapshotInfo>
|
||||||
where
|
where
|
||||||
P: AsRef<Path>,
|
P: AsRef<Path>,
|
||||||
{
|
{
|
||||||
match fs::read_dir(&snapshots_dir) {
|
match fs::read_dir(&bank_snapshots_dir) {
|
||||||
Ok(paths) => paths
|
Ok(paths) => paths
|
||||||
.filter_map(|entry| {
|
.filter_map(|entry| {
|
||||||
entry.ok().and_then(|e| {
|
entry.ok().and_then(|e| {
|
||||||
|
@ -411,7 +411,7 @@ where
|
||||||
let snapshot_file_name = get_snapshot_file_name(slot);
|
let snapshot_file_name = get_snapshot_file_name(slot);
|
||||||
// So nice I join-ed it twice! The redundant `snapshot_file_name` is unintentional
|
// So nice I join-ed it twice! The redundant `snapshot_file_name` is unintentional
|
||||||
// and should be simplified. Kept for compatibility.
|
// and should be simplified. Kept for compatibility.
|
||||||
let snapshot_path = snapshots_dir
|
let snapshot_path = bank_snapshots_dir
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.join(&snapshot_file_name)
|
.join(&snapshot_file_name)
|
||||||
.join(snapshot_file_name);
|
.join(snapshot_file_name);
|
||||||
|
@ -424,7 +424,7 @@ where
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
info!(
|
info!(
|
||||||
"Unable to read snapshots directory {}: {}",
|
"Unable to read snapshots directory {}: {}",
|
||||||
snapshots_dir.as_ref().display(),
|
bank_snapshots_dir.as_ref().display(),
|
||||||
err
|
err
|
||||||
);
|
);
|
||||||
vec![]
|
vec![]
|
||||||
|
@ -433,11 +433,11 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the bank snapshot with the highest slot in a directory
|
/// Get the bank snapshot with the highest slot in a directory
|
||||||
pub fn get_highest_bank_snapshot_info<P>(snapshots_dir: P) -> Option<BankSnapshotInfo>
|
pub fn get_highest_bank_snapshot_info<P>(bank_snapshots_dir: P) -> Option<BankSnapshotInfo>
|
||||||
where
|
where
|
||||||
P: AsRef<Path>,
|
P: AsRef<Path>,
|
||||||
{
|
{
|
||||||
let mut bank_snapshot_infos = get_bank_snapshots(snapshots_dir);
|
let mut bank_snapshot_infos = get_bank_snapshots(bank_snapshots_dir);
|
||||||
bank_snapshot_infos.sort_unstable();
|
bank_snapshot_infos.sort_unstable();
|
||||||
bank_snapshot_infos.into_iter().rev().next()
|
bank_snapshot_infos.into_iter().rev().next()
|
||||||
}
|
}
|
||||||
|
@ -613,17 +613,17 @@ where
|
||||||
|
|
||||||
/// Serialize a bank to a snapshot
|
/// Serialize a bank to a snapshot
|
||||||
pub fn add_bank_snapshot<P: AsRef<Path>>(
|
pub fn add_bank_snapshot<P: AsRef<Path>>(
|
||||||
snapshots_dir: P,
|
bank_snapshots_dir: P,
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
snapshot_storages: &[SnapshotStorage],
|
snapshot_storages: &[SnapshotStorage],
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
) -> Result<BankSnapshotInfo> {
|
) -> Result<BankSnapshotInfo> {
|
||||||
let slot = bank.slot();
|
let slot = bank.slot();
|
||||||
// snapshots_dir/slot
|
// bank_snapshots_dir/slot
|
||||||
let bank_snapshots_dir = get_bank_snapshots_dir(snapshots_dir, slot);
|
let bank_snapshots_dir = get_bank_snapshots_dir(bank_snapshots_dir, slot);
|
||||||
fs::create_dir_all(&bank_snapshots_dir)?;
|
fs::create_dir_all(&bank_snapshots_dir)?;
|
||||||
|
|
||||||
// the bank snapshot is stored as snapshots_dir/slot/slot
|
// the bank snapshot is stored as bank_snapshots_dir/slot/slot
|
||||||
let snapshot_bank_file_path = bank_snapshots_dir.join(get_snapshot_file_name(slot));
|
let snapshot_bank_file_path = bank_snapshots_dir.join(get_snapshot_file_name(slot));
|
||||||
info!(
|
info!(
|
||||||
"Creating snapshot for slot {}, path: {:?}",
|
"Creating snapshot for slot {}, path: {:?}",
|
||||||
|
@ -689,11 +689,11 @@ fn serialize_status_cache(
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Remove the snapshot directory for this slot
|
/// Remove the snapshot directory for this slot
|
||||||
pub fn remove_bank_snapshot<P>(slot: Slot, snapshots_dir: P) -> Result<()>
|
pub fn remove_bank_snapshot<P>(slot: Slot, bank_snapshots_dir: P) -> Result<()>
|
||||||
where
|
where
|
||||||
P: AsRef<Path>,
|
P: AsRef<Path>,
|
||||||
{
|
{
|
||||||
let bank_snapshot_dir = get_bank_snapshots_dir(&snapshots_dir, slot);
|
let bank_snapshot_dir = get_bank_snapshots_dir(&bank_snapshots_dir, slot);
|
||||||
fs::remove_dir_all(bank_snapshot_dir)?;
|
fs::remove_dir_all(bank_snapshot_dir)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -715,7 +715,7 @@ const PARALLEL_UNTAR_READERS_DEFAULT: usize = 4;
|
||||||
pub fn bank_from_snapshot_archives(
|
pub fn bank_from_snapshot_archives(
|
||||||
account_paths: &[PathBuf],
|
account_paths: &[PathBuf],
|
||||||
frozen_account_pubkeys: &[Pubkey],
|
frozen_account_pubkeys: &[Pubkey],
|
||||||
snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
full_snapshot_archive_info: &FullSnapshotArchiveInfo,
|
full_snapshot_archive_info: &FullSnapshotArchiveInfo,
|
||||||
incremental_snapshot_archive_info: Option<&IncrementalSnapshotArchiveInfo>,
|
incremental_snapshot_archive_info: Option<&IncrementalSnapshotArchiveInfo>,
|
||||||
genesis_config: &GenesisConfig,
|
genesis_config: &GenesisConfig,
|
||||||
|
@ -741,7 +741,7 @@ pub fn bank_from_snapshot_archives(
|
||||||
);
|
);
|
||||||
|
|
||||||
let unarchived_full_snapshot = unarchive_snapshot(
|
let unarchived_full_snapshot = unarchive_snapshot(
|
||||||
&snapshots_dir,
|
&bank_snapshots_dir,
|
||||||
TMP_FULL_SNAPSHOT_PREFIX,
|
TMP_FULL_SNAPSHOT_PREFIX,
|
||||||
full_snapshot_archive_info.path(),
|
full_snapshot_archive_info.path(),
|
||||||
"snapshot untar",
|
"snapshot untar",
|
||||||
|
@ -753,7 +753,7 @@ pub fn bank_from_snapshot_archives(
|
||||||
let mut unarchived_incremental_snapshot =
|
let mut unarchived_incremental_snapshot =
|
||||||
if let Some(incremental_snapshot_archive_info) = incremental_snapshot_archive_info {
|
if let Some(incremental_snapshot_archive_info) = incremental_snapshot_archive_info {
|
||||||
let unarchived_incremental_snapshot = unarchive_snapshot(
|
let unarchived_incremental_snapshot = unarchive_snapshot(
|
||||||
&snapshots_dir,
|
&bank_snapshots_dir,
|
||||||
TMP_INCREMENTAL_SNAPSHOT_PREFIX,
|
TMP_INCREMENTAL_SNAPSHOT_PREFIX,
|
||||||
incremental_snapshot_archive_info.path(),
|
incremental_snapshot_archive_info.path(),
|
||||||
"incremental snapshot untar",
|
"incremental snapshot untar",
|
||||||
|
@ -824,7 +824,7 @@ pub fn bank_from_snapshot_archives(
|
||||||
/// highest full snapshot and highest corresponding incremental snapshot, then rebuilds the bank.
|
/// highest full snapshot and highest corresponding incremental snapshot, then rebuilds the bank.
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub fn bank_from_latest_snapshot_archives(
|
pub fn bank_from_latest_snapshot_archives(
|
||||||
snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
snapshot_archives_dir: impl AsRef<Path>,
|
||||||
account_paths: &[PathBuf],
|
account_paths: &[PathBuf],
|
||||||
frozen_account_pubkeys: &[Pubkey],
|
frozen_account_pubkeys: &[Pubkey],
|
||||||
|
@ -863,7 +863,7 @@ pub fn bank_from_latest_snapshot_archives(
|
||||||
let (bank, timings) = bank_from_snapshot_archives(
|
let (bank, timings) = bank_from_snapshot_archives(
|
||||||
account_paths,
|
account_paths,
|
||||||
frozen_account_pubkeys,
|
frozen_account_pubkeys,
|
||||||
snapshots_dir.as_ref(),
|
bank_snapshots_dir.as_ref(),
|
||||||
&full_snapshot_archive_info,
|
&full_snapshot_archive_info,
|
||||||
incremental_snapshot_archive_info.as_ref(),
|
incremental_snapshot_archive_info.as_ref(),
|
||||||
genesis_config,
|
genesis_config,
|
||||||
|
@ -918,7 +918,7 @@ fn verify_bank_against_expected_slot_hash(
|
||||||
/// directories, untaring, reading the version file, and then returning those fields plus the
|
/// directories, untaring, reading the version file, and then returning those fields plus the
|
||||||
/// unpacked append vec map.
|
/// unpacked append vec map.
|
||||||
fn unarchive_snapshot<P, Q>(
|
fn unarchive_snapshot<P, Q>(
|
||||||
snapshots_dir: P,
|
bank_snapshots_dir: P,
|
||||||
unpacked_snapshots_dir_prefix: &'static str,
|
unpacked_snapshots_dir_prefix: &'static str,
|
||||||
snapshot_archive_path: Q,
|
snapshot_archive_path: Q,
|
||||||
measure_name: &'static str,
|
measure_name: &'static str,
|
||||||
|
@ -932,7 +932,7 @@ where
|
||||||
{
|
{
|
||||||
let unpack_dir = tempfile::Builder::new()
|
let unpack_dir = tempfile::Builder::new()
|
||||||
.prefix(unpacked_snapshots_dir_prefix)
|
.prefix(unpacked_snapshots_dir_prefix)
|
||||||
.tempdir_in(snapshots_dir)?;
|
.tempdir_in(bank_snapshots_dir)?;
|
||||||
let unpacked_snapshots_dir = unpack_dir.path().join("snapshots");
|
let unpacked_snapshots_dir = unpack_dir.path().join("snapshots");
|
||||||
|
|
||||||
let mut measure_untar = Measure::start(measure_name);
|
let mut measure_untar = Measure::start(measure_name);
|
||||||
|
@ -1509,18 +1509,18 @@ pub fn verify_snapshot_archive<P, Q, R>(
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Remove outdated bank snapshots
|
/// Remove outdated bank snapshots
|
||||||
pub fn purge_old_bank_snapshots<P>(snapshots_dir: P)
|
pub fn purge_old_bank_snapshots<P>(bank_snapshots_dir: P)
|
||||||
where
|
where
|
||||||
P: AsRef<Path>,
|
P: AsRef<Path>,
|
||||||
{
|
{
|
||||||
let mut bank_snapshot_infos = get_bank_snapshots(&snapshots_dir);
|
let mut bank_snapshot_infos = get_bank_snapshots(&bank_snapshots_dir);
|
||||||
bank_snapshot_infos.sort_unstable();
|
bank_snapshot_infos.sort_unstable();
|
||||||
bank_snapshot_infos
|
bank_snapshot_infos
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.rev()
|
.rev()
|
||||||
.skip(MAX_BANK_SNAPSHOTS)
|
.skip(MAX_BANK_SNAPSHOTS)
|
||||||
.for_each(|bank_snapshot_info| {
|
.for_each(|bank_snapshot_info| {
|
||||||
let r = remove_bank_snapshot(bank_snapshot_info.slot, &snapshots_dir);
|
let r = remove_bank_snapshot(bank_snapshot_info.slot, &bank_snapshots_dir);
|
||||||
if r.is_err() {
|
if r.is_err() {
|
||||||
warn!(
|
warn!(
|
||||||
"Couldn't remove snapshot at: {}",
|
"Couldn't remove snapshot at: {}",
|
||||||
|
@ -1535,28 +1535,28 @@ pub fn snapshot_bank(
|
||||||
root_bank: &Bank,
|
root_bank: &Bank,
|
||||||
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
||||||
accounts_package_sender: &AccountsPackageSender,
|
accounts_package_sender: &AccountsPackageSender,
|
||||||
snapshots_dir: &Path,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
snapshot_package_output_path: &Path,
|
snapshot_archives_dir: impl AsRef<Path>,
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
archive_format: &ArchiveFormat,
|
archive_format: &ArchiveFormat,
|
||||||
hash_for_testing: Option<Hash>,
|
hash_for_testing: Option<Hash>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let storages = root_bank.get_snapshot_storages(None);
|
let storages = root_bank.get_snapshot_storages(None);
|
||||||
let mut add_snapshot_time = Measure::start("add-snapshot-ms");
|
let mut add_snapshot_time = Measure::start("add-snapshot-ms");
|
||||||
add_bank_snapshot(snapshots_dir, root_bank, &storages, snapshot_version)?;
|
add_bank_snapshot(&bank_snapshots_dir, root_bank, &storages, snapshot_version)?;
|
||||||
add_snapshot_time.stop();
|
add_snapshot_time.stop();
|
||||||
inc_new_counter_info!("add-snapshot-ms", add_snapshot_time.as_ms() as usize);
|
inc_new_counter_info!("add-snapshot-ms", add_snapshot_time.as_ms() as usize);
|
||||||
|
|
||||||
// Package the relevant snapshots
|
// Package the relevant snapshots
|
||||||
let highest_bank_snapshot_info = get_highest_bank_snapshot_info(snapshots_dir)
|
let highest_bank_snapshot_info = get_highest_bank_snapshot_info(&bank_snapshots_dir)
|
||||||
.expect("no snapshots found in config snapshots_dir");
|
.expect("no snapshots found in config bank_snapshots_dir");
|
||||||
|
|
||||||
let accounts_package = AccountsPackage::new_for_full_snapshot(
|
let accounts_package = AccountsPackage::new_for_full_snapshot(
|
||||||
root_bank,
|
root_bank,
|
||||||
&highest_bank_snapshot_info,
|
&highest_bank_snapshot_info,
|
||||||
snapshots_dir,
|
bank_snapshots_dir,
|
||||||
status_cache_slot_deltas,
|
status_cache_slot_deltas,
|
||||||
snapshot_package_output_path,
|
snapshot_archives_dir,
|
||||||
storages,
|
storages,
|
||||||
*archive_format,
|
*archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -1574,10 +1574,10 @@ pub fn snapshot_bank(
|
||||||
/// Requires:
|
/// Requires:
|
||||||
/// - `bank` is complete
|
/// - `bank` is complete
|
||||||
pub fn bank_to_full_snapshot_archive(
|
pub fn bank_to_full_snapshot_archive(
|
||||||
snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
snapshot_version: Option<SnapshotVersion>,
|
snapshot_version: Option<SnapshotVersion>,
|
||||||
snapshot_package_output_path: impl AsRef<Path>,
|
snapshot_archives_dir: impl AsRef<Path>,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
thread_pool: Option<&ThreadPool>,
|
thread_pool: Option<&ThreadPool>,
|
||||||
maximum_snapshots_to_retain: usize,
|
maximum_snapshots_to_retain: usize,
|
||||||
|
@ -1591,7 +1591,7 @@ pub fn bank_to_full_snapshot_archive(
|
||||||
bank.update_accounts_hash();
|
bank.update_accounts_hash();
|
||||||
bank.rehash(); // Bank accounts may have been manually modified by the caller
|
bank.rehash(); // Bank accounts may have been manually modified by the caller
|
||||||
|
|
||||||
let temp_dir = tempfile::tempdir_in(snapshots_dir)?;
|
let temp_dir = tempfile::tempdir_in(bank_snapshots_dir)?;
|
||||||
let storages = bank.get_snapshot_storages(None);
|
let storages = bank.get_snapshot_storages(None);
|
||||||
let bank_snapshot_info = add_bank_snapshot(&temp_dir, bank, &storages, snapshot_version)?;
|
let bank_snapshot_info = add_bank_snapshot(&temp_dir, bank, &storages, snapshot_version)?;
|
||||||
|
|
||||||
|
@ -1599,7 +1599,7 @@ pub fn bank_to_full_snapshot_archive(
|
||||||
bank,
|
bank,
|
||||||
&bank_snapshot_info,
|
&bank_snapshot_info,
|
||||||
&temp_dir,
|
&temp_dir,
|
||||||
snapshot_package_output_path,
|
snapshot_archives_dir,
|
||||||
storages,
|
storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -1615,11 +1615,11 @@ pub fn bank_to_full_snapshot_archive(
|
||||||
/// - `bank` is complete
|
/// - `bank` is complete
|
||||||
/// - `bank`'s slot is greater than `full_snapshot_slot`
|
/// - `bank`'s slot is greater than `full_snapshot_slot`
|
||||||
pub fn bank_to_incremental_snapshot_archive(
|
pub fn bank_to_incremental_snapshot_archive(
|
||||||
snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
full_snapshot_slot: Slot,
|
full_snapshot_slot: Slot,
|
||||||
snapshot_version: Option<SnapshotVersion>,
|
snapshot_version: Option<SnapshotVersion>,
|
||||||
snapshot_package_output_path: impl AsRef<Path>,
|
snapshot_archives_dir: impl AsRef<Path>,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
thread_pool: Option<&ThreadPool>,
|
thread_pool: Option<&ThreadPool>,
|
||||||
maximum_snapshots_to_retain: usize,
|
maximum_snapshots_to_retain: usize,
|
||||||
|
@ -1634,7 +1634,7 @@ pub fn bank_to_incremental_snapshot_archive(
|
||||||
bank.update_accounts_hash();
|
bank.update_accounts_hash();
|
||||||
bank.rehash(); // Bank accounts may have been manually modified by the caller
|
bank.rehash(); // Bank accounts may have been manually modified by the caller
|
||||||
|
|
||||||
let temp_dir = tempfile::tempdir_in(snapshots_dir)?;
|
let temp_dir = tempfile::tempdir_in(bank_snapshots_dir)?;
|
||||||
let storages = {
|
let storages = {
|
||||||
let mut storages = bank.get_snapshot_storages(Some(full_snapshot_slot));
|
let mut storages = bank.get_snapshot_storages(Some(full_snapshot_slot));
|
||||||
filter_snapshot_storages_for_incremental_snapshot(&mut storages, full_snapshot_slot);
|
filter_snapshot_storages_for_incremental_snapshot(&mut storages, full_snapshot_slot);
|
||||||
|
@ -1647,7 +1647,7 @@ pub fn bank_to_incremental_snapshot_archive(
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
&bank_snapshot_info,
|
&bank_snapshot_info,
|
||||||
&temp_dir,
|
&temp_dir,
|
||||||
snapshot_package_output_path,
|
snapshot_archives_dir,
|
||||||
storages,
|
storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -1660,8 +1660,8 @@ pub fn bank_to_incremental_snapshot_archive(
|
||||||
pub fn package_process_and_archive_full_snapshot(
|
pub fn package_process_and_archive_full_snapshot(
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
bank_snapshot_info: &BankSnapshotInfo,
|
bank_snapshot_info: &BankSnapshotInfo,
|
||||||
snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
snapshot_package_output_path: impl AsRef<Path>,
|
snapshot_archives_dir: impl AsRef<Path>,
|
||||||
snapshot_storages: SnapshotStorages,
|
snapshot_storages: SnapshotStorages,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
|
@ -1671,9 +1671,9 @@ pub fn package_process_and_archive_full_snapshot(
|
||||||
let accounts_package = AccountsPackage::new_for_full_snapshot(
|
let accounts_package = AccountsPackage::new_for_full_snapshot(
|
||||||
bank,
|
bank,
|
||||||
bank_snapshot_info,
|
bank_snapshot_info,
|
||||||
snapshots_dir,
|
bank_snapshots_dir,
|
||||||
bank.src.slot_deltas(&bank.src.roots()),
|
bank.src.slot_deltas(&bank.src.roots()),
|
||||||
snapshot_package_output_path,
|
snapshot_archives_dir,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -1698,8 +1698,8 @@ pub fn package_process_and_archive_incremental_snapshot(
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
incremental_snapshot_base_slot: Slot,
|
incremental_snapshot_base_slot: Slot,
|
||||||
bank_snapshot_info: &BankSnapshotInfo,
|
bank_snapshot_info: &BankSnapshotInfo,
|
||||||
snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
snapshot_package_output_path: impl AsRef<Path>,
|
snapshot_archives_dir: impl AsRef<Path>,
|
||||||
snapshot_storages: SnapshotStorages,
|
snapshot_storages: SnapshotStorages,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
|
@ -1710,9 +1710,9 @@ pub fn package_process_and_archive_incremental_snapshot(
|
||||||
bank,
|
bank,
|
||||||
incremental_snapshot_base_slot,
|
incremental_snapshot_base_slot,
|
||||||
bank_snapshot_info,
|
bank_snapshot_info,
|
||||||
snapshots_dir,
|
bank_snapshots_dir,
|
||||||
bank.src.slot_deltas(&bank.src.roots()),
|
bank.src.slot_deltas(&bank.src.roots()),
|
||||||
snapshot_package_output_path,
|
snapshot_archives_dir,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -1782,13 +1782,13 @@ pub fn process_accounts_package(
|
||||||
|
|
||||||
let snapshot_archive_path = match incremental_snapshot_base_slot {
|
let snapshot_archive_path = match incremental_snapshot_base_slot {
|
||||||
None => build_full_snapshot_archive_path(
|
None => build_full_snapshot_archive_path(
|
||||||
accounts_package.snapshot_output_dir,
|
accounts_package.snapshot_archives_dir,
|
||||||
accounts_package.slot,
|
accounts_package.slot,
|
||||||
&hash,
|
&hash,
|
||||||
accounts_package.archive_format,
|
accounts_package.archive_format,
|
||||||
),
|
),
|
||||||
Some(incremental_snapshot_base_slot) => build_incremental_snapshot_archive_path(
|
Some(incremental_snapshot_base_slot) => build_incremental_snapshot_archive_path(
|
||||||
accounts_package.snapshot_output_dir,
|
accounts_package.snapshot_archives_dir,
|
||||||
incremental_snapshot_base_slot,
|
incremental_snapshot_base_slot,
|
||||||
accounts_package.slot,
|
accounts_package.slot,
|
||||||
&hash,
|
&hash,
|
||||||
|
@ -2146,9 +2146,13 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A test heler function that creates bank snapshot files
|
/// A test heler function that creates bank snapshot files
|
||||||
fn common_create_bank_snapshot_files(snapshots_dir: &Path, min_slot: Slot, max_slot: Slot) {
|
fn common_create_bank_snapshot_files(
|
||||||
|
bank_snapshots_dir: &Path,
|
||||||
|
min_slot: Slot,
|
||||||
|
max_slot: Slot,
|
||||||
|
) {
|
||||||
for slot in min_slot..max_slot {
|
for slot in min_slot..max_slot {
|
||||||
let snapshot_dir = get_bank_snapshots_dir(snapshots_dir, slot);
|
let snapshot_dir = get_bank_snapshots_dir(bank_snapshots_dir, slot);
|
||||||
fs::create_dir_all(&snapshot_dir).unwrap();
|
fs::create_dir_all(&snapshot_dir).unwrap();
|
||||||
|
|
||||||
let snapshot_filename = get_snapshot_file_name(slot);
|
let snapshot_filename = get_snapshot_file_name(slot);
|
||||||
|
@ -2494,12 +2498,12 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshots_dir = tempfile::TempDir::new().unwrap();
|
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::Tar;
|
let snapshot_archive_format = ArchiveFormat::Tar;
|
||||||
|
|
||||||
let snapshot_archive_info = bank_to_full_snapshot_archive(
|
let snapshot_archive_info = bank_to_full_snapshot_archive(
|
||||||
&snapshots_dir,
|
&bank_snapshots_dir,
|
||||||
&original_bank,
|
&original_bank,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
snapshot_archives_dir.path(),
|
||||||
|
@ -2512,7 +2516,7 @@ mod tests {
|
||||||
let (roundtrip_bank, _) = bank_from_snapshot_archives(
|
let (roundtrip_bank, _) = bank_from_snapshot_archives(
|
||||||
&[PathBuf::from(accounts_dir.path())],
|
&[PathBuf::from(accounts_dir.path())],
|
||||||
&[],
|
&[],
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&snapshot_archive_info,
|
&snapshot_archive_info,
|
||||||
None,
|
None,
|
||||||
&genesis_config,
|
&genesis_config,
|
||||||
|
@ -2585,12 +2589,12 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshots_dir = tempfile::TempDir::new().unwrap();
|
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::TarGzip;
|
let snapshot_archive_format = ArchiveFormat::TarGzip;
|
||||||
|
|
||||||
let full_snapshot_archive_info = bank_to_full_snapshot_archive(
|
let full_snapshot_archive_info = bank_to_full_snapshot_archive(
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&bank4,
|
&bank4,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
snapshot_archives_dir.path(),
|
||||||
|
@ -2603,7 +2607,7 @@ mod tests {
|
||||||
let (roundtrip_bank, _) = bank_from_snapshot_archives(
|
let (roundtrip_bank, _) = bank_from_snapshot_archives(
|
||||||
&[PathBuf::from(accounts_dir.path())],
|
&[PathBuf::from(accounts_dir.path())],
|
||||||
&[],
|
&[],
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&full_snapshot_archive_info,
|
&full_snapshot_archive_info,
|
||||||
None,
|
None,
|
||||||
&genesis_config,
|
&genesis_config,
|
||||||
|
@ -2661,13 +2665,13 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshots_dir = tempfile::TempDir::new().unwrap();
|
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::TarZstd;
|
let snapshot_archive_format = ArchiveFormat::TarZstd;
|
||||||
|
|
||||||
let full_snapshot_slot = slot;
|
let full_snapshot_slot = slot;
|
||||||
let full_snapshot_archive_info = bank_to_full_snapshot_archive(
|
let full_snapshot_archive_info = bank_to_full_snapshot_archive(
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&bank1,
|
&bank1,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
snapshot_archives_dir.path(),
|
||||||
|
@ -2699,7 +2703,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive(
|
let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive(
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&bank4,
|
&bank4,
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
None,
|
None,
|
||||||
|
@ -2713,7 +2717,7 @@ mod tests {
|
||||||
let (roundtrip_bank, _) = bank_from_snapshot_archives(
|
let (roundtrip_bank, _) = bank_from_snapshot_archives(
|
||||||
&[PathBuf::from(accounts_dir.path())],
|
&[PathBuf::from(accounts_dir.path())],
|
||||||
&[],
|
&[],
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&full_snapshot_archive_info,
|
&full_snapshot_archive_info,
|
||||||
Some(&incremental_snapshot_archive_info),
|
Some(&incremental_snapshot_archive_info),
|
||||||
&genesis_config,
|
&genesis_config,
|
||||||
|
@ -2761,13 +2765,13 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshots_dir = tempfile::TempDir::new().unwrap();
|
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::Tar;
|
let snapshot_archive_format = ArchiveFormat::Tar;
|
||||||
|
|
||||||
let full_snapshot_slot = slot;
|
let full_snapshot_slot = slot;
|
||||||
bank_to_full_snapshot_archive(
|
bank_to_full_snapshot_archive(
|
||||||
&snapshots_dir,
|
&bank_snapshots_dir,
|
||||||
&bank1,
|
&bank1,
|
||||||
None,
|
None,
|
||||||
&snapshot_archives_dir,
|
&snapshot_archives_dir,
|
||||||
|
@ -2799,7 +2803,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
bank_to_incremental_snapshot_archive(
|
bank_to_incremental_snapshot_archive(
|
||||||
&snapshots_dir,
|
&bank_snapshots_dir,
|
||||||
&bank4,
|
&bank4,
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
None,
|
None,
|
||||||
|
@ -2811,7 +2815,7 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let (deserialized_bank, _) = bank_from_latest_snapshot_archives(
|
let (deserialized_bank, _) = bank_from_latest_snapshot_archives(
|
||||||
&snapshots_dir,
|
&bank_snapshots_dir,
|
||||||
&snapshot_archives_dir,
|
&snapshot_archives_dir,
|
||||||
&[accounts_dir.as_ref().to_path_buf()],
|
&[accounts_dir.as_ref().to_path_buf()],
|
||||||
&[],
|
&[],
|
||||||
|
@ -2863,7 +2867,7 @@ mod tests {
|
||||||
let key2 = Keypair::new();
|
let key2 = Keypair::new();
|
||||||
|
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshots_dir = tempfile::TempDir::new().unwrap();
|
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::Tar;
|
let snapshot_archive_format = ArchiveFormat::Tar;
|
||||||
|
|
||||||
|
@ -2899,7 +2903,7 @@ mod tests {
|
||||||
|
|
||||||
let full_snapshot_slot = slot;
|
let full_snapshot_slot = slot;
|
||||||
let full_snapshot_archive_info = bank_to_full_snapshot_archive(
|
let full_snapshot_archive_info = bank_to_full_snapshot_archive(
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&bank1,
|
&bank1,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
snapshot_archives_dir.path(),
|
||||||
|
@ -2940,7 +2944,7 @@ mod tests {
|
||||||
// Take an incremental snapshot and then do a roundtrip on the bank and ensure it
|
// Take an incremental snapshot and then do a roundtrip on the bank and ensure it
|
||||||
// deserializes correctly.
|
// deserializes correctly.
|
||||||
let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive(
|
let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive(
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&bank2,
|
&bank2,
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
None,
|
None,
|
||||||
|
@ -2953,7 +2957,7 @@ mod tests {
|
||||||
let (deserialized_bank, _) = bank_from_snapshot_archives(
|
let (deserialized_bank, _) = bank_from_snapshot_archives(
|
||||||
&[accounts_dir.path().to_path_buf()],
|
&[accounts_dir.path().to_path_buf()],
|
||||||
&[],
|
&[],
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&full_snapshot_archive_info,
|
&full_snapshot_archive_info,
|
||||||
Some(&incremental_snapshot_archive_info),
|
Some(&incremental_snapshot_archive_info),
|
||||||
&genesis_config,
|
&genesis_config,
|
||||||
|
@ -3001,7 +3005,7 @@ mod tests {
|
||||||
// Take an incremental snapshot and then do a roundtrip on the bank and ensure it
|
// Take an incremental snapshot and then do a roundtrip on the bank and ensure it
|
||||||
// deserializes correctly
|
// deserializes correctly
|
||||||
let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive(
|
let incremental_snapshot_archive_info = bank_to_incremental_snapshot_archive(
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&bank4,
|
&bank4,
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
None,
|
None,
|
||||||
|
@ -3015,7 +3019,7 @@ mod tests {
|
||||||
let (deserialized_bank, _) = bank_from_snapshot_archives(
|
let (deserialized_bank, _) = bank_from_snapshot_archives(
|
||||||
&[accounts_dir.path().to_path_buf()],
|
&[accounts_dir.path().to_path_buf()],
|
||||||
&[],
|
&[],
|
||||||
snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&full_snapshot_archive_info,
|
&full_snapshot_archive_info,
|
||||||
Some(&incremental_snapshot_archive_info),
|
Some(&incremental_snapshot_archive_info),
|
||||||
&genesis_config,
|
&genesis_config,
|
||||||
|
|
|
@ -403,7 +403,7 @@ fn get_rpc_node(
|
||||||
blacklisted_rpc_nodes: &mut HashSet<Pubkey>,
|
blacklisted_rpc_nodes: &mut HashSet<Pubkey>,
|
||||||
snapshot_not_required: bool,
|
snapshot_not_required: bool,
|
||||||
no_untrusted_rpc: bool,
|
no_untrusted_rpc: bool,
|
||||||
snapshot_output_dir: &Path,
|
snapshot_archives_dir: &Path,
|
||||||
) -> Option<(ContactInfo, Option<(Slot, Hash)>)> {
|
) -> Option<(ContactInfo, Option<(Slot, Hash)>)> {
|
||||||
let mut blacklist_timeout = Instant::now();
|
let mut blacklist_timeout = Instant::now();
|
||||||
let mut newer_cluster_snapshot_timeout = None;
|
let mut newer_cluster_snapshot_timeout = None;
|
||||||
|
@ -482,7 +482,7 @@ fn get_rpc_node(
|
||||||
blacklist_timeout = Instant::now();
|
blacklist_timeout = Instant::now();
|
||||||
|
|
||||||
let mut highest_snapshot_hash: Option<(Slot, Hash)> =
|
let mut highest_snapshot_hash: Option<(Slot, Hash)> =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_output_dir).map(
|
snapshot_utils::get_highest_full_snapshot_archive_info(snapshot_archives_dir).map(
|
||||||
|snapshot_archive_info| {
|
|snapshot_archive_info| {
|
||||||
(snapshot_archive_info.slot(), *snapshot_archive_info.hash())
|
(snapshot_archive_info.slot(), *snapshot_archive_info.hash())
|
||||||
},
|
},
|
||||||
|
@ -756,7 +756,7 @@ fn rpc_bootstrap(
|
||||||
node: &Node,
|
node: &Node,
|
||||||
identity_keypair: &Arc<Keypair>,
|
identity_keypair: &Arc<Keypair>,
|
||||||
ledger_path: &Path,
|
ledger_path: &Path,
|
||||||
snapshot_output_dir: &Path,
|
snapshot_archives_dir: &Path,
|
||||||
vote_account: &Pubkey,
|
vote_account: &Pubkey,
|
||||||
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
|
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
|
||||||
cluster_entrypoints: &[ContactInfo],
|
cluster_entrypoints: &[ContactInfo],
|
||||||
|
@ -817,7 +817,7 @@ fn rpc_bootstrap(
|
||||||
&mut blacklisted_rpc_nodes,
|
&mut blacklisted_rpc_nodes,
|
||||||
bootstrap_config.no_snapshot_fetch,
|
bootstrap_config.no_snapshot_fetch,
|
||||||
bootstrap_config.no_untrusted_rpc,
|
bootstrap_config.no_untrusted_rpc,
|
||||||
snapshot_output_dir,
|
snapshot_archives_dir,
|
||||||
);
|
);
|
||||||
if rpc_node_details.is_none() {
|
if rpc_node_details.is_none() {
|
||||||
return;
|
return;
|
||||||
|
@ -874,7 +874,7 @@ fn rpc_bootstrap(
|
||||||
let mut use_local_snapshot = false;
|
let mut use_local_snapshot = false;
|
||||||
|
|
||||||
if let Some(highest_local_snapshot_slot) =
|
if let Some(highest_local_snapshot_slot) =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_slot(snapshot_output_dir)
|
snapshot_utils::get_highest_full_snapshot_archive_slot(snapshot_archives_dir)
|
||||||
{
|
{
|
||||||
if highest_local_snapshot_slot
|
if highest_local_snapshot_slot
|
||||||
> snapshot_hash.0.saturating_sub(maximum_local_snapshot_age)
|
> snapshot_hash.0.saturating_sub(maximum_local_snapshot_age)
|
||||||
|
@ -920,7 +920,7 @@ fn rpc_bootstrap(
|
||||||
};
|
};
|
||||||
let ret = download_snapshot(
|
let ret = download_snapshot(
|
||||||
&rpc_contact_info.rpc,
|
&rpc_contact_info.rpc,
|
||||||
snapshot_output_dir,
|
snapshot_archives_dir,
|
||||||
snapshot_hash,
|
snapshot_hash,
|
||||||
use_progress_bar,
|
use_progress_bar,
|
||||||
maximum_snapshots_to_retain,
|
maximum_snapshots_to_retain,
|
||||||
|
@ -2575,16 +2575,16 @@ pub fn main() {
|
||||||
let maximum_snapshot_download_abort =
|
let maximum_snapshot_download_abort =
|
||||||
value_t_or_exit!(matches, "maximum_snapshot_download_abort", u64);
|
value_t_or_exit!(matches, "maximum_snapshot_download_abort", u64);
|
||||||
|
|
||||||
let snapshot_output_dir = if matches.is_present("snapshots") {
|
let snapshot_archives_dir = if matches.is_present("snapshots") {
|
||||||
PathBuf::from(matches.value_of("snapshots").unwrap())
|
PathBuf::from(matches.value_of("snapshots").unwrap())
|
||||||
} else {
|
} else {
|
||||||
ledger_path.clone()
|
ledger_path.clone()
|
||||||
};
|
};
|
||||||
let snapshot_path = snapshot_output_dir.join("snapshot");
|
let bank_snapshots_dir = snapshot_archives_dir.join("snapshot");
|
||||||
fs::create_dir_all(&snapshot_path).unwrap_or_else(|err| {
|
fs::create_dir_all(&bank_snapshots_dir).unwrap_or_else(|err| {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"Failed to create snapshots directory {:?}: {}",
|
"Failed to create snapshots directory {:?}: {}",
|
||||||
snapshot_path, err
|
bank_snapshots_dir, err
|
||||||
);
|
);
|
||||||
exit(1);
|
exit(1);
|
||||||
});
|
});
|
||||||
|
@ -2616,8 +2616,8 @@ pub fn main() {
|
||||||
std::u64::MAX
|
std::u64::MAX
|
||||||
},
|
},
|
||||||
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_path,
|
bank_snapshots_dir,
|
||||||
snapshot_package_output_path: snapshot_output_dir.clone(),
|
snapshot_archives_dir: snapshot_archives_dir.clone(),
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
maximum_snapshots_to_retain,
|
maximum_snapshots_to_retain,
|
||||||
|
@ -2784,7 +2784,7 @@ pub fn main() {
|
||||||
solana_metrics::set_panic_hook("validator");
|
solana_metrics::set_panic_hook("validator");
|
||||||
|
|
||||||
solana_entry::entry::init_poh();
|
solana_entry::entry::init_poh();
|
||||||
solana_runtime::snapshot_utils::remove_tmp_snapshot_archives(&snapshot_output_dir);
|
solana_runtime::snapshot_utils::remove_tmp_snapshot_archives(&snapshot_archives_dir);
|
||||||
|
|
||||||
let identity_keypair = Arc::new(identity_keypair);
|
let identity_keypair = Arc::new(identity_keypair);
|
||||||
|
|
||||||
|
@ -2794,7 +2794,7 @@ pub fn main() {
|
||||||
&node,
|
&node,
|
||||||
&identity_keypair,
|
&identity_keypair,
|
||||||
&ledger_path,
|
&ledger_path,
|
||||||
&snapshot_output_dir,
|
&snapshot_archives_dir,
|
||||||
&vote_account,
|
&vote_account,
|
||||||
authorized_voter_keypairs.clone(),
|
authorized_voter_keypairs.clone(),
|
||||||
&cluster_entrypoints,
|
&cluster_entrypoints,
|
||||||
|
|
Loading…
Reference in New Issue