Add options to store full and/or incremental snapshots in separate locations (#24247)
This commit is contained in:
parent
dc6e28099b
commit
2fa9bc3e70
|
@ -384,7 +384,8 @@ mod tests {
|
||||||
snapshot_storages: vec![],
|
snapshot_storages: vec![],
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
snapshot_version: SnapshotVersion::default(),
|
snapshot_version: SnapshotVersion::default(),
|
||||||
snapshot_archives_dir: PathBuf::default(),
|
full_snapshot_archives_dir: PathBuf::default(),
|
||||||
|
incremental_snapshot_archives_dir: PathBuf::default(),
|
||||||
expected_capitalization: 0,
|
expected_capitalization: 0,
|
||||||
accounts_hash_for_testing: None,
|
accounts_hash_for_testing: None,
|
||||||
cluster_type: ClusterType::MainnetBeta,
|
cluster_type: ClusterType::MainnetBeta,
|
||||||
|
|
|
@ -84,6 +84,8 @@ impl SnapshotPackagerService {
|
||||||
// last_full_snapshot_slot that requires this archive call to succeed.
|
// last_full_snapshot_slot that requires this archive call to succeed.
|
||||||
snapshot_utils::archive_snapshot_package(
|
snapshot_utils::archive_snapshot_package(
|
||||||
&snapshot_package,
|
&snapshot_package,
|
||||||
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
|
&snapshot_config.incremental_snapshot_archives_dir,
|
||||||
snapshot_config.maximum_full_snapshot_archives_to_retain,
|
snapshot_config.maximum_full_snapshot_archives_to_retain,
|
||||||
snapshot_config.maximum_incremental_snapshot_archives_to_retain,
|
snapshot_config.maximum_incremental_snapshot_archives_to_retain,
|
||||||
)
|
)
|
||||||
|
@ -259,8 +261,10 @@ mod tests {
|
||||||
fn create_and_verify_snapshot(temp_dir: &Path) {
|
fn create_and_verify_snapshot(temp_dir: &Path) {
|
||||||
let accounts_dir = temp_dir.join("accounts");
|
let accounts_dir = temp_dir.join("accounts");
|
||||||
let snapshots_dir = temp_dir.join("snapshots");
|
let snapshots_dir = temp_dir.join("snapshots");
|
||||||
let snapshot_archives_dir = temp_dir.join("snapshots_output");
|
let full_snapshot_archives_dir = temp_dir.join("full_snapshot_archives");
|
||||||
fs::create_dir_all(&snapshot_archives_dir).unwrap();
|
let incremental_snapshot_archives_dir = temp_dir.join("incremental_snapshot_archives");
|
||||||
|
fs::create_dir_all(&full_snapshot_archives_dir).unwrap();
|
||||||
|
fs::create_dir_all(&incremental_snapshot_archives_dir).unwrap();
|
||||||
|
|
||||||
fs::create_dir_all(&accounts_dir).unwrap();
|
fs::create_dir_all(&accounts_dir).unwrap();
|
||||||
// Create some storage entries
|
// Create some storage entries
|
||||||
|
@ -302,7 +306,7 @@ mod tests {
|
||||||
let hash = Hash::default();
|
let hash = Hash::default();
|
||||||
let archive_format = ArchiveFormat::TarBzip2;
|
let archive_format = ArchiveFormat::TarBzip2;
|
||||||
let output_tar_path = snapshot_utils::build_full_snapshot_archive_path(
|
let output_tar_path = snapshot_utils::build_full_snapshot_archive_path(
|
||||||
snapshot_archives_dir,
|
&full_snapshot_archives_dir,
|
||||||
slot,
|
slot,
|
||||||
&hash,
|
&hash,
|
||||||
archive_format,
|
archive_format,
|
||||||
|
@ -325,6 +329,8 @@ mod tests {
|
||||||
// Make tarball from packageable snapshot
|
// Make tarball from packageable snapshot
|
||||||
snapshot_utils::archive_snapshot_package(
|
snapshot_utils::archive_snapshot_package(
|
||||||
&snapshot_package,
|
&snapshot_package,
|
||||||
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
)
|
)
|
||||||
|
|
|
@ -1676,7 +1676,8 @@ fn maybe_warp_slot(
|
||||||
ledger_path,
|
ledger_path,
|
||||||
&bank_forks.root_bank(),
|
&bank_forks.root_bank(),
|
||||||
None,
|
None,
|
||||||
&snapshot_config.snapshot_archives_dir,
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
|
&snapshot_config.incremental_snapshot_archives_dir,
|
||||||
snapshot_config.archive_format,
|
snapshot_config.archive_format,
|
||||||
snapshot_config.maximum_full_snapshot_archives_to_retain,
|
snapshot_config.maximum_full_snapshot_archives_to_retain,
|
||||||
snapshot_config.maximum_incremental_snapshot_archives_to_retain,
|
snapshot_config.maximum_incremental_snapshot_archives_to_retain,
|
||||||
|
|
|
@ -107,7 +107,8 @@ mod tests {
|
||||||
struct SnapshotTestConfig {
|
struct SnapshotTestConfig {
|
||||||
accounts_dir: TempDir,
|
accounts_dir: TempDir,
|
||||||
bank_snapshots_dir: TempDir,
|
bank_snapshots_dir: TempDir,
|
||||||
snapshot_archives_dir: TempDir,
|
full_snapshot_archives_dir: TempDir,
|
||||||
|
incremental_snapshot_archives_dir: TempDir,
|
||||||
snapshot_config: SnapshotConfig,
|
snapshot_config: SnapshotConfig,
|
||||||
bank_forks: BankForks,
|
bank_forks: BankForks,
|
||||||
genesis_config_info: GenesisConfigInfo,
|
genesis_config_info: GenesisConfigInfo,
|
||||||
|
@ -123,7 +124,8 @@ mod tests {
|
||||||
) -> SnapshotTestConfig {
|
) -> SnapshotTestConfig {
|
||||||
let accounts_dir = TempDir::new().unwrap();
|
let accounts_dir = TempDir::new().unwrap();
|
||||||
let bank_snapshots_dir = TempDir::new().unwrap();
|
let bank_snapshots_dir = TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = TempDir::new().unwrap();
|
let full_snapshot_archives_dir = TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = TempDir::new().unwrap();
|
||||||
// validator_stake_lamports should be non-zero otherwise stake
|
// validator_stake_lamports should be non-zero otherwise stake
|
||||||
// account will not be stored in accounts-db but still cached in
|
// account will not be stored in accounts-db but still cached in
|
||||||
// bank stakes which results in mismatch when banks are loaded from
|
// bank stakes which results in mismatch when banks are loaded from
|
||||||
|
@ -151,7 +153,10 @@ mod tests {
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots,
|
full_snapshot_archive_interval_slots,
|
||||||
incremental_snapshot_archive_interval_slots,
|
incremental_snapshot_archive_interval_slots,
|
||||||
snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(),
|
full_snapshot_archives_dir: full_snapshot_archives_dir.path().to_path_buf(),
|
||||||
|
incremental_snapshot_archives_dir: incremental_snapshot_archives_dir
|
||||||
|
.path()
|
||||||
|
.to_path_buf(),
|
||||||
bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
|
bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
..SnapshotConfig::default()
|
..SnapshotConfig::default()
|
||||||
|
@ -160,7 +165,8 @@ mod tests {
|
||||||
SnapshotTestConfig {
|
SnapshotTestConfig {
|
||||||
accounts_dir,
|
accounts_dir,
|
||||||
bank_snapshots_dir,
|
bank_snapshots_dir,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
snapshot_config,
|
snapshot_config,
|
||||||
bank_forks,
|
bank_forks,
|
||||||
genesis_config_info,
|
genesis_config_info,
|
||||||
|
@ -174,17 +180,17 @@ mod tests {
|
||||||
old_genesis_config: &GenesisConfig,
|
old_genesis_config: &GenesisConfig,
|
||||||
account_paths: &[PathBuf],
|
account_paths: &[PathBuf],
|
||||||
) {
|
) {
|
||||||
let snapshot_archives_dir = old_bank_forks
|
let full_snapshot_archives_dir = old_bank_forks
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|c| &c.snapshot_archives_dir)
|
.map(|c| &c.full_snapshot_archives_dir)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
|
let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
|
||||||
|
|
||||||
let check_hash_calculation = false;
|
let check_hash_calculation = false;
|
||||||
let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path(
|
let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path(
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
old_last_bank.slot(),
|
old_last_bank.slot(),
|
||||||
&old_last_bank.get_accounts_hash(),
|
&old_last_bank.get_accounts_hash(),
|
||||||
ArchiveFormat::TarBzip2,
|
ArchiveFormat::TarBzip2,
|
||||||
|
@ -280,7 +286,8 @@ mod tests {
|
||||||
&last_bank_snapshot_info,
|
&last_bank_snapshot_info,
|
||||||
bank_snapshots_dir,
|
bank_snapshots_dir,
|
||||||
last_bank.src.slot_deltas(&last_bank.src.roots()),
|
last_bank.src.slot_deltas(&last_bank.src.roots()),
|
||||||
&snapshot_config.snapshot_archives_dir,
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
|
&snapshot_config.incremental_snapshot_archives_dir,
|
||||||
last_bank.get_snapshot_storages(None),
|
last_bank.get_snapshot_storages(None),
|
||||||
ArchiveFormat::TarBzip2,
|
ArchiveFormat::TarBzip2,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -297,6 +304,8 @@ mod tests {
|
||||||
SnapshotPackage::new(accounts_package, last_bank.get_accounts_hash());
|
SnapshotPackage::new(accounts_package, last_bank.get_accounts_hash());
|
||||||
snapshot_utils::archive_snapshot_package(
|
snapshot_utils::archive_snapshot_package(
|
||||||
&snapshot_package,
|
&snapshot_package,
|
||||||
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
|
&snapshot_config.incremental_snapshot_archives_dir,
|
||||||
snapshot_config.maximum_full_snapshot_archives_to_retain,
|
snapshot_config.maximum_full_snapshot_archives_to_retain,
|
||||||
snapshot_config.maximum_incremental_snapshot_archives_to_retain,
|
snapshot_config.maximum_incremental_snapshot_archives_to_retain,
|
||||||
)
|
)
|
||||||
|
@ -360,7 +369,8 @@ mod tests {
|
||||||
let bank_forks = &mut snapshot_test_config.bank_forks;
|
let bank_forks = &mut snapshot_test_config.bank_forks;
|
||||||
let snapshot_config = &snapshot_test_config.snapshot_config;
|
let snapshot_config = &snapshot_test_config.snapshot_config;
|
||||||
let bank_snapshots_dir = &snapshot_config.bank_snapshots_dir;
|
let bank_snapshots_dir = &snapshot_config.bank_snapshots_dir;
|
||||||
let snapshot_archives_dir = &snapshot_config.snapshot_archives_dir;
|
let full_snapshot_archives_dir = &snapshot_config.full_snapshot_archives_dir;
|
||||||
|
let incremental_snapshot_archives_dir = &snapshot_config.incremental_snapshot_archives_dir;
|
||||||
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
||||||
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
|
let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
|
||||||
|
|
||||||
|
@ -414,7 +424,8 @@ mod tests {
|
||||||
vec![],
|
vec![],
|
||||||
pending_accounts_package,
|
pending_accounts_package,
|
||||||
bank_snapshots_dir,
|
bank_snapshots_dir,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
snapshot_config.snapshot_version,
|
snapshot_config.snapshot_version,
|
||||||
snapshot_config.archive_format,
|
snapshot_config.archive_format,
|
||||||
None,
|
None,
|
||||||
|
@ -464,7 +475,7 @@ mod tests {
|
||||||
fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
|
fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
|
||||||
|
|
||||||
saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path(
|
saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path(
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
slot,
|
slot,
|
||||||
// this needs to match the hash value that we reserialize with later. It is complicated, so just use default.
|
// this needs to match the hash value that we reserialize with later. It is complicated, so just use default.
|
||||||
// This hash value is just used to build the file name. Since this is mocked up test code, it is sufficient to pass default here.
|
// This hash value is just used to build the file name. Since this is mocked up test code, it is sufficient to pass default here.
|
||||||
|
@ -679,7 +690,8 @@ mod tests {
|
||||||
FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
|
FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
|
||||||
INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
|
INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
|
||||||
);
|
);
|
||||||
trace!("SnapshotTestConfig:\naccounts_dir: {}\nbank_snapshots_dir: {}\nsnapshot_archives_dir: {}", snapshot_test_config.accounts_dir.path().display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.snapshot_archives_dir.path().display());
|
trace!("SnapshotTestConfig:\naccounts_dir: {}\nbank_snapshots_dir: {}\nfull_snapshot_archives_dir: {}\nincremental_snapshot_archives_dir: {}",
|
||||||
|
snapshot_test_config.accounts_dir.path().display(), snapshot_test_config.bank_snapshots_dir.path().display(), snapshot_test_config.full_snapshot_archives_dir.path().display(), snapshot_test_config.incremental_snapshot_archives_dir.path().display());
|
||||||
|
|
||||||
let bank_forks = &mut snapshot_test_config.bank_forks;
|
let bank_forks = &mut snapshot_test_config.bank_forks;
|
||||||
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
|
||||||
|
@ -783,7 +795,8 @@ mod tests {
|
||||||
bank,
|
bank,
|
||||||
&bank_snapshot_info,
|
&bank_snapshot_info,
|
||||||
&snapshot_config.bank_snapshots_dir,
|
&snapshot_config.bank_snapshots_dir,
|
||||||
&snapshot_config.snapshot_archives_dir,
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
|
&snapshot_config.incremental_snapshot_archives_dir,
|
||||||
bank.get_snapshot_storages(None),
|
bank.get_snapshot_storages(None),
|
||||||
snapshot_config.archive_format,
|
snapshot_config.archive_format,
|
||||||
snapshot_config.snapshot_version,
|
snapshot_config.snapshot_version,
|
||||||
|
@ -820,7 +833,8 @@ mod tests {
|
||||||
incremental_snapshot_base_slot,
|
incremental_snapshot_base_slot,
|
||||||
&bank_snapshot_info,
|
&bank_snapshot_info,
|
||||||
&snapshot_config.bank_snapshots_dir,
|
&snapshot_config.bank_snapshots_dir,
|
||||||
&snapshot_config.snapshot_archives_dir,
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
|
&snapshot_config.incremental_snapshot_archives_dir,
|
||||||
storages,
|
storages,
|
||||||
snapshot_config.archive_format,
|
snapshot_config.archive_format,
|
||||||
snapshot_config.snapshot_version,
|
snapshot_config.snapshot_version,
|
||||||
|
@ -839,7 +853,8 @@ mod tests {
|
||||||
) -> snapshot_utils::Result<()> {
|
) -> snapshot_utils::Result<()> {
|
||||||
let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives(
|
let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives(
|
||||||
&snapshot_config.bank_snapshots_dir,
|
&snapshot_config.bank_snapshots_dir,
|
||||||
&snapshot_config.snapshot_archives_dir,
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
|
&snapshot_config.incremental_snapshot_archives_dir,
|
||||||
&[accounts_dir],
|
&[accounts_dir],
|
||||||
genesis_config,
|
genesis_config,
|
||||||
None,
|
None,
|
||||||
|
@ -1015,7 +1030,12 @@ mod tests {
|
||||||
|
|
||||||
let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives(
|
let (deserialized_bank, ..) = snapshot_utils::bank_from_latest_snapshot_archives(
|
||||||
&snapshot_test_config.snapshot_config.bank_snapshots_dir,
|
&snapshot_test_config.snapshot_config.bank_snapshots_dir,
|
||||||
&snapshot_test_config.snapshot_config.snapshot_archives_dir,
|
&snapshot_test_config
|
||||||
|
.snapshot_config
|
||||||
|
.full_snapshot_archives_dir,
|
||||||
|
&snapshot_test_config
|
||||||
|
.snapshot_config
|
||||||
|
.incremental_snapshot_archives_dir,
|
||||||
&[snapshot_test_config.accounts_dir.as_ref().to_path_buf()],
|
&[snapshot_test_config.accounts_dir.as_ref().to_path_buf()],
|
||||||
&snapshot_test_config.genesis_config_info.genesis_config,
|
&snapshot_test_config.genesis_config_info.genesis_config,
|
||||||
None,
|
None,
|
||||||
|
|
|
@ -254,7 +254,8 @@ pub fn download_genesis_if_missing(
|
||||||
/// a full snapshot or an incremental snapshot.
|
/// a full snapshot or an incremental snapshot.
|
||||||
pub fn download_snapshot_archive<'a, 'b>(
|
pub fn download_snapshot_archive<'a, 'b>(
|
||||||
rpc_addr: &SocketAddr,
|
rpc_addr: &SocketAddr,
|
||||||
snapshot_archives_dir: &Path,
|
full_snapshot_archives_dir: &Path,
|
||||||
|
incremental_snapshot_archives_dir: &Path,
|
||||||
desired_snapshot_hash: (Slot, Hash),
|
desired_snapshot_hash: (Slot, Hash),
|
||||||
snapshot_type: SnapshotType,
|
snapshot_type: SnapshotType,
|
||||||
maximum_full_snapshot_archives_to_retain: usize,
|
maximum_full_snapshot_archives_to_retain: usize,
|
||||||
|
@ -263,13 +264,17 @@ pub fn download_snapshot_archive<'a, 'b>(
|
||||||
progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>,
|
progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
snapshot_utils::purge_old_snapshot_archives(
|
snapshot_utils::purge_old_snapshot_archives(
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
maximum_incremental_snapshot_archives_to_retain,
|
maximum_incremental_snapshot_archives_to_retain,
|
||||||
);
|
);
|
||||||
|
|
||||||
let snapshot_archives_remote_dir =
|
let snapshot_archives_remote_dir =
|
||||||
snapshot_utils::build_snapshot_archives_remote_dir(snapshot_archives_dir);
|
snapshot_utils::build_snapshot_archives_remote_dir(match snapshot_type {
|
||||||
|
SnapshotType::FullSnapshot => full_snapshot_archives_dir,
|
||||||
|
SnapshotType::IncrementalSnapshot(_) => incremental_snapshot_archives_dir,
|
||||||
|
});
|
||||||
fs::create_dir_all(&snapshot_archives_remote_dir).unwrap();
|
fs::create_dir_all(&snapshot_archives_remote_dir).unwrap();
|
||||||
|
|
||||||
for archive_format in [
|
for archive_format in [
|
||||||
|
|
|
@ -731,6 +731,7 @@ fn load_bank_forks(
|
||||||
blockstore: &Blockstore,
|
blockstore: &Blockstore,
|
||||||
process_options: ProcessOptions,
|
process_options: ProcessOptions,
|
||||||
snapshot_archive_path: Option<PathBuf>,
|
snapshot_archive_path: Option<PathBuf>,
|
||||||
|
incremental_snapshot_archive_path: Option<PathBuf>,
|
||||||
) -> Result<(Arc<RwLock<BankForks>>, Option<StartingSnapshotHashes>), BlockstoreProcessorError> {
|
) -> Result<(Arc<RwLock<BankForks>>, Option<StartingSnapshotHashes>), BlockstoreProcessorError> {
|
||||||
let bank_snapshots_dir = blockstore
|
let bank_snapshots_dir = blockstore
|
||||||
.ledger_path()
|
.ledger_path()
|
||||||
|
@ -742,12 +743,15 @@ fn load_bank_forks(
|
||||||
let snapshot_config = if arg_matches.is_present("no_snapshot") {
|
let snapshot_config = if arg_matches.is_present("no_snapshot") {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
let snapshot_archives_dir =
|
let full_snapshot_archives_dir =
|
||||||
snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf());
|
snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf());
|
||||||
|
let incremental_snapshot_archives_dir =
|
||||||
|
incremental_snapshot_archive_path.unwrap_or_else(|| full_snapshot_archives_dir.clone());
|
||||||
Some(SnapshotConfig {
|
Some(SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots: Slot::MAX,
|
full_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
bank_snapshots_dir,
|
bank_snapshots_dir,
|
||||||
..SnapshotConfig::default()
|
..SnapshotConfig::default()
|
||||||
})
|
})
|
||||||
|
@ -1102,7 +1106,15 @@ fn main() {
|
||||||
.value_name("DIR")
|
.value_name("DIR")
|
||||||
.takes_value(true)
|
.takes_value(true)
|
||||||
.global(true)
|
.global(true)
|
||||||
.help("Use DIR for ledger location"),
|
.help("Use DIR for snapshot location"),
|
||||||
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("incremental_snapshot_archive_path")
|
||||||
|
.long("incremental-snapshot-archive-path")
|
||||||
|
.value_name("DIR")
|
||||||
|
.takes_value(true)
|
||||||
|
.global(true)
|
||||||
|
.help("Use DIR for separate incremental snapshot location"),
|
||||||
)
|
)
|
||||||
.arg(
|
.arg(
|
||||||
Arg::with_name("output_format")
|
Arg::with_name("output_format")
|
||||||
|
@ -1713,6 +1725,10 @@ fn main() {
|
||||||
let snapshot_archive_path = value_t!(matches, "snapshot_archive_path", String)
|
let snapshot_archive_path = value_t!(matches, "snapshot_archive_path", String)
|
||||||
.ok()
|
.ok()
|
||||||
.map(PathBuf::from);
|
.map(PathBuf::from);
|
||||||
|
let incremental_snapshot_archive_path =
|
||||||
|
value_t!(matches, "incremental_snapshot_archive_path", String)
|
||||||
|
.ok()
|
||||||
|
.map(PathBuf::from);
|
||||||
|
|
||||||
let wal_recovery_mode = matches
|
let wal_recovery_mode = matches
|
||||||
.value_of("wal_recovery_mode")
|
.value_of("wal_recovery_mode")
|
||||||
|
@ -1828,6 +1844,7 @@ fn main() {
|
||||||
&blockstore,
|
&blockstore,
|
||||||
process_options,
|
process_options,
|
||||||
snapshot_archive_path,
|
snapshot_archive_path,
|
||||||
|
incremental_snapshot_archive_path,
|
||||||
) {
|
) {
|
||||||
Ok((bank_forks, ..)) => {
|
Ok((bank_forks, ..)) => {
|
||||||
println!(
|
println!(
|
||||||
|
@ -1909,6 +1926,7 @@ fn main() {
|
||||||
&blockstore,
|
&blockstore,
|
||||||
process_options,
|
process_options,
|
||||||
snapshot_archive_path,
|
snapshot_archive_path,
|
||||||
|
incremental_snapshot_archive_path,
|
||||||
) {
|
) {
|
||||||
Ok((bank_forks, ..)) => {
|
Ok((bank_forks, ..)) => {
|
||||||
println!("{}", &bank_forks.read().unwrap().working_bank().hash());
|
println!("{}", &bank_forks.read().unwrap().working_bank().hash());
|
||||||
|
@ -2148,6 +2166,7 @@ fn main() {
|
||||||
&blockstore,
|
&blockstore,
|
||||||
process_options,
|
process_options,
|
||||||
snapshot_archive_path,
|
snapshot_archive_path,
|
||||||
|
incremental_snapshot_archive_path,
|
||||||
)
|
)
|
||||||
.unwrap_or_else(|err| {
|
.unwrap_or_else(|err| {
|
||||||
eprintln!("Ledger verification failed: {:?}", err);
|
eprintln!("Ledger verification failed: {:?}", err);
|
||||||
|
@ -2179,6 +2198,7 @@ fn main() {
|
||||||
&blockstore,
|
&blockstore,
|
||||||
process_options,
|
process_options,
|
||||||
snapshot_archive_path,
|
snapshot_archive_path,
|
||||||
|
incremental_snapshot_archive_path,
|
||||||
) {
|
) {
|
||||||
Ok((bank_forks, ..)) => {
|
Ok((bank_forks, ..)) => {
|
||||||
let dot = graph_forks(
|
let dot = graph_forks(
|
||||||
|
@ -2208,10 +2228,20 @@ fn main() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
("create-snapshot", Some(arg_matches)) => {
|
("create-snapshot", Some(arg_matches)) => {
|
||||||
|
let is_incremental = arg_matches.is_present("incremental");
|
||||||
let output_directory = value_t!(arg_matches, "output_directory", PathBuf)
|
let output_directory = value_t!(arg_matches, "output_directory", PathBuf)
|
||||||
.unwrap_or_else(|_| match &snapshot_archive_path {
|
.unwrap_or_else(|_| {
|
||||||
Some(snapshot_archive_path) => snapshot_archive_path.clone(),
|
match (
|
||||||
None => ledger_path.clone(),
|
is_incremental,
|
||||||
|
&snapshot_archive_path,
|
||||||
|
&incremental_snapshot_archive_path,
|
||||||
|
) {
|
||||||
|
(true, _, Some(incremental_snapshot_archive_path)) => {
|
||||||
|
incremental_snapshot_archive_path.clone()
|
||||||
|
}
|
||||||
|
(_, Some(snapshot_archive_path), _) => snapshot_archive_path.clone(),
|
||||||
|
(_, _, _) => ledger_path.clone(),
|
||||||
|
}
|
||||||
});
|
});
|
||||||
let mut warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok();
|
let mut warp_slot = value_t!(arg_matches, "warp_slot", Slot).ok();
|
||||||
let remove_stake_accounts = arg_matches.is_present("remove_stake_accounts");
|
let remove_stake_accounts = arg_matches.is_present("remove_stake_accounts");
|
||||||
|
@ -2266,7 +2296,6 @@ fn main() {
|
||||||
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
|
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
|
||||||
let blockstore =
|
let blockstore =
|
||||||
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
|
open_blockstore(&ledger_path, AccessType::Secondary, wal_recovery_mode);
|
||||||
let is_incremental = arg_matches.is_present("incremental");
|
|
||||||
|
|
||||||
let snapshot_slot = if Some("ROOT") == arg_matches.value_of("snapshot_slot") {
|
let snapshot_slot = if Some("ROOT") == arg_matches.value_of("snapshot_slot") {
|
||||||
blockstore
|
blockstore
|
||||||
|
@ -2296,6 +2325,7 @@ fn main() {
|
||||||
..ProcessOptions::default()
|
..ProcessOptions::default()
|
||||||
},
|
},
|
||||||
snapshot_archive_path,
|
snapshot_archive_path,
|
||||||
|
incremental_snapshot_archive_path,
|
||||||
) {
|
) {
|
||||||
Ok((bank_forks, starting_snapshot_hashes)) => {
|
Ok((bank_forks, starting_snapshot_hashes)) => {
|
||||||
let mut bank = bank_forks
|
let mut bank = bank_forks
|
||||||
|
@ -2530,6 +2560,7 @@ fn main() {
|
||||||
&bank,
|
&bank,
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
Some(snapshot_version),
|
Some(snapshot_version),
|
||||||
|
output_directory.clone(),
|
||||||
output_directory,
|
output_directory,
|
||||||
ArchiveFormat::TarZstd,
|
ArchiveFormat::TarZstd,
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
|
@ -2553,6 +2584,7 @@ fn main() {
|
||||||
ledger_path,
|
ledger_path,
|
||||||
&bank,
|
&bank,
|
||||||
Some(snapshot_version),
|
Some(snapshot_version),
|
||||||
|
output_directory.clone(),
|
||||||
output_directory,
|
output_directory,
|
||||||
ArchiveFormat::TarZstd,
|
ArchiveFormat::TarZstd,
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
|
@ -2603,6 +2635,7 @@ fn main() {
|
||||||
&blockstore,
|
&blockstore,
|
||||||
process_options,
|
process_options,
|
||||||
snapshot_archive_path,
|
snapshot_archive_path,
|
||||||
|
incremental_snapshot_archive_path,
|
||||||
)
|
)
|
||||||
.unwrap_or_else(|err| {
|
.unwrap_or_else(|err| {
|
||||||
eprintln!("Failed to load ledger: {:?}", err);
|
eprintln!("Failed to load ledger: {:?}", err);
|
||||||
|
@ -2662,6 +2695,7 @@ fn main() {
|
||||||
&blockstore,
|
&blockstore,
|
||||||
process_options,
|
process_options,
|
||||||
snapshot_archive_path,
|
snapshot_archive_path,
|
||||||
|
incremental_snapshot_archive_path,
|
||||||
) {
|
) {
|
||||||
Ok((bank_forks, ..)) => {
|
Ok((bank_forks, ..)) => {
|
||||||
let bank_forks = bank_forks.read().unwrap();
|
let bank_forks = bank_forks.read().unwrap();
|
||||||
|
|
|
@ -98,7 +98,7 @@ pub fn load_bank_forks(
|
||||||
.expect("Couldn't create snapshot directory");
|
.expect("Couldn't create snapshot directory");
|
||||||
|
|
||||||
if snapshot_utils::get_highest_full_snapshot_archive_info(
|
if snapshot_utils::get_highest_full_snapshot_archive_info(
|
||||||
&snapshot_config.snapshot_archives_dir,
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
)
|
)
|
||||||
.is_some()
|
.is_some()
|
||||||
{
|
{
|
||||||
|
@ -188,7 +188,8 @@ fn bank_forks_from_snapshot(
|
||||||
let (mut deserialized_bank, full_snapshot_archive_info, incremental_snapshot_archive_info) =
|
let (mut deserialized_bank, full_snapshot_archive_info, incremental_snapshot_archive_info) =
|
||||||
snapshot_utils::bank_from_latest_snapshot_archives(
|
snapshot_utils::bank_from_latest_snapshot_archives(
|
||||||
&snapshot_config.bank_snapshots_dir,
|
&snapshot_config.bank_snapshots_dir,
|
||||||
&snapshot_config.snapshot_archives_dir,
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
|
&snapshot_config.incremental_snapshot_archives_dir,
|
||||||
&account_paths,
|
&account_paths,
|
||||||
genesis_config,
|
genesis_config,
|
||||||
process_options.debug_keys.clone(),
|
process_options.debug_keys.clone(),
|
||||||
|
|
|
@ -13,11 +13,18 @@ use {
|
||||||
|
|
||||||
impl LocalCluster {
|
impl LocalCluster {
|
||||||
/// Return the next full snapshot archive info after the cluster's last processed slot
|
/// Return the next full snapshot archive info after the cluster's last processed slot
|
||||||
pub fn wait_for_next_full_snapshot(
|
pub fn wait_for_next_full_snapshot<T>(
|
||||||
&self,
|
&self,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: T,
|
||||||
) -> FullSnapshotArchiveInfo {
|
) -> FullSnapshotArchiveInfo
|
||||||
match self.wait_for_next_snapshot(snapshot_archives_dir, NextSnapshotType::FullSnapshot) {
|
where
|
||||||
|
T: AsRef<Path>,
|
||||||
|
{
|
||||||
|
match self.wait_for_next_snapshot(
|
||||||
|
full_snapshot_archives_dir,
|
||||||
|
None::<T>,
|
||||||
|
NextSnapshotType::FullSnapshot,
|
||||||
|
) {
|
||||||
NextSnapshotResult::FullSnapshot(full_snapshot_archive_info) => {
|
NextSnapshotResult::FullSnapshot(full_snapshot_archive_info) => {
|
||||||
full_snapshot_archive_info
|
full_snapshot_archive_info
|
||||||
}
|
}
|
||||||
|
@ -29,10 +36,12 @@ impl LocalCluster {
|
||||||
/// after the cluster's last processed slot
|
/// after the cluster's last processed slot
|
||||||
pub fn wait_for_next_incremental_snapshot(
|
pub fn wait_for_next_incremental_snapshot(
|
||||||
&self,
|
&self,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
) -> (IncrementalSnapshotArchiveInfo, FullSnapshotArchiveInfo) {
|
) -> (IncrementalSnapshotArchiveInfo, FullSnapshotArchiveInfo) {
|
||||||
match self.wait_for_next_snapshot(
|
match self.wait_for_next_snapshot(
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
Some(incremental_snapshot_archives_dir),
|
||||||
NextSnapshotType::IncrementalAndFullSnapshot,
|
NextSnapshotType::IncrementalAndFullSnapshot,
|
||||||
) {
|
) {
|
||||||
NextSnapshotResult::IncrementalAndFullSnapshot(
|
NextSnapshotResult::IncrementalAndFullSnapshot(
|
||||||
|
@ -47,9 +56,10 @@ impl LocalCluster {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return the next snapshot archive infos after the cluster's last processed slot
|
/// Return the next snapshot archive infos after the cluster's last processed slot
|
||||||
pub fn wait_for_next_snapshot(
|
fn wait_for_next_snapshot(
|
||||||
&self,
|
&self,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: Option<impl AsRef<Path>>,
|
||||||
next_snapshot_type: NextSnapshotType,
|
next_snapshot_type: NextSnapshotType,
|
||||||
) -> NextSnapshotResult {
|
) -> NextSnapshotResult {
|
||||||
// Get slot after which this was generated
|
// Get slot after which this was generated
|
||||||
|
@ -69,7 +79,7 @@ impl LocalCluster {
|
||||||
);
|
);
|
||||||
loop {
|
loop {
|
||||||
if let Some(full_snapshot_archive_info) =
|
if let Some(full_snapshot_archive_info) =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir)
|
snapshot_utils::get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir)
|
||||||
{
|
{
|
||||||
match next_snapshot_type {
|
match next_snapshot_type {
|
||||||
NextSnapshotType::FullSnapshot => {
|
NextSnapshotType::FullSnapshot => {
|
||||||
|
@ -80,7 +90,7 @@ impl LocalCluster {
|
||||||
NextSnapshotType::IncrementalAndFullSnapshot => {
|
NextSnapshotType::IncrementalAndFullSnapshot => {
|
||||||
if let Some(incremental_snapshot_archive_info) =
|
if let Some(incremental_snapshot_archive_info) =
|
||||||
snapshot_utils::get_highest_incremental_snapshot_archive_info(
|
snapshot_utils::get_highest_incremental_snapshot_archive_info(
|
||||||
&snapshot_archives_dir,
|
incremental_snapshot_archives_dir.as_ref().unwrap(),
|
||||||
full_snapshot_archive_info.slot(),
|
full_snapshot_archive_info.slot(),
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
|
|
|
@ -441,7 +441,8 @@ pub fn generate_account_paths(num_account_paths: usize) -> (Vec<TempDir>, Vec<Pa
|
||||||
|
|
||||||
pub struct SnapshotValidatorConfig {
|
pub struct SnapshotValidatorConfig {
|
||||||
pub bank_snapshots_dir: TempDir,
|
pub bank_snapshots_dir: TempDir,
|
||||||
pub snapshot_archives_dir: TempDir,
|
pub full_snapshot_archives_dir: TempDir,
|
||||||
|
pub incremental_snapshot_archives_dir: TempDir,
|
||||||
pub account_storage_dirs: Vec<TempDir>,
|
pub account_storage_dirs: Vec<TempDir>,
|
||||||
pub validator_config: ValidatorConfig,
|
pub validator_config: ValidatorConfig,
|
||||||
}
|
}
|
||||||
|
@ -470,11 +471,15 @@ impl SnapshotValidatorConfig {
|
||||||
// Create the snapshot config
|
// Create the snapshot config
|
||||||
let _ = fs::create_dir_all(farf_dir());
|
let _ = fs::create_dir_all(farf_dir());
|
||||||
let bank_snapshots_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let bank_snapshots_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let full_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
full_snapshot_archive_interval_slots,
|
full_snapshot_archive_interval_slots,
|
||||||
incremental_snapshot_archive_interval_slots,
|
incremental_snapshot_archive_interval_slots,
|
||||||
snapshot_archives_dir: snapshot_archives_dir.path().to_path_buf(),
|
full_snapshot_archives_dir: full_snapshot_archives_dir.path().to_path_buf(),
|
||||||
|
incremental_snapshot_archives_dir: incremental_snapshot_archives_dir
|
||||||
|
.path()
|
||||||
|
.to_path_buf(),
|
||||||
bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
|
bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
|
||||||
maximum_full_snapshot_archives_to_retain: usize::MAX,
|
maximum_full_snapshot_archives_to_retain: usize::MAX,
|
||||||
maximum_incremental_snapshot_archives_to_retain: usize::MAX,
|
maximum_incremental_snapshot_archives_to_retain: usize::MAX,
|
||||||
|
@ -495,7 +500,8 @@ impl SnapshotValidatorConfig {
|
||||||
|
|
||||||
SnapshotValidatorConfig {
|
SnapshotValidatorConfig {
|
||||||
bank_snapshots_dir,
|
bank_snapshots_dir,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
account_storage_dirs,
|
account_storage_dirs,
|
||||||
validator_config,
|
validator_config,
|
||||||
}
|
}
|
||||||
|
|
|
@ -467,21 +467,29 @@ fn test_snapshot_download() {
|
||||||
|
|
||||||
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
|
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
|
||||||
|
|
||||||
let snapshot_archives_dir = &leader_snapshot_test_config
|
let full_snapshot_archives_dir = &leader_snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_archives_dir;
|
.full_snapshot_archives_dir;
|
||||||
|
let incremental_snapshot_archives_dir = &leader_snapshot_test_config
|
||||||
|
.validator_config
|
||||||
|
.snapshot_config
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.incremental_snapshot_archives_dir;
|
||||||
|
|
||||||
trace!("Waiting for snapshot");
|
trace!("Waiting for snapshot");
|
||||||
let full_snapshot_archive_info = cluster.wait_for_next_full_snapshot(snapshot_archives_dir);
|
let full_snapshot_archive_info =
|
||||||
|
cluster.wait_for_next_full_snapshot(full_snapshot_archives_dir);
|
||||||
trace!("found: {}", full_snapshot_archive_info.path().display());
|
trace!("found: {}", full_snapshot_archive_info.path().display());
|
||||||
|
|
||||||
// Download the snapshot, then boot a validator from it.
|
// Download the snapshot, then boot a validator from it.
|
||||||
download_snapshot_archive(
|
download_snapshot_archive(
|
||||||
&cluster.entry_point_info.rpc,
|
&cluster.entry_point_info.rpc,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
(
|
(
|
||||||
full_snapshot_archive_info.slot(),
|
full_snapshot_archive_info.slot(),
|
||||||
*full_snapshot_archive_info.hash(),
|
*full_snapshot_archive_info.hash(),
|
||||||
|
@ -549,43 +557,60 @@ fn test_incremental_snapshot_download() {
|
||||||
|
|
||||||
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
|
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
|
||||||
|
|
||||||
let snapshot_archives_dir = &leader_snapshot_test_config
|
let full_snapshot_archives_dir = &leader_snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_archives_dir;
|
.full_snapshot_archives_dir;
|
||||||
|
let incremental_snapshot_archives_dir = &leader_snapshot_test_config
|
||||||
|
.validator_config
|
||||||
|
.snapshot_config
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.incremental_snapshot_archives_dir;
|
||||||
|
|
||||||
debug!("snapshot config:\n\tfull snapshot interval: {}\n\tincremental snapshot interval: {}\n\taccounts hash interval: {}",
|
debug!("snapshot config:\n\tfull snapshot interval: {}\n\tincremental snapshot interval: {}\n\taccounts hash interval: {}",
|
||||||
full_snapshot_interval,
|
full_snapshot_interval,
|
||||||
incremental_snapshot_interval,
|
incremental_snapshot_interval,
|
||||||
accounts_hash_interval);
|
accounts_hash_interval);
|
||||||
debug!(
|
debug!(
|
||||||
"leader config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}",
|
"leader config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}",
|
||||||
leader_snapshot_test_config
|
leader_snapshot_test_config
|
||||||
.bank_snapshots_dir
|
.bank_snapshots_dir
|
||||||
.path()
|
.path()
|
||||||
.display(),
|
.display(),
|
||||||
leader_snapshot_test_config
|
leader_snapshot_test_config
|
||||||
.snapshot_archives_dir
|
.full_snapshot_archives_dir
|
||||||
|
.path()
|
||||||
|
.display(),
|
||||||
|
leader_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
.path()
|
.path()
|
||||||
.display(),
|
.display(),
|
||||||
);
|
);
|
||||||
debug!(
|
debug!(
|
||||||
"validator config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}",
|
"validator config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}",
|
||||||
validator_snapshot_test_config
|
validator_snapshot_test_config
|
||||||
.bank_snapshots_dir
|
.bank_snapshots_dir
|
||||||
.path()
|
.path()
|
||||||
.display(),
|
.display(),
|
||||||
validator_snapshot_test_config
|
validator_snapshot_test_config
|
||||||
.snapshot_archives_dir
|
.full_snapshot_archives_dir
|
||||||
|
.path()
|
||||||
|
.display(),
|
||||||
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
.path()
|
.path()
|
||||||
.display(),
|
.display(),
|
||||||
);
|
);
|
||||||
|
|
||||||
trace!("Waiting for snapshots");
|
trace!("Waiting for snapshots");
|
||||||
let (incremental_snapshot_archive_info, full_snapshot_archive_info) =
|
let (incremental_snapshot_archive_info, full_snapshot_archive_info) = cluster
|
||||||
cluster.wait_for_next_incremental_snapshot(snapshot_archives_dir);
|
.wait_for_next_incremental_snapshot(
|
||||||
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
|
);
|
||||||
trace!(
|
trace!(
|
||||||
"found: {} and {}",
|
"found: {} and {}",
|
||||||
full_snapshot_archive_info.path().display(),
|
full_snapshot_archive_info.path().display(),
|
||||||
|
@ -595,7 +620,8 @@ fn test_incremental_snapshot_download() {
|
||||||
// Download the snapshots, then boot a validator from them.
|
// Download the snapshots, then boot a validator from them.
|
||||||
download_snapshot_archive(
|
download_snapshot_archive(
|
||||||
&cluster.entry_point_info.rpc,
|
&cluster.entry_point_info.rpc,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
(
|
(
|
||||||
full_snapshot_archive_info.slot(),
|
full_snapshot_archive_info.slot(),
|
||||||
*full_snapshot_archive_info.hash(),
|
*full_snapshot_archive_info.hash(),
|
||||||
|
@ -620,7 +646,8 @@ fn test_incremental_snapshot_download() {
|
||||||
|
|
||||||
download_snapshot_archive(
|
download_snapshot_archive(
|
||||||
&cluster.entry_point_info.rpc,
|
&cluster.entry_point_info.rpc,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
(
|
(
|
||||||
incremental_snapshot_archive_info.slot(),
|
incremental_snapshot_archive_info.slot(),
|
||||||
*incremental_snapshot_archive_info.hash(),
|
*incremental_snapshot_archive_info.hash(),
|
||||||
|
@ -708,24 +735,32 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
incremental_snapshot_interval,
|
incremental_snapshot_interval,
|
||||||
accounts_hash_interval);
|
accounts_hash_interval);
|
||||||
debug!(
|
debug!(
|
||||||
"leader config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}",
|
"leader config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}",
|
||||||
leader_snapshot_test_config
|
leader_snapshot_test_config
|
||||||
.bank_snapshots_dir
|
.bank_snapshots_dir
|
||||||
.path()
|
.path()
|
||||||
.display(),
|
.display(),
|
||||||
leader_snapshot_test_config
|
leader_snapshot_test_config
|
||||||
.snapshot_archives_dir
|
.full_snapshot_archives_dir
|
||||||
|
.path()
|
||||||
|
.display(),
|
||||||
|
leader_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
.path()
|
.path()
|
||||||
.display(),
|
.display(),
|
||||||
);
|
);
|
||||||
debug!(
|
debug!(
|
||||||
"validator config:\n\tbank snapshots dir: {}\n\tsnapshot archives dir: {}",
|
"validator config:\n\tbank snapshots dir: {}\n\tfull snapshot archives dir: {}\n\tincremental snapshot archives dir: {}",
|
||||||
validator_snapshot_test_config
|
validator_snapshot_test_config
|
||||||
.bank_snapshots_dir
|
.bank_snapshots_dir
|
||||||
.path()
|
.path()
|
||||||
.display(),
|
.display(),
|
||||||
validator_snapshot_test_config
|
validator_snapshot_test_config
|
||||||
.snapshot_archives_dir
|
.full_snapshot_archives_dir
|
||||||
|
.path()
|
||||||
|
.display(),
|
||||||
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
.path()
|
.path()
|
||||||
.display(),
|
.display(),
|
||||||
);
|
);
|
||||||
|
@ -734,7 +769,12 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
let (incremental_snapshot_archive, full_snapshot_archive) =
|
let (incremental_snapshot_archive, full_snapshot_archive) =
|
||||||
LocalCluster::wait_for_next_incremental_snapshot(
|
LocalCluster::wait_for_next_incremental_snapshot(
|
||||||
&cluster,
|
&cluster,
|
||||||
leader_snapshot_test_config.snapshot_archives_dir.path(),
|
leader_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
leader_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
);
|
);
|
||||||
info!(
|
info!(
|
||||||
"Found snapshots:\n\tfull snapshot: {}\n\tincremental snapshot: {}",
|
"Found snapshots:\n\tfull snapshot: {}\n\tincremental snapshot: {}",
|
||||||
|
@ -751,7 +791,12 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
info!("Downloading full snapshot to validator...");
|
info!("Downloading full snapshot to validator...");
|
||||||
download_snapshot_archive(
|
download_snapshot_archive(
|
||||||
&cluster.entry_point_info.rpc,
|
&cluster.entry_point_info.rpc,
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
(full_snapshot_archive.slot(), *full_snapshot_archive.hash()),
|
(full_snapshot_archive.slot(), *full_snapshot_archive.hash()),
|
||||||
SnapshotType::FullSnapshot,
|
SnapshotType::FullSnapshot,
|
||||||
validator_snapshot_test_config
|
validator_snapshot_test_config
|
||||||
|
@ -771,7 +816,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let downloaded_full_snapshot_archive = snapshot_utils::get_highest_full_snapshot_archive_info(
|
let downloaded_full_snapshot_archive = snapshot_utils::get_highest_full_snapshot_archive_info(
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
info!(
|
info!(
|
||||||
|
@ -782,7 +829,12 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
info!("Downloading incremental snapshot to validator...");
|
info!("Downloading incremental snapshot to validator...");
|
||||||
download_snapshot_archive(
|
download_snapshot_archive(
|
||||||
&cluster.entry_point_info.rpc,
|
&cluster.entry_point_info.rpc,
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
(
|
(
|
||||||
incremental_snapshot_archive.slot(),
|
incremental_snapshot_archive.slot(),
|
||||||
*incremental_snapshot_archive.hash(),
|
*incremental_snapshot_archive.hash(),
|
||||||
|
@ -806,7 +858,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let downloaded_incremental_snapshot_archive =
|
let downloaded_incremental_snapshot_archive =
|
||||||
snapshot_utils::get_highest_incremental_snapshot_archive_info(
|
snapshot_utils::get_highest_incremental_snapshot_archive_info(
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
full_snapshot_archive.slot(),
|
full_snapshot_archive.slot(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
@ -876,14 +930,30 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
// restart the node and guarantee that the only snapshots present are these initial ones. So,
|
// restart the node and guarantee that the only snapshots present are these initial ones. So,
|
||||||
// the easiest way to do that is create a backup now, delete the ones on the node before
|
// the easiest way to do that is create a backup now, delete the ones on the node before
|
||||||
// restart, then copy the backup ones over again.
|
// restart, then copy the backup ones over again.
|
||||||
let backup_validator_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let backup_validator_full_snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
trace!(
|
trace!(
|
||||||
"Backing up validator snapshots to dir: {}...",
|
"Backing up validator full snapshots to dir: {}...",
|
||||||
backup_validator_snapshot_archives_dir.path().display()
|
backup_validator_full_snapshot_archives_dir.path().display()
|
||||||
);
|
);
|
||||||
copy_files_with_remote(
|
copy_files_with_remote(
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
backup_validator_snapshot_archives_dir.path(),
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
backup_validator_full_snapshot_archives_dir.path(),
|
||||||
|
);
|
||||||
|
let backup_validator_incremental_snapshot_archives_dir =
|
||||||
|
tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
|
trace!(
|
||||||
|
"Backing up validator incremental snapshots to dir: {}...",
|
||||||
|
backup_validator_incremental_snapshot_archives_dir
|
||||||
|
.path()
|
||||||
|
.display()
|
||||||
|
);
|
||||||
|
copy_files_with_remote(
|
||||||
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
backup_validator_incremental_snapshot_archives_dir.path(),
|
||||||
);
|
);
|
||||||
|
|
||||||
info!("Starting the validator...");
|
info!("Starting the validator...");
|
||||||
|
@ -931,13 +1001,17 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
// Putting this all in its own block so its clear we're only intended to keep the leader's info
|
// Putting this all in its own block so its clear we're only intended to keep the leader's info
|
||||||
let leader_full_snapshot_archive_for_comparison = {
|
let leader_full_snapshot_archive_for_comparison = {
|
||||||
let validator_full_snapshot = snapshot_utils::get_highest_full_snapshot_archive_info(
|
let validator_full_snapshot = snapshot_utils::get_highest_full_snapshot_archive_info(
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
// Now get the same full snapshot on the LEADER that we just got from the validator
|
// Now get the same full snapshot on the LEADER that we just got from the validator
|
||||||
let mut leader_full_snapshots = snapshot_utils::get_full_snapshot_archives(
|
let mut leader_full_snapshots = snapshot_utils::get_full_snapshot_archives(
|
||||||
leader_snapshot_test_config.snapshot_archives_dir.path(),
|
leader_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
);
|
);
|
||||||
leader_full_snapshots.retain(|full_snapshot| {
|
leader_full_snapshots.retain(|full_snapshot| {
|
||||||
full_snapshot.slot() == validator_full_snapshot.slot()
|
full_snapshot.slot() == validator_full_snapshot.slot()
|
||||||
|
@ -959,10 +1033,27 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
info!("leader full snapshot archive for comparison: {leader_full_snapshot_archive_for_comparison:#?}");
|
info!("leader full snapshot archive for comparison: {leader_full_snapshot_archive_for_comparison:#?}");
|
||||||
|
|
||||||
info!("Delete all the snapshots on the validator and restore the originals from the backup...");
|
info!("Delete all the snapshots on the validator and restore the originals from the backup...");
|
||||||
delete_files_with_remote(validator_snapshot_test_config.snapshot_archives_dir.path());
|
delete_files_with_remote(
|
||||||
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
);
|
||||||
|
delete_files_with_remote(
|
||||||
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
);
|
||||||
copy_files_with_remote(
|
copy_files_with_remote(
|
||||||
backup_validator_snapshot_archives_dir.path(),
|
backup_validator_full_snapshot_archives_dir.path(),
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
);
|
||||||
|
copy_files_with_remote(
|
||||||
|
backup_validator_incremental_snapshot_archives_dir.path(),
|
||||||
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
);
|
);
|
||||||
info!(
|
info!(
|
||||||
"Delete all the snapshots on the validator and restore the originals from the backup... DONE"
|
"Delete all the snapshots on the validator and restore the originals from the backup... DONE"
|
||||||
|
@ -971,7 +1062,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
// Get the highest full snapshot slot *before* restarting, as a comparison
|
// Get the highest full snapshot slot *before* restarting, as a comparison
|
||||||
let validator_full_snapshot_slot_at_startup =
|
let validator_full_snapshot_slot_at_startup =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_slot(
|
snapshot_utils::get_highest_full_snapshot_archive_slot(
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
@ -998,12 +1091,16 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
let timer = Instant::now();
|
let timer = Instant::now();
|
||||||
loop {
|
loop {
|
||||||
if let Some(full_snapshot_slot) = snapshot_utils::get_highest_full_snapshot_archive_slot(
|
if let Some(full_snapshot_slot) = snapshot_utils::get_highest_full_snapshot_archive_slot(
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
) {
|
) {
|
||||||
if full_snapshot_slot >= validator_next_full_snapshot_slot {
|
if full_snapshot_slot >= validator_next_full_snapshot_slot {
|
||||||
if let Some(incremental_snapshot_slot) =
|
if let Some(incremental_snapshot_slot) =
|
||||||
snapshot_utils::get_highest_incremental_snapshot_archive_slot(
|
snapshot_utils::get_highest_incremental_snapshot_archive_slot(
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
|
@ -1034,7 +1131,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
// Check to make sure that the full snapshot the validator created during startup is the same
|
// Check to make sure that the full snapshot the validator created during startup is the same
|
||||||
// or one greater than the snapshot the leader created.
|
// or one greater than the snapshot the leader created.
|
||||||
let validator_full_snapshot_archives = snapshot_utils::get_full_snapshot_archives(
|
let validator_full_snapshot_archives = snapshot_utils::get_full_snapshot_archives(
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
);
|
);
|
||||||
info!("validator full snapshot archives: {validator_full_snapshot_archives:#?}");
|
info!("validator full snapshot archives: {validator_full_snapshot_archives:#?}");
|
||||||
let validator_full_snapshot_archive_for_comparison = validator_full_snapshot_archives
|
let validator_full_snapshot_archive_for_comparison = validator_full_snapshot_archives
|
||||||
|
@ -1071,12 +1170,29 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
|
||||||
// Copy over the snapshots to the new node, but need to remove the tmp snapshot dir so it
|
// Copy over the snapshots to the new node, but need to remove the tmp snapshot dir so it
|
||||||
// doesn't break the simple copy_files closure.
|
// doesn't break the simple copy_files closure.
|
||||||
snapshot_utils::remove_tmp_snapshot_archives(
|
snapshot_utils::remove_tmp_snapshot_archives(
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
);
|
||||||
|
snapshot_utils::remove_tmp_snapshot_archives(
|
||||||
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
);
|
);
|
||||||
copy_files(
|
copy_files(
|
||||||
validator_snapshot_test_config.snapshot_archives_dir.path(),
|
validator_snapshot_test_config
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
final_validator_snapshot_test_config
|
final_validator_snapshot_test_config
|
||||||
.snapshot_archives_dir
|
.full_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
);
|
||||||
|
copy_files(
|
||||||
|
validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.path(),
|
||||||
|
final_validator_snapshot_test_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
.path(),
|
.path(),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -1128,19 +1244,20 @@ fn test_snapshot_restart_tower() {
|
||||||
let validator_info = cluster.exit_node(&validator_id);
|
let validator_info = cluster.exit_node(&validator_id);
|
||||||
|
|
||||||
// Get slot after which this was generated
|
// Get slot after which this was generated
|
||||||
let snapshot_archives_dir = &leader_snapshot_test_config
|
let full_snapshot_archives_dir = &leader_snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_archives_dir;
|
.full_snapshot_archives_dir;
|
||||||
|
|
||||||
let full_snapshot_archive_info = cluster.wait_for_next_full_snapshot(snapshot_archives_dir);
|
let full_snapshot_archive_info =
|
||||||
|
cluster.wait_for_next_full_snapshot(full_snapshot_archives_dir);
|
||||||
|
|
||||||
// Copy archive to validator's snapshot output directory
|
// Copy archive to validator's snapshot output directory
|
||||||
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
|
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
|
||||||
validator_snapshot_test_config
|
validator_snapshot_test_config
|
||||||
.snapshot_archives_dir
|
.full_snapshot_archives_dir
|
||||||
.into_path(),
|
.into_path(),
|
||||||
full_snapshot_archive_info.slot(),
|
full_snapshot_archive_info.slot(),
|
||||||
full_snapshot_archive_info.hash(),
|
full_snapshot_archive_info.hash(),
|
||||||
|
@ -1178,12 +1295,12 @@ fn test_snapshots_blockstore_floor() {
|
||||||
let mut validator_snapshot_test_config =
|
let mut validator_snapshot_test_config =
|
||||||
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
||||||
|
|
||||||
let snapshot_archives_dir = &leader_snapshot_test_config
|
let full_snapshot_archives_dir = &leader_snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_archives_dir;
|
.full_snapshot_archives_dir;
|
||||||
|
|
||||||
let mut config = ClusterConfig {
|
let mut config = ClusterConfig {
|
||||||
node_stakes: vec![DEFAULT_NODE_STAKE],
|
node_stakes: vec![DEFAULT_NODE_STAKE],
|
||||||
|
@ -1201,7 +1318,7 @@ fn test_snapshots_blockstore_floor() {
|
||||||
|
|
||||||
let archive_info = loop {
|
let archive_info = loop {
|
||||||
let archive =
|
let archive =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir);
|
snapshot_utils::get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir);
|
||||||
if archive.is_some() {
|
if archive.is_some() {
|
||||||
trace!("snapshot exists");
|
trace!("snapshot exists");
|
||||||
break archive.unwrap();
|
break archive.unwrap();
|
||||||
|
@ -1212,7 +1329,7 @@ fn test_snapshots_blockstore_floor() {
|
||||||
// Copy archive to validator's snapshot output directory
|
// Copy archive to validator's snapshot output directory
|
||||||
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
|
let validator_archive_path = snapshot_utils::build_full_snapshot_archive_path(
|
||||||
validator_snapshot_test_config
|
validator_snapshot_test_config
|
||||||
.snapshot_archives_dir
|
.full_snapshot_archives_dir
|
||||||
.into_path(),
|
.into_path(),
|
||||||
archive_info.slot(),
|
archive_info.slot(),
|
||||||
archive_info.hash(),
|
archive_info.hash(),
|
||||||
|
@ -1280,12 +1397,12 @@ fn test_snapshots_restart_validity() {
|
||||||
let num_account_paths = 1;
|
let num_account_paths = 1;
|
||||||
let mut snapshot_test_config =
|
let mut snapshot_test_config =
|
||||||
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
|
||||||
let snapshot_archives_dir = &snapshot_test_config
|
let full_snapshot_archives_dir = &snapshot_test_config
|
||||||
.validator_config
|
.validator_config
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.snapshot_archives_dir;
|
.full_snapshot_archives_dir;
|
||||||
|
|
||||||
// Set up the cluster with 1 snapshotting validator
|
// Set up the cluster with 1 snapshotting validator
|
||||||
let mut all_account_storage_dirs = vec![vec![]];
|
let mut all_account_storage_dirs = vec![vec![]];
|
||||||
|
@ -1323,7 +1440,7 @@ fn test_snapshots_restart_validity() {
|
||||||
|
|
||||||
expected_balances.extend(new_balances);
|
expected_balances.extend(new_balances);
|
||||||
|
|
||||||
cluster.wait_for_next_full_snapshot(snapshot_archives_dir);
|
cluster.wait_for_next_full_snapshot(full_snapshot_archives_dir);
|
||||||
|
|
||||||
// Create new account paths since validator exit is not guaranteed to cleanup RPC threads,
|
// Create new account paths since validator exit is not guaranteed to cleanup RPC threads,
|
||||||
// which may delete the old accounts on exit at any point
|
// which may delete the old accounts on exit at any point
|
||||||
|
|
|
@ -2546,17 +2546,22 @@ pub mod rpc_minimal {
|
||||||
return Err(RpcCustomError::NoSnapshot.into());
|
return Err(RpcCustomError::NoSnapshot.into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let snapshot_archives_dir = meta
|
let (full_snapshot_archives_dir, incremental_snapshot_archives_dir) = meta
|
||||||
.snapshot_config
|
.snapshot_config
|
||||||
.map(|snapshot_config| snapshot_config.snapshot_archives_dir)
|
.map(|snapshot_config| {
|
||||||
|
(
|
||||||
|
snapshot_config.full_snapshot_archives_dir,
|
||||||
|
snapshot_config.incremental_snapshot_archives_dir,
|
||||||
|
)
|
||||||
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let full_snapshot_slot =
|
let full_snapshot_slot =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_slot(&snapshot_archives_dir)
|
snapshot_utils::get_highest_full_snapshot_archive_slot(&full_snapshot_archives_dir)
|
||||||
.ok_or(RpcCustomError::NoSnapshot)?;
|
.ok_or(RpcCustomError::NoSnapshot)?;
|
||||||
let incremental_snapshot_slot =
|
let incremental_snapshot_slot =
|
||||||
snapshot_utils::get_highest_incremental_snapshot_archive_slot(
|
snapshot_utils::get_highest_incremental_snapshot_archive_slot(
|
||||||
&snapshot_archives_dir,
|
&incremental_snapshot_archives_dir,
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -3925,7 +3930,7 @@ pub mod rpc_deprecated_v1_9 {
|
||||||
meta.snapshot_config
|
meta.snapshot_config
|
||||||
.and_then(|snapshot_config| {
|
.and_then(|snapshot_config| {
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_slot(
|
snapshot_utils::get_highest_full_snapshot_archive_slot(
|
||||||
&snapshot_config.snapshot_archives_dir,
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.ok_or_else(|| RpcCustomError::NoSnapshot.into())
|
.ok_or_else(|| RpcCustomError::NoSnapshot.into())
|
||||||
|
|
|
@ -160,7 +160,22 @@ impl RpcRequestMiddleware {
|
||||||
where
|
where
|
||||||
P: AsRef<Path>,
|
P: AsRef<Path>,
|
||||||
{
|
{
|
||||||
let root = &self.snapshot_config.as_ref().unwrap().snapshot_archives_dir;
|
let root = if self
|
||||||
|
.full_snapshot_archive_path_regex
|
||||||
|
.is_match(Path::new("").join(&stem).to_str().unwrap())
|
||||||
|
{
|
||||||
|
&self
|
||||||
|
.snapshot_config
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.full_snapshot_archives_dir
|
||||||
|
} else {
|
||||||
|
&self
|
||||||
|
.snapshot_config
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
};
|
||||||
let local_path = root.join(&stem);
|
let local_path = root.join(&stem);
|
||||||
if local_path.exists() {
|
if local_path.exists() {
|
||||||
local_path
|
local_path
|
||||||
|
@ -236,7 +251,7 @@ impl RequestMiddleware for RpcRequestMiddleware {
|
||||||
// Convenience redirect to the latest snapshot
|
// Convenience redirect to the latest snapshot
|
||||||
let full_snapshot_archive_info =
|
let full_snapshot_archive_info =
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(
|
snapshot_utils::get_highest_full_snapshot_archive_info(
|
||||||
&snapshot_config.snapshot_archives_dir,
|
&snapshot_config.full_snapshot_archives_dir,
|
||||||
);
|
);
|
||||||
let snapshot_archive_info =
|
let snapshot_archive_info =
|
||||||
if let Some(full_snapshot_archive_info) = full_snapshot_archive_info {
|
if let Some(full_snapshot_archive_info) = full_snapshot_archive_info {
|
||||||
|
@ -244,7 +259,7 @@ impl RequestMiddleware for RpcRequestMiddleware {
|
||||||
Some(full_snapshot_archive_info.snapshot_archive_info().clone())
|
Some(full_snapshot_archive_info.snapshot_archive_info().clone())
|
||||||
} else {
|
} else {
|
||||||
snapshot_utils::get_highest_incremental_snapshot_archive_info(
|
snapshot_utils::get_highest_incremental_snapshot_archive_info(
|
||||||
&snapshot_config.snapshot_archives_dir,
|
&snapshot_config.incremental_snapshot_archives_dir,
|
||||||
full_snapshot_archive_info.slot(),
|
full_snapshot_archive_info.slot(),
|
||||||
)
|
)
|
||||||
.map(|incremental_snapshot_archive_info| {
|
.map(|incremental_snapshot_archive_info| {
|
||||||
|
|
|
@ -285,7 +285,8 @@ impl SnapshotRequestHandler {
|
||||||
status_cache_slot_deltas,
|
status_cache_slot_deltas,
|
||||||
&self.pending_accounts_package,
|
&self.pending_accounts_package,
|
||||||
&self.snapshot_config.bank_snapshots_dir,
|
&self.snapshot_config.bank_snapshots_dir,
|
||||||
&self.snapshot_config.snapshot_archives_dir,
|
&self.snapshot_config.full_snapshot_archives_dir,
|
||||||
|
&self.snapshot_config.incremental_snapshot_archives_dir,
|
||||||
self.snapshot_config.snapshot_version,
|
self.snapshot_config.snapshot_version,
|
||||||
self.snapshot_config.archive_format,
|
self.snapshot_config.archive_format,
|
||||||
hash_for_testing,
|
hash_for_testing,
|
||||||
|
|
|
@ -13,8 +13,11 @@ pub struct SnapshotConfig {
|
||||||
/// Generate a new incremental snapshot archive every this many slots
|
/// Generate a new incremental snapshot archive every this many slots
|
||||||
pub incremental_snapshot_archive_interval_slots: Slot,
|
pub incremental_snapshot_archive_interval_slots: Slot,
|
||||||
|
|
||||||
/// Path to the directory where snapshot archives are stored
|
/// Path to the directory where full snapshot archives are stored
|
||||||
pub snapshot_archives_dir: PathBuf,
|
pub full_snapshot_archives_dir: PathBuf,
|
||||||
|
|
||||||
|
/// Path to the directory where incremental snapshot archives are stored
|
||||||
|
pub incremental_snapshot_archives_dir: PathBuf,
|
||||||
|
|
||||||
/// Path to the directory where bank snapshots are stored
|
/// Path to the directory where bank snapshots are stored
|
||||||
pub bank_snapshots_dir: PathBuf,
|
pub bank_snapshots_dir: PathBuf,
|
||||||
|
@ -46,7 +49,8 @@ impl Default for SnapshotConfig {
|
||||||
snapshot_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
|
snapshot_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
|
||||||
incremental_snapshot_archive_interval_slots:
|
incremental_snapshot_archive_interval_slots:
|
||||||
snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
|
snapshot_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS,
|
||||||
snapshot_archives_dir: PathBuf::default(),
|
full_snapshot_archives_dir: PathBuf::default(),
|
||||||
|
incremental_snapshot_archives_dir: PathBuf::default(),
|
||||||
bank_snapshots_dir: PathBuf::default(),
|
bank_snapshots_dir: PathBuf::default(),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
snapshot_version: SnapshotVersion::default(),
|
snapshot_version: SnapshotVersion::default(),
|
||||||
|
|
|
@ -39,7 +39,8 @@ pub struct AccountsPackage {
|
||||||
pub snapshot_storages: SnapshotStorages,
|
pub snapshot_storages: SnapshotStorages,
|
||||||
pub archive_format: ArchiveFormat,
|
pub archive_format: ArchiveFormat,
|
||||||
pub snapshot_version: SnapshotVersion,
|
pub snapshot_version: SnapshotVersion,
|
||||||
pub snapshot_archives_dir: PathBuf,
|
pub full_snapshot_archives_dir: PathBuf,
|
||||||
|
pub incremental_snapshot_archives_dir: PathBuf,
|
||||||
pub expected_capitalization: u64,
|
pub expected_capitalization: u64,
|
||||||
pub accounts_hash_for_testing: Option<Hash>,
|
pub accounts_hash_for_testing: Option<Hash>,
|
||||||
pub cluster_type: ClusterType,
|
pub cluster_type: ClusterType,
|
||||||
|
@ -57,7 +58,8 @@ impl AccountsPackage {
|
||||||
bank_snapshot_info: &BankSnapshotInfo,
|
bank_snapshot_info: &BankSnapshotInfo,
|
||||||
bank_snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
slot_deltas: Vec<BankSlotDelta>,
|
slot_deltas: Vec<BankSlotDelta>,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
snapshot_storages: SnapshotStorages,
|
snapshot_storages: SnapshotStorages,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
|
@ -105,7 +107,10 @@ impl AccountsPackage {
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
snapshot_archives_dir: snapshot_archives_dir.as_ref().to_path_buf(),
|
full_snapshot_archives_dir: full_snapshot_archives_dir.as_ref().to_path_buf(),
|
||||||
|
incremental_snapshot_archives_dir: incremental_snapshot_archives_dir
|
||||||
|
.as_ref()
|
||||||
|
.to_path_buf(),
|
||||||
expected_capitalization: bank.capitalization(),
|
expected_capitalization: bank.capitalization(),
|
||||||
accounts_hash_for_testing,
|
accounts_hash_for_testing,
|
||||||
cluster_type: bank.cluster_type(),
|
cluster_type: bank.cluster_type(),
|
||||||
|
@ -137,7 +142,7 @@ impl SnapshotPackage {
|
||||||
let mut snapshot_storages = accounts_package.snapshot_storages;
|
let mut snapshot_storages = accounts_package.snapshot_storages;
|
||||||
let snapshot_archive_path = match accounts_package.snapshot_type.unwrap() {
|
let snapshot_archive_path = match accounts_package.snapshot_type.unwrap() {
|
||||||
SnapshotType::FullSnapshot => snapshot_utils::build_full_snapshot_archive_path(
|
SnapshotType::FullSnapshot => snapshot_utils::build_full_snapshot_archive_path(
|
||||||
accounts_package.snapshot_archives_dir,
|
accounts_package.full_snapshot_archives_dir,
|
||||||
accounts_package.slot,
|
accounts_package.slot,
|
||||||
&accounts_hash,
|
&accounts_hash,
|
||||||
accounts_package.archive_format,
|
accounts_package.archive_format,
|
||||||
|
@ -156,7 +161,7 @@ impl SnapshotPackage {
|
||||||
"Incremental snapshot package must only contain storage entries where slot > incremental snapshot base slot (i.e. full snapshot slot)!"
|
"Incremental snapshot package must only contain storage entries where slot > incremental snapshot base slot (i.e. full snapshot slot)!"
|
||||||
);
|
);
|
||||||
snapshot_utils::build_incremental_snapshot_archive_path(
|
snapshot_utils::build_incremental_snapshot_archive_path(
|
||||||
accounts_package.snapshot_archives_dir,
|
accounts_package.incremental_snapshot_archives_dir,
|
||||||
incremental_snapshot_base_slot,
|
incremental_snapshot_base_slot,
|
||||||
accounts_package.slot,
|
accounts_package.slot,
|
||||||
&accounts_hash,
|
&accounts_hash,
|
||||||
|
|
|
@ -249,6 +249,8 @@ pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: impl AsRef<Path>) {
|
||||||
/// Make a snapshot archive out of the snapshot package
|
/// Make a snapshot archive out of the snapshot package
|
||||||
pub fn archive_snapshot_package(
|
pub fn archive_snapshot_package(
|
||||||
snapshot_package: &SnapshotPackage,
|
snapshot_package: &SnapshotPackage,
|
||||||
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
maximum_full_snapshot_archives_to_retain: usize,
|
maximum_full_snapshot_archives_to_retain: usize,
|
||||||
maximum_incremental_snapshot_archives_to_retain: usize,
|
maximum_incremental_snapshot_archives_to_retain: usize,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
@ -379,7 +381,8 @@ pub fn archive_snapshot_package(
|
||||||
.map_err(|e| SnapshotError::IoWithSource(e, "archive path rename"))?;
|
.map_err(|e| SnapshotError::IoWithSource(e, "archive path rename"))?;
|
||||||
|
|
||||||
purge_old_snapshot_archives(
|
purge_old_snapshot_archives(
|
||||||
tar_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
maximum_incremental_snapshot_archives_to_retain,
|
maximum_incremental_snapshot_archives_to_retain,
|
||||||
);
|
);
|
||||||
|
@ -892,12 +895,13 @@ pub fn bank_from_snapshot_archives(
|
||||||
Ok((bank, timings))
|
Ok((bank, timings))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Rebuild bank from snapshot archives. This function searches `snapshot_archives_dir` for the
|
/// Rebuild bank from snapshot archives. This function searches `full_snapshot_archives_dir` and `incremental_snapshot_archives_dir` for the
|
||||||
/// highest full snapshot and highest corresponding incremental snapshot, then rebuilds the bank.
|
/// highest full snapshot and highest corresponding incremental snapshot, then rebuilds the bank.
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub fn bank_from_latest_snapshot_archives(
|
pub fn bank_from_latest_snapshot_archives(
|
||||||
bank_snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
account_paths: &[PathBuf],
|
account_paths: &[PathBuf],
|
||||||
genesis_config: &GenesisConfig,
|
genesis_config: &GenesisConfig,
|
||||||
debug_keys: Option<Arc<HashSet<Pubkey>>>,
|
debug_keys: Option<Arc<HashSet<Pubkey>>>,
|
||||||
|
@ -916,11 +920,12 @@ pub fn bank_from_latest_snapshot_archives(
|
||||||
FullSnapshotArchiveInfo,
|
FullSnapshotArchiveInfo,
|
||||||
Option<IncrementalSnapshotArchiveInfo>,
|
Option<IncrementalSnapshotArchiveInfo>,
|
||||||
)> {
|
)> {
|
||||||
let full_snapshot_archive_info = get_highest_full_snapshot_archive_info(&snapshot_archives_dir)
|
let full_snapshot_archive_info =
|
||||||
.ok_or(SnapshotError::NoSnapshotArchives)?;
|
get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir)
|
||||||
|
.ok_or(SnapshotError::NoSnapshotArchives)?;
|
||||||
|
|
||||||
let incremental_snapshot_archive_info = get_highest_incremental_snapshot_archive_info(
|
let incremental_snapshot_archive_info = get_highest_incremental_snapshot_archive_info(
|
||||||
&snapshot_archives_dir,
|
&incremental_snapshot_archives_dir,
|
||||||
full_snapshot_archive_info.slot(),
|
full_snapshot_archive_info.slot(),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -1125,12 +1130,12 @@ pub fn build_snapshot_archives_remote_dir(snapshot_archives_dir: impl AsRef<Path
|
||||||
/// Build the full snapshot archive path from its components: the snapshot archives directory, the
|
/// Build the full snapshot archive path from its components: the snapshot archives directory, the
|
||||||
/// snapshot slot, the accounts hash, and the archive format.
|
/// snapshot slot, the accounts hash, and the archive format.
|
||||||
pub fn build_full_snapshot_archive_path(
|
pub fn build_full_snapshot_archive_path(
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
hash: &Hash,
|
hash: &Hash,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
) -> PathBuf {
|
) -> PathBuf {
|
||||||
snapshot_archives_dir.as_ref().join(format!(
|
full_snapshot_archives_dir.as_ref().join(format!(
|
||||||
"snapshot-{}-{}.{}",
|
"snapshot-{}-{}.{}",
|
||||||
slot,
|
slot,
|
||||||
hash,
|
hash,
|
||||||
|
@ -1142,13 +1147,13 @@ pub fn build_full_snapshot_archive_path(
|
||||||
/// directory, the snapshot base slot, the snapshot slot, the accounts hash, and the archive
|
/// directory, the snapshot base slot, the snapshot slot, the accounts hash, and the archive
|
||||||
/// format.
|
/// format.
|
||||||
pub fn build_incremental_snapshot_archive_path(
|
pub fn build_incremental_snapshot_archive_path(
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
base_slot: Slot,
|
base_slot: Slot,
|
||||||
slot: Slot,
|
slot: Slot,
|
||||||
hash: &Hash,
|
hash: &Hash,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
) -> PathBuf {
|
) -> PathBuf {
|
||||||
snapshot_archives_dir.as_ref().join(format!(
|
incremental_snapshot_archives_dir.as_ref().join(format!(
|
||||||
"incremental-snapshot-{}-{}-{}.{}",
|
"incremental-snapshot-{}-{}-{}.{}",
|
||||||
base_slot,
|
base_slot,
|
||||||
slot,
|
slot,
|
||||||
|
@ -1256,74 +1261,66 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get a list of the full snapshot archives from a directory
|
/// Get a list of the full snapshot archives from a directory
|
||||||
pub fn get_full_snapshot_archives<P>(snapshot_archives_dir: P) -> Vec<FullSnapshotArchiveInfo>
|
pub fn get_full_snapshot_archives(
|
||||||
where
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
P: AsRef<Path>,
|
) -> Vec<FullSnapshotArchiveInfo> {
|
||||||
{
|
|
||||||
get_snapshot_archives(
|
get_snapshot_archives(
|
||||||
snapshot_archives_dir.as_ref(),
|
full_snapshot_archives_dir.as_ref(),
|
||||||
FullSnapshotArchiveInfo::new_from_path,
|
FullSnapshotArchiveInfo::new_from_path,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get a list of the incremental snapshot archives from a directory
|
/// Get a list of the incremental snapshot archives from a directory
|
||||||
pub fn get_incremental_snapshot_archives<P>(
|
pub fn get_incremental_snapshot_archives(
|
||||||
snapshot_archives_dir: P,
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
) -> Vec<IncrementalSnapshotArchiveInfo>
|
) -> Vec<IncrementalSnapshotArchiveInfo> {
|
||||||
where
|
|
||||||
P: AsRef<Path>,
|
|
||||||
{
|
|
||||||
get_snapshot_archives(
|
get_snapshot_archives(
|
||||||
snapshot_archives_dir.as_ref(),
|
incremental_snapshot_archives_dir.as_ref(),
|
||||||
IncrementalSnapshotArchiveInfo::new_from_path,
|
IncrementalSnapshotArchiveInfo::new_from_path,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the highest slot of the full snapshot archives in a directory
|
/// Get the highest slot of the full snapshot archives in a directory
|
||||||
pub fn get_highest_full_snapshot_archive_slot<P>(snapshot_archives_dir: P) -> Option<Slot>
|
pub fn get_highest_full_snapshot_archive_slot(
|
||||||
where
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
P: AsRef<Path>,
|
) -> Option<Slot> {
|
||||||
{
|
get_highest_full_snapshot_archive_info(full_snapshot_archives_dir)
|
||||||
get_highest_full_snapshot_archive_info(snapshot_archives_dir)
|
|
||||||
.map(|full_snapshot_archive_info| full_snapshot_archive_info.slot())
|
.map(|full_snapshot_archive_info| full_snapshot_archive_info.slot())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the highest slot of the incremental snapshot archives in a directory, for a given full
|
/// Get the highest slot of the incremental snapshot archives in a directory, for a given full
|
||||||
/// snapshot slot
|
/// snapshot slot
|
||||||
pub fn get_highest_incremental_snapshot_archive_slot<P: AsRef<Path>>(
|
pub fn get_highest_incremental_snapshot_archive_slot(
|
||||||
snapshot_archives_dir: P,
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
full_snapshot_slot: Slot,
|
full_snapshot_slot: Slot,
|
||||||
) -> Option<Slot> {
|
) -> Option<Slot> {
|
||||||
get_highest_incremental_snapshot_archive_info(snapshot_archives_dir, full_snapshot_slot)
|
get_highest_incremental_snapshot_archive_info(
|
||||||
.map(|incremental_snapshot_archive_info| incremental_snapshot_archive_info.slot())
|
incremental_snapshot_archives_dir,
|
||||||
|
full_snapshot_slot,
|
||||||
|
)
|
||||||
|
.map(|incremental_snapshot_archive_info| incremental_snapshot_archive_info.slot())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the path (and metadata) for the full snapshot archive with the highest slot in a directory
|
/// Get the path (and metadata) for the full snapshot archive with the highest slot in a directory
|
||||||
pub fn get_highest_full_snapshot_archive_info<P>(
|
pub fn get_highest_full_snapshot_archive_info(
|
||||||
snapshot_archives_dir: P,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
) -> Option<FullSnapshotArchiveInfo>
|
) -> Option<FullSnapshotArchiveInfo> {
|
||||||
where
|
let mut full_snapshot_archives = get_full_snapshot_archives(full_snapshot_archives_dir);
|
||||||
P: AsRef<Path>,
|
|
||||||
{
|
|
||||||
let mut full_snapshot_archives = get_full_snapshot_archives(snapshot_archives_dir);
|
|
||||||
full_snapshot_archives.sort_unstable();
|
full_snapshot_archives.sort_unstable();
|
||||||
full_snapshot_archives.into_iter().rev().next()
|
full_snapshot_archives.into_iter().rev().next()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the path for the incremental snapshot archive with the highest slot, for a given full
|
/// Get the path for the incremental snapshot archive with the highest slot, for a given full
|
||||||
/// snapshot slot, in a directory
|
/// snapshot slot, in a directory
|
||||||
pub fn get_highest_incremental_snapshot_archive_info<P>(
|
pub fn get_highest_incremental_snapshot_archive_info(
|
||||||
snapshot_archives_dir: P,
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
full_snapshot_slot: Slot,
|
full_snapshot_slot: Slot,
|
||||||
) -> Option<IncrementalSnapshotArchiveInfo>
|
) -> Option<IncrementalSnapshotArchiveInfo> {
|
||||||
where
|
|
||||||
P: AsRef<Path>,
|
|
||||||
{
|
|
||||||
// Since we want to filter down to only the incremental snapshot archives that have the same
|
// Since we want to filter down to only the incremental snapshot archives that have the same
|
||||||
// full snapshot slot as the value passed in, perform the filtering before sorting to avoid
|
// full snapshot slot as the value passed in, perform the filtering before sorting to avoid
|
||||||
// doing unnecessary work.
|
// doing unnecessary work.
|
||||||
let mut incremental_snapshot_archives =
|
let mut incremental_snapshot_archives =
|
||||||
get_incremental_snapshot_archives(snapshot_archives_dir)
|
get_incremental_snapshot_archives(incremental_snapshot_archives_dir)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|incremental_snapshot_archive_info| {
|
.filter(|incremental_snapshot_archive_info| {
|
||||||
incremental_snapshot_archive_info.base_slot() == full_snapshot_slot
|
incremental_snapshot_archive_info.base_slot() == full_snapshot_slot
|
||||||
|
@ -1333,21 +1330,19 @@ where
|
||||||
incremental_snapshot_archives.into_iter().rev().next()
|
incremental_snapshot_archives.into_iter().rev().next()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn purge_old_snapshot_archives<P>(
|
pub fn purge_old_snapshot_archives(
|
||||||
snapshot_archives_dir: P,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
maximum_full_snapshot_archives_to_retain: usize,
|
maximum_full_snapshot_archives_to_retain: usize,
|
||||||
maximum_incremental_snapshot_archives_to_retain: usize,
|
maximum_incremental_snapshot_archives_to_retain: usize,
|
||||||
) where
|
) {
|
||||||
P: AsRef<Path>,
|
|
||||||
{
|
|
||||||
info!(
|
info!(
|
||||||
"Purging old snapshot archives in {}, retaining up to {} full snapshots and up to {} incremental snapshots",
|
"Purging old full snapshot archives in {}, retaining up to {} full snapshots",
|
||||||
snapshot_archives_dir.as_ref().display(),
|
full_snapshot_archives_dir.as_ref().display(),
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain
|
||||||
maximum_incremental_snapshot_archives_to_retain
|
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut full_snapshot_archives = get_full_snapshot_archives(&snapshot_archives_dir);
|
let mut full_snapshot_archives = get_full_snapshot_archives(&full_snapshot_archives_dir);
|
||||||
full_snapshot_archives.sort_unstable();
|
full_snapshot_archives.sort_unstable();
|
||||||
full_snapshot_archives.reverse();
|
full_snapshot_archives.reverse();
|
||||||
|
|
||||||
|
@ -1383,8 +1378,15 @@ pub fn purge_old_snapshot_archives<P>(
|
||||||
}
|
}
|
||||||
remove_archives(full_snapshot_archives_to_remove);
|
remove_archives(full_snapshot_archives_to_remove);
|
||||||
|
|
||||||
|
info!(
|
||||||
|
"Purging old incremental snapshot archives in {}, retaining up to {} incremental snapshots",
|
||||||
|
incremental_snapshot_archives_dir.as_ref().display(),
|
||||||
|
maximum_incremental_snapshot_archives_to_retain
|
||||||
|
);
|
||||||
let mut incremental_snapshot_archives_by_base_slot = HashMap::<Slot, Vec<_>>::new();
|
let mut incremental_snapshot_archives_by_base_slot = HashMap::<Slot, Vec<_>>::new();
|
||||||
for incremental_snapshot_archive in get_incremental_snapshot_archives(&snapshot_archives_dir) {
|
for incremental_snapshot_archive in
|
||||||
|
get_incremental_snapshot_archives(&incremental_snapshot_archives_dir)
|
||||||
|
{
|
||||||
incremental_snapshot_archives_by_base_slot
|
incremental_snapshot_archives_by_base_slot
|
||||||
.entry(incremental_snapshot_archive.base_slot())
|
.entry(incremental_snapshot_archive.base_slot())
|
||||||
.or_default()
|
.or_default()
|
||||||
|
@ -1724,12 +1726,14 @@ pub fn purge_old_bank_snapshots(bank_snapshots_dir: impl AsRef<Path>) {
|
||||||
/// function is called from AccountsBackgroundService to handle snapshot requests. Since taking a
|
/// function is called from AccountsBackgroundService to handle snapshot requests. Since taking a
|
||||||
/// snapshot is not permitted to fail, any errors returned here will trigger the node to shutdown.
|
/// snapshot is not permitted to fail, any errors returned here will trigger the node to shutdown.
|
||||||
/// So, be careful whenever adding new code that may return errors.
|
/// So, be careful whenever adding new code that may return errors.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub fn snapshot_bank(
|
pub fn snapshot_bank(
|
||||||
root_bank: &Bank,
|
root_bank: &Bank,
|
||||||
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
status_cache_slot_deltas: Vec<BankSlotDelta>,
|
||||||
pending_accounts_package: &PendingAccountsPackage,
|
pending_accounts_package: &PendingAccountsPackage,
|
||||||
bank_snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
hash_for_testing: Option<Hash>,
|
hash_for_testing: Option<Hash>,
|
||||||
|
@ -1752,7 +1756,8 @@ pub fn snapshot_bank(
|
||||||
&bank_snapshot_info,
|
&bank_snapshot_info,
|
||||||
bank_snapshots_dir,
|
bank_snapshots_dir,
|
||||||
status_cache_slot_deltas,
|
status_cache_slot_deltas,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -1810,7 +1815,8 @@ pub fn bank_to_full_snapshot_archive(
|
||||||
bank_snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
snapshot_version: Option<SnapshotVersion>,
|
snapshot_version: Option<SnapshotVersion>,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
maximum_full_snapshot_archives_to_retain: usize,
|
maximum_full_snapshot_archives_to_retain: usize,
|
||||||
maximum_incremental_snapshot_archives_to_retain: usize,
|
maximum_incremental_snapshot_archives_to_retain: usize,
|
||||||
|
@ -1833,7 +1839,8 @@ pub fn bank_to_full_snapshot_archive(
|
||||||
bank,
|
bank,
|
||||||
&bank_snapshot_info,
|
&bank_snapshot_info,
|
||||||
&temp_dir,
|
&temp_dir,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -1853,7 +1860,8 @@ pub fn bank_to_incremental_snapshot_archive(
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
full_snapshot_slot: Slot,
|
full_snapshot_slot: Slot,
|
||||||
snapshot_version: Option<SnapshotVersion>,
|
snapshot_version: Option<SnapshotVersion>,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
maximum_full_snapshot_archives_to_retain: usize,
|
maximum_full_snapshot_archives_to_retain: usize,
|
||||||
maximum_incremental_snapshot_archives_to_retain: usize,
|
maximum_incremental_snapshot_archives_to_retain: usize,
|
||||||
|
@ -1878,7 +1886,8 @@ pub fn bank_to_incremental_snapshot_archive(
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
&bank_snapshot_info,
|
&bank_snapshot_info,
|
||||||
&temp_dir,
|
&temp_dir,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -1888,11 +1897,13 @@ pub fn bank_to_incremental_snapshot_archive(
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Helper function to hold shared code to package, process, and archive full snapshots
|
/// Helper function to hold shared code to package, process, and archive full snapshots
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub fn package_and_archive_full_snapshot(
|
pub fn package_and_archive_full_snapshot(
|
||||||
bank: &Bank,
|
bank: &Bank,
|
||||||
bank_snapshot_info: &BankSnapshotInfo,
|
bank_snapshot_info: &BankSnapshotInfo,
|
||||||
bank_snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
snapshot_storages: SnapshotStorages,
|
snapshot_storages: SnapshotStorages,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
|
@ -1904,7 +1915,8 @@ pub fn package_and_archive_full_snapshot(
|
||||||
bank_snapshot_info,
|
bank_snapshot_info,
|
||||||
bank_snapshots_dir,
|
bank_snapshots_dir,
|
||||||
bank.src.slot_deltas(&bank.src.roots()),
|
bank.src.slot_deltas(&bank.src.roots()),
|
||||||
snapshot_archives_dir,
|
&full_snapshot_archives_dir,
|
||||||
|
&incremental_snapshot_archives_dir,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -1921,6 +1933,8 @@ pub fn package_and_archive_full_snapshot(
|
||||||
let snapshot_package = SnapshotPackage::new(accounts_package, bank.get_accounts_hash());
|
let snapshot_package = SnapshotPackage::new(accounts_package, bank.get_accounts_hash());
|
||||||
archive_snapshot_package(
|
archive_snapshot_package(
|
||||||
&snapshot_package,
|
&snapshot_package,
|
||||||
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
maximum_incremental_snapshot_archives_to_retain,
|
maximum_incremental_snapshot_archives_to_retain,
|
||||||
)?;
|
)?;
|
||||||
|
@ -1937,7 +1951,8 @@ pub fn package_and_archive_incremental_snapshot(
|
||||||
incremental_snapshot_base_slot: Slot,
|
incremental_snapshot_base_slot: Slot,
|
||||||
bank_snapshot_info: &BankSnapshotInfo,
|
bank_snapshot_info: &BankSnapshotInfo,
|
||||||
bank_snapshots_dir: impl AsRef<Path>,
|
bank_snapshots_dir: impl AsRef<Path>,
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
snapshot_storages: SnapshotStorages,
|
snapshot_storages: SnapshotStorages,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
snapshot_version: SnapshotVersion,
|
snapshot_version: SnapshotVersion,
|
||||||
|
@ -1949,7 +1964,8 @@ pub fn package_and_archive_incremental_snapshot(
|
||||||
bank_snapshot_info,
|
bank_snapshot_info,
|
||||||
bank_snapshots_dir,
|
bank_snapshots_dir,
|
||||||
bank.src.slot_deltas(&bank.src.roots()),
|
bank.src.slot_deltas(&bank.src.roots()),
|
||||||
snapshot_archives_dir,
|
&full_snapshot_archives_dir,
|
||||||
|
&incremental_snapshot_archives_dir,
|
||||||
snapshot_storages,
|
snapshot_storages,
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
|
@ -1968,6 +1984,8 @@ pub fn package_and_archive_incremental_snapshot(
|
||||||
let snapshot_package = SnapshotPackage::new(accounts_package, bank.get_accounts_hash());
|
let snapshot_package = SnapshotPackage::new(accounts_package, bank.get_accounts_hash());
|
||||||
archive_snapshot_package(
|
archive_snapshot_package(
|
||||||
&snapshot_package,
|
&snapshot_package,
|
||||||
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
maximum_incremental_snapshot_archives_to_retain,
|
maximum_incremental_snapshot_archives_to_retain,
|
||||||
)?;
|
)?;
|
||||||
|
@ -2414,13 +2432,15 @@ mod tests {
|
||||||
/// `max_incremental_snapshot_slot`]. Additionally, "bad" files are created for both full and
|
/// `max_incremental_snapshot_slot`]. Additionally, "bad" files are created for both full and
|
||||||
/// incremental snapshots to ensure the tests properly filter them out.
|
/// incremental snapshots to ensure the tests properly filter them out.
|
||||||
fn common_create_snapshot_archive_files(
|
fn common_create_snapshot_archive_files(
|
||||||
snapshot_archives_dir: &Path,
|
full_snapshot_archives_dir: &Path,
|
||||||
|
incremental_snapshot_archives_dir: &Path,
|
||||||
min_full_snapshot_slot: Slot,
|
min_full_snapshot_slot: Slot,
|
||||||
max_full_snapshot_slot: Slot,
|
max_full_snapshot_slot: Slot,
|
||||||
min_incremental_snapshot_slot: Slot,
|
min_incremental_snapshot_slot: Slot,
|
||||||
max_incremental_snapshot_slot: Slot,
|
max_incremental_snapshot_slot: Slot,
|
||||||
) {
|
) {
|
||||||
fs::create_dir_all(snapshot_archives_dir).unwrap();
|
fs::create_dir_all(full_snapshot_archives_dir).unwrap();
|
||||||
|
fs::create_dir_all(incremental_snapshot_archives_dir).unwrap();
|
||||||
for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot {
|
for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot {
|
||||||
for incremental_snapshot_slot in
|
for incremental_snapshot_slot in
|
||||||
min_incremental_snapshot_slot..max_incremental_snapshot_slot
|
min_incremental_snapshot_slot..max_incremental_snapshot_slot
|
||||||
|
@ -2431,13 +2451,13 @@ mod tests {
|
||||||
incremental_snapshot_slot,
|
incremental_snapshot_slot,
|
||||||
Hash::default()
|
Hash::default()
|
||||||
);
|
);
|
||||||
let snapshot_filepath = snapshot_archives_dir.join(snapshot_filename);
|
let snapshot_filepath = incremental_snapshot_archives_dir.join(snapshot_filename);
|
||||||
File::create(snapshot_filepath).unwrap();
|
File::create(snapshot_filepath).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
let snapshot_filename =
|
let snapshot_filename =
|
||||||
format!("snapshot-{}-{}.tar", full_snapshot_slot, Hash::default());
|
format!("snapshot-{}-{}.tar", full_snapshot_slot, Hash::default());
|
||||||
let snapshot_filepath = snapshot_archives_dir.join(snapshot_filename);
|
let snapshot_filepath = full_snapshot_archives_dir.join(snapshot_filename);
|
||||||
File::create(snapshot_filepath).unwrap();
|
File::create(snapshot_filepath).unwrap();
|
||||||
|
|
||||||
// Add in an incremental snapshot with a bad filename and high slot to ensure filename are filtered and sorted correctly
|
// Add in an incremental snapshot with a bad filename and high slot to ensure filename are filtered and sorted correctly
|
||||||
|
@ -2446,50 +2466,54 @@ mod tests {
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
max_incremental_snapshot_slot + 1,
|
max_incremental_snapshot_slot + 1,
|
||||||
);
|
);
|
||||||
let bad_filepath = snapshot_archives_dir.join(bad_filename);
|
let bad_filepath = incremental_snapshot_archives_dir.join(bad_filename);
|
||||||
File::create(bad_filepath).unwrap();
|
File::create(bad_filepath).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add in a snapshot with a bad filename and high slot to ensure filename are filtered and
|
// Add in a snapshot with a bad filename and high slot to ensure filename are filtered and
|
||||||
// sorted correctly
|
// sorted correctly
|
||||||
let bad_filename = format!("snapshot-{}-bad!hash.tar", max_full_snapshot_slot + 1);
|
let bad_filename = format!("snapshot-{}-bad!hash.tar", max_full_snapshot_slot + 1);
|
||||||
let bad_filepath = snapshot_archives_dir.join(bad_filename);
|
let bad_filepath = full_snapshot_archives_dir.join(bad_filename);
|
||||||
File::create(bad_filepath).unwrap();
|
File::create(bad_filepath).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_full_snapshot_archives() {
|
fn test_get_full_snapshot_archives() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let min_slot = 123;
|
let min_slot = 123;
|
||||||
let max_slot = 456;
|
let max_slot = 456;
|
||||||
common_create_snapshot_archive_files(
|
common_create_snapshot_archive_files(
|
||||||
temp_snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
min_slot,
|
min_slot,
|
||||||
max_slot,
|
max_slot,
|
||||||
0,
|
0,
|
||||||
0,
|
0,
|
||||||
);
|
);
|
||||||
|
|
||||||
let snapshot_archives = get_full_snapshot_archives(temp_snapshot_archives_dir);
|
let snapshot_archives = get_full_snapshot_archives(full_snapshot_archives_dir);
|
||||||
assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot);
|
assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_full_snapshot_archives_remote() {
|
fn test_get_full_snapshot_archives_remote() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let min_slot = 123;
|
let min_slot = 123;
|
||||||
let max_slot = 456;
|
let max_slot = 456;
|
||||||
common_create_snapshot_archive_files(
|
common_create_snapshot_archive_files(
|
||||||
&temp_snapshot_archives_dir.path().join("remote"),
|
&full_snapshot_archives_dir.path().join("remote"),
|
||||||
|
&incremental_snapshot_archives_dir.path().join("remote"),
|
||||||
min_slot,
|
min_slot,
|
||||||
max_slot,
|
max_slot,
|
||||||
0,
|
0,
|
||||||
0,
|
0,
|
||||||
);
|
);
|
||||||
|
|
||||||
let snapshot_archives = get_full_snapshot_archives(temp_snapshot_archives_dir);
|
let snapshot_archives = get_full_snapshot_archives(full_snapshot_archives_dir);
|
||||||
assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot);
|
assert_eq!(snapshot_archives.len() as Slot, max_slot - min_slot);
|
||||||
assert!(snapshot_archives.iter().all(|info| info.is_remote()));
|
assert!(snapshot_archives.iter().all(|info| info.is_remote()));
|
||||||
}
|
}
|
||||||
|
@ -2497,13 +2521,15 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_incremental_snapshot_archives() {
|
fn test_get_incremental_snapshot_archives() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let min_full_snapshot_slot = 12;
|
let min_full_snapshot_slot = 12;
|
||||||
let max_full_snapshot_slot = 23;
|
let max_full_snapshot_slot = 23;
|
||||||
let min_incremental_snapshot_slot = 34;
|
let min_incremental_snapshot_slot = 34;
|
||||||
let max_incremental_snapshot_slot = 45;
|
let max_incremental_snapshot_slot = 45;
|
||||||
common_create_snapshot_archive_files(
|
common_create_snapshot_archive_files(
|
||||||
temp_snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
min_full_snapshot_slot,
|
min_full_snapshot_slot,
|
||||||
max_full_snapshot_slot,
|
max_full_snapshot_slot,
|
||||||
min_incremental_snapshot_slot,
|
min_incremental_snapshot_slot,
|
||||||
|
@ -2511,7 +2537,7 @@ mod tests {
|
||||||
);
|
);
|
||||||
|
|
||||||
let incremental_snapshot_archives =
|
let incremental_snapshot_archives =
|
||||||
get_incremental_snapshot_archives(temp_snapshot_archives_dir);
|
get_incremental_snapshot_archives(incremental_snapshot_archives_dir);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
incremental_snapshot_archives.len() as Slot,
|
incremental_snapshot_archives.len() as Slot,
|
||||||
(max_full_snapshot_slot - min_full_snapshot_slot)
|
(max_full_snapshot_slot - min_full_snapshot_slot)
|
||||||
|
@ -2522,13 +2548,15 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_incremental_snapshot_archives_remote() {
|
fn test_get_incremental_snapshot_archives_remote() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let min_full_snapshot_slot = 12;
|
let min_full_snapshot_slot = 12;
|
||||||
let max_full_snapshot_slot = 23;
|
let max_full_snapshot_slot = 23;
|
||||||
let min_incremental_snapshot_slot = 34;
|
let min_incremental_snapshot_slot = 34;
|
||||||
let max_incremental_snapshot_slot = 45;
|
let max_incremental_snapshot_slot = 45;
|
||||||
common_create_snapshot_archive_files(
|
common_create_snapshot_archive_files(
|
||||||
&temp_snapshot_archives_dir.path().join("remote"),
|
&full_snapshot_archives_dir.path().join("remote"),
|
||||||
|
&incremental_snapshot_archives_dir.path().join("remote"),
|
||||||
min_full_snapshot_slot,
|
min_full_snapshot_slot,
|
||||||
max_full_snapshot_slot,
|
max_full_snapshot_slot,
|
||||||
min_incremental_snapshot_slot,
|
min_incremental_snapshot_slot,
|
||||||
|
@ -2536,7 +2564,7 @@ mod tests {
|
||||||
);
|
);
|
||||||
|
|
||||||
let incremental_snapshot_archives =
|
let incremental_snapshot_archives =
|
||||||
get_incremental_snapshot_archives(temp_snapshot_archives_dir);
|
get_incremental_snapshot_archives(incremental_snapshot_archives_dir);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
incremental_snapshot_archives.len() as Slot,
|
incremental_snapshot_archives.len() as Slot,
|
||||||
(max_full_snapshot_slot - min_full_snapshot_slot)
|
(max_full_snapshot_slot - min_full_snapshot_slot)
|
||||||
|
@ -2550,11 +2578,13 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_highest_full_snapshot_archive_slot() {
|
fn test_get_highest_full_snapshot_archive_slot() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let min_slot = 123;
|
let min_slot = 123;
|
||||||
let max_slot = 456;
|
let max_slot = 456;
|
||||||
common_create_snapshot_archive_files(
|
common_create_snapshot_archive_files(
|
||||||
temp_snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
min_slot,
|
min_slot,
|
||||||
max_slot,
|
max_slot,
|
||||||
0,
|
0,
|
||||||
|
@ -2562,7 +2592,7 @@ mod tests {
|
||||||
);
|
);
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
get_highest_full_snapshot_archive_slot(temp_snapshot_archives_dir.path()),
|
get_highest_full_snapshot_archive_slot(full_snapshot_archives_dir.path()),
|
||||||
Some(max_slot - 1)
|
Some(max_slot - 1)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
@ -2570,13 +2600,15 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_highest_incremental_snapshot_slot() {
|
fn test_get_highest_incremental_snapshot_slot() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let temp_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let min_full_snapshot_slot = 12;
|
let min_full_snapshot_slot = 12;
|
||||||
let max_full_snapshot_slot = 23;
|
let max_full_snapshot_slot = 23;
|
||||||
let min_incremental_snapshot_slot = 34;
|
let min_incremental_snapshot_slot = 34;
|
||||||
let max_incremental_snapshot_slot = 45;
|
let max_incremental_snapshot_slot = 45;
|
||||||
common_create_snapshot_archive_files(
|
common_create_snapshot_archive_files(
|
||||||
temp_snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
min_full_snapshot_slot,
|
min_full_snapshot_slot,
|
||||||
max_full_snapshot_slot,
|
max_full_snapshot_slot,
|
||||||
min_incremental_snapshot_slot,
|
min_incremental_snapshot_slot,
|
||||||
|
@ -2586,7 +2618,7 @@ mod tests {
|
||||||
for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot {
|
for full_snapshot_slot in min_full_snapshot_slot..max_full_snapshot_slot {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
get_highest_incremental_snapshot_archive_slot(
|
get_highest_incremental_snapshot_archive_slot(
|
||||||
temp_snapshot_archives_dir.path(),
|
incremental_snapshot_archives_dir.path(),
|
||||||
full_snapshot_slot
|
full_snapshot_slot
|
||||||
),
|
),
|
||||||
Some(max_incremental_snapshot_slot - 1)
|
Some(max_incremental_snapshot_slot - 1)
|
||||||
|
@ -2595,7 +2627,7 @@ mod tests {
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
get_highest_incremental_snapshot_archive_slot(
|
get_highest_incremental_snapshot_archive_slot(
|
||||||
temp_snapshot_archives_dir.path(),
|
incremental_snapshot_archives_dir.path(),
|
||||||
max_full_snapshot_slot
|
max_full_snapshot_slot
|
||||||
),
|
),
|
||||||
None
|
None
|
||||||
|
@ -2615,6 +2647,7 @@ mod tests {
|
||||||
let mut _snap_file = File::create(snap_path);
|
let mut _snap_file = File::create(snap_path);
|
||||||
}
|
}
|
||||||
purge_old_snapshot_archives(
|
purge_old_snapshot_archives(
|
||||||
|
temp_snap_dir.path(),
|
||||||
temp_snap_dir.path(),
|
temp_snap_dir.path(),
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
maximum_incremental_snapshot_archives_to_retain,
|
maximum_incremental_snapshot_archives_to_retain,
|
||||||
|
@ -2691,14 +2724,15 @@ mod tests {
|
||||||
/// snapshot archives on disk are correct.
|
/// snapshot archives on disk are correct.
|
||||||
#[test]
|
#[test]
|
||||||
fn test_purge_old_full_snapshot_archives_in_the_loop() {
|
fn test_purge_old_full_snapshot_archives_in_the_loop() {
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let maximum_snapshots_to_retain = 5;
|
let maximum_snapshots_to_retain = 5;
|
||||||
let starting_slot: Slot = 42;
|
let starting_slot: Slot = 42;
|
||||||
|
|
||||||
for slot in (starting_slot..).take(100) {
|
for slot in (starting_slot..).take(100) {
|
||||||
let full_snapshot_archive_file_name =
|
let full_snapshot_archive_file_name =
|
||||||
format!("snapshot-{}-{}.tar", slot, Hash::default());
|
format!("snapshot-{}-{}.tar", slot, Hash::default());
|
||||||
let full_snapshot_archive_path = snapshot_archives_dir
|
let full_snapshot_archive_path = full_snapshot_archives_dir
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.join(full_snapshot_archive_file_name);
|
.join(full_snapshot_archive_file_name);
|
||||||
File::create(full_snapshot_archive_path).unwrap();
|
File::create(full_snapshot_archive_path).unwrap();
|
||||||
|
@ -2714,11 +2748,13 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
purge_old_snapshot_archives(
|
purge_old_snapshot_archives(
|
||||||
&snapshot_archives_dir,
|
&full_snapshot_archives_dir,
|
||||||
|
&incremental_snapshot_archives_dir,
|
||||||
maximum_snapshots_to_retain,
|
maximum_snapshots_to_retain,
|
||||||
usize::MAX,
|
usize::MAX,
|
||||||
);
|
);
|
||||||
let mut full_snapshot_archives = get_full_snapshot_archives(&snapshot_archives_dir);
|
let mut full_snapshot_archives =
|
||||||
|
get_full_snapshot_archives(&full_snapshot_archives_dir);
|
||||||
full_snapshot_archives.sort_unstable();
|
full_snapshot_archives.sort_unstable();
|
||||||
assert_eq!(full_snapshot_archives.len(), maximum_snapshots_to_retain);
|
assert_eq!(full_snapshot_archives.len(), maximum_snapshots_to_retain);
|
||||||
assert_eq!(full_snapshot_archives.last().unwrap().slot(), slot);
|
assert_eq!(full_snapshot_archives.last().unwrap().slot(), slot);
|
||||||
|
@ -2731,7 +2767,8 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn test_purge_old_incremental_snapshot_archives() {
|
fn test_purge_old_incremental_snapshot_archives() {
|
||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let starting_slot = 100_000;
|
let starting_slot = 100_000;
|
||||||
|
|
||||||
let maximum_incremental_snapshot_archives_to_retain =
|
let maximum_incremental_snapshot_archives_to_retain =
|
||||||
|
@ -2751,7 +2788,7 @@ mod tests {
|
||||||
.for_each(|full_snapshot_slot| {
|
.for_each(|full_snapshot_slot| {
|
||||||
let snapshot_filename =
|
let snapshot_filename =
|
||||||
format!("snapshot-{}-{}.tar", full_snapshot_slot, Hash::default());
|
format!("snapshot-{}-{}.tar", full_snapshot_slot, Hash::default());
|
||||||
let snapshot_path = snapshot_archives_dir.path().join(&snapshot_filename);
|
let snapshot_path = full_snapshot_archives_dir.path().join(&snapshot_filename);
|
||||||
File::create(snapshot_path).unwrap();
|
File::create(snapshot_path).unwrap();
|
||||||
snapshot_filenames.push(snapshot_filename);
|
snapshot_filenames.push(snapshot_filename);
|
||||||
|
|
||||||
|
@ -2766,14 +2803,17 @@ mod tests {
|
||||||
incremental_snapshot_slot,
|
incremental_snapshot_slot,
|
||||||
Hash::default()
|
Hash::default()
|
||||||
);
|
);
|
||||||
let snapshot_path = snapshot_archives_dir.path().join(&snapshot_filename);
|
let snapshot_path = incremental_snapshot_archives_dir
|
||||||
|
.path()
|
||||||
|
.join(&snapshot_filename);
|
||||||
File::create(snapshot_path).unwrap();
|
File::create(snapshot_path).unwrap();
|
||||||
snapshot_filenames.push(snapshot_filename);
|
snapshot_filenames.push(snapshot_filename);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
purge_old_snapshot_archives(
|
purge_old_snapshot_archives(
|
||||||
snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
maximum_incremental_snapshot_archives_to_retain,
|
maximum_incremental_snapshot_archives_to_retain,
|
||||||
);
|
);
|
||||||
|
@ -2781,7 +2821,7 @@ mod tests {
|
||||||
// Ensure correct number of full snapshot archives are purged/retained
|
// Ensure correct number of full snapshot archives are purged/retained
|
||||||
// NOTE: One extra full snapshot is always kept (the oldest), hence the `+1`
|
// NOTE: One extra full snapshot is always kept (the oldest), hence the `+1`
|
||||||
let mut remaining_full_snapshot_archives =
|
let mut remaining_full_snapshot_archives =
|
||||||
get_full_snapshot_archives(snapshot_archives_dir.path());
|
get_full_snapshot_archives(full_snapshot_archives_dir.path());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
remaining_full_snapshot_archives.len(),
|
remaining_full_snapshot_archives.len(),
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
|
@ -2792,7 +2832,7 @@ mod tests {
|
||||||
|
|
||||||
// Ensure correct number of incremental snapshot archives are purged/retained
|
// Ensure correct number of incremental snapshot archives are purged/retained
|
||||||
let mut remaining_incremental_snapshot_archives =
|
let mut remaining_incremental_snapshot_archives =
|
||||||
get_incremental_snapshot_archives(snapshot_archives_dir.path());
|
get_incremental_snapshot_archives(incremental_snapshot_archives_dir.path());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
remaining_incremental_snapshot_archives.len(),
|
remaining_incremental_snapshot_archives.len(),
|
||||||
maximum_incremental_snapshot_archives_to_retain
|
maximum_incremental_snapshot_archives_to_retain
|
||||||
|
@ -2846,7 +2886,8 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_purge_all_incremental_snapshot_archives_when_no_full_snapshot_archives() {
|
fn test_purge_all_incremental_snapshot_archives_when_no_full_snapshot_archives() {
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
|
||||||
for snapshot_filenames in [
|
for snapshot_filenames in [
|
||||||
format!("incremental-snapshot-100-120-{}.tar", Hash::default()),
|
format!("incremental-snapshot-100-120-{}.tar", Hash::default()),
|
||||||
|
@ -2858,14 +2899,21 @@ mod tests {
|
||||||
format!("incremental-snapshot-200-260-{}.tar", Hash::default()),
|
format!("incremental-snapshot-200-260-{}.tar", Hash::default()),
|
||||||
format!("incremental-snapshot-200-280-{}.tar", Hash::default()),
|
format!("incremental-snapshot-200-280-{}.tar", Hash::default()),
|
||||||
] {
|
] {
|
||||||
let snapshot_path = snapshot_archives_dir.path().join(&snapshot_filenames);
|
let snapshot_path = incremental_snapshot_archives_dir
|
||||||
|
.path()
|
||||||
|
.join(&snapshot_filenames);
|
||||||
File::create(snapshot_path).unwrap();
|
File::create(snapshot_path).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
purge_old_snapshot_archives(snapshot_archives_dir.path(), usize::MAX, usize::MAX);
|
purge_old_snapshot_archives(
|
||||||
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
|
usize::MAX,
|
||||||
|
usize::MAX,
|
||||||
|
);
|
||||||
|
|
||||||
let remaining_incremental_snapshot_archives =
|
let remaining_incremental_snapshot_archives =
|
||||||
get_incremental_snapshot_archives(snapshot_archives_dir.path());
|
get_incremental_snapshot_archives(incremental_snapshot_archives_dir.path());
|
||||||
assert!(remaining_incremental_snapshot_archives.is_empty());
|
assert!(remaining_incremental_snapshot_archives.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2883,14 +2931,16 @@ mod tests {
|
||||||
|
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::Tar;
|
let snapshot_archive_format = ArchiveFormat::Tar;
|
||||||
|
|
||||||
let snapshot_archive_info = bank_to_full_snapshot_archive(
|
let snapshot_archive_info = bank_to_full_snapshot_archive(
|
||||||
&bank_snapshots_dir,
|
&bank_snapshots_dir,
|
||||||
&original_bank,
|
&original_bank,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
snapshot_archive_format,
|
snapshot_archive_format,
|
||||||
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -2974,14 +3024,16 @@ mod tests {
|
||||||
|
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::TarGzip;
|
let snapshot_archive_format = ArchiveFormat::TarGzip;
|
||||||
|
|
||||||
let full_snapshot_archive_info = bank_to_full_snapshot_archive(
|
let full_snapshot_archive_info = bank_to_full_snapshot_archive(
|
||||||
bank_snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&bank4,
|
&bank4,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
snapshot_archive_format,
|
snapshot_archive_format,
|
||||||
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -3050,7 +3102,8 @@ mod tests {
|
||||||
|
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::TarZstd;
|
let snapshot_archive_format = ArchiveFormat::TarZstd;
|
||||||
|
|
||||||
let full_snapshot_slot = slot;
|
let full_snapshot_slot = slot;
|
||||||
|
@ -3058,7 +3111,8 @@ mod tests {
|
||||||
bank_snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&bank1,
|
&bank1,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
snapshot_archive_format,
|
snapshot_archive_format,
|
||||||
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -3091,7 +3145,8 @@ mod tests {
|
||||||
&bank4,
|
&bank4,
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
snapshot_archive_format,
|
snapshot_archive_format,
|
||||||
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -3150,7 +3205,8 @@ mod tests {
|
||||||
|
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::Tar;
|
let snapshot_archive_format = ArchiveFormat::Tar;
|
||||||
|
|
||||||
let full_snapshot_slot = slot;
|
let full_snapshot_slot = slot;
|
||||||
|
@ -3158,7 +3214,8 @@ mod tests {
|
||||||
&bank_snapshots_dir,
|
&bank_snapshots_dir,
|
||||||
&bank1,
|
&bank1,
|
||||||
None,
|
None,
|
||||||
&snapshot_archives_dir,
|
&full_snapshot_archives_dir,
|
||||||
|
&incremental_snapshot_archives_dir,
|
||||||
snapshot_archive_format,
|
snapshot_archive_format,
|
||||||
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -3191,7 +3248,8 @@ mod tests {
|
||||||
&bank4,
|
&bank4,
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
None,
|
None,
|
||||||
&snapshot_archives_dir,
|
&full_snapshot_archives_dir,
|
||||||
|
&incremental_snapshot_archives_dir,
|
||||||
snapshot_archive_format,
|
snapshot_archive_format,
|
||||||
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -3200,7 +3258,8 @@ mod tests {
|
||||||
|
|
||||||
let (deserialized_bank, ..) = bank_from_latest_snapshot_archives(
|
let (deserialized_bank, ..) = bank_from_latest_snapshot_archives(
|
||||||
&bank_snapshots_dir,
|
&bank_snapshots_dir,
|
||||||
&snapshot_archives_dir,
|
&full_snapshot_archives_dir,
|
||||||
|
&incremental_snapshot_archives_dir,
|
||||||
&[accounts_dir.as_ref().to_path_buf()],
|
&[accounts_dir.as_ref().to_path_buf()],
|
||||||
&genesis_config,
|
&genesis_config,
|
||||||
None,
|
None,
|
||||||
|
@ -3252,7 +3311,8 @@ mod tests {
|
||||||
|
|
||||||
let accounts_dir = tempfile::TempDir::new().unwrap();
|
let accounts_dir = tempfile::TempDir::new().unwrap();
|
||||||
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
let bank_snapshots_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
let full_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = tempfile::TempDir::new().unwrap();
|
||||||
let snapshot_archive_format = ArchiveFormat::Tar;
|
let snapshot_archive_format = ArchiveFormat::Tar;
|
||||||
|
|
||||||
let (genesis_config, mint_keypair) = create_genesis_config(1_000_000);
|
let (genesis_config, mint_keypair) = create_genesis_config(1_000_000);
|
||||||
|
@ -3289,7 +3349,8 @@ mod tests {
|
||||||
bank_snapshots_dir.path(),
|
bank_snapshots_dir.path(),
|
||||||
&bank1,
|
&bank1,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
snapshot_archive_format,
|
snapshot_archive_format,
|
||||||
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -3329,7 +3390,8 @@ mod tests {
|
||||||
&bank2,
|
&bank2,
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
snapshot_archive_format,
|
snapshot_archive_format,
|
||||||
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -3390,7 +3452,8 @@ mod tests {
|
||||||
&bank4,
|
&bank4,
|
||||||
full_snapshot_slot,
|
full_snapshot_slot,
|
||||||
None,
|
None,
|
||||||
snapshot_archives_dir.path(),
|
full_snapshot_archives_dir.path(),
|
||||||
|
incremental_snapshot_archives_dir.path(),
|
||||||
snapshot_archive_format,
|
snapshot_archive_format,
|
||||||
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
||||||
|
@ -3455,7 +3518,8 @@ mod tests {
|
||||||
snapshot_storages: SnapshotStorages::default(),
|
snapshot_storages: SnapshotStorages::default(),
|
||||||
archive_format: ArchiveFormat::Tar,
|
archive_format: ArchiveFormat::Tar,
|
||||||
snapshot_version: SnapshotVersion::default(),
|
snapshot_version: SnapshotVersion::default(),
|
||||||
snapshot_archives_dir: PathBuf::default(),
|
full_snapshot_archives_dir: PathBuf::default(),
|
||||||
|
incremental_snapshot_archives_dir: PathBuf::default(),
|
||||||
expected_capitalization: u64::default(),
|
expected_capitalization: u64::default(),
|
||||||
accounts_hash_for_testing: None,
|
accounts_hash_for_testing: None,
|
||||||
cluster_type: solana_sdk::genesis_config::ClusterType::Development,
|
cluster_type: solana_sdk::genesis_config::ClusterType::Development,
|
||||||
|
|
|
@ -711,7 +711,8 @@ impl TestValidator {
|
||||||
full_snapshot_archive_interval_slots: 100,
|
full_snapshot_archive_interval_slots: 100,
|
||||||
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
bank_snapshots_dir: ledger_path.join("snapshot"),
|
bank_snapshots_dir: ledger_path.join("snapshot"),
|
||||||
snapshot_archives_dir: ledger_path.to_path_buf(),
|
full_snapshot_archives_dir: ledger_path.to_path_buf(),
|
||||||
|
incremental_snapshot_archives_dir: ledger_path.to_path_buf(),
|
||||||
..SnapshotConfig::default()
|
..SnapshotConfig::default()
|
||||||
}),
|
}),
|
||||||
enforce_ulimit_nofile: false,
|
enforce_ulimit_nofile: false,
|
||||||
|
|
|
@ -56,7 +56,8 @@ pub fn rpc_bootstrap(
|
||||||
node: &Node,
|
node: &Node,
|
||||||
identity_keypair: &Arc<Keypair>,
|
identity_keypair: &Arc<Keypair>,
|
||||||
ledger_path: &Path,
|
ledger_path: &Path,
|
||||||
snapshot_archives_dir: &Path,
|
full_snapshot_archives_dir: &Path,
|
||||||
|
incremental_snapshot_archives_dir: &Path,
|
||||||
vote_account: &Pubkey,
|
vote_account: &Pubkey,
|
||||||
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
|
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
|
||||||
cluster_entrypoints: &[ContactInfo],
|
cluster_entrypoints: &[ContactInfo],
|
||||||
|
@ -96,7 +97,8 @@ pub fn rpc_bootstrap(
|
||||||
node,
|
node,
|
||||||
identity_keypair,
|
identity_keypair,
|
||||||
ledger_path,
|
ledger_path,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
vote_account,
|
vote_account,
|
||||||
authorized_voter_keypairs,
|
authorized_voter_keypairs,
|
||||||
cluster_entrypoints,
|
cluster_entrypoints,
|
||||||
|
@ -116,7 +118,7 @@ pub fn rpc_bootstrap(
|
||||||
node,
|
node,
|
||||||
identity_keypair,
|
identity_keypair,
|
||||||
ledger_path,
|
ledger_path,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
vote_account,
|
vote_account,
|
||||||
authorized_voter_keypairs,
|
authorized_voter_keypairs,
|
||||||
cluster_entrypoints,
|
cluster_entrypoints,
|
||||||
|
@ -384,32 +386,6 @@ fn check_vote_account(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the Slot and Hash of the local snapshot with the highest slot. Can be either a full
|
|
||||||
/// snapshot or an incremental snapshot.
|
|
||||||
fn get_highest_local_snapshot_hash(
|
|
||||||
snapshot_archives_dir: impl AsRef<Path>,
|
|
||||||
) -> Option<(Slot, Hash)> {
|
|
||||||
if let Some(full_snapshot_info) =
|
|
||||||
snapshot_utils::get_highest_full_snapshot_archive_info(&snapshot_archives_dir)
|
|
||||||
{
|
|
||||||
if let Some(incremental_snapshot_info) =
|
|
||||||
snapshot_utils::get_highest_incremental_snapshot_archive_info(
|
|
||||||
&snapshot_archives_dir,
|
|
||||||
full_snapshot_info.slot(),
|
|
||||||
)
|
|
||||||
{
|
|
||||||
Some((
|
|
||||||
incremental_snapshot_info.slot(),
|
|
||||||
*incremental_snapshot_info.hash(),
|
|
||||||
))
|
|
||||||
} else {
|
|
||||||
Some((full_snapshot_info.slot(), *full_snapshot_info.hash()))
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
mod without_incremental_snapshots {
|
mod without_incremental_snapshots {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
@ -418,7 +394,7 @@ mod without_incremental_snapshots {
|
||||||
node: &Node,
|
node: &Node,
|
||||||
identity_keypair: &Arc<Keypair>,
|
identity_keypair: &Arc<Keypair>,
|
||||||
ledger_path: &Path,
|
ledger_path: &Path,
|
||||||
snapshot_archives_dir: &Path,
|
full_snapshot_archives_dir: &Path,
|
||||||
vote_account: &Pubkey,
|
vote_account: &Pubkey,
|
||||||
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
|
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
|
||||||
cluster_entrypoints: &[ContactInfo],
|
cluster_entrypoints: &[ContactInfo],
|
||||||
|
@ -458,7 +434,7 @@ mod without_incremental_snapshots {
|
||||||
validator_config,
|
validator_config,
|
||||||
&mut blacklisted_rpc_nodes,
|
&mut blacklisted_rpc_nodes,
|
||||||
&bootstrap_config,
|
&bootstrap_config,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
);
|
);
|
||||||
if rpc_node_details.is_none() {
|
if rpc_node_details.is_none() {
|
||||||
return;
|
return;
|
||||||
|
@ -512,7 +488,7 @@ mod without_incremental_snapshots {
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(snapshot_hash) = snapshot_hash {
|
if let Some(snapshot_hash) = snapshot_hash {
|
||||||
let use_local_snapshot = match get_highest_local_snapshot_hash(snapshot_archives_dir) {
|
let use_local_snapshot = match get_highest_local_snapshot_hash(full_snapshot_archives_dir) {
|
||||||
None => {
|
None => {
|
||||||
info!("Downloading snapshot for slot {} since there is not a local snapshot", snapshot_hash.0);
|
info!("Downloading snapshot for slot {} since there is not a local snapshot", snapshot_hash.0);
|
||||||
false
|
false
|
||||||
|
@ -555,16 +531,17 @@ mod without_incremental_snapshots {
|
||||||
gossip.take().unwrap();
|
gossip.take().unwrap();
|
||||||
cluster_info.save_contact_info();
|
cluster_info.save_contact_info();
|
||||||
gossip_exit_flag.store(true, Ordering::Relaxed);
|
gossip_exit_flag.store(true, Ordering::Relaxed);
|
||||||
let (maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain) = if let Some(snapshot_config) =
|
let (maximum_full_snapshot_archives_to_retain, maximum_incremental_snapshot_archives_to_retain, incremental_snapshot_archives_dir) = if let Some(snapshot_config) =
|
||||||
validator_config.snapshot_config.as_ref()
|
validator_config.snapshot_config.as_ref()
|
||||||
{
|
{
|
||||||
(snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain)
|
(snapshot_config.maximum_full_snapshot_archives_to_retain, snapshot_config.maximum_incremental_snapshot_archives_to_retain, snapshot_config.incremental_snapshot_archives_dir.as_path())
|
||||||
} else {
|
} else {
|
||||||
(DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN)
|
(DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN, DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, full_snapshot_archives_dir)
|
||||||
};
|
};
|
||||||
let ret = download_snapshot_archive(
|
let ret = download_snapshot_archive(
|
||||||
&rpc_contact_info.rpc,
|
&rpc_contact_info.rpc,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
snapshot_hash,
|
snapshot_hash,
|
||||||
SnapshotType::FullSnapshot,
|
SnapshotType::FullSnapshot,
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
|
@ -668,7 +645,7 @@ mod without_incremental_snapshots {
|
||||||
validator_config: &ValidatorConfig,
|
validator_config: &ValidatorConfig,
|
||||||
blacklisted_rpc_nodes: &mut HashSet<Pubkey>,
|
blacklisted_rpc_nodes: &mut HashSet<Pubkey>,
|
||||||
bootstrap_config: &RpcBootstrapConfig,
|
bootstrap_config: &RpcBootstrapConfig,
|
||||||
snapshot_archives_dir: &Path,
|
full_snapshot_archives_dir: &Path,
|
||||||
) -> Option<(ContactInfo, Option<(Slot, Hash)>)> {
|
) -> Option<(ContactInfo, Option<(Slot, Hash)>)> {
|
||||||
let mut blacklist_timeout = Instant::now();
|
let mut blacklist_timeout = Instant::now();
|
||||||
let mut newer_cluster_snapshot_timeout = None;
|
let mut newer_cluster_snapshot_timeout = None;
|
||||||
|
@ -692,7 +669,8 @@ mod without_incremental_snapshots {
|
||||||
let rpc_peers = rpc_peers.unwrap();
|
let rpc_peers = rpc_peers.unwrap();
|
||||||
blacklist_timeout = Instant::now();
|
blacklist_timeout = Instant::now();
|
||||||
|
|
||||||
let mut highest_snapshot_hash = get_highest_local_snapshot_hash(snapshot_archives_dir);
|
let mut highest_snapshot_hash =
|
||||||
|
get_highest_local_snapshot_hash(full_snapshot_archives_dir);
|
||||||
let eligible_rpc_peers = if bootstrap_config.no_snapshot_fetch {
|
let eligible_rpc_peers = if bootstrap_config.no_snapshot_fetch {
|
||||||
rpc_peers
|
rpc_peers
|
||||||
} else {
|
} else {
|
||||||
|
@ -795,6 +773,14 @@ mod without_incremental_snapshots {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Get the Slot and Hash of the local snapshot with the highest slot.
|
||||||
|
fn get_highest_local_snapshot_hash(
|
||||||
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
) -> Option<(Slot, Hash)> {
|
||||||
|
snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir)
|
||||||
|
.map(|full_snapshot_info| (full_snapshot_info.slot(), *full_snapshot_info.hash()))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
mod with_incremental_snapshots {
|
mod with_incremental_snapshots {
|
||||||
|
@ -829,7 +815,8 @@ mod with_incremental_snapshots {
|
||||||
node: &Node,
|
node: &Node,
|
||||||
identity_keypair: &Arc<Keypair>,
|
identity_keypair: &Arc<Keypair>,
|
||||||
ledger_path: &Path,
|
ledger_path: &Path,
|
||||||
snapshot_archives_dir: &Path,
|
full_snapshot_archives_dir: &Path,
|
||||||
|
incremental_snapshot_archives_dir: &Path,
|
||||||
vote_account: &Pubkey,
|
vote_account: &Pubkey,
|
||||||
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
|
authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
|
||||||
cluster_entrypoints: &[ContactInfo],
|
cluster_entrypoints: &[ContactInfo],
|
||||||
|
@ -935,7 +922,8 @@ mod with_incremental_snapshots {
|
||||||
info!("RPC node root slot: {}", rpc_client_slot);
|
info!("RPC node root slot: {}", rpc_client_slot);
|
||||||
|
|
||||||
download_snapshots(
|
download_snapshots(
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
validator_config,
|
validator_config,
|
||||||
&bootstrap_config,
|
&bootstrap_config,
|
||||||
use_progress_bar,
|
use_progress_bar,
|
||||||
|
@ -1088,6 +1076,33 @@ mod with_incremental_snapshots {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Get the Slot and Hash of the local snapshot with the highest slot. Can be either a full
|
||||||
|
/// snapshot or an incremental snapshot.
|
||||||
|
fn get_highest_local_snapshot_hash(
|
||||||
|
full_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
incremental_snapshot_archives_dir: impl AsRef<Path>,
|
||||||
|
) -> Option<(Slot, Hash)> {
|
||||||
|
if let Some(full_snapshot_info) =
|
||||||
|
snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir)
|
||||||
|
{
|
||||||
|
if let Some(incremental_snapshot_info) =
|
||||||
|
snapshot_utils::get_highest_incremental_snapshot_archive_info(
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
|
full_snapshot_info.slot(),
|
||||||
|
)
|
||||||
|
{
|
||||||
|
Some((
|
||||||
|
incremental_snapshot_info.slot(),
|
||||||
|
*incremental_snapshot_info.hash(),
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
Some((full_snapshot_info.slot(), *full_snapshot_info.hash()))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Get peer snapshot hashes
|
/// Get peer snapshot hashes
|
||||||
///
|
///
|
||||||
/// The result is a vector of peers with snapshot hashes that:
|
/// The result is a vector of peers with snapshot hashes that:
|
||||||
|
@ -1420,7 +1435,8 @@ mod with_incremental_snapshots {
|
||||||
/// Check to see if we can use our local snapshots, otherwise download newer ones.
|
/// Check to see if we can use our local snapshots, otherwise download newer ones.
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn download_snapshots(
|
fn download_snapshots(
|
||||||
snapshot_archives_dir: &Path,
|
full_snapshot_archives_dir: &Path,
|
||||||
|
incremental_snapshot_archives_dir: &Path,
|
||||||
validator_config: &ValidatorConfig,
|
validator_config: &ValidatorConfig,
|
||||||
bootstrap_config: &RpcBootstrapConfig,
|
bootstrap_config: &RpcBootstrapConfig,
|
||||||
use_progress_bar: bool,
|
use_progress_bar: bool,
|
||||||
|
@ -1442,7 +1458,8 @@ mod with_incremental_snapshots {
|
||||||
|
|
||||||
// If the local snapshots are new enough, then use 'em; no need to download new snapshots
|
// If the local snapshots are new enough, then use 'em; no need to download new snapshots
|
||||||
if should_use_local_snapshot(
|
if should_use_local_snapshot(
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
maximum_local_snapshot_age,
|
maximum_local_snapshot_age,
|
||||||
full_snapshot_hash,
|
full_snapshot_hash,
|
||||||
incremental_snapshot_hash,
|
incremental_snapshot_hash,
|
||||||
|
@ -1451,7 +1468,7 @@ mod with_incremental_snapshots {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check and see if we've already got the full snapshot; if not, download it
|
// Check and see if we've already got the full snapshot; if not, download it
|
||||||
if snapshot_utils::get_full_snapshot_archives(snapshot_archives_dir)
|
if snapshot_utils::get_full_snapshot_archives(full_snapshot_archives_dir)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.any(|snapshot_archive| {
|
.any(|snapshot_archive| {
|
||||||
snapshot_archive.slot() == full_snapshot_hash.0
|
snapshot_archive.slot() == full_snapshot_hash.0
|
||||||
|
@ -1464,7 +1481,8 @@ mod with_incremental_snapshots {
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
download_snapshot(
|
download_snapshot(
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
validator_config,
|
validator_config,
|
||||||
bootstrap_config,
|
bootstrap_config,
|
||||||
use_progress_bar,
|
use_progress_bar,
|
||||||
|
@ -1480,7 +1498,7 @@ mod with_incremental_snapshots {
|
||||||
|
|
||||||
// Check and see if we've already got the incremental snapshot; if not, download it
|
// Check and see if we've already got the incremental snapshot; if not, download it
|
||||||
if let Some(incremental_snapshot_hash) = incremental_snapshot_hash {
|
if let Some(incremental_snapshot_hash) = incremental_snapshot_hash {
|
||||||
if snapshot_utils::get_incremental_snapshot_archives(snapshot_archives_dir)
|
if snapshot_utils::get_incremental_snapshot_archives(incremental_snapshot_archives_dir)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.any(|snapshot_archive| {
|
.any(|snapshot_archive| {
|
||||||
snapshot_archive.slot() == incremental_snapshot_hash.0
|
snapshot_archive.slot() == incremental_snapshot_hash.0
|
||||||
|
@ -1494,7 +1512,8 @@ mod with_incremental_snapshots {
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
download_snapshot(
|
download_snapshot(
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
validator_config,
|
validator_config,
|
||||||
bootstrap_config,
|
bootstrap_config,
|
||||||
use_progress_bar,
|
use_progress_bar,
|
||||||
|
@ -1515,7 +1534,8 @@ mod with_incremental_snapshots {
|
||||||
/// Download a snapshot
|
/// Download a snapshot
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn download_snapshot(
|
fn download_snapshot(
|
||||||
snapshot_archives_dir: &Path,
|
full_snapshot_archives_dir: &Path,
|
||||||
|
incremental_snapshot_archives_dir: &Path,
|
||||||
validator_config: &ValidatorConfig,
|
validator_config: &ValidatorConfig,
|
||||||
bootstrap_config: &RpcBootstrapConfig,
|
bootstrap_config: &RpcBootstrapConfig,
|
||||||
use_progress_bar: bool,
|
use_progress_bar: bool,
|
||||||
|
@ -1547,7 +1567,8 @@ mod with_incremental_snapshots {
|
||||||
};
|
};
|
||||||
download_snapshot_archive(
|
download_snapshot_archive(
|
||||||
&rpc_contact_info.rpc,
|
&rpc_contact_info.rpc,
|
||||||
snapshot_archives_dir,
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
desired_snapshot_hash,
|
desired_snapshot_hash,
|
||||||
snapshot_type,
|
snapshot_type,
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
|
@ -1591,7 +1612,8 @@ mod with_incremental_snapshots {
|
||||||
/// Check to see if bootstrap should load from its local snapshots or not. If not, then snapshots
|
/// Check to see if bootstrap should load from its local snapshots or not. If not, then snapshots
|
||||||
/// will be downloaded.
|
/// will be downloaded.
|
||||||
fn should_use_local_snapshot(
|
fn should_use_local_snapshot(
|
||||||
snapshot_archives_dir: &Path,
|
full_snapshot_archives_dir: &Path,
|
||||||
|
incremental_snapshot_archives_dir: &Path,
|
||||||
maximum_local_snapshot_age: Slot,
|
maximum_local_snapshot_age: Slot,
|
||||||
full_snapshot_hash: (Slot, Hash),
|
full_snapshot_hash: (Slot, Hash),
|
||||||
incremental_snapshot_hash: Option<(Slot, Hash)>,
|
incremental_snapshot_hash: Option<(Slot, Hash)>,
|
||||||
|
@ -1600,7 +1622,10 @@ mod with_incremental_snapshots {
|
||||||
.map(|(slot, _)| slot)
|
.map(|(slot, _)| slot)
|
||||||
.unwrap_or(full_snapshot_hash.0);
|
.unwrap_or(full_snapshot_hash.0);
|
||||||
|
|
||||||
match get_highest_local_snapshot_hash(snapshot_archives_dir) {
|
match get_highest_local_snapshot_hash(
|
||||||
|
full_snapshot_archives_dir,
|
||||||
|
incremental_snapshot_archives_dir,
|
||||||
|
) {
|
||||||
None => {
|
None => {
|
||||||
info!(
|
info!(
|
||||||
"Downloading a snapshot for slot {} since there is not a local snapshot.",
|
"Downloading a snapshot for slot {} since there is not a local snapshot.",
|
||||||
|
|
|
@ -727,6 +727,14 @@ pub fn main() {
|
||||||
.takes_value(true)
|
.takes_value(true)
|
||||||
.help("Use DIR as snapshot location [default: --ledger value]"),
|
.help("Use DIR as snapshot location [default: --ledger value]"),
|
||||||
)
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::with_name("incremental_snapshot_archive_path")
|
||||||
|
.long("incremental-snapshot-archive-path")
|
||||||
|
.conflicts_with("no-incremental-snapshots")
|
||||||
|
.value_name("DIR")
|
||||||
|
.takes_value(true)
|
||||||
|
.help("Use DIR as separate location for incremental snapshot archives [default: --snapshots value]"),
|
||||||
|
)
|
||||||
.arg(
|
.arg(
|
||||||
Arg::with_name("tower")
|
Arg::with_name("tower")
|
||||||
.long("tower")
|
.long("tower")
|
||||||
|
@ -2602,16 +2610,36 @@ pub fn main() {
|
||||||
let maximum_snapshot_download_abort =
|
let maximum_snapshot_download_abort =
|
||||||
value_t_or_exit!(matches, "maximum_snapshot_download_abort", u64);
|
value_t_or_exit!(matches, "maximum_snapshot_download_abort", u64);
|
||||||
|
|
||||||
let snapshot_archives_dir = if matches.is_present("snapshots") {
|
let full_snapshot_archives_dir = if matches.is_present("snapshots") {
|
||||||
PathBuf::from(matches.value_of("snapshots").unwrap())
|
PathBuf::from(matches.value_of("snapshots").unwrap())
|
||||||
} else {
|
} else {
|
||||||
ledger_path.clone()
|
ledger_path.clone()
|
||||||
};
|
};
|
||||||
let bank_snapshots_dir = snapshot_archives_dir.join("snapshot");
|
let incremental_snapshot_archives_dir =
|
||||||
|
if matches.is_present("incremental_snapshot_archive_path") {
|
||||||
|
let incremental_snapshot_archives_dir = PathBuf::from(
|
||||||
|
matches
|
||||||
|
.value_of("incremental_snapshot_archive_path")
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
fs::create_dir_all(&incremental_snapshot_archives_dir).unwrap_or_else(|err| {
|
||||||
|
eprintln!(
|
||||||
|
"Failed to create incremental snapshot archives directory {:?}: {}",
|
||||||
|
incremental_snapshot_archives_dir.display(),
|
||||||
|
err
|
||||||
|
);
|
||||||
|
exit(1);
|
||||||
|
});
|
||||||
|
incremental_snapshot_archives_dir
|
||||||
|
} else {
|
||||||
|
full_snapshot_archives_dir.clone()
|
||||||
|
};
|
||||||
|
let bank_snapshots_dir = incremental_snapshot_archives_dir.join("snapshot");
|
||||||
fs::create_dir_all(&bank_snapshots_dir).unwrap_or_else(|err| {
|
fs::create_dir_all(&bank_snapshots_dir).unwrap_or_else(|err| {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"Failed to create snapshots directory {:?}: {}",
|
"Failed to create snapshots directory {:?}: {}",
|
||||||
bank_snapshots_dir, err
|
bank_snapshots_dir.display(),
|
||||||
|
err
|
||||||
);
|
);
|
||||||
exit(1);
|
exit(1);
|
||||||
});
|
});
|
||||||
|
@ -2657,7 +2685,8 @@ pub fn main() {
|
||||||
full_snapshot_archive_interval_slots,
|
full_snapshot_archive_interval_slots,
|
||||||
incremental_snapshot_archive_interval_slots,
|
incremental_snapshot_archive_interval_slots,
|
||||||
bank_snapshots_dir,
|
bank_snapshots_dir,
|
||||||
snapshot_archives_dir: snapshot_archives_dir.clone(),
|
full_snapshot_archives_dir: full_snapshot_archives_dir.clone(),
|
||||||
|
incremental_snapshot_archives_dir: incremental_snapshot_archives_dir.clone(),
|
||||||
archive_format,
|
archive_format,
|
||||||
snapshot_version,
|
snapshot_version,
|
||||||
maximum_full_snapshot_archives_to_retain,
|
maximum_full_snapshot_archives_to_retain,
|
||||||
|
@ -2887,7 +2916,8 @@ pub fn main() {
|
||||||
Some(version)
|
Some(version)
|
||||||
});
|
});
|
||||||
solana_entry::entry::init_poh();
|
solana_entry::entry::init_poh();
|
||||||
snapshot_utils::remove_tmp_snapshot_archives(&snapshot_archives_dir);
|
snapshot_utils::remove_tmp_snapshot_archives(&full_snapshot_archives_dir);
|
||||||
|
snapshot_utils::remove_tmp_snapshot_archives(&incremental_snapshot_archives_dir);
|
||||||
|
|
||||||
let identity_keypair = Arc::new(identity_keypair);
|
let identity_keypair = Arc::new(identity_keypair);
|
||||||
|
|
||||||
|
@ -2897,7 +2927,8 @@ pub fn main() {
|
||||||
&node,
|
&node,
|
||||||
&identity_keypair,
|
&identity_keypair,
|
||||||
&ledger_path,
|
&ledger_path,
|
||||||
&snapshot_archives_dir,
|
&full_snapshot_archives_dir,
|
||||||
|
&incremental_snapshot_archives_dir,
|
||||||
&vote_account,
|
&vote_account,
|
||||||
authorized_voter_keypairs.clone(),
|
authorized_voter_keypairs.clone(),
|
||||||
&cluster_entrypoints,
|
&cluster_entrypoints,
|
||||||
|
|
Loading…
Reference in New Issue