2023-04-11 12:51:52 -07:00
|
|
|
mod snapshot_gossip_manager;
|
2021-12-03 09:00:31 -08:00
|
|
|
use {
|
2023-02-21 19:36:29 -08:00
|
|
|
crossbeam_channel::{Receiver, Sender},
|
2023-04-11 12:51:52 -07:00
|
|
|
snapshot_gossip_manager::SnapshotGossipManager,
|
2023-04-12 09:45:31 -07:00
|
|
|
solana_gossip::cluster_info::{ClusterInfo, MAX_LEGACY_SNAPSHOT_HASHES},
|
2023-02-21 19:36:29 -08:00
|
|
|
solana_measure::measure_us,
|
2021-12-03 09:00:31 -08:00
|
|
|
solana_perf::thread::renice_this_thread,
|
|
|
|
solana_runtime::{
|
|
|
|
snapshot_archive_info::SnapshotArchiveInfoGetter,
|
|
|
|
snapshot_config::SnapshotConfig,
|
2023-04-11 12:51:52 -07:00
|
|
|
snapshot_hash::StartingSnapshotHashes,
|
|
|
|
snapshot_package::{self, SnapshotPackage},
|
2021-12-03 09:00:31 -08:00
|
|
|
snapshot_utils,
|
|
|
|
},
|
|
|
|
std::{
|
|
|
|
sync::{
|
|
|
|
atomic::{AtomicBool, Ordering},
|
|
|
|
Arc,
|
|
|
|
},
|
|
|
|
thread::{self, Builder, JoinHandle},
|
|
|
|
time::Duration,
|
2020-01-23 09:20:34 -08:00
|
|
|
},
|
|
|
|
};
|
2019-10-18 14:58:16 -07:00
|
|
|
|
|
|
|
pub struct SnapshotPackagerService {
|
|
|
|
t_snapshot_packager: JoinHandle<()>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl SnapshotPackagerService {
|
2023-02-21 19:36:29 -08:00
|
|
|
/// If there are no snapshot packages to handle, limit how often we re-check
|
|
|
|
const LOOP_LIMITER: Duration = Duration::from_millis(100);
|
|
|
|
|
2020-02-20 11:46:13 -08:00
|
|
|
pub fn new(
|
2023-02-21 19:36:29 -08:00
|
|
|
snapshot_package_sender: Sender<SnapshotPackage>,
|
|
|
|
snapshot_package_receiver: Receiver<SnapshotPackage>,
|
2021-10-08 13:14:56 -07:00
|
|
|
starting_snapshot_hashes: Option<StartingSnapshotHashes>,
|
2023-04-19 08:09:49 -07:00
|
|
|
exit: Arc<AtomicBool>,
|
|
|
|
cluster_info: Arc<ClusterInfo>,
|
2021-09-03 14:42:32 -07:00
|
|
|
snapshot_config: SnapshotConfig,
|
2021-10-20 11:07:29 -07:00
|
|
|
enable_gossip_push: bool,
|
2020-02-20 11:46:13 -08:00
|
|
|
) -> Self {
|
2021-10-08 13:14:56 -07:00
|
|
|
let max_full_snapshot_hashes = std::cmp::min(
|
2023-04-10 14:52:20 -07:00
|
|
|
MAX_LEGACY_SNAPSHOT_HASHES,
|
2023-03-30 08:16:36 -07:00
|
|
|
snapshot_config
|
|
|
|
.maximum_full_snapshot_archives_to_retain
|
|
|
|
.get(),
|
2021-10-01 13:59:45 -07:00
|
|
|
);
|
2020-03-05 22:52:31 -08:00
|
|
|
|
2019-10-18 14:58:16 -07:00
|
|
|
let t_snapshot_packager = Builder::new()
|
2022-08-17 08:40:23 -07:00
|
|
|
.name("solSnapshotPkgr".to_string())
|
2020-02-21 18:42:24 -08:00
|
|
|
.spawn(move || {
|
2023-03-28 05:32:18 -07:00
|
|
|
info!("SnapshotPackagerService has started");
|
2021-10-26 18:56:16 -07:00
|
|
|
renice_this_thread(snapshot_config.packager_thread_niceness_adj).unwrap();
|
2023-07-06 20:45:29 -07:00
|
|
|
let mut snapshot_gossip_manager = enable_gossip_push.then(|| {
|
2023-03-06 13:50:45 -08:00
|
|
|
SnapshotGossipManager::new(
|
2021-10-20 11:07:29 -07:00
|
|
|
cluster_info,
|
|
|
|
max_full_snapshot_hashes,
|
2023-04-13 08:49:17 -07:00
|
|
|
starting_snapshot_hashes,
|
2023-03-06 13:50:45 -08:00
|
|
|
)
|
2023-07-06 20:45:29 -07:00
|
|
|
});
|
2021-10-08 13:14:56 -07:00
|
|
|
|
2020-02-21 18:42:24 -08:00
|
|
|
loop {
|
|
|
|
if exit.load(Ordering::Relaxed) {
|
|
|
|
break;
|
|
|
|
}
|
2020-01-23 10:20:37 -08:00
|
|
|
|
2023-02-21 19:36:29 -08:00
|
|
|
let Some((
|
|
|
|
snapshot_package,
|
|
|
|
num_outstanding_snapshot_packages,
|
|
|
|
num_re_enqueued_snapshot_packages,
|
2023-07-06 20:45:29 -07:00
|
|
|
)) = Self::get_next_snapshot_package(
|
|
|
|
&snapshot_package_sender,
|
|
|
|
&snapshot_package_receiver,
|
|
|
|
)
|
|
|
|
else {
|
2023-02-21 19:36:29 -08:00
|
|
|
std::thread::sleep(Self::LOOP_LIMITER);
|
2021-08-31 16:33:27 -07:00
|
|
|
continue;
|
2023-02-21 19:36:29 -08:00
|
|
|
};
|
|
|
|
info!("handling snapshot package: {snapshot_package:?}");
|
|
|
|
let enqueued_time = snapshot_package.enqueued.elapsed();
|
|
|
|
|
2023-05-15 10:53:48 -07:00
|
|
|
let (purge_bank_snapshots_time_us, handling_time_us) = measure_us!({
|
2023-02-21 19:36:29 -08:00
|
|
|
// Archiving the snapshot package is not allowed to fail.
|
|
|
|
// AccountsBackgroundService calls `clean_accounts()` with a value for
|
|
|
|
// last_full_snapshot_slot that requires this archive call to succeed.
|
|
|
|
snapshot_utils::archive_snapshot_package(
|
|
|
|
&snapshot_package,
|
|
|
|
&snapshot_config.full_snapshot_archives_dir,
|
|
|
|
&snapshot_config.incremental_snapshot_archives_dir,
|
|
|
|
snapshot_config.maximum_full_snapshot_archives_to_retain,
|
|
|
|
snapshot_config.maximum_incremental_snapshot_archives_to_retain,
|
|
|
|
)
|
|
|
|
.expect("failed to archive snapshot package");
|
|
|
|
|
|
|
|
if let Some(snapshot_gossip_manager) = snapshot_gossip_manager.as_mut() {
|
|
|
|
snapshot_gossip_manager.push_snapshot_hash(
|
2023-08-19 16:55:29 -07:00
|
|
|
snapshot_package.snapshot_kind,
|
2023-03-06 13:50:45 -08:00
|
|
|
(snapshot_package.slot(), *snapshot_package.hash()),
|
2023-02-21 19:36:29 -08:00
|
|
|
);
|
|
|
|
}
|
2023-05-12 12:39:14 -07:00
|
|
|
|
|
|
|
// Now that this snapshot package has been archived, it is safe to remove
|
|
|
|
// all bank snapshots older than this slot. We want to keep the bank
|
|
|
|
// snapshot *at this slot* so that it can be used during restarts, when
|
|
|
|
// booting from local state.
|
2023-05-15 10:53:48 -07:00
|
|
|
measure_us!(snapshot_utils::purge_bank_snapshots_older_than_slot(
|
2023-05-12 12:39:14 -07:00
|
|
|
&snapshot_config.bank_snapshots_dir,
|
|
|
|
snapshot_package.slot(),
|
2023-07-06 20:45:29 -07:00
|
|
|
))
|
|
|
|
.1
|
2023-02-21 19:36:29 -08:00
|
|
|
});
|
|
|
|
|
|
|
|
datapoint_info!(
|
|
|
|
"snapshot_packager_service",
|
|
|
|
(
|
2023-05-15 09:02:43 -07:00
|
|
|
"num_outstanding_snapshot_packages",
|
2023-02-21 19:36:29 -08:00
|
|
|
num_outstanding_snapshot_packages,
|
|
|
|
i64
|
|
|
|
),
|
|
|
|
(
|
2023-05-15 09:02:43 -07:00
|
|
|
"num_re_enqueued_snapshot_packages",
|
2023-02-21 19:36:29 -08:00
|
|
|
num_re_enqueued_snapshot_packages,
|
|
|
|
i64
|
|
|
|
),
|
2023-05-15 09:02:43 -07:00
|
|
|
("enqueued_time_us", enqueued_time.as_micros(), i64),
|
|
|
|
("handling_time_us", handling_time_us, i64),
|
2023-07-06 20:45:29 -07:00
|
|
|
(
|
|
|
|
"purge_old_snapshots_time_us",
|
|
|
|
purge_bank_snapshots_time_us,
|
|
|
|
i64
|
|
|
|
),
|
2023-02-21 19:36:29 -08:00
|
|
|
);
|
2019-10-18 14:58:16 -07:00
|
|
|
}
|
2023-03-28 05:32:18 -07:00
|
|
|
info!("SnapshotPackagerService has stopped");
|
2019-10-18 14:58:16 -07:00
|
|
|
})
|
|
|
|
.unwrap();
|
2021-01-11 10:21:15 -08:00
|
|
|
|
2019-10-18 14:58:16 -07:00
|
|
|
Self {
|
|
|
|
t_snapshot_packager,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-13 10:12:09 -08:00
|
|
|
pub fn join(self) -> thread::Result<()> {
|
2019-10-18 14:58:16 -07:00
|
|
|
self.t_snapshot_packager.join()
|
|
|
|
}
|
2023-02-21 19:36:29 -08:00
|
|
|
|
|
|
|
/// Get the next snapshot package to handle
|
|
|
|
///
|
|
|
|
/// Look through the snapshot package channel to find the highest priority one to handle next.
|
|
|
|
/// If there are no snapshot packages in the channel, return None. Otherwise return the
|
|
|
|
/// highest priority one. Unhandled snapshot packages with slots GREATER-THAN the handled one
|
|
|
|
/// will be re-enqueued. The remaining will be dropped.
|
|
|
|
///
|
|
|
|
/// Also return the number of snapshot packages initially in the channel, and the number of
|
|
|
|
/// ones re-enqueued.
|
|
|
|
fn get_next_snapshot_package(
|
|
|
|
snapshot_package_sender: &Sender<SnapshotPackage>,
|
|
|
|
snapshot_package_receiver: &Receiver<SnapshotPackage>,
|
|
|
|
) -> Option<(
|
|
|
|
SnapshotPackage,
|
|
|
|
/*num outstanding snapshot packages*/ usize,
|
|
|
|
/*num re-enqueued snapshot packages*/ usize,
|
|
|
|
)> {
|
|
|
|
let mut snapshot_packages: Vec<_> = snapshot_package_receiver.try_iter().collect();
|
|
|
|
// `select_nth()` panics if the slice is empty, so return if that's the case
|
|
|
|
if snapshot_packages.is_empty() {
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
let snapshot_packages_len = snapshot_packages.len();
|
|
|
|
debug!("outstanding snapshot packages ({snapshot_packages_len}): {snapshot_packages:?}");
|
|
|
|
|
|
|
|
snapshot_packages.select_nth_unstable_by(
|
|
|
|
snapshot_packages_len - 1,
|
|
|
|
snapshot_package::cmp_snapshot_packages_by_priority,
|
|
|
|
);
|
|
|
|
// SAFETY: We know `snapshot_packages` is not empty, so its len is >= 1,
|
|
|
|
// therefore there is always an element to pop.
|
|
|
|
let snapshot_package = snapshot_packages.pop().unwrap();
|
|
|
|
let handled_snapshot_package_slot = snapshot_package.slot();
|
|
|
|
// re-enqueue any remaining snapshot packages for slots GREATER-THAN the snapshot package
|
|
|
|
// that will be handled
|
|
|
|
let num_re_enqueued_snapshot_packages = snapshot_packages
|
|
|
|
.into_iter()
|
|
|
|
.filter(|snapshot_package| snapshot_package.slot() > handled_snapshot_package_slot)
|
|
|
|
.map(|snapshot_package| {
|
|
|
|
snapshot_package_sender
|
|
|
|
.try_send(snapshot_package)
|
|
|
|
.expect("re-enqueue snapshot package")
|
|
|
|
})
|
|
|
|
.count();
|
|
|
|
|
|
|
|
Some((
|
|
|
|
snapshot_package,
|
|
|
|
snapshot_packages_len,
|
|
|
|
num_re_enqueued_snapshot_packages,
|
|
|
|
))
|
|
|
|
}
|
2019-10-18 14:58:16 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
2021-12-03 09:00:31 -08:00
|
|
|
use {
|
|
|
|
super::*,
|
2023-02-21 19:36:29 -08:00
|
|
|
rand::seq::SliceRandom,
|
2021-12-03 09:00:31 -08:00
|
|
|
solana_runtime::{
|
|
|
|
snapshot_archive_info::SnapshotArchiveInfo,
|
2023-07-24 16:31:03 -07:00
|
|
|
snapshot_bank_utils,
|
2022-10-31 11:28:35 -07:00
|
|
|
snapshot_hash::SnapshotHash,
|
2023-08-19 16:55:29 -07:00
|
|
|
snapshot_package::{SnapshotKind, SnapshotPackage},
|
2023-04-24 18:52:50 -07:00
|
|
|
snapshot_utils::{self, ArchiveFormat, SnapshotVersion},
|
2021-12-03 09:00:31 -08:00
|
|
|
},
|
2023-04-24 18:52:50 -07:00
|
|
|
solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash},
|
2021-12-03 09:00:31 -08:00
|
|
|
std::{
|
2023-04-24 18:52:50 -07:00
|
|
|
fs::{self, remove_dir_all},
|
2021-12-03 09:00:31 -08:00
|
|
|
path::{Path, PathBuf},
|
2023-02-21 19:36:29 -08:00
|
|
|
time::Instant,
|
2021-12-03 09:00:31 -08:00
|
|
|
},
|
|
|
|
tempfile::TempDir,
|
2019-12-05 11:58:02 -08:00
|
|
|
};
|
2019-10-18 14:58:16 -07:00
|
|
|
|
2019-12-05 11:58:02 -08:00
|
|
|
// Create temporary placeholder directory for all test files
|
|
|
|
fn make_tmp_dir_path() -> PathBuf {
|
|
|
|
let out_dir = std::env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
|
2022-12-06 06:30:06 -08:00
|
|
|
let path = PathBuf::from(format!("{out_dir}/tmp/test_package_snapshots"));
|
2019-12-05 11:58:02 -08:00
|
|
|
|
|
|
|
// whack any possible collision
|
|
|
|
let _ignored = std::fs::remove_dir_all(&path);
|
|
|
|
// whack any possible collision
|
|
|
|
let _ignored = std::fs::remove_file(&path);
|
|
|
|
|
|
|
|
path
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_package_snapshots_relative_ledger_path() {
|
|
|
|
let temp_dir = make_tmp_dir_path();
|
|
|
|
create_and_verify_snapshot(&temp_dir);
|
|
|
|
remove_dir_all(temp_dir).expect("should remove tmp dir");
|
|
|
|
}
|
|
|
|
|
2019-10-18 14:58:16 -07:00
|
|
|
#[test]
|
|
|
|
fn test_package_snapshots() {
|
2019-12-05 11:58:02 -08:00
|
|
|
create_and_verify_snapshot(TempDir::new().unwrap().path())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn create_and_verify_snapshot(temp_dir: &Path) {
|
2023-04-24 18:52:50 -07:00
|
|
|
let bank_snapshots_dir = temp_dir.join("snapshots");
|
|
|
|
fs::create_dir_all(&bank_snapshots_dir).unwrap();
|
2022-05-10 13:37:41 -07:00
|
|
|
let full_snapshot_archives_dir = temp_dir.join("full_snapshot_archives");
|
|
|
|
let incremental_snapshot_archives_dir = temp_dir.join("incremental_snapshot_archives");
|
|
|
|
fs::create_dir_all(&full_snapshot_archives_dir).unwrap();
|
|
|
|
fs::create_dir_all(&incremental_snapshot_archives_dir).unwrap();
|
2019-10-18 14:58:16 -07:00
|
|
|
|
2023-04-24 18:52:50 -07:00
|
|
|
let num_snapshots = 1;
|
2019-10-18 14:58:16 -07:00
|
|
|
|
2023-04-24 18:52:50 -07:00
|
|
|
let genesis_config = GenesisConfig::default();
|
2023-07-24 16:31:03 -07:00
|
|
|
let bank = snapshot_bank_utils::create_snapshot_dirs_for_tests(
|
2023-04-24 18:52:50 -07:00
|
|
|
&genesis_config,
|
|
|
|
&bank_snapshots_dir,
|
|
|
|
num_snapshots,
|
|
|
|
num_snapshots,
|
|
|
|
);
|
2019-10-18 14:58:16 -07:00
|
|
|
|
2023-04-24 18:52:50 -07:00
|
|
|
let bank_snapshot_info =
|
|
|
|
snapshot_utils::get_highest_bank_snapshot(&bank_snapshots_dir).unwrap();
|
|
|
|
let snapshot_storages = bank.get_snapshot_storages(None);
|
2021-08-31 16:33:27 -07:00
|
|
|
let archive_format = ArchiveFormat::TarBzip2;
|
2019-10-18 14:58:16 -07:00
|
|
|
|
2023-07-24 16:31:03 -07:00
|
|
|
let full_archive = snapshot_bank_utils::package_and_archive_full_snapshot(
|
2023-04-24 18:52:50 -07:00
|
|
|
&bank,
|
|
|
|
&bank_snapshot_info,
|
2022-05-10 13:37:41 -07:00
|
|
|
full_snapshot_archives_dir,
|
|
|
|
incremental_snapshot_archives_dir,
|
2023-04-24 18:52:50 -07:00
|
|
|
snapshot_storages,
|
|
|
|
archive_format,
|
|
|
|
SnapshotVersion::default(),
|
2021-07-22 12:40:37 -07:00
|
|
|
snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
2021-09-06 16:01:56 -07:00
|
|
|
snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN,
|
2021-05-12 10:32:27 -07:00
|
|
|
)
|
|
|
|
.unwrap();
|
2019-10-18 14:58:16 -07:00
|
|
|
|
2020-01-23 10:20:37 -08:00
|
|
|
// Check archive is correct
|
2020-04-03 13:13:49 -07:00
|
|
|
snapshot_utils::verify_snapshot_archive(
|
2023-04-24 18:52:50 -07:00
|
|
|
full_archive.path(),
|
|
|
|
bank_snapshots_dir,
|
2021-08-31 16:33:27 -07:00
|
|
|
archive_format,
|
2022-04-06 19:55:44 -07:00
|
|
|
snapshot_utils::VerifyBank::Deterministic,
|
2023-04-24 18:52:50 -07:00
|
|
|
bank_snapshot_info.slot,
|
2020-04-03 13:13:49 -07:00
|
|
|
);
|
2019-10-18 14:58:16 -07:00
|
|
|
}
|
2023-02-21 19:36:29 -08:00
|
|
|
|
|
|
|
/// Ensure that unhandled snapshot packages are properly re-enqueued or dropped
|
|
|
|
///
|
|
|
|
/// The snapshot package handler should re-enqueue unhandled snapshot packages, if those
|
|
|
|
/// unhandled snapshot packages are for slots GREATER-THAN the last handled snapshot package.
|
|
|
|
/// Otherwise, they should be dropped.
|
|
|
|
#[test]
|
|
|
|
fn test_get_next_snapshot_package() {
|
2023-08-19 16:55:29 -07:00
|
|
|
fn new(snapshot_kind: SnapshotKind, slot: Slot) -> SnapshotPackage {
|
2023-02-21 19:36:29 -08:00
|
|
|
SnapshotPackage {
|
|
|
|
snapshot_archive_info: SnapshotArchiveInfo {
|
|
|
|
path: PathBuf::default(),
|
|
|
|
slot,
|
|
|
|
hash: SnapshotHash(Hash::default()),
|
|
|
|
archive_format: ArchiveFormat::Tar,
|
|
|
|
},
|
|
|
|
block_height: slot,
|
2023-05-01 11:24:59 -07:00
|
|
|
bank_snapshot_dir: PathBuf::default(),
|
2023-02-21 19:36:29 -08:00
|
|
|
snapshot_storages: Vec::default(),
|
|
|
|
snapshot_version: SnapshotVersion::default(),
|
2023-08-19 16:55:29 -07:00
|
|
|
snapshot_kind,
|
2023-02-21 19:36:29 -08:00
|
|
|
enqueued: Instant::now(),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fn new_full(slot: Slot) -> SnapshotPackage {
|
2023-08-19 16:55:29 -07:00
|
|
|
new(SnapshotKind::FullSnapshot, slot)
|
2023-02-21 19:36:29 -08:00
|
|
|
}
|
|
|
|
fn new_incr(slot: Slot, base: Slot) -> SnapshotPackage {
|
2023-08-19 16:55:29 -07:00
|
|
|
new(SnapshotKind::IncrementalSnapshot(base), slot)
|
2023-02-21 19:36:29 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
let (snapshot_package_sender, snapshot_package_receiver) = crossbeam_channel::unbounded();
|
|
|
|
|
|
|
|
// Populate the channel so that re-enqueueing and dropping will be tested
|
|
|
|
let mut snapshot_packages = [
|
|
|
|
new_full(100),
|
|
|
|
new_incr(110, 100),
|
|
|
|
new_incr(210, 100),
|
|
|
|
new_full(300),
|
|
|
|
new_incr(310, 300),
|
|
|
|
new_full(400), // <-- handle 1st
|
|
|
|
new_incr(410, 400),
|
|
|
|
new_incr(420, 400), // <-- handle 2nd
|
|
|
|
];
|
|
|
|
// Shuffle the snapshot packages to simulate receiving new snapshot packages from AHV
|
|
|
|
// simultaneously as SPS is handling them.
|
|
|
|
snapshot_packages.shuffle(&mut rand::thread_rng());
|
|
|
|
snapshot_packages
|
|
|
|
.into_iter()
|
|
|
|
.for_each(|snapshot_package| snapshot_package_sender.send(snapshot_package).unwrap());
|
|
|
|
|
|
|
|
// The Full Snapshot from slot 400 is handled 1st
|
|
|
|
// (the older full snapshots are skipped and dropped)
|
|
|
|
let (
|
|
|
|
snapshot_package,
|
|
|
|
_num_outstanding_snapshot_packages,
|
|
|
|
num_re_enqueued_snapshot_packages,
|
|
|
|
) = SnapshotPackagerService::get_next_snapshot_package(
|
|
|
|
&snapshot_package_sender,
|
|
|
|
&snapshot_package_receiver,
|
|
|
|
)
|
|
|
|
.unwrap();
|
2023-08-19 16:55:29 -07:00
|
|
|
assert_eq!(snapshot_package.snapshot_kind, SnapshotKind::FullSnapshot,);
|
2023-02-21 19:36:29 -08:00
|
|
|
assert_eq!(snapshot_package.slot(), 400);
|
|
|
|
assert_eq!(num_re_enqueued_snapshot_packages, 2);
|
|
|
|
|
|
|
|
// The Incremental Snapshot from slot 420 is handled 2nd
|
|
|
|
// (the older incremental snapshot from slot 410 is skipped and dropped)
|
|
|
|
let (
|
|
|
|
snapshot_package,
|
|
|
|
_num_outstanding_snapshot_packages,
|
|
|
|
num_re_enqueued_snapshot_packages,
|
|
|
|
) = SnapshotPackagerService::get_next_snapshot_package(
|
|
|
|
&snapshot_package_sender,
|
|
|
|
&snapshot_package_receiver,
|
|
|
|
)
|
|
|
|
.unwrap();
|
|
|
|
assert_eq!(
|
2023-08-19 16:55:29 -07:00
|
|
|
snapshot_package.snapshot_kind,
|
|
|
|
SnapshotKind::IncrementalSnapshot(400),
|
2023-02-21 19:36:29 -08:00
|
|
|
);
|
|
|
|
assert_eq!(snapshot_package.slot(), 420);
|
|
|
|
assert_eq!(num_re_enqueued_snapshot_packages, 0);
|
|
|
|
|
|
|
|
// And now the snapshot package channel is empty!
|
|
|
|
assert!(SnapshotPackagerService::get_next_snapshot_package(
|
|
|
|
&snapshot_package_sender,
|
|
|
|
&snapshot_package_receiver
|
|
|
|
)
|
|
|
|
.is_none());
|
|
|
|
}
|
2019-10-18 14:58:16 -07:00
|
|
|
}
|