Add incremental_snapshot_archive_interval_slots to SnapshotConfig (#19026)
This commit also renames `snapshot_interval_slots` to `full_snapshot_archive_interval_slots`, updates the comments on the fields, and make appropriate updates where SnapshotConfig is used.
This commit is contained in:
parent
cc27b8a5a7
commit
ca14475085
|
@ -497,7 +497,8 @@ impl TestValidator {
|
||||||
account_paths: vec![ledger_path.join("accounts")],
|
account_paths: vec![ledger_path.join("accounts")],
|
||||||
poh_verify: false, // Skip PoH verification of ledger on startup for speed
|
poh_verify: false, // Skip PoH verification of ledger on startup for speed
|
||||||
snapshot_config: Some(SnapshotConfig {
|
snapshot_config: Some(SnapshotConfig {
|
||||||
snapshot_interval_slots: 100,
|
full_snapshot_archive_interval_slots: 100,
|
||||||
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_path: ledger_path.join("snapshot"),
|
snapshot_path: ledger_path.join("snapshot"),
|
||||||
snapshot_package_output_path: ledger_path.to_path_buf(),
|
snapshot_package_output_path: ledger_path.to_path_buf(),
|
||||||
archive_format: ArchiveFormat::Tar,
|
archive_format: ArchiveFormat::Tar,
|
||||||
|
|
|
@ -199,7 +199,7 @@ impl Tvu {
|
||||||
|
|
||||||
let snapshot_interval_slots = {
|
let snapshot_interval_slots = {
|
||||||
if let Some(config) = bank_forks.read().unwrap().snapshot_config() {
|
if let Some(config) = bank_forks.read().unwrap().snapshot_config() {
|
||||||
config.snapshot_interval_slots
|
config.full_snapshot_archive_interval_slots
|
||||||
} else {
|
} else {
|
||||||
std::u64::MAX
|
std::u64::MAX
|
||||||
}
|
}
|
||||||
|
|
|
@ -608,7 +608,7 @@ impl Validator {
|
||||||
let (snapshot_packager_service, snapshot_config_and_pending_package) =
|
let (snapshot_packager_service, snapshot_config_and_pending_package) =
|
||||||
if let Some(snapshot_config) = config.snapshot_config.clone() {
|
if let Some(snapshot_config) = config.snapshot_config.clone() {
|
||||||
if is_snapshot_config_invalid(
|
if is_snapshot_config_invalid(
|
||||||
snapshot_config.snapshot_interval_slots,
|
snapshot_config.full_snapshot_archive_interval_slots,
|
||||||
config.accounts_hash_interval_slots,
|
config.accounts_hash_interval_slots,
|
||||||
) {
|
) {
|
||||||
error!("Snapshot config is invalid");
|
error!("Snapshot config is invalid");
|
||||||
|
|
|
@ -126,7 +126,8 @@ mod tests {
|
||||||
bank_forks.accounts_hash_interval_slots = accounts_hash_interval_slots;
|
bank_forks.accounts_hash_interval_slots = accounts_hash_interval_slots;
|
||||||
|
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
snapshot_interval_slots,
|
full_snapshot_archive_interval_slots: snapshot_interval_slots,
|
||||||
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
|
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
|
||||||
snapshot_path: bank_snapshots_dir.path().to_path_buf(),
|
snapshot_path: bank_snapshots_dir.path().to_path_buf(),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
|
|
|
@ -694,7 +694,8 @@ fn load_bank_forks(
|
||||||
let snapshot_package_output_path =
|
let snapshot_package_output_path =
|
||||||
snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf());
|
snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf());
|
||||||
Some(SnapshotConfig {
|
Some(SnapshotConfig {
|
||||||
snapshot_interval_slots: 0, // Value doesn't matter
|
full_snapshot_archive_interval_slots: 0, // Value doesn't matter
|
||||||
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_package_output_path,
|
snapshot_package_output_path,
|
||||||
snapshot_path,
|
snapshot_path,
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
|
|
|
@ -3451,7 +3451,8 @@ fn setup_snapshot_validator_config(
|
||||||
let snapshot_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let snapshot_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
snapshot_interval_slots,
|
full_snapshot_archive_interval_slots: snapshot_interval_slots,
|
||||||
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
|
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
|
||||||
snapshot_path: snapshot_dir.path().to_path_buf(),
|
snapshot_path: snapshot_dir.path().to_path_buf(),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
|
|
|
@ -254,7 +254,8 @@ impl ReplicaNode {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
snapshot_interval_slots: std::u64::MAX,
|
full_snapshot_archive_interval_slots: std::u64::MAX,
|
||||||
|
incremental_snapshot_archive_interval_slots: std::u64::MAX,
|
||||||
snapshot_package_output_path: replica_config.snapshot_output_dir.clone(),
|
snapshot_package_output_path: replica_config.snapshot_output_dir.clone(),
|
||||||
snapshot_path: replica_config.snapshot_path.clone(),
|
snapshot_path: replica_config.snapshot_path.clone(),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
|
|
|
@ -119,7 +119,8 @@ fn setup_snapshot_validator_config(
|
||||||
let snapshot_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let snapshot_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
let snapshot_archives_dir = tempfile::tempdir_in(farf_dir()).unwrap();
|
||||||
let snapshot_config = SnapshotConfig {
|
let snapshot_config = SnapshotConfig {
|
||||||
snapshot_interval_slots,
|
full_snapshot_archive_interval_slots: snapshot_interval_slots,
|
||||||
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
|
snapshot_package_output_path: snapshot_archives_dir.path().to_path_buf(),
|
||||||
snapshot_path: snapshot_dir.path().to_path_buf(),
|
snapshot_path: snapshot_dir.path().to_path_buf(),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
|
|
|
@ -599,7 +599,8 @@ mod tests {
|
||||||
let rrm_with_snapshot_config = RpcRequestMiddleware::new(
|
let rrm_with_snapshot_config = RpcRequestMiddleware::new(
|
||||||
PathBuf::from("/"),
|
PathBuf::from("/"),
|
||||||
Some(SnapshotConfig {
|
Some(SnapshotConfig {
|
||||||
snapshot_interval_slots: 0,
|
full_snapshot_archive_interval_slots: 0,
|
||||||
|
incremental_snapshot_archive_interval_slots: u64::MAX,
|
||||||
snapshot_package_output_path: PathBuf::from("/"),
|
snapshot_package_output_path: PathBuf::from("/"),
|
||||||
snapshot_path: PathBuf::from("/"),
|
snapshot_path: PathBuf::from("/"),
|
||||||
archive_format: ArchiveFormat::TarBzip2,
|
archive_format: ArchiveFormat::TarBzip2,
|
||||||
|
|
|
@ -6,13 +6,16 @@ use std::path::PathBuf;
|
||||||
/// Snapshot configuration and runtime information
|
/// Snapshot configuration and runtime information
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct SnapshotConfig {
|
pub struct SnapshotConfig {
|
||||||
/// Generate a new snapshot every this many slots
|
/// Generate a new full snapshot archive every this many slots
|
||||||
pub snapshot_interval_slots: Slot,
|
pub full_snapshot_archive_interval_slots: Slot,
|
||||||
|
|
||||||
/// Where to store the latest packaged snapshot
|
/// Generate a new incremental snapshot archive every this many slots
|
||||||
|
pub incremental_snapshot_archive_interval_slots: Slot,
|
||||||
|
|
||||||
|
/// Where to store the latest packaged snapshot archives
|
||||||
pub snapshot_package_output_path: PathBuf,
|
pub snapshot_package_output_path: PathBuf,
|
||||||
|
|
||||||
/// Where to place the snapshots for recent slots
|
/// Where to place the bank snapshots for recent slots
|
||||||
pub snapshot_path: PathBuf,
|
pub snapshot_path: PathBuf,
|
||||||
|
|
||||||
/// The archive format to use for snapshots
|
/// The archive format to use for snapshots
|
||||||
|
@ -21,6 +24,6 @@ pub struct SnapshotConfig {
|
||||||
/// Snapshot version to generate
|
/// Snapshot version to generate
|
||||||
pub snapshot_version: SnapshotVersion,
|
pub snapshot_version: SnapshotVersion,
|
||||||
|
|
||||||
/// Maximum number of snapshots to retain
|
/// Maximum number of full snapshot archives to retain
|
||||||
pub maximum_snapshots_to_retain: usize,
|
pub maximum_snapshots_to_retain: usize,
|
||||||
}
|
}
|
||||||
|
|
|
@ -2494,11 +2494,12 @@ pub fn main() {
|
||||||
})
|
})
|
||||||
});
|
});
|
||||||
validator_config.snapshot_config = Some(SnapshotConfig {
|
validator_config.snapshot_config = Some(SnapshotConfig {
|
||||||
snapshot_interval_slots: if snapshot_interval_slots > 0 {
|
full_snapshot_archive_interval_slots: if snapshot_interval_slots > 0 {
|
||||||
snapshot_interval_slots
|
snapshot_interval_slots
|
||||||
} else {
|
} else {
|
||||||
std::u64::MAX
|
std::u64::MAX
|
||||||
},
|
},
|
||||||
|
incremental_snapshot_archive_interval_slots: Slot::MAX,
|
||||||
snapshot_path,
|
snapshot_path,
|
||||||
snapshot_package_output_path: snapshot_output_dir.clone(),
|
snapshot_package_output_path: snapshot_output_dir.clone(),
|
||||||
archive_format,
|
archive_format,
|
||||||
|
|
Loading…
Reference in New Issue