Checks if bank snapshot is loadable before fastbooting (#343)
This commit is contained in:
parent
b1919bd9e4
commit
182d27f718
|
@ -24,6 +24,7 @@ use {
|
||||||
hash::Hash,
|
hash::Hash,
|
||||||
},
|
},
|
||||||
std::{
|
std::{
|
||||||
|
io::{Error as IoError, Result as IoResult},
|
||||||
sync::{
|
sync::{
|
||||||
atomic::{AtomicBool, Ordering},
|
atomic::{AtomicBool, Ordering},
|
||||||
Arc,
|
Arc,
|
||||||
|
@ -71,12 +72,17 @@ impl AccountsHashVerifier {
|
||||||
info!("handling accounts package: {accounts_package:?}");
|
info!("handling accounts package: {accounts_package:?}");
|
||||||
let enqueued_time = accounts_package.enqueued.elapsed();
|
let enqueued_time = accounts_package.enqueued.elapsed();
|
||||||
|
|
||||||
let (_, handling_time_us) = measure_us!(Self::process_accounts_package(
|
let (result, handling_time_us) = measure_us!(Self::process_accounts_package(
|
||||||
accounts_package,
|
accounts_package,
|
||||||
snapshot_package_sender.as_ref(),
|
snapshot_package_sender.as_ref(),
|
||||||
&snapshot_config,
|
&snapshot_config,
|
||||||
&exit,
|
&exit,
|
||||||
));
|
));
|
||||||
|
if let Err(err) = result {
|
||||||
|
error!("Stopping AccountsHashVerifier! Fatal error while processing accounts package: {err}");
|
||||||
|
exit.store(true, Ordering::Relaxed);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
datapoint_info!(
|
datapoint_info!(
|
||||||
"accounts_hash_verifier",
|
"accounts_hash_verifier",
|
||||||
|
@ -208,9 +214,9 @@ impl AccountsHashVerifier {
|
||||||
snapshot_package_sender: Option<&Sender<SnapshotPackage>>,
|
snapshot_package_sender: Option<&Sender<SnapshotPackage>>,
|
||||||
snapshot_config: &SnapshotConfig,
|
snapshot_config: &SnapshotConfig,
|
||||||
exit: &AtomicBool,
|
exit: &AtomicBool,
|
||||||
) {
|
) -> IoResult<()> {
|
||||||
let accounts_hash =
|
let accounts_hash =
|
||||||
Self::calculate_and_verify_accounts_hash(&accounts_package, snapshot_config);
|
Self::calculate_and_verify_accounts_hash(&accounts_package, snapshot_config)?;
|
||||||
|
|
||||||
Self::save_epoch_accounts_hash(&accounts_package, accounts_hash);
|
Self::save_epoch_accounts_hash(&accounts_package, accounts_hash);
|
||||||
|
|
||||||
|
@ -221,13 +227,15 @@ impl AccountsHashVerifier {
|
||||||
accounts_hash,
|
accounts_hash,
|
||||||
exit,
|
exit,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// returns calculated accounts hash
|
/// returns calculated accounts hash
|
||||||
fn calculate_and_verify_accounts_hash(
|
fn calculate_and_verify_accounts_hash(
|
||||||
accounts_package: &AccountsPackage,
|
accounts_package: &AccountsPackage,
|
||||||
snapshot_config: &SnapshotConfig,
|
snapshot_config: &SnapshotConfig,
|
||||||
) -> AccountsHashKind {
|
) -> IoResult<AccountsHashKind> {
|
||||||
let accounts_hash_calculation_kind = match accounts_package.package_kind {
|
let accounts_hash_calculation_kind = match accounts_package.package_kind {
|
||||||
AccountsPackageKind::AccountsHashVerifier => CalcAccountsHashKind::Full,
|
AccountsPackageKind::AccountsHashVerifier => CalcAccountsHashKind::Full,
|
||||||
AccountsPackageKind::EpochAccountsHash => CalcAccountsHashKind::Full,
|
AccountsPackageKind::EpochAccountsHash => CalcAccountsHashKind::Full,
|
||||||
|
@ -303,6 +311,23 @@ impl AccountsHashVerifier {
|
||||||
&accounts_hash_for_reserialize,
|
&accounts_hash_for_reserialize,
|
||||||
bank_incremental_snapshot_persistence.as_ref(),
|
bank_incremental_snapshot_persistence.as_ref(),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// now write the full snapshot slot file after reserializing so this bank snapshot is loadable
|
||||||
|
let full_snapshot_archive_slot = match accounts_package.package_kind {
|
||||||
|
AccountsPackageKind::Snapshot(SnapshotKind::IncrementalSnapshot(base_slot)) => {
|
||||||
|
base_slot
|
||||||
|
}
|
||||||
|
_ => accounts_package.slot,
|
||||||
|
};
|
||||||
|
snapshot_utils::write_full_snapshot_slot_file(
|
||||||
|
&snapshot_info.bank_snapshot_dir,
|
||||||
|
full_snapshot_archive_slot,
|
||||||
|
)
|
||||||
|
.map_err(|err| {
|
||||||
|
IoError::other(format!(
|
||||||
|
"failed to calculate accounts hash for {accounts_package:?}: {err}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
}
|
}
|
||||||
|
|
||||||
if accounts_package.package_kind
|
if accounts_package.package_kind
|
||||||
|
@ -340,7 +365,7 @@ impl AccountsHashVerifier {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
accounts_hash_kind
|
Ok(accounts_hash_kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn _calculate_full_accounts_hash(
|
fn _calculate_full_accounts_hash(
|
||||||
|
|
|
@ -244,20 +244,70 @@ fn bank_forks_from_snapshot(
|
||||||
.map(SnapshotArchiveInfoGetter::slot)
|
.map(SnapshotArchiveInfoGetter::slot)
|
||||||
.unwrap_or(0),
|
.unwrap_or(0),
|
||||||
);
|
);
|
||||||
let latest_bank_snapshot =
|
|
||||||
snapshot_utils::get_highest_bank_snapshot_post(&snapshot_config.bank_snapshots_dir);
|
|
||||||
|
|
||||||
let will_startup_from_snapshot_archives = match process_options.use_snapshot_archives_at_startup
|
let fastboot_snapshot = match process_options.use_snapshot_archives_at_startup {
|
||||||
{
|
UseSnapshotArchivesAtStartup::Always => None,
|
||||||
UseSnapshotArchivesAtStartup::Always => true,
|
UseSnapshotArchivesAtStartup::Never => {
|
||||||
UseSnapshotArchivesAtStartup::Never => false,
|
let Some(bank_snapshot) =
|
||||||
UseSnapshotArchivesAtStartup::WhenNewest => latest_bank_snapshot
|
snapshot_utils::get_highest_loadable_bank_snapshot(snapshot_config)
|
||||||
.as_ref()
|
else {
|
||||||
.map(|bank_snapshot| latest_snapshot_archive_slot > bank_snapshot.slot)
|
return Err(BankForksUtilsError::NoBankSnapshotDirectory {
|
||||||
.unwrap_or(true),
|
flag: use_snapshot_archives_at_startup::cli::LONG_ARG.to_string(),
|
||||||
|
value: UseSnapshotArchivesAtStartup::Never.to_string(),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
// If a newer snapshot archive was downloaded, it is possible that its slot is
|
||||||
|
// higher than the local state we will load. Did the user intend for this?
|
||||||
|
if bank_snapshot.slot < latest_snapshot_archive_slot {
|
||||||
|
warn!(
|
||||||
|
"Starting up from local state at slot {}, which is *older* than \
|
||||||
|
the latest snapshot archive at slot {}. If this is not desired, \
|
||||||
|
change the --{} CLI option to *not* \"{}\" and restart.",
|
||||||
|
bank_snapshot.slot,
|
||||||
|
latest_snapshot_archive_slot,
|
||||||
|
use_snapshot_archives_at_startup::cli::LONG_ARG,
|
||||||
|
UseSnapshotArchivesAtStartup::Never.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Some(bank_snapshot)
|
||||||
|
}
|
||||||
|
UseSnapshotArchivesAtStartup::WhenNewest => {
|
||||||
|
snapshot_utils::get_highest_loadable_bank_snapshot(snapshot_config)
|
||||||
|
.filter(|bank_snapshot| bank_snapshot.slot >= latest_snapshot_archive_slot)
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let bank = if will_startup_from_snapshot_archives {
|
let bank = if let Some(fastboot_snapshot) = fastboot_snapshot {
|
||||||
|
let (bank, _) = snapshot_bank_utils::bank_from_snapshot_dir(
|
||||||
|
&account_paths,
|
||||||
|
&fastboot_snapshot,
|
||||||
|
genesis_config,
|
||||||
|
&process_options.runtime_config,
|
||||||
|
process_options.debug_keys.clone(),
|
||||||
|
None,
|
||||||
|
process_options.account_indexes.clone(),
|
||||||
|
process_options.limit_load_slot_count_from_snapshot,
|
||||||
|
process_options.shrink_ratio,
|
||||||
|
process_options.verify_index,
|
||||||
|
process_options.accounts_db_config.clone(),
|
||||||
|
accounts_update_notifier,
|
||||||
|
exit,
|
||||||
|
)
|
||||||
|
.map_err(|err| BankForksUtilsError::BankFromSnapshotsDirectory {
|
||||||
|
source: err,
|
||||||
|
path: fastboot_snapshot.snapshot_path(),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// If the node crashes before taking the next bank snapshot, the next startup will attempt
|
||||||
|
// to load from the same bank snapshot again. And if `shrink` has run, the account storage
|
||||||
|
// files that are hard linked in bank snapshot will be *different* than what the bank
|
||||||
|
// snapshot expects. This would cause the node to crash again. To prevent that, purge all
|
||||||
|
// the bank snapshots here. In the above scenario, this will cause the node to load from a
|
||||||
|
// snapshot archive next time, which is safe.
|
||||||
|
snapshot_utils::purge_all_bank_snapshots(&snapshot_config.bank_snapshots_dir);
|
||||||
|
|
||||||
|
bank
|
||||||
|
} else {
|
||||||
// Given that we are going to boot from an archive, the append vecs held in the snapshot dirs for fast-boot should
|
// Given that we are going to boot from an archive, the append vecs held in the snapshot dirs for fast-boot should
|
||||||
// be released. They will be released by the account_background_service anyway. But in the case of the account_paths
|
// be released. They will be released by the account_background_service anyway. But in the case of the account_paths
|
||||||
// using memory-mounted file system, they are not released early enough to give space for the new append-vecs from
|
// using memory-mounted file system, they are not released early enough to give space for the new append-vecs from
|
||||||
|
@ -292,60 +342,6 @@ fn bank_forks_from_snapshot(
|
||||||
.map(|archive| archive.path().display().to_string())
|
.map(|archive| archive.path().display().to_string())
|
||||||
.unwrap_or("none".to_string()),
|
.unwrap_or("none".to_string()),
|
||||||
})?;
|
})?;
|
||||||
bank
|
|
||||||
} else {
|
|
||||||
let bank_snapshot =
|
|
||||||
latest_bank_snapshot.ok_or_else(|| BankForksUtilsError::NoBankSnapshotDirectory {
|
|
||||||
flag: use_snapshot_archives_at_startup::cli::LONG_ARG.to_string(),
|
|
||||||
value: UseSnapshotArchivesAtStartup::Never.to_string(),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// If a newer snapshot archive was downloaded, it is possible that its slot is
|
|
||||||
// higher than the local bank we will load. Did the user intend for this?
|
|
||||||
if bank_snapshot.slot < latest_snapshot_archive_slot {
|
|
||||||
assert_eq!(
|
|
||||||
process_options.use_snapshot_archives_at_startup,
|
|
||||||
UseSnapshotArchivesAtStartup::Never,
|
|
||||||
);
|
|
||||||
warn!(
|
|
||||||
"Starting up from local state at slot {}, which is *older* than \
|
|
||||||
the latest snapshot archive at slot {}. If this is not desired, \
|
|
||||||
change the --{} CLI option to *not* \"{}\" and restart.",
|
|
||||||
bank_snapshot.slot,
|
|
||||||
latest_snapshot_archive_slot,
|
|
||||||
use_snapshot_archives_at_startup::cli::LONG_ARG,
|
|
||||||
UseSnapshotArchivesAtStartup::Never.to_string(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
let (bank, _) = snapshot_bank_utils::bank_from_snapshot_dir(
|
|
||||||
&account_paths,
|
|
||||||
&bank_snapshot,
|
|
||||||
genesis_config,
|
|
||||||
&process_options.runtime_config,
|
|
||||||
process_options.debug_keys.clone(),
|
|
||||||
None,
|
|
||||||
process_options.account_indexes.clone(),
|
|
||||||
process_options.limit_load_slot_count_from_snapshot,
|
|
||||||
process_options.shrink_ratio,
|
|
||||||
process_options.verify_index,
|
|
||||||
process_options.accounts_db_config.clone(),
|
|
||||||
accounts_update_notifier,
|
|
||||||
exit,
|
|
||||||
)
|
|
||||||
.map_err(|err| BankForksUtilsError::BankFromSnapshotsDirectory {
|
|
||||||
source: err,
|
|
||||||
path: bank_snapshot.snapshot_path(),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// If the node crashes before taking the next bank snapshot, the next startup will attempt
|
|
||||||
// to load from the same bank snapshot again. And if `shrink` has run, the account storage
|
|
||||||
// files that are hard linked in bank snapshot will be *different* than what the bank
|
|
||||||
// snapshot expects. This would cause the node to crash again. To prevent that, purge all
|
|
||||||
// the bank snapshots here. In the above scenario, this will cause the node to load from a
|
|
||||||
// snapshot archive next time, which is safe.
|
|
||||||
snapshot_utils::purge_all_bank_snapshots(&snapshot_config.bank_snapshots_dir);
|
|
||||||
|
|
||||||
bank
|
bank
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -5067,6 +5067,105 @@ fn test_boot_from_local_state() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Test fastboot to ensure a node can boot in case it crashed while archiving a full snapshot
|
||||||
|
///
|
||||||
|
/// 1. Start a node and wait for it to take at least two full snapshots and one more
|
||||||
|
/// bank snapshot POST afterwards (for simplicity, wait for 2 full and 1 incremental).
|
||||||
|
/// 2. To simulate a node crashing while archiving a full snapshot, stop the node and
|
||||||
|
/// then delete the latest full snapshot archive.
|
||||||
|
/// 3. Restart the node. This should succeed, and boot from the older full snapshot archive,
|
||||||
|
/// *not* the latest bank snapshot POST.
|
||||||
|
/// 4. Take another incremental snapshot. This ensures the correct snapshot was loaded,
|
||||||
|
/// AND ensures the correct accounts hashes are present (which are needed when making
|
||||||
|
/// the bank snapshot POST for the new incremental snapshot).
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn test_boot_from_local_state_missing_archive() {
|
||||||
|
solana_logger::setup_with_default(RUST_LOG_FILTER);
|
||||||
|
const FULL_SNAPSHOT_INTERVAL: Slot = 20;
|
||||||
|
const INCREMENTAL_SNAPSHOT_INTERVAL: Slot = 10;
|
||||||
|
|
||||||
|
let validator_config = SnapshotValidatorConfig::new(
|
||||||
|
FULL_SNAPSHOT_INTERVAL,
|
||||||
|
INCREMENTAL_SNAPSHOT_INTERVAL,
|
||||||
|
INCREMENTAL_SNAPSHOT_INTERVAL,
|
||||||
|
7,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut cluster_config = ClusterConfig {
|
||||||
|
node_stakes: vec![100 * DEFAULT_NODE_STAKE],
|
||||||
|
cluster_lamports: DEFAULT_CLUSTER_LAMPORTS,
|
||||||
|
validator_configs: make_identical_validator_configs(&validator_config.validator_config, 1),
|
||||||
|
..ClusterConfig::default()
|
||||||
|
};
|
||||||
|
let mut cluster = LocalCluster::new(&mut cluster_config, SocketAddrSpace::Unspecified);
|
||||||
|
|
||||||
|
// we need two full snapshots and an incremental snapshot for this test
|
||||||
|
info!("Waiting for validator to create snapshots...");
|
||||||
|
LocalCluster::wait_for_next_full_snapshot(
|
||||||
|
&cluster,
|
||||||
|
&validator_config.full_snapshot_archives_dir,
|
||||||
|
Some(Duration::from_secs(5 * 60)),
|
||||||
|
);
|
||||||
|
LocalCluster::wait_for_next_full_snapshot(
|
||||||
|
&cluster,
|
||||||
|
&validator_config.full_snapshot_archives_dir,
|
||||||
|
Some(Duration::from_secs(5 * 60)),
|
||||||
|
);
|
||||||
|
LocalCluster::wait_for_next_incremental_snapshot(
|
||||||
|
&cluster,
|
||||||
|
&validator_config.full_snapshot_archives_dir,
|
||||||
|
&validator_config.incremental_snapshot_archives_dir,
|
||||||
|
Some(Duration::from_secs(5 * 60)),
|
||||||
|
);
|
||||||
|
debug!(
|
||||||
|
"snapshot archives:\n\tfull: {:?}\n\tincr: {:?}",
|
||||||
|
snapshot_utils::get_full_snapshot_archives(
|
||||||
|
validator_config.full_snapshot_archives_dir.path()
|
||||||
|
),
|
||||||
|
snapshot_utils::get_incremental_snapshot_archives(
|
||||||
|
validator_config.incremental_snapshot_archives_dir.path()
|
||||||
|
),
|
||||||
|
);
|
||||||
|
info!("Waiting for validator to create snapshots... DONE");
|
||||||
|
|
||||||
|
// now delete the latest full snapshot archive and restart, to simulate a crash while archiving
|
||||||
|
// a full snapshot package
|
||||||
|
info!("Stopping validator...");
|
||||||
|
let validator_pubkey = cluster.get_node_pubkeys()[0];
|
||||||
|
let mut validator_info = cluster.exit_node(&validator_pubkey);
|
||||||
|
info!("Stopping validator... DONE");
|
||||||
|
|
||||||
|
info!("Deleting latest full snapshot archive...");
|
||||||
|
let highest_full_snapshot = snapshot_utils::get_highest_full_snapshot_archive_info(
|
||||||
|
validator_config.full_snapshot_archives_dir.path(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fs::remove_file(highest_full_snapshot.path()).unwrap();
|
||||||
|
info!("Deleting latest full snapshot archive... DONE");
|
||||||
|
|
||||||
|
info!("Restarting validator...");
|
||||||
|
// if we set this to `Never`, the validator should not boot
|
||||||
|
validator_info.config.use_snapshot_archives_at_startup =
|
||||||
|
UseSnapshotArchivesAtStartup::WhenNewest;
|
||||||
|
cluster.restart_node(
|
||||||
|
&validator_pubkey,
|
||||||
|
validator_info,
|
||||||
|
SocketAddrSpace::Unspecified,
|
||||||
|
);
|
||||||
|
info!("Restarting validator... DONE");
|
||||||
|
|
||||||
|
// ensure we can create new incremental snapshots, since that is what used to fail
|
||||||
|
info!("Waiting for validator to create snapshots...");
|
||||||
|
LocalCluster::wait_for_next_incremental_snapshot(
|
||||||
|
&cluster,
|
||||||
|
&validator_config.full_snapshot_archives_dir,
|
||||||
|
&validator_config.incremental_snapshot_archives_dir,
|
||||||
|
Some(Duration::from_secs(5 * 60)),
|
||||||
|
);
|
||||||
|
info!("Waiting for validator to create snapshots... DONE");
|
||||||
|
}
|
||||||
|
|
||||||
// We want to simulate the following:
|
// We want to simulate the following:
|
||||||
// /--- 1 --- 3 (duplicate block)
|
// /--- 1 --- 3 (duplicate block)
|
||||||
// 0
|
// 0
|
||||||
|
|
|
@ -1257,13 +1257,16 @@ mod tests {
|
||||||
crate::{
|
crate::{
|
||||||
bank_forks::BankForks,
|
bank_forks::BankForks,
|
||||||
genesis_utils,
|
genesis_utils,
|
||||||
|
snapshot_config::SnapshotConfig,
|
||||||
snapshot_utils::{
|
snapshot_utils::{
|
||||||
clean_orphaned_account_snapshot_dirs, create_tmp_accounts_dir_for_tests,
|
clean_orphaned_account_snapshot_dirs, create_tmp_accounts_dir_for_tests,
|
||||||
get_bank_snapshots, get_bank_snapshots_post, get_bank_snapshots_pre,
|
get_bank_snapshots, get_bank_snapshots_post, get_bank_snapshots_pre,
|
||||||
get_highest_bank_snapshot, purge_all_bank_snapshots, purge_bank_snapshot,
|
get_highest_bank_snapshot, get_highest_bank_snapshot_pre,
|
||||||
|
get_highest_loadable_bank_snapshot, purge_all_bank_snapshots, purge_bank_snapshot,
|
||||||
purge_bank_snapshots_older_than_slot, purge_incomplete_bank_snapshots,
|
purge_bank_snapshots_older_than_slot, purge_incomplete_bank_snapshots,
|
||||||
purge_old_bank_snapshots, purge_old_bank_snapshots_at_startup,
|
purge_old_bank_snapshots, purge_old_bank_snapshots_at_startup,
|
||||||
snapshot_storage_rebuilder::get_slot_and_append_vec_id, ArchiveFormat,
|
snapshot_storage_rebuilder::get_slot_and_append_vec_id,
|
||||||
|
write_full_snapshot_slot_file, ArchiveFormat, SNAPSHOT_FULL_SNAPSHOT_SLOT_FILENAME,
|
||||||
},
|
},
|
||||||
status_cache::Status,
|
status_cache::Status,
|
||||||
},
|
},
|
||||||
|
@ -2638,4 +2641,136 @@ mod tests {
|
||||||
Err(VerifySlotDeltasError::SlotNotFoundInDeltas(333)),
|
Err(VerifySlotDeltasError::SlotNotFoundInDeltas(333)),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_get_highest_loadable_bank_snapshot() {
|
||||||
|
let bank_snapshots_dir = TempDir::new().unwrap();
|
||||||
|
let full_snapshot_archives_dir = TempDir::new().unwrap();
|
||||||
|
let incremental_snapshot_archives_dir = TempDir::new().unwrap();
|
||||||
|
|
||||||
|
let snapshot_config = SnapshotConfig {
|
||||||
|
bank_snapshots_dir: bank_snapshots_dir.as_ref().to_path_buf(),
|
||||||
|
full_snapshot_archives_dir: full_snapshot_archives_dir.as_ref().to_path_buf(),
|
||||||
|
incremental_snapshot_archives_dir: incremental_snapshot_archives_dir
|
||||||
|
.as_ref()
|
||||||
|
.to_path_buf(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let load_only_snapshot_config = SnapshotConfig {
|
||||||
|
bank_snapshots_dir: snapshot_config.bank_snapshots_dir.clone(),
|
||||||
|
full_snapshot_archives_dir: snapshot_config.full_snapshot_archives_dir.clone(),
|
||||||
|
incremental_snapshot_archives_dir: snapshot_config
|
||||||
|
.incremental_snapshot_archives_dir
|
||||||
|
.clone(),
|
||||||
|
..SnapshotConfig::new_load_only()
|
||||||
|
};
|
||||||
|
|
||||||
|
let genesis_config = GenesisConfig::default();
|
||||||
|
let mut bank = Arc::new(Bank::new_for_tests(&genesis_config));
|
||||||
|
|
||||||
|
// take some snapshots, and archive them
|
||||||
|
for _ in 0..snapshot_config
|
||||||
|
.maximum_full_snapshot_archives_to_retain
|
||||||
|
.get()
|
||||||
|
{
|
||||||
|
let slot = bank.slot() + 1;
|
||||||
|
bank = Arc::new(Bank::new_from_parent(bank, &Pubkey::default(), slot));
|
||||||
|
bank.fill_bank_with_ticks_for_tests();
|
||||||
|
bank.squash();
|
||||||
|
bank.force_flush_accounts_cache();
|
||||||
|
bank.update_accounts_hash(CalcAccountsHashDataSource::Storages, false, false);
|
||||||
|
let snapshot_storages = bank.get_snapshot_storages(None);
|
||||||
|
let slot_deltas = bank.status_cache.read().unwrap().root_slot_deltas();
|
||||||
|
let bank_snapshot_info = add_bank_snapshot(
|
||||||
|
&bank_snapshots_dir,
|
||||||
|
&bank,
|
||||||
|
&snapshot_storages,
|
||||||
|
snapshot_config.snapshot_version,
|
||||||
|
slot_deltas,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
crate::serde_snapshot::reserialize_bank_with_new_accounts_hash(
|
||||||
|
&bank_snapshot_info.snapshot_dir,
|
||||||
|
bank.slot(),
|
||||||
|
&bank.get_accounts_hash().unwrap(),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
);
|
||||||
|
write_full_snapshot_slot_file(&bank_snapshot_info.snapshot_dir, slot).unwrap();
|
||||||
|
package_and_archive_full_snapshot(
|
||||||
|
&bank,
|
||||||
|
&bank_snapshot_info,
|
||||||
|
&full_snapshot_archives_dir,
|
||||||
|
&incremental_snapshot_archives_dir,
|
||||||
|
snapshot_storages,
|
||||||
|
snapshot_config.archive_format,
|
||||||
|
snapshot_config.snapshot_version,
|
||||||
|
snapshot_config.maximum_full_snapshot_archives_to_retain,
|
||||||
|
snapshot_config.maximum_incremental_snapshot_archives_to_retain,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
// take another snapshot, but leave it as PRE
|
||||||
|
let slot = bank.slot() + 1;
|
||||||
|
bank = Arc::new(Bank::new_from_parent(bank, &Pubkey::default(), slot));
|
||||||
|
bank.fill_bank_with_ticks_for_tests();
|
||||||
|
bank.squash();
|
||||||
|
bank.force_flush_accounts_cache();
|
||||||
|
let snapshot_storages = bank.get_snapshot_storages(None);
|
||||||
|
let slot_deltas = bank.status_cache.read().unwrap().root_slot_deltas();
|
||||||
|
add_bank_snapshot(
|
||||||
|
&bank_snapshots_dir,
|
||||||
|
&bank,
|
||||||
|
&snapshot_storages,
|
||||||
|
SnapshotVersion::default(),
|
||||||
|
slot_deltas,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let highest_full_snapshot_archive =
|
||||||
|
get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir).unwrap();
|
||||||
|
let highest_bank_snapshot_post =
|
||||||
|
get_highest_bank_snapshot_post(&bank_snapshots_dir).unwrap();
|
||||||
|
let highest_bank_snapshot_pre = get_highest_bank_snapshot_pre(&bank_snapshots_dir).unwrap();
|
||||||
|
|
||||||
|
// we want a bank snapshot PRE with the highest slot to ensure get_highest_loadable()
|
||||||
|
// correctly skips bank snapshots PRE
|
||||||
|
assert!(highest_bank_snapshot_pre.slot > highest_bank_snapshot_post.slot);
|
||||||
|
|
||||||
|
// 1. call get_highest_loadable() but bad snapshot dir, so returns None
|
||||||
|
assert!(get_highest_loadable_bank_snapshot(&SnapshotConfig::default()).is_none());
|
||||||
|
|
||||||
|
// 2. get_highest_loadable(), should return highest_bank_snapshot_post_slot
|
||||||
|
let bank_snapshot = get_highest_loadable_bank_snapshot(&snapshot_config).unwrap();
|
||||||
|
assert_eq!(bank_snapshot, highest_bank_snapshot_post);
|
||||||
|
|
||||||
|
// 3. delete highest full snapshot archive, get_highest_loadable() should return NONE
|
||||||
|
fs::remove_file(highest_full_snapshot_archive.path()).unwrap();
|
||||||
|
assert!(get_highest_loadable_bank_snapshot(&snapshot_config).is_none());
|
||||||
|
|
||||||
|
// 4. get_highest_loadable(), but with a load-only snapshot config, should return Some()
|
||||||
|
let bank_snapshot = get_highest_loadable_bank_snapshot(&load_only_snapshot_config).unwrap();
|
||||||
|
assert_eq!(bank_snapshot, highest_bank_snapshot_post);
|
||||||
|
|
||||||
|
// 5. delete highest bank snapshot, get_highest_loadable() should return Some() again, with slot-1
|
||||||
|
fs::remove_dir_all(&highest_bank_snapshot_post.snapshot_dir).unwrap();
|
||||||
|
let bank_snapshot = get_highest_loadable_bank_snapshot(&snapshot_config).unwrap();
|
||||||
|
assert_eq!(bank_snapshot.slot, highest_bank_snapshot_post.slot - 1);
|
||||||
|
|
||||||
|
// 6. delete the full snapshot slot file, get_highest_loadable() should return NONE
|
||||||
|
fs::remove_file(
|
||||||
|
bank_snapshot
|
||||||
|
.snapshot_dir
|
||||||
|
.join(SNAPSHOT_FULL_SNAPSHOT_SLOT_FILENAME),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(get_highest_loadable_bank_snapshot(&snapshot_config).is_none());
|
||||||
|
|
||||||
|
// 7. however, a load-only snapshot config should return Some() again
|
||||||
|
let bank_snapshot2 =
|
||||||
|
get_highest_loadable_bank_snapshot(&load_only_snapshot_config).unwrap();
|
||||||
|
assert_eq!(bank_snapshot2, bank_snapshot);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,7 @@ use {
|
||||||
snapshot_archive_info::{
|
snapshot_archive_info::{
|
||||||
FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter,
|
FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter,
|
||||||
},
|
},
|
||||||
|
snapshot_config::SnapshotConfig,
|
||||||
snapshot_hash::SnapshotHash,
|
snapshot_hash::SnapshotHash,
|
||||||
snapshot_package::SnapshotPackage,
|
snapshot_package::SnapshotPackage,
|
||||||
snapshot_utils::snapshot_storage_rebuilder::{
|
snapshot_utils::snapshot_storage_rebuilder::{
|
||||||
|
@ -58,6 +59,7 @@ pub const SNAPSHOT_VERSION_FILENAME: &str = "version";
|
||||||
pub const SNAPSHOT_STATE_COMPLETE_FILENAME: &str = "state_complete";
|
pub const SNAPSHOT_STATE_COMPLETE_FILENAME: &str = "state_complete";
|
||||||
pub const SNAPSHOT_ACCOUNTS_HARDLINKS: &str = "accounts_hardlinks";
|
pub const SNAPSHOT_ACCOUNTS_HARDLINKS: &str = "accounts_hardlinks";
|
||||||
pub const SNAPSHOT_ARCHIVE_DOWNLOAD_DIR: &str = "remote";
|
pub const SNAPSHOT_ARCHIVE_DOWNLOAD_DIR: &str = "remote";
|
||||||
|
pub const SNAPSHOT_FULL_SNAPSHOT_SLOT_FILENAME: &str = "full_snapshot_slot";
|
||||||
pub const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
|
pub const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
|
||||||
const MAX_SNAPSHOT_VERSION_FILE_SIZE: u64 = 8; // byte
|
const MAX_SNAPSHOT_VERSION_FILE_SIZE: u64 = 8; // byte
|
||||||
const VERSION_STRING_V1_2_0: &str = "1.2.0";
|
const VERSION_STRING_V1_2_0: &str = "1.2.0";
|
||||||
|
@ -625,6 +627,76 @@ fn is_bank_snapshot_complete(bank_snapshot_dir: impl AsRef<Path>) -> bool {
|
||||||
state_complete_path.is_file()
|
state_complete_path.is_file()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Writes the full snapshot slot file into the bank snapshot dir
|
||||||
|
pub fn write_full_snapshot_slot_file(
|
||||||
|
bank_snapshot_dir: impl AsRef<Path>,
|
||||||
|
full_snapshot_slot: Slot,
|
||||||
|
) -> IoResult<()> {
|
||||||
|
let full_snapshot_slot_path = bank_snapshot_dir
|
||||||
|
.as_ref()
|
||||||
|
.join(SNAPSHOT_FULL_SNAPSHOT_SLOT_FILENAME);
|
||||||
|
fs::write(
|
||||||
|
&full_snapshot_slot_path,
|
||||||
|
Slot::to_le_bytes(full_snapshot_slot),
|
||||||
|
)
|
||||||
|
.map_err(|err| {
|
||||||
|
IoError::other(format!(
|
||||||
|
"failed to write full snapshot slot file '{}': {err}",
|
||||||
|
full_snapshot_slot_path.display(),
|
||||||
|
))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reads the full snapshot slot file from the bank snapshot dir
|
||||||
|
pub fn read_full_snapshot_slot_file(bank_snapshot_dir: impl AsRef<Path>) -> IoResult<Slot> {
|
||||||
|
const SLOT_SIZE: usize = std::mem::size_of::<Slot>();
|
||||||
|
let full_snapshot_slot_path = bank_snapshot_dir
|
||||||
|
.as_ref()
|
||||||
|
.join(SNAPSHOT_FULL_SNAPSHOT_SLOT_FILENAME);
|
||||||
|
let full_snapshot_slot_file_metadata = fs::metadata(&full_snapshot_slot_path)?;
|
||||||
|
if full_snapshot_slot_file_metadata.len() != SLOT_SIZE as u64 {
|
||||||
|
let error_message = format!(
|
||||||
|
"invalid full snapshot slot file size: '{}' has {} bytes (should be {} bytes)",
|
||||||
|
full_snapshot_slot_path.display(),
|
||||||
|
full_snapshot_slot_file_metadata.len(),
|
||||||
|
SLOT_SIZE,
|
||||||
|
);
|
||||||
|
return Err(IoError::other(error_message));
|
||||||
|
}
|
||||||
|
let mut full_snapshot_slot_file = fs::File::open(&full_snapshot_slot_path)?;
|
||||||
|
let mut buffer = [0; SLOT_SIZE];
|
||||||
|
full_snapshot_slot_file.read_exact(&mut buffer)?;
|
||||||
|
let slot = Slot::from_le_bytes(buffer);
|
||||||
|
Ok(slot)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Gets the highest, loadable, bank snapshot
|
||||||
|
///
|
||||||
|
/// The highest bank snapshot is the one with the highest slot.
|
||||||
|
/// To be loadable, the bank snapshot must be a BankSnapshotKind::Post.
|
||||||
|
/// And if we're generating snapshots (e.g. running a normal validator), then
|
||||||
|
/// the full snapshot file's slot must match the highest full snapshot archive's.
|
||||||
|
pub fn get_highest_loadable_bank_snapshot(
|
||||||
|
snapshot_config: &SnapshotConfig,
|
||||||
|
) -> Option<BankSnapshotInfo> {
|
||||||
|
let highest_bank_snapshot =
|
||||||
|
get_highest_bank_snapshot_post(&snapshot_config.bank_snapshots_dir)?;
|
||||||
|
|
||||||
|
// If we're *not* generating snapshots, e.g. running ledger-tool, then we *can* load
|
||||||
|
// this bank snapshot, and we do not need to check for anything else.
|
||||||
|
if !snapshot_config.should_generate_snapshots() {
|
||||||
|
return Some(highest_bank_snapshot);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise, the bank snapshot's full snapshot slot *must* be the same as
|
||||||
|
// the highest full snapshot archive's slot.
|
||||||
|
let highest_full_snapshot_archive_slot =
|
||||||
|
get_highest_full_snapshot_archive_slot(&snapshot_config.full_snapshot_archives_dir)?;
|
||||||
|
let full_snapshot_file_slot =
|
||||||
|
read_full_snapshot_slot_file(&highest_bank_snapshot.snapshot_dir).ok()?;
|
||||||
|
(full_snapshot_file_slot == highest_full_snapshot_archive_slot).then_some(highest_bank_snapshot)
|
||||||
|
}
|
||||||
|
|
||||||
/// If the validator halts in the middle of `archive_snapshot_package()`, the temporary staging
|
/// If the validator halts in the middle of `archive_snapshot_package()`, the temporary staging
|
||||||
/// directory won't be cleaned up. Call this function to clean them up.
|
/// directory won't be cleaned up. Call this function to clean them up.
|
||||||
pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: impl AsRef<Path>) {
|
pub fn remove_tmp_snapshot_archives(snapshot_archives_dir: impl AsRef<Path>) {
|
||||||
|
@ -2269,6 +2341,7 @@ mod tests {
|
||||||
std::{convert::TryFrom, mem::size_of},
|
std::{convert::TryFrom, mem::size_of},
|
||||||
tempfile::NamedTempFile,
|
tempfile::NamedTempFile,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_serialize_snapshot_data_file_under_limit() {
|
fn test_serialize_snapshot_data_file_under_limit() {
|
||||||
let temp_dir = tempfile::TempDir::new().unwrap();
|
let temp_dir = tempfile::TempDir::new().unwrap();
|
||||||
|
@ -3211,4 +3284,34 @@ mod tests {
|
||||||
Err(GetSnapshotAccountsHardLinkDirError::GetAccountPath(_))
|
Err(GetSnapshotAccountsHardLinkDirError::GetAccountPath(_))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_full_snapshot_slot_file_good() {
|
||||||
|
let slot_written = 123_456_789;
|
||||||
|
let bank_snapshot_dir = TempDir::new().unwrap();
|
||||||
|
write_full_snapshot_slot_file(&bank_snapshot_dir, slot_written).unwrap();
|
||||||
|
|
||||||
|
let slot_read = read_full_snapshot_slot_file(&bank_snapshot_dir).unwrap();
|
||||||
|
assert_eq!(slot_read, slot_written);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_full_snapshot_slot_file_bad() {
|
||||||
|
const SLOT_SIZE: usize = std::mem::size_of::<Slot>();
|
||||||
|
let too_small = [1u8; SLOT_SIZE - 1];
|
||||||
|
let too_large = [1u8; SLOT_SIZE + 1];
|
||||||
|
|
||||||
|
for contents in [too_small.as_slice(), too_large.as_slice()] {
|
||||||
|
let bank_snapshot_dir = TempDir::new().unwrap();
|
||||||
|
let full_snapshot_slot_path = bank_snapshot_dir
|
||||||
|
.as_ref()
|
||||||
|
.join(SNAPSHOT_FULL_SNAPSHOT_SLOT_FILENAME);
|
||||||
|
fs::write(full_snapshot_slot_path, contents).unwrap();
|
||||||
|
|
||||||
|
let err = read_full_snapshot_slot_file(&bank_snapshot_dir).unwrap_err();
|
||||||
|
assert!(err
|
||||||
|
.to_string()
|
||||||
|
.starts_with("invalid full snapshot slot file size"));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue