blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Tune test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_startup to be less flaky

This commit is contained in:
Michael Vines 2022-03-24 11:16:26 -07:00
parent d0a8a16a57
commit 78d9bd6bb6
1 changed files with 44 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
local-cluster/tests/local_cluster.rs
View File

@ -867,7 +867,7 @@ fn test_incremental_snapshot_download() {
#[test] #[test]
#[serial] #[serial]
fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_startup() { fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_startup() {
solana_logger::setup_with_default("error,local_cluster=info"); solana_logger::setup_with_default(RUST_LOG_FILTER);
// If these intervals change, also make sure to change the loop timers accordingly. // If these intervals change, also make sure to change the loop timers accordingly.
let accounts_hash_interval = 3; let accounts_hash_interval = 3;
let incremental_snapshot_interval = accounts_hash_interval * 3; let incremental_snapshot_interval = accounts_hash_interval * 3;
@ -899,7 +899,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified); let mut cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
debug!("snapshot config:\n\tfull snapshot interval: {}\n\tincremental snapshot interval: {}\n\taccounts hash interval: {}", info!("snapshot config:\n\tfull snapshot interval: {}\n\tincremental snapshot interval: {}\n\taccounts hash interval: {}",
full_snapshot_interval, full_snapshot_interval,
incremental_snapshot_interval, incremental_snapshot_interval,
accounts_hash_interval); accounts_hash_interval);
@ -926,13 +926,13 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
.display(), .display(),
); );
info!("Waiting for leader to create snapshots..."); info!("Waiting for leader to create the next incremental snapshot...");
let (incremental_snapshot_archive, full_snapshot_archive) = let (incremental_snapshot_archive, full_snapshot_archive) =
LocalCluster::wait_for_next_incremental_snapshot( LocalCluster::wait_for_next_incremental_snapshot(
&cluster, &cluster,
leader_snapshot_test_config.snapshot_archives_dir.path(), leader_snapshot_test_config.snapshot_archives_dir.path(),
); );
debug!( info!(
"Found snapshots:\n\tfull snapshot: {}\n\tincremental snapshot: {}", "Found snapshots:\n\tfull snapshot: {}\n\tincremental snapshot: {}",
full_snapshot_archive.path().display(), full_snapshot_archive.path().display(),
incremental_snapshot_archive.path().display() incremental_snapshot_archive.path().display()
@ -970,11 +970,10 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config.snapshot_archives_dir.path(),
) )
.unwrap(); .unwrap();
debug!( info!(
"Downloaded full snapshot, slot: {}", "Downloaded full snapshot, slot: {}",
downloaded_full_snapshot_archive.slot() downloaded_full_snapshot_archive.slot()
); );
info!("Downloading full snapshot to validator... DONE");
info!("Downloading incremental snapshot to validator..."); info!("Downloading incremental snapshot to validator...");
download_snapshot_archive( download_snapshot_archive(
@ -1007,7 +1006,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
full_snapshot_archive.slot(), full_snapshot_archive.slot(),
) )
.unwrap(); .unwrap();
debug!( info!(
"Downloaded incremental snapshot, slot: {}, base slot: {}", "Downloaded incremental snapshot, slot: {}, base slot: {}",
downloaded_incremental_snapshot_archive.slot(), downloaded_incremental_snapshot_archive.slot(),
downloaded_incremental_snapshot_archive.base_slot(), downloaded_incremental_snapshot_archive.base_slot(),
@ -1016,7 +1015,6 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
downloaded_full_snapshot_archive.slot(), downloaded_full_snapshot_archive.slot(),
downloaded_incremental_snapshot_archive.base_slot() downloaded_incremental_snapshot_archive.base_slot()
); );
info!("Downloading incremental snapshot to validator... DONE");
// closure to copy files in a directory to another directory // closure to copy files in a directory to another directory
let copy_files = |from: &Path, to: &Path| { let copy_files = |from: &Path, to: &Path| {
@ -1097,10 +1095,9 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
// To ensure that a snapshot will be taken during startup, the blockstore needs to have roots // To ensure that a snapshot will be taken during startup, the blockstore needs to have roots
// that cross a full snapshot interval. // that cross a full snapshot interval.
info!("Waiting for the validator to see enough slots to cross a full snapshot interval...");
let starting_slot = incremental_snapshot_archive.slot(); let starting_slot = incremental_snapshot_archive.slot();
let next_full_snapshot_slot = starting_slot + full_snapshot_interval; let next_full_snapshot_slot = starting_slot + full_snapshot_interval;
trace!("next full snapshot slot: {next_full_snapshot_slot}"); info!("Waiting for the validator to see enough slots to cross a full snapshot interval ({next_full_snapshot_slot})...");
let timer = Instant::now(); let timer = Instant::now();
loop { loop {
let validator_current_slot = cluster let validator_current_slot = cluster
@ -1109,7 +1106,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
.get_slot_with_commitment(CommitmentConfig::finalized()) .get_slot_with_commitment(CommitmentConfig::finalized())
.unwrap(); .unwrap();
trace!("validator current slot: {validator_current_slot}"); trace!("validator current slot: {validator_current_slot}");
if validator_current_slot > (starting_slot + full_snapshot_interval) { if validator_current_slot > next_full_snapshot_slot {
break; break;
} }
assert!( assert!(
@ -1118,9 +1115,8 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
); );
std::thread::yield_now(); std::thread::yield_now();
} }
debug!("Waited {:?}", timer.elapsed());
info!( info!(
"Waiting for the validator to see enough slots to cross a full snapshot interval... DONE" "Waited {:?} for the validator to see enough slots to cross a full snapshot interval... DONE", timer.elapsed()
); );
// Get the highest full snapshot archive info for the validator, now that it has crossed the // Get the highest full snapshot archive info for the validator, now that it has crossed the
@ -1156,7 +1152,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
leader_full_snapshot.clone() leader_full_snapshot.clone()
}; };
trace!("leader full snapshot archive for comparison: {leader_full_snapshot_archive_for_comparison:#?}"); info!("leader full snapshot archive for comparison: {leader_full_snapshot_archive_for_comparison:#?}");
info!("Delete all the snapshots on the validator and restore the originals from the backup..."); info!("Delete all the snapshots on the validator and restore the originals from the backup...");
delete_files_with_remote(validator_snapshot_test_config.snapshot_archives_dir.path()); delete_files_with_remote(validator_snapshot_test_config.snapshot_archives_dir.path());
@ -1174,9 +1170,10 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config.snapshot_archives_dir.path(),
) )
.unwrap(); .unwrap();
trace!("validator full snapshot slot at startup: {validator_full_snapshot_slot_at_startup}");
info!("Restarting the validator..."); info!(
"Restarting the validator with full snapshot {validator_full_snapshot_slot_at_startup}..."
);
let validator_info = cluster.exit_node(&validator_identity.pubkey()); let validator_info = cluster.exit_node(&validator_identity.pubkey());
cluster.restart_node( cluster.restart_node(
&validator_identity.pubkey(), &validator_identity.pubkey(),
@ -1192,8 +1189,8 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
validator_full_snapshot_slot_at_startup + full_snapshot_interval; validator_full_snapshot_slot_at_startup + full_snapshot_interval;
let validator_next_incremental_snapshot_slot = let validator_next_incremental_snapshot_slot =
validator_next_full_snapshot_slot + incremental_snapshot_interval; validator_next_full_snapshot_slot + incremental_snapshot_interval;
trace!("Waiting for validator next full snapshot slot: {validator_next_full_snapshot_slot}"); info!("Waiting for validator next full snapshot slot: {validator_next_full_snapshot_slot}");
trace!("Waiting for validator next incremental snapshot slot: {validator_next_incremental_snapshot_slot}"); info!("Waiting for validator next incremental snapshot slot: {validator_next_incremental_snapshot_slot}");
let timer = Instant::now(); let timer = Instant::now();
loop { loop {
if let Some(full_snapshot_slot) = snapshot_utils::get_highest_full_snapshot_archive_slot( if let Some(full_snapshot_slot) = snapshot_utils::get_highest_full_snapshot_archive_slot(
@ -1208,7 +1205,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
{ {
if incremental_snapshot_slot >= validator_next_incremental_snapshot_slot { if incremental_snapshot_slot >= validator_next_incremental_snapshot_slot {
// specific incremental snapshot is not important, just that one was created // specific incremental snapshot is not important, just that one was created
debug!( info!(
"Validator made new snapshots, full snapshot slot: {}, incremental snapshot slot: {}", "Validator made new snapshots, full snapshot slot: {}, incremental snapshot slot: {}",
full_snapshot_slot, full_snapshot_slot,
incremental_snapshot_slot, incremental_snapshot_slot,
@ -1220,28 +1217,44 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
} }
assert!( assert!(
timer.elapsed() < Duration::from_secs(10), timer.elapsed() < Duration::from_secs(30),
"It should not take longer than 10 seconds to cross the next incremental snapshot interval." "It should not take longer than 30 seconds to cross the next incremental snapshot interval."
); );
std::thread::yield_now(); std::thread::yield_now();
} }
debug!("Waited {:?}", timer.elapsed()); info!(
info!("Waiting for the validator to make new snapshots... DONE"); "Waited {:?} for the validator to make new snapshots... DONE",
timer.elapsed()
);
// Check to make sure that the full snapshot the validator created during startup is the same // Check to make sure that the full snapshot the validator created during startup is the same
// as the snapshot the leader created. // or one greater than the snapshot the leader created.
let validator_full_snapshot_archives = snapshot_utils::get_full_snapshot_archives( let validator_full_snapshot_archives = snapshot_utils::get_full_snapshot_archives(
validator_snapshot_test_config.snapshot_archives_dir.path(), validator_snapshot_test_config.snapshot_archives_dir.path(),
); );
info!("validator full snapshot archives: {validator_full_snapshot_archives:#?}");
let validator_full_snapshot_archive_for_comparison = validator_full_snapshot_archives let validator_full_snapshot_archive_for_comparison = validator_full_snapshot_archives
.into_iter() .into_iter()
.find(|validator_full_snapshot_archive| validator_full_snapshot_archive.slot() == leader_full_snapshot_archive_for_comparison.slot()) .find(|validator_full_snapshot_archive| {
.expect("validator shall have a full snapshot with the same slot as the one that will be used for comparison with the leader"); [
trace!("Validator full snapshot archive for comparison: {validator_full_snapshot_archive_for_comparison:#?}"); leader_full_snapshot_archive_for_comparison.slot(),
assert_eq!( leader_full_snapshot_archive_for_comparison.slot() + full_snapshot_interval,
validator_full_snapshot_archive_for_comparison.hash(), ]
leader_full_snapshot_archive_for_comparison.hash(), .contains(&validator_full_snapshot_archive.slot())
); })
.expect("validator created an unexpected full snapshot");
info!("Validator full snapshot archive for comparison: {validator_full_snapshot_archive_for_comparison:#?}");
if validator_full_snapshot_archive_for_comparison.slot()
== leader_full_snapshot_archive_for_comparison.slot()
{
assert_eq!(
validator_full_snapshot_archive_for_comparison.hash(),
leader_full_snapshot_archive_for_comparison.hash(),
);
} else {
info!("Hash check skipped due to slot mismatch");
}
// And lastly, startup another node with the new snapshots to ensure they work // And lastly, startup another node with the new snapshots to ensure they work
let final_validator_snapshot_test_config = SnapshotValidatorConfig::new( let final_validator_snapshot_test_config = SnapshotValidatorConfig::new(
@ -1273,8 +1286,6 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st
SocketAddrSpace::Unspecified, SocketAddrSpace::Unspecified,
); );
info!("Starting final validator... DONE"); info!("Starting final validator... DONE");
// Success!
} }
#[allow(unused_attributes)] #[allow(unused_attributes)]