compare contents of serialized banks instead of exact file format (#24141)

* compare contents of serialized banks instead of exact file format

* Update runtime/src/snapshot_utils.rs

Co-authored-by: Brooks Prumo <brooks@prumo.org>

* Update runtime/src/snapshot_utils.rs

Co-authored-by: Brooks Prumo <brooks@prumo.org>

* pr feedback

* get rid of clone

* pr feedback

Co-authored-by: Brooks Prumo <brooks@prumo.org>
This commit is contained in:
Jeff Washington (jwash) 2022-04-06 21:55:44 -05:00 committed by GitHub
parent fddd162645
commit 550ca7bf92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 45 additions and 3 deletions

View File

@ -349,6 +349,7 @@ mod tests {
snapshots_dir,
accounts_dir,
archive_format,
snapshot_utils::VerifyBank::Deterministic,
);
}
}

View File

@ -558,6 +558,7 @@ mod tests {
saved_snapshots_dir.path(),
saved_accounts_dir.path(),
ArchiveFormat::TarBzip2,
snapshot_utils::VerifyBank::NonDeterministic(saved_slot),
);
}

View File

@ -894,7 +894,7 @@ impl NonceInfo for NonceFull {
// Bank's common fields shared by all supported snapshot versions for deserialization.
// Sync fields with BankFieldsToSerialize! This is paired with it.
// All members are made public to remain Bank's members private and to make versioned deserializer workable on this
#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, PartialEq)]
pub(crate) struct BankFieldsToDeserialize {
pub(crate) blockhash_queue: BlockhashQueue,
pub(crate) ancestors: AncestorsForSerialization,

View File

@ -63,7 +63,7 @@ pub(crate) enum SerdeStyle {
const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024;
#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)]
#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample, PartialEq)]
struct AccountsDbFields<T>(
HashMap<Slot, Vec<T>>,
StoredMetaWriteVersion,
@ -137,7 +137,7 @@ impl<T> SnapshotAccountsDbFields<T> {
}
}
trait TypeContext<'a> {
trait TypeContext<'a>: PartialEq {
type SerializableAccountStorageEntry: Serialize
+ DeserializeOwned
+ From<&'a AccountStorageEntry>
@ -189,6 +189,23 @@ where
.deserialize_from::<R, T>(reader)
}
/// used by tests to compare contents of serialized bank fields
/// serialized format is not deterministic - likely due to randomness in structs like hashmaps
pub(crate) fn compare_two_serialized_banks(
path1: impl AsRef<Path>,
path2: impl AsRef<Path>,
) -> std::result::Result<bool, Error> {
use std::fs::File;
let file1 = File::open(path1)?;
let mut stream1 = BufReader::new(file1);
let file2 = File::open(path2)?;
let mut stream2 = BufReader::new(file2);
let fields1 = newer::Context::deserialize_bank_fields(&mut stream1)?;
let fields2 = newer::Context::deserialize_bank_fields(&mut stream2)?;
Ok(fields1 == fields2)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn bank_from_streams<R>(
serde_style: SerdeStyle,

View File

@ -176,6 +176,7 @@ impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedB
#[cfg(RUSTC_WITH_SPECIALIZATION)]
impl<'a> solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableVersionedBank<'a> {}
#[derive(PartialEq)]
pub(super) struct Context {}
impl<'a> TypeContext<'a> for Context {

View File

@ -1649,11 +1649,22 @@ fn get_io_error(error: &str) -> SnapshotError {
SnapshotError::Io(IoError::new(ErrorKind::Other, error))
}
#[derive(Debug, Copy, Clone)]
/// allow tests to specify what happened to the serialized format
pub enum VerifyBank {
/// the bank's serialized format is expected to be identical to what we are comparing against
Deterministic,
/// the serialized bank was 'reserialized' into a non-deterministic format at the specified slot
/// so, deserialize both files and compare deserialized results
NonDeterministic(Slot),
}
pub fn verify_snapshot_archive<P, Q, R>(
snapshot_archive: P,
snapshots_to_verify: Q,
storages_to_verify: R,
archive_format: ArchiveFormat,
verify_bank: VerifyBank,
) where
P: AsRef<Path>,
Q: AsRef<Path>,
@ -1672,6 +1683,17 @@ pub fn verify_snapshot_archive<P, Q, R>(
// Check snapshots are the same
let unpacked_snapshots = unpack_dir.join("snapshots");
if let VerifyBank::NonDeterministic(slot) = verify_bank {
// file contents may be different, but deserialized structs should be equal
let slot = slot.to_string();
let p1 = snapshots_to_verify.as_ref().join(&slot).join(&slot);
let p2 = unpacked_snapshots.join(&slot).join(&slot);
assert!(crate::serde_snapshot::compare_two_serialized_banks(&p1, &p2).unwrap());
std::fs::remove_file(p1).unwrap();
std::fs::remove_file(p2).unwrap();
}
assert!(!dir_diff::is_different(&snapshots_to_verify, unpacked_snapshots).unwrap());
// Check the account entries are the same