compare contents of serialized banks instead of exact file format (#24141)
* compare contents of serialized banks instead of exact file format * Update runtime/src/snapshot_utils.rs Co-authored-by: Brooks Prumo <brooks@prumo.org> * Update runtime/src/snapshot_utils.rs Co-authored-by: Brooks Prumo <brooks@prumo.org> * pr feedback * get rid of clone * pr feedback Co-authored-by: Brooks Prumo <brooks@prumo.org>
This commit is contained in:
parent
fddd162645
commit
550ca7bf92
|
@ -349,6 +349,7 @@ mod tests {
|
|||
snapshots_dir,
|
||||
accounts_dir,
|
||||
archive_format,
|
||||
snapshot_utils::VerifyBank::Deterministic,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -558,6 +558,7 @@ mod tests {
|
|||
saved_snapshots_dir.path(),
|
||||
saved_accounts_dir.path(),
|
||||
ArchiveFormat::TarBzip2,
|
||||
snapshot_utils::VerifyBank::NonDeterministic(saved_slot),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
@ -894,7 +894,7 @@ impl NonceInfo for NonceFull {
|
|||
// Bank's common fields shared by all supported snapshot versions for deserialization.
|
||||
// Sync fields with BankFieldsToSerialize! This is paired with it.
|
||||
// All members are made public to remain Bank's members private and to make versioned deserializer workable on this
|
||||
#[derive(Clone, Debug, Default)]
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub(crate) struct BankFieldsToDeserialize {
|
||||
pub(crate) blockhash_queue: BlockhashQueue,
|
||||
pub(crate) ancestors: AncestorsForSerialization,
|
||||
|
|
|
@ -63,7 +63,7 @@ pub(crate) enum SerdeStyle {
|
|||
|
||||
const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)]
|
||||
#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample, PartialEq)]
|
||||
struct AccountsDbFields<T>(
|
||||
HashMap<Slot, Vec<T>>,
|
||||
StoredMetaWriteVersion,
|
||||
|
@ -137,7 +137,7 @@ impl<T> SnapshotAccountsDbFields<T> {
|
|||
}
|
||||
}
|
||||
|
||||
trait TypeContext<'a> {
|
||||
trait TypeContext<'a>: PartialEq {
|
||||
type SerializableAccountStorageEntry: Serialize
|
||||
+ DeserializeOwned
|
||||
+ From<&'a AccountStorageEntry>
|
||||
|
@ -189,6 +189,23 @@ where
|
|||
.deserialize_from::<R, T>(reader)
|
||||
}
|
||||
|
||||
/// used by tests to compare contents of serialized bank fields
|
||||
/// serialized format is not deterministic - likely due to randomness in structs like hashmaps
|
||||
pub(crate) fn compare_two_serialized_banks(
|
||||
path1: impl AsRef<Path>,
|
||||
path2: impl AsRef<Path>,
|
||||
) -> std::result::Result<bool, Error> {
|
||||
use std::fs::File;
|
||||
let file1 = File::open(path1)?;
|
||||
let mut stream1 = BufReader::new(file1);
|
||||
let file2 = File::open(path2)?;
|
||||
let mut stream2 = BufReader::new(file2);
|
||||
|
||||
let fields1 = newer::Context::deserialize_bank_fields(&mut stream1)?;
|
||||
let fields2 = newer::Context::deserialize_bank_fields(&mut stream2)?;
|
||||
Ok(fields1 == fields2)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn bank_from_streams<R>(
|
||||
serde_style: SerdeStyle,
|
||||
|
|
|
@ -176,6 +176,7 @@ impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedB
|
|||
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
||||
impl<'a> solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableVersionedBank<'a> {}
|
||||
|
||||
#[derive(PartialEq)]
|
||||
pub(super) struct Context {}
|
||||
|
||||
impl<'a> TypeContext<'a> for Context {
|
||||
|
|
|
@ -1649,11 +1649,22 @@ fn get_io_error(error: &str) -> SnapshotError {
|
|||
SnapshotError::Io(IoError::new(ErrorKind::Other, error))
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
/// allow tests to specify what happened to the serialized format
|
||||
pub enum VerifyBank {
|
||||
/// the bank's serialized format is expected to be identical to what we are comparing against
|
||||
Deterministic,
|
||||
/// the serialized bank was 'reserialized' into a non-deterministic format at the specified slot
|
||||
/// so, deserialize both files and compare deserialized results
|
||||
NonDeterministic(Slot),
|
||||
}
|
||||
|
||||
pub fn verify_snapshot_archive<P, Q, R>(
|
||||
snapshot_archive: P,
|
||||
snapshots_to_verify: Q,
|
||||
storages_to_verify: R,
|
||||
archive_format: ArchiveFormat,
|
||||
verify_bank: VerifyBank,
|
||||
) where
|
||||
P: AsRef<Path>,
|
||||
Q: AsRef<Path>,
|
||||
|
@ -1672,6 +1683,17 @@ pub fn verify_snapshot_archive<P, Q, R>(
|
|||
|
||||
// Check snapshots are the same
|
||||
let unpacked_snapshots = unpack_dir.join("snapshots");
|
||||
if let VerifyBank::NonDeterministic(slot) = verify_bank {
|
||||
// file contents may be different, but deserialized structs should be equal
|
||||
let slot = slot.to_string();
|
||||
let p1 = snapshots_to_verify.as_ref().join(&slot).join(&slot);
|
||||
let p2 = unpacked_snapshots.join(&slot).join(&slot);
|
||||
|
||||
assert!(crate::serde_snapshot::compare_two_serialized_banks(&p1, &p2).unwrap());
|
||||
std::fs::remove_file(p1).unwrap();
|
||||
std::fs::remove_file(p2).unwrap();
|
||||
}
|
||||
|
||||
assert!(!dir_diff::is_different(&snapshots_to_verify, unpacked_snapshots).unwrap());
|
||||
|
||||
// Check the account entries are the same
|
||||
|
|
Loading…
Reference in New Issue