compare contents of serialized banks instead of exact file format (#24141)

* compare contents of serialized banks instead of exact file format

* Update runtime/src/snapshot_utils.rs

Co-authored-by: Brooks Prumo <brooks@prumo.org>

* Update runtime/src/snapshot_utils.rs

Co-authored-by: Brooks Prumo <brooks@prumo.org>

* pr feedback

* get rid of clone

* pr feedback

Co-authored-by: Brooks Prumo <brooks@prumo.org>

core/src/snapshot_packager_service.rs

@@ -349,6 +349,7 @@ mod tests {
             snapshots_dir,
             accounts_dir,
             archive_format,
+            snapshot_utils::VerifyBank::Deterministic,
         );
     }
 }

core/tests/snapshots.rs

@@ -558,6 +558,7 @@ mod tests {
             saved_snapshots_dir.path(),
             saved_accounts_dir.path(),
             ArchiveFormat::TarBzip2,
+            snapshot_utils::VerifyBank::NonDeterministic(saved_slot),
         );
     }
 

runtime/src/bank.rs

@@ -894,7 +894,7 @@ impl NonceInfo for NonceFull {
 // Bank's common fields shared by all supported snapshot versions for deserialization.
 // Sync fields with BankFieldsToSerialize! This is paired with it.
 // All members are made public to remain Bank's members private and to make versioned deserializer workable on this
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, PartialEq)]
 pub(crate) struct BankFieldsToDeserialize {
     pub(crate) blockhash_queue: BlockhashQueue,
     pub(crate) ancestors: AncestorsForSerialization,

runtime/src/serde_snapshot.rs

@@ -63,7 +63,7 @@ pub(crate) enum SerdeStyle {
 
 const MAX_STREAM_SIZE: u64 = 32 * 1024 * 1024 * 1024;
 
-#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample)]
+#[derive(Clone, Debug, Default, Deserialize, Serialize, AbiExample, PartialEq)]
 struct AccountsDbFields<T>(
     HashMap<Slot, Vec<T>>,
     StoredMetaWriteVersion,
@@ -137,7 +137,7 @@ impl<T> SnapshotAccountsDbFields<T> {
     }
 }
 
-trait TypeContext<'a> {
+trait TypeContext<'a>: PartialEq {
     type SerializableAccountStorageEntry: Serialize
         + DeserializeOwned
         + From<&'a AccountStorageEntry>
@@ -189,6 +189,23 @@ where
         .deserialize_from::<R, T>(reader)
 }
 
+/// used by tests to compare contents of serialized bank fields
+/// serialized format is not deterministic - likely due to randomness in structs like hashmaps
+pub(crate) fn compare_two_serialized_banks(
+    path1: impl AsRef<Path>,
+    path2: impl AsRef<Path>,
+) -> std::result::Result<bool, Error> {
+    use std::fs::File;
+    let file1 = File::open(path1)?;
+    let mut stream1 = BufReader::new(file1);
+    let file2 = File::open(path2)?;
+    let mut stream2 = BufReader::new(file2);
+
+    let fields1 = newer::Context::deserialize_bank_fields(&mut stream1)?;
+    let fields2 = newer::Context::deserialize_bank_fields(&mut stream2)?;
+    Ok(fields1 == fields2)
+}
+
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn bank_from_streams<R>(
     serde_style: SerdeStyle,

runtime/src/serde_snapshot/newer.rs

@@ -176,6 +176,7 @@ impl<'a> From<crate::bank::BankFieldsToSerialize<'a>> for SerializableVersionedB
 #[cfg(RUSTC_WITH_SPECIALIZATION)]
 impl<'a> solana_frozen_abi::abi_example::IgnoreAsHelper for SerializableVersionedBank<'a> {}
 
+#[derive(PartialEq)]
 pub(super) struct Context {}
 
 impl<'a> TypeContext<'a> for Context {

runtime/src/snapshot_utils.rs

@@ -1649,11 +1649,22 @@ fn get_io_error(error: &str) -> SnapshotError {
     SnapshotError::Io(IoError::new(ErrorKind::Other, error))
 }
 
+#[derive(Debug, Copy, Clone)]
+/// allow tests to specify what happened to the serialized format
+pub enum VerifyBank {
+    /// the bank's serialized format is expected to be identical to what we are comparing against
+    Deterministic,
+    /// the serialized bank was 'reserialized' into a non-deterministic format at the specified slot
+    /// so, deserialize both files and compare deserialized results
+    NonDeterministic(Slot),
+}
+
 pub fn verify_snapshot_archive<P, Q, R>(
     snapshot_archive: P,
     snapshots_to_verify: Q,
     storages_to_verify: R,
     archive_format: ArchiveFormat,
+    verify_bank: VerifyBank,
 ) where
     P: AsRef<Path>,
     Q: AsRef<Path>,
@@ -1672,6 +1683,17 @@ pub fn verify_snapshot_archive<P, Q, R>(
 
     // Check snapshots are the same
     let unpacked_snapshots = unpack_dir.join("snapshots");
+    if let VerifyBank::NonDeterministic(slot) = verify_bank {
+        // file contents may be different, but deserialized structs should be equal
+        let slot = slot.to_string();
+        let p1 = snapshots_to_verify.as_ref().join(&slot).join(&slot);
+        let p2 = unpacked_snapshots.join(&slot).join(&slot);
+
+        assert!(crate::serde_snapshot::compare_two_serialized_banks(&p1, &p2).unwrap());
+        std::fs::remove_file(p1).unwrap();
+        std::fs::remove_file(p2).unwrap();
+    }
+
     assert!(!dir_diff::is_different(&snapshots_to_verify, unpacked_snapshots).unwrap());
 
     // Check the account entries are the same