add(tests): Add snapshot tests for sprout database formats (#7057)

* Add methods for loading entire column families from the database

* Add a method that loads all the sprout trees from the database

* Add snapshot tests for sprout note commitment trees

* Add round-trip proptests for tree root database serialization

* Add a manual sprout note commitment tree database serialization snapshot test

* Add tests for 1,2,4,8 note commitments in a tree

* Remove redundant "rand" package rename in dependencies

* Randomly cache roots rather than only caching even roots

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Cargo.lock

@@ -5881,6 +5881,7 @@ dependencies = [
  "once_cell",
  "proptest",
  "proptest-derive",
+ "rand 0.8.5",
  "rayon",
  "regex",
  "rlimit",

tower-batch-control/Cargo.toml

@@ -39,7 +39,7 @@ color-eyre = "0.6.2"
 tinyvec = { version = "1.6.0", features = ["rustc_1_55"] }
 
 ed25519-zebra = "4.0.0"
-rand = { version = "0.8.5", package = "rand" }
+rand = "0.8.5"
 
 tokio = { version = "1.28.2", features = ["full", "tracing", "test-util"] }
 tokio-test = "0.4.2"

zebra-chain/Cargo.toml

@@ -114,7 +114,7 @@ zcash_address = { version = "0.2.1", optional = true }
 proptest = { version = "1.2.0", optional = true }
 proptest-derive = { version = "0.3.0", optional = true }
 
-rand = { version = "0.8.5", optional = true, package = "rand" }
+rand = { version = "0.8.5", optional = true }
 rand_chacha = { version = "0.3.1", optional = true }
 
 tokio = { version = "1.28.2", features = ["tracing"], optional = true }
@@ -137,7 +137,7 @@ tracing = "0.1.37"
 proptest = "1.2.0"
 proptest-derive = "0.3.0"
 
-rand = { version = "0.8.5", package = "rand" }
+rand = "0.8.5"
 rand_chacha = "0.3.1"
 
 tokio = { version = "1.28.2", features = ["full", "tracing", "test-util"] }

zebra-consensus/Cargo.toml

@@ -40,7 +40,7 @@ bellman = "0.14.0"
 bls12_381 = "0.8.0"
 halo2 = { package = "halo2_proofs", version = "0.3.0" }
 jubjub = "0.10.0"
-rand = { version = "0.8.5", package = "rand" }
+rand = "0.8.5"
 rayon = "1.7.0"
 
 chrono = { version = "0.4.26", default-features = false, features = ["clock", "std"] }

zebra-network/Cargo.toml

@@ -53,7 +53,7 @@ lazy_static = "1.4.0"
 num-integer = "0.1.45"
 ordered-map = "0.4.2"
 pin-project = "1.1.0"
-rand = { version = "0.8.5", package = "rand" }
+rand = "0.8.5"
 rayon = "1.7.0"
 regex = "1.8.4"
 serde = { version = "1.0.164", features = ["serde_derive"] }

zebra-rpc/Cargo.toml

@@ -63,7 +63,7 @@ hex = { version = "0.4.3", features = ["serde"] }
 serde = { version = "1.0.164", features = ["serde_derive"] }
 
 # Experimental feature getblocktemplate-rpcs
-rand = { version = "0.8.5", package = "rand", optional = true }
+rand = { version = "0.8.5", optional = true }
 # ECC deps used by getblocktemplate-rpcs feature
 zcash_address = { version = "0.2.1", optional = true }
 

zebra-state/Cargo.toml

@@ -91,10 +91,11 @@ once_cell = "1.18.0"
 spandoc = "0.2.2"
 
 hex = { version = "0.4.3", features = ["serde"] }
-insta = { version = "1.30.0", features = ["ron"] }
+insta = { version = "1.30.0", features = ["ron", "redactions"] }
 
 proptest = "1.2.0"
 proptest-derive = "0.3.0"
+rand = "0.8.5"
 
 halo2 = { package = "halo2_proofs", version = "0.3.0" }
 jubjub = "0.10.0"

zebra-state/src/service/finalized_state/disk_db.rs

@@ -10,7 +10,14 @@
 //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must
 //! be incremented each time the database format (column, serialization, etc) changes.
 
-use std::{cmp::Ordering, fmt::Debug, path::Path, sync::Arc};
+use std::{
+    cmp::Ordering,
+    collections::{BTreeMap, HashMap},
+    fmt::Debug,
+    ops::RangeBounds,
+    path::Path,
+    sync::Arc,
+};
 
 use itertools::Itertools;
 use rlimit::increase_nofile_limit;
@@ -146,6 +153,7 @@ impl WriteDisk for DiskWriteBatch {
 /// defined format
 //
 // TODO: just implement these methods directly on DiskDb
+//       move this trait, its methods, and support methods to another module
 pub trait ReadDisk {
     /// Returns true if a rocksdb column family `cf` does not contain any entries.
     fn zs_is_empty<C>(&self, cf: &C) -> bool
@@ -202,6 +210,26 @@ pub trait ReadDisk {
         C: rocksdb::AsColumnFamilyRef,
         K: IntoDisk + FromDisk,
         V: FromDisk;
+
+    /// Returns the keys and values in `cf` in `range`, in an ordered `BTreeMap`.
+    ///
+    /// Holding this iterator open might delay block commit transactions.
+    fn zs_items_in_range_ordered<C, K, V, R>(&self, cf: &C, range: R) -> BTreeMap<K, V>
+    where
+        C: rocksdb::AsColumnFamilyRef,
+        K: IntoDisk + FromDisk + Ord,
+        V: FromDisk,
+        R: RangeBounds<K>;
+
+    /// Returns the keys and values in `cf` in `range`, in an unordered `HashMap`.
+    ///
+    /// Holding this iterator open might delay block commit transactions.
+    fn zs_items_in_range_unordered<C, K, V, R>(&self, cf: &C, range: R) -> HashMap<K, V>
+    where
+        C: rocksdb::AsColumnFamilyRef,
+        K: IntoDisk + FromDisk + Eq + std::hash::Hash,
+        V: FromDisk,
+        R: RangeBounds<K>;
 }
 
 impl PartialEq for DiskDb {
@@ -342,6 +370,26 @@ impl ReadDisk for DiskDb {
             })
             .expect("unexpected database failure")
     }
+
+    fn zs_items_in_range_ordered<C, K, V, R>(&self, cf: &C, range: R) -> BTreeMap<K, V>
+    where
+        C: rocksdb::AsColumnFamilyRef,
+        K: IntoDisk + FromDisk + Ord,
+        V: FromDisk,
+        R: RangeBounds<K>,
+    {
+        self.zs_range_iter(cf, range).collect()
+    }
+
+    fn zs_items_in_range_unordered<C, K, V, R>(&self, cf: &C, range: R) -> HashMap<K, V>
+    where
+        C: rocksdb::AsColumnFamilyRef,
+        K: IntoDisk + FromDisk + Eq + std::hash::Hash,
+        V: FromDisk,
+        R: RangeBounds<K>,
+    {
+        self.zs_range_iter(cf, range).collect()
+    }
 }
 
 impl DiskWriteBatch {
@@ -366,6 +414,58 @@ impl DiskWriteBatch {
 }
 
 impl DiskDb {
+    /// Returns an iterator over the items in `cf` in `range`.
+    ///
+    /// Holding this iterator open might delay block commit transactions.
+    fn zs_range_iter<C, K, V, R>(&self, cf: &C, range: R) -> impl Iterator<Item = (K, V)> + '_
+    where
+        C: rocksdb::AsColumnFamilyRef,
+        K: IntoDisk + FromDisk,
+        V: FromDisk,
+        R: RangeBounds<K>,
+    {
+        use std::ops::Bound::{self, *};
+
+        // Replace with map() when it stabilises:
+        // https://github.com/rust-lang/rust/issues/86026
+        let map_to_vec = |bound: Bound<&K>| -> Bound<Vec<u8>> {
+            match bound {
+                Unbounded => Unbounded,
+                Included(x) => Included(x.as_bytes().as_ref().to_vec()),
+                Excluded(x) => Excluded(x.as_bytes().as_ref().to_vec()),
+            }
+        };
+
+        let start_bound = map_to_vec(range.start_bound());
+        let end_bound = map_to_vec(range.end_bound());
+        let range = (start_bound.clone(), end_bound);
+
+        let start_bound_vec =
+            if let Included(ref start_bound) | Excluded(ref start_bound) = start_bound {
+                start_bound.clone()
+            } else {
+                // Actually unused
+                Vec::new()
+            };
+
+        let start_mode = if matches!(start_bound, Unbounded) {
+            // Unbounded iterators start at the first item
+            rocksdb::IteratorMode::Start
+        } else {
+            rocksdb::IteratorMode::From(start_bound_vec.as_slice(), rocksdb::Direction::Forward)
+        };
+
+        // Reading multiple items from iterators has caused database hangs,
+        // in previous RocksDB versions
+        self.db
+            .iterator_cf(cf, start_mode)
+            .map(|result| result.expect("unexpected database failure"))
+            .map(|(key, value)| (key.to_vec(), value))
+            // Handle Excluded start and the end bound
+            .filter(move |(key, _value)| range.contains(key))
+            .map(|(key, value)| (K::from_bytes(key), V::from_bytes(value)))
+    }
+
     /// The ideal open file limit for Zebra
     const IDEAL_OPEN_FILE_LIMIT: u64 = 1024;
 

zebra-state/src/service/finalized_state/disk_format/shielded.rs

@@ -44,6 +44,13 @@ impl IntoDisk for sprout::tree::Root {
     }
 }
 
+impl FromDisk for sprout::tree::Root {
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
+        let array: [u8; 32] = bytes.as_ref().try_into().unwrap();
+        array.into()
+    }
+}
+
 impl IntoDisk for sapling::tree::Root {
     type Bytes = [u8; 32];
 
@@ -52,6 +59,13 @@ impl IntoDisk for sapling::tree::Root {
     }
 }
 
+impl FromDisk for sapling::tree::Root {
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
+        let array: [u8; 32] = bytes.as_ref().try_into().unwrap();
+        array.try_into().expect("finalized data must be valid")
+    }
+}
+
 impl IntoDisk for orchard::tree::Root {
     type Bytes = [u8; 32];
 
@@ -60,6 +74,13 @@ impl IntoDisk for orchard::tree::Root {
     }
 }
 
+impl FromDisk for orchard::tree::Root {
+    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
+        let array: [u8; 32] = bytes.as_ref().try_into().unwrap();
+        array.try_into().expect("finalized data must be valid")
+    }
+}
+
 // The following implementations for the note commitment trees use `serde` and
 // `bincode` because currently the inner Merkle tree frontier (from
 // `incrementalmerkletree`) only supports `serde` for serialization. `bincode`

zebra-state/src/service/finalized_state/disk_format/tests/prop.rs

@@ -279,6 +279,13 @@ fn serialized_sprout_tree_root_equal() {
     );
 }
 
+#[test]
+fn roundtrip_sprout_tree_root() {
+    let _init_guard = zebra_test::init();
+
+    proptest!(|(val in any::<sprout::tree::Root>())| assert_value_properties(val));
+}
+
 // TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary
 
 // Sapling
@@ -347,6 +354,13 @@ fn serialized_sapling_tree_root_equal() {
     );
 }
 
+#[test]
+fn roundtrip_sapling_tree_root() {
+    let _init_guard = zebra_test::init();
+
+    proptest!(|(val in any::<sapling::tree::Root>())| assert_value_properties(val));
+}
+
 // TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary
 
 // Orchard
@@ -415,6 +429,13 @@ fn serialized_orchard_tree_root_equal() {
     );
 }
 
+#[test]
+fn roundtrip_orchard_tree_root() {
+    let _init_guard = zebra_test::init();
+
+    proptest!(|(val in any::<orchard::tree::Root>())| assert_value_properties(val));
+}
+
 // TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary
 
 // Chain

zebra-state/src/service/finalized_state/tests/vectors.rs

@@ -1,11 +1,142 @@
 //! Fixed test vectors for the finalized state.
+//! These tests contain snapshots of the note commitment tree serialization format.
+//!
+//! We don't need to check empty trees, because the database format snapshot tests
+//! use empty trees.
 
 use halo2::pasta::{group::ff::PrimeField, pallas};
 use hex::FromHex;
+use rand::random;
+
+use zebra_chain::{orchard, sapling, sprout};
 
 use crate::service::finalized_state::disk_format::{FromDisk, IntoDisk};
-use zebra_chain::{orchard, sapling};
 
+/// Check that the sprout tree database serialization format has not changed.
+#[test]
+fn sprout_note_commitment_tree_serialization() {
+    let _init_guard = zebra_test::init();
+
+    let mut incremental_tree = sprout::tree::NoteCommitmentTree::default();
+
+    // Some commitments from zebra-chain/src/sprout/tests/test_vectors.rs
+    let hex_commitments = [
+        "62fdad9bfbf17c38ea626a9c9b8af8a748e6b4367c8494caf0ca592999e8b6ba",
+        "68eb35bc5e1ddb80a761718e63a1ecf4d4977ae22cc19fa732b85515b2a4c943",
+        "836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb",
+    ];
+
+    for (idx, cm_hex) in hex_commitments.iter().enumerate() {
+        let bytes = <[u8; 32]>::from_hex(cm_hex).unwrap();
+
+        let cm = sprout::NoteCommitment::from(bytes);
+        incremental_tree.append(cm).unwrap();
+        if random() {
+            info!(?idx, "randomly caching root for note commitment tree index");
+            // Cache the root half of the time to make sure it works in both cases
+            let _ = incremental_tree.root();
+        }
+    }
+
+    // Make sure the last root is cached
+    let _ = incremental_tree.root();
+
+    // This test vector was generated by the code itself.
+    // The purpose of this test is to make sure the serialization format does
+    // not change by accident.
+    let expected_serialized_tree_hex = "010200836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb019f5b2b1e4bf7e7318d0a1f417ca6bca36077025b3d11e074b94cd55ce9f3861801c45297124f50dcd3f78eed017afd1e30764cd74cdf0a57751978270fd0721359";
+    let serialized_tree = incremental_tree.as_bytes();
+    assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex);
+
+    let deserialized_tree = sprout::tree::NoteCommitmentTree::from_bytes(serialized_tree);
+
+    assert_eq!(incremental_tree.root(), deserialized_tree.root());
+}
+
+/// Check that the sprout tree database serialization format has not changed for one commitment.
+#[test]
+fn sprout_note_commitment_tree_serialization_one() {
+    let _init_guard = zebra_test::init();
+
+    let mut incremental_tree = sprout::tree::NoteCommitmentTree::default();
+
+    // Some commitments from zebra-chain/src/sprout/tests/test_vectors.rs
+    let hex_commitments = ["836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb"];
+
+    for (idx, cm_hex) in hex_commitments.iter().enumerate() {
+        let bytes = <[u8; 32]>::from_hex(cm_hex).unwrap();
+
+        let cm = sprout::NoteCommitment::from(bytes);
+        incremental_tree.append(cm).unwrap();
+        if random() {
+            info!(?idx, "randomly caching root for note commitment tree index");
+            // Cache the root half of the time to make sure it works in both cases
+            let _ = incremental_tree.root();
+        }
+    }
+
+    // Make sure the last root is cached
+    let _ = incremental_tree.root();
+
+    // This test vector was generated by the code itself.
+    // The purpose of this test is to make sure the serialization format does
+    // not change by accident.
+    let expected_serialized_tree_hex = "010000836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb000193e5f97ce1d5d94d0c6e1b66a4a262c9ae89e56e28f3f6e4a557b6fb70e173a8";
+    let serialized_tree = incremental_tree.as_bytes();
+    assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex);
+
+    let deserialized_tree = sprout::tree::NoteCommitmentTree::from_bytes(serialized_tree);
+
+    assert_eq!(incremental_tree.root(), deserialized_tree.root());
+}
+
+/// Check that the sprout tree database serialization format has not changed when the number of
+/// commitments is a power of two.
+///
+/// Some trees have special handling for even numbers of roots, or powers of two,
+/// so we also check that case.
+#[test]
+fn sprout_note_commitment_tree_serialization_pow2() {
+    let _init_guard = zebra_test::init();
+
+    let mut incremental_tree = sprout::tree::NoteCommitmentTree::default();
+
+    // Some commitments from zebra-chain/src/sprout/tests/test_vectors.rs
+    let hex_commitments = [
+        "62fdad9bfbf17c38ea626a9c9b8af8a748e6b4367c8494caf0ca592999e8b6ba",
+        "68eb35bc5e1ddb80a761718e63a1ecf4d4977ae22cc19fa732b85515b2a4c943",
+        "836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb",
+        "92498a8295ea36d593eaee7cb8b55be3a3e37b8185d3807693184054cd574ae4",
+    ];
+
+    for (idx, cm_hex) in hex_commitments.iter().enumerate() {
+        let bytes = <[u8; 32]>::from_hex(cm_hex).unwrap();
+
+        let cm = sprout::NoteCommitment::from(bytes);
+        incremental_tree.append(cm).unwrap();
+        if random() {
+            info!(?idx, "randomly caching root for note commitment tree index");
+            // Cache the root half of the time to make sure it works in both cases
+            let _ = incremental_tree.root();
+        }
+    }
+
+    // Make sure the last root is cached
+    let _ = incremental_tree.root();
+
+    // This test vector was generated by the code itself.
+    // The purpose of this test is to make sure the serialization format does
+    // not change by accident.
+    let expected_serialized_tree_hex = "010301836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb92498a8295ea36d593eaee7cb8b55be3a3e37b8185d3807693184054cd574ae4019f5b2b1e4bf7e7318d0a1f417ca6bca36077025b3d11e074b94cd55ce9f3861801b61f588fcba9cea79e94376adae1c49583f716d2f20367141f1369a235b95c98";
+    let serialized_tree = incremental_tree.as_bytes();
+    assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex);
+
+    let deserialized_tree = sprout::tree::NoteCommitmentTree::from_bytes(serialized_tree);
+
+    assert_eq!(incremental_tree.root(), deserialized_tree.root());
+}
+
+/// Check that the sapling tree database serialization format has not changed.
 #[test]
 fn sapling_note_commitment_tree_serialization() {
     let _init_guard = zebra_test::init();
@@ -24,7 +155,8 @@ fn sapling_note_commitment_tree_serialization() {
 
         let cm_u = jubjub::Fq::from_bytes(&bytes).unwrap();
         incremental_tree.append(cm_u).unwrap();
-        if idx % 2 == 0 {
+        if random() {
+            info!(?idx, "randomly caching root for note commitment tree index");
             // Cache the root half of the time to make sure it works in both cases
             let _ = incremental_tree.root();
         }
@@ -45,6 +177,94 @@ fn sapling_note_commitment_tree_serialization() {
     assert_eq!(incremental_tree.root(), deserialized_tree.root());
 }
 
+/// Check that the sapling tree database serialization format has not changed for one commitment.
+#[test]
+fn sapling_note_commitment_tree_serialization_one() {
+    let _init_guard = zebra_test::init();
+
+    let mut incremental_tree = sapling::tree::NoteCommitmentTree::default();
+
+    // Some commitments from zebra-chain/src/sapling/tests/test_vectors.rs
+    let hex_commitments = ["225747f3b5d5dab4e5a424f81f85c904ff43286e0f3fd07ef0b8c6a627b11458"];
+
+    for (idx, cm_u_hex) in hex_commitments.iter().enumerate() {
+        let bytes = <[u8; 32]>::from_hex(cm_u_hex).unwrap();
+
+        let cm_u = jubjub::Fq::from_bytes(&bytes).unwrap();
+        incremental_tree.append(cm_u).unwrap();
+        if random() {
+            info!(?idx, "randomly caching root for note commitment tree index");
+            // Cache the root half of the time to make sure it works in both cases
+            let _ = incremental_tree.root();
+        }
+    }
+
+    // Make sure the last root is cached
+    let _ = incremental_tree.root();
+
+    // This test vector was generated by the code itself.
+    // The purpose of this test is to make sure the serialization format does
+    // not change by accident.
+    let expected_serialized_tree_hex = "010000225747f3b5d5dab4e5a424f81f85c904ff43286e0f3fd07ef0b8c6a627b1145800012c60c7de033d7539d123fb275011edfe08d57431676981d162c816372063bc71";
+    let serialized_tree = incremental_tree.as_bytes();
+    assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex);
+
+    let deserialized_tree = sapling::tree::NoteCommitmentTree::from_bytes(serialized_tree);
+
+    assert_eq!(incremental_tree.root(), deserialized_tree.root());
+}
+
+/// Check that the sapling tree database serialization format has not changed when the number of
+/// commitments is a power of two.
+///
+/// Some trees have special handling for even numbers of roots, or powers of two,
+/// so we also check that case.
+#[test]
+fn sapling_note_commitment_tree_serialization_pow2() {
+    let _init_guard = zebra_test::init();
+
+    let mut incremental_tree = sapling::tree::NoteCommitmentTree::default();
+
+    // Some commitments from zebra-chain/src/sapling/tests/test_vectors.rs
+    let hex_commitments = [
+        "3a27fed5dbbc475d3880360e38638c882fd9b273b618fc433106896083f77446",
+        "c7ca8f7df8fd997931d33985d935ee2d696856cc09cc516d419ea6365f163008",
+        "f0fa37e8063b139d342246142fc48e7c0c50d0a62c97768589e06466742c3702",
+        "e6d4d7685894d01b32f7e081ab188930be6c2b9f76d6847b7f382e3dddd7c608",
+        "8cebb73be883466d18d3b0c06990520e80b936440a2c9fd184d92a1f06c4e826",
+        "22fab8bcdb88154dbf5877ad1e2d7f1b541bc8a5ec1b52266095381339c27c03",
+        "f43e3aac61e5a753062d4d0508c26ceaf5e4c0c58ba3c956e104b5d2cf67c41c",
+        "3a3661bc12b72646c94bc6c92796e81953985ee62d80a9ec3645a9a95740ac15",
+    ];
+
+    for (idx, cm_u_hex) in hex_commitments.iter().enumerate() {
+        let bytes = <[u8; 32]>::from_hex(cm_u_hex).unwrap();
+
+        let cm_u = jubjub::Fq::from_bytes(&bytes).unwrap();
+        incremental_tree.append(cm_u).unwrap();
+        if random() {
+            info!(?idx, "randomly caching root for note commitment tree index");
+            // Cache the root half of the time to make sure it works in both cases
+            let _ = incremental_tree.root();
+        }
+    }
+
+    // Make sure the last root is cached
+    let _ = incremental_tree.root();
+
+    // This test vector was generated by the code itself.
+    // The purpose of this test is to make sure the serialization format does
+    // not change by accident.
+    let expected_serialized_tree_hex = "010701f43e3aac61e5a753062d4d0508c26ceaf5e4c0c58ba3c956e104b5d2cf67c41c3a3661bc12b72646c94bc6c92796e81953985ee62d80a9ec3645a9a95740ac15025991131c5c25911b35fcea2a8343e2dfd7a4d5b45493390e0cb184394d91c349002df68503da9247dfde6585cb8c9fa94897cf21735f8fc1b32116ef474de05c01d23765f3d90dfd97817ed6d995bd253d85967f77b9f1eaef6ecbcb0ef6796812";
+    let serialized_tree = incremental_tree.as_bytes();
+    assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex);
+
+    let deserialized_tree = sapling::tree::NoteCommitmentTree::from_bytes(serialized_tree);
+
+    assert_eq!(incremental_tree.root(), deserialized_tree.root());
+}
+
+/// Check that the orchard tree database serialization format has not changed.
 #[test]
 fn orchard_note_commitment_tree_serialization() {
     let _init_guard = zebra_test::init();
@@ -73,7 +293,8 @@ fn orchard_note_commitment_tree_serialization() {
     for (idx, cm_x_bytes) in commitments.iter().enumerate() {
         let cm_x = pallas::Base::from_repr(*cm_x_bytes).unwrap();
         incremental_tree.append(cm_x).unwrap();
-        if idx % 2 == 0 {
+        if random() {
+            info!(?idx, "randomly caching root for note commitment tree index");
             // Cache the root half of the time to make sure it works in both cases
             let _ = incremental_tree.root();
         }
@@ -93,3 +314,92 @@ fn orchard_note_commitment_tree_serialization() {
 
     assert_eq!(incremental_tree.root(), deserialized_tree.root());
 }
+
+/// Check that the orchard tree database serialization format has not changed for one commitment.
+#[test]
+fn orchard_note_commitment_tree_serialization_one() {
+    let _init_guard = zebra_test::init();
+
+    let mut incremental_tree = orchard::tree::NoteCommitmentTree::default();
+
+    // Some commitments from zebra-chain/src/orchard/tests/tree.rs
+    let commitments = [[
+        0x68, 0x13, 0x5c, 0xf4, 0x99, 0x33, 0x22, 0x90, 0x99, 0xa4, 0x4e, 0xc9, 0x9a, 0x75, 0xe1,
+        0xe1, 0xcb, 0x46, 0x40, 0xf9, 0xb5, 0xbd, 0xec, 0x6b, 0x32, 0x23, 0x85, 0x6f, 0xea, 0x16,
+        0x39, 0x0a,
+    ]];
+
+    for (idx, cm_x_bytes) in commitments.iter().enumerate() {
+        let cm_x = pallas::Base::from_repr(*cm_x_bytes).unwrap();
+        incremental_tree.append(cm_x).unwrap();
+        if random() {
+            info!(?idx, "randomly caching root for note commitment tree index");
+            // Cache the root half of the time to make sure it works in both cases
+            let _ = incremental_tree.root();
+        }
+    }
+
+    // Make sure the last root is cached
+    let _ = incremental_tree.root();
+
+    // This test vector was generated by the code itself.
+    // The purpose of this test is to make sure the serialization format does
+    // not change by accident.
+    let expected_serialized_tree_hex = "01000068135cf49933229099a44ec99a75e1e1cb4640f9b5bdec6b3223856fea16390a000178afd4da59c541e9c2f317f9aff654f1fb38d14dc99431cbbfa93601c7068117";
+    let serialized_tree = incremental_tree.as_bytes();
+    assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex);
+
+    let deserialized_tree = orchard::tree::NoteCommitmentTree::from_bytes(serialized_tree);
+
+    assert_eq!(incremental_tree.root(), deserialized_tree.root());
+}
+
+/// Check that the orchard tree database serialization format has not changed when the number of
+/// commitments is a power of two.
+///
+/// Some trees have special handling for even numbers of roots, or powers of two,
+/// so we also check that case.
+#[test]
+fn orchard_note_commitment_tree_serialization_pow2() {
+    let _init_guard = zebra_test::init();
+
+    let mut incremental_tree = orchard::tree::NoteCommitmentTree::default();
+
+    // Some commitments from zebra-chain/src/orchard/tests/tree.rs
+    let commitments = [
+        [
+            0x78, 0x31, 0x50, 0x08, 0xfb, 0x29, 0x98, 0xb4, 0x30, 0xa5, 0x73, 0x1d, 0x67, 0x26,
+            0x20, 0x7d, 0xc0, 0xf0, 0xec, 0x81, 0xea, 0x64, 0xaf, 0x5c, 0xf6, 0x12, 0x95, 0x69,
+            0x01, 0xe7, 0x2f, 0x0e,
+        ],
+        [
+            0xee, 0x94, 0x88, 0x05, 0x3a, 0x30, 0xc5, 0x96, 0xb4, 0x30, 0x14, 0x10, 0x5d, 0x34,
+            0x77, 0xe6, 0xf5, 0x78, 0xc8, 0x92, 0x40, 0xd1, 0xd1, 0xee, 0x17, 0x43, 0xb7, 0x7b,
+            0xb6, 0xad, 0xc4, 0x0a,
+        ],
+    ];
+
+    for (idx, cm_x_bytes) in commitments.iter().enumerate() {
+        let cm_x = pallas::Base::from_repr(*cm_x_bytes).unwrap();
+        incremental_tree.append(cm_x).unwrap();
+        if random() {
+            info!(?idx, "randomly caching root for note commitment tree index");
+            // Cache the root half of the time to make sure it works in both cases
+            let _ = incremental_tree.root();
+        }
+    }
+
+    // Make sure the last root is cached
+    let _ = incremental_tree.root();
+
+    // This test vector was generated by the code itself.
+    // The purpose of this test is to make sure the serialization format does
+    // not change by accident.
+    let expected_serialized_tree_hex = "01010178315008fb2998b430a5731d6726207dc0f0ec81ea64af5cf612956901e72f0eee9488053a30c596b43014105d3477e6f578c89240d1d1ee1743b77bb6adc40a0001d3d525931005e45f5a29bc82524e871e5ee1b6d77839deb741a6e50cd99fdf1a";
+    let serialized_tree = incremental_tree.as_bytes();
+    assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex);
+
+    let deserialized_tree = orchard::tree::NoteCommitmentTree::from_bytes(serialized_tree);
+
+    assert_eq!(incremental_tree.root(), deserialized_tree.root());
+}

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs

@@ -217,6 +217,8 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
     if let Some((max_height, tip_block_hash)) = tip {
         // Check that the database returns empty note commitment trees for the
         // genesis block.
+        //
+        // We only store the sprout tree for the tip by height, so we can't check sprout here.
         let sapling_tree = state
             .sapling_note_commitment_tree_by_height(&block::Height::MIN)
             .expect("the genesis block in the database has a Sapling tree");
@@ -241,9 +243,11 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
 
         // Shielded
 
+        let stored_sprout_trees = state.sprout_note_commitments_full_map();
         let mut stored_sapling_trees = Vec::new();
         let mut stored_orchard_trees = Vec::new();
 
+        let sprout_tree_at_tip = state.sprout_note_commitment_tree();
         let sapling_tree_at_tip = state.sapling_note_commitment_tree();
         let orchard_tree_at_tip = state.orchard_note_commitment_tree();
 
@@ -268,9 +272,11 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
                 .block(query_height.into())
                 .expect("heights up to tip have blocks");
 
-            // Check the sapling and orchard note commitment trees.
+            // Check the shielded note commitment trees.
             //
-            // TODO: test the rest of the shielded data (anchors, nullifiers, sprout)
+            // We only store the sprout tree for the tip by height, so we can't check sprout here.
+            //
+            // TODO: test the rest of the shielded data (anchors, nullifiers)
             let sapling_tree_by_height = state
                 .sapling_note_commitment_tree_by_height(&query_height)
                 .expect("heights up to tip have Sapling trees");
@@ -297,6 +303,18 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
             if query_height == max_height {
                 assert_eq!(stored_block_hash, tip_block_hash);
 
+                // We only store the sprout tree for the tip by height,
+                // so the sprout check is less strict.
+                // We enforce the tip tree order by snapshotting it as well.
+                if let Some(stored_tree) = stored_sprout_trees.get(&sprout_tree_at_tip.root()) {
+                    assert_eq!(
+                        &sprout_tree_at_tip, stored_tree,
+                        "unexpected missing sprout tip tree:\n\
+                         all trees: {stored_sprout_trees:?}"
+                    );
+                } else {
+                    assert_eq!(sprout_tree_at_tip, Default::default());
+                }
                 assert_eq!(sapling_tree_at_tip, sapling_tree_by_height);
                 assert_eq!(orchard_tree_at_tip, orchard_tree_by_height);
 
@@ -427,6 +445,14 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
         // These snapshots will change if the trees do not have cached roots.
         // But we expect them to always have cached roots,
         // because those roots are used to populate the anchor column families.
+        insta::assert_ron_snapshot!("sprout_tree_at_tip", sprout_tree_at_tip);
+        insta::assert_ron_snapshot!(
+            "sprout_trees",
+            stored_sprout_trees,
+            {
+                "." => insta::sorted_redaction()
+            }
+        );
         insta::assert_ron_snapshot!("sapling_trees", stored_sapling_trees);
         insta::assert_ron_snapshot!("orchard_trees", stored_orchard_trees);
 

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_0.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: sprout_tree_at_tip
+---
+NoteCommitmentTree(
+  inner: Frontier(
+    frontier: None,
+  ),
+  cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))),
+)

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_1.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: sprout_tree_at_tip
+---
+NoteCommitmentTree(
+  inner: Frontier(
+    frontier: None,
+  ),
+  cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))),
+)

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_2.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: sprout_tree_at_tip
+---
+NoteCommitmentTree(
+  inner: Frontier(
+    frontier: None,
+  ),
+  cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))),
+)

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_0.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: sprout_tree_at_tip
+---
+NoteCommitmentTree(
+  inner: Frontier(
+    frontier: None,
+  ),
+  cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))),
+)

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_1.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: sprout_tree_at_tip
+---
+NoteCommitmentTree(
+  inner: Frontier(
+    frontier: None,
+  ),
+  cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))),
+)

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_2.snap

@@ -0,0 +1,10 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: sprout_tree_at_tip
+---
+NoteCommitmentTree(
+  inner: Frontier(
+    frontier: None,
+  ),
+  cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))),
+)

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_0.snap

@@ -0,0 +1,5 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: stored_sprout_trees
+---
+{}

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_1.snap

@@ -0,0 +1,12 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: stored_sprout_trees
+---
+{
+  Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89)): NoteCommitmentTree(
+    inner: Frontier(
+      frontier: None,
+    ),
+    cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))),
+  ),
+}

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_2.snap

@@ -0,0 +1,12 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: stored_sprout_trees
+---
+{
+  Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89)): NoteCommitmentTree(
+    inner: Frontier(
+      frontier: None,
+    ),
+    cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))),
+  ),
+}

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_0.snap

@@ -0,0 +1,5 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: stored_sprout_trees
+---
+{}

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_1.snap

@@ -0,0 +1,12 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: stored_sprout_trees
+---
+{
+  Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89)): NoteCommitmentTree(
+    inner: Frontier(
+      frontier: None,
+    ),
+    cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))),
+  ),
+}

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_2.snap

@@ -0,0 +1,12 @@
+---
+source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
+expression: stored_sprout_trees
+---
+{
+  Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89)): NoteCommitmentTree(
+    inner: Frontier(
+      frontier: None,
+    ),
+    cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))),
+  ),
+}

zebra-state/src/service/finalized_state/zebra_db/shielded.rs

@@ -12,7 +12,7 @@
 //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must
 //! be incremented each time the database format (column, serialization, etc) changes.
 
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
 
 use zebra_chain::{
     block::Height, orchard, parallel::tree::NoteCommitmentTrees, sapling, sprout,
@@ -99,6 +99,19 @@ impl ZebraDb {
             .map(Arc::new)
     }
 
+    /// Returns all the Sprout note commitment trees in the database.
+    ///
+    /// Calling this method can load a lot of data into RAM, and delay block commit transactions.
+    #[allow(dead_code, clippy::unwrap_in_result)]
+    pub fn sprout_note_commitments_full_map(
+        &self,
+    ) -> HashMap<sprout::tree::Root, Arc<sprout::tree::NoteCommitmentTree>> {
+        let sprout_anchors_handle = self.db.cf_handle("sprout_anchors").unwrap();
+
+        self.db
+            .zs_items_in_range_unordered(&sprout_anchors_handle, ..)
+    }
+
     /// Returns the Sapling note commitment tree of the finalized tip
     /// or the empty tree if the state is empty.
     pub fn sapling_note_commitment_tree(&self) -> Arc<sapling::tree::NoteCommitmentTree> {

zebra-test/Cargo.toml

@@ -21,7 +21,7 @@ lazy_static = "1.4.0"
 insta = "1.30.0"
 proptest = "1.2.0"
 once_cell = "1.18.0"
-rand = { version = "0.8.5", package = "rand" }
+rand = "0.8.5"
 regex = "1.8.4"
 
 tokio = { version = "1.28.2", features = ["full", "tracing", "test-util"] }

zebrad/Cargo.toml

@@ -172,7 +172,7 @@ dirs = "5.0.1"
 atty = "0.2.14"
 
 num-integer = "0.1.45"
-rand = { version = "0.8.5", package = "rand" }
+rand = "0.8.5"
 
 # prod feature sentry
 sentry = { version = "0.31.5", default-features = false, features = ["backtrace", "contexts", "reqwest", "rustls", "tracing"], optional = true }