change(scan): Refactor scanning tests (#8047)

* Derive & impl helper traits from `std`

* Create `compact_to_v4` fn

* Create `fake_block` fn

* Refactor existing tests to use the new functions

* Cosmetics

* Refactor docs

* Put `Default` behind `cfg_attr(test)`

Rationale
---------

We avoid implementing `Default` on consensus-critical types because it's
easy to miss an incorrect use in a review. It's easy to hide a
`default()` in a call like `unwrap_or_default()` or even more subtle
methods.

---------

Co-authored-by: teor <teor@riseup.net>

Cargo.lock

@@ -5798,6 +5798,7 @@ name = "zebra-scan"
 version = "0.1.0-alpha.0"
 dependencies = [
  "bls12_381",
+ "chrono",
  "color-eyre",
  "ff",
  "group",

zebra-chain/src/amount.rs

@@ -491,7 +491,7 @@ impl Error {
 ///     -MAX_MONEY..=MAX_MONEY,
 /// );
 /// ```
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Default)]
 pub struct NegativeAllowed;
 
 impl Constraint for NegativeAllowed {

zebra-chain/src/block/hash.rs

@@ -21,7 +21,7 @@ use proptest_derive::Arbitrary;
 /// Note: Zebra displays transaction and block hashes in big-endian byte-order,
 /// following the u256 convention set by Bitcoin and zcashd.
 #[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
-#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
+#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary, Default))]
 pub struct Hash(pub [u8; 32]);
 
 impl Hash {

zebra-chain/src/block/merkle.rs

@@ -70,7 +70,7 @@ use proptest_derive::Arbitrary;
 ///
 /// [ZIP-244]: https://zips.z.cash/zip-0244
 #[derive(Clone, Copy, Eq, PartialEq, Serialize, Deserialize)]
-#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
+#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary, Default))]
 pub struct Root(pub [u8; 32]);
 
 impl fmt::Debug for Root {

zebra-chain/src/fmt.rs

@@ -163,7 +163,7 @@ where
 
 /// Wrapper to override `Debug`, redirecting it to hex-encode the type.
 /// The type must implement `AsRef<[u8]>`.
-#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default)]
 #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))]
 #[serde(transparent)]
 pub struct HexDebug<T: AsRef<[u8]>>(pub T);

zebra-chain/src/sapling.rs

@@ -24,7 +24,9 @@ pub mod shielded_data;
 pub mod spend;
 pub mod tree;
 
-pub use commitment::{CommitmentRandomness, NoteCommitment, ValueCommitment};
+pub use commitment::{
+    CommitmentRandomness, NotSmallOrderValueCommitment, NoteCommitment, ValueCommitment,
+};
 pub use keys::Diversifier;
 pub use note::{EncryptedNote, Note, Nullifier, WrappedNoteKey};
 pub use output::{Output, OutputInTransactionV4, OutputPrefixInTransactionV5};

zebra-chain/src/sapling/commitment.rs

@@ -158,6 +158,7 @@ impl NoteCommitment {
 ///
 /// <https://zips.z.cash/protocol/protocol.pdf#concretehomomorphiccommit>
 #[derive(Clone, Copy, Deserialize, PartialEq, Eq, Serialize)]
+#[cfg_attr(any(test, feature = "proptest-impl"), derive(Default))]
 pub struct ValueCommitment(#[serde(with = "serde_helpers::AffinePoint")] jubjub::AffinePoint);
 
 impl<'a> std::ops::Add<&'a ValueCommitment> for ValueCommitment {
@@ -302,6 +303,7 @@ lazy_static! {
 /// <https://zips.z.cash/protocol/protocol.pdf#spenddesc>
 /// <https://zips.z.cash/protocol/protocol.pdf#outputdesc>
 #[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq, Serialize)]
+#[cfg_attr(any(test, feature = "proptest-impl"), derive(Default))]
 pub struct NotSmallOrderValueCommitment(ValueCommitment);
 
 impl TryFrom<ValueCommitment> for NotSmallOrderValueCommitment {

zebra-chain/src/sapling/note/ciphertexts.rs

@@ -12,6 +12,12 @@ use crate::serialization::{SerializationError, ZcashDeserialize, ZcashSerialize}
 #[derive(Deserialize, Serialize)]
 pub struct EncryptedNote(#[serde(with = "BigArray")] pub(crate) [u8; 580]);
 
+impl From<[u8; 580]> for EncryptedNote {
+    fn from(byte_array: [u8; 580]) -> Self {
+        Self(byte_array)
+    }
+}
+
 impl fmt::Debug for EncryptedNote {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         f.debug_tuple("EncryptedNote")
@@ -59,6 +65,12 @@ impl ZcashDeserialize for EncryptedNote {
 #[derive(Deserialize, Serialize)]
 pub struct WrappedNoteKey(#[serde(with = "BigArray")] pub(crate) [u8; 80]);
 
+impl From<[u8; 80]> for WrappedNoteKey {
+    fn from(byte_array: [u8; 80]) -> Self {
+        Self(byte_array)
+    }
+}
+
 impl fmt::Debug for WrappedNoteKey {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         f.debug_tuple("WrappedNoteKey")

zebra-chain/src/work/difficulty.rs

@@ -63,6 +63,7 @@ mod tests;
 ///
 /// [section 7.7.4]: https://zips.z.cash/protocol/protocol.pdf#nbits
 #[derive(Clone, Copy, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(any(test, feature = "proptest-impl"), derive(Default))]
 pub struct CompactDifficulty(pub(crate) u32);
 
 /// An invalid CompactDifficulty value, for testing.

zebra-chain/src/work/equihash.rs

@@ -93,6 +93,13 @@ impl Clone for Solution {
 
 impl Eq for Solution {}
 
+#[cfg(any(test, feature = "proptest-impl"))]
+impl Default for Solution {
+    fn default() -> Self {
+        Self([0; SOLUTION_SIZE])
+    }
+}
+
 impl ZcashSerialize for Solution {
     fn zcash_serialize<W: io::Write>(&self, writer: W) -> Result<(), io::Error> {
         zcash_serialize_bytes(&self.0.to_vec(), writer)

zebra-scan/Cargo.toml

@@ -35,6 +35,8 @@ zcash_primitives = "0.13.0-rc.1"
 zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.31" }
 zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31", features = ["shielded-scan"] }
 
+chrono = { version = "0.4.31", default-features = false, features = ["clock", "std", "serde"] }
+
 [dev-dependencies]
 
 bls12_381 = "0.8.0"

zebra-scan/src/tests.rs

@@ -5,10 +5,12 @@
 
 use std::sync::Arc;
 
-use color_eyre::Result;
+use chrono::{DateTime, Utc};
+
+use color_eyre::{Report, Result};
 use ff::{Field, PrimeField};
 use group::GroupEncoding;
-use rand::{rngs::OsRng, RngCore};
+use rand::{rngs::OsRng, thread_rng, RngCore};
 
 use zcash_client_backend::{
     encoding::decode_extended_full_viewing_key,
@@ -26,16 +28,23 @@ use zcash_primitives::{
         note_encryption::{sapling_note_encryption, SaplingDomain},
         util::generate_random_rseed,
         value::NoteValue,
-        Note, Nullifier, SaplingIvk,
+        Note, Nullifier,
     },
-    zip32::{AccountId, DiversifiableFullViewingKey, ExtendedSpendingKey},
+    zip32::{DiversifiableFullViewingKey, ExtendedSpendingKey},
 };
 
 use zebra_chain::{
-    block::{Block, Height},
+    amount::{Amount, NegativeAllowed},
+    block::{self, merkle, Block, Header, Height},
     chain_tip::ChainTip,
+    fmt::HexDebug,
     parameters::Network,
-    serialization::ZcashDeserializeInto,
+    primitives::{redjubjub, Groth16Proof},
+    sapling::{self, PerSpendAnchor, Spend, TransferData},
+    serialization::{AtLeastOne, ZcashDeserializeInto},
+    transaction::{LockTime, Transaction},
+    transparent::{CoinbaseData, Input},
+    work::{difficulty::CompactDifficulty, equihash::Solution},
 };
 use zebra_state::SaplingScannedResult;
 
@@ -44,49 +53,40 @@ use crate::{
     scan::{block_to_compact, scan_block},
 };
 
-/// Prove that we can create fake blocks with fake notes and scan them using the
+/// This test:
-/// `zcash_client_backend::scanning::scan_block` function:
+/// - Creates a viewing key and a fake block containing a Sapling output decryptable by the key.
-/// - Function `fake_compact_block` will generate 1 block with one pre created fake nullifier in
+/// - Scans the block.
-/// the transaction and one additional random transaction without it.
+/// - Checks that the result contains the txid of the tx containing the Sapling output.
-/// - Verify one relevant transaction is found in the chain when scanning for the pre created fake
+#[tokio::test]
-/// account's nullifier.
+async fn scanning_from_fake_generated_blocks() -> Result<()> {
-#[test]
-fn scanning_from_fake_generated_blocks() -> Result<()> {
-    let account = AccountId::from(12);
     let extsk = ExtendedSpendingKey::master(&[]);
     let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key();
-    let vks: Vec<(&AccountId, &SaplingIvk)> = vec![];
     let nf = Nullifier([7; 32]);
 
-    let cb = fake_compact_block(
+    let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0));
-        1u32.into(),
-        BlockHash([0; 32]),
-        nf,
-        &dfvk,
-        1,
-        false,
-        Some(0),
-    );
 
-    // The fake block function will have our transaction and a random one.
+    assert_eq!(block.transactions.len(), 4);
-    assert_eq!(cb.vtx.len(), 2);
 
-    let res = zcash_client_backend::scanning::scan_block(
+    let res = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap();
-        &zcash_primitives::consensus::MainNetwork,
-        cb.clone(),
-        &vks[..],
-        &[(account, nf)],
-        None,
-    )
-    .unwrap();
 
     // The response should have one transaction relevant to the key we provided.
     assert_eq!(res.transactions().len(), 1);
-    // The transaction should be the one we provided, second one in the block.
+
-    // (random transaction is added before ours in `fake_compact_block` function)
+    // Check that the original block contains the txid in the scanning result.
-    assert_eq!(res.transactions()[0].txid, cb.vtx[1].txid());
+    assert!(block
+        .transactions
+        .iter()
+        .map(|tx| tx.hash().bytes_in_display_order())
+        .any(|txid| &txid == res.transactions()[0].txid.as_ref()));
+
+    // Check that the txid in the scanning result matches the third tx in the original block.
+    assert_eq!(
+        res.transactions()[0].txid.as_ref(),
+        &block.transactions[2].hash().bytes_in_display_order()
+    );
+
     // The block hash of the response should be the same as the one provided.
-    assert_eq!(res.block_hash(), cb.hash());
+    assert_eq!(res.block_hash().0, block.hash().0);
 
     Ok(())
 }
@@ -111,7 +111,7 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> {
     let ivk = fvk.vk.ivk();
     let ivks = vec![ivk];
 
-    let network = zebra_chain::parameters::Network::Mainnet;
+    let network = Network::Mainnet;
 
     // Create a continuous chain of mainnet blocks from genesis
     let blocks: Vec<Arc<Block>> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS
@@ -170,13 +170,14 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> {
     Ok(())
 }
 
-/// In this test we generate a viewing key and manually add it to the database. Also we send results to the Storage database.
+/// Creates a viewing key and a fake block containing a Sapling output decryptable by the key, scans
+/// the block using the key, and adds the results to the database.
+///
 /// The purpose of this test is to check if our database and our scanning code are compatible.
 #[test]
 #[allow(deprecated)]
 fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
     // Generate a key
-    let account = AccountId::from(12);
     let extsk = ExtendedSpendingKey::master(&[]);
     // TODO: find out how to do it with `to_diversifiable_full_viewing_key` as `to_extended_full_viewing_key` is deprecated.
     let extfvk = extsk.to_extended_full_viewing_key();
@@ -197,28 +198,11 @@ fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
         Some(&s.min_sapling_birthday_height())
     );
 
-    let vks: Vec<(&AccountId, &SaplingIvk)> = vec![];
     let nf = Nullifier([7; 32]);
 
-    // Add key to fake block
+    let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0));
-    let cb = fake_compact_block(
-        1u32.into(),
-        BlockHash([0; 32]),
-        nf,
-        &dfvk,
-        1,
-        false,
-        Some(0),
-    );
 
-    let result = zcash_client_backend::scanning::scan_block(
+    let result = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap();
-        &zcash_primitives::consensus::MainNetwork,
-        cb.clone(),
-        &vks[..],
-        &[(account, nf)],
-        None,
-    )
-    .unwrap();
 
     // The response should have one transaction relevant to the key we provided.
     assert_eq!(result.transactions().len(), 1);
@@ -237,6 +221,81 @@ fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
     Ok(())
 }
 
+/// Generates a fake block containing a Sapling output decryptable by `dfvk`.
+///
+/// The fake block has the following transactions in this order:
+/// 1. a transparent coinbase tx,
+/// 2. a V4 tx containing a random Sapling output,
+/// 3. a V4 tx containing a Sapling output decryptable by `dfvk`,
+/// 4. depending on the value of `tx_after`, another V4 tx containing a random Sapling output.
+fn fake_block(
+    height: BlockHeight,
+    nf: Nullifier,
+    dfvk: &DiversifiableFullViewingKey,
+    value: u64,
+    tx_after: bool,
+    initial_sapling_tree_size: Option<u32>,
+) -> (Block, u32) {
+    let header = Header {
+        version: 4,
+        previous_block_hash: block::Hash::default(),
+        merkle_root: merkle::Root::default(),
+        commitment_bytes: HexDebug::default(),
+        time: DateTime::<Utc>::default(),
+        difficulty_threshold: CompactDifficulty::default(),
+        nonce: HexDebug::default(),
+        solution: Solution::default(),
+    };
+
+    let block = fake_compact_block(
+        height,
+        BlockHash([0; 32]),
+        nf,
+        dfvk,
+        value,
+        tx_after,
+        initial_sapling_tree_size,
+    );
+
+    let mut transactions: Vec<Arc<Transaction>> = block
+        .vtx
+        .iter()
+        .map(|tx| compact_to_v4(tx).expect("A fake compact tx should be convertible to V4."))
+        .map(Arc::new)
+        .collect();
+
+    let coinbase_input = Input::Coinbase {
+        height: Height(1),
+        data: CoinbaseData::new(vec![]),
+        sequence: u32::MAX,
+    };
+
+    let coinbase = Transaction::V4 {
+        inputs: vec![coinbase_input],
+        outputs: vec![],
+        lock_time: LockTime::Height(Height(1)),
+        expiry_height: Height(1),
+        joinsplit_data: None,
+        sapling_shielded_data: None,
+    };
+
+    transactions.insert(0, Arc::new(coinbase));
+
+    let sapling_tree_size = block
+        .chain_metadata
+        .as_ref()
+        .unwrap()
+        .sapling_commitment_tree_size;
+
+    (
+        Block {
+            header: Arc::new(header),
+            transactions,
+        },
+        sapling_tree_size,
+    )
+}
+
 /// Create a fake compact block with provided fake account data.
 // This is a copy of zcash_primitives `fake_compact_block` where the `value` argument was changed to
 // be a number for easier conversion:
@@ -362,3 +421,85 @@ fn random_compact_tx(mut rng: impl RngCore) -> CompactTx {
     ctx.outputs.push(cout);
     ctx
 }
+
+/// Converts [`CompactTx`] to [`Transaction::V4`].
+fn compact_to_v4(tx: &CompactTx) -> Result<Transaction> {
+    let sk = redjubjub::SigningKey::<redjubjub::SpendAuth>::new(thread_rng());
+    let vk = redjubjub::VerificationKey::from(&sk);
+    let dummy_rk = sapling::keys::ValidatingKey::try_from(vk)
+        .expect("Internally generated verification key should be convertible to a validating key.");
+
+    let spends = tx
+        .spends
+        .iter()
+        .map(|spend| {
+            Ok(Spend {
+                cv: sapling::NotSmallOrderValueCommitment::default(),
+                per_spend_anchor: sapling::tree::Root::default(),
+                nullifier: sapling::Nullifier::from(
+                    spend.nf().map_err(|_| Report::msg("Invalid nullifier."))?.0,
+                ),
+                rk: dummy_rk.clone(),
+                zkproof: Groth16Proof([0; 192]),
+                spend_auth_sig: redjubjub::Signature::<redjubjub::SpendAuth>::from([0; 64]),
+            })
+        })
+        .collect::<Result<Vec<Spend<PerSpendAnchor>>>>()?;
+
+    let spends = AtLeastOne::<Spend<PerSpendAnchor>>::try_from(spends)?;
+
+    let maybe_outputs = tx
+        .outputs
+        .iter()
+        .map(|output| {
+            let mut ciphertext = output.ciphertext.clone();
+            ciphertext.resize(580, 0);
+            let ciphertext: [u8; 580] = ciphertext
+                .try_into()
+                .map_err(|_| Report::msg("Could not convert ciphertext to `[u8; 580]`"))?;
+            let enc_ciphertext = sapling::EncryptedNote::from(ciphertext);
+
+            Ok(sapling::Output {
+                cv: sapling::NotSmallOrderValueCommitment::default(),
+                cm_u: Option::from(jubjub::Fq::from_bytes(
+                    &output
+                        .cmu()
+                        .map_err(|_| Report::msg("Invalid commitment."))?
+                        .to_bytes(),
+                ))
+                .ok_or(Report::msg("Invalid commitment."))?,
+                ephemeral_key: sapling::keys::EphemeralPublicKey::try_from(
+                    output
+                        .ephemeral_key()
+                        .map_err(|_| Report::msg("Invalid ephemeral key."))?
+                        .0,
+                )
+                .map_err(Report::msg)?,
+                enc_ciphertext,
+                out_ciphertext: sapling::WrappedNoteKey::from([0; 80]),
+                zkproof: Groth16Proof([0; 192]),
+            })
+        })
+        .collect::<Result<Vec<sapling::Output>>>()?;
+
+    let transfers = TransferData::SpendsAndMaybeOutputs {
+        shared_anchor: sapling::FieldNotPresent,
+        spends,
+        maybe_outputs,
+    };
+
+    let shielded_data = sapling::ShieldedData {
+        value_balance: Amount::<NegativeAllowed>::default(),
+        transfers,
+        binding_sig: redjubjub::Signature::<redjubjub::Binding>::from([0; 64]),
+    };
+
+    Ok(Transaction::V4 {
+        inputs: vec![],
+        outputs: vec![],
+        lock_time: LockTime::Height(Height(0)),
+        expiry_height: Height(0),
+        joinsplit_data: None,
+        sapling_shielded_data: (Some(shielded_data)),
+    })
+}