Store orchard nullifiers into the state (#2185)

* add nullifier methods to orchard
* store orchard nullifiers
* bump database version
* update `IntoDisk`
* support V5 in `UpdateWith`
* add a test for finalized state
* Use the latest network upgrade in state proptests

book/src/dev/rfcs/0005-state-updates.md

@@ -280,6 +280,7 @@ struct Chain {
     sapling_anchors: HashSet<sapling::tree::Root>,
     sprout_nullifiers: HashSet<sprout::Nullifier>,
     sapling_nullifiers: HashSet<sapling::Nullifier>,
+    orchard_nullifiers: HashSet<orchard::Nullifier>,
     partial_cumulative_work: PartialCumulativeWork,
 }
 ```
@@ -608,6 +609,7 @@ We use the following rocksdb column families:
 | `utxo_by_outpoint`   | `OutPoint`            | `TransparentOutput`                 |
 | `sprout_nullifiers`  | `sprout::Nullifier`   | `()`                                |
 | `sapling_nullifiers` | `sapling::Nullifier`  | `()`                                |
+| `orchard_nullifiers` | `orchard::Nullifier`  | `()`                                |
 | `sprout_anchors`     | `sprout::tree::Root`  | `()`                                |
 | `sapling_anchors`    | `sapling::tree::Root` | `()`                                |
 
@@ -694,6 +696,9 @@ check that `block`'s parent hash is `null` (all zeroes) and its height is `0`.
    5. For each [`Spend`] description in the transaction, insert
    `(nullifier,())` into `sapling_nullifiers`.
 
+   6. For each [`Action`] description in the transaction, insert
+   `(nullifier,())` into `orchard_nullifiers`.
+
 **Note**: The Sprout and Sapling anchors are the roots of the Sprout and
 Sapling note commitment trees that have already been calculated for the last
 transaction(s) in the block that have `JoinSplit`s in the Sprout case and/or
@@ -708,8 +713,9 @@ irrelevant for the mainnet and testnet chains.
 Hypothetically, if Sapling were activated from genesis, the specification requires
 a Sapling anchor, but `zcashd` would ignore that anchor.
 
-[`JoinSplit`]: https://doc.zebra.zfnd.org/zebra_chain/transaction/struct.JoinSplit.html
+[`JoinSplit`]: https://doc.zebra.zfnd.org/zebra_chain/sprout/struct.JoinSplit.html
-[`Spend`]: https://doc.zebra.zfnd.org/zebra_chain/transaction/struct.Spend.html
+[`Spend`]: https://doc.zebra.zfnd.org/zebra_chain/sapling/spend/struct.Spend.html
+[`Action`]: https://doc.zebra.zfnd.org/zebra_chain/orchard/struct.Action.html
 
 These updates can be performed in a batch or without necessarily iterating
 over all transactions, if the data is available by other means; they're

zebra-chain/src/orchard/note/nullifiers.rs

@@ -9,6 +9,8 @@ use super::super::{
     commitment::NoteCommitment, keys::NullifierDerivingKey, note::Note, sinsemilla::*,
 };
 
+use std::hash::{Hash, Hasher};
+
 /// A cryptographic permutation, defined in [poseidonhash].
 ///
 /// PoseidonHash(x, y) = f([x, y, 0])_1 (using 1-based indexing).
@@ -35,15 +37,27 @@ fn prf_nf(nk: pallas::Base, rho: pallas::Base) -> pallas::Base {
 }
 
 /// A Nullifier for Orchard transactions
-#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, Eq, Serialize, Deserialize)]
 pub struct Nullifier(#[serde(with = "serde_helpers::Base")] pallas::Base);
 
+impl Hash for Nullifier {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.0.to_bytes().hash(state);
+    }
+}
+
 impl From<[u8; 32]> for Nullifier {
     fn from(bytes: [u8; 32]) -> Self {
         Self(pallas::Base::from_bytes(&bytes).unwrap())
     }
 }
 
+impl PartialEq for Nullifier {
+    fn eq(&self, other: &Self) -> bool {
+        self.0 == other.0
+    }
+}
+
 impl From<(NullifierDerivingKey, Note, NoteCommitment)> for Nullifier {
     /// Derive a `Nullifier` for an Orchard _note_.
     ///

zebra-chain/src/transaction.rs

@@ -393,5 +393,28 @@ impl Transaction {
         }
     }
 
-    // TODO: orchard
+    // orchard
+
+    /// Access the orchard::Nullifiers in this transaction, regardless of version.
+    pub fn orchard_nullifiers(&self) -> Box<dyn Iterator<Item = &orchard::Nullifier> + '_> {
+        // This function returns a boxed iterator because the different
+        // transaction variants can have different iterator types
+        match self {
+            // Actions
+            Transaction::V5 {
+                orchard_shielded_data: Some(orchard_shielded_data),
+                ..
+            } => Box::new(orchard_shielded_data.nullifiers()),
+
+            // No Actions
+            Transaction::V1 { .. }
+            | Transaction::V2 { .. }
+            | Transaction::V3 { .. }
+            | Transaction::V4 { .. }
+            | Transaction::V5 {
+                orchard_shielded_data: None,
+                ..
+            } => Box::new(std::iter::empty()),
+        }
+    }
 }

zebra-state/src/constants.rs

@@ -14,7 +14,7 @@ pub const MIN_TRANSPARENT_COINBASE_MATURITY: u32 = 100;
 pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
 
 /// The database format version, incremented each time the database format changes.
-pub const DATABASE_FORMAT_VERSION: u32 = 4;
+pub const DATABASE_FORMAT_VERSION: u32 = 5;
 
 use lazy_static::lazy_static;
 use regex::Regex;

zebra-state/src/service/finalized_state.rs

@@ -2,6 +2,9 @@
 
 mod disk_format;
 
+#[cfg(test)]
+mod tests;
+
 use std::{collections::HashMap, convert::TryInto, sync::Arc};
 
 use zebra_chain::transparent;
@@ -44,6 +47,7 @@ impl FinalizedState {
             rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new("sapling_nullifiers", db_options.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("orchard_nullifiers", db_options.clone()),
         ];
         let db_result = rocksdb::DB::open_cf_descriptors(&db_options, &path, column_families);
 
@@ -194,6 +198,7 @@ impl FinalizedState {
         let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap();
         let sprout_nullifiers = self.db.cf_handle("sprout_nullifiers").unwrap();
         let sapling_nullifiers = self.db.cf_handle("sapling_nullifiers").unwrap();
+        let orchard_nullifiers = self.db.cf_handle("orchard_nullifiers").unwrap();
 
         // Assert that callers (including unit tests) get the chain order correct
         if self.is_empty(hash_by_height) {
@@ -273,13 +278,16 @@ impl FinalizedState {
                     }
                 }
 
-                // Mark sprout and sapling nullifiers as spent
+                // Mark sprout, sapling and orchard nullifiers as spent
                 for sprout_nullifier in transaction.sprout_nullifiers() {
                     batch.zs_insert(sprout_nullifiers, sprout_nullifier, ());
                 }
                 for sapling_nullifier in transaction.sapling_nullifiers() {
                     batch.zs_insert(sapling_nullifiers, sapling_nullifier, ());
                 }
+                for orchard_nullifier in transaction.orchard_nullifiers() {
+                    batch.zs_insert(orchard_nullifiers, orchard_nullifier, ());
+                }
             }
 
             batch
@@ -431,6 +439,12 @@ fn block_precommit_metrics(finalized: &FinalizedBlock) {
         .flat_map(|t| t.sapling_nullifiers())
         .count();
 
+    let orchard_nullifier_count = block
+        .transactions
+        .iter()
+        .flat_map(|t| t.orchard_nullifiers())
+        .count();
+
     tracing::debug!(
         ?hash,
         ?height,
@@ -439,6 +453,7 @@ fn block_precommit_metrics(finalized: &FinalizedBlock) {
         transparent_newout_count,
         sprout_nullifier_count,
         sapling_nullifier_count,
+        orchard_nullifier_count,
         "preparing to commit finalized block"
     );
     metrics::counter!(
@@ -461,4 +476,8 @@ fn block_precommit_metrics(finalized: &FinalizedBlock) {
         "state.finalized.cumulative.sapling_nullifiers",
         sapling_nullifier_count as u64
     );
+    metrics::counter!(
+        "state.finalized.cumulative.orchard_nullifiers",
+        orchard_nullifier_count as u64
+    );
 }

zebra-state/src/service/finalized_state/disk_format.rs

@@ -4,7 +4,7 @@ use std::{convert::TryInto, fmt::Debug, sync::Arc};
 use zebra_chain::{
     block,
     block::Block,
-    sapling,
+    orchard, sapling,
     serialization::{ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize},
     sprout, transaction, transparent,
 };
@@ -163,6 +163,15 @@ impl IntoDisk for sapling::Nullifier {
     }
 }
 
+impl IntoDisk for orchard::Nullifier {
+    type Bytes = [u8; 32];
+
+    fn as_bytes(&self) -> Self::Bytes {
+        let nullifier: orchard::Nullifier = *self;
+        nullifier.into()
+    }
+}
+
 impl IntoDisk for () {
     type Bytes = [u8; 0];
 

zebra-state/src/service/finalized_state/tests.rs

@@ -0,0 +1 @@
+mod prop;

zebra-state/src/service/finalized_state/tests/prop.rs

@@ -0,0 +1,37 @@
+use std::env;
+
+use zebra_chain::block::Height;
+use zebra_test::prelude::*;
+
+use crate::{
+    config::Config,
+    service::{
+        finalized_state::{FinalizedBlock, FinalizedState},
+        non_finalized_state::arbitrary::PreparedChain,
+    },
+};
+
+const DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES: u32 = 32;
+
+#[test]
+fn blocks_with_v5_transactions() -> Result<()> {
+    zebra_test::init();
+    proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)),
+        |((chain, count, network) in PreparedChain::default())| {
+            let mut state = FinalizedState::new(&Config::ephemeral(), network);
+            let mut height = Height(0);
+            // use `count` to minimize test failures, so they are easier to diagnose
+            for block in chain.iter().take(count) {
+                let hash = state.commit_finalized_direct(FinalizedBlock::from(block.clone()));
+                prop_assert_eq!(Some(height), state.finalized_tip_height());
+                prop_assert_eq!(hash.unwrap(), block.hash);
+                // TODO: check that the nullifiers were correctly inserted (#2230)
+                height = Height(height.0 + 1);
+            }
+    });
+
+    Ok(())
+}

zebra-state/src/service/non_finalized_state.rs

@@ -6,7 +6,7 @@ mod chain;
 mod queued_blocks;
 
 #[cfg(any(test, feature = "proptest-impl"))]
-mod arbitrary;
+pub mod arbitrary;
 #[cfg(test)]
 mod tests;
 

zebra-state/src/service/non_finalized_state/arbitrary.rs

@@ -5,7 +5,7 @@ use proptest::{
 };
 use std::sync::Arc;
 
-use zebra_chain::{block::Block, LedgerState};
+use zebra_chain::{block::Block, parameters::NetworkUpgrade::Nu5, LedgerState};
 use zebra_test::prelude::*;
 
 use crate::tests::Prepare;
@@ -54,9 +54,8 @@ impl Strategy for PreparedChain {
     fn new_tree(&self, runner: &mut TestRunner) -> NewTree<Self> {
         let mut chain = self.chain.lock().unwrap();
         if chain.is_none() {
-            // Disable NU5 for now
+            // TODO: use the latest network upgrade (#1974)
-            // `genesis_strategy(None)` re-enables the default Nu5 override
+            let ledger_strategy = LedgerState::genesis_strategy(Nu5);
-            let ledger_strategy = LedgerState::genesis_strategy(Canopy);
 
             let (network, blocks) = ledger_strategy
                 .prop_flat_map(|ledger| {

zebra-state/src/service/non_finalized_state/chain.rs

@@ -6,8 +6,8 @@ use std::{
 
 use tracing::{debug_span, instrument, trace};
 use zebra_chain::{
-    block, primitives::Groth16Proof, sapling, sprout, transaction, transparent,
+    block, orchard, primitives::Groth16Proof, sapling, sprout, transaction,
-    work::difficulty::PartialCumulativeWork,
+    transaction::Transaction::*, transparent, work::difficulty::PartialCumulativeWork,
 };
 
 use crate::{PreparedBlock, Utxo};
@@ -20,10 +20,12 @@ pub struct Chain {
 
     pub created_utxos: HashMap<transparent::OutPoint, Utxo>,
     spent_utxos: HashSet<transparent::OutPoint>,
+    // TODO: add sprout, sapling and orchard anchors (#1320)
     sprout_anchors: HashSet<sprout::tree::Root>,
     sapling_anchors: HashSet<sapling::tree::Root>,
     sprout_nullifiers: HashSet<sprout::Nullifier>,
     sapling_nullifiers: HashSet<sapling::Nullifier>,
+    orchard_nullifiers: HashSet<orchard::Nullifier>,
     partial_cumulative_work: PartialCumulativeWork,
 }
 
@@ -165,17 +167,33 @@ impl UpdateWith<PreparedBlock> for Chain {
             .zip(transaction_hashes.iter().cloned())
             .enumerate()
         {
-            use transaction::Transaction::*;
+            let (
-            let (inputs, joinsplit_data, sapling_shielded_data) = match transaction.deref() {
+                inputs,
+                joinsplit_data,
+                sapling_shielded_data_per_spend_anchor,
+                sapling_shielded_data_shared_anchor,
+                orchard_shielded_data,
+            ) = match transaction.deref() {
                 V4 {
                     inputs,
                     joinsplit_data,
                     sapling_shielded_data,
                     ..
-                } => (inputs, joinsplit_data, sapling_shielded_data),
+                } => (inputs, joinsplit_data, sapling_shielded_data, &None, &None),
-                V5 { .. } => unimplemented!("v5 transaction format as specified in ZIP-225"),
+                V5 {
+                    inputs,
+                    sapling_shielded_data,
+                    orchard_shielded_data,
+                    ..
+                } => (
+                    inputs,
+                    &None,
+                    &None,
+                    sapling_shielded_data,
+                    orchard_shielded_data,
+                ),
                 V1 { .. } | V2 { .. } | V3 { .. } => unreachable!(
-                    "older transaction versions only exist in finalized blocks pre sapling",
+                    "older transaction versions only exist in finalized blocks, because of the mandatory canopy checkpoint",
                 ),
             };
 
@@ -192,10 +210,12 @@ impl UpdateWith<PreparedBlock> for Chain {
             self.update_chain_state_with(&prepared.new_outputs);
             // add the utxos this consumed
             self.update_chain_state_with(inputs);
-            // add sprout anchor and nullifiers
+
+            // add the shielded data
             self.update_chain_state_with(joinsplit_data);
-            // add sapling anchor and nullifier
+            self.update_chain_state_with(sapling_shielded_data_per_spend_anchor);
-            self.update_chain_state_with(sapling_shielded_data);
+            self.update_chain_state_with(sapling_shielded_data_shared_anchor);
+            self.update_chain_state_with(orchard_shielded_data);
         }
     }
 
@@ -225,17 +245,33 @@ impl UpdateWith<PreparedBlock> for Chain {
         for (transaction, transaction_hash) in
             block.transactions.iter().zip(transaction_hashes.iter())
         {
-            use transaction::Transaction::*;
+            let (
-            let (inputs, joinsplit_data, sapling_shielded_data) = match transaction.deref() {
+                inputs,
+                joinsplit_data,
+                sapling_shielded_data_per_spend_anchor,
+                sapling_shielded_data_shared_anchor,
+                orchard_shielded_data,
+            ) = match transaction.deref() {
                 V4 {
                     inputs,
                     joinsplit_data,
                     sapling_shielded_data,
                     ..
-                } => (inputs, joinsplit_data, sapling_shielded_data),
+                } => (inputs, joinsplit_data, sapling_shielded_data, &None, &None),
-                V5 { .. } => unimplemented!("v5 transaction format as specified in ZIP-225"),
+                V5 {
+                    inputs,
+                    sapling_shielded_data,
+                    orchard_shielded_data,
+                    ..
+                } => (
+                    inputs,
+                    &None,
+                    &None,
+                    sapling_shielded_data,
+                    orchard_shielded_data,
+                ),
                 V1 { .. } | V2 { .. } | V3 { .. } => unreachable!(
-                    "older transaction versions only exist in finalized blocks pre sapling",
+                    "older transaction versions only exist in finalized blocks, because of the mandatory canopy checkpoint",
                 ),
             };
 
@@ -249,10 +285,12 @@ impl UpdateWith<PreparedBlock> for Chain {
             self.revert_chain_state_with(&prepared.new_outputs);
             // remove the utxos this consumed
             self.revert_chain_state_with(inputs);
-            // remove sprout anchor and nullifiers
+
+            // remove the shielded data
             self.revert_chain_state_with(joinsplit_data);
-            // remove sapling anchor and nullfier
+            self.revert_chain_state_with(sapling_shielded_data_per_spend_anchor);
-            self.revert_chain_state_with(sapling_shielded_data);
+            self.revert_chain_state_with(sapling_shielded_data_shared_anchor);
+            self.revert_chain_state_with(orchard_shielded_data);
         }
     }
 }
@@ -366,6 +404,27 @@ where
     }
 }
 
+impl UpdateWith<Option<orchard::ShieldedData>> for Chain {
+    fn update_chain_state_with(&mut self, orchard_shielded_data: &Option<orchard::ShieldedData>) {
+        if let Some(orchard_shielded_data) = orchard_shielded_data {
+            for nullifier in orchard_shielded_data.nullifiers() {
+                self.orchard_nullifiers.insert(*nullifier);
+            }
+        }
+    }
+
+    fn revert_chain_state_with(&mut self, orchard_shielded_data: &Option<orchard::ShieldedData>) {
+        if let Some(orchard_shielded_data) = orchard_shielded_data {
+            for nullifier in orchard_shielded_data.nullifiers() {
+                assert!(
+                    self.orchard_nullifiers.remove(nullifier),
+                    "nullifier must be present if block was"
+                );
+            }
+        }
+    }
+}
+
 impl PartialEq for Chain {
     fn eq(&self, other: &Self) -> bool {
         self.partial_cmp(other) == Some(Ordering::Equal)