diff --git a/book/src/dev/rfcs/0005-state-updates.md b/book/src/dev/rfcs/0005-state-updates.md index b073666a2..e169a33d5 100644 --- a/book/src/dev/rfcs/0005-state-updates.md +++ b/book/src/dev/rfcs/0005-state-updates.md @@ -280,6 +280,7 @@ struct Chain { sapling_anchors: HashSet, sprout_nullifiers: HashSet, sapling_nullifiers: HashSet, + orchard_nullifiers: HashSet, partial_cumulative_work: PartialCumulativeWork, } ``` @@ -608,6 +609,7 @@ We use the following rocksdb column families: | `utxo_by_outpoint` | `OutPoint` | `TransparentOutput` | | `sprout_nullifiers` | `sprout::Nullifier` | `()` | | `sapling_nullifiers` | `sapling::Nullifier` | `()` | +| `orchard_nullifiers` | `orchard::Nullifier` | `()` | | `sprout_anchors` | `sprout::tree::Root` | `()` | | `sapling_anchors` | `sapling::tree::Root` | `()` | @@ -694,6 +696,9 @@ check that `block`'s parent hash is `null` (all zeroes) and its height is `0`. 5. For each [`Spend`] description in the transaction, insert `(nullifier,())` into `sapling_nullifiers`. + 6. For each [`Action`] description in the transaction, insert + `(nullifier,())` into `orchard_nullifiers`. + **Note**: The Sprout and Sapling anchors are the roots of the Sprout and Sapling note commitment trees that have already been calculated for the last transaction(s) in the block that have `JoinSplit`s in the Sprout case and/or @@ -708,8 +713,9 @@ irrelevant for the mainnet and testnet chains. Hypothetically, if Sapling were activated from genesis, the specification requires a Sapling anchor, but `zcashd` would ignore that anchor. -[`JoinSplit`]: https://doc.zebra.zfnd.org/zebra_chain/transaction/struct.JoinSplit.html -[`Spend`]: https://doc.zebra.zfnd.org/zebra_chain/transaction/struct.Spend.html +[`JoinSplit`]: https://doc.zebra.zfnd.org/zebra_chain/sprout/struct.JoinSplit.html +[`Spend`]: https://doc.zebra.zfnd.org/zebra_chain/sapling/spend/struct.Spend.html +[`Action`]: https://doc.zebra.zfnd.org/zebra_chain/orchard/struct.Action.html These updates can be performed in a batch or without necessarily iterating over all transactions, if the data is available by other means; they're diff --git a/zebra-chain/src/orchard/note/nullifiers.rs b/zebra-chain/src/orchard/note/nullifiers.rs index 0ef180a3d..897ce6482 100644 --- a/zebra-chain/src/orchard/note/nullifiers.rs +++ b/zebra-chain/src/orchard/note/nullifiers.rs @@ -9,6 +9,8 @@ use super::super::{ commitment::NoteCommitment, keys::NullifierDerivingKey, note::Note, sinsemilla::*, }; +use std::hash::{Hash, Hasher}; + /// A cryptographic permutation, defined in [poseidonhash]. /// /// PoseidonHash(x, y) = f([x, y, 0])_1 (using 1-based indexing). @@ -35,15 +37,27 @@ fn prf_nf(nk: pallas::Base, rho: pallas::Base) -> pallas::Base { } /// A Nullifier for Orchard transactions -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, Serialize, Deserialize)] pub struct Nullifier(#[serde(with = "serde_helpers::Base")] pallas::Base); +impl Hash for Nullifier { + fn hash(&self, state: &mut H) { + self.0.to_bytes().hash(state); + } +} + impl From<[u8; 32]> for Nullifier { fn from(bytes: [u8; 32]) -> Self { Self(pallas::Base::from_bytes(&bytes).unwrap()) } } +impl PartialEq for Nullifier { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + impl From<(NullifierDerivingKey, Note, NoteCommitment)> for Nullifier { /// Derive a `Nullifier` for an Orchard _note_. /// diff --git a/zebra-chain/src/transaction.rs b/zebra-chain/src/transaction.rs index 212867312..ee28f655a 100644 --- a/zebra-chain/src/transaction.rs +++ b/zebra-chain/src/transaction.rs @@ -393,5 +393,28 @@ impl Transaction { } } - // TODO: orchard + // orchard + + /// Access the orchard::Nullifiers in this transaction, regardless of version. + pub fn orchard_nullifiers(&self) -> Box + '_> { + // This function returns a boxed iterator because the different + // transaction variants can have different iterator types + match self { + // Actions + Transaction::V5 { + orchard_shielded_data: Some(orchard_shielded_data), + .. + } => Box::new(orchard_shielded_data.nullifiers()), + + // No Actions + Transaction::V1 { .. } + | Transaction::V2 { .. } + | Transaction::V3 { .. } + | Transaction::V4 { .. } + | Transaction::V5 { + orchard_shielded_data: None, + .. + } => Box::new(std::iter::empty()), + } + } } diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs index e51ac559a..99126dcec 100644 --- a/zebra-state/src/constants.rs +++ b/zebra-state/src/constants.rs @@ -14,7 +14,7 @@ pub const MIN_TRANSPARENT_COINBASE_MATURITY: u32 = 100; pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1; /// The database format version, incremented each time the database format changes. -pub const DATABASE_FORMAT_VERSION: u32 = 4; +pub const DATABASE_FORMAT_VERSION: u32 = 5; use lazy_static::lazy_static; use regex::Regex; diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index a418608b3..8de272769 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -2,6 +2,9 @@ mod disk_format; +#[cfg(test)] +mod tests; + use std::{collections::HashMap, convert::TryInto, sync::Arc}; use zebra_chain::transparent; @@ -44,6 +47,7 @@ impl FinalizedState { rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("sapling_nullifiers", db_options.clone()), + rocksdb::ColumnFamilyDescriptor::new("orchard_nullifiers", db_options.clone()), ]; let db_result = rocksdb::DB::open_cf_descriptors(&db_options, &path, column_families); @@ -194,6 +198,7 @@ impl FinalizedState { let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap(); let sprout_nullifiers = self.db.cf_handle("sprout_nullifiers").unwrap(); let sapling_nullifiers = self.db.cf_handle("sapling_nullifiers").unwrap(); + let orchard_nullifiers = self.db.cf_handle("orchard_nullifiers").unwrap(); // Assert that callers (including unit tests) get the chain order correct if self.is_empty(hash_by_height) { @@ -273,13 +278,16 @@ impl FinalizedState { } } - // Mark sprout and sapling nullifiers as spent + // Mark sprout, sapling and orchard nullifiers as spent for sprout_nullifier in transaction.sprout_nullifiers() { batch.zs_insert(sprout_nullifiers, sprout_nullifier, ()); } for sapling_nullifier in transaction.sapling_nullifiers() { batch.zs_insert(sapling_nullifiers, sapling_nullifier, ()); } + for orchard_nullifier in transaction.orchard_nullifiers() { + batch.zs_insert(orchard_nullifiers, orchard_nullifier, ()); + } } batch @@ -431,6 +439,12 @@ fn block_precommit_metrics(finalized: &FinalizedBlock) { .flat_map(|t| t.sapling_nullifiers()) .count(); + let orchard_nullifier_count = block + .transactions + .iter() + .flat_map(|t| t.orchard_nullifiers()) + .count(); + tracing::debug!( ?hash, ?height, @@ -439,6 +453,7 @@ fn block_precommit_metrics(finalized: &FinalizedBlock) { transparent_newout_count, sprout_nullifier_count, sapling_nullifier_count, + orchard_nullifier_count, "preparing to commit finalized block" ); metrics::counter!( @@ -461,4 +476,8 @@ fn block_precommit_metrics(finalized: &FinalizedBlock) { "state.finalized.cumulative.sapling_nullifiers", sapling_nullifier_count as u64 ); + metrics::counter!( + "state.finalized.cumulative.orchard_nullifiers", + orchard_nullifier_count as u64 + ); } diff --git a/zebra-state/src/service/finalized_state/disk_format.rs b/zebra-state/src/service/finalized_state/disk_format.rs index 92ec14abf..31546de48 100644 --- a/zebra-state/src/service/finalized_state/disk_format.rs +++ b/zebra-state/src/service/finalized_state/disk_format.rs @@ -4,7 +4,7 @@ use std::{convert::TryInto, fmt::Debug, sync::Arc}; use zebra_chain::{ block, block::Block, - sapling, + orchard, sapling, serialization::{ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize}, sprout, transaction, transparent, }; @@ -163,6 +163,15 @@ impl IntoDisk for sapling::Nullifier { } } +impl IntoDisk for orchard::Nullifier { + type Bytes = [u8; 32]; + + fn as_bytes(&self) -> Self::Bytes { + let nullifier: orchard::Nullifier = *self; + nullifier.into() + } +} + impl IntoDisk for () { type Bytes = [u8; 0]; diff --git a/zebra-state/src/service/finalized_state/tests.rs b/zebra-state/src/service/finalized_state/tests.rs new file mode 100644 index 000000000..2bf82ef4e --- /dev/null +++ b/zebra-state/src/service/finalized_state/tests.rs @@ -0,0 +1 @@ +mod prop; diff --git a/zebra-state/src/service/finalized_state/tests/prop.rs b/zebra-state/src/service/finalized_state/tests/prop.rs new file mode 100644 index 000000000..232a90a54 --- /dev/null +++ b/zebra-state/src/service/finalized_state/tests/prop.rs @@ -0,0 +1,37 @@ +use std::env; + +use zebra_chain::block::Height; +use zebra_test::prelude::*; + +use crate::{ + config::Config, + service::{ + finalized_state::{FinalizedBlock, FinalizedState}, + non_finalized_state::arbitrary::PreparedChain, + }, +}; + +const DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES: u32 = 32; + +#[test] +fn blocks_with_v5_transactions() -> Result<()> { + zebra_test::init(); + proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + |((chain, count, network) in PreparedChain::default())| { + let mut state = FinalizedState::new(&Config::ephemeral(), network); + let mut height = Height(0); + // use `count` to minimize test failures, so they are easier to diagnose + for block in chain.iter().take(count) { + let hash = state.commit_finalized_direct(FinalizedBlock::from(block.clone())); + prop_assert_eq!(Some(height), state.finalized_tip_height()); + prop_assert_eq!(hash.unwrap(), block.hash); + // TODO: check that the nullifiers were correctly inserted (#2230) + height = Height(height.0 + 1); + } + }); + + Ok(()) +} diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index 021a1e66f..5e695b067 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -6,7 +6,7 @@ mod chain; mod queued_blocks; #[cfg(any(test, feature = "proptest-impl"))] -mod arbitrary; +pub mod arbitrary; #[cfg(test)] mod tests; diff --git a/zebra-state/src/service/non_finalized_state/arbitrary.rs b/zebra-state/src/service/non_finalized_state/arbitrary.rs index 949c3cf8d..eb16d1558 100644 --- a/zebra-state/src/service/non_finalized_state/arbitrary.rs +++ b/zebra-state/src/service/non_finalized_state/arbitrary.rs @@ -5,7 +5,7 @@ use proptest::{ }; use std::sync::Arc; -use zebra_chain::{block::Block, LedgerState}; +use zebra_chain::{block::Block, parameters::NetworkUpgrade::Nu5, LedgerState}; use zebra_test::prelude::*; use crate::tests::Prepare; @@ -54,9 +54,8 @@ impl Strategy for PreparedChain { fn new_tree(&self, runner: &mut TestRunner) -> NewTree { let mut chain = self.chain.lock().unwrap(); if chain.is_none() { - // Disable NU5 for now - // `genesis_strategy(None)` re-enables the default Nu5 override - let ledger_strategy = LedgerState::genesis_strategy(Canopy); + // TODO: use the latest network upgrade (#1974) + let ledger_strategy = LedgerState::genesis_strategy(Nu5); let (network, blocks) = ledger_strategy .prop_flat_map(|ledger| { diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs index 852775c66..abe1f6374 100644 --- a/zebra-state/src/service/non_finalized_state/chain.rs +++ b/zebra-state/src/service/non_finalized_state/chain.rs @@ -6,8 +6,8 @@ use std::{ use tracing::{debug_span, instrument, trace}; use zebra_chain::{ - block, primitives::Groth16Proof, sapling, sprout, transaction, transparent, - work::difficulty::PartialCumulativeWork, + block, orchard, primitives::Groth16Proof, sapling, sprout, transaction, + transaction::Transaction::*, transparent, work::difficulty::PartialCumulativeWork, }; use crate::{PreparedBlock, Utxo}; @@ -20,10 +20,12 @@ pub struct Chain { pub created_utxos: HashMap, spent_utxos: HashSet, + // TODO: add sprout, sapling and orchard anchors (#1320) sprout_anchors: HashSet, sapling_anchors: HashSet, sprout_nullifiers: HashSet, sapling_nullifiers: HashSet, + orchard_nullifiers: HashSet, partial_cumulative_work: PartialCumulativeWork, } @@ -165,17 +167,33 @@ impl UpdateWith for Chain { .zip(transaction_hashes.iter().cloned()) .enumerate() { - use transaction::Transaction::*; - let (inputs, joinsplit_data, sapling_shielded_data) = match transaction.deref() { + let ( + inputs, + joinsplit_data, + sapling_shielded_data_per_spend_anchor, + sapling_shielded_data_shared_anchor, + orchard_shielded_data, + ) = match transaction.deref() { V4 { inputs, joinsplit_data, sapling_shielded_data, .. - } => (inputs, joinsplit_data, sapling_shielded_data), - V5 { .. } => unimplemented!("v5 transaction format as specified in ZIP-225"), + } => (inputs, joinsplit_data, sapling_shielded_data, &None, &None), + V5 { + inputs, + sapling_shielded_data, + orchard_shielded_data, + .. + } => ( + inputs, + &None, + &None, + sapling_shielded_data, + orchard_shielded_data, + ), V1 { .. } | V2 { .. } | V3 { .. } => unreachable!( - "older transaction versions only exist in finalized blocks pre sapling", + "older transaction versions only exist in finalized blocks, because of the mandatory canopy checkpoint", ), }; @@ -192,10 +210,12 @@ impl UpdateWith for Chain { self.update_chain_state_with(&prepared.new_outputs); // add the utxos this consumed self.update_chain_state_with(inputs); - // add sprout anchor and nullifiers + + // add the shielded data self.update_chain_state_with(joinsplit_data); - // add sapling anchor and nullifier - self.update_chain_state_with(sapling_shielded_data); + self.update_chain_state_with(sapling_shielded_data_per_spend_anchor); + self.update_chain_state_with(sapling_shielded_data_shared_anchor); + self.update_chain_state_with(orchard_shielded_data); } } @@ -225,17 +245,33 @@ impl UpdateWith for Chain { for (transaction, transaction_hash) in block.transactions.iter().zip(transaction_hashes.iter()) { - use transaction::Transaction::*; - let (inputs, joinsplit_data, sapling_shielded_data) = match transaction.deref() { + let ( + inputs, + joinsplit_data, + sapling_shielded_data_per_spend_anchor, + sapling_shielded_data_shared_anchor, + orchard_shielded_data, + ) = match transaction.deref() { V4 { inputs, joinsplit_data, sapling_shielded_data, .. - } => (inputs, joinsplit_data, sapling_shielded_data), - V5 { .. } => unimplemented!("v5 transaction format as specified in ZIP-225"), + } => (inputs, joinsplit_data, sapling_shielded_data, &None, &None), + V5 { + inputs, + sapling_shielded_data, + orchard_shielded_data, + .. + } => ( + inputs, + &None, + &None, + sapling_shielded_data, + orchard_shielded_data, + ), V1 { .. } | V2 { .. } | V3 { .. } => unreachable!( - "older transaction versions only exist in finalized blocks pre sapling", + "older transaction versions only exist in finalized blocks, because of the mandatory canopy checkpoint", ), }; @@ -249,10 +285,12 @@ impl UpdateWith for Chain { self.revert_chain_state_with(&prepared.new_outputs); // remove the utxos this consumed self.revert_chain_state_with(inputs); - // remove sprout anchor and nullifiers + + // remove the shielded data self.revert_chain_state_with(joinsplit_data); - // remove sapling anchor and nullfier - self.revert_chain_state_with(sapling_shielded_data); + self.revert_chain_state_with(sapling_shielded_data_per_spend_anchor); + self.revert_chain_state_with(sapling_shielded_data_shared_anchor); + self.revert_chain_state_with(orchard_shielded_data); } } } @@ -366,6 +404,27 @@ where } } +impl UpdateWith> for Chain { + fn update_chain_state_with(&mut self, orchard_shielded_data: &Option) { + if let Some(orchard_shielded_data) = orchard_shielded_data { + for nullifier in orchard_shielded_data.nullifiers() { + self.orchard_nullifiers.insert(*nullifier); + } + } + } + + fn revert_chain_state_with(&mut self, orchard_shielded_data: &Option) { + if let Some(orchard_shielded_data) = orchard_shielded_data { + for nullifier in orchard_shielded_data.nullifiers() { + assert!( + self.orchard_nullifiers.remove(nullifier), + "nullifier must be present if block was" + ); + } + } + } +} + impl PartialEq for Chain { fn eq(&self, other: &Self) -> bool { self.partial_cmp(other) == Some(Ordering::Equal)