From e719c46b1b1b700da7674028bc607d53917ac7a0 Mon Sep 17 00:00:00 2001 From: Deirdre Connolly Date: Thu, 29 Jul 2021 09:37:18 -0400 Subject: [PATCH] Track anchors and note commitment trees in zebra-state (#2458) * Tidy chain Cargo.toml * Organize imports * Add method to get note commitments from all Actions in Orchard shielded data * Add method to get note commitments from all JoinSplits in Sprout JoinSplitData * Add Request and Response variants for awaiting anchors * Add anchors and note commitment trees to finalized state db * Add (From|Into)Disk impls for tree::Roots and stubs for NoteCommitmentTrees * Track anchors and note commitment trees in Chain Append note commitments to their trees when doing update_chain_state_with, then use the resulting Sapling and Orchard roots to pass to history_tree, and add new roots to the anchor sets. * Handle errors when appending to note commitment trees * Add comments explaining why note commitment are not removed from the tree in revert_chain_state_with * Implementing note commitments in finalized state * Finish serialization of Orchard tree; remove old tree when updating finalize state * Add serialization and finalized state updates for Sprout and Sapling trees * Partially handle trees in non-finalized state. Use Option for trees in Chain * Rebuild trees when forking; change finalized state tree getters to not require height * Pass empty trees to tests; use empty trees by default in Chain * Also rebuild anchor sets when forking * Use empty tree as default in finalized state tree getters (for now) * Use HashMultiSet for anchors in order to make pop_root() work correctly * Reduce DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES and MAX_PARTIAL_CHAIN_BLOCKS * Reduce DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES and MAX_PARTIAL_CHAIN_BLOCKS even more * Apply suggestions from code review * Add comments about order of note commitments and related methods/fields * Don't use Option for trees * Set DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES=1 and restore MAX_PARTIAL_CHAIN_BLOCKS * Remove unneeded anchor set rebuilding in fork() * Improve proptest formatting * Add missing comparisons to eq_internal_state * Renamed sprout::tree::NoteCommitmentTree::hash() to root() * Improve comments * Add asserts, add issues to TODOs * Remove impl Default for Chain since it was only used by tests * Improve documentation and assertions; add tree serialization tests * Remove Sprout code, which will be moved to another branch * Add todo! in Sprout tree append() * Remove stub request, response *Anchor* handling for now * Add test for validating Sapling note commitment tree using test blocks * Increase database version (new columns added for note commitment trees and anchors) * Update test to make sure the order of sapling_note_commitments() is being tested * Improve comments and structure of the test * Improve variable names again * Rustfmt Co-authored-by: Deirdre Connolly Co-authored-by: Conrado P. L. Gouvea Co-authored-by: Conrado Gouvea Co-authored-by: teor --- Cargo.lock | 19 ++ zebra-chain/Cargo.toml | 8 +- zebra-chain/src/orchard/shielded_data.rs | 28 +-- zebra-chain/src/orchard/tree.rs | 31 +++- zebra-chain/src/sapling/shielded_data.rs | 13 +- zebra-chain/src/sapling/tests/tree.rs | 94 +++++++++- zebra-chain/src/sapling/tree.rs | 25 ++- zebra-chain/src/sprout/tree.rs | 28 ++- zebra-chain/src/transaction.rs | 40 +++++ zebra-chain/src/transaction/joinsplit.rs | 18 +- zebra-state/Cargo.toml | 8 + zebra-state/src/constants.rs | 2 +- zebra-state/src/error.rs | 6 + zebra-state/src/service/finalized_state.rs | 105 ++++++++++- .../service/finalized_state/disk_format.rs | 73 ++++++++ .../src/service/finalized_state/tests.rs | 1 + .../src/service/finalized_state/tests/prop.rs | 2 +- .../service/finalized_state/tests/vectors.rs | 80 +++++++++ .../src/service/non_finalized_state.rs | 30 +++- .../src/service/non_finalized_state/chain.rs | 167 ++++++++++++++++-- .../service/non_finalized_state/tests/prop.rs | 39 ++-- .../non_finalized_state/tests/vectors.rs | 10 +- 22 files changed, 751 insertions(+), 76 deletions(-) create mode 100644 zebra-state/src/service/finalized_state/tests/vectors.rs diff --git a/Cargo.lock b/Cargo.lock index 52d7d288f..cce202b30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -300,6 +300,15 @@ dependencies = [ "crunchy 0.1.6", ] +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bindgen" version = "0.57.0" @@ -2131,6 +2140,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "multiset" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce8738c9ddd350996cb8b8b718192851df960803764bcdaa3afb44a63b1ddb5c" + [[package]] name = "net2" version = "0.2.36" @@ -4623,15 +4638,19 @@ dependencies = [ name = "zebra-state" version = "1.0.0-alpha.13" dependencies = [ + "bincode", "chrono", "color-eyre", "dirs", "displaydoc", "futures 0.3.15", + "halo2", "hex", "itertools 0.10.1", + "jubjub", "lazy_static", "metrics", + "multiset", "once_cell", "proptest", "proptest-derive", diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index c3ab51564..f3d09f190 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -15,10 +15,12 @@ bench = ["zebra-test"] [dependencies] aes = "0.6" bech32 = "0.8.1" +bigint = "4" bitflags = "1.2.1" bitvec = "0.22" blake2b_simd = "0.5.11" blake2s_simd = "0.5.11" +bls12_381 = "0.5.0" bs58 = { version = "0.4", features = ["check"] } byteorder = "1.4" chrono = { version = "0.4", features = ["serde"] } @@ -30,6 +32,7 @@ group = "0.10" # Note: if updating this, also update the workspace Cargo.toml to match. halo2 = { git = "https://github.com/zcash/halo2.git", rev = "236115917df9db45282fec24d1e1e36f275f71ab" } hex = "0.4" +incrementalmerkletree = "0.1.0" jubjub = "0.7.0" lazy_static = "1.4.0" rand_core = "0.6" @@ -40,13 +43,10 @@ serde-big-array = "0.3.2" sha2 = { version = "0.9.5", features=["compress"] } subtle = "2.4" thiserror = "1" +uint = "0.9.1" x25519-dalek = { version = "1.1", features = ["serde"] } zcash_history = { git = "https://github.com/zcash/librustzcash.git", rev = "0c3ed159985affa774e44d10172d4471d798a85a" } zcash_primitives = { git = "https://github.com/zcash/librustzcash.git", rev = "0c3ed159985affa774e44d10172d4471d798a85a" } -bigint = "4" -uint = "0.9.1" -bls12_381 = "0.5.0" -incrementalmerkletree = "0.1.0" proptest = { version = "0.10", optional = true } proptest-derive = { version = "0.3.0", optional = true } diff --git a/zebra-chain/src/orchard/shielded_data.rs b/zebra-chain/src/orchard/shielded_data.rs index d9eb49e05..0fd447e0e 100644 --- a/zebra-chain/src/orchard/shielded_data.rs +++ b/zebra-chain/src/orchard/shielded_data.rs @@ -1,5 +1,14 @@ //! Orchard shielded data for `V5` `Transaction`s. +use std::{ + cmp::{Eq, PartialEq}, + fmt::Debug, + io, +}; + +use byteorder::{ReadBytesExt, WriteBytesExt}; +use halo2::pasta::pallas; + use crate::{ amount::{Amount, NegativeAllowed}, block::MAX_BLOCK_BYTES, @@ -13,14 +22,6 @@ use crate::{ }, }; -use byteorder::{ReadBytesExt, WriteBytesExt}; - -use std::{ - cmp::{Eq, PartialEq}, - fmt::Debug, - io, -}; - /// A bundle of [`Action`] descriptions and signature data. #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] pub struct ShieldedData { @@ -32,14 +33,15 @@ pub struct ShieldedData { pub shared_anchor: tree::Root, /// The aggregated zk-SNARK proof for all the actions in this transaction. pub proof: Halo2Proof, - /// The Orchard Actions. + /// The Orchard Actions, in the order they appear in the transaction. pub actions: AtLeastOne, /// A signature on the transaction `sighash`. pub binding_sig: Signature, } impl ShieldedData { - /// Iterate over the [`Action`]s for the [`AuthorizedAction`]s in this transaction. + /// Iterate over the [`Action`]s for the [`AuthorizedAction`]s in this + /// transaction, in the order they appear in it. pub fn actions(&self) -> impl Iterator { self.actions.actions() } @@ -55,6 +57,12 @@ impl ShieldedData { pub fn value_balance(&self) -> Amount { self.value_balance } + + /// Collect the cm_x's for this transaction, if it contains [`Action`]s with + /// outputs, in the order they appear in the transaction. + pub fn note_commitments(&self) -> impl Iterator { + self.actions().map(|action| &action.cm_x) + } } impl AtLeastOne { diff --git a/zebra-chain/src/orchard/tree.rs b/zebra-chain/src/orchard/tree.rs index 2df186855..c8d48052e 100644 --- a/zebra-chain/src/orchard/tree.rs +++ b/zebra-chain/src/orchard/tree.rs @@ -111,6 +111,12 @@ impl From for [u8; 32] { } } +impl From<&Root> for [u8; 32] { + fn from(root: &Root) -> Self { + (*root).into() + } +} + impl Hash for Root { fn hash(&self, state: &mut H) { self.0.to_bytes().hash(state) @@ -177,15 +183,36 @@ impl From for Node { } } +impl serde::Serialize for Node { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.0.to_bytes().serialize(serializer) + } +} + +impl<'de> serde::Deserialize<'de> for Node { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bytes = <[u8; 32]>::deserialize(deserializer)?; + Option::::from(pallas::Base::from_bytes(&bytes)) + .map(Node) + .ok_or_else(|| serde::de::Error::custom("invalid Pallas field element")) + } +} + #[allow(dead_code, missing_docs)] -#[derive(Error, Debug, PartialEq)] +#[derive(Error, Debug, PartialEq, Eq)] pub enum NoteCommitmentTreeError { #[error("The note commitment tree is full")] FullTree, } /// Orchard Incremental Note Commitment Tree -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct NoteCommitmentTree { /// The tree represented as a Frontier. /// diff --git a/zebra-chain/src/sapling/shielded_data.rs b/zebra-chain/src/sapling/shielded_data.rs index b32a0cbb0..4a468cf49 100644 --- a/zebra-chain/src/sapling/shielded_data.rs +++ b/zebra-chain/src/sapling/shielded_data.rs @@ -104,7 +104,7 @@ where pub value_balance: Amount, /// A bundle of spends and outputs, containing at least one spend or - /// output. + /// output, in the order they appear in the transaction. /// /// In V5 transactions, also contains a shared anchor, if there are any /// spends. @@ -154,7 +154,8 @@ where /// [`Spend`]s in this `TransferData`. spends: AtLeastOne>, - /// Maybe some outputs (can be empty). + /// Maybe some outputs (can be empty), in the order they appear in the + /// transaction. /// /// Use the [`ShieldedData::outputs`] method to get an iterator over the /// [`Outputs`]s in this `TransferData`. @@ -167,7 +168,7 @@ where /// In Transaction::V5, if there are no spends, there must not be a shared /// anchor. JustOutputs { - /// At least one output. + /// At least one output, in the order they appear in the transaction. /// /// Use the [`ShieldedData::outputs`] method to get an iterator over the /// [`Outputs`]s in this `TransferData`. @@ -205,7 +206,8 @@ where self.transfers.spends() } - /// Iterate over the [`Output`]s for this transaction. + /// Iterate over the [`Output`]s for this transaction, in the order they + /// appear in it. pub fn outputs(&self) -> impl Iterator { self.transfers.outputs() } @@ -225,7 +227,8 @@ where self.spends().map(|spend| &spend.nullifier) } - /// Collect the cm_u's for this transaction, if it contains [`Output`]s. + /// Collect the cm_u's for this transaction, if it contains [`Output`]s, + /// in the order they appear in the transaction. pub fn note_commitments(&self) -> impl Iterator { self.outputs().map(|output| &output.cm_u) } diff --git a/zebra-chain/src/sapling/tests/tree.rs b/zebra-chain/src/sapling/tests/tree.rs index b7708f4a2..afa5f86fb 100644 --- a/zebra-chain/src/sapling/tests/tree.rs +++ b/zebra-chain/src/sapling/tests/tree.rs @@ -1,7 +1,17 @@ +use std::sync::Arc; + +use color_eyre::eyre; +use eyre::Result; use hex::FromHex; -use crate::sapling::tests::test_vectors; -use crate::sapling::tree::*; +use crate::block::Block; +use crate::parameters::NetworkUpgrade; +use crate::sapling::{self, tree::*}; +use crate::serialization::ZcashDeserializeInto; +use crate::{parameters::Network, sapling::tests::test_vectors}; +use zebra_test::vectors::{ + MAINNET_BLOCKS, MAINNET_FINAL_SAPLING_ROOTS, TESTNET_BLOCKS, TESTNET_FINAL_SAPLING_ROOTS, +}; #[test] fn empty_roots() { @@ -41,3 +51,83 @@ fn incremental_roots() { ); } } + +#[test] +fn incremental_roots_with_blocks() -> Result<()> { + incremental_roots_with_blocks_for_network(Network::Mainnet)?; + incremental_roots_with_blocks_for_network(Network::Testnet)?; + Ok(()) +} + +fn incremental_roots_with_blocks_for_network(network: Network) -> Result<()> { + let (blocks, sapling_roots) = match network { + Network::Mainnet => (&*MAINNET_BLOCKS, &*MAINNET_FINAL_SAPLING_ROOTS), + Network::Testnet => (&*TESTNET_BLOCKS, &*TESTNET_FINAL_SAPLING_ROOTS), + }; + let height = NetworkUpgrade::Sapling + .activation_height(network) + .unwrap() + .0; + + // Build empty note commitment tree + let mut tree = sapling::tree::NoteCommitmentTree::default(); + + // Load Sapling activation block + let sapling_activation_block = Arc::new( + blocks + .get(&height) + .expect("test vector exists") + .zcash_deserialize_into::() + .expect("block is structurally valid"), + ); + + // Add note commitments from the Sapling activation block to the tree + for transaction in sapling_activation_block.transactions.iter() { + for sapling_note_commitment in transaction.sapling_note_commitments() { + tree.append(*sapling_note_commitment) + .expect("test vector is correct"); + } + } + + // Check if root of the tree of the activation block is correct + let sapling_activation_block_root = + sapling::tree::Root(**sapling_roots.get(&height).expect("test vector exists")); + assert_eq!(sapling_activation_block_root, tree.root()); + + // Load the block immediately after Sapling activation (activation + 1) + let block_after_sapling_activation = Arc::new( + blocks + .get(&(height + 1)) + .expect("test vector exists") + .zcash_deserialize_into::() + .expect("block is structurally valid"), + ); + let block_after_sapling_activation_root = sapling::tree::Root( + **sapling_roots + .get(&(height + 1)) + .expect("test vector exists"), + ); + + // Add note commitments from the block after Sapling activatoin to the tree + let mut appended_count = 0; + for transaction in block_after_sapling_activation.transactions.iter() { + for sapling_note_commitment in transaction.sapling_note_commitments() { + tree.append(*sapling_note_commitment) + .expect("test vector is correct"); + appended_count += 1; + } + } + // We also want to make sure that sapling_note_commitments() is returning + // the commitments in the right order. But this will only be actually tested + // if there are more than one note commitment in a block. + // In the test vectors this applies only for the block 1 in mainnet, + // so we make this explicit in this assert. + if network == Network::Mainnet { + assert!(appended_count > 1); + } + + // Check if root of the second block is correct + assert_eq!(block_after_sapling_activation_root, tree.root()); + + Ok(()) +} diff --git a/zebra-chain/src/sapling/tree.rs b/zebra-chain/src/sapling/tree.rs index bd029c6ae..4acce7653 100644 --- a/zebra-chain/src/sapling/tree.rs +++ b/zebra-chain/src/sapling/tree.rs @@ -147,15 +147,36 @@ impl From for Node { } } +impl serde::Serialize for Node { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.0.serialize(serializer) + } +} + +impl<'de> serde::Deserialize<'de> for Node { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bytes = <[u8; 32]>::deserialize(deserializer)?; + Option::::from(jubjub::Fq::from_bytes(&bytes)) + .map(Node::from) + .ok_or_else(|| serde::de::Error::custom("invalid JubJub field element")) + } +} + #[allow(dead_code, missing_docs)] -#[derive(Error, Debug, PartialEq)] +#[derive(Error, Debug, PartialEq, Eq)] pub enum NoteCommitmentTreeError { #[error("The note commitment tree is full")] FullTree, } /// Sapling Incremental Note Commitment Tree. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct NoteCommitmentTree { /// The tree represented as a Frontier. /// diff --git a/zebra-chain/src/sprout/tree.rs b/zebra-chain/src/sprout/tree.rs index f8672c737..62dbc2743 100644 --- a/zebra-chain/src/sprout/tree.rs +++ b/zebra-chain/src/sprout/tree.rs @@ -104,10 +104,22 @@ impl From for [u8; 32] { } } +impl From<&[u8; 32]> for Root { + fn from(bytes: &[u8; 32]) -> Root { + (*bytes).into() + } +} + +impl From<&Root> for [u8; 32] { + fn from(root: &Root) -> Self { + (*root).into() + } +} + /// Sprout Note Commitment Tree -#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] -struct NoteCommitmentTree { +pub struct NoteCommitmentTree { /// The root node of the tree (often used as an anchor). root: Root, /// The height of the tree (maximum height for Sprout is 29). @@ -164,8 +176,14 @@ impl From> for NoteCommitmentTree { impl NoteCommitmentTree { /// Get the Jubjub-based Pedersen hash of root node of this merkle tree of /// commitment notes. - pub fn hash(&self) -> [u8; 32] { - self.root.0 + pub fn root(&self) -> Root { + self.root + } + + /// Add a note commitment to the tree. + pub fn append(&mut self, _cm: &NoteCommitment) { + // TODO: https://github.com/ZcashFoundation/zebra/issues/2485 + todo!("implement sprout note commitment trees #2485"); } } @@ -277,7 +295,7 @@ mod tests { let tree = NoteCommitmentTree::from(leaves.clone()); - assert_eq!(hex::encode(tree.hash()), roots[i]); + assert_eq!(hex::encode(<[u8; 32]>::from(tree.root())), roots[i]); } } } diff --git a/zebra-chain/src/transaction.rs b/zebra-chain/src/transaction.rs index 8a86b2945..496ca60bd 100644 --- a/zebra-chain/src/transaction.rs +++ b/zebra-chain/src/transaction.rs @@ -1,5 +1,6 @@ //! Transactions and transaction-related structures. +use halo2::pasta::pallas; use serde::{Deserialize, Serialize}; mod hash; @@ -590,6 +591,36 @@ impl Transaction { } } + /// Access the note commitments in this transaction, regardless of version. + pub fn sapling_note_commitments(&self) -> Box + '_> { + // This function returns a boxed iterator because the different + // transaction variants end up having different iterator types + match self { + // Spends with Groth16 Proofs + Transaction::V4 { + sapling_shielded_data: Some(sapling_shielded_data), + .. + } => Box::new(sapling_shielded_data.note_commitments()), + Transaction::V5 { + sapling_shielded_data: Some(sapling_shielded_data), + .. + } => Box::new(sapling_shielded_data.note_commitments()), + + // No Spends + Transaction::V1 { .. } + | Transaction::V2 { .. } + | Transaction::V3 { .. } + | Transaction::V4 { + sapling_shielded_data: None, + .. + } + | Transaction::V5 { + sapling_shielded_data: None, + .. + } => Box::new(std::iter::empty()), + } + } + /// Return if the transaction has any Sapling shielded data. pub fn has_sapling_shielded_data(&self) -> bool { match self { @@ -643,6 +674,15 @@ impl Transaction { .flatten() } + /// Access the note commitments in this transaction, if there are any, + /// regardless of version. + pub fn orchard_note_commitments(&self) -> impl Iterator { + self.orchard_shielded_data() + .into_iter() + .map(orchard::ShieldedData::note_commitments) + .flatten() + } + /// Access the [`orchard::Flags`] in this transaction, if there is any, /// regardless of version. pub fn orchard_flags(&self) -> Option { diff --git a/zebra-chain/src/transaction/joinsplit.rs b/zebra-chain/src/transaction/joinsplit.rs index 1d583b47c..fec3a8439 100644 --- a/zebra-chain/src/transaction/joinsplit.rs +++ b/zebra-chain/src/transaction/joinsplit.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use crate::{ amount::{Amount, Error}, primitives::{ed25519, ZkSnarkProof}, - sprout::{JoinSplit, Nullifier}, + sprout::{self, JoinSplit, Nullifier}, }; /// A bundle of [`JoinSplit`] descriptions and signature data. @@ -16,7 +16,8 @@ use crate::{ /// JoinSplit data. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct JoinSplitData { - /// The first JoinSplit description, using proofs of type `P`. + /// The first JoinSplit description in the transaction, + /// using proofs of type `P`. /// /// Storing this separately from `rest` ensures that it is impossible /// to construct an invalid `JoinSplitData` with no `JoinSplit`s. @@ -29,7 +30,8 @@ pub struct JoinSplitData { deserialize = "JoinSplit

: Deserialize<'de>" ))] pub first: JoinSplit

, - /// The rest of the JoinSplit descriptions, using proofs of type `P`. + /// The rest of the JoinSplit descriptions, using proofs of type `P`, + /// in the order they appear in the transaction. /// /// The [`JoinSplitData::joinsplits`] method provides an iterator over /// all `JoinSplit`s. @@ -45,7 +47,8 @@ pub struct JoinSplitData { } impl JoinSplitData

{ - /// Iterate over the [`JoinSplit`]s in `self`. + /// Iterate over the [`JoinSplit`]s in `self`, in the order they appear + /// in the transaction. pub fn joinsplits(&self) -> impl Iterator> { std::iter::once(&self.first).chain(self.rest.iter()) } @@ -64,4 +67,11 @@ impl JoinSplitData

{ .flat_map(|j| j.vpub_old.constrain() - j.vpub_new.constrain()?) .sum() } + + /// Collect the Sprout note commitments for this transaction, if it contains [`Output`]s, + /// in the order they appear in the transaction. + pub fn note_commitments(&self) -> impl Iterator { + self.joinsplits() + .flat_map(|joinsplit| &joinsplit.commitments) + } } diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index f14764349..b264ebe86 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -16,6 +16,7 @@ hex = "0.4.3" lazy_static = "1.4.0" regex = "1" serde = { version = "1", features = ["serde_derive"] } +bincode = "1" futures = "0.3.15" metrics = "0.13.0-alpha.8" @@ -28,6 +29,9 @@ rocksdb = "0.16.0" tempdir = "0.3.7" chrono = "0.4.19" rlimit = "0.5.4" +# TODO: this crate is not maintained anymore. Replace it? +# https://github.com/ZcashFoundation/zebra/issues/2523 +multiset = "0.0.5" proptest = { version = "0.10.1", optional = true } zebra-test = { path = "../zebra-test/", optional = true } @@ -42,6 +46,10 @@ itertools = "0.10.1" spandoc = "0.2" tempdir = "0.3.7" tokio = { version = "0.3.6", features = ["full"] } +# TODO: replace w/ crate version when released: https://github.com/ZcashFoundation/zebra/issues/2083 +# Note: if updating this, also update the workspace Cargo.toml to match. +halo2 = { git = "https://github.com/zcash/halo2.git", rev = "236115917df9db45282fec24d1e1e36f275f71ab" } +jubjub = "0.7.0" proptest = "0.10.1" proptest-derive = "0.3" diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs index b915374ec..e4e2bb1b2 100644 --- a/zebra-state/src/constants.rs +++ b/zebra-state/src/constants.rs @@ -25,7 +25,7 @@ pub const MIN_TRANSPARENT_COINBASE_MATURITY: u32 = 100; pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1; /// The database format version, incremented each time the database format changes. -pub const DATABASE_FORMAT_VERSION: u32 = 5; +pub const DATABASE_FORMAT_VERSION: u32 = 6; use lazy_static::lazy_static; use regex::Regex; diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index afa799d92..4127a7779 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -146,6 +146,12 @@ pub enum ValidateContextError { transaction_hash: zebra_chain::transaction::Hash, in_finalized_state: bool, }, + + #[error("error in Sapling note commitment tree")] + SaplingNoteCommitmentTreeError(#[from] zebra_chain::sapling::tree::NoteCommitmentTreeError), + + #[error("error in Orchard note commitment tree")] + OrchardNoteCommitmentTreeError(#[from] zebra_chain::orchard::tree::NoteCommitmentTreeError), } /// Trait for creating the corresponding duplicate nullifier error from a nullifier. diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index bd2076043..0736ddd0d 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -50,6 +50,16 @@ impl FinalizedState { rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("sapling_nullifiers", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("orchard_nullifiers", db_options.clone()), + rocksdb::ColumnFamilyDescriptor::new("sapling_anchors", db_options.clone()), + rocksdb::ColumnFamilyDescriptor::new("orchard_anchors", db_options.clone()), + rocksdb::ColumnFamilyDescriptor::new( + "sapling_note_commitment_tree", + db_options.clone(), + ), + rocksdb::ColumnFamilyDescriptor::new( + "orchard_note_commitment_tree", + db_options.clone(), + ), ]; let db_result = rocksdb::DB::open_cf_descriptors(&db_options, &path, column_families); @@ -196,15 +206,26 @@ impl FinalizedState { transaction_hashes, } = finalized; + let finalized_tip_height = self.finalized_tip_height(); + let hash_by_height = self.db.cf_handle("hash_by_height").unwrap(); let height_by_hash = self.db.cf_handle("height_by_hash").unwrap(); let block_by_height = self.db.cf_handle("block_by_height").unwrap(); let tx_by_hash = self.db.cf_handle("tx_by_hash").unwrap(); let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap(); + let sprout_nullifiers = self.db.cf_handle("sprout_nullifiers").unwrap(); let sapling_nullifiers = self.db.cf_handle("sapling_nullifiers").unwrap(); let orchard_nullifiers = self.db.cf_handle("orchard_nullifiers").unwrap(); + let sapling_anchors = self.db.cf_handle("sapling_anchors").unwrap(); + let orchard_anchors = self.db.cf_handle("orchard_anchors").unwrap(); + + let sapling_note_commitment_tree_cf = + self.db.cf_handle("sapling_note_commitment_tree").unwrap(); + let orchard_note_commitment_tree_cf = + self.db.cf_handle("orchard_note_commitment_tree").unwrap(); + // Assert that callers (including unit tests) get the chain order correct if self.is_empty(hash_by_height) { assert_eq!( @@ -220,9 +241,7 @@ impl FinalizedState { ); } else { assert_eq!( - self.finalized_tip_height() - .expect("state must have a genesis block committed") - + 1, + finalized_tip_height.expect("state must have a genesis block committed") + 1, Some(height), "committed block height must be 1 more than the finalized tip height, source: {}", source, @@ -236,6 +255,11 @@ impl FinalizedState { ); } + // Read the current note commitment trees. If there are no blocks in the + // state, these will contain the empty trees. + let mut sapling_note_commitment_tree = self.sapling_note_commitment_tree(); + let mut orchard_note_commitment_tree = self.orchard_note_commitment_tree(); + // We use a closure so we can use an early return for control flow in // the genesis case let prepare_commit = || -> rocksdb::WriteBatch { @@ -246,12 +270,24 @@ impl FinalizedState { batch.zs_insert(height_by_hash, hash, height); batch.zs_insert(block_by_height, height, &block); - // TODO: sprout and sapling anchors (per block) - // "A transaction MUST NOT spend an output of the genesis block coinbase transaction. // (There is one such zero-valued output, on each of Testnet and Mainnet .)" // https://zips.z.cash/protocol/protocol.pdf#txnconsensus if block.header.previous_block_hash == GENESIS_PREVIOUS_BLOCK_HASH { + // Insert empty note commitment trees. Note that these can't be + // used too early (e.g. the Orchard tree before Nu5 activates) + // since the block validation will make sure only appropriate + // transactions are allowed in a block. + batch.zs_insert( + sapling_note_commitment_tree_cf, + height, + sapling_note_commitment_tree, + ); + batch.zs_insert( + orchard_note_commitment_tree_cf, + height, + orchard_note_commitment_tree, + ); return batch; } @@ -297,8 +333,39 @@ impl FinalizedState { for orchard_nullifier in transaction.orchard_nullifiers() { batch.zs_insert(orchard_nullifiers, orchard_nullifier, ()); } + + for sapling_note_commitment in transaction.sapling_note_commitments() { + sapling_note_commitment_tree + .append(*sapling_note_commitment) + .expect("must work since it was already appended before in the non-finalized state"); + } + for orchard_note_commitment in transaction.orchard_note_commitments() { + orchard_note_commitment_tree + .append(*orchard_note_commitment) + .expect("must work since it was already appended before in the non-finalized state"); + } } + // Compute the new anchors and index them + batch.zs_insert(sapling_anchors, height, sapling_note_commitment_tree.root()); + batch.zs_insert(orchard_anchors, height, orchard_note_commitment_tree.root()); + + // Update the note commitment trees + if let Some(h) = finalized_tip_height { + batch.zs_delete(sapling_note_commitment_tree_cf, h); + batch.zs_delete(orchard_note_commitment_tree_cf, h); + } + batch.zs_insert( + sapling_note_commitment_tree_cf, + height, + sapling_note_commitment_tree, + ); + batch.zs_insert( + orchard_note_commitment_tree_cf, + height, + orchard_note_commitment_tree, + ); + batch }; @@ -408,6 +475,34 @@ impl FinalizedState { }) } + /// Returns the Sapling note commitment tree of the finalized tip + /// or the empty tree if the state is empty. + pub fn sapling_note_commitment_tree(&self) -> sapling::tree::NoteCommitmentTree { + let height = match self.finalized_tip_height() { + Some(h) => h, + None => return Default::default(), + }; + let sapling_note_commitment_tree = + self.db.cf_handle("sapling_note_commitment_tree").unwrap(); + self.db + .zs_get(sapling_note_commitment_tree, &height) + .expect("note commitment tree must exist if there is a finalized tip") + } + + /// Returns the Orchard note commitment tree of the finalized tip + /// or the empty tree if the state is empty. + pub fn orchard_note_commitment_tree(&self) -> orchard::tree::NoteCommitmentTree { + let height = match self.finalized_tip_height() { + Some(h) => h, + None => return Default::default(), + }; + let orchard_note_commitment_tree = + self.db.cf_handle("orchard_note_commitment_tree").unwrap(); + self.db + .zs_get(orchard_note_commitment_tree, &height) + .expect("note commitment tree must exist if there is a finalized tip") + } + /// If the database is `ephemeral`, delete it. fn delete_ephemeral(&self) { if self.ephemeral { diff --git a/zebra-state/src/service/finalized_state/disk_format.rs b/zebra-state/src/service/finalized_state/disk_format.rs index 241582e36..432e81ee3 100644 --- a/zebra-state/src/service/finalized_state/disk_format.rs +++ b/zebra-state/src/service/finalized_state/disk_format.rs @@ -1,6 +1,7 @@ //! Module defining exactly how to move types in and out of rocksdb use std::{convert::TryInto, fmt::Debug, sync::Arc}; +use bincode::Options; use zebra_chain::{ block, block::Block, @@ -232,6 +233,65 @@ impl IntoDisk for transparent::OutPoint { } } +impl IntoDisk for sapling::tree::Root { + type Bytes = [u8; 32]; + + fn as_bytes(&self) -> Self::Bytes { + self.into() + } +} + +impl IntoDisk for orchard::tree::Root { + type Bytes = [u8; 32]; + + fn as_bytes(&self) -> Self::Bytes { + self.into() + } +} + +// The following implementations for the note commitment trees use `serde` and +// `bincode` because currently the inner Merkle tree frontier (from +// `incrementalmerkletree`) only supports `serde` for serialization. `bincode` +// was chosen because it is small and fast. We explicitly use `DefaultOptions` +// in particular to disallow trailing bytes; see +// https://docs.rs/bincode/1.3.3/bincode/config/index.html#options-struct-vs-bincode-functions + +impl IntoDisk for sapling::tree::NoteCommitmentTree { + type Bytes = Vec; + + fn as_bytes(&self) -> Self::Bytes { + bincode::DefaultOptions::new() + .serialize(self) + .expect("serialization to vec doesn't fail") + } +} + +impl FromDisk for sapling::tree::NoteCommitmentTree { + fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { + bincode::DefaultOptions::new() + .deserialize(bytes.as_ref()) + .expect("deserialization format should match the serialization format used by IntoDisk") + } +} + +impl IntoDisk for orchard::tree::NoteCommitmentTree { + type Bytes = Vec; + + fn as_bytes(&self) -> Self::Bytes { + bincode::DefaultOptions::new() + .serialize(self) + .expect("serialization to vec doesn't fail") + } +} + +impl FromDisk for orchard::tree::NoteCommitmentTree { + fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { + bincode::DefaultOptions::new() + .deserialize(bytes.as_ref()) + .expect("deserialization format should match the serialization format used by IntoDisk") + } +} + /// Helper trait for inserting (Key, Value) pairs into rocksdb with a consistently /// defined format pub trait DiskSerialize { @@ -241,6 +301,11 @@ pub trait DiskSerialize { where K: IntoDisk + Debug, V: IntoDisk; + + /// Remove the given key form rocksdb column family if it exists. + fn zs_delete(&mut self, cf: &rocksdb::ColumnFamily, key: K) + where + K: IntoDisk + Debug; } impl DiskSerialize for rocksdb::WriteBatch { @@ -253,6 +318,14 @@ impl DiskSerialize for rocksdb::WriteBatch { let value_bytes = value.as_bytes(); self.put_cf(cf, key_bytes, value_bytes); } + + fn zs_delete(&mut self, cf: &rocksdb::ColumnFamily, key: K) + where + K: IntoDisk + Debug, + { + let key_bytes = key.as_bytes(); + self.delete_cf(cf, key_bytes); + } } /// Helper trait for retrieving values from rocksdb column familys with a consistently diff --git a/zebra-state/src/service/finalized_state/tests.rs b/zebra-state/src/service/finalized_state/tests.rs index 2bf82ef4e..cc95d9d45 100644 --- a/zebra-state/src/service/finalized_state/tests.rs +++ b/zebra-state/src/service/finalized_state/tests.rs @@ -1 +1,2 @@ mod prop; +mod vectors; diff --git a/zebra-state/src/service/finalized_state/tests/prop.rs b/zebra-state/src/service/finalized_state/tests/prop.rs index f321079e9..34386cc3f 100644 --- a/zebra-state/src/service/finalized_state/tests/prop.rs +++ b/zebra-state/src/service/finalized_state/tests/prop.rs @@ -12,7 +12,7 @@ use crate::{ ContextuallyValidBlock, }; -const DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES: u32 = 16; +const DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES: u32 = 1; #[test] fn blocks_with_v5_transactions() -> Result<()> { diff --git a/zebra-state/src/service/finalized_state/tests/vectors.rs b/zebra-state/src/service/finalized_state/tests/vectors.rs new file mode 100644 index 000000000..6501f0911 --- /dev/null +++ b/zebra-state/src/service/finalized_state/tests/vectors.rs @@ -0,0 +1,80 @@ +use halo2::arithmetic::FieldExt; +use halo2::pasta::pallas; +use hex::FromHex; + +use crate::service::finalized_state::disk_format::{FromDisk, IntoDisk}; +use zebra_chain::{orchard, sapling}; + +#[test] +fn sapling_note_commitment_tree_serialization() { + zebra_test::init(); + + let mut incremental_tree = sapling::tree::NoteCommitmentTree::default(); + + // Some commitments from zebra-chain/src/sapling/tests/test_vectors.rs + let hex_commitments = [ + "b02310f2e087e55bfd07ef5e242e3b87ee5d00c9ab52f61e6bd42542f93a6f55", + "225747f3b5d5dab4e5a424f81f85c904ff43286e0f3fd07ef0b8c6a627b11458", + "7c3ea01a6e3a3d90cf59cd789e467044b5cd78eb2c84cc6816f960746d0e036c", + ]; + + for cm_u_hex in hex_commitments { + let bytes = <[u8; 32]>::from_hex(cm_u_hex).unwrap(); + + let cm_u = jubjub::Fq::from_bytes(&bytes).unwrap(); + incremental_tree.append(cm_u).unwrap(); + } + + // This test vector was generated by the code itself. + // The purpose of this test is to make sure the serialization format does + // not change by accident. + let expected_serialized_tree_hex = "0102007c3ea01a6e3a3d90cf59cd789e467044b5cd78eb2c84cc6816f960746d0e036c0162324ff2c329e99193a74d28a585a3c167a93bf41a255135529c913bd9b1e666"; + let serialized_tree = incremental_tree.as_bytes(); + assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex); + + let deserialized_tree = sapling::tree::NoteCommitmentTree::from_bytes(serialized_tree); + + assert_eq!(incremental_tree.root(), deserialized_tree.root()); +} + +#[test] +fn orchard_note_commitment_tree_serialization() { + zebra_test::init(); + + let mut incremental_tree = orchard::tree::NoteCommitmentTree::default(); + + // Some commitments from zebra-chain/src/orchard/tests/tree.rs + let commitments = [ + [ + 0x68, 0x13, 0x5c, 0xf4, 0x99, 0x33, 0x22, 0x90, 0x99, 0xa4, 0x4e, 0xc9, 0x9a, 0x75, + 0xe1, 0xe1, 0xcb, 0x46, 0x40, 0xf9, 0xb5, 0xbd, 0xec, 0x6b, 0x32, 0x23, 0x85, 0x6f, + 0xea, 0x16, 0x39, 0x0a, + ], + [ + 0x78, 0x31, 0x50, 0x08, 0xfb, 0x29, 0x98, 0xb4, 0x30, 0xa5, 0x73, 0x1d, 0x67, 0x26, + 0x20, 0x7d, 0xc0, 0xf0, 0xec, 0x81, 0xea, 0x64, 0xaf, 0x5c, 0xf6, 0x12, 0x95, 0x69, + 0x01, 0xe7, 0x2f, 0x0e, + ], + [ + 0xee, 0x94, 0x88, 0x05, 0x3a, 0x30, 0xc5, 0x96, 0xb4, 0x30, 0x14, 0x10, 0x5d, 0x34, + 0x77, 0xe6, 0xf5, 0x78, 0xc8, 0x92, 0x40, 0xd1, 0xd1, 0xee, 0x17, 0x43, 0xb7, 0x7b, + 0xb6, 0xad, 0xc4, 0x0a, + ], + ]; + + for cm_x_bytes in &commitments { + let cm_x = pallas::Base::from_bytes(cm_x_bytes).unwrap(); + incremental_tree.append(cm_x).unwrap(); + } + + // This test vector was generated by the code itself. + // The purpose of this test is to make sure the serialization format does + // not change by accident. + let expected_serialized_tree_hex = "010200ee9488053a30c596b43014105d3477e6f578c89240d1d1ee1743b77bb6adc40a01a34b69a4e4d9ccf954d46e5da1004d361a5497f511aeb4d481d23c0be1778133"; + let serialized_tree = incremental_tree.as_bytes(); + assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex); + + let deserialized_tree = orchard::tree::NoteCommitmentTree::from_bytes(serialized_tree); + + assert_eq!(incremental_tree.root(), deserialized_tree.root()); +} diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index 3e2cf97c5..b1bcf18ef 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -14,13 +14,15 @@ use std::{collections::BTreeSet, mem, ops::Deref, sync::Arc}; use zebra_chain::{ block::{self, Block}, + orchard, parameters::Network, + sapling, transaction::{self, Transaction}, transparent, }; #[cfg(test)] -use zebra_chain::{orchard, sapling, sprout}; +use zebra_chain::sprout; use crate::{FinalizedBlock, HashOrHeight, PreparedBlock, ValidateContextError}; @@ -123,7 +125,11 @@ impl NonFinalizedState { let parent_hash = prepared.block.header.previous_block_hash; let (height, hash) = (prepared.height, prepared.hash); - let parent_chain = self.parent_chain(parent_hash)?; + let parent_chain = self.parent_chain( + parent_hash, + finalized_state.sapling_note_commitment_tree(), + finalized_state.orchard_note_commitment_tree(), + )?; // We might have taken a chain, so all validation must happen within // validate_and_commit, so that the chain is restored correctly. @@ -154,7 +160,10 @@ impl NonFinalizedState { prepared: PreparedBlock, finalized_state: &FinalizedState, ) -> Result<(), ValidateContextError> { - let chain = Chain::default(); + let chain = Chain::new( + finalized_state.sapling_note_commitment_tree(), + finalized_state.orchard_note_commitment_tree(), + ); let (height, hash) = (prepared.height, prepared.hash); // if the block is invalid, drop the newly created chain fork @@ -345,9 +354,14 @@ impl NonFinalizedState { /// /// The chain can be an existing chain in the non-finalized state or a freshly /// created fork, if needed. + /// + /// The note commitment trees must be the trees of the finalized tip. + /// They are used to recreate the trees if a fork is needed. fn parent_chain( &mut self, parent_hash: block::Hash, + sapling_note_commitment_tree: sapling::tree::NoteCommitmentTree, + orchard_note_commitment_tree: orchard::tree::NoteCommitmentTree, ) -> Result, ValidateContextError> { match self.take_chain_if(|chain| chain.non_finalized_tip_hash() == parent_hash) { // An existing chain in the non-finalized state @@ -356,7 +370,15 @@ impl NonFinalizedState { None => Ok(Box::new( self.chain_set .iter() - .find_map(|chain| chain.fork(parent_hash).transpose()) + .find_map(|chain| { + chain + .fork( + parent_hash, + sapling_note_commitment_tree.clone(), + orchard_note_commitment_tree.clone(), + ) + .transpose() + }) .expect( "commit_block is only called with blocks that are ready to be commited", )?, diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs index 70f343572..0cf3c165c 100644 --- a/zebra-state/src/service/non_finalized_state/chain.rs +++ b/zebra-state/src/service/non_finalized_state/chain.rs @@ -4,6 +4,7 @@ use std::{ ops::Deref, }; +use multiset::HashMultiSet; use tracing::instrument; use zebra_chain::{ @@ -13,7 +14,7 @@ use zebra_chain::{ use crate::{service::check, ContextuallyValidBlock, PreparedBlock, ValidateContextError}; -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] pub struct Chain { /// The contextually valid blocks which form this non-finalized partial chain, in height order. pub(crate) blocks: BTreeMap, @@ -32,20 +33,25 @@ pub struct Chain { /// including those created by earlier transactions or blocks in the chain. pub(crate) spent_utxos: HashSet, - /// The sprout anchors created by `blocks`. - /// - /// TODO: does this include intersitial anchors? - pub(super) sprout_anchors: HashSet, - /// The sapling anchors created by `blocks`. - pub(super) sapling_anchors: HashSet, - /// The orchard anchors created by `blocks`. - pub(super) orchard_anchors: HashSet, + /// The Sapling note commitment tree of the tip of this Chain. + pub(super) sapling_note_commitment_tree: sapling::tree::NoteCommitmentTree, + /// The Orchard note commitment tree of the tip of this Chain. + pub(super) orchard_note_commitment_tree: orchard::tree::NoteCommitmentTree, - /// The sprout nullifiers revealed by `blocks`. + /// The Sapling anchors created by `blocks`. + pub(super) sapling_anchors: HashMultiSet, + /// The Sapling anchors created by each block in the chain. + pub(super) sapling_anchors_by_height: BTreeMap, + /// The Orchard anchors created by `blocks`. + pub(super) orchard_anchors: HashMultiSet, + /// The Orchard anchors created by each block in the chain. + pub(super) orchard_anchors_by_height: BTreeMap, + + /// The Sprout nullifiers revealed by `blocks`. pub(super) sprout_nullifiers: HashSet, - /// The sapling nullifiers revealed by `blocks`. + /// The Sapling nullifiers revealed by `blocks`. pub(super) sapling_nullifiers: HashSet, - /// The orchard nullifiers revealed by `blocks`. + /// The Orchard nullifiers revealed by `blocks`. pub(super) orchard_nullifiers: HashSet, /// The cumulative work represented by this partial non-finalized chain. @@ -53,6 +59,30 @@ pub struct Chain { } impl Chain { + // Create a new Chain with the given note commitment trees. + pub(crate) fn new( + sapling_note_commitment_tree: sapling::tree::NoteCommitmentTree, + orchard_note_commitment_tree: orchard::tree::NoteCommitmentTree, + ) -> Self { + Self { + blocks: Default::default(), + height_by_hash: Default::default(), + tx_by_hash: Default::default(), + created_utxos: Default::default(), + sapling_note_commitment_tree, + orchard_note_commitment_tree, + spent_utxos: Default::default(), + sapling_anchors: HashMultiSet::new(), + sapling_anchors_by_height: Default::default(), + orchard_anchors: HashMultiSet::new(), + orchard_anchors_by_height: Default::default(), + sprout_nullifiers: Default::default(), + sapling_nullifiers: Default::default(), + orchard_nullifiers: Default::default(), + partial_cumulative_work: Default::default(), + } + } + /// Is the internal state of `self` the same as `other`? /// /// [`Chain`] has custom [`Eq`] and [`Ord`] implementations based on proof of work, @@ -76,10 +106,15 @@ impl Chain { self.created_utxos == other.created_utxos && self.spent_utxos == other.spent_utxos && + // note commitment trees + self.sapling_note_commitment_tree.root() == other.sapling_note_commitment_tree.root() && + self.orchard_note_commitment_tree.root() == other.orchard_note_commitment_tree.root() && + // anchors - self.sprout_anchors == other.sprout_anchors && self.sapling_anchors == other.sapling_anchors && + self.sapling_anchors_by_height == other.sapling_anchors_by_height && self.orchard_anchors == other.orchard_anchors && + self.orchard_anchors_by_height == other.orchard_anchors_by_height && // nullifiers self.sprout_nullifiers == other.sprout_nullifiers && @@ -133,17 +168,46 @@ impl Chain { /// Fork a chain at the block with the given hash, if it is part of this /// chain. - pub fn fork(&self, fork_tip: block::Hash) -> Result, ValidateContextError> { + /// + /// The note commitment trees must be the trees of the finalized tip. + pub fn fork( + &self, + fork_tip: block::Hash, + sapling_note_commitment_tree: sapling::tree::NoteCommitmentTree, + orchard_note_commitment_tree: orchard::tree::NoteCommitmentTree, + ) -> Result, ValidateContextError> { if !self.height_by_hash.contains_key(&fork_tip) { return Ok(None); } - let mut forked = self.clone(); + let mut forked = + self.with_trees(sapling_note_commitment_tree, orchard_note_commitment_tree); while forked.non_finalized_tip_hash() != fork_tip { forked.pop_tip(); } + // Rebuild the note commitment trees, starting from the finalized tip tree. + // TODO: change to a more efficient approach by removing nodes + // from the tree of the original chain (in `pop_tip()`). + // See https://github.com/ZcashFoundation/zebra/issues/2378 + for block in forked.blocks.values() { + for transaction in block.block.transactions.iter() { + for sapling_note_commitment in transaction.sapling_note_commitments() { + forked + .sapling_note_commitment_tree + .append(*sapling_note_commitment) + .expect("must work since it was already appended before the fork"); + } + for orchard_note_commitment in transaction.orchard_note_commitments() { + forked + .orchard_note_commitment_tree + .append(*orchard_note_commitment) + .expect("must work since it was already appended before the fork"); + } + } + } + Ok(Some(forked)) } @@ -194,6 +258,34 @@ impl Chain { unspent_utxos.retain(|out_point, _utxo| !self.spent_utxos.contains(out_point)); unspent_utxos } + + /// Clone the Chain but not the history and note commitment trees, using + /// the specified trees instead. + /// + /// Useful when forking, where the trees are rebuilt anyway. + fn with_trees( + &self, + sapling_note_commitment_tree: sapling::tree::NoteCommitmentTree, + orchard_note_commitment_tree: orchard::tree::NoteCommitmentTree, + ) -> Self { + Chain { + blocks: self.blocks.clone(), + height_by_hash: self.height_by_hash.clone(), + tx_by_hash: self.tx_by_hash.clone(), + created_utxos: self.created_utxos.clone(), + spent_utxos: self.spent_utxos.clone(), + sapling_note_commitment_tree, + orchard_note_commitment_tree, + sapling_anchors: self.sapling_anchors.clone(), + orchard_anchors: self.orchard_anchors.clone(), + sapling_anchors_by_height: self.sapling_anchors_by_height.clone(), + orchard_anchors_by_height: self.orchard_anchors_by_height.clone(), + sprout_nullifiers: self.sprout_nullifiers.clone(), + sapling_nullifiers: self.sapling_nullifiers.clone(), + orchard_nullifiers: self.orchard_nullifiers.clone(), + partial_cumulative_work: self.partial_cumulative_work, + } + } } /// Helper trait to organize inverse operations done on the `Chain` type. Used to @@ -300,14 +392,25 @@ impl UpdateWith for Chain { self.update_chain_state_with(orchard_shielded_data)?; } + // Having updated all the note commitment trees and nullifier sets in + // this block, the roots of the note commitment trees as of the last + // transaction are the treestates of this block. + let root = self.sapling_note_commitment_tree.root(); + self.sapling_anchors.insert(root); + self.sapling_anchors_by_height.insert(height, root); + let root = self.orchard_note_commitment_tree.root(); + self.orchard_anchors.insert(root); + self.orchard_anchors_by_height.insert(height, root); + Ok(()) } #[instrument(skip(self, contextually_valid), fields(block = %contextually_valid.block))] fn revert_chain_state_with(&mut self, contextually_valid: &ContextuallyValidBlock) { - let (block, hash, new_outputs, transaction_hashes) = ( + let (block, hash, height, new_outputs, transaction_hashes) = ( contextually_valid.block.as_ref(), contextually_valid.hash, + contextually_valid.height, &contextually_valid.new_outputs, &contextually_valid.transaction_hashes, ); @@ -377,6 +480,22 @@ impl UpdateWith for Chain { self.revert_chain_state_with(sapling_shielded_data_shared_anchor); self.revert_chain_state_with(orchard_shielded_data); } + let anchor = self + .sapling_anchors_by_height + .remove(&height) + .expect("Sapling anchor must be present if block was added to chain"); + assert!( + self.sapling_anchors.remove(&anchor), + "Sapling anchor must be present if block was added to chain" + ); + let anchor = self + .orchard_anchors_by_height + .remove(&height) + .expect("Orchard anchor must be present if block was added to chain"); + assert!( + self.orchard_anchors.remove(&anchor), + "Orchard anchor must be present if block was added to chain" + ); } } @@ -474,6 +593,10 @@ where sapling_shielded_data: &Option>, ) -> Result<(), ValidateContextError> { if let Some(sapling_shielded_data) = sapling_shielded_data { + for cm_u in sapling_shielded_data.note_commitments() { + self.sapling_note_commitment_tree.append(*cm_u)?; + } + check::nullifier::add_to_non_finalized_chain_unique( &mut self.sapling_nullifiers, sapling_shielded_data.nullifiers(), @@ -493,6 +616,10 @@ where sapling_shielded_data: &Option>, ) { if let Some(sapling_shielded_data) = sapling_shielded_data { + // Note commitments are not removed from the tree here because we + // don't support that operation yet. Instead, we recreate the tree + // from the finalized tip in NonFinalizedState. + check::nullifier::remove_from_non_finalized_chain( &mut self.sapling_nullifiers, sapling_shielded_data.nullifiers(), @@ -508,6 +635,10 @@ impl UpdateWith> for Chain { orchard_shielded_data: &Option, ) -> Result<(), ValidateContextError> { if let Some(orchard_shielded_data) = orchard_shielded_data { + for cm_x in orchard_shielded_data.note_commitments() { + self.orchard_note_commitment_tree.append(*cm_x)?; + } + check::nullifier::add_to_non_finalized_chain_unique( &mut self.orchard_nullifiers, orchard_shielded_data.nullifiers(), @@ -524,6 +655,10 @@ impl UpdateWith> for Chain { #[instrument(skip(self, orchard_shielded_data))] fn revert_chain_state_with(&mut self, orchard_shielded_data: &Option) { if let Some(orchard_shielded_data) = orchard_shielded_data { + // Note commitments are not removed from the tree here because we + // don't support that operation yet. Instead, we recreate the tree + // from the finalized tip in NonFinalizedState. + check::nullifier::remove_from_non_finalized_chain( &mut self.orchard_nullifiers, orchard_shielded_data.nullifiers(), diff --git a/zebra-state/src/service/non_finalized_state/tests/prop.rs b/zebra-state/src/service/non_finalized_state/tests/prop.rs index 86a5b8fe8..1989fb03c 100644 --- a/zebra-state/src/service/non_finalized_state/tests/prop.rs +++ b/zebra-state/src/service/non_finalized_state/tests/prop.rs @@ -19,7 +19,7 @@ use crate::{ Config, }; -const DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES: u32 = 16; +const DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES: u32 = 1; /// Check that a forked chain is the same as a chain that had the same blocks appended. /// @@ -37,8 +37,9 @@ fn forked_equals_pushed() -> Result<()> { |((chain, fork_at_count, _network) in PreparedChain::default())| { // use `fork_at_count` as the fork tip let fork_tip_hash = chain[fork_at_count - 1].hash; - let mut full_chain = Chain::default(); - let mut partial_chain = Chain::default(); + + let mut full_chain = Chain::new(Default::default(), Default::default()); + let mut partial_chain = Chain::new(Default::default(), Default::default()); let mut has_prevouts = false; for block in chain.iter().take(fork_at_count) { @@ -72,7 +73,14 @@ fn forked_equals_pushed() -> Result<()> { .is_some(); } - let forked = full_chain.fork(fork_tip_hash).expect("fork works").expect("hash is present"); + let forked = full_chain + .fork( + fork_tip_hash, + Default::default(), + Default::default(), + ) + .expect("fork works") + .expect("hash is present"); // the first check is redundant, but it's useful for debugging prop_assert_eq!(forked.blocks.len(), partial_chain.blocks.len()); @@ -99,13 +107,19 @@ fn finalized_equals_pushed() -> Result<()> { |((chain, end_count, _network) in PreparedChain::default())| { // use `end_count` as the number of non-finalized blocks at the end of the chain let finalized_count = chain.len() - end_count; - let mut full_chain = Chain::default(); - let mut partial_chain = Chain::default(); + let mut full_chain = Chain::new(Default::default(), Default::default()); + for block in chain.iter().take(finalized_count) { + full_chain = full_chain.push(block.clone())?; + } + let mut partial_chain = Chain::new( + full_chain.sapling_note_commitment_tree.clone(), + full_chain.orchard_note_commitment_tree.clone(), + ); for block in chain.iter().skip(finalized_count) { partial_chain = partial_chain.push(block.clone())?; } - for block in chain.iter() { + for block in chain.iter().skip(finalized_count) { full_chain = full_chain.push(block.clone())?; } @@ -215,8 +229,8 @@ fn different_blocks_different_chains() -> Result<()> { .prop_flat_map(|block_strategy| (block_strategy.clone(), block_strategy)) .prop_map(|(block1, block2)| (DisplayToDebug(block1), DisplayToDebug(block2))) )| { - let chain1 = Chain::default(); - let chain2 = Chain::default(); + let chain1 = Chain::new(Default::default(), Default::default()); + let chain2 = Chain::new(Default::default(), Default::default()); let block1 = Arc::new(block1.0).prepare(); let block2 = Arc::new(block2.0).prepare(); @@ -250,10 +264,15 @@ fn different_blocks_different_chains() -> Result<()> { chain1.created_utxos = chain2.created_utxos.clone(); chain1.spent_utxos = chain2.spent_utxos.clone(); + // note commitment trees + chain1.sapling_note_commitment_tree = chain2.sapling_note_commitment_tree.clone(); + chain1.orchard_note_commitment_tree = chain2.orchard_note_commitment_tree.clone(); + // anchors - chain1.sprout_anchors = chain2.sprout_anchors.clone(); chain1.sapling_anchors = chain2.sapling_anchors.clone(); + chain1.sapling_anchors_by_height = chain2.sapling_anchors_by_height.clone(); chain1.orchard_anchors = chain2.orchard_anchors.clone(); + chain1.orchard_anchors_by_height = chain2.orchard_anchors_by_height.clone(); // nullifiers chain1.sprout_nullifiers = chain2.sprout_nullifiers.clone(); diff --git a/zebra-state/src/service/non_finalized_state/tests/vectors.rs b/zebra-state/src/service/non_finalized_state/tests/vectors.rs index f4fff81b7..4be78298c 100644 --- a/zebra-state/src/service/non_finalized_state/tests/vectors.rs +++ b/zebra-state/src/service/non_finalized_state/tests/vectors.rs @@ -18,7 +18,7 @@ use self::assert_eq; #[test] fn construct_empty() { zebra_test::init(); - let _chain = Chain::default(); + let _chain = Chain::new(Default::default(), Default::default()); } #[test] @@ -27,7 +27,7 @@ fn construct_single() -> Result<()> { let block: Arc = zebra_test::vectors::BLOCK_MAINNET_434873_BYTES.zcash_deserialize_into()?; - let mut chain = Chain::default(); + let mut chain = Chain::new(Default::default(), Default::default()); chain = chain.push(block.prepare())?; assert_eq!(1, chain.blocks.len()); @@ -49,7 +49,7 @@ fn construct_many() -> Result<()> { block = next_block; } - let mut chain = Chain::default(); + let mut chain = Chain::new(Default::default(), Default::default()); for block in blocks { chain = chain.push(block.prepare())?; @@ -68,10 +68,10 @@ fn ord_matches_work() -> Result<()> { .set_work(1); let more_block = less_block.clone().set_work(10); - let mut lesser_chain = Chain::default(); + let mut lesser_chain = Chain::new(Default::default(), Default::default()); lesser_chain = lesser_chain.push(less_block.prepare())?; - let mut bigger_chain = Chain::default(); + let mut bigger_chain = Chain::new(Default::default(), Default::default()); bigger_chain = bigger_chain.push(more_block.prepare())?; assert!(bigger_chain > lesser_chain);