From ff8095b7edebd997f39e4ff33f1f64d4c5bce0b4 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 15 Feb 2023 16:32:30 -0700 Subject: [PATCH 1/7] Remove `serde` serialization & deserialization The structure of the `incrementalmerkletree` and `bridgetree` types has not historically been stable, and as such it is not appropriate to automatically derive serialization for these types. --- bridgetree/CHANGELOG.md | 1 + bridgetree/Cargo.toml | 1 - bridgetree/src/lib.rs | 11 +++++------ incrementalmerkletree/CHANGELOG.md | 1 + incrementalmerkletree/Cargo.toml | 1 - incrementalmerkletree/src/lib.rs | 7 +++---- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/bridgetree/CHANGELOG.md b/bridgetree/CHANGELOG.md index a6830ac..2cf1efe 100644 --- a/bridgetree/CHANGELOG.md +++ b/bridgetree/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to Rust's notion of - The `testing` module has been removed in favor of depending on `incrementalmerkletree::testing`. +- `serde` serialization and parsing are no longer supported. ## [bridgetree-v0.2.0] - 2022-05-10 diff --git a/bridgetree/Cargo.toml b/bridgetree/Cargo.toml index cdbc431..a1a5960 100644 --- a/bridgetree/Cargo.toml +++ b/bridgetree/Cargo.toml @@ -15,7 +15,6 @@ rust-version = "1.60" [dependencies] incrementalmerkletree = { version = "0.3", path = "../incrementalmerkletree" } -serde = { version = "1", features = ["derive"] } proptest = { version = "1.0.0", optional = true } [dev-dependencies] diff --git a/bridgetree/src/lib.rs b/bridgetree/src/lib.rs index cc2ee22..da34f7a 100644 --- a/bridgetree/src/lib.rs +++ b/bridgetree/src/lib.rs @@ -30,7 +30,6 @@ //! reset the state to. //! //! In this module, the term "ommer" is used as for the sibling of a parent node in a binary tree. -use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::convert::TryFrom; use std::fmt::Debug; @@ -130,7 +129,7 @@ impl Iterator for WitnessAddrsIter { /// A [`NonEmptyFrontier`] is a reduced representation of a Merkle tree, containing a single leaf /// value, along with the vector of hashes produced by the reduction of previously appended leaf /// values that will be required when producing a witness for the current leaf. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct NonEmptyFrontier { position: Position, leaf: H, @@ -274,7 +273,7 @@ impl NonEmptyFrontier { } /// A possibly-empty Merkle frontier. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Frontier { frontier: Option>, } @@ -366,7 +365,7 @@ impl Frontier { /// [`MerkleBridge`] values have a semigroup, such that the sum (`fuse`d) value of two successive /// bridges, along with a [`NonEmptyFrontier`] with its tip at the prior position of the first bridge /// being fused, can be used to produce a witness for the leaf at the tip of the prior frontier. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct MerkleBridge { /// The position of the final leaf in the frontier of the bridge that this bridge is the /// successor of, or None if this is the first bridge in a tree. @@ -612,7 +611,7 @@ impl<'a, H: Hashable + Ord + Clone + 'a> MerkleBridge { /// crosses [`MerkleBridge`] boundaries, and so it is not sufficient to just truncate the list of /// bridges; instead, we use [`Checkpoint`] values to be able to rapidly restore the cache to its /// previous state. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct Checkpoint { /// The unique identifier for this checkpoint. id: usize, @@ -713,7 +712,7 @@ impl Checkpoint { /// A sparse representation of a Merkle tree with linear appending of leaves that contains enough /// information to produce a witness for any `mark`ed leaf. -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq)] pub struct BridgeTree { /// The ordered list of Merkle bridges representing the history /// of the tree. There will be one bridge for each saved leaf. diff --git a/incrementalmerkletree/CHANGELOG.md b/incrementalmerkletree/CHANGELOG.md index f938ab8..6d0b90a 100644 --- a/incrementalmerkletree/CHANGELOG.md +++ b/incrementalmerkletree/CHANGELOG.md @@ -57,6 +57,7 @@ is not another good use case for polymorphism over tree implementations. - `Position::max_altitude` - `Position::ommer_altitudes` - `impl Sub for Altitude` +- `serde` serialization and parsing are no longer supported. ## [0.3.0] - 2022-05-10 diff --git a/incrementalmerkletree/Cargo.toml b/incrementalmerkletree/Cargo.toml index e05e83f..92c709f 100644 --- a/incrementalmerkletree/Cargo.toml +++ b/incrementalmerkletree/Cargo.toml @@ -15,7 +15,6 @@ rust-version = "1.60" [dependencies] either = "1.8" -serde = { version = "1", features = ["derive"] } proptest = { version = "1.0.0", optional = true } [dev-dependencies] diff --git a/incrementalmerkletree/src/lib.rs b/incrementalmerkletree/src/lib.rs index 08382d4..14aee4b 100644 --- a/incrementalmerkletree/src/lib.rs +++ b/incrementalmerkletree/src/lib.rs @@ -1,7 +1,6 @@ //! Common types and utilities used in incremental Merkle tree implementations. use either::Either; -use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::convert::{TryFrom, TryInto}; use std::num::TryFromIntError; @@ -44,7 +43,7 @@ impl Retention { } /// A type representing the position of a leaf in a Merkle tree. -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(transparent)] pub struct Position(usize); @@ -128,7 +127,7 @@ impl TryFrom for Position { /// nodes at level `0` are leaves, nodes at level `1` are parents of nodes at /// level `0`, and so forth. This type is capable of representing levels in /// trees containing up to 2^255 leaves. -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(transparent)] pub struct Level(u8); @@ -178,7 +177,7 @@ impl Sub for Level { /// The address of an internal node of the Merkle tree. /// When `level == 0`, the index has the same value as the /// position. -#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Address { level: Level, index: usize, From 4e3c6a6378f4be5c6744444281f46df0553e2c4e Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 15 Feb 2023 17:03:19 -0700 Subject: [PATCH 2/7] Move frontier types from `bridgetree` to `incrementalmerkletree` These types are reusable outside of the context of the `bridgetree` crate. --- bridgetree/CHANGELOG.md | 2 + bridgetree/src/lib.rs | 444 ++------------------------ incrementalmerkletree/CHANGELOG.md | 6 + incrementalmerkletree/src/frontier.rs | 322 +++++++++++++++++++ incrementalmerkletree/src/lib.rs | 111 ++++++- 5 files changed, 464 insertions(+), 421 deletions(-) create mode 100644 incrementalmerkletree/src/frontier.rs diff --git a/bridgetree/CHANGELOG.md b/bridgetree/CHANGELOG.md index 2cf1efe..21eb2e6 100644 --- a/bridgetree/CHANGELOG.md +++ b/bridgetree/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to Rust's notion of ### Removed +- The `NonEmptyFrontier`, `Frontier`, and `FrontierError` types have + been moved to the `incrementalmerkletree` crate. - The `testing` module has been removed in favor of depending on `incrementalmerkletree::testing`. - `serde` serialization and parsing are no longer supported. diff --git a/bridgetree/src/lib.rs b/bridgetree/src/lib.rs index da34f7a..ee63108 100644 --- a/bridgetree/src/lib.rs +++ b/bridgetree/src/lib.rs @@ -30,27 +30,14 @@ //! reset the state to. //! //! In this module, the term "ommer" is used as for the sibling of a parent node in a binary tree. +pub use incrementalmerkletree::{ + frontier::{Frontier, NonEmptyFrontier}, + Address, Hashable, Level, Position, Retention, Source, +}; use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::convert::TryFrom; use std::fmt::Debug; -use std::mem::size_of; use std::ops::Range; -pub use incrementalmerkletree::{Address, Hashable, Level, Position, Retention}; - -/// Validation errors that can occur during reconstruction of a Merkle frontier from -/// its constituent parts. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum FrontierError { - /// An error representing that the number of ommers provided in frontier construction does not - /// the expected length of the ommers list given the position. - PositionMismatch { expected_ommers: usize }, - /// An error representing that the position and/or list of ommers provided to frontier - /// construction would result in a frontier that exceeds the maximum statically allowed depth - /// of the tree. - MaxDepthExceeded { depth: u8 }, -} - /// Errors that can be discovered during checks that verify the compatibility of adjacent bridges. #[derive(Clone, Debug, PartialEq, Eq)] pub enum ContinuityError { @@ -73,284 +60,6 @@ pub enum WitnessingError { BridgeAddressInvalid(Address), } -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum Source { - /// The sibling to the address can be derived from the incremental frontier - /// at the contained ommer index - Past(usize), - /// The sibling to the address must be obtained from values discovered by - /// the addition of more nodes to the tree - Future, -} - -#[must_use = "iterators are lazy and do nothing unless consumed"] -struct WitnessAddrsIter { - root_level: Level, - current: Address, - ommer_count: usize, -} - -/// Returns an iterator over the addresses of nodes required to create a witness for this -/// position, beginning with the sibling of the leaf at this position and ending with the -/// sibling of the ancestor of the leaf at this position that is required to compute a root at -/// the specified level. -fn witness_addrs(position: Position, root_level: Level) -> impl Iterator { - WitnessAddrsIter { - root_level, - current: Address::from(position), - ommer_count: 0, - } -} - -impl Iterator for WitnessAddrsIter { - type Item = (Address, Source); - - fn next(&mut self) -> Option<(Address, Source)> { - if self.current.level() < self.root_level { - let current = self.current; - let source = if current.is_right_child() { - Source::Past(self.ommer_count) - } else { - Source::Future - }; - - self.current = current.parent(); - if matches!(source, Source::Past(_)) { - self.ommer_count += 1; - } - - Some((current.sibling(), source)) - } else { - None - } - } -} - -/// A [`NonEmptyFrontier`] is a reduced representation of a Merkle tree, containing a single leaf -/// value, along with the vector of hashes produced by the reduction of previously appended leaf -/// values that will be required when producing a witness for the current leaf. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct NonEmptyFrontier { - position: Position, - leaf: H, - ommers: Vec, -} - -impl NonEmptyFrontier { - /// Constructs a new frontier with the specified value at position 0. - pub fn new(leaf: H) -> Self { - Self { - position: 0.into(), - leaf, - ommers: vec![], - } - } - - /// Constructs a new frontier from its constituent parts - pub fn from_parts(position: Position, leaf: H, ommers: Vec) -> Result { - let expected_ommers = position.past_ommer_count(); - if ommers.len() == expected_ommers { - Ok(Self { - position, - leaf, - ommers, - }) - } else { - Err(FrontierError::PositionMismatch { expected_ommers }) - } - } - - /// Returns the position of the most recently appended leaf. - pub fn position(&self) -> Position { - self.position - } - - /// Returns the leaf most recently appended to the frontier - pub fn leaf(&self) -> &H { - &self.leaf - } - - /// Returns the list of past hashes required to construct a witness for the - /// leaf most recently appended to the frontier. - pub fn ommers(&self) -> &[H] { - &self.ommers - } -} - -impl NonEmptyFrontier { - /// Append a new leaf to the frontier, and recompute recompute ommers by hashing together full - /// subtrees until an empty ommer slot is found. - pub fn append(&mut self, leaf: H) { - let prior_position = self.position; - let prior_leaf = self.leaf.clone(); - self.position += 1; - self.leaf = leaf; - if self.position.is_odd() { - // if the new position is odd, the current leaf will directly become - // an ommer at level 0, and there is no other mutation made to the tree. - self.ommers.insert(0, prior_leaf); - } else { - // if the new position is even, then the current leaf will be hashed - // with the first ommer, and so forth up the tree. - let new_root_level = self.position.root_level(); - - let mut carry = Some((prior_leaf, 0.into())); - let mut new_ommers = Vec::with_capacity(self.position.past_ommer_count()); - for (addr, source) in witness_addrs(prior_position, new_root_level) { - if let Source::Past(i) = source { - if let Some((carry_ommer, carry_lvl)) = carry.as_ref() { - if *carry_lvl == addr.level() { - carry = Some(( - H::combine(addr.level(), &self.ommers[i], carry_ommer), - addr.level() + 1, - )) - } else { - // insert the carry at the first empty slot; then the rest of the - // ommers will remain unchanged - new_ommers.push(carry_ommer.clone()); - new_ommers.push(self.ommers[i].clone()); - carry = None; - } - } else { - // when there's no carry, just push on the ommer value - new_ommers.push(self.ommers[i].clone()); - } - } - } - - // we carried value out, so we need to push on one more ommer. - if let Some((carry_ommer, _)) = carry { - new_ommers.push(carry_ommer); - } - - self.ommers = new_ommers; - } - } - - /// Generate the root of the Merkle tree by hashing against empty subtree roots. - pub fn root(&self, root_level: Option) -> H { - let max_level = root_level.unwrap_or_else(|| self.position.root_level()); - witness_addrs(self.position, max_level) - .fold( - (self.leaf.clone(), Level::from(0)), - |(digest, complete_lvl), (addr, source)| { - // fold up from complete_lvl to addr.level() pairing with empty roots; if - // complete_lvl == addr.level() this is just the complete digest to this point - let digest = complete_lvl - .iter_to(addr.level()) - .fold(digest, |d, l| H::combine(l, &d, &H::empty_root(l))); - - let res_digest = match source { - Source::Past(i) => H::combine(addr.level(), &self.ommers[i], &digest), - Source::Future => { - H::combine(addr.level(), &digest, &H::empty_root(addr.level())) - } - }; - - (res_digest, addr.level() + 1) - }, - ) - .0 - } - - /// Constructs a witness for the leaf at the tip of this - /// frontier, given a source of node values that complement this frontier. - pub fn witness(&self, depth: u8, bridge_value_at: F) -> Result, WitnessingError> - where - F: Fn(Address) -> Option, - { - // construct a complete trailing edge that includes the data from - // the following frontier not yet included in the trailing edge. - witness_addrs(self.position(), depth.into()) - .map(|(addr, source)| match source { - Source::Past(i) => Ok(self.ommers[i].clone()), - Source::Future => { - bridge_value_at(addr).ok_or(WitnessingError::BridgeAddressInvalid(addr)) - } - }) - .collect::, _>>() - } -} - -/// A possibly-empty Merkle frontier. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Frontier { - frontier: Option>, -} - -impl TryFrom> for Frontier { - type Error = FrontierError; - fn try_from(f: NonEmptyFrontier) -> Result { - if f.position.root_level() <= Level::from(DEPTH) { - Ok(Frontier { frontier: Some(f) }) - } else { - Err(FrontierError::MaxDepthExceeded { - depth: f.position.root_level().into(), - }) - } - } -} - -impl Frontier { - /// Constructs a new empty frontier. - pub fn empty() -> Self { - Self { frontier: None } - } - - /// Constructs a new frontier from its constituent parts. - /// - /// Returns `None` if the new frontier would exceed the maximum - /// allowed depth or if the list of ommers provided is not consistent - /// with the position of the leaf. - pub fn from_parts(position: Position, leaf: H, ommers: Vec) -> Result { - NonEmptyFrontier::from_parts(position, leaf, ommers).and_then(Self::try_from) - } - - /// Return the wrapped NonEmptyFrontier reference, or None if - /// the frontier is empty. - pub fn value(&self) -> Option<&NonEmptyFrontier> { - self.frontier.as_ref() - } - - /// Returns the amount of memory dynamically allocated for ommer - /// values within the frontier. - pub fn dynamic_memory_usage(&self) -> usize { - self.frontier.as_ref().map_or(0, |f| { - size_of::() + (f.ommers.capacity() + 1) * size_of::() - }) - } -} - -impl Frontier { - /// Appends a new value to the frontier at the next available slot. - /// Returns true if successful and false if the frontier would exceed - /// the maximum allowed depth. - pub fn append(&mut self, value: H) -> bool { - if let Some(frontier) = self.frontier.as_mut() { - if frontier.position().is_complete_subtree(DEPTH.into()) { - false - } else { - frontier.append(value); - true - } - } else { - self.frontier = Some(NonEmptyFrontier::new(value)); - true - } - } - - /// Obtains the current root of this Merkle frontier by hashing - /// against empty nodes up to the maximum height of the pruned - /// tree that the frontier represents. - pub fn root(&self) -> H { - self.frontier - .as_ref() - .map_or(H::empty_root(DEPTH.into()), |frontier| { - frontier.root(Some(DEPTH.into())) - }) - } -} - /// The information required to "update" witnesses from one state of a Merkle tree to another. /// /// The witness for a particular leaf of a Merkle tree consists of the siblings of that leaf, plus @@ -581,19 +290,21 @@ impl<'a, H: Hashable + Ord + Clone + 'a> MerkleBridge { ) -> Result, WitnessingError> { assert!(Some(prior_frontier.position()) == self.prior_position); - prior_frontier.witness(depth, |addr| { - let r = addr.position_range(); - if self.frontier.position() < r.start { - Some(H::empty_root(addr.level())) - } else if r.contains(&self.frontier.position()) { - Some(self.frontier.root(Some(addr.level()))) - } else { - // the frontier's position is after the end of the requested - // range, so the requested value should exist in a stored - // fragment - self.ommers.get(&addr).cloned() - } - }) + prior_frontier + .witness(depth, |addr| { + let r = addr.position_range(); + if self.frontier.position() < r.start { + Some(H::empty_root(addr.level())) + } else if r.contains(&self.frontier.position()) { + Some(self.frontier.root(Some(addr.level()))) + } else { + // the frontier's position is after the end of the requested + // range, so the requested value should exist in a stored + // fragment + self.ommers.get(&addr).cloned() + } + }) + .map_err(WitnessingError::BridgeAddressInvalid) } fn retain(&mut self, ommer_addrs: &BTreeSet
) { @@ -1229,8 +940,10 @@ impl BridgeTree { // Add the elements of the auth path to the set of addresses we should // continue to track and retain information for - for (addr, source) in - witness_addrs(cur_bridge.frontier.position(), Level::from(DEPTH)) + for (addr, source) in cur_bridge + .frontier + .position() + .witness_addrs(Level::from(DEPTH)) { if source == Source::Future { ommer_addrs.insert(addr); @@ -1283,24 +996,14 @@ mod tests { use super::*; use incrementalmerkletree::{ testing::{ - apply_operation, arb_operation, check_checkpoint_rewind, check_operations, + self, apply_operation, arb_operation, check_checkpoint_rewind, check_operations, check_remove_mark, check_rewind_remove_mark, check_root_hashes, check_witnesses, - complete_tree::CompleteTree, CombinedTree, Frontier, SipHashable, Tree, + complete_tree::CompleteTree, CombinedTree, SipHashable, }, Hashable, }; - impl Frontier for super::Frontier { - fn append(&mut self, value: H) -> bool { - super::Frontier::append(self, value) - } - - fn root(&self) -> H { - super::Frontier::root(self) - } - } - - impl Tree for BridgeTree { + impl testing::Tree for BridgeTree { fn append(&mut self, value: H, retention: Retention) -> bool { let appended = BridgeTree::append(self, value); if appended { @@ -1351,101 +1054,6 @@ mod tests { } } - #[test] - fn position_witness_addrs() { - use Source::*; - let path_elem = |l, i, s| (Address::from_parts(Level::from(l), i), s); - assert_eq!( - vec![path_elem(0, 1, Future), path_elem(1, 1, Future)], - witness_addrs(Position::from(0), Level::from(2)).collect::>() - ); - assert_eq!( - vec![path_elem(0, 3, Future), path_elem(1, 0, Past(0))], - witness_addrs(Position::from(2), Level::from(2)).collect::>() - ); - assert_eq!( - vec![ - path_elem(0, 2, Past(0)), - path_elem(1, 0, Past(1)), - path_elem(2, 1, Future) - ], - witness_addrs(Position::from(3), Level::from(3)).collect::>() - ); - assert_eq!( - vec![ - path_elem(0, 5, Future), - path_elem(1, 3, Future), - path_elem(2, 0, Past(0)), - path_elem(3, 1, Future) - ], - witness_addrs(Position::from(4), Level::from(4)).collect::>() - ); - assert_eq!( - vec![ - path_elem(0, 7, Future), - path_elem(1, 2, Past(0)), - path_elem(2, 0, Past(1)), - path_elem(3, 1, Future) - ], - witness_addrs(Position::from(6), Level::from(4)).collect::>() - ); - } - - #[test] - fn nonempty_frontier_root() { - let mut frontier = NonEmptyFrontier::new("a".to_string()); - assert_eq!(frontier.root(None), "a"); - - frontier.append("b".to_string()); - assert_eq!(frontier.root(None), "ab"); - - frontier.append("c".to_string()); - assert_eq!(frontier.root(None), "abc_"); - } - - #[test] - fn frontier_from_parts() { - assert!(super::Frontier::<(), 1>::from_parts(0.into(), (), vec![]).is_ok()); - assert!(super::Frontier::<(), 1>::from_parts(1.into(), (), vec![()]).is_ok()); - assert!(super::Frontier::<(), 1>::from_parts(0.into(), (), vec![()]).is_err()); - } - - #[test] - fn frontier_root() { - let mut frontier: super::Frontier = super::Frontier::empty(); - assert_eq!(frontier.root().len(), 16); - assert_eq!(frontier.root(), "________________"); - - frontier.append("a".to_string()); - assert_eq!(frontier.root(), "a_______________"); - - frontier.append("b".to_string()); - assert_eq!(frontier.root(), "ab______________"); - - frontier.append("c".to_string()); - assert_eq!(frontier.root(), "abc_____________"); - } - - #[test] - fn frontier_witness() { - let mut frontier = NonEmptyFrontier::::new("a".to_string()); - for c in 'b'..'h' { - frontier.append(c.to_string()); - } - let bridge_value_at = |addr: Address| match ::from(addr.level()) { - 0 => Some("h".to_string()), - 3 => Some("xxxxxxxx".to_string()), - _ => None, - }; - - assert_eq!( - Ok(["h", "ef", "abcd", "xxxxxxxx"] - .map(|v| v.to_string()) - .to_vec()), - frontier.witness(4, bridge_value_at) - ); - } - #[test] fn tree_depth() { let mut tree = BridgeTree::::new(100, 0); diff --git a/incrementalmerkletree/CHANGELOG.md b/incrementalmerkletree/CHANGELOG.md index 6d0b90a..39135aa 100644 --- a/incrementalmerkletree/CHANGELOG.md +++ b/incrementalmerkletree/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to Rust's notion of ## [Unreleased] +### Added +- `incrementalmerkletree::frontier` Types that model the state at the rightmost + node of a Merkle tree that is filled sequentially from the left. These have + been migrated here from the `bridgetree` crate as they are useful outside + of the context of the `bridgetree` data structures. + ## [0.3.1] - 2023-02-28 ### Fixed diff --git a/incrementalmerkletree/src/frontier.rs b/incrementalmerkletree/src/frontier.rs new file mode 100644 index 0000000..6d64a48 --- /dev/null +++ b/incrementalmerkletree/src/frontier.rs @@ -0,0 +1,322 @@ +use std::convert::TryFrom; +use std::mem::size_of; + +use crate::{Address, Hashable, Level, Position, Source}; + +/// Validation errors that can occur during reconstruction of a Merkle frontier from +/// its constituent parts. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FrontierError { + /// An error representing that the number of ommers provided in frontier construction does not + /// the expected length of the ommers list given the position. + PositionMismatch { expected_ommers: usize }, + /// An error representing that the position and/or list of ommers provided to frontier + /// construction would result in a frontier that exceeds the maximum statically allowed depth + /// of the tree. + MaxDepthExceeded { depth: u8 }, +} + +/// A [`NonEmptyFrontier`] is a reduced representation of a Merkle tree, containing a single leaf +/// value, along with the vector of hashes produced by the reduction of previously appended leaf +/// values that will be required when producing a witness for the current leaf. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NonEmptyFrontier { + position: Position, + leaf: H, + ommers: Vec, +} + +impl NonEmptyFrontier { + /// Constructs a new frontier with the specified value at position 0. + pub fn new(leaf: H) -> Self { + Self { + position: 0.into(), + leaf, + ommers: vec![], + } + } + + /// Constructs a new frontier from its constituent parts + pub fn from_parts(position: Position, leaf: H, ommers: Vec) -> Result { + let expected_ommers = position.past_ommer_count(); + if ommers.len() == expected_ommers { + Ok(Self { + position, + leaf, + ommers, + }) + } else { + Err(FrontierError::PositionMismatch { expected_ommers }) + } + } + + /// Returns the position of the most recently appended leaf. + pub fn position(&self) -> Position { + self.position + } + + /// Returns the leaf most recently appended to the frontier + pub fn leaf(&self) -> &H { + &self.leaf + } + + /// Returns the list of past hashes required to construct a witness for the + /// leaf most recently appended to the frontier. + pub fn ommers(&self) -> &[H] { + &self.ommers + } +} + +impl NonEmptyFrontier { + /// Append a new leaf to the frontier, and recompute recompute ommers by hashing together full + /// subtrees until an empty ommer slot is found. + pub fn append(&mut self, leaf: H) { + let prior_position = self.position; + let prior_leaf = self.leaf.clone(); + self.position += 1; + self.leaf = leaf; + if self.position.is_odd() { + // if the new position is odd, the current leaf will directly become + // an ommer at level 0, and there is no other mutation made to the tree. + self.ommers.insert(0, prior_leaf); + } else { + // if the new position is even, then the current leaf will be hashed + // with the first ommer, and so forth up the tree. + let new_root_level = self.position.root_level(); + + let mut carry = Some((prior_leaf, 0.into())); + let mut new_ommers = Vec::with_capacity(self.position.past_ommer_count()); + for (addr, source) in prior_position.witness_addrs(new_root_level) { + if let Source::Past(i) = source { + if let Some((carry_ommer, carry_lvl)) = carry.as_ref() { + if *carry_lvl == addr.level() { + carry = Some(( + H::combine(addr.level(), &self.ommers[i], carry_ommer), + addr.level() + 1, + )) + } else { + // insert the carry at the first empty slot; then the rest of the + // ommers will remain unchanged + new_ommers.push(carry_ommer.clone()); + new_ommers.push(self.ommers[i].clone()); + carry = None; + } + } else { + // when there's no carry, just push on the ommer value + new_ommers.push(self.ommers[i].clone()); + } + } + } + + // we carried value out, so we need to push on one more ommer. + if let Some((carry_ommer, _)) = carry { + new_ommers.push(carry_ommer); + } + + self.ommers = new_ommers; + } + } + + /// Generate the root of the Merkle tree by hashing against empty subtree roots. + pub fn root(&self, root_level: Option) -> H { + let max_level = root_level.unwrap_or_else(|| self.position.root_level()); + self.position + .witness_addrs(max_level) + .fold( + (self.leaf.clone(), Level::from(0)), + |(digest, complete_lvl), (addr, source)| { + // fold up from complete_lvl to addr.level() pairing with empty roots; if + // complete_lvl == addr.level() this is just the complete digest to this point + let digest = complete_lvl + .iter_to(addr.level()) + .fold(digest, |d, l| H::combine(l, &d, &H::empty_root(l))); + + let res_digest = match source { + Source::Past(i) => H::combine(addr.level(), &self.ommers[i], &digest), + Source::Future => { + H::combine(addr.level(), &digest, &H::empty_root(addr.level())) + } + }; + + (res_digest, addr.level() + 1) + }, + ) + .0 + } + + /// Constructs a witness for the leaf at the tip of this frontier, given a source of node + /// values that complement this frontier. + /// + /// If the `complement_nodes` function returns `None` when the value is requested at a given + /// tree address, the address at which the failure occurs will be returned as an error. + pub fn witness(&self, depth: u8, complement_nodes: F) -> Result, Address> + where + F: Fn(Address) -> Option, + { + // construct a complete trailing edge that includes the data from + // the following frontier not yet included in the trailing edge. + self.position() + .witness_addrs(depth.into()) + .map(|(addr, source)| match source { + Source::Past(i) => Ok(self.ommers[i].clone()), + Source::Future => complement_nodes(addr).ok_or(addr), + }) + .collect::, _>>() + } +} + +/// A possibly-empty Merkle frontier. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frontier { + frontier: Option>, +} + +impl TryFrom> for Frontier { + type Error = FrontierError; + fn try_from(f: NonEmptyFrontier) -> Result { + if f.position.root_level() <= Level::from(DEPTH) { + Ok(Frontier { frontier: Some(f) }) + } else { + Err(FrontierError::MaxDepthExceeded { + depth: f.position.root_level().into(), + }) + } + } +} + +impl Frontier { + /// Constructs a new empty frontier. + pub fn empty() -> Self { + Self { frontier: None } + } + + /// Constructs a new frontier from its constituent parts. + /// + /// Returns `None` if the new frontier would exceed the maximum + /// allowed depth or if the list of ommers provided is not consistent + /// with the position of the leaf. + pub fn from_parts(position: Position, leaf: H, ommers: Vec) -> Result { + NonEmptyFrontier::from_parts(position, leaf, ommers).and_then(Self::try_from) + } + + /// Return the wrapped NonEmptyFrontier reference, or None if + /// the frontier is empty. + pub fn value(&self) -> Option<&NonEmptyFrontier> { + self.frontier.as_ref() + } + + /// Returns the amount of memory dynamically allocated for ommer + /// values within the frontier. + pub fn dynamic_memory_usage(&self) -> usize { + self.frontier.as_ref().map_or(0, |f| { + size_of::() + (f.ommers.capacity() + 1) * size_of::() + }) + } +} + +impl Frontier { + /// Appends a new value to the frontier at the next available slot. + /// Returns true if successful and false if the frontier would exceed + /// the maximum allowed depth. + pub fn append(&mut self, value: H) -> bool { + if let Some(frontier) = self.frontier.as_mut() { + if frontier.position().is_complete_subtree(DEPTH.into()) { + false + } else { + frontier.append(value); + true + } + } else { + self.frontier = Some(NonEmptyFrontier::new(value)); + true + } + } + + /// Obtains the current root of this Merkle frontier by hashing + /// against empty nodes up to the maximum height of the pruned + /// tree that the frontier represents. + pub fn root(&self) -> H { + self.frontier + .as_ref() + .map_or(H::empty_root(DEPTH.into()), |frontier| { + frontier.root(Some(DEPTH.into())) + }) + } +} + +#[cfg(feature = "test-dependencies")] +pub mod testing { + use crate::Hashable; + + impl crate::testing::Frontier + for super::Frontier + { + fn append(&mut self, value: H) -> bool { + super::Frontier::append(self, value) + } + + fn root(&self) -> H { + super::Frontier::root(self) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn nonempty_frontier_root() { + let mut frontier = NonEmptyFrontier::new("a".to_string()); + assert_eq!(frontier.root(None), "a"); + + frontier.append("b".to_string()); + assert_eq!(frontier.root(None), "ab"); + + frontier.append("c".to_string()); + assert_eq!(frontier.root(None), "abc_"); + } + + #[test] + fn frontier_from_parts() { + assert!(super::Frontier::<(), 1>::from_parts(0.into(), (), vec![]).is_ok()); + assert!(super::Frontier::<(), 1>::from_parts(1.into(), (), vec![()]).is_ok()); + assert!(super::Frontier::<(), 1>::from_parts(0.into(), (), vec![()]).is_err()); + } + + #[test] + fn frontier_root() { + let mut frontier: super::Frontier = super::Frontier::empty(); + assert_eq!(frontier.root().len(), 16); + assert_eq!(frontier.root(), "________________"); + + frontier.append("a".to_string()); + assert_eq!(frontier.root(), "a_______________"); + + frontier.append("b".to_string()); + assert_eq!(frontier.root(), "ab______________"); + + frontier.append("c".to_string()); + assert_eq!(frontier.root(), "abc_____________"); + } + + #[test] + fn frontier_witness() { + let mut frontier = NonEmptyFrontier::::new("a".to_string()); + for c in 'b'..'h' { + frontier.append(c.to_string()); + } + let bridge_value_at = |addr: Address| match ::from(addr.level()) { + 0 => Some("h".to_string()), + 3 => Some("xxxxxxxx".to_string()), + _ => None, + }; + + assert_eq!( + Ok(["h", "ef", "abcd", "xxxxxxxx"] + .map(|v| v.to_string()) + .to_vec()), + frontier.witness(4, bridge_value_at) + ); + } +} diff --git a/incrementalmerkletree/src/lib.rs b/incrementalmerkletree/src/lib.rs index 14aee4b..31f05b3 100644 --- a/incrementalmerkletree/src/lib.rs +++ b/incrementalmerkletree/src/lib.rs @@ -6,6 +6,8 @@ use std::convert::{TryFrom, TryInto}; use std::num::TryFromIntError; use std::ops::{Add, AddAssign, Range, Sub}; +pub mod frontier; + #[cfg(feature = "test-dependencies")] pub mod testing; @@ -42,6 +44,47 @@ impl Retention { } } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum Source { + /// The sibling to the address can be derived from the incremental frontier + /// at the contained ommer index + Past(usize), + /// The sibling to the address must be obtained from values discovered by + /// the addition of more nodes to the tree + Future, +} + +#[must_use = "iterators are lazy and do nothing unless consumed"] +struct WitnessAddrsIter { + root_level: Level, + current: Address, + ommer_count: usize, +} + +impl Iterator for WitnessAddrsIter { + type Item = (Address, Source); + + fn next(&mut self) -> Option<(Address, Source)> { + if self.current.level() < self.root_level { + let current = self.current; + let source = if current.is_right_child() { + Source::Past(self.ommer_count) + } else { + Source::Future + }; + + self.current = current.parent(); + if matches!(source, Source::Past(_)) { + self.ommer_count += 1; + } + + Some((current.sibling(), source)) + } else { + None + } + } +} + /// A type representing the position of a leaf in a Merkle tree. #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(transparent)] @@ -73,6 +116,18 @@ impl Position { pub fn is_complete_subtree(&self, root_level: Level) -> bool { !(0..(root_level.0)).any(|l| self.0 & (1 << l) == 0) } + + /// Returns an iterator over the addresses of nodes required to create a witness for this + /// position, beginning with the sibling of the leaf at this position and ending with the + /// sibling of the ancestor of the leaf at this position that is required to compute a root at + /// the specified level. + pub fn witness_addrs(&self, root_level: Level) -> impl Iterator { + WitnessAddrsIter { + root_level, + current: Address::from(*self), + ommer_count: 0, + } + } } impl From for usize { @@ -419,7 +474,7 @@ pub trait Hashable: Sized + core::fmt::Debug { #[cfg(test)] pub(crate) mod tests { - use super::{Address, Level, Position}; + use super::{Address, Level, Position, Source}; use core::ops::Range; use either::Either; @@ -458,7 +513,57 @@ pub(crate) mod tests { } #[test] - fn current_incomplete() { + fn position_witness_addrs() { + use Source::*; + let path_elem = |l, i, s| (Address::from_parts(Level::from(l), i), s); + assert_eq!( + vec![path_elem(0, 1, Future), path_elem(1, 1, Future)], + Position::from(0) + .witness_addrs(Level::from(2)) + .collect::>() + ); + assert_eq!( + vec![path_elem(0, 3, Future), path_elem(1, 0, Past(0))], + Position::from(2) + .witness_addrs(Level::from(2)) + .collect::>() + ); + assert_eq!( + vec![ + path_elem(0, 2, Past(0)), + path_elem(1, 0, Past(1)), + path_elem(2, 1, Future) + ], + Position::from(3) + .witness_addrs(Level::from(3)) + .collect::>() + ); + assert_eq!( + vec![ + path_elem(0, 5, Future), + path_elem(1, 3, Future), + path_elem(2, 0, Past(0)), + path_elem(3, 1, Future) + ], + Position::from(4) + .witness_addrs(Level::from(4)) + .collect::>() + ); + assert_eq!( + vec![ + path_elem(0, 7, Future), + path_elem(1, 2, Past(0)), + path_elem(2, 0, Past(1)), + path_elem(3, 1, Future) + ], + Position::from(6) + .witness_addrs(Level::from(4)) + .collect::>() + ); + } + + #[test] + fn address_current_incomplete() { let addr = |l, i| Address::from_parts(Level(l), i); assert_eq!(addr(0, 0), addr(0, 0).current_incomplete()); assert_eq!(addr(1, 0), addr(0, 1).current_incomplete()); @@ -467,7 +572,7 @@ pub(crate) mod tests { } #[test] - fn next_incomplete_parent() { + fn address_next_incomplete_parent() { let addr = |l, i| Address::from_parts(Level(l), i); assert_eq!(addr(1, 0), addr(0, 0).next_incomplete_parent()); assert_eq!(addr(1, 0), addr(0, 1).next_incomplete_parent()); From 7cb2bdcc8f2c8de771e743c63926fbba609cd058 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 16 Feb 2023 08:43:29 -0700 Subject: [PATCH 3/7] Add legacy `CommitmentTree` and `IncrementalWitness` types. These types were previously part of the `zcash_primitives` crate and are being included here to provide a migration path for when these types are removed from `zcash_primitives`. --- incrementalmerkletree/CHANGELOG.md | 7 +- incrementalmerkletree/Cargo.toml | 1 + incrementalmerkletree/src/frontier.rs | 345 +++++++++++++++++++++++++- incrementalmerkletree/src/lib.rs | 10 + incrementalmerkletree/src/witness.rs | 239 ++++++++++++++++++ 5 files changed, 600 insertions(+), 2 deletions(-) create mode 100644 incrementalmerkletree/src/witness.rs diff --git a/incrementalmerkletree/CHANGELOG.md b/incrementalmerkletree/CHANGELOG.md index 39135aa..08281ff 100644 --- a/incrementalmerkletree/CHANGELOG.md +++ b/incrementalmerkletree/CHANGELOG.md @@ -11,7 +11,12 @@ and this project adheres to Rust's notion of - `incrementalmerkletree::frontier` Types that model the state at the rightmost node of a Merkle tree that is filled sequentially from the left. These have been migrated here from the `bridgetree` crate as they are useful outside - of the context of the `bridgetree` data structures. + of the context of the `bridgetree` data structures. Additional legacy types + used for this modeling have been moved here from the `librustzcash` crate; + these migrated types are available under a `legacy-api` feature flag. +- `incrementalmerkletree::witness` Types migrated from `librustzcash` under + the `legacy-api` feature flag related to constructing witnesses for leaves + of a Merkle tree. ## [0.3.1] - 2023-02-28 diff --git a/incrementalmerkletree/Cargo.toml b/incrementalmerkletree/Cargo.toml index 92c709f..45a7c39 100644 --- a/incrementalmerkletree/Cargo.toml +++ b/incrementalmerkletree/Cargo.toml @@ -21,4 +21,5 @@ proptest = { version = "1.0.0", optional = true } proptest = "1.0.0" [features] +legacy-api = [] test-dependencies = ["proptest"] diff --git a/incrementalmerkletree/src/frontier.rs b/incrementalmerkletree/src/frontier.rs index 6d64a48..e60e21e 100644 --- a/incrementalmerkletree/src/frontier.rs +++ b/incrementalmerkletree/src/frontier.rs @@ -3,6 +3,9 @@ use std::mem::size_of; use crate::{Address, Hashable, Level, Position, Source}; +#[cfg(feature = "legacy-api")] +use {std::collections::VecDeque, std::iter::repeat}; + /// Validation errors that can occur during reconstruction of a Merkle frontier from /// its constituent parts. #[derive(Clone, Debug, PartialEq, Eq)] @@ -244,9 +247,227 @@ impl Frontier { } } +#[cfg(feature = "legacy-api")] +pub(crate) struct PathFiller { + pub(crate) queue: VecDeque, +} + +#[cfg(feature = "legacy-api")] +impl PathFiller { + pub(crate) fn empty() -> Self { + PathFiller { + queue: VecDeque::new(), + } + } + + pub(crate) fn next(&mut self, level: Level) -> H { + self.queue + .pop_front() + .unwrap_or_else(|| H::empty_root(level)) + } +} + +/// A Merkle tree of note commitments. +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg(feature = "legacy-api")] +pub struct CommitmentTree { + pub(crate) left: Option, + pub(crate) right: Option, + pub(crate) parents: Vec>, +} + +#[cfg(feature = "legacy-api")] +impl CommitmentTree { + /// Creates an empty tree. + pub fn empty() -> Self { + CommitmentTree { + left: None, + right: None, + parents: vec![], + } + } + + pub fn from_frontier(frontier: &Frontier) -> Self + where + H: Clone, + { + frontier.value().map_or_else(Self::empty, |f| { + let mut ommers_iter = f.ommers().iter().cloned(); + let (left, right) = if f.position().is_odd() { + ( + ommers_iter + .next() + .expect("An ommer must exist if the frontier position is odd"), + Some(f.leaf().clone()), + ) + } else { + (f.leaf().clone(), None) + }; + + let upos: usize = f.position().into(); + Self { + left: Some(left), + right, + parents: (1u8..DEPTH) + .into_iter() + .map(|i| { + if upos & (1 << i) == 0 { + None + } else { + ommers_iter.next() + } + }) + .collect(), + } + }) + } + + pub fn to_frontier(&self) -> Frontier + where + H: Hashable + Clone, + { + if self.size() == 0 { + Frontier::empty() + } else { + let ommers_iter = self.parents.iter().filter_map(|v| v.as_ref()).cloned(); + let (leaf, ommers) = match (self.left.as_ref(), self.right.as_ref()) { + (Some(a), None) => (a.clone(), ommers_iter.collect()), + (Some(a), Some(b)) => ( + b.clone(), + Some(a.clone()).into_iter().chain(ommers_iter).collect(), + ), + _ => unreachable!(), + }; + + // If a frontier cannot be successfully constructed from the + // parts of a commitment tree, it is a programming error. + Frontier::from_parts((self.size() - 1).into(), leaf, ommers) + .expect("Frontier should be constructable from CommitmentTree.") + } + } + + /// Returns the number of leaf nodes in the tree. + pub fn size(&self) -> usize { + self.parents.iter().enumerate().fold( + match (self.left.as_ref(), self.right.as_ref()) { + (None, None) => 0, + (Some(_), None) => 1, + (Some(_), Some(_)) => 2, + (None, Some(_)) => unreachable!(), + }, + |acc, (i, p)| { + // Treat occupation of parents array as a binary number + // (right-shifted by 1) + acc + if p.is_some() { 1 << (i + 1) } else { 0 } + }, + ) + } + + pub(crate) fn is_complete(&self, depth: u8) -> bool { + if depth == 0 { + self.left.is_some() && self.right.is_none() && self.parents.is_empty() + } else { + self.left.is_some() + && self.right.is_some() + && self + .parents + .iter() + .chain(repeat(&None)) + .take((depth - 1).into()) + .all(|p| p.is_some()) + } + } +} + +#[cfg(feature = "legacy-api")] +impl CommitmentTree { + /// Adds a leaf node to the tree. + /// + /// Returns an error if the tree is full. + #[allow(clippy::result_unit_err)] + pub fn append(&mut self, node: H) -> Result<(), ()> { + if self.is_complete(DEPTH) { + // Tree is full + return Err(()); + } + + match (&self.left, &self.right) { + (None, _) => self.left = Some(node), + (_, None) => self.right = Some(node), + (Some(l), Some(r)) => { + let mut combined = H::combine(0.into(), l, r); + self.left = Some(node); + self.right = None; + + for i in 0..DEPTH { + let i_usize = usize::from(i); + if i_usize < self.parents.len() { + if let Some(p) = &self.parents[i_usize] { + combined = H::combine((i + 1).into(), p, &combined); + self.parents[i_usize] = None; + } else { + self.parents[i_usize] = Some(combined); + break; + } + } else { + self.parents.push(Some(combined)); + break; + } + } + } + } + + Ok(()) + } + + /// Returns the current root of the tree. + pub fn root(&self) -> H { + self.root_inner(DEPTH, PathFiller::empty()) + } + + pub(crate) fn root_inner(&self, depth: u8, mut filler: PathFiller) -> H { + assert!(depth > 0); + + // 1) Hash left and right leaves together. + // - Empty leaves are used as needed. + // - Note that `filler.next` is side-effecting and so cannot be factored out. + let leaf_root = H::combine( + 0.into(), + &self + .left + .as_ref() + .map_or_else(|| filler.next(0.into()), |n| n.clone()), + &self + .right + .as_ref() + .map_or_else(|| filler.next(0.into()), |n| n.clone()), + ); + + // 2) Extend the parents to the desired depth with None values, then hash from leaf to + // root. Roots of the empty subtrees are used as needed. + self.parents + .iter() + .chain(repeat(&None)) + .take((depth - 1).into()) + .enumerate() + .fold(leaf_root, |root, (i, p)| { + let level = Level::from(i as u8 + 1); + match p { + Some(node) => H::combine(level, node, &root), + None => H::combine(level, &root, &filler.next(level)), + } + }) + } +} + #[cfg(feature = "test-dependencies")] pub mod testing { - use crate::Hashable; + use core::fmt::Debug; + use proptest::prelude::*; + use std::collections::hash_map::DefaultHasher; + use std::hash::Hasher; + + use crate::{Hashable, Level}; impl crate::testing::Frontier for super::Frontier @@ -259,12 +480,65 @@ pub mod testing { super::Frontier::root(self) } } + + #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] + pub struct TestNode(pub u64); + + impl Hashable for TestNode { + fn empty_leaf() -> Self { + Self(0) + } + + fn combine(level: Level, a: &Self, b: &Self) -> Self { + let mut hasher = DefaultHasher::new(); + hasher.write_u8(level.into()); + hasher.write_u64(a.0); + hasher.write_u64(b.0); + Self(hasher.finish()) + } + } + + prop_compose! { + pub fn arb_test_node()(i in any::()) -> TestNode { + TestNode(i) + } + } + + #[cfg(feature = "legacy-api")] + use {crate::frontier::CommitmentTree, proptest::collection::vec}; + + #[cfg(feature = "legacy-api")] + pub fn arb_commitment_tree< + H: Hashable + Clone + Debug, + T: Strategy, + const DEPTH: u8, + >( + min_size: usize, + arb_node: T, + ) -> impl Strategy> { + assert!((1 << DEPTH) >= min_size + 100); + vec(arb_node, min_size..(min_size + 100)).prop_map(move |v| { + let mut tree = CommitmentTree::empty(); + for node in v.into_iter() { + tree.append(node).unwrap(); + } + tree.parents.resize_with((DEPTH - 1).into(), || None); + tree + }) + } } #[cfg(test)] mod tests { + use super::*; + #[cfg(feature = "legacy-api")] + use { + super::testing::{arb_commitment_tree, arb_test_node, TestNode}, + proptest::prelude::*, + }; + #[test] fn nonempty_frontier_root() { let mut frontier = NonEmptyFrontier::new("a".to_string()); @@ -319,4 +593,73 @@ mod tests { frontier.witness(4, bridge_value_at) ); } + + #[test] + #[cfg(feature = "legacy-api")] + fn test_commitment_tree_complete() { + let mut t: CommitmentTree = CommitmentTree::empty(); + for n in 1u64..=32 { + t.append(TestNode(n)).unwrap(); + // every tree of a power-of-two height is complete + let is_complete = n.count_ones() == 1; + let level = usize::BITS - 1 - n.leading_zeros(); //log2 + assert_eq!( + is_complete, + t.is_complete(level.try_into().unwrap()), + "Tree {:?} {} complete at height {}", + t, + if is_complete { + "should be" + } else { + "should not be" + }, + n + ); + } + } + + #[test] + #[cfg(feature = "legacy-api")] + fn test_commitment_tree_roundtrip() { + let ct = CommitmentTree { + left: Some("a".to_string()), + right: Some("b".to_string()), + parents: vec![ + Some("c".to_string()), + Some("d".to_string()), + Some("e".to_string()), + Some("f".to_string()), + None, + None, + None, + ], + }; + + let frontier: Frontier = ct.to_frontier(); + let ct0 = CommitmentTree::from_frontier(&frontier); + assert_eq!(ct, ct0); + let frontier0: Frontier = ct0.to_frontier(); + assert_eq!(frontier, frontier0); + } + + #[cfg(feature = "legacy-api")] + proptest! { + #[test] + fn prop_commitment_tree_roundtrip(ct in arb_commitment_tree(32, arb_test_node())) { + let frontier: Frontier = ct.to_frontier(); + let ct0 = CommitmentTree::from_frontier(&frontier); + assert_eq!(ct, ct0); + let frontier0: Frontier = ct0.to_frontier(); + assert_eq!(frontier, frontier0); + } + + #[test] + fn prop_commitment_tree_roundtrip_str(ct in arb_commitment_tree::<_, _, 8>(32, any::().prop_map(|c| c.to_string()))) { + let frontier: Frontier = ct.to_frontier(); + let ct0 = CommitmentTree::from_frontier(&frontier); + assert_eq!(ct, ct0); + let frontier0: Frontier = ct0.to_frontier(); + assert_eq!(frontier, frontier0); + } + } } diff --git a/incrementalmerkletree/src/lib.rs b/incrementalmerkletree/src/lib.rs index 31f05b3..c88ddfc 100644 --- a/incrementalmerkletree/src/lib.rs +++ b/incrementalmerkletree/src/lib.rs @@ -8,6 +8,9 @@ use std::ops::{Add, AddAssign, Range, Sub}; pub mod frontier; +#[cfg(feature = "legacy-api")] +pub mod witness; + #[cfg(feature = "test-dependencies")] pub mod testing; @@ -219,6 +222,13 @@ impl From for usize { } } +impl TryFrom for Level { + type Error = TryFromIntError; + fn try_from(sz: usize) -> Result { + ::try_from(sz).map(Self) + } +} + impl Sub for Level { type Output = Self; fn sub(self, value: u8) -> Self { diff --git a/incrementalmerkletree/src/witness.rs b/incrementalmerkletree/src/witness.rs new file mode 100644 index 0000000..97147b4 --- /dev/null +++ b/incrementalmerkletree/src/witness.rs @@ -0,0 +1,239 @@ +use std::convert::TryInto; +use std::iter::repeat; + +use crate::{ + frontier::{CommitmentTree, PathFiller}, + Hashable, Level, +}; + +/// A path from a position in a particular commitment tree to the root of that tree. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MerklePath { + auth_path: Vec<(H, bool)>, + position: u64, +} + +impl MerklePath { + pub fn auth_path(&self) -> &[(H, bool)] { + &self.auth_path + } + + pub fn position(&self) -> u64 { + self.position + } + + /// Constructs a Merkle path directly from a path and position. + pub fn from_path(auth_path: Vec<(H, bool)>, position: u64) -> Result { + if auth_path.len() == usize::from(DEPTH) { + Ok(MerklePath { + auth_path, + position, + }) + } else { + Err(()) + } + } +} + +impl MerklePath { + /// Returns the root of the tree corresponding to this path applied to `leaf`. + pub fn root(&self, leaf: H) -> H { + self.auth_path + .iter() + .enumerate() + .fold(leaf, |root, (i, (p, leaf_is_on_right))| { + let level = u8::try_from(i) + .expect("Parents list length may not exceed what is representable by an u8") + .into(); + match leaf_is_on_right { + false => H::combine(level, &root, p), + true => H::combine(level, p, &root), + } + }) + } +} + +/// An updatable witness to a path from a position in a particular [`CommitmentTree`]. +/// +/// Appending the same commitments in the same order to both the original +/// [`CommitmentTree`] and this `IncrementalWitness` will result in a witness to the path +/// from the target position to the root of the updated tree. +/// +/// # Examples +/// +/// ``` +/// use incrementalmerkletree::{ +/// frontier::{CommitmentTree, testing::TestNode}, +/// witness::IncrementalWitness, +/// }; +/// +/// let mut tree = CommitmentTree::::empty(); +/// +/// tree.append(TestNode(0)); +/// tree.append(TestNode(1)); +/// let mut witness = IncrementalWitness::from_tree(tree.clone()); +/// assert_eq!(witness.position(), 1); +/// assert_eq!(tree.root(), witness.root()); +/// +/// let next = TestNode(2); +/// tree.append(next.clone()); +/// witness.append(next); +/// assert_eq!(tree.root(), witness.root()); +/// ``` +#[derive(Clone, Debug)] +pub struct IncrementalWitness { + tree: CommitmentTree, + filled: Vec, + cursor_depth: u8, + cursor: Option>, +} + +impl IncrementalWitness { + /// Creates an `IncrementalWitness` for the most recent commitment added to the given + /// [`CommitmentTree`]. + pub fn from_tree(tree: CommitmentTree) -> IncrementalWitness { + IncrementalWitness { + tree, + filled: vec![], + cursor_depth: 0, + cursor: None, + } + } + + /// Returns the position of the witnessed leaf node in the commitment tree. + pub fn position(&self) -> usize { + self.tree.size() - 1 + } + + /// Finds the next "depth" of an unfilled subtree. + fn next_depth(&self) -> u8 { + let mut skip: u8 = self + .filled + .len() + .try_into() + .expect("Merkle tree depths may not exceed the bounds of a u8"); + + if self.tree.left.is_none() { + if skip > 0 { + skip -= 1; + } else { + return 0; + } + } + + if self.tree.right.is_none() { + if skip > 0 { + skip -= 1; + } else { + return 0; + } + } + + let mut d = 1; + for p in &self.tree.parents { + if p.is_none() { + if skip > 0 { + skip -= 1; + } else { + return d; + } + } + d += 1; + } + + d + skip + } +} + +impl IncrementalWitness { + fn filler(&self) -> PathFiller { + let cursor_root = self + .cursor + .as_ref() + .map(|c| c.root_inner(self.cursor_depth, PathFiller::empty())); + + PathFiller { + queue: self.filled.iter().cloned().chain(cursor_root).collect(), + } + } + + /// Tracks a leaf node that has been added to the underlying tree. + /// + /// Returns an error if the tree is full. + #[allow(clippy::result_unit_err)] + pub fn append(&mut self, node: H) -> Result<(), ()> { + if let Some(mut cursor) = self.cursor.take() { + cursor.append(node).expect("cursor should not be full"); + if cursor.is_complete(self.cursor_depth) { + self.filled + .push(cursor.root_inner(self.cursor_depth, PathFiller::empty())); + } else { + self.cursor = Some(cursor); + } + } else { + self.cursor_depth = self.next_depth(); + if self.cursor_depth >= DEPTH { + // Tree is full + return Err(()); + } + + if self.cursor_depth == 0 { + self.filled.push(node); + } else { + let mut cursor = CommitmentTree::empty(); + cursor.append(node).expect("cursor should not be full"); + self.cursor = Some(cursor); + } + } + + Ok(()) + } + + /// Returns the current root of the tree corresponding to the witness. + pub fn root(&self) -> H { + self.root_inner(DEPTH) + } + + fn root_inner(&self, depth: u8) -> H { + self.tree.root_inner(depth, self.filler()) + } + + /// Returns the current witness, or None if the tree is empty. + pub fn path(&self) -> Option> { + self.path_inner(DEPTH) + } + + fn path_inner(&self, depth: u8) -> Option> { + let mut filler = self.filler(); + let mut auth_path = Vec::new(); + + if let Some(node) = &self.tree.left { + if self.tree.right.is_some() { + auth_path.push((node.clone(), true)); + } else { + auth_path.push((filler.next(0.into()), false)); + } + } else { + // Can't create an authentication path for the beginning of the tree + return None; + } + + for (i, p) in self + .tree + .parents + .iter() + .chain(repeat(&None)) + .take((depth - 1).into()) + .enumerate() + { + auth_path.push(match p { + Some(node) => (node.clone(), true), + None => (filler.next(Level::from((i + 1) as u8)), false), + }); + } + + assert_eq!(auth_path.len(), usize::from(depth)); + + MerklePath::from_path(auth_path, self.position() as u64).ok() + } +} From 71557cc0e363dbdd514b84a4907d44efeea42d46 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 16 Mar 2023 11:53:48 -0600 Subject: [PATCH 4/7] Expose operations on `legacy-api` types required for serialization & testing. --- incrementalmerkletree/src/frontier.rs | 144 +++++++++++++++----------- incrementalmerkletree/src/witness.rs | 65 ++++++++---- 2 files changed, 129 insertions(+), 80 deletions(-) diff --git a/incrementalmerkletree/src/frontier.rs b/incrementalmerkletree/src/frontier.rs index e60e21e..a531aeb 100644 --- a/incrementalmerkletree/src/frontier.rs +++ b/incrementalmerkletree/src/frontier.rs @@ -248,19 +248,23 @@ impl Frontier { } #[cfg(feature = "legacy-api")] -pub(crate) struct PathFiller { - pub(crate) queue: VecDeque, +pub struct PathFiller { + queue: VecDeque, } #[cfg(feature = "legacy-api")] impl PathFiller { - pub(crate) fn empty() -> Self { + pub fn empty() -> Self { PathFiller { queue: VecDeque::new(), } } - pub(crate) fn next(&mut self, level: Level) -> H { + pub fn new(queue: VecDeque) -> Self { + Self { queue } + } + + pub fn next(&mut self, level: Level) -> H { self.queue .pop_front() .unwrap_or_else(|| H::empty_root(level)) @@ -287,63 +291,32 @@ impl CommitmentTree { } } - pub fn from_frontier(frontier: &Frontier) -> Self - where - H: Clone, - { - frontier.value().map_or_else(Self::empty, |f| { - let mut ommers_iter = f.ommers().iter().cloned(); - let (left, right) = if f.position().is_odd() { - ( - ommers_iter - .next() - .expect("An ommer must exist if the frontier position is odd"), - Some(f.leaf().clone()), - ) - } else { - (f.leaf().clone(), None) - }; - - let upos: usize = f.position().into(); - Self { - left: Some(left), + pub fn from_parts( + left: Option, + right: Option, + parents: Vec>, + ) -> Result { + if parents.len() < usize::from(DEPTH) { + Ok(CommitmentTree { + left, right, - parents: (1u8..DEPTH) - .into_iter() - .map(|i| { - if upos & (1 << i) == 0 { - None - } else { - ommers_iter.next() - } - }) - .collect(), - } - }) + parents, + }) + } else { + Err(()) + } } - pub fn to_frontier(&self) -> Frontier - where - H: Hashable + Clone, - { - if self.size() == 0 { - Frontier::empty() - } else { - let ommers_iter = self.parents.iter().filter_map(|v| v.as_ref()).cloned(); - let (leaf, ommers) = match (self.left.as_ref(), self.right.as_ref()) { - (Some(a), None) => (a.clone(), ommers_iter.collect()), - (Some(a), Some(b)) => ( - b.clone(), - Some(a.clone()).into_iter().chain(ommers_iter).collect(), - ), - _ => unreachable!(), - }; + pub fn left(&self) -> &Option { + &self.left + } - // If a frontier cannot be successfully constructed from the - // parts of a commitment tree, it is a programming error. - Frontier::from_parts((self.size() - 1).into(), leaf, ommers) - .expect("Frontier should be constructable from CommitmentTree.") - } + pub fn right(&self) -> &Option { + &self.right + } + + pub fn parents(&self) -> &Vec> { + &self.parents } /// Returns the number of leaf nodes in the tree. @@ -381,6 +354,59 @@ impl CommitmentTree { #[cfg(feature = "legacy-api")] impl CommitmentTree { + pub fn from_frontier(frontier: &Frontier) -> Self { + frontier.value().map_or_else(Self::empty, |f| { + let mut ommers_iter = f.ommers().iter().cloned(); + let (left, right) = if f.position().is_odd() { + ( + ommers_iter + .next() + .expect("An ommer must exist if the frontier position is odd"), + Some(f.leaf().clone()), + ) + } else { + (f.leaf().clone(), None) + }; + + let upos: usize = f.position().into(); + Self { + left: Some(left), + right, + parents: (1u8..DEPTH) + .into_iter() + .map(|i| { + if upos & (1 << i) == 0 { + None + } else { + ommers_iter.next() + } + }) + .collect(), + } + }) + } + + pub fn to_frontier(&self) -> Frontier { + if self.size() == 0 { + Frontier::empty() + } else { + let ommers_iter = self.parents.iter().filter_map(|v| v.as_ref()).cloned(); + let (leaf, ommers) = match (self.left.as_ref(), self.right.as_ref()) { + (Some(a), None) => (a.clone(), ommers_iter.collect()), + (Some(a), Some(b)) => ( + b.clone(), + Some(a.clone()).into_iter().chain(ommers_iter).collect(), + ), + _ => unreachable!(), + }; + + // If a frontier cannot be successfully constructed from the + // parts of a commitment tree, it is a programming error. + Frontier::from_parts((self.size() - 1).into(), leaf, ommers) + .expect("Frontier should be constructable from CommitmentTree.") + } + } + /// Adds a leaf node to the tree. /// /// Returns an error if the tree is full. @@ -422,10 +448,10 @@ impl CommitmentTree { /// Returns the current root of the tree. pub fn root(&self) -> H { - self.root_inner(DEPTH, PathFiller::empty()) + self.root_at_depth(DEPTH, PathFiller::empty()) } - pub(crate) fn root_inner(&self, depth: u8, mut filler: PathFiller) -> H { + pub fn root_at_depth(&self, depth: u8, mut filler: PathFiller) -> H { assert!(depth > 0); // 1) Hash left and right leaves together. diff --git a/incrementalmerkletree/src/witness.rs b/incrementalmerkletree/src/witness.rs index 97147b4..cf9d765 100644 --- a/incrementalmerkletree/src/witness.rs +++ b/incrementalmerkletree/src/witness.rs @@ -14,16 +14,8 @@ pub struct MerklePath { } impl MerklePath { - pub fn auth_path(&self) -> &[(H, bool)] { - &self.auth_path - } - - pub fn position(&self) -> u64 { - self.position - } - /// Constructs a Merkle path directly from a path and position. - pub fn from_path(auth_path: Vec<(H, bool)>, position: u64) -> Result { + pub fn from_parts(auth_path: Vec<(H, bool)>, position: u64) -> Result { if auth_path.len() == usize::from(DEPTH) { Ok(MerklePath { auth_path, @@ -33,6 +25,14 @@ impl MerklePath { Err(()) } } + + pub fn auth_path(&self) -> &[(H, bool)] { + &self.auth_path + } + + pub fn position(&self) -> u64 { + self.position + } } impl MerklePath { @@ -91,7 +91,7 @@ pub struct IncrementalWitness { impl IncrementalWitness { /// Creates an `IncrementalWitness` for the most recent commitment added to the given /// [`CommitmentTree`]. - pub fn from_tree(tree: CommitmentTree) -> IncrementalWitness { + pub fn from_tree(tree: CommitmentTree) -> Self { IncrementalWitness { tree, filled: vec![], @@ -100,6 +100,35 @@ impl IncrementalWitness { } } + pub fn from_parts( + tree: CommitmentTree, + filled: Vec, + cursor: Option>, + ) -> Self { + let mut witness = IncrementalWitness { + tree, + filled, + cursor_depth: 0, + cursor, + }; + + witness.cursor_depth = witness.next_depth(); + + witness + } + + pub fn tree(&self) -> &CommitmentTree { + &self.tree + } + + pub fn filled(&self) -> &Vec { + &self.filled + } + + pub fn cursor(&self) -> &Option> { + &self.cursor + } + /// Returns the position of the witnessed leaf node in the commitment tree. pub fn position(&self) -> usize { self.tree.size() - 1 @@ -150,11 +179,9 @@ impl IncrementalWitness { let cursor_root = self .cursor .as_ref() - .map(|c| c.root_inner(self.cursor_depth, PathFiller::empty())); + .map(|c| c.root_at_depth(self.cursor_depth, PathFiller::empty())); - PathFiller { - queue: self.filled.iter().cloned().chain(cursor_root).collect(), - } + PathFiller::new(self.filled.iter().cloned().chain(cursor_root).collect()) } /// Tracks a leaf node that has been added to the underlying tree. @@ -166,7 +193,7 @@ impl IncrementalWitness { cursor.append(node).expect("cursor should not be full"); if cursor.is_complete(self.cursor_depth) { self.filled - .push(cursor.root_inner(self.cursor_depth, PathFiller::empty())); + .push(cursor.root_at_depth(self.cursor_depth, PathFiller::empty())); } else { self.cursor = Some(cursor); } @@ -191,11 +218,7 @@ impl IncrementalWitness { /// Returns the current root of the tree corresponding to the witness. pub fn root(&self) -> H { - self.root_inner(DEPTH) - } - - fn root_inner(&self, depth: u8) -> H { - self.tree.root_inner(depth, self.filler()) + self.tree.root_at_depth(DEPTH, self.filler()) } /// Returns the current witness, or None if the tree is empty. @@ -234,6 +257,6 @@ impl IncrementalWitness { assert_eq!(auth_path.len(), usize::from(depth)); - MerklePath::from_path(auth_path, self.position() as u64).ok() + MerklePath::from_parts(auth_path, self.position() as u64).ok() } } From 30a1b7b62b0aa3b6fedb9a44e61270aa2d54061d Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 16 Mar 2023 17:19:08 -0600 Subject: [PATCH 5/7] Fix clippy complaints. --- incrementalmerkletree/src/frontier.rs | 1 + incrementalmerkletree/src/witness.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/incrementalmerkletree/src/frontier.rs b/incrementalmerkletree/src/frontier.rs index a531aeb..30e6d04 100644 --- a/incrementalmerkletree/src/frontier.rs +++ b/incrementalmerkletree/src/frontier.rs @@ -291,6 +291,7 @@ impl CommitmentTree { } } + #[allow(clippy::result_unit_err)] pub fn from_parts( left: Option, right: Option, diff --git a/incrementalmerkletree/src/witness.rs b/incrementalmerkletree/src/witness.rs index cf9d765..88a9abd 100644 --- a/incrementalmerkletree/src/witness.rs +++ b/incrementalmerkletree/src/witness.rs @@ -15,6 +15,7 @@ pub struct MerklePath { impl MerklePath { /// Constructs a Merkle path directly from a path and position. + #[allow(clippy::result_unit_err)] pub fn from_parts(auth_path: Vec<(H, bool)>, position: u64) -> Result { if auth_path.len() == usize::from(DEPTH) { Ok(MerklePath { From 02341ece264d189cbdfe17210246937d68c03d13 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 27 Mar 2023 09:54:21 -0600 Subject: [PATCH 6/7] Ensure that `BridgeTree` state is checkpointed when constructing from a frontier. --- bridgetree/src/lib.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bridgetree/src/lib.rs b/bridgetree/src/lib.rs index 7cba360..151e8ce 100644 --- a/bridgetree/src/lib.rs +++ b/bridgetree/src/lib.rs @@ -536,8 +536,8 @@ impl BridgeTree { impl BridgeTree { /// Construct a new BridgeTree that will start recording changes from the state of /// the specified frontier. - pub fn from_frontier(max_checkpoints: usize, frontier: NonEmptyFrontier) -> Self { - Self { + pub fn from_frontier(max_checkpoints: usize, frontier: NonEmptyFrontier, checkpoint_id: C) -> Self { + let mut bridge = Self { prior_bridges: vec![], current_bridge: Some(MerkleBridge::from_parts( None, @@ -548,7 +548,9 @@ impl BridgeTree Date: Fri, 31 Mar 2023 16:24:15 -0600 Subject: [PATCH 7/7] Apply suggestions from code review. Co-authored-by: Daira Hopwood --- bridgetree/src/lib.rs | 6 +++++- incrementalmerkletree/src/frontier.rs | 11 ++++++----- incrementalmerkletree/src/lib.rs | 6 +----- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/bridgetree/src/lib.rs b/bridgetree/src/lib.rs index 40860fa..acb815d 100644 --- a/bridgetree/src/lib.rs +++ b/bridgetree/src/lib.rs @@ -535,7 +535,11 @@ impl BridgeTree { impl BridgeTree { /// Construct a new BridgeTree that will start recording changes from the state of /// the specified frontier. - pub fn from_frontier(max_checkpoints: usize, frontier: NonEmptyFrontier, checkpoint_id: C) -> Self { + pub fn from_frontier( + max_checkpoints: usize, + frontier: NonEmptyFrontier, + checkpoint_id: C, + ) -> Self { let mut bridge = Self { prior_bridges: vec![], current_bridge: Some(MerkleBridge::from_parts( diff --git a/incrementalmerkletree/src/frontier.rs b/incrementalmerkletree/src/frontier.rs index 30e6d04..5cb8c1f 100644 --- a/incrementalmerkletree/src/frontier.rs +++ b/incrementalmerkletree/src/frontier.rs @@ -15,7 +15,8 @@ pub enum FrontierError { PositionMismatch { expected_ommers: usize }, /// An error representing that the position and/or list of ommers provided to frontier /// construction would result in a frontier that exceeds the maximum statically allowed depth - /// of the tree. + /// of the tree. `depth` is the minimum tree depth that would be required in order for that + /// tree to contain the position in question. MaxDepthExceeded { depth: u8 }, } @@ -39,7 +40,7 @@ impl NonEmptyFrontier { } } - /// Constructs a new frontier from its constituent parts + /// Constructs a new frontier from its constituent parts. pub fn from_parts(position: Position, leaf: H, ommers: Vec) -> Result { let expected_ommers = position.past_ommer_count(); if ommers.len() == expected_ommers { @@ -58,7 +59,7 @@ impl NonEmptyFrontier { self.position } - /// Returns the leaf most recently appended to the frontier + /// Returns the leaf most recently appended to the frontier. pub fn leaf(&self) -> &H { &self.leaf } @@ -71,8 +72,8 @@ impl NonEmptyFrontier { } impl NonEmptyFrontier { - /// Append a new leaf to the frontier, and recompute recompute ommers by hashing together full - /// subtrees until an empty ommer slot is found. + /// Append a new leaf to the frontier, and recompute ommers by hashing together full subtrees + /// until an empty ommer slot is found. pub fn append(&mut self, leaf: H) { let prior_position = self.position; let prior_leaf = self.leaf.clone(); diff --git a/incrementalmerkletree/src/lib.rs b/incrementalmerkletree/src/lib.rs index 1352379..236b4e7 100644 --- a/incrementalmerkletree/src/lib.rs +++ b/incrementalmerkletree/src/lib.rs @@ -289,11 +289,7 @@ impl Address { pub fn sibling(&self) -> Address { Address { level: self.level, - index: if self.index & 0x1 == 0 { - self.index + 1 - } else { - self.index - 1 - }, + index: self.index ^ 1, } }