From fef054d43a9581d3b3cf4424b6cb71e66de4b844 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 15 Apr 2022 10:00:03 -0600 Subject: [PATCH 1/3] Move-only: group witness/remove_witness code together. --- src/bridgetree.rs | 140 +++++++++++++++++++++++----------------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/src/bridgetree.rs b/src/bridgetree.rs index 37907f8..115fb90 100644 --- a/src/bridgetree.rs +++ b/src/bridgetree.rs @@ -896,12 +896,6 @@ impl Tree for BridgeTree< self.current_bridge.as_ref().map(|b| b.current_leaf()) } - fn get_witnessed_leaf(&self, position: Position) -> Option<&H> { - self.saved - .get(&position) - .and_then(|idx| self.prior_bridges.get(*idx).map(|b| b.current_leaf())) - } - fn witness(&mut self) -> Option { match self.current_bridge.take() { Some(mut cur_b) => { @@ -940,70 +934,10 @@ impl Tree for BridgeTree< self.saved.keys().cloned().collect() } - fn authentication_path(&self, position: Position) -> Option> { - self.saved.get(&position).and_then(|idx| { - let frontier = &self.prior_bridges[*idx].frontier; - - // Fuse the following bridges to obtain a bridge that has all - // of the data to the right of the selected value in the tree. - // The unwrap here is safe because a witnessed leaf always - // generates a subsequent bridge in the tree. - MerkleBridge::fuse_all( - self.prior_bridges[(idx + 1)..] - .iter() - .chain(self.current_bridge.iter()), - ) - .map(|fused| { - // construct a complete trailing edge that includes the data from - // the following frontier not yet included in the trailing edge. - let auth_fragment = fused.auth_fragments.get(&frontier.position()); - let rest_frontier = fused.frontier; - - let mut auth_values = auth_fragment.iter().flat_map(|auth_fragment| { - let last_altitude = auth_fragment.next_required_altitude(); - let last_digest = - last_altitude.and_then(|lvl| rest_frontier.witness_incomplete(lvl)); - - // TODO: can we eliminate this .cloned()? - auth_fragment.values.iter().cloned().chain(last_digest) - }); - - let mut result = vec![]; - match &frontier.leaf { - Leaf::Left(_) => { - result.push(auth_values.next().unwrap_or_else(H::empty_leaf)); - } - Leaf::Right(a, _) => { - result.push(a.clone()); - } - } - - for (ommer, ommer_lvl) in frontier - .ommers - .iter() - .zip(frontier.position.ommer_altitudes()) - { - for synth_lvl in (result.len() as u8)..(ommer_lvl.into()) { - result.push( - auth_values - .next() - .unwrap_or_else(|| H::empty_root(Altitude(synth_lvl))), - ) - } - result.push(ommer.clone()); - } - - for synth_lvl in (result.len() as u8)..DEPTH { - result.push( - auth_values - .next() - .unwrap_or_else(|| H::empty_root(Altitude(synth_lvl))), - ); - } - - result - }) - }) + fn get_witnessed_leaf(&self, position: Position) -> Option<&H> { + self.saved + .get(&position) + .and_then(|idx| self.prior_bridges.get(*idx).map(|b| b.current_leaf())) } fn remove_witness(&mut self, position: Position) -> bool { @@ -1073,6 +1007,72 @@ impl Tree for BridgeTree< } } + fn authentication_path(&self, position: Position) -> Option> { + self.saved.get(&position).and_then(|idx| { + let frontier = &self.prior_bridges[*idx].frontier; + + // Fuse the following bridges to obtain a bridge that has all + // of the data to the right of the selected value in the tree. + // The unwrap here is safe because a witnessed leaf always + // generates a subsequent bridge in the tree. + MerkleBridge::fuse_all( + self.prior_bridges[(idx + 1)..] + .iter() + .chain(self.current_bridge.iter()), + ) + .map(|fused| { + // construct a complete trailing edge that includes the data from + // the following frontier not yet included in the trailing edge. + let auth_fragment = fused.auth_fragments.get(&frontier.position()); + let rest_frontier = fused.frontier; + + let mut auth_values = auth_fragment.iter().flat_map(|auth_fragment| { + let last_altitude = auth_fragment.next_required_altitude(); + let last_digest = + last_altitude.and_then(|lvl| rest_frontier.witness_incomplete(lvl)); + + // TODO: can we eliminate this .cloned()? + auth_fragment.values.iter().cloned().chain(last_digest) + }); + + let mut result = vec![]; + match &frontier.leaf { + Leaf::Left(_) => { + result.push(auth_values.next().unwrap_or_else(H::empty_leaf)); + } + Leaf::Right(a, _) => { + result.push(a.clone()); + } + } + + for (ommer, ommer_lvl) in frontier + .ommers + .iter() + .zip(frontier.position.ommer_altitudes()) + { + for synth_lvl in (result.len() as u8)..(ommer_lvl.into()) { + result.push( + auth_values + .next() + .unwrap_or_else(|| H::empty_root(Altitude(synth_lvl))), + ) + } + result.push(ommer.clone()); + } + + for synth_lvl in (result.len() as u8)..DEPTH { + result.push( + auth_values + .next() + .unwrap_or_else(|| H::empty_root(Altitude(synth_lvl))), + ); + } + + result + }) + }) + } + fn garbage_collect(&mut self) { // Only garbage collect once we have more bridges than the maximum number of // checkpoints; we cannot remove information that we might need to restore in From 06728b9499d2b10e86c98d57ef74eb80ce6aaeeb Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 18 Apr 2022 16:01:53 -0600 Subject: [PATCH 2/3] Add an as_of_root argument to Tree::authentication_path This requires a user of this API to specify a root of the tree that identifies the state of the tree at which the user wishes to construct an authentication path. This must be equal to either the current root of the tree, or to the root of the tree at a previous checkpoint. --- CHANGELOG.md | 23 ++ proptest-regressions/lib.txt | 14 -- src/bridgetree.rs | 236 ++++++++++++++---- src/lib.rs | 452 +++++++++++++++++++++++++---------- src/sample.rs | 75 ++++-- 5 files changed, 600 insertions(+), 200 deletions(-) delete mode 100644 proptest-regressions/lib.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index eaadf62..8215209 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,29 @@ and this project adheres to Rust's notion of ## [Unreleased] +### Added + +- `incrementalmerkletree::bridgetree`: + - `Checkpoint::witnessed` returns the set of positions that have been marked with + `BridgeTree::witness` while this checkpoint was the current checkpoint. + +### Changed + +- `incrementalmerkletree`: + - `Tree::authentication_path` has been changed to take an `as_of_root` parameter. + This allows computation of the authentication path as of previous tree states, in + addition to the previous behavior which only allowed computation of the path as of the + most recent tree state. The provided `as_of_root` value must be equal to either the + current root of the tree, or to the root of the tree at a previous checkpoint. + +### Removed + +- `incrementalmerkletree::bridgetree`: + - `Checkpoint::rewrite_indices` was an internal utility method that had inadvertently + been made a part of the public API. + +## [0.3.0-beta.2] - 2022-04-06 + ### Added - `incrementalmerkletree`: - `Tree::get_witnessed_leaf`, to allow a user to query for the leaf value of diff --git a/proptest-regressions/lib.txt b/proptest-regressions/lib.txt deleted file mode 100644 index 07c7718..0000000 --- a/proptest-regressions/lib.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc 6a5f61f970da957baf3be8b5383d71a1510b42e9457cc00509163fde8430f198 # shrinks to ops = [Append(SipHashable(0)), Checkpoint, Witness, Rewind] -cc ed57a2e3d91f2da644f695b8698822f9eaa2b3fd1f5bd039a5753e7027623ed9 # shrinks to ops = [Append("a"), Unwitness("a"), Checkpoint, Witness, Rewind] -cc aa498d80d63c58720a913f2edcc6ba19da462f7a824f5be906e63c7ca1566076 # shrinks to ops = [Append(SipHashable(0)), Checkpoint, Checkpoint, Rewind, Append(SipHashable(0)), Rewind, Append(SipHashable(1))] -cc 7cebce1664b804d55e259c466ebcc140c4a3708f4ec49ed865d6009736b0a568 # shrinks to ops = [Append("d"), Checkpoint, Witness, Unwitness("d"), Rewind, Unwitness("d")] -cc 9078a200bceb14aa2f78743f545dab9719db29ddbbb5f1f67fae01805da90f4b # shrinks to ops = [Append("a"), Unwitness("a"), Checkpoint, Checkpoint, Rewind, Append("a"), Rewind, Append("a")] -cc 20e43678bbf38f4c40fdd9e6f5bbc2b8eebcb1756b599f443f3268d6cf8e9d22 # shrinks to ops = [Append("o"), Checkpoint, Witness, Checkpoint, Unwitness("o"), Rewind, Rewind] -cc 2b82b514115d3ddb4990853b0178e997632226be6b8f7c9cf07e94a3e9f41690 # shrinks to ops = [Checkpoint, Witness, Rewind, Unwitness(Position(0), "x"), Checkpoint, Rewind, Checkpoint, Unwitness(Position(0), "x"), Unwitness(Position(0), "x"), Witness, Unwitness(Position(0), "x"), Append("x"), Append("b"), Authpath(Position(0), "x"), Authpath(Position(0), "x"), Rewind, Rewind, Authpath(Position(0), "x"), Witness, Rewind, Witness, Rewind, Append("x"), Rewind, Authpath(Position(0), "x"), Checkpoint, Authpath(Position(0), "x"), Witness, Checkpoint, Witness, Checkpoint, Rewind, Unwitness(Position(0), "x"), Unwitness(Position(0), "x"), Rewind, Witness, Rewind, Authpath(Position(0), "x"), Unwitness(Position(0), "x"), Authpath(Position(0), "x"), Witness, Witness, Checkpoint, Rewind, Checkpoint] -cc 248258b4e2fd558b56611179f6c8493326a8a35bd3fd3ffb55634c14f6fe417a # shrinks to ops = [Append(SipHashable(0)), Checkpoint, Witness, Rewind, Rewind, Authpath(Position(0), SipHashable(0)), Authpath(Position(0), SipHashable(0)), Append(SipHashable(0)), Witness, Authpath(Position(0), SipHashable(0)), Unwitness(Position(0), SipHashable(0)), Unwitness(Position(0), SipHashable(0)), Authpath(Position(0), SipHashable(0)), Authpath(Position(0), SipHashable(0)), Unwitness(Position(0), SipHashable(0)), Witness, Rewind, Checkpoint, Authpath(Position(0), SipHashable(0)), Witness, Rewind, Authpath(Position(0), SipHashable(0)), Authpath(Position(0), SipHashable(0)), Unwitness(Position(0), SipHashable(0)), Rewind, Append(SipHashable(0)), Witness, Authpath(Position(1), SipHashable(0)), Witness, Witness, Unwitness(Position(3), SipHashable(0)), Append(SipHashable(0)), Checkpoint, Append(SipHashable(0)), Witness, Witness, Checkpoint, Unwitness(Position(5), SipHashable(0)), Unwitness(Position(1), SipHashable(0)), Append(SipHashable(0)), Checkpoint, Authpath(Position(2), SipHashable(0)), Witness, Append(SipHashable(0)), Unwitness(Position(6), SipHashable(0)), Authpath(Position(2), SipHashable(0)), Witness, Unwitness(Position(5), SipHashable(0)), Checkpoint, Rewind, Unwitness(Position(1), SipHashable(0)), Checkpoint, Rewind, Witness, Checkpoint, Unwitness(Position(5), SipHashable(0)), Authpath(Position(2), SipHashable(0)), Append(SipHashable(0)), Checkpoint, Append(SipHashable(0)), Checkpoint, Append(SipHashable(0)), Unwitness(Position(4), SipHashable(0)), Rewind, Append(SipHashable(7)), Checkpoint, Append(SipHashable(9)), Authpath(Position(5), SipHashable(0)), Checkpoint, Authpath(Position(10), SipHashable(7)), Rewind, Checkpoint, Rewind] diff --git a/src/bridgetree.rs b/src/bridgetree.rs index 115fb90..7305ea0 100644 --- a/src/bridgetree.rs +++ b/src/bridgetree.rs @@ -564,6 +564,16 @@ impl<'a, H: Hashable + Ord + Clone + 'a> MerkleBridge { self.frontier.root() } + fn root_at_altitude(&self, alt: Altitude) -> H { + // fold from the current height, combining with empty branches, + // up to the specified altitude + (self.max_altitude() + 1) + .iter_to(alt) + .fold(self.frontier.root(), |d, lvl| { + H::combine(lvl, &d, &H::empty_root(lvl)) + }) + } + /// Returns a single MerkleBridge that contains the aggregate information /// of this bridge and `next`, or None if `next` is not a valid successor /// to this bridge. The resulting Bridge will have the same state as though @@ -620,6 +630,9 @@ pub struct Checkpoint { /// A flag indicating whether or not the current state of the tree /// had been witnessed at the time the checkpoint was created. is_witnessed: bool, + /// A set of the positions that have been witnessed during the period that this + /// checkpoint is the current checkpoint. + witnessed: BTreeSet, /// When a witness is forgotten, if the index of the forgotten witness is <= bridge_idx we /// record it in the current checkpoint so that on rollback, we restore the forgotten /// witnesses to the BridgeTree's "saved" list. If the witness was newly created since the @@ -629,39 +642,89 @@ pub struct Checkpoint { } impl Checkpoint { + /// Creates a new checkpoint from its constituent parts. pub fn from_parts( bridges_len: usize, is_witnessed: bool, + witnessed: BTreeSet, forgotten: BTreeMap, ) -> Self { Self { bridges_len, is_witnessed, + witnessed, forgotten, } } + /// Creates a new empty checkpoint for the specified [`BridgeTree`] state. pub fn at_length(bridges_len: usize, is_witnessed: bool) -> Self { Checkpoint { bridges_len, is_witnessed, + witnessed: BTreeSet::new(), forgotten: BTreeMap::new(), } } + /// Returns the length of the [`prior_bridges`] vector of the [`BridgeTree`] to which + /// this checkpoint refers. + /// + /// This is the number of bridges that will be retained in the event of a rewind to this + /// checkpoint. pub fn bridges_len(&self) -> usize { self.bridges_len } + /// Returns whether the current state of the tree had been witnessed at the point that + /// this checkpoint was made. + /// + /// In the event of a rewind, the rewind logic will ensure that witness information is + /// properly reconstituted for the checkpointed tree state. pub fn is_witnessed(&self) -> bool { self.is_witnessed } + /// Returns a set of the positions that have been witnessed during the period that this + /// checkpoint is the current checkpoint. + pub fn witnessed(&self) -> &BTreeSet { + &self.witnessed + } + + /// Returns the set of previously-witnessed positions that have had their witnesses removed + /// during the period that this checkpoint is the current checkpoint. pub fn forgotten(&self) -> &BTreeMap { &self.forgotten } - pub fn rewrite_indices usize>(&mut self, f: F) { + // A private convenience method that returns the root of the bridge corresponding to + // this checkpoint at a specified depth, given the slice of bridges from which this checkpoint + // was derived. + fn root( + &self, + bridges: &[MerkleBridge], + altitude: Altitude, + ) -> H { + if self.bridges_len == 0 { + H::empty_root(altitude) + } else { + bridges[self.bridges_len - 1].root_at_altitude(altitude) + } + } + + // A private convenience method that returns the position of the bridge corresponding + // to this checkpoint, if the checkpoint is not for the empty bridge. + fn position(&self, bridges: &[MerkleBridge]) -> Option { + if self.bridges_len == 0 { + None + } else { + Some(bridges[self.bridges_len - 1].position()) + } + } + + // A private method that rewrites the indices of each forgotten witness record + // using the specified rewrite function. Used during garbage collection. + fn rewrite_indices usize>(&mut self, f: F) { self.bridges_len = f(self.bridges_len); for v in self.forgotten.values_mut() { *v = f(*v) @@ -857,7 +920,7 @@ impl BridgeTree { } } -impl crate::Frontier for BridgeTree { +impl Tree for BridgeTree { fn append(&mut self, value: &H) -> bool { if let Some(bridge) = self.current_bridge.as_mut() { if bridge.frontier.position().is_complete(Altitude(DEPTH)) { @@ -872,22 +935,26 @@ impl crate::Frontier for BridgeTr } } - fn root(&self) -> H { - self.current_bridge - .as_ref() - .map_or(H::empty_root(Altitude(DEPTH)), |bridge| { - // fold from the current height, combining with empty branches, - // up to the maximum height of the tree - (bridge.max_altitude() + 1) - .iter_to(Altitude(DEPTH)) - .fold(bridge.root(), |d, lvl| { - H::combine(lvl, &d, &H::empty_root(lvl)) - }) - }) + fn root(&self, checkpoint_depth: usize) -> Option { + let altitude = Altitude(DEPTH); + if checkpoint_depth == 0 { + Some( + self.current_bridge + .as_ref() + .map_or(H::empty_root(altitude), |bridge| { + bridge.root_at_altitude(altitude) + }), + ) + } else if self.checkpoints.len() >= checkpoint_depth { + let checkpoint_idx = self.checkpoints.len() - checkpoint_depth; + self.checkpoints + .get(checkpoint_idx) + .map(|c| c.root(&self.prior_bridges, altitude)) + } else { + None + } } -} -impl Tree for BridgeTree { fn current_position(&self) -> Option { self.current_bridge.as_ref().map(|b| b.position()) } @@ -924,6 +991,14 @@ impl Tree for BridgeTree< self.saved .entry(pos) .or_insert(self.prior_bridges.len() - 1); + + // mark the position as having been witnessed in the current checkpoint + if let Some(c) = self.checkpoints.last_mut() { + if !c.is_witnessed { + c.witnessed.insert(pos); + } + } + Some(pos) } None => None, @@ -942,12 +1017,16 @@ impl Tree for BridgeTree< fn remove_witness(&mut self, position: Position) -> bool { if let Some(idx) = self.saved.remove(&position) { - // If the index of the saved value is one that could have been known - // at the last checkpoint, then add it to the set of those forgotten - // during the current checkpoint span so that it can be restored - // on rollback. + // Stop tracking auth fragments for the removed position + if let Some(cur_b) = self.current_bridge.as_mut() { + cur_b.auth_fragments.remove(&position); + } + + // If the position is one that has *not* just been witnessed since the last checkpoint, + // then add it to the set of those forgotten during the current checkpoint span so that + // it can be restored on rollback. if let Some(c) = self.checkpoints.last_mut() { - if c.bridges_len > 0 && idx < c.bridges_len - 1 { + if !c.witnessed.contains(&position) { c.forgotten.insert(position, idx); } } @@ -997,7 +1076,10 @@ impl Tree for BridgeTree< self.saved.append(&mut c.forgotten); self.saved.retain(|_, i| *i + 1 < c.bridges_len); self.prior_bridges.truncate(c.bridges_len); - self.current_bridge = self.prior_bridges.last().map(|b| b.successor(false)); + self.current_bridge = self + .prior_bridges + .last() + .map(|b| b.successor(c.is_witnessed)); if c.is_witnessed { self.witness(); } @@ -1007,24 +1089,92 @@ impl Tree for BridgeTree< } } - fn authentication_path(&self, position: Position) -> Option> { - self.saved.get(&position).and_then(|idx| { + fn authentication_path(&self, position: Position, as_of_root: &H) -> Option> { + #[derive(Debug)] + enum AuthBase<'a> { + Current, + Checkpoint(usize, &'a Checkpoint), + NotFound, + } + + let max_alt = Altitude(DEPTH); + + // Find the earliest checkpoint having a matching root, or the current + // root if it matches and there is no earlier matching checkpoint. + let auth_base = self + .checkpoints + .iter() + .enumerate() + .rev() + .take_while(|(_, c)| c.position(&self.prior_bridges) >= Some(position)) + .filter(|(_, c)| &c.root(&self.prior_bridges, max_alt) == as_of_root) + .last() + .map(|(i, c)| AuthBase::Checkpoint(i, c)) + .unwrap_or_else(|| { + if self.root(0).as_ref() == Some(as_of_root) { + AuthBase::Current + } else { + AuthBase::NotFound + } + }); + + let saved_idx = self.saved.get(&position).or_else(|| { + if let AuthBase::Checkpoint(i, _) = auth_base { + // The saved position might have been forgotten since the checkpoint, + // so look for it in each of the subsequent checkpoints' forgotten + // items. + self.checkpoints[i..].iter().find_map(|c| { + // restore the forgotten position, if that position was not also witnessed + // in the same checkpoint + c.forgotten + .get(&position) + .filter(|_| !c.witnessed.contains(&position)) + }) + } else { + None + } + }); + + saved_idx.and_then(|idx| { let frontier = &self.prior_bridges[*idx].frontier; // Fuse the following bridges to obtain a bridge that has all - // of the data to the right of the selected value in the tree. - // The unwrap here is safe because a witnessed leaf always - // generates a subsequent bridge in the tree. - MerkleBridge::fuse_all( - self.prior_bridges[(idx + 1)..] - .iter() - .chain(self.current_bridge.iter()), - ) - .map(|fused| { + // of the data to the right of the selected value in the tree, + // up to the specified checkpoint depth. + let fuse_from = idx + 1; + let fused = match auth_base { + AuthBase::Current => MerkleBridge::fuse_all( + self.prior_bridges[fuse_from..] + .iter() + .chain(&self.current_bridge), + ), + AuthBase::Checkpoint(_, checkpoint) if fuse_from < checkpoint.bridges_len => { + MerkleBridge::fuse_all( + self.prior_bridges[fuse_from..checkpoint.bridges_len].iter(), + ) + } + AuthBase::Checkpoint(_, checkpoint) if fuse_from == checkpoint.bridges_len => { + // The successor bridge should just be the empty successor to the + // checkpointed bridge. + if checkpoint.bridges_len > 0 { + Some(self.prior_bridges[checkpoint.bridges_len - 1].successor(false)) + } else { + None + } + } + AuthBase::Checkpoint(_, _) => { + // if the saved index is after the checkpoint, we can't generate + // an auth path + None + } + AuthBase::NotFound => None, + }; + + fused.map(|successor| { // construct a complete trailing edge that includes the data from // the following frontier not yet included in the trailing edge. - let auth_fragment = fused.auth_fragments.get(&frontier.position()); - let rest_frontier = fused.frontier; + let auth_fragment = successor.auth_fragments.get(&frontier.position()); + let rest_frontier = successor.frontier; let mut auth_values = auth_fragment.iter().flat_map(|auth_fragment| { let last_altitude = auth_fragment.next_required_altitude(); @@ -1147,7 +1297,7 @@ mod tests { use super::*; use crate::tests::{apply_operation, arb_operation}; - use crate::{Frontier, Tree}; + use crate::Tree; #[test] fn tree_depth() { @@ -1209,8 +1359,8 @@ mod tests { for pos in tree.saved.keys() { assert_eq!( - tree.authentication_path(*pos), - tree_mut.authentication_path(*pos) + tree.authentication_path(*pos, &tree.root(0).unwrap()), + tree_mut.authentication_path(*pos, &tree.root(0).unwrap()) ); } } @@ -1309,7 +1459,7 @@ mod tests { let auth_paths = has_auth_path .iter() .map(|pos| { - t.authentication_path(*pos) + t.authentication_path(*pos, &t.root(0).unwrap()) .expect("Must be able to get auth path") }) .collect::>(); @@ -1319,7 +1469,7 @@ mod tests { let retained_auth_paths = has_auth_path .iter() .map(|pos| { - t.authentication_path(*pos) + t.authentication_path(*pos, &t.root(0).unwrap()) .expect("Must be able to get auth path") }) .collect::>(); @@ -1329,14 +1479,14 @@ mod tests { #[test] fn garbage_collect_idx() { let mut tree: BridgeTree = BridgeTree::new(100); - let empty_root = tree.root(); + let empty_root = tree.root(0); tree.append(&"a".to_string()); for _ in 0..100 { tree.checkpoint(); } tree.garbage_collect(); - assert!(tree.root() != empty_root); + assert!(tree.root(0) != empty_root); tree.rewind(); - assert!(tree.root() != empty_root); + assert!(tree.root(0) != empty_root); } } diff --git a/src/lib.rs b/src/lib.rs index 14b97f3..c0918fa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -232,7 +232,12 @@ pub trait Frontier { /// A Merkle tree that supports incremental appends, witnessing of /// leaf nodes, checkpoints and rollbacks. -pub trait Tree: Frontier { +pub trait Tree { + /// Appends a new value to the tree at the next available slot. + /// Returns true if successful and false if the tree would exceed + /// the maximum allowed depth. + fn append(&mut self, value: &H) -> bool; + /// Returns the most recently appended leaf value. fn current_position(&self) -> Option; @@ -252,10 +257,18 @@ pub trait Tree: Frontier { /// Return a set of all the positions for which we have witnessed. fn witnessed_positions(&self) -> BTreeSet; - /// Obtains an authentication path to the value at the specified position. + /// Obtains the root of the Merkle tree at the specified checkpoint depth + /// by hashing against empty nodes up to the maximum height of the tree. + /// Returns `None` if there are not enough checkpoints available to reach the + /// requested checkpoint depth. + fn root(&self, checkpoint_depth: usize) -> Option; + + /// Obtains an authentication path to the value at the specified position, + /// as of the tree state corresponding to the given root. /// Returns `None` if there is no available authentication path to that - /// value. - fn authentication_path(&self, position: Position) -> Option>; + /// position or if the root does not correspond to a checkpointed + /// root of the tree. + fn authentication_path(&self, position: Position, as_of_root: &H) -> Option>; /// Marks the value at the specified position as a value we're no longer /// interested in maintaining a witness for. Returns true if successful and @@ -292,7 +305,7 @@ pub(crate) mod tests { use super::bridgetree::BridgeTree; use super::sample::{lazy_root, CompleteTree}; - use super::{Altitude, Frontier, Hashable, Position, Tree}; + use super::{Altitude, Hashable, Position, Tree}; #[test] fn position_altitudes() { @@ -341,17 +354,17 @@ pub(crate) mod tests { pub(crate) fn check_root_hashes, F: Fn(usize) -> T>(new_tree: F) { let mut tree = new_tree(100); - assert_eq!(tree.root(), "________________"); + assert_eq!(tree.root(0).unwrap(), "________________"); tree.append(&"a".to_string()); - assert_eq!(tree.root().len(), 16); - assert_eq!(tree.root(), "a_______________"); + assert_eq!(tree.root(0).unwrap().len(), 16); + assert_eq!(tree.root(0).unwrap(), "a_______________"); tree.append(&"b".to_string()); - assert_eq!(tree.root(), "ab______________"); + assert_eq!(tree.root(0).unwrap(), "ab______________"); tree.append(&"c".to_string()); - assert_eq!(tree.root(), "abc_____________"); + assert_eq!(tree.root(0).unwrap(), "abc_____________"); let mut t = new_tree(100); t.append(&"a".to_string()); @@ -360,7 +373,7 @@ pub(crate) mod tests { t.append(&"a".to_string()); t.append(&"a".to_string()); t.append(&"a".to_string()); - assert_eq!(t.root(), "aaaa____________"); + assert_eq!(t.root(0).unwrap(), "aaaa____________"); } pub(crate) fn check_auth_paths + std::fmt::Debug, F: Fn(usize) -> T>( @@ -370,7 +383,7 @@ pub(crate) mod tests { tree.append(&"a".to_string()); tree.witness(); assert_eq!( - tree.authentication_path(Position::from(0)), + tree.authentication_path(Position::from(0), &tree.root(0).unwrap()), Some(vec![ "_".to_string(), "__".to_string(), @@ -381,7 +394,7 @@ pub(crate) mod tests { tree.append(&"b".to_string()); assert_eq!( - tree.authentication_path(Position::zero()), + tree.authentication_path(Position::zero(), &tree.root(0).unwrap()), Some(vec![ "b".to_string(), "__".to_string(), @@ -393,7 +406,7 @@ pub(crate) mod tests { tree.append(&"c".to_string()); tree.witness(); assert_eq!( - tree.authentication_path(Position::from(2)), + tree.authentication_path(Position::from(2), &tree.root(0).unwrap()), Some(vec![ "_".to_string(), "ab".to_string(), @@ -404,7 +417,7 @@ pub(crate) mod tests { tree.append(&"d".to_string()); assert_eq!( - tree.authentication_path(Position::from(2)), + tree.authentication_path(Position::from(2), &tree.root(0).unwrap()), Some(vec![ "d".to_string(), "ab".to_string(), @@ -415,7 +428,7 @@ pub(crate) mod tests { tree.append(&"e".to_string()); assert_eq!( - tree.authentication_path(Position::from(2)), + tree.authentication_path(Position::from(2), &tree.root(0).unwrap()), Some(vec![ "d".to_string(), "ab".to_string(), @@ -434,7 +447,7 @@ pub(crate) mod tests { tree.append(&"h".to_string()); assert_eq!( - tree.authentication_path(Position::zero()), + tree.authentication_path(Position::zero(), &tree.root(0).unwrap()), Some(vec![ "b".to_string(), "cd".to_string(), @@ -457,7 +470,7 @@ pub(crate) mod tests { tree.append(&"g".to_string()); assert_eq!( - tree.authentication_path(Position::from(5)), + tree.authentication_path(Position::from(5), &tree.root(0).unwrap()), Some(vec![ "e".to_string(), "g_".to_string(), @@ -474,7 +487,7 @@ pub(crate) mod tests { tree.append(&'l'.to_string()); assert_eq!( - tree.authentication_path(Position::from(10)), + tree.authentication_path(Position::from(10), &tree.root(0).unwrap()), Some(vec![ "l".to_string(), "ij".to_string(), @@ -497,7 +510,7 @@ pub(crate) mod tests { } assert_eq!( - tree.authentication_path(Position::zero()), + tree.authentication_path(Position::zero(), &tree.root(0).unwrap()), Some(vec![ "b".to_string(), "cd".to_string(), @@ -521,7 +534,7 @@ pub(crate) mod tests { assert!(tree.rewind()); assert_eq!( - tree.authentication_path(Position::from(2)), + tree.authentication_path(Position::from(2), &tree.root(0).unwrap()), Some(vec![ "d".to_string(), "ab".to_string(), @@ -534,7 +547,10 @@ pub(crate) mod tests { tree.append(&'a'.to_string()); tree.append(&'b'.to_string()); tree.witness(); - assert_eq!(tree.authentication_path(Position::from(0)), None); + assert_eq!( + tree.authentication_path(Position::from(0), &tree.root(0).unwrap()), + None + ); let mut tree = new_tree(100); for c in 'a'..'n' { @@ -547,7 +563,7 @@ pub(crate) mod tests { tree.append(&'p'.to_string()); assert_eq!( - tree.authentication_path(Position::from(12)), + tree.authentication_path(Position::from(12), &tree.root(0).unwrap()), Some(vec![ "n".to_string(), "op".to_string(), @@ -562,7 +578,7 @@ pub(crate) mod tests { .chain(Some(Witness)) .chain(Some(Append('m'.to_string()))) .chain(Some(Append('n'.to_string()))) - .chain(Some(Authpath(11usize.into()))) + .chain(Some(Authpath(11usize.into(), 0))) .collect::>(); let mut tree = new_tree(100); @@ -618,7 +634,7 @@ pub(crate) mod tests { t.append(&"b".to_string()); assert!(t.rewind()); t.append(&"b".to_string()); - assert_eq!(t.root(), "ab______________"); + assert_eq!(t.root(0).unwrap(), "ab______________"); } pub(crate) fn check_rewind_remove_witness, F: Fn(usize) -> T>(new_tree: F) { @@ -671,69 +687,46 @@ pub(crate) mod tests { // test framework itself previously did not correctly handle // chain state restoration. - fn append(x: &str) -> Operation { - Append(x.to_string()) + let samples = vec![ + vec![append("x"), Checkpoint, Witness, Rewind, unwitness(0)], + vec![ + append("d"), + Checkpoint, + Witness, + unwitness(0), + Rewind, + unwitness(0), + ], + vec![ + append("o"), + Checkpoint, + Witness, + Checkpoint, + unwitness(0), + Rewind, + Rewind, + ], + vec![ + append("s"), + Witness, + append("m"), + Checkpoint, + unwitness(0), + Rewind, + unwitness(0), + unwitness(0), + ], + ]; + + for (i, sample) in samples.iter().enumerate() { + let result = check_operations(sample); + assert!( + matches!(result, Ok(())), + "Reference/Test mismatch at index {}: {:?}", + i, + result + ); } - - fn unwitness(pos: usize) -> Operation { - Unwitness(Position::from(pos)) - } - - let ops = vec![append("x"), Checkpoint, Witness, Rewind, unwitness(0)]; - let result = check_operations(ops); - assert!( - matches!(result, Ok(())), - "Reference/Test mismatch: {:?}", - result - ); - - let ops = vec![ - append("d"), - Checkpoint, - Witness, - unwitness(0), - Rewind, - unwitness(0), - ]; - let result = check_operations(ops); - assert!( - matches!(result, Ok(())), - "Reference/Test mismatch: {:?}", - result - ); - - let ops = vec![ - append("o"), - Checkpoint, - Witness, - Checkpoint, - unwitness(0), - Rewind, - Rewind, - ]; - let result = check_operations(ops); - assert!( - matches!(result, Ok(())), - "Reference/Test mismatch: {:?}", - result - ); - - let ops = vec![ - append("s"), - Witness, - append("m"), - Checkpoint, - unwitness(0), - Rewind, - unwitness(0), - unwitness(0), - ]; - let result = check_operations(ops); - assert!( - matches!(result, Ok(())), - "Reference/Test mismatch: {:?}", - result - ); } // @@ -755,7 +748,7 @@ pub(crate) mod tests { } } - impl Frontier for CombinedTree { + impl Tree for CombinedTree { fn append(&mut self, value: &H) -> bool { let a = self.inefficient.append(value); let b = self.efficient.append(value); @@ -763,16 +756,13 @@ pub(crate) mod tests { a } - /// Obtains the current root of this Merkle tree. - fn root(&self) -> H { - let a = self.inefficient.root(); - let b = self.efficient.root(); + fn root(&self, checkpoint_depth: usize) -> Option { + let a = self.inefficient.root(checkpoint_depth); + let b = self.efficient.root(checkpoint_depth); assert_eq!(a, b); a } - } - impl Tree for CombinedTree { fn current_position(&self) -> Option { let a = self.inefficient.current_position(); let b = self.efficient.current_position(); @@ -811,9 +801,9 @@ pub(crate) mod tests { a } - fn authentication_path(&self, position: Position) -> Option> { - let a = self.inefficient.authentication_path(position); - let b = self.efficient.authentication_path(position); + fn authentication_path(&self, position: Position, as_of_root: &H) -> Option> { + let a = self.inefficient.authentication_path(position, as_of_root); + let b = self.efficient.authentication_path(position, as_of_root); assert_eq!(a, b); a } @@ -843,6 +833,10 @@ pub(crate) mod tests { } } + // + // Operations + // + #[derive(Clone, Debug)] pub enum Operation { Append(A), @@ -854,12 +848,24 @@ pub(crate) mod tests { Unwitness(Position), Checkpoint, Rewind, - Authpath(Position), + Authpath(Position, usize), GarbageCollect, } use Operation::*; + fn append(x: &str) -> Operation { + Operation::Append(x.to_string()) + } + + fn unwitness(pos: usize) -> Operation { + Operation::Unwitness(Position::from(pos)) + } + + fn authpath(pos: usize, depth: usize) -> Operation { + Operation::Authpath(Position::from(pos), depth) + } + impl Operation { pub fn apply>(&self, tree: &mut T) -> Option<(Position, Vec)> { match self { @@ -887,7 +893,10 @@ pub(crate) mod tests { assert!(tree.rewind(), "rewind failed"); None } - Authpath(p) => tree.authentication_path(*p).map(|xs| (*p, xs)), + Authpath(p, d) => tree + .root(*d) + .and_then(|root| tree.authentication_path(*p, &root)) + .map(|xs| (*p, xs)), GarbageCollect => None, } } @@ -977,6 +986,192 @@ pub(crate) mod tests { ); } + #[test] + fn test_auth_path_consistency() { + let samples = vec![ + // Reduced examples + vec![ + append("a"), + append("b"), + Checkpoint, + Witness, + authpath(0, 1), + ], + vec![ + append("c"), + append("d"), + Witness, + Checkpoint, + authpath(1, 1), + ], + vec![ + append("e"), + Checkpoint, + Witness, + append("f"), + authpath(0, 1), + ], + vec![ + append("g"), + Witness, + Checkpoint, + unwitness(0), + append("h"), + authpath(0, 0), + ], + vec![ + append("i"), + Checkpoint, + Witness, + unwitness(0), + append("j"), + authpath(0, 0), + ], + vec![ + append("i"), + Witness, + append("j"), + Checkpoint, + append("k"), + authpath(0, 1), + ], + vec![ + append("l"), + Checkpoint, + Witness, + Checkpoint, + append("m"), + Checkpoint, + authpath(0, 2), + ], + vec![Checkpoint, append("n"), Witness, authpath(0, 1)], + vec![ + append("a"), + Witness, + Checkpoint, + unwitness(0), + Checkpoint, + append("b"), + authpath(0, 1), + ], + vec![ + append("a"), + Witness, + append("b"), + unwitness(0), + Checkpoint, + authpath(0, 0), + ], + vec![ + append("a"), + Witness, + Checkpoint, + unwitness(0), + Checkpoint, + Rewind, + append("b"), + authpath(0, 0), + ], + vec![ + append("a"), + Witness, + Checkpoint, + Checkpoint, + Rewind, + append("a"), + unwitness(0), + authpath(0, 1), + ], + // Unreduced examples + vec![ + append("o"), + append("p"), + Witness, + append("q"), + Checkpoint, + unwitness(1), + authpath(1, 1), + ], + vec![ + append("r"), + append("s"), + append("t"), + Witness, + Checkpoint, + unwitness(2), + Checkpoint, + authpath(2, 2), + ], + vec![ + append("u"), + Witness, + append("v"), + append("w"), + Checkpoint, + unwitness(0), + append("x"), + Checkpoint, + Checkpoint, + authpath(0, 3), + ], + ]; + + for (i, sample) in samples.iter().enumerate() { + let result = check_operations(sample); + assert!( + matches!(result, Ok(())), + "Reference/Test mismatch at index {}: {:?}", + i, + result + ); + } + } + + // These check_operations tests cover errors where the test framework itself previously did not + // correctly handle chain state restoration. + #[test] + fn test_rewind_remove_witness_consistency() { + let samples = vec![ + vec![append("x"), Checkpoint, Witness, Rewind, unwitness(0)], + vec![ + append("d"), + Checkpoint, + Witness, + unwitness(0), + Rewind, + unwitness(0), + ], + vec![ + append("o"), + Checkpoint, + Witness, + Checkpoint, + unwitness(0), + Rewind, + Rewind, + ], + vec![ + append("s"), + Witness, + append("m"), + Checkpoint, + unwitness(0), + Rewind, + unwitness(0), + unwitness(0), + ], + ]; + for (i, sample) in samples.iter().enumerate() { + let result = check_operations(sample); + assert!( + matches!(result, Ok(())), + "Reference/Test mismatch at index {}: {:?}", + i, + result + ); + } + } + use proptest::prelude::*; pub fn arb_operation( @@ -1003,7 +1198,8 @@ pub(crate) mod tests { .prop_map(|i| Operation::Unwitness(Position::from(i))), Just(Operation::Checkpoint), Just(Operation::Rewind), - pos_gen.prop_map(|i| Operation::Authpath(Position::from(i))), + pos_gen.prop_flat_map(|i| (0usize..10) + .prop_map(move |depth| Operation::Authpath(Position::from(i), depth))), ] } @@ -1026,7 +1222,7 @@ pub(crate) mod tests { } CurrentPosition => {} CurrentLeaf => {} - Authpath(_) => {} + Authpath(_, _) => {} WitnessedLeaf(_) => {} WitnessedPositions => {} GarbageCollect => {} @@ -1034,7 +1230,7 @@ pub(crate) mod tests { } fn check_operations( - ops: Vec>, + ops: &[Operation], ) -> Result<(), TestCaseError> { const DEPTH: u8 = 4; let mut tree = CombinedTree::::new(); @@ -1048,7 +1244,7 @@ pub(crate) mod tests { prop_assert_eq!(tree_size, tree_values.len()); match op { Append(value) => { - if tree.append(&value) { + if tree.append(value) { prop_assert!(tree_size < (1 << DEPTH)); tree_size += 1; tree_values.push(value.clone()); @@ -1073,12 +1269,12 @@ pub(crate) mod tests { } } WitnessedLeaf(position) => { - if tree.get_witnessed_leaf(position).is_some() { - prop_assert!(::from(position) < tree_size); + if tree.get_witnessed_leaf(*position).is_some() { + prop_assert!(::from(*position) < tree_size); } } Unwitness(position) => { - tree.remove_witness(position); + tree.remove_witness(*position); } WitnessedPositions => {} Checkpoint => { @@ -1093,20 +1289,36 @@ pub(crate) mod tests { tree_size = checkpointed_tree_size; } } - Authpath(position) => { - if let Some(path) = tree.authentication_path(position) { - let value: H = tree_values.get(::from(position)).unwrap().clone(); - let mut extended_tree_values = tree_values.clone(); - extended_tree_values.resize(1 << DEPTH, H::empty_leaf()); - let expected_root = lazy_root::(extended_tree_values); + Authpath(position, depth) => { + if let Some(path) = tree + .root(*depth) + .and_then(|r| tree.authentication_path(*position, &r)) + { + let value: H = tree_values[::from(*position)].clone(); + let tree_root = tree.root(*depth); - let tree_root = tree.root(); - prop_assert_eq!(&tree_root, &expected_root); + if tree_checkpoints.len() >= *depth { + let mut extended_tree_values = tree_values.clone(); + if *depth > 0 { + // prune the tree back to the checkpointed size. + if let Some(checkpointed_tree_size) = + tree_checkpoints.get(tree_checkpoints.len() - depth) + { + extended_tree_values.truncate(*checkpointed_tree_size); + } + } + // extend the tree with empty leaves until it is full + extended_tree_values.resize(1 << DEPTH, H::empty_leaf()); - prop_assert_eq!( - &compute_root_from_auth_path(value, position, &path), - &expected_root - ); + // compute the root + let expected_root = lazy_root::(extended_tree_values); + prop_assert_eq!(&tree_root.unwrap(), &expected_root); + + prop_assert_eq!( + &compute_root_from_auth_path(value, *position, &path), + &expected_root + ); + } } } GarbageCollect => {} @@ -1126,7 +1338,7 @@ pub(crate) mod tests { 1..100 ) ) { - check_operations(ops)?; + check_operations(&ops)?; } #[test] @@ -1136,7 +1348,7 @@ pub(crate) mod tests { 1..100 ) ) { - check_operations::(ops)?; + check_operations::(&ops)?; } } } diff --git a/src/sample.rs b/src/sample.rs index 9570427..115da6c 100644 --- a/src/sample.rs +++ b/src/sample.rs @@ -69,9 +69,7 @@ impl TreeState { /// witnessing. Returns the current position if the tree is non-empty. fn witness(&mut self) -> Option { self.current_position().map(|pos| { - if !self.witnesses.contains(&pos) { - self.witnesses.insert(pos); - } + self.witnesses.insert(pos); pos }) } @@ -80,7 +78,7 @@ impl TreeState { /// Returns `None` if there is no available authentication path to that /// value. fn authentication_path(&self, position: Position) -> Option> { - if self.witnesses.contains(&position) { + if Some(position) <= self.current_position() { let mut path = vec![]; let mut leaf_idx: usize = position.into(); @@ -117,7 +115,7 @@ impl CompleteTree { /// Creates a new, empty binary tree of specified depth. #[cfg(test)] pub fn new(depth: usize, max_checkpoints: usize) -> Self { - CompleteTree { + Self { tree_state: TreeState::new(depth), checkpoints: vec![], max_checkpoints, @@ -125,16 +123,6 @@ impl CompleteTree { } } -impl Frontier for CompleteTree { - fn append(&mut self, value: &H) -> bool { - self.tree_state.append(value) - } - - fn root(&self) -> H { - self.tree_state.root() - } -} - impl CompleteTree { /// Removes the oldest checkpoint. Returns true if successful and false if /// there are no checkpoints. @@ -146,9 +134,30 @@ impl CompleteTree { true } } + + /// Retrieve the tree state at the specified checkpoint depth. This + /// is the current tree state if the depth is 0, and this will return + /// None if not enough checkpoints exist to obtain the requested depth. + fn tree_state_at_checkpoint_depth(&self, checkpoint_depth: usize) -> Option<&TreeState> { + if self.checkpoints.len() < checkpoint_depth { + None + } else if checkpoint_depth == 0 { + Some(&self.tree_state) + } else { + self.checkpoints + .get(self.checkpoints.len() - checkpoint_depth) + } + } } -impl Tree for CompleteTree { +impl Tree for CompleteTree { + /// Appends a new value to the tree at the next available slot. Returns true + /// if successful and false if the tree is full. + fn append(&mut self, value: &H) -> bool { + self.tree_state.append(value) + } + + /// Returns the most recently appended leaf value. fn current_position(&self) -> Option { self.tree_state.current_position() } @@ -169,8 +178,26 @@ impl Tree for CompleteTree { self.tree_state.witnesses.clone() } - fn authentication_path(&self, position: Position) -> Option> { - self.tree_state.authentication_path(position) + fn root(&self, checkpoint_depth: usize) -> Option { + self.tree_state_at_checkpoint_depth(checkpoint_depth) + .map(|s| s.root()) + } + + fn authentication_path(&self, position: Position, root: &H) -> Option> { + // Search for the checkpointed state corresponding to the provided root, and if one is + // found, compute the authentication path as of that root. + self.checkpoints + .iter() + .chain(Some(&self.tree_state)) + .rev() + .skip_while(|c| !c.witnesses.contains(&position)) + .find_map(|c| { + if &c.root() == root { + c.authentication_path(position) + } else { + None + } + }) } fn remove_witness(&mut self, position: Position) -> bool { @@ -225,7 +252,7 @@ pub(crate) fn lazy_root(mut leaves: Vec) -> H { #[cfg(test)] mod tests { use crate::tests::{compute_root_from_auth_path, SipHashable}; - use crate::{Altitude, Frontier, Hashable, Position, Tree}; + use crate::{Altitude, Hashable, Position, Tree}; use std::convert::TryFrom; use super::CompleteTree; @@ -239,7 +266,7 @@ mod tests { } let tree = CompleteTree::::new(DEPTH as usize, 100); - assert_eq!(tree.root(), expected); + assert_eq!(tree.root(0).unwrap(), expected); } #[test] @@ -267,7 +294,7 @@ mod tests { ), ); - assert_eq!(tree.root(), expected); + assert_eq!(tree.root(0).unwrap(), expected); } #[test] @@ -306,11 +333,13 @@ mod tests { ), ); - assert_eq!(tree.root(), expected); + assert_eq!(tree.root(0).unwrap(), expected); for i in 0u64..(1 << DEPTH) { let position = Position::try_from(i).unwrap(); - let path = tree.authentication_path(position).unwrap(); + let path = tree + .authentication_path(position, &tree.root(0).unwrap()) + .unwrap(); assert_eq!( compute_root_from_auth_path(SipHashable(i), position, &path), expected From 9d94e9d6e88fa6c5bb960dc618b13ebae77781ca Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 22 Apr 2022 09:57:23 -0600 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: ebfull --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8215209..9c49938 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ and this project adheres to Rust's notion of This allows computation of the authentication path as of previous tree states, in addition to the previous behavior which only allowed computation of the path as of the most recent tree state. The provided `as_of_root` value must be equal to either the - current root of the tree, or to the root of the tree at a previous checkpoint. + current root of the tree, or to the root of the tree at a previous checkpoint that + contained a note at the given position. ### Removed