Merge pull request #52 from nuttycom/witness_at_depth

Compute witnesses as of checkpoint depth rather than as of a root hash.

bridgetree/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to Rust's notion of
 
 ## [Unreleased]
 
+### Changed
+
+- `BridgeTree::witness` now takes a checkpoint depth rather than a root hash to
+  identify the tree state with respect to which the witness should be constructed.
+
 ### Removed
 
 - The `testing` module has been removed in favor of depending on

bridgetree/proptest-regressions/bridgetree.txt

@@ -1,8 +0,0 @@
-# Seeds for failure cases proptest has generated in the past. It is
-# automatically read and these particular cases re-run before any
-# novel cases are generated.
-#
-# It is recommended to check this file in to source control so that
-# everyone who runs the test benefits from these saved cases.
-cc a64e0d40c9287f2975a27447d266e4dd7228e44de99d000df39c4610598f486a # shrinks to tree = BridgeTree {   depth: 8,   prior_bridges: [MerkleBridge { prior_position: None, auth_fragments: {}, frontier: NonEmptyFrontier { position: Position(0), leaf: Left("a"), ommers: [] } }, MerkleBridge { prior_position: Some(Position(0)), auth_fragments: {}, frontier: NonEmptyFrontier { position: Position(1), leaf: Right("a", "a"), ommers: [] } }],   current_bridge: Some(MerkleBridge { prior_position: Some(Position(1)), auth_fragments: {Position(1): AuthFragment { position: Position(1), altitudes_observed: 0, values: [] }}, frontier: NonEmptyFrontier { position: Position(1), leaf: Right("a", "a"), ommers: [] } }),   saved: {Position(1): 1},   checkpoints: [Checkpoint { bridges_len: 1, is_witnessed: false, forgotten: {} }],   max_checkpoints: 10 }
-cc 736aee7c92f418b3b7391b0ae253ca4dc18f9b6cc625c0c34f0e568d26421e92 # shrinks to tree = BridgeTree {   depth: 8,   prior_bridges: [],   current_bridge: None,   saved: {},   checkpoints: [],   max_checkpoints: 10 }

bridgetree/proptest-regressions/lib.txt

@@ -4,5 +4,11 @@
 #
 # It is recommended to check this file in to source control so that
 # everyone who runs the test benefits from these saved cases.
-cc 190d23d28fc081e651e779d6209951363ee8a21752233cb72471626d14dd8bad # shrinks to ops = [Append("a"), Append("a"), Append("a")]
+cc a64e0d40c9287f2975a27447d266e4dd7228e44de99d000df39c4610598f486a # shrinks to tree = BridgeTree {   depth: 8,   prior_bridges: [MerkleBridge { prior_position: None, auth_fragments: {}, frontier: NonEmptyFrontier { position: Position(0), leaf: Left("a"), ommers: [] } }, MerkleBridge { prior_position: Some(Position(0)), auth_fragments: {}, frontier: NonEmptyFrontier { position: Position(1), leaf: Right("a", "a"), ommers: [] } }],   current_bridge: Some(MerkleBridge { prior_position: Some(Position(1)), auth_fragments: {Position(1): AuthFragment { position: Position(1), altitudes_observed: 0, values: [] }}, frontier: NonEmptyFrontier { position: Position(1), leaf: Right("a", "a"), ommers: [] } }),   saved: {Position(1): 1},   checkpoints: [Checkpoint { bridges_len: 1, is_witnessed: false, forgotten: {} }],   max_checkpoints: 10 }
-cc 81533fad8faadbdfdc9547e07f0491e40172a2f5fe4e8768d289389ed2e3cbcb # shrinks to ops = [Append(SipHashable(0)), Append(SipHashable(0)), Append(SipHashable(0))]
+cc 736aee7c92f418b3b7391b0ae253ca4dc18f9b6cc625c0c34f0e568d26421e92 # shrinks to tree = BridgeTree {   depth: 8,   prior_bridges: [],   current_bridge: None,   saved: {},   checkpoints: [],   max_checkpoints: 10 }
+cc f1a0c0e8114919f9f675e0b31cdecf37af31579e365119312a7ebefcabb639f1 # shrinks to ops = [Append(SipHashable(0)), Checkpoint, Checkpoint, Mark, Checkpoint, Checkpoint, Checkpoint, Checkpoint, Checkpoint, Authpath(Position(0), 6)]
+cc ac8dad1a92cb9563a291802cbd3dfca2a89da35fe4ed377701f5ad85b43700f4 # shrinks to ops = [Append("a"), Append("a"), Checkpoint, Checkpoint, Checkpoint, Mark, Checkpoint, Authpath(Position(1), 2)]
+cc cebb95fe896dc1d1e3c9a65efd50e786e6a4a3503c86fb2a6817bb05d25e751e # shrinks to tree = BridgeTree {   depth: 8,   prior_bridges: [MerkleBridge { prior_position: None, tracking: {Address { level: Level(2), index: 0 }}, ommers: {}, frontier: NonEmptyFrontier { position: Position(3), leaf: "a", ommers: ["a", "aa"] } }],   current_bridge: Some(MerkleBridge { prior_position: Some(Position(3)), tracking: {Address { level: Level(2), index: 0 }}, ommers: {}, frontier: NonEmptyFrontier { position: Position(3), leaf: "a", ommers: ["a", "aa"] } }),   saved: {Position(3): 0},   checkpoints: [Checkpoint { bridges_len: 0, marked: {Position(3)}, forgotten: {Position(3): 0} }],   max_checkpoints: 10 }
+cc cdab33688ef9270768481b72d1615ff1d209fdb3d8d45be2ad564a8d5e0addc1 # shrinks to ops = [Append(SipHashable(0)), Mark, Mark, Checkpoint, Mark, Rewind, Append(SipHashable(0)), Mark]
+cc 4058fadeb645f982bb83d666ea20c9541c9b920438877c145ee750cb09e22337 # shrinks to ops = [Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), CurrentLeaf, CurrentLeaf, Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Checkpoint { id: (), is_marked: true }), Unmark(Position(4))]
+cc befc65ab8360534feebaa4c0e3da24518d134b03326235f1ccede15ad998d066 # shrinks to ops = [Append("a", Ephemeral), Append("a", Checkpoint { id: (), is_marked: true }), Append("a", Ephemeral), Append("a", Checkpoint { id: (), is_marked: false }), Authpath(Position(1), 0)]

bridgetree/src/lib.rs

@@ -31,13 +31,13 @@
 //!
 //! In this module, the term "ommer" is used as for the sibling of a parent node in a binary tree.
 use serde::{Deserialize, Serialize};
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::convert::TryFrom;
 use std::fmt::Debug;
 use std::mem::size_of;
 use std::ops::Range;
 
-pub use incrementalmerkletree::{Address, Hashable, Level, Position};
+pub use incrementalmerkletree::{Address, Hashable, Level, Position, Retention};
 
 /// Validation errors that can occur during reconstruction of a Merkle frontier from
 /// its constituent parts.
@@ -482,7 +482,7 @@ impl<'a, H: Hashable + Ord + Clone + 'a> MerkleBridge<H> {
     /// will track the information necessary to create a witness for the leaf most
     /// recently appended to this bridge's frontier.
     #[must_use]
-    pub fn successor(&self, mark_current_leaf: bool) -> Self {
+    pub fn successor(&self, track_current_leaf: bool) -> Self {
         let mut result = Self {
             prior_position: Some(self.frontier.position()),
             tracking: self.tracking.clone(),
@@ -490,7 +490,7 @@ impl<'a, H: Hashable + Ord + Clone + 'a> MerkleBridge<H> {
             frontier: self.frontier.clone(),
         };
 
-        if mark_current_leaf {
+        if track_current_leaf {
             result.track_current_leaf();
         }
 
@@ -614,48 +614,51 @@ impl<'a, H: Hashable + Ord + Clone + 'a> MerkleBridge<H> {
 /// previous state.
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct Checkpoint {
+    /// The unique identifier for this checkpoint.
+    id: usize,
     /// The number of bridges that will be retained in a rewind.
     bridges_len: usize,
-    /// A flag indicating whether or not the current state of the tree
-    /// had been marked at the time the checkpoint was created.
-    is_marked: bool,
     /// A set of the positions that have been marked during the period that this
     /// checkpoint is the current checkpoint.
     marked: BTreeSet<Position>,
-    /// When a mark is forgotten, if the index of the forgotten mark is <= bridge_idx we
+    /// When a mark is forgotten, we add it to the checkpoint's forgotten set but
-    /// record it in the current checkpoint so that on rollback, we restore the forgotten
+    /// don't immediately remove it from the `saved` map; that removal occurs when
-    /// marks to the BridgeTree's "saved" list. If the mark was newly created since the
+    /// the checkpoint is eventually dropped.
-    /// checkpoint, we don't need to remember when we forget it because both the mark
+    forgotten: BTreeSet<Position>,
-    /// creation and removal will be reverted in the rollback.
-    forgotten: BTreeMap<Position, usize>,
 }
 
 impl Checkpoint {
     /// Creates a new checkpoint from its constituent parts.
     pub fn from_parts(
+        id: usize,
         bridges_len: usize,
-        is_marked: bool,
         marked: BTreeSet<Position>,
-        forgotten: BTreeMap<Position, usize>,
+        forgotten: BTreeSet<Position>,
     ) -> Self {
         Self {
+            id,
             bridges_len,
-            is_marked,
             marked,
             forgotten,
         }
     }
 
     /// Creates a new empty checkpoint for the specified [`BridgeTree`] state.
-    pub fn at_length(bridges_len: usize, is_marked: bool) -> Self {
+    pub fn at_length(bridges_len: usize, id: usize) -> Self {
         Checkpoint {
+            id,
             bridges_len,
-            is_marked,
             marked: BTreeSet::new(),
-            forgotten: BTreeMap::new(),
+            forgotten: BTreeSet::new(),
         }
     }
 
+    /// The unique identifier for the checkpoint, which is simply an automatically incrementing
+    /// index over all checkpoints that have ever been created in the history of the tree.
+    pub fn id(&self) -> usize {
+        self.id
+    }
+
     /// Returns the length of the [`BridgeTree::prior_bridges`] vector of the [`BridgeTree`] to
     /// which this checkpoint refers.
     ///
@@ -665,15 +668,6 @@ impl Checkpoint {
         self.bridges_len
     }
 
-    /// Returns whether the current state of the tree had been marked at the point that
-    /// this checkpoint was made.
-    ///
-    /// In the event of a rewind, the rewind logic will ensure that mark information is
-    /// properly reconstituted for the checkpointed tree state.
-    pub fn is_marked(&self) -> bool {
-        self.is_marked
-    }
-
     /// Returns a set of the positions that have been marked during the period that this
     /// checkpoint is the current checkpoint.
     pub fn marked(&self) -> &BTreeSet<Position> {
@@ -682,7 +676,7 @@ impl Checkpoint {
 
     /// Returns the set of previously-marked positions that have had their marks removed
     /// during the period that this checkpoint is the current checkpoint.
-    pub fn forgotten(&self) -> &BTreeMap<Position, usize> {
+    pub fn forgotten(&self) -> &BTreeSet<Position> {
         &self.forgotten
     }
 
@@ -714,9 +708,6 @@ impl Checkpoint {
     // using the specified rewrite function. Used during garbage collection.
     fn rewrite_indices<F: Fn(usize) -> usize>(&mut self, f: F) {
         self.bridges_len = f(self.bridges_len);
-        for v in self.forgotten.values_mut() {
-            *v = f(*v)
-        }
     }
 }
 
@@ -732,8 +723,11 @@ pub struct BridgeTree<H, const DEPTH: u8> {
     /// A map from positions for which we wish to be able to compute a
     /// witness to index in the bridges vector.
     saved: BTreeMap<Position, usize>,
-    /// A stack of bridge indices to which it's possible to rewind directly.
+    /// A deque of bridge indices to which it's possible to rewind directly.
-    checkpoints: Vec<Checkpoint>,
+    /// This deque must be maintained to have a minimum size of 1 and a maximum
+    /// size of `max_checkpoints` in order to correctly maintain mark & rewind
+    /// semantics.
+    checkpoints: VecDeque<Checkpoint>,
     /// The maximum number of checkpoints to retain. If this number is
     /// exceeded, the oldest checkpoint will be dropped when creating
     /// a new checkpoint.
@@ -764,24 +758,34 @@ pub enum BridgeTreeError {
 
 impl<H, const DEPTH: u8> BridgeTree<H, DEPTH> {
     /// Construct an empty BridgeTree value with the specified maximum number of checkpoints.
-    pub fn new(max_checkpoints: usize) -> Self {
+    ///
+    /// Panics if `max_checkpoints < 1` because mark/rewind logic depends upon the presence
+    /// of checkpoints to function.
+    pub fn new(max_checkpoints: usize, initial_checkpoint_id: usize) -> Self {
+        assert!(max_checkpoints >= 1);
         Self {
             prior_bridges: vec![],
             current_bridge: None,
             saved: BTreeMap::new(),
-            checkpoints: vec![],
+            checkpoints: VecDeque::from(vec![Checkpoint::at_length(0, initial_checkpoint_id)]),
             max_checkpoints,
         }
     }
 
-    /// Removes the oldest checkpoint. Returns true if successful and false if
+    /// Removes the oldest checkpoint if there are more than `max_checkpoints`. Returns true if
-    /// there are no checkpoints.
+    /// successful and false if there are not enough checkpoints.
     fn drop_oldest_checkpoint(&mut self) -> bool {
-        if self.checkpoints.is_empty() {
+        if self.checkpoints.len() > self.max_checkpoints {
-            false
+            let c = self
-        } else {
+                .checkpoints
-            self.checkpoints.remove(0);
+                .pop_front()
+                .expect("Checkpoints deque is known to be non-empty.");
+            for pos in c.forgotten.iter() {
+                self.saved.remove(pos);
+            }
             true
+        } else {
+            false
         }
     }
 
@@ -802,7 +806,7 @@ impl<H, const DEPTH: u8> BridgeTree<H, DEPTH> {
     }
 
     /// Returns the checkpoints to which this tree may be rewound.
-    pub fn checkpoints(&self) -> &[Checkpoint] {
+    pub fn checkpoints(&self) -> &VecDeque<Checkpoint> {
         &self.checkpoints
     }
 
@@ -832,7 +836,7 @@ impl<H: Hashable + Ord + Clone, const DEPTH: u8> BridgeTree<H, DEPTH> {
                 frontier,
             )),
             saved: BTreeMap::new(),
-            checkpoints: vec![],
+            checkpoints: VecDeque::new(),
             max_checkpoints,
         }
     }
@@ -843,7 +847,7 @@ impl<H: Hashable + Ord + Clone, const DEPTH: u8> BridgeTree<H, DEPTH> {
         prior_bridges: Vec<MerkleBridge<H>>,
         current_bridge: Option<MerkleBridge<H>>,
         saved: BTreeMap<Position, usize>,
-        checkpoints: Vec<Checkpoint>,
+        checkpoints: VecDeque<Checkpoint>,
         max_checkpoints: usize,
     ) -> Result<Self, BridgeTreeError> {
         Self::check_consistency_internal(
@@ -876,7 +880,7 @@ impl<H: Hashable + Ord + Clone, const DEPTH: u8> BridgeTree<H, DEPTH> {
         prior_bridges: &[MerkleBridge<H>],
         current_bridge: &Option<MerkleBridge<H>>,
         saved: &BTreeMap<Position, usize>,
-        checkpoints: &[Checkpoint],
+        checkpoints: &VecDeque<Checkpoint>,
         max_checkpoints: usize,
     ) -> Result<(), BridgeTreeError> {
         // check that saved values correspond to bridges
@@ -969,10 +973,10 @@ impl<H: Hashable + Ord + Clone, const DEPTH: u8> BridgeTree<H, DEPTH> {
         self.current_bridge.as_ref().map(|b| b.current_leaf())
     }
 
-    /// Marks the current leaf as one for which we're interested in producing
+    /// Marks the current leaf as one for which we're interested in producing a witness.
-    /// a witness. Returns an optional value containing the
+    ///
-    /// current position if successful or if the current value was already
+    /// Returns an optional value containing the current position if successful or if the current
-    /// marked, or None if the tree is empty.
+    /// value was already marked, or None if the tree is empty.
     pub fn mark(&mut self) -> Option<Position> {
         match self.current_bridge.take() {
             Some(mut cur_b) => {
@@ -995,15 +999,14 @@ impl<H: Hashable + Ord + Clone, const DEPTH: u8> BridgeTree<H, DEPTH> {
                     self.current_bridge = Some(successor);
                 }
 
-                self.saved
-                    .entry(pos)
-                    .or_insert(self.prior_bridges.len() - 1);
-
                 // mark the position as having been marked in the current checkpoint
-                if let Some(c) = self.checkpoints.last_mut() {
+                if let std::collections::btree_map::Entry::Vacant(e) = self.saved.entry(pos) {
-                    if !c.is_marked {
+                    let c = self
-                        c.marked.insert(pos);
+                        .checkpoints
-                    }
+                        .back_mut()
+                        .expect("Checkpoints deque must never be empty");
+                    c.marked.insert(pos);
+                    e.insert(self.prior_bridges.len() - 1);
                 }
 
                 Some(pos)
@@ -1029,51 +1032,54 @@ impl<H: Hashable + Ord + Clone, const DEPTH: u8> BridgeTree<H, DEPTH> {
     /// interested in maintaining a mark for. Returns true if successful and
     /// false if we were already not maintaining a mark at this position.
     pub fn remove_mark(&mut self, position: Position) -> bool {
-        if let Some(idx) = self.saved.remove(&position) {
+        if self.saved.contains_key(&position) {
-            // If the position is one that has *not* just been marked since the last checkpoint,
+            let c = self
-            // then add it to the set of those forgotten during the current checkpoint span so that
+                .checkpoints
-            // it can be restored on rollback.
+                .back_mut()
-            if let Some(c) = self.checkpoints.last_mut() {
+                .expect("Checkpoints deque must never be empty.");
-                if !c.marked.contains(&position) {
+            c.forgotten.insert(position);
-                    c.forgotten.insert(position, idx);
-                }
-            }
             true
         } else {
             false
         }
     }
 
-    /// Creates a new checkpoint for the current tree state. It is valid to
+    /// Creates a new checkpoint for the current tree state, with the given identifier.
-    /// have multiple checkpoints for the same tree state, and each `rewind`
+    ///
-    /// call will remove a single checkpoint.
+    /// It is valid to have multiple checkpoints for the same tree state, and each `rewind` call
-    pub fn checkpoint(&mut self) {
+    /// will remove a single checkpoint. Successive checkpoint identifiers must always be provided
-        match self.current_bridge.take() {
+    /// in increasing order.
-            Some(cur_b) => {
+    pub fn checkpoint(&mut self, id: usize) -> bool {
-                let is_marked = self.get_marked_leaf(cur_b.position()).is_some();
+        if Some(id) > self.checkpoints.back().map(|c| c.id) {
+            match self.current_bridge.take() {
+                Some(cur_b) => {
+                    // Do not create a duplicate bridge
+                    if self
+                        .prior_bridges
+                        .last()
+                        .map_or(false, |pb| pb.position() == cur_b.position())
+                    {
+                        self.current_bridge = Some(cur_b);
+                    } else {
+                        self.current_bridge = Some(cur_b.successor(false));
+                        self.prior_bridges.push(cur_b);
+                    }
 
-                // Do not create a duplicate bridge
+                    self.checkpoints
-                if self
+                        .push_back(Checkpoint::at_length(self.prior_bridges.len(), id));
-                    .prior_bridges
+                }
-                    .last()
+                None => {
-                    .map_or(false, |pb| pb.position() == cur_b.position())
+                    self.checkpoints.push_back(Checkpoint::at_length(0, id));
-                {
-                    self.current_bridge = Some(cur_b);
-                } else {
-                    self.current_bridge = Some(cur_b.successor(false));
-                    self.prior_bridges.push(cur_b);
                 }
-
-                self.checkpoints
-                    .push(Checkpoint::at_length(self.prior_bridges.len(), is_marked));
             }
-            None => {
-                self.checkpoints.push(Checkpoint::at_length(0, false));
-            }
-        }
 
-        if self.checkpoints.len() > self.max_checkpoints {
+            if self.checkpoints.len() > self.max_checkpoints {
-            self.drop_oldest_checkpoint();
+                self.drop_oldest_checkpoint();
+            }
+
+            true
+        } else {
+            false
         }
     }
 
@@ -1083,80 +1089,72 @@ impl<H: Hashable + Ord + Clone, const DEPTH: u8> BridgeTree<H, DEPTH> {
     /// at that tree state have been removed using `rewind`. This function
     /// return false and leave the tree unmodified if no checkpoints exist.
     pub fn rewind(&mut self) -> bool {
-        match self.checkpoints.pop() {
+        if self.checkpoints.len() > 1 {
-            Some(mut c) => {
+            let c = self
-                // drop marked values at and above the checkpoint height;
+                .checkpoints
-                // we will re-mark if necessary.
+                .pop_back()
-                self.saved.append(&mut c.forgotten);
+                .expect("Checkpoints deque is known to be non-empty.");
-                self.saved.retain(|_, i| *i + 1 < c.bridges_len);
+
-                self.prior_bridges.truncate(c.bridges_len);
+            // Remove marks for positions that were marked during the lifetime of this checkpoint.
-                self.current_bridge = self.prior_bridges.last().map(|b| b.successor(c.is_marked));
+            for pos in c.marked {
-                if c.is_marked {
+                self.saved.remove(&pos);
-                    self.mark();
-                }
-                true
             }
-            None => false,
+
+            self.prior_bridges.truncate(c.bridges_len);
+            self.current_bridge = self
+                .prior_bridges
+                .last()
+                .map(|b| b.successor(self.saved.contains_key(&b.position())));
+            true
+        } else {
+            false
         }
     }
 
-    /// Obtains a witness to the value at the specified position,
+    /// Obtains a witness for the value at the specified leaf position, as of the tree state at the
-    /// as of the tree state corresponding to the given root.
+    /// given checkpoint depth. Returns `None` if there is no witness information for the requested
-    /// Returns `None` if there is no available witness to that
+    /// position or if no checkpoint is available at the specified depth.
-    /// position or if the root does not correspond to a checkpointed
+    pub fn witness(
-    /// root of the tree.
+        &self,
-    pub fn witness(&self, position: Position, as_of_root: &H) -> Option<Vec<H>> {
+        position: Position,
-        self.witness_inner(position, as_of_root).ok()
+        checkpoint_depth: usize,
-    }
+    ) -> Result<Vec<H>, WitnessingError> {
-
-    fn witness_inner(&self, position: Position, as_of_root: &H) -> Result<Vec<H>, WitnessingError> {
         #[derive(Debug)]
         enum AuthBase<'a> {
             Current,
             Checkpoint(usize, &'a Checkpoint),
-            NotFound,
         }
 
-        let max_alt = Level::from(DEPTH);
-
         // Find the earliest checkpoint having a matching root, or the current
         // root if it matches and there is no earlier matching checkpoint.
-        let auth_base = self
+        let auth_base = if checkpoint_depth == 0 {
-            .checkpoints
+            Ok(AuthBase::Current)
-            .iter()
+        } else if self.checkpoints.len() >= checkpoint_depth {
-            .enumerate()
+            let c_idx = self.checkpoints.len() - checkpoint_depth;
-            .rev()
+            if self
-            .take_while(|(_, c)| c.position(&self.prior_bridges) >= Some(position))
+                .checkpoints
-            .filter(|(_, c)| &c.root(&self.prior_bridges, max_alt) == as_of_root)
+                .iter()
-            .last()
+                .skip(c_idx)
-            .map(|(i, c)| AuthBase::Checkpoint(i, c))
+                .take_while(|c| {
-            .unwrap_or_else(|| {
+                    c.position(&self.prior_bridges)
-                if self.root(0).as_ref() == Some(as_of_root) {
+                        .iter()
-                    AuthBase::Current
+                        .any(|p| p <= &position)
-                } else {
+                })
-                    AuthBase::NotFound
+                .any(|c| c.marked.contains(&position))
-                }
+            {
-            });
+                // The mark had not yet been established at the point the checkpoint was
+                // created, so we can't treat it as marked.
+                Err(WitnessingError::PositionNotMarked(position))
+            } else {
+                Ok(AuthBase::Checkpoint(c_idx, &self.checkpoints[c_idx]))
+            }
+        } else {
+            Err(WitnessingError::CheckpointInvalid)
+        }?;
 
         let saved_idx = self
             .saved
             .get(&position)
-            .or_else(|| {
-                if let AuthBase::Checkpoint(i, _) = auth_base {
-                    // The saved position might have been forgotten since the checkpoint,
-                    // so look for it in each of the subsequent checkpoints' forgotten
-                    // items.
-                    self.checkpoints[i..].iter().find_map(|c| {
-                        // restore the forgotten position, if that position was not also marked
-                        // in the same checkpoint
-                        c.forgotten
-                            .get(&position)
-                            .filter(|_| !c.marked.contains(&position))
-                    })
-                } else {
-                    None
-                }
-            })
             .ok_or(WitnessingError::PositionNotMarked(position))?;
 
         let prior_frontier = &self.prior_bridges[*saved_idx].frontier;
@@ -1198,10 +1196,6 @@ impl<H: Hashable + Ord + Clone, const DEPTH: u8> BridgeTree<H, DEPTH> {
                     fuse_from - checkpoint.bridges_len,
                 ))
             }
-            AuthBase::NotFound => {
-                // we didn't find any suitable auth base
-                Err(WitnessingError::AuthBaseNotFound)
-            }
         }?;
 
         successor.witness(DEPTH, prior_frontier)
@@ -1216,17 +1210,7 @@ impl<H: Hashable + Ord + Clone, const DEPTH: u8> BridgeTree<H, DEPTH> {
         // checkpoints; we cannot remove information that we might need to restore in
         // a rewind.
         if self.checkpoints.len() == self.max_checkpoints {
-            let gc_len = self.checkpoints.first().unwrap().bridges_len;
+            let gc_len = self.checkpoints.front().unwrap().bridges_len;
-            // Get a list of the leaf positions that we need to retain. This consists of
-            // all the saved leaves, plus all the leaves that have been forgotten since
-            // the most distant checkpoint to which we could rewind.
-            let remember: BTreeSet<Position> = self
-                .saved
-                .keys()
-                .chain(self.checkpoints.iter().flat_map(|c| c.forgotten.keys()))
-                .cloned()
-                .collect();
-
             let mut cur: Option<MerkleBridge<H>> = None;
             let mut merged = 0;
             let mut ommer_addrs: BTreeSet<Address> = BTreeSet::new();
@@ -1236,7 +1220,7 @@ impl<H: Hashable + Ord + Clone, const DEPTH: u8> BridgeTree<H, DEPTH> {
             {
                 if let Some(cur_bridge) = cur {
                     let pos = cur_bridge.position();
-                    let mut new_cur = if remember.contains(&pos) || i > gc_len {
+                    let mut new_cur = if self.saved.contains_key(&pos) || i > gc_len {
                         // We need to remember cur_bridge; update its save index & put next_bridge
                         // on the chopping block
                         if let Some(idx) = self.saved.get_mut(&pos) {
@@ -1300,9 +1284,8 @@ mod tests {
     use incrementalmerkletree::{
         testing::{
             apply_operation, arb_operation, check_checkpoint_rewind, check_operations,
-            check_rewind_remove_mark, check_rewind_remove_mark_consistency, check_root_hashes,
+            check_remove_mark, check_rewind_remove_mark, check_root_hashes, check_witnesses,
-            check_witnesses, complete_tree::CompleteTree, CombinedTree, Frontier, SipHashable,
+            complete_tree::CompleteTree, CombinedTree, Frontier, SipHashable, Tree,
-            Tree,
         },
         Hashable,
     };
@@ -1317,27 +1300,32 @@ mod tests {
         }
     }
 
-    impl<H: Hashable + Ord + Clone, const DEPTH: u8> Tree<H> for BridgeTree<H, DEPTH> {
+    impl<H: Hashable + Ord + Clone, const DEPTH: u8> Tree<H, usize> for BridgeTree<H, DEPTH> {
-        fn append(&mut self, value: H) -> bool {
+        fn append(&mut self, value: H, retention: Retention<usize>) -> bool {
-            BridgeTree::append(self, value)
+            let appended = BridgeTree::append(self, value);
+            if appended {
+                if retention.is_marked() {
+                    BridgeTree::mark(self);
+                }
+                if let Retention::Checkpoint { id, .. } = retention {
+                    BridgeTree::checkpoint(self, id);
+                }
+            }
+            appended
+        }
+
+        fn depth(&self) -> u8 {
+            DEPTH
         }
 
         fn current_position(&self) -> Option<Position> {
             BridgeTree::current_position(self)
         }
 
-        fn current_leaf(&self) -> Option<&H> {
-            BridgeTree::current_leaf(self)
-        }
-
         fn get_marked_leaf(&self, position: Position) -> Option<&H> {
             BridgeTree::get_marked_leaf(self, position)
         }
 
-        fn mark(&mut self) -> Option<Position> {
-            BridgeTree::mark(self)
-        }
-
         fn marked_positions(&self) -> BTreeSet<Position> {
             BridgeTree::marked_positions(self)
         }
@@ -1346,16 +1334,16 @@ mod tests {
             BridgeTree::root(self, checkpoint_depth)
         }
 
-        fn witness(&self, position: Position, as_of_root: &H) -> Option<Vec<H>> {
+        fn witness(&self, position: Position, checkpoint_depth: usize) -> Option<Vec<H>> {
-            BridgeTree::witness(self, position, as_of_root)
+            BridgeTree::witness(self, position, checkpoint_depth).ok()
         }
 
         fn remove_mark(&mut self, position: Position) -> bool {
             BridgeTree::remove_mark(self, position)
         }
 
-        fn checkpoint(&mut self) {
+        fn checkpoint(&mut self, id: usize) -> bool {
-            BridgeTree::checkpoint(self)
+            BridgeTree::checkpoint(self, id)
         }
 
         fn rewind(&mut self) -> bool {
@@ -1460,13 +1448,29 @@ mod tests {
 
     #[test]
     fn tree_depth() {
-        let mut tree = BridgeTree::<String, 3>::new(100);
+        let mut tree = BridgeTree::<String, 3>::new(100, 0);
         for c in 'a'..'i' {
             assert!(tree.append(c.to_string()))
         }
         assert!(!tree.append('i'.to_string()));
     }
 
+    fn check_garbage_collect<H: Hashable + Clone + Ord, const DEPTH: u8>(
+        mut tree: BridgeTree<H, DEPTH>,
+    ) {
+        // Add checkpoints until we're sure everything that can be gc'ed will be gc'ed
+        for i in 0..tree.max_checkpoints {
+            tree.checkpoint(i + 1);
+        }
+
+        let mut tree_mut = tree.clone();
+        tree_mut.garbage_collect();
+
+        for pos in tree.saved.keys() {
+            assert_eq!(tree.witness(*pos, 0), tree_mut.witness(*pos, 0));
+        }
+    }
+
     fn arb_bridgetree<G: Strategy + Clone>(
         item_gen: G,
         max_count: usize,
@@ -1476,9 +1480,9 @@ mod tests {
     {
         proptest::collection::vec(arb_operation(item_gen, 0..max_count), 0..max_count).prop_map(
             |ops| {
-                let mut tree: BridgeTree<G::Value, 8> = BridgeTree::new(10);
+                let mut tree: BridgeTree<G::Value, 8> = BridgeTree::new(10, 0);
-                for op in ops {
+                for (i, op) in ops.into_iter().enumerate() {
-                    apply_operation(&mut tree, op);
+                    apply_operation(&mut tree, op.map_checkpoint_id(|_| i));
                 }
                 tree
             },
@@ -1506,70 +1510,53 @@ mod tests {
         fn prop_garbage_collect(
             tree in arb_bridgetree((97u8..123).prop_map(|c| char::from(c).to_string()), 100)
         ) {
-            let mut tree_mut = tree.clone();
+            check_garbage_collect(tree);
-            // ensure we have enough checkpoints to not rewind past the state `tree` is in
-            for _ in 0..10 {
-                tree_mut.checkpoint();
-            }
-
-            tree_mut.garbage_collect();
-
-            tree_mut.rewind();
-
-            for pos in tree.saved.keys() {
-                assert_eq!(
-                    tree.witness(*pos, &tree.root(0).unwrap()),
-                    tree_mut.witness(*pos, &tree.root(0).unwrap())
-                );
-            }
         }
     }
 
-    #[test]
-    fn drop_oldest_checkpoint() {
-        let mut t = BridgeTree::<String, 6>::new(100);
-        t.checkpoint();
-        t.append("a".to_string());
-        t.mark();
-        t.append("b".to_string());
-        t.append("c".to_string());
-        assert!(
-            t.drop_oldest_checkpoint(),
-            "Checkpoint drop is expected to succeed"
-        );
-        assert!(!t.rewind(), "Rewind is expected to fail.");
-    }
-
     #[test]
     fn root_hashes() {
-        check_root_hashes(BridgeTree::<String, 4>::new);
+        check_root_hashes(|max_checkpoints| BridgeTree::<String, 4>::new(max_checkpoints, 0));
     }
 
     #[test]
-    fn witnesss() {
+    fn witness() {
-        check_witnesses(BridgeTree::<String, 4>::new);
+        check_witnesses(|max_checkpoints| BridgeTree::<String, 4>::new(max_checkpoints, 0));
     }
 
     #[test]
     fn checkpoint_rewind() {
-        check_checkpoint_rewind(BridgeTree::<String, 4>::new);
+        check_checkpoint_rewind(|max_checkpoints| BridgeTree::<String, 4>::new(max_checkpoints, 0));
     }
 
     #[test]
     fn rewind_remove_mark() {
-        check_rewind_remove_mark(BridgeTree::<String, 4>::new);
+        check_rewind_remove_mark(|max_checkpoints| {
+            BridgeTree::<String, 4>::new(max_checkpoints, 0)
+        });
     }
 
     #[test]
     fn garbage_collect() {
-        let mut t = BridgeTree::<String, 7>::new(10);
+        let mut tree: BridgeTree<String, 7> = BridgeTree::new(1000, 0);
+        let empty_root = tree.root(0);
+        tree.append("a".to_string());
+        for i in 0..100 {
+            tree.checkpoint(i + 1);
+        }
+        tree.garbage_collect();
+        assert!(tree.root(0) != empty_root);
+        tree.rewind();
+        assert!(tree.root(0) != empty_root);
+
+        let mut t = BridgeTree::<String, 7>::new(10, 0);
         let mut to_unmark = vec![];
         let mut has_witness = vec![];
         for i in 0usize..100 {
             let elem: String = format!("{},", i);
             assert!(t.append(elem), "Append should succeed.");
             if i % 5 == 0 {
-                t.checkpoint();
+                t.checkpoint(i + 1);
             }
             if i % 7 == 0 {
                 t.mark();
@@ -1586,9 +1573,9 @@ mod tests {
         }
         // 32 = 20 (checkpointed) + 14 (marked) - 2 (marked & checkpointed)
         assert_eq!(t.prior_bridges().len(), 20 + 14 - 2);
-        let witnesss = has_witness
+        let witness = has_witness
             .iter()
-            .map(|pos| match t.witness_inner(*pos, &t.root(0).unwrap()) {
+            .map(|pos| match t.witness(*pos, 0) {
                 Ok(path) => path,
                 Err(e) => panic!("Failed to get auth path: {:?}", e),
             })
@@ -1596,48 +1583,31 @@ mod tests {
         t.garbage_collect();
         // 20 = 32 - 10 (removed checkpoints) + 1 (not removed due to mark) - 3 (removed marks)
         assert_eq!(t.prior_bridges().len(), 32 - 10 + 1 - 3);
-        let retained_witnesss = has_witness
+        let retained_witness = has_witness
             .iter()
-            .map(|pos| {
+            .map(|pos| t.witness(*pos, 0).expect("Must be able to get auth path"))
-                t.witness(*pos, &t.root(0).unwrap())
-                    .expect("Must be able to get auth path")
-            })
             .collect::<Vec<_>>();
-        assert_eq!(witnesss, retained_witnesss);
+        assert_eq!(witness, retained_witness);
-    }
-
-    #[test]
-    fn garbage_collect_idx() {
-        let mut tree: BridgeTree<String, 7> = BridgeTree::new(100);
-        let empty_root = tree.root(0);
-        tree.append("a".to_string());
-        for _ in 0..100 {
-            tree.checkpoint();
-        }
-        tree.garbage_collect();
-        assert!(tree.root(0) != empty_root);
-        tree.rewind();
-        assert!(tree.root(0) != empty_root);
     }
 
     // Combined tree tests
     fn new_combined_tree<H: Hashable + Ord + Clone + Debug>(
         max_checkpoints: usize,
-    ) -> CombinedTree<H, CompleteTree<H>, BridgeTree<H, 4>> {
+    ) -> CombinedTree<H, usize, CompleteTree<H, usize, 4>, BridgeTree<H, 4>> {
         CombinedTree::new(
-            CompleteTree::new(4, max_checkpoints),
+            CompleteTree::<H, usize, 4>::new(max_checkpoints, 0),
-            BridgeTree::<H, 4>::new(max_checkpoints),
+            BridgeTree::<H, 4>::new(max_checkpoints, 0),
         )
     }
 
     #[test]
-    fn test_rewind_remove_mark() {
+    fn combined_remove_mark() {
-        check_rewind_remove_mark(new_combined_tree);
+        check_remove_mark(new_combined_tree);
     }
 
     #[test]
-    fn test_rewind_remove_mark_consistency() {
+    fn combined_rewind_remove_mark() {
-        check_rewind_remove_mark_consistency(new_combined_tree);
+        check_rewind_remove_mark(new_combined_tree);
     }
 
     proptest! {
@@ -1651,7 +1621,8 @@ mod tests {
             )
         ) {
             let tree = new_combined_tree(100);
-            check_operations(tree, 4, &ops)?;
+            let indexed_ops = ops.iter().enumerate().map(|(i, op)| op.map_checkpoint_id(|_| i + 1)).collect::<Vec<_>>();
+            check_operations(tree, &indexed_ops)?;
         }
 
         #[test]
@@ -1662,7 +1633,8 @@ mod tests {
             )
         ) {
             let tree = new_combined_tree(100);
-            check_operations(tree, 4, &ops)?;
+            let indexed_ops = ops.iter().enumerate().map(|(i, op)| op.map_checkpoint_id(|_| i + 1)).collect::<Vec<_>>();
+            check_operations(tree, &indexed_ops)?;
         }
     }
 }

incrementalmerkletree/CHANGELOG.md

@@ -41,7 +41,9 @@ is not another good use case for polymorphism over tree implementations.
   - `Tree::witnessed_positions` has been renamed to `Tree::marked_positions`
   - `Tree::get_witnessed_leaf` has been renamed to `Tree::get_marked_leaf`
   - `Tree::remove_witness` has been renamed to `Tree::remove_mark`
-  - `Tree::authentication_path` has been renamed to `Tree::witness`
+  - `Tree::authentication_path` has been renamed to `Tree::witness`. Also, this method
+    now takes a checkpoint depth as its second argument rather than a Merkle root,
+    to better support future changes.
 - `Tree::append` now takes ownership of the value being appended instead of a value passed
   by reference.
 

incrementalmerkletree/src/lib.rs

@@ -10,6 +10,39 @@ use std::ops::{Add, AddAssign, Range, Sub};
 #[cfg(feature = "test-dependencies")]
 pub mod testing;
 
+/// A type for metadata that is used to determine when and how a leaf can be pruned from a tree.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Retention<C> {
+    Ephemeral,
+    Checkpoint { id: C, is_marked: bool },
+    Marked,
+}
+
+impl<C> Retention<C> {
+    pub fn is_checkpoint(&self) -> bool {
+        matches!(self, Retention::Checkpoint { .. })
+    }
+
+    pub fn is_marked(&self) -> bool {
+        match self {
+            Retention::Ephemeral => false,
+            Retention::Checkpoint { is_marked, .. } => *is_marked,
+            Retention::Marked => true,
+        }
+    }
+
+    pub fn map<'a, D, F: Fn(&'a C) -> D>(&'a self, f: F) -> Retention<D> {
+        match self {
+            Retention::Ephemeral => Retention::Ephemeral,
+            Retention::Checkpoint { id, is_marked } => Retention::Checkpoint {
+                id: f(id),
+                is_marked: *is_marked,
+            },
+            Retention::Marked => Retention::Marked,
+        }
+    }
+}
+
 /// A type representing the position of a leaf in a Merkle tree.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 #[repr(transparent)]
@@ -373,7 +406,7 @@ impl<'a> From<&'a Address> for Option<Position> {
 
 /// A trait describing the operations that make a type suitable for use as
 /// a leaf or node value in a merkle tree.
-pub trait Hashable: Sized {
+pub trait Hashable: Sized + core::fmt::Debug {
     fn empty_leaf() -> Self;
 
     fn combine(level: Level, a: &Self, b: &Self) -> Self;

incrementalmerkletree/src/testing/complete_tree.rs

@@ -1,13 +1,17 @@
 //! Sample implementation of the Tree interface.
-use std::collections::BTreeSet;
+use std::cmp::min;
+use std::collections::{BTreeMap, BTreeSet};
 
-use crate::{
+use crate::{testing::Tree, Hashable, Level, Position, Retention};
-    testing::{Frontier, Tree},
-    Hashable, Level, Position,
-};
 
-pub(crate) fn root<H: Hashable + Clone>(mut leaves: Vec<H>) -> H {
+pub(crate) fn root<H: Hashable + Clone>(leaves: &[H], depth: u8) -> H {
-    leaves.resize(leaves.len().next_power_of_two(), H::empty_leaf());
+    let empty_leaf = H::empty_leaf();
+    let mut leaves = leaves
+        .iter()
+        .chain(std::iter::repeat(&empty_leaf))
+        .take(1 << depth)
+        .cloned()
+        .collect::<Vec<H>>();
 
     //leaves are always at level zero, so we start there.
     let mut level = Level::from(0);
@@ -32,214 +36,277 @@ pub(crate) fn root<H: Hashable + Clone>(mut leaves: Vec<H>) -> H {
     leaves[0].clone()
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Checkpoint {
+    /// The number of leaves in the tree when the checkpoint was created.
+    leaves_len: usize,
+    /// A set of the positions that have been marked during the period that this
+    /// checkpoint is the current checkpoint.
+    marked: BTreeSet<Position>,
+    /// When a mark is forgotten, we add it to the checkpoint's forgotten set but
+    /// don't immediately remove it from the `marked` set; that removal occurs when
+    /// the checkpoint is eventually dropped.
+    forgotten: BTreeSet<Position>,
+}
+
+impl Checkpoint {
+    fn at_length(leaves_len: usize) -> Self {
+        Checkpoint {
+            leaves_len,
+            marked: BTreeSet::new(),
+            forgotten: BTreeSet::new(),
+        }
+    }
+}
+
 #[derive(Clone, Debug)]
-pub struct TreeState<H: Hashable> {
+pub struct CompleteTree<H, C: Ord, const DEPTH: u8> {
-    leaves: Vec<H>,
+    leaves: Vec<Option<H>>,
-    current_offset: usize,
     marks: BTreeSet<Position>,
-    depth: usize,
+    checkpoints: BTreeMap<C, Checkpoint>,
+    max_checkpoints: usize,
 }
 
-impl<H: Hashable + Clone> TreeState<H> {
+impl<H: Hashable, C: Clone + Ord + core::fmt::Debug, const DEPTH: u8> CompleteTree<H, C, DEPTH> {
-    /// Creates a new, empty binary tree of specified depth.
+    /// Creates a new, empty binary tree
-    pub fn new(depth: usize) -> Self {
+    pub fn new(max_checkpoints: usize, initial_checkpoint_id: C) -> Self {
         Self {
-            leaves: vec![H::empty_leaf(); 1 << depth],
+            leaves: vec![],
-            current_offset: 0,
             marks: BTreeSet::new(),
-            depth,
+            checkpoints: BTreeMap::from([(initial_checkpoint_id, Checkpoint::at_length(0))]),
-        }
+            max_checkpoints,
-    }
-}
-
-impl<H: Hashable + Clone> Frontier<H> for TreeState<H> {
-    fn append(&mut self, value: H) -> bool {
-        if self.current_offset == (1 << self.depth) {
-            false
-        } else {
-            self.leaves[self.current_offset] = value;
-            self.current_offset += 1;
-            true
         }
     }
 
-    /// Obtains the current root of this Merkle tree.
+    /// Appends a new value to the tree at the next available slot.
-    fn root(&self) -> H {
+    ///
-        root(self.leaves.clone())
+    /// Returns true if successful and false if the tree is full or, for values with `Checkpoint`
-    }
+    /// retention, if a checkpoint id would be introduced that is less than or equal to the current
-}
+    /// maximum checkpoint id.
+    fn append(&mut self, value: H, retention: Retention<C>) -> Result<(), AppendError<C>> {
+        fn append<H, C>(
+            leaves: &mut Vec<Option<H>>,
+            value: H,
+            depth: u8,
+        ) -> Result<(), AppendError<C>> {
+            if leaves.len() < (1 << depth) {
+                leaves.push(Some(value));
+                Ok(())
+            } else {
+                Err(AppendError::TreeFull)
+            }
+        }
+
+        match retention {
+            Retention::Marked => {
+                append(&mut self.leaves, value, DEPTH)?;
+                self.mark();
+            }
+            Retention::Checkpoint { id, is_marked } => {
+                let latest_checkpoint = self.checkpoints.keys().rev().next();
+                if Some(&id) > latest_checkpoint {
+                    append(&mut self.leaves, value, DEPTH)?;
+                    if is_marked {
+                        self.mark();
+                    }
+                    self.checkpoint(id, self.current_position());
+                } else {
+                    return Err(AppendError::CheckpointOutOfOrder {
+                        current_max: latest_checkpoint.cloned(),
+                        checkpoint: id,
+                    });
+                }
+            }
+            Retention::Ephemeral => {
+                append(&mut self.leaves, value, DEPTH)?;
+            }
+        }
+
+        Ok(())
+    }
 
-impl<H: Hashable + PartialEq + Clone> TreeState<H> {
     fn current_position(&self) -> Option<Position> {
-        if self.current_offset == 0 {
+        if self.leaves.is_empty() {
             None
         } else {
-            Some((self.current_offset - 1).into())
+            Some((self.leaves.len() - 1).into())
-        }
-    }
-
-    /// Returns the leaf most recently appended to the tree
-    fn current_leaf(&self) -> Option<&H> {
-        self.current_position()
-            .map(|p| &self.leaves[<usize>::from(p)])
-    }
-
-    /// Returns the leaf at the specified position if the tree can produce
-    /// a witness for it.
-    fn get_marked_leaf(&self, position: Position) -> Option<&H> {
-        if self.marks.contains(&position) {
-            self.leaves.get(<usize>::from(position))
-        } else {
-            None
         }
     }
 
     /// Marks the current tree state leaf as a value that we're interested in
     /// marking. Returns the current position if the tree is non-empty.
     fn mark(&mut self) -> Option<Position> {
-        self.current_position().map(|pos| {
+        match self.current_position() {
-            self.marks.insert(pos);
+            Some(pos) => {
-            pos
+                if !self.marks.contains(&pos) {
-        })
+                    self.marks.insert(pos);
-    }
+                    self.checkpoints
-
+                        .iter_mut()
-    /// Obtains a witness to the value at the specified position.
+                        .rev()
-    /// Returns `None` if there is no available witness to that
+                        .next()
-    /// value.
+                        .unwrap()
-    fn witness(&self, position: Position) -> Option<Vec<H>> {
+                        .1
-        if Some(position) <= self.current_position() {
+                        .marked
-            let mut path = vec![];
+                        .insert(pos);
-
-            let mut leaf_idx: usize = position.into();
-            for bit in 0..self.depth {
-                leaf_idx ^= 1 << bit;
-                path.push(root::<H>(self.leaves[leaf_idx..][0..(1 << bit)].to_vec()));
-                leaf_idx &= usize::MAX << (bit + 1);
-            }
-
-            Some(path)
-        } else {
-            None
-        }
-    }
-
-    /// Marks the value at the specified position as a value we're no longer
-    /// interested in maintaining a mark for. Returns true if successful and
-    /// false if we were already not maintaining a mark at this position.
-    fn remove_mark(&mut self, position: Position) -> bool {
-        self.marks.remove(&position)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct CompleteTree<H: Hashable> {
-    tree_state: TreeState<H>,
-    checkpoints: Vec<TreeState<H>>,
-    max_checkpoints: usize,
-}
-
-impl<H: Hashable + Clone> CompleteTree<H> {
-    /// Creates a new, empty binary tree of specified depth.
-    pub fn new(depth: usize, max_checkpoints: usize) -> Self {
-        Self {
-            tree_state: TreeState::new(depth),
-            checkpoints: vec![],
-            max_checkpoints,
-        }
-    }
-}
-
-impl<H: Hashable + PartialEq + Clone> CompleteTree<H> {
-    /// Removes the oldest checkpoint. Returns true if successful and false if
-    /// there are no checkpoints.
-    fn drop_oldest_checkpoint(&mut self) -> bool {
-        if self.checkpoints.is_empty() {
-            false
-        } else {
-            self.checkpoints.remove(0);
-            true
-        }
-    }
-
-    /// Retrieve the tree state at the specified checkpoint depth. This
-    /// is the current tree state if the depth is 0, and this will return
-    /// None if not enough checkpoints exist to obtain the requested depth.
-    fn tree_state_at_checkpoint_depth(&self, checkpoint_depth: usize) -> Option<&TreeState<H>> {
-        if self.checkpoints.len() < checkpoint_depth {
-            None
-        } else if checkpoint_depth == 0 {
-            Some(&self.tree_state)
-        } else {
-            self.checkpoints
-                .get(self.checkpoints.len() - checkpoint_depth)
-        }
-    }
-}
-
-impl<H: Hashable + PartialEq + Clone + std::fmt::Debug> Tree<H> for CompleteTree<H> {
-    /// Appends a new value to the tree at the next available slot. Returns true
-    /// if successful and false if the tree is full.
-    fn append(&mut self, value: H) -> bool {
-        self.tree_state.append(value)
-    }
-
-    /// Returns the most recently appended leaf value.
-    fn current_position(&self) -> Option<Position> {
-        self.tree_state.current_position()
-    }
-
-    fn current_leaf(&self) -> Option<&H> {
-        self.tree_state.current_leaf()
-    }
-
-    fn get_marked_leaf(&self, position: Position) -> Option<&H> {
-        self.tree_state.get_marked_leaf(position)
-    }
-
-    fn mark(&mut self) -> Option<Position> {
-        self.tree_state.mark()
-    }
-
-    fn marked_positions(&self) -> BTreeSet<Position> {
-        self.tree_state.marks.clone()
-    }
-
-    fn root(&self, checkpoint_depth: usize) -> Option<H> {
-        self.tree_state_at_checkpoint_depth(checkpoint_depth)
-            .map(|s| s.root())
-    }
-
-    fn witness(&self, position: Position, root: &H) -> Option<Vec<H>> {
-        // Search for the checkpointed state corresponding to the provided root, and if one is
-        // found, compute the witness as of that root.
-        self.checkpoints
-            .iter()
-            .chain(Some(&self.tree_state))
-            .rev()
-            .skip_while(|c| !c.marks.contains(&position))
-            .find_map(|c| {
-                if &c.root() == root {
-                    c.witness(position)
-                } else {
-                    None
                 }
-            })
+                Some(pos)
+            }
+            None => None,
+        }
     }
 
-    fn remove_mark(&mut self, position: Position) -> bool {
+    fn checkpoint(&mut self, id: C, pos: Option<Position>) {
-        self.tree_state.remove_mark(position)
+        self.checkpoints.insert(
-    }
+            id,
-
+            Checkpoint::at_length(pos.map_or_else(|| 0, |p| usize::from(p) + 1)),
-    fn checkpoint(&mut self) {
+        );
-        self.checkpoints.push(self.tree_state.clone());
         if self.checkpoints.len() > self.max_checkpoints {
             self.drop_oldest_checkpoint();
         }
     }
 
+    fn leaves_at_checkpoint_depth(&self, checkpoint_depth: usize) -> Option<usize> {
+        if checkpoint_depth == 0 {
+            Some(self.leaves.len())
+        } else {
+            self.checkpoints
+                .iter()
+                .rev()
+                .skip(checkpoint_depth - 1)
+                .map(|(_, c)| c.leaves_len)
+                .next()
+        }
+    }
+
+    /// Removes the oldest checkpoint. Returns true if successful and false if
+    /// there are fewer than `self.max_checkpoints` checkpoints.
+    fn drop_oldest_checkpoint(&mut self) -> bool {
+        if self.checkpoints.len() > self.max_checkpoints {
+            let (id, c) = self.checkpoints.iter().next().unwrap();
+            for pos in c.forgotten.iter() {
+                self.marks.remove(pos);
+            }
+            let id = id.clone(); // needed to avoid mutable/immutable borrow conflict
+            self.checkpoints.remove(&id);
+            true
+        } else {
+            false
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+enum AppendError<C> {
+    TreeFull,
+    CheckpointOutOfOrder {
+        current_max: Option<C>,
+        checkpoint: C,
+    },
+}
+
+impl<H: Hashable + PartialEq + Clone, C: Ord + Clone + core::fmt::Debug, const DEPTH: u8> Tree<H, C>
+    for CompleteTree<H, C, DEPTH>
+{
+    fn depth(&self) -> u8 {
+        DEPTH
+    }
+
+    fn append(&mut self, value: H, retention: Retention<C>) -> bool {
+        Self::append(self, value, retention).is_ok()
+    }
+
+    fn current_position(&self) -> Option<Position> {
+        Self::current_position(self)
+    }
+
+    fn marked_positions(&self) -> BTreeSet<Position> {
+        self.marks.clone()
+    }
+
+    fn get_marked_leaf(&self, position: Position) -> Option<&H> {
+        if self.marks.contains(&position) {
+            self.leaves
+                .get(usize::from(position))
+                .and_then(|opt: &Option<H>| opt.as_ref())
+        } else {
+            None
+        }
+    }
+
+    fn root(&self, checkpoint_depth: usize) -> Option<H> {
+        self.leaves_at_checkpoint_depth(checkpoint_depth)
+            .and_then(|len| root(&self.leaves[0..len], DEPTH))
+    }
+
+    fn witness(&self, position: Position, checkpoint_depth: usize) -> Option<Vec<H>> {
+        if self.marks.contains(&position) && checkpoint_depth <= self.checkpoints.len() {
+            let leaves_len = self.leaves_at_checkpoint_depth(checkpoint_depth)?;
+            let c_idx = self.checkpoints.len() - checkpoint_depth;
+            if self
+                .checkpoints
+                .iter()
+                .skip(c_idx)
+                .any(|(_, c)| c.marked.contains(&position))
+            {
+                // The requested position was marked after the checkpoint was created, so we
+                // cannot create a witness.
+                None
+            } else {
+                let mut path = vec![];
+
+                let mut leaf_idx: usize = position.into();
+                for bit in 0..DEPTH {
+                    leaf_idx ^= 1 << bit;
+                    path.push(if leaf_idx < leaves_len {
+                        let subtree_end = min(leaf_idx + (1 << bit), leaves_len);
+                        root(&self.leaves[leaf_idx..subtree_end], bit)?
+                    } else {
+                        H::empty_root(Level::from(bit))
+                    });
+                    leaf_idx &= usize::MAX << (bit + 1);
+                }
+
+                Some(path)
+            }
+        } else {
+            None
+        }
+    }
+
+    fn remove_mark(&mut self, position: Position) -> bool {
+        if self.marks.contains(&position) {
+            self.checkpoints
+                .iter_mut()
+                .rev()
+                .next()
+                .unwrap()
+                .1
+                .forgotten
+                .insert(position);
+            true
+        } else {
+            false
+        }
+    }
+
+    fn checkpoint(&mut self, id: C) -> bool {
+        if Some(&id) > self.checkpoints.iter().rev().next().map(|(id, _)| id) {
+            Self::checkpoint(self, id, self.current_position());
+            true
+        } else {
+            false
+        }
+    }
+
     fn rewind(&mut self) -> bool {
-        if let Some(checkpointed_state) = self.checkpoints.pop() {
+        if self.checkpoints.len() > 1 {
-            self.tree_state = checkpointed_state;
+            let (id, c) = self.checkpoints.iter().rev().next().unwrap();
+            self.leaves.truncate(c.leaves_len);
+            for pos in c.marked.iter() {
+                self.marks.remove(pos);
+            }
+            let id = id.clone(); // needed to avoid mutable/immutable borrow conflict
+            self.checkpoints.remove(&id);
             true
         } else {
             false
@@ -257,7 +324,7 @@ mod tests {
             check_checkpoint_rewind, check_rewind_remove_mark, check_root_hashes, check_witnesses,
             compute_root_from_witness, SipHashable, Tree,
         },
-        Hashable, Level, Position,
+        Hashable, Level, Position, Retention,
     };
 
     #[test]
@@ -268,20 +335,20 @@ mod tests {
             expected = SipHashable::combine(lvl.into(), &expected, &expected);
         }
 
-        let tree = CompleteTree::<SipHashable>::new(DEPTH as usize, 100);
+        let tree = CompleteTree::<SipHashable, (), DEPTH>::new(100, ());
         assert_eq!(tree.root(0).unwrap(), expected);
     }
 
     #[test]
     fn correct_root() {
-        const DEPTH: usize = 3;
+        const DEPTH: u8 = 3;
         let values = (0..(1 << DEPTH)).into_iter().map(SipHashable);
 
-        let mut tree = CompleteTree::<SipHashable>::new(DEPTH, 100);
+        let mut tree = CompleteTree::<SipHashable, (), DEPTH>::new(100, ());
         for value in values {
-            assert!(tree.append(value));
+            assert!(tree.append(value, Retention::Ephemeral).is_ok());
         }
-        assert!(!tree.append(SipHashable(0)));
+        assert!(tree.append(SipHashable(0), Retention::Ephemeral).is_err());
 
         let expected = SipHashable::combine(
             Level::from(2),
@@ -302,25 +369,30 @@ mod tests {
 
     #[test]
     fn root_hashes() {
-        check_root_hashes(|max_c| CompleteTree::<String>::new(4, max_c));
+        check_root_hashes(|max_checkpoints| {
+            CompleteTree::<String, usize, 4>::new(max_checkpoints, 0)
+        });
     }
 
     #[test]
-    fn witnesss() {
+    fn witness() {
-        check_witnesses(|max_c| CompleteTree::<String>::new(4, max_c));
+        check_witnesses(|max_checkpoints| {
+            CompleteTree::<String, usize, 4>::new(max_checkpoints, 0)
+        });
     }
 
     #[test]
     fn correct_witness() {
-        const DEPTH: usize = 3;
+        use crate::{testing::Tree, Retention};
+
+        const DEPTH: u8 = 3;
         let values = (0..(1 << DEPTH)).into_iter().map(SipHashable);
 
-        let mut tree = CompleteTree::<SipHashable>::new(DEPTH, 100);
+        let mut tree = CompleteTree::<SipHashable, (), DEPTH>::new(100, ());
         for value in values {
-            assert!(tree.append(value));
+            assert!(Tree::append(&mut tree, value, Retention::Marked));
-            tree.mark();
         }
-        assert!(!tree.append(SipHashable(0)));
+        assert!(tree.append(SipHashable(0), Retention::Ephemeral).is_err());
 
         let expected = SipHashable::combine(
             <Level>::from(2),
@@ -340,7 +412,7 @@ mod tests {
 
         for i in 0u64..(1 << DEPTH) {
             let position = Position::try_from(i).unwrap();
-            let path = tree.witness(position, &tree.root(0).unwrap()).unwrap();
+            let path = tree.witness(position, 0).unwrap();
             assert_eq!(
                 compute_root_from_witness(SipHashable(i), position, &path),
                 expected
@@ -350,11 +422,15 @@ mod tests {
 
     #[test]
     fn checkpoint_rewind() {
-        check_checkpoint_rewind(|max_c| CompleteTree::<String>::new(4, max_c));
+        check_checkpoint_rewind(|max_checkpoints| {
+            CompleteTree::<String, usize, 4>::new(max_checkpoints, 0)
+        });
     }
 
     #[test]
     fn rewind_remove_mark() {
-        check_rewind_remove_mark(|max_c| CompleteTree::<String>::new(4, max_c));
+        check_rewind_remove_mark(|max_checkpoints| {
+            CompleteTree::<String, usize, 4>::new(max_checkpoints, 0)
+        });
     }
 }