From e55ff2d7f2a9b135567d7647b91e9e2e9817ff08 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 20 May 2024 11:14:40 -0600 Subject: [PATCH] incrementalmerkletree: add `Reference` retention type. This new retention type is intended to be used when inserting frontiers that should not automatically be pruned. Also, improve documentation for the `Retention` type. --- incrementalmerkletree/CHANGELOG.md | 9 ++++ incrementalmerkletree/src/lib.rs | 49 ++++++++++++++++--- incrementalmerkletree/src/testing.rs | 42 ++++++++++------ .../src/testing/complete_tree.rs | 7 +-- shardtree/src/legacy.rs | 6 ++- shardtree/src/lib.rs | 20 ++++---- shardtree/src/prunable.rs | 24 ++++++--- shardtree/src/store/caching.rs | 28 +++++------ shardtree/src/testing.rs | 21 +++++--- 9 files changed, 143 insertions(+), 63 deletions(-) diff --git a/incrementalmerkletree/CHANGELOG.md b/incrementalmerkletree/CHANGELOG.md index 81e5fd3..efa8906 100644 --- a/incrementalmerkletree/CHANGELOG.md +++ b/incrementalmerkletree/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to Rust's notion of ## Unreleased +### Added +- `incrementalmerkletree::Marking` + +### Changed +- `incrementalmerkletree::Retention` + - Has added variant `Retention::Reference` + - `Retention::Checkpoint::is_marked` has been replaced by `Retention::Checkpoint::marking` + to permit checkpoints with `Reference` retention to be represented. + ## [0.5.1] - 2024-03-25 ### Added diff --git a/incrementalmerkletree/src/lib.rs b/incrementalmerkletree/src/lib.rs index 2391dba..a300c67 100644 --- a/incrementalmerkletree/src/lib.rs +++ b/incrementalmerkletree/src/lib.rs @@ -62,35 +62,72 @@ pub mod witness; #[cfg_attr(docsrs, doc(cfg(feature = "test-dependencies")))] pub mod testing; +/// An enumeration of the additional marking states that can be applied +/// to leaves with [`Retention::Checkpoint`] retention. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Marking { + /// A checkpoint with `Marked` marking will have `Marked` retention after `Checkpoint` + /// retention is removed. + Marked, + /// A checkpoint with `Reference` marking will have `Reference` retention after `Checkpoint` + /// retention is removed. + Reference, + /// A checkpoint with `None` marking will have `Ephemeral` retention after `Checkpoint` + /// retention is removed. + None, +} + /// A type for metadata that is used to determine when and how a leaf can be pruned from a tree. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum Retention { + /// A leaf with `Ephemeral` retention will be pruned whenever its sibling is also a leaf with + /// `Ephemeral` retention. Ephemeral, - Checkpoint { id: C, is_marked: bool }, + /// A leaf with `Checkpoint` retention will be retained in the tree during pruning. If + /// `Checkpoint` retention is removed from the leaf, then the retention for the leaf will + /// become either `Ephemeral`, `Marked`, or `Reference` depending upon the value of the + /// `marking` field. + Checkpoint { id: C, marking: Marking }, + /// A leaf with `Marked` retention will be retained in the tree during pruning. `Marked` + /// retention may only be explicitly removed. Marked, + /// A leaf with `Reference` retention will be retained in the tree during pruning. `Reference` + /// retention is removed whenever the associated leaf is overwritten with a tree node having + /// `Ephemeral`, `Checkpoint`, or `Marked` retention. + Reference, } impl Retention { + /// Returns whether the associated node has [`Retention::Checkpoint`] retention. pub fn is_checkpoint(&self) -> bool { matches!(self, Retention::Checkpoint { .. }) } + /// Returns whether the associated node has [`Retention::Marked`] retention + /// or [`Retention::Checkpoint`] retention with [`Marking::Marked`] marking. pub fn is_marked(&self) -> bool { match self { - Retention::Ephemeral => false, - Retention::Checkpoint { is_marked, .. } => *is_marked, - Retention::Marked => true, + Retention::Marked + | Retention::Checkpoint { + marking: Marking::Marked, + .. + } => true, + _ => false, } } + /// Applies the provided function to the checkpoint identifier, if any, and returns a new + /// `Retention` value having the same structure with the resulting value used as the checkpoint + /// identifier. pub fn map<'a, D, F: Fn(&'a C) -> D>(&'a self, f: F) -> Retention { match self { Retention::Ephemeral => Retention::Ephemeral, - Retention::Checkpoint { id, is_marked } => Retention::Checkpoint { + Retention::Checkpoint { id, marking } => Retention::Checkpoint { id: f(id), - is_marked: *is_marked, + marking: *marking, }, Retention::Marked => Retention::Marked, + Retention::Reference => Retention::Reference, } } } diff --git a/incrementalmerkletree/src/testing.rs b/incrementalmerkletree/src/testing.rs index f1232e8..0a724d2 100644 --- a/incrementalmerkletree/src/testing.rs +++ b/incrementalmerkletree/src/testing.rs @@ -5,7 +5,7 @@ use core::marker::PhantomData; use proptest::prelude::*; use std::collections::BTreeSet; -use crate::{Hashable, Level, Position, Retention}; +use crate::{Hashable, Level, Marking, Position, Retention}; pub mod complete_tree; @@ -208,10 +208,22 @@ impl Operation { } } +/// Returns a strategy for creating a uniformly-distributed [`Marking`] +/// value. +pub fn arb_marking() -> impl Strategy { + prop_oneof![ + Just(Marking::Marked), + Just(Marking::Reference), + Just(Marking::None) + ] +} + +/// Returns a strategy for creating a uniformly-distributed [`Retention`] +/// value. pub fn arb_retention() -> impl Strategy> { prop_oneof![ Just(Retention::Ephemeral), - any::().prop_map(|is_marked| Retention::Checkpoint { id: (), is_marked }), + arb_marking().prop_map(|marking| Retention::Checkpoint { id: (), marking }), Just(Retention::Marked), ] } @@ -557,7 +569,7 @@ pub fn check_root_hashes, F: F 0, Retention::Checkpoint { id: 1, - is_marked: true, + marking: Marking::Marked, }, ); for _ in 0..3 { @@ -735,7 +747,7 @@ pub fn check_witnesses, F: Fn( 0, Checkpoint { id: 1, - is_marked: true, + marking: Marking::Marked, }, ); assert!(tree.rewind()); @@ -769,7 +781,7 @@ pub fn check_witnesses, F: Fn( 6, Checkpoint { id: 1, - is_marked: true, + marking: Marking::Marked, }, ); tree.assert_append(7, Ephemeral); @@ -838,7 +850,7 @@ pub fn check_witnesses, F: Fn( H::from_u64(3), Checkpoint { id: C::from_u64(1), - is_marked: true, + marking: Marking::Marked, }, ), Append(H::from_u64(4), Marked), @@ -847,21 +859,21 @@ pub fn check_witnesses, F: Fn( H::from_u64(5), Checkpoint { id: C::from_u64(3), - is_marked: false, + marking: Marking::None, }, ), Append( H::from_u64(6), Checkpoint { id: C::from_u64(4), - is_marked: false, + marking: Marking::None, }, ), Append( H::from_u64(7), Checkpoint { id: C::from_u64(5), - is_marked: false, + marking: Marking::None, }, ), Witness(3u64.into(), 5), @@ -890,7 +902,7 @@ pub fn check_witnesses, F: Fn( H::from_u64(0), Checkpoint { id: C::from_u64(1), - is_marked: true, + marking: Marking::Marked, }, ), Append(H::from_u64(0), Ephemeral), @@ -900,7 +912,7 @@ pub fn check_witnesses, F: Fn( H::from_u64(0), Checkpoint { id: C::from_u64(2), - is_marked: false, + marking: Marking::None, }, ), Append(H::from_u64(0), Ephemeral), @@ -939,7 +951,7 @@ pub fn check_witnesses, F: Fn( H::from_u64(0), Checkpoint { id: C::from_u64(4), - is_marked: false, + marking: Marking::None, }, ), Rewind, @@ -958,14 +970,14 @@ pub fn check_witnesses, F: Fn( H::from_u64(0), Checkpoint { id: C::from_u64(1), - is_marked: true, + marking: Marking::Marked, }, ), Append( H::from_u64(0), Checkpoint { id: C::from_u64(4), - is_marked: false, + marking: Marking::None, }, ), Witness(Position(2), 2), @@ -1034,7 +1046,7 @@ pub fn check_remove_mark, F: Fn(usize) -> "a", Retention::Checkpoint { id: C::from_u64(1), - is_marked: true, + marking: Marking::Marked, }, ), witness(1, 1), diff --git a/incrementalmerkletree/src/testing/complete_tree.rs b/incrementalmerkletree/src/testing/complete_tree.rs index e6984ae..79c189d 100644 --- a/incrementalmerkletree/src/testing/complete_tree.rs +++ b/incrementalmerkletree/src/testing/complete_tree.rs @@ -2,6 +2,7 @@ use std::cmp::min; use std::collections::{BTreeMap, BTreeSet}; +use crate::Marking; use crate::{testing::Tree, Hashable, Level, Position, Retention}; const MAX_COMPLETE_SIZE_ERROR: &str = "Positions of a `CompleteTree` must fit into the platform word size, because larger complete trees are not representable."; @@ -104,11 +105,11 @@ impl CompleteTr append(&mut self.leaves, value, DEPTH)?; self.mark(); } - Retention::Checkpoint { id, is_marked } => { + Retention::Checkpoint { id, marking } => { let latest_checkpoint = self.checkpoints.keys().rev().next(); if Some(&id) > latest_checkpoint { append(&mut self.leaves, value, DEPTH)?; - if is_marked { + if marking == Marking::Marked { self.mark(); } self.checkpoint(id, self.current_position()); @@ -119,7 +120,7 @@ impl CompleteTr }); } } - Retention::Ephemeral => { + Retention::Ephemeral | Retention::Reference => { append(&mut self.leaves, value, DEPTH)?; } } diff --git a/shardtree/src/legacy.rs b/shardtree/src/legacy.rs index 3957061..19e6818 100644 --- a/shardtree/src/legacy.rs +++ b/shardtree/src/legacy.rs @@ -1,6 +1,8 @@ use std::fmt; -use incrementalmerkletree::{witness::IncrementalWitness, Address, Hashable, Level, Retention}; +use incrementalmerkletree::{ + witness::IncrementalWitness, Address, Hashable, Level, Marking, Retention, +}; use crate::{ store::ShardStore, InsertionError, LocatedPrunableTree, LocatedTree, PrunableTree, @@ -198,7 +200,7 @@ impl LocatedPrunableTree { c.ommers_iter().cloned(), &Retention::Checkpoint { id: checkpoint_id, - is_marked: false, + marking: Marking::None, }, self.root_addr.level(), ) diff --git a/shardtree/src/lib.rs b/shardtree/src/lib.rs index 6f1e213..39be92e 100644 --- a/shardtree/src/lib.rs +++ b/shardtree/src/lib.rs @@ -25,6 +25,7 @@ use core::fmt::Debug; use either::Either; use incrementalmerkletree::frontier::Frontier; +use incrementalmerkletree::Marking; use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use tracing::{debug, trace}; @@ -281,18 +282,19 @@ impl< self.insert_frontier_nodes(nonempty_frontier, leaf_retention) } else { match leaf_retention { - Retention::Ephemeral => Ok(()), + Retention::Ephemeral | Retention::Reference => Ok(()), Retention::Checkpoint { id, - is_marked: false, + marking: Marking::None | Marking::Reference, } => self .store .add_checkpoint(id, Checkpoint::tree_empty()) .map_err(ShardTreeError::Storage), - Retention::Checkpoint { - is_marked: true, .. - } - | Retention::Marked => Err(ShardTreeError::Insert( + Retention::Marked + | Retention::Checkpoint { + marking: Marking::Marked, + .. + } => Err(ShardTreeError::Insert( InsertionError::MarkedRetentionInvalid, )), } @@ -344,7 +346,7 @@ impl< .map_err(ShardTreeError::Storage)?; } - if let Retention::Checkpoint { id, is_marked: _ } = leaf_retention { + if let Retention::Checkpoint { id, .. } = leaf_retention { trace!("Adding checkpoint {:?} at {:?}", id, leaf_position); self.store .add_checkpoint(id, Checkpoint::at_position(leaf_position)) @@ -1310,7 +1312,7 @@ mod tests { check_witness_consistency, check_witnesses, complete_tree::CompleteTree, CombinedTree, SipHashable, }, - Address, Hashable, Level, Position, Retention, + Address, Hashable, Level, Marking, Position, Retention, }; use crate::{ @@ -1411,7 +1413,7 @@ mod tests { 'd'.to_string(), Retention::Checkpoint { id: 11, - is_marked: false + marking: Marking::None }, ), Ok(()), diff --git a/shardtree/src/prunable.rs b/shardtree/src/prunable.rs index 1054c50..b50e84f 100644 --- a/shardtree/src/prunable.rs +++ b/shardtree/src/prunable.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use bitflags::bitflags; +use incrementalmerkletree::Marking; use incrementalmerkletree::{ frontier::NonEmptyFrontier, Address, Hashable, Level, Position, Retention, }; @@ -30,6 +31,12 @@ bitflags! { /// A leaf with `MARKED` retention can be pruned only as a consequence of an explicit deletion /// action. const MARKED = 0b00000010; + + /// A leaf with `REFERENCE` retention will not be considered prunable until the `REFERENCE` + /// flag is removed from the leaf. The `REFERENCE` flag will be removed at any point that + /// the leaf is overwritten without `REFERENCE` retention, and `REFERENCE` retention cannot + /// be added to an existing leaf. + const REFERENCE = 0b00000100; } } @@ -47,14 +54,17 @@ impl<'a, C> From<&'a Retention> for RetentionFlags { fn from(retention: &'a Retention) -> Self { match retention { Retention::Ephemeral => RetentionFlags::EPHEMERAL, - Retention::Checkpoint { is_marked, .. } => { - if *is_marked { - RetentionFlags::CHECKPOINT | RetentionFlags::MARKED - } else { - RetentionFlags::CHECKPOINT - } - } + Retention::Checkpoint { + marking: Marking::Marked, + .. + } => RetentionFlags::CHECKPOINT | RetentionFlags::MARKED, + Retention::Checkpoint { + marking: Marking::Reference, + .. + } => RetentionFlags::CHECKPOINT | RetentionFlags::REFERENCE, + Retention::Checkpoint { .. } => RetentionFlags::CHECKPOINT, Retention::Marked => RetentionFlags::MARKED, + Retention::Reference => RetentionFlags::REFERENCE, } } } diff --git a/shardtree/src/store/caching.rs b/shardtree/src/store/caching.rs index ab898fe..104106d 100644 --- a/shardtree/src/store/caching.rs +++ b/shardtree/src/store/caching.rs @@ -219,7 +219,7 @@ mod tests { append_str, check_operations, unmark, witness, CombinedTree, Operation, TestHashable, Tree, }, - Hashable, Position, Retention, + Hashable, Marking, Position, Retention, }; use super::CachingShardStore; @@ -300,7 +300,7 @@ mod tests { String::from_u64(0), Checkpoint { id: 1, - is_marked: true, + marking: Marking::Marked, }, )); for _ in 0..3 { @@ -542,7 +542,7 @@ mod tests { String::from_u64(0), Checkpoint { id: 1, - is_marked: true, + marking: Marking::Marked, }, )); assert!(tree.rewind()); @@ -584,7 +584,7 @@ mod tests { String::from_u64(6), Checkpoint { id: 1, - is_marked: true, + marking: Marking::Marked, }, )); assert!(tree.append(String::from_u64(7), Ephemeral)); @@ -671,7 +671,7 @@ mod tests { String::from_u64(3), Checkpoint { id: 1, - is_marked: true, + marking: Marking::Marked, }, ), Append(String::from_u64(4), Marked), @@ -680,21 +680,21 @@ mod tests { String::from_u64(5), Checkpoint { id: 3, - is_marked: false, + marking: Marking::None, }, ), Append( String::from_u64(6), Checkpoint { id: 4, - is_marked: false, + marking: Marking::None, }, ), Append( String::from_u64(7), Checkpoint { id: 5, - is_marked: false, + marking: Marking::None, }, ), Witness(3u64.into(), 5), @@ -732,7 +732,7 @@ mod tests { String::from_u64(0), Checkpoint { id: 1, - is_marked: true, + marking: Marking::Marked, }, ), Append(String::from_u64(0), Ephemeral), @@ -742,7 +742,7 @@ mod tests { String::from_u64(0), Checkpoint { id: 2, - is_marked: false, + marking: Marking::None, }, ), Append(String::from_u64(0), Ephemeral), @@ -790,7 +790,7 @@ mod tests { String::from_u64(0), Checkpoint { id: 4, - is_marked: false, + marking: Marking::None, }, ), Rewind, @@ -818,14 +818,14 @@ mod tests { String::from_u64(0), Checkpoint { id: 1, - is_marked: true, + marking: Marking::Marked, }, ), Append( String::from_u64(0), Checkpoint { id: 4, - is_marked: false, + marking: Marking::None, }, ), Witness(Position::from(2), 2), @@ -962,7 +962,7 @@ mod tests { "a", Retention::Checkpoint { id: 1, - is_marked: true, + marking: Marking::Marked, }, ), witness(1, 1), diff --git a/shardtree/src/testing.rs b/shardtree/src/testing.rs index 4771a66..1cd3046 100644 --- a/shardtree/src/testing.rs +++ b/shardtree/src/testing.rs @@ -86,7 +86,14 @@ where leaf, match (is_checkpoint, is_marked) { (false, false) => Retention::Ephemeral, - (true, is_marked) => Retention::Checkpoint { id, is_marked }, + (true, is_marked) => Retention::Checkpoint { + id, + marking: if is_marked { + Marking::Marked + } else { + Marking::None + }, + }, (false, true) => Retention::Marked, }, ) @@ -121,10 +128,10 @@ where .map(|(id, (leaf, retention))| { let pos = Position::try_from(id).unwrap(); match retention { - Retention::Ephemeral => (), - Retention::Checkpoint { is_marked, .. } => { + Retention::Ephemeral | Retention::Reference => (), + Retention::Checkpoint { marking, .. } => { checkpoint_positions.push(pos); - if is_marked { + if marking == Marking::Marked { marked_positions.push(pos); } } @@ -234,7 +241,7 @@ pub fn check_shardtree_insertion< tree.batch_insert( Position::from(1), vec![ - ("b".to_string(), Retention::Checkpoint { id: 1, is_marked: false }), + ("b".to_string(), Retention::Checkpoint { id: 1, marking: Marking::None }), ("c".to_string(), Retention::Ephemeral), ("d".to_string(), Retention::Marked), ].into_iter() @@ -285,7 +292,7 @@ pub fn check_shardtree_insertion< Position::from(10), vec![ ("k".to_string(), Retention::Ephemeral), - ("l".to_string(), Retention::Checkpoint { id: 2, is_marked: false }), + ("l".to_string(), Retention::Checkpoint { id: 2, marking: Marking::None }), ("m".to_string(), Retention::Ephemeral), ].into_iter() ), @@ -386,7 +393,7 @@ pub fn check_witness_with_pruned_subtrees< 'c' => Retention::Marked, 'h' => Retention::Checkpoint { id: 3, - is_marked: false, + marking: Marking::None, }, _ => Retention::Ephemeral, },