zec
/
incrementalmerkletree
mirror of https://github.com/zcash/incrementalmerkletree.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

incrementalmerkletree: add `Reference` retention type. 

This new retention type is intended to be used when inserting frontiers
that should not automatically be pruned.

Also, improve documentation for the `Retention` type.
This commit is contained in:
Kris Nuttycombe 2024-05-20 11:14:40 -06:00
parent 57b6e8999f
commit e55ff2d7f2
9 changed files with 143 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
incrementalmerkletree/CHANGELOG.md
View File

@ -7,6 +7,15 @@ and this project adheres to Rust's notion of
## Unreleased
### Added
- `incrementalmerkletree::Marking`
### Changed
- `incrementalmerkletree::Retention`
- Has added variant `Retention::Reference`
- `Retention::Checkpoint::is_marked` has been replaced by `Retention::Checkpoint::marking`
to permit checkpoints with `Reference` retention to be represented.
## [0.5.1] - 2024-03-25
### Added

49
incrementalmerkletree/src/lib.rs
View File

@ -62,35 +62,72 @@ pub mod witness;
#[cfg_attr(docsrs, doc(cfg(feature = "test-dependencies")))]
pub mod testing;
/// An enumeration of the additional marking states that can be applied
/// to leaves with [`Retention::Checkpoint`] retention.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum Marking {
/// A checkpoint with `Marked` marking will have `Marked` retention after `Checkpoint`
/// retention is removed.
Marked,
/// A checkpoint with `Reference` marking will have `Reference` retention after `Checkpoint`
/// retention is removed.
Reference,
/// A checkpoint with `None` marking will have `Ephemeral` retention after `Checkpoint`
/// retention is removed.
None,
}
/// A type for metadata that is used to determine when and how a leaf can be pruned from a tree.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum Retention<C> {
/// A leaf with `Ephemeral` retention will be pruned whenever its sibling is also a leaf with
/// `Ephemeral` retention.
Ephemeral,
Checkpoint { id: C, is_marked: bool },
/// A leaf with `Checkpoint` retention will be retained in the tree during pruning. If
/// `Checkpoint` retention is removed from the leaf, then the retention for the leaf will
/// become either `Ephemeral`, `Marked`, or `Reference` depending upon the value of the
/// `marking` field.
Checkpoint { id: C, marking: Marking },
/// A leaf with `Marked` retention will be retained in the tree during pruning. `Marked`
/// retention may only be explicitly removed.
Marked,
/// A leaf with `Reference` retention will be retained in the tree during pruning. `Reference`
/// retention is removed whenever the associated leaf is overwritten with a tree node having
/// `Ephemeral`, `Checkpoint`, or `Marked` retention.
Reference,
}
impl<C> Retention<C> {
/// Returns whether the associated node has [`Retention::Checkpoint`] retention.
pub fn is_checkpoint(&self) -> bool {
matches!(self, Retention::Checkpoint { .. })
}
/// Returns whether the associated node has [`Retention::Marked`] retention
/// or [`Retention::Checkpoint`] retention with [`Marking::Marked`] marking.
pub fn is_marked(&self) -> bool {
match self {
Retention::Ephemeral => false,
Retention::Checkpoint { is_marked, .. } => *is_marked,
Retention::Marked => true,
Retention::Marked
| Retention::Checkpoint {
marking: Marking::Marked,
..
} => true,
_ => false,
}
}
/// Applies the provided function to the checkpoint identifier, if any, and returns a new
/// `Retention` value having the same structure with the resulting value used as the checkpoint
/// identifier.
pub fn map<'a, D, F: Fn(&'a C) -> D>(&'a self, f: F) -> Retention<D> {
match self {
Retention::Ephemeral => Retention::Ephemeral,
Retention::Checkpoint { id, is_marked } => Retention::Checkpoint {
Retention::Checkpoint { id, marking } => Retention::Checkpoint {
id: f(id),
is_marked: *is_marked,
marking: *marking,
},
Retention::Marked => Retention::Marked,
Retention::Reference => Retention::Reference,
}
}
}

42
incrementalmerkletree/src/testing.rs
View File

@ -5,7 +5,7 @@ use core::marker::PhantomData;
use proptest::prelude::*;
use std::collections::BTreeSet;
use crate::{Hashable, Level, Position, Retention};
use crate::{Hashable, Level, Marking, Position, Retention};
pub mod complete_tree;
@ -208,10 +208,22 @@ impl<H: Hashable + Clone, C: Clone> Operation<H, C> {
}
}
/// Returns a strategy for creating a uniformly-distributed [`Marking`]
/// value.
pub fn arb_marking() -> impl Strategy<Value = Marking> {
prop_oneof![
Just(Marking::Marked),
Just(Marking::Reference),
Just(Marking::None)
]
}
/// Returns a strategy for creating a uniformly-distributed [`Retention`]
/// value.
pub fn arb_retention() -> impl Strategy<Value = Retention<()>> {
prop_oneof![
Just(Retention::Ephemeral),
any::<bool>().prop_map(|is_marked| Retention::Checkpoint { id: (), is_marked }),
arb_marking().prop_map(|marking| Retention::Checkpoint { id: (), marking }),
Just(Retention::Marked),
]
}
@ -557,7 +569,7 @@ pub fn check_root_hashes<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: F
0,
Retention::Checkpoint {
id: 1,
is_marked: true,
marking: Marking::Marked,
},
);
for _ in 0..3 {
@ -735,7 +747,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
0,
Checkpoint {
id: 1,
is_marked: true,
marking: Marking::Marked,
},
);
assert!(tree.rewind());
@ -769,7 +781,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
6,
Checkpoint {
id: 1,
is_marked: true,
marking: Marking::Marked,
},
);
tree.assert_append(7, Ephemeral);
@ -838,7 +850,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
H::from_u64(3),
Checkpoint {
id: C::from_u64(1),
is_marked: true,
marking: Marking::Marked,
},
),
Append(H::from_u64(4), Marked),
@ -847,21 +859,21 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
H::from_u64(5),
Checkpoint {
id: C::from_u64(3),
is_marked: false,
marking: Marking::None,
},
),
Append(
H::from_u64(6),
Checkpoint {
id: C::from_u64(4),
is_marked: false,
marking: Marking::None,
},
),
Append(
H::from_u64(7),
Checkpoint {
id: C::from_u64(5),
is_marked: false,
marking: Marking::None,
},
),
Witness(3u64.into(), 5),
@ -890,7 +902,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
H::from_u64(0),
Checkpoint {
id: C::from_u64(1),
is_marked: true,
marking: Marking::Marked,
},
),
Append(H::from_u64(0), Ephemeral),
@ -900,7 +912,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
H::from_u64(0),
Checkpoint {
id: C::from_u64(2),
is_marked: false,
marking: Marking::None,
},
),
Append(H::from_u64(0), Ephemeral),
@ -939,7 +951,7 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
H::from_u64(0),
Checkpoint {
id: C::from_u64(4),
is_marked: false,
marking: Marking::None,
},
),
Rewind,
@ -958,14 +970,14 @@ pub fn check_witnesses<H: TestHashable, C: TestCheckpoint, T: Tree<H, C>, F: Fn(
H::from_u64(0),
Checkpoint {
id: C::from_u64(1),
is_marked: true,
marking: Marking::Marked,
},
),
Append(
H::from_u64(0),
Checkpoint {
id: C::from_u64(4),
is_marked: false,
marking: Marking::None,
},
),
Witness(Position(2), 2),
@ -1034,7 +1046,7 @@ pub fn check_remove_mark<C: TestCheckpoint, T: Tree<String, C>, F: Fn(usize) ->
"a",
Retention::Checkpoint {
id: C::from_u64(1),
is_marked: true,
marking: Marking::Marked,
},
),
witness(1, 1),

7
incrementalmerkletree/src/testing/complete_tree.rs
View File

@ -2,6 +2,7 @@
use std::cmp::min;
use std::collections::{BTreeMap, BTreeSet};
use crate::Marking;
use crate::{testing::Tree, Hashable, Level, Position, Retention};
const MAX_COMPLETE_SIZE_ERROR: &str = "Positions of a `CompleteTree` must fit into the platform word size, because larger complete trees are not representable.";
@ -104,11 +105,11 @@ impl<H: Hashable, C: Clone + Ord + core::fmt::Debug, const DEPTH: u8> CompleteTr
append(&mut self.leaves, value, DEPTH)?;
self.mark();
}
Retention::Checkpoint { id, is_marked } => {
Retention::Checkpoint { id, marking } => {
let latest_checkpoint = self.checkpoints.keys().rev().next();
if Some(&id) > latest_checkpoint {
append(&mut self.leaves, value, DEPTH)?;
if is_marked {
if marking == Marking::Marked {
self.mark();
}
self.checkpoint(id, self.current_position());
@ -119,7 +120,7 @@ impl<H: Hashable, C: Clone + Ord + core::fmt::Debug, const DEPTH: u8> CompleteTr
});
}
}
Retention::Ephemeral => {
Retention::Ephemeral | Retention::Reference => {
append(&mut self.leaves, value, DEPTH)?;
}
}

6
shardtree/src/legacy.rs
View File

@ -1,6 +1,8 @@
use std::fmt;
use incrementalmerkletree::{witness::IncrementalWitness, Address, Hashable, Level, Retention};
use incrementalmerkletree::{
witness::IncrementalWitness, Address, Hashable, Level, Marking, Retention,
};
use crate::{
store::ShardStore, InsertionError, LocatedPrunableTree, LocatedTree, PrunableTree,
@ -198,7 +200,7 @@ impl<H: Hashable + Clone + PartialEq> LocatedPrunableTree<H> {
c.ommers_iter().cloned(),
&Retention::Checkpoint {
id: checkpoint_id,
is_marked: false,
marking: Marking::None,
},
self.root_addr.level(),
)

20
shardtree/src/lib.rs
View File

@ -25,6 +25,7 @@
use core::fmt::Debug;
use either::Either;
use incrementalmerkletree::frontier::Frontier;
use incrementalmerkletree::Marking;
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use tracing::{debug, trace};
@ -281,18 +282,19 @@ impl<
self.insert_frontier_nodes(nonempty_frontier, leaf_retention)
} else {
match leaf_retention {
Retention::Ephemeral => Ok(()),
Retention::Ephemeral | Retention::Reference => Ok(()),
Retention::Checkpoint {
id,
is_marked: false,
marking: Marking::None | Marking::Reference,
} => self
.store
.add_checkpoint(id, Checkpoint::tree_empty())
.map_err(ShardTreeError::Storage),
Retention::Checkpoint {
is_marked: true, ..
}
| Retention::Marked => Err(ShardTreeError::Insert(
Retention::Marked
| Retention::Checkpoint {
marking: Marking::Marked,
..
} => Err(ShardTreeError::Insert(
InsertionError::MarkedRetentionInvalid,
)),
}
@ -344,7 +346,7 @@ impl<
.map_err(ShardTreeError::Storage)?;
}
if let Retention::Checkpoint { id, is_marked: _ } = leaf_retention {
if let Retention::Checkpoint { id, .. } = leaf_retention {
trace!("Adding checkpoint {:?} at {:?}", id, leaf_position);
self.store
.add_checkpoint(id, Checkpoint::at_position(leaf_position))
@ -1310,7 +1312,7 @@ mod tests {
check_witness_consistency, check_witnesses, complete_tree::CompleteTree, CombinedTree,
SipHashable,
},
Address, Hashable, Level, Position, Retention,
Address, Hashable, Level, Marking, Position, Retention,
};
use crate::{
@ -1411,7 +1413,7 @@ mod tests {
'd'.to_string(),
Retention::Checkpoint {
id: 11,
is_marked: false
marking: Marking::None
},
),
Ok(()),

24
shardtree/src/prunable.rs
View File

@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use bitflags::bitflags;
use incrementalmerkletree::Marking;
use incrementalmerkletree::{
frontier::NonEmptyFrontier, Address, Hashable, Level, Position, Retention,
};
@ -30,6 +31,12 @@ bitflags! {
/// A leaf with `MARKED` retention can be pruned only as a consequence of an explicit deletion
/// action.
const MARKED = 0b00000010;
/// A leaf with `REFERENCE` retention will not be considered prunable until the `REFERENCE`
/// flag is removed from the leaf. The `REFERENCE` flag will be removed at any point that
/// the leaf is overwritten without `REFERENCE` retention, and `REFERENCE` retention cannot
/// be added to an existing leaf.
const REFERENCE = 0b00000100;
}
}
@ -47,14 +54,17 @@ impl<'a, C> From<&'a Retention<C>> for RetentionFlags {
fn from(retention: &'a Retention<C>) -> Self {
match retention {
Retention::Ephemeral => RetentionFlags::EPHEMERAL,
Retention::Checkpoint { is_marked, .. } => {
if *is_marked {
RetentionFlags::CHECKPOINT | RetentionFlags::MARKED
} else {
RetentionFlags::CHECKPOINT
}
}
Retention::Checkpoint {
marking: Marking::Marked,
..
} => RetentionFlags::CHECKPOINT | RetentionFlags::MARKED,
Retention::Checkpoint {
marking: Marking::Reference,
..
} => RetentionFlags::CHECKPOINT | RetentionFlags::REFERENCE,
Retention::Checkpoint { .. } => RetentionFlags::CHECKPOINT,
Retention::Marked => RetentionFlags::MARKED,
Retention::Reference => RetentionFlags::REFERENCE,
}
}
}

28
shardtree/src/store/caching.rs
View File

@ -219,7 +219,7 @@ mod tests {
append_str, check_operations, unmark, witness, CombinedTree, Operation, TestHashable,
Tree,
},
Hashable, Position, Retention,
Hashable, Marking, Position, Retention,
};
use super::CachingShardStore;
@ -300,7 +300,7 @@ mod tests {
String::from_u64(0),
Checkpoint {
id: 1,
is_marked: true,
marking: Marking::Marked,
},
));
for _ in 0..3 {
@ -542,7 +542,7 @@ mod tests {
String::from_u64(0),
Checkpoint {
id: 1,
is_marked: true,
marking: Marking::Marked,
},
));
assert!(tree.rewind());
@ -584,7 +584,7 @@ mod tests {
String::from_u64(6),
Checkpoint {
id: 1,
is_marked: true,
marking: Marking::Marked,
},
));
assert!(tree.append(String::from_u64(7), Ephemeral));
@ -671,7 +671,7 @@ mod tests {
String::from_u64(3),
Checkpoint {
id: 1,
is_marked: true,
marking: Marking::Marked,
},
),
Append(String::from_u64(4), Marked),
@ -680,21 +680,21 @@ mod tests {
String::from_u64(5),
Checkpoint {
id: 3,
is_marked: false,
marking: Marking::None,
},
),
Append(
String::from_u64(6),
Checkpoint {
id: 4,
is_marked: false,
marking: Marking::None,
},
),
Append(
String::from_u64(7),
Checkpoint {
id: 5,
is_marked: false,
marking: Marking::None,
},
),
Witness(3u64.into(), 5),
@ -732,7 +732,7 @@ mod tests {
String::from_u64(0),
Checkpoint {
id: 1,
is_marked: true,
marking: Marking::Marked,
},
),
Append(String::from_u64(0), Ephemeral),
@ -742,7 +742,7 @@ mod tests {
String::from_u64(0),
Checkpoint {
id: 2,
is_marked: false,
marking: Marking::None,
},
),
Append(String::from_u64(0), Ephemeral),
@ -790,7 +790,7 @@ mod tests {
String::from_u64(0),
Checkpoint {
id: 4,
is_marked: false,
marking: Marking::None,
},
),
Rewind,
@ -818,14 +818,14 @@ mod tests {
String::from_u64(0),
Checkpoint {
id: 1,
is_marked: true,
marking: Marking::Marked,
},
),
Append(
String::from_u64(0),
Checkpoint {
id: 4,
is_marked: false,
marking: Marking::None,
},
),
Witness(Position::from(2), 2),
@ -962,7 +962,7 @@ mod tests {
"a",
Retention::Checkpoint {
id: 1,
is_marked: true,
marking: Marking::Marked,
},
),
witness(1, 1),

21
shardtree/src/testing.rs
View File

@ -86,7 +86,14 @@ where
leaf,
match (is_checkpoint, is_marked) {
(false, false) => Retention::Ephemeral,
(true, is_marked) => Retention::Checkpoint { id, is_marked },
(true, is_marked) => Retention::Checkpoint {
id,
marking: if is_marked {
Marking::Marked
} else {
Marking::None
},
},
(false, true) => Retention::Marked,
},
)
@ -121,10 +128,10 @@ where
.map(|(id, (leaf, retention))| {
let pos = Position::try_from(id).unwrap();
match retention {
Retention::Ephemeral => (),
Retention::Checkpoint { is_marked, .. } => {
Retention::Ephemeral | Retention::Reference => (),
Retention::Checkpoint { marking, .. } => {
checkpoint_positions.push(pos);
if is_marked {
if marking == Marking::Marked {
marked_positions.push(pos);
}
}
@ -234,7 +241,7 @@ pub fn check_shardtree_insertion<
tree.batch_insert(
Position::from(1),
vec![
("b".to_string(), Retention::Checkpoint { id: 1, is_marked: false }),
("b".to_string(), Retention::Checkpoint { id: 1, marking: Marking::None }),
("c".to_string(), Retention::Ephemeral),
("d".to_string(), Retention::Marked),
].into_iter()
@ -285,7 +292,7 @@ pub fn check_shardtree_insertion<
Position::from(10),
vec![
("k".to_string(), Retention::Ephemeral),
("l".to_string(), Retention::Checkpoint { id: 2, is_marked: false }),
("l".to_string(), Retention::Checkpoint { id: 2, marking: Marking::None }),
("m".to_string(), Retention::Ephemeral),
].into_iter()
),
@ -386,7 +393,7 @@ pub fn check_witness_with_pruned_subtrees<
'c' => Retention::Marked,
'h' => Retention::Checkpoint {
id: 3,
is_marked: false,
marking: Marking::None,
},
_ => Retention::Ephemeral,
},