From 081937162ce7cea128d7fd6adea30d35ef14b584 Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Mon, 19 Aug 2019 11:29:06 +0300 Subject: [PATCH 01/76] Initial commit --- README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..6eefe94d2 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# zcash-mmr \ No newline at end of file From ae364f3ae7d284ccb1e00be21c0654a785d727a3 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 20 Aug 2019 18:05:14 +0300 Subject: [PATCH 02/76] initial commit --- .gitignore | 4 + Cargo.toml | 10 ++ src/lib.rs | 66 ++++++++++++ src/tree.rs | 290 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 370 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 src/lib.rs create mode 100644 src/tree.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..293dd90a8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/target +**/*.rs.bk +Cargo.lock +.idea \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..c64cb6a8b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "zcash-mmr" +version = "0.1.0" +authors = ["NikVolf "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +owning_ref = "0.3" \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 000000000..abc0d45ad --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,66 @@ +//! MMR library for Zcash +//! +//! To be used in zebra and via FFI bindings in zcashd + +extern crate owning_ref; + +mod tree; + +#[repr(C)] +#[derive(Debug)] +pub struct NodeData { + subtree_commitment: [u8; 32], + start_time: u32, + end_time: u32, + start_target: u32, + end_target: u32, + start_sapling_root: [u8; 32], + end_sapling_root: [u8; 32], + subtree_total_work: u64, + start_height: u32, + end_height: u32, + shielded_tx: u64, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub enum NodeLink { + Stored(u32), + Generated(u32), +} + +#[repr(C)] +#[derive(Debug)] +pub struct MMRNode { + left: Option, + right: Option, + data: NodeData, +} + +impl MMRNode { + fn complete(&self) -> bool { + let leaves = self.data.end_height - self.data.start_height + 1; + leaves & (leaves - 1) == 0 + } +} + +impl From for MMRNode { + fn from(s: NodeData) -> Self { + MMRNode { left: None, right: None, data: s } + } +} + +#[no_mangle] +pub extern fn append( + stored: *const MMRNode, + stored_count: u32, + generated: *const MMRNode, + generated_count: u32, + append_count: *mut u32, + append_buffer: *mut MMRNode, +) -> MMRNode { + + // TODO: construct tree and write to (append_count, append_buffer) + // TODO: also return generated?? + unimplemented!() +} diff --git a/src/tree.rs b/src/tree.rs new file mode 100644 index 000000000..9027f21cd --- /dev/null +++ b/src/tree.rs @@ -0,0 +1,290 @@ +use std::collections::HashMap; + +use crate::{MMRNode, NodeLink, NodeData}; + +#[derive(Default)] +struct Tree { + stored: HashMap, + + generated: HashMap, + + // number of persistent(!) tree entries + stored_count: u32, + + // number of virtual nodes generated + generated_count: u32, +} + +impl Tree { + fn resolve_link(&self, link: NodeLink) -> IndexedNode { + match link { + NodeLink::Generated(index) => { + // TODO: maybe graceful error? + let node = self.generated.get(&index).expect("caller should ensure id generated"); + IndexedNode { + node, + link, + } + }, + NodeLink::Stored(index) => { + // TODO: maybe graceful error? + let node = self.stored.get(&index).expect("caller should ensure id stored"); + IndexedNode { + node, + link, + } + }, + } + } + + fn push(&mut self, data: MMRNode) -> NodeLink { + let idx = self.stored_count; + self.stored_count = self.stored_count + 1; + self.stored.insert(idx, data); + NodeLink::Stored(idx) + } + + fn push_generated(&mut self, data: MMRNode) -> NodeLink { + let idx = self.generated_count; + self.generated_count = self.generated_count + 1; + self.generated.insert(idx, data); + NodeLink::Generated(idx) + } + + pub fn populate(loaded: Vec) -> Self { + let mut result = Tree::default(); + result.stored_count = loaded.len() as u32; + for (idx, item) in loaded.into_iter().enumerate() { + result.stored.insert(idx as u32, item); + } + + result + } +} + +/// plain list of nodes that has to be appended to the end of the tree as the result of append operation +/// along with new root +pub struct AppendTransaction { + pub appended: Vec, + pub new_root: NodeLink, +} + +struct IndexedNode<'a> { + node: &'a MMRNode, + link: NodeLink, +} + +fn combine_data(left: &NodeData, right: &NodeData) -> NodeData { + NodeData { + // TODO: hash children + subtree_commitment: [0u8; 32], + start_time: left.start_time, + end_time: right.end_time, + start_target: left.start_target, + end_target: right.end_target, + start_sapling_root: left.start_sapling_root, + end_sapling_root: right.end_sapling_root, + + // TODO: sum work? + subtree_total_work: 0, + start_height: left.start_height, + end_height: right.end_height, + shielded_tx: left.shielded_tx + right.shielded_tx, + } +} + +fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> MMRNode { + MMRNode { + left: Some(left.link), + right: Some(right.link), + data: combine_data(&left.node.data, &right.node.data), + } +} + +fn append(tree: &mut Tree, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { + + let is_complete= tree.resolve_link(root).node.complete(); + + let (new_root_node, mut appended) = if is_complete { + let new_leaf_link = tree.push(new_leaf.into()); + + let mut appended = Vec::new(); + appended.push(new_leaf_link); + + // since we dethrone stored root, new one is always generated + let new_root_node = combine_nodes( + tree.resolve_link(root), + tree.resolve_link(new_leaf_link), + ); + + (new_root_node, appended) + + + } else { + let (root_left_child, root_right_child) = { + let root = tree.resolve_link(root).node; + ( + root.left.expect("Root should always have left child"), + root.right.expect("Root should always have right child"), + ) + }; + + let nested_append = append(tree, root_right_child, new_leaf); + let mut appended = nested_append.appended; + let subtree_root = nested_append.new_root; + + let new_root_node = combine_nodes( + tree.resolve_link(root_left_child), + tree.resolve_link(subtree_root), + ); + + (new_root_node, appended) + }; + + let new_root = if new_root_node.complete() { + let new_root= tree.push(new_root_node); + appended.push(new_root); + new_root + } else { + tree.push_generated(new_root_node) + }; + + AppendTransaction { + new_root, + appended, + } +} + +#[cfg(test)] +mod tests { + + use super::{MMRNode, NodeData, Tree, append, NodeLink}; + + fn leaf(height: u32) -> NodeData { + NodeData { + // TODO: hash children + subtree_commitment: [0u8; 32], + start_time: 0, + end_time: 0, + start_target: 0, + end_target: 0, + start_sapling_root: [0u8; 32], + end_sapling_root: [0u8; 32], + + // TODO: sum work? + subtree_total_work: 0, + start_height: height, + end_height: height, + shielded_tx: 7, + } + } + + fn node(start_height: u32, end_height: u32) -> NodeData { + NodeData { + // TODO: hash children + subtree_commitment: [0u8; 32], + start_time: 0, + end_time: 0, + start_target: 0, + end_target: 0, + start_sapling_root: [0u8; 32], + end_sapling_root: [0u8; 32], + + // TODO: sum work? + subtree_total_work: 0, + start_height: start_height, + end_height: end_height, + shielded_tx: 7, + } + } + + // size should be power of 2-1 + fn initial() -> Tree { + let node1: MMRNode = leaf(1).into(); + let node2: MMRNode = leaf(2).into(); + + let node3 = MMRNode { + data: node(1, 2), + left: Some(NodeLink::Stored(0)), + right: Some(NodeLink::Stored(1)), + }; + + Tree::populate(vec![node1, node2, node3]) + } + + #[test] + fn discrete_append() { + let mut tree = initial(); + let append_tx = append( + &mut tree, NodeLink::Stored(2), + leaf(3) + ); + let new_root_link = append_tx.new_root; + let new_root = tree.resolve_link(new_root_link).node; + + // initial tree: (2) + // / \ + // (0) (1) + // + // new tree: + // (4g) + // / \ + // (2) \ + // / \ \ + // (0) (1) (3) + // + // so only (3) is added as real leaf + // while new root, (4g) is generated one + assert_eq!(new_root.data.end_height, 3); + assert_eq!(append_tx.appended.len(), 1); + + let append_tx = append(&mut tree, new_root_link, leaf(4)); + let new_root_link = append_tx.new_root; + let new_root = tree.resolve_link(new_root_link).node; + + // intermediate tree: + // (4g) + // / \ + // (2) \ + // / \ \ + // (0) (1) (3) + // + // new tree: + // ( 6 ) + // / \ + // (2) (5) + // / \ / \ + // (0) (1) (3) (4) + // + // so (4), (5), (6) are added as real leaves + // and new root, (6) is stored one + assert_eq!(new_root.data.end_height, 4); + assert_eq!(append_tx.appended.len(), 3); + + let append_tx = append(&mut tree, new_root_link, leaf(5)); + let new_root_link = append_tx.new_root; + let new_root = tree.resolve_link(new_root_link).node; + + // intermediate tree: + // ( 6 ) + // / \ + // (2) (5) + // / \ / \ + // (0) (1) (3) (4) + // + // new tree: + // ( 8g ) + // / \ + // ( 6 ) \ + // / \ \ + // (2) (5) \ + // / \ / \ \ + // (0) (1) (3) (4) (7) + // + // so (7) is added as real leaf + // and new root, (8g) is generated one + assert_eq!(new_root.data.end_height, 5); + assert_eq!(append_tx.appended.len(), 1); + } + +} \ No newline at end of file From c794cdc6803ba01f00aa6790dd2d7d45b7c6f5d3 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 20 Aug 2019 18:09:09 +0300 Subject: [PATCH 03/76] fix signature and generate h --- bindings.h | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 bindings.h diff --git a/bindings.h b/bindings.h new file mode 100644 index 000000000..818dfc86f --- /dev/null +++ b/bindings.h @@ -0,0 +1,59 @@ +#include +#include +#include +#include + +template +struct Option; + +struct NodeLink { + enum class Tag { + Stored, + Generated, + }; + + struct Stored_Body { + uint32_t _0; + }; + + struct Generated_Body { + uint32_t _0; + }; + + Tag tag; + union { + Stored_Body stored; + Generated_Body generated; + }; +}; + +struct NodeData { + uint8_t subtree_commitment[32]; + uint32_t start_time; + uint32_t end_time; + uint32_t start_target; + uint32_t end_target; + uint8_t start_sapling_root[32]; + uint8_t end_sapling_root[32]; + uint64_t subtree_total_work; + uint32_t start_height; + uint32_t end_height; + uint64_t shielded_tx; +}; + +struct MMRNode { + Option left; + Option right; + NodeData data; +}; + +extern "C" { + +void append(const MMRNode *stored, + uint32_t stored_count, + const MMRNode *generated, + uint32_t generated_count, + uint32_t *append_count, + MMRNode *append_buffer); + +} // extern "C" diff --git a/src/lib.rs b/src/lib.rs index abc0d45ad..726e473fa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,7 +58,7 @@ pub extern fn append( generated_count: u32, append_count: *mut u32, append_buffer: *mut MMRNode, -) -> MMRNode { +) { // TODO: construct tree and write to (append_count, append_buffer) // TODO: also return generated?? From 845babc3c213a1de8fdd79f7c5b665be9c2f3d25 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 20 Aug 2019 18:42:44 +0300 Subject: [PATCH 04/76] remove unuse dep --- Cargo.toml | 5 ----- src/lib.rs | 2 -- 2 files changed, 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c64cb6a8b..0157039bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,8 +3,3 @@ name = "zcash-mmr" version = "0.1.0" authors = ["NikVolf "] edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -owning_ref = "0.3" \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 726e473fa..f164295c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,8 +2,6 @@ //! //! To be used in zebra and via FFI bindings in zcashd -extern crate owning_ref; - mod tree; #[repr(C)] From fec7e077649f1d89936b024367abed4568b63b1b Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 20 Aug 2019 20:07:50 +0300 Subject: [PATCH 05/76] refactor and fix warnings --- src/lib.rs | 14 +++--- src/tree.rs | 137 ++++++++++++++++++++++++++-------------------------- 2 files changed, 76 insertions(+), 75 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f164295c0..cc7309892 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,8 @@ mod tree; +pub use tree::Tree; + #[repr(C)] #[derive(Debug)] pub struct NodeData { @@ -50,12 +52,12 @@ impl From for MMRNode { #[no_mangle] pub extern fn append( - stored: *const MMRNode, - stored_count: u32, - generated: *const MMRNode, - generated_count: u32, - append_count: *mut u32, - append_buffer: *mut MMRNode, + _stored: *const MMRNode, + _stored_count: u32, + _generated: *const MMRNode, + _generated_count: u32, + _append_count: *mut u32, + _append_buffer: *mut MMRNode, ) { // TODO: construct tree and write to (append_count, append_buffer) diff --git a/src/tree.rs b/src/tree.rs index 9027f21cd..5495b49b4 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use crate::{MMRNode, NodeLink, NodeData}; #[derive(Default)] -struct Tree { +pub struct Tree { stored: HashMap, generated: HashMap, @@ -15,6 +15,13 @@ struct Tree { generated_count: u32, } +/// plain list of nodes that has to be appended to the end of the tree as the result of append operation +/// along with new root +pub struct AppendTransaction { + pub appended: Vec, + pub new_root: NodeLink, +} + impl Tree { fn resolve_link(&self, link: NodeLink) -> IndexedNode { match link { @@ -60,14 +67,64 @@ impl Tree { result } + + + pub fn append(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { + + let is_complete= self.resolve_link(root).node.complete(); + + let (new_root_node, mut appended) = if is_complete { + let new_leaf_link = self.push(new_leaf.into()); + + let mut appended = Vec::new(); + appended.push(new_leaf_link); + + // since we dethrone stored root, new one is always generated + let new_root_node = combine_nodes( + self.resolve_link(root), + self.resolve_link(new_leaf_link), + ); + + (new_root_node, appended) + + + } else { + let (root_left_child, root_right_child) = { + let root = self.resolve_link(root).node; + ( + root.left.expect("Root should always have left child"), + root.right.expect("Root should always have right child"), + ) + }; + + let nested_append = self.append(root_right_child, new_leaf); + let appended = nested_append.appended; + let subtree_root = nested_append.new_root; + + let new_root_node = combine_nodes( + self.resolve_link(root_left_child), + self.resolve_link(subtree_root), + ); + + (new_root_node, appended) + }; + + let new_root = if new_root_node.complete() { + let new_root= self.push(new_root_node); + appended.push(new_root); + new_root + } else { + self.push_generated(new_root_node) + }; + + AppendTransaction { + new_root, + appended, + } + } + } -/// plain list of nodes that has to be appended to the end of the tree as the result of append operation -/// along with new root -pub struct AppendTransaction { - pub appended: Vec, - pub new_root: NodeLink, -} struct IndexedNode<'a> { node: &'a MMRNode, @@ -101,64 +158,10 @@ fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> MMRNode { } } -fn append(tree: &mut Tree, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { - - let is_complete= tree.resolve_link(root).node.complete(); - - let (new_root_node, mut appended) = if is_complete { - let new_leaf_link = tree.push(new_leaf.into()); - - let mut appended = Vec::new(); - appended.push(new_leaf_link); - - // since we dethrone stored root, new one is always generated - let new_root_node = combine_nodes( - tree.resolve_link(root), - tree.resolve_link(new_leaf_link), - ); - - (new_root_node, appended) - - - } else { - let (root_left_child, root_right_child) = { - let root = tree.resolve_link(root).node; - ( - root.left.expect("Root should always have left child"), - root.right.expect("Root should always have right child"), - ) - }; - - let nested_append = append(tree, root_right_child, new_leaf); - let mut appended = nested_append.appended; - let subtree_root = nested_append.new_root; - - let new_root_node = combine_nodes( - tree.resolve_link(root_left_child), - tree.resolve_link(subtree_root), - ); - - (new_root_node, appended) - }; - - let new_root = if new_root_node.complete() { - let new_root= tree.push(new_root_node); - appended.push(new_root); - new_root - } else { - tree.push_generated(new_root_node) - }; - - AppendTransaction { - new_root, - appended, - } -} - #[cfg(test)] mod tests { - use super::{MMRNode, NodeData, Tree, append, NodeLink}; + use super::{MMRNode, NodeData, Tree, NodeLink}; fn leaf(height: u32) -> NodeData { NodeData { @@ -198,7 +201,6 @@ mod tests { } } - // size should be power of 2-1 fn initial() -> Tree { let node1: MMRNode = leaf(1).into(); let node2: MMRNode = leaf(2).into(); @@ -215,10 +217,7 @@ mod tests { #[test] fn discrete_append() { let mut tree = initial(); - let append_tx = append( - &mut tree, NodeLink::Stored(2), - leaf(3) - ); + let append_tx = tree.append(NodeLink::Stored(2), leaf(3)); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).node; @@ -238,7 +237,7 @@ mod tests { assert_eq!(new_root.data.end_height, 3); assert_eq!(append_tx.appended.len(), 1); - let append_tx = append(&mut tree, new_root_link, leaf(4)); + let append_tx = tree.append(new_root_link, leaf(4)); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).node; @@ -261,7 +260,7 @@ mod tests { assert_eq!(new_root.data.end_height, 4); assert_eq!(append_tx.appended.len(), 3); - let append_tx = append(&mut tree, new_root_link, leaf(5)); + let append_tx = tree.append(new_root_link, leaf(5)); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).node; From 66c31be6c500951c099673132586199b52e1391a Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 20 Aug 2019 20:10:29 +0300 Subject: [PATCH 06/76] commit from cbindgen --- bindings.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bindings.h b/bindings.h index 818dfc86f..fca8fa6ca 100644 --- a/bindings.h +++ b/bindings.h @@ -49,11 +49,11 @@ struct MMRNode { extern "C" { -void append(const MMRNode *stored, - uint32_t stored_count, - const MMRNode *generated, - uint32_t generated_count, - uint32_t *append_count, - MMRNode *append_buffer); +void append(const MMRNode *_stored, + uint32_t _stored_count, + const MMRNode *_generated, + uint32_t _generated_count, + uint32_t *_append_count, + MMRNode *_append_buffer); } // extern "C" From 261ad90d337e10aacfa18b6327eabf13a8191c61 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 22 Aug 2019 14:36:57 +0300 Subject: [PATCH 07/76] truncate leaf and test --- src/tree.rs | 104 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 91 insertions(+), 13 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index 5495b49b4..119673eee 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -22,6 +22,11 @@ pub struct AppendTransaction { pub new_root: NodeLink, } +pub struct DeleteTransaction { + pub truncated: u32, + pub new_root: NodeLink, +} + impl Tree { fn resolve_link(&self, link: NodeLink) -> IndexedNode { match link { @@ -58,6 +63,7 @@ impl Tree { NodeLink::Generated(idx) } + // TODO: populate both stored and generated nodes? pub fn populate(loaded: Vec) -> Self { let mut result = Tree::default(); result.stored_count = loaded.len() as u32; @@ -69,7 +75,7 @@ impl Tree { } - pub fn append(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { + pub fn append_leaf(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { let is_complete= self.resolve_link(root).node.complete(); @@ -86,8 +92,6 @@ impl Tree { ); (new_root_node, appended) - - } else { let (root_left_child, root_right_child) = { let root = self.resolve_link(root).node; @@ -97,7 +101,7 @@ impl Tree { ) }; - let nested_append = self.append(root_right_child, new_leaf); + let nested_append = self.append_leaf(root_right_child, new_leaf); let appended = nested_append.appended; let subtree_root = nested_append.new_root; @@ -123,6 +127,61 @@ impl Tree { } } + fn pop(&mut self) { + self.stored.remove(&(self.stored_count-1)); + self.stored_count = self.stored_count - 1; + } + + pub fn truncate_leaf(&mut self, root: NodeLink) -> DeleteTransaction { + let root = { + let n = self.resolve_link(root); + let leaves = n.node.data.end_height - n.node.data.start_height + 1; + if leaves & 1 != 0 { + return DeleteTransaction { + truncated: 1, + new_root: n.node.left.expect("Root should have left child while deleting"), + } + } else { + n + } + }; + + let mut peaks = vec![root.node.left.expect("Root should have left child")]; + let mut subtree_root_link = root.node.right.expect("Root should have right child"); + let mut truncated = 1; + + loop { + let left_link = self.resolve_link(subtree_root_link).node.left; + if let Some(left_link) = left_link { + peaks.push(left_link); + subtree_root_link = self + .resolve_link(subtree_root_link).node.right + .expect("If left exists, right should exist as well"); + truncated += 1; + } else { + break; + } + } + + let root = peaks.drain(0..1).nth(0).expect("At lest 2 elements in peaks"); + let new_root = peaks.into_iter().fold( + root, + |root, next_peak| + self.push_generated( + combine_nodes( + self.resolve_link(root), + self.resolve_link(next_peak) + ) + ) + ); + + for _ in 0..truncated { self.pop(); } + + DeleteTransaction { + new_root, + truncated, + } + } } @@ -165,7 +224,6 @@ mod tests { fn leaf(height: u32) -> NodeData { NodeData { - // TODO: hash children subtree_commitment: [0u8; 32], start_time: 0, end_time: 0, @@ -173,8 +231,6 @@ mod tests { end_target: 0, start_sapling_root: [0u8; 32], end_sapling_root: [0u8; 32], - - // TODO: sum work? subtree_total_work: 0, start_height: height, end_height: height, @@ -184,7 +240,6 @@ mod tests { fn node(start_height: u32, end_height: u32) -> NodeData { NodeData { - // TODO: hash children subtree_commitment: [0u8; 32], start_time: 0, end_time: 0, @@ -192,8 +247,6 @@ mod tests { end_target: 0, start_sapling_root: [0u8; 32], end_sapling_root: [0u8; 32], - - // TODO: sum work? subtree_total_work: 0, start_height: start_height, end_height: end_height, @@ -214,10 +267,23 @@ mod tests { Tree::populate(vec![node1, node2, node3]) } + // returns tree with specified number of leafs and it's root + fn generated(length: u32) -> (Tree, NodeLink) { + assert!(length > 3); + let mut tree = initial(); + let mut root = NodeLink::Stored(2); + + for i in 2..length { + root = tree.append_leaf(root, leaf(i+1).into()).new_root; + } + + (tree, root) + } + #[test] fn discrete_append() { let mut tree = initial(); - let append_tx = tree.append(NodeLink::Stored(2), leaf(3)); + let append_tx = tree.append_leaf(NodeLink::Stored(2), leaf(3)); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).node; @@ -237,7 +303,7 @@ mod tests { assert_eq!(new_root.data.end_height, 3); assert_eq!(append_tx.appended.len(), 1); - let append_tx = tree.append(new_root_link, leaf(4)); + let append_tx = tree.append_leaf(new_root_link, leaf(4)); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).node; @@ -260,7 +326,7 @@ mod tests { assert_eq!(new_root.data.end_height, 4); assert_eq!(append_tx.appended.len(), 3); - let append_tx = tree.append(new_root_link, leaf(5)); + let append_tx = tree.append_leaf(new_root_link, leaf(5)); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).node; @@ -286,4 +352,16 @@ mod tests { assert_eq!(append_tx.appended.len(), 1); } + #[test] + fn truncate_simple() { + let (mut tree, root) = generated(9); + let delete_tx = tree.truncate_leaf(root); + + match delete_tx.new_root { + NodeLink::Stored(14) => { /* ok */ }, + _ => panic!("Root should be stored(14)") + } + + } + } \ No newline at end of file From f316c1b439f81ec373e25964e2a5b24a62e260d8 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 22 Aug 2019 14:37:26 +0300 Subject: [PATCH 08/76] remove whitespaces --- src/tree.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index 119673eee..aeeb2cd2b 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -361,7 +361,5 @@ mod tests { NodeLink::Stored(14) => { /* ok */ }, _ => panic!("Root should be stored(14)") } - } - } \ No newline at end of file From 5c6d85671458a60d4d071958119e7697e0fdf79c Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 22 Aug 2019 15:47:49 +0300 Subject: [PATCH 09/76] extra test --- src/tree.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/tree.rs b/src/tree.rs index aeeb2cd2b..1ab941d2b 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -74,7 +74,6 @@ impl Tree { result } - pub fn append_leaf(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { let is_complete= self.resolve_link(root).node.complete(); @@ -352,6 +351,8 @@ mod tests { assert_eq!(append_tx.appended.len(), 1); } + // TODO: use assert_matches below + #[test] fn truncate_simple() { let (mut tree, root) = generated(9); @@ -362,4 +363,29 @@ mod tests { _ => panic!("Root should be stored(14)") } } + + #[test] + fn truncate_generated() { + let (mut tree, root) = generated(10); + let delete_tx = tree.truncate_leaf(root); + + match delete_tx.new_root { + NodeLink::Generated(_) => { /* ok */ }, + _ => panic!("Root now should be generated") + } + + let (left_root_child, right_root_child) = { + let root = tree.resolve_link(delete_tx.new_root); + + ( + root.node.left.expect("there should be left child for root"), + root.node.right.expect("there should be right child for root"), + ) + }; + + match (left_root_child, right_root_child) { + (NodeLink::Stored(14), NodeLink::Stored(15)) => { /* ok */ }, + _ => panic!("Root should have s(14) and s(15) children") + }; + } } \ No newline at end of file From 767d73f777758f2c39ec22eff975d7b1635664e2 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 22 Aug 2019 15:49:43 +0300 Subject: [PATCH 10/76] extra asserts --- src/tree.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tree.rs b/src/tree.rs index 1ab941d2b..aade99609 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -181,6 +181,10 @@ impl Tree { truncated, } } + + pub fn len(&self) -> u32 { + self.stored_count + } } @@ -387,5 +391,8 @@ mod tests { (NodeLink::Stored(14), NodeLink::Stored(15)) => { /* ok */ }, _ => panic!("Root should have s(14) and s(15) children") }; + + assert_eq!(delete_tx.truncated, 2); + assert_eq!(tree.len(), 16); } } \ No newline at end of file From 3a09eef6b3aa20de3e09ef6c4764f1e9cc21cdf8 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 22 Aug 2019 15:59:18 +0300 Subject: [PATCH 11/76] some ascii fun --- src/tree.rs | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index aade99609..8b4648d8d 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -362,10 +362,35 @@ mod tests { let (mut tree, root) = generated(9); let delete_tx = tree.truncate_leaf(root); + // initial tree: + // + // (-------16g------) + // / \ + // (--------14-------) \ + // / \ \ + // ( 6 ) ( 13 ) \ + // / \ / \ \ + // (2) (5) (9) (12) \ + // / \ / \ / \ / \ \ + // (0) (1) (3) (4) (7) (8) (10) (11) (15) + // + // new tree: + // (--------14-------) + // / \ + // ( 6 ) ( 13 ) + // / \ / \ + // (2) (5) (9) (12) + // / \ / \ / \ / \ + // (0) (1) (3) (4) (7) (8) (10) (11) + // + // so (15) is truncated + // and new root, (14) is a stored one now + match delete_tx.new_root { NodeLink::Stored(14) => { /* ok */ }, _ => panic!("Root should be stored(14)") } + assert_eq!(tree.len(), 15); } #[test] @@ -373,11 +398,36 @@ mod tests { let (mut tree, root) = generated(10); let delete_tx = tree.truncate_leaf(root); + // initial tree: + // + // (--------18g--------) + // / \ + // (--------14-------) \ + // / \ \ + // ( 6 ) ( 13 ) \ + // / \ / \ \ + // (2) (5) (9) (12) (17) + // / \ / \ / \ / \ / \ + // (0) (1) (3) (4) (7) (8) (10) (11) (15) (16) + // + // new tree: + // (-------16g------) + // / \ + // (--------14-------) \ + // / \ \ + // ( 6 ) ( 13 ) \ + // / \ / \ \ + // (2) (5) (9) (12) \ + // / \ / \ / \ / \ \ + // (0) (1) (3) (4) (7) (8) (10) (11) (15) + + // new root is generated match delete_tx.new_root { NodeLink::Generated(_) => { /* ok */ }, _ => panic!("Root now should be generated") } + // left is 14 and right is 15 let (left_root_child, right_root_child) = { let root = tree.resolve_link(delete_tx.new_root); @@ -386,12 +436,10 @@ mod tests { root.node.right.expect("there should be right child for root"), ) }; - match (left_root_child, right_root_child) { (NodeLink::Stored(14), NodeLink::Stored(15)) => { /* ok */ }, _ => panic!("Root should have s(14) and s(15) children") }; - assert_eq!(delete_tx.truncated, 2); assert_eq!(tree.len(), 16); } From bd8eea97b7d81ef85181803883d0a718e3b84680 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 22 Aug 2019 17:36:05 +0300 Subject: [PATCH 12/76] add docs/comments and fix bug --- src/lib.rs | 6 ++++++ src/tree.rs | 38 +++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index cc7309892..923322bee 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ mod tree; pub use tree::Tree; +/// Node metadata. #[repr(C)] #[derive(Debug)] pub struct NodeData { @@ -22,15 +23,20 @@ pub struct NodeData { shielded_tx: u64, } +/// Reference to to the tree node. #[repr(C)] #[derive(Clone, Copy, Debug)] pub enum NodeLink { + /// Reference to the stored (in the array representation) leaf/node. Stored(u32), + /// Reference to the generated leaf/node. Generated(u32), } +/// MMR Node. It is leaf when `left`, `right` are `None` and node when they are not. #[repr(C)] #[derive(Debug)] +// TODO: Better layout would be enum (node, leaf), with left, right set only for nodes? pub struct MMRNode { left: Option, right: Option, diff --git a/src/tree.rs b/src/tree.rs index 8b4648d8d..cd45287d1 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -2,6 +2,13 @@ use std::collections::HashMap; use crate::{MMRNode, NodeLink, NodeData}; +/// Represents partially loaded tree. +/// +/// Some kind of "view" into the array representation of the MMR tree. +/// With only some of the leaves/nodes pre-loaded / pre-generated. +/// Exact amount of the loaded data can be calculated by the constructing party, +/// depending on the length of the tree and maximum amount of operations that are going +/// to happen after construction. #[derive(Default)] pub struct Tree { stored: HashMap, @@ -15,15 +22,21 @@ pub struct Tree { generated_count: u32, } -/// plain list of nodes that has to be appended to the end of the tree as the result of append operation -/// along with new root +/// Result of appending one or several leaves. pub struct AppendTransaction { + /// Plain list of nodes that has to be appended to the end of the array representation + /// of the tree as the result of append operation. pub appended: Vec, + + /// New root as a result of the operation (can be generated one). pub new_root: NodeLink, } +/// Result of truncating one or severl leaves. pub struct DeleteTransaction { + /// Number of leaves that should be dropped from the end of the list. pub truncated: u32, + /// New root as the result of the operation (can be generated one). pub new_root: NodeLink, } @@ -63,7 +76,8 @@ impl Tree { NodeLink::Generated(idx) } - // TODO: populate both stored and generated nodes? + /// Populate tree with plain list of the leaves/nodes. Mostly for test, + /// since this `Tree` structure is for partially loaded tree. pub fn populate(loaded: Vec) -> Self { let mut result = Tree::default(); result.stored_count = loaded.len() as u32; @@ -74,6 +88,7 @@ impl Tree { result } + /// Append one leaf to the tree. pub fn append_leaf(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { let is_complete= self.resolve_link(root).node.complete(); @@ -131,17 +146,24 @@ impl Tree { self.stored_count = self.stored_count - 1; } + /// Truncate one leaf from the end of the tree. pub fn truncate_leaf(&mut self, root: NodeLink) -> DeleteTransaction { let root = { - let n = self.resolve_link(root); - let leaves = n.node.data.end_height - n.node.data.start_height + 1; + let (leaves, root_left_child) = { + let n = self.resolve_link(root); + ( + n.node.data.end_height - n.node.data.start_height + 1, + n.node.left.expect("Root should have left child while deleting") + ) + }; if leaves & 1 != 0 { + self.pop(); return DeleteTransaction { truncated: 1, - new_root: n.node.left.expect("Root should have left child while deleting"), + new_root: root_left_child, } } else { - n + self.resolve_link(root) } }; @@ -182,6 +204,7 @@ impl Tree { } } + /// Length of array representation of the tree. pub fn len(&self) -> u32 { self.stored_count } @@ -440,6 +463,7 @@ mod tests { (NodeLink::Stored(14), NodeLink::Stored(15)) => { /* ok */ }, _ => panic!("Root should have s(14) and s(15) children") }; + // two stored nodes should leave us (leaf 16 and no longer needed node 17) assert_eq!(delete_tx.truncated, 2); assert_eq!(tree.len(), 16); } From 27337f1bb00eced4e48c29fd17a9d95e15c0b821 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 22 Aug 2019 17:43:03 +0300 Subject: [PATCH 13/76] use assert_matches! --- Cargo.toml | 3 +++ src/lib.rs | 2 ++ src/tree.rs | 23 +++++++++-------------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0157039bd..522af41ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,3 +3,6 @@ name = "zcash-mmr" version = "0.1.0" authors = ["NikVolf "] edition = "2018" + +[dev-dependencies] +assert_matches = "1.3.0" \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 923322bee..1b5ad7934 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,8 @@ //! //! To be used in zebra and via FFI bindings in zcashd +#[cfg(test)] #[macro_use] extern crate assert_matches; + mod tree; pub use tree::Tree; diff --git a/src/tree.rs b/src/tree.rs index cd45287d1..09fff6908 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -378,8 +378,6 @@ mod tests { assert_eq!(append_tx.appended.len(), 1); } - // TODO: use assert_matches below - #[test] fn truncate_simple() { let (mut tree, root) = generated(9); @@ -409,10 +407,7 @@ mod tests { // so (15) is truncated // and new root, (14) is a stored one now - match delete_tx.new_root { - NodeLink::Stored(14) => { /* ok */ }, - _ => panic!("Root should be stored(14)") - } + assert_matches!(delete_tx.new_root, NodeLink::Stored(14)); assert_eq!(tree.len(), 15); } @@ -445,10 +440,8 @@ mod tests { // (0) (1) (3) (4) (7) (8) (10) (11) (15) // new root is generated - match delete_tx.new_root { - NodeLink::Generated(_) => { /* ok */ }, - _ => panic!("Root now should be generated") - } + + assert_matches!(delete_tx.new_root, NodeLink::Generated(_)); // left is 14 and right is 15 let (left_root_child, right_root_child) = { @@ -459,10 +452,12 @@ mod tests { root.node.right.expect("there should be right child for root"), ) }; - match (left_root_child, right_root_child) { - (NodeLink::Stored(14), NodeLink::Stored(15)) => { /* ok */ }, - _ => panic!("Root should have s(14) and s(15) children") - }; + + assert_matches!( + (left_root_child, right_root_child), + (NodeLink::Stored(14), NodeLink::Stored(15)) + ); + // two stored nodes should leave us (leaf 16 and no longer needed node 17) assert_eq!(delete_tx.truncated, 2); assert_eq!(tree.len(), 16); From 5a479363ff08742ec2cbb79dfc56dbd11ffb1a92 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 22 Aug 2019 20:35:56 +0300 Subject: [PATCH 14/76] new initializer --- src/tree.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/tree.rs b/src/tree.rs index 09fff6908..136c9cf8f 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -76,7 +76,7 @@ impl Tree { NodeLink::Generated(idx) } - /// Populate tree with plain list of the leaves/nodes. Mostly for test, + /// Populate tree with plain list of the leaves/nodes. Mostly for tests, /// since this `Tree` structure is for partially loaded tree. pub fn populate(loaded: Vec) -> Self { let mut result = Tree::default(); @@ -88,6 +88,27 @@ impl Tree { result } + pub fn new( + length: u32, + stored: Vec<(u32, MMRNode)>, + generated: Vec, + ) -> Self { + let mut result = Tree::default(); + result.stored_count = length; + + for (idx, node) in stored.into_iter() { + result.stored.insert(idx, node); + } + + result.generated_count = generated.len() as u32; + + for (idx, node) in generated.into_iter().enumerate() { + result.generated.insert(idx as u32, node); + } + + result + } + /// Append one leaf to the tree. pub fn append_leaf(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { From e7d67364cca6c63d40eb25ad3abed2688d87bee6 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 23 Aug 2019 16:25:34 +0300 Subject: [PATCH 15/76] add quickcheck(failing) also failing append --- Cargo.toml | 3 +- src/lib.rs | 1 + src/tree.rs | 128 ++++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 122 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 522af41ea..839ef380b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,4 +5,5 @@ authors = ["NikVolf "] edition = "2018" [dev-dependencies] -assert_matches = "1.3.0" \ No newline at end of file +assert_matches = "1.3.0" +quickcheck = "0.8" \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 1b5ad7934..e398029e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ //! To be used in zebra and via FFI bindings in zcashd #[cfg(test)] #[macro_use] extern crate assert_matches; +#[cfg(test)] #[macro_use] extern crate quickcheck; mod tree; diff --git a/src/tree.rs b/src/tree.rs index 136c9cf8f..5a387403e 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -111,7 +111,6 @@ impl Tree { /// Append one leaf to the tree. pub fn append_leaf(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { - let is_complete= self.resolve_link(root).node.complete(); let (new_root_node, mut appended) = if is_complete { @@ -162,6 +161,18 @@ impl Tree { } } + #[cfg(test)] + fn for_children(&mut self, node: NodeLink, mut f: F) { + let (left, right) = { + let link = self.resolve_link(node); + ( + link.node.left.expect("test use only (l)"), + link.node.right.expect("test use only (r)"), + ) + }; + f(left, right) + } + fn pop(&mut self) { self.stored.remove(&(self.stored_count-1)); self.stored_count = self.stored_count - 1; @@ -268,6 +279,7 @@ fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> MMRNode { mod tests { use super::{MMRNode, NodeData, Tree, NodeLink}; + use quickcheck::TestResult; fn leaf(height: u32) -> NodeData { NodeData { @@ -301,7 +313,7 @@ mod tests { } } - fn initial() -> Tree { + fn initial() -> (NodeLink, Tree) { let node1: MMRNode = leaf(1).into(); let node2: MMRNode = leaf(2).into(); @@ -311,15 +323,13 @@ mod tests { right: Some(NodeLink::Stored(1)), }; - Tree::populate(vec![node1, node2, node3]) + (NodeLink::Stored(2), Tree::populate(vec![node1, node2, node3])) } // returns tree with specified number of leafs and it's root fn generated(length: u32) -> (Tree, NodeLink) { - assert!(length > 3); - let mut tree = initial(); - let mut root = NodeLink::Stored(2); - + assert!(length >= 3); + let (mut root, mut tree) = initial(); for i in 2..length { root = tree.append_leaf(root, leaf(i+1).into()).new_root; } @@ -329,8 +339,10 @@ mod tests { #[test] fn discrete_append() { - let mut tree = initial(); - let append_tx = tree.append_leaf(NodeLink::Stored(2), leaf(3)); + let (root, mut tree) = initial(); + + // ** APPEND 3 ** + let append_tx = tree.append_leaf(root, leaf(3)); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).node; @@ -350,6 +362,7 @@ mod tests { assert_eq!(new_root.data.end_height, 3); assert_eq!(append_tx.appended.len(), 1); + // ** APPEND 4 ** let append_tx = tree.append_leaf(new_root_link, leaf(4)); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).node; @@ -372,6 +385,9 @@ mod tests { // and new root, (6) is stored one assert_eq!(new_root.data.end_height, 4); assert_eq!(append_tx.appended.len(), 3); + assert_matches!(new_root_link, NodeLink::Stored(6)); + + // ** APPEND 5 ** let append_tx = tree.append_leaf(new_root_link, leaf(5)); let new_root_link = append_tx.new_root; @@ -397,6 +413,81 @@ mod tests { // and new root, (8g) is generated one assert_eq!(new_root.data.end_height, 5); assert_eq!(append_tx.appended.len(), 1); + assert_matches!(new_root_link, NodeLink::Generated(_)); + tree.for_children(new_root_link, |l, r| { + assert_matches!(l, NodeLink::Stored(6)); + assert_matches!(r, NodeLink::Stored(7)); + }); + + // *** APPEND #6 *** + + let append_tx = tree.append_leaf(new_root_link, leaf(6)); + let new_root_link = append_tx.new_root; + let new_root = tree.resolve_link(new_root_link).node; + + // intermediate tree: + // ( 8g ) + // / \ + // ( 6 ) \ + // / \ \ + // (2) (5) \ + // / \ / \ \ + // (0) (1) (3) (4) (7) + // + // new tree: + // (---8g---) + // / \ + // ( 6 ) \ + // / \ \ + // (2) (5) (9) + // / \ / \ / \ + // (0) (1) (3) (4) (7) (8) + // + // so (7) is added as real leaf + // and new root, (8g) is generated one + assert_eq!(new_root.data.end_height, 6); + assert_eq!(append_tx.appended.len(), 2); + assert_matches!(new_root_link, NodeLink::Generated(_)); + tree.for_children(new_root_link, |l, r| { + assert_matches!(l, NodeLink::Stored(6)); + assert_matches!(r, NodeLink::Stored(9)); + }); + + // *** APPEND #7 *** + + let append_tx = tree.append_leaf(new_root_link, leaf(7)); + let new_root_link = append_tx.new_root; + let new_root = tree.resolve_link(new_root_link).node; + + // intermediate tree: + // (---8g---) + // / \ + // ( 6 ) \ + // / \ \ + // (2) (5) (9) + // / \ / \ / \ + // (0) (1) (3) (4) (7) (8) + // + // new tree: + // (---12g--) + // / \ + // (---11g---) \ + // / \ \ + // ( 6 ) \ \ + // / \ \ \ + // (2) (5) (9) \ + // / \ / \ / \ \ + // (0) (1) (3) (4) (7) (8) (10) + // + // so (7) is added as real leaf + // and new root, (8g) is generated one + assert_eq!(new_root.data.end_height, 7); + assert_eq!(append_tx.appended.len(), 1); + assert_matches!(new_root_link, NodeLink::Generated(_)); + tree.for_children(new_root_link, |l, r| { + assert_matches!(l, NodeLink::Generated(_)); + assert_matches!(r, NodeLink::Stored(10)); + }); } #[test] @@ -483,4 +574,23 @@ mod tests { assert_eq!(delete_tx.truncated, 2); assert_eq!(tree.len(), 16); } + + + quickcheck! { + fn there_and_back(number: u32) -> TestResult { + if (number > 1024*1024) { + TestResult::discard() + } else { + let (mut root, mut tree) = initial(); + for i in 0..number { + root = tree.append_leaf(root, leaf(i+3)).new_root; + } + for i in 0..number { + root = tree.truncate_leaf(root).new_root; + } + + TestResult::from_bool(if let NodeLink::Stored(2) = root { true } else { false }) + } + } + } } \ No newline at end of file From c05446d2aee99bc47662619500ec90be388a2748 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 29 Aug 2019 19:34:27 +0300 Subject: [PATCH 16/76] add get_peaks method --- src/tree.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/tree.rs b/src/tree.rs index 5a387403e..f199461f3 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -109,6 +109,34 @@ impl Tree { result } + fn get_peaks(&self, root: NodeLink) -> Vec { + let (left_child_link, right_child_link) = { + let root = self.resolve_link(root); + ( + root.node.left.expect("It would stop before when root is leaf"), + root.node.right.expect("It would stop before when root is leaf"), + ) + }; + + let mut result = Vec::new(); + + let left_child = self.resolve_link(left_child_link); + if left_child.node.complete() { + result.push(left_child_link); + } else { + result.extend(self.get_peaks(left_child_link)); + } + + let right_child = self.resolve_link(right_child_link); + if right_child.node.complete() { + result.push(right_child_link); + } else { + result.extend(self.get_peaks(right_child_link)); + } + + result + } + /// Append one leaf to the tree. pub fn append_leaf(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { let is_complete= self.resolve_link(root).node.complete(); From fa04929891e96b218ed8febbe5c9f2b1c85f9161 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 2 Sep 2019 15:11:23 +0300 Subject: [PATCH 17/76] refactored append and fixed tests --- src/lib.rs | 6 ++- src/tree.rs | 107 +++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 86 insertions(+), 27 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e398029e3..e55c308d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,9 +48,13 @@ pub struct MMRNode { impl MMRNode { fn complete(&self) -> bool { - let leaves = self.data.end_height - self.data.start_height + 1; + let leaves = self.leaf_count(); leaves & (leaves - 1) == 0 } + + fn leaf_count(&self) -> u32 { + self.data.end_height - self.data.start_height + 1 + } } impl From for MMRNode { diff --git a/src/tree.rs b/src/tree.rs index f199461f3..c26696b4a 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -141,7 +141,7 @@ impl Tree { pub fn append_leaf(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { let is_complete= self.resolve_link(root).node.complete(); - let (new_root_node, mut appended) = if is_complete { + let (new_root, mut appended) = if is_complete { let new_leaf_link = self.push(new_leaf.into()); let mut appended = Vec::new(); @@ -153,35 +153,77 @@ impl Tree { self.resolve_link(new_leaf_link), ); - (new_root_node, appended) + (self.push_generated(new_root_node), appended) } else { - let (root_left_child, root_right_child) = { - let root = self.resolve_link(root).node; - ( - root.left.expect("Root should always have left child"), - root.right.expect("Root should always have right child"), +// let (root_left_child, root_right_child) = { +// let root = self.resolve_link(root).node; +// ( +// root.left.expect("Root should always have left child"), +// root.right.expect("Root should always have right child"), +// ) +// }; +// +// let nested_append = self.append_leaf(root_right_child, new_leaf); +// let appended = nested_append.appended; +// let subtree_root = nested_append.new_root; +// +// let new_root_node = combine_nodes( +// self.resolve_link(root_left_child), +// self.resolve_link(subtree_root), +// ); +// +// (new_root_node, appended) + + let new_leaf_link = self.push(new_leaf.into()); + let mut appended = Vec::new(); + appended.push(new_leaf_link); + + let mut peaks = self.get_peaks(root); + + let mut merge_stack = Vec::new(); + merge_stack.push(new_leaf_link); + + while let Some(next_peak) = peaks.pop() { + dbg!(next_peak); + let next_merge = merge_stack.pop().expect("there should be at least one, checked below"); + + if let Some(stored) = { + let peak = self.resolve_link(next_peak); + let m = self.resolve_link(next_merge); + if peak.node.leaf_count() == m.node.leaf_count() { + Some(combine_nodes(peak, m)) + } else { None } + } { + let link = self.push(stored); + merge_stack.push(link); + appended.push(link); + dbg!(link); + continue; + } + merge_stack.push(next_merge); + merge_stack.push(next_peak); + } + + let mut root = merge_stack.pop().expect("There should be at least one node in stack"); + while let Some(next_child) = merge_stack.pop() { + root = self.push_generated( + combine_nodes( + self.resolve_link(root), + self.resolve_link(next_child), + ) ) - }; + } - let nested_append = self.append_leaf(root_right_child, new_leaf); - let appended = nested_append.appended; - let subtree_root = nested_append.new_root; - - let new_root_node = combine_nodes( - self.resolve_link(root_left_child), - self.resolve_link(subtree_root), - ); - - (new_root_node, appended) + (root, appended) }; - let new_root = if new_root_node.complete() { - let new_root= self.push(new_root_node); - appended.push(new_root); - new_root - } else { - self.push_generated(new_root_node) - }; +// let new_root = if new_root_node.complete() { +// let new_root= self.push(new_root_node); +// appended.push(new_root); +// new_root +// } else { +// self.push_generated(new_root_node) +// }; AppendTransaction { new_root, @@ -448,7 +490,7 @@ mod tests { }); // *** APPEND #6 *** - + dbg!("go"); let append_tx = tree.append_leaf(new_root_link, leaf(6)); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).node; @@ -620,5 +662,18 @@ mod tests { TestResult::from_bool(if let NodeLink::Stored(2) = root { true } else { false }) } } + + fn leaf_count(number: u32) -> TestResult { + if (number > 1024 * 1024 || number < 3) { + TestResult::discard() + } else { + let (mut root, mut tree) = initial(); + for i in 0..(number-2) { + root = tree.append_leaf(root, leaf(i+3)).new_root; + } + + TestResult::from_bool(tree.resolve_link(root).node.leaf_count() == number) + } + } } } \ No newline at end of file From 68983dc0c1a1322cedca50ba63f5f5821a1634b0 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 2 Sep 2019 15:28:51 +0300 Subject: [PATCH 18/76] extra tests and notes --- src/tree.rs | 122 +++++++++++++++++----------------------------------- 1 file changed, 39 insertions(+), 83 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index c26696b4a..c3e0bedff 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -110,8 +110,12 @@ impl Tree { } fn get_peaks(&self, root: NodeLink) -> Vec { + let (left_child_link, right_child_link) = { let root = self.resolve_link(root); + if root.node.complete() { + return vec![root.link]; + } ( root.node.left.expect("It would stop before when root is leaf"), root.node.right.expect("It would stop before when root is leaf"), @@ -139,91 +143,44 @@ impl Tree { /// Append one leaf to the tree. pub fn append_leaf(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { - let is_complete= self.resolve_link(root).node.complete(); + let new_leaf_link = self.push(new_leaf.into()); + let mut appended = Vec::new(); + appended.push(new_leaf_link); - let (new_root, mut appended) = if is_complete { - let new_leaf_link = self.push(new_leaf.into()); + let mut peaks = self.get_peaks(root); - let mut appended = Vec::new(); - appended.push(new_leaf_link); + let mut merge_stack = Vec::new(); + merge_stack.push(new_leaf_link); - // since we dethrone stored root, new one is always generated - let new_root_node = combine_nodes( - self.resolve_link(root), - self.resolve_link(new_leaf_link), - ); + while let Some(next_peak) = peaks.pop() { + let next_merge = merge_stack.pop().expect("there should be at least one, checked below"); - (self.push_generated(new_root_node), appended) - } else { -// let (root_left_child, root_right_child) = { -// let root = self.resolve_link(root).node; -// ( -// root.left.expect("Root should always have left child"), -// root.right.expect("Root should always have right child"), -// ) -// }; -// -// let nested_append = self.append_leaf(root_right_child, new_leaf); -// let appended = nested_append.appended; -// let subtree_root = nested_append.new_root; -// -// let new_root_node = combine_nodes( -// self.resolve_link(root_left_child), -// self.resolve_link(subtree_root), -// ); -// -// (new_root_node, appended) - - let new_leaf_link = self.push(new_leaf.into()); - let mut appended = Vec::new(); - appended.push(new_leaf_link); - - let mut peaks = self.get_peaks(root); - - let mut merge_stack = Vec::new(); - merge_stack.push(new_leaf_link); - - while let Some(next_peak) = peaks.pop() { - dbg!(next_peak); - let next_merge = merge_stack.pop().expect("there should be at least one, checked below"); - - if let Some(stored) = { - let peak = self.resolve_link(next_peak); - let m = self.resolve_link(next_merge); - if peak.node.leaf_count() == m.node.leaf_count() { - Some(combine_nodes(peak, m)) - } else { None } - } { - let link = self.push(stored); - merge_stack.push(link); - appended.push(link); - dbg!(link); - continue; - } - merge_stack.push(next_merge); - merge_stack.push(next_peak); + if let Some(stored) = { + let peak = self.resolve_link(next_peak); + let m = self.resolve_link(next_merge); + if peak.node.leaf_count() == m.node.leaf_count() { + Some(combine_nodes(peak, m)) + } else { None } + } { + let link = self.push(stored); + merge_stack.push(link); + appended.push(link); + continue; } + merge_stack.push(next_merge); + merge_stack.push(next_peak); + } - let mut root = merge_stack.pop().expect("There should be at least one node in stack"); - while let Some(next_child) = merge_stack.pop() { - root = self.push_generated( - combine_nodes( - self.resolve_link(root), - self.resolve_link(next_child), - ) + let mut new_root = merge_stack.pop().expect("There should be at least one node in stack"); + while let Some(next_child) = merge_stack.pop() { + new_root = self.push_generated( + combine_nodes( + self.resolve_link(new_root), + self.resolve_link(next_child), ) - } + ) + } - (root, appended) - }; - -// let new_root = if new_root_node.complete() { -// let new_root= self.push(new_root_node); -// appended.push(new_root); -// new_root -// } else { -// self.push_generated(new_root_node) -// }; AppendTransaction { new_root, @@ -490,7 +447,6 @@ mod tests { }); // *** APPEND #6 *** - dbg!("go"); let append_tx = tree.append_leaf(new_root_link, leaf(6)); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).node; @@ -648,14 +604,14 @@ mod tests { quickcheck! { fn there_and_back(number: u32) -> TestResult { - if (number > 1024*1024) { + if number > 1024*1024 { TestResult::discard() } else { let (mut root, mut tree) = initial(); for i in 0..number { root = tree.append_leaf(root, leaf(i+3)).new_root; } - for i in 0..number { + for _ in 0..number { root = tree.truncate_leaf(root).new_root; } @@ -664,12 +620,12 @@ mod tests { } fn leaf_count(number: u32) -> TestResult { - if (number > 1024 * 1024 || number < 3) { + if number > 1024 * 1024 || number < 3 { TestResult::discard() } else { let (mut root, mut tree) = initial(); - for i in 0..(number-2) { - root = tree.append_leaf(root, leaf(i+3)).new_root; + for i in 1..(number-1) { + root = tree.append_leaf(root, leaf(i+2)).new_root; } TestResult::from_bool(tree.resolve_link(root).node.leaf_count() == number) From 4c49f09515308b5794e1a1bebbf08a0210c625f9 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 2 Sep 2019 18:05:20 +0300 Subject: [PATCH 19/76] extra quick-checks and fixes for deleting from full root --- src/tree.rs | 105 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 94 insertions(+), 11 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index c3e0bedff..008a19ac7 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -153,7 +153,7 @@ impl Tree { merge_stack.push(new_leaf_link); while let Some(next_peak) = peaks.pop() { - let next_merge = merge_stack.pop().expect("there should be at least one, checked below"); + let next_merge = merge_stack.pop().expect("there should be at least one, initial or re-pushed"); if let Some(stored) = { let peak = self.resolve_link(next_peak); @@ -171,7 +171,7 @@ impl Tree { merge_stack.push(next_peak); } - let mut new_root = merge_stack.pop().expect("There should be at least one node in stack"); + let mut new_root = merge_stack.pop().expect("Loop above cannot reduce the merge_stack"); while let Some(next_child) = merge_stack.pop() { new_root = self.push_generated( combine_nodes( @@ -211,7 +211,7 @@ impl Tree { let (leaves, root_left_child) = { let n = self.resolve_link(root); ( - n.node.data.end_height - n.node.data.start_height + 1, + n.node.leaf_count(), n.node.left.expect("Root should have left child while deleting") ) }; @@ -239,21 +239,21 @@ impl Tree { .expect("If left exists, right should exist as well"); truncated += 1; } else { + if root.node.complete() { truncated += 1; } break; } } - let root = peaks.drain(0..1).nth(0).expect("At lest 2 elements in peaks"); - let new_root = peaks.into_iter().fold( - root, - |root, next_peak| - self.push_generated( + let mut new_root = peaks.drain(0..1).nth(0).expect("At lest 2 elements in peaks"); + + for next_peak in peaks.into_iter() { + new_root = self.push_generated( combine_nodes( - self.resolve_link(root), + self.resolve_link(new_root), self.resolve_link(next_peak) ) - ) - ); + ); + } for _ in 0..truncated { self.pop(); } @@ -601,6 +601,22 @@ mod tests { assert_eq!(tree.len(), 16); } + #[test] + fn tree_len() { + let (mut root, mut tree) = initial(); + + assert_eq!(tree.len(), 3); + + for i in 0..2 { + root = tree.append_leaf(root, leaf(i+3)).new_root; + } + assert_eq!(tree.len(), 7); + + root = tree.truncate_leaf(root).new_root; + + assert_eq!(tree.len(), 4); + } + quickcheck! { fn there_and_back(number: u32) -> TestResult { @@ -631,5 +647,72 @@ mod tests { TestResult::from_bool(tree.resolve_link(root).node.leaf_count() == number) } } + + fn parity(number: u32) -> TestResult { + if number > 2048 * 2048 || number < 3 { + TestResult::discard() + } else { + let (mut root, mut tree) = initial(); + for i in 1..(number-1) { + root = tree.append_leaf(root, leaf(i+2)).new_root; + } + + TestResult::from_bool( + if number & number - 1 == 0 { + if let NodeLink::Stored(_) = root { true } + else { false } + } else { + if let NodeLink::Generated(_) = root { true } + else { false } + } + ) + } + } + + fn parity_with_truncate(add: u32, delete: u32) -> TestResult { + // First we add `add` number of leaves, then delete `delete` number of leaves + // What is left should be consistent with generated-stored structure + if add > 2048 * 2048 || add < delete { + TestResult::discard() + } else { + let (mut root, mut tree) = initial(); + for i in 0..add { + root = tree.append_leaf(root, leaf(i+3)).new_root; + } + for _ in 0..delete { + root = tree.truncate_leaf(root).new_root; + } + + let total = add - delete + 2; + + TestResult::from_bool( + if total & total - 1 == 0 { + if let NodeLink::Stored(_) = root { true } + else { false } + } else { + if let NodeLink::Generated(_) = root { true } + else { false } + } + ) + } + } + + fn stored_length(add: u32, delete: u32) -> TestResult { + if add > 2048 * 2048 || add < delete { + TestResult::discard() + } else { + let (mut root, mut tree) = initial(); + for i in 0..add { + root = tree.append_leaf(root, leaf(i+3)).new_root; + } + for _ in 0..delete { + root = tree.truncate_leaf(root).new_root; + } + + let total = add - delete + 2; + + TestResult::from_bool(total * total > tree.len()) + } + } } } \ No newline at end of file From d58b33fb78a2ca7d5435a9c83d41585f4c47ddb2 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 2 Sep 2019 18:18:33 +0300 Subject: [PATCH 20/76] simplify and optimize get_peaks --- src/tree.rs | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index 008a19ac7..22abd366b 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -109,12 +109,13 @@ impl Tree { result } - fn get_peaks(&self, root: NodeLink) -> Vec { + fn get_peaks(&self, root: NodeLink, target: &mut Vec) { let (left_child_link, right_child_link) = { let root = self.resolve_link(root); if root.node.complete() { - return vec![root.link]; + target.push(root.link); + return; } ( root.node.left.expect("It would stop before when root is leaf"), @@ -122,23 +123,8 @@ impl Tree { ) }; - let mut result = Vec::new(); - - let left_child = self.resolve_link(left_child_link); - if left_child.node.complete() { - result.push(left_child_link); - } else { - result.extend(self.get_peaks(left_child_link)); - } - - let right_child = self.resolve_link(right_child_link); - if right_child.node.complete() { - result.push(right_child_link); - } else { - result.extend(self.get_peaks(right_child_link)); - } - - result + self.get_peaks(left_child_link, target); + self.get_peaks(right_child_link, target); } /// Append one leaf to the tree. @@ -147,7 +133,8 @@ impl Tree { let mut appended = Vec::new(); appended.push(new_leaf_link); - let mut peaks = self.get_peaks(root); + let mut peaks = Vec::new(); + self.get_peaks(root, &mut peaks); let mut merge_stack = Vec::new(); merge_stack.push(new_leaf_link); @@ -181,7 +168,6 @@ impl Tree { ) } - AppendTransaction { new_root, appended, @@ -612,7 +598,7 @@ mod tests { } assert_eq!(tree.len(), 7); - root = tree.truncate_leaf(root).new_root; + tree.truncate_leaf(root); assert_eq!(tree.len(), 4); } From 49f20e67359cbfd5680b4da193d325df05c64c27 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 2 Sep 2019 18:26:51 +0300 Subject: [PATCH 21/76] extra long truncate --- src/tree.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/tree.rs b/src/tree.rs index 22abd366b..1dd7b81ad 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -603,6 +603,24 @@ mod tests { assert_eq!(tree.len(), 4); } + #[test] + fn tree_len_long() { + let (mut root, mut tree) = initial(); + + assert_eq!(tree.len(), 3); + + for i in 0..4094 { + root = tree.append_leaf(root, leaf(i+3)).new_root; + } + assert_eq!(tree.len(), 8191); // 4096*2-1 (full tree) + + for _ in 0..2049 { + root = tree.truncate_leaf(root).new_root; + } + + assert_eq!(tree.len(), 4083); // 4095 - log2(4096) + } + quickcheck! { fn there_and_back(number: u32) -> TestResult { From 615c4f662e42544e3624d37030b29643e20f5454 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 2 Sep 2019 19:51:00 +0300 Subject: [PATCH 22/76] refactor to rust-only structures --- bindings.h | 59 ---------- src/lib.rs | 72 ++++++++----- src/tree.rs | 303 ++++++++++++++++++++++++++++++---------------------- 3 files changed, 225 insertions(+), 209 deletions(-) delete mode 100644 bindings.h diff --git a/bindings.h b/bindings.h deleted file mode 100644 index fca8fa6ca..000000000 --- a/bindings.h +++ /dev/null @@ -1,59 +0,0 @@ -#include -#include -#include -#include - -template -struct Option; - -struct NodeLink { - enum class Tag { - Stored, - Generated, - }; - - struct Stored_Body { - uint32_t _0; - }; - - struct Generated_Body { - uint32_t _0; - }; - - Tag tag; - union { - Stored_Body stored; - Generated_Body generated; - }; -}; - -struct NodeData { - uint8_t subtree_commitment[32]; - uint32_t start_time; - uint32_t end_time; - uint32_t start_target; - uint32_t end_target; - uint8_t start_sapling_root[32]; - uint8_t end_sapling_root[32]; - uint64_t subtree_total_work; - uint32_t start_height; - uint32_t end_height; - uint64_t shielded_tx; -}; - -struct MMRNode { - Option left; - Option right; - NodeData data; -}; - -extern "C" { - -void append(const MMRNode *_stored, - uint32_t _stored_count, - const MMRNode *_generated, - uint32_t _generated_count, - uint32_t *_append_count, - MMRNode *_append_buffer); - -} // extern "C" diff --git a/src/lib.rs b/src/lib.rs index e55c308d9..f6c555b5f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,10 +26,16 @@ pub struct NodeData { shielded_tx: u64, } +#[derive(Debug)] +pub enum Error { + ExpectedInMemory(EntryLink), + ExpectedNode(Option), +} + /// Reference to to the tree node. #[repr(C)] #[derive(Clone, Copy, Debug)] -pub enum NodeLink { +pub enum EntryLink { /// Reference to the stored (in the array representation) leaf/node. Stored(u32), /// Reference to the generated leaf/node. @@ -39,41 +45,57 @@ pub enum NodeLink { /// MMR Node. It is leaf when `left`, `right` are `None` and node when they are not. #[repr(C)] #[derive(Debug)] -// TODO: Better layout would be enum (node, leaf), with left, right set only for nodes? -pub struct MMRNode { - left: Option, - right: Option, +pub enum EntryKind { + Leaf, + Node(EntryLink, EntryLink), +} + +pub struct Entry { + kind: EntryKind, data: NodeData, } -impl MMRNode { - fn complete(&self) -> bool { +impl Entry { + pub fn complete(&self) -> bool { let leaves = self.leaf_count(); leaves & (leaves - 1) == 0 } - fn leaf_count(&self) -> u32 { + pub fn leaf_count(&self) -> u32 { self.data.end_height - self.data.start_height + 1 } -} -impl From for MMRNode { - fn from(s: NodeData) -> Self { - MMRNode { left: None, right: None, data: s } + pub fn is_leaf(&self) -> bool { + if let EntryKind::Leaf = self.kind { true } else { false } + } + + pub fn left(&self) -> Result { + match self.kind { + EntryKind::Leaf => { Err(Error::ExpectedNode(None)) } + EntryKind::Node(left, _) => Ok(left) + } + } + + pub fn right(&self) -> Result { + match self.kind { + EntryKind::Leaf => { Err(Error::ExpectedNode(None)) } + EntryKind::Node(_, right) => Ok(right) + } } } -#[no_mangle] -pub extern fn append( - _stored: *const MMRNode, - _stored_count: u32, - _generated: *const MMRNode, - _generated_count: u32, - _append_count: *mut u32, - _append_buffer: *mut MMRNode, -) { - - // TODO: construct tree and write to (append_count, append_buffer) - // TODO: also return generated?? - unimplemented!() +impl From for Entry { + fn from(s: NodeData) -> Self { + Entry { kind: EntryKind::Leaf, data: s } + } } + +impl Error { + pub (crate) fn augment(self, link: EntryLink) -> Self { + match self { + Error::ExpectedNode(None) => Error::ExpectedNode(Some(link)), + val => val + } + } +} + diff --git a/src/tree.rs b/src/tree.rs index 1dd7b81ad..2faebb60e 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use crate::{MMRNode, NodeLink, NodeData}; +use crate::{Entry, EntryLink, NodeData, Error, EntryKind}; /// Represents partially loaded tree. /// @@ -11,9 +11,9 @@ use crate::{MMRNode, NodeLink, NodeData}; /// to happen after construction. #[derive(Default)] pub struct Tree { - stored: HashMap, + stored: HashMap, - generated: HashMap, + generated: HashMap, // number of persistent(!) tree entries stored_count: u32, @@ -26,10 +26,10 @@ pub struct Tree { pub struct AppendTransaction { /// Plain list of nodes that has to be appended to the end of the array representation /// of the tree as the result of append operation. - pub appended: Vec, + pub appended: Vec, /// New root as a result of the operation (can be generated one). - pub new_root: NodeLink, + pub new_root: EntryLink, } /// Result of truncating one or severl leaves. @@ -37,48 +37,48 @@ pub struct DeleteTransaction { /// Number of leaves that should be dropped from the end of the list. pub truncated: u32, /// New root as the result of the operation (can be generated one). - pub new_root: NodeLink, + pub new_root: EntryLink, } impl Tree { - fn resolve_link(&self, link: NodeLink) -> IndexedNode { + fn resolve_link(&self, link: EntryLink) -> Result { match link { - NodeLink::Generated(index) => { + EntryLink::Generated(index) => { // TODO: maybe graceful error? - let node = self.generated.get(&index).expect("caller should ensure id generated"); - IndexedNode { + let node = self.generated.get(&index).ok_or(Error::ExpectedInMemory(link))?; + Ok(IndexedNode { node, link, - } + }) }, - NodeLink::Stored(index) => { + EntryLink::Stored(index) => { // TODO: maybe graceful error? - let node = self.stored.get(&index).expect("caller should ensure id stored"); - IndexedNode { + let node = self.stored.get(&index).ok_or(Error::ExpectedInMemory(link))?; + Ok(IndexedNode { node, link, - } + }) }, } } - fn push(&mut self, data: MMRNode) -> NodeLink { + fn push(&mut self, data: Entry) -> EntryLink { let idx = self.stored_count; self.stored_count = self.stored_count + 1; self.stored.insert(idx, data); - NodeLink::Stored(idx) + EntryLink::Stored(idx) } - fn push_generated(&mut self, data: MMRNode) -> NodeLink { + fn push_generated(&mut self, data: Entry) -> EntryLink { let idx = self.generated_count; self.generated_count = self.generated_count + 1; self.generated.insert(idx, data); - NodeLink::Generated(idx) + EntryLink::Generated(idx) } /// Populate tree with plain list of the leaves/nodes. Mostly for tests, /// since this `Tree` structure is for partially loaded tree. - pub fn populate(loaded: Vec) -> Self { + pub fn populate(loaded: Vec) -> Self { let mut result = Tree::default(); result.stored_count = loaded.len() as u32; for (idx, item) in loaded.into_iter().enumerate() { @@ -90,8 +90,8 @@ impl Tree { pub fn new( length: u32, - stored: Vec<(u32, MMRNode)>, - generated: Vec, + stored: Vec<(u32, Entry)>, + generated: Vec, ) -> Self { let mut result = Tree::default(); result.stored_count = length; @@ -109,32 +109,33 @@ impl Tree { result } - fn get_peaks(&self, root: NodeLink, target: &mut Vec) { + fn get_peaks(&self, root: EntryLink, target: &mut Vec) -> Result<(), Error> { let (left_child_link, right_child_link) = { - let root = self.resolve_link(root); + let root = self.resolve_link(root)?; if root.node.complete() { target.push(root.link); - return; + return Ok(()); } ( - root.node.left.expect("It would stop before when root is leaf"), - root.node.right.expect("It would stop before when root is leaf"), + root.left()?, + root.right()?, ) }; - self.get_peaks(left_child_link, target); - self.get_peaks(right_child_link, target); + self.get_peaks(left_child_link, target)?; + self.get_peaks(right_child_link, target)?; + Ok(()) } /// Append one leaf to the tree. - pub fn append_leaf(&mut self, root: NodeLink, new_leaf: NodeData) -> AppendTransaction { + pub fn append_leaf(&mut self, root: EntryLink, new_leaf: NodeData) -> Result { let new_leaf_link = self.push(new_leaf.into()); let mut appended = Vec::new(); appended.push(new_leaf_link); let mut peaks = Vec::new(); - self.get_peaks(root, &mut peaks); + self.get_peaks(root, &mut peaks)?; let mut merge_stack = Vec::new(); merge_stack.push(new_leaf_link); @@ -143,8 +144,8 @@ impl Tree { let next_merge = merge_stack.pop().expect("there should be at least one, initial or re-pushed"); if let Some(stored) = { - let peak = self.resolve_link(next_peak); - let m = self.resolve_link(next_merge); + let peak = self.resolve_link(next_peak)?; + let m = self.resolve_link(next_merge)?; if peak.node.leaf_count() == m.node.leaf_count() { Some(combine_nodes(peak, m)) } else { None } @@ -162,28 +163,28 @@ impl Tree { while let Some(next_child) = merge_stack.pop() { new_root = self.push_generated( combine_nodes( - self.resolve_link(new_root), - self.resolve_link(next_child), + self.resolve_link(new_root)?, + self.resolve_link(next_child)?, ) ) } - AppendTransaction { + Ok(AppendTransaction { new_root, appended, - } + }) } #[cfg(test)] - fn for_children(&mut self, node: NodeLink, mut f: F) { + fn for_children(&mut self, node: EntryLink, mut f: F) { let (left, right) = { - let link = self.resolve_link(node); + let link = self.resolve_link(node).expect("Failed to resolve link in test"); ( - link.node.left.expect("test use only (l)"), - link.node.right.expect("test use only (r)"), + link.left().expect("Failed to find node in test"), + link.right().expect("Failed to find node in test"), ) }; - f(left, right) + f(left, right); } fn pop(&mut self) { @@ -192,37 +193,35 @@ impl Tree { } /// Truncate one leaf from the end of the tree. - pub fn truncate_leaf(&mut self, root: NodeLink) -> DeleteTransaction { + pub fn truncate_leaf(&mut self, root: EntryLink) -> Result { let root = { let (leaves, root_left_child) = { - let n = self.resolve_link(root); + let n = self.resolve_link(root)?; ( n.node.leaf_count(), - n.node.left.expect("Root should have left child while deleting") + n.node.left()?, ) }; if leaves & 1 != 0 { self.pop(); - return DeleteTransaction { + return Ok(DeleteTransaction { truncated: 1, new_root: root_left_child, - } + }) } else { - self.resolve_link(root) + self.resolve_link(root)? } }; - let mut peaks = vec![root.node.left.expect("Root should have left child")]; - let mut subtree_root_link = root.node.right.expect("Root should have right child"); + let mut peaks = vec![root.left()?]; + let mut subtree_root_link = root.right()?; let mut truncated = 1; loop { - let left_link = self.resolve_link(subtree_root_link).node.left; - if let Some(left_link) = left_link { - peaks.push(left_link); - subtree_root_link = self - .resolve_link(subtree_root_link).node.right - .expect("If left exists, right should exist as well"); + let left_link = self.resolve_link(subtree_root_link)?.node; + if let EntryKind::Node(left, right) = left_link.kind { + peaks.push(left); + subtree_root_link = right; truncated += 1; } else { if root.node.complete() { truncated += 1; } @@ -234,19 +233,19 @@ impl Tree { for next_peak in peaks.into_iter() { new_root = self.push_generated( - combine_nodes( - self.resolve_link(new_root), - self.resolve_link(next_peak) - ) - ); + combine_nodes( + self.resolve_link(new_root)?, + self.resolve_link(next_peak)?, + ) + ); } for _ in 0..truncated { self.pop(); } - DeleteTransaction { + Ok(DeleteTransaction { new_root, truncated, - } + }) } /// Length of array representation of the tree. @@ -257,8 +256,20 @@ impl Tree { struct IndexedNode<'a> { - node: &'a MMRNode, - link: NodeLink, + node: &'a Entry, + link: EntryLink, +} + +impl<'a> IndexedNode<'a> { + + fn left(&self) -> Result { + self.node.left().map_err(|e| e.augment(self.link)) + } + + fn right(&self) -> Result { + self.node.right().map_err(|e| e.augment(self.link)) + } + } fn combine_data(left: &NodeData, right: &NodeData) -> NodeData { @@ -280,10 +291,9 @@ fn combine_data(left: &NodeData, right: &NodeData) -> NodeData { } } -fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> MMRNode { - MMRNode { - left: Some(left.link), - right: Some(right.link), +fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> Entry { + Entry { + kind: EntryKind::Node(left.link, right.link), data: combine_data(&left.node.data, &right.node.data), } } @@ -291,7 +301,7 @@ fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> MMRNode { #[cfg(test)] mod tests { - use super::{MMRNode, NodeData, Tree, NodeLink}; + use super::{Entry, NodeData, Tree, EntryLink, EntryKind}; use quickcheck::TestResult; fn leaf(height: u32) -> NodeData { @@ -326,25 +336,27 @@ mod tests { } } - fn initial() -> (NodeLink, Tree) { - let node1: MMRNode = leaf(1).into(); - let node2: MMRNode = leaf(2).into(); + fn initial() -> (EntryLink, Tree) { + let node1: Entry = leaf(1).into(); + let node2: Entry = leaf(2).into(); - let node3 = MMRNode { + let node3 = Entry { data: node(1, 2), - left: Some(NodeLink::Stored(0)), - right: Some(NodeLink::Stored(1)), + kind: EntryKind::Leaf, }; - (NodeLink::Stored(2), Tree::populate(vec![node1, node2, node3])) + (EntryLink::Stored(2), Tree::populate(vec![node1, node2, node3])) } // returns tree with specified number of leafs and it's root - fn generated(length: u32) -> (Tree, NodeLink) { + fn generated(length: u32) -> (Tree, EntryLink) { assert!(length >= 3); let (mut root, mut tree) = initial(); for i in 2..length { - root = tree.append_leaf(root, leaf(i+1).into()).new_root; + root = tree + .append_leaf(root, leaf(i+1).into()) + .expect("Failed to append") + .new_root; } (tree, root) @@ -355,9 +367,11 @@ mod tests { let (root, mut tree) = initial(); // ** APPEND 3 ** - let append_tx = tree.append_leaf(root, leaf(3)); + let append_tx = tree + .append_leaf(root, leaf(3)) + .expect("Failed to append"); let new_root_link = append_tx.new_root; - let new_root = tree.resolve_link(new_root_link).node; + let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; // initial tree: (2) // / \ @@ -376,9 +390,12 @@ mod tests { assert_eq!(append_tx.appended.len(), 1); // ** APPEND 4 ** - let append_tx = tree.append_leaf(new_root_link, leaf(4)); + let append_tx = tree + .append_leaf(new_root_link, leaf(4)) + .expect("Failed to append"); + let new_root_link = append_tx.new_root; - let new_root = tree.resolve_link(new_root_link).node; + let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; // intermediate tree: // (4g) @@ -398,13 +415,15 @@ mod tests { // and new root, (6) is stored one assert_eq!(new_root.data.end_height, 4); assert_eq!(append_tx.appended.len(), 3); - assert_matches!(new_root_link, NodeLink::Stored(6)); + assert_matches!(new_root_link, EntryLink::Stored(6)); // ** APPEND 5 ** - let append_tx = tree.append_leaf(new_root_link, leaf(5)); + let append_tx = tree + .append_leaf(new_root_link, leaf(5)) + .expect("Failed to append"); let new_root_link = append_tx.new_root; - let new_root = tree.resolve_link(new_root_link).node; + let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; // intermediate tree: // ( 6 ) @@ -426,16 +445,18 @@ mod tests { // and new root, (8g) is generated one assert_eq!(new_root.data.end_height, 5); assert_eq!(append_tx.appended.len(), 1); - assert_matches!(new_root_link, NodeLink::Generated(_)); + assert_matches!(new_root_link, EntryLink::Generated(_)); tree.for_children(new_root_link, |l, r| { - assert_matches!(l, NodeLink::Stored(6)); - assert_matches!(r, NodeLink::Stored(7)); + assert_matches!(l, EntryLink::Stored(6)); + assert_matches!(r, EntryLink::Stored(7)); }); // *** APPEND #6 *** - let append_tx = tree.append_leaf(new_root_link, leaf(6)); + let append_tx = tree + .append_leaf(new_root_link, leaf(6)) + .expect("Failed to append"); let new_root_link = append_tx.new_root; - let new_root = tree.resolve_link(new_root_link).node; + let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; // intermediate tree: // ( 8g ) @@ -459,17 +480,22 @@ mod tests { // and new root, (8g) is generated one assert_eq!(new_root.data.end_height, 6); assert_eq!(append_tx.appended.len(), 2); - assert_matches!(new_root_link, NodeLink::Generated(_)); + assert_matches!(new_root_link, EntryLink::Generated(_)); tree.for_children(new_root_link, |l, r| { - assert_matches!(l, NodeLink::Stored(6)); - assert_matches!(r, NodeLink::Stored(9)); + assert_matches!(l, EntryLink::Stored(6)); + assert_matches!(r, EntryLink::Stored(9)); }); // *** APPEND #7 *** - let append_tx = tree.append_leaf(new_root_link, leaf(7)); + let append_tx = tree + .append_leaf(new_root_link, leaf(7)) + .expect("Failed to append"); let new_root_link = append_tx.new_root; - let new_root = tree.resolve_link(new_root_link).node; + let new_root = tree + .resolve_link(new_root_link) + .expect("Failed to resolve root") + .node; // intermediate tree: // (---8g---) @@ -495,17 +521,19 @@ mod tests { // and new root, (8g) is generated one assert_eq!(new_root.data.end_height, 7); assert_eq!(append_tx.appended.len(), 1); - assert_matches!(new_root_link, NodeLink::Generated(_)); + assert_matches!(new_root_link, EntryLink::Generated(_)); tree.for_children(new_root_link, |l, r| { - assert_matches!(l, NodeLink::Generated(_)); - assert_matches!(r, NodeLink::Stored(10)); + assert_matches!(l, EntryLink::Generated(_)); + assert_matches!(r, EntryLink::Stored(10)); }); } #[test] fn truncate_simple() { let (mut tree, root) = generated(9); - let delete_tx = tree.truncate_leaf(root); + let delete_tx = tree + .truncate_leaf(root) + .expect("Failed to truncate"); // initial tree: // @@ -531,14 +559,16 @@ mod tests { // so (15) is truncated // and new root, (14) is a stored one now - assert_matches!(delete_tx.new_root, NodeLink::Stored(14)); + assert_matches!(delete_tx.new_root, EntryLink::Stored(14)); assert_eq!(tree.len(), 15); } #[test] fn truncate_generated() { let (mut tree, root) = generated(10); - let delete_tx = tree.truncate_leaf(root); + let delete_tx = tree + .truncate_leaf(root) + .expect("Failed to truncate"); // initial tree: // @@ -565,21 +595,21 @@ mod tests { // new root is generated - assert_matches!(delete_tx.new_root, NodeLink::Generated(_)); + assert_matches!(delete_tx.new_root, EntryLink::Generated(_)); // left is 14 and right is 15 let (left_root_child, right_root_child) = { - let root = tree.resolve_link(delete_tx.new_root); + let root = tree.resolve_link(delete_tx.new_root).expect("Failed to resolve"); ( - root.node.left.expect("there should be left child for root"), - root.node.right.expect("there should be right child for root"), + root.left().expect("Expected node"), + root.right().expect("Expected node"), ) }; assert_matches!( (left_root_child, right_root_child), - (NodeLink::Stored(14), NodeLink::Stored(15)) + (EntryLink::Stored(14), EntryLink::Stored(15)) ); // two stored nodes should leave us (leaf 16 and no longer needed node 17) @@ -594,11 +624,14 @@ mod tests { assert_eq!(tree.len(), 3); for i in 0..2 { - root = tree.append_leaf(root, leaf(i+3)).new_root; + root = tree + .append_leaf(root, leaf(i+3)) + .expect("Failed to append") + .new_root; } assert_eq!(tree.len(), 7); - tree.truncate_leaf(root); + tree.truncate_leaf(root).expect("Failed to truncate"); assert_eq!(tree.len(), 4); } @@ -610,12 +643,18 @@ mod tests { assert_eq!(tree.len(), 3); for i in 0..4094 { - root = tree.append_leaf(root, leaf(i+3)).new_root; + root = tree + .append_leaf(root, leaf(i+3)) + .expect("Failed to append") + .new_root; } assert_eq!(tree.len(), 8191); // 4096*2-1 (full tree) for _ in 0..2049 { - root = tree.truncate_leaf(root).new_root; + root = tree + .truncate_leaf(root) + .expect("Failed to truncate") + .new_root; } assert_eq!(tree.len(), 4083); // 4095 - log2(4096) @@ -629,13 +668,16 @@ mod tests { } else { let (mut root, mut tree) = initial(); for i in 0..number { - root = tree.append_leaf(root, leaf(i+3)).new_root; + root = tree + .append_leaf(root, leaf(i+3)) + .expect("Failed to append") + .new_root; } for _ in 0..number { - root = tree.truncate_leaf(root).new_root; + root = tree.truncate_leaf(root).expect("Failed to truncate").new_root; } - TestResult::from_bool(if let NodeLink::Stored(2) = root { true } else { false }) + TestResult::from_bool(if let EntryLink::Stored(2) = root { true } else { false }) } } @@ -645,10 +687,21 @@ mod tests { } else { let (mut root, mut tree) = initial(); for i in 1..(number-1) { - root = tree.append_leaf(root, leaf(i+2)).new_root; + root = tree + .append_leaf(root, leaf(i+2)) + .expect("Failed to append") + .new_root; } - TestResult::from_bool(tree.resolve_link(root).node.leaf_count() == number) + TestResult::from_bool( + tree + .resolve_link(root) + .expect("Failed to resolve root") + .node + .leaf_count() + == + number + ) } } @@ -658,15 +711,15 @@ mod tests { } else { let (mut root, mut tree) = initial(); for i in 1..(number-1) { - root = tree.append_leaf(root, leaf(i+2)).new_root; + root = tree.append_leaf(root, leaf(i+2)).expect("Failed to append").new_root; } TestResult::from_bool( if number & number - 1 == 0 { - if let NodeLink::Stored(_) = root { true } + if let EntryLink::Stored(_) = root { true } else { false } } else { - if let NodeLink::Generated(_) = root { true } + if let EntryLink::Generated(_) = root { true } else { false } } ) @@ -681,20 +734,20 @@ mod tests { } else { let (mut root, mut tree) = initial(); for i in 0..add { - root = tree.append_leaf(root, leaf(i+3)).new_root; + root = tree.append_leaf(root, leaf(i+3)).expect("Failed to append").new_root; } for _ in 0..delete { - root = tree.truncate_leaf(root).new_root; + root = tree.truncate_leaf(root).expect("Failed to truncate").new_root; } let total = add - delete + 2; TestResult::from_bool( if total & total - 1 == 0 { - if let NodeLink::Stored(_) = root { true } + if let EntryLink::Stored(_) = root { true } else { false } } else { - if let NodeLink::Generated(_) = root { true } + if let EntryLink::Generated(_) = root { true } else { false } } ) @@ -707,10 +760,10 @@ mod tests { } else { let (mut root, mut tree) = initial(); for i in 0..add { - root = tree.append_leaf(root, leaf(i+3)).new_root; + root = tree.append_leaf(root, leaf(i+3)).expect("Failed to append").new_root; } for _ in 0..delete { - root = tree.truncate_leaf(root).new_root; + root = tree.truncate_leaf(root).expect("Failed to truncate").new_root; } let total = add - delete + 2; From 771aa867c1f100a232b357b2a6a86062c570fbd5 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 3 Sep 2019 12:52:50 +0300 Subject: [PATCH 23/76] derive display for errors --- Cargo.toml | 5 ++++- src/lib.rs | 20 ++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 839ef380b..219918bf4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,4 +6,7 @@ edition = "2018" [dev-dependencies] assert_matches = "1.3.0" -quickcheck = "0.8" \ No newline at end of file +quickcheck = "0.8" + +[dependencies] +derive_more = "0.15" \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index f6c555b5f..f7afa462d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,8 @@ #[cfg(test)] #[macro_use] extern crate assert_matches; #[cfg(test)] #[macro_use] extern crate quickcheck; +extern crate derive_more; + mod tree; @@ -26,19 +28,25 @@ pub struct NodeData { shielded_tx: u64, } -#[derive(Debug)] +#[derive(Debug, derive_more::Display)] pub enum Error { + #[display(fmt="Node/leaf expected to be in memory: {}", _0)] ExpectedInMemory(EntryLink), - ExpectedNode(Option), + #[display(fmt="Node expected")] + ExpectedNode, + #[display(fmt="Node expected, not leaf: {}", _0)] + ExpectedNodeForLink(EntryLink), } /// Reference to to the tree node. #[repr(C)] -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, derive_more::Display)] pub enum EntryLink { /// Reference to the stored (in the array representation) leaf/node. + #[display(fmt="stored(@{})", _0)] Stored(u32), /// Reference to the generated leaf/node. + #[display(fmt="generated(@{})", _0)] Generated(u32), } @@ -71,14 +79,14 @@ impl Entry { pub fn left(&self) -> Result { match self.kind { - EntryKind::Leaf => { Err(Error::ExpectedNode(None)) } + EntryKind::Leaf => { Err(Error::ExpectedNode) } EntryKind::Node(left, _) => Ok(left) } } pub fn right(&self) -> Result { match self.kind { - EntryKind::Leaf => { Err(Error::ExpectedNode(None)) } + EntryKind::Leaf => { Err(Error::ExpectedNode) } EntryKind::Node(_, right) => Ok(right) } } @@ -93,7 +101,7 @@ impl From for Entry { impl Error { pub (crate) fn augment(self, link: EntryLink) -> Self { match self { - Error::ExpectedNode(None) => Error::ExpectedNode(Some(link)), + Error::ExpectedNode => Error::ExpectedNodeForLink(link), val => val } } From bce88797f4cc2ccb1d7526df222569c67d5baf36 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 3 Sep 2019 12:56:24 +0300 Subject: [PATCH 24/76] move node data to separate mod --- src/lib.rs | 20 ++------------------ src/node_data.rs | 37 +++++++++++++++++++++++++++++++++++++ src/tree.rs | 21 +-------------------- 3 files changed, 40 insertions(+), 38 deletions(-) create mode 100644 src/node_data.rs diff --git a/src/lib.rs b/src/lib.rs index f7afa462d..1511ea7b1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,27 +6,11 @@ #[cfg(test)] #[macro_use] extern crate quickcheck; extern crate derive_more; - mod tree; +mod node_data; pub use tree::Tree; - -/// Node metadata. -#[repr(C)] -#[derive(Debug)] -pub struct NodeData { - subtree_commitment: [u8; 32], - start_time: u32, - end_time: u32, - start_target: u32, - end_target: u32, - start_sapling_root: [u8; 32], - end_sapling_root: [u8; 32], - subtree_total_work: u64, - start_height: u32, - end_height: u32, - shielded_tx: u64, -} +pub use node_data::NodeData; #[derive(Debug, derive_more::Display)] pub enum Error { diff --git a/src/node_data.rs b/src/node_data.rs new file mode 100644 index 000000000..8d81c71b3 --- /dev/null +++ b/src/node_data.rs @@ -0,0 +1,37 @@ +/// Node metadata. +#[repr(C)] +#[derive(Debug)] +pub struct NodeData { + pub subtree_commitment: [u8; 32], + pub start_time: u32, + pub end_time: u32, + pub start_target: u32, + pub end_target: u32, + pub start_sapling_root: [u8; 32], + pub end_sapling_root: [u8; 32], + pub subtree_total_work: u64, + pub start_height: u32, + pub end_height: u32, + pub shielded_tx: u64, +} + +impl NodeData { + pub fn combine(left: &NodeData, right: &NodeData) -> NodeData { + NodeData { + // TODO: hash children + subtree_commitment: [0u8; 32], + start_time: left.start_time, + end_time: right.end_time, + start_target: left.start_target, + end_target: right.end_target, + start_sapling_root: left.start_sapling_root, + end_sapling_root: right.end_sapling_root, + + // TODO: sum work? + subtree_total_work: 0, + start_height: left.start_height, + end_height: right.end_height, + shielded_tx: left.shielded_tx + right.shielded_tx, + } + } +} \ No newline at end of file diff --git a/src/tree.rs b/src/tree.rs index 2faebb60e..c5fc08f84 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -272,29 +272,10 @@ impl<'a> IndexedNode<'a> { } -fn combine_data(left: &NodeData, right: &NodeData) -> NodeData { - NodeData { - // TODO: hash children - subtree_commitment: [0u8; 32], - start_time: left.start_time, - end_time: right.end_time, - start_target: left.start_target, - end_target: right.end_target, - start_sapling_root: left.start_sapling_root, - end_sapling_root: right.end_sapling_root, - - // TODO: sum work? - subtree_total_work: 0, - start_height: left.start_height, - end_height: right.end_height, - shielded_tx: left.shielded_tx + right.shielded_tx, - } -} - fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> Entry { Entry { kind: EntryKind::Node(left.link, right.link), - data: combine_data(&left.node.data, &right.node.data), + data: NodeData::combine(&left.node.data, &right.node.data), } } From f432983f09b82b872bc78ccc4bca74174422d2db Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 3 Sep 2019 13:11:10 +0300 Subject: [PATCH 25/76] add notes and remove todos --- src/tree.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index c5fc08f84..9be1b5e3a 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -44,7 +44,6 @@ impl Tree { fn resolve_link(&self, link: EntryLink) -> Result { match link { EntryLink::Generated(index) => { - // TODO: maybe graceful error? let node = self.generated.get(&index).ok_or(Error::ExpectedInMemory(link))?; Ok(IndexedNode { node, @@ -52,7 +51,6 @@ impl Tree { }) }, EntryLink::Stored(index) => { - // TODO: maybe graceful error? let node = self.stored.get(&index).ok_or(Error::ExpectedInMemory(link))?; Ok(IndexedNode { node, @@ -735,6 +733,7 @@ mod tests { } } + // Length of tree is always less than number of leaves squared fn stored_length(add: u32, delete: u32) -> TestResult { if add > 2048 * 2048 || add < delete { TestResult::discard() From d8c04e8143e2ee1bc958ea3996bfe3bdc8454b45 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 3 Sep 2019 17:22:21 +0300 Subject: [PATCH 26/76] node serialization --- Cargo.toml | 4 ++- src/lib.rs | 4 ++- src/node_data.rs | 65 +++++++++++++++++++++++++++++++++++++++++++++--- src/tree.rs | 12 ++++----- 4 files changed, 73 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 219918bf4..355bd0914 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,4 +9,6 @@ assert_matches = "1.3.0" quickcheck = "0.8" [dependencies] -derive_more = "0.15" \ No newline at end of file +derive_more = "0.15" +bigint = "4" +byteorder = "1" \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 1511ea7b1..dbc5b91b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,8 @@ #[cfg(test)] #[macro_use] extern crate assert_matches; #[cfg(test)] #[macro_use] extern crate quickcheck; extern crate derive_more; +extern crate bigint; +extern crate byteorder; mod tree; mod node_data; @@ -53,7 +55,7 @@ impl Entry { leaves & (leaves - 1) == 0 } - pub fn leaf_count(&self) -> u32 { + pub fn leaf_count(&self) -> u64 { self.data.end_height - self.data.start_height + 1 } diff --git a/src/node_data.rs b/src/node_data.rs index 8d81c71b3..ba7877f9d 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -1,3 +1,7 @@ +use byteorder::{LittleEndian, WriteBytesExt}; + +use bigint::U256; + /// Node metadata. #[repr(C)] #[derive(Debug)] @@ -9,13 +13,15 @@ pub struct NodeData { pub end_target: u32, pub start_sapling_root: [u8; 32], pub end_sapling_root: [u8; 32], - pub subtree_total_work: u64, - pub start_height: u32, - pub end_height: u32, + pub subtree_total_work: U256, + pub start_height: u64, + pub end_height: u64, pub shielded_tx: u64, } impl NodeData { + pub const MAX_SERIALIZED_SIZE: usize = 32 + 4 + 4 + 4 + 4 + 32 + 32 + 32 + 9 + 9 + 9; // =171; + pub fn combine(left: &NodeData, right: &NodeData) -> NodeData { NodeData { // TODO: hash children @@ -28,10 +34,61 @@ impl NodeData { end_sapling_root: right.end_sapling_root, // TODO: sum work? - subtree_total_work: 0, + subtree_total_work: left.subtree_total_work + right.subtree_total_work, start_height: left.start_height, end_height: right.end_height, shielded_tx: left.shielded_tx + right.shielded_tx, } } + + fn write_compact(w: &mut W, compact: u64) -> std::io::Result<()> { + match compact { + 0..=0xfc => { + w.write_all(&[compact as u8])? + }, + 0xfd..=0xffff => { + w.write_all(&[0xfd])?; + w.write_u16::(compact as u16)?; + }, + 0x10000..=0xffff_ffff => { + w.write_all(&[0xfe])?; + w.write_u32::(compact as u32)?; + }, + _ => { + w.write_all(&[0xff])?; + w.write_u64::(compact)?; + } + } + Ok(()) + } + + pub fn write(&self, w: &mut W) -> std::io::Result<()> { + w.write_all(&self.subtree_commitment)?; + w.write_u32::(self.start_time)?; + w.write_u32::(self.end_time)?; + w.write_u32::(self.start_target)?; + w.write_u32::(self.end_target)?; + w.write_all(&self.start_sapling_root)?; + w.write_all(&self.end_sapling_root)?; + + let mut work_buf = [0u8; 32]; + self.subtree_total_work.to_little_endian(&mut work_buf[..]); + w.write_all(&work_buf)?; + + Self::write_compact(w, self.start_height)?; + Self::write_compact(w, self.end_height)?; + Self::write_compact(w, self.shielded_tx)?; + Ok(()) + } + + pub fn to_bytes(&self) -> Vec { + let mut buf = [0u8; Self::MAX_SERIALIZED_SIZE]; + let pos = { + let mut cursor = std::io::Cursor::new(&mut buf[..]); + self.write(&mut cursor).expect("Cursor cannot fail"); + cursor.position() as usize + }; + + buf[0..pos].to_vec() + } } \ No newline at end of file diff --git a/src/tree.rs b/src/tree.rs index 9be1b5e3a..e2b1c5bb7 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -292,14 +292,14 @@ mod tests { end_target: 0, start_sapling_root: [0u8; 32], end_sapling_root: [0u8; 32], - subtree_total_work: 0, - start_height: height, - end_height: height, + subtree_total_work: 0.into(), + start_height: height as u64, + end_height: height as u64, shielded_tx: 7, } } - fn node(start_height: u32, end_height: u32) -> NodeData { + fn node(start_height: u64, end_height: u64) -> NodeData { NodeData { subtree_commitment: [0u8; 32], start_time: 0, @@ -308,7 +308,7 @@ mod tests { end_target: 0, start_sapling_root: [0u8; 32], end_sapling_root: [0u8; 32], - subtree_total_work: 0, + subtree_total_work: 0.into(), start_height: start_height, end_height: end_height, shielded_tx: 7, @@ -679,7 +679,7 @@ mod tests { .node .leaf_count() == - number + number as u64 ) } } From b03b4cf958d2c87f0cc95752697a50cc6c0eca52 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 3 Sep 2019 17:47:21 +0300 Subject: [PATCH 27/76] hashing with blake2 --- Cargo.toml | 3 ++- src/lib.rs | 1 + src/node_data.rs | 39 +++++++++++++++++++++++++++++++++++---- src/tree.rs | 2 ++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 355bd0914..b2d0d811e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,4 +11,5 @@ quickcheck = "0.8" [dependencies] derive_more = "0.15" bigint = "4" -byteorder = "1" \ No newline at end of file +byteorder = "1" +blake2-rfc = { git = "https://github.com/gtank/blake2-rfc.git", rev = "7a5b5fc99ae483a0043db7547fb79a6fa44b88a9" } diff --git a/src/lib.rs b/src/lib.rs index dbc5b91b7..5ba7e161d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ extern crate derive_more; extern crate bigint; extern crate byteorder; +extern crate blake2_rfc as blake2; mod tree; mod node_data; diff --git a/src/node_data.rs b/src/node_data.rs index ba7877f9d..10bb5b3cb 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -1,11 +1,12 @@ -use byteorder::{LittleEndian, WriteBytesExt}; - +use byteorder::{LittleEndian, WriteBytesExt, ByteOrder}; use bigint::U256; +use blake2::blake2b::Blake2b; /// Node metadata. #[repr(C)] #[derive(Debug)] pub struct NodeData { + pub consensus_branch_id: u32, pub subtree_commitment: [u8; 32], pub start_time: u32, pub end_time: u32, @@ -19,13 +20,44 @@ pub struct NodeData { pub shielded_tx: u64, } +pub fn blake2b_personal(personalization: &[u8], input: &[u8]) -> [u8; 32] { + let mut hasher = Blake2b::with_params(32, &[], &[], personalization); + hasher.update(input); + let mut result = [0u8; 32]; + result.copy_from_slice(hasher.finalize().as_bytes()); + result +} + +pub fn personalization(branch_id: u32) -> [u8; 16] { + let mut result = [0u8; 16]; + result[..12].copy_from_slice(b"ZcashHistory"); + LittleEndian::write_u32(&mut result[12..], branch_id); + result +} + impl NodeData { pub const MAX_SERIALIZED_SIZE: usize = 32 + 4 + 4 + 4 + 4 + 32 + 32 + 32 + 9 + 9 + 9; // =171; pub fn combine(left: &NodeData, right: &NodeData) -> NodeData { + assert_eq!(left.consensus_branch_id, right.consensus_branch_id); + + let mut hash_buf = [0u8; Self::MAX_SERIALIZED_SIZE * 2]; + let size = { + let mut cursor = ::std::io::Cursor::new(&mut hash_buf[..]); + left.write(&mut cursor).expect("Writing to memory buf with enough length cannot fail; qed"); + right.write(&mut cursor).expect("Writing to memory buf with enough length cannot fail; qed"); + cursor.position() as usize + }; + + let hash = blake2b_personal( + &personalization(left.consensus_branch_id), + &hash_buf[..size] + ); + NodeData { // TODO: hash children - subtree_commitment: [0u8; 32], + consensus_branch_id: left.consensus_branch_id, + subtree_commitment: hash, start_time: left.start_time, end_time: right.end_time, start_target: left.start_target, @@ -33,7 +65,6 @@ impl NodeData { start_sapling_root: left.start_sapling_root, end_sapling_root: right.end_sapling_root, - // TODO: sum work? subtree_total_work: left.subtree_total_work + right.subtree_total_work, start_height: left.start_height, end_height: right.end_height, diff --git a/src/tree.rs b/src/tree.rs index e2b1c5bb7..efb59e370 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -285,6 +285,7 @@ mod tests { fn leaf(height: u32) -> NodeData { NodeData { + consensus_branch_id: 1, subtree_commitment: [0u8; 32], start_time: 0, end_time: 0, @@ -301,6 +302,7 @@ mod tests { fn node(start_height: u64, end_height: u64) -> NodeData { NodeData { + consensus_branch_id: 1, subtree_commitment: [0u8; 32], start_time: 0, end_time: 0, From efac432128ad0fbc37803170307bb15aaffbc7eb Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 3 Sep 2019 18:13:33 +0300 Subject: [PATCH 28/76] remove todo and de-pub --- src/node_data.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/node_data.rs b/src/node_data.rs index 10bb5b3cb..23452c2ff 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -20,7 +20,7 @@ pub struct NodeData { pub shielded_tx: u64, } -pub fn blake2b_personal(personalization: &[u8], input: &[u8]) -> [u8; 32] { +fn blake2b_personal(personalization: &[u8], input: &[u8]) -> [u8; 32] { let mut hasher = Blake2b::with_params(32, &[], &[], personalization); hasher.update(input); let mut result = [0u8; 32]; @@ -28,7 +28,7 @@ pub fn blake2b_personal(personalization: &[u8], input: &[u8]) -> [u8; 32] { result } -pub fn personalization(branch_id: u32) -> [u8; 16] { +fn personalization(branch_id: u32) -> [u8; 16] { let mut result = [0u8; 16]; result[..12].copy_from_slice(b"ZcashHistory"); LittleEndian::write_u32(&mut result[12..], branch_id); @@ -55,7 +55,6 @@ impl NodeData { ); NodeData { - // TODO: hash children consensus_branch_id: left.consensus_branch_id, subtree_commitment: hash, start_time: left.start_time, @@ -64,7 +63,6 @@ impl NodeData { end_target: right.end_target, start_sapling_root: left.start_sapling_root, end_sapling_root: right.end_sapling_root, - subtree_total_work: left.subtree_total_work + right.subtree_total_work, start_height: left.start_height, end_height: right.end_height, From 872ac5af7b96b6b2ccc2231f38f06661ac7f00c9 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 3 Sep 2019 18:31:51 +0300 Subject: [PATCH 29/76] avoid drain of vec --- src/tree.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index efb59e370..bc8541621 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -227,9 +227,9 @@ impl Tree { } } - let mut new_root = peaks.drain(0..1).nth(0).expect("At lest 2 elements in peaks"); + let mut new_root = *peaks.iter().nth(0).expect("At lest 2 elements in peaks"); - for next_peak in peaks.into_iter() { + for next_peak in peaks.into_iter().skip(1) { new_root = self.push_generated( combine_nodes( self.resolve_link(new_root)?, From 6b36cb5a51bc32ccda167aa062a0a0e7cfccba4e Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 6 Sep 2019 15:40:26 +0300 Subject: [PATCH 30/76] example and neccessary fixes --- examples/producer.rs | 219 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 5 + src/node_data.rs | 2 +- src/tree.rs | 27 +++--- 4 files changed, 241 insertions(+), 12 deletions(-) create mode 100644 examples/producer.rs diff --git a/examples/producer.rs b/examples/producer.rs new file mode 100644 index 000000000..b439947f1 --- /dev/null +++ b/examples/producer.rs @@ -0,0 +1,219 @@ +extern crate zcash_mmr as mmr; + +use mmr::{NodeData, Tree, EntryLink, Entry}; + +fn prepare_tree(vec: Vec) -> (Tree, EntryLink) { + + assert!(vec.len() > 0); + + // integer log2 of (vec.len()+1), -1 + let mut h = (32 - ((vec.len()+1) as u32).leading_zeros() - 1)-1; + let mut peak_pos = (1 << (h+1)) - 1; + + let mut nodes = Vec::new(); + let mut root_peak: Entry = vec[peak_pos-1].clone().into(); + + root_peak.update_siblings( + EntryLink::Stored((peak_pos - (1< vec.len() { + // left child, -2^h + peak_pos = peak_pos - (1< bool { let leaves = self.leaf_count(); leaves & (leaves - 1) == 0 diff --git a/src/node_data.rs b/src/node_data.rs index 23452c2ff..d5cec6132 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -4,7 +4,7 @@ use blake2::blake2b::Blake2b; /// Node metadata. #[repr(C)] -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct NodeData { pub consensus_branch_id: u32, pub subtree_commitment: [u8; 32], diff --git a/src/tree.rs b/src/tree.rs index bc8541621..11e93e367 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -88,23 +88,28 @@ impl Tree { pub fn new( length: u32, - stored: Vec<(u32, Entry)>, - generated: Vec, - ) -> Self { + peaks: Vec<(u32, Entry)>, + extra: Vec<(u32, Entry)>, + ) -> (Self, EntryLink) { let mut result = Tree::default(); result.stored_count = length; - for (idx, node) in stored.into_iter() { + let mut gen = 0; + let mut root = EntryLink::Stored(peaks[0].0); + for (idx, node) in peaks.into_iter() { result.stored.insert(idx, node); + if gen != 0 { + let next_generated = + combine_nodes(result. + resolve_link(root).expect("Inserted before, cannot fail; qed"), + result.resolve_link(EntryLink::Stored(idx)).expect("Inserted before, cannot fail; qed") + ); + root = result.push_generated(next_generated); + } + gen += 1; } - result.generated_count = generated.len() as u32; - - for (idx, node) in generated.into_iter().enumerate() { - result.generated.insert(idx as u32, node); - } - - result + (result, root) } fn get_peaks(&self, root: EntryLink, target: &mut Vec) -> Result<(), Error> { From 5d2f84a1542ea30a03b38b078a36af10cc3ec1a6 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 6 Sep 2019 18:52:27 +0300 Subject: [PATCH 31/76] carry root with tree --- examples/producer.rs | 6 ++-- src/tree.rs | 85 ++++++++++++++++++++++++++++---------------- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/examples/producer.rs b/examples/producer.rs index b439947f1..5c669857c 100644 --- a/examples/producer.rs +++ b/examples/producer.rs @@ -2,7 +2,7 @@ extern crate zcash_mmr as mmr; use mmr::{NodeData, Tree, EntryLink, Entry}; -fn prepare_tree(vec: Vec) -> (Tree, EntryLink) { +fn prepare_tree(vec: Vec) -> Tree { assert!(vec.len() > 0); @@ -213,7 +213,7 @@ fn main() { }, ); - let (_tree, root) = prepare_tree(initial_tree_vec); - println!("root: {}", root); + let tree = prepare_tree(initial_tree_vec); + println!("root: {}", tree.root()); } \ No newline at end of file diff --git a/src/tree.rs b/src/tree.rs index 11e93e367..cf0bf9339 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -9,7 +9,6 @@ use crate::{Entry, EntryLink, NodeData, Error, EntryKind}; /// Exact amount of the loaded data can be calculated by the constructing party, /// depending on the length of the tree and maximum amount of operations that are going /// to happen after construction. -#[derive(Default)] pub struct Tree { stored: HashMap, @@ -20,6 +19,8 @@ pub struct Tree { // number of virtual nodes generated generated_count: u32, + + root: EntryLink, } /// Result of appending one or several leaves. @@ -76,22 +77,34 @@ impl Tree { /// Populate tree with plain list of the leaves/nodes. Mostly for tests, /// since this `Tree` structure is for partially loaded tree. - pub fn populate(loaded: Vec) -> Self { - let mut result = Tree::default(); + pub fn populate(loaded: Vec, root: EntryLink) -> Self { + let mut result = Tree::invalid(); result.stored_count = loaded.len() as u32; for (idx, item) in loaded.into_iter().enumerate() { result.stored.insert(idx as u32, item); } + result.root = root; result } + fn invalid() -> Self { + Tree { + root: EntryLink::Generated(0), + generated: Default::default(), + stored: Default::default(), + generated_count: 0, + stored_count: 0, + } + } + pub fn new( length: u32, peaks: Vec<(u32, Entry)>, extra: Vec<(u32, Entry)>, - ) -> (Self, EntryLink) { - let mut result = Tree::default(); + ) -> Self { + let mut result = Tree::invalid(); + result.stored_count = length; let mut gen = 0; @@ -109,7 +122,9 @@ impl Tree { gen += 1; } - (result, root) + result.root = root; + + result } fn get_peaks(&self, root: EntryLink, target: &mut Vec) -> Result<(), Error> { @@ -132,7 +147,8 @@ impl Tree { } /// Append one leaf to the tree. - pub fn append_leaf(&mut self, root: EntryLink, new_leaf: NodeData) -> Result { + pub fn append_leaf(&mut self, new_leaf: NodeData) -> Result { + let root = self.root; let new_leaf_link = self.push(new_leaf.into()); let mut appended = Vec::new(); appended.push(new_leaf_link); @@ -172,6 +188,8 @@ impl Tree { ) } + self.root = new_root; + Ok(AppendTransaction { new_root, appended, @@ -196,10 +214,10 @@ impl Tree { } /// Truncate one leaf from the end of the tree. - pub fn truncate_leaf(&mut self, root: EntryLink) -> Result { + pub fn truncate_leaf(&mut self) -> Result { let root = { let (leaves, root_left_child) = { - let n = self.resolve_link(root)?; + let n = self.resolve_link(self.root)?; ( n.node.leaf_count(), n.node.left()?, @@ -207,12 +225,13 @@ impl Tree { }; if leaves & 1 != 0 { self.pop(); + self.root = root_left_child; return Ok(DeleteTransaction { truncated: 1, new_root: root_left_child, }) } else { - self.resolve_link(root)? + self.resolve_link(self.root)? } }; @@ -245,6 +264,8 @@ impl Tree { for _ in 0..truncated { self.pop(); } + self.root = new_root; + Ok(DeleteTransaction { new_root, truncated, @@ -255,6 +276,8 @@ impl Tree { pub fn len(&self) -> u32 { self.stored_count } + + pub fn root(&self) -> EntryLink { self.root } } @@ -331,7 +354,7 @@ mod tests { kind: EntryKind::Leaf, }; - (EntryLink::Stored(2), Tree::populate(vec![node1, node2, node3])) + (EntryLink::Stored(2), Tree::populate(vec![node1, node2, node3], EntryLink::Stored(2))) } // returns tree with specified number of leafs and it's root @@ -340,7 +363,7 @@ mod tests { let (mut root, mut tree) = initial(); for i in 2..length { root = tree - .append_leaf(root, leaf(i+1).into()) + .append_leaf(leaf(i+1).into()) .expect("Failed to append") .new_root; } @@ -354,7 +377,7 @@ mod tests { // ** APPEND 3 ** let append_tx = tree - .append_leaf(root, leaf(3)) + .append_leaf(leaf(3)) .expect("Failed to append"); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; @@ -377,7 +400,7 @@ mod tests { // ** APPEND 4 ** let append_tx = tree - .append_leaf(new_root_link, leaf(4)) + .append_leaf(leaf(4)) .expect("Failed to append"); let new_root_link = append_tx.new_root; @@ -406,7 +429,7 @@ mod tests { // ** APPEND 5 ** let append_tx = tree - .append_leaf(new_root_link, leaf(5)) + .append_leaf(leaf(5)) .expect("Failed to append"); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; @@ -439,7 +462,7 @@ mod tests { // *** APPEND #6 *** let append_tx = tree - .append_leaf(new_root_link, leaf(6)) + .append_leaf(leaf(6)) .expect("Failed to append"); let new_root_link = append_tx.new_root; let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; @@ -475,7 +498,7 @@ mod tests { // *** APPEND #7 *** let append_tx = tree - .append_leaf(new_root_link, leaf(7)) + .append_leaf(leaf(7)) .expect("Failed to append"); let new_root_link = append_tx.new_root; let new_root = tree @@ -518,7 +541,7 @@ mod tests { fn truncate_simple() { let (mut tree, root) = generated(9); let delete_tx = tree - .truncate_leaf(root) + .truncate_leaf() .expect("Failed to truncate"); // initial tree: @@ -553,7 +576,7 @@ mod tests { fn truncate_generated() { let (mut tree, root) = generated(10); let delete_tx = tree - .truncate_leaf(root) + .truncate_leaf() .expect("Failed to truncate"); // initial tree: @@ -611,13 +634,13 @@ mod tests { for i in 0..2 { root = tree - .append_leaf(root, leaf(i+3)) + .append_leaf(leaf(i+3)) .expect("Failed to append") .new_root; } assert_eq!(tree.len(), 7); - tree.truncate_leaf(root).expect("Failed to truncate"); + tree.truncate_leaf().expect("Failed to truncate"); assert_eq!(tree.len(), 4); } @@ -630,7 +653,7 @@ mod tests { for i in 0..4094 { root = tree - .append_leaf(root, leaf(i+3)) + .append_leaf(leaf(i+3)) .expect("Failed to append") .new_root; } @@ -638,7 +661,7 @@ mod tests { for _ in 0..2049 { root = tree - .truncate_leaf(root) + .truncate_leaf() .expect("Failed to truncate") .new_root; } @@ -655,12 +678,12 @@ mod tests { let (mut root, mut tree) = initial(); for i in 0..number { root = tree - .append_leaf(root, leaf(i+3)) + .append_leaf(leaf(i+3)) .expect("Failed to append") .new_root; } for _ in 0..number { - root = tree.truncate_leaf(root).expect("Failed to truncate").new_root; + root = tree.truncate_leaf().expect("Failed to truncate").new_root; } TestResult::from_bool(if let EntryLink::Stored(2) = root { true } else { false }) @@ -674,7 +697,7 @@ mod tests { let (mut root, mut tree) = initial(); for i in 1..(number-1) { root = tree - .append_leaf(root, leaf(i+2)) + .append_leaf(leaf(i+2)) .expect("Failed to append") .new_root; } @@ -697,7 +720,7 @@ mod tests { } else { let (mut root, mut tree) = initial(); for i in 1..(number-1) { - root = tree.append_leaf(root, leaf(i+2)).expect("Failed to append").new_root; + root = tree.append_leaf(leaf(i+2)).expect("Failed to append").new_root; } TestResult::from_bool( @@ -720,10 +743,10 @@ mod tests { } else { let (mut root, mut tree) = initial(); for i in 0..add { - root = tree.append_leaf(root, leaf(i+3)).expect("Failed to append").new_root; + root = tree.append_leaf(leaf(i+3)).expect("Failed to append").new_root; } for _ in 0..delete { - root = tree.truncate_leaf(root).expect("Failed to truncate").new_root; + root = tree.truncate_leaf().expect("Failed to truncate").new_root; } let total = add - delete + 2; @@ -747,10 +770,10 @@ mod tests { } else { let (mut root, mut tree) = initial(); for i in 0..add { - root = tree.append_leaf(root, leaf(i+3)).expect("Failed to append").new_root; + root = tree.append_leaf(leaf(i+3)).expect("Failed to append").new_root; } for _ in 0..delete { - root = tree.truncate_leaf(root).expect("Failed to truncate").new_root; + root = tree.truncate_leaf().expect("Failed to truncate").new_root; } let total = add - delete + 2; From 942a976ef5a8959997580b94b2cd1c89f3016bd7 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 6 Sep 2019 19:14:53 +0300 Subject: [PATCH 32/76] refactor numerous tests --- src/tree.rs | 212 +++++++++++++++++++++------------------------------- 1 file changed, 85 insertions(+), 127 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index cf0bf9339..3e2d4b078 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -23,24 +23,6 @@ pub struct Tree { root: EntryLink, } -/// Result of appending one or several leaves. -pub struct AppendTransaction { - /// Plain list of nodes that has to be appended to the end of the array representation - /// of the tree as the result of append operation. - pub appended: Vec, - - /// New root as a result of the operation (can be generated one). - pub new_root: EntryLink, -} - -/// Result of truncating one or severl leaves. -pub struct DeleteTransaction { - /// Number of leaves that should be dropped from the end of the list. - pub truncated: u32, - /// New root as the result of the operation (can be generated one). - pub new_root: EntryLink, -} - impl Tree { fn resolve_link(&self, link: EntryLink) -> Result { match link { @@ -122,6 +104,10 @@ impl Tree { gen += 1; } + for (idx, node) in extra { + result.stored.insert(idx, node); + } + result.root = root; result @@ -147,7 +133,9 @@ impl Tree { } /// Append one leaf to the tree. - pub fn append_leaf(&mut self, new_leaf: NodeData) -> Result { + /// + /// Returns links to actual nodes that has to be persisted as the result of the append. + pub fn append_leaf(&mut self, new_leaf: NodeData) -> Result, Error> { let root = self.root; let new_leaf_link = self.push(new_leaf.into()); let mut appended = Vec::new(); @@ -190,10 +178,7 @@ impl Tree { self.root = new_root; - Ok(AppendTransaction { - new_root, - appended, - }) + Ok(appended) } #[cfg(test)] @@ -214,7 +199,9 @@ impl Tree { } /// Truncate one leaf from the end of the tree. - pub fn truncate_leaf(&mut self) -> Result { + /// + /// Returns actual number of nodes that has to be removed from the array representation. + pub fn truncate_leaf(&mut self) -> Result { let root = { let (leaves, root_left_child) = { let n = self.resolve_link(self.root)?; @@ -226,10 +213,7 @@ impl Tree { if leaves & 1 != 0 { self.pop(); self.root = root_left_child; - return Ok(DeleteTransaction { - truncated: 1, - new_root: root_left_child, - }) + return Ok(1); } else { self.resolve_link(self.root)? } @@ -266,10 +250,7 @@ impl Tree { self.root = new_root; - Ok(DeleteTransaction { - new_root, - truncated, - }) + Ok(truncated) } /// Length of array representation of the tree. @@ -277,11 +258,17 @@ impl Tree { self.stored_count } + /// Link to the root node pub fn root(&self) -> EntryLink { self.root } + + /// Reference to the root ndoe + pub fn root_node(&self) -> Result { + self.resolve_link(self.root) + } } -struct IndexedNode<'a> { +pub struct IndexedNode<'a> { node: &'a Entry, link: EntryLink, } @@ -296,6 +283,10 @@ impl<'a> IndexedNode<'a> { self.node.right().map_err(|e| e.augment(self.link)) } + pub fn node(&self) -> &Entry { + self.node + } + } fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> Entry { @@ -345,7 +336,7 @@ mod tests { } } - fn initial() -> (EntryLink, Tree) { + fn initial() -> Tree { let node1: Entry = leaf(1).into(); let node2: Entry = leaf(2).into(); @@ -354,33 +345,29 @@ mod tests { kind: EntryKind::Leaf, }; - (EntryLink::Stored(2), Tree::populate(vec![node1, node2, node3], EntryLink::Stored(2))) + Tree::populate(vec![node1, node2, node3], EntryLink::Stored(2)) } // returns tree with specified number of leafs and it's root - fn generated(length: u32) -> (Tree, EntryLink) { + fn generated(length: u32) -> Tree { assert!(length >= 3); - let (mut root, mut tree) = initial(); + let mut tree = initial(); for i in 2..length { - root = tree - .append_leaf(leaf(i+1).into()) - .expect("Failed to append") - .new_root; + tree.append_leaf(leaf(i+1).into()).expect("Failed to append"); } - (tree, root) + tree } #[test] fn discrete_append() { - let (root, mut tree) = initial(); + let mut tree = initial(); // ** APPEND 3 ** - let append_tx = tree + let appended = tree .append_leaf(leaf(3)) .expect("Failed to append"); - let new_root_link = append_tx.new_root; - let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; + let new_root = tree.root_node().expect("Failed to resolve root").node; // initial tree: (2) // / \ @@ -396,15 +383,14 @@ mod tests { // so only (3) is added as real leaf // while new root, (4g) is generated one assert_eq!(new_root.data.end_height, 3); - assert_eq!(append_tx.appended.len(), 1); + assert_eq!(appended.len(), 1); // ** APPEND 4 ** - let append_tx = tree + let appended = tree .append_leaf(leaf(4)) .expect("Failed to append"); - let new_root_link = append_tx.new_root; - let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; + let new_root = tree.root_node().expect("Failed to resolve root").node; // intermediate tree: // (4g) @@ -423,16 +409,15 @@ mod tests { // so (4), (5), (6) are added as real leaves // and new root, (6) is stored one assert_eq!(new_root.data.end_height, 4); - assert_eq!(append_tx.appended.len(), 3); - assert_matches!(new_root_link, EntryLink::Stored(6)); + assert_eq!(appended.len(), 3); + assert_matches!(tree.root(), EntryLink::Stored(6)); // ** APPEND 5 ** - let append_tx = tree + let appended = tree .append_leaf(leaf(5)) .expect("Failed to append"); - let new_root_link = append_tx.new_root; - let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; + let new_root = tree.root_node().expect("Failed to resolve root").node; // intermediate tree: // ( 6 ) @@ -453,19 +438,18 @@ mod tests { // so (7) is added as real leaf // and new root, (8g) is generated one assert_eq!(new_root.data.end_height, 5); - assert_eq!(append_tx.appended.len(), 1); - assert_matches!(new_root_link, EntryLink::Generated(_)); - tree.for_children(new_root_link, |l, r| { + assert_eq!(appended.len(), 1); + assert_matches!(tree.root(), EntryLink::Generated(_)); + tree.for_children(tree.root(), |l, r| { assert_matches!(l, EntryLink::Stored(6)); assert_matches!(r, EntryLink::Stored(7)); }); // *** APPEND #6 *** - let append_tx = tree + let appended = tree .append_leaf(leaf(6)) .expect("Failed to append"); - let new_root_link = append_tx.new_root; - let new_root = tree.resolve_link(new_root_link).expect("Failed to resolve root").node; + let new_root = tree.root_node().expect("Failed to resolve root").node; // intermediate tree: // ( 8g ) @@ -488,21 +472,20 @@ mod tests { // so (7) is added as real leaf // and new root, (8g) is generated one assert_eq!(new_root.data.end_height, 6); - assert_eq!(append_tx.appended.len(), 2); - assert_matches!(new_root_link, EntryLink::Generated(_)); - tree.for_children(new_root_link, |l, r| { + assert_eq!(appended.len(), 2); + assert_matches!(tree.root(), EntryLink::Generated(_)); + tree.for_children(tree.root(), |l, r| { assert_matches!(l, EntryLink::Stored(6)); assert_matches!(r, EntryLink::Stored(9)); }); // *** APPEND #7 *** - let append_tx = tree + let appended = tree .append_leaf(leaf(7)) .expect("Failed to append"); - let new_root_link = append_tx.new_root; let new_root = tree - .resolve_link(new_root_link) + .root_node() .expect("Failed to resolve root") .node; @@ -529,9 +512,9 @@ mod tests { // so (7) is added as real leaf // and new root, (8g) is generated one assert_eq!(new_root.data.end_height, 7); - assert_eq!(append_tx.appended.len(), 1); - assert_matches!(new_root_link, EntryLink::Generated(_)); - tree.for_children(new_root_link, |l, r| { + assert_eq!(appended.len(), 1); + assert_matches!(tree.root(), EntryLink::Generated(_)); + tree.for_children(tree.root(), |l, r| { assert_matches!(l, EntryLink::Generated(_)); assert_matches!(r, EntryLink::Stored(10)); }); @@ -539,10 +522,8 @@ mod tests { #[test] fn truncate_simple() { - let (mut tree, root) = generated(9); - let delete_tx = tree - .truncate_leaf() - .expect("Failed to truncate"); + let mut tree = generated(9); + tree.truncate_leaf().expect("Failed to truncate"); // initial tree: // @@ -568,16 +549,14 @@ mod tests { // so (15) is truncated // and new root, (14) is a stored one now - assert_matches!(delete_tx.new_root, EntryLink::Stored(14)); + assert_matches!(tree.root(), EntryLink::Stored(14)); assert_eq!(tree.len(), 15); } #[test] fn truncate_generated() { - let (mut tree, root) = generated(10); - let delete_tx = tree - .truncate_leaf() - .expect("Failed to truncate"); + let mut tree = generated(10); + let deleted = tree.truncate_leaf().expect("Failed to truncate"); // initial tree: // @@ -604,11 +583,11 @@ mod tests { // new root is generated - assert_matches!(delete_tx.new_root, EntryLink::Generated(_)); + assert_matches!(tree.root(), EntryLink::Generated(_)); // left is 14 and right is 15 let (left_root_child, right_root_child) = { - let root = tree.resolve_link(delete_tx.new_root).expect("Failed to resolve"); + let root = tree.root_node().expect("Failed to resolve"); ( root.left().expect("Expected node"), @@ -622,21 +601,18 @@ mod tests { ); // two stored nodes should leave us (leaf 16 and no longer needed node 17) - assert_eq!(delete_tx.truncated, 2); + assert_eq!(deleted, 2); assert_eq!(tree.len(), 16); } #[test] fn tree_len() { - let (mut root, mut tree) = initial(); + let mut tree = initial(); assert_eq!(tree.len(), 3); for i in 0..2 { - root = tree - .append_leaf(leaf(i+3)) - .expect("Failed to append") - .new_root; + tree.append_leaf(leaf(i+3)).expect("Failed to append"); } assert_eq!(tree.len(), 7); @@ -647,23 +623,17 @@ mod tests { #[test] fn tree_len_long() { - let (mut root, mut tree) = initial(); + let mut tree = initial(); assert_eq!(tree.len(), 3); for i in 0..4094 { - root = tree - .append_leaf(leaf(i+3)) - .expect("Failed to append") - .new_root; + tree.append_leaf(leaf(i+3)).expect("Failed to append"); } assert_eq!(tree.len(), 8191); // 4096*2-1 (full tree) for _ in 0..2049 { - root = tree - .truncate_leaf() - .expect("Failed to truncate") - .new_root; + tree.truncate_leaf().expect("Failed to truncate"); } assert_eq!(tree.len(), 4083); // 4095 - log2(4096) @@ -675,18 +645,15 @@ mod tests { if number > 1024*1024 { TestResult::discard() } else { - let (mut root, mut tree) = initial(); + let mut tree = initial(); for i in 0..number { - root = tree - .append_leaf(leaf(i+3)) - .expect("Failed to append") - .new_root; + tree.append_leaf(leaf(i+3)).expect("Failed to append"); } for _ in 0..number { - root = tree.truncate_leaf().expect("Failed to truncate").new_root; + tree.truncate_leaf().expect("Failed to truncate"); } - TestResult::from_bool(if let EntryLink::Stored(2) = root { true } else { false }) + TestResult::from_bool(if let EntryLink::Stored(2) = tree.root() { true } else { false }) } } @@ -694,22 +661,13 @@ mod tests { if number > 1024 * 1024 || number < 3 { TestResult::discard() } else { - let (mut root, mut tree) = initial(); + let mut tree = initial(); for i in 1..(number-1) { - root = tree - .append_leaf(leaf(i+2)) - .expect("Failed to append") - .new_root; + tree.append_leaf(leaf(i+2)).expect("Failed to append"); } TestResult::from_bool( - tree - .resolve_link(root) - .expect("Failed to resolve root") - .node - .leaf_count() - == - number as u64 + tree.root_node().expect("no root").node.leaf_count() == number as u64 ) } } @@ -718,17 +676,17 @@ mod tests { if number > 2048 * 2048 || number < 3 { TestResult::discard() } else { - let (mut root, mut tree) = initial(); + let mut tree = initial(); for i in 1..(number-1) { - root = tree.append_leaf(leaf(i+2)).expect("Failed to append").new_root; + tree.append_leaf(leaf(i+2)).expect("Failed to append"); } TestResult::from_bool( if number & number - 1 == 0 { - if let EntryLink::Stored(_) = root { true } + if let EntryLink::Stored(_) = tree.root() { true } else { false } } else { - if let EntryLink::Generated(_) = root { true } + if let EntryLink::Generated(_) = tree.root() { true } else { false } } ) @@ -741,22 +699,22 @@ mod tests { if add > 2048 * 2048 || add < delete { TestResult::discard() } else { - let (mut root, mut tree) = initial(); + let mut tree = initial(); for i in 0..add { - root = tree.append_leaf(leaf(i+3)).expect("Failed to append").new_root; + tree.append_leaf(leaf(i+3)).expect("Failed to append"); } for _ in 0..delete { - root = tree.truncate_leaf().expect("Failed to truncate").new_root; + tree.truncate_leaf().expect("Failed to truncate"); } let total = add - delete + 2; TestResult::from_bool( if total & total - 1 == 0 { - if let EntryLink::Stored(_) = root { true } + if let EntryLink::Stored(_) = tree.root() { true } else { false } } else { - if let EntryLink::Generated(_) = root { true } + if let EntryLink::Generated(_) = tree.root() { true } else { false } } ) @@ -768,12 +726,12 @@ mod tests { if add > 2048 * 2048 || add < delete { TestResult::discard() } else { - let (mut root, mut tree) = initial(); + let mut tree = initial(); for i in 0..add { - root = tree.append_leaf(leaf(i+3)).expect("Failed to append").new_root; + tree.append_leaf(leaf(i+3)).expect("Failed to append"); } for _ in 0..delete { - root = tree.truncate_leaf().expect("Failed to truncate").new_root; + tree.truncate_leaf().expect("Failed to truncate"); } let total = add - delete + 2; From a46ace4c2d2dc83c3ec5fbdc39624b88a22d9455 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 6 Sep 2019 19:19:58 +0300 Subject: [PATCH 33/76] simplify tree preparation --- examples/producer.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/examples/producer.rs b/examples/producer.rs index 5c669857c..2ef7598ca 100644 --- a/examples/producer.rs +++ b/examples/producer.rs @@ -9,18 +9,7 @@ fn prepare_tree(vec: Vec) -> Tree { // integer log2 of (vec.len()+1), -1 let mut h = (32 - ((vec.len()+1) as u32).leading_zeros() - 1)-1; let mut peak_pos = (1 << (h+1)) - 1; - let mut nodes = Vec::new(); - let mut root_peak: Entry = vec[peak_pos-1].clone().into(); - - root_peak.update_siblings( - EntryLink::Stored((peak_pos - (1< Date: Sat, 7 Sep 2019 11:19:06 +0300 Subject: [PATCH 34/76] missing deserialization bits --- src/node_data.rs | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/src/node_data.rs b/src/node_data.rs index d5cec6132..ab08d4a56 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -1,10 +1,10 @@ -use byteorder::{LittleEndian, WriteBytesExt, ByteOrder}; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt, ByteOrder}; use bigint::U256; use blake2::blake2b::Blake2b; /// Node metadata. #[repr(C)] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct NodeData { pub consensus_branch_id: u32, pub subtree_commitment: [u8; 32], @@ -91,6 +91,17 @@ impl NodeData { Ok(()) } + fn read_compact(reader: &mut R) -> std::io::Result { + let result = match reader.read_u8()? { + i @ 0..=0xfc => i.into(), + 0xfd => reader.read_u16::()?.into(), + 0xfe => reader.read_u32::()?.into(), + _ => reader.read_u64::()?.into(), + }; + + Ok(result) + } + pub fn write(&self, w: &mut W) -> std::io::Result<()> { w.write_all(&self.subtree_commitment)?; w.write_u32::(self.start_time)?; @@ -110,6 +121,28 @@ impl NodeData { Ok(()) } + pub fn read(&self, consensus_branch_id: u32, r: &mut R) -> std::io::Result { + let mut data = Self::default(); + data.consensus_branch_id = consensus_branch_id; + r.read_exact(&mut data.subtree_commitment)?; + data.start_time = r.read_u32::()?; + data.end_time = r.read_u32::()?; + data.start_target= r.read_u32::()?; + data.end_target= r.read_u32::()?; + r.read_exact(&mut data.start_sapling_root)?; + r.read_exact(&mut data.end_sapling_root)?; + + let mut work_buf = [0u8; 32]; + r.read_exact(&mut work_buf)?; + data.subtree_total_work = U256::from_little_endian(&work_buf); + + data.start_height = Self::read_compact(r)?; + data.end_height = Self::read_compact(r)?; + data.shielded_tx = Self::read_compact(r)?; + + Ok(data) + } + pub fn to_bytes(&self) -> Vec { let mut buf = [0u8; Self::MAX_SERIALIZED_SIZE]; let pos = { From c4f8f8ea0437309b0ae0d5d8bfb013c57c164fd6 Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Sat, 7 Sep 2019 10:31:01 +0200 Subject: [PATCH 35/76] fix: use rust2018 idioms --- Cargo.toml | 2 +- src/lib.rs | 7 ------- src/tree.rs | 5 +++-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b2d0d811e..637341706 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,4 +12,4 @@ quickcheck = "0.8" derive_more = "0.15" bigint = "4" byteorder = "1" -blake2-rfc = { git = "https://github.com/gtank/blake2-rfc.git", rev = "7a5b5fc99ae483a0043db7547fb79a6fa44b88a9" } +blake2 = { package = "blake2-rfc", git = "https://github.com/gtank/blake2-rfc.git", rev = "7a5b5fc99ae483a0043db7547fb79a6fa44b88a9" } diff --git a/src/lib.rs b/src/lib.rs index 5c3b673b2..cac231376 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,13 +2,6 @@ //! //! To be used in zebra and via FFI bindings in zcashd -#[cfg(test)] #[macro_use] extern crate assert_matches; -#[cfg(test)] #[macro_use] extern crate quickcheck; -extern crate derive_more; -extern crate bigint; -extern crate byteorder; -extern crate blake2_rfc as blake2; - mod tree; mod node_data; diff --git a/src/tree.rs b/src/tree.rs index 3e2d4b078..d5210e54b 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -300,7 +300,8 @@ fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> Entry { mod tests { use super::{Entry, NodeData, Tree, EntryLink, EntryKind}; - use quickcheck::TestResult; + use quickcheck::{quickcheck, TestResult}; + use assert_matches::assert_matches; fn leaf(height: u32) -> NodeData { NodeData { @@ -740,4 +741,4 @@ mod tests { } } } -} \ No newline at end of file +} From 03524ba7d033db79d093fb2de40b47b36cad82eb Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sat, 7 Sep 2019 13:52:19 +0300 Subject: [PATCH 36/76] entry to the dedicated module --- src/entry.rs | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 47 ++--------------------------------------------- 2 files changed, 50 insertions(+), 45 deletions(-) create mode 100644 src/entry.rs diff --git a/src/entry.rs b/src/entry.rs new file mode 100644 index 000000000..da0b1319f --- /dev/null +++ b/src/entry.rs @@ -0,0 +1,48 @@ +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt, ByteOrder}; + +use crate::{EntryKind, NodeData, Error, EntryLink}; + +#[derive(Debug)] +pub struct Entry { + pub(crate) kind: EntryKind, + pub(crate) data: NodeData, +} + +impl Entry { + pub fn update_siblings(&mut self, left: EntryLink, right: EntryLink) { + self.kind = EntryKind::Node(left, right); + } + + pub fn complete(&self) -> bool { + let leaves = self.leaf_count(); + leaves & (leaves - 1) == 0 + } + + pub fn leaf_count(&self) -> u64 { + self.data.end_height - self.data.start_height + 1 + } + + pub fn is_leaf(&self) -> bool { + if let EntryKind::Leaf = self.kind { true } else { false } + } + + pub fn left(&self) -> Result { + match self.kind { + EntryKind::Leaf => { Err(Error::ExpectedNode) } + EntryKind::Node(left, _) => Ok(left) + } + } + + pub fn right(&self) -> Result { + match self.kind { + EntryKind::Leaf => { Err(Error::ExpectedNode) } + EntryKind::Node(_, right) => Ok(right) + } + } +} + +impl From for Entry { + fn from(s: NodeData) -> Self { + Entry { kind: EntryKind::Leaf, data: s } + } +} diff --git a/src/lib.rs b/src/lib.rs index cac231376..552dee293 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,9 +4,11 @@ mod tree; mod node_data; +mod entry; pub use tree::Tree; pub use node_data::NodeData; +pub use entry::Entry; #[derive(Debug, derive_more::Display)] pub enum Error { @@ -38,51 +40,6 @@ pub enum EntryKind { Node(EntryLink, EntryLink), } -#[derive(Debug)] -pub struct Entry { - kind: EntryKind, - data: NodeData, -} - -impl Entry { - pub fn update_siblings(&mut self, left: EntryLink, right: EntryLink) { - self.kind = EntryKind::Node(left, right); - } - - pub fn complete(&self) -> bool { - let leaves = self.leaf_count(); - leaves & (leaves - 1) == 0 - } - - pub fn leaf_count(&self) -> u64 { - self.data.end_height - self.data.start_height + 1 - } - - pub fn is_leaf(&self) -> bool { - if let EntryKind::Leaf = self.kind { true } else { false } - } - - pub fn left(&self) -> Result { - match self.kind { - EntryKind::Leaf => { Err(Error::ExpectedNode) } - EntryKind::Node(left, _) => Ok(left) - } - } - - pub fn right(&self) -> Result { - match self.kind { - EntryKind::Leaf => { Err(Error::ExpectedNode) } - EntryKind::Node(_, right) => Ok(right) - } - } -} - -impl From for Entry { - fn from(s: NodeData) -> Self { - Entry { kind: EntryKind::Leaf, data: s } - } -} - impl Error { pub (crate) fn augment(self, link: EntryLink) -> Self { match self { From de053e1d8f23ca7ccf60619ef3443d709ad8ff6e Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sat, 7 Sep 2019 14:00:34 +0300 Subject: [PATCH 37/76] reading for Entry --- src/entry.rs | 27 ++++++++++++++++++++++++++- src/node_data.rs | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/entry.rs b/src/entry.rs index da0b1319f..3ed1c71ab 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -1,4 +1,4 @@ -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt, ByteOrder}; +use byteorder::{LittleEndian, ReadBytesExt}; use crate::{EntryKind, NodeData, Error, EntryLink}; @@ -39,6 +39,31 @@ impl Entry { EntryKind::Node(_, right) => Ok(right) } } + + pub fn read(&self, consensus_branch_id: u32, r: &mut R) -> std::io::Result { + let kind = { + match r.read_u8()? { + 0 => { + let left = r.read_u32::()?; + let right = r.read_u32::()?; + EntryKind::Node(EntryLink::Stored(left), EntryLink::Stored(right)) + }, + 1 => { + EntryKind::Leaf + }, + _ => { + return Err(std::io::Error::from(std::io::ErrorKind::InvalidData)) + }, + } + }; + + let data = NodeData::read(consensus_branch_id, r)?; + + Ok(Entry { + kind, + data, + }) + } } impl From for Entry { diff --git a/src/node_data.rs b/src/node_data.rs index ab08d4a56..b95f643cd 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -121,7 +121,7 @@ impl NodeData { Ok(()) } - pub fn read(&self, consensus_branch_id: u32, r: &mut R) -> std::io::Result { + pub fn read(consensus_branch_id: u32, r: &mut R) -> std::io::Result { let mut data = Self::default(); data.consensus_branch_id = consensus_branch_id; r.read_exact(&mut data.subtree_commitment)?; From 49763d1c0189713f4c9e51827cb31b9574361355 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sat, 7 Sep 2019 14:04:51 +0300 Subject: [PATCH 38/76] arrange constants --- examples/producer.rs | 1 - src/entry.rs | 4 +++- src/lib.rs | 4 ++-- src/node_data.rs | 8 ++++---- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/examples/producer.rs b/examples/producer.rs index 2ef7598ca..7d93cba8d 100644 --- a/examples/producer.rs +++ b/examples/producer.rs @@ -204,5 +204,4 @@ fn main() { let tree = prepare_tree(initial_tree_vec); println!("root: {}", tree.root()); - } \ No newline at end of file diff --git a/src/entry.rs b/src/entry.rs index 3ed1c71ab..485b36486 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -1,6 +1,8 @@ use byteorder::{LittleEndian, ReadBytesExt}; -use crate::{EntryKind, NodeData, Error, EntryLink}; +use crate::{EntryKind, NodeData, Error, EntryLink, MAX_NODE_DATA_SIZE}; + +pub const MAX_ENTRY_SIZE: usize = MAX_NODE_DATA_SIZE + 9; #[derive(Debug)] pub struct Entry { diff --git a/src/lib.rs b/src/lib.rs index 552dee293..f0eb3f466 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,8 +7,8 @@ mod node_data; mod entry; pub use tree::Tree; -pub use node_data::NodeData; -pub use entry::Entry; +pub use node_data::{NodeData, MAX_NODE_DATA_SIZE}; +pub use entry::{Entry, MAX_ENTRY_SIZE}; #[derive(Debug, derive_more::Display)] pub enum Error { diff --git a/src/node_data.rs b/src/node_data.rs index b95f643cd..59591e532 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -2,6 +2,8 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt, ByteOrder}; use bigint::U256; use blake2::blake2b::Blake2b; +pub const MAX_NODE_DATA_SIZE: usize = 32 + 4 + 4 + 4 + 4 + 32 + 32 + 32 + 9 + 9 + 9; // 171 + /// Node metadata. #[repr(C)] #[derive(Debug, Clone, Default)] @@ -36,12 +38,10 @@ fn personalization(branch_id: u32) -> [u8; 16] { } impl NodeData { - pub const MAX_SERIALIZED_SIZE: usize = 32 + 4 + 4 + 4 + 4 + 32 + 32 + 32 + 9 + 9 + 9; // =171; - pub fn combine(left: &NodeData, right: &NodeData) -> NodeData { assert_eq!(left.consensus_branch_id, right.consensus_branch_id); - let mut hash_buf = [0u8; Self::MAX_SERIALIZED_SIZE * 2]; + let mut hash_buf = [0u8; MAX_NODE_DATA_SIZE * 2]; let size = { let mut cursor = ::std::io::Cursor::new(&mut hash_buf[..]); left.write(&mut cursor).expect("Writing to memory buf with enough length cannot fail; qed"); @@ -144,7 +144,7 @@ impl NodeData { } pub fn to_bytes(&self) -> Vec { - let mut buf = [0u8; Self::MAX_SERIALIZED_SIZE]; + let mut buf = [0u8; MAX_NODE_DATA_SIZE]; let pos = { let mut cursor = std::io::Cursor::new(&mut buf[..]); self.write(&mut cursor).expect("Cursor cannot fail"); From 6d9deefb934f11c91f751618fd86a5a0dbe86570 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sat, 7 Sep 2019 14:10:08 +0300 Subject: [PATCH 39/76] fix read and add from_bytes --- src/entry.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/entry.rs b/src/entry.rs index 485b36486..59663fcdb 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -42,7 +42,7 @@ impl Entry { } } - pub fn read(&self, consensus_branch_id: u32, r: &mut R) -> std::io::Result { + pub fn read(consensus_branch_id: u32, r: &mut R) -> std::io::Result { let kind = { match r.read_u8()? { 0 => { @@ -66,6 +66,11 @@ impl Entry { data, }) } + + pub fn from_bytes(consensus_branch_id: u32, buf: &[u8]) -> std::io::Result { + let mut cursor = std::io::Cursor::new(buf); + Self::read(consensus_branch_id, &mut cursor) + } } impl From for Entry { From a9d2ce71505bd50b98ebb6278b9ecdd7e5420f56 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sat, 7 Sep 2019 18:15:18 +0300 Subject: [PATCH 40/76] from_bytes for NodeData --- src/node_data.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/node_data.rs b/src/node_data.rs index 59591e532..fd01c3a9a 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -153,4 +153,9 @@ impl NodeData { buf[0..pos].to_vec() } + + pub fn from_bytes(consensus_branch_id: u32, buf: &[u8]) -> std::io::Result { + let mut cursor = std::io::Cursor::new(buf); + Self::read(consensus_branch_id, &mut cursor) + } } \ No newline at end of file From ad403f1cca1c6cd5a6f1ddfce4f3bdc89a290fd8 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sat, 7 Sep 2019 18:22:37 +0300 Subject: [PATCH 41/76] add data reader --- src/tree.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tree.rs b/src/tree.rs index d5210e54b..c4ab9826a 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -287,6 +287,9 @@ impl<'a> IndexedNode<'a> { self.node } + pub fn data(&self) -> &NodeData { + &self.node.data + } } fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> Entry { From 0bfd1d6b0d1aaaad4c5f4c3e97243cd9f5495bdc Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sat, 7 Sep 2019 18:30:57 +0300 Subject: [PATCH 42/76] resolve_link is of course public --- src/tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tree.rs b/src/tree.rs index c4ab9826a..36c3e4ec5 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -24,7 +24,7 @@ pub struct Tree { } impl Tree { - fn resolve_link(&self, link: EntryLink) -> Result { + pub fn resolve_link(&self, link: EntryLink) -> Result { match link { EntryLink::Generated(index) => { let node = self.generated.get(&index).ok_or(Error::ExpectedInMemory(link))?; From 0afa12297043e3abab6a81fad988603153105db3 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sat, 7 Sep 2019 19:18:49 +0300 Subject: [PATCH 43/76] add .travis.yml --- src/.travis.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/.travis.yml diff --git a/src/.travis.yml b/src/.travis.yml new file mode 100644 index 000000000..00e7694a3 --- /dev/null +++ b/src/.travis.yml @@ -0,0 +1,4 @@ +language: rust +rust: +- nightly +- stable \ No newline at end of file From 9470610b754accee4de274acfda37b2a6d07ec71 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sat, 7 Sep 2019 19:23:13 +0300 Subject: [PATCH 44/76] license and readme --- LICENSE-APACHE | 201 +++++++++++++++++++++++++++++++++++++++++++++++++ LICENSE-MIT | 25 ++++++ README.md | 19 ++++- 3 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 000000000..16fe87b06 --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 000000000..5ee6ad60c --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2019 Nikolay Volf + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index 6eefe94d2..db76689f0 100644 --- a/README.md +++ b/README.md @@ -1 +1,18 @@ -# zcash-mmr \ No newline at end of file +# zcash-mmr + +Special implementation of merkle mountain ranges (MMR) for ZCash! + +[![Build Status](https://travis-ci.org/NikVolf/zcash-mmr.svg?branch=master)](https://travis-ci.org/NikVolf/zcash-mmr) + +# License + +`zcash-mmr` is primarily distributed under the terms of both the MIT +license and the Apache License (Version 2.0), at your choice. + +See LICENSE-APACHE, and LICENSE-MIT for details. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in `zcash-mmr` by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. From c87122561f5ca9fa25a2ab634ffde3dca2c95ebc Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sun, 8 Sep 2019 00:17:40 +0300 Subject: [PATCH 45/76] add .travis.yml --- src/.travis.yml => .travis.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/.travis.yml => .travis.yml (100%) diff --git a/src/.travis.yml b/.travis.yml similarity index 100% rename from src/.travis.yml rename to .travis.yml From a5c4d51652a08876aec2a69a1b420f22f8109879 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Sun, 8 Sep 2019 09:32:47 +0300 Subject: [PATCH 46/76] improve on api --- src/entry.rs | 2 +- src/node_data.rs | 2 +- src/tree.rs | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/entry.rs b/src/entry.rs index 59663fcdb..982b879fd 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -67,7 +67,7 @@ impl Entry { }) } - pub fn from_bytes(consensus_branch_id: u32, buf: &[u8]) -> std::io::Result { + pub fn from_bytes>(consensus_branch_id: u32, buf: T) -> std::io::Result { let mut cursor = std::io::Cursor::new(buf); Self::read(consensus_branch_id, &mut cursor) } diff --git a/src/node_data.rs b/src/node_data.rs index fd01c3a9a..dcc8ded09 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -154,7 +154,7 @@ impl NodeData { buf[0..pos].to_vec() } - pub fn from_bytes(consensus_branch_id: u32, buf: &[u8]) -> std::io::Result { + pub fn from_bytes>(consensus_branch_id: u32, buf: T) -> std::io::Result { let mut cursor = std::io::Cursor::new(buf); Self::read(consensus_branch_id, &mut cursor) } diff --git a/src/tree.rs b/src/tree.rs index 36c3e4ec5..b816287c4 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -267,7 +267,7 @@ impl Tree { } } - +#[derive(Debug)] pub struct IndexedNode<'a> { node: &'a Entry, link: EntryLink, @@ -290,6 +290,10 @@ impl<'a> IndexedNode<'a> { pub fn data(&self) -> &NodeData { &self.node.data } + + pub fn link(&self) -> EntryLink { + self.link + } } fn combine_nodes<'a>(left: IndexedNode<'a>, right: IndexedNode<'a>) -> Entry { From b73f49c96835b455b3d8203d1c70458fe99e078a Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 9 Sep 2019 11:50:41 +0300 Subject: [PATCH 47/76] add optional file generation --- examples/producer.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/examples/producer.rs b/examples/producer.rs index 7d93cba8d..1bc5824e4 100644 --- a/examples/producer.rs +++ b/examples/producer.rs @@ -1,8 +1,7 @@ -extern crate zcash_mmr as mmr; +use zcash_mmr::{NodeData, Tree, EntryLink, Entry}; +use std::io::Write; -use mmr::{NodeData, Tree, EntryLink, Entry}; - -fn prepare_tree(vec: Vec) -> Tree { +fn prepare_tree(vec: &Vec) -> Tree { assert!(vec.len() > 0); @@ -202,6 +201,22 @@ fn main() { }, ); - let tree = prepare_tree(initial_tree_vec); + let tree = prepare_tree(&initial_tree_vec); + + let mut buf = Vec::new(); + if let Some(out_file_path) = ::std::env::args().nth(1) { + for node in initial_tree_vec.into_iter() { + node.write(&mut buf); + } + + let mut file = std::fs::File::create(&out_file_path) + .expect("Failed to create output file"); + + file.write_all(&buf[..]) + .expect("Failed to write data to file"); + } + println!("root: {}", tree.root()); + + } \ No newline at end of file From 636f3e3751a4d95a2c2d4ceaa3e985f3d9ed1333 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 9 Sep 2019 11:52:57 +0300 Subject: [PATCH 48/76] update readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index db76689f0..6f753461e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,14 @@ Special implementation of merkle mountain ranges (MMR) for ZCash! [![Build Status](https://travis-ci.org/NikVolf/zcash-mmr.svg?branch=master)](https://travis-ci.org/NikVolf/zcash-mmr) +The main design goals of this mmr implementation were + +- Allow zero-cache and avoid db callbacks. As it is implemented, calling side must just smartly pre-load MMR nodes from the database (about log2(tree length) for append, twice as much for deletion). + +- Reuse as much logic between rust and c++ clients and place it here and librustzcash. + +- Close to zero memory consumption. + # License `zcash-mmr` is primarily distributed under the terms of both the MIT From 96b130e034cc277586f27879a17a8d974f9f6193 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 9 Sep 2019 14:06:05 +0300 Subject: [PATCH 49/76] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f753461e..26532bb9c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Special implementation of merkle mountain ranges (MMR) for ZCash! [![Build Status](https://travis-ci.org/NikVolf/zcash-mmr.svg?branch=master)](https://travis-ci.org/NikVolf/zcash-mmr) -The main design goals of this mmr implementation were +The main design goals of this mmr implementation are - Allow zero-cache and avoid db callbacks. As it is implemented, calling side must just smartly pre-load MMR nodes from the database (about log2(tree length) for append, twice as much for deletion). From a0c33945ab3b534eb47ab0f000bbf15972b24b06 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 9 Sep 2019 14:33:49 +0300 Subject: [PATCH 50/76] write for entry --- examples/producer.rs | 2 +- src/entry.rs | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/examples/producer.rs b/examples/producer.rs index 1bc5824e4..108bc2517 100644 --- a/examples/producer.rs +++ b/examples/producer.rs @@ -206,7 +206,7 @@ fn main() { let mut buf = Vec::new(); if let Some(out_file_path) = ::std::env::args().nth(1) { for node in initial_tree_vec.into_iter() { - node.write(&mut buf); + node.write(&mut buf).expect("Failed to write data"); } let mut file = std::fs::File::create(&out_file_path) diff --git a/src/entry.rs b/src/entry.rs index 982b879fd..72582c240 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -1,4 +1,4 @@ -use byteorder::{LittleEndian, ReadBytesExt}; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use crate::{EntryKind, NodeData, Error, EntryLink, MAX_NODE_DATA_SIZE}; @@ -67,6 +67,24 @@ impl Entry { }) } + pub fn write(&self, w: &mut W) -> std::io::Result<()> { + match self.kind { + EntryKind::Node(EntryLink::Stored(left), EntryLink::Stored(right)) => { + w.write_u8(0)?; + w.write_u32::(left)?; + w.write_u32::(right)?; + }, + EntryKind::Leaf => { + w.write_u8(1)?; + }, + _ => { return Err(std::io::Error::from(std::io::ErrorKind::InvalidData)); } + } + + self.data.write(w)?; + + Ok(()) + } + pub fn from_bytes>(consensus_branch_id: u32, buf: T) -> std::io::Result { let mut cursor = std::io::Cursor::new(buf); Self::read(consensus_branch_id, &mut cursor) From f5c1381c4c644f76a298534f74e200a0b86a080d Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 9 Sep 2019 18:23:00 +0300 Subject: [PATCH 51/76] arrange data generator example properly --- examples/producer.rs | 222 ------------------------------------- examples/producer/main.rs | 4 + examples/producer/share.rs | 75 +++++++++++++ examples/write.rs | 41 +++++++ 4 files changed, 120 insertions(+), 222 deletions(-) delete mode 100644 examples/producer.rs create mode 100644 examples/producer/main.rs create mode 100644 examples/producer/share.rs create mode 100644 examples/write.rs diff --git a/examples/producer.rs b/examples/producer.rs deleted file mode 100644 index 108bc2517..000000000 --- a/examples/producer.rs +++ /dev/null @@ -1,222 +0,0 @@ -use zcash_mmr::{NodeData, Tree, EntryLink, Entry}; -use std::io::Write; - -fn prepare_tree(vec: &Vec) -> Tree { - - assert!(vec.len() > 0); - - // integer log2 of (vec.len()+1), -1 - let mut h = (32 - ((vec.len()+1) as u32).leading_zeros() - 1)-1; - let mut peak_pos = (1 << (h+1)) - 1; - let mut nodes = Vec::new(); - - loop { - - if peak_pos > vec.len() { - // left child, -2^h - peak_pos = peak_pos - (1<, + tree: Tree, + cursor: usize, + leaf_cursor: usize, +} + +impl Iterator for NodeDataIterator { + type Item = NodeData; + + fn next(&mut self) -> Option { + let result = if self.cursor == 1 { + self.leaf_cursor = 2; + Some(leaf(1)) + } else if self.cursor == 2 { + self.leaf_cursor = 3; + Some(leaf(2)) + } else if self.cursor == 3 { + Some(self.tree.root_node().expect("always exists").data().clone()) + } else if self.return_stack.len() > 0 { + self.return_stack.pop() + } else { + for n_append in + self.tree.append_leaf(leaf(self.leaf_cursor as u32)) + .expect("full tree cannot fail").into_iter().rev() + { + self.return_stack.push(self.tree.resolve_link(n_append).expect("just pushed").data().clone()) + } + self.leaf_cursor += 1; + self.return_stack.pop() + }; + + self.cursor += 1; + result + } +} + +impl NodeDataIterator { + pub fn new() -> Self { + let mut root: Entry = NodeData::combine(&leaf(1), &leaf(2)).into(); + root.update_siblings(EntryLink::Stored(0), EntryLink::Stored(1)); + let tree = + Tree::new( + 3, + vec![(2, root)], + vec![(0, leaf(1).into()), (1, leaf(2).into())] + ); + + NodeDataIterator { + return_stack: Vec::new(), + tree, + cursor: 1, + leaf_cursor: 1, + } + } +} + +fn leaf(height: u32) -> NodeData { + NodeData { + consensus_branch_id: 0, + subtree_commitment: [0u8; 32], + start_time: height*10+1, + end_time: (height+1)*10, + start_target: 100 + height*10, + end_target: 100 + (height+1)*10, + start_sapling_root: [0u8; 32], + end_sapling_root: [0u8; 32], + subtree_total_work: 0.into(), + start_height: height as u64, + end_height: height as u64, + shielded_tx: 5 + height as u64, + } +} diff --git a/examples/write.rs b/examples/write.rs new file mode 100644 index 000000000..f51e2a7bf --- /dev/null +++ b/examples/write.rs @@ -0,0 +1,41 @@ +#[path= "producer/share.rs"] +mod share; + +// Test data generator +// $ cargo run --example writer -- 16 nodes.dat +// or +// $ cargo run --example writer -- 16 +// to preview + +fn main() { + let mut args = std::env::args().skip(1); + + let (number, out_file) = match args.next() { + None => { eprintln!("writer []"); std::process::exit(1); }, + Some(number) => { + (number.parse::().expect("invalid number"), args.next()) + } + }; + + let iterator = share::NodeDataIterator::new().take(number); + + if let Some(out_file_path) = out_file { + use std::io::Write; + + let mut buf = Vec::new(); + + for node in iterator{ + node.write(&mut buf).expect("Failed to write data"); + } + + let mut file = std::fs::File::create(&out_file_path) + .expect("Failed to create output file"); + + file.write_all(&buf[..]) + .expect("Failed to write data to file"); + } else { + for n in iterator { + println!("{:?}", n); + } + } +} \ No newline at end of file From 4ec651d172900a112500e9359964bb6a50fc2a5e Mon Sep 17 00:00:00 2001 From: NikVolf Date: Mon, 9 Sep 2019 18:28:23 +0300 Subject: [PATCH 52/76] update naming --- examples/{producer => lib}/main.rs | 0 examples/{producer/share.rs => lib/shared.rs} | 0 examples/write.rs | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename examples/{producer => lib}/main.rs (100%) rename examples/{producer/share.rs => lib/shared.rs} (100%) diff --git a/examples/producer/main.rs b/examples/lib/main.rs similarity index 100% rename from examples/producer/main.rs rename to examples/lib/main.rs diff --git a/examples/producer/share.rs b/examples/lib/shared.rs similarity index 100% rename from examples/producer/share.rs rename to examples/lib/shared.rs diff --git a/examples/write.rs b/examples/write.rs index f51e2a7bf..c029a85ce 100644 --- a/examples/write.rs +++ b/examples/write.rs @@ -1,4 +1,4 @@ -#[path= "producer/share.rs"] +#[path= "lib/shared.rs"] mod share; // Test data generator From 79cba2e500b29f77cacfe0fcc68402d221f2330d Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 10 Sep 2019 12:51:23 +0300 Subject: [PATCH 53/76] run against long examples --- examples/long.rs | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 examples/long.rs diff --git a/examples/long.rs b/examples/long.rs new file mode 100644 index 000000000..bb2f2d481 --- /dev/null +++ b/examples/long.rs @@ -0,0 +1,73 @@ +use zcash_mmr::{Entry, EntryLink, NodeData, Tree}; + +#[path= "lib/shared.rs"] +mod share; + +fn prepare_tree(vec: &Vec) -> Tree { + + assert!(vec.len() > 0); + + // integer log2 of (vec.len()+1), -1 + let mut h = (32 - ((vec.len()+1) as u32).leading_zeros() - 1)-1; + let mut peak_pos = (1 << (h+1)) - 1; + let mut nodes = Vec::new(); + + loop { + + if peak_pos > vec.len() { + // left child, -2^h + peak_pos = peak_pos - (1< { eprintln!("writer []"); std::process::exit(1); }, + Some(number) => { + number.parse::().expect("invalid number") + } + }; + + let long_vec = share::NodeDataIterator::new().take(number) + .collect::>(); + + let now = std::time::Instant::now(); + + let tree = prepare_tree(&long_vec); + + println!("Tree final root: {}-{}", + tree.root_node().expect("root").data().start_height, + tree.root_node().expect("root").data().end_height, + ); + + println!("Prepare tree of {} length: {} ns / {} mcs / {} ms", + number, + now.elapsed().as_nanos(), now.elapsed().as_micros(), now.elapsed().as_millis() + ); +} From 4cbc0451c1f4e713886df833466923931c802f8e Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 24 Sep 2019 10:34:27 +0200 Subject: [PATCH 54/76] switch to blake2_simd --- Cargo.toml | 2 +- src/node_data.rs | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 637341706..ff2a2f8db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,4 +12,4 @@ quickcheck = "0.8" derive_more = "0.15" bigint = "4" byteorder = "1" -blake2 = { package = "blake2-rfc", git = "https://github.com/gtank/blake2-rfc.git", rev = "7a5b5fc99ae483a0043db7547fb79a6fa44b88a9" } +blake2 = { package = "blake2b_simd", version = "0.5" } diff --git a/src/node_data.rs b/src/node_data.rs index dcc8ded09..0adc0f2f6 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -1,6 +1,6 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt, ByteOrder}; use bigint::U256; -use blake2::blake2b::Blake2b; +use blake2::Params as Blake2Params; pub const MAX_NODE_DATA_SIZE: usize = 32 + 4 + 4 + 4 + 4 + 32 + 32 + 32 + 9 + 9 + 9; // 171 @@ -23,10 +23,14 @@ pub struct NodeData { } fn blake2b_personal(personalization: &[u8], input: &[u8]) -> [u8; 32] { - let mut hasher = Blake2b::with_params(32, &[], &[], personalization); - hasher.update(input); + let hash_result = Blake2Params::new() + .hash_length(32) + .personal(personalization) + .to_state() + .update(input) + .finalize(); let mut result = [0u8; 32]; - result.copy_from_slice(hasher.finalize().as_bytes()); + result.copy_from_slice(hash_result.as_bytes()); result } From 38aa09b6a1d3d017b14d070f18d8567a8e5d3b19 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 24 Sep 2019 10:35:44 +0200 Subject: [PATCH 55/76] change package name --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ff2a2f8db..ce1763818 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "zcash-mmr" +name = "zcash_mmr" version = "0.1.0" authors = ["NikVolf "] edition = "2018" From fb6fef4e9ce94c5d7d0adabb3e84cdeb4dea221a Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 24 Sep 2019 10:36:28 +0200 Subject: [PATCH 56/76] update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 26532bb9c..088208c58 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# zcash-mmr +# zcash_mmr Special implementation of merkle mountain ranges (MMR) for ZCash! @@ -14,7 +14,7 @@ The main design goals of this mmr implementation are # License -`zcash-mmr` is primarily distributed under the terms of both the MIT +`zcash_mmr` is primarily distributed under the terms of both the MIT license and the Apache License (Version 2.0), at your choice. See LICENSE-APACHE, and LICENSE-MIT for details. @@ -22,5 +22,5 @@ See LICENSE-APACHE, and LICENSE-MIT for details. ### Contribution Unless you explicitly state otherwise, any contribution intentionally submitted -for inclusion in `zcash-mmr` by you, as defined in the Apache-2.0 license, shall be +for inclusion in `zcash_mmr` by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. From ce2416623fd95f1307fb5f1058431b3a51d45e30 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 24 Sep 2019 12:19:25 +0200 Subject: [PATCH 57/76] get rid of derive_more --- Cargo.toml | 1 - src/lib.rs | 28 +++++++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ce1763818..d527675e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,6 @@ assert_matches = "1.3.0" quickcheck = "0.8" [dependencies] -derive_more = "0.15" bigint = "4" byteorder = "1" blake2 = { package = "blake2b_simd", version = "0.5" } diff --git a/src/lib.rs b/src/lib.rs index f0eb3f466..da0cc5d32 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,28 +10,42 @@ pub use tree::Tree; pub use node_data::{NodeData, MAX_NODE_DATA_SIZE}; pub use entry::{Entry, MAX_ENTRY_SIZE}; -#[derive(Debug, derive_more::Display)] +#[derive(Debug)] pub enum Error { - #[display(fmt="Node/leaf expected to be in memory: {}", _0)] ExpectedInMemory(EntryLink), - #[display(fmt="Node expected")] ExpectedNode, - #[display(fmt="Node expected, not leaf: {}", _0)] ExpectedNodeForLink(EntryLink), } +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + Self::ExpectedInMemory(l) => write!(f, "Node/leaf expected to be in memory: {}", l), + Self::ExpectedNode => write!(f, "Node expected"), + Self::ExpectedNodeForLink(l) => write!(f, "Node expected, not leaf: {}", l), + } + } +} + /// Reference to to the tree node. #[repr(C)] -#[derive(Clone, Copy, Debug, derive_more::Display)] +#[derive(Clone, Copy, Debug)] pub enum EntryLink { /// Reference to the stored (in the array representation) leaf/node. - #[display(fmt="stored(@{})", _0)] Stored(u32), /// Reference to the generated leaf/node. - #[display(fmt="generated(@{})", _0)] Generated(u32), } +impl std::fmt::Display for EntryLink { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + Self::Stored(v) => write!(f, "stored({})", v), + Self::Generated(v) => write!(f, "generated({})", v), + } + } +} + /// MMR Node. It is leaf when `left`, `right` are `None` and node when they are not. #[repr(C)] #[derive(Debug)] From 7879b6332132108bdda7476a574eb5ef9cc5f9a3 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 24 Sep 2019 12:45:23 +0200 Subject: [PATCH 58/76] add docs --- src/entry.rs | 11 +++++++++++ src/lib.rs | 8 ++++++++ src/node_data.rs | 18 ++++++++++++++++++ src/tree.rs | 11 +++++++++++ 4 files changed, 48 insertions(+) diff --git a/src/entry.rs b/src/entry.rs index 72582c240..04490b3c5 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -2,8 +2,10 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use crate::{EntryKind, NodeData, Error, EntryLink, MAX_NODE_DATA_SIZE}; +/// Max serialized length of entry data. pub const MAX_ENTRY_SIZE: usize = MAX_NODE_DATA_SIZE + 9; +/// MMR Entry. #[derive(Debug)] pub struct Entry { pub(crate) kind: EntryKind, @@ -11,23 +13,28 @@ pub struct Entry { } impl Entry { + /// Update siblings of the entry (to promote it from leaf to node) pub fn update_siblings(&mut self, left: EntryLink, right: EntryLink) { self.kind = EntryKind::Node(left, right); } + /// Returns if is this node complete (has total of 2^N leaves) pub fn complete(&self) -> bool { let leaves = self.leaf_count(); leaves & (leaves - 1) == 0 } + /// Number of leaves under this node. pub fn leaf_count(&self) -> u64 { self.data.end_height - self.data.start_height + 1 } + /// Is this node a leaf. pub fn is_leaf(&self) -> bool { if let EntryKind::Leaf = self.kind { true } else { false } } + /// Left child pub fn left(&self) -> Result { match self.kind { EntryKind::Leaf => { Err(Error::ExpectedNode) } @@ -35,6 +42,7 @@ impl Entry { } } + /// Right child. pub fn right(&self) -> Result { match self.kind { EntryKind::Leaf => { Err(Error::ExpectedNode) } @@ -42,6 +50,7 @@ impl Entry { } } + /// Read from byte representation. pub fn read(consensus_branch_id: u32, r: &mut R) -> std::io::Result { let kind = { match r.read_u8()? { @@ -67,6 +76,7 @@ impl Entry { }) } + /// Write to byte representation. pub fn write(&self, w: &mut W) -> std::io::Result<()> { match self.kind { EntryKind::Node(EntryLink::Stored(left), EntryLink::Stored(right)) => { @@ -85,6 +95,7 @@ impl Entry { Ok(()) } + /// Convert from byte representation. pub fn from_bytes>(consensus_branch_id: u32, buf: T) -> std::io::Result { let mut cursor = std::io::Cursor::new(buf); Self::read(consensus_branch_id, &mut cursor) diff --git a/src/lib.rs b/src/lib.rs index da0cc5d32..c0c781cc6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,19 +1,25 @@ //! MMR library for Zcash //! //! To be used in zebra and via FFI bindings in zcashd +#![warn(missing_docs)] mod tree; mod node_data; mod entry; + pub use tree::Tree; pub use node_data::{NodeData, MAX_NODE_DATA_SIZE}; pub use entry::{Entry, MAX_ENTRY_SIZE}; +/// Crate-level error type #[derive(Debug)] pub enum Error { + /// Entry expected to be presented in the tree view while it was not. ExpectedInMemory(EntryLink), + /// Entry expected to be a node. ExpectedNode, + /// Entry expected to be a node (specifying for which link this is not true). ExpectedNodeForLink(EntryLink), } @@ -50,7 +56,9 @@ impl std::fmt::Display for EntryLink { #[repr(C)] #[derive(Debug)] pub enum EntryKind { + /// Leaf entry. Leaf, + /// Node entry with children links. Node(EntryLink, EntryLink), } diff --git a/src/node_data.rs b/src/node_data.rs index 0adc0f2f6..a018daa88 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -2,23 +2,36 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt, ByteOrder}; use bigint::U256; use blake2::Params as Blake2Params; +/// Maximum serialized size of the node metadata. pub const MAX_NODE_DATA_SIZE: usize = 32 + 4 + 4 + 4 + 4 + 32 + 32 + 32 + 9 + 9 + 9; // 171 /// Node metadata. #[repr(C)] #[derive(Debug, Clone, Default)] pub struct NodeData { + /// Consensus branch id, should be provided by deserializing node. pub consensus_branch_id: u32, + /// Subtree commitment - either block hash for leaves or hashsum of children for nodes. pub subtree_commitment: [u8; 32], + /// Start time. pub start_time: u32, + /// End time. pub end_time: u32, + /// Start target. pub start_target: u32, + /// End target. pub end_target: u32, + /// Start sapling tree root. pub start_sapling_root: [u8; 32], + /// End sapling tree root. pub end_sapling_root: [u8; 32], + /// Part of tree total work. pub subtree_total_work: U256, + /// Start height. pub start_height: u64, + /// End height pub end_height: u64, + /// Number of shielded transactions. pub shielded_tx: u64, } @@ -42,6 +55,7 @@ fn personalization(branch_id: u32) -> [u8; 16] { } impl NodeData { + /// Combine two nodes metadata. pub fn combine(left: &NodeData, right: &NodeData) -> NodeData { assert_eq!(left.consensus_branch_id, right.consensus_branch_id); @@ -106,6 +120,7 @@ impl NodeData { Ok(result) } + /// Write to the byte representation. pub fn write(&self, w: &mut W) -> std::io::Result<()> { w.write_all(&self.subtree_commitment)?; w.write_u32::(self.start_time)?; @@ -125,6 +140,7 @@ impl NodeData { Ok(()) } + /// Read from the byte representation. pub fn read(consensus_branch_id: u32, r: &mut R) -> std::io::Result { let mut data = Self::default(); data.consensus_branch_id = consensus_branch_id; @@ -147,6 +163,7 @@ impl NodeData { Ok(data) } + /// Convert to byte representation. pub fn to_bytes(&self) -> Vec { let mut buf = [0u8; MAX_NODE_DATA_SIZE]; let pos = { @@ -158,6 +175,7 @@ impl NodeData { buf[0..pos].to_vec() } + /// Convert from byte representation. pub fn from_bytes>(consensus_branch_id: u32, buf: T) -> std::io::Result { let mut cursor = std::io::Cursor::new(buf); Self::read(consensus_branch_id, &mut cursor) diff --git a/src/tree.rs b/src/tree.rs index b816287c4..8b65c5244 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -24,6 +24,7 @@ pub struct Tree { } impl Tree { + /// Resolve link originated from this tree pub fn resolve_link(&self, link: EntryLink) -> Result { match link { EntryLink::Generated(index) => { @@ -80,6 +81,12 @@ impl Tree { } } + /// New view into the the tree array representation + /// + /// `length` is total length of the array representation + /// `peaks` is peaks of the mmr tree + /// `extra` is some extra nodes that calculated to be required during next one or more + /// operations on the tree. pub fn new( length: u32, peaks: Vec<(u32, Entry)>, @@ -267,6 +274,7 @@ impl Tree { } } +/// Reference to the node with link attached. #[derive(Debug)] pub struct IndexedNode<'a> { node: &'a Entry, @@ -283,14 +291,17 @@ impl<'a> IndexedNode<'a> { self.node.right().map_err(|e| e.augment(self.link)) } + /// Reference to the entry struct. pub fn node(&self) -> &Entry { self.node } + /// Reference to the entry metadata. pub fn data(&self) -> &NodeData { &self.node.data } + /// Actual link by what this node was resolved. pub fn link(&self) -> EntryLink { self.link } From 1eb4fb91f6ce68bf54e408f4a2737faa61473a0e Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 24 Sep 2019 13:15:35 +0200 Subject: [PATCH 59/76] add invariant about number of returned links --- src/tree.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tree.rs b/src/tree.rs index 8b65c5244..4482cd808 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -142,6 +142,8 @@ impl Tree { /// Append one leaf to the tree. /// /// Returns links to actual nodes that has to be persisted as the result of the append. + /// If completed without error, at least one link to the appended + /// node (with metadata provided in `new_leaf`) will be returned. pub fn append_leaf(&mut self, new_leaf: NodeData) -> Result, Error> { let root = self.root; let new_leaf_link = self.push(new_leaf.into()); From 08806cc1097967e5b3a5f0f4f351e7ed4b579efb Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 24 Sep 2019 13:47:32 +0200 Subject: [PATCH 60/76] proper elapsed time calc --- examples/long.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/long.rs b/examples/long.rs index bb2f2d481..45e66c2cf 100644 --- a/examples/long.rs +++ b/examples/long.rs @@ -60,6 +60,7 @@ fn main() { let now = std::time::Instant::now(); let tree = prepare_tree(&long_vec); + let elapsed = now.elapsed(); println!("Tree final root: {}-{}", tree.root_node().expect("root").data().start_height, @@ -68,6 +69,6 @@ fn main() { println!("Prepare tree of {} length: {} ns / {} mcs / {} ms", number, - now.elapsed().as_nanos(), now.elapsed().as_micros(), now.elapsed().as_millis() + elapsed.as_nanos(), elapsed.as_micros(), elapsed.as_millis() ); } From 43efdf992b3a135b8016122db6500d54377573de Mon Sep 17 00:00:00 2001 From: NikVolf Date: Tue, 24 Sep 2019 15:28:34 +0200 Subject: [PATCH 61/76] prepare tree for deleting also --- examples/long.rs | 60 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/examples/long.rs b/examples/long.rs index 45e66c2cf..85f2c13ce 100644 --- a/examples/long.rs +++ b/examples/long.rs @@ -12,6 +12,10 @@ fn prepare_tree(vec: &Vec) -> Tree { let mut peak_pos = (1 << (h+1)) - 1; let mut nodes = Vec::new(); + // used later + let mut last_peak_pos = 0; + let mut last_peak_h = 0; + loop { if peak_pos > vec.len() { @@ -22,17 +26,22 @@ fn prepare_tree(vec: &Vec) -> Tree { if peak_pos <= vec.len() { let mut peak: Entry = vec[peak_pos-1].clone().into(); - let left_idx = (peak_pos - (1<) -> Tree { } } - Tree::new(vec.len() as u32, nodes, vec![]) + // for deletion, everything on the right slope of the last peak should be pre-loaded + let mut extra = Vec::new(); + let mut h = last_peak_h; + let mut peak_pos = last_peak_pos; + + while h > 0 { + let left_pos = peak_pos - (1< Date: Wed, 25 Sep 2019 09:50:59 +0200 Subject: [PATCH 62/76] dry example a bit and reduce api --- examples/long.rs | 55 +++++++++++++++++------------------------------- src/entry.rs | 18 +++++++++++++--- 2 files changed, 34 insertions(+), 39 deletions(-) diff --git a/examples/long.rs b/examples/long.rs index 85f2c13ce..951d2262e 100644 --- a/examples/long.rs +++ b/examples/long.rs @@ -3,6 +3,22 @@ use zcash_mmr::{Entry, EntryLink, NodeData, Tree}; #[path= "lib/shared.rs"] mod share; +fn draft(into: &mut Vec<(u32, Entry)>, vec: &Vec, peak_pos: usize, h: u32) { + let node_data = vec[peak_pos-1].clone(); + let peak: Entry = match h { + 0 => node_data.into(), + _ => Entry::new( + node_data, + EntryLink::Stored((peak_pos - (1 << h) - 1) as u32), + EntryLink::Stored((peak_pos - 2) as u32), + ), + }; + + println!("Entry #{}: {}", into.len(), peak); + + into.push(((peak_pos-1) as u32, peak)); +} + fn prepare_tree(vec: &Vec) -> Tree { assert!(vec.len() > 0); @@ -25,19 +41,7 @@ fn prepare_tree(vec: &Vec) -> Tree { } if peak_pos <= vec.len() { - let mut peak: Entry = vec[peak_pos-1].clone().into(); - if h != 0 { - let left_idx = (peak_pos - (1<) -> Tree { h = h - 1; // drafting left child - let mut peak: Entry = vec[left_pos-1].clone().into(); - if h != 0 { - let left_idx = (left_pos - (1<) -> Tree { println!("Total extra of {} required for deletion!", extra.len()); - Tree::new(vec.len() as u32, nodes, extra) } diff --git a/src/entry.rs b/src/entry.rs index 04490b3c5..b20855523 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -13,9 +13,12 @@ pub struct Entry { } impl Entry { - /// Update siblings of the entry (to promote it from leaf to node) - pub fn update_siblings(&mut self, left: EntryLink, right: EntryLink) { - self.kind = EntryKind::Node(left, right); + /// New entry of type node. + pub fn new(data: NodeData, left: EntryLink, right: EntryLink) -> Self { + Entry { + kind: EntryKind::Node(left, right), + data, + } } /// Returns if is this node complete (has total of 2^N leaves) @@ -107,3 +110,12 @@ impl From for Entry { Entry { kind: EntryKind::Leaf, data: s } } } + +impl std::fmt::Display for Entry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.kind { + EntryKind::Node(l, r) => write!(f, "node({}, {}, ..)", l, r), + EntryKind::Leaf => write!(f, "leaf(..)"), + } + } +} From da0d0a669a049109ee96354072ebc86a116bce25 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Wed, 25 Sep 2019 09:53:06 +0200 Subject: [PATCH 63/76] more idiomatic naming --- examples/lib/shared.rs | 7 +++++-- src/entry.rs | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/lib/shared.rs b/examples/lib/shared.rs index 7ead49ddf..4cd926212 100644 --- a/examples/lib/shared.rs +++ b/examples/lib/shared.rs @@ -39,8 +39,11 @@ impl Iterator for NodeDataIterator { impl NodeDataIterator { pub fn new() -> Self { - let mut root: Entry = NodeData::combine(&leaf(1), &leaf(2)).into(); - root.update_siblings(EntryLink::Stored(0), EntryLink::Stored(1)); + let root = Entry::new( + NodeData::combine(&leaf(1), &leaf(2)), + EntryLink::Stored(0), + EntryLink::Stored(1) + ); let tree = Tree::new( 3, diff --git a/src/entry.rs b/src/entry.rs index b20855523..a3f14a8bb 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -33,7 +33,7 @@ impl Entry { } /// Is this node a leaf. - pub fn is_leaf(&self) -> bool { + pub fn leaf(&self) -> bool { if let EntryKind::Leaf = self.kind { true } else { false } } From 84dc3bf73c71a499245a8a360ad1ac188576f7e2 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 07:58:29 +0300 Subject: [PATCH 64/76] license notice --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 088208c58..defdafe26 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The main design goals of this mmr implementation are # License -`zcash_mmr` is primarily distributed under the terms of both the MIT +`zcash_mmr` is distributed under the terms of both the MIT license and the Apache License (Version 2.0), at your choice. See LICENSE-APACHE, and LICENSE-MIT for details. From 9d4412103b80f25818f880bd9f9679d300fc8e50 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 08:00:07 +0300 Subject: [PATCH 65/76] more clear leaf_count --- src/entry.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/entry.rs b/src/entry.rs index a3f14a8bb..257b9ddb2 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -29,7 +29,7 @@ impl Entry { /// Number of leaves under this node. pub fn leaf_count(&self) -> u64 { - self.data.end_height - self.data.start_height + 1 + self.data.end_height - (self.data.start_height - 1) } /// Is this node a leaf. From cb818ecbe3798d35ba78b0ccb74b11abdd7d4c2b Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 08:04:42 +0300 Subject: [PATCH 66/76] serialization sizes notice --- src/node_data.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/node_data.rs b/src/node_data.rs index a018daa88..5d7535bbd 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -3,7 +3,19 @@ use bigint::U256; use blake2::Params as Blake2Params; /// Maximum serialized size of the node metadata. -pub const MAX_NODE_DATA_SIZE: usize = 32 + 4 + 4 + 4 + 4 + 32 + 32 + 32 + 9 + 9 + 9; // 171 +pub const MAX_NODE_DATA_SIZE: usize = + 32 + // subtree commitment + 4 + // start time + 4 + // end time + 4 + // start target + 4 + // end target + 32 + // start sapling tree root + 32 + // end sapling tree root + 32 + // subtree total work + 9 + // start height (compact uint) + 9 + // end height (compact uint) + 9; // shielded tx count (compact uint) + // = total of 171 /// Node metadata. #[repr(C)] From 29b8d7a7563753e958e54a996d7393eb52276583 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 08:18:36 +0300 Subject: [PATCH 67/76] serialization roundtrip test --- src/node_data.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/node_data.rs b/src/node_data.rs index 5d7535bbd..bbaa98c07 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -20,6 +20,7 @@ pub const MAX_NODE_DATA_SIZE: usize = /// Node metadata. #[repr(C)] #[derive(Debug, Clone, Default)] +#[cfg_attr(test, derive(PartialEq))] pub struct NodeData { /// Consensus branch id, should be provided by deserializing node. pub consensus_branch_id: u32, @@ -192,4 +193,40 @@ impl NodeData { let mut cursor = std::io::Cursor::new(buf); Self::read(consensus_branch_id, &mut cursor) } +} + +#[cfg(test)] +impl quickcheck::Arbitrary for NodeData { + fn arbitrary(gen: &mut G) -> Self { + let mut node_data = NodeData::default(); + node_data.consensus_branch_id = 0; + gen.fill_bytes(&mut node_data.subtree_commitment[..]); + node_data.start_time = gen.next_u32(); + node_data.end_time = gen.next_u32(); + node_data.start_target = gen.next_u32(); + node_data.end_target = gen.next_u32(); + gen.fill_bytes(&mut node_data.start_sapling_root[..]); + gen.fill_bytes(&mut node_data.end_sapling_root[..]); + let mut number = [0u8; 32]; + gen.fill_bytes(&mut number[..]); + node_data.subtree_total_work = U256::from_little_endian(&number[..]); + node_data.start_height = gen.next_u64(); + node_data.end_height = gen.next_u64(); + node_data.shielded_tx = gen.next_u64(); + + node_data + } +} + +#[cfg(test)] +mod tests { + use super::NodeData; + use quickcheck::{quickcheck, TestResult}; + + + quickcheck! { + fn serialization_round_trip(node_data: NodeData) -> TestResult { + TestResult::from_bool(NodeData::from_bytes(0, &node_data.to_bytes()).unwrap() == node_data) + } + } } \ No newline at end of file From e701687b691ea920bc51c867e8a870ef34cb92bf Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 08:28:50 +0300 Subject: [PATCH 68/76] various small fixes --- README.md | 4 ++-- src/node_data.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index defdafe26..f034137ec 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # zcash_mmr -Special implementation of merkle mountain ranges (MMR) for ZCash! +Special implementation of Merkle mountain ranges (MMR) for Zcash! [![Build Status](https://travis-ci.org/NikVolf/zcash-mmr.svg?branch=master)](https://travis-ci.org/NikVolf/zcash-mmr) -The main design goals of this mmr implementation are +The main design goals of this MMR implementation are - Allow zero-cache and avoid db callbacks. As it is implemented, calling side must just smartly pre-load MMR nodes from the database (about log2(tree length) for append, twice as much for deletion). diff --git a/src/node_data.rs b/src/node_data.rs index bbaa98c07..ac545e24f 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -127,7 +127,7 @@ impl NodeData { i @ 0..=0xfc => i.into(), 0xfd => reader.read_u16::()?.into(), 0xfe => reader.read_u32::()?.into(), - _ => reader.read_u64::()?.into(), + _ => reader.read_u64::()?, }; Ok(result) From 6b7a3dec9ca47837e61a455b8cb560e8851b7ac6 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 08:44:20 +0300 Subject: [PATCH 69/76] store generated as vec --- src/tree.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index 4482cd808..d1bf93666 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -12,14 +12,11 @@ use crate::{Entry, EntryLink, NodeData, Error, EntryKind}; pub struct Tree { stored: HashMap, - generated: HashMap, + generated: Vec, // number of persistent(!) tree entries stored_count: u32, - // number of virtual nodes generated - generated_count: u32, - root: EntryLink, } @@ -28,7 +25,7 @@ impl Tree { pub fn resolve_link(&self, link: EntryLink) -> Result { match link { EntryLink::Generated(index) => { - let node = self.generated.get(&index).ok_or(Error::ExpectedInMemory(link))?; + let node = self.generated.get(index as usize).ok_or(Error::ExpectedInMemory(link))?; Ok(IndexedNode { node, link, @@ -52,14 +49,13 @@ impl Tree { } fn push_generated(&mut self, data: Entry) -> EntryLink { - let idx = self.generated_count; - self.generated_count = self.generated_count + 1; - self.generated.insert(idx, data); - EntryLink::Generated(idx) + self.generated.push(data); + EntryLink::Generated(self.generated.len() as u32 - 1) } - /// Populate tree with plain list of the leaves/nodes. Mostly for tests, - /// since this `Tree` structure is for partially loaded tree. + /// Populate tree with plain list of the leaves/nodes. For now, only for tests, + /// since this `Tree` structure is for partially loaded tree (but it might change) + #[cfg(test)] pub fn populate(loaded: Vec, root: EntryLink) -> Self { let mut result = Tree::invalid(); result.stored_count = loaded.len() as u32; @@ -71,12 +67,12 @@ impl Tree { result } + // Empty tree with invalid root fn invalid() -> Self { Tree { root: EntryLink::Generated(0), generated: Default::default(), stored: Default::default(), - generated_count: 0, stored_count: 0, } } From 3082593fccfed7c99fd7779eff6a115bcd48a386 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 08:50:44 +0300 Subject: [PATCH 70/76] add intented use of the api --- src/tree.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tree.rs b/src/tree.rs index d1bf93666..65e0220d6 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -8,10 +8,15 @@ use crate::{Entry, EntryLink, NodeData, Error, EntryKind}; /// With only some of the leaves/nodes pre-loaded / pre-generated. /// Exact amount of the loaded data can be calculated by the constructing party, /// depending on the length of the tree and maximum amount of operations that are going -/// to happen after construction. +/// to happen after construction. `Tree` should not be used as self-contained data structure, +/// since it's internal state can grow indefinitely after serial operations. +/// Intended use of this `Tree` is to instantiate it based on partially loaded data (see example +/// how to pick right nodes from the array representation of MMR Tree), perform several operations +/// (append-s/delete-s) and then drop it. pub struct Tree { stored: HashMap, + // This can grow indefinitely if `Tree` is misused as a self-contained data structure generated: Vec, // number of persistent(!) tree entries From b9bfc07146f2576d47e9c14bf1df0f2f540c1bc5 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 08:55:43 +0300 Subject: [PATCH 71/76] style fixes --- src/tree.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index 65e0220d6..1d3db06da 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -48,7 +48,7 @@ impl Tree { fn push(&mut self, data: Entry) -> EntryLink { let idx = self.stored_count; - self.stored_count = self.stored_count + 1; + self.stored_count += 1; self.stored.insert(idx, data); EntryLink::Stored(idx) } @@ -84,15 +84,22 @@ impl Tree { /// New view into the the tree array representation /// - /// `length` is total length of the array representation + /// `length` is total length of the array representation (is generally not a sum of + /// peaks.len + extra.len) /// `peaks` is peaks of the mmr tree /// `extra` is some extra nodes that calculated to be required during next one or more /// operations on the tree. + /// + /// # Panics + /// + /// Will panic if `peaks` is empty. pub fn new( length: u32, peaks: Vec<(u32, Entry)>, extra: Vec<(u32, Entry)>, ) -> Self { + assert!(peaks.len() > 0); + let mut result = Tree::invalid(); result.stored_count = length; @@ -157,6 +164,9 @@ impl Tree { let mut merge_stack = Vec::new(); merge_stack.push(new_leaf_link); + // Scan the peaks right-to-left, merging together equal-sized adjacent + // complete subtrees. After this, merge_stack only contains peaks of + // unequal-sized subtrees. while let Some(next_peak) = peaks.pop() { let next_merge = merge_stack.pop().expect("there should be at least one, initial or re-pushed"); @@ -171,12 +181,15 @@ impl Tree { merge_stack.push(link); appended.push(link); continue; + } else { + merge_stack.push(next_merge); + merge_stack.push(next_peak); } - merge_stack.push(next_merge); - merge_stack.push(next_peak); } let mut new_root = merge_stack.pop().expect("Loop above cannot reduce the merge_stack"); + // Scan the peaks left-to-right, producing new generated nodes that + // connect the subtrees while let Some(next_child) = merge_stack.pop() { new_root = self.push_generated( combine_nodes( @@ -210,7 +223,8 @@ impl Tree { /// Truncate one leaf from the end of the tree. /// - /// Returns actual number of nodes that has to be removed from the array representation. + /// Returns actual number of nodes that should be removed by the caller + /// from the end of the array representation. pub fn truncate_leaf(&mut self) -> Result { let root = { let (leaves, root_left_child) = { @@ -245,7 +259,7 @@ impl Tree { } } - let mut new_root = *peaks.iter().nth(0).expect("At lest 2 elements in peaks"); + let mut new_root = *peaks.get(0).expect("At lest 1 elements in peaks"); for next_peak in peaks.into_iter().skip(1) { new_root = self.push_generated( From f24ec04340d8bb816ae95939f14193a15959a09f Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 09:00:47 +0300 Subject: [PATCH 72/76] add is_empty --- src/tree.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tree.rs b/src/tree.rs index 1d3db06da..d1d1d6f13 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -289,6 +289,10 @@ impl Tree { pub fn root_node(&self) -> Result { self.resolve_link(self.root) } + + pub fn is_empty(&self) -> bool { + self.stored_count == 0 + } } /// Reference to the node with link attached. From 481e43689cd2d2aa8e0d856970d74384e06a8c2c Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 09:04:04 +0300 Subject: [PATCH 73/76] use NodeData::combine --- src/tree.rs | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index d1d1d6f13..fbadaba81 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -285,11 +285,12 @@ impl Tree { /// Link to the root node pub fn root(&self) -> EntryLink { self.root } - /// Reference to the root ndoe + /// Reference to the root node. pub fn root_node(&self) -> Result { self.resolve_link(self.root) } + /// If this tree is empty. pub fn is_empty(&self) -> bool { self.stored_count == 0 } @@ -359,29 +360,12 @@ mod tests { } } - fn node(start_height: u64, end_height: u64) -> NodeData { - NodeData { - consensus_branch_id: 1, - subtree_commitment: [0u8; 32], - start_time: 0, - end_time: 0, - start_target: 0, - end_target: 0, - start_sapling_root: [0u8; 32], - end_sapling_root: [0u8; 32], - subtree_total_work: 0.into(), - start_height: start_height, - end_height: end_height, - shielded_tx: 7, - } - } - fn initial() -> Tree { let node1: Entry = leaf(1).into(); let node2: Entry = leaf(2).into(); let node3 = Entry { - data: node(1, 2), + data: NodeData::combine(&node1.data, &node2.data), kind: EntryKind::Leaf, }; From acad37924b7cd4fccaa7988e697d3747e8f87f42 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 11 Oct 2019 09:14:19 +0300 Subject: [PATCH 74/76] test updates --- src/tree.rs | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/tree.rs b/src/tree.rs index fbadaba81..d3b6dcd7a 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -205,7 +205,7 @@ impl Tree { } #[cfg(test)] - fn for_children(&mut self, node: EntryLink, mut f: F) { + fn for_children(&self, node: EntryLink, f: F) { let (left, right) = { let link = self.resolve_link(node).expect("Failed to resolve link in test"); ( @@ -485,7 +485,7 @@ mod tests { // (0) (1) (3) (4) (7) // // new tree: - // (---8g---) + // (---10g--) // / \ // ( 6 ) \ // / \ \ @@ -494,7 +494,7 @@ mod tests { // (0) (1) (3) (4) (7) (8) // // so (7) is added as real leaf - // and new root, (8g) is generated one + // and new root, (10g) is generated one assert_eq!(new_root.data.end_height, 6); assert_eq!(appended.len(), 2); assert_matches!(tree.root(), EntryLink::Generated(_)); @@ -533,13 +533,16 @@ mod tests { // / \ / \ / \ \ // (0) (1) (3) (4) (7) (8) (10) // - // so (7) is added as real leaf - // and new root, (8g) is generated one + // so (10) is added as real leaf + // and new root, (12g) is generated one assert_eq!(new_root.data.end_height, 7); assert_eq!(appended.len(), 1); assert_matches!(tree.root(), EntryLink::Generated(_)); tree.for_children(tree.root(), |l, r| { assert_matches!(l, EntryLink::Generated(_)); + tree.for_children(l, |l, r| + assert_matches!((l, r), (EntryLink::Stored(6), EntryLink::Stored(9))) + ); assert_matches!(r, EntryLink::Stored(10)); }); } @@ -547,7 +550,7 @@ mod tests { #[test] fn truncate_simple() { let mut tree = generated(9); - tree.truncate_leaf().expect("Failed to truncate"); + let total_truncated = tree.truncate_leaf().expect("Failed to truncate"); // initial tree: // @@ -574,6 +577,7 @@ mod tests { // and new root, (14) is a stored one now assert_matches!(tree.root(), EntryLink::Stored(14)); + assert_eq!(total_truncated, 1); assert_eq!(tree.len(), 15); } @@ -609,19 +613,11 @@ mod tests { assert_matches!(tree.root(), EntryLink::Generated(_)); - // left is 14 and right is 15 - let (left_root_child, right_root_child) = { - let root = tree.root_node().expect("Failed to resolve"); - - ( - root.left().expect("Expected node"), - root.right().expect("Expected node"), + tree.for_children(tree.root(),|left, right| + assert_matches!( + (left, right), + (EntryLink::Stored(14), EntryLink::Stored(15)) ) - }; - - assert_matches!( - (left_root_child, right_root_child), - (EntryLink::Stored(14), EntryLink::Stored(15)) ); // two stored nodes should leave us (leaf 16 and no longer needed node 17) @@ -706,7 +702,7 @@ mod tests { } TestResult::from_bool( - if number & number - 1 == 0 { + if number & (number - 1) == 0 { if let EntryLink::Stored(_) = tree.root() { true } else { false } } else { From 26be46573ee7e0094f19d861fd29793adda6298e Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 28 Nov 2019 10:31:16 +0300 Subject: [PATCH 75/76] add hash of the node method --- src/node_data.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/node_data.rs b/src/node_data.rs index ac545e24f..0e972f933 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -193,6 +193,14 @@ impl NodeData { let mut cursor = std::io::Cursor::new(buf); Self::read(consensus_branch_id, &mut cursor) } + + pub fn hash(&self) -> [u8; 32] { + let mut buf = [0u8; 32]; + + let bytes = self.to_bytes(); + + blake2b_personal(&personalization(self.consensus_branch_id), &bytes) + } } #[cfg(test)] From 9059f53873fcc59e4c6b0c4c9fb7caddc5f9f128 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 28 Nov 2019 15:31:23 +0300 Subject: [PATCH 76/76] fix review notes and other issues --- src/entry.rs | 4 ++-- src/lib.rs | 16 ++++++++++------ src/node_data.rs | 3 +-- src/tree.rs | 25 +++++++++---------------- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/entry.rs b/src/entry.rs index 257b9ddb2..ab95a38b1 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -40,7 +40,7 @@ impl Entry { /// Left child pub fn left(&self) -> Result { match self.kind { - EntryKind::Leaf => { Err(Error::ExpectedNode) } + EntryKind::Leaf => { Err(Error::node_expected()) } EntryKind::Node(left, _) => Ok(left) } } @@ -48,7 +48,7 @@ impl Entry { /// Right child. pub fn right(&self) -> Result { match self.kind { - EntryKind::Leaf => { Err(Error::ExpectedNode) } + EntryKind::Leaf => { Err(Error::node_expected()) } EntryKind::Node(_, right) => Ok(right) } } diff --git a/src/lib.rs b/src/lib.rs index c0c781cc6..0c26dfa94 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,18 +17,16 @@ pub use entry::{Entry, MAX_ENTRY_SIZE}; pub enum Error { /// Entry expected to be presented in the tree view while it was not. ExpectedInMemory(EntryLink), - /// Entry expected to be a node. - ExpectedNode, /// Entry expected to be a node (specifying for which link this is not true). - ExpectedNodeForLink(EntryLink), + ExpectedNode(Option), } impl std::fmt::Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match *self { Self::ExpectedInMemory(l) => write!(f, "Node/leaf expected to be in memory: {}", l), - Self::ExpectedNode => write!(f, "Node expected"), - Self::ExpectedNodeForLink(l) => write!(f, "Node expected, not leaf: {}", l), + Self::ExpectedNode(None) => write!(f, "Node expected"), + Self::ExpectedNode(Some(l)) => write!(f, "Node expected, not leaf: {}", l), } } } @@ -63,9 +61,15 @@ pub enum EntryKind { } impl Error { + /// Entry expected to be a node (specifying for which link this is not true). + pub fn link_node_expected(link: EntryLink) -> Self { Self::ExpectedNode(Some(link)) } + + /// Some entry is expected to be node + pub fn node_expected() -> Self { Self::ExpectedNode(None) } + pub (crate) fn augment(self, link: EntryLink) -> Self { match self { - Error::ExpectedNode => Error::ExpectedNodeForLink(link), + Error::ExpectedNode(_) => Error::ExpectedNode(Some(link)), val => val } } diff --git a/src/node_data.rs b/src/node_data.rs index 0e972f933..55b2df21f 100644 --- a/src/node_data.rs +++ b/src/node_data.rs @@ -194,9 +194,8 @@ impl NodeData { Self::read(consensus_branch_id, &mut cursor) } + /// Hash node metadata pub fn hash(&self) -> [u8; 32] { - let mut buf = [0u8; 32]; - let bytes = self.to_bytes(); blake2b_personal(&personalization(self.consensus_branch_id), &bytes) diff --git a/src/tree.rs b/src/tree.rs index d3b6dcd7a..c020c72f0 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -28,22 +28,15 @@ pub struct Tree { impl Tree { /// Resolve link originated from this tree pub fn resolve_link(&self, link: EntryLink) -> Result { - match link { - EntryLink::Generated(index) => { - let node = self.generated.get(index as usize).ok_or(Error::ExpectedInMemory(link))?; - Ok(IndexedNode { - node, - link, - }) - }, - EntryLink::Stored(index) => { - let node = self.stored.get(&index).ok_or(Error::ExpectedInMemory(link))?; - Ok(IndexedNode { - node, - link, - }) - }, - } + match link { + EntryLink::Generated(index) => self.generated.get(index as usize), + EntryLink::Stored(index) => self.stored.get(&index), + } + .map(|node| IndexedNode { + node, + link, + }) + .ok_or(Error::ExpectedInMemory(link)) } fn push(&mut self, data: Entry) -> EntryLink {