From ea6b031298f2a3c6515bec7522588c38d68aef9e Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Sun, 3 Sep 2017 09:11:14 +0200 Subject: [PATCH 01/12] triehash is separated from util (#6428) --- Cargo.toml | 9 ++ src/lib.rs | 371 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+) create mode 100644 Cargo.toml create mode 100644 src/lib.rs diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..cc18a6e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "triehash" +version = "0.1.0" +authors = ["Parity Technologies "] + +[dependencies] +rlp = { path = "../rlp" } +ethcore-bigint = { path = "../bigint" } +hash = { path = "../hash" } diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..d166aa5 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,371 @@ +// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Generetes trie root. +//! +//! This module should be used to generate trie root hash. + +extern crate ethcore_bigint; +extern crate hash; +extern crate rlp; + +use std::collections::BTreeMap; +use std::cmp; +use ethcore_bigint::hash::H256; +use hash::keccak; +use rlp::RlpStream; + +fn shared_prefix_len(first: &[T], second: &[T]) -> usize { + let len = cmp::min(first.len(), second.len()); + (0..len).take_while(|&i| first[i] == second[i]).count() +} + +/// Generates a trie root hash for a vector of values +/// +/// ```rust +/// extern crate triehash; +/// use triehash::ordered_trie_root; +/// +/// fn main() { +/// let v = vec![From::from("doe"), From::from("reindeer")]; +/// let root = "e766d5d51b89dc39d981b41bda63248d7abce4f0225eefd023792a540bcffee3"; +/// assert_eq!(ordered_trie_root(v), root.parse().unwrap()); +/// } +/// ``` +pub fn ordered_trie_root(input: I) -> H256 + where I: IntoIterator> +{ + let gen_input = input + // first put elements into btree to sort them by nibbles + // optimize it later + .into_iter() + .enumerate() + .map(|(i, vec)| (rlp::encode(&i).into_vec(), vec)) + .collect::>() + // then move them to a vector + .into_iter() + .map(|(k, v)| (as_nibbles(&k), v) ) + .collect(); + + gen_trie_root(gen_input) +} + +/// Generates a trie root hash for a vector of key-values +/// +/// ```rust +/// extern crate triehash; +/// use triehash::trie_root; +/// +/// fn main() { +/// let v = vec![ +/// (From::from("doe"), From::from("reindeer")), +/// (From::from("dog"), From::from("puppy")), +/// (From::from("dogglesworth"), From::from("cat")), +/// ]; +/// +/// let root = "8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3"; +/// assert_eq!(trie_root(v), root.parse().unwrap()); +/// } +/// ``` +pub fn trie_root(input: I) -> H256 + where I: IntoIterator, Vec)> +{ + let gen_input = input + // first put elements into btree to sort them and to remove duplicates + .into_iter() + .collect::>() + // then move them to a vector + .into_iter() + .map(|(k, v)| (as_nibbles(&k), v) ) + .collect(); + + gen_trie_root(gen_input) +} + +/// Generates a key-hashed (secure) trie root hash for a vector of key-values. +/// +/// ```rust +/// extern crate triehash; +/// use triehash::sec_trie_root; +/// +/// fn main() { +/// let v = vec![ +/// (From::from("doe"), From::from("reindeer")), +/// (From::from("dog"), From::from("puppy")), +/// (From::from("dogglesworth"), From::from("cat")), +/// ]; +/// +/// let root = "d4cd937e4a4368d7931a9cf51686b7e10abb3dce38a39000fd7902a092b64585"; +/// assert_eq!(sec_trie_root(v), root.parse().unwrap()); +/// } +/// ``` +pub fn sec_trie_root(input: Vec<(Vec, Vec)>) -> H256 { + let gen_input = input + // first put elements into btree to sort them and to remove duplicates + .into_iter() + .map(|(k, v)| (keccak(k), v)) + .collect::>() + // then move them to a vector + .into_iter() + .map(|(k, v)| (as_nibbles(&k), v) ) + .collect(); + + gen_trie_root(gen_input) +} + +fn gen_trie_root(input: Vec<(Vec, Vec)>) -> H256 { + let mut stream = RlpStream::new(); + hash256rlp(&input, 0, &mut stream); + keccak(stream.out()) +} + +/// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. +/// +/// The "termination marker" and "leaf-node" specifier are completely equivalent. +/// +/// Input values are in range `[0, 0xf]`. +/// +/// ```markdown +/// [0,0,1,2,3,4,5] 0x10012345 // 7 > 4 +/// [0,1,2,3,4,5] 0x00012345 // 6 > 4 +/// [1,2,3,4,5] 0x112345 // 5 > 3 +/// [0,0,1,2,3,4] 0x00001234 // 6 > 3 +/// [0,1,2,3,4] 0x101234 // 5 > 3 +/// [1,2,3,4] 0x001234 // 4 > 3 +/// [0,0,1,2,3,4,5,T] 0x30012345 // 7 > 4 +/// [0,0,1,2,3,4,T] 0x20001234 // 6 > 4 +/// [0,1,2,3,4,5,T] 0x20012345 // 6 > 4 +/// [1,2,3,4,5,T] 0x312345 // 5 > 3 +/// [1,2,3,4,T] 0x201234 // 4 > 3 +/// ``` +fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { + let inlen = nibbles.len(); + let oddness_factor = inlen % 2; + // next even number divided by two + let reslen = (inlen + 2) >> 1; + let mut res = Vec::with_capacity(reslen); + + let first_byte = { + let mut bits = ((inlen as u8 & 1) + (2 * leaf as u8)) << 4; + if oddness_factor == 1 { + bits += nibbles[0]; + } + bits + }; + + res.push(first_byte); + + let mut offset = oddness_factor; + while offset < inlen { + let byte = (nibbles[offset] << 4) + nibbles[offset + 1]; + res.push(byte); + offset += 2; + } + + res +} + +/// Converts slice of bytes to nibbles. +fn as_nibbles(bytes: &[u8]) -> Vec { + let mut res = Vec::with_capacity(bytes.len() * 2); + for i in 0..bytes.len() { + let byte = bytes[i]; + res.push(byte >> 4); + res.push(byte & 0b1111); + } + res +} + +fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { + let inlen = input.len(); + + // in case of empty slice, just append empty data + if inlen == 0 { + stream.append_empty_data(); + return; + } + + // take slices + let key: &[u8] = &input[0].0; + let value: &[u8] = &input[0].1; + + // if the slice contains just one item, append the suffix of the key + // and then append value + if inlen == 1 { + stream.begin_list(2); + stream.append(&hex_prefix_encode(&key[pre_len..], true)); + stream.append(&value); + return; + } + + // get length of the longest shared prefix in slice keys + let shared_prefix = input.iter() + // skip first element + .skip(1) + // get minimum number of shared nibbles between first and each successive + .fold(key.len(), | acc, &(ref k, _) | { + cmp::min(shared_prefix_len(key, k), acc) + }); + + // if shared prefix is higher than current prefix append its + // new part of the key to the stream + // then recursively append suffixes of all items who had this key + if shared_prefix > pre_len { + stream.begin_list(2); + stream.append(&hex_prefix_encode(&key[pre_len..shared_prefix], false)); + hash256aux(input, shared_prefix, stream); + return; + } + + // an item for every possible nibble/suffix + // + 1 for data + stream.begin_list(17); + + // if first key len is equal to prefix_len, move to next element + let mut begin = match pre_len == key.len() { + true => 1, + false => 0 + }; + + // iterate over all possible nibbles + for i in 0..16 { + // cout how many successive elements have same next nibble + let len = match begin < input.len() { + true => input[begin..].iter() + .take_while(| pair | pair.0[pre_len] == i ) + .count(), + false => 0 + }; + + // if at least 1 successive element has the same nibble + // append their suffixes + match len { + 0 => { stream.append_empty_data(); }, + _ => hash256aux(&input[begin..(begin + len)], pre_len + 1, stream) + } + begin += len; + } + + // if fist key len is equal prefix, append its value + match pre_len == key.len() { + true => { stream.append(&value); }, + false => { stream.append_empty_data(); } + }; +} + +fn hash256aux(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { + let mut s = RlpStream::new(); + hash256rlp(input, pre_len, &mut s); + let out = s.out(); + match out.len() { + 0...31 => stream.append_raw(&out, 1), + _ => stream.append(&keccak(out)) + }; +} + + +#[test] +fn test_nibbles() { + let v = vec![0x31, 0x23, 0x45]; + let e = vec![3, 1, 2, 3, 4, 5]; + assert_eq!(as_nibbles(&v), e); + + // A => 65 => 0x41 => [4, 1] + let v: Vec = From::from("A"); + let e = vec![4, 1]; + assert_eq!(as_nibbles(&v), e); +} + + +#[cfg(test)] +mod tests { + use super::{trie_root, shared_prefix_len, hex_prefix_encode}; + + #[test] + fn test_hex_prefix_encode() { + let v = vec![0, 0, 1, 2, 3, 4, 5]; + let e = vec![0x10, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v, false); + assert_eq!(h, e); + + let v = vec![0, 1, 2, 3, 4, 5]; + let e = vec![0x00, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v, false); + assert_eq!(h, e); + + let v = vec![0, 1, 2, 3, 4, 5]; + let e = vec![0x20, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v, true); + assert_eq!(h, e); + + let v = vec![1, 2, 3, 4, 5]; + let e = vec![0x31, 0x23, 0x45]; + let h = hex_prefix_encode(&v, true); + assert_eq!(h, e); + + let v = vec![1, 2, 3, 4]; + let e = vec![0x00, 0x12, 0x34]; + let h = hex_prefix_encode(&v, false); + assert_eq!(h, e); + + let v = vec![4, 1]; + let e = vec![0x20, 0x41]; + let h = hex_prefix_encode(&v, true); + assert_eq!(h, e); + } + + #[test] + fn simple_test() { + assert_eq!(trie_root(vec![ + (b"A".to_vec(), b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_vec()) + ]), "d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab".parse().unwrap()); + } + + #[test] + fn test_triehash_out_of_order() { + assert!(trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ]) == + trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + ])); + } + + #[test] + fn test_shared_prefix() { + let a = vec![1,2,3,4,5,6]; + let b = vec![4,2,3,4,5,6]; + assert_eq!(shared_prefix_len(&a, &b), 0); + } + + #[test] + fn test_shared_prefix2() { + let a = vec![1,2,3,3,5]; + let b = vec![1,2,3]; + assert_eq!(shared_prefix_len(&a, &b), 3); + } + + #[test] + fn test_shared_prefix3() { + let a = vec![1,2,3,4,5,6]; + let b = vec![1,2,3,4,5,6]; + assert_eq!(shared_prefix_len(&a, &b), 6); + } +} From 0e542861667ab7e0594067ed13f4481b687301aa Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 10 Nov 2017 18:31:31 +0100 Subject: [PATCH 02/12] prepare cargo configuration for upload of crates --- Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index cc18a6e..1fae40d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,8 @@ name = "triehash" version = "0.1.0" authors = ["Parity Technologies "] +description = "in memory patricia trie operations" +license = "GPL-3.0" [dependencies] rlp = { path = "../rlp" } From abf3f0d9b9b79a994b3185d04ef28a3e7b4ff414 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 10 Nov 2017 18:46:35 +0100 Subject: [PATCH 03/12] update ethcore-bigint version --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1fae40d..2731a46 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,6 @@ description = "in memory patricia trie operations" license = "GPL-3.0" [dependencies] -rlp = { path = "../rlp" } -ethcore-bigint = { path = "../bigint" } -hash = { path = "../hash" } +rlp = { version = "0.2", path = "../rlp" } +ethcore-bigint = { version = "0.2.1", path = "../bigint" } +hash = { version = "0.1", path = "../hash" } From 33f32ae3a4d55f32cfba7744137aabdd6104626c Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 10 Nov 2017 19:04:55 +0100 Subject: [PATCH 04/12] rename hash crate to keccak-hash --- Cargo.toml | 2 +- src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2731a46..d9ff480 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,4 +8,4 @@ license = "GPL-3.0" [dependencies] rlp = { version = "0.2", path = "../rlp" } ethcore-bigint = { version = "0.2.1", path = "../bigint" } -hash = { version = "0.1", path = "../hash" } +keccak-hash = { version = "0.1", path = "../hash" } diff --git a/src/lib.rs b/src/lib.rs index d166aa5..78c22fb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,7 +19,7 @@ //! This module should be used to generate trie root hash. extern crate ethcore_bigint; -extern crate hash; +extern crate keccak_hash as hash; extern crate rlp; use std::collections::BTreeMap; From 88b0b422582b34c3a0a2989379294a5c241cabe4 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 10 Nov 2017 20:21:24 +0100 Subject: [PATCH 05/12] update rlp --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d9ff480..53d6b56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,6 @@ description = "in memory patricia trie operations" license = "GPL-3.0" [dependencies] -rlp = { version = "0.2", path = "../rlp" } +rlp = { version = "0.2.1", path = "../rlp" } ethcore-bigint = { version = "0.2.1", path = "../bigint" } keccak-hash = { version = "0.1", path = "../hash" } From 76fcce1f036fdc4d0333b22b5ec1557b5848bacd Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Wed, 10 Jan 2018 13:35:18 +0100 Subject: [PATCH 06/12] dissolve util (#7460) * ethereum-types refactor in progress * ethereum-types refactor in progress * ethereum-types refactor in progress * ethereum-types refactor in progress * ethereum-types refactor finished * removed obsolete util/src/lib.rs * removed commented out code --- Cargo.toml | 2 +- src/lib.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 53d6b56..a19a4da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,5 +7,5 @@ license = "GPL-3.0" [dependencies] rlp = { version = "0.2.1", path = "../rlp" } -ethcore-bigint = { version = "0.2.1", path = "../bigint" } +ethereum-types = "0.1" keccak-hash = { version = "0.1", path = "../hash" } diff --git a/src/lib.rs b/src/lib.rs index 78c22fb..13ed491 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,13 +18,13 @@ //! //! This module should be used to generate trie root hash. -extern crate ethcore_bigint; +extern crate ethereum_types; extern crate keccak_hash as hash; extern crate rlp; use std::collections::BTreeMap; use std::cmp; -use ethcore_bigint::hash::H256; +use ethereum_types::H256; use hash::keccak; use rlp::RlpStream; From b3c085a10191e51a5360b51789b6e7f027013424 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Fri, 9 Feb 2018 09:32:06 +0100 Subject: [PATCH 07/12] ethabi version 5 (#7723) * Refactor updater to use ethabi-derive * Grumble: do_call type alias * Empty commit to trigger test re-run * migration to ethabi-5.0 * migration to ethabi-5.0 in progress * use ethabi_deriven to generate TransactAcl contract * use ethabi_deriven to generate Registry contract * hash-fetch uses ethabi_derive, removed retain cycle from updater, fixed #7720 * node-filter crate uses ethabi_derive to generate peer_set contract interface * use LruCache in node-filter instead of HashMap * validator_set engine uses ethabi_derive * ethcore does not depend on native_contracts * miner does no depend on native_contracts * secret_store does not use native_contracts (in progress) * removed native-contracts * ethcore and updater does not depend on futures * updated ethereum-types * fixed all warnings caused by using new version of ethereum-types * updated ethabi_derive && ethabi_contract to get rid of warnings * removed another retain cycle in updater, fixed following minor version on update * moved contracts out of native_contracts res * updated ethabi_contract * fixed failing test * fixed failing test * there is no need to create two contracts of the same kind any more * simplify updater::ReleaseTrack conversion into u8 and add several tests for it * applied review suggestions * applied review suggestions --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a19a4da..1667827 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,5 +7,5 @@ license = "GPL-3.0" [dependencies] rlp = { version = "0.2.1", path = "../rlp" } -ethereum-types = "0.1" +ethereum-types = "0.2" keccak-hash = { version = "0.1", path = "../hash" } From 8ac8d57573ca07a43bf5f3676710129118952298 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Fri, 16 Feb 2018 20:24:16 +0100 Subject: [PATCH 08/12] triehash optimisations (#7920) --- src/lib.rs | 73 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 13ed491..e958e82 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,27 +40,28 @@ fn shared_prefix_len(first: &[T], second: &[T]) -> usize { /// use triehash::ordered_trie_root; /// /// fn main() { -/// let v = vec![From::from("doe"), From::from("reindeer")]; +/// let v = &["doe", "reindeer"]; /// let root = "e766d5d51b89dc39d981b41bda63248d7abce4f0225eefd023792a540bcffee3"; -/// assert_eq!(ordered_trie_root(v), root.parse().unwrap()); +/// assert_eq!(ordered_trie_root(v), root.into()); /// } /// ``` -pub fn ordered_trie_root(input: I) -> H256 - where I: IntoIterator> +pub fn ordered_trie_root(input: I) -> H256 + where I: IntoIterator, + A: AsRef<[u8]>, { - let gen_input = input + let gen_input: Vec<_> = input // first put elements into btree to sort them by nibbles // optimize it later .into_iter() .enumerate() - .map(|(i, vec)| (rlp::encode(&i).into_vec(), vec)) + .map(|(i, slice)| (rlp::encode(&i), slice)) .collect::>() // then move them to a vector .into_iter() .map(|(k, v)| (as_nibbles(&k), v) ) .collect(); - gen_trie_root(gen_input) + gen_trie_root(&gen_input) } /// Generates a trie root hash for a vector of key-values @@ -71,28 +72,30 @@ pub fn ordered_trie_root(input: I) -> H256 /// /// fn main() { /// let v = vec![ -/// (From::from("doe"), From::from("reindeer")), -/// (From::from("dog"), From::from("puppy")), -/// (From::from("dogglesworth"), From::from("cat")), +/// ("doe", "reindeer"), +/// ("dog", "puppy"), +/// ("dogglesworth", "cat"), /// ]; /// /// let root = "8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3"; -/// assert_eq!(trie_root(v), root.parse().unwrap()); +/// assert_eq!(trie_root(v), root.into()); /// } /// ``` -pub fn trie_root(input: I) -> H256 - where I: IntoIterator, Vec)> +pub fn trie_root(input: I) -> H256 + where I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, { - let gen_input = input + let gen_input: Vec<_> = input // first put elements into btree to sort them and to remove duplicates .into_iter() .collect::>() // then move them to a vector .into_iter() - .map(|(k, v)| (as_nibbles(&k), v) ) + .map(|(k, v)| (as_nibbles(k.as_ref()), v) ) .collect(); - gen_trie_root(gen_input) + gen_trie_root(&gen_input) } /// Generates a key-hashed (secure) trie root hash for a vector of key-values. @@ -103,17 +106,21 @@ pub fn trie_root(input: I) -> H256 /// /// fn main() { /// let v = vec![ -/// (From::from("doe"), From::from("reindeer")), -/// (From::from("dog"), From::from("puppy")), -/// (From::from("dogglesworth"), From::from("cat")), +/// ("doe", "reindeer"), +/// ("dog", "puppy"), +/// ("dogglesworth", "cat"), /// ]; /// /// let root = "d4cd937e4a4368d7931a9cf51686b7e10abb3dce38a39000fd7902a092b64585"; -/// assert_eq!(sec_trie_root(v), root.parse().unwrap()); +/// assert_eq!(sec_trie_root(v), root.into()); /// } /// ``` -pub fn sec_trie_root(input: Vec<(Vec, Vec)>) -> H256 { - let gen_input = input +pub fn sec_trie_root(input: I) -> H256 + where I: IntoIterator, + A: AsRef<[u8]>, + B: AsRef<[u8]>, +{ + let gen_input: Vec<_> = input // first put elements into btree to sort them and to remove duplicates .into_iter() .map(|(k, v)| (keccak(k), v)) @@ -123,12 +130,12 @@ pub fn sec_trie_root(input: Vec<(Vec, Vec)>) -> H256 { .map(|(k, v)| (as_nibbles(&k), v) ) .collect(); - gen_trie_root(gen_input) + gen_trie_root(&gen_input) } -fn gen_trie_root(input: Vec<(Vec, Vec)>) -> H256 { +fn gen_trie_root, B: AsRef<[u8]>>(input: &[(A, B)]) -> H256 { let mut stream = RlpStream::new(); - hash256rlp(&input, 0, &mut stream); + hash256rlp(input, 0, &mut stream); keccak(stream.out()) } @@ -189,7 +196,7 @@ fn as_nibbles(bytes: &[u8]) -> Vec { res } -fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { +fn hash256rlp, B: AsRef<[u8]>>(input: &[(A, B)], pre_len: usize, stream: &mut RlpStream) { let inlen = input.len(); // in case of empty slice, just append empty data @@ -199,8 +206,8 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre } // take slices - let key: &[u8] = &input[0].0; - let value: &[u8] = &input[0].1; + let key: &[u8] = &input[0].0.as_ref(); + let value: &[u8] = &input[0].1.as_ref(); // if the slice contains just one item, append the suffix of the key // and then append value @@ -217,7 +224,7 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre .skip(1) // get minimum number of shared nibbles between first and each successive .fold(key.len(), | acc, &(ref k, _) | { - cmp::min(shared_prefix_len(key, k), acc) + cmp::min(shared_prefix_len(key, k.as_ref()), acc) }); // if shared prefix is higher than current prefix append its @@ -245,7 +252,7 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre // cout how many successive elements have same next nibble let len = match begin < input.len() { true => input[begin..].iter() - .take_while(| pair | pair.0[pre_len] == i ) + .take_while(| pair | pair.0.as_ref()[pre_len] == i ) .count(), false => 0 }; @@ -266,7 +273,7 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre }; } -fn hash256aux(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { +fn hash256aux, B: AsRef<[u8]>>(input: &[(A, B)], pre_len: usize, stream: &mut RlpStream) { let mut s = RlpStream::new(); hash256rlp(input, pre_len, &mut s); let out = s.out(); @@ -330,8 +337,8 @@ mod tests { #[test] fn simple_test() { assert_eq!(trie_root(vec![ - (b"A".to_vec(), b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_vec()) - ]), "d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab".parse().unwrap()); + (b"A", b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" as &[u8]) + ]), "d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab".into()); } #[test] From 61cdc504296baf458646674be859dcd1c2160c8b Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Mon, 19 Feb 2018 12:03:54 +0100 Subject: [PATCH 09/12] bring back trie and triehash benches (#7926) --- Cargo.toml | 3 + benches/triehash.rs | 147 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 benches/triehash.rs diff --git a/Cargo.toml b/Cargo.toml index 1667827..c7b6242 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,3 +9,6 @@ license = "GPL-3.0" rlp = { version = "0.2.1", path = "../rlp" } ethereum-types = "0.2" keccak-hash = { version = "0.1", path = "../hash" } + +[dev-dependencies] +trie-standardmap = { path = "../trie-standardmap" } diff --git a/benches/triehash.rs b/benches/triehash.rs new file mode 100644 index 0000000..505ea12 --- /dev/null +++ b/benches/triehash.rs @@ -0,0 +1,147 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +#![feature(test)] + +extern crate ethereum_types; +extern crate keccak_hash; +extern crate test; +extern crate trie_standardmap; +extern crate triehash; + +use ethereum_types::H256; +use keccak_hash::keccak; +use test::Bencher; +use trie_standardmap::{Alphabet, ValueMode, StandardMap}; +use triehash::trie_root; + +fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + diff_count <= 32); + *seed = keccak(&seed); + let r = min_count + (seed[31] as usize % (diff_count + 1)); + let mut ret: Vec = Vec::with_capacity(r); + for i in 0..r { + ret.push(alphabet[seed[i] as usize % alphabet.len()]); + } + ret +} + +fn random_bytes(min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + diff_count <= 32); + *seed = keccak(&seed); + let r = min_count + (seed[31] as usize % (diff_count + 1)); + seed[0..r].to_vec() +} + +fn random_value(seed: &mut H256) -> Vec { + *seed = keccak(&seed); + match seed[0] % 2 { + 1 => vec![seed[31];1], + _ => seed.to_vec(), + } +} + +#[bench] +fn triehash_insertions_32_mir_1k(b: &mut Bencher) { + let st = StandardMap { + alphabet: Alphabet::All, + min_key: 32, + journal_key: 0, + value_mode: ValueMode::Mirror, + count: 1000, + }; + let d = st.make(); + b.iter(&mut ||{ + trie_root(d.clone()).clone(); + }); +} + +#[bench] +fn triehash_insertions_32_ran_1k(b: &mut Bencher) { + let st = StandardMap { + alphabet: Alphabet::All, + min_key: 32, + journal_key: 0, + value_mode: ValueMode::Random, + count: 1000, + }; + let d = st.make(); + b.iter(&mut ||{ + trie_root(d.clone()).clone(); + }); +} + +#[bench] +fn triehash_insertions_six_high(b: &mut Bencher) { + let mut d: Vec<(Vec, Vec)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_bytes(6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(&||{ + trie_root(d.clone()); + }) +} + +#[bench] +fn triehash_insertions_six_mid(b: &mut Bencher) { + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + let mut d: Vec<(Vec, Vec)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + b.iter(||{ + trie_root(d.clone()); + }) +} + +#[bench] +fn triehash_insertions_random_mid(b: &mut Bencher) { + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + let mut d: Vec<(Vec, Vec)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 1, 5, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + trie_root(d.clone()); + }) +} + +#[bench] +fn triehash_insertions_six_low(b: &mut Bencher) { + let alphabet = b"abcdef"; + let mut d: Vec<(Vec, Vec)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + trie_root(d.clone()); + }) +} From 70cf288545c3b86d8c9c923b45117997c197833b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Thu, 22 Mar 2018 09:24:46 +0100 Subject: [PATCH 10/12] Avoid allocations when computing triehash. (#8176) * Avoid allocations when computing triehash. * Bump elastic-array to 0.10 --- Cargo.toml | 1 + src/lib.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c7b6242..52af591 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ description = "in memory patricia trie operations" license = "GPL-3.0" [dependencies] +elastic-array = "0.10" rlp = { version = "0.2.1", path = "../rlp" } ethereum-types = "0.2" keccak-hash = { version = "0.1", path = "../hash" } diff --git a/src/lib.rs b/src/lib.rs index e958e82..7f20d39 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,12 +18,14 @@ //! //! This module should be used to generate trie root hash. +extern crate elastic_array; extern crate ethereum_types; extern crate keccak_hash as hash; extern crate rlp; use std::collections::BTreeMap; use std::cmp; +use elastic_array::{ElasticArray4, ElasticArray8}; use ethereum_types::H256; use hash::keccak; use rlp::RlpStream; @@ -158,12 +160,10 @@ fn gen_trie_root, B: AsRef<[u8]>>(input: &[(A, B)]) -> H256 { /// [1,2,3,4,5,T] 0x312345 // 5 > 3 /// [1,2,3,4,T] 0x201234 // 4 > 3 /// ``` -fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { +fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> ElasticArray4 { let inlen = nibbles.len(); let oddness_factor = inlen % 2; - // next even number divided by two - let reslen = (inlen + 2) >> 1; - let mut res = Vec::with_capacity(reslen); + let mut res = ElasticArray4::new(); let first_byte = { let mut bits = ((inlen as u8 & 1) + (2 * leaf as u8)) << 4; @@ -186,8 +186,8 @@ fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { } /// Converts slice of bytes to nibbles. -fn as_nibbles(bytes: &[u8]) -> Vec { - let mut res = Vec::with_capacity(bytes.len() * 2); +fn as_nibbles(bytes: &[u8]) -> ElasticArray8 { + let mut res = ElasticArray8::new(); for i in 0..bytes.len() { let byte = bytes[i]; res.push(byte >> 4); @@ -213,7 +213,7 @@ fn hash256rlp, B: AsRef<[u8]>>(input: &[(A, B)], pre_len: usize, // and then append value if inlen == 1 { stream.begin_list(2); - stream.append(&hex_prefix_encode(&key[pre_len..], true)); + stream.append(&&*hex_prefix_encode(&key[pre_len..], true)); stream.append(&value); return; } @@ -232,7 +232,7 @@ fn hash256rlp, B: AsRef<[u8]>>(input: &[(A, B)], pre_len: usize, // then recursively append suffixes of all items who had this key if shared_prefix > pre_len { stream.begin_list(2); - stream.append(&hex_prefix_encode(&key[pre_len..shared_prefix], false)); + stream.append(&&*hex_prefix_encode(&key[pre_len..shared_prefix], false)); hash256aux(input, shared_prefix, stream); return; } From e4037e26b638412bc8ee1b181f883f93ad91a1ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Mon, 2 Apr 2018 13:12:52 +0200 Subject: [PATCH 11/12] Bump ethabi & ethereum-types. (#8258) * Bump ethabi & ethereum-types. * Fix test. * Fix hex encodings. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 52af591..ee42b9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" rlp = { version = "0.2.1", path = "../rlp" } -ethereum-types = "0.2" +ethereum-types = "0.3" keccak-hash = { version = "0.1", path = "../hash" } [dev-dependencies] From 1aa25358ac3b483945316b294667f103827dca1e Mon Sep 17 00:00:00 2001 From: Niklas Adolfsson Date: Mon, 4 Jun 2018 10:19:50 +0200 Subject: [PATCH 12/12] Update `license header` and `scripts` (#8666) * Update `add_license` script * run script * add `remove duplicate lines script` and run it * Revert changes `English spaces` * strip whitespaces * Revert `GPL` in files with `apache/mit license` * don't append `gpl license` in files with other lic * Don't append `gpl header` in files with other lic. * re-ran script * include c and cpp files too * remove duplicate header * rebase nit --- src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7f20d39..c78ed0c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Parity Technologies (UK) Ltd. +// Copyright 2015-2018 Parity Technologies (UK) Ltd. // This file is part of Parity. // Parity is free software: you can redistribute it and/or modify @@ -283,7 +283,6 @@ fn hash256aux, B: AsRef<[u8]>>(input: &[(A, B)], pre_len: usize, }; } - #[test] fn test_nibbles() { let v = vec![0x31, 0x23, 0x45]; @@ -296,7 +295,6 @@ fn test_nibbles() { assert_eq!(as_nibbles(&v), e); } - #[cfg(test)] mod tests { use super::{trie_root, shared_prefix_len, hex_prefix_encode};