Introduce alternative codec candidate for performance tweaking
This commit is contained in:
Gav
parent
1d2ab202df
commit
5d32d3d3b8
|
|
@ -17,7 +17,7 @@
|
|||
#[macro_use]
|
||||
extern crate criterion;
|
||||
use criterion::{Criterion, black_box, Fun};
|
||||
criterion_group!(benches, trie_insertions_32_mir_1k, trie_iter, trie_insertions_32_ran_1k, trie_insertions_six_high, trie_insertions_six_mid, trie_insertions_random_mid, trie_insertions_six_low, nibble_common_prefix);
|
||||
criterion_group!(benches, trie_insertions_32_mir_1k, trie_insertions_32_ran_1k, trie_insertions_six_high, trie_insertions_six_mid, trie_insertions_random_mid, trie_insertions_six_low, nibble_common_prefix);
|
||||
criterion_main!(benches);
|
||||
|
||||
extern crate memorydb;
|
||||
|
|
@ -27,18 +27,22 @@ extern crate substrate_trie;
|
|||
extern crate keccak_hasher;
|
||||
extern crate trie_standardmap;
|
||||
extern crate hashdb;
|
||||
extern crate triehash;
|
||||
|
||||
use memorydb::MemoryDB;
|
||||
use trie::{DBValue, NibbleSlice, TrieMut, Trie};
|
||||
use ethtrie::RlpNodeCodec;
|
||||
use ethtrie::{RlpNodeCodec, RlpTrieStream};
|
||||
use substrate_trie::{ParityNodeCodec, CodecTrieStream, ParityNodeCodecAlt, CodecTrieStreamAlt};
|
||||
use trie_standardmap::{Alphabet, ValueMode, StandardMap};
|
||||
use substrate_trie::ParityNodeCodec;
|
||||
use keccak_hasher::KeccakHasher;
|
||||
use hashdb::Hasher;
|
||||
use triehash::trie_root;
|
||||
|
||||
type H256 = <KeccakHasher as Hasher>::Out;
|
||||
type TrieDB<'a> = trie::TrieDB<'a, KeccakHasher, ParityNodeCodec<KeccakHasher>>;
|
||||
type TrieDBMut<'a> = trie::TrieDBMut<'a, KeccakHasher, ParityNodeCodec<KeccakHasher>>;
|
||||
type AltTrieDB<'a> = trie::TrieDB<'a, KeccakHasher, ParityNodeCodecAlt<KeccakHasher>>;
|
||||
type AltTrieDBMut<'a> = trie::TrieDBMut<'a, KeccakHasher, ParityNodeCodecAlt<KeccakHasher>>;
|
||||
type RlpTrieDB<'a> = trie::TrieDB<'a, KeccakHasher, RlpNodeCodec<KeccakHasher>>;
|
||||
type RlpTrieDBMut<'a> = trie::TrieDBMut<'a, KeccakHasher, RlpNodeCodec<KeccakHasher>>;
|
||||
|
||||
|
|
@ -76,6 +80,23 @@ impl ::std::fmt::Display for TrieInsertionList {
|
|||
}
|
||||
|
||||
fn bench_insertions(b: &mut Criterion, name: &str, d: Vec<(Vec<u8>, Vec<u8>)>) {
|
||||
let mut rlp_memdb = MemoryDB::<KeccakHasher, DBValue>::new();
|
||||
let mut rlp_root = H256::default();
|
||||
let mut codec_memdb = MemoryDB::<KeccakHasher, DBValue>::new_codec();
|
||||
let mut codec_root = H256::default();
|
||||
let mut alt_memdb = MemoryDB::<KeccakHasher, DBValue>::new_codec();
|
||||
let mut alt_root = H256::default();
|
||||
{
|
||||
let mut rlp_t = RlpTrieDBMut::new(&mut rlp_memdb, &mut rlp_root);
|
||||
let mut codec_t = TrieDBMut::new(&mut codec_memdb, &mut codec_root);
|
||||
let mut alt_t = TrieDBMut::new(&mut alt_memdb, &mut alt_root);
|
||||
for i in d.iter() {
|
||||
rlp_t.insert(&i.0, &i.1).unwrap();
|
||||
codec_t.insert(&i.0, &i.1).unwrap();
|
||||
alt_t.insert(&i.0, &i.1).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
let funs = vec![
|
||||
Fun::new("Rlp", |b, d: &TrieInsertionList| b.iter(&mut ||{
|
||||
let mut memdb = MemoryDB::<KeccakHasher, DBValue>::new();
|
||||
|
|
@ -93,43 +114,44 @@ fn bench_insertions(b: &mut Criterion, name: &str, d: Vec<(Vec<u8>, Vec<u8>)>) {
|
|||
t.insert(&i.0, &i.1).unwrap();
|
||||
}
|
||||
})),
|
||||
Fun::new("-", |b, _d: &TrieInsertionList| b.iter(&mut ||{}))
|
||||
];
|
||||
|
||||
b.bench_functions(name, funs, &TrieInsertionList(d));
|
||||
}
|
||||
|
||||
fn bench_iteration(b: &mut Criterion, name: &str, d: Vec<(Vec<u8>, Vec<u8>)>) {
|
||||
let mut rlp_memdb = MemoryDB::<KeccakHasher, DBValue>::new();
|
||||
let mut rlp_root = H256::default();
|
||||
let mut codec_memdb = MemoryDB::<KeccakHasher, DBValue>::new_codec();
|
||||
let mut codec_root = H256::default();
|
||||
{
|
||||
let mut rlp_t = RlpTrieDBMut::new(&mut rlp_memdb, &mut rlp_root);
|
||||
let mut codec_t = TrieDBMut::new(&mut codec_memdb, &mut codec_root);
|
||||
for i in d.iter() {
|
||||
rlp_t.insert(&i.0, &i.1).unwrap();
|
||||
codec_t.insert(&i.0, &i.1).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
let funs = vec![
|
||||
Fun::new("Rlp", move |b, d: &u8| b.iter(&mut ||{
|
||||
Fun::new("Alt", |b, d: &TrieInsertionList| b.iter(&mut ||{
|
||||
let mut memdb = MemoryDB::<KeccakHasher, DBValue>::new_codec();
|
||||
let mut root = H256::default();
|
||||
let mut t = AltTrieDBMut::new(&mut memdb, &mut root);
|
||||
for i in d.0.iter() {
|
||||
t.insert(&i.0, &i.1).unwrap();
|
||||
}
|
||||
})),
|
||||
Fun::new("IterRlp", move |b, _d| b.iter(&mut ||{
|
||||
let t = RlpTrieDB::new(&rlp_memdb, &rlp_root).unwrap();
|
||||
for n in t.iter().unwrap() {
|
||||
black_box(n).unwrap();
|
||||
}
|
||||
})),
|
||||
Fun::new("Codec", move |b, d: &u8| b.iter(&mut ||{
|
||||
Fun::new("IterCodec", move |b, _d| b.iter(&mut ||{
|
||||
let t = TrieDB::new(&codec_memdb, &codec_root).unwrap();
|
||||
for n in t.iter().unwrap() {
|
||||
black_box(n).unwrap();
|
||||
}
|
||||
})),
|
||||
Fun::new("-", |b, _d: &u8| b.iter(&mut ||{}))
|
||||
Fun::new("IterAlt", move |b, _d| b.iter(&mut ||{
|
||||
let t = AltTrieDB::new(&alt_memdb, &codec_root).unwrap();
|
||||
for n in t.iter().unwrap() {
|
||||
black_box(n).unwrap();
|
||||
}
|
||||
})),
|
||||
Fun::new("ClosedRlp", |b, d: &TrieInsertionList| b.iter(&mut ||{
|
||||
trie_root::<KeccakHasher, RlpTrieStream, _, _, _>(d.0.clone())
|
||||
})),
|
||||
Fun::new("ClosedCodec", |b, d: &TrieInsertionList| b.iter(&mut ||{
|
||||
trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(d.0.clone())
|
||||
})),
|
||||
Fun::new("ClosedAlt", |b, d: &TrieInsertionList| b.iter(&mut ||{
|
||||
trie_root::<KeccakHasher, CodecTrieStreamAlt, _, _, _>(d.0.clone())
|
||||
}))
|
||||
];
|
||||
|
||||
b.bench_functions(name, funs, &0u8);
|
||||
b.bench_functions(name, funs, &TrieInsertionList(d));
|
||||
}
|
||||
|
||||
fn trie_insertions_32_mir_1k(b: &mut Criterion) {
|
||||
|
|
@ -201,17 +223,6 @@ fn trie_insertions_six_low(b: &mut Criterion) {
|
|||
bench_insertions(b, "trie_ins_six_low", d);
|
||||
}
|
||||
|
||||
fn trie_iter(b: &mut Criterion) {
|
||||
let st = StandardMap {
|
||||
alphabet: Alphabet::All,
|
||||
min_key: 32,
|
||||
journal_key: 0,
|
||||
value_mode: ValueMode::Mirror,
|
||||
count: 1000,
|
||||
};
|
||||
bench_iteration(b, "iteration", st.make());
|
||||
}
|
||||
|
||||
fn nibble_common_prefix(b: &mut Criterion) {
|
||||
let st = StandardMap {
|
||||
alphabet: Alphabet::Custom(b"abcdef".to_vec()),
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
|
||||
#[macro_use]
|
||||
extern crate criterion;
|
||||
use criterion::Criterion;
|
||||
use criterion::{Criterion, Fun};
|
||||
criterion_group!(benches, triehash_insertions_32_mir_1k, triehash_insertions_32_ran_1k, triehash_insertions_six_high, triehash_insertions_six_mid, triehash_insertions_random_mid, triehash_insertions_six_low, typical_tx_payload);
|
||||
criterion_main!(benches);
|
||||
|
||||
|
|
@ -34,6 +34,7 @@ use trie_standardmap::{Alphabet, ValueMode, StandardMap};
|
|||
use triehash::trie_root;
|
||||
use codec::{Encode, Compact};
|
||||
use substrate_trie::CodecTrieStream;
|
||||
use patricia_trie_ethereum::RlpTrieStream;
|
||||
|
||||
type H256 = <KeccakHasher as Hasher>::Out;
|
||||
|
||||
|
|
@ -63,6 +64,27 @@ fn random_value(seed: &mut H256) -> Vec<u8> {
|
|||
}
|
||||
}
|
||||
|
||||
struct TrieInsertionList(Vec<(Vec<u8>, Vec<u8>)>);
|
||||
impl ::std::fmt::Display for TrieInsertionList {
|
||||
fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
|
||||
write!(fmt, "{} items", self.0.len())
|
||||
}
|
||||
}
|
||||
|
||||
fn bench_root(b: &mut Criterion, name: &str, d: Vec<(Vec<u8>, Vec<u8>)>) {
|
||||
let funs = vec![
|
||||
Fun::new("Rlp", |b, d: &TrieInsertionList| b.iter(&mut ||{
|
||||
trie_root::<KeccakHasher, RlpTrieStream, _, _, _>(d.0.clone())
|
||||
})),
|
||||
Fun::new("Codec", |b, d: &TrieInsertionList| b.iter(&mut ||{
|
||||
trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(d.0.clone())
|
||||
})),
|
||||
Fun::new("-", |b, _d: &TrieInsertionList| b.iter(&mut ||{}))
|
||||
];
|
||||
|
||||
b.bench_functions(name, funs, &TrieInsertionList(d));
|
||||
}
|
||||
|
||||
fn triehash_insertions_32_mir_1k(b: &mut Criterion) {
|
||||
let st = StandardMap {
|
||||
alphabet: Alphabet::All,
|
||||
|
|
@ -71,10 +93,7 @@ fn triehash_insertions_32_mir_1k(b: &mut Criterion) {
|
|||
value_mode: ValueMode::Mirror,
|
||||
count: 1000,
|
||||
};
|
||||
let d = st.make();
|
||||
b.bench_function("triehash_insertions_32_mir_1k", |b| b.iter(&mut ||{
|
||||
let _ = trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(d.clone()).clone();
|
||||
}));
|
||||
bench_root(b, "triehash_insertions_32_mir_1k", st.make());
|
||||
}
|
||||
|
||||
fn triehash_insertions_32_ran_1k(b: &mut Criterion) {
|
||||
|
|
@ -85,10 +104,7 @@ fn triehash_insertions_32_ran_1k(b: &mut Criterion) {
|
|||
value_mode: ValueMode::Random,
|
||||
count: 1000,
|
||||
};
|
||||
let d = st.make();
|
||||
b.bench_function("triehash_insertions_32_ran_1k", |b| b.iter(&mut ||{
|
||||
let _ = trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(d.clone()).clone();
|
||||
}));
|
||||
bench_root(b, "triehash_insertions_32_ran_1k", st.make());
|
||||
}
|
||||
|
||||
fn triehash_insertions_six_high(b: &mut Criterion) {
|
||||
|
|
@ -100,9 +116,7 @@ fn triehash_insertions_six_high(b: &mut Criterion) {
|
|||
d.push((k, v))
|
||||
}
|
||||
|
||||
b.bench_function("triehash_insertions_six_high", |b| b.iter(&mut ||{
|
||||
let _ = trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(d.clone());
|
||||
}));
|
||||
bench_root(b, "triehash_insertions_six_high", d);
|
||||
}
|
||||
|
||||
fn triehash_insertions_six_mid(b: &mut Criterion) {
|
||||
|
|
@ -114,9 +128,7 @@ fn triehash_insertions_six_mid(b: &mut Criterion) {
|
|||
let v = random_value(&mut seed);
|
||||
d.push((k, v))
|
||||
}
|
||||
b.bench_function("triehash_insertions_six_mid", |b| b.iter(&mut ||{
|
||||
let _ = trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(d.clone());
|
||||
}));
|
||||
bench_root(b, "triehash_insertions_six_mid", d);
|
||||
}
|
||||
|
||||
fn triehash_insertions_random_mid(b: &mut Criterion) {
|
||||
|
|
@ -129,9 +141,7 @@ fn triehash_insertions_random_mid(b: &mut Criterion) {
|
|||
d.push((k, v))
|
||||
}
|
||||
|
||||
b.bench_function("triehash_insertions_random_mid", |b| b.iter(&mut ||{
|
||||
let _ = trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(d.clone());
|
||||
}));
|
||||
bench_root(b, "triehash_insertions_random_mid", d);
|
||||
}
|
||||
|
||||
fn triehash_insertions_six_low(b: &mut Criterion) {
|
||||
|
|
@ -144,9 +154,7 @@ fn triehash_insertions_six_low(b: &mut Criterion) {
|
|||
d.push((k, v))
|
||||
}
|
||||
|
||||
b.bench_function("triehash_insertions_six_low", |b| b.iter(&mut ||{
|
||||
let _ = trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(d.clone());
|
||||
}));
|
||||
bench_root(b, "triehash_insertions_six_low", d);
|
||||
}
|
||||
|
||||
fn typical_tx_payload(b: &mut Criterion) {
|
||||
|
|
@ -295,18 +303,9 @@ fn typical_tx_payload(b: &mut Criterion) {
|
|||
vec![0xf8, 0x6b, 0xf, 0x85, 0x1, 0x3f, 0x2e, 0xd0, 0xc0, 0x82, 0x52, 0x8, 0x94, 0x4f, 0xed, 0x1f, 0xc4, 0x14, 0x4c, 0x22, 0x3a, 0xe3, 0xc1, 0x55, 0x3b, 0xe2, 0x3, 0xcd, 0xfc, 0xbd, 0x38, 0xc5, 0x81, 0x87, 0xad, 0xf4, 0x88, 0xb7, 0xb3, 0xd8, 0x30, 0x80, 0x25, 0xa0, 0xb4, 0x31, 0xe4, 0x56, 0xc7, 0xdf, 0x1f, 0x17, 0xf2, 0xdd, 0x40, 0xf7, 0x7a, 0x10, 0xeb, 0xa7, 0xa4, 0xc1, 0x6d, 0x91, 0x1a, 0xda, 0xc1, 0xe9, 0x24, 0xf6, 0xe, 0x75, 0x58, 0xd0, 0x5e, 0x64, 0xa0, 0x69, 0xa4, 0xa, 0x93, 0xd1, 0xb, 0x5d, 0xda, 0xf6, 0xf4, 0xb7, 0xea, 0xc2, 0x4, 0x34, 0xf1, 0x80, 0xf4, 0xdf, 0x58, 0xac, 0xaa, 0x44, 0xe3, 0xfe, 0x2d, 0x62, 0x8a, 0x4, 0x51, 0x59, 0x8e]
|
||||
];
|
||||
|
||||
fn ordered_trie_root<I, V>(input: I) -> H256
|
||||
where
|
||||
I: IntoIterator<Item = V>,
|
||||
V: AsRef<[u8]> + std::fmt::Debug, // TODO: remove the debug bound when cleaning up
|
||||
{
|
||||
let input = input.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| (Compact(i as u32).encode(), v) );
|
||||
trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(input)
|
||||
}
|
||||
|
||||
b.bench_function("typical_tx_payload", |b| b.iter(&mut ||{
|
||||
let _ = ordered_trie_root(&tx_payload);
|
||||
}));
|
||||
let d = tx_payload.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, v)| (Compact(i as u32).encode(), v) )
|
||||
.collect::<Vec<_>>();
|
||||
bench_root(b, "typical_tx_payload", d);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,120 @@
|
|||
// Copyright 2015-2018 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Parity.
|
||||
|
||||
// Parity is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Parity is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
|
||||
use std::iter::once;
|
||||
use triehash::{Hasher, TrieStream};
|
||||
use codec::Encode;
|
||||
|
||||
/// Codec-flavoured TrieStream
|
||||
pub struct CodecTrieStreamAlt {
|
||||
buffer: Vec<u8>
|
||||
}
|
||||
|
||||
pub const EMPTY_TRIE: u8 = 0;
|
||||
pub const LEAF_NODE_OFFSET: u8 = 1;
|
||||
pub const LEAF_NODE_BIG: u8 = 127;
|
||||
pub const EXTENSION_NODE_OFFSET: u8 = 128;
|
||||
pub const EXTENSION_NODE_BIG: u8 = 253;
|
||||
pub const BRANCH_NODE_NO_VALUE: u8 = 254;
|
||||
pub const BRANCH_NODE_WITH_VALUE: u8 = 255;
|
||||
|
||||
impl CodecTrieStreamAlt {
|
||||
// useful for debugging but not used otherwise
|
||||
pub fn as_raw(&self) -> &[u8] { &self.buffer }
|
||||
}
|
||||
|
||||
/// Create a leaf/extension node, encoding a number of nibbles. Note that this
|
||||
/// cannot handle a number of nibbles that is zero or greater than 127 and if
|
||||
/// you attempt to do so *IT WILL PANIC*.
|
||||
fn fuse_nibbles_node<'a>(nibbles: &'a [u8], leaf: bool) -> impl Iterator<Item = u8> + 'a {
|
||||
debug_assert!(nibbles.len() < 255 + 126, "nibbles length too long. what kind of size of key are you trying to include in the trie!?!");
|
||||
// We use two ranges of possible values; one for leafs and the other for extensions.
|
||||
// Each range encodes zero following nibbles up to some maximum. If the maximum is
|
||||
// reached, then it is considered "big" and a second byte follows it in order to
|
||||
// encode a further offset to the number of nibbles of up to 255. Beyond that, we
|
||||
// cannot encode. This shouldn't be a problem though since that allows for keys of
|
||||
// up to 380 nibbles (190 bytes) and we expect key sizes to be generally 128-bit (16
|
||||
// bytes) or, at a push, 384-bit (48 bytes).
|
||||
|
||||
let (first_byte_small, big_threshold) = if leaf {
|
||||
(LEAF_NODE_OFFSET, (LEAF_NODE_BIG - LEAF_NODE_OFFSET) as usize)
|
||||
} else {
|
||||
(EXTENSION_NODE_OFFSET, (EXTENSION_NODE_BIG - EXTENSION_NODE_OFFSET) as usize)
|
||||
};
|
||||
let first_byte = first_byte_small + nibbles.len().min(big_threshold) as u8;
|
||||
once(first_byte)
|
||||
.chain(if nibbles.len() >= big_threshold { Some((nibbles.len() - big_threshold) as u8) } else { None })
|
||||
.chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None })
|
||||
.chain(nibbles[nibbles.len() % 2..].chunks(2).map(|ch| ch[0] << 4 | ch[1]))
|
||||
}
|
||||
|
||||
pub fn branch_node(has_value: bool, has_children: impl Iterator<Item = bool>) -> [u8; 3] {
|
||||
let first = if has_value {
|
||||
BRANCH_NODE_WITH_VALUE
|
||||
} else {
|
||||
BRANCH_NODE_NO_VALUE
|
||||
};
|
||||
let mut bitmap: u16 = 0;
|
||||
let mut cursor: u16 = 1;
|
||||
for v in has_children {
|
||||
if v { bitmap |= cursor }
|
||||
cursor <<= 1;
|
||||
}
|
||||
[first, (bitmap % 256 ) as u8, (bitmap / 256 ) as u8]
|
||||
}
|
||||
|
||||
impl TrieStream for CodecTrieStreamAlt {
|
||||
fn new() -> Self { Self {buffer: Vec::new() } }
|
||||
fn append_empty_data(&mut self) {
|
||||
self.buffer.push(EMPTY_TRIE);
|
||||
}
|
||||
|
||||
fn append_leaf(&mut self, key: &[u8], value: &[u8]) {
|
||||
self.buffer.extend(fuse_nibbles_node(key, true));
|
||||
// OPTIMISATION: I'd like to do `hpe.encode_to(&mut self.buffer);` here; need an `impl<'a> Encode for impl Iterator<Item = u8> + 'a`?
|
||||
value.encode_to(&mut self.buffer);
|
||||
}
|
||||
fn begin_branch(&mut self, maybe_value: Option<&[u8]>, has_children: impl Iterator<Item = bool>) {
|
||||
// println!("[begin_branch] pushing BRANCH_NODE");
|
||||
self.buffer.extend(&branch_node(maybe_value.is_some(), has_children));
|
||||
// Push the value if one exists.
|
||||
if let Some(value) = maybe_value {
|
||||
value.encode_to(&mut self.buffer);
|
||||
}
|
||||
// println!("[begin_branch] buffer so far: {:#x?}", self.buffer);
|
||||
}
|
||||
fn append_extension(&mut self, key: &[u8]) {
|
||||
self.buffer.extend(fuse_nibbles_node(key, false));
|
||||
}
|
||||
fn append_substream<H: Hasher>(&mut self, other: Self) {
|
||||
let data = other.out();
|
||||
// println!("[append_substream] START own buffer: {:x?}", self.buffer);
|
||||
// println!("[append_substream] START other buffer: {:x?}", data);
|
||||
match data.len() {
|
||||
0...31 => {
|
||||
// println!("[append_substream] appending data, because data.len() = {}", data.len());
|
||||
data.encode_to(&mut self.buffer)
|
||||
},
|
||||
_ => {
|
||||
// println!("[append_substream] would have hashed, because data.len() = {}", data.len());
|
||||
// data.encode_to(&mut self.buffer)
|
||||
// TODO: re-enable hashing before merging
|
||||
H::hash(&data).as_ref().encode_to(&mut self.buffer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn out(self) -> Vec<u8> { self.buffer }
|
||||
}
|
||||
|
|
@ -29,11 +29,15 @@ extern crate keccak_hasher;
|
|||
|
||||
mod codec_error;
|
||||
mod parity_node_codec;
|
||||
mod parity_node_codec_alt;
|
||||
mod codec_triestream;
|
||||
mod codec_triestream_alt;
|
||||
|
||||
pub use codec_error::CodecError;
|
||||
pub use parity_node_codec::ParityNodeCodec;
|
||||
pub use parity_node_codec_alt::ParityNodeCodecAlt;
|
||||
pub use codec_triestream::CodecTrieStream;
|
||||
pub use codec_triestream_alt::CodecTrieStreamAlt;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
|
@ -45,19 +49,36 @@ mod tests {
|
|||
use patricia_trie::{Hasher, DBValue, TrieMut, TrieDBMut};
|
||||
|
||||
fn check_equivalent(input: Vec<(&[u8], &[u8])>) {
|
||||
let closed_form = trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(input.clone());
|
||||
let d = unhashed_trie::<KeccakHasher, CodecTrieStream, _, _, _>(input.clone());
|
||||
println!("Data: {:#x?}, {:#x?}", d, KeccakHasher::hash(&d[..]));
|
||||
let persistent = {
|
||||
let mut memdb = MemoryDB::<KeccakHasher, DBValue>::from_null_node(&[0u8][..], [0u8][..].into());
|
||||
let mut root = <KeccakHasher as Hasher>::Out::default();
|
||||
let mut t = TrieDBMut::<KeccakHasher, ParityNodeCodec<KeccakHasher>>::new(&mut memdb, &mut root);
|
||||
for (x, y) in input {
|
||||
t.insert(x, y).unwrap();
|
||||
}
|
||||
t.root().clone()
|
||||
};
|
||||
assert_eq!(closed_form, persistent);
|
||||
{
|
||||
let closed_form = trie_root::<KeccakHasher, CodecTrieStream, _, _, _>(input.clone());
|
||||
let d = unhashed_trie::<KeccakHasher, CodecTrieStream, _, _, _>(input.clone());
|
||||
println!("Data: {:#x?}, {:#x?}", d, KeccakHasher::hash(&d[..]));
|
||||
let persistent = {
|
||||
let mut memdb = MemoryDB::<KeccakHasher, DBValue>::from_null_node(&[0u8][..], [0u8][..].into());
|
||||
let mut root = <KeccakHasher as Hasher>::Out::default();
|
||||
let mut t = TrieDBMut::<KeccakHasher, ParityNodeCodec<KeccakHasher>>::new(&mut memdb, &mut root);
|
||||
for (x, y) in input.clone() {
|
||||
t.insert(x, y).unwrap();
|
||||
}
|
||||
t.root().clone()
|
||||
};
|
||||
assert_eq!(closed_form, persistent);
|
||||
}
|
||||
{
|
||||
let closed_form = trie_root::<KeccakHasher, CodecTrieStreamAlt, _, _, _>(input.clone());
|
||||
let d = unhashed_trie::<KeccakHasher, CodecTrieStreamAlt, _, _, _>(input.clone());
|
||||
println!("Data: {:#x?}, {:#x?}", d, KeccakHasher::hash(&d[..]));
|
||||
let persistent = {
|
||||
let mut memdb = MemoryDB::<KeccakHasher, DBValue>::from_null_node(&[0u8][..], [0u8][..].into());
|
||||
let mut root = <KeccakHasher as Hasher>::Out::default();
|
||||
let mut t = TrieDBMut::<KeccakHasher, ParityNodeCodecAlt<KeccakHasher>>::new(&mut memdb, &mut root);
|
||||
for (x, y) in input {
|
||||
t.insert(x, y).unwrap();
|
||||
}
|
||||
t.root().clone()
|
||||
};
|
||||
assert_eq!(closed_form, persistent);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,227 @@
|
|||
// Copyright 2015-2018 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Parity.
|
||||
|
||||
// Parity is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Parity is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Parity. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! `NodeCodec` implementation for Rlp
|
||||
|
||||
use std::marker::PhantomData;
|
||||
use patricia_trie::{DBValue, NibbleSlice, NodeCodec, node::Node, ChildReference, Hasher};
|
||||
use codec::{Encode, Decode, Input, Output, Compact};
|
||||
use codec_error::CodecError;
|
||||
use codec_triestream::{EMPTY_TRIE, LEAF_NODE_OFFSET, LEAF_NODE_BIG, EXTENSION_NODE_OFFSET,
|
||||
EXTENSION_NODE_BIG, BRANCH_NODE_NO_VALUE, BRANCH_NODE_WITH_VALUE, branch_node};
|
||||
|
||||
/// Concrete implementation of a `NodeCodec` with Parity Codec encoding, generic over the `Hasher`
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ParityNodeCodecAlt<H: Hasher>(PhantomData<H>);
|
||||
|
||||
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
|
||||
enum NodeHeader {
|
||||
Null,
|
||||
Branch(bool),
|
||||
Extension(usize),
|
||||
Leaf(usize),
|
||||
}
|
||||
|
||||
const LEAF_NODE_THRESHOLD: u8 = LEAF_NODE_BIG - LEAF_NODE_OFFSET;
|
||||
const EXTENSION_NODE_THRESHOLD: u8 = EXTENSION_NODE_BIG - EXTENSION_NODE_OFFSET; //125
|
||||
const LEAF_NODE_SMALL_MAX: u8 = LEAF_NODE_BIG - 1;
|
||||
const EXTENSION_NODE_SMALL_MAX: u8 = EXTENSION_NODE_BIG - 1;
|
||||
|
||||
impl Encode for NodeHeader {
|
||||
fn encode_to<T: Output>(&self, output: &mut T) {
|
||||
match self {
|
||||
NodeHeader::Null => output.push_byte(EMPTY_TRIE),
|
||||
|
||||
NodeHeader::Branch(true) => output.push_byte(BRANCH_NODE_WITH_VALUE),
|
||||
NodeHeader::Branch(false) => output.push_byte(BRANCH_NODE_NO_VALUE),
|
||||
|
||||
NodeHeader::Leaf(nibble_count) if *nibble_count < LEAF_NODE_THRESHOLD as usize =>
|
||||
output.push_byte(LEAF_NODE_OFFSET + *nibble_count as u8),
|
||||
NodeHeader::Leaf(nibble_count) => {
|
||||
output.push_byte(LEAF_NODE_BIG);
|
||||
output.push_byte((*nibble_count - LEAF_NODE_THRESHOLD as usize) as u8);
|
||||
}
|
||||
|
||||
NodeHeader::Extension(nibble_count) if *nibble_count < EXTENSION_NODE_THRESHOLD as usize =>
|
||||
output.push_byte(EXTENSION_NODE_OFFSET + *nibble_count as u8),
|
||||
NodeHeader::Extension(nibble_count) => {
|
||||
output.push_byte(EXTENSION_NODE_BIG);
|
||||
output.push_byte((*nibble_count - EXTENSION_NODE_THRESHOLD as usize) as u8);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Decode for NodeHeader {
|
||||
fn decode<I: Input>(input: &mut I) -> Option<Self> {
|
||||
Some(match input.read_byte()? {
|
||||
EMPTY_TRIE => NodeHeader::Null, // 0
|
||||
|
||||
i @ LEAF_NODE_OFFSET ... LEAF_NODE_SMALL_MAX => // 1 ... (127 - 1)
|
||||
NodeHeader::Leaf((i - LEAF_NODE_OFFSET) as usize),
|
||||
LEAF_NODE_BIG => // 127
|
||||
NodeHeader::Leaf(input.read_byte()? as usize + LEAF_NODE_THRESHOLD as usize),
|
||||
|
||||
i @ EXTENSION_NODE_OFFSET ... EXTENSION_NODE_SMALL_MAX =>// 128 ... (253 - 1)
|
||||
NodeHeader::Extension((i - EXTENSION_NODE_OFFSET) as usize),
|
||||
EXTENSION_NODE_BIG => // 253
|
||||
NodeHeader::Extension(input.read_byte()? as usize + EXTENSION_NODE_THRESHOLD as usize),
|
||||
|
||||
BRANCH_NODE_NO_VALUE => NodeHeader::Branch(false), // 254
|
||||
BRANCH_NODE_WITH_VALUE => NodeHeader::Branch(true), // 255
|
||||
|
||||
_ => unreachable!(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// encode branch as 3 bytes: header including value existence + 16-bit bitmap for branch existence
|
||||
|
||||
fn take<'a>(input: &mut &'a[u8], count: usize) -> Option<&'a[u8]> {
|
||||
if input.len() < count {
|
||||
return None
|
||||
}
|
||||
let r = &(*input)[..count];
|
||||
*input = &(*input)[count..];
|
||||
Some(r)
|
||||
}
|
||||
|
||||
fn partial_to_key(partial: &[u8], offset: u8, big: u8) -> Vec<u8> {
|
||||
let nibble_count = (partial.len() - 1) * 2 + if partial[0] & 16 == 16 { 1 } else { 0 };
|
||||
let (first_byte_small, big_threshold) = (offset, (big - offset) as usize);
|
||||
let mut output = vec![first_byte_small + nibble_count.min(big_threshold) as u8];
|
||||
if nibble_count >= big_threshold { output.push((nibble_count - big_threshold) as u8) }
|
||||
if nibble_count % 2 == 1 {
|
||||
output.push(partial[0] & 0x0f);
|
||||
}
|
||||
output.extend_from_slice(&partial[1..]);
|
||||
output
|
||||
}
|
||||
|
||||
// NOTE: what we'd really like here is:
|
||||
// `impl<H: Hasher> NodeCodec<H> for RlpNodeCodec<H> where H::Out: Decodable`
|
||||
// but due to the current limitations of Rust const evaluation we can't
|
||||
// do `const HASHED_NULL_NODE: H::Out = H::Out( … … )`. Perhaps one day soon?
|
||||
impl<H: Hasher> NodeCodec<H> for ParityNodeCodecAlt<H> {
|
||||
type Error = CodecError;
|
||||
|
||||
fn hashed_null_node() -> H::Out {
|
||||
H::hash(&[0u8][..])
|
||||
}
|
||||
|
||||
fn decode(data: &[u8]) -> ::std::result::Result<Node, Self::Error> {
|
||||
let input = &mut &*data;
|
||||
match NodeHeader::decode(input).ok_or(CodecError::BadFormat)? {
|
||||
NodeHeader::Null => Ok(Node::Empty),
|
||||
NodeHeader::Branch(has_value) => {
|
||||
let bitmap = u16::decode(input).ok_or(CodecError::BadFormat)?;
|
||||
let value = if has_value {
|
||||
let count = <Compact<u32>>::decode(input).ok_or(CodecError::BadFormat)?.0 as usize;
|
||||
Some(take(input, count).ok_or(CodecError::BadFormat)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut children = [None; 16];
|
||||
let mut pot_cursor = 1;
|
||||
for i in 0..16 {
|
||||
if bitmap & pot_cursor != 0 {
|
||||
let count = <Compact<u32>>::decode(input).ok_or(CodecError::BadFormat)?.0 as usize;
|
||||
children[i] = Some(take(input, count).ok_or(CodecError::BadFormat)?);
|
||||
}
|
||||
pot_cursor <<= 1;
|
||||
}
|
||||
Ok(Node::Branch(children, value))
|
||||
}
|
||||
NodeHeader::Extension(nibble_count) => {
|
||||
let nibble_data = take(input, (nibble_count + 1) / 2).ok_or(CodecError::BadFormat)?;
|
||||
let nibble_slice = NibbleSlice::new_offset(nibble_data, nibble_count % 2);
|
||||
let count = <Compact<u32>>::decode(input).ok_or(CodecError::BadFormat)?.0 as usize;
|
||||
Ok(Node::Extension(nibble_slice, take(input, count).ok_or(CodecError::BadFormat)?))
|
||||
}
|
||||
NodeHeader::Leaf(nibble_count) => {
|
||||
let nibble_data = take(input, (nibble_count + 1) / 2).ok_or(CodecError::BadFormat)?;
|
||||
let nibble_slice = NibbleSlice::new_offset(nibble_data, nibble_count % 2);
|
||||
let count = <Compact<u32>>::decode(input).ok_or(CodecError::BadFormat)?.0 as usize;
|
||||
Ok(Node::Leaf(nibble_slice, take(input, count).ok_or(CodecError::BadFormat)?))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn try_decode_hash(data: &[u8]) -> Option<H::Out> {
|
||||
if data.len() == H::LENGTH {
|
||||
let mut r = H::Out::default();
|
||||
r.as_mut().copy_from_slice(data);
|
||||
Some(r)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn is_empty_node(data: &[u8]) -> bool {
|
||||
data == &[EMPTY_TRIE][..]
|
||||
}
|
||||
fn empty_node() -> Vec<u8> {
|
||||
vec![EMPTY_TRIE]
|
||||
}
|
||||
|
||||
// TODO: refactor this so that `partial` isn't already encoded with HPE. Should just be an `impl Iterator<Item=u8>`.
|
||||
fn leaf_node(partial: &[u8], value: &[u8]) -> Vec<u8> {
|
||||
let mut output = partial_to_key(partial, LEAF_NODE_OFFSET, LEAF_NODE_BIG);
|
||||
value.encode_to(&mut output);
|
||||
// println!("leaf_node: {:#x?}", output);
|
||||
output
|
||||
|
||||
}
|
||||
|
||||
// TODO: refactor this so that `partial` isn't already encoded with HPE. Should just be an `impl Iterator<Item=u8>`.
|
||||
fn ext_node(partial: &[u8], child: ChildReference<H::Out>) -> Vec<u8> {
|
||||
let mut output = partial_to_key(partial, EXTENSION_NODE_OFFSET, EXTENSION_NODE_BIG);
|
||||
match child {
|
||||
ChildReference::Hash(h) =>
|
||||
h.as_ref().encode_to(&mut output),
|
||||
ChildReference::Inline(inline_data, len) =>
|
||||
(&AsRef::<[u8]>::as_ref(&inline_data)[..len]).encode_to(&mut output),
|
||||
};
|
||||
// println!("ext_node: {:#x?}", output);
|
||||
output
|
||||
}
|
||||
|
||||
fn branch_node<I>(children: I, maybe_value: Option<DBValue>) -> Vec<u8>
|
||||
where I: IntoIterator<Item=Option<ChildReference<H::Out>>> + Iterator<Item=Option<ChildReference<H::Out>>>
|
||||
{
|
||||
let mut output = vec![0, 0, 0];
|
||||
let have_value = if let Some(value) = maybe_value {
|
||||
(&*value).encode_to(&mut output);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let prefix = branch_node(have_value, children.map(|maybe_child| match maybe_child {
|
||||
Some(ChildReference::Hash(h)) => {
|
||||
h.as_ref().encode_to(&mut output);
|
||||
true
|
||||
}
|
||||
Some(ChildReference::Inline(inline_data, len)) => {
|
||||
(&AsRef::<[u8]>::as_ref(&inline_data)[..len]).encode_to(&mut output);
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
}));
|
||||
output[0..3].copy_from_slice(&prefix[..]);
|
||||
// println!("branch_node: {:#x?}", output);
|
||||
output
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue