From edb18349c9135916878b15b20e289770f284c147 Mon Sep 17 00:00:00 2001 From: Ryan Zhu Date: Sun, 23 Feb 2020 02:12:37 +0800 Subject: [PATCH] Improve merkle-tree nodes capacity computing (#8273) * Improve merkle-tree nodes capacity computing * Add test cases for math compute of merkle-tree nodes capacity --- Cargo.lock | 16 +++++++++ merkle-tree/Cargo.toml | 1 + merkle-tree/src/merkle_tree.rs | 64 ++++++++++++++++++++++++++++++---- 3 files changed, 75 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e00f51059..3fdb0bb2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1120,6 +1120,14 @@ name = "fake-simd" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "fast-math" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ieee754 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "feature-probe" version = "0.1.1" @@ -1637,6 +1645,11 @@ dependencies = [ "unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ieee754" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "indexed" version = "0.1.1" @@ -4226,6 +4239,7 @@ dependencies = [ name = "solana-merkle-tree" version = "1.1.0" dependencies = [ + "fast-math 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "solana-sdk 1.1.0", ] @@ -6179,6 +6193,7 @@ dependencies = [ "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +"checksum fast-math 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2465292146cdfc2011350fe3b1c616ac83cf0faeedb33463ba1c332ed8948d66" "checksum feature-probe 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da" "checksum filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "450537dc346f0c4d738dda31e790da1da5d4bd12145aad4da0d03d713cb3794f" "checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" @@ -6237,6 +6252,7 @@ dependencies = [ "checksum hyper-rustls 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89109920197f2c90d75e82addbb96bf424570790d310cc2b18f0b33f4a9cc43" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +"checksum ieee754 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9007da9cacbd3e6343da136e98b0d2df013f553d35bdec8b518f07bea768e19c" "checksum indexed 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d480125acf340d6a6e59dab69ae19d6fca3a906e1eade277671272cc8f73794b" "checksum indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" "checksum indicatif 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49a68371cf417889c9d7f98235b7102ea7c54fc59bcbd22f3dea785be9d27e40" diff --git a/merkle-tree/Cargo.toml b/merkle-tree/Cargo.toml index b045b5e41..96bcbd76e 100644 --- a/merkle-tree/Cargo.toml +++ b/merkle-tree/Cargo.toml @@ -10,6 +10,7 @@ edition = "2018" [dependencies] solana-sdk = { path = "../sdk", version = "1.1.0" } +fast-math = "0.1" [dev-dependencies] hex = "0.4.0" diff --git a/merkle-tree/src/merkle_tree.rs b/merkle-tree/src/merkle_tree.rs index 982a21523..281566f82 100644 --- a/merkle-tree/src/merkle_tree.rs +++ b/merkle-tree/src/merkle_tree.rs @@ -75,13 +75,27 @@ impl MerkleTree { } } - fn calculate_vec_capacity(mut leaf_count: usize) -> usize { - let mut capacity = 0; - while leaf_count > 0 { - capacity += leaf_count; - leaf_count = MerkleTree::next_level_len(leaf_count); + fn calculate_vec_capacity(leaf_count: usize) -> usize { + // the most nodes consuming case is when n-1 is full balanced binary tree + // then n will cause the previous tree add a left only path to the root + // this cause the total nodes number increased by tree height, we use this + // condition as the max nodes consuming case. + // n is current leaf nodes number + // asuming n-1 is a full balanced binary tree, n-1 tree nodes number will be + // 2(n-1) - 1, n tree height is closed to log2(n) + 1 + // so the max nodes number is 2(n-1) - 1 + log2(n) + 1, finally we can use + // 2n + log2(n+1) as a safe capacity value. + // test results: + // 8192 leaf nodes(full balanced): + // computed cap is 16398, actually using is 16383 + // 8193 leaf nodes:(full balanced plus 1 leaf): + // computed cap is 16400, actually using is 16398 + // about performance: current used fast_math log2 code is constant algo time + if leaf_count > 0 { + fast_math::log2_raw(leaf_count as f32) as usize + 2 * leaf_count + 1 + } else { + 0 } - capacity } pub fn new>(items: &[T]) -> Self { @@ -247,6 +261,44 @@ mod tests { ProofEntry::new(&Hash::default(), None, Some(&Hash::default())); } + #[test] + fn test_nodes_capacity_compute() { + let iteration_count = |mut leaf_count: usize| -> usize { + let mut capacity = 0; + while leaf_count > 0 { + capacity += leaf_count; + leaf_count = MerkleTree::next_level_len(leaf_count); + } + capacity + }; + + // test max 64k leaf nodes compute + for leaf_count in 0..65536 { + let math_count = MerkleTree::calculate_vec_capacity(leaf_count); + let iter_count = iteration_count(leaf_count); + assert!(math_count >= iter_count); + } + } + + #[test] + fn test_node_capacity_constant_time() { + use std::time::Instant; + + // trigger function invoking optimize once + MerkleTree::calculate_vec_capacity(2); + + // record time spending + let mut time_record = Instant::now(); + MerkleTree::calculate_vec_capacity(4); + let small_leaf_count_duration = time_record.elapsed(); + + time_record = Instant::now(); + MerkleTree::calculate_vec_capacity(65536); + let large_leaf_count_duration = time_record.elapsed(); + // large leafs should not bring time inceasing + assert!(large_leaf_count_duration < 2 * small_leaf_count_duration); + } + #[test] #[should_panic] fn test_proof_entry_instantiation_both_clear() {