diff --git a/Cargo.lock b/Cargo.lock index b070acd39..40ca04aad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1853,6 +1853,7 @@ dependencies = [ "bv 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index fb1eca89f..9baeb8b8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ bs58 = "0.2.0" bv = { version = "0.11.0", features = ["serde"] } byteorder = "1.2.1" chrono = { version = "0.4.0", features = ["serde"] } +fnv = "1.0.6" hashbrown = "0.1.8" indexmap = "1.0" itertools = "0.8.0" diff --git a/benches/bloom.rs b/benches/bloom.rs new file mode 100644 index 000000000..ab52b1334 --- /dev/null +++ b/benches/bloom.rs @@ -0,0 +1,101 @@ +#![feature(test)] + +extern crate test; +use bv::BitVec; +use fnv::FnvHasher; +use solana::bloom::{Bloom, BloomHashIndex}; +use solana_sdk::hash::{hash, Hash}; +use solana_sdk::signature::Signature; +//use std::collections::HashSet; +use hashbrown::HashSet; +use std::hash::Hasher; +use test::Bencher; + +#[bench] +#[ignore] +fn bench_bits_set(bencher: &mut Bencher) { + let mut bits: BitVec = BitVec::new_fill(false, 38_340_234 as u64); + let mut hasher: FnvHasher = Default::default(); + + bencher.iter(|| { + let idx = hasher.finish() % bits.len(); + bits.set(idx, true); + hasher.write_u64(idx); + }); + // subtract the next bencher result from this one to get a number for raw + // bits.set() +} + +#[bench] +#[ignore] +fn bench_bits_set_hasher(bencher: &mut Bencher) { + let bits: BitVec = BitVec::new_fill(false, 38_340_234 as u64); + let mut hasher: FnvHasher = Default::default(); + + bencher.iter(|| { + let idx = hasher.finish() % bits.len(); + hasher.write_u64(idx); + }); +} + +#[bench] +#[ignore] +fn bench_sigs_bloom(bencher: &mut Bencher) { + // 1M TPS * 1s (length of block in sigs) == 1M items in filter + // 1.0E-8 false positive rate + // https://hur.st/bloomfilter/?n=1000000&p=1.0E-8&m=&k= + let last_id = hash(Hash::default().as_ref()); + // eprintln!("last_id = {:?}", last_id); + let keys = (0..27) + .into_iter() + .map(|i| last_id.hash_at_index(i)) + .collect(); + let mut sigs: Bloom = Bloom::new(38_340_234, keys); + + let mut id = last_id; + let mut falses = 0; + let mut iterations = 0; + bencher.iter(|| { + id = hash(id.as_ref()); + let mut sigbytes = Vec::from(id.as_ref()); + id = hash(id.as_ref()); + sigbytes.extend(id.as_ref()); + + let sig = Signature::new(&sigbytes); + if sigs.contains(&sig) { + falses += 1; + } + sigs.add(&sig); + sigs.contains(&sig); + iterations += 1; + }); + assert_eq!(falses, 0); +} + +#[bench] +#[ignore] +fn bench_sigs_hashmap(bencher: &mut Bencher) { + // same structure as above, new + let last_id = hash(Hash::default().as_ref()); + // eprintln!("last_id = {:?}", last_id); + let mut sigs: HashSet = HashSet::new(); + + let mut id = last_id; + let mut falses = 0; + let mut iterations = 0; + bencher.iter(|| { + id = hash(id.as_ref()); + let mut sigbytes = Vec::from(id.as_ref()); + id = hash(id.as_ref()); + sigbytes.extend(id.as_ref()); + + let sig = Signature::new(&sigbytes); + if sigs.contains(&sig) { + falses += 1; + } + sigs.insert(sig); + sigs.contains(&sig); + iterations += 1; + }); + assert_eq!(falses, 0); +} diff --git a/src/bloom.rs b/src/bloom.rs index 3027db5d6..f81f0730f 100644 --- a/src/bloom.rs +++ b/src/bloom.rs @@ -1,8 +1,9 @@ //! Simple Bloom Filter use bv::BitVec; +use fnv::FnvHasher; use rand::{self, Rng}; -use solana_sdk::hash::hashv; use std::cmp; +use std::hash::Hasher; use std::marker::PhantomData; /// Generate a stable hash of `self` for each `hash_index` @@ -63,33 +64,34 @@ impl Bloom { } } -fn to_slice(v: u64) -> [u8; 8] { - [ - v as u8, - (v >> 8) as u8, - (v >> 16) as u8, - (v >> 24) as u8, - (v >> 32) as u8, - (v >> 40) as u8, - (v >> 48) as u8, - (v >> 56) as u8, - ] -} - -fn from_slice(v: &[u8]) -> u64 { - u64::from(v[0]) - | u64::from(v[1]) << 8 - | u64::from(v[2]) << 16 - | u64::from(v[3]) << 24 - | u64::from(v[4]) << 32 - | u64::from(v[5]) << 40 - | u64::from(v[6]) << 48 - | u64::from(v[7]) << 56 -} +//fn to_slice(v: u64) -> [u8; 8] { +// [ +// v as u8, +// (v >> 8) as u8, +// (v >> 16) as u8, +// (v >> 24) as u8, +// (v >> 32) as u8, +// (v >> 40) as u8, +// (v >> 48) as u8, +// (v >> 56) as u8, +// ] +//} +//fn from_slice(v: &[u8]) -> u64 { +// u64::from(v[0]) +// | u64::from(v[1]) << 8 +// | u64::from(v[2]) << 16 +// | u64::from(v[3]) << 24 +// | u64::from(v[4]) << 32 +// | u64::from(v[5]) << 40 +// | u64::from(v[6]) << 48 +// | u64::from(v[7]) << 56 +//} +// fn slice_hash(slice: &[u8], hash_index: u64) -> u64 { - let hash = hashv(&[slice, &to_slice(hash_index)]); - from_slice(hash.as_ref()) + let mut hasher = FnvHasher::with_key(hash_index); + hasher.write(slice); + hasher.finish() } impl> BloomHashIndex for T { @@ -102,15 +104,15 @@ impl> BloomHashIndex for T { mod test { use super::*; use solana_sdk::hash::{hash, Hash}; - #[test] - fn test_slice() { - assert_eq!(from_slice(&to_slice(10)), 10); - assert_eq!(from_slice(&to_slice(0x7fff7fff)), 0x7fff7fff); - assert_eq!( - from_slice(&to_slice(0x7fff7fff7fff7fff)), - 0x7fff7fff7fff7fff - ); - } + // #[test] + // fn test_slice() { + // assert_eq!(from_slice(&to_slice(10)), 10); + // assert_eq!(from_slice(&to_slice(0x7fff7fff)), 0x7fff7fff); + // assert_eq!( + // from_slice(&to_slice(0x7fff7fff7fff7fff)), + // 0x7fff7fff7fff7fff + // ); + // } #[test] fn test_bloom_filter() {