From b19408a0a8f44f4f9901198a1879635076af1782 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 24 Nov 2016 13:18:18 +0300 Subject: [PATCH 1/6] dedicated bencer --- Cargo.lock | 14 ++++ Cargo.toml | 1 + bencher/Cargo.toml | 19 ++++++ bencher/src/database.rs | 145 ++++++++++++++++++++++++++++++++++++++++ bencher/src/main.rs | 60 +++++++++++++++++ 5 files changed, 239 insertions(+) create mode 100644 bencher/Cargo.toml create mode 100644 bencher/src/database.rs create mode 100644 bencher/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 8489662d..554f7191 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,6 +64,19 @@ name = "base58" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bencher" +version = "0.1.0" +dependencies = [ + "chain 0.1.0", + "db 0.1.0", + "ethcore-devtools 1.3.0", + "primitives 0.1.0", + "test-data 0.1.0", + "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "verification 0.1.0", +] + [[package]] name = "bit-vec" version = "0.4.3" @@ -482,6 +495,7 @@ name = "pbtc" version = "0.1.0" dependencies = [ "app_dirs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bencher 0.1.0", "chain 0.1.0", "clap 2.18.0 (registry+https://github.com/rust-lang/crates.io-index)", "db 0.1.0", diff --git a/Cargo.toml b/Cargo.toml index 6a16dd68..07eb1989 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ db = { path = "db" } verification = { path = "verification" } sync = { path = "sync" } import = { path = "import" } +bencher = { path = "bencher" } [[bin]] path = "pbtc/main.rs" diff --git a/bencher/Cargo.toml b/bencher/Cargo.toml new file mode 100644 index 00000000..0aec1566 --- /dev/null +++ b/bencher/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bencher" +version = "0.1.0" +license = "GPL-3.0" +authors = ["Ethcore "] +description = "Parity bitcoin client." + +[dependencies] +db = { path = "../db" } +verification = { path = "../verification" } +chain = { path = "../chain" } +primitives = { path = "../primitives" } +ethcore-devtools = { path = "../devtools" } +test-data = { path = "../test-data" } +time = "*" + +[[bin]] +path = "src/main.rs" +name = "bencher" diff --git a/bencher/src/database.rs b/bencher/src/database.rs new file mode 100644 index 00000000..7f33dcc9 --- /dev/null +++ b/bencher/src/database.rs @@ -0,0 +1,145 @@ +use devtools::RandomTempPath; +use db::{Storage, BlockStapler, BlockProvider, BlockRef, BlockInsertedChain}; +use test_data; + +use super::Benchmark; + +pub fn fetch(benchmark: &mut Benchmark) { + // params + const BLOCKS: usize = 10000; + + // test setup + let path = RandomTempPath::create_dir(); + let store = Storage::new(path.as_path()).unwrap(); + + let genesis = test_data::genesis(); + store.insert_block(&genesis).unwrap(); + + let genesis = test_data::genesis(); + store.insert_block(&genesis).unwrap(); + + let mut rolling_hash = genesis.hash(); + let mut blocks = Vec::new(); + let mut hashes = Vec::new(); + + for x in 0..BLOCKS { + let next_block = test_data::block_builder() + .transaction().coinbase().build() + .transaction().output().value(5000000000).build().build() + .merkled_header().parent(rolling_hash.clone()).nonce(x as u32).build() + .build(); + rolling_hash = next_block.hash(); + blocks.push(next_block); + hashes.push(rolling_hash.clone()); + } + + for block in blocks.iter() { store.insert_block(block).unwrap(); } + + // bench + benchmark.start(); + for _ in 0..BLOCKS { + let block = store.block(BlockRef::Hash(hashes[0].clone())).unwrap(); + assert_eq!(&block.hash(), &hashes[0]); + } + benchmark.stop(); +} + +pub fn write(benchmark: &mut Benchmark) { + // params + const BLOCKS: usize = 1000; + + // setup + let path = RandomTempPath::create_dir(); + let store = Storage::new(path.as_path()).unwrap(); + + let genesis = test_data::genesis(); + store.insert_block(&genesis).unwrap(); + + let mut rolling_hash = genesis.hash(); + + let mut blocks = Vec::new(); + + for x in 0..BLOCKS { + let next_block = test_data::block_builder() + .transaction().coinbase().build() + .transaction().output().value(5000000000).build().build() + .merkled_header().parent(rolling_hash.clone()).nonce(x as u32).build() + .build(); + rolling_hash = next_block.hash(); + blocks.push(next_block); + } + + // bench + benchmark.start(); + for idx in 0..BLOCKS { + store.insert_block(&blocks[idx]).unwrap(); + } + benchmark.stop(); +} + +pub fn reorg_short(benchmark: &mut Benchmark) { + // params + const BLOCKS: usize = 1000; + + // setup + let path = RandomTempPath::create_dir(); + let store = Storage::new(path.as_path()).unwrap(); + + let genesis = test_data::genesis(); + store.insert_block(&genesis).unwrap(); + + let mut rolling_hash = genesis.hash(); + + let mut blocks = Vec::new(); + + for x in 0..1000 { + let base = rolling_hash.clone(); + + let next_block = test_data::block_builder() + .transaction().coinbase().build() + .transaction().output().value(5000000000).build().build() + .merkled_header().parent(rolling_hash.clone()).nonce(x*4).build() + .build(); + rolling_hash = next_block.hash(); + blocks.push(next_block); + + let next_block_side = test_data::block_builder() + .transaction().coinbase().build() + .transaction().output().value(5000000000).build().build() + .merkled_header().parent(base).nonce(x * 4 + 2).build() + .build(); + let next_base = next_block_side.hash(); + blocks.push(next_block_side); + + let next_block_side_continue = test_data::block_builder() + .transaction().coinbase().build() + .transaction().output().value(5000000000).build().build() + .merkled_header().parent(next_base).nonce(x * 4 + 3).build() + .build(); + blocks.push(next_block_side_continue); + + let next_block_continue = test_data::block_builder() + .transaction().coinbase().build() + .transaction().output().value(5000000000).build().build() + .merkled_header().parent(rolling_hash.clone()).nonce(x * 4+1).build() + .build(); + rolling_hash = next_block_continue.hash(); + blocks.push(next_block_continue); + } + + let mut total: usize = 0; + let mut reorgs: usize = 0; + + // bench + benchmark.start(); + for idx in 0..BLOCKS { + total += 1; + if let BlockInsertedChain::Reorganized(_) = store.insert_block(&blocks[idx]).unwrap() { + reorgs += 1; + } + } + benchmark.stop(); + + assert_eq!(1000, total); + assert_eq!(499, reorgs); +} diff --git a/bencher/src/main.rs b/bencher/src/main.rs new file mode 100644 index 00000000..b0706323 --- /dev/null +++ b/bencher/src/main.rs @@ -0,0 +1,60 @@ +extern crate db; +extern crate chain; +extern crate ethcore_devtools as devtools; +extern crate test_data; +extern crate time; + +mod database; + +use time::{PreciseTime, Duration}; +use std::io::Write; +use std::str; + +#[derive(Default)] +pub struct Benchmark { + start: Option, + end: Option, +} + +impl Benchmark { + pub fn start(&mut self) { + self.start = Some(PreciseTime::now()); + } + + pub fn stop(&mut self) { + self.end = Some(PreciseTime::now()); + } + + pub fn evaluate(&self) -> Duration { + self.start.expect("benchmarch never ended").to(self.end.expect("benchmark never started")) + } +} + +fn decimal_mark(s: String) -> String { + let bytes: Vec<_> = s.bytes().rev().collect(); + let chunks: Vec<_> = bytes.chunks(3).map(|chunk| str::from_utf8(chunk).unwrap()).collect(); + let result: Vec<_> = chunks.join(",").bytes().rev().collect(); + String::from_utf8(result).unwrap() +} + + +fn run_benchmark(name: &str, f: F) where F: FnOnce(&mut Benchmark) { + print!("{}: ", name); + ::std::io::stdout().flush().unwrap(); + + let mut benchmark = Benchmark::default(); + f(&mut benchmark); + println!("{} ns", decimal_mark(format!("{}", benchmark.evaluate().num_nanoseconds().unwrap()))); +} + +macro_rules! benchmark { + ($t:expr) => { + run_benchmark(stringify!($t), $t); + }; +} + +fn main() { + benchmark!(database::fetch); + benchmark!(database::write); + benchmark!(database::reorg_short); +} From 3597c2e8cc9999c2ad64aba70d14a5f774259181 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 24 Nov 2016 13:28:18 +0300 Subject: [PATCH 2/6] use const --- bencher/src/database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bencher/src/database.rs b/bencher/src/database.rs index 7f33dcc9..c3e3c491 100644 --- a/bencher/src/database.rs +++ b/bencher/src/database.rs @@ -92,7 +92,7 @@ pub fn reorg_short(benchmark: &mut Benchmark) { let mut blocks = Vec::new(); - for x in 0..1000 { + for x in 0..BLOCKS { let base = rolling_hash.clone(); let next_block = test_data::block_builder() From 320970aac3824fdf3ecede0515e71de65f4539c6 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 24 Nov 2016 16:05:01 +0300 Subject: [PATCH 3/6] more tests --- bencher/src/database.rs | 73 ++++++++++++++++++++++++++++++++++++++--- bencher/src/main.rs | 1 + 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/bencher/src/database.rs b/bencher/src/database.rs index c3e3c491..e242ca00 100644 --- a/bencher/src/database.rs +++ b/bencher/src/database.rs @@ -6,7 +6,7 @@ use super::Benchmark; pub fn fetch(benchmark: &mut Benchmark) { // params - const BLOCKS: usize = 10000; + const BLOCKS: usize = 1000; // test setup let path = RandomTempPath::create_dir(); @@ -98,7 +98,7 @@ pub fn reorg_short(benchmark: &mut Benchmark) { let next_block = test_data::block_builder() .transaction().coinbase().build() .transaction().output().value(5000000000).build().build() - .merkled_header().parent(rolling_hash.clone()).nonce(x*4).build() + .merkled_header().parent(rolling_hash.clone()).nonce(x as u32 * 4).build() .build(); rolling_hash = next_block.hash(); blocks.push(next_block); @@ -106,7 +106,7 @@ pub fn reorg_short(benchmark: &mut Benchmark) { let next_block_side = test_data::block_builder() .transaction().coinbase().build() .transaction().output().value(5000000000).build().build() - .merkled_header().parent(base).nonce(x * 4 + 2).build() + .merkled_header().parent(base).nonce(x as u32 * 4 + 2).build() .build(); let next_base = next_block_side.hash(); blocks.push(next_block_side); @@ -114,14 +114,14 @@ pub fn reorg_short(benchmark: &mut Benchmark) { let next_block_side_continue = test_data::block_builder() .transaction().coinbase().build() .transaction().output().value(5000000000).build().build() - .merkled_header().parent(next_base).nonce(x * 4 + 3).build() + .merkled_header().parent(next_base).nonce(x as u32 * 4 + 3).build() .build(); blocks.push(next_block_side_continue); let next_block_continue = test_data::block_builder() .transaction().coinbase().build() .transaction().output().value(5000000000).build().build() - .merkled_header().parent(rolling_hash.clone()).nonce(x * 4+1).build() + .merkled_header().parent(rolling_hash.clone()).nonce(x as u32 * 4 + 1).build() .build(); rolling_hash = next_block_continue.hash(); blocks.push(next_block_continue); @@ -140,6 +140,69 @@ pub fn reorg_short(benchmark: &mut Benchmark) { } benchmark.stop(); + // reorgs occur twice per iteration except last one where there only one, blocks are inserted with rate 4/iteration + // so reorgs = total/2 - 1 assert_eq!(1000, total); assert_eq!(499, reorgs); } + +// 1. write 12000 blocks +// 2. write 100 blocks that has 100 transaction each spending outputs from first 1000 blocks +pub fn write_heavy(benchmark: &mut Benchmark) { + // params + const BLOCKS_INITIAL: usize = 12000; + const BLOCKS: usize = 100; + const TRANSACTIONS: usize = 100; + + // test setup + let path = RandomTempPath::create_dir(); + let store = Storage::new(path.as_path()).unwrap(); + + let genesis = test_data::genesis(); + store.insert_block(&genesis).unwrap(); + + let genesis = test_data::genesis(); + store.insert_block(&genesis).unwrap(); + + let mut rolling_hash = genesis.hash(); + let mut blocks = Vec::new(); + let mut hashes = Vec::new(); + + for x in 0..BLOCKS_INITIAL { + let next_block = test_data::block_builder() + .transaction() + .coinbase() + .output().value(5000000000).build() + .build() + .merkled_header().parent(rolling_hash.clone()).nonce(x as u32).build() + .build(); + rolling_hash = next_block.hash(); + blocks.push(next_block); + hashes.push(rolling_hash.clone()); + } + + for b in 0..BLOCKS { + let mut builder = test_data::block_builder() + .transaction().coinbase().build(); + + for t in 0..TRANSACTIONS { + builder = builder.transaction() + .input().hash(blocks[b*TRANSACTIONS+t].transactions()[0].hash()).build() // default index is 0 which is ok + .output().value(1000).build() + .build(); + } + + let next_block = builder.merkled_header().parent(rolling_hash).build().build(); + + rolling_hash = next_block.hash(); + blocks.push(next_block); + hashes.push(rolling_hash.clone()); + } + + for block in blocks[..BLOCKS_INITIAL].iter() { store.insert_block(block).unwrap(); } + + // bench + benchmark.start(); + for block in blocks[BLOCKS_INITIAL+1..].iter() { store.insert_block(block).unwrap(); } + benchmark.stop(); +} diff --git a/bencher/src/main.rs b/bencher/src/main.rs index b0706323..7749d8f5 100644 --- a/bencher/src/main.rs +++ b/bencher/src/main.rs @@ -57,4 +57,5 @@ fn main() { benchmark!(database::fetch); benchmark!(database::write); benchmark!(database::reorg_short); + benchmark!(database::write_heavy); } From 9d896064a1dc804ff46f6af36aa4dc2e3f712ce0 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 24 Nov 2016 16:06:22 +0300 Subject: [PATCH 4/6] modify transaction gen --- bencher/src/database.rs | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/bencher/src/database.rs b/bencher/src/database.rs index e242ca00..f3a8b46e 100644 --- a/bencher/src/database.rs +++ b/bencher/src/database.rs @@ -24,8 +24,10 @@ pub fn fetch(benchmark: &mut Benchmark) { for x in 0..BLOCKS { let next_block = test_data::block_builder() - .transaction().coinbase().build() - .transaction().output().value(5000000000).build().build() + .transaction() + .coinbase() + .output().value(5000000000).build() + .build() .merkled_header().parent(rolling_hash.clone()).nonce(x as u32).build() .build(); rolling_hash = next_block.hash(); @@ -61,8 +63,10 @@ pub fn write(benchmark: &mut Benchmark) { for x in 0..BLOCKS { let next_block = test_data::block_builder() - .transaction().coinbase().build() - .transaction().output().value(5000000000).build().build() + .transaction() + .coinbase() + .output().value(5000000000).build() + .build() .merkled_header().parent(rolling_hash.clone()).nonce(x as u32).build() .build(); rolling_hash = next_block.hash(); @@ -96,31 +100,39 @@ pub fn reorg_short(benchmark: &mut Benchmark) { let base = rolling_hash.clone(); let next_block = test_data::block_builder() - .transaction().coinbase().build() - .transaction().output().value(5000000000).build().build() + .transaction() + .coinbase() + .output().value(5000000000).build() + .build() .merkled_header().parent(rolling_hash.clone()).nonce(x as u32 * 4).build() .build(); rolling_hash = next_block.hash(); blocks.push(next_block); let next_block_side = test_data::block_builder() - .transaction().coinbase().build() - .transaction().output().value(5000000000).build().build() + .transaction() + .coinbase() + .output().value(5000000000).build() + .build() .merkled_header().parent(base).nonce(x as u32 * 4 + 2).build() .build(); let next_base = next_block_side.hash(); blocks.push(next_block_side); let next_block_side_continue = test_data::block_builder() - .transaction().coinbase().build() - .transaction().output().value(5000000000).build().build() + .transaction() + .coinbase() + .output().value(5000000000).build() + .build() .merkled_header().parent(next_base).nonce(x as u32 * 4 + 3).build() .build(); blocks.push(next_block_side_continue); let next_block_continue = test_data::block_builder() - .transaction().coinbase().build() - .transaction().output().value(5000000000).build().build() + .transaction() + .coinbase() + .output().value(5000000000).build() + .build() .merkled_header().parent(rolling_hash.clone()).nonce(x as u32 * 4 + 1).build() .build(); rolling_hash = next_block_continue.hash(); From 7cba4673427133ca1fd37c048fa6a1d2f8316b5e Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 24 Nov 2016 16:17:55 +0300 Subject: [PATCH 5/6] ci --- .travis.yml | 1 + tools/bench.sh | 3 +++ 2 files changed, 4 insertions(+) create mode 100755 tools/bench.sh diff --git a/.travis.yml b/.travis.yml index 756d5b33..1d1e8535 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ addons: - g++-4.8 script: - ./tools/test.sh + - ./tools/bench.sh after_success: | [ false ] && [ $TRAVIS_BRANCH = master ] && diff --git a/tools/bench.sh b/tools/bench.sh new file mode 100755 index 00000000..30a63ef8 --- /dev/null +++ b/tools/bench.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +cargo run --manifest-path ./bencher/Cargo.toml --release From 76ea2abf85a03e8c62ff38b95c7e46cc3fef7231 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Thu, 24 Nov 2016 21:32:36 +0300 Subject: [PATCH 6/6] per-sample option in bench --- bencher/src/database.rs | 6 ++++++ bencher/src/main.rs | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/bencher/src/database.rs b/bencher/src/database.rs index f3a8b46e..48bc9c4c 100644 --- a/bencher/src/database.rs +++ b/bencher/src/database.rs @@ -8,6 +8,8 @@ pub fn fetch(benchmark: &mut Benchmark) { // params const BLOCKS: usize = 1000; + benchmark.samples(BLOCKS); + // test setup let path = RandomTempPath::create_dir(); let store = Storage::new(path.as_path()).unwrap(); @@ -49,6 +51,7 @@ pub fn fetch(benchmark: &mut Benchmark) { pub fn write(benchmark: &mut Benchmark) { // params const BLOCKS: usize = 1000; + benchmark.samples(BLOCKS); // setup let path = RandomTempPath::create_dir(); @@ -84,6 +87,7 @@ pub fn write(benchmark: &mut Benchmark) { pub fn reorg_short(benchmark: &mut Benchmark) { // params const BLOCKS: usize = 1000; + benchmark.samples(BLOCKS); // setup let path = RandomTempPath::create_dir(); @@ -166,6 +170,8 @@ pub fn write_heavy(benchmark: &mut Benchmark) { const BLOCKS: usize = 100; const TRANSACTIONS: usize = 100; + benchmark.samples(BLOCKS); + // test setup let path = RandomTempPath::create_dir(); let store = Storage::new(path.as_path()).unwrap(); diff --git a/bencher/src/main.rs b/bencher/src/main.rs index 7749d8f5..e37f119e 100644 --- a/bencher/src/main.rs +++ b/bencher/src/main.rs @@ -14,6 +14,7 @@ use std::str; pub struct Benchmark { start: Option, end: Option, + samples: Option, } impl Benchmark { @@ -28,6 +29,10 @@ impl Benchmark { pub fn evaluate(&self) -> Duration { self.start.expect("benchmarch never ended").to(self.end.expect("benchmark never started")) } + + pub fn samples(&mut self, samples: usize) { + self.samples = Some(samples); + } } fn decimal_mark(s: String) -> String { @@ -44,7 +49,14 @@ fn run_benchmark(name: &str, f: F) where F: FnOnce(&mut Benchmark) { let mut benchmark = Benchmark::default(); f(&mut benchmark); - println!("{} ns", decimal_mark(format!("{}", benchmark.evaluate().num_nanoseconds().unwrap()))); + if let Some(samples) = benchmark.samples { + println!("{} ns/sample", + decimal_mark(format!("{}", benchmark.evaluate().num_nanoseconds().unwrap() / samples as i64)), + ); + } + else { + println!("{} ns", decimal_mark(format!("{}", benchmark.evaluate().num_nanoseconds().unwrap()))); + } } macro_rules! benchmark {