diff --git a/Cargo.lock b/Cargo.lock index 84394ac5..eca8a053 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,6 +10,7 @@ dependencies = [ "miner 0.1.0", "p2p 0.1.0", "script 0.1.0", + "verification 0.1.0", ] [[package]] @@ -111,6 +112,7 @@ dependencies = [ "primitives 0.1.0", "rocksdb 0.4.5 (git+https://github.com/ethcore/rust-rocksdb)", "serialization 0.1.0", + "test-data 0.1.0", ] [[package]] @@ -225,6 +227,11 @@ name = "libc" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "linked-hash-map" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "log" version = "0.3.6" @@ -522,6 +529,13 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "test-data" +version = "0.1.0" +dependencies = [ + "chain 0.1.0", +] + [[package]] name = "thread-id" version = "2.0.0" @@ -581,6 +595,20 @@ name = "vec_map" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "verification" +version = "0.1.0" +dependencies = [ + "chain 0.1.0", + "db 0.1.0", + "ethcore-devtools 1.3.0", + "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "primitives 0.1.0", + "serialization 0.1.0", + "test-data 0.1.0", +] + [[package]] name = "void" version = "1.0.2" @@ -633,6 +661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "49247ec2a285bb3dcb23cbd9c35193c025e7251bfce77c1d5da97e6362dffe7f" "checksum lazycell 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce12306c4739d86ee97c23139f3a34ddf0387bbf181bc7929d287025a8c3ef6b" "checksum libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "408014cace30ee0f767b1c4517980646a573ec61a57957aeeabcac8ac0a02e8d" +"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" "checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054" "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" "checksum mio 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2dadd39d4b47343e10513ac2a731c979517a4761224ecb6bbd243602300c9537" diff --git a/Cargo.toml b/Cargo.toml index 2333ecf9..b2cc74bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ miner = { path = "miner" } p2p = { path = "p2p" } script = { path = "script" } db = { path = "db" } +verification = { path = "verification" } [[bin]] path = "pbtc/main.rs" diff --git a/db/Cargo.toml b/db/Cargo.toml index f249c418..9962bc54 100644 --- a/db/Cargo.toml +++ b/db/Cargo.toml @@ -12,3 +12,4 @@ byteorder = "0.5" chain = { path = "../chain" } serialization = { path = "../serialization" } parking_lot = "0.3" +test-data = { path = "../test-data" } diff --git a/db/src/lib.rs b/db/src/lib.rs index bb67073b..3b341cf2 100644 --- a/db/src/lib.rs +++ b/db/src/lib.rs @@ -10,6 +10,8 @@ extern crate serialization; #[cfg(test)] extern crate ethcore_devtools as devtools; +#[cfg(test)] +extern crate test_data; mod kvdb; mod storage; diff --git a/db/src/storage.rs b/db/src/storage.rs index 10ef3f6e..cd5fe36c 100644 --- a/db/src/storage.rs +++ b/db/src/storage.rs @@ -264,6 +264,7 @@ mod tests { use devtools::RandomTempPath; use chain::Block; use super::super::BlockRef; + use test_data; #[test] fn open_store() { @@ -276,7 +277,7 @@ mod tests { let path = RandomTempPath::create_dir(); let store = Storage::new(path.as_path()).unwrap(); - let block: Block = "01000000ba8b9cda965dd8e536670f9ddec10e53aab14b20bacad27b9137190000000000190760b278fe7b8565fda3b968b918d5fd997f993b23674c0af3b6fde300b38f33a5914ce6ed5b1b01e32f570201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704e6ed5b1b014effffffff0100f2052a01000000434104b68a50eaa0287eff855189f949c1c6e5f58b37c88231373d8a59809cbae83059cc6469d65c665ccfd1cfeb75c6e8e19413bba7fbff9bc762419a76d87b16086eac000000000100000001a6b97044d03da79c005b20ea9c0e1a6d9dc12d9f7b91a5911c9030a439eed8f5000000004948304502206e21798a42fae0e854281abd38bacd1aeed3ee3738d9e1446618c4571d1090db022100e2ac980643b0b82c0e88ffdfec6b64e3e6ba35e7ba5fdd7d5d6cc8d25c6b241501ffffffff0100f2052a010000001976a914404371705fa9bd789a2fcd52d2c580b65d35549d88ac00000000".into(); + let block: Block = test_data::block1(); store.insert_block(&block).unwrap(); let loaded_block = store.block(BlockRef::Hash(block.hash())).unwrap(); @@ -288,7 +289,7 @@ mod tests { let path = RandomTempPath::create_dir(); let store = Storage::new(path.as_path()).unwrap(); - let block: Block = "01000000ba8b9cda965dd8e536670f9ddec10e53aab14b20bacad27b9137190000000000190760b278fe7b8565fda3b968b918d5fd997f993b23674c0af3b6fde300b38f33a5914ce6ed5b1b01e32f570201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704e6ed5b1b014effffffff0100f2052a01000000434104b68a50eaa0287eff855189f949c1c6e5f58b37c88231373d8a59809cbae83059cc6469d65c665ccfd1cfeb75c6e8e19413bba7fbff9bc762419a76d87b16086eac000000000100000001a6b97044d03da79c005b20ea9c0e1a6d9dc12d9f7b91a5911c9030a439eed8f5000000004948304502206e21798a42fae0e854281abd38bacd1aeed3ee3738d9e1446618c4571d1090db022100e2ac980643b0b82c0e88ffdfec6b64e3e6ba35e7ba5fdd7d5d6cc8d25c6b241501ffffffff0100f2052a010000001976a914404371705fa9bd789a2fcd52d2c580b65d35549d88ac00000000".into(); + let block: Block = test_data::block1(); let tx1 = block.transactions()[0].hash(); store.insert_block(&block).unwrap(); diff --git a/primitives/src/hash.rs b/primitives/src/hash.rs index a656db31..11145eb6 100644 --- a/primitives/src/hash.rs +++ b/primitives/src/hash.rs @@ -1,6 +1,6 @@ use std::{fmt, ops, cmp, str}; -use std::hash::{Hash, Hasher}; use hex::{ToHex, FromHex, FromHexError}; +use std::hash::{Hash, Hasher}; macro_rules! impl_hash { ($name: ident, $size: expr) => { @@ -97,8 +97,6 @@ macro_rules! impl_hash { } } - impl Eq for $name {} - impl cmp::PartialEq for $name { fn eq(&self, other: &Self) -> bool { let self_ref: &[u8] = &self.0; @@ -114,6 +112,8 @@ macro_rules! impl_hash { } } + impl Eq for $name { } + impl $name { pub fn reversed(&self) -> Self { let mut result = self.clone(); diff --git a/test-data/Cargo.toml b/test-data/Cargo.toml new file mode 100644 index 00000000..9e06b1d7 --- /dev/null +++ b/test-data/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "test-data" +version = "0.1.0" +authors = ["Nikolay Volf "] + +[dependencies] +chain = { path = "../chain" } diff --git a/test-data/src/lib.rs b/test-data/src/lib.rs new file mode 100644 index 00000000..9499ca0e --- /dev/null +++ b/test-data/src/lib.rs @@ -0,0 +1,24 @@ +//! Various chain-specific test dummies + +extern crate chain; + +use chain::Block; + +pub fn block1() -> Block { + let block: Block = "01000000ba8b9cda965dd8e536670f9ddec10e53aab14b20bacad27b9137190000000000190760b278fe7b8565fda3b968b918d5fd997f993b23674c0af3b6fde300b38f33a5914ce6ed5b1b01e32f570201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704e6ed5b1b014effffffff0100f2052a01000000434104b68a50eaa0287eff855189f949c1c6e5f58b37c88231373d8a59809cbae83059cc6469d65c665ccfd1cfeb75c6e8e19413bba7fbff9bc762419a76d87b16086eac000000000100000001a6b97044d03da79c005b20ea9c0e1a6d9dc12d9f7b91a5911c9030a439eed8f5000000004948304502206e21798a42fae0e854281abd38bacd1aeed3ee3738d9e1446618c4571d1090db022100e2ac980643b0b82c0e88ffdfec6b64e3e6ba35e7ba5fdd7d5d6cc8d25c6b241501ffffffff0100f2052a010000001976a914404371705fa9bd789a2fcd52d2c580b65d35549d88ac00000000".into(); + block +} + +// https://webbtc.com/block/00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048 +// height 1 +pub fn block_h1() -> Block { + let block: Block = "010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e362990101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000".into(); + block +} + +// https://webbtc.com/block/000000006a625f06636b8bb6ac7b960a8d03705d1ace08b1a19da3fdcc99ddbd.hex +// height 2 +pub fn block_h2() -> Block { + let block: Block = "010000004860eb18bf1b1620e37e9490fc8a427514416fd75159ab86688e9a8300000000d5fdcc541e25de1c7a5addedf24858b8bb665c9f36ef744ee42c316022c90f9bb0bc6649ffff001d08d2bd610101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d010bffffffff0100f2052a010000004341047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77ac00000000".into(); + block +} diff --git a/tools/test.sh b/tools/test.sh index 1e433887..8cfb0779 100755 --- a/tools/test.sh +++ b/tools/test.sh @@ -12,4 +12,5 @@ cargo test\ -p primitives\ -p script\ -p serialization\ - -p pbtc + -p pbtc\ + -p verification diff --git a/verification/Cargo.toml b/verification/Cargo.toml new file mode 100644 index 00000000..1bf4b4b1 --- /dev/null +++ b/verification/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "verification" +version = "0.1.0" +authors = ["Nikolay Volf "] + +[dependencies] +db = { path = "../db" } +ethcore-devtools = { path = "../devtools" } +primitives = { path = "../primitives" } +chain = { path = "../chain" } +serialization = { path = "../serialization" } +parking_lot = "0.3" +linked-hash-map = "0.3" +test-data = { path = "../test-data" } diff --git a/verification/src/lib.rs b/verification/src/lib.rs new file mode 100644 index 00000000..37e188b7 --- /dev/null +++ b/verification/src/lib.rs @@ -0,0 +1,75 @@ +//! Bitcoin blocks verification + +extern crate db; +extern crate primitives; +extern crate chain; +extern crate serialization; +extern crate parking_lot; +extern crate linked_hash_map; + +#[cfg(test)] +extern crate ethcore_devtools as devtools; +#[cfg(test)] +extern crate test_data; + +mod queue; + +pub use queue::Queue; + +#[derive(Debug)] +/// All possible verification errors +pub enum Error { + /// has an equal duplicate in the chain + Duplicate, + /// No transactions in block + Empty, + /// Invalid proof-of-work (Block hash does not satisfy nBits) + Pow, + /// Invalid timestamp + Timestamp, + /// First transaction is not a coinbase transaction + Coinbase, + /// One of the transactions is invalid (corresponding index and specific transaction error) + Transaction(usize, TransactionError), + /// nBits do not match difficulty rules + Difficulty +} + +#[derive(Debug)] +/// Possible transactions verification errors +pub enum TransactionError { + /// Not found corresponding output for transaction input + Input, + /// Referenced coinbase output for the transaction input is not mature enough + Maturity, + /// Signature invalid + Signature, +} + +/// Block verification chain +pub enum Chain { + /// Main chain + Main, + /// Side chain + Side, + /// Orphan (no known parent) + Orphan, +} + +#[derive(PartialEq, Debug)] +/// block status within the queue +pub enum BlockStatus { + Valid, + Invalid, + Pending, + Absent, + Verifying, +} + +/// Verification result +pub type VerificationResult = Result; + +/// Interface for block verification +pub trait Verify : Send + Sync { + fn verify(&self, block: &chain::Block) -> VerificationResult; +} diff --git a/verification/src/queue.rs b/verification/src/queue.rs new file mode 100644 index 00000000..ddacf8e4 --- /dev/null +++ b/verification/src/queue.rs @@ -0,0 +1,230 @@ +//! Blocks verification queue + +use chain::Block; +use primitives::hash::H256; +use super::{Chain, Verify, BlockStatus}; +use linked_hash_map::LinkedHashMap; +use parking_lot::RwLock; +use std::collections::HashSet; + +const MAX_PENDING_PRESET: usize = 128; + +pub struct VerifiedBlock { + pub chain: Chain, + pub block: Block, +} + +impl VerifiedBlock { + fn new(chain: Chain, block: Block) -> Self { + VerifiedBlock { chain: chain, block: block } + } +} + +#[derive(Debug)] +/// Queue errors +pub enum Error { + /// Queue is currently full + Full, + /// There is already block in the queue + Duplicate, +} + +/// Verification queue +pub struct Queue { + verifier: Box, + items: RwLock>, + verified: RwLock>, + invalid: RwLock>, + processing: RwLock>, +} + +impl Queue { + + /// New verification queue + pub fn new(verifier: Box) -> Self { + Queue { + verifier: verifier, + items: RwLock::new(LinkedHashMap::new()), + verified: RwLock::new(LinkedHashMap::new()), + invalid: RwLock::new(HashSet::new()), + processing: RwLock::new(HashSet::new()), + } + } + + /// Process one block in the queue + pub fn process(&self) { + let (hash, block) = { + let mut processing = self.processing.write(); + let mut items = self.items.write(); + match items.pop_front() { + Some((hash, block)) => { + processing.insert(hash.clone()); + (hash, block) + }, + /// nothing to verify + None => { return; }, + } + }; + + match self.verifier.verify(&block) { + Ok(chain) => { + let mut verified = self.verified.write(); + let mut processing = self.processing.write(); + processing.remove(&hash); + verified.insert(hash, VerifiedBlock::new(chain, block)); + }, + Err(e) => { + println!("Verification failed: {:?}", e); + let mut invalid = self.invalid.write(); + let mut processing = self.processing.write(); + + processing.remove(&hash); + invalid.insert(hash); + } + } + } + + /// Query block status + pub fn block_status(&self, hash: &H256) -> BlockStatus { + if self.invalid.read().contains(hash) { BlockStatus::Invalid } + else if self.processing.read().contains(hash) { BlockStatus::Verifying } + else if self.verified.read().contains_key(hash) { BlockStatus::Valid } + else if self.items.read().contains_key(hash) { BlockStatus::Pending } + else { BlockStatus::Absent } + } + + pub fn max_pending(&self) -> usize { + // todo: later might be calculated with lazy-static here based on memory usage + MAX_PENDING_PRESET + } + + pub fn push(&self, block: Block) -> Result<(), Error> { + let hash = block.hash(); + + if self.block_status(&hash) != BlockStatus::Absent { return Err(Error::Duplicate) } + + let mut items = self.items.write(); + if items.len() > self.max_pending() { return Err(Error::Full) } + items.insert(hash, block); + + Ok(()) + } + + pub fn pop_valid(&self) -> Option<(H256, VerifiedBlock)> { + self.verified.write().pop_front() + } +} + +#[cfg(test)] +mod tests { + use super::Queue; + use super::super::{BlockStatus, VerificationResult, Verify, Chain, Error as VerificationError}; + use chain::Block; + use primitives::hash::H256; + use test_data; + + struct FacileVerifier; + impl Verify for FacileVerifier { + fn verify(&self, _block: &Block) -> VerificationResult { Ok(Chain::Main) } + } + + struct EvilVerifier; + impl Verify for EvilVerifier { + fn verify(&self, _block: &Block) -> VerificationResult { Err(VerificationError::Empty) } + } + + #[test] + fn new() { + let queue = Queue::new(Box::new(FacileVerifier)); + assert_eq!(queue.block_status(&H256::from(0u8)), BlockStatus::Absent); + } + + #[test] + fn push() { + let queue = Queue::new(Box::new(FacileVerifier)); + let block = test_data::block1(); + let hash = block.hash(); + + queue.push(block).unwrap(); + + assert_eq!(queue.block_status(&hash), BlockStatus::Pending); + } + + #[test] + fn push_duplicate() { + let queue = Queue::new(Box::new(FacileVerifier)); + let block = test_data::block1(); + let dup_block = test_data::block1(); + + queue.push(block).unwrap(); + let second_push = queue.push(dup_block); + + assert!(second_push.is_err()); + } + + #[test] + fn process_happy() { + let queue = Queue::new(Box::new(FacileVerifier)); + let block = test_data::block1(); + let hash = block.hash(); + + queue.push(block).unwrap(); + queue.process(); + + assert_eq!(queue.block_status(&hash), BlockStatus::Valid); + } + + #[test] + fn process_unhappy() { + let queue = Queue::new(Box::new(EvilVerifier)); + let block = test_data::block1(); + let hash = block.hash(); + + queue.push(block).unwrap(); + queue.process(); + + assert_eq!(queue.block_status(&hash), BlockStatus::Invalid); + } + + #[test] + fn process_async() { + use std::thread; + use std::sync::Arc; + + let queue = Arc::new(Queue::new(Box::new(FacileVerifier))); + + let t1_queue = queue.clone(); + let t1_handle = thread::spawn(move || { + let block_h1 = test_data::block_h1(); + t1_queue.push(block_h1).unwrap(); + t1_queue.process(); + }); + + let t2_queue = queue.clone(); + let t2_handle = thread::spawn(move || { + let block_h2 = test_data::block_h2(); + t2_queue.push(block_h2).unwrap(); + t2_queue.process(); + }); + + t1_handle.join().unwrap(); + t2_handle.join().unwrap(); + + assert_eq!(queue.block_status(&test_data::block_h1().hash()), BlockStatus::Valid); + assert_eq!(queue.block_status(&test_data::block_h2().hash()), BlockStatus::Valid); + } + + #[test] + fn pop() { + let queue = Queue::new(Box::new(FacileVerifier)); + let block = test_data::block1(); + let hash = block.hash(); + + queue.push(block).unwrap(); + queue.process(); + let (h, _b) = queue.pop_valid().unwrap(); + + assert_eq!(queue.block_status(&hash), BlockStatus::Absent); + assert_eq!(h, hash); + } +}