Merge branch 'master' into session

Cargo.lock

@@ -10,6 +10,7 @@ dependencies = [
  "miner 0.1.0",
  "p2p 0.1.0",
  "script 0.1.0",
+ "verification 0.1.0",
 ]
 
 [[package]]
@@ -111,6 +112,7 @@ dependencies = [
  "primitives 0.1.0",
  "rocksdb 0.4.5 (git+https://github.com/ethcore/rust-rocksdb)",
  "serialization 0.1.0",
+ "test-data 0.1.0",
 ]
 
 [[package]]
@@ -225,6 +227,11 @@ name = "libc"
 version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "linked-hash-map"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "log"
 version = "0.3.6"
@@ -522,6 +529,13 @@ dependencies = [
  "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "test-data"
+version = "0.1.0"
+dependencies = [
+ "chain 0.1.0",
+]
+
 [[package]]
 name = "thread-id"
 version = "2.0.0"
@@ -581,6 +595,20 @@ name = "vec_map"
 version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "verification"
+version = "0.1.0"
+dependencies = [
+ "chain 0.1.0",
+ "db 0.1.0",
+ "ethcore-devtools 1.3.0",
+ "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parking_lot 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "primitives 0.1.0",
+ "serialization 0.1.0",
+ "test-data 0.1.0",
+]
+
 [[package]]
 name = "void"
 version = "1.0.2"
@@ -633,6 +661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "49247ec2a285bb3dcb23cbd9c35193c025e7251bfce77c1d5da97e6362dffe7f"
 "checksum lazycell 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce12306c4739d86ee97c23139f3a34ddf0387bbf181bc7929d287025a8c3ef6b"
 "checksum libc 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "408014cace30ee0f767b1c4517980646a573ec61a57957aeeabcac8ac0a02e8d"
+"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
 "checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054"
 "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20"
 "checksum mio 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2dadd39d4b47343e10513ac2a731c979517a4761224ecb6bbd243602300c9537"

Cargo.toml

@@ -14,6 +14,7 @@ miner = { path = "miner" }
 p2p = { path = "p2p" }
 script = { path = "script" }
 db = { path = "db" }
+verification = { path = "verification" }
 
 [[bin]]
 path = "pbtc/main.rs"

db/Cargo.toml

@@ -12,3 +12,4 @@ byteorder = "0.5"
 chain = { path = "../chain"  }
 serialization = { path = "../serialization" }
 parking_lot = "0.3"
+test-data = { path = "../test-data" }

db/src/lib.rs

@@ -10,6 +10,8 @@ extern crate serialization;
 
 #[cfg(test)]
 extern crate ethcore_devtools as devtools;
+#[cfg(test)]
+extern crate test_data;
 
 mod kvdb;
 mod storage;

db/src/storage.rs

@@ -264,6 +264,7 @@ mod tests {
 	use devtools::RandomTempPath;
 	use chain::Block;
 	use super::super::BlockRef;
+	use test_data;
 
 	#[test]
 	fn open_store() {
@@ -276,7 +277,7 @@ mod tests {
 		let path = RandomTempPath::create_dir();
 		let store = Storage::new(path.as_path()).unwrap();
 
-		let block: Block = "01000000ba8b9cda965dd8e536670f9ddec10e53aab14b20bacad27b9137190000000000190760b278fe7b8565fda3b968b918d5fd997f993b23674c0af3b6fde300b38f33a5914ce6ed5b1b01e32f570201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704e6ed5b1b014effffffff0100f2052a01000000434104b68a50eaa0287eff855189f949c1c6e5f58b37c88231373d8a59809cbae83059cc6469d65c665ccfd1cfeb75c6e8e19413bba7fbff9bc762419a76d87b16086eac000000000100000001a6b97044d03da79c005b20ea9c0e1a6d9dc12d9f7b91a5911c9030a439eed8f5000000004948304502206e21798a42fae0e854281abd38bacd1aeed3ee3738d9e1446618c4571d1090db022100e2ac980643b0b82c0e88ffdfec6b64e3e6ba35e7ba5fdd7d5d6cc8d25c6b241501ffffffff0100f2052a010000001976a914404371705fa9bd789a2fcd52d2c580b65d35549d88ac00000000".into();
+		let block: Block = test_data::block1();
 		store.insert_block(&block).unwrap();
 
 		let loaded_block = store.block(BlockRef::Hash(block.hash())).unwrap();
@@ -288,7 +289,7 @@ mod tests {
 		let path = RandomTempPath::create_dir();
 		let store = Storage::new(path.as_path()).unwrap();
 
-		let block: Block = "01000000ba8b9cda965dd8e536670f9ddec10e53aab14b20bacad27b9137190000000000190760b278fe7b8565fda3b968b918d5fd997f993b23674c0af3b6fde300b38f33a5914ce6ed5b1b01e32f570201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704e6ed5b1b014effffffff0100f2052a01000000434104b68a50eaa0287eff855189f949c1c6e5f58b37c88231373d8a59809cbae83059cc6469d65c665ccfd1cfeb75c6e8e19413bba7fbff9bc762419a76d87b16086eac000000000100000001a6b97044d03da79c005b20ea9c0e1a6d9dc12d9f7b91a5911c9030a439eed8f5000000004948304502206e21798a42fae0e854281abd38bacd1aeed3ee3738d9e1446618c4571d1090db022100e2ac980643b0b82c0e88ffdfec6b64e3e6ba35e7ba5fdd7d5d6cc8d25c6b241501ffffffff0100f2052a010000001976a914404371705fa9bd789a2fcd52d2c580b65d35549d88ac00000000".into();
+		let block: Block = test_data::block1();
 		let tx1 = block.transactions()[0].hash();
 		store.insert_block(&block).unwrap();
 

primitives/src/hash.rs

@@ -1,6 +1,6 @@
 use std::{fmt, ops, cmp, str};
-use std::hash::{Hash, Hasher};
 use hex::{ToHex, FromHex, FromHexError};
+use std::hash::{Hash, Hasher};
 
 macro_rules! impl_hash {
 	($name: ident, $size: expr) => {
@@ -97,8 +97,6 @@ macro_rules! impl_hash {
 			}
 		}
 
-		impl Eq for $name {}
-
 		impl cmp::PartialEq for $name {
 			fn eq(&self, other: &Self) -> bool {
 				let self_ref: &[u8] = &self.0;
@@ -114,6 +112,8 @@ macro_rules! impl_hash {
 			}
 		}
 
+		impl Eq for $name { }
+
 		impl $name {
 			pub fn reversed(&self) -> Self {
 				let mut result = self.clone();

test-data/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "test-data"
+version = "0.1.0"
+authors = ["Nikolay Volf <nikvolf@gmail.com>"]
+
+[dependencies]
+chain = { path = "../chain"  }

test-data/src/lib.rs

@@ -0,0 +1,24 @@
+//! Various chain-specific test dummies
+
+extern crate chain;
+
+use chain::Block;
+
+pub fn block1() -> Block {
+	let block: Block = "01000000ba8b9cda965dd8e536670f9ddec10e53aab14b20bacad27b9137190000000000190760b278fe7b8565fda3b968b918d5fd997f993b23674c0af3b6fde300b38f33a5914ce6ed5b1b01e32f570201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704e6ed5b1b014effffffff0100f2052a01000000434104b68a50eaa0287eff855189f949c1c6e5f58b37c88231373d8a59809cbae83059cc6469d65c665ccfd1cfeb75c6e8e19413bba7fbff9bc762419a76d87b16086eac000000000100000001a6b97044d03da79c005b20ea9c0e1a6d9dc12d9f7b91a5911c9030a439eed8f5000000004948304502206e21798a42fae0e854281abd38bacd1aeed3ee3738d9e1446618c4571d1090db022100e2ac980643b0b82c0e88ffdfec6b64e3e6ba35e7ba5fdd7d5d6cc8d25c6b241501ffffffff0100f2052a010000001976a914404371705fa9bd789a2fcd52d2c580b65d35549d88ac00000000".into();
+	block
+}
+
+// https://webbtc.com/block/00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048
+// height 1
+pub fn block_h1() -> Block {
+	let block: Block = "010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e362990101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000".into();
+	block
+}
+
+// https://webbtc.com/block/000000006a625f06636b8bb6ac7b960a8d03705d1ace08b1a19da3fdcc99ddbd.hex
+// height 2
+pub fn block_h2() -> Block {
+	let block: Block = "010000004860eb18bf1b1620e37e9490fc8a427514416fd75159ab86688e9a8300000000d5fdcc541e25de1c7a5addedf24858b8bb665c9f36ef744ee42c316022c90f9bb0bc6649ffff001d08d2bd610101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d010bffffffff0100f2052a010000004341047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77ac00000000".into();
+	block
+}

tools/test.sh

@@ -12,4 +12,5 @@ cargo test\
 	-p primitives\
 	-p script\
 	-p serialization\
-	-p pbtc
+	-p pbtc\
+	-p verification

verification/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "verification"
+version = "0.1.0"
+authors = ["Nikolay Volf <nikvolf@gmail.com>"]
+
+[dependencies]
+db = { path = "../db" }
+ethcore-devtools = { path = "../devtools" }
+primitives = { path = "../primitives" }
+chain = { path = "../chain"  }
+serialization = { path = "../serialization" }
+parking_lot = "0.3"
+linked-hash-map = "0.3"
+test-data = { path = "../test-data" }

verification/src/lib.rs

@@ -0,0 +1,75 @@
+//! Bitcoin blocks verification
+
+extern crate db;
+extern crate primitives;
+extern crate chain;
+extern crate serialization;
+extern crate parking_lot;
+extern crate linked_hash_map;
+
+#[cfg(test)]
+extern crate ethcore_devtools as devtools;
+#[cfg(test)]
+extern crate test_data;
+
+mod queue;
+
+pub use queue::Queue;
+
+#[derive(Debug)]
+/// All possible verification errors
+pub enum Error {
+	/// has an equal duplicate in the chain
+	Duplicate,
+	/// No transactions in block
+	Empty,
+	/// Invalid proof-of-work (Block hash does not satisfy nBits)
+	Pow,
+	/// Invalid timestamp
+	Timestamp,
+	/// First transaction is not a coinbase transaction
+	Coinbase,
+	/// One of the transactions is invalid (corresponding index and specific transaction error)
+	Transaction(usize, TransactionError),
+	/// nBits do not match difficulty rules
+	Difficulty
+}
+
+#[derive(Debug)]
+/// Possible transactions verification errors
+pub enum TransactionError {
+	/// Not found corresponding output for transaction input
+	Input,
+	/// Referenced coinbase output for the transaction input is not mature enough
+	Maturity,
+	/// Signature invalid
+	Signature,
+}
+
+/// Block verification chain
+pub enum Chain {
+	/// Main chain
+	Main,
+	/// Side chain
+	Side,
+	/// Orphan (no known parent)
+	Orphan,
+}
+
+#[derive(PartialEq, Debug)]
+/// block status within the queue
+pub enum BlockStatus {
+	Valid,
+	Invalid,
+	Pending,
+	Absent,
+	Verifying,
+}
+
+/// Verification result
+pub type VerificationResult = Result<Chain, Error>;
+
+/// Interface for block verification
+pub trait Verify : Send + Sync {
+	fn verify(&self, block: &chain::Block) -> VerificationResult;
+}

verification/src/queue.rs

@@ -0,0 +1,230 @@
+//! Blocks verification queue
+
+use chain::Block;
+use primitives::hash::H256;
+use super::{Chain, Verify, BlockStatus};
+use linked_hash_map::LinkedHashMap;
+use parking_lot::RwLock;
+use std::collections::HashSet;
+
+const MAX_PENDING_PRESET: usize = 128;
+
+pub struct VerifiedBlock {
+	pub chain: Chain,
+	pub block: Block,
+}
+
+impl VerifiedBlock {
+	fn new(chain: Chain, block: Block) -> Self {
+		VerifiedBlock { chain: chain, block: block }
+	}
+}
+
+#[derive(Debug)]
+/// Queue errors
+pub enum Error {
+	/// Queue is currently full
+	Full,
+	/// There is already block in the queue
+	Duplicate,
+}
+
+/// Verification queue
+pub struct Queue {
+	verifier: Box<Verify>,
+	items: RwLock<LinkedHashMap<H256, Block>>,
+	verified: RwLock<LinkedHashMap<H256, VerifiedBlock>>,
+	invalid: RwLock<HashSet<H256>>,
+	processing: RwLock<HashSet<H256>>,
+}
+
+impl Queue {
+
+	/// New verification queue
+	pub fn new(verifier: Box<Verify>) -> Self {
+		Queue {
+			verifier: verifier,
+			items: RwLock::new(LinkedHashMap::new()),
+			verified: RwLock::new(LinkedHashMap::new()),
+			invalid: RwLock::new(HashSet::new()),
+			processing: RwLock::new(HashSet::new()),
+		}
+	}
+
+	/// Process one block in the queue
+	pub fn process(&self) {
+		let (hash, block) = {
+			let mut processing = self.processing.write();
+			let mut items = self.items.write();
+			match items.pop_front() {
+				Some((hash, block)) => {
+					processing.insert(hash.clone());
+					(hash, block)
+				},
+				/// nothing to verify
+				None => { return; },
+			}
+		};
+
+		match self.verifier.verify(&block) {
+			Ok(chain) => {
+				let mut verified = self.verified.write();
+				let mut processing = self.processing.write();
+				processing.remove(&hash);
+				verified.insert(hash, VerifiedBlock::new(chain, block));
+			},
+			Err(e) => {
+				println!("Verification failed: {:?}", e);
+				let mut invalid = self.invalid.write();
+				let mut processing = self.processing.write();
+
+				processing.remove(&hash);
+				invalid.insert(hash);
+			}
+		}
+	}
+
+	/// Query block status
+	pub fn block_status(&self, hash: &H256) -> BlockStatus {
+		if self.invalid.read().contains(hash) { BlockStatus::Invalid }
+		else if self.processing.read().contains(hash) { BlockStatus::Verifying }
+		else if self.verified.read().contains_key(hash) { BlockStatus::Valid }
+		else if self.items.read().contains_key(hash) { BlockStatus::Pending }
+		else { BlockStatus::Absent }
+	}
+
+	pub fn max_pending(&self) -> usize {
+		// todo: later might be calculated with lazy-static here based on memory usage
+		MAX_PENDING_PRESET
+	}
+
+	pub fn push(&self, block: Block) -> Result<(), Error> {
+		let hash = block.hash();
+
+		if self.block_status(&hash) != BlockStatus::Absent { return Err(Error::Duplicate) }
+
+		let mut items = self.items.write();
+		if items.len() > self.max_pending() { return Err(Error::Full) }
+		items.insert(hash, block);
+
+		Ok(())
+	}
+
+	pub fn pop_valid(&self) -> Option<(H256, VerifiedBlock)> {
+		self.verified.write().pop_front()
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::Queue;
+	use super::super::{BlockStatus, VerificationResult, Verify, Chain, Error as VerificationError};
+	use chain::Block;
+	use primitives::hash::H256;
+	use test_data;
+
+	struct FacileVerifier;
+	impl Verify for FacileVerifier {
+		fn verify(&self, _block: &Block) -> VerificationResult { Ok(Chain::Main) }
+	}
+
+	struct EvilVerifier;
+	impl Verify for EvilVerifier {
+		fn verify(&self, _block: &Block) -> VerificationResult { Err(VerificationError::Empty) }
+	}
+
+	#[test]
+	fn new() {
+		let queue = Queue::new(Box::new(FacileVerifier));
+		assert_eq!(queue.block_status(&H256::from(0u8)), BlockStatus::Absent);
+	}
+
+	#[test]
+	fn push() {
+		let queue = Queue::new(Box::new(FacileVerifier));
+		let block = test_data::block1();
+		let hash = block.hash();
+
+		queue.push(block).unwrap();
+
+		assert_eq!(queue.block_status(&hash), BlockStatus::Pending);
+	}
+
+	#[test]
+	fn push_duplicate() {
+		let queue = Queue::new(Box::new(FacileVerifier));
+		let block = test_data::block1();
+		let dup_block = test_data::block1();
+
+		queue.push(block).unwrap();
+		let second_push = queue.push(dup_block);
+
+		assert!(second_push.is_err());
+	}
+
+	#[test]
+	fn process_happy() {
+		let queue = Queue::new(Box::new(FacileVerifier));
+		let block = test_data::block1();
+		let hash = block.hash();
+
+		queue.push(block).unwrap();
+		queue.process();
+
+		assert_eq!(queue.block_status(&hash), BlockStatus::Valid);
+	}
+
+	#[test]
+	fn process_unhappy() {
+		let queue = Queue::new(Box::new(EvilVerifier));
+		let block = test_data::block1();
+		let hash = block.hash();
+
+		queue.push(block).unwrap();
+		queue.process();
+
+		assert_eq!(queue.block_status(&hash), BlockStatus::Invalid);
+	}
+
+	#[test]
+	fn process_async() {
+		use std::thread;
+		use std::sync::Arc;
+
+		let queue = Arc::new(Queue::new(Box::new(FacileVerifier)));
+
+		let t1_queue = queue.clone();
+		let t1_handle = thread::spawn(move || {
+			let block_h1 = test_data::block_h1();
+			t1_queue.push(block_h1).unwrap();
+			t1_queue.process();
+		});
+
+		let t2_queue = queue.clone();
+		let t2_handle = thread::spawn(move || {
+			let block_h2 = test_data::block_h2();
+			t2_queue.push(block_h2).unwrap();
+			t2_queue.process();
+		});
+
+		t1_handle.join().unwrap();
+		t2_handle.join().unwrap();
+
+		assert_eq!(queue.block_status(&test_data::block_h1().hash()), BlockStatus::Valid);
+		assert_eq!(queue.block_status(&test_data::block_h2().hash()), BlockStatus::Valid);
+	}
+
+	#[test]
+	fn pop() {
+		let queue = Queue::new(Box::new(FacileVerifier));
+		let block = test_data::block1();
+		let hash = block.hash();
+
+		queue.push(block).unwrap();
+		queue.process();
+		let (h, _b) = queue.pop_valid().unwrap();
+
+		assert_eq!(queue.block_status(&hash), BlockStatus::Absent);
+		assert_eq!(h, hash);
+	}
+}