Merge pull request #85 from ethcore/import_command

work on import command in progress

Cargo.lock

@@ -2,12 +2,14 @@
 name = "pbtc"
 version = "0.1.0"
 dependencies = [
+ "app_dirs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "chain 0.1.0",
  "clap 2.16.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "db 0.1.0",
  "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "import 0.1.0",
  "keys 0.1.0",
+ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "message 0.1.0",
  "miner 0.1.0",
  "p2p 0.1.0",
@@ -39,6 +41,17 @@ name = "ansi_term"
 version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "app_dirs"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ole32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "shell32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "xdg 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "arrayvec"
 version = "0.3.20"
@@ -227,6 +240,7 @@ name = "import"
 version = "0.1.0"
 dependencies = [
  "chain 0.1.0",
+ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "primitives 0.1.0",
  "serialization 0.1.0",
 ]
@@ -392,6 +406,15 @@ name = "odds"
 version = "0.2.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "ole32-sys"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "owning_ref"
 version = "0.2.2"
@@ -558,6 +581,15 @@ dependencies = [
  "primitives 0.1.0",
 ]
 
+[[package]]
+name = "shell32-sys"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "slab"
 version = "0.3.0"
@@ -709,6 +741,11 @@ dependencies = [
  "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "xdg"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "yaml-rust"
 version = "0.3.4"
@@ -718,6 +755,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum abstract-ns 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "179c25175ddfb957808aab46c48dd7d0c66a366bff79363813b527447c9f620c"
 "checksum aho-corasick 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ca972c2ea5f742bfce5687b9aef75506a764f61d37f8f649047846a9686ddb66"
 "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"
+"checksum app_dirs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b7d1c0d48a81bbb13043847f957971f4d87c81542d80ece5e84ba3cba4058fd4"
 "checksum arrayvec 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d89f1b0e242270b5b797778af0c8d182a1a2ccac5d8d6fadf414223cc0fab096"
 "checksum base58 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83"
 "checksum bit-vec 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5b97c2c8e8bbb4251754f559df8af22fb264853c7d009084a576cdf12565089d"
@@ -751,6 +789,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum ns-dns-tokio 0.1.0 (git+https://github.com/debris/abstract-ns)" = "<none>"
 "checksum num_cpus 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8890e6084723d57d0df8d2720b0d60c6ee67d6c93e7169630e4371e88765dcad"
 "checksum odds 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)" = "e04630a62b3f1cc8c58b4d8f2555a40136f02b420e158242936ef286a72d33a0"
+"checksum ole32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2c49021782e5233cd243168edfa8037574afed4eba4bbaf538b3d8d1789d8c"
 "checksum owning_ref 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8d91377085359426407a287ab16884a0111ba473aa6844ff01d4ec20ce3d75e7"
 "checksum parking_lot 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "e1435e7a2a00dfebededd6c6bdbd54008001e94b4a2aadd6aef0dc4c56317621"
 "checksum parking_lot_core 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fb1b97670a2ffadce7c397fb80a3d687c4f3060140b885621ef1653d0e5d5068"
@@ -766,6 +805,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084"
 "checksum scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f417c22df063e9450888a7561788e9bd46d3bb3c1466435b4eccb903807f147d"
 "checksum semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac"
+"checksum shell32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72f20b8f3c060374edb8046591ba28f62448c369ccbdc7b02075103fb3a9e38d"
 "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23"
 "checksum smallvec 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "fcc8d19212aacecf95e4a7a2179b26f7aeb9732a915cf01f05b0d3e044865410"
 "checksum strsim 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "50c069df92e4b01425a8bf3576d5d417943a6a7272fbabaf5bd80b1aaa76442e"
@@ -782,4 +822,5 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
+"checksum xdg 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77b831a5ba77110f438f0ac5583aafeb087f70432998ba6b7dcb1d32185db453"
 "checksum yaml-rust 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "371cea3a33a58d11dc83c0992fb37e44f651ebdf2df12f9d939f6cb24be2a8fd"

Cargo.toml

@@ -6,7 +6,9 @@ authors = ["Ethcore <admin@ethcore.io>"]
 description = "Parity bitcoin client."
 
 [dependencies]
+log = "0.3"
 env_logger = "0.3"
+app_dirs = "^1.1.1"
 clap = { version = "2", features = ["yaml"] }
 chain = { path = "chain" }
 keys = { path = "keys" }

import/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 authors = ["debris <marek.kotewicz@gmail.com>"]
 
 [dependencies]
+log = "0.3"
 primitives = { path = "../primitives" }
 chain = { path = "../chain" }
 serialization = { path = "../serialization" }

import/src/blk.rs

@@ -4,6 +4,7 @@ use block::Block;
 use fs::read_blk_dir;
 
 pub fn open_blk_file<P>(path: P) -> Result<BlkFile, io::Error> where P: AsRef<path::Path> {
+	trace!("Opening blk file: {:?}", path.as_ref());
 	let file = try!(fs::File::open(path));
 	let blk_file = BlkFile {
 		reader: deserialize_iterator(file),

import/src/lib.rs

@@ -1,3 +1,5 @@
+#[macro_use]
+extern crate log;
 extern crate primitives;
 extern crate serialization as ser;
 extern crate chain;

pbtc/commands/import.rs

@@ -0,0 +1,37 @@
+use clap::ArgMatches;
+use sync::{create_sync_blocks_writer, Error};
+use config::Config;
+use util::{open_db, init_db};
+
+pub fn import(cfg: Config, matches: &ArgMatches) -> Result<(), String> {
+	let db = open_db(cfg.use_disk_database);
+	// TODO: this might be unnecessary here!
+	init_db(&db);
+
+	let mut writer = create_sync_blocks_writer(db);
+
+	let blk_path = matches.value_of("PATH").expect("PATH is required in cli.yml; qed");
+	let blk_dir = try!(::import::open_blk_dir(blk_path).map_err(|_| "Import directory does not exist".to_owned()));
+	let mut counter = 0;
+	let mut skipped = 0;
+	for blk in blk_dir {
+		// TODO: verify magic!
+		let blk = try!(blk.map_err(|_| "Cannot read block".to_owned()));
+		match writer.append_block(blk.block) {
+			Ok(_) => {
+				counter += 1;
+				if counter % 1000 == 0 {
+					info!("Imported {} blocks", counter);
+				}
+			}
+			Err(Error::OutOfOrderBlock) => {
+				skipped += 1;
+			},
+			Err(_) => return Err(format!("Cannot append block")),
+		}
+	}
+
+	info!("Finished import of {} blocks. Skipped {} blocks.", counter, skipped);
+
+	Ok(())
+}

pbtc/commands/mod.rs

@@ -0,0 +1,5 @@
+mod import;
+mod start;
+
+pub use self::import::import;
+pub use self::start::start;

pbtc/commands/start.rs

@@ -0,0 +1,36 @@
+use std::net::SocketAddr;
+use sync::create_sync_connection_factory;
+use util::{open_db, init_db};
+use {config, p2p};
+
+pub fn start(cfg: config::Config) -> Result<(), String> {
+	let mut el = p2p::event_loop();
+
+	let p2p_cfg = p2p::Config {
+		threads: 4,
+		protocol_minimum: 70001,
+		protocol_maximum: 70017,
+		inbound_connections: 10,
+		outbound_connections: 10,
+		connection: p2p::NetConfig {
+			magic: cfg.magic,
+			local_address: SocketAddr::new("127.0.0.1".parse().unwrap(), cfg.port),
+			services: Default::default(),
+			user_agent: "pbtc".into(),
+			start_height: 0,
+			relay: false,
+		},
+		peers: cfg.connect.map_or_else(|| vec![], |x| vec![x]),
+		seeds: cfg.seednode.map_or_else(|| vec![], |x| vec![x]),
+	};
+
+	let db = open_db(cfg.use_disk_database);
+	init_db(&db);
+
+	let sync_connection_factory = create_sync_connection_factory(db);
+
+	let p2p = p2p::P2P::new(p2p_cfg, sync_connection_factory, el.handle());
+	try!(p2p.run().map_err(|_| "Failed to start p2p module"));
+	el.run(p2p::forever()).unwrap();
+	Ok(())
+}

pbtc/config.rs

@@ -1,6 +1,6 @@
 use std::net;
 use clap;
-use message::common::Magic;
+use message::Magic;
 
 pub struct Config {
 	pub magic: Magic,

pbtc/main.rs

@@ -2,7 +2,10 @@
 
 #[macro_use]
 extern crate clap;
+#[macro_use]
+extern crate log;
 extern crate env_logger;
+extern crate app_dirs;
 
 extern crate db;
 extern crate chain;
@@ -13,14 +16,13 @@ extern crate p2p;
 extern crate sync;
 extern crate import;
 
+mod commands;
 mod config;
+mod util;
 
-use std::env;
+use app_dirs::AppInfo;
-use std::sync::Arc;
+
-use std::net::SocketAddr;
+pub const APP_INFO: AppInfo = AppInfo { name: "pbtc", author: "Parity" };
-use p2p::{P2P, event_loop, forever, NetConfig};
-use sync::create_sync_connection_factory;
-use chain::Block;
 
 fn main() {
 	env_logger::init().unwrap();
@@ -30,79 +32,16 @@ fn main() {
 	}
 }
 
-fn open_db(use_disk_database: bool) -> Arc<db::Store> {
-	match use_disk_database {
-		true => {
-			let mut db_path = env::home_dir().expect("Failed to get home dir");
-			db_path.push(".pbtc");
-			db_path.push("db");
-			Arc::new(db::Storage::new(db_path).expect("Failed to open database"))
-		},
-		false => {
-			Arc::new(db::TestStorage::default())
-		}
-	}
-}
-
-fn init_db(db: &Arc<db::Store>) {
-	// insert genesis block if db is empty
-	if db.best_block().is_none() {
-		// TODO: move to config
-		let genesis_block: Block = "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000".into();
-		db.insert_block(&genesis_block)
-			.expect("Failed to insert genesis block to the database");
-	}
-}
 
 fn run() -> Result<(), String> {
 	let yaml = load_yaml!("cli.yml");
 	let matches = clap::App::from_yaml(yaml).get_matches();
+	let cfg = try!(config::parse(&matches));
 
 	match matches.subcommand() {
-		("import", Some(import_matches)) => import_blockchain(import_matches),
+		("import", Some(import_matches)) => commands::import(cfg, import_matches),
-		_ => start_pbtc(&matches),
+		_ => commands::start(cfg),
 	}
 }
 
-fn import_blockchain(matches: &clap::ArgMatches) -> Result<(), String> {
-	let db_path = matches.value_of("PATH").expect("PATH is required in cli.yml; qed");
-	let blk_dir = try!(import::open_blk_dir(db_path).map_err(|_| "Import directory does not exist".to_owned()));
-	for (_i, _blk) in blk_dir.enumerate() {
-		// import logic goes here
-	}
-	Ok(())
-}
 
-fn start_pbtc(matches: &clap::ArgMatches) -> Result<(), String> {
-	let cfg = try!(config::parse(matches));
-
-	let mut el = event_loop();
-
-	let p2p_cfg = p2p::Config {
-		threads: 4,
-		protocol_minimum: 70001,
-		protocol_maximum: 70017,
-		inbound_connections: 10,
-		outbound_connections: 10,
-		connection: NetConfig {
-			magic: cfg.magic,
-			local_address: SocketAddr::new("127.0.0.1".parse().unwrap(), cfg.port),
-			services: Default::default(),
-			user_agent: "pbtc".into(),
-			start_height: 0,
-			relay: false,
-		},
-		peers: cfg.connect.map_or_else(|| vec![], |x| vec![x]),
-		seeds: cfg.seednode.map_or_else(|| vec![], |x| vec![x]),
-	};
-
-	let db = open_db(cfg.use_disk_database);
-	init_db(&db);
-
-	let sync_connection_factory = create_sync_connection_factory(db);
-
-	let p2p = P2P::new(p2p_cfg, sync_connection_factory, el.handle());
-	try!(p2p.run().map_err(|_| "Failed to start p2p module"));
-	el.run(forever()).unwrap();
-	Ok(())
-}

pbtc/util.rs

@@ -0,0 +1,26 @@
+use std::sync::Arc;
+use app_dirs::{app_dir, AppDataType};
+use chain::Block;
+use {db, APP_INFO};
+
+pub fn open_db(use_disk_database: bool) -> Arc<db::Store> {
+	match use_disk_database {
+		true => {
+			let db_path = app_dir(AppDataType::UserData, &APP_INFO, "db").expect("Failed to get app dir");
+			Arc::new(db::Storage::new(db_path).expect("Failed to open database"))
+		},
+		false => {
+			Arc::new(db::TestStorage::default())
+		}
+	}
+}
+
+pub fn init_db(db: &Arc<db::Store>) {
+	// insert genesis block if db is empty
+	if db.best_block().is_none() {
+		// TODO: move to config
+		let genesis_block: Block = "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000".into();
+		db.insert_block(&genesis_block)
+			.expect("Failed to insert genesis block to the database");
+	}
+}