diff --git a/lib/Sync.js b/lib/Sync.js index 2408cf3..af05493 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -7,206 +7,206 @@ require('classtool'); var isSyncTxEnabled = 0; function spec() { - var mongoose = require('mongoose'); - var util = require('util'); + var mongoose = require('mongoose'); + var util = require('util'); - var RpcClient = require('bitcore/RpcClient').class(); - var networks = require('bitcore/networks'); - var async = require('async'); + var RpcClient = require('bitcore/RpcClient').class(); + var networks = require('bitcore/networks'); + var async = require('async'); - var config = require('../config/config'); - var Block = require('../app/models/Block'); - var Transaction = require('../app/models/Transaction'); + var config = require('../config/config'); + var Block = require('../app/models/Block'); + var Transaction = require('../app/models/Transaction'); - function Sync(config) { - this.network = config.networkName === 'testnet' ? networks.testnet: networks.livenet; - } + function Sync(config) { + this.network = config.networkName === 'testnet' ? networks.testnet: networks.livenet; + } - var progress_bar = function(string, current, total) { - console.log(util.format('\t%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total))); - }; + var progress_bar = function(string, current, total) { + console.log(util.format('\t%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total))); + }; - Sync.prototype.getNextBlock = function(blockHash, cb) { - var that = this; - if (!blockHash) { - return cb(); - } - this.rpc.getBlock(blockHash, function(err, blockInfo) { - if (err) return cb(err); - if (blockInfo.result.height % 1000 === 0) { - var h = blockInfo.result.height, - d = blockInfo.result.confirmations; - progress_bar('height', h, h + d); - } + Sync.prototype.getNextBlock = function(blockHash, cb) { + var that = this; + if (!blockHash) { + return cb(); + } + this.rpc.getBlock(blockHash, function(err, blockInfo) { + if (err) return cb(err); + if (blockInfo.result.height % 1000 === 0) { + var h = blockInfo.result.height, + d = blockInfo.result.confirmations; + progress_bar('height', h, h + d); + } - that.storeBlock(blockInfo.result, function(err) { - if (!err) { - var txs = blockInfo.result.tx; - that.storeTxs(txs, function(err) { - if (!err) { - return that.getNextBlock(blockInfo.result.nextblockhash, cb); - } - }); - } - }); - }); - }; + that.storeBlock(blockInfo.result, function(err) { + if (!err) { + var txs = blockInfo.result.tx; + that.storeTxs(txs, function(err) { + if (!err) { + return that.getNextBlock(blockInfo.result.nextblockhash, cb); + } + }); + } + }); + }); + }; - Sync.prototype.storeBlock = function(block, cb) { - Block.create(block, function(err, inBlock) { - // E11000 => already exists - if (err && ! err.toString().match(/E11000/)) { - return cb(err); - } - cb(); - }); - }; + Sync.prototype.storeBlock = function(block, cb) { + Block.create(block, function(err, inBlock) { + // E11000 => already exists + if (err && ! err.toString().match(/E11000/)) { + return cb(err); + } + cb(); + }); + }; - Sync.prototype.storeTxs = function(txs, cb) { - Transaction.createFromArray(txs, cb); - }; + Sync.prototype.storeTxs = function(txs, cb) { + Transaction.createFromArray(txs, cb); + }; - Sync.prototype.syncBlocks = function(reindex, cb) { - var that = this; - var genesisHash = this.network.genesisBlock.hash.reverse().toString('hex'); + Sync.prototype.syncBlocks = function(reindex, cb) { + var that = this; + var genesisHash = this.network.genesisBlock.hash.reverse().toString('hex'); - console.log('Syncing Blocks... ' + reindex); - if (reindex) { - return this.getNextBlock(genesisHash, cb); - } + console.log('Syncing Blocks... ' + reindex); + if (reindex) { + return this.getNextBlock(genesisHash, cb); + } - Block.findOne({}, - {}, - { - sort: { - 'time': - 1 - } - }, - function(err, block) { - if (err) return cb(err); + Block.findOne({}, + {}, + { + sort: { + 'time': - 1 + } + }, + function(err, block) { + if (err) return cb(err); - var nextHash = block && block.hash ? block.hash: genesisHash; + var nextHash = block && block.hash ? block.hash: genesisHash; - console.log('\tStarting at hash: ' + nextHash); - return that.getNextBlock(nextHash, cb); - }); - }; + console.log('\tStarting at hash: ' + nextHash); + return that.getNextBlock(nextHash, cb); + }); + }; - // This is not currently used. Transactions are represented by txid only - // in mongodb - Sync.prototype.syncTXs = function(reindex, cb) { + // This is not currently used. Transactions are represented by txid only + // in mongodb + Sync.prototype.syncTXs = function(reindex, cb) { - var that = this; + var that = this; - console.log('Syncing TXs...'); - if (reindex) { - // TODO? - } + console.log('Syncing TXs...'); + if (reindex) { + // TODO? + } - Transaction.find({ - blockhash: null - }, - function(err, txs) { - if (err) return cb(err); + Transaction.find({ + blockhash: null + }, + function(err, txs) { + if (err) return cb(err); - var read = 0; - var pull = 0; - var write = 0; - var total = txs.length; - console.log('\tneed to pull %d txs', total); + var read = 0; + var pull = 0; + var write = 0; + var total = txs.length; + console.log('\tneed to pull %d txs', total); - if (!total) return cb(); + if (!total) return cb(); - async.each(txs, function(tx, next) { - if (!tx.txid) { - console.log('NO TXID skipping...', tx); - return next(); - } + async.each(txs, function(tx, next) { + if (!tx.txid) { + console.log('NO TXID skipping...', tx); + return next(); + } - if (read++ % 1000 === 0) progress_bar('read', read, total); + if (read++ % 1000 === 0) progress_bar('read', read, total); - that.rpc.getRawTransaction(tx.txid, 1, function(err, txInfo) { + that.rpc.getRawTransaction(tx.txid, 1, function(err, txInfo) { - if (pull++ % 1000 === 0) progress_bar('\tpull', pull, total); + if (pull++ % 1000 === 0) progress_bar('\tpull', pull, total); - if (!err && txInfo) { - Transaction.update({ - txid: tx.txid - }, - txInfo.result, function(err) { - if (err) return next(err); + if (!err && txInfo) { + Transaction.update({ + txid: tx.txid + }, + txInfo.result, function(err) { + if (err) return next(err); - if (write++ % 1000 === 0) progress_bar('\t\twrite', write, total); + if (write++ % 1000 === 0) progress_bar('\t\twrite', write, total); - return next(); - }); - } - else return next(); - }); - }, - function(err) { - if (err) return cb(err); - return cb(err); - }); - }); - }; + return next(); + }); + } + else return next(); + }); + }, + function(err) { + if (err) return cb(err); + return cb(err); + }); + }); + }; - Sync.prototype.init = function(opts) { + Sync.prototype.init = function(opts) { - mongoose.connect(config.db); - this.db = mongoose.connection; - this.rpc = new RpcClient(config.bitcoind); + mongoose.connect(config.db); + this.db = mongoose.connection; + this.rpc = new RpcClient(config.bitcoind); - this.db.on('error', console.error.bind(console, 'connection error:')); + this.db.on('error', console.error.bind(console, 'connection error:')); - }; + }; - Sync.prototype.import_history = function(opts, next) { + Sync.prototype.import_history = function(opts, next) { - var that = this; - this.db.once('open', function() { - async.series([ - function(cb) { - if (opts.destroy) { - console.log('Deleting Blocks...'); - that.db.collections.blocks.drop(cb); - } else { - cb(); - } - }, - function(cb) { - if (opts.destroy) { - console.log('Deleting TXs...'); - that.db.collections.transactions.drop(cb); - } else { - cb(); - } - }, - function(cb) { - if (!opts.skip_blocks) { - that.syncBlocks(opts.reindex, cb); - } else { - cb(); - } - }, - function(cb) { - if (isSyncTxEnabled && ! opts.skip_txs) { - that.syncTXs(opts.reindex, cb); - } - else { - return cb(); - } - }], function(err) { - return next(err); - }); - }); - }; + var that = this; + this.db.once('open', function() { + async.series([ + function(cb) { + if (opts.destroy) { + console.log('Deleting Blocks...'); + that.db.collections.blocks.drop(cb); + } else { + cb(); + } + }, + function(cb) { + if (opts.destroy) { + console.log('Deleting TXs...'); + that.db.collections.transactions.drop(cb); + } else { + cb(); + } + }, + function(cb) { + if (!opts.skip_blocks) { + that.syncBlocks(opts.reindex, cb); + } else { + cb(); + } + }, + function(cb) { + if (isSyncTxEnabled && ! opts.skip_txs) { + that.syncTXs(opts.reindex, cb); + } + else { + return cb(); + } + }], function(err) { + return next(err); + }); + }); + }; - Sync.prototype.close = function() { - console.log("closing connection"); - this.db.close(); - }; - return Sync; + Sync.prototype.close = function() { + console.log("closing connection"); + this.db.close(); + }; + return Sync; } module.defineClass(spec); diff --git a/p2p.js b/p2p.js index 8b42c2c..bfbc768 100755 --- a/p2p.js +++ b/p2p.js @@ -18,153 +18,153 @@ var peerdb = undefined; var hdrdb = undefined; var network = networks.testnet; var config = { - network: network.name + network: network.name }; var PeerManager = require('bitcore/PeerManager').createClass({ - config: config + config: config }); var Peer = require('bitcore/Peer').class(); function peerdb_load() { - try { - peerdb = JSON.parse(fs.readFileSync(peerdb_fn)); - } catch(d) { - console.warn('Unable to read peer db', peerdb_fn, 'creating new one.'); - peerdb = [{ - ipv4: '127.0.0.1', - port: 18333 - }, - ]; + try { + peerdb = JSON.parse(fs.readFileSync(peerdb_fn)); + } catch(d) { + console.warn('Unable to read peer db', peerdb_fn, 'creating new one.'); + peerdb = [{ + ipv4: '127.0.0.1', + port: 18333 + }, + ]; - fs.writeFileSync(peerdb_fn, JSON.stringify(peerdb)); - } + fs.writeFileSync(peerdb_fn, JSON.stringify(peerdb)); + } } function hdrdb_load() { - hdrdb = new HeaderDB({ - network: network - }); + hdrdb = new HeaderDB({ + network: network + }); } function get_more_headers(info) { - var conn = info.conn; - var loc = hdrdb.locator(); - conn.sendGetHeaders(loc, coinUtil.NULL_HASH); + var conn = info.conn; + var loc = hdrdb.locator(); + conn.sendGetHeaders(loc, coinUtil.NULL_HASH); } function add_header(info, block) { - var hashStr = coinUtil.formatHashFull(block.calcHash()); + var hashStr = coinUtil.formatHashFull(block.calcHash()); - try { - hdrdb.add(block); - } catch(e) { - return; - } + try { + hdrdb.add(block); + } catch(e) { + return; + } } function handle_headers(info) { - console.log('handle headers'); - var headers = info.message.headers; + console.log('handle headers'); + var headers = info.message.headers; - headers.forEach(function(hdr) { - add_header(info, hdr); - }); + headers.forEach(function(hdr) { + add_header(info, hdr); + }); - // We persist the header DB after each batch - //hdrdb.writeFile(hdrdb_fn); - // Only one request per batch of headers we receive. - get_more_headers(info); + // We persist the header DB after each batch + //hdrdb.writeFile(hdrdb_fn); + // Only one request per batch of headers we receive. + get_more_headers(info); } function handle_verack(info) { - var inv = { - type: CoinConst.MSG.BLOCK, - hash: network.genesisBlock.hash, - }; - var invs = [inv]; + var inv = { + type: CoinConst.MSG.BLOCK, + hash: network.genesisBlock.hash, + }; + var invs = [inv]; - // Asks for the genesis block - // console.log('p2psync: Asking for the genesis block'); - // info.conn.sendGetData(invs); + // Asks for the genesis block + // console.log('p2psync: Asking for the genesis block'); + // info.conn.sendGetData(invs); } function handle_inv(info) { - // TODO: should limit the invs to objects we haven't seen yet - var invs = info.message.invs; - invs.forEach(function(inv) { - console.log('Handle inv for a ' + CoinConst.MSG.to_str(inv.type)); - }); - info.conn.sendGetData(invs); + // TODO: should limit the invs to objects we haven't seen yet + var invs = info.message.invs; + invs.forEach(function(inv) { + console.log('Handle inv for a ' + CoinConst.MSG.to_str(inv.type)); + }); + info.conn.sendGetData(invs); } var sync = new Sync({ - networkName: networks.testnet + networkName: networks.testnet }); sync.init(); function handle_tx(info) { - var tx = info.message.tx.getStandardizedObject(); - console.log('Handle tx: ' + tx.hash); - sync.storeTxs([tx.hash], function(err) { - if (err) { - console.log('error in handle TX: ' + err); - } - }); + var tx = info.message.tx.getStandardizedObject(); + console.log('Handle tx: ' + tx.hash); + sync.storeTxs([tx.hash], function(err) { + if (err) { + console.log('error in handle TX: ' + err); + } + }); } function handle_block(info) { - var block = info.message.block; - var now = Math.round(new Date().getTime() / 1000); - var blockHash = coinUtil.formatHashFull(block.calcHash()); - console.log('Handle block: ' + blockHash); - sync.storeBlock({ - 'hash': blockHash, - 'time': now - }, - function(err) { - if (err) { - console.log('error in handle Block: ' + err); - } else { - var hashes = block.txs.map(function(tx) { - return coinUtil.formatHashFull(tx.hash); - }); - sync.storeTxs(hashes, function() {}); - } - }); + var block = info.message.block; + var now = Math.round(new Date().getTime() / 1000); + var blockHash = coinUtil.formatHashFull(block.calcHash()); + console.log('Handle block: ' + blockHash); + sync.storeBlock({ + 'hash': blockHash, + 'time': now + }, + function(err) { + if (err) { + console.log('error in handle Block: ' + err); + } else { + var hashes = block.txs.map(function(tx) { + return coinUtil.formatHashFull(tx.hash); + }); + sync.storeTxs(hashes, function() {}); + } + }); } function handle_connected(data) { - var peerman = data.pm; - var peers_n = peerman.peers.length; - console.log('p2psync: Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's': '')); + var peerman = data.pm; + var peers_n = peerman.peers.length; + console.log('p2psync: Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's': '')); } function p2psync() { - var peerman = new PeerManager(); + var peerman = new PeerManager(); - peerdb.forEach(function(datum) { - var peer = new Peer(datum.ipv4, datum.port); - peerman.addPeer(peer); - }); + peerdb.forEach(function(datum) { + var peer = new Peer(datum.ipv4, datum.port); + peerman.addPeer(peer); + }); - peerman.on('connection', function(conn) { - conn.on('verack', handle_verack); - conn.on('block', handle_block); - conn.on('headers', handle_headers); - conn.on('inv', handle_inv); - conn.on('tx', handle_tx); - }); - peerman.on('connect', handle_connected); + peerman.on('connection', function(conn) { + conn.on('verack', handle_verack); + conn.on('block', handle_block); + conn.on('headers', handle_headers); + conn.on('inv', handle_inv); + conn.on('tx', handle_tx); + }); + peerman.on('connect', handle_connected); - peerman.start(); + peerman.start(); } function main() { - peerdb_load(); - hdrdb_load(); + peerdb_load(); + hdrdb_load(); - p2psync(); + p2psync(); } main(); diff --git a/util/sync.js b/util/sync.js index e26166a..1c58afa 100755 --- a/util/sync.js +++ b/util/sync.js @@ -1,4 +1,4 @@ -# ! /usr/bin / env node +#! /usr/bin/env node 'use strict'; @@ -14,7 +14,7 @@ var async = require('async'); program.version(SYNC_VERSION).option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet').option('-R --reindex', 'Force reindexing', '0').option('-D --destroy', 'Remove current DB', '0').option('--skip_blocks', 'Sync blocks').option('--skip_txs', 'Sync transactions').parse(process.argv); var sync = new Sync({ - networkName: program.network + networkName: program.network }); if (program.remove) { @@ -23,22 +23,22 @@ if (program.remove) { async.series([ function(cb) { - sync.init(program); - cb(); + sync.init(program); + cb(); }, function(cb) { - sync.import_history(program, function(err) { - if (err) { - console.log('CRITICAL ERROR: ', err); - } - else { - console.log('Done!'); - } - cb(); - }); + sync.import_history(program, function(err) { + if (err) { + console.log('CRITICAL ERROR: ', err); + } + else { + console.log('Done!'); + } + cb(); + }); }, function(cb) { - sync.close(); - cb(); + sync.close(); + cb(); }]);