diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 4cb79573..b664bc6b 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -5,23 +5,21 @@ require('classtool'); function spec(b) { - var TIMESTAMP_PREFIX = 'bts-'; // b-ts- => + var TIMESTAMP_PREFIX = 'bts-'; // b-ts- => var PREV_PREFIX = 'bpr-'; // b-prev- => var NEXT_PREFIX = 'bne-'; // b-next- => var MAIN_PREFIX = 'bma-'; // b-main- => 1/0 - var TIP = 'bti-'; // last block on the chain + var TIP = 'bti-'; // last block on the chain + var LAST_FILE_INDEX = 'file-'; // last processed file index /** * Module dependencies. */ - var RpcClient = require('bitcore/RpcClient').class(), - util = require('bitcore/util/util'), - levelup = require('levelup'), - BitcoreBlock= require('bitcore/Block').class(), + var levelup = require('levelup'), config = require('../config/config'); - var db = b.db || levelup(config.leveldb + '/blocks'); - var rpc = b.rpc || new RpcClient(config.bitcoind); + var db = b.db || levelup(config.leveldb + '/blocks'); + var Rpc = b.rpc || require('./Rpc').class(); var BlockDb = function() { }; @@ -80,6 +78,24 @@ function spec(b) { }); }; + + BlockDb.prototype.setLastFileIndex = function(idx, cb) { + var self = this; + if (this.lastFileIndexSaved === idx) return cb(); + + db.put(LAST_FILE_INDEX, idx, function(err) { + self.lastFileIndexSaved = idx; + return cb(err); + }); + }; + + BlockDb.prototype.getLastFileIndex = function(cb) { + db.get(LAST_FILE_INDEX, function(err,val) { + if (err && err.notFound) { err = null; val = null;} + return cb(err,val); + }); + }; + BlockDb.prototype.getNext = function(hash, cb) { db.get(NEXT_PREFIX + hash, function(err,val) { if (err && err.notFound) { err = null; val = null;} @@ -138,13 +154,8 @@ function spec(b) { BlockDb.prototype.fromHashWithInfo = function(hash, cb) { var self = this; - rpc.getBlock(hash, function(err, info) { - // Not found? - if (err && err.code === -5) return cb(); - if (err) return cb(err); - - if (info.result.height) - info.result.reward = BitcoreBlock.getBlockValue(info.result.height) / util.COIN ; + Rpc.getBlock(hash, function(err, info) { + if (err || !info) return cb(err); self.isMain(hash, function(err, val) { if (err) return cb(err); @@ -182,12 +193,7 @@ function spec(b) { }; BlockDb.prototype.blockIndex = function(height, cb) { - var rpc = new RpcClient(config.bitcoind); - rpc.getBlockHash(height, function(err, bh){ - if (err) return cb(err); - - cb(null, { blockHash: bh.result }); - }); + return Rpc.blockIndex(height,cb); }; return BlockDb; diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 753e2de0..91f412a4 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -14,7 +14,8 @@ function spec() { var Sync = require('./Sync').class(); var sockets = require('../app/controllers/socket.js'); var BlockExtractor = require('./BlockExtractor.js').class(); - //var Deserialize = require('bitcore/Deserialize'); + // var bitcoreUtil = require('bitcore/util/util'); + // var Deserialize = require('bitcore/Deserialize'); var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:'; @@ -122,11 +123,10 @@ function spec() { if (self.opts.shouldBroadcastSync) { sockets.broadcastSyncInfo(self.info()); } - -//TODO -// if (self.syncPercentage > 10) { -// process.exit(-1); -// } +// +// if (self.syncPercentage > 10) { +// process.exit(-1); +// } }; HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) { @@ -229,7 +229,6 @@ function spec() { //get Info self.blockExtractor.getNextBlock(function(err, b) { if (err || ! b) return cb(err); - blockInfo = b.getStandardizedObject(b.txs, self.network); // blockInfo.curWork = Deserialize.intFromCompact(b.bits); // We keep the RPC field names @@ -277,19 +276,24 @@ function spec() { } self.sync.storeTipBlock(blockInfo, function(err) { - if (blockInfo && blockInfo.hash) { - self.syncedBlocks++; - } else - self.status = 'finished'; - if (err) { self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); + return cb(err); } - return cb(err); + + self.sync.bDb.setLastFileIndex(self.blockExtractor.currentFileIndex, function(err) { + if (err) return cb(err); + + if (blockInfo && blockInfo.hash) { + self.syncedBlocks++; + } else + self.status = 'finished'; + + return cb(err); + }); }); }); - }; @@ -307,7 +311,7 @@ function spec() { }; - HistoricSync.prototype.getBlockCount = function(cb) { + HistoricSync.prototype.updateBlockCount = function(cb) { var self = this; if (self.blockChainHeight) return cb(); @@ -319,64 +323,110 @@ function spec() { }); }; - HistoricSync.prototype.importHistory = function(scanOpts, next) { + HistoricSync.prototype.smartImport = function(scanOpts, next) { var self = this; + var genesis, count; var lastBlock; var tip; async.series([ - function(cb) { - if (scanOpts.destroy) { + function(s_c) { + if (!scanOpts.destroy) return s_c(); + p('Deleting DB...'); - return self.sync.destroy(cb); - } - return cb(); - }, - // We are not using getBestBlockHash, because is not available in all clients - function (cb) { return self.getBlockCount(cb); }, - function(cb) { - if (!scanOpts.reverse) return cb(); - self.rpc.getBlockHash(self.blockChainHeight, function(err, res) { - if (err) return cb(err); - lastBlock = res.result; - return cb(); - }); - }, - function(cb) { - if (!scanOpts.reverse) return cb(); - self.sync.bDb.getTip(function(err, res) { - if (err) return cb(err); - tip = res; + return self.sync.destroy(s_c); + }, + function(s_c) { + self.sync.bDb.has(self.genesis, function(err, b) { + genesis = b; + return s_c(err); + }); + }, + function(s_c) { + self.countNotOrphan(function(err, c) { + count = c; + return s_c(err); + }); + }, + function(s_c) { + if (!config.bitcoind.dataDir) return s_c(); + if (scanOpts.startFile) { + self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); + self.blockExtractor.currentFileIndex = scanOpts.startFile; + return s_c(); + } + self.sync.bDb.getLastFileIndex(function(err, idx) { + self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); + if (idx) self.blockExtractor.currentFileIndex = idx; + return s_c(err); + }); + }, + function(s_c) { + self.updateBlockCount(s_c); + }, - console.log('Old Tip:', tip); - return cb(); - }); - }, - - function(cb) { - if (scanOpts.reverse) { + // define sync strategy + function(s_c) { + if (!genesis || scanOpts.destroy || count < self.blockChainHeight * 0.9 ) { + + // Full sync. + if (!genesis) + p('Could not find Genesis block. Running FULL SYNC'); + else + p('Less that 90% of current blockchain is stored. Running FULL SYNC', + parseInt(count/self.blockChainHeight*100)); + + if (config.bitcoind.dataDir) { + p('bitcoind dataDir configured...importing blocks from .dat files'); + p('Starting from file: ' + self.blockExtractor.currentFileIndex); + scanOpts.fromFiles = true; + } + else { + scanOpts.reverse = true; + } + } + else { + p('Genesis block found. Syncing upto old TIP.'); + p('Got ' + count + ' out of ' + self.blockChainHeight + ' blocks'); + scanOpts.reverse = true; + } + + if (!scanOpts.reverse) return s_c(); + + self.rpc.getBlockHash(self.blockChainHeight, function(err, res) { + if (err) return s_c(err); + lastBlock = res.result; + return s_c(); + }); + }, + function(s_c) { + if (!scanOpts.reverse) return s_c(); + self.sync.bDb.getTip(function(err, res) { + if (err) return s_c(err); + tip = res; + + console.log('Old Tip:', tip); + return s_c(); + }); + }, + function(s_c) { + if (!scanOpts.reverse) return s_c(); self.countNotOrphan(function(err, count) { - if (err) return cb(err); + if (err) return s_c(err); self.syncedBlocks = count || 0; - return cb(); + return s_c(); }); - } - else { - return cb(); - } - }, - ], function(err) { + }], + function(err) { + // SETUP Sync params + var start, end; if (err) { self.setError(err); return next(err, 0); } - - // SETUP Sync params - var start, end; - if (!self.step) { var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000); @@ -428,46 +478,6 @@ function spec() { return next(err); }); } - - }); - }; - - // upto if we have genesis block? - HistoricSync.prototype.smartImport = function(scanOpts, next) { - var self = this; - - self.sync.bDb.has(self.genesis, function(err, b) { - if (err) return next(err); - self.countNotOrphan(function(err, count) { - if (err) return next(err); - self.getBlockCount(function(err) { - if (err) return next(err); - - if (!b || scanOpts.destroy || count < self.blockChainHeight * 0.8 ) { - - if (!b) - p('Could not find Genesis block. Running FULL SYNC'); - else - p('Less that 80% of current blockchain is stored. Running FULL SYNC', - parseInt(count/self.blockChainHeight*100)); - - if (config.bitcoind.dataDir) { - p('bitcoind dataDir configured...importing blocks from .dat files'); - scanOpts.fromFiles = true; - self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); - } - else { - scanOpts.reverse = true; - } - } - else { - p('Genesis block found. Syncing upto old TIP.'); - p('Got ' + count + ' out of ' + self.blockChainHeight + ' blocks'); - scanOpts.reverse = true; - } - return self.importHistory(scanOpts, next); - }); - }); }); }; diff --git a/lib/Rpc.js b/lib/Rpc.js new file mode 100644 index 00000000..fe8735b9 --- /dev/null +++ b/lib/Rpc.js @@ -0,0 +1,103 @@ +'use strict'; + +require('classtool'); + + +function spec(b) { + var RpcClient = require('bitcore/RpcClient').class(), + BitcoreTransaction = require('bitcore/Transaction').class(), + BitcoreBlock = require('bitcore/Block').class(), + bitcoreUtil = require('bitcore/util/util'), + util = require('util'), + config = require('../config/config'); + + var bitcoreRpc = b.bitcoreRpc || new RpcClient(config.bitcoind); + + function Rpc() { + } + + Rpc._parseRpcResult = function(info) { + var b = new Buffer(info.hex,'hex'); + var tx = new BitcoreTransaction(); + tx.parse(b); + + // Inputs + if (tx.isCoinBase()) { + info.isCoinBase = true; + } + + var n =0; + info.vin.forEach(function(i) { + i.n = n++; + }); + + // Outputs + var valueOutSat = 0; + info.vout.forEach( function(o) { + valueOutSat += o.value * bitcoreUtil.COIN; + }); + info.valueOut = parseInt(valueOutSat) / bitcoreUtil.COIN; + info.size = b.length; + + return info; + }; + + + Rpc.errMsg = function(err) { + var e = err; + e.message += util.format(' [Host: %s:%d User:%s Using password:%s]', + bitcoreRpc.host, + bitcoreRpc.port, + bitcoreRpc.user, + bitcoreRpc.pass?'yes':'no' + ); + return e; + }; + + Rpc.getRpcInfo = function(txid, cb) { + var self = this; + + bitcoreRpc.getRawTransaction(txid, 1, function(err, txInfo) { + + // Not found? + if (err && err.code === -5) return cb(); + if (err) return cb(self.errMsg(err)); + + var info = self._parseRpcResult(txInfo.result); + + return cb(null,info); + }); + }; + + + Rpc.blockIndex = function(height, cb) { + var self = this; + + bitcoreRpc.getBlockHash(height, function(err, bh){ + if (err) return cb(self.errMsg(err)); + cb(null, { blockHash: bh.result }); + }); + }; + + Rpc.getBlock = function(hash, cb) { + var self = this; + + bitcoreRpc.getBlock(hash, function(err,info) { + // Not found? + if (err && err.code === -5) return cb(); + if (err) return cb(self.errMsg(err)); + + + if (info.result.height) + info.result.reward = BitcoreBlock.getBlockValue(info.result.height) / util.COIN ; + + return cb(err,info); + }); + }; + + + return Rpc; +} +module.defineClass(spec); + + diff --git a/lib/Sync.js b/lib/Sync.js index d343fd0d..0102512a 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -143,7 +143,6 @@ function spec() { function(err) { if (!err) self._handleBroadcast(b.hash, null, updatedAddrs); if (err && err.toString().match(/WARN/) ) { - console.log(err); err=null; } return cb(err); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 607d10c9..720e62fc 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -25,7 +25,7 @@ function spec(b) { /** * Module dependencies. */ - var TransactionRpc = require('./TransactionRpc').class(), + var Rpc = b.rpc || require('./Rpc').class(), util = require('bitcore/util/util'), levelup = require('levelup'), async = require('async'), @@ -224,7 +224,7 @@ function spec(b) { TransactionDb.prototype._getInfo = function(txid, next) { var self = this; - TransactionRpc.getRpcInfo(txid, function(err, info) { + Rpc.getRpcInfo(txid, function(err, info) { if (err) return next(err); self._fillOutpoints(info, function() { @@ -573,7 +573,7 @@ function spec(b) { // TODO: parse it from networks.genesisTX? if (t === genesisTXID) return each_cb(); - TransactionRpc.getRpcInfo(t, function(err, inInfo) { + Rpc.getRpcInfo(t, function(err, inInfo) { if (!inInfo) return each_cb(err); return self.add(inInfo, blockHash, each_cb); diff --git a/lib/TransactionRpc.js b/lib/TransactionRpc.js deleted file mode 100644 index f249a7f9..00000000 --- a/lib/TransactionRpc.js +++ /dev/null @@ -1,62 +0,0 @@ -'use strict'; - -require('classtool'); - - -function spec(b) { - var RpcClient = require('bitcore/RpcClient').class(), - BitcoreTransaction = require('bitcore/Transaction').class(), - util = require('bitcore/util/util'), - config = require('../config/config'); - - var rpc = b.rpc || new RpcClient(config.bitcoind); - - function TransactionRpc() { - } - - TransactionRpc._parseRpcResult = function(info) { - var b = new Buffer(info.hex,'hex'); - var tx = new BitcoreTransaction(); - tx.parse(b); - - // Inputs - if (tx.isCoinBase()) { - info.isCoinBase = true; - } - - var n =0; - info.vin.forEach(function(i) { - i.n = n++; - }); - - // Outputs - var valueOutSat = 0; - info.vout.forEach( function(o) { - valueOutSat += o.value * util.COIN; - }); - info.valueOut = parseInt(valueOutSat) / util.COIN; - info.size = b.length; - - return info; - }; - - TransactionRpc.getRpcInfo = function(txid, cb) { - var Self = this; - - rpc.getRawTransaction(txid, 1, function(err, txInfo) { - - // Not found? - if (err && err.code === -5) return cb(); - if (err) return cb(err); - - var info = Self._parseRpcResult(txInfo.result); - - return cb(null,info); - }); - }; - - return TransactionRpc; -} -module.defineClass(spec); - - diff --git a/util/sync.js b/util/sync.js index bfa7c940..61be7e88 100755 --- a/util/sync.js +++ b/util/sync.js @@ -12,12 +12,8 @@ var async = require('async'); program .version(SYNC_VERSION) - .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet') .option('-D --destroy', 'Remove current DB (and start from there)', 0) - .option('-R --reverse', 'Sync backwards', 0) - .option('-U --uptoexisting', 'Sync only until old Tip block is found', 0) - .option('-F --fromfiles', 'Sync using bitcoind .dat block files (faster)', 0) - .option('-S --smart', 'genesis stored? uptoexisting = 1, fromFiles=1 [default]', true) + .option('-S --startfile', 'Number of file from bitcoind to start(default=0)') .option('-v --verbose', 'Verbose 0/1', 0) .parse(process.argv); @@ -33,19 +29,10 @@ async.series([ historicSync.init(program, cb); }, function(cb) { - - if (typeof program.smart === 'undefined' || parseInt(program.smart) ) { - historicSync.smartImport({ - destroy: program.destroy, - },cb); - } - else { - historicSync.importHistory({ - destroy: program.destroy, - reverse: program.reverse, - fromFiles: program.fromfiles, - }, cb); - } + historicSync.smartImport({ + destroy: program.destroy, + startFile: program.startfile, + },cb); }, ], function(err) {