diff --git a/app/models/TransactionOut.js b/app/models/TransactionOut.js index 56548dc9..8247c5e2 100644 --- a/app/models/TransactionOut.js +++ b/app/models/TransactionOut.js @@ -196,6 +196,8 @@ TransactionOutSchema.statics.createFromTxs = function(txs, next) { }); }, function(a_cb) { + if (txid === genesisTXID) return a_cb(); + Self.storeTransactionOuts(txInfo, function(err, addrs) { if (err) return a_cb(err); diff --git a/lib/BlockExtractor.js b/lib/BlockExtractor.js index 5f1b7d30..1802788b 100644 --- a/lib/BlockExtractor.js +++ b/lib/BlockExtractor.js @@ -80,11 +80,9 @@ function spec() { var fd = fs.openSync(fname, 'r'); -// if (status) return cb(new Error(status.message)); - var buffer = new Buffer(size); - var num = fs.readSync(fd, buffer, 0, size, 0); + fs.readSync(fd, buffer, 0, size, 0); self.currentBuffer = buffer; self.currentParser = new Parser(buffer); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 0fcf69b7..f1aec64a 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -13,6 +13,7 @@ function spec() { var Block = require('../app/models/Block'); var Sync = require('./Sync').class(); var sockets = require('../app/controllers/socket.js'); + var BlockExtractor = require('./BlockExtractor.js').class(); var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:'; @@ -125,7 +126,6 @@ if (self.syncPercentage>5) { var existed = false; var blockInfo; - var blockObj; async.series([ // Already got it? @@ -137,7 +137,6 @@ if (self.syncPercentage>5) { } if (block) { existed = true; - blockObj = block; } return c(); }); @@ -150,22 +149,20 @@ if (self.syncPercentage>5) { return c(); }, - //get Info from RPC - function(c) { - // TODO: if we store prev/next, no need to go to RPC - // if (blockObj && blockObj.nextBlockHash) return c(); + function(c) { self.rpc.getBlock(blockHash, function(err, ret) { if (err) return c(err); - blockInfo = ret; + blockInfo = ret ? ret.result : null; return c(); }); }, //store it function(c) { if (existed) return c(); - self.sync.storeBlock(blockInfo.result, function(err) { + + self.sync.storeBlock(blockInfo, function(err) { existed = err && err.toString().match(/E11000/); @@ -190,6 +187,7 @@ if (self.syncPercentage>5) { self.status = 'aborted'; self.showProgress(); p(self.err); + return cb(err); } else { self.err = null; @@ -205,7 +203,7 @@ if (self.syncPercentage>5) { } // Continue - if (blockInfo && blockInfo.result) { + if (blockInfo) { if (existed) self.skippedBlocks++; @@ -213,16 +211,98 @@ if (self.syncPercentage>5) { self.syncedBlocks++; // recursion - if (scanOpts.prev && blockInfo.result.previousblockhash) - return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, scanOpts, cb); + if (scanOpts.prev && blockInfo.previousblockhash) + return self.getPrevNextBlock(blockInfo.previousblockhash, blockEnd, scanOpts, cb); - if (scanOpts.next && blockInfo.result.nextblockhash) - return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, scanOpts, cb); + if (scanOpts.next && blockInfo.nextblockhash) + return self.getPrevNextBlock(blockInfo.nextblockhash, blockEnd, scanOpts, cb); } return cb(err); }); }; + + HistoricSync.prototype.getBlockFromFile = function(height, scanOpts, cb) { + var self = this; + + var nextHash; + var blockInfo; + var isMainChain; + var existed; + + async.series([ + // Is it in mainchain? + function(c) { + self.rpc.getBlockHash(height, function(err, res) { + if (err) return cb(err); + + nextHash = res.result; +//console.log('[HistoricSync.js.235:nextHash:]',nextHash); //TODO + return c(); + }); + }, + //show some (inacurate) status + function(c) { + if ( ( self.syncedBlocks + self.skippedBlocks) % self.step === 1) { + self.showProgress(); + } + + return c(); + }, + //get Info + function(c) { + self.blockExtractor.getNextBlock(function(err, b) { + if (err || ! b) return c(err); + + blockInfo = b.getStandardizedObject(b.txs); + return c(); + }); + }, + //store it + function(c) { + + isMainChain = blockInfo.hash === nextHash; + + //TODO + blockInfo.isOrphan = !isMainChain; + + self.sync.storeBlock(blockInfo, function(err) { + existed = err && err.toString().match(/E11000/); + + if (err && ! existed) return c(err); + return c(); + }); + }, + ], function(err) { + + if (err) { + self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo.hash, err, self.syncedBlocks); + self.status = 'aborted'; + self.showProgress(); + p(err); + return cb(err); + } + else { + self.err = null; + self.status = 'syncing'; + } + + // Continue + if (blockInfo) { + + // mainchain + if (isMainChain) height++; + + self.syncedBlocks++; + + return self.getBlockFromFile(height, scanOpts, cb); + } + return cb(err); + }); + }; + + + HistoricSync.prototype.importHistory = function(scanOpts, next) { var self = this; @@ -290,16 +370,23 @@ if (self.syncPercentage>5) { p(' to : ', end); p(' scanOpts: ', JSON.stringify(scanOpts)); - self.getPrevNextBlock(start, end, scanOpts, function(err) { - if (err && err.message.match(/ECONNREFUSED/)) { - setTimeout(function() { - p('Retrying in %d secs', retry_secs); - sync(); - }, - retry_secs * 1000); - } - else return next(err); - }); + if (scanOpts.fromFiles) { + self.getBlockFromFile(0, scanOpts, function(err) { + return next(err); + }); + } + else { + self.getPrevNextBlock(start, end, scanOpts, function(err) { + if (err && err.message.match(/ECONNREFUSED/)) { + setTimeout(function() { + p('Retrying in %d secs', retry_secs); + sync(); + }, + retry_secs * 1000); + } + else return next(err); + }); + } } @@ -326,7 +413,7 @@ if (self.syncPercentage>5) { }; // upto if we have genesis block? - HistoricSync.prototype.smartImport = function(next) { + HistoricSync.prototype.smartImport = function(scanOpts, next) { var self = this; Block.fromHash(self.genesis, function(err, b) { @@ -334,13 +421,12 @@ if (self.syncPercentage>5) { if (err) return next(err); - var scanOpts = {}; - - if (!b) { + if (!b || scanOpts.destroy) { p('Could not find Genesis block. Running FULL SYNC'); if (config.bitcoind.dataDir) { p('bitcoind dataDir configured...importing blocks from .dat files'); scanOpts.fromFiles = true; + self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); } else { scanOpts.reverse = true; diff --git a/lib/Sync.js b/lib/Sync.js index ab70e4d9..bb445fad 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -65,6 +65,7 @@ function spec() { function(b) { try {self.db.collections.transactionouts.drop(b);} catch (e) { return b(); } }, ], next); }; + Sync.prototype.storeBlock = function(block, cb) { var self = this; diff --git a/util/sync.js b/util/sync.js index 48a4307c..3c1af7d9 100755 --- a/util/sync.js +++ b/util/sync.js @@ -16,6 +16,7 @@ program .option('-D --destroy', 'Remove current DB (and start from there)', 0) .option('-R --reverse', 'Sync backwards', 0) .option('-U --uptoexisting', 'Sync only until an existing block is found', 0) + .option('-F --fromfiles', 'Sync using bitcoind .dat block files (faster)', 0) .parse(process.argv); var historicSync = new HistoricSync(); @@ -32,13 +33,16 @@ async.series([ function(cb) { if (program.smart) { - historicSync.smartImport(cb); + historicSync.smartImport({ + destroy: program.destroy, + },cb); } else { historicSync.importHistory({ destroy: program.destroy, reverse: program.reverse, upToExisting: program.uptoexisting, + fromFiles: program.fromfiles, }, cb); } },