From 444249763800fb8473d5d7b1457e0ac89669347b Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 17 Jan 2014 10:22:00 -0300 Subject: [PATCH 1/4] implements uptosync --- lib/HistoricSync.js | 32 +++++++++++++++++++------------- util/sync.js | 12 +++++++++--- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 2c432d9..908f341 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -111,6 +111,11 @@ function spec() { ], function (err){ + if (opts.uptoexisting && existed) { + p('DONE. Found existing block: %s ', blockHash); + return cb(err); + } + if (err) p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count); @@ -126,19 +131,17 @@ function spec() { }); }; - HistoricSync.prototype.syncBlocks = function(start, end, isForward, cb) { + HistoricSync.prototype.syncBlocks = function(start, end, opts, cb) { var that = this; p('Starting from: ', start); p(' to : ', end); - p(' isForward: ', isForward); + p(' opts: ', JSON.stringify(opts)); - - return that.getPrevNextBlock( start, end, - isForward ? { next: 1 } : { prev: 1}, cb); + return that.getPrevNextBlock( start, end, opts , cb); }; - HistoricSync.prototype.do_import_history = function(opts, next) { + HistoricSync.prototype.import_history = function(opts, next) { var that = this; var retry_attemps = 100; @@ -203,21 +206,20 @@ function spec() { ], function(err) { + var start, end; function sync() { - var start, end, isForward; - if (opts.reverse) { start = block_best; end = that.network.genesisBlock.hash.reverse().toString('hex'); - isForward = false; + opts.prev = true; } else { start = that.network.genesisBlock.hash.reverse().toString('hex'); end = null; - isForward = true; + opts.next = true; } - that.syncBlocks(start, end, isForward, function(err) { + that.syncBlocks(start, end, opts, function(err) { if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){ setTimeout(function() { @@ -236,9 +238,13 @@ function spec() { }); }; - HistoricSync.prototype.import_history = function(opts, next) { + // Reverse Imports (upto if we have genesis block?) + HistoricSync.prototype.smart_import = function(next) { var that = this; - that.do_import_history(opts, next); + var opts = { + prev: 1, + }; + that.import_history(opts, next); }; diff --git a/util/sync.js b/util/sync.js index 569ca78..a4e3f44 100755 --- a/util/sync.js +++ b/util/sync.js @@ -16,6 +16,7 @@ program .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet') .option('-D --destroy', 'Remove current DB (and start from there)', 0) .option('-R --reverse', 'Sync backwards', 0) + .option('-U --uptoexisting', 'Sync only until an existing block is found', 0) .parse(process.argv); var historicSync = new HistoricSync({ @@ -23,7 +24,7 @@ var historicSync = new HistoricSync({ }); if (program.remove) { - + // TODO: Sure? } async.series([ @@ -31,12 +32,17 @@ async.series([ historicSync.init(program, cb); }, function(cb) { - historicSync.import_history(program, function(err, count) { + historicSync.import_history({ + network: program.network, + destroy: program.destroy, + reverse: program.reverse, + uptoexisting: program.uptoexisting, + }, function(err, count) { if (err) { console.log('CRITICAL ERROR: ', err); } else { - console.log('Done! [%d blocks]', count, err); + console.log('Finished. [%d blocks]', count); } cb(); }); From 82beb27a3cd8f241dec123d74d15013dd3775c88 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 17 Jan 2014 10:44:14 -0300 Subject: [PATCH 2/4] better block_total accounting --- lib/HistoricSync.js | 12 ++++++------ util/sync.js | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 908f341..49d6763 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -70,7 +70,7 @@ function spec() { }, //show some (inacurate) status function(c) { - if (that.block_count++ % 1000 === 0) { + if (that.block_count % 1000 === 1) { progress_bar('sync status:', that.block_count, that.block_total); } return c(); @@ -91,7 +91,6 @@ function spec() { //store it function(c) { if (existed) return c(); - that.sync.storeBlock(blockInfo.result, function(err) { existed = err && err.toString().match(/E11000/); if (err && ! existed) return c(err); @@ -112,7 +111,7 @@ function spec() { function (err){ if (opts.uptoexisting && existed) { - p('DONE. Found existing block: %s ', blockHash); + p('DONE. Found existing block: ', blockHash); return cb(err); } @@ -120,6 +119,7 @@ function spec() { p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count); if (blockInfo && blockInfo.result) { + block_total++; if (opts.prev && blockInfo.result.previousblockhash) { return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); } @@ -177,7 +177,7 @@ function spec() { }, function(cb) { that.rpc.getInfo(function(err, res) { - if (err) cb(err); + if (err) return cb(err); that.block_total = res.result.blocks; return cb(); @@ -188,7 +188,7 @@ function spec() { if (!opts.reverse) return cb(); that.rpc.getBlockCount(function(err, res) { - if (err) cb(err); + if (err) return cb(err); block_height = res.result; return cb(); }); @@ -197,7 +197,7 @@ function spec() { if (!opts.reverse) return cb(); that.rpc.getBlockHash(block_height, function(err, res) { - if (err) cb(err); + if (err) return cb(err); block_best = res.result; return cb(); diff --git a/util/sync.js b/util/sync.js index a4e3f44..6e53120 100755 --- a/util/sync.js +++ b/util/sync.js @@ -14,6 +14,7 @@ var async = require('async'); program .version(SYNC_VERSION) .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet') + .option('-S --smart', 'genesis stored? uptoexisting = 1', 1) .option('-D --destroy', 'Remove current DB (and start from there)', 0) .option('-R --reverse', 'Sync backwards', 0) .option('-U --uptoexisting', 'Sync only until an existing block is found', 0) @@ -37,6 +38,7 @@ async.series([ destroy: program.destroy, reverse: program.reverse, uptoexisting: program.uptoexisting, + smart: program.smart, }, function(err, count) { if (err) { console.log('CRITICAL ERROR: ', err); From 80d3c2475567be9192e9a9c8f19796f560227957 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 17 Jan 2014 16:36:34 -0300 Subject: [PATCH 3/4] smart sync. Skips genesisTX --- app/models/Block.js | 1 + app/models/Transaction.js | 7 +++ lib/HistoricSync.js | 106 ++++++++++++++++++++++---------------- lib/Sync.js | 12 +---- server.js | 4 +- util/sync.js | 43 +++++++++------- 6 files changed, 97 insertions(+), 76 deletions(-) diff --git a/app/models/Block.js b/app/models/Block.js index edb80f9..709fbcf 100644 --- a/app/models/Block.js +++ b/app/models/Block.js @@ -55,6 +55,7 @@ BlockSchema.statics.customCreate = function(block, cb) { newBlock.hash = block.hash; newBlock.nextBlockHash = block.nextBlockHash; + Transaction.createFromArray(block.tx, newBlock.time, function(err, inserted_txs) { if (err) return cb(err); diff --git a/app/models/Transaction.js b/app/models/Transaction.js index fbb3f35..84c8614 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -19,6 +19,9 @@ var mongoose = require('mongoose'), var CONCURRENCY = 5; +// TODO: use bitcore networks module +var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; + /** */ var TransactionSchema = new Schema({ @@ -113,6 +116,10 @@ TransactionSchema.statics.createFromArray = function(txs, time, next) { TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) { + // Is it from genesis block? (testnet==livenet) + // TODO: parse it from networks.genesisTX + if (txid === genesisTXID) return cb(); + this.queryInfo(txid, function(err, info) { if (err || !info) return cb(err); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 49d6763..fdadfcc 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -16,6 +16,10 @@ function spec() { this.block_count= 0; this.block_total= 0; this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; + + var genesisHashReversed = new Buffer(32); + this.network.genesisBlock.hash.copy(genesisHashReversed); + this.genesis = genesisHashReversed.reverse().toString('hex'); this.sync = new Sync(opts); } @@ -44,12 +48,10 @@ function spec() { HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { - var that = this; + var self = this; // recursion end. - if (!blockHash || (blockEnd && blockEnd === blockHash) ) { - return cb(); - } + if (!blockHash ) return cb(); var existed = 0; var blockInfo; @@ -61,17 +63,16 @@ function spec() { Block.findOne({hash:blockHash}, function(err,block){ if (err) { p(err); return c(err); } if (block) { - existed = 1; - blockObj = block; + existed =1; + blockObj =block; } - return c(); }); }, //show some (inacurate) status function(c) { - if (that.block_count % 1000 === 1) { - progress_bar('sync status:', that.block_count, that.block_total); + if (self.block_count % 1000 === 1) { + progress_bar('sync status:', self.block_count, self.block_total); } return c(); }, @@ -81,7 +82,7 @@ function spec() { // TODO: if we store prev/next, no need to go to RPC // if (blockObj && blockObj.nextBlockHash) return c(); - that.rpc.getBlock(blockHash, function(err, ret) { + self.rpc.getBlock(blockHash, function(err, ret) { if (err) return c(err); blockInfo = ret; @@ -91,8 +92,10 @@ function spec() { //store it function(c) { if (existed) return c(); - that.sync.storeBlock(blockInfo.result, function(err) { + self.sync.storeBlock(blockInfo.result, function(err) { + existed = err && err.toString().match(/E11000/); + if (err && ! existed) return c(err); return c(); }); @@ -110,39 +113,36 @@ function spec() { ], function (err){ + if (err) + p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, self.block_count); + if (opts.uptoexisting && existed) { p('DONE. Found existing block: ', blockHash); return cb(err); } - if (err) - p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count); + if (blockEnd && blockEnd === blockHash) { + p('DONE. Found END block: ', blockHash); + return cb(err); + } + + // Continue if (blockInfo && blockInfo.result) { - block_total++; + self.block_count++; if (opts.prev && blockInfo.result.previousblockhash) { - return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); + return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); } if (opts.next && blockInfo.result.nextblockhash) - return that.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb); + return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb); } return cb(err); }); }; - HistoricSync.prototype.syncBlocks = function(start, end, opts, cb) { - var that = this; - - p('Starting from: ', start); - p(' to : ', end); - p(' opts: ', JSON.stringify(opts)); - - return that.getPrevNextBlock( start, end, opts , cb); - }; - HistoricSync.prototype.import_history = function(opts, next) { - var that = this; + var self = this; var retry_attemps = 100; var retry_secs = 2; @@ -154,7 +154,7 @@ function spec() { function(cb) { if (opts.destroy) { p('Deleting Blocks...'); - that.db.collections.blocks.drop(cb); + self.db.collections.blocks.drop(cb); } else { return cb(); } @@ -162,7 +162,7 @@ function spec() { function(cb) { if (opts.destroy) { p('Deleting TXs...'); - that.db.collections.transactions.drop(cb); + self.db.collections.transactions.drop(cb); } else { return cb(); } @@ -170,16 +170,16 @@ function spec() { function(cb) { if (opts.destroy) { p('Deleting TXItems...'); - that.db.collections.transactionitems.drop(cb); + self.db.collections.transactionitems.drop(cb); } else { return cb(); } }, function(cb) { - that.rpc.getInfo(function(err, res) { + self.rpc.getInfo(function(err, res) { if (err) return cb(err); - that.block_total = res.result.blocks; + self.block_total = res.result.blocks; return cb(); }); }, @@ -187,7 +187,7 @@ function spec() { function(cb) { if (!opts.reverse) return cb(); - that.rpc.getBlockCount(function(err, res) { + self.rpc.getBlockCount(function(err, res) { if (err) return cb(err); block_height = res.result; return cb(); @@ -196,7 +196,7 @@ function spec() { function(cb) { if (!opts.reverse) return cb(); - that.rpc.getBlockHash(block_height, function(err, res) { + self.rpc.getBlockHash(block_height, function(err, res) { if (err) return cb(err); block_best = res.result; @@ -210,17 +210,20 @@ function spec() { function sync() { if (opts.reverse) { start = block_best; - end = that.network.genesisBlock.hash.reverse().toString('hex'); + end = self.genesis; opts.prev = true; } else { - start = that.network.genesisBlock.hash.reverse().toString('hex'); + start = self.genesis; end = null; opts.next = true; } - that.syncBlocks(start, end, opts, function(err) { + p('Starting from: ', start); + p(' to : ', end); + p(' opts: ', JSON.stringify(opts)); + self.getPrevNextBlock( start, end, opts , function(err) { if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){ setTimeout(function() { p('Retrying in %d secs', retry_secs); @@ -228,7 +231,7 @@ function spec() { }, retry_secs * 1000); } else - return next(err, that.block_count); + return next(err, self.block_count); }); } if (!err) @@ -238,13 +241,28 @@ function spec() { }); }; - // Reverse Imports (upto if we have genesis block?) + // upto if we have genesis block? HistoricSync.prototype.smart_import = function(next) { - var that = this; - var opts = { - prev: 1, - }; - that.import_history(opts, next); + var self = this; + + Block.findOne({hash:self.genesis}, function(err, b){ + if (err) return next(err); + + + if (!b) { + p('Could not find Genesis block. Running FULL SYNC'); + } + else { + p('Genesis block found. Syncing upto know blocks.'); + } + + var opts = { + reverse: 1, + uptoexisting: b ? true: false, + }; + + return self.import_history(opts, next); + }); }; diff --git a/lib/Sync.js b/lib/Sync.js index 941ac37..1cebb38 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -19,6 +19,7 @@ function spec() { var that = this; Block.customCreate(block, function(err, block, inserted_txs){ + if (err) return cb(err); if (block && that.opts.broadcast_blocks) { sockets.broadcast_block(block); @@ -56,17 +57,6 @@ function spec() { }; - Sync.prototype.syncBlocks = function(start, end, isForward, cb) { - var that = this; - - console.log('Syncing Blocks, starting \n\tfrom: %s \n\tend: %s \n\tisForward:', - start, end, isForward); - - - return that.getPrevNextBlock( start, end, - isForward ? { next: 1 } : { prev: 1}, cb); - }; - Sync.prototype.init = function(opts, cb) { var that = this; diff --git a/server.js b/server.js index 1d61a87..004e6ea 100644 --- a/server.js +++ b/server.js @@ -50,9 +50,7 @@ if (!config.disableHistoricSync) { skip_db_connection: true, networkName: config.network }, function() { - hs.import_history({ - reverse: 1, - }, function(){ + hs.smart_import(function(){ console.log('historic_sync finished!'); }); }); diff --git a/util/sync.js b/util/sync.js index 6e53120..beccf0f 100755 --- a/util/sync.js +++ b/util/sync.js @@ -24,33 +24,40 @@ var historicSync = new HistoricSync({ networkName: program.network }); +/* TODO: Sure? if (program.remove) { - // TODO: Sure? + } +*/ async.series([ function(cb) { historicSync.init(program, cb); }, function(cb) { - historicSync.import_history({ - network: program.network, - destroy: program.destroy, - reverse: program.reverse, - uptoexisting: program.uptoexisting, - smart: program.smart, - }, function(err, count) { - if (err) { - console.log('CRITICAL ERROR: ', err); - } - else { - console.log('Finished. [%d blocks]', count); - } - cb(); - }); + if (program.smart) { + historicSync.smart_import(cb); + } + else { + historicSync.import_history({ + destroy: program.destroy, + reverse: program.reverse, + uptoexisting: program.uptoexisting, + }, cb); + } }, function(cb) { historicSync.close(); - cb(); -}]); + return cb(); + }, + ], + function(err, count) { + if (err) { + console.log('CRITICAL ERROR: ', err); + } + else { + console.log('Finished. [%d blocks synced]', count[1]); + } + return; +}); From b02e85c45da0fc2586a71a3a2be7de77b74f16c8 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 17 Jan 2014 16:42:07 -0300 Subject: [PATCH 4/4] fix message --- server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.js b/server.js index 004e6ea..84e659a 100644 --- a/server.js +++ b/server.js @@ -51,7 +51,7 @@ if (!config.disableHistoricSync) { networkName: config.network }, function() { hs.smart_import(function(){ - console.log('historic_sync finished!'); + console.log('[historic_sync] finished!'); }); }); }