From d9f6de4e0caf7789de7f340aa01d51b9ee154a9f Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 4 Feb 2014 19:55:58 -0300 Subject: [PATCH] sync fixes --- lib/BlockDb.js | 8 ++++---- lib/BlockExtractor.js | 9 +++++++-- lib/HistoricSync.js | 29 ++++++----------------------- lib/Sync.js | 34 ++++++++++++++++++++++++++-------- lib/TransactionDb.js | 2 +- test/integration/sync.js | 25 +++++++++++++++++++++---- 6 files changed, 65 insertions(+), 42 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 13dd2e8..551d8df 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -5,8 +5,8 @@ require('classtool'); function spec() { - var TIMESTAMP_ROOT = 'b-ts-'; - var PREV_ROOT = 'b-prev-'; // b-prev- => (0 if orphan) + var TIMESTAMP_ROOT = 'b-ts-'; // b-ts- => + var PREV_ROOT = 'b-prev-'; // b-prev- => (0 if orphan) /** @@ -49,6 +49,7 @@ function spec() { }; + BlockDb.prototype.setOrphan = function(hash, cb) { var self = this; @@ -80,9 +81,8 @@ function spec() { BlockDb.prototype.countNotOrphan = function(cb) { - var self = this; var c = 0; - this.db.createReadStream({start: PREV_ROOT}) + this.db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' }) .on('data', function (data) { if (data.value !== 0) c++; }) diff --git a/lib/BlockExtractor.js b/lib/BlockExtractor.js index 857ed71..92c6740 100644 --- a/lib/BlockExtractor.js +++ b/lib/BlockExtractor.js @@ -113,7 +113,7 @@ function spec() { magic = self.currentParser ? self.currentParser.buffer(4).toString('hex') : null ; - if (!self.currentParser || self.currentParser.eof()) { + if (!self.currentParser || self.currentParser.eof() || magic === '00000000') { magic = null; if (self.nextFile()) { console.log('Moving forward to file:' + self.currentFile() ); @@ -121,7 +121,8 @@ function spec() { } else { console.log('Finished all files'); - return cb(); + magic = null; + return w_cb(); } } else { @@ -130,6 +131,8 @@ function spec() { }, a_cb); }, function (a_cb) { + if (!magic) return a_cb(); + if (magic !== self.magic) { var e = new Error('CRITICAL ERROR: Magic number mismatch: ' + magic + '!=' + self.magic); @@ -141,6 +144,8 @@ function spec() { return a_cb(); }, function (a_cb) { + if (!magic) return a_cb(); + b = new Block(); b.parse(self.currentParser); b.getHash(); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index b84ca9e..95c6b14 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -115,13 +115,6 @@ function spec() { if (self.opts.shouldBroadcast) { sockets.broadcastSyncInfo(self.info()); } - -//TODO - /* -if (self.syncPercentage>10) { - console.log(self.info()); - process.exit(1); -}*/ }; HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) { @@ -247,9 +240,7 @@ if (self.syncPercentage>10) { addrStrs = [ addr.toString() ]; break; case Script.TX_MULTISIG: - var addrs = []; var chunks = s.capture(); - chunks.forEach(function(chunk) { var a = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk)); addrStrs.push(a.toString()); @@ -262,15 +253,10 @@ if (self.syncPercentage>10) { return addrStrs; }; -var kk=0; - HistoricSync.prototype.getBlockFromFile = function(scanOpts, cb) { var self = this; - - var nextHash; var blockInfo; var existed; - async.series([ //show some (inacurate) status function(c) { @@ -313,18 +299,18 @@ var kk=0; }, //store it function(c) { - if (self.prevHash && blockInfo.prev_block !== self.prevHash) { - - console.log('Orphans found: @%s', blockInfo.hash); + if (blockInfo && self.prevHash && blockInfo.prev_block !== self.prevHash) { + console.log('Hole found @%s', blockInfo.hash); console.log('From: %s To: %s', self.prevHash, blockInfo.prev_block); -process.exit(1); - self.sync.setOrphan(self.prevHash, blockInfo.prev_block, c); + self.sync.checkOrphan(self.prevHash, blockInfo.prev_block, c); } else return c(); }, //store it function(c) { + if (!blockInfo) return c(); + self.sync.storeBlock(blockInfo, function(err) { existed = err && err.toString().match(/E11000/); @@ -343,7 +329,7 @@ process.exit(1); }, ], function(err) { if (err) { - self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo.hash, err, self.syncedBlocks)); + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); } return cb(err); }); @@ -437,9 +423,6 @@ process.exit(1); p(' scanOpts: ', JSON.stringify(scanOpts)); if (scanOpts.fromFiles) { - - var height = 0; - self.status = 'syncing'; async.whilst(function() { return self.status === 'syncing'; diff --git a/lib/Sync.js b/lib/Sync.js index 17c524e..ffd9f06 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -94,23 +94,41 @@ function spec() { }); }; - Sync.prototype.setOrphan = function(fromBlock, toBlock, c) { + Sync.prototype.checkOrphan = function(fromBlock, toBlock, c) { var self = this; var hash = fromBlock; + var co = 0; + var limit = 10; + var cont = 1; + async.whilst( function () { - return hash && hash !== toBlock; + if (++co > limit) { +console.log('[Sync.js.109] WARN: Reach reog depth limit'); //TODO + } + return cont && hash && hash !== toBlock && co < limit; }, function (w_c) { - self.txDb.setOrphan(hash, function(err, insertedTxs, updateAddrs) { - if (err) return w_c(err); - self.blockDb.setOrphan(hash, function(err, prevHash){ + //check with RPC if the block is mainchain + self.blockDb.fromHashWithInfo(hash, function (err, info) { - hash = prevHash; - return w_c(err); - }); + if (!info) { + console.log('[Sync.js.107:hash:ORPHAN]',hash); //TODO + self.txDb.setOrphan(hash, function(err) { + if (err) return w_c(err); + self.blockDb.setOrphan(hash, function(err, prevHash){ + hash = prevHash; + return w_c(err); + }); + }); + } + else { + console.log('[Sync.js.107:hash:NOT ORPHAN]',hash); //TODO + cont = 0; + return w_c(); + } }); }, function (err) { diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index f3ab69b..38924e9 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -368,7 +368,7 @@ console.log('[TransactionDb.js.165]', ret); //TODO // txs can be a [hashes] or [txObjects] TransactionDb.prototype.createFromBlock = function(b, next) { var self = this; - if (!b.tx) return next(); + if (!b || !b.tx) return next(); return self.createFromArray(b.tx, b.hash, next); }; diff --git a/test/integration/sync.js b/test/integration/sync.js index f0e2589..be6a078 100644 --- a/test/integration/sync.js +++ b/test/integration/sync.js @@ -42,8 +42,25 @@ var test = function(s,cb) { +var testNo = function(s,cb) { + async.each([2,3,4], function(i,c) { + s.blockDb.getPrev(b[i], function(err, p) { + assert.equal(p,b[i-1]); + return c(); + }); + }, function() { + s.blockDb.getPrev(b[1], function(err, p) { + assert.equal(p,b[0]); + return cb(); + }); + }); +}; + + + + var s; -describe('Sync setOrphan', function(){ +describe('Sync checkOrphan', function(){ before(function(done) { s = new Sync(); @@ -54,15 +71,15 @@ describe('Sync setOrphan', function(){ fix(s,done); }); - it('setOrphan', function(done) { + it('checkOrphan', function(done) { this.timeout(100000); s.blockDb.has(b[0], function(err, has) { assert(has); s.blockDb.has(b[1], function(err, has) { assert(has); - s.setOrphan(b[4],b[1], function() { - test(s,done); + s.checkOrphan(b[4],b[1], function() { + testNo(s,done); }); }); });