From 97e1903dd16b2f56fa1bf8943456372dfbead3c7 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sat, 8 Feb 2014 16:20:51 -0300 Subject: [PATCH] complete sync process from files working --- lib/BlockDb.js | 2 +- lib/HistoricSync.js | 56 +++--- lib/Sync.js | 78 +++++--- lib/TransactionDb.js | 2 +- test/integration/sync.js | 384 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 449 insertions(+), 73 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index b253ca8..05c0650 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -90,7 +90,7 @@ function spec(b) { }; BlockDb.prototype.setMain = function(hash, isMain, cb) { - if (!isMain) console.log('ORPHAN: %s',hash); + if (!isMain) console.log('\tNew orphan: %s',hash); db.put(MAIN_PREFIX + hash, isMain?1:0, function(err) { return cb(err); }); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 8534ecf..5b829e8 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -119,6 +119,11 @@ function spec() { if (self.opts.shouldBroadcast) { sockets.broadcastSyncInfo(self.info()); } + +//TODO +// if (self.syncPercentage > 10) { +// process.exit(-1); +// } }; HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) { @@ -156,7 +161,8 @@ function spec() { blockInfo = ret.result; // this is to match block retreived from file if (blockInfo.hash === self.genesis) - blockInfo.previousblockhash='0000000000000000000000000000000000000000000000000000000000000000'; + blockInfo.previousblockhash = + '0000000000000000000000000000000000000000000000000000000000000000'; } else { blockInfo = null; @@ -295,38 +301,30 @@ function spec() { HistoricSync.prototype.nextBlockFromFile = function(scanOpts, cb) { var self = this; - var blockInfo; - async.series([ - function(c) { - self.showProgress(); - return c(); - }, - function(c) { - self.getBlockFromFile(function(err, inBlockInfo) { - blockInfo = inBlockInfo; - return c(err); - }); - }, - function(c) { - self.sync.storeTipBlock(blockInfo, function(err) { - return c(err); - }); - }, - function(c) { - // continue - if (blockInfo && blockInfo.hash) { - self.syncedBlocks++; - } else - self.status = 'finished'; - return c(); - }, - ], function(err) { + self.showProgress(); + + self.getBlockFromFile(function(err, blockInfo) { if (err) { - self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', + blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); + return cb(err); } - return cb(err); + + self.sync.storeTipBlock(blockInfo, function(err) { + if (blockInfo && blockInfo.hash) { + self.syncedBlocks++; + } else + self.status = 'finished'; + + if (err) { + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', + blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); + } + return cb(err); + }); }); + }; diff --git a/lib/Sync.js b/lib/Sync.js index e923db2..58671d8 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -62,18 +62,32 @@ function spec() { * * A-B-C-D-E(TIP) ... NEW * - * 1) Get NEW.prev recusively until existing block - * then case 0) / 1) / 2) + * NEW is ignored * */ Sync.prototype.storeTipBlock = function(b, cb) { + if (!b) return cb(); + var self = this; var oldTip, oldNext, needReorg = true; var newPrev = b.previousblockhash; var updatedTxs, updatedAddrs; async.series([ + function(c) { + self.bDb.has(b.hash, function(err, val) { + return c(err || + (val ? new Error('WARN: Ignoring already existing block:' + b.hash) : null )); + }); + }, + function(c) { + self.bDb.has(newPrev, function(err, val) { + if (!val && newPrev.match(/^0+$/)) return c(); + return c(err || + (!val ? new Error('WARN: Ignoring block with non existing prev:' + b.hash) : null )); + }); + }, function(c) { self.txDb.createFromBlock(b, function(err, txs, addrs) { updatedTxs = txs; @@ -105,9 +119,9 @@ function spec() { function(c) { if (!needReorg) return c(); - console.log('NEW TIP: %s NEED REORG', b.hash, oldTip); + console.log('NEW TIP: %s NEED REORG (old tip: %s)', b.hash, oldTip); // TODO should modify updatedTxs and addrs. - self.processReorg(oldTip, oldNext, newPrev, cb); + self.processReorg(oldTip, oldNext, newPrev, c); }, function(c) { self.bDb.setNext(newPrev, b.hash, function(err) { @@ -115,7 +129,11 @@ function spec() { }); }], function(err) { - self._handleBroadcast(b, updatedTxs, updatedAddrs); + if (!err) self._handleBroadcast(b, updatedTxs, updatedAddrs); + if (err && err.toString().match(/WARN/) ) { + console.log(err); + err=null; + } return cb(err); }); }; @@ -125,23 +143,14 @@ function spec() { Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, cb) { var self = this; - var newPrevExisted, orphanizeFrom; + var orphanizeFrom; async.series([ - function(c) { - self.bDb.has(newPrev, function(err, ret) { - newPrevExisted = ret; - return c(); - }); - }, - function(c) { - if (newPrevExisted) return c(); - console.log('[BlockDb.js.133] case 3) not implemented yet in reorg'); //TODO - process.exit(1); - }, function(c) { self.bDb.isMain(newPrev, function(err,val) { if (!val) return c(); + + console.log('# Reorg Case 1)'); // case 1 orphanizeFrom = oldNext; return c(err); @@ -150,11 +159,14 @@ function spec() { function(c) { if (orphanizeFrom) return c(); - self.setBranchConnectedBackwards(newPrev, function(err, yHash) { + console.log('# Reorg Case 2)'); + self.setBranchConnectedBackwards(newPrev, function(err, yHash, newYHashNext) { if (err) return c(err); self.bDb.getNext(yHash, function(err, yHashNext) { orphanizeFrom = yHashNext; - return c(err); + self.bDb.setNext(yHash, newYHashNext, function(err) { + return c(err); + }); }); }); }, @@ -170,6 +182,14 @@ function spec() { }); }; + Sync.prototype.setBlockMain = function(hash, isMain, cb) { + var self = this; + self.bDb.setMain(hash, isMain, function(err) { + if (err) return cb(err); + return self.txDb.handleBlockChange(hash, isMain, cb); + }); + }; + Sync.prototype.setBranchOrphan = function(fromHash, cb) { var self = this, hashInterator = fromHash; @@ -187,26 +207,19 @@ function spec() { }, cb); }; - Sync.prototype.setBlockMain = function(hash, isMain, cb) { - var self = this; - - self.bDb.setMain(hash, isMain, function(err) { - if (err) return cb(err); - return self.txDb.handleBlockChange(hash, isMain, cb); - }); - }; - Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { var self = this, hashInterator = fromHash, + lastHash = fromHash, isMain; async.doWhilst( function(c) { - self.setConnected(hashInterator, function (err) { + self.setBlockMain(hashInterator, true, function (err) { if (err) return c(err); self.bDb.getPrev(hashInterator, function (err, val) { if (err) return c(err); + lastHash = hashInterator; hashInterator = val; self.bDb.isMain(hashInterator, function (err, val) { isMain = val; @@ -215,7 +228,12 @@ function spec() { }); }); }, - function() { return hashInterator; }, cb); + function() { return hashInterator && !isMain; }, + function(err) { + console.log('\tFound yBlock:', hashInterator); + return cb(err, hashInterator, lastHash); + } + ); }; Sync.prototype._handleBroadcast = function(hash, updatedTxs, updatedAddrs) { diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index d2fa9ef..734f846 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -432,7 +432,7 @@ function spec(b) { TransactionDb.prototype.handleBlockChange = function(hash, isMain, cb) { var k = IN_BLK_PREFIX; var toChange = []; - console.log('Searching Txs from block:' + hash); + console.log('\tSearching Txs from block:' + hash); // This is slow, but prevent us to create a new block->tx index. db.createReadStream({start: k, end: k + '~'}) diff --git a/test/integration/sync.js b/test/integration/sync.js index 9433f50..9d4e47c 100644 --- a/test/integration/sync.js +++ b/test/integration/sync.js @@ -1,6 +1,5 @@ #!/usr/bin/env node 'use strict'; - process.env.NODE_ENV = process.env.NODE_ENV || 'development'; @@ -20,9 +19,9 @@ var b = [ ]; var t = [ 'd08582d3711f75d085c618874fb0d049ae09d5ec95ec6f5abd289f4b54712c54', // TX from B#16 - '1729001087e0cebea8d14de1653d5cf59628d9746bc1ae65f776f1cbaff7ebad', - 'cf53d7ccd83a099acfbc319ee10c1e3b10e3d42ba675b569fdd6b69cb8d2db4e', - 'cf53d7ccd83a099acfbc319ee10c1e3b10e3d42ba675b569fdd6b69cb8d2db4e', + '1729001087e0cebea8d14de1653d5cf59628d9746bc1ae65f776f1cbaff7ebad', //1 + 'cf53d7ccd83a099acfbc319ee10c1e3b10e3d42ba675b569fdd6b69cb8d2db4e', //2 + '73a4988adf462b6540cfa59097804174b298cfa439f73c1a072c2c6fbdbe57c7', //3 'd45f9da73619799e9d7bd03cc290e70875ea4cbad56b8bffa15135fbbb3df9ea', //4 Tx from B20 ]; @@ -49,7 +48,34 @@ var test = function(cb) { }); }; -describe('Sync checkOrphan', function(){ +/* + * TEST CASES + * + * Blocks: 0-1-2-3-4 + * case 1) + * 0-1-2-3-4 + * \ + * C1* + * + * case 2) + * 0-1-2---3-4 + * \ \ + * C1 C2* + * + * case 2b) + * 0-1-2---3-4 + * \ \ + * C1 C2-C2b(TX=C1.TX)* + * case 2c) + * 0-1-2---3-4 + * \ \ + * C1 C2-C2b(TX=C1.TX) + * \ + * C2c(TX=C2.TX)* + * + */ + +describe('Sync Reorgs', function(){ before(function(done) { s = new HistoricSync(); @@ -68,15 +94,15 @@ describe('Sync checkOrphan', function(){ }); }); + var case1 = { + hash: '0000000000000000000000000000000000000000000000000000000000000001', + tx: [ 'f0596531810160d090813673b4a397f4617aab44eb26c7f06c8a766eac984b91' ], + time: 1296690099, + previousblockhash: b[2], + }; + it('reorg, case 1', function(done) { - var case1 = { - hash: '0000000000000000000000000000000000000000000000000000000000000001', - tx: [ '1000000000000000000000000000000000000000000000000000000000000000' ], - time: 1296690099, - previousblockhash: b[2], - }; - async.series([ function (c) { s.sync.txDb.isConfirmed(t[0], function(err,is) { @@ -85,6 +111,13 @@ describe('Sync checkOrphan', function(){ return c(); }); }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, function (c) { s.sync.txDb.isConfirmed(t[4], function(err,is) { assert(!err); @@ -126,6 +159,13 @@ describe('Sync checkOrphan', function(){ return c(); }); }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, function (c) { s.sync.txDb.isConfirmed(t[4], function(err,is) { assert(!err); @@ -133,6 +173,326 @@ describe('Sync checkOrphan', function(){ return c(); }); }, + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + ], done ); + }); + + it('reorg, case 1 (repeat)', function(done) { + s.sync.storeTipBlock(case1, function(err) { + assert(!err, 'shouldnt return error' + err); + return done(); + }); + }); + + var case2 = { + hash: '0000000000000000000000000000000000000000000000000000000000000002', + tx: [ '99bb359a4b12a588fcb9e59e5e8d92d593ce7a56d2ba42085fe86d9a0b4fde15' ], + time: 1296690099, + previousblockhash: b[3], + }; + + + it('reorg, case 2', function(done) { + async.series([ + function (c) { + s.sync.txDb.isConfirmed(t[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[4], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.storeTipBlock(case2, function(err) { + assert(!err, 'shouldnt return error' + err); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(b[3], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(b[4], function(err,is) { + assert(!err); + assert(!is, b[3] + 'should not be on main chain'); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(case1.hash, function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(case2.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(is, 'transaction t[3] should be valid:' + t[3]); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[4], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.bDb.getNext(b[2], function(err, val) { + assert(!err); + assert.equal(val,b[3]); + return c(); + }); + }, + + + + ], done ); + }); + + + var case2b = { + hash: '0000000000000000000000000000000000000000000000000000000000000003', + tx: case1.tx, + time: 1296690099, + previousblockhash: case2.hash, + }; + + it('reorg, case 2b', function(done) { + async.series([ + function (c) { + s.sync.txDb.isConfirmed(case2b.tx[0], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.storeTipBlock(case2b, function(err) { + assert(!err, 'shouldnt return error' + err); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(is, 'transaction t[3] should be valid:' + t[3]); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2b.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + ], done ); + }); + + + + var case2c = { + hash: '0000000000000000000000000000000000000000000000000000000000000004', + tx: case2.tx, + time: 1296690099, + previousblockhash: case1.hash, + }; + + it('reorg, case 2c', function(done) { + async.series([ + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(case1.hash, function(err,is) { + assert(!err); + assert(!is, 'case1 block shouldnt be main:' + case1.hash); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2c.tx[0], function(err,is) { + assert(!err); + assert(is); //It was there before (from case2) + return c(); + }); + }, + function (c) { + s.sync.storeTipBlock(case2c, function(err) { + assert(!err, 'shouldnt return error' + err); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.has(case1.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.has(case2c.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2c.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(!is, 'TX t[3]: shouldnt be confirmed:' + t[3] +':'+ is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[4], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + + ], done ); + }); + + var case3 = { + hash: '0000000000000000000000000000000000000000000000000000000000000005', + tx: case2.tx, + time: 1296690099, + previousblockhash: '666', + }; + + it('reorg, case 3)', function(done) { + async.series([ + function (c) { + s.sync.storeTipBlock(case3, function(err) { + assert(!err, 'shouldnt return error' + err); + return c(); + }); + }, + + //shoudnt change anything + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.has(case1.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.has(case2c.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2c.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(!is, 'TX t[3]: shouldnt be confirmed:' + t[3] +':'+ is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[4], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, ], done ); });