From 9a09050646258999ed7912bff67bba8729974379 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 29 Jul 2016 08:52:36 -0300 Subject: [PATCH] better tests --- lib/server.js | 72 +++++++++++++----- lib/storage.js | 54 +++++++++----- test/integration/server.js | 148 ++++++++++++++++++++++++++++++++++--- 3 files changed, 223 insertions(+), 51 deletions(-) diff --git a/lib/server.js b/lib/server.js index 9c22a34..f7dc855 100644 --- a/lib/server.js +++ b/lib/server.js @@ -2775,32 +2775,65 @@ WalletService.prototype.getTxHistory = function(opts, cb) { var addressStrs = _.pluck(addresses, 'address'); var networkName = Bitcore.Address(addressStrs[0]).toObject().network; + var useCache = addresses.length >= Defaults.HISTORY_CACHE_ADDRESS_THRESOLD; var bc = self._getBlockchainExplorer(networkName); + var from = opts.skip || 0; + var to = from + opts.limit; + var normalizedTxs, fromCache; + async.parallel([ function(next) { self.storage.fetchTxs(self.walletId, {}, next); }, function(next) { - var from = opts.skip || 0; - var to = from + opts.limit; - bc.getTransactions(addressStrs, from, to, function(err, txs, total) { - if (err) return cb(err); - var txsNormalized = self._normalizeTxHistory(txs); + var totalItems; - if (addresses.length < Defaults.HISTORY_CACHE_ADDRESS_THRESOLD) - return next(err, txsNormalized); + async.series([ + function(nextSerie) { + if (!useCache) return nextSerie(); - var txsToCache = _.filter(txsNormalized, function(i) { - return i.confirmations >= Defaults.CONFIRMATIONS_TO_START_CACHING; - }).reverse(); - var index = total - to; - if (index < 0) index = 0; - self.storage.storeTxHistoryCache(self.walletId, total, index, txsToCache, function(err) { - next(err, txsNormalized); - }) - }); + self.storage.getTxHistoryCache(self.walletId, from, to, function(err, res) { + if (err) return nextSerie(err); + if (!res || !res[0] ) return nextSerie(); + + normalizedTxs = res; + fromCache = true; + + return nextSerie() + }); + }, + function(nextSerie) { + if (normalizedTxs) return nextSerie(); + bc.getTransactions(addressStrs, from, to, function(err, rawTxs, total) { + if (err) return cb(err); + normalizedTxs = self._normalizeTxHistory(rawTxs); + totalItems = total; + return nextSerie(); + }); + }, + function(nextSerie) { + if (!useCache || fromCache) return nextSerie(); + + var txsToCache = _.filter(normalizedTxs, function(i) { + return i.confirmations >= Defaults.CONFIRMATIONS_TO_START_CACHING; + }).reverse(); + + if (!txsToCache.length) + return nextSerie(err); + + var fwdIndex = totalItems - to; + if (fwdIndex < 0) fwdIndex = 0; + self.storage.storeTxHistoryCache(self.walletId, totalItems, fwdIndex, txsToCache, function(err) { + nextSerie(err); + }) + } + ], + function(err) { + if (err) return next(err); + return next(); + }); }, function(next) { self.storage.fetchTxNotes(self.walletId, {}, next); @@ -2809,12 +2842,11 @@ WalletService.prototype.getTxHistory = function(opts, cb) { if (err) return cb(err); var proposals = res[0]; - var txs = res[1]; - var notes = res[2]; + var notes = res[3]; - txs = decorate(txs, addresses, proposals, notes); + var finalTxs = decorate(normalizedTxs, addresses, proposals, notes); - return cb(null, txs); + return cb(null, finalTxs, !!fromCache); }); }); }; diff --git a/lib/storage.js b/lib/storage.js index bf68348..7b01c24 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -612,7 +612,7 @@ Storage.prototype.storeActiveAddresses = function(walletId, addresses, cb) { // -------- --------------------------- Total // > Time > // ^to <= ^from -// ^start => ^end +// ^fwdIndex => ^end Storage.prototype.getTxHistoryCache = function(walletId, from, to, cb) { var self = this; @@ -624,14 +624,16 @@ Storage.prototype.getTxHistoryCache = function(walletId, from, to, cb) { }, function(err, result) { if (err) return cb(err); if (!result) return cb(); + if (!result.isUpdated) return cb(); // Reverse indexes - var start = result.totalItems - from; - var end = start + to - from; + var fwdIndex = result.totalItems - to; + if (fwdIndex < 0) fwdIndex = 0; + var end = fwdIndex + to - from; console.log('[storage.js.632] from,to:', from, to); //TODO - console.log('[storage.js.632] start,end:', start, end); //TODO + console.log('[storage.js.632] fwsIndex,end:', fwdIndex, end); //TODO // Cache is OK. self.db.collection(collections.CACHE).findOne({ @@ -639,14 +641,20 @@ Storage.prototype.getTxHistoryCache = function(walletId, from, to, cb) { type: 'historyCache', key: null }, function(err, result) { - console.log('[storage.js.641:result:]', result); //TODO if (err) return cb(err); - if (!_.any(result.history, function(i) { - return !!i; - })) return cb(); // some items are not yet defined. - var ret = result.history.slice(start, end); - return cb(null, ret); + if (result.history.length < fwdIndex) + return cb(); + + var ret = result.history.slice(fwdIndex, end); + + if (!_.any(ret, function(i) { + return !!i; + })) { + console.log('[storage.js.650] history has holes. not using cache'); //TODO + return cb(); // some items are not yet defined. + } + return cb(null, ret.reverse()); }); }) }; @@ -657,7 +665,7 @@ Storage.prototype.softResetTxHistoryCache = function(walletId, cb) { type: 'historyCacheStatus', key: null }, { - cacheIsUpdated: false, + isUpdated: false, }, { w: 1, upsert: true, @@ -700,39 +708,47 @@ Storage.prototype.storeTxHistoryCache = function(walletId, totalItems, firstPosi key: null }, function(err, result) { if (err) return cb(err); - result = result || []; + result = result || {}; + var h = result.history || []; + +console.log('[storage.js.723] STORING FROM', firstPosition); //TODO //create a sparce array, from the input _.each(items, function(i) { - result[firstPosition++] = i; + h[firstPosition++] = i; }); - // TODO: check txid uniqness? + // TODO: check txid uniqness? self.db.collection(collections.CACHE).update({ walletId: walletId, type: 'historyCache', key: null }, { - history: result + walletId: walletId, + type: 'historyCache', + key: null, + history:h }, { w: 1, upsert: true, }, function(err) { if (err) return cb(err); - var cacheIsComplete = !!result[0]; + var cacheIsComplete = !!h[0]; var now = Date.now(); - self.db.collection(collections.CACHE).update({ walletId: walletId, type: 'historyCacheStatus', key: null }, { + walletId: walletId, + type: 'historyCacheStatus', + key: null, totalItems: totalItems, updatedOn: now, - cacheIsComplete: cacheIsComplete, - cacheIsUpdated: true, + isComplete: cacheIsComplete, + isUpdated: true, }, { w: 1, upsert: true, diff --git a/test/integration/server.js b/test/integration/server.js index da5bad2..80c36d8 100644 --- a/test/integration/server.js +++ b/test/integration/server.js @@ -6241,10 +6241,8 @@ describe('Wallet service', function() { it('should store partial cache tx history from insight', function(done) { var h = helpers.historyCacheTest(200); helpers.stubHistory(h); - var spy = sinon.spy(server.storage, 'storeTxHistoryCache'); - var toCache = _.filter(h, function(i) { - return i.confirmations >= server.confirmationsToStartCaching; - }); + var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache'); + var skip = 95; var limit = 10; @@ -6259,7 +6257,7 @@ describe('Wallet service', function() { should.not.exist(err); should.exist(txs); txs.length.should.equal(limit); - var calls = spy.getCalls(); + var calls = storeTxHistoryCacheSpy.getCalls(); calls.length.should.equal(1); calls[0].args[1].should.equal(200); // total @@ -6278,14 +6276,14 @@ describe('Wallet service', function() { it('should not cache tx history from insight', function(done) { var h = helpers.historyCacheTest(200); helpers.stubHistory(h); - var spy = sinon.spy(server.storage, 'storeTxHistoryCache'); + var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache'); server.getTxHistory({ skip: 0, limit: 10, }, function(err, txs) { should.not.exist(err); should.exist(txs); - var calls = spy.getCalls(); + var calls = storeTxHistoryCacheSpy.getCalls(); calls.length.should.equal(1); calls[0].args[3].length.should.equal(0); server.storage.storeTxHistoryCache.restore(); @@ -6297,10 +6295,7 @@ describe('Wallet service', function() { it('should store cache all tx history from insight', function(done) { var h = helpers.historyCacheTest(200); helpers.stubHistory(h); - var spy = sinon.spy(server.storage, 'storeTxHistoryCache'); - var toCache = _.filter(h, function(i) { - return i.confirmations >= server.confirmationsToStartCaching; - }); + var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache'); var skip = 195; var limit = 5; @@ -6312,7 +6307,7 @@ describe('Wallet service', function() { should.not.exist(err); should.exist(txs); txs.length.should.equal(limit); - var calls = spy.getCalls(); + var calls = storeTxHistoryCacheSpy.getCalls(); calls.length.should.equal(1); calls[0].args[1].should.equal(200); // total @@ -6326,6 +6321,135 @@ describe('Wallet service', function() { done(); }); }); + + describe.only('Downloading history', function() { + var h; + beforeEach(function() { + h = helpers.historyCacheTest(200); + helpers.stubHistory(h); + }); + + it('from 0 to 200, two times, in order', function(done) { + async.eachSeries(_.range(0, 200, 5), function(i, next) { + console.log('FIRST ', i); //TODO + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(false); + next(); + }); + }, function() { + async.eachSeries(_.range(0, 200, 5), function(i, next) { + console.log('SECOND ', i); //TODO + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(i >= 100); + next(); + }); + }, done); + }); + }); + + it('from 0 to 200, two times, random', function(done) { + var indexes = _.range(0, 200, 5); + async.eachSeries(_.shuffle(indexes), function(i, next) { + console.log('FIRST ', i); //TODO + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(false); + next(); + }); + }, function() { + async.eachSeries(_.range(0, 200, 5), function(i, next) { + console.log('SECOND ', i); //TODO + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(i >= 100); + next(); + }); + }, done); + }); + }); + + + it('from 0 to 200, two times, random, with resets', function(done) { + var indexes = _.range(0, 200, 5); + async.eachSeries(_.shuffle(indexes), function(i, next) { + console.log('FIRST ', i); //TODO + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(false); + next(); + }); + }, function() { + async.eachSeries(_.range(0, 200, 5), function(i, next) { + console.log('SECOND ', i); //TODO + + function resetCache(cb) { + if (!(i%25)) { +console.log('[server.js.6424] RESET CACHE!!!!!!!!!!!!!!!!'); //TODO + storage.softResetTxHistoryCache(server.walletId, function() { + return cb(true); + }); + } else { + return cb(false); + } + } + + resetCache(function(reset) { + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(i >= 100 && !reset); + next(); + }); + }, done); + }); + }); + }); + + + }); }); describe('#scan', function() {