From 35f44b46365e11d4006ea93de231ccf2fa74204f Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 26 Jul 2016 21:10:49 -0300 Subject: [PATCH] store history cache --- config.js | 2 +- lib/blockchainexplorers/insight.js | 12 ++- lib/server.js | 12 ++- lib/storage.js | 124 ++++++++++++++++++++-- test/integration/helpers.js | 5 +- test/integration/server.js | 55 +++++++++- test/testdata.js | 159 +++++++++++++++++++++++++++++ 7 files changed, 349 insertions(+), 20 deletions(-) diff --git a/config.js b/config.js index 56fdd5c..7caac96 100644 --- a/config.js +++ b/config.js @@ -81,6 +81,6 @@ var config = { // api_user: xxx, // api_key: xxx, // }); - + confirmationsToStartCaching: 100, }; module.exports = config; diff --git a/lib/blockchainexplorers/insight.js b/lib/blockchainexplorers/insight.js index 121dd49..2c3e01d 100644 --- a/lib/blockchainexplorers/insight.js +++ b/lib/blockchainexplorers/insight.js @@ -97,6 +97,7 @@ Insight.prototype.getTransaction = function(txid, cb) { Insight.prototype.getTransactions = function(addresses, from, to, cb) { var qs = []; + var total; if (_.isNumber(from)) qs.push('from=' + from); if (_.isNumber(to)) qs.push('to=' + to); @@ -116,13 +117,18 @@ Insight.prototype.getTransactions = function(addresses, from, to, cb) { this._doRequest(args, function(err, res, txs) { if (err || res.statusCode !== 200) return cb(_parseErr(err, res)); - if (_.isObject(txs) && txs.items) - txs = txs.items; + if (_.isObject(txs)) { + if (txs.totalItems) + total = txs.totalItems; + + if (txs.items) + txs = txs.items; + } // NOTE: Whenever Insight breaks communication with bitcoind, it returns invalid data but no error code. if (!_.isArray(txs) || (txs.length != _.compact(txs).length)) return cb(new Error('Could not retrieve transactions from blockchain. Request was:' + JSON.stringify(args))); - return cb(null, txs); + return cb(null, txs, total); }); }; diff --git a/lib/server.js b/lib/server.js index 4fdd829..91e5b84 100644 --- a/lib/server.js +++ b/lib/server.js @@ -57,6 +57,7 @@ function WalletService() { this.messageBroker = messageBroker; this.fiatRateService = fiatRateService; this.notifyTicker = 0; + this.confirmationsToStartCaching = config.confirmationsToStartCaching || 100; }; function checkRequired(obj, args, cb) { @@ -2785,9 +2786,16 @@ WalletService.prototype.getTxHistory = function(opts, cb) { function(next) { var from = opts.skip || 0; var to = from + opts.limit; - bc.getTransactions(addressStrs, from, to, function(err, txs) { + bc.getTransactions(addressStrs, from, to, function(err, txs, total) { if (err) return cb(err); - next(null, self._normalizeTxHistory(txs)); + var txsNormalized = self._normalizeTxHistory(txs); + var txsToCache = _.filter(txsNormalized, function(i){ + return i.confirmations >= self.confirmationsToStartCaching; + }); + var index = total - from; + self.storage.storeTxHistoryCache(self.walletId, total, index, txsToCache, function(err) { + next(err, txsNormalized); + }) }); }, function(next) { diff --git a/lib/storage.js b/lib/storage.js index b4c1960..ad2e788 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -609,27 +609,129 @@ Storage.prototype.storeActiveAddresses = function(walletId, addresses, cb) { }, cb); }; +// -------- --------------------------- Total +// > Time > +// ^to <= ^from +// ^start => ^end -Storage.prototype.storeTxHistoryCache = function(walletId, firstPosition, items, cb) { +Storage.prototype.getTxHistoryCache = function(walletId, from, to, cb) { var self = this; self.db.collection(collections.CACHE).findOne({ - walletId: record.walletId, - type: 'historyCache' + walletId: walletId, + type: 'historyCacheStatus', + key: null + }, function(err, result) { + if (err) return cb(err); + if (!result) return cb(); + + // Reverse indexes + var start = result.totalItems - from; + var end = start + to - from; + + + console.log('[storage.js.632] from,to:', from, to); //TODO + console.log('[storage.js.632] start,end:', start, end); //TODO + + // Cache is OK. + self.db.collection(collections.CACHE).findOne({ + walletId: walletId, + type: 'historyCache', + key: null + }, function(err, result) { + console.log('[storage.js.641:result:]', result); //TODO + if (err) return cb(err); + if (!_.any(result.history, function(i) { + return !!i; + })) return cb(); // some items are not yet defined. + + var ret = result.history.slice(start, end); + return cb(null, ret); + }); + }) +}; + +Storage.prototype.softResetTxHistoryCache = function(walletId, cb) { + this.db.collection(collections.CACHE).remove({ + walletId: walletId, + type: 'historyCacheStatus', + key: null + }, { + w: 1 + }, cb); +}; + + +Storage.prototype.clearTxHistoryCache = function(walletId, cb) { + var self = this; + self.db.collection(collections.CACHE).remove({ + walletId: walletId, + type: 'historyCache', + key: null + }, { + w: 1 + }, function(err) { + self.db.collection(collections.CACHE).remove({ + walletId: walletId, + type: 'historyCacheStatus', + key: null + }, { + w: 1 + }, cb); + }); +}; + +Storage.prototype.storeTxHistoryCache = function(walletId, totalItems, firstPosition, itemsLastFirst, cb) { + $.shouldBeNumber(firstPosition); + $.checkArgument(firstPosition >= 0); + $.shouldBeNumber(totalItems); + $.checkArgument(totalItems >= 0); + + var self = this; + + self.db.collection(collections.CACHE).findOne({ + walletId: walletId, + type: 'historyCache', + key: null }, function(err, result) { if (err) return cb(err); result = result || []; - result - self.db.collection(collections.CACHE).update({ - walletId: record.walletId, - type: record.type, - key: record.key, - }, record, { + //create a sparce array, from the reversed input + _.each(itemsLastFirst.reverse(), function(i) { + result[firstPosition++] = i; + }); + + var cacheIsReady = !!result[0]; + var now = Date.now(); + + self.db.collection(collections.CACHE).update({ + walletId: walletId, + type: 'historyCacheStatus', + key: null + }, { + totalItems: totalItems, + updatedOn: now, + }, { w: 1, upsert: true, - }, next); - }, cb); + }, function(err) { + if (err) return cb(err); + + self.db.collection(collections.CACHE).update({ + walletId: walletId, + type: 'historyCache', + key: null + }, { + history: result + }, { + w: 1, + upsert: true, + }, cb); + + }); + + }); }; diff --git a/test/integration/helpers.js b/test/integration/helpers.js index f0d0cac..2f66e3b 100644 --- a/test/integration/helpers.js +++ b/test/integration/helpers.js @@ -322,7 +322,8 @@ helpers.stubBroadcast = function(thirdPartyBroadcast) { blockchainExplorer.getTransaction = sinon.stub().callsArgWith(1, null, null); }; -helpers.stubHistory = function(txs) { +helpers.stubHistory = function(txs, totalItems) { + totalItems = totalItems || 100; blockchainExplorer.getTransactions = function(addresses, from, to, cb) { var MAX_BATCH_SIZE = 100; var nbTxs = txs.length; @@ -343,7 +344,7 @@ helpers.stubHistory = function(txs) { if (to > nbTxs) to = nbTxs; var page = txs.slice(from, to); - return cb(null, page); + return cb(null, page, totalItems); }; }; diff --git a/test/integration/server.js b/test/integration/server.js index 36dc2a3..b4555b1 100644 --- a/test/integration/server.js +++ b/test/integration/server.js @@ -4092,7 +4092,7 @@ describe('Wallet service', function() { amount: 200, }], }]; - helpers.stubHistory(txs); + helpers.stubHistory(txs, 100); server.editTxNote({ txid: '123', body: 'just some note' @@ -5923,6 +5923,59 @@ describe('Wallet service', function() { done(); }); }); + + it('should cache tx history from insight', function(done) { + helpers.stubHistory(TestData.historyToCache); + var spy = sinon.spy(server.storage, 'storeTxHistoryCache'); + var toCache = _.filter(TestData.historyToCache, function(i) { + return i.confirmations >= server.confirmationsToStartCaching; + }); + var skip = 1; + + server.getTxHistory({ + skip: skip, + }, function(err, txs) { + + // FROM the END, we are getting items + // End-1, end-2, end-3. + + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(TestData.historyToCache.length - skip); + var calls = spy.getCalls(); + calls.length.should.equal(1); + calls[0].args[1].should.equal(100); // total + calls[0].args[2].should.equal(100 - skip); // position + calls[0].args[3].length.should.equal(toCache.length - skip); + + // should be reversed! + calls[0].args[3][0].txid.should.equal(toCache[toCache.length-1].txid); + server.storage.storeTxHistoryCache.restore(); + done(); + }); + }); + + + it('should not cache tx history from insight', function(done) { + helpers.stubHistory(TestData.history); + var spy = sinon.spy(server.storage, 'storeTxHistoryCache'); + var total = _.filter(TestData.history, function(i) { + return i.confirmations >= server.confirmationsToStartCaching; + }); + server.getTxHistory({}, function(err, txs) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(TestData.history.length); + var calls = spy.getCalls(); + calls.length.should.equal(1); + calls[0].args[1].should.equal(100); + calls[0].args[2].should.equal(100); + calls[0].args[3].length.should.deep.equal(total.length); + server.storage.storeTxHistoryCache.restore(); + done(); + }); + }); + it('should get tx history for incoming txs', function(done) { server._normalizeTxHistory = sinon.stub().returnsArg(0); var txs = [{ diff --git a/test/testdata.js b/test/testdata.js index f5c466e..3ebfd01 100644 --- a/test/testdata.js +++ b/test/testdata.js @@ -251,6 +251,165 @@ var history = [ fees: 0.00014299 }]; +var historyToCache = [ + { + txid: "0279ef7b21630f859deb723e28beac9e7011660bd1346c2da40321d2f7e34f04", + vin: [{ + txid: "c8e221141e8bb60977896561b77fa59d6dacfcc10db82bf6f5f923048b11c70d", + vout: 0, + n: 0, + addr: "2N6Zutg26LEC4iYVxi7SHhopVLP1iZPU1rZ", + valueSat: 485645, + value: 0.00485645, + }, { + txid: "6e599eea3e2898b91087eb87e041c5d8dec5362447a8efba185ed593f6dc64c0", + vout: 1, + n: 1, + addr: "2MyqmcWjmVxW8i39wdk1CVPdEqKyFSY9H1S", + valueSat: 885590, + value: 0.0088559, + }], + vout: [{ + value: "0.00045753", + n: 0, + scriptPubKey: { + addresses: [ + "2NAVFnsHqy5JvqDJydbHPx393LFqFFBQ89V" + ] + }, + }, { + value: "0.01300000", + n: 1, + scriptPubKey: { + addresses: [ + "mq4D3Va5mYHohMEHrgHNGzCjKhBKvuEhPE" + ] + } + }], + confirmations: 120, + firstSeenTs: 1424471000, + valueOut: 0.01345753, + valueIn: 0.01371235, + fees: 0.00025482 +}, + { + txid: "0279ef7b21630f959deb723e28beac9e7011660bd1346c2da40321d2f7e34f04", + vin: [{ + txid: "c8e221141e8bb60977896561b77fa59d6dacfcc10db82bf6f5f923048b11c70d", + vout: 0, + n: 0, + addr: "2N6Zutg26LEC4iYVxi7SHhopVLP1iZPU1rZ", + valueSat: 485645, + value: 0.00485645, + }, { + txid: "6e599eea3e2898b91087eb87e041c5d8dec5362447a8efba185ed593f6dc64c0", + vout: 1, + n: 1, + addr: "2MyqmcWjmVxW8i39wdk1CVPdEqKyFSY9H1S", + valueSat: 885590, + value: 0.0088559, + }], + vout: [{ + value: "0.00045753", + n: 0, + scriptPubKey: { + addresses: [ + "2NAVFnsHqy5JvqDJydbHPx393LFqFFBQ89V" + ] + }, + }, { + value: "0.01300000", + n: 1, + scriptPubKey: { + addresses: [ + "mq4D3Va5mYHohMEHrgHNGzCjKhBKvuEhPE" + ] + } + }], + confirmations: 120, + firstSeenTs: 1424471000, + valueOut: 0.01345753, + valueIn: 0.01371235, + fees: 0.00025482 +}, + + { + txid: "0279ef7b21630f859deb723e28beac9e7011660bd1346c2da40321d2f7e34f04", + vin: [{ + txid: "c8e221141e8bb60977896561b77fa59d6dacfcc10db82bf6f5f923048b11c70d", + vout: 0, + n: 0, + addr: "2N6Zutg26LEC4iYVxi7SHhopVLP1iZPU1rZ", + valueSat: 485645, + value: 0.00485645, + }, { + txid: "6e599eea3e2898b91087eb87e041c5d8dec5362447a8efba185ed593f6dc64c0", + vout: 1, + n: 1, + addr: "2MyqmcWjmVxW8i39wdk1CVPdEqKyFSY9H1S", + valueSat: 885590, + value: 0.0088559, + }], + vout: [{ + value: "0.00045753", + n: 0, + scriptPubKey: { + addresses: [ + "2NAVFnsHqy5JvqDJydbHPx393LFqFFBQ89V" + ] + }, + }, { + value: "0.01300000", + n: 1, + scriptPubKey: { + addresses: [ + "mq4D3Va5mYHohMEHrgHNGzCjKhBKvuEhPE" + ] + } + }], + confirmations: 2, + firstSeenTs: 1424471041, + blocktime: 1424471051, + valueOut: 0.01345753, + valueIn: 0.01371235, + fees: 0.00025482 +}, { + txid: "fad88682ccd2ff34cac6f7355fe9ecd8addd9ef167e3788455972010e0d9d0de", + vin: [{ + txid: "0279ef7b21630f859deb723e28beac9e7011660bd1346c2da40321d2f7e34f04", + vout: 0, + n: 0, + addr: "2NAVFnsHqy5JvqDJydbHPx393LFqFFBQ89V", + valueSat: 45753, + value: 0.00045753, + }], + vout: [{ + value: "0.00011454", + n: 0, + scriptPubKey: { + addresses: [ + "2N7GT7XaN637eBFMmeczton2aZz5rfRdZso" + ] + } + }, { + value: "0.00020000", + n: 1, + scriptPubKey: { + addresses: [ + "mq4D3Va5mYHohMEHrgHNGzCjKhBKvuEhPE" + ] + } + }], + confirmations: 101, + time: 1424472242, + blocktime: 1424472242, + valueOut: 0.00031454, + valueIn: 0.00045753, + fees: 0.00014299 +}]; + + module.exports.keyPair = keyPair; module.exports.copayers = copayers; module.exports.history = history; +module.exports.historyToCache = historyToCache;