diff --git a/.travis.yml b/.travis.yml index 80a5abe..66e3f5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: node_js node_js: - - '0.10' - - '0.12' - '4' install: - npm install diff --git a/config.js b/config.js index 56fdd5c..3e6a902 100644 --- a/config.js +++ b/config.js @@ -81,6 +81,5 @@ var config = { // api_user: xxx, // api_key: xxx, // }); - }; module.exports = config; diff --git a/lib/blockchainexplorers/insight.js b/lib/blockchainexplorers/insight.js index 4ae1ef3..b8e7db0 100644 --- a/lib/blockchainexplorers/insight.js +++ b/lib/blockchainexplorers/insight.js @@ -97,6 +97,7 @@ Insight.prototype.getTransaction = function(txid, cb) { Insight.prototype.getTransactions = function(addresses, from, to, cb) { var qs = []; + var total; if (_.isNumber(from)) qs.push('from=' + from); if (_.isNumber(to)) qs.push('to=' + to); @@ -116,13 +117,18 @@ Insight.prototype.getTransactions = function(addresses, from, to, cb) { this._doRequest(args, function(err, res, txs) { if (err || res.statusCode !== 200) return cb(_parseErr(err, res)); - if (_.isObject(txs) && txs.items) - txs = txs.items; + if (_.isObject(txs)) { + if (txs.totalItems) + total = txs.totalItems; + + if (txs.items) + txs = txs.items; + } // NOTE: Whenever Insight breaks communication with bitcoind, it returns invalid data but no error code. if (!_.isArray(txs) || (txs.length != _.compact(txs).length)) return cb(new Error('Could not retrieve transactions from blockchain. Request was:' + JSON.stringify(args))); - return cb(null, txs); + return cb(null, txs, total); }); }; diff --git a/lib/blockchainmonitor.js b/lib/blockchainmonitor.js index d1c48f1..0be4200 100644 --- a/lib/blockchainmonitor.js +++ b/lib/blockchainmonitor.js @@ -109,22 +109,25 @@ BlockchainMonitor.prototype._handleTxId = function(data, processIt) { log.info('Processing accepted txp [' + txp.id + '] for wallet ' + walletId + ' [' + txp.amount + 'sat ]'); txp.setBroadcasted(); - self.storage.storeTx(self.walletId, txp, function(err) { - if (err) - log.error('Could not save TX'); - var args = { - txProposalId: txp.id, - txid: data.txid, - amount: txp.getTotalAmount(), - }; + self.storage.softResetTxHistoryCache(walletId, function() { + self.storage.storeTx(self.walletId, txp, function(err) { + if (err) + log.error('Could not save TX'); - var notification = Notification.create({ - type: 'NewOutgoingTxByThirdParty', - data: args, - walletId: walletId, + var args = { + txProposalId: txp.id, + txid: data.txid, + amount: txp.getTotalAmount(), + }; + + var notification = Notification.create({ + type: 'NewOutgoingTxByThirdParty', + data: args, + walletId: walletId, + }); + self._storeAndBroadcastNotification(notification); }); - self._storeAndBroadcastNotification(notification); }); }); }; @@ -166,8 +169,10 @@ BlockchainMonitor.prototype._handleTxOuts = function(data) { }, walletId: walletId, }); - self._updateActiveAddresses(address, function() { - self._storeAndBroadcastNotification(notification, next); + self.storage.softResetTxHistoryCache(walletId, function() { + self._updateActiveAddresses(address, function() { + self._storeAndBroadcastNotification(notification, next); + }); }); }); }, function(err) { @@ -202,8 +207,11 @@ BlockchainMonitor.prototype._handleNewBlock = function(network, hash) { hash: hash, }, }); - self._storeAndBroadcastNotification(notification, function(err) { - return; + + self.storage.softResetAllTxHistoryCache(function() { + self._storeAndBroadcastNotification(notification, function(err) { + return; + }); }); }; diff --git a/lib/common/defaults.js b/lib/common/defaults.js index 34c14e6..30ed443 100644 --- a/lib/common/defaults.js +++ b/lib/common/defaults.js @@ -73,4 +73,14 @@ Defaults.UTXO_SELECTION_MAX_FEE_VS_SINGLE_UTXO_FEE_FACTOR = 5; // Minimum allowed amount for tx outputs (including change) in SAT Defaults.MIN_OUTPUT_AMOUNT = 5000; +// Number of confirmations from which tx in history will be cached +// (ie we consider them inmutables) +Defaults.CONFIRMATIONS_TO_START_CACHING = 6 * 6; // ~ 6hrs + +// Number of addresses from which tx history is enabled in a wallet +Defaults.HISTORY_CACHE_ADDRESS_THRESOLD = 100; + +// Cache time for blockchain height (in seconds) +Defaults.BLOCKHEIGHT_CACHE_TIME = 10 * 60; + module.exports = Defaults; diff --git a/lib/server.js b/lib/server.js index 4aedd09..b6dcaba 100644 --- a/lib/server.js +++ b/lib/server.js @@ -108,11 +108,11 @@ WalletService.initialize = function(opts, cb) { }; function initMessageBroker(cb) { - if (opts.messageBroker) { - messageBroker = opts.messageBroker; - } else { - messageBroker = new MessageBroker(opts.messageBrokerOpts); + messageBroker = opts.messageBroker || new MessageBroker(opts.messageBrokerOpts); + if (messageBroker) { + messageBroker.onMessage(WalletService.handleIncomingNotification); } + return cb(); }; @@ -153,6 +153,14 @@ WalletService.initialize = function(opts, cb) { }); }; +WalletService.handleIncomingNotification = function(notification, cb) { + cb = cb || function() {}; + + if (!notification || notification.type != 'NewBlock') return cb(); + WalletService._clearBlockchainHeightCache(); + return cb(); +}; + WalletService.shutDown = function(cb) { if (!initialized) return cb(); @@ -2196,7 +2204,9 @@ WalletService.prototype._processBroadcast = function(txp, opts, cb) { self._notify('NewOutgoingTx', args); } - return cb(err, txp); + self.storage.softResetTxHistoryCache(self.walletId, function() { + return cb(err, txp); + }); }); }; @@ -2421,6 +2431,7 @@ WalletService.prototype._normalizeTxHistory = function(txs) { return { txid: tx.txid, confirmations: tx.confirmations, + blockheight: tx.blockheight, fees: parseInt((tx.fees * 1e8).toFixed(0)), time: t, inputs: inputs, @@ -2429,6 +2440,37 @@ WalletService.prototype._normalizeTxHistory = function(txs) { }); }; +WalletService._cachedBlockheight; +WalletService._clearBlockchainHeightCache = function() { + WalletService._cachedBlockheight.current = null; +}; + +WalletService.prototype._getBlockchainHeight = function(network, cb) { + var self = this; + + var now = Date.now(); + if (!WalletService._cachedBlockheight) WalletService._cachedBlockheight = {}; + var cache = WalletService._cachedBlockheight; + + function fetchFromBlockchain(cb) { + var bc = self._getBlockchainExplorer(network); + bc.getBlockchainHeight(function(err, height) { + if (!err && height > 0) { + cache.current = height; + cache.last = height; + cache.updatedOn = now; + } + return cb(null, cache.last); + }); + }; + + if (!cache.current || (now - cache.updatedOn) > Defaults.BLOCKHEIGHT_CACHE_TIME * 1000) { + return fetchFromBlockchain(cb); + } + + return cb(null, cache.current); +}; + /** * Retrieves all transactions (incoming & outgoing) * Times are in UNIX EPOCH @@ -2442,6 +2484,7 @@ WalletService.prototype._normalizeTxHistory = function(txs) { WalletService.prototype.getTxHistory = function(opts, cb) { var self = this; + opts = opts || {}; opts.limit = (_.isUndefined(opts.limit) ? Defaults.HISTORY_LIMIT : opts.limit); if (opts.limit > Defaults.HISTORY_LIMIT) @@ -2574,27 +2617,90 @@ WalletService.prototype.getTxHistory = function(opts, cb) { }); }; + function getNormalizedTxs(addresses, from, to, cb) { + var txs, fromCache, totalItems; + var useCache = addresses.length >= Defaults.HISTORY_CACHE_ADDRESS_THRESOLD; + var network = Bitcore.Address(addresses[0].address).toObject().network; + + async.series([ + + function(next) { + if (!useCache) return next(); + + self.storage.getTxHistoryCache(self.walletId, from, to, function(err, res) { + if (err) return next(err); + if (!res || !res[0]) return next(); + + txs = res; + fromCache = true; + + return next() + }); + }, + function(next) { + if (txs) return next(); + + var addressStrs = _.pluck(addresses, 'address'); + var bc = self._getBlockchainExplorer(network); + bc.getTransactions(addressStrs, from, to, function(err, rawTxs, total) { + if (err) return next(err); + txs = self._normalizeTxHistory(rawTxs); + totalItems = total; + return next(); + }); + }, + function(next) { + if (!useCache || fromCache) return next(); + + var txsToCache = _.filter(txs, function(i) { + return i.confirmations >= Defaults.CONFIRMATIONS_TO_START_CACHING; + }).reverse(); + + if (!txsToCache.length) return next(); + + var fwdIndex = totalItems - to; + if (fwdIndex < 0) fwdIndex = 0; + self.storage.storeTxHistoryCache(self.walletId, totalItems, fwdIndex, txsToCache, next); + }, + function(next) { + if (!txs) return next(); + + // Fix tx confirmations + self._getBlockchainHeight(network, function(err, height) { + if (err || !height) return next(err); + _.each(txs, function(tx) { + if (tx.blockheight >= 0) { + tx.confirmations = height - tx.blockheight + 1; + } + }); + next(); + }); + }, + ], function(err) { + if (err) return cb(err); + + return cb(null, { + items: txs, + fromCache: fromCache + }); + }); + }; + // Get addresses for this wallet self.storage.fetchAddresses(self.walletId, function(err, addresses) { if (err) return cb(err); if (addresses.length == 0) return cb(null, []); - var addressStrs = _.pluck(addresses, 'address'); - var networkName = Bitcore.Address(addressStrs[0]).toObject().network; + var from = opts.skip || 0; + var to = from + opts.limit; - var bc = self._getBlockchainExplorer(networkName); async.parallel([ function(next) { - self.storage.fetchTxs(self.walletId, {}, next); + getNormalizedTxs(addresses, from, to, next); }, function(next) { - var from = opts.skip || 0; - var to = from + opts.limit; - bc.getTransactions(addressStrs, from, to, function(err, txs) { - if (err) return cb(err); - next(null, self._normalizeTxHistory(txs)); - }); + self.storage.fetchTxs(self.walletId, {}, next); }, function(next) { self.storage.fetchTxNotes(self.walletId, {}, next); @@ -2602,13 +2708,12 @@ WalletService.prototype.getTxHistory = function(opts, cb) { ], function(err, res) { if (err) return cb(err); - var proposals = res[0]; - var txs = res[1]; - var notes = res[2]; + var finalTxs = decorate(res[0].items, addresses, res[1], res[2]); - txs = decorate(txs, addresses, proposals, notes); + if (res[0].fromCache) + log.debug("History from cache for:", self.walletId, from, to); - return cb(null, txs); + return cb(null, finalTxs, !!res[0].fromCache); }); }); }; @@ -2659,40 +2764,43 @@ WalletService.prototype.scan = function(opts, cb) { if (!wallet.isComplete()) return cb(Errors.WALLET_NOT_COMPLETE); wallet.scanStatus = 'running'; - self.storage.storeWallet(wallet, function(err) { - if (err) return cb(err); - var derivators = []; - _.each([false, true], function(isChange) { - derivators.push({ - derive: _.bind(wallet.createAddress, wallet, isChange), - rewind: _.bind(wallet.addressManager.rewindIndex, wallet.addressManager, isChange), - }); - if (opts.includeCopayerBranches) { - _.each(wallet.copayers, function(copayer) { - if (copayer.addressManager) { - derivators.push({ - derive: _.bind(copayer.createAddress, copayer, wallet, isChange), - rewind: _.bind(copayer.addressManager.rewindIndex, copayer.addressManager, isChange), - }); - } - }); - } - }); + self.storage.clearTxHistoryCache(self.walletId, function() { + self.storage.storeWallet(wallet, function(err) { + if (err) return cb(err); - async.eachSeries(derivators, function(derivator, next) { - scanBranch(derivator, function(err, addresses) { - if (err) return next(err); - self.storage.storeAddressAndWallet(wallet, addresses, next); - }); - }, function(error) { - self.storage.fetchWallet(wallet.id, function(err, wallet) { - if (err) return cb(err); - wallet.scanStatus = error ? 'error' : 'success'; - self.storage.storeWallet(wallet, function() { - return cb(error); + var derivators = []; + _.each([false, true], function(isChange) { + derivators.push({ + derive: _.bind(wallet.createAddress, wallet, isChange), + rewind: _.bind(wallet.addressManager.rewindIndex, wallet.addressManager, isChange), }); - }) + if (opts.includeCopayerBranches) { + _.each(wallet.copayers, function(copayer) { + if (copayer.addressManager) { + derivators.push({ + derive: _.bind(copayer.createAddress, copayer, wallet, isChange), + rewind: _.bind(copayer.addressManager.rewindIndex, copayer.addressManager, isChange), + }); + } + }); + } + }); + + async.eachSeries(derivators, function(derivator, next) { + scanBranch(derivator, function(err, addresses) { + if (err) return next(err); + self.storage.storeAddressAndWallet(wallet, addresses, next); + }); + }, function(error) { + self.storage.fetchWallet(wallet.id, function(err, wallet) { + if (err) return cb(err); + wallet.scanStatus = error ? 'error' : 'success'; + self.storage.storeWallet(wallet, function() { + return cb(error); + }); + }) + }); }); }); }); diff --git a/lib/storage.js b/lib/storage.js index 7cf6a7f..6682679 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -609,6 +609,161 @@ Storage.prototype.storeActiveAddresses = function(walletId, addresses, cb) { }, cb); }; +// -------- --------------------------- Total +// > Time > +// ^to <= ^from +// ^fwdIndex => ^end +Storage.prototype.getTxHistoryCache = function(walletId, from, to, cb) { + var self = this; + $.checkArgument(from >= 0); + $.checkArgument(from <= to); + + self.db.collection(collections.CACHE).findOne({ + walletId: walletId, + type: 'historyCacheStatus', + key: null + }, function(err, result) { + if (err) return cb(err); + if (!result) return cb(); + if (!result.isUpdated) return cb(); + + // Reverse indexes + var fwdIndex = result.totalItems - to; + + if (fwdIndex < 0) { + fwdIndex = 0; + } + + var end = result.totalItems - from; + + // nothing to return + if (end <= 0) return cb(null, []); + + // Cache is OK. + self.db.collection(collections.CACHE).find({ + walletId: walletId, + type: 'historyCache', + key: { + $gte: fwdIndex, + $lt: end + }, + }).sort({ + key: -1, + }).toArray(function(err, result) { + if (err) return cb(err); + + if (!result) return cb(); + + if (result.length < end - fwdIndex) { + // some items are not yet defined. + return cb(); + } + + var txs = _.pluck(result, 'tx'); + return cb(null, txs); + }); + }) +}; + +Storage.prototype.softResetAllTxHistoryCache = function(cb) { + this.db.collection(collections.CACHE).update({ + type: 'historyCacheStatus', + }, { + isUpdated: false, + }, { + multi: true, + }, cb); +}; + +Storage.prototype.softResetTxHistoryCache = function(walletId, cb) { + this.db.collection(collections.CACHE).update({ + walletId: walletId, + type: 'historyCacheStatus', + key: null + }, { + isUpdated: false, + }, { + w: 1, + upsert: true, + }, cb); +}; + +Storage.prototype.clearTxHistoryCache = function(walletId, cb) { + var self = this; + self.db.collection(collections.CACHE).remove({ + walletId: walletId, + type: 'historyCache', + }, { + multi: 1 + }, function(err) { + if (err) return cb(err); + self.db.collection(collections.CACHE).remove({ + walletId: walletId, + type: 'historyCacheStatus', + key: null + }, { + w: 1 + }, cb); + }); +}; + +// items should be in CHRONOLOGICAL order +Storage.prototype.storeTxHistoryCache = function(walletId, totalItems, firstPosition, items, cb) { + $.shouldBeNumber(firstPosition); + $.checkArgument(firstPosition >= 0); + $.shouldBeNumber(totalItems); + $.checkArgument(totalItems >= 0); + + var self = this; + + _.each(items, function(item, i) { + item.position = firstPosition + i; + }); + var cacheIsComplete = (firstPosition == 0); + + // TODO: check txid uniqness? + async.each(items, function(item, next) { + var pos = item.position; + delete item.position; + self.db.collection(collections.CACHE).update({ + walletId: walletId, + type: 'historyCache', + key: pos, + }, { + walletId: walletId, + type: 'historyCache', + key: pos, + tx: item, + }, { + w: 1, + upsert: true, + }, next); + }, function(err) { + if (err) return cb(err); + + self.db.collection(collections.CACHE).update({ + walletId: walletId, + type: 'historyCacheStatus', + key: null + }, { + walletId: walletId, + type: 'historyCacheStatus', + key: null, + totalItems: totalItems, + updatedOn: Date.now(), + isComplete: cacheIsComplete, + isUpdated: true, + }, { + w: 1, + upsert: true, + }, cb); + }); +}; + + + + + Storage.prototype.fetchActiveAddresses = function(walletId, cb) { var self = this; diff --git a/test/integration/helpers.js b/test/integration/helpers.js index 60cee8f..ec49d5b 100644 --- a/test/integration/helpers.js +++ b/test/integration/helpers.js @@ -323,6 +323,7 @@ helpers.stubBroadcast = function(thirdPartyBroadcast) { }; helpers.stubHistory = function(txs) { + var totalItems = txs.length; blockchainExplorer.getTransactions = function(addresses, from, to, cb) { var MAX_BATCH_SIZE = 100; var nbTxs = txs.length; @@ -343,7 +344,7 @@ helpers.stubHistory = function(txs) { if (to > nbTxs) to = nbTxs; var page = txs.slice(from, to); - return cb(null, page); + return cb(null, page, totalItems); }; }; @@ -428,4 +429,55 @@ helpers.createAndPublishTx = function(server, txOpts, signingKey, cb) { }); }; + +helpers.historyCacheTest = function(items) { + var template = { + txid: "fad88682ccd2ff34cac6f7355fe9ecd8addd9ef167e3788455972010e0d9d0de", + vin: [{ + txid: "0279ef7b21630f859deb723e28beac9e7011660bd1346c2da40321d2f7e34f04", + vout: 0, + n: 0, + addr: "2NAVFnsHqy5JvqDJydbHPx393LFqFFBQ89V", + valueSat: 45753, + value: 0.00045753, + }], + vout: [{ + value: "0.00011454", + n: 0, + scriptPubKey: { + addresses: [ + "2N7GT7XaN637eBFMmeczton2aZz5rfRdZso" + ] + } + }, { + value: "0.00020000", + n: 1, + scriptPubKey: { + addresses: [ + "mq4D3Va5mYHohMEHrgHNGzCjKhBKvuEhPE" + ] + } + }], + confirmations: 1, + blockheight: 423499, + time: 1424472242, + blocktime: 1424472242, + valueOut: 0.00031454, + valueIn: 0.00045753, + fees: 0.00014299 + }; + + var ret = []; + _.each(_.range(0, items), function(i) { + var t = _.clone(template); + t.txid = 'txid:' + i; + t.confirmations = items - i - 1; + t.blockheight = i; + t.time = t.blocktime = i; + ret.unshift(t); + }); + + return ret; +}; + module.exports = helpers; diff --git a/test/integration/server.js b/test/integration/server.js index 8a73a3f..b032293 100644 --- a/test/integration/server.js +++ b/test/integration/server.js @@ -3727,6 +3727,7 @@ describe('Wallet service', function() { }); it('should include the note in tx history listing', function(done) { helpers.createAddresses(server, wallet, 1, 1, function(mainAddresses, changeAddress) { + blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 1000); server._normalizeTxHistory = sinon.stub().returnsArg(0); var txs = [{ txid: '123', @@ -5494,6 +5495,7 @@ describe('Wallet service', function() { describe('#getTxHistory', function() { var server, wallet, mainAddresses, changeAddresses; beforeEach(function(done) { + blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 1000); helpers.createAndJoinWallet(1, 1, function(s, w) { server = s; wallet = w; @@ -5519,6 +5521,7 @@ describe('Wallet service', function() { done(); }); }); + it('should get tx history for incoming txs', function(done) { server._normalizeTxHistory = sinon.stub().returnsArg(0); var txs = [{ @@ -5813,6 +5816,341 @@ describe('Wallet service', function() { }); }); + describe('#getTxHistory cache', function() { + var server, wallet, mainAddresses, changeAddresses; + var _threshold = Defaults.HISTORY_CACHE_ADDRESS_THRESOLD; + beforeEach(function(done) { + Defaults.HISTORY_CACHE_ADDRESS_THRESOLD = 1; + helpers.createAndJoinWallet(1, 1, function(s, w) { + server = s; + wallet = w; + helpers.createAddresses(server, wallet, 1, 1, function(main, change) { + mainAddresses = main; + changeAddresses = change; + done(); + }); + }); + }); + afterEach(function() { + Defaults.HISTORY_CACHE_ADDRESS_THRESOLD = _threshold; + }); + + it('should store partial cache tx history from insight', function(done) { + var skip = 31; + var limit = 10; + var totalItems = 200; + var currentHeight = 1000; + + var h = helpers.historyCacheTest(totalItems); + helpers.stubHistory(h); + blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, currentHeight); + var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache'); + + + server.getTxHistory({ + skip: skip, + limit: limit, + }, function(err, txs) { + + // FROM the END, we are getting items + // End-1, end-2, end-3. + + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(limit); + var calls = storeTxHistoryCacheSpy.getCalls(); + calls.length.should.equal(1); + + calls[0].args[1].should.equal(totalItems); // total + calls[0].args[2].should.equal(totalItems - skip - limit); // position + calls[0].args[3].length.should.equal(5); // 5 txs have confirmations>= 100 + + // should be reversed! + calls[0].args[3][0].confirmations.should.equal(currentHeight - (totalItems - (skip + limit)) + 1); + calls[0].args[3][0].txid.should.equal(h[skip + limit - 1].txid); + server.storage.storeTxHistoryCache.restore(); + done(); + }); + }); + + + it('should not cache tx history when requesting txs with low # of confirmations', function(done) { + var h = helpers.historyCacheTest(200); + helpers.stubHistory(h); + blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 1000); + var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache'); + server.getTxHistory({ + skip: 0, + limit: 10, + }, function(err, txs) { + should.not.exist(err); + should.exist(txs); + var calls = storeTxHistoryCacheSpy.getCalls(); + calls.length.should.equal(0); + server.storage.storeTxHistoryCache.restore(); + done(); + }); + }); + + + it('should store cache all tx history from insight', function(done) { + var skip = 195; + var limit = 5; + var totalItems = 200; + var currentHeight = 1000; + + var h = helpers.historyCacheTest(totalItems); + helpers.stubHistory(h); + blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, currentHeight); + var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache'); + + server.getTxHistory({ + skip: skip, + limit: limit, + }, function(err, txs) { + + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(limit); + var calls = storeTxHistoryCacheSpy.getCalls(); + calls.length.should.equal(1); + + calls[0].args[1].should.equal(totalItems); // total + calls[0].args[2].should.equal(totalItems - skip - limit); // position + calls[0].args[3].length.should.equal(5); + + // should be reversed! + calls[0].args[3][0].confirmations.should.equal(currentHeight + 1); + calls[0].args[3][0].txid.should.equal(h[totalItems - 1].txid); + server.storage.storeTxHistoryCache.restore(); + done(); + }); + }); + + it('should get real # of confirmations based on current block height', function(done) { + var _confirmations = Defaults.CONFIRMATIONS_TO_START_CACHING; + Defaults.CONFIRMATIONS_TO_START_CACHING = 6; + WalletService._cachedBlockheight = null; + + var h = helpers.historyCacheTest(20); + _.each(h, function(x, i) { + x.blockheight = 100 - i; + }); + helpers.stubHistory(h); + var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache'); + + blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 100); + + // Cache txs + server.getTxHistory({ + skip: 0, + limit: 30, + }, function(err, txs) { + should.not.exist(err); + should.exist(txs); + var calls = storeTxHistoryCacheSpy.getCalls(); + calls.length.should.equal(1); + + server.getTxHistory({ + skip: 0, + limit: 30, + }, function(err, txs) { + should.not.exist(err); + txs.length.should.equal(20); + _.first(txs).confirmations.should.equal(1); + _.last(txs).confirmations.should.equal(20); + + blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 200); + server._notify('NewBlock', { + hash: 'dummy hash', + }, { + isGlobal: true + }, function(err) { + should.not.exist(err); + setTimeout(function() { + server.getTxHistory({ + skip: 0, + limit: 30, + }, function(err, txs) { + should.not.exist(err); + _.first(txs).confirmations.should.equal(101); + _.last(txs).confirmations.should.equal(120); + + server.storage.storeTxHistoryCache.restore(); + Defaults.CONFIRMATIONS_TO_START_CACHING = _confirmations; + done(); + }); + }, 100); + }); + }); + }); + }); + + it('should get cached # of confirmations if current height unknown', function(done) { + var _confirmations = Defaults.CONFIRMATIONS_TO_START_CACHING; + Defaults.CONFIRMATIONS_TO_START_CACHING = 6; + WalletService._cachedBlockheight = null; + + var h = helpers.historyCacheTest(20); + helpers.stubHistory(h); + var storeTxHistoryCacheSpy = sinon.spy(server.storage, 'storeTxHistoryCache'); + + blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, null); + + // Cache txs + server.getTxHistory({ + skip: 0, + limit: 30, + }, function(err, txs) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(20); + var calls = storeTxHistoryCacheSpy.getCalls(); + calls.length.should.equal(1); + + server.getTxHistory({ + skip: 0, + limit: 30, + }, function(err, txs) { + should.not.exist(err); + txs.length.should.equal(20); + _.first(txs).confirmations.should.equal(0); + _.last(txs).confirmations.should.equal(19); + + server.storage.storeTxHistoryCache.restore(); + Defaults.CONFIRMATIONS_TO_START_CACHING = _confirmations; + done(); + }); + }); + }); + + describe('Downloading history', function() { + var h; + beforeEach(function(done) { + blockchainExplorer.getBlockchainHeight = sinon.stub().callsArgWith(0, null, 1000); + h = helpers.historyCacheTest(200); + helpers.stubHistory(h); + server.storage.clearTxHistoryCache(server.walletId, function() { + done(); + }); + }); + + it('from 0 to 200, two times, in order', function(done) { + async.eachSeries(_.range(0, 200, 5), function(i, next) { + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(false); + next(); + }); + }, function() { + async.eachSeries(_.range(0, 200, 5), function(i, next) { + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(i >= Defaults.CONFIRMATIONS_TO_START_CACHING && i < 200); + next(); + }); + }, done); + }); + }); + + it('from 0 to 200, two times, random', function(done) { + var indexes = _.range(0, 200, 5); + async.eachSeries(_.shuffle(indexes), function(i, next) { + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(false); + next(); + }); + }, function() { + async.eachSeries(_.range(0, 190, 7), function(i, next) { + server.getTxHistory({ + skip: i, + limit: 7, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(7); + var s = h.slice(i, i + 7); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(i >= Defaults.CONFIRMATIONS_TO_START_CACHING); + next(); + }); + }, done); + }); + }); + + + it('from 0 to 200, two times, random, with resets', function(done) { + var indexes = _.range(0, 200, 5); + async.eachSeries(_.shuffle(indexes), function(i, next) { + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(false); + next(); + }); + }, function() { + async.eachSeries(_.range(0, 200, 5), function(i, next) { + + function resetCache(cb) { + if (!(i % 25)) { + storage.softResetTxHistoryCache(server.walletId, function() { + return cb(true); + }); + } else { + return cb(false); + } + } + + resetCache(function(reset) { + server.getTxHistory({ + skip: i, + limit: 5, + }, function(err, txs, fromCache) { + should.not.exist(err); + should.exist(txs); + txs.length.should.equal(5); + var s = h.slice(i, i + 5); + _.pluck(txs, 'txid').should.deep.equal(_.pluck(s, 'txid')); + fromCache.should.equal(i >= Defaults.CONFIRMATIONS_TO_START_CACHING && !reset); + next(); + }); + }); + }, done); + }); + }); + + + }); + }); + describe('#scan', function() { var server, wallet; diff --git a/test/testdata.js b/test/testdata.js index f5c466e..97eb999 100644 --- a/test/testdata.js +++ b/test/testdata.js @@ -251,6 +251,7 @@ var history = [ fees: 0.00014299 }]; + module.exports.keyPair = keyPair; module.exports.copayers = copayers; module.exports.history = history;