From bf2e281787e303f29f5f06baf8779716b7b7b57f Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 2 Nov 2017 10:35:34 -0300 Subject: [PATCH 1/4] WIP short address cache --- lib/common/defaults.js | 6 +++++ lib/server.js | 16 +++++++++-- lib/storage.js | 60 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/lib/common/defaults.js b/lib/common/defaults.js index 30444b5..4181ef7 100644 --- a/lib/common/defaults.js +++ b/lib/common/defaults.js @@ -96,6 +96,12 @@ Defaults.CONFIRMATIONS_TO_START_CACHING = 6 * 6; // ~ 6hrs // Number of addresses from which tx history is enabled in a wallet Defaults.HISTORY_CACHE_ADDRESS_THRESOLD = 100; +// Number of addresses from which balance in cache for a few seconds +Defaults.BALANCE_CACHE_ADDRESS_THRESOLD = Defaults.HISTORY_CACHE_ADDRESS_THRESOLD; + +// Duration in seconds of the balance cache +Defaults.BALANCE_CACHE_DURATION = 5 + // Cache time for blockchain height (in seconds) Defaults.BLOCKHEIGHT_CACHE_TIME = 10 * 60; diff --git a/lib/server.js b/lib/server.js index ac953f5..7f1ef7e 100644 --- a/lib/server.js +++ b/lib/server.js @@ -1243,11 +1243,18 @@ WalletService.prototype._totalizeUtxos = function(utxos) { WalletService.prototype._getBalanceFromAddresses = function(opts, cb, i) { var self = this; - var opts = opts || {}; + var isBig = opts.addresses.length > Default.BALANCE_CACHE_ADDRESS_THRESOLD; // This lock is to prevent server starvation on big wallets self._runLocked(cb, function(cb) { + + if (isBig && self.storage.checkAndUseBalanceCache(addresses,cb)) + log.info('Using UTXO Cache'); + return; + } + + self._getUtxosForCurrentWallet({ coin: opts.coin, addresses: opts.addresses @@ -1271,7 +1278,12 @@ WalletService.prototype._getBalanceFromAddresses = function(opts, cb, i) { }); balance.byAddress = _.values(byAddress); - return cb(null, balance); + + if (isBig) { + return self.storage.storeBalanceCache(addresses, balance, cb); + } else { + return cb(null, balance); + } }); }); }; diff --git a/lib/storage.js b/lib/storage.js index fe0ee75..3e6a691 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -7,6 +7,7 @@ var log = require('npmlog'); log.debug = log.verbose; log.disableColor(); var util = require('util'); +var Bitcore = require('bitcore-lib'); var mongodb = require('mongodb'); @@ -1065,5 +1066,64 @@ Storage.prototype._dump = function(cb, fn) { }); }; + +Storage.prototype._addressHash = function(addresses) { + var all = addresses.join(); + return Bitcore.crypto.Hash.ripemd160(new Buffer(all)).toString('hex'); +}; + +Storage.prototype.checkAndUseBalanceCache = function(addresses, cb) { + var key = ths._addressHash(addresses); + var now = Date.now(); + + self.db.collection(collections.CACHE).findOne({ + walletId: walletId, + type: 'balanceCache', + key: key, + }, function(err, ret) { + if (err) return cb(err); + if (!ret) return cb(); + + var validFor = ret.ts + Defauls.BALANCE_CACHE_DURATION * 1000 - now; + + if (validFor > 0) { + log.debug('','Using Balance Cache valid for %d ms more', validFor); + return cb(null, ret.result); + } + + return cb(); + + log.debug('','Balance cache expired, deleting'); + self.db.collection(collections.CACHE).remove({ + walletId: walletId, + type: 'balanceCache', + key: key, + }, {}, function() { + }); + }); +}); + + + +Storage.prototype.storeBalanceCache = function (addresses, balance, cb) { + var key = ths._addressHash(addresses); + var now = Date.now(); + + self.db.collection(collections.CACHE).update( { + walletId: walletId, + type: 'balanceCache', + key: key, + }, { + "$set": + { + ts: now, + result: balance, + } + }, { + w: 1, + upsert: true, + }, cb); +}; + Storage.collections = collections; module.exports = Storage; From 1db21481d62a6852c69d2f9953fc6d2d2a0917d6 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 2 Nov 2017 13:03:52 -0300 Subject: [PATCH 2/4] fast balance cache --- .jshintrc | 1 + lib/common/defaults.js | 3 -- lib/server.js | 95 ++++++++++++++++++++++---------------- lib/storage.js | 37 +++++++-------- test/integration/server.js | 84 +++++++++++++++++++++++++++++++++ 5 files changed, 160 insertions(+), 60 deletions(-) diff --git a/.jshintrc b/.jshintrc index 844dac3..afed196 100644 --- a/.jshintrc +++ b/.jshintrc @@ -16,6 +16,7 @@ "maxlen": 120, "maxparams": 4, "maxstatements": 15, + "no-extra-semi": 0, "mocha": true, "newcap": true, "noarg": true, diff --git a/lib/common/defaults.js b/lib/common/defaults.js index 4181ef7..4455695 100644 --- a/lib/common/defaults.js +++ b/lib/common/defaults.js @@ -99,9 +99,6 @@ Defaults.HISTORY_CACHE_ADDRESS_THRESOLD = 100; // Number of addresses from which balance in cache for a few seconds Defaults.BALANCE_CACHE_ADDRESS_THRESOLD = Defaults.HISTORY_CACHE_ADDRESS_THRESOLD; -// Duration in seconds of the balance cache -Defaults.BALANCE_CACHE_DURATION = 5 - // Cache time for blockchain height (in seconds) Defaults.BLOCKHEIGHT_CACHE_TIME = 10 * 60; diff --git a/lib/server.js b/lib/server.js index 7f1ef7e..f882dea 100644 --- a/lib/server.js +++ b/lib/server.js @@ -1244,46 +1244,64 @@ WalletService.prototype._totalizeUtxos = function(utxos) { WalletService.prototype._getBalanceFromAddresses = function(opts, cb, i) { var self = this; var opts = opts || {}; - var isBig = opts.addresses.length > Default.BALANCE_CACHE_ADDRESS_THRESOLD; + opts.addresses = opts.addresses || []; + + + function checkBalanceCache(cb) { + if (opts.addresses.length < Defaults.BALANCE_CACHE_ADDRESS_THRESOLD) + return cb(); + + self.storage.checkAndUseBalanceCache(self.walletId, opts.addresses, cb); + }; + + function storeBalanceCache(balance, cb) { + if (opts.addresses.length < Defaults.BALANCE_CACHE_ADDRESS_THRESOLD) + return cb(null, balance); + + self.storage.storeBalanceCache(self.walletId, opts.addresses, balance, function(err) { + if (err) + log.warn('Could not save cache:',err); + + return cb(null, balance); + }); + }; // This lock is to prevent server starvation on big wallets self._runLocked(cb, function(cb) { - - if (isBig && self.storage.checkAndUseBalanceCache(addresses,cb)) - log.info('Using UTXO Cache'); - return; - } - - - self._getUtxosForCurrentWallet({ - coin: opts.coin, - addresses: opts.addresses - }, function(err, utxos) { + checkBalanceCache(function(err, cache) { if (err) return cb(err); - var balance = self._totalizeUtxos(utxos); - - // Compute balance by address - var byAddress = {}; - _.each(_.indexBy(_.sortBy(utxos, 'address'), 'address'), function(value, key) { - byAddress[key] = { - address: key, - path: value.path, - amount: 0, - }; - }); - - _.each(utxos, function(utxo) { - byAddress[utxo.address].amount += utxo.satoshis; - }); - - balance.byAddress = _.values(byAddress); - - if (isBig) { - return self.storage.storeBalanceCache(addresses, balance, cb); - } else { - return cb(null, balance); + if (cache) { + log.info('Using UTXO Cache'); + return cb(null, cache, true); } + + self._getUtxosForCurrentWallet({ + coin: opts.coin, + addresses: opts.addresses + }, function(err, utxos) { + if (err) return cb(err); + + var balance = self._totalizeUtxos(utxos); + + // Compute balance by address + var byAddress = {}; + _.each(_.indexBy(_.sortBy(utxos, 'address'), 'address'), function(value, key) { + byAddress[key] = { + address: key, + path: value.path, + amount: 0, + }; + }); + + _.each(utxos, function(utxo) { + byAddress[utxo.address].amount += utxo.satoshis; + }); + + balance.byAddress = _.values(byAddress); + + storeBalanceCache(balance, cb); + }); }); }); }; @@ -1296,7 +1314,7 @@ WalletService.prototype._getBalanceOneStep = function(opts, cb) { self._getBalanceFromAddresses({ coin: opts.coin, addresses: addresses - }, function(err, balance) { + }, function(err, balance, cacheUsed) { if (err) return cb(err); // Update cache @@ -1305,7 +1323,7 @@ WalletService.prototype._getBalanceOneStep = function(opts, cb) { if (err) { log.warn('Could not update wallet cache', err); } - return cb(null, balance); + return cb(null, balance, cacheUsed); }); }); }); @@ -1382,7 +1400,6 @@ WalletService.prototype.getBalance = function(opts, cb, i) { return self._getBalanceOneStep(opts, cb); } - self.storage.getTwoStepCache(self.walletId, function(err, twoStepCache) { if (err) return cb(err); twoStepCache = twoStepCache || {}; @@ -1404,9 +1421,9 @@ WalletService.prototype.getBalance = function(opts, cb, i) { self._getBalanceFromAddresses({ coin: opts.coin, addresses: activeAddresses - }, function(err, partialBalance) { + }, function(err, partialBalance, cacheUsed) { if (err) return cb(err); - cb(null, partialBalance); + cb(null, partialBalance, cacheUsed); var now = Math.floor(Date.now() / 1000); diff --git a/lib/storage.js b/lib/storage.js index 3e6a691..15f17a7 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -8,7 +8,6 @@ log.debug = log.verbose; log.disableColor(); var util = require('util'); var Bitcore = require('bitcore-lib'); - var mongodb = require('mongodb'); var Model = require('./model'); @@ -551,7 +550,7 @@ Storage.prototype.fetchAddressByCoin = function(coin, address, cb) { if (!result || _.isEmpty(result)) return cb(); if (result.length > 1) { result = _.find(result, function(address) { - return coin == (address.coin || Defaults.COIN); + return coin == (address.coin || 'btc'); }); } else { result = _.first(result); @@ -1072,45 +1071,47 @@ Storage.prototype._addressHash = function(addresses) { return Bitcore.crypto.Hash.ripemd160(new Buffer(all)).toString('hex'); }; -Storage.prototype.checkAndUseBalanceCache = function(addresses, cb) { - var key = ths._addressHash(addresses); +Storage.prototype.checkAndUseBalanceCache = function(walletId, addresses, cb) { + var self = this; + var key = self._addressHash(addresses); var now = Date.now(); + var BALANCE_CACHE_DURATION = 10; + self.db.collection(collections.CACHE).findOne({ - walletId: walletId, + walletId: walletId || key, type: 'balanceCache', key: key, }, function(err, ret) { if (err) return cb(err); if (!ret) return cb(); - var validFor = ret.ts + Defauls.BALANCE_CACHE_DURATION * 1000 - now; + var validFor = ret.ts + 10 * 1000 - now; if (validFor > 0) { log.debug('','Using Balance Cache valid for %d ms more', validFor); - return cb(null, ret.result); + cb(null, ret.result); + return true; } - - return cb(); + cb(); log.debug('','Balance cache expired, deleting'); self.db.collection(collections.CACHE).remove({ walletId: walletId, type: 'balanceCache', key: key, - }, {}, function() { - }); + }, {}, function() {}); + + return false; }); -}); +}; - - -Storage.prototype.storeBalanceCache = function (addresses, balance, cb) { - var key = ths._addressHash(addresses); +Storage.prototype.storeBalanceCache = function (walletId, addresses, balance, cb) { + var key = this._addressHash(addresses); var now = Date.now(); - self.db.collection(collections.CACHE).update( { - walletId: walletId, + this.db.collection(collections.CACHE).update({ + walletId: walletId || key, type: 'balanceCache', key: key, }, { diff --git a/test/integration/server.js b/test/integration/server.js index bfc9889..ee0860a 100644 --- a/test/integration/server.js +++ b/test/integration/server.js @@ -2617,6 +2617,90 @@ describe('Wallet service', function() { }); + + describe('#getBalance fast cache', function() { + var server, wallet, clock; + var _old = Defaults.BALANCE_CACHE_ADDRESS_THRESOLD; + beforeEach(function(done) { + clock = sinon.useFakeTimers(Date.now(), 'Date'); + Defaults.BALANCE_CACHE_ADDRESS_THRESOLD = 0; + + helpers.createAndJoinWallet(1, 1, function(s, w) { + server = s; + wallet = w; + done(); + }); + }); + afterEach(function() { + clock.restore(); + Defaults.BALANCE_CACHE_ADDRESS_THRESOLD = _old; + }); + + function checkBalance(balance) { + should.exist(balance); + balance.totalAmount.should.equal(helpers.toSatoshi(6)); + should.exist(balance.byAddress); + balance.byAddress.length.should.equal(2); + balance.byAddress[1].amount.should.equal(helpers.toSatoshi(2)); + }; + + it('should get balance from insight and store cache', function(done) { + helpers.stubUtxos(server, wallet, [1, 'u2', 3], function() { + server.getBalance({ + twoStep: false + }, function(err, balance, cacheUsed) { + should.not.exist(err); + should.not.exist(cacheUsed); + checkBalance(balance); + done(); + }); + }); + }); + + it('should get balance from cache', function(done) { + helpers.stubUtxos(server, wallet, [1, 'u2', 3], function() { + server.getBalance({ + twoStep: false + }, function(err, balance, cacheUsed) { + should.not.exist(err); + should.not.exist(cacheUsed); + server.getBalance({ + twoStep: false + }, function(err, balance, cacheUsed) { + should.not.exist(err); + cacheUsed.should.equal(true); + checkBalance(balance); + done(); + }); + }); + }); + }); + + + it('should not get balance from cache, after 11secs', function(done) { + helpers.stubUtxos(server, wallet, [1, 'u2', 3], function() { + server.getBalance({ + twoStep: false + }, function(err, balance, cacheUsed) { + should.not.exist(err); + should.not.exist(cacheUsed); + clock.tick((10+1) * 1000); + server.getBalance({ + twoStep: false + }, function(err, balance, cacheUsed) { + should.not.exist(err); + should.not.exist(cacheUsed); + checkBalance(balance); + done(); + }); + }); + }); + }); + + + }); + + describe('#getFeeLevels', function() { var server, wallet, levels; before(function() { From 23278a06fb7d84049df1eeac9b9ca28ba9abc1b5 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 2 Nov 2017 13:35:39 -0300 Subject: [PATCH 3/4] skip doing 2 queries if all addresses are active (only in new imported wallets, mainly) --- lib/server.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/server.js b/lib/server.js index f882dea..c3002bb 100644 --- a/lib/server.js +++ b/lib/server.js @@ -1311,6 +1311,12 @@ WalletService.prototype._getBalanceOneStep = function(opts, cb) { self.storage.fetchAddresses(self.walletId, function(err, addresses) { if (err) return cb(err); + + if (addresses.length == opts.alreadyQueriedLength) { + log.info('Query Skipped, all active addresses'); + return cb(null,null, true); + } + self._getBalanceFromAddresses({ coin: opts.coin, addresses: addresses @@ -1435,13 +1441,16 @@ WalletService.prototype.getBalance = function(opts, cb, i) { setTimeout(function() { log.debug('Running full balance query'); - self._getBalanceOneStep(opts, function(err, fullBalance) { + opts.alreadyQueriedLength = activeAddresses.length; + self._getBalanceOneStep(opts, function(err, fullBalance, skipped) { if (err) return; - if (!_.isEqual(partialBalance, fullBalance)) { + if (!skipped && !_.isEqual(partialBalance, fullBalance)) { log.info('Balance in active addresses differs from final balance'); self._notify('BalanceUpdated', fullBalance, { isGlobal: true }); + } else if (skipped) { + return; } else { // updates cache twoStepCache.lastEmpty = now; From 5c2285232849e2d6290c9e56172788a427e16f83 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 2 Nov 2017 14:31:56 -0300 Subject: [PATCH 4/4] better tests --- lib/common/defaults.js | 3 +++ lib/server.js | 11 ++++++++--- lib/storage.js | 5 ++--- test/integration/server.js | 25 ++++++++++++++++++++++--- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/lib/common/defaults.js b/lib/common/defaults.js index 4455695..ff14564 100644 --- a/lib/common/defaults.js +++ b/lib/common/defaults.js @@ -99,6 +99,9 @@ Defaults.HISTORY_CACHE_ADDRESS_THRESOLD = 100; // Number of addresses from which balance in cache for a few seconds Defaults.BALANCE_CACHE_ADDRESS_THRESOLD = Defaults.HISTORY_CACHE_ADDRESS_THRESOLD; +Defaults.BALANCE_CACHE_DIRECT_DURATION = 60; +Defaults.BALANCE_CACHE_DURATION = 10; + // Cache time for blockchain height (in seconds) Defaults.BLOCKHEIGHT_CACHE_TIME = 10 * 60; diff --git a/lib/server.js b/lib/server.js index c3002bb..0aa4c18 100644 --- a/lib/server.js +++ b/lib/server.js @@ -1248,10 +1248,10 @@ WalletService.prototype._getBalanceFromAddresses = function(opts, cb, i) { function checkBalanceCache(cb) { - if (opts.addresses.length < Defaults.BALANCE_CACHE_ADDRESS_THRESOLD) + if (opts.addresses.length < Defaults.BALANCE_CACHE_ADDRESS_THRESOLD || !opts.fastCache) return cb(); - self.storage.checkAndUseBalanceCache(self.walletId, opts.addresses, cb); + self.storage.checkAndUseBalanceCache(self.walletId, opts.addresses, opts.fastCache, cb); }; function storeBalanceCache(balance, cb) { @@ -1319,7 +1319,8 @@ WalletService.prototype._getBalanceOneStep = function(opts, cb) { self._getBalanceFromAddresses({ coin: opts.coin, - addresses: addresses + addresses: addresses, + fastCache: opts.fastCache, }, function(err, balance, cacheUsed) { if (err) return cb(err); @@ -1403,6 +1404,7 @@ WalletService.prototype.getBalance = function(opts, cb, i) { } if (!opts.twoStep) { + opts.fastCache = Defaults.BALANCE_CACHE_DIRECT_DURATION; return self._getBalanceOneStep(opts, cb); } @@ -1441,7 +1443,10 @@ WalletService.prototype.getBalance = function(opts, cb, i) { setTimeout(function() { log.debug('Running full balance query'); + opts.alreadyQueriedLength = activeAddresses.length; + opts.fastCache = Defaults.BALANCE_CACHE_DURATION; + self._getBalanceOneStep(opts, function(err, fullBalance, skipped) { if (err) return; if (!skipped && !_.isEqual(partialBalance, fullBalance)) { diff --git a/lib/storage.js b/lib/storage.js index 15f17a7..feadde3 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -1071,11 +1071,10 @@ Storage.prototype._addressHash = function(addresses) { return Bitcore.crypto.Hash.ripemd160(new Buffer(all)).toString('hex'); }; -Storage.prototype.checkAndUseBalanceCache = function(walletId, addresses, cb) { +Storage.prototype.checkAndUseBalanceCache = function(walletId, addresses, duration, cb) { var self = this; var key = self._addressHash(addresses); var now = Date.now(); - var BALANCE_CACHE_DURATION = 10; self.db.collection(collections.CACHE).findOne({ @@ -1086,7 +1085,7 @@ Storage.prototype.checkAndUseBalanceCache = function(walletId, addresses, cb) { if (err) return cb(err); if (!ret) return cb(); - var validFor = ret.ts + 10 * 1000 - now; + var validFor = ret.ts + duration * 1000 - now; if (validFor > 0) { log.debug('','Using Balance Cache valid for %d ms more', validFor); diff --git a/test/integration/server.js b/test/integration/server.js index ee0860a..783eb77 100644 --- a/test/integration/server.js +++ b/test/integration/server.js @@ -2677,14 +2677,14 @@ describe('Wallet service', function() { }); - it('should not get balance from cache, after 11secs', function(done) { + it('should not get balance from cache, after X secs, on a direct hit', function(done) { helpers.stubUtxos(server, wallet, [1, 'u2', 3], function() { server.getBalance({ twoStep: false }, function(err, balance, cacheUsed) { should.not.exist(err); should.not.exist(cacheUsed); - clock.tick((10+1) * 1000); + clock.tick(( Defaults.BALANCE_CACHE_DIRECT_DURATION +1) * 1000); server.getBalance({ twoStep: false }, function(err, balance, cacheUsed) { @@ -2698,6 +2698,25 @@ describe('Wallet service', function() { }); + it('should not get balance from cache, after X secs, on a twostep hit', function(done) { + helpers.stubUtxos(server, wallet, [1, 'u2', 3], function() { + server.getBalance({ + twoStep: false + }, function(err, balance, cacheUsed) { + should.not.exist(err); + should.not.exist(cacheUsed); + clock.tick(( Defaults.BALANCE_CACHE_DIRECT_DURATION - 1) * 1000); + server.getBalance({ + twoStep: true + }, function(err, balance, cacheUsed) { + should.not.exist(err); + should.not.exist(cacheUsed); + checkBalance(balance); + done(); + }); + }); + }); + }); }); @@ -7579,7 +7598,7 @@ describe('Wallet service', function() { function(server, wallet, next) { server.scan({}, function(err) { should.not.exist(err); - server.getBalance(wallet.id, function(err, balance) { + server.getBalance({}, function(err, balance) { balance.totalAmount.should.equal(helpers.toSatoshi(6)); next(); })