diff --git a/lib/common/defaults.js b/lib/common/defaults.js index d386345..7706a50 100644 --- a/lib/common/defaults.js +++ b/lib/common/defaults.js @@ -136,7 +136,5 @@ Defaults.RateLimit = { }; Defaults.COIN = 'btc'; - -Defaults.INSIGHT_REQUEST_POOL_SIZE = 20; - +Defaults.INSIGHT_REQUEST_POOL_SIZE = 10; module.exports = Defaults; diff --git a/lib/server.js b/lib/server.js index 22d719b..35c10cd 100644 --- a/lib/server.js +++ b/lib/server.js @@ -507,7 +507,6 @@ WalletService.prototype.getWalletFromIdentifier = function(opts, cb) { /** * Retrieves wallet status. * @param {Object} opts - * @param {Object} opts.twoStep[=false] - Optional: use 2-step balance computation for improved performance * @param {Object} opts.includeExtendedInfo - Include PKR info & address managers for wallet & copayers * @returns {Object} status */ @@ -945,6 +944,8 @@ WalletService.prototype._canCreateAddress = function(ignoreMaxGap, cb) { if (ignoreMaxGap) return cb(null, true); + + // TODO: improve performace... do not got to bc because of this! self.storage.fetchAddresses(self.walletId, function(err, addresses) { if (err) return cb(err); var latestAddresses = _.takeRight(_.reject(addresses, { @@ -1005,6 +1006,7 @@ WalletService.prototype.createAddress = function(opts, cb) { }; function getFirstAddress(wallet, cb) { + // TODO pull only last one! self.storage.fetchAddresses(self.walletId, function(err, addresses) { if (err) return cb(err); if (!_.isEmpty(addresses)) return cb(null, _.first(addresses)) @@ -1096,134 +1098,202 @@ WalletService.prototype._getBlockchainExplorer = function(coin, network) { return bc; }; -WalletService.prototype._getUtxos = function(coin, addresses, cb) { +WalletService.prototype._getUtxos = function(coin, opts, cb) { + var addresses = opts.addresses || []; + $.checkArgument(coin); var self = this; - if (addresses.length == 0) return cb(null, []); + function checkUtxoCache(addresses, next) { + if (addresses.length < Defaults.UTXO_CACHE_ADDRESS_THRESOLD || !opts.fastCache) + return next(); + + self.storage.checkAndUseUtxoCache(self.walletId, addresses, opts.fastCache, next); + }; + + function storeUtxoCache(addresses, utxos, next) { + if (addresses.length < Defaults.UTXO_CACHE_ADDRESS_THRESOLD) + return next(null, utxos); + + self.storage.storeUtxoCache(self.walletId, addresses, utxos, function(err) { + if (err) + self.logw('Could not save cache:',err); + + return next(null, utxos); + }); + }; + var networkName = Bitcore_[coin].Address(addresses[0]).toObject().network; var bc = self._getBlockchainExplorer(coin, networkName); if (!bc) return cb(new Error('Could not get blockchain explorer instance')); - self.logi('','Querying utxos: %s addrs', addresses.length); + self.logi('Querying utxos addrs --:' + addresses.length); - bc.getUtxos(addresses, function(err, utxos) { + checkUtxoCache(addresses, function(err, cache) { if (err) return cb(err); - var utxos = _.map(utxos, function(utxo) { - var u = _.pick(utxo, ['txid', 'vout', 'address', 'scriptPubKey', 'amount', 'satoshis', 'confirmations']); - u.confirmations = u.confirmations || 0; - u.locked = false; - u.satoshis = _.isNumber(u.satoshis) ? +u.satoshis : Utils.strip(u.amount * 1e8); - delete u.amount; - return u; - }); + if (cache) { + self.logi('Using UTXO Cache'); + return cb(null, cache, true); + } - return cb(null, utxos); + bc.getUtxos(addresses, function(err, utxos) { + if (err) return cb(err); + + var utxos = _.map(utxos, function(utxo) { + var u = _.pick(utxo, ['txid', 'vout', 'address', 'scriptPubKey', 'amount', 'satoshis', 'confirmations']); + u.confirmations = u.confirmations || 0; + u.locked = false; + u.satoshis = _.isNumber(u.satoshis) ? +u.satoshis : Utils.strip(u.amount * 1e8); + delete u.amount; + return u; + }); + + return storeUtxoCache(addresses, utxos, cb); + }); }); }; + +/** + * Get UTXOs from a wallet removing pending utxos from pending TXs, recently broadcasted, etc. + * @param {Object} opts + * @param {string} opts.alternateCoin - Use coin instead of wallet's coin + * @param {Array} opts.addresses (optional) - Pull Utxos from the given subset of wallet's addr + * @returns {Address[]} + */ + WalletService.prototype._getUtxosForCurrentWallet = function(opts, cb) { var self = this; - var opts = opts || {}; function utxoKey(utxo) { return utxo.txid + '|' + utxo.vout }; - var coin, allAddresses, allUtxos, utxoIndex, addressStrs; + var coin, allAddresses, addressObj, allUtxos, utxoIndex, addressStrs, fromCache; - async.series([ - function(next) { - self.getWallet({}, function(err, wallet) { - if (err) return next(err); + function conditionalLock(main) { + if (opts.noLock) return main(cb); + return self._runLocked(cb, main); + }; - coin = wallet.coin; - return next(); - }); - }, - function(next) { - if (_.isArray(opts.addresses)) { - allAddresses = opts.addresses; - return next(); - } - self.storage.fetchAddresses(self.walletId, function(err, addresses) { - allAddresses = addresses; - if (allAddresses.length == 0) return cb(null, []); + // to prevent server overload + conditionalLock(function (cb) { + async.series([ + function(next) { - return next(); - }); - }, - function(next) { - addressStrs = _.map(allAddresses, 'address'); - if (!opts.coin) return next(); + if (_.isArray(opts.addresses)) { + $.checkState(_.isObject(opts.addresses[0])); + allAddresses = opts.addresses; + return next(); + } - coin = opts.coin; - addressStrs = _.map(addressStrs, function(a) { - return Utils.translateAddress(a, coin); - }); - next(); - }, - function(next) { - self._getUtxos(coin, addressStrs, function(err, utxos) { - if (err) return next(err); - - if (utxos.length == 0) return cb(null, []); - allUtxos = utxos; - utxoIndex = _.indexBy(allUtxos, utxoKey); - return next(); - }); - }, - function(next) { - self.getPendingTxs({}, function(err, txps) { - if (err) return next(err); - - var lockedInputs = _.map(_.flatten(_.map(txps, 'inputs')), utxoKey); - _.each(lockedInputs, function(input) { - if (utxoIndex[input]) { - utxoIndex[input].locked = true; + self._getAliveAddresses(function(err, addresses) { + allAddresses = addresses; + if (!allAddresses || !allAddresses.length) { + return cb(null, []); + } else { + return next(); } }); + }, + function(next) { + var localCoin = Utils.getAddressCoin(allAddresses[0].address); + addressObj = _.indexBy(allAddresses, 'address'); + addressStrs = _.map(allAddresses, 'address'); + + + if (opts.alternateCoin && localCoin != opts.alternateCoin) { + self.logi('Translating addresses to '+ opts.alternateCoin); + addressStrs = _.map(addressStrs, function(a) { + return Utils.translateAddress(a, opts.alternateCoin); + }); + } + + coin = opts.alternateCoin || localCoin; return next(); - }); - }, - function(next) { - var now = Math.floor(Date.now() / 1000); - // Fetch latest broadcasted txs and remove any spent inputs from the - // list of UTXOs returned by the block explorer. This counteracts any out-of-sync - // effects between broadcasting a tx and getting the list of UTXOs. - // This is especially true in the case of having multiple instances of the block explorer. - self.storage.fetchBroadcastedTxs(self.walletId, { - minTs: now - 24 * 3600, - limit: 100 - }, function(err, txs) { - if (err) return next(err); - var spentInputs = _.map(_.flatten(_.map(txs, 'inputs')), utxoKey); - _.each(spentInputs, function(input) { - if (utxoIndex[input]) { - utxoIndex[input].spent = true; - } + }, + + function(next) { + self._getUtxos(coin, { + addresses: addressStrs, + fastCache: Defaults.UTXO_CACHE_DURATION, + }, function(err, utxos, inFromCache) { + if (err) return next(err); + + if (utxos.length == 0) + return cb(null, [], inFromCache, addressStrs.length); + + allUtxos = utxos; + utxoIndex = _.indexBy(allUtxos, utxoKey); + fromCache = inFromCache; + return next(); }); - allUtxos = _.reject(allUtxos, { - spent: true + }, + + function(next) { + if (fromCache) return next(); + + + // Update addresses with utxo + var withUtxo = _.uniq(_.map(allUtxos, 'address')); + var withUtxoObj = _.map(withUtxo, function(x){ + return addressObj[x]; + }); + + self.storage.updateAddressesWithUtxo(self.walletId, withUtxoObj, {}, next); + }, + + function(next) { + self.getPendingTxs({}, function(err, txps) { + if (err) return next(err); + + var lockedInputs = _.map(_.flatten(_.map(txps, 'inputs')), utxoKey); + _.each(lockedInputs, function(input) { + if (utxoIndex[input]) { + utxoIndex[input].locked = true; + } + }); + return next(); + }); + }, + function(next) { + var now = Math.floor(Date.now() / 1000); + // Fetch latest broadcasted txs and remove any spent inputs from the + // list of UTXOs returned by the block explorer. This counteracts any out-of-sync + // effects between broadcasting a tx and getting the list of UTXOs. + // This is especially true in the case of having multiple instances of the block explorer. + self.storage.fetchBroadcastedTxs(self.walletId, { + minTs: now - 24 * 3600, + limit: 100 + }, function(err, txs) { + if (err) return next(err); + var spentInputs = _.map(_.flatten(_.map(txs, 'inputs')), utxoKey); + _.each(spentInputs, function(input) { + if (utxoIndex[input]) { + utxoIndex[input].spent = true; + } + }); + allUtxos = _.reject(allUtxos, { + spent: true + }); + return next(); + }); + }, + function(next) { + if (!addressObj) return next(); + // Needed for the clients to sign UTXOs + _.each(allUtxos, function(utxo) { + utxo.path = addressObj[utxo.address].path; + utxo.publicKeys = addressObj[utxo.address].publicKeys; }); return next(); - }); - }, - function(next) { - if (opts.coin) return next(); - // Needed for the clients to sign UTXOs - var addressToPath = _.indexBy(allAddresses, 'address'); - _.each(allUtxos, function(utxo) { - utxo.path = addressToPath[utxo.address].path; - utxo.publicKeys = addressToPath[utxo.address].publicKeys; - }); - return next(); - }, - ], function(err) { - return cb(err, allUtxos); + }, + ], function(err) { + return cb(err, allUtxos, fromCache, allAddresses.length); + }); }); }; @@ -1233,6 +1303,7 @@ WalletService.prototype._getUtxosForCurrentWallet = function(opts, cb) { * @param {String} [opts.coin='btc'] (optional) * @param {Array} opts.addresses (optional) - List of addresses from where to fetch UTXOs. * @returns {Array} utxos - List of UTXOs. + * */ WalletService.prototype.getUtxos = function(opts, cb) { var self = this; @@ -1246,10 +1317,14 @@ WalletService.prototype.getUtxos = function(opts, cb) { if (_.isUndefined(opts.addresses)) { self._getUtxosForCurrentWallet({ - coin: opts.coin + fastCache: Defaults.UTXO_CACHE_DIRECT_DURATION, }, cb); } else { - self._getUtxos(Utils.getAddressCoin(opts.addresses[0]), opts.addresses, cb); + + if (opts.addresses.length > Defaults.MAX_ADDRS_UTXO ) + return cb(new ClientError('Too many addresses in query')); + + self._getUtxos(Utils.getAddressCoin(opts.addresses[0]), opts, cb); } }; @@ -1267,146 +1342,130 @@ WalletService.prototype._totalizeUtxos = function(utxos) { }; -WalletService.prototype._getBalanceFromAddresses = function(opts, cb, i) { +WalletService.prototype._getBalanceFromCurrentWallet = function(opts, cb, i) { var self = this; var opts = opts || {}; opts.addresses = opts.addresses || []; - - function checkBalanceCache(cb) { - if (opts.addresses.length < Defaults.BALANCE_CACHE_ADDRESS_THRESOLD || !opts.fastCache) - return cb(); - - self.storage.checkAndUseBalanceCache(self.walletId, opts.addresses, opts.fastCache, cb); - }; - - function storeBalanceCache(balance, cb) { - if (opts.addresses.length < Defaults.BALANCE_CACHE_ADDRESS_THRESOLD) - return cb(null, balance); - - self.storage.storeBalanceCache(self.walletId, opts.addresses, balance, function(err) { - if (err) - self.logw('Could not save cache:',err); - - return cb(null, balance); - }); - }; - - // This lock is to prevent server starvation on big wallets - self._runLocked(cb, function(cb) { - checkBalanceCache(function(err, cache) { - if (err) return cb(err); - - if (cache) { - self.logi('Using UTXO Cache'); - return cb(null, cache, true); - } - - self._getUtxosForCurrentWallet({ - coin: opts.coin, - addresses: opts.addresses - }, function(err, utxos) { - if (err) return cb(err); - - var balance = self._totalizeUtxos(utxos); - - // Compute balance by address - var byAddress = {}; - _.each(_.indexBy(_.sortBy(utxos, 'address'), 'address'), function(value, key) { - byAddress[key] = { - address: key, - path: value.path, - amount: 0, - }; - }); - - _.each(utxos, function(utxo) { - byAddress[utxo.address].amount += utxo.satoshis; - }); - - balance.byAddress = _.values(byAddress); - - storeBalanceCache(balance, cb); - }); - }); - }); -}; - -WalletService.prototype._getBalanceOneStep = function(opts, cb) { - var self = this; - - self.storage.fetchAddresses(self.walletId, function(err, addresses) { - if (err) return cb(err); - - if (addresses.length == opts.alreadyQueriedLength) { - self.logi('Query Skipped, all active addresses'); - return cb(null,null, true); - } - - self._getBalanceFromAddresses({ - coin: opts.coin, - addresses: addresses, - fastCache: opts.fastCache, - }, function(err, balance, cacheUsed) { - if (err) return cb(err); - - // Update cache - var withBalance = _.map(balance.byAddress, 'address') - self.storage.storeAddressesWithBalance(self.walletId, withBalance, function(err) { - if (err) { - self.logw('Could not update wallet cache', err); - } - return cb(null, balance, cacheUsed); - }); - }); - }); -}; - - -WalletService.prototype._getActiveAddresses = function(cb) { - var self = this; - - self.storage.fetchAddressesWithBalance(self.walletId, function(err, addressesWB) { - if (err) { - self.logw('Could not fetch active addresses from cache', err); - return cb(); - } - if (!_.isArray(addressesWB)) - addressesWB = []; - - var now = Math.floor(Date.now() / 1000); - var fromTs = now - Defaults.TWO_STEP_CREATION_HOURS * 3600; - - self.storage.fetchNewAddresses(self.walletId, fromTs, function(err, recent) { - if (err) return cb(err); - - var result = _.uniq(_.union(addressesWB, recent), 'address'); - return cb(null, result); - }); - }); -}; - -WalletService.prototype._checkAndUpdateAddressCount = function(twoStepCache, cb) { - var self = this; - - if (twoStepCache.addressCount > Defaults.TWO_STEP_BALANCE_THRESHOLD) { - self.logi('Not counting addresses'); - return cb(null, true); - } - - self.storage.countAddresses(self.walletId, function(err, addressCount) { + self._getUtxosForCurrentWallet({ + fastCache: Defaults.UTXO_CACHE_DURATION, + alternateCoin: opts.alternateCoin, + }, function(err, utxos, fromCache, usedAddresses) { if (err) return cb(err); - if (addressCount < Defaults.TWO_STEP_BALANCE_THRESHOLD) - return cb(null, false); + var balance = self._totalizeUtxos(utxos); - twoStepCache.addressCount = addressCount; + // Compute balance by address + var byAddress = {}; + _.each(_.indexBy(_.sortBy(utxos, 'address'), 'address'), function(value, key) { + byAddress[key] = { + address: key, + path: value.path, + amount: 0, + }; + }); - // updates cache - self.storage.storeTwoStepCache(self.walletId, twoStepCache, function(err) { + _.each(utxos, function(utxo) { + byAddress[utxo.address].amount += utxo.satoshis; + }); + + balance.byAddress = _.values(byAddress); + + // fromCache, and finalAddrNr are mainly for tests + return cb(null, balance, fromCache, usedAddresses); + }); +}; + + +/* + + HAD balance in the last TIME and is main + + HAD balance in the last TIME2 and is change + 3. addr = addr + [ addr main && createdOn > now - TIME ] + 3. addr = addr + [ addr change && createdOn > now - TIME2 ] + */ + +WalletService.prototype._getAliveAddresses = function(cb) { + var self = this; + + self._checkAndUpdateAddressCacheStatus(function(err, da_active ) { + if (err) return cb(err); + + if (!da_active) { + self.logi(' * Small wallet, querying full address set'); + return self.storage.fetchAddresses(self.walletId, cb); + } + + var addressWB = []; + async.parallel([ + function(next) { + self.storage.fetchAddressesWithUtxo(self.walletId, function(err, res) { + if (err) { + self.logw('Could not fetch active addresses from cache', err); + return next(err); + } +// self.logi(' * Adding addresses with recent balance:', res.length); + return next(null, res); + }); + }, + // Recent addresses + function(next) { + // Get all types of addresses from main ts. + self.storage.fetchNewAddresses(self.walletId, function(err, recent, nrMain, nrChange) { + if (err) return next(err); + recent = recent || []; + +// self.logi(' * Adding recent main addresses:', nrMain); +// self.logi(' * Adding recent change addresses:', nrChange); + + return next(null, recent); + }); + }, + ], function(err, res) { if (err) return cb(err); + if (!res[0].length) { + self.logi(' * No prev utxo info. Querying ALL addresses'); + return self.storage.fetchAddresses(self.walletId, cb); + } + + // TODO: warning with lodash 4: replace with 'uniqBy' keys: _.contains _.all + res = _.uniq(_.compact(_.flatten(res)), 'address'); + self.logi('Total addresses alive : ' + res.length); + + if (_.isEmpty(res)) + return cb(); + + return cb(null, res); + }); + }); +}; + +WalletService.prototype._checkAndUpdateAddressCacheStatus = function(cb) { + var self = this; + + + self.storage.getAddressCacheStatus(self.walletId, function(err, cacheStatus) { + cacheStatus = cacheStatus || {}; + + if (cacheStatus.addressCount > Defaults.DA_MIN_ADDR) { + self.logi('Not counting addresses'); return cb(null, true); + } + + self.storage.countAddresses(self.walletId, function(err, addressCount) { + if (err) return cb(err); + + if (addressCount < Defaults.DA_MIN_ADDR) + return cb(null, false); + + cacheStatus.addressCount = addressCount; + + // updates cache + self.storage.storeAddressCacheStatus(self.walletId, cacheStatus, function(err) { + if (err) return cb(err); + + return cb(null, true); + }); }); }); }; @@ -1415,13 +1474,11 @@ WalletService.prototype._checkAndUpdateAddressCount = function(twoStepCache, cb) * Get wallet balance. * @param {Object} opts * @param {string} [opts.coin] - Override wallet coin (default wallet's coin). - * @param {Boolean} opts.twoStep[=false] - Optional - Use 2 step balance computation for improved performance * @returns {Object} balance - Total amount & locked amount. */ WalletService.prototype.getBalance = function(opts, cb, i) { var self = this; - opts = opts || {}; if (opts.coin) { @@ -1429,77 +1486,9 @@ WalletService.prototype.getBalance = function(opts, cb, i) { return cb(new ClientError('Invalid coin')); } - if (!opts.twoStep) { - opts.fastCache = Defaults.BALANCE_CACHE_DIRECT_DURATION; - return self._getBalanceOneStep(opts, cb); - } - - self.storage.getTwoStepCache(self.walletId, function(err, twoStepCache) { - if (err) return cb(err); - twoStepCache = twoStepCache || {}; - - self._checkAndUpdateAddressCount(twoStepCache, function(err, needsTwoStep ) { - if (err) return cb(err); - - if (!needsTwoStep) { - return self._getBalanceOneStep(opts, cb); - } - - self._getActiveAddresses(function(err, activeAddresses) { - if (err) return cb(err); - if (!_.isArray(activeAddresses)) { - return self._getBalanceOneStep(opts, cb); - } else { - self.logi('Requesting partial balance for ' + activeAddresses.length + ' addresses'); - - self._getBalanceFromAddresses({ - coin: opts.coin, - addresses: activeAddresses - }, function(err, partialBalance, cacheUsed) { - if (err) return cb(err); - cb(null, partialBalance, cacheUsed); - - var now = Math.floor(Date.now() / 1000); - - if (twoStepCache.lastEmpty > now - Defaults.TWO_STEP_INACTIVE_CLEAN_DURATION_MIN * 60 ) { - self.logi('Not running the FULL balance query due to TWO_STEP_INACTIVE_CLEAN_DURATION_MIN '); - return; - } - - - setTimeout(function() { - self.logi('Running full balance query'); - - opts.alreadyQueriedLength = activeAddresses.length; - opts.fastCache = Defaults.BALANCE_CACHE_DURATION; - - self._getBalanceOneStep(opts, function(err, fullBalance, skipped) { - if (err) return; - if (!skipped && !_.isEqual(partialBalance, fullBalance)) { - self.logi('Balance in active addresses differs from final balance'); - self._notify('BalanceUpdated', fullBalance, { - isGlobal: true - }); - } else if (skipped) { - return; - } else { - // updates cache - twoStepCache.lastEmpty = now; - - // updates cache - return self.storage.storeTwoStepCache(self.walletId, twoStepCache, function(err) { - return; - }); - - } - }); - }, 1); - return; - }, i); - } - }); - }); - }); + return self._getBalanceFromCurrentWallet({ + alternateCoin: opts.coin, + }, cb); }; /** @@ -1541,6 +1530,7 @@ WalletService.prototype.getSendMaxInfo = function(opts, cb) { return cb(new ClientError('Invalid fee per KB')); } + self._getUtxosForCurrentWallet({}, function(err, utxos) { if (err) return cb(err); @@ -1673,58 +1663,77 @@ WalletService.prototype.getFeeLevels = function(opts, cb) { if (!Utils.checkValueInCollection(opts.network, Constants.NETWORKS)) return cb(new ClientError('Invalid network')); - var feeLevels = Defaults.FEE_LEVELS[opts.coin]; - - function samplePoints() { - var definedPoints = _.uniq(_.map(feeLevels, 'nbBlocks')); - return _.uniq(_.flatten(_.map(definedPoints, function(p) { - return _.range(p, p + Defaults.FEE_LEVELS_FALLBACK + 1); - }))); + + function checkAndUseFeeLevelsCache (next) { + self.storage.checkAndUseFeeLevelsCache(opts, next); }; - function getFeeLevel(feeSamples, level, n, fallback) { - var result; + function storeFeeLevelsCache(values,next) { + self.storage.storeFeeLevelsCache(opts, values, next); + }; - if (feeSamples[n] >= 0) { - result = { - nbBlocks: n, - feePerKb: feeSamples[n], - }; - } else { - if (fallback > 0) { - result = getFeeLevel(feeSamples, level, n + 1, fallback - 1); - } else { + checkAndUseFeeLevelsCache(function(err, values) { + if (err) return cb(err); + if (values) return cb(null, values, true); + + var feeLevels = Defaults.FEE_LEVELS[opts.coin]; + function samplePoints() { + var definedPoints = _.uniq(_.map(feeLevels, 'nbBlocks')); + return _.uniq(_.flatten(_.map(definedPoints, function(p) { + return _.range(p, p + Defaults.FEE_LEVELS_FALLBACK + 1); + }))); + }; + + function getFeeLevel(feeSamples, level, n, fallback) { + var result; + + if (feeSamples[n] >= 0) { result = { - feePerKb: level.defaultValue, - nbBlocks: null, + nbBlocks: n, + feePerKb: feeSamples[n], }; - } - } - return result; - }; - - self._sampleFeeLevels(opts.coin, opts.network, samplePoints(), function(err, feeSamples) { - var values = _.map(feeLevels, function(level) { - var result = { - level: level.name, - }; - if (err) { - result.feePerKb = level.defaultValue; - result.nbBlocks = null; } else { - var feeLevel = getFeeLevel(feeSamples, level, level.nbBlocks, Defaults.FEE_LEVELS_FALLBACK); - result.feePerKb = +(feeLevel.feePerKb * (level.multiplier || 1)).toFixed(0); - result.nbBlocks = feeLevel.nbBlocks; + if (fallback > 0) { + result = getFeeLevel(feeSamples, level, n + 1, fallback - 1); + } else { + result = { + feePerKb: level.defaultValue, + nbBlocks: null, + }; + } } return result; + }; + + self._sampleFeeLevels(opts.coin, opts.network, samplePoints(), function(err, feeSamples) { + var values = _.map(feeLevels, function(level) { + var result = { + level: level.name, + }; + if (err) { + result.feePerKb = level.defaultValue; + result.nbBlocks = null; + } else { + var feeLevel = getFeeLevel(feeSamples, level, level.nbBlocks, Defaults.FEE_LEVELS_FALLBACK); + result.feePerKb = +(feeLevel.feePerKb * (level.multiplier || 1)).toFixed(0); + result.nbBlocks = feeLevel.nbBlocks; + } + return result; + }); + + // Ensure monotonically decreasing values + for (var i = 1; i < values.length; i++) { + values[i].feePerKb = Math.min(values[i].feePerKb, values[i - 1].feePerKb); + } + + storeFeeLevelsCache(values, function(err) { + if (err) { + log.warn('Could not store fee level cache'); + } + return cb(null, values); + }); + }); - - // Ensure monotonically decreasing values - for (var i = 1; i < values.length; i++) { - values[i].feePerKb = Math.min(values[i].feePerKb, values[i - 1].feePerKb); - } - - return cb(null, values); }); }; @@ -1922,8 +1931,9 @@ WalletService.prototype._selectTxInputs = function(txp, utxosToExclude, cb) { }; //log.debug('Selecting inputs for a ' + Utils.formatAmountInBtc(txp.getTotalAmount()) + ' txp'); - - self._getUtxosForCurrentWallet({}, function(err, utxos) { + self._getUtxosForCurrentWallet({ + noLock: true, + }, function(err, utxos) { if (err) return cb(err); var totalAmount; @@ -2350,10 +2360,11 @@ WalletService.prototype.publishTx = function(opts, cb) { txp.proposalSignaturePubKeySig = signingKey.signature; } + // Verify UTXOs are still available self._getUtxosForCurrentWallet({ addresses: txp.inputs, - coin: txp.coin, + noLock: true, }, function(err, utxos) { if (err) return cb(err); @@ -3119,7 +3130,7 @@ WalletService.prototype.getTxHistory = function(opts, cb) { var bc = self._getBlockchainExplorer(wallet.coin, wallet.network); if (!bc) return next(new Error('Could not get blockchain explorer instance')); - self.logi('','Querying tx for: %s addrs', addresses.length); + self.logi('Querying tx for addrs:' + addresses.length); bc.getTransactions(addressStrs, from, to, function(err, rawTxs, total) { if (err) return next(err); @@ -3339,9 +3350,7 @@ WalletService.prototype.scan = function(opts, cb) { if (err) return cb(err); wallet.scanStatus = error ? 'error' : 'success'; self.storage.storeWallet(wallet, function() { - self.storage.storeTwoStepCache(self.walletId, {}, function(err) { - return cb(error); - }); + return cb(error); }); }) }); diff --git a/lib/storage.js b/lib/storage.js index feadde3..f1be1b9 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -1125,5 +1125,48 @@ Storage.prototype.storeBalanceCache = function (walletId, addresses, balance, cb }, cb); }; +// FEE_LEVEL_DURATION = 5min +var FEE_LEVEL_DURATION = 5 * 60 * 1000; +Storage.prototype.checkAndUseFeeLevelsCache = function(opts, cb) { + var self = this; + var key = JSON.stringify(opts); + var now = Date.now(); + + self.db.collection(collections.CACHE).findOne({ + walletId: null, + type: 'feeLevels', + key: key, + }, function(err, ret) { + if (err) return cb(err); + if (!ret) return cb(); + + var validFor = ret.ts + FEE_LEVEL_DURATION - now; + return cb(null, validFor > 0 ? ret.result : null); + }); +}; + + +Storage.prototype.storeFeeLevelsCache = function (opts, values, cb) { + var key = JSON.stringify(opts); + var now = Date.now(); + this.db.collection(collections.CACHE).update({ + walletId: null, + type: 'feeLevels', + key: key, + }, { + "$set": + { + ts: now, + result: values, + } + }, { + w: 1, + upsert: true, + }, cb); +}; + + + + Storage.collections = collections; module.exports = Storage; diff --git a/scripts/deleteWallet.mongo b/scripts/deleteWallet.mongo new file mode 100644 index 0000000..c6228c3 --- /dev/null +++ b/scripts/deleteWallet.mongo @@ -0,0 +1,17 @@ + +a='bd0e9eed-4712-42e5-bb21-193cde4c9e21'; +b= {'walletId':a}; + +db.addresses.remove(b); +db.cache.remove(b); +db.copayers_lookup.remove(b); +db.notifications.remove(b); +db.preferences.remove(b); +db.sessions.remove(b); +db.tx_confirmation_subs.remove(b); +db.txs.remove(b); +db.tx_notes.remove(b); +db.wallets.remove({'id':a}); + +db.push_notification_subs.remove(b); +db.email_queue.remove(b); diff --git a/test/integration/server.js b/test/integration/server.js index 783eb77..5161ae2 100644 --- a/test/integration/server.js +++ b/test/integration/server.js @@ -1763,6 +1763,7 @@ describe('Wallet service', function() { should.not.exist(err); bal.totalAmount.should.equal(1e8); getAuthServer(opts.copayerId, reqPrivKey, function(err, server2) { + server2.getBalance(res.wallet.walletId, function(err, bal2) { should.not.exist(err); bal2.totalAmount.should.equal(1e8); @@ -2752,14 +2753,21 @@ describe('Wallet service', function() { after(function() { Defaults.FEE_LEVELS = levels; }); + var clock; beforeEach(function(done) { helpers.createAndJoinWallet(1, 1, function(s, w) { server = s; wallet = w; done(); }); + clock = sinon.useFakeTimers(Date.now(), 'Date'); }); + afterEach(function() { + clock.restore(); + }); + + it('should get current fee levels', function(done) { helpers.stubFeeLevels({ 1: 40000, @@ -2890,6 +2898,83 @@ describe('Wallet service', function() { done(); }); }); + + it('should get current fee levels FROM CACHE', function(done) { + helpers.stubFeeLevels({ + 1: 40000, + 2: 20000, + }); + server.getFeeLevels({}, function(err, fees, fromCache) { + should.not.exist(err); + fees = _.zipObject(_.map(fees, function(item) { + return [item.level, item]; + })); + fees.urgent.feePerKb.should.equal(60000); + fees.priority.feePerKb.should.equal(40000); + should.not.exist(fromCache); + server.getFeeLevels({}, function(err, fees, fromCache) { + should.not.exist(err); + fees = _.zipObject(_.map(fees, function(item) { + return [item.level, item]; + })); + fees.urgent.feePerKb.should.equal(60000); + fees.priority.feePerKb.should.equal(40000); + fromCache.should.equal(true); + done(); + }); + }); + }); + + + it('should expire CACHE', function(done) { + helpers.stubFeeLevels({ + 1: 40000, + 2: 20000, + }); + server.getFeeLevels({}, function(err, fees, fromCache) { + should.not.exist(err); + fees = _.zipObject(_.map(fees, function(item) { + return [item.level, item]; + })); + fees.urgent.feePerKb.should.equal(60000); + fees.priority.feePerKb.should.equal(40000); + should.not.exist(fromCache); + clock.tick(6*60*1000); + server.getFeeLevels({}, function(err, fees, fromCache) { + should.not.exist(err); + fees = _.zipObject(_.map(fees, function(item) { + return [item.level, item]; + })); + fees.urgent.feePerKb.should.equal(60000); + fees.priority.feePerKb.should.equal(40000); + should.not.exist(fromCache); + done(); + }); + }); + }); + + + it('should not use cache on different opts', function(done) { + helpers.stubFeeLevels({ + 1: 40000, + 2: 20000, + }); + server.getFeeLevels({}, function(err, fees, fromCache) { + should.not.exist(err); + should.not.exist(fromCache); + server.getFeeLevels({coin:'bch'}, function(err, fees, fromCache) { + should.not.exist(err); + should.not.exist(fromCache); + server.getFeeLevels({coin:'bch', network:'testnet'}, function(err, fees, fromCache) { + should.not.exist(err); + should.not.exist(fromCache); + done(); + }); + }); + }); + }); + + }); describe('Wallet not complete tests', function() {