diff --git a/lib/blockchainexplorers/insight.js b/lib/blockchainexplorers/insight.js index 614c68b..04a4ddb 100644 --- a/lib/blockchainexplorers/insight.js +++ b/lib/blockchainexplorers/insight.js @@ -41,6 +41,16 @@ Insight.prototype._doRequest = function(args, cb) { 'User-Agent': this.userAgent, } }; + + if (log.level == 'verbose') { + var s = JSON.stringify(args); + + if ( s.length > 100 ) + s= s.substr(0,100) + '...'; + + log.debug('', 'Insight Q: %s', s); + } + requestList(_.defaults(args, opts), cb); }; @@ -59,9 +69,10 @@ Insight.prototype.getUtxos = function(addresses, cb) { json: { addrs: _.uniq([].concat(addresses)).join(',') }, - timeout: 120000, }; + log.info('','Querying utxos: %s addrs', addresses.length); + this._doRequest(args, function(err, res, unspent) { if (err || res.statusCode !== 200) return cb(_parseErr(err, res)); return cb(null, unspent); @@ -122,6 +133,9 @@ Insight.prototype.getTransactions = function(addresses, from, to, cb) { timeout: 120000, }; + + log.info('','Querying addresses: %s addrs', addresses.length); + this._doRequest(args, function(err, res, txs) { if (err || res.statusCode !== 200) return cb(_parseErr(err, res)); diff --git a/lib/blockchainexplorers/request-list.js b/lib/blockchainexplorers/request-list.js index 93fb650..8c7f4c5 100644 --- a/lib/blockchainexplorers/request-list.js +++ b/lib/blockchainexplorers/request-list.js @@ -31,11 +31,23 @@ var requestList = function(args, cb) { async.whilst( function() { nextUrl = urls.shift(); + if (!nextUrl && success === 'false') + log.warn('no more servers to test for the request'); return nextUrl && !success; }, function(a_cb) { args.uri = nextUrl; + + var time = 0; + var interval = setInterval(function() { + time += 10; + log.debug('', 'Delayed insight query: %s, time: %d s', args.uri, time); + }, 10000); + request(args, function(err, res, body) { + clearInterval(interval); + sucess = false; + if (err) { log.warn('REQUEST FAIL: ' + nextUrl + ' ERROR: ' + err); } diff --git a/lib/blockchainmonitor.js b/lib/blockchainmonitor.js index 0671981..3a008ac 100644 --- a/lib/blockchainmonitor.js +++ b/lib/blockchainmonitor.js @@ -152,7 +152,6 @@ BlockchainMonitor.prototype._handleThirdPartyBroadcasts = function(data, process BlockchainMonitor.prototype._handleIncomingPayments = function(coin, network, data) { var self = this; - if (!data || !data.vout) return; var outs = _.compact(_.map(data.vout, function(v) { @@ -197,7 +196,7 @@ BlockchainMonitor.prototype._handleIncomingPayments = function(coin, network, da walletId: walletId, }); self.storage.softResetTxHistoryCache(walletId, function() { - self._updateActiveAddresses(address, function() { + self._updateAddressesWithBalance(address, function() { self._storeAndBroadcastNotification(notification, next); }); }); @@ -208,14 +207,29 @@ BlockchainMonitor.prototype._handleIncomingPayments = function(coin, network, da }); }; -BlockchainMonitor.prototype._updateActiveAddresses = function(address, cb) { +BlockchainMonitor.prototype._updateAddressesWithBalance = function(address, cb) { + var self = this; - self.storage.storeActiveAddresses(address.walletId, address.address, function(err) { + self.storage.fetchAddressesWithBalance(address.walletId, function(err, result) { if (err) { log.warn('Could not update wallet cache', err); + return cb(err); } - return cb(err); + var addresses = _.map(result,'address'); + + if (_.indexOf(addresses, address.address) >= 0) { + return cb(); + } + + addresses.push(address.address); + log.info('Activating address '+address); + self.storage.storeAddressesWithBalance(address.walletId, addresses, function(err) { + if (err) { + log.warn('Could not update wallet cache', err); + } + return cb(err); + }); }); }; diff --git a/lib/common/defaults.js b/lib/common/defaults.js index 130237b..30444b5 100644 --- a/lib/common/defaults.js +++ b/lib/common/defaults.js @@ -60,6 +60,12 @@ Defaults.FEE_LEVELS_FALLBACK = 2; // Minimum nb of addresses a wallet must have to start using 2-step balance optimization Defaults.TWO_STEP_BALANCE_THRESHOLD = 100; +// Age Limit for addresses to be considered 'active' always +Defaults.TWO_STEP_CREATION_HOURS = 24; + +// Time to prevent re-quering inactive addresses (MIN) +Defaults.TWO_STEP_INACTIVE_CLEAN_DURATION_MIN = 60; + Defaults.FIAT_RATE_PROVIDER = 'BitPay'; Defaults.FIAT_RATE_FETCH_INTERVAL = 10; // In minutes Defaults.FIAT_RATE_MAX_LOOK_BACK_TIME = 120; // In minutes diff --git a/lib/errors/errordefinitions.js b/lib/errors/errordefinitions.js index f5c8c2f..a69017e 100644 --- a/lib/errors/errordefinitions.js +++ b/lib/errors/errordefinitions.js @@ -33,7 +33,7 @@ var errors = { UPGRADE_NEEDED: 'Client app needs to be upgraded', WALLET_ALREADY_EXISTS: 'Wallet already exists', WALLET_FULL: 'Wallet full', - WALLET_LOCKED: 'Wallet is locked', + WALLET_BUSY: 'Wallet is busy, try later', WALLET_NOT_COMPLETE: 'Wallet is not complete', WALLET_NOT_FOUND: 'Wallet not found', }; diff --git a/lib/expressapp.js b/lib/expressapp.js index ab03335..8236273 100644 --- a/lib/expressapp.js +++ b/lib/expressapp.js @@ -17,7 +17,7 @@ var Stats = require('./stats'); log.disableColor(); log.debug = log.verbose; -log.level = 'info'; +log.level = 'verbose'; var ExpressApp = function() { this.app = express(); @@ -75,14 +75,10 @@ ExpressApp.prototype.start = function(opts, cb) { } else { var morgan = require('morgan'); morgan.token('walletId', function getId(req) { - return req.walletId + return req.walletId ? '<' + req.walletId + '>' : '<>'; }); - morgan.token('copayerId', function getId(req) { - return req.copayerId - }); - - var logFormat = ':remote-addr :date[iso] ":method :url" :status :res[content-length] :response-time ":user-agent" :walletId :copayerId'; + var logFormat = ':walletId :remote-addr :date[iso] ":method :url" :status :res[content-length] :response-time ":user-agent" '; var logOpts = { skip: function(req, res) { if (res.statusCode != 200) return false; @@ -141,6 +137,7 @@ ExpressApp.prototype.start = function(opts, cb) { }; function getServer(req, res) { + log.heading = '<>'; var opts = { clientVersion: req.header('x-client-version'), }; @@ -183,6 +180,8 @@ ExpressApp.prototype.start = function(opts, cb) { req.walletId = server.walletId; req.copayerId = server.copayerId; + log.heading = '<' + req.walletId + '>'; + return cb(server); }); }; diff --git a/lib/lock.js b/lib/lock.js index 0dd2e69..caeb18d 100644 --- a/lib/lock.js +++ b/lib/lock.js @@ -31,7 +31,7 @@ Lock.prototype.runLocked = function(token, cb, task) { $.shouldBeDefined(token); this.lock.locked(token, 5 * 1000, 5 * 60 * 1000, function(err, release) { - if (err) return cb(Errors.WALLET_LOCKED); + if (err) return cb(Errors.WALLET_BUSY); var _cb = function() { cb.apply(null, arguments); release(); diff --git a/lib/model/wallet.js b/lib/model/wallet.js index bf29cec..ab6b5fd 100644 --- a/lib/model/wallet.js +++ b/lib/model/wallet.js @@ -2,6 +2,7 @@ var _ = require('lodash'); var util = require('util'); +var log = require('npmlog'); var $ = require('preconditions').singleton(); var Uuid = require('uuid'); @@ -156,6 +157,7 @@ Wallet.prototype.createAddress = function(isChange) { var self = this; var path = this.addressManager.getNewAddressPath(isChange); + log.verbose('Deriving addr:' + path); var address = Address.derive(self.id, this.addressType, this.publicKeyRing, path, this.m, this.coin, this.network, isChange); return address; }; diff --git a/lib/server.js b/lib/server.js index bd03c67..ac953f5 100644 --- a/lib/server.js +++ b/lib/server.js @@ -463,7 +463,7 @@ WalletService.prototype.getWalletFromIdentifier = function(opts, cb) { bc.getTransaction(opts.identifier, function(err, tx) { if (err || !tx) return nextCoinNetwork(false); var outputs = _.first(self._normalizeTxHistory(tx)).outputs; - var toAddresses = _.pluck(outputs, 'address'); + var toAddresses = _.map(outputs, 'address'); async.detect(toAddresses, function(addressStr, nextAddress) { self.storage.fetchAddressByCoin(coinNetwork.coin, addressStr, function(err, address) { if (err || !address) return nextAddress(false); @@ -873,7 +873,7 @@ WalletService.prototype.savePreferences = function(opts, cb) { }, }]; - opts = _.pick(opts, _.pluck(preferences, 'name')); + opts = _.pick(opts, _.map(preferences, 'name')); try { _.each(preferences, function(preference) { var value = opts[preference.name]; @@ -1131,7 +1131,7 @@ WalletService.prototype._getUtxosForCurrentWallet = function(opts, cb) { }); }, function(next) { - addressStrs = _.pluck(allAddresses, 'address'); + addressStrs = _.map(allAddresses, 'address'); if (!opts.coin) return next(); coin = opts.coin; @@ -1141,7 +1141,6 @@ WalletService.prototype._getUtxosForCurrentWallet = function(opts, cb) { next(); }, function(next) { - self._getUtxos(coin, addressStrs, function(err, utxos) { if (err) return next(err); @@ -1155,7 +1154,7 @@ WalletService.prototype._getUtxosForCurrentWallet = function(opts, cb) { self.getPendingTxs({}, function(err, txps) { if (err) return next(err); - var lockedInputs = _.map(_.flatten(_.pluck(txps, 'inputs')), utxoKey); + var lockedInputs = _.map(_.flatten(_.map(txps, 'inputs')), utxoKey); _.each(lockedInputs, function(input) { if (utxoIndex[input]) { utxoIndex[input].locked = true; @@ -1175,7 +1174,7 @@ WalletService.prototype._getUtxosForCurrentWallet = function(opts, cb) { limit: 100 }, function(err, txs) { if (err) return next(err); - var spentInputs = _.map(_.flatten(_.pluck(txs, 'inputs')), utxoKey); + var spentInputs = _.map(_.flatten(_.map(txs, 'inputs')), utxoKey); _.each(spentInputs, function(input) { if (utxoIndex[input]) { utxoIndex[input].spent = true; @@ -1242,101 +1241,113 @@ WalletService.prototype._totalizeUtxos = function(utxos) { }; -WalletService.prototype._getBalanceFromAddresses = function(opts, cb) { +WalletService.prototype._getBalanceFromAddresses = function(opts, cb, i) { var self = this; var opts = opts || {}; - self._getUtxosForCurrentWallet({ - coin: opts.coin, - addresses: opts.addresses - }, function(err, utxos) { - if (err) return cb(err); + // This lock is to prevent server starvation on big wallets + self._runLocked(cb, function(cb) { + self._getUtxosForCurrentWallet({ + coin: opts.coin, + addresses: opts.addresses + }, function(err, utxos) { + if (err) return cb(err); - var balance = self._totalizeUtxos(utxos); + var balance = self._totalizeUtxos(utxos); - // Compute balance by address - var byAddress = {}; - _.each(_.indexBy(_.sortBy(utxos, 'address'), 'address'), function(value, key) { - byAddress[key] = { - address: key, - path: value.path, - amount: 0, - }; + // Compute balance by address + var byAddress = {}; + _.each(_.indexBy(_.sortBy(utxos, 'address'), 'address'), function(value, key) { + byAddress[key] = { + address: key, + path: value.path, + amount: 0, + }; + }); + + _.each(utxos, function(utxo) { + byAddress[utxo.address].amount += utxo.satoshis; + }); + + balance.byAddress = _.values(byAddress); + return cb(null, balance); }); - - _.each(utxos, function(utxo) { - byAddress[utxo.address].amount += utxo.satoshis; - }); - - balance.byAddress = _.values(byAddress); - - return cb(null, balance); }); }; WalletService.prototype._getBalanceOneStep = function(opts, cb) { var self = this; - self.storage.fetchAddresses(self.walletId, function(err, addresses) { - if (err) return cb(err); - self._getBalanceFromAddresses({ - coin: opts.coin, - addresses: addresses - }, function(err, balance) { + self.storage.fetchAddresses(self.walletId, function(err, addresses) { if (err) return cb(err); + self._getBalanceFromAddresses({ + coin: opts.coin, + addresses: addresses + }, function(err, balance) { + if (err) return cb(err); - // Update cache - async.series([ - - function(next) { - self.storage.cleanActiveAddresses(self.walletId, next); - }, - function(next) { - var active = _.pluck(balance.byAddress, 'address') - self.storage.storeActiveAddresses(self.walletId, active, next); - }, - ], function(err) { - if (err) { - log.warn('Could not update wallet cache', err); - } - return cb(null, balance); + // Update cache + var withBalance = _.map(balance.byAddress, 'address') + self.storage.storeAddressesWithBalance(self.walletId, withBalance, function(err) { + if (err) { + log.warn('Could not update wallet cache', err); + } + return cb(null, balance); + }); }); }); - }); }; WalletService.prototype._getActiveAddresses = function(cb) { var self = this; - self.storage.fetchActiveAddresses(self.walletId, function(err, active) { + self.storage.fetchAddressesWithBalance(self.walletId, function(err, addressesWB) { if (err) { log.warn('Could not fetch active addresses from cache', err); return cb(); } + if (!_.isArray(addressesWB)) + addressesWB = []; - if (!_.isArray(active)) return cb(); + var now = Math.floor(Date.now() / 1000); + var fromTs = now - Defaults.TWO_STEP_CREATION_HOURS * 3600; - self.storage.fetchAddresses(self.walletId, function(err, allAddresses) { + self.storage.fetchNewAddresses(self.walletId, fromTs, function(err, recent) { if (err) return cb(err); - var now = Math.floor(Date.now() / 1000); - var recent = _.pluck(_.filter(allAddresses, function(address) { - return address.createdOn > (now - 24 * 3600); - }), 'address'); - - var result = _.union(active, recent); - - var index = _.indexBy(allAddresses, 'address'); - result = _.compact(_.map(result, function(r) { - return index[r]; - })); + var result = _.uniq(_.union(addressesWB, recent), 'address'); return cb(null, result); }); }); }; +WalletService.prototype._checkAndUpdateAddressCount = function(twoStepCache, cb) { + var self = this; + + if (twoStepCache.addressCount > Defaults.TWO_STEP_BALANCE_THRESHOLD) { + log.info('Not counting addresses for '+ self.walletId); + return cb(null, true); + } + + self.storage.countAddresses(self.walletId, function(err, addressCount) { + if (err) return cb(err); + + if (addressCount < Defaults.TWO_STEP_BALANCE_THRESHOLD) + return cb(null, false); + + twoStepCache.addressCount = addressCount; + + // updates cache + self.storage.storeTwoStepCache(self.walletId, twoStepCache, function(err) { + if (err) return cb(err); + + return cb(null, true); + }); + }); +}; + /** * Get wallet balance. * @param {Object} opts @@ -1344,7 +1355,8 @@ WalletService.prototype._getActiveAddresses = function(cb) { * @param {Boolean} opts.twoStep[=false] - Optional - Use 2 step balance computation for improved performance * @returns {Object} balance - Total amount & locked amount. */ -WalletService.prototype.getBalance = function(opts, cb) { + +WalletService.prototype.getBalance = function(opts, cb, i) { var self = this; opts = opts || {}; @@ -1354,40 +1366,69 @@ WalletService.prototype.getBalance = function(opts, cb) { return cb(new ClientError('Invalid coin')); } - if (!opts.twoStep) + if (!opts.twoStep) { return self._getBalanceOneStep(opts, cb); + } - self.storage.countAddresses(self.walletId, function(err, nbAddresses) { + + self.storage.getTwoStepCache(self.walletId, function(err, twoStepCache) { if (err) return cb(err); - if (nbAddresses < Defaults.TWO_STEP_BALANCE_THRESHOLD) { - return self._getBalanceOneStep(opts, cb); - } - self._getActiveAddresses(function(err, activeAddresses) { + twoStepCache = twoStepCache || {}; + + self._checkAndUpdateAddressCount(twoStepCache, function(err, needsTwoStep ) { if (err) return cb(err); - if (!_.isArray(activeAddresses)) { + + if (!needsTwoStep) { return self._getBalanceOneStep(opts, cb); - } else { - log.debug('Requesting partial balance for ' + activeAddresses.length + ' out of ' + nbAddresses + ' addresses'); - self._getBalanceFromAddresses({ - coin: opts.coin, - addresses: activeAddresses - }, function(err, partialBalance) { - if (err) return cb(err); - cb(null, partialBalance); - setTimeout(function() { - self._getBalanceOneStep(opts, function(err, fullBalance) { - if (err) return; - if (!_.isEqual(partialBalance, fullBalance)) { - log.info('Balance in active addresses differs from final balance'); - self._notify('BalanceUpdated', fullBalance, { - isGlobal: true - }); - } - }); - }, 1); - return; - }); } + + self._getActiveAddresses(function(err, activeAddresses) { + if (err) return cb(err); + if (!_.isArray(activeAddresses)) { + return self._getBalanceOneStep(opts, cb); + } else { + log.debug('Requesting partial balance for ' + activeAddresses.length + ' addresses'); + + self._getBalanceFromAddresses({ + coin: opts.coin, + addresses: activeAddresses + }, function(err, partialBalance) { + if (err) return cb(err); + cb(null, partialBalance); + + var now = Math.floor(Date.now() / 1000); + + if (twoStepCache.lastEmpty > now - Defaults.TWO_STEP_INACTIVE_CLEAN_DURATION_MIN * 60 ) { + log.debug('Not running the FULL balance query due to TWO_STEP_INACTIVE_CLEAN_DURATION_MIN '); + return; + } + + + setTimeout(function() { + log.debug('Running full balance query'); + self._getBalanceOneStep(opts, function(err, fullBalance) { + if (err) return; + if (!_.isEqual(partialBalance, fullBalance)) { + log.info('Balance in active addresses differs from final balance'); + self._notify('BalanceUpdated', fullBalance, { + isGlobal: true + }); + } else { + // updates cache + twoStepCache.lastEmpty = now; + + // updates cache + return self.storage.storeTwoStepCache(self.walletId, twoStepCache, function(err) { + return; + }); + + } + }); + }, 1); + return; + }, i); + } + }); }); }); }; @@ -1424,7 +1465,7 @@ WalletService.prototype.getSendMaxInfo = function(opts, cb) { if (!_.any(feeLevels, { name: opts.feeLevel })) - return cb(new ClientError('Invalid fee level. Valid values are ' + _.pluck(feeLevels, 'name').join(', '))); + return cb(new ClientError('Invalid fee level. Valid values are ' + _.map(feeLevels, 'name').join(', '))); } if (_.isNumber(opts.feePerKb)) { @@ -1567,7 +1608,7 @@ WalletService.prototype.getFeeLevels = function(opts, cb) { var feeLevels = Defaults.FEE_LEVELS[opts.coin]; function samplePoints() { - var definedPoints = _.uniq(_.pluck(feeLevels, 'nbBlocks')); + var definedPoints = _.uniq(_.map(feeLevels, 'nbBlocks')); return _.uniq(_.flatten(_.map(definedPoints, function(p) { return _.range(p, p + Defaults.FEE_LEVELS_FALLBACK + 1); }))); @@ -1986,7 +2027,7 @@ WalletService.prototype._validateAndSanitizeTxOpts = function(wallet, opts, cb) if (!_.any(feeLevels, { name: opts.feeLevel })) - return next(new ClientError('Invalid fee level. Valid values are ' + _.pluck(feeLevels, 'name').join(', '))); + return next(new ClientError('Invalid fee level. Valid values are ' + _.map(feeLevels, 'name').join(', '))); } if (_.isNumber(opts.feePerKb)) { @@ -2642,7 +2683,7 @@ WalletService.prototype.rejectTx = function(opts, cb) { }, function(next) { if (txp.status == 'rejected') { - var rejectedBy = _.pluck(_.filter(txp.actions, { + var rejectedBy = _.map(_.filter(txp.actions, { type: 'reject' }), 'copayerId'); @@ -3005,7 +3046,7 @@ WalletService.prototype.getTxHistory = function(opts, cb) { function(next) { if (txs) return next(); - var addressStrs = _.pluck(addresses, 'address'); + var addressStrs = _.map(addresses, 'address'); var bc = self._getBlockchainExplorer(wallet.coin, wallet.network); if (!bc) return next(new Error('Could not get blockchain explorer instance')); bc.getTransactions(addressStrs, from, to, function(err, rawTxs, total) { @@ -3169,6 +3210,7 @@ WalletService.prototype.scan = function(opts, cb) { var gap = Defaults.SCAN_ADDRESS_GAP; async.whilst(function() { + log.debug('Scanning addr gap:'+ inactiveCounter); return inactiveCounter < gap; }, function(next) { var address = derivator.derive(); @@ -3225,7 +3267,9 @@ WalletService.prototype.scan = function(opts, cb) { if (err) return cb(err); wallet.scanStatus = error ? 'error' : 'success'; self.storage.storeWallet(wallet, function() { - return cb(error); + self.storage.storeTwoStepCache(self.walletId, {}, function(err) { + return cb(error); + }); }); }) }); diff --git a/lib/storage.js b/lib/storage.js index 94fe4a2..fe0ee75 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -70,6 +70,10 @@ Storage.prototype._createIndexes = function() { this.db.collection(collections.ADDRESSES).createIndex({ address: 1, }); + this.db.collection(collections.ADDRESSES).createIndex({ + walletId: 1, + address: 1, + }); this.db.collection(collections.EMAIL_QUEUE).createIndex({ id: 1, }); @@ -468,6 +472,28 @@ Storage.prototype.fetchAddresses = function(walletId, cb) { }); }; + +Storage.prototype.fetchNewAddresses = function(walletId, fromTs, cb) { + var self = this; + + this.db.collection(collections.ADDRESSES).find({ + walletId: walletId, + createdOn: { + $gte: fromTs, + }, + }).sort({ + createdOn: 1 + }).toArray(function(err, result) { + if (err) return cb(err); + if (!result) return cb(); + var addresses = _.map(result, function(address) { + return Model.Address.fromObj(address); + }); + return cb(null, addresses); + }); +}; + + Storage.prototype.countAddresses = function(walletId, cb) { this.db.collection(collections.ADDRESSES).find({ walletId: walletId, @@ -598,52 +624,91 @@ Storage.prototype.fetchEmailByNotification = function(notificationId, cb) { }); }; -Storage.prototype.cleanActiveAddresses = function(walletId, cb) { +Storage.prototype.storeTwoStepCache = function(walletId, cacheStatus, cb) { var self = this; - - async.series([ - - function(next) { - self.db.collection(collections.CACHE).remove({ - walletId: walletId, - type: 'activeAddresses', - }, { - w: 1 - }, next); - }, - function(next) { - self.db.collection(collections.CACHE).insert({ - walletId: walletId, - type: 'activeAddresses', - key: null - }, { - w: 1 - }, next); - }, - ], cb); -}; - -Storage.prototype.storeActiveAddresses = function(walletId, addresses, cb) { - var self = this; - - if (_.isEmpty(addresses)) return cb(); - async.each(addresses, function(address, next) { - var record = { - walletId: walletId, - type: 'activeAddresses', - key: address, - }; - self.db.collection(collections.CACHE).update({ - walletId: record.walletId, - type: record.type, - key: record.key, - }, record, { - w: 1, - upsert: true, - }, next); + self.db.collection(collections.CACHE).update( { + walletId: walletId, + type: 'twoStep', + key: null, + }, { + "$set": + { + addressCount: cacheStatus.addressCount, + lastEmpty: cacheStatus.lastEmpty, + } + }, { + w: 1, + upsert: true, }, cb); }; +Storage.prototype.getTwoStepCache = function(walletId, cb) { + var self = this; + + + self.db.collection(collections.CACHE).findOne({ + walletId: walletId, + type: 'twoStep', + key: null + }, function(err, result) { + if (err) return cb(err); + if (!result) return cb(); + return cb(null, result); + }); +}; + + +Storage.prototype.storeAddressesWithBalance = function(walletId, addresses, cb) { + var self = this; + + if (_.isEmpty(addresses)) + addresses = []; + + self.db.collection(collections.CACHE).update({ + walletId: walletId, + type: 'addressesWithBalance', + key: null, + }, { + "$set": + { + addresses: addresses, + } + }, { + w: 1, + upsert: true, + }, cb); +}; + +Storage.prototype.fetchAddressesWithBalance = function(walletId, cb) { + var self = this; + + + self.db.collection(collections.CACHE).findOne({ + walletId: walletId, + type: 'addressesWithBalance', + key: null, + }, function(err, result) { + if (err) return cb(err); + if (_.isEmpty(result)) return cb(null, []); + + + self.db.collection(collections.ADDRESSES).find({ + walletId: walletId, + address: { $in: result.addresses }, + }).toArray(function(err, result2) { + if (err) return cb(err); + if (!result2) return cb(null, []); + + var addresses = _.map(result2, function(address) { + return Model.Address.fromObj(address); + }); + return cb(null, addresses); + }); + + }); +}; + + // -------- --------------------------- Total // > Time > // ^to <= ^from @@ -694,7 +759,7 @@ Storage.prototype.getTxHistoryCache = function(walletId, from, to, cb) { return cb(); } - var txs = _.pluck(result, 'tx'); + var txs = _.map(result, 'tx'); return cb(null, txs); }); }) @@ -796,23 +861,6 @@ Storage.prototype.storeTxHistoryCache = function(walletId, totalItems, firstPosi }; - - - -Storage.prototype.fetchActiveAddresses = function(walletId, cb) { - var self = this; - - self.db.collection(collections.CACHE).find({ - walletId: walletId, - type: 'activeAddresses', - }).toArray(function(err, result) { - if (err) return cb(err); - if (_.isEmpty(result)) return cb(); - - return cb(null, _.compact(_.pluck(result, 'key'))); - }); -}; - Storage.prototype.storeFiatRate = function(providerName, rates, cb) { var self = this; diff --git a/package-lock.json b/package-lock.json index 3dc42d2..2383ba2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2709,9 +2709,9 @@ } }, "safe": { - "version": "0.4.5", - "resolved": "https://registry.npmjs.org/safe/-/safe-0.4.5.tgz", - "integrity": "sha1-MWEo64HpZFEe3OKAd55aP/N/LVg=", + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/safe/-/safe-0.3.9.tgz", + "integrity": "sha1-F4FZvuRXkawhYoruLau3SlLV0HI=", "dev": true }, "safe-buffer": { @@ -3167,15 +3167,14 @@ } }, "tingodb": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/tingodb/-/tingodb-0.3.5.tgz", - "integrity": "sha1-xs0G2WxzuCp4GyjDCC/EoZBaRWg=", + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/tingodb/-/tingodb-0.5.1.tgz", + "integrity": "sha1-U8rlLLTUMQgImMwZ/F0EMzoDAck=", "dev": true, "requires": { - "async": "0.9.2", "bson": "0.2.22", - "lodash": "2.4.2", - "safe": "0.4.5" + "lodash": "4.11.2", + "safe": "0.3.9" }, "dependencies": { "bson": { @@ -3189,9 +3188,9 @@ } }, "lodash": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz", - "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4=", + "version": "4.11.2", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.11.2.tgz", + "integrity": "sha1-1rQzixEKWOIdrlzrz9u/0rxM2zs=", "dev": true }, "nan": { diff --git a/package.json b/package.json index 5dce2db..c6df78e 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "proxyquire": "^1.7.2", "sinon": "1.10.3", "supertest": "*", - "tingodb": "^0.3.4" + "tingodb": "^0.5.1" }, "scripts": { "start": "./start.sh", diff --git a/test/integration/bcmonitor.js b/test/integration/bcmonitor.js index db2c0e4..5b125cf 100644 --- a/test/integration/bcmonitor.js +++ b/test/integration/bcmonitor.js @@ -13,7 +13,6 @@ log.level = 'info'; var WalletService = require('../../lib/server'); var BlockchainMonitor = require('../../lib/blockchainmonitor'); -var TestData = require('../testdata'); var helpers = require('./helpers'); var storage, blockchainExplorer; @@ -90,6 +89,112 @@ describe('Blockchain monitor', function() { }); }); + it('should update addressWithBalance cache on 1 incoming tx', function(done) { + server.createAddress({}, function(err, address) { + should.not.exist(err); + var incoming = { + txid: '123', + vout: [{}], + }; + server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) { + should.not.exist(err); + _.isEmpty(ret).should.equal(true); + incoming.vout[0][address.address] = 1500; + socket.handlers['tx'](incoming); + setTimeout(function() { + server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) { + should.not.exist(err); + ret.length.should.equal(1); + ret[0].address.should.equal(address.address); + done(); + }); + }, 100); + }); + }); + }); + + it('should update addressWithBalance cache on 2 incoming tx, same address', function(done) { + server.createAddress({}, function(err, address) { + should.not.exist(err); + + server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) { + should.not.exist(err); + _.isEmpty(ret).should.equal(true); + + var incoming = { + txid: '123', + vout: [{}], + }; + incoming.vout[0][address.address] = 1500; + + socket.handlers['tx'](incoming); + setTimeout(function() { + + + var incoming2 = { + txid: '456', + vout: [{}], + }; + incoming2.vout[0][address.address] = 2500; + socket.handlers['tx'](incoming2); + + setTimeout(function() { + server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) { + should.not.exist(err); + ret.length.should.equal(1); + ret[0].address.should.equal(address.address); + done(); + }); + }, 100); + }, 100); + }); + }); + }); + + + it('should update addressWithBalance cache on 2 incoming tx, different address', function(done) { + server.createAddress({}, function(err, address) { + should.not.exist(err); + server.createAddress({}, function(err, address2) { + should.not.exist(err); + server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) { + should.not.exist(err); + _.isEmpty(ret).should.equal(true); + + var incoming = { + txid: '123', + vout: [{}], + }; + incoming.vout[0][address.address] = 1500; + socket.handlers['tx'](incoming); + + setTimeout(function() { + + var incoming2 = { + txid: '456', + vout: [{}], + }; + incoming2.vout[0][address2.address] = 500; + socket.handlers['tx'](incoming2); + + setTimeout(function() { + server.storage.fetchAddressesWithBalance(wallet.id, function(err,ret) { + should.not.exist(err); + ret.length.should.equal(2); + ret[0].address.should.equal(address.address); + done(); + }); + }, 100); + }, 100); + }); + }); + }); + }); + + + + + it('should not notify copayers of incoming txs more than once', function(done) { server.createAddress({}, function(err, address) { should.not.exist(err); diff --git a/test/integration/server.js b/test/integration/server.js index 918988a..bfc9889 100644 --- a/test/integration/server.js +++ b/test/integration/server.js @@ -2027,12 +2027,13 @@ describe('Wallet service', function() { }); }, function(next) { - clock.tick(7 * 24 * 3600 * 1000); + clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000); helpers.createAddresses(server, wallet, 2, 0, function(addrs) { newAddrs = addrs; server._getActiveAddresses(function(err, active) { should.not.exist(err); - should.not.exist(active); + active.length.should.equal(2); + helpers.stubUtxos(server, wallet, [1, 2], { addresses: [oldAddrs[0], newAddrs[0]], }, function() { @@ -2047,7 +2048,9 @@ describe('Wallet service', function() { }, function(err, balance) { should.not.exist(err); should.exist(balance); - balance.totalAmount.should.equal(helpers.toSatoshi(3)); + + // Only should see newAddr[0] + balance.totalAmount.should.equal(helpers.toSatoshi(2)); next(); }); }, @@ -2058,6 +2061,7 @@ describe('Wallet service', function() { server._getActiveAddresses(function(err, active) { should.not.exist(err); should.exist(active); + // 1 old (with balance) + 2 news active.length.should.equal(3); next(); }); @@ -2099,6 +2103,76 @@ describe('Wallet service', function() { }); }); + it('should not count addresses if wallet have already passed the threshold', function(done) { + var oldAddrs, newAddrs, spy; + async.series([ + + function(next) { + spy = sinon.spy(server.storage, 'countAddresses'); + helpers.createAddresses(server, wallet, 2, 0, function(addrs) { + oldAddrs = addrs; + next(); + }); + }, + function(next) { + clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000); + helpers.createAddresses(server, wallet, 2, 0, function(addrs) { + newAddrs = addrs; + server._getActiveAddresses(function(err, active) { + should.not.exist(err); + // the new 2 + active.length.should.equal(2); + helpers.stubUtxos(server, wallet, [1, 2], { + addresses: [oldAddrs[0], newAddrs[0]], + }, function() { + next(); + }); + }); + }); + }, + function(next) { + server.getBalance({ + twoStep: true + }, function(err, balance) { + should.not.exist(err); + should.exist(balance); + balance.totalAmount.should.equal(helpers.toSatoshi(2)); + next(); + }); + }, + function(next) { + setTimeout(next, 100); + }, + function(next) { + spy.calledOnce.should.equal(true); + next(); + }, + function(next) { + server.getBalance({ + twoStep: true + }, function(err, balance) { + should.not.exist(err); + should.exist(balance); + balance.totalAmount.should.equal(helpers.toSatoshi(3)); + next(); + }); + }, + function(next) { + setTimeout(next, 100); + }, + // should NOT count addresses again! (still only one call) + function(next) { + spy.calledOnce.should.equal(true); + next(); + }, + + ], function(err) { + should.not.exist(err); + done(); + }); + }); + + it('should not trigger notification when only balance of prioritary addresses is updated', function(done) { var oldAddrs, newAddrs; @@ -2111,7 +2185,7 @@ describe('Wallet service', function() { }); }, function(next) { - clock.tick(7 * 24 * 3600 * 1000); + clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000); helpers.createAddresses(server, wallet, 2, 0, function(addrs) { newAddrs = addrs; helpers.stubUtxos(server, wallet, [1, 2], { @@ -2121,6 +2195,17 @@ describe('Wallet service', function() { }); }); }, + function(next) { + setTimeout(next, 100); + }, + function(next) { + server.getNotifications({}, function(err, notifications) { + should.not.exist(err); + var last = _.last(notifications); + last.type.should.not.equal('BalanceUpdated'); + next(); + }); + }, function(next) { server.getBalance({ twoStep: true @@ -2169,6 +2254,251 @@ describe('Wallet service', function() { }); }); + it('should not do 2 steps if called 2 times given the first one there found no funds on non-active addresses', function(done) { + var oldAddrs, newAddrs, spy; + + async.series([ + function(next) { + spy = sinon.spy(server, '_getBalanceOneStep'); + helpers.createAddresses(server, wallet, 2, 0, function(addrs) { + oldAddrs = addrs; + next(); + }); + }, + function(next) { + clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000); + helpers.createAddresses(server, wallet, 2, 0, function(addrs) { + newAddrs = addrs; + helpers.stubUtxos(server, wallet, [1, 2], { + addresses: newAddrs, + }, function() { + next(); + }); + }); + }, + function(next) { + server.getBalance({ + twoStep: true + }, function(err, balance) { + should.not.exist(err); + should.exist(balance); + balance.totalAmount.should.equal(helpers.toSatoshi(3)); + next(); + }); + }, + function(next) { + setTimeout(next, 100); + }, + // Should _oneStep should be called once + function(next) { + spy.calledOnce.should.equal(true); + next(); + }, + function(next) { + helpers.stubUtxos(server, wallet, 0.5, { + addresses: newAddrs[0], + keepUtxos: true, + }, function() { + next(); + }); + }, + function(next) { + server.getBalance({ + twoStep: true + }, function(err, balance) { + should.not.exist(err); + should.exist(balance); + balance.totalAmount.should.equal(helpers.toSatoshi(3.5)); + next(); + }); + }, + function(next) { + setTimeout(next, 100); + }, + // Should _oneStep should be called once + function(next) { + spy.calledOnce.should.equal(true); + next(); + }, + // Should not trigger notification either + function(next) { + server.getNotifications({}, function(err, notifications) { + should.not.exist(err); + var last = _.last(notifications); + last.type.should.not.equal('BalanceUpdated'); + next(); + }); + }, + ], function(err) { + should.not.exist(err); + done(); + }); + }); + + it('should do 2 steps if called 2 times given the first one there found no funds on non-active addresses, but times passes', function(done) { + var oldAddrs, newAddrs, spy; + + async.series([ + function(next) { + spy = sinon.spy(server, '_getBalanceOneStep'); + helpers.createAddresses(server, wallet, 2, 0, function(addrs) { + oldAddrs = addrs; + next(); + }); + }, + function(next) { + clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000); + helpers.createAddresses(server, wallet, 2, 0, function(addrs) { + newAddrs = addrs; + helpers.stubUtxos(server, wallet, [1, 2], { + addresses: newAddrs, + }, function() { + next(); + }); + }); + }, + function(next) { + server.getBalance({ + twoStep: true + }, function(err, balance) { + should.not.exist(err); + should.exist(balance); + balance.totalAmount.should.equal(helpers.toSatoshi(3)); + next(); + }); + }, + function(next) { + setTimeout(next, 100); + }, + // Should _oneStep should be called once + function(next) { + spy.calledOnce.should.equal(true); + next(); + }, + function(next) { + helpers.stubUtxos(server, wallet, 0.5, { + addresses: newAddrs[0], + keepUtxos: true, + }, function() { + clock.tick(2 * Defaults.TWO_STEP_INACTIVE_CLEAN_DURATION_MIN * 60 * 1000); + next(); + }); + }, + function(next) { + server.getBalance({ + twoStep: true + }, function(err, balance) { + should.not.exist(err); + should.exist(balance); + balance.totalAmount.should.equal(helpers.toSatoshi(3.5)); + next(); + }); + }, + function(next) { + setTimeout(next, 100); + }, + function(next) { + spy.calledTwice.should.equal(true); + next(); + }, + ], function(err) { + should.not.exist(err); + done(); + }); + }); + + it('should do 2 steps if called 2 times given the first one there found funds on non-active addresses', function(done) { + var oldAddrs, newAddrs, spy, notificationCount; + + async.series([ + function(next) { + spy = sinon.spy(server, '_getBalanceOneStep'); + helpers.createAddresses(server, wallet, 2, 0, function(addrs) { + oldAddrs = addrs; + next(); + }); + }, + function(next) { + clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000); + helpers.createAddresses(server, wallet, 2, 0, function(addrs) { + newAddrs = addrs; + helpers.stubUtxos(server, wallet, [1, 2], { + addresses: [ oldAddrs[0], newAddrs[0] ], + }, function() { + next(); + }); + }); + }, + function(next) { + server.getBalance({ + twoStep: true + }, function(err, balance) { + should.not.exist(err); + should.exist(balance); + balance.totalAmount.should.equal(helpers.toSatoshi(2)); + next(); + }); + }, + function(next) { + setTimeout(next, 100); + }, + // Should not trigger notification either + function(next) { + server.getNotifications({}, function(err, notifications) { + notificationCount = notifications.length; + + should.not.exist(err); + var last = _.last(notifications); + last.type.should.equal('BalanceUpdated'); + next(); + }); + }, + // Should _oneStep should be called once + function(next) { + spy.calledOnce.should.equal(true); + next(); + }, + function(next) { + helpers.stubUtxos(server, wallet, 0.5, { + addresses: newAddrs[0], + keepUtxos: true, + }, function() { + next(); + }); + }, + function(next) { + server.getBalance({ + twoStep: true + }, function(err, balance) { + should.not.exist(err); + should.exist(balance); + balance.totalAmount.should.equal(helpers.toSatoshi(3.5)); + next(); + }); + }, + function(next) { + setTimeout(next, 100); + }, + // Should _oneStep should be called TWICE + function(next) { + spy.calledTwice.should.equal(true); + next(); + }, + // Should not trigger notification either + function(next) { + server.getNotifications({}, function(err, notifications) { + should.not.exist(err); + notifications.length.should.equal(notificationCount); + next(); + }); + }, + ], function(err) { + should.not.exist(err); + done(); + }); + }); + + it('should resolve balance of new addresses immediately', function(done) { var addresses; @@ -2192,7 +2522,10 @@ describe('Wallet service', function() { should.exist(balance); balance.totalAmount.should.equal(helpers.toSatoshi(3)); next(); - }); + }, 1); + }, + function(next) { + setTimeout(next, 100); }, function(next) { server.createAddress({}, function(err, addr) { @@ -2212,7 +2545,7 @@ describe('Wallet service', function() { should.exist(balance); balance.totalAmount.should.equal(helpers.toSatoshi(3.5)); next(); - }); + }, 2); }, function(next) { setTimeout(next, 100); @@ -2244,7 +2577,7 @@ describe('Wallet service', function() { }); }, function(next) { - clock.tick(7 * 24 * 3600 * 1000); + clock.tick(7 * Defaults.TWO_STEP_CREATION_HOURS * 3600 * 1000); helpers.createAddresses(server, wallet, 2, 0, function(addrs) { newAddrs = addrs; helpers.stubUtxos(server, wallet, [1, 2], { @@ -2280,6 +2613,8 @@ describe('Wallet service', function() { done(); }); }); + + }); describe('#getFeeLevels', function() {