new input selection algorithm

lib/model/txproposal.js

@@ -219,13 +219,16 @@ TxProposal.prototype.getRawTx = function() {
   return t.uncheckedSerialize();
 };
 
+TxProposal.prototype.getEstimatedSizeForSingleInput = function() {
+  return this.requiredSignatures * 72 + this.walletN * 36 + 44;
+};
+
 TxProposal.prototype.getEstimatedSize = function() {
   // Note: found empirically based on all multisig P2SH inputs and within m & n allowed limits.
   var safetyMargin = 0.05;
-  var walletM = this.requiredSignatures;
 
   var overhead = 4 + 4 + 9 + 9;
-  var inputSize = walletM * 72 + this.walletN * 36 + 44;
+  var inputSize = this.getEstimatedSizeForSingleInput();
   var outputSize = 34;
   var nbInputs = this.inputs.length;
   var nbOutputs = (_.isArray(this.outputs) ? Math.max(1, this.outputs.length) : 1) + 1;

lib/server.js

@@ -1347,6 +1347,171 @@ WalletService.prototype._selectTxInputs = function(txp, utxosToExclude, cb) {
   });
 };
 
+
+var UTXO_SELECTION_MAX_SINGLE_UTXO_FACTOR = 2;
+
+WalletService.prototype._selectTxInputs2 = function(txp, utxosToExclude, cb) {
+  var self = this;
+
+  //todo: check inputs are ours and has enough value
+  if (txp.inputs && txp.inputs.length > 0) {
+    return cb(self._checkTxAndEstimateFee(txp));
+  }
+
+  function excludeUtxos(utxos) {
+    var excludeIndex = _.reduce(utxosToExclude, function(res, val) {
+      res[val] = val;
+      return res;
+    }, {});
+
+    return _.reject(utxos, function(utxo) {
+      return excludeIndex[utxo.txid + ":" + utxo.vout];
+    });
+  };
+
+  function partitionUtxos(utxos) {
+    return _.groupBy(utxos, function(utxo) {
+      if (utxo.confirmations == 0) return '0'
+      if (utxo.confirmations < 6) return '<6';
+      return '6+';
+    });
+  };
+
+  function select(utxos) {
+    var txpAmount = txp.getTotalAmount();
+    var i = 0;
+    var total = 0;
+    var selected = [];
+
+    console.log('*** [server.js ln1362] ----------------------- select for amount of:', txpAmount); // TODO
+
+    // TODO: fix for when fee is specified instead of feePerKb
+    var feePerInput = txp.getEstimatedSizeForSingleInput() * txp.feePerKb / 1000.;
+
+    console.log('*** [server.js ln1375] feePerInput:', feePerInput); // TODO
+
+    var partitions = _.partition(utxos, function(utxo) {
+      return utxo.satoshis > txpAmount * UTXO_SELECTION_MAX_SINGLE_UTXO_FACTOR;
+    });
+
+    var bigInputs = _.sortBy(partitions[0], 'satoshis');
+    var smallInputs = _.sortBy(partitions[1], function(utxo) {
+      return -utxo.satoshis;
+    });
+
+    console.log('*** [server.js ln1386] bigInputs:', _.pluck(bigInputs, 'satoshis')); // TODO
+    console.log('*** [server.js ln1386] smallInputs:', _.pluck(smallInputs, 'satoshis')); // TODO
+
+
+    _.each(smallInputs, function(input) {
+      if (input.satoshis < feePerInput) return false;
+      selected.push(input);
+
+      console.log('*** [server.js ln1380] input:', input.satoshis, '  aporta ->>> ', input.satoshis - feePerInput); // TODO
+
+      total += input.satoshis - feePerInput;
+      if (total >= txpAmount) return false;
+    });
+
+    console.log('*** [server.js ln1400] total, txpAmount:', total, txpAmount); // TODO
+
+    if (total < txpAmount) {
+      console.log('*** [server.js ln1401] no alcanzó:'); // TODO
+
+      selected = [];
+      if (!_.isEmpty(bigInputs)) {
+        console.log('*** [server.js ln1405] pero hay bigInputs!:', _.first(bigInputs).satoshis); // TODO
+
+        selected = [_.first(bigInputs)];
+      }
+    }
+    return selected;
+  };
+
+  self._getUtxosForCurrentWallet(null, function(err, utxos) {
+    if (err) return cb(err);
+
+    utxos = excludeUtxos(utxos);
+
+    var totalAmount;
+    var availableAmount;
+
+    var balance = self._totalizeUtxos(utxos);
+    if (txp.excludeUnconfirmedUtxos) {
+      totalAmount = balance.totalConfirmedAmount;
+      availableAmount = balance.availableConfirmedAmount;
+    } else {
+      totalAmount = balance.totalAmount;
+      availableAmount = balance.availableAmount;
+    }
+
+    if (totalAmount < txp.getTotalAmount()) return cb(Errors.INSUFFICIENT_FUNDS);
+    if (availableAmount < txp.getTotalAmount()) return cb(Errors.LOCKED_FUNDS);
+
+    // Prepare UTXOs list
+    utxos = _.reject(utxos, 'locked');
+    if (txp.excludeUnconfirmedUtxos) {
+      utxos = _.filter(utxos, 'confirmations');
+    }
+
+    var inputs = [];
+    var groups = [6, 1, 0];
+    var lastGroupLength;
+    _.each(groups, function(group) {
+      var candidateUtxos = _.filter(utxos, function(utxo) {
+        return utxo.confirmations >= group;
+      });
+
+      // If this group does not have any new elements, skip it
+      if (lastGroupLength === candidateUtxos.length) return;
+      lastGroupLength = candidateUtxos.length;
+
+      console.log('*** [server.js ln1415] group >=', group, '\n', _.map(candidateUtxos, function(u) {
+        return _.pick(u, 'satoshis', 'confirmations')
+      })); // TODO
+
+      inputs = select(candidateUtxos);
+
+      console.log('*** [server.js ln1418] inputs:', _.pluck(inputs, 'satoshis')); // TODO
+
+      if (!_.isEmpty(inputs)) return false;
+    });
+
+    if (_.isEmpty(inputs)) return cb(Errors.INSUFFICIENT_FUNDS);
+
+    txp.setInputs(inputs);
+    if (txp.getEstimatedSize() / 1000 > Defaults.MAX_TX_SIZE_IN_KB)
+      return cb(Errors.TX_MAX_SIZE_EXCEEDED);
+
+    var bitcoreError = self._checkTxAndEstimateFee(txp);
+    return cb(bitcoreError);
+
+    // var i = 0;
+    // var total = 0;
+    // var selected = [];
+
+    // var bitcoreTx, bitcoreError;
+
+    // function select() {
+    //   if (i >= inputs.length) return cb(bitcoreError || new Error('Could not select tx inputs'));
+
+    //   var input = inputs[i++];
+    //   selected.push(input);
+    //   total += input.satoshis;
+    //   if (total >= txp.getTotalAmount()) {
+    //     txp.setInputs(selected);
+    //     bitcoreError = self._checkTxAndEstimateFee(txp);
+    //     if (!bitcoreError) return cb();
+    //     if (txp.getEstimatedSize() / 1000 > Defaults.MAX_TX_SIZE_IN_KB)
+    //       return cb(Errors.TX_MAX_SIZE_EXCEEDED);
+    //   }
+    //   setTimeout(select, 0);
+    // };
+
+    // select();
+  });
+};
+
 WalletService.prototype._canCreateTx = function(cb) {
   var self = this;
   self.storage.fetchLastTxs(self.walletId, self.copayerId, 5 + Defaults.BACKOFF_OFFSET, function(err, txs) {
@@ -1527,7 +1692,7 @@ WalletService.prototype.createTxLegacy = function(opts, cb) {
           txp.version = '1.0.1';
         }
 
-        self._selectTxInputs(txp, opts.utxosToExclude, function(err) {
+        self._selectTxInputs2(txp, opts.utxosToExclude, function(err) {
           if (err) return cb(err);
 
           $.checkState(txp.inputs);
@@ -1610,7 +1775,7 @@ WalletService.prototype.createTx = function(opts, cb) {
 
         var txp = Model.TxProposal.create(txOpts);
 
-        self._selectTxInputs(txp, opts.utxosToExclude, function(err) {
+        self._selectTxInputs2(txp, opts.utxosToExclude, function(err) {
           if (err) return cb(err);
 
           self.storage.storeAddressAndWallet(wallet, txp.changeAddress, function(err) {
@@ -2213,7 +2378,7 @@ WalletService.prototype.getTxHistory = function(opts, cb) {
         }
 
         amount = Math.abs(amount);
-        if (action == 'sent' || action == 'moved') {
+        if (action == 'sent' || xaction == 'moved') {
           var firstExternalOutput = _.find(outputs, {
             isMine: false
           });

test/integration/helpers.js

@@ -233,12 +233,15 @@ helpers.stubUtxos = function(server, wallet, amounts, opts, cb) {
       addresses.should.not.be.empty;
 
       var utxos = _.compact(_.map([].concat(amounts), function(amount, i) {
-        var confirmations;
+        var confirmations = _.random(6, 100);
-        if (_.isString(amount) && _.startsWith(amount, 'u')) {
+        if (_.isString(amount)) {
+          if (_.startsWith(amount, 'u')) {
             amount = parseFloat(amount.substring(1));
             confirmations = 0;
-        } else {
+          } else if (_.startsWith(amount, '<6')) {
-          confirmations = Math.floor(Math.random() * 100 + 1);
+            amount = parseFloat(amount.substring(2));
+            confirmations = _.random(1, 5);
+          }
         }
         if (amount <= 0) return null;
 
@@ -257,7 +260,7 @@ helpers.stubUtxos = function(server, wallet, amounts, opts, cb) {
 
         return {
           txid: helpers.randomTXID(),
-          vout: Math.floor(Math.random() * 10 + 1),
+          vout: _.random(0, 10),
           satoshis: helpers.toSatoshi(amount),
           scriptPubKey: scriptPubKey.toBuffer().toString('hex'),
           address: address.address,

test/integration/server.js

@@ -5730,4 +5730,76 @@ describe('Wallet service', function() {
       });
     });
   });
+
+  describe('UTXO Selection', function() {
+    var server, wallet;
+    beforeEach(function(done) {
+      helpers.createAndJoinWallet(2, 3, function(s, w) {
+        server = s;
+        wallet = w;
+        done();
+      });
+    });
+
+    it.skip('should select a single utxo if within thresholds relative to tx amount', function(done) {});
+
+    it('should select smaller utxos if within max fee constraints', function(done) {
+      helpers.stubUtxos(server, wallet, [1, 0.0001, 0.0001, 0.0001], function() {
+        var txOpts = {
+          outputs: [{
+            toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+            amount: 20000,
+          }],
+          feePerKb: 1000,
+        };
+        server.createTx(txOpts, function(err, txp) {
+          should.not.exist(err);
+          should.exist(txp);
+          done();
+        });
+      });
+    });
+    it('should select smallest big utxo if small utxos are insufficient', function(done) {
+      helpers.stubUtxos(server, wallet, [3, 1, 2, 0.0001, 0.0001, 0.0001], function() {
+        var txOpts = {
+          outputs: [{
+            toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+            amount: 30000,
+          }],
+          feePerKb: 1000,
+        };
+        server.createTx(txOpts, function(err, txp) {
+          should.not.exist(err);
+          should.exist(txp);
+          txp.inputs.length.should.equal(1);
+          txp.inputs[0].satoshis.should.equal(1e8);
+          done();
+        });
+      });
+    });
+    it.skip('should select smallest big utxo if small utxos exceed maximum fee', function(done) {});
+    it.only('should ignore utxos not contributing enough to cover increase in fee', function(done) {
+      helpers.stubUtxos(server, wallet, [0.0001, 0.0001, 0.0001], function() {
+        var txOpts = {
+          outputs: [{
+            toAddress: '18PzpUFkFZE8zKWUPvfykkTxmB9oMR8qP7',
+            amount: 20000,
+          }],
+          feePerKb: 8000,
+        };
+        server.createTx(txOpts, function(err, txp) {
+          should.not.exist(err);
+          should.exist(txp);
+          txp.inputs.length.should.equal(3);
+          txOpts.feePerKb = 12000;
+          server.createTx(txOpts, function(err, txp) {
+            should.exist(err);
+            should.not.exist(txp);
+            done();
+          });
+        });
+      });
+    });
+  });
+
 });