Merge pull request #100 from matiu/feature/refactorAddress2

Feature/refactor address2

README.md

@@ -23,6 +23,18 @@ must be stopped.
 
 Alternatively, a total resync can be made, running `$ util/sync.js -D`
 
+## IMPORTANT: v0.2 Caching schema
+
+In v0.2 a new cache schema has been introduced. Only information from transactions with
+SAFE_CONFIRMATIONS+ settings will be cached (by default SAFE_CONFIRMATIONS=6). There 
+are 3 different caches:
+ * nr. of confirmations 
+ * transaction spent information
+ * scriptPubKey for unspent transactions
+
+Cache data is only completed on request, i.e., only after accessing the required data for
+the first time, the information is cached, there is not pre-caching procedure.
+
 ## Prerequisites
 
 * **bitcoind** - Download and Install [Bitcoin](http://bitcoin.org/en/download)
@@ -79,6 +91,7 @@ BITCOIND_PASS         # RPC password
 BITCOIND_DATADIR      # bitcoind datadir for livenet, or datadir/testnet3 for testnet
 INSIGHT_NETWORK [= 'livenet' | 'testnet']
 INSIGHT_DB            # Path where to store insight's internal DB. (defaults to $HOME/.insight)
+SAFE_CONFIRMATIONS=6  # Nr. of confirmation needed to start caching transaction information   
 ```
 
 Make sure that bitcoind is configured to [accept incoming connections using 'rpcallowip'](https://en.bitcoin.it/wiki/Running_Bitcoin).
@@ -157,6 +170,33 @@ The end-points are:
 ```
   /api/addr/[:addr]/utxo
 ```
+Sample return:
+``` json
+[
+    {
+    address: "n2PuaAguxZqLddRbTnAoAuwKYgN2w2hZk7",
+    txid: "dbfdc2a0d22a8282c4e7be0452d595695f3a39173bed4f48e590877382b112fc",
+    vout: 0,
+    ts: 1401276201,
+    scriptPubKey: "76a914e50575162795cd77366fb80d728e3216bd52deac88ac",
+    amount: 0.001,
+    confirmations: 3
+    },
+    {
+    address: "n2PuaAguxZqLddRbTnAoAuwKYgN2w2hZk7",
+    txid: "e2b82af55d64f12fd0dd075d0922ee7d6a300f58fe60a23cbb5831b31d1d58b4",
+    vout: 0,
+    ts: 1401226410,
+    scriptPubKey: "76a914e50575162795cd77366fb80d728e3216bd52deac88ac",
+    amount: 0.001,
+    confirmations: "6+"
+    }
+]
+```
+Please not that in case confirmations are cached and are more that SAFE_CONFIRMATIONS setting, the
+return can be a string of the form 'SAFE_CONFIRMATIONS+'
+
+
 ### Unspent Outputs for multiple addresses
 GET method:
 ```

app/controllers/addresses.js

@@ -53,7 +53,7 @@ exports.show = function(req, res, next) {
       } else {
         return res.jsonp(a.getObj());
       }
-    }, req.query.noTxList);
+    }, {noTxList: req.query.noTxList});
   }
 };
 
@@ -62,13 +62,13 @@ exports.show = function(req, res, next) {
 exports.utxo = function(req, res, next) {
   var a = getAddr(req, res, next);
   if (a) {
-    a.getUtxo(function(err, utxo) {
+    a.update(function(err) {
       if (err)
         return common.handleErrors(err, res);
       else {
-        return res.jsonp(utxo);
+        return res.jsonp(a.unspent);
       }
-    });
+    }, {onlyUnspent: 1});
   }
 };
 
@@ -77,11 +77,11 @@ exports.multiutxo = function(req, res, next) {
   if (as) {
     var utxos = [];
     async.each(as, function(a, callback) {
-      a.getUtxo(function(err, utxo) {
+      a.update(function(err) {
         if (err) callback(err);
-        utxos = utxos.concat(utxo);
+        utxos = utxos.concat(a.unspent);
         callback();
-      });
+      }, {onlyUnspent:1});
     }, function(err) { // finished callback
       if (err) return common.handleErrors(err, res);
       res.jsonp(utxos);

app/models/Address.js

@@ -26,6 +26,7 @@ function Address(addrStr) {
 
   // TODO store only txids? +index? +all?
   this.transactions   = [];
+  this.unspent   = [];
 
   var a = new BitcoreAddress(addrStr);
   a.validate();
@@ -92,56 +93,24 @@ Address.prototype.getObj = function() {
   };
 };
 
-Address.prototype.getUtxo = function(next) {
+Address.prototype._addTxItem = function(txItem, txList) {
-  var self = this;
-  var tDb   = TransactionDb;
-  var bDb   = BlockDb;
-  var ret;
-  if (!self.addrStr) return next(new Error('no error'));
-
-  tDb.fromAddr(self.addrStr, function(err,txOut){
-    if (err) return next(err);
-    var unspent = txOut.filter(function(x){
-      return !x.spentTxId;
-    });
-
-    bDb.fillConfirmations(unspent, function() {
-      tDb.fillScriptPubKey(unspent, function() {
-        ret = unspent.map(function(x){
-          return {
-            address: self.addrStr,
-            txid: x.txid,
-            vout: x.index,
-            ts: x.ts,
-            scriptPubKey: x.scriptPubKey,
-            amount: x.value_sat / BitcoreUtil.COIN,
-            confirmations: x.isConfirmedCached ? (config.safeConfirmations+'+') : x.confirmations,
-          };
-        });
-        return next(null, ret);
-      });
-    });
-  });
-};
-
-
-Address.prototype._addTxItem = function(txItem, notxlist) {
   var add=0, addSpend=0;
   var v = txItem.value_sat;
   var seen = this.seen;
-  var txs = [];
 
+  // Founding tx
   if ( !seen[txItem.txid] ) {
-    if (!notxlist) {
-      txs.push({txid: txItem.txid, ts: txItem.ts});
-    }
     seen[txItem.txid]=1;
     add=1;
+
+    if (txList)
+      txList.push({txid: txItem.txid, ts: txItem.ts});
   }
 
+  // Spent tx
   if (txItem.spentTxId && !seen[txItem.spentTxId]  ) {
-    if (!notxlist) {
+    if (txList) {
-      txs.push({txid: txItem.spentTxId, ts: txItem.spentTs});
+      txList.push({txid: txItem.spentTxId, ts: txItem.spentTs});
     }
     seen[txItem.spentTxId]=1;
     addSpend=1;
@@ -169,8 +138,6 @@ Address.prototype._addTxItem = function(txItem, notxlist) {
     this.unconfirmedBalanceSat += v;
     this.unconfirmedTxApperances += add;
   }
-
-  return txs;
 };
 
 Address.prototype._setTxs = function(txs) {
@@ -186,11 +153,16 @@ Address.prototype._setTxs = function(txs) {
   this.transactions = txs.map(function(i) { return i.txid; } );
 };
 
-Address.prototype.update = function(next, notxlist) {
+// opts are
+// .noTxList
+// .onlyUnspent
+// .noSortTxs
+Address.prototype.update = function(next, opts) {
   var self = this;
   if (!self.addrStr) return next();
+  opts = opts || {};
 
-  var txs  = [];
+  var txList  = opts.noTxList ? null : [];
   var tDb   = TransactionDb;
   var bDb   = BlockDb;
   tDb.fromAddr(self.addrStr, function(err,txOut){
@@ -198,20 +170,59 @@ Address.prototype.update = function(next, notxlist) {
 
     bDb.fillConfirmations(txOut, function(err) {
       if (err) return next(err);
+
       tDb.cacheConfirmations(txOut, function(err) {
         if (err) return next(err);
 
-        txOut.forEach(function(txItem){
+        if (opts.onlyUnspent) {
-          txs=txs.concat(self._addTxItem(txItem, notxlist));
+          txOut  = txOut.filter(function(x){
-        });
+            return !x.spentTxId;
+          });
+          tDb.fillScriptPubKey(txOut, function() {
+            self.unspent = txOut.map(function(x){
+              return {
+                address: self.addrStr,
+                txid: x.txid,
+                vout: x.index,
+                ts: x.ts,
+                scriptPubKey: x.scriptPubKey,
+                amount: x.value_sat / BitcoreUtil.COIN,
+                confirmations: x.isConfirmedCached ? (config.safeConfirmations+'+') : x.confirmations,
+              };
+            });
+            return next();
+          });
+        }
+        else {
+          txOut.forEach(function(txItem){
+            self._addTxItem(txItem, txList);
+          });
 
-        if (!notxlist)
+          if (txList && !opts.noSortTxs)
-          self._setTxs(txs);
+            self._setTxs(txList);
-        return next();
+          return next();
+        }
       });
     });
   });
 };
+Address.prototype.getUtxo = function(next) {
+  var self = this;
+  var tDb   = TransactionDb;
+  var bDb   = BlockDb;
+  var ret;
+  if (!self.addrStr) return next(new Error('no error'));
+
+  tDb.fromAddr(self.addrStr, function(err,txOut){
+    if (err) return next(err);
+    var unspent = txOut.filter(function(x){
+      return !x.spentTxId;
+    });
+
+    bDb.fillConfirmations(unspent, function() {
+    });
+  });
+};
 
 module.exports = require('soop')(Address);
 

lib/BlockDb.js

@@ -366,8 +366,8 @@ BlockDb.prototype.fillConfirmations = function(txouts, cb) {
   var self = this;
   this.getTip(function(err, hash, height){
     var txs = txouts.filter(function(x){
-      return !x.spentIsConfirmedCached // not 100%cached
+      return !x.spentIsConfirmedCached              // not 100%cached
-        && !(x.isConfirmedCached && !x.spentTxId);  // and not 50%cached but not spent 
+        && !(x.isConfirmedCached && !x.spentTxId);  // and not partial cached but not spent 
     });
     //console.log('[BlockDb.js.373:txs:]',txs.length, txs.slice(0,5)); //TODO
 

test/integration/addr.js

@@ -68,7 +68,7 @@ describe('Address balances', function() {
           if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent);
           if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance);
           done();
-        },1);
+        },{noTxList:1});
       });
     }
   });
@@ -76,7 +76,7 @@ describe('Address balances', function() {
 });
 
 //tested against https://api.biteasy.com/testnet/v1/addresses/2N1pLkosf6o8Ciqs573iwwgVpuFS6NbNKx5/unspent-outputs?per_page=40
-describe('Address utxo', function() {
+describe('Address unspent', function() {
 
   before(function(c) {
     txDb = TransactionDb;
@@ -96,15 +96,15 @@ describe('Address utxo', function() {
     if (v.disabled) {
       console.log(v.addr + ' => disabled in JSON');
     } else {
-      it('Address utxo for: ' + v.addr, function(done) {
+      it('Address unspent for: ' + v.addr, function(done) {
         this.timeout(2000);
         var a = new Address(v.addr, txDb);
-        a.getUtxo(function(err, utxo) {
+        a.update(function(err) {
           if (err) done(err);
           assert.equal(v.addr, a.addrStr);
-          if (v.length) utxo.length.should.equal(v.length, 'Unspent count');
+          if (v.length) a.unspent.length.should.equal(v.length, 'Unspent count');
           if (v.tx0id) {
-            var x=utxo.filter(function(x){
+            var x=a.unspent.filter(function(x){
                 return x.txid === v.tx0id;
             });
             assert(x,'found output');
@@ -113,17 +113,17 @@ describe('Address utxo', function() {
             x[0].amount.should.equal(v.tx0amount,'amount');
           }
           done();
-        });
+        }, {onlyUnspent:1});
       });
-      it('Address utxo (cached) for: ' + v.addr, function(done) {
+      it('Address unspent (cached) for: ' + v.addr, function(done) {
         this.timeout(2000);
         var a = new Address(v.addr, txDb);
-        a.getUtxo(function(err, utxo) {
+        a.update(function(err) {
           if (err) done(err);
           assert.equal(v.addr, a.addrStr);
-          if (v.length) utxo.length.should.equal(v.length, 'Unspent count');
+          if (v.length) a.unspent.length.should.equal(v.length, 'Unspent count');
           if (v.tx0id) {
-            var x=utxo.filter(function(x){
+            var x=a.unspent.filter(function(x){
                 return x.txid === v.tx0id;
             });
             assert(x,'found output');
@@ -132,7 +132,7 @@ describe('Address utxo', function() {
             x[0].amount.should.equal(v.tx0amount,'amount');
           }
           done();
-        });
+        }, {onlyUnspent:1});
       });
     }
   });

test/integration/addrCache.js

@@ -91,7 +91,7 @@ describe('Address cache ', function() {
       a.totalReceived.should.equal(1376000, 'totalReceived');
       a.txApperances.should.equal(8003, 'txApperances');
       return done();
-    },1);
+    },{noTxList:1});
   });
 });