diff --git a/README.md b/README.md index eecd3489..ca29c6c8 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,18 @@ must be stopped. Alternatively, a total resync can be made, running `$ util/sync.js -D` +## IMPORTANT: v0.2 Caching schema + +In v0.2 a new cache schema has been introduced. Only information from transactions with +SAFE_CONFIRMATIONS+ settings will be cached (by default SAFE_CONFIRMATIONS=6). There +are 3 different caches: + * nr. of confirmations + * transaction spent information + * scriptPubKey for unspent transactions + +Cache data is only completed on request, i.e., only after accessing the required data for +the first time, the information is cached, there is not pre-caching procedure. + ## Prerequisites * **bitcoind** - Download and Install [Bitcoin](http://bitcoin.org/en/download) @@ -79,6 +91,7 @@ BITCOIND_PASS # RPC password BITCOIND_DATADIR # bitcoind datadir for livenet, or datadir/testnet3 for testnet INSIGHT_NETWORK [= 'livenet' | 'testnet'] INSIGHT_DB # Path where to store insight's internal DB. (defaults to $HOME/.insight) +SAFE_CONFIRMATIONS=6 # Nr. of confirmation needed to start caching transaction information ``` Make sure that bitcoind is configured to [accept incoming connections using 'rpcallowip'](https://en.bitcoin.it/wiki/Running_Bitcoin). @@ -157,6 +170,33 @@ The end-points are: ``` /api/addr/[:addr]/utxo ``` +Sample return: +``` json +[ + { + address: "n2PuaAguxZqLddRbTnAoAuwKYgN2w2hZk7", + txid: "dbfdc2a0d22a8282c4e7be0452d595695f3a39173bed4f48e590877382b112fc", + vout: 0, + ts: 1401276201, + scriptPubKey: "76a914e50575162795cd77366fb80d728e3216bd52deac88ac", + amount: 0.001, + confirmations: 3 + }, + { + address: "n2PuaAguxZqLddRbTnAoAuwKYgN2w2hZk7", + txid: "e2b82af55d64f12fd0dd075d0922ee7d6a300f58fe60a23cbb5831b31d1d58b4", + vout: 0, + ts: 1401226410, + scriptPubKey: "76a914e50575162795cd77366fb80d728e3216bd52deac88ac", + amount: 0.001, + confirmations: "6+" + } +] +``` +Please not that in case confirmations are cached and are more that SAFE_CONFIRMATIONS setting, the +return can be a string of the form 'SAFE_CONFIRMATIONS+' + + ### Unspent Outputs for multiple addresses GET method: ``` diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 4d5b984a..089990a4 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -53,7 +53,7 @@ exports.show = function(req, res, next) { } else { return res.jsonp(a.getObj()); } - }, req.query.noTxList); + }, {noTxList: req.query.noTxList}); } }; @@ -62,13 +62,13 @@ exports.show = function(req, res, next) { exports.utxo = function(req, res, next) { var a = getAddr(req, res, next); if (a) { - a.getUtxo(function(err, utxo) { + a.update(function(err) { if (err) return common.handleErrors(err, res); else { - return res.jsonp(utxo); + return res.jsonp(a.unspent); } - }); + }, {onlyUnspent: 1}); } }; @@ -77,11 +77,11 @@ exports.multiutxo = function(req, res, next) { if (as) { var utxos = []; async.each(as, function(a, callback) { - a.getUtxo(function(err, utxo) { + a.update(function(err) { if (err) callback(err); - utxos = utxos.concat(utxo); + utxos = utxos.concat(a.unspent); callback(); - }); + }, {onlyUnspent:1}); }, function(err) { // finished callback if (err) return common.handleErrors(err, res); res.jsonp(utxos); diff --git a/app/models/Address.js b/app/models/Address.js index 1e5e8b31..8629c6ab 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -26,6 +26,7 @@ function Address(addrStr) { // TODO store only txids? +index? +all? this.transactions = []; + this.unspent = []; var a = new BitcoreAddress(addrStr); a.validate(); @@ -92,56 +93,24 @@ Address.prototype.getObj = function() { }; }; -Address.prototype.getUtxo = function(next) { - var self = this; - var tDb = TransactionDb; - var bDb = BlockDb; - var ret; - if (!self.addrStr) return next(new Error('no error')); - - tDb.fromAddr(self.addrStr, function(err,txOut){ - if (err) return next(err); - var unspent = txOut.filter(function(x){ - return !x.spentTxId; - }); - - bDb.fillConfirmations(unspent, function() { - tDb.fillScriptPubKey(unspent, function() { - ret = unspent.map(function(x){ - return { - address: self.addrStr, - txid: x.txid, - vout: x.index, - ts: x.ts, - scriptPubKey: x.scriptPubKey, - amount: x.value_sat / BitcoreUtil.COIN, - confirmations: x.isConfirmedCached ? (config.safeConfirmations+'+') : x.confirmations, - }; - }); - return next(null, ret); - }); - }); - }); -}; - - -Address.prototype._addTxItem = function(txItem, notxlist) { +Address.prototype._addTxItem = function(txItem, txList) { var add=0, addSpend=0; var v = txItem.value_sat; var seen = this.seen; - var txs = []; + // Founding tx if ( !seen[txItem.txid] ) { - if (!notxlist) { - txs.push({txid: txItem.txid, ts: txItem.ts}); - } seen[txItem.txid]=1; add=1; + + if (txList) + txList.push({txid: txItem.txid, ts: txItem.ts}); } + // Spent tx if (txItem.spentTxId && !seen[txItem.spentTxId] ) { - if (!notxlist) { - txs.push({txid: txItem.spentTxId, ts: txItem.spentTs}); + if (txList) { + txList.push({txid: txItem.spentTxId, ts: txItem.spentTs}); } seen[txItem.spentTxId]=1; addSpend=1; @@ -169,8 +138,6 @@ Address.prototype._addTxItem = function(txItem, notxlist) { this.unconfirmedBalanceSat += v; this.unconfirmedTxApperances += add; } - - return txs; }; Address.prototype._setTxs = function(txs) { @@ -186,11 +153,16 @@ Address.prototype._setTxs = function(txs) { this.transactions = txs.map(function(i) { return i.txid; } ); }; -Address.prototype.update = function(next, notxlist) { +// opts are +// .noTxList +// .onlyUnspent +// .noSortTxs +Address.prototype.update = function(next, opts) { var self = this; if (!self.addrStr) return next(); + opts = opts || {}; - var txs = []; + var txList = opts.noTxList ? null : []; var tDb = TransactionDb; var bDb = BlockDb; tDb.fromAddr(self.addrStr, function(err,txOut){ @@ -198,20 +170,59 @@ Address.prototype.update = function(next, notxlist) { bDb.fillConfirmations(txOut, function(err) { if (err) return next(err); + tDb.cacheConfirmations(txOut, function(err) { if (err) return next(err); - txOut.forEach(function(txItem){ - txs=txs.concat(self._addTxItem(txItem, notxlist)); - }); + if (opts.onlyUnspent) { + txOut = txOut.filter(function(x){ + return !x.spentTxId; + }); + tDb.fillScriptPubKey(txOut, function() { + self.unspent = txOut.map(function(x){ + return { + address: self.addrStr, + txid: x.txid, + vout: x.index, + ts: x.ts, + scriptPubKey: x.scriptPubKey, + amount: x.value_sat / BitcoreUtil.COIN, + confirmations: x.isConfirmedCached ? (config.safeConfirmations+'+') : x.confirmations, + }; + }); + return next(); + }); + } + else { + txOut.forEach(function(txItem){ + self._addTxItem(txItem, txList); + }); - if (!notxlist) - self._setTxs(txs); - return next(); + if (txList && !opts.noSortTxs) + self._setTxs(txList); + return next(); + } }); }); }); }; +Address.prototype.getUtxo = function(next) { + var self = this; + var tDb = TransactionDb; + var bDb = BlockDb; + var ret; + if (!self.addrStr) return next(new Error('no error')); + + tDb.fromAddr(self.addrStr, function(err,txOut){ + if (err) return next(err); + var unspent = txOut.filter(function(x){ + return !x.spentTxId; + }); + + bDb.fillConfirmations(unspent, function() { + }); + }); +}; module.exports = require('soop')(Address); diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 80562705..a172c0e8 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -366,8 +366,8 @@ BlockDb.prototype.fillConfirmations = function(txouts, cb) { var self = this; this.getTip(function(err, hash, height){ var txs = txouts.filter(function(x){ - return !x.spentIsConfirmedCached // not 100%cached - && !(x.isConfirmedCached && !x.spentTxId); // and not 50%cached but not spent + return !x.spentIsConfirmedCached // not 100%cached + && !(x.isConfirmedCached && !x.spentTxId); // and not partial cached but not spent }); //console.log('[BlockDb.js.373:txs:]',txs.length, txs.slice(0,5)); //TODO diff --git a/test/integration/addr.js b/test/integration/addr.js index 8b527357..8e7ee396 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -68,7 +68,7 @@ describe('Address balances', function() { if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent); if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance); done(); - },1); + },{noTxList:1}); }); } }); @@ -76,7 +76,7 @@ describe('Address balances', function() { }); //tested against https://api.biteasy.com/testnet/v1/addresses/2N1pLkosf6o8Ciqs573iwwgVpuFS6NbNKx5/unspent-outputs?per_page=40 -describe('Address utxo', function() { +describe('Address unspent', function() { before(function(c) { txDb = TransactionDb; @@ -96,15 +96,15 @@ describe('Address utxo', function() { if (v.disabled) { console.log(v.addr + ' => disabled in JSON'); } else { - it('Address utxo for: ' + v.addr, function(done) { + it('Address unspent for: ' + v.addr, function(done) { this.timeout(2000); var a = new Address(v.addr, txDb); - a.getUtxo(function(err, utxo) { + a.update(function(err) { if (err) done(err); assert.equal(v.addr, a.addrStr); - if (v.length) utxo.length.should.equal(v.length, 'Unspent count'); + if (v.length) a.unspent.length.should.equal(v.length, 'Unspent count'); if (v.tx0id) { - var x=utxo.filter(function(x){ + var x=a.unspent.filter(function(x){ return x.txid === v.tx0id; }); assert(x,'found output'); @@ -113,17 +113,17 @@ describe('Address utxo', function() { x[0].amount.should.equal(v.tx0amount,'amount'); } done(); - }); + }, {onlyUnspent:1}); }); - it('Address utxo (cached) for: ' + v.addr, function(done) { + it('Address unspent (cached) for: ' + v.addr, function(done) { this.timeout(2000); var a = new Address(v.addr, txDb); - a.getUtxo(function(err, utxo) { + a.update(function(err) { if (err) done(err); assert.equal(v.addr, a.addrStr); - if (v.length) utxo.length.should.equal(v.length, 'Unspent count'); + if (v.length) a.unspent.length.should.equal(v.length, 'Unspent count'); if (v.tx0id) { - var x=utxo.filter(function(x){ + var x=a.unspent.filter(function(x){ return x.txid === v.tx0id; }); assert(x,'found output'); @@ -132,7 +132,7 @@ describe('Address utxo', function() { x[0].amount.should.equal(v.tx0amount,'amount'); } done(); - }); + }, {onlyUnspent:1}); }); } }); diff --git a/test/integration/addrCache.js b/test/integration/addrCache.js index 556775e2..2e15d6bb 100644 --- a/test/integration/addrCache.js +++ b/test/integration/addrCache.js @@ -91,7 +91,7 @@ describe('Address cache ', function() { a.totalReceived.should.equal(1376000, 'totalReceived'); a.txApperances.should.equal(8003, 'txApperances'); return done(); - },1); + },{noTxList:1}); }); });