From b1261bbf3b11bdc0dbfdbb8e87b3355fc9a97b6c Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 28 May 2014 09:31:59 -0300 Subject: [PATCH 1/2] refactor addr update opts --- app/controllers/addresses.js | 2 +- app/models/Address.js | 177 +++++++++++++++++----------------- lib/BlockDb.js | 4 +- test/integration/addr.js | 2 +- test/integration/addrCache.js | 2 +- 5 files changed, 93 insertions(+), 94 deletions(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 4d5b984a..03ae89aa 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -53,7 +53,7 @@ exports.show = function(req, res, next) { } else { return res.jsonp(a.getObj()); } - }, req.query.noTxList); + }, {noTxList: req.query.noTxList}); } }; diff --git a/app/models/Address.js b/app/models/Address.js index 1e5e8b31..75f975c0 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -92,6 +92,94 @@ Address.prototype.getObj = function() { }; }; +Address.prototype._addTxItem = function(txItem, txList) { + var add=0, addSpend=0; + var v = txItem.value_sat; + var seen = this.seen; + + // Founding tx + if ( !seen[txItem.txid] ) { + seen[txItem.txid]=1; + add=1; + + if (txList) + txList.push({txid: txItem.txid, ts: txItem.ts}); + } + + // Spent tx + if (txItem.spentTxId && !seen[txItem.spentTxId] ) { + if (txList) { + txList.push({txid: txItem.spentTxId, ts: txItem.spentTs}); + } + seen[txItem.spentTxId]=1; + addSpend=1; + } + if (txItem.isConfirmed) { + this.txApperances += add; + this.totalReceivedSat += v; + if (! txItem.spentTxId ) { + //unspent + this.balanceSat += v; + } + else if(!txItem.spentIsConfirmed) { + // unspent + this.balanceSat += v; + this.unconfirmedBalanceSat -= v; + this.unconfirmedTxApperances += addSpend; + } + else { + // spent + this.totalSentSat += v; + this.txApperances += addSpend; + } + } + else { + this.unconfirmedBalanceSat += v; + this.unconfirmedTxApperances += add; + } +}; + +Address.prototype._setTxs = function(txs) { + + // sort input and outputs togheter + txs.sort( + function compare(a,b) { + if (a.ts < b.ts) return 1; + if (a.ts > b.ts) return -1; + return 0; + }); + + this.transactions = txs.map(function(i) { return i.txid; } ); +}; + +Address.prototype.update = function(next, opts) { + var self = this; + if (!self.addrStr) return next(); + opts = opts || {}; + + var txList = opts.noTxList ? null : []; + var tDb = TransactionDb; + var bDb = BlockDb; + tDb.fromAddr(self.addrStr, function(err,txOut){ + if (err) return next(err); + + bDb.fillConfirmations(txOut, function(err) { + if (err) return next(err); + tDb.cacheConfirmations(txOut, function(err) { + if (err) return next(err); + + txOut.forEach(function(txItem){ + self._addTxItem(txItem, txList); + }); + + if (txList) + self._setTxs(txList); + return next(); + }); + }); + }); +}; + Address.prototype.getUtxo = function(next) { var self = this; var tDb = TransactionDb; @@ -124,94 +212,5 @@ Address.prototype.getUtxo = function(next) { }); }; - -Address.prototype._addTxItem = function(txItem, notxlist) { - var add=0, addSpend=0; - var v = txItem.value_sat; - var seen = this.seen; - var txs = []; - - if ( !seen[txItem.txid] ) { - if (!notxlist) { - txs.push({txid: txItem.txid, ts: txItem.ts}); - } - seen[txItem.txid]=1; - add=1; - } - - if (txItem.spentTxId && !seen[txItem.spentTxId] ) { - if (!notxlist) { - txs.push({txid: txItem.spentTxId, ts: txItem.spentTs}); - } - seen[txItem.spentTxId]=1; - addSpend=1; - } - if (txItem.isConfirmed) { - this.txApperances += add; - this.totalReceivedSat += v; - if (! txItem.spentTxId ) { - //unspent - this.balanceSat += v; - } - else if(!txItem.spentIsConfirmed) { - // unspent - this.balanceSat += v; - this.unconfirmedBalanceSat -= v; - this.unconfirmedTxApperances += addSpend; - } - else { - // spent - this.totalSentSat += v; - this.txApperances += addSpend; - } - } - else { - this.unconfirmedBalanceSat += v; - this.unconfirmedTxApperances += add; - } - - return txs; -}; - -Address.prototype._setTxs = function(txs) { - - // sort input and outputs togheter - txs.sort( - function compare(a,b) { - if (a.ts < b.ts) return 1; - if (a.ts > b.ts) return -1; - return 0; - }); - - this.transactions = txs.map(function(i) { return i.txid; } ); -}; - -Address.prototype.update = function(next, notxlist) { - var self = this; - if (!self.addrStr) return next(); - - var txs = []; - var tDb = TransactionDb; - var bDb = BlockDb; - tDb.fromAddr(self.addrStr, function(err,txOut){ - if (err) return next(err); - - bDb.fillConfirmations(txOut, function(err) { - if (err) return next(err); - tDb.cacheConfirmations(txOut, function(err) { - if (err) return next(err); - - txOut.forEach(function(txItem){ - txs=txs.concat(self._addTxItem(txItem, notxlist)); - }); - - if (!notxlist) - self._setTxs(txs); - return next(); - }); - }); - }); -}; - module.exports = require('soop')(Address); diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 80562705..a172c0e8 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -366,8 +366,8 @@ BlockDb.prototype.fillConfirmations = function(txouts, cb) { var self = this; this.getTip(function(err, hash, height){ var txs = txouts.filter(function(x){ - return !x.spentIsConfirmedCached // not 100%cached - && !(x.isConfirmedCached && !x.spentTxId); // and not 50%cached but not spent + return !x.spentIsConfirmedCached // not 100%cached + && !(x.isConfirmedCached && !x.spentTxId); // and not partial cached but not spent }); //console.log('[BlockDb.js.373:txs:]',txs.length, txs.slice(0,5)); //TODO diff --git a/test/integration/addr.js b/test/integration/addr.js index 8b527357..d18cf2fe 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -68,7 +68,7 @@ describe('Address balances', function() { if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent); if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance); done(); - },1); + },{noTxList:1}); }); } }); diff --git a/test/integration/addrCache.js b/test/integration/addrCache.js index 556775e2..2e15d6bb 100644 --- a/test/integration/addrCache.js +++ b/test/integration/addrCache.js @@ -91,7 +91,7 @@ describe('Address cache ', function() { a.totalReceived.should.equal(1376000, 'totalReceived'); a.txApperances.should.equal(8003, 'txApperances'); return done(); - },1); + },{noTxList:1}); }); }); From 3ab0213d5bd9fd74aec98c72d0ec68b685a50628 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 28 May 2014 09:50:01 -0300 Subject: [PATCH 2/2] add cache info on readme --- README.md | 40 ++++++++++++++++++++++++++ app/controllers/addresses.js | 12 ++++---- app/models/Address.js | 54 ++++++++++++++++++++++-------------- test/integration/addr.js | 22 +++++++-------- 4 files changed, 90 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index eecd3489..ca29c6c8 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,18 @@ must be stopped. Alternatively, a total resync can be made, running `$ util/sync.js -D` +## IMPORTANT: v0.2 Caching schema + +In v0.2 a new cache schema has been introduced. Only information from transactions with +SAFE_CONFIRMATIONS+ settings will be cached (by default SAFE_CONFIRMATIONS=6). There +are 3 different caches: + * nr. of confirmations + * transaction spent information + * scriptPubKey for unspent transactions + +Cache data is only completed on request, i.e., only after accessing the required data for +the first time, the information is cached, there is not pre-caching procedure. + ## Prerequisites * **bitcoind** - Download and Install [Bitcoin](http://bitcoin.org/en/download) @@ -79,6 +91,7 @@ BITCOIND_PASS # RPC password BITCOIND_DATADIR # bitcoind datadir for livenet, or datadir/testnet3 for testnet INSIGHT_NETWORK [= 'livenet' | 'testnet'] INSIGHT_DB # Path where to store insight's internal DB. (defaults to $HOME/.insight) +SAFE_CONFIRMATIONS=6 # Nr. of confirmation needed to start caching transaction information ``` Make sure that bitcoind is configured to [accept incoming connections using 'rpcallowip'](https://en.bitcoin.it/wiki/Running_Bitcoin). @@ -157,6 +170,33 @@ The end-points are: ``` /api/addr/[:addr]/utxo ``` +Sample return: +``` json +[ + { + address: "n2PuaAguxZqLddRbTnAoAuwKYgN2w2hZk7", + txid: "dbfdc2a0d22a8282c4e7be0452d595695f3a39173bed4f48e590877382b112fc", + vout: 0, + ts: 1401276201, + scriptPubKey: "76a914e50575162795cd77366fb80d728e3216bd52deac88ac", + amount: 0.001, + confirmations: 3 + }, + { + address: "n2PuaAguxZqLddRbTnAoAuwKYgN2w2hZk7", + txid: "e2b82af55d64f12fd0dd075d0922ee7d6a300f58fe60a23cbb5831b31d1d58b4", + vout: 0, + ts: 1401226410, + scriptPubKey: "76a914e50575162795cd77366fb80d728e3216bd52deac88ac", + amount: 0.001, + confirmations: "6+" + } +] +``` +Please not that in case confirmations are cached and are more that SAFE_CONFIRMATIONS setting, the +return can be a string of the form 'SAFE_CONFIRMATIONS+' + + ### Unspent Outputs for multiple addresses GET method: ``` diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 03ae89aa..089990a4 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -62,13 +62,13 @@ exports.show = function(req, res, next) { exports.utxo = function(req, res, next) { var a = getAddr(req, res, next); if (a) { - a.getUtxo(function(err, utxo) { + a.update(function(err) { if (err) return common.handleErrors(err, res); else { - return res.jsonp(utxo); + return res.jsonp(a.unspent); } - }); + }, {onlyUnspent: 1}); } }; @@ -77,11 +77,11 @@ exports.multiutxo = function(req, res, next) { if (as) { var utxos = []; async.each(as, function(a, callback) { - a.getUtxo(function(err, utxo) { + a.update(function(err) { if (err) callback(err); - utxos = utxos.concat(utxo); + utxos = utxos.concat(a.unspent); callback(); - }); + }, {onlyUnspent:1}); }, function(err) { // finished callback if (err) return common.handleErrors(err, res); res.jsonp(utxos); diff --git a/app/models/Address.js b/app/models/Address.js index 75f975c0..8629c6ab 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -26,6 +26,7 @@ function Address(addrStr) { // TODO store only txids? +index? +all? this.transactions = []; + this.unspent = []; var a = new BitcoreAddress(addrStr); a.validate(); @@ -152,6 +153,10 @@ Address.prototype._setTxs = function(txs) { this.transactions = txs.map(function(i) { return i.txid; } ); }; +// opts are +// .noTxList +// .onlyUnspent +// .noSortTxs Address.prototype.update = function(next, opts) { var self = this; if (!self.addrStr) return next(); @@ -165,21 +170,42 @@ Address.prototype.update = function(next, opts) { bDb.fillConfirmations(txOut, function(err) { if (err) return next(err); + tDb.cacheConfirmations(txOut, function(err) { if (err) return next(err); - txOut.forEach(function(txItem){ - self._addTxItem(txItem, txList); - }); + if (opts.onlyUnspent) { + txOut = txOut.filter(function(x){ + return !x.spentTxId; + }); + tDb.fillScriptPubKey(txOut, function() { + self.unspent = txOut.map(function(x){ + return { + address: self.addrStr, + txid: x.txid, + vout: x.index, + ts: x.ts, + scriptPubKey: x.scriptPubKey, + amount: x.value_sat / BitcoreUtil.COIN, + confirmations: x.isConfirmedCached ? (config.safeConfirmations+'+') : x.confirmations, + }; + }); + return next(); + }); + } + else { + txOut.forEach(function(txItem){ + self._addTxItem(txItem, txList); + }); - if (txList) - self._setTxs(txList); - return next(); + if (txList && !opts.noSortTxs) + self._setTxs(txList); + return next(); + } }); }); }); }; - Address.prototype.getUtxo = function(next) { var self = this; var tDb = TransactionDb; @@ -194,20 +220,6 @@ Address.prototype.getUtxo = function(next) { }); bDb.fillConfirmations(unspent, function() { - tDb.fillScriptPubKey(unspent, function() { - ret = unspent.map(function(x){ - return { - address: self.addrStr, - txid: x.txid, - vout: x.index, - ts: x.ts, - scriptPubKey: x.scriptPubKey, - amount: x.value_sat / BitcoreUtil.COIN, - confirmations: x.isConfirmedCached ? (config.safeConfirmations+'+') : x.confirmations, - }; - }); - return next(null, ret); - }); }); }); }; diff --git a/test/integration/addr.js b/test/integration/addr.js index d18cf2fe..8e7ee396 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -76,7 +76,7 @@ describe('Address balances', function() { }); //tested against https://api.biteasy.com/testnet/v1/addresses/2N1pLkosf6o8Ciqs573iwwgVpuFS6NbNKx5/unspent-outputs?per_page=40 -describe('Address utxo', function() { +describe('Address unspent', function() { before(function(c) { txDb = TransactionDb; @@ -96,15 +96,15 @@ describe('Address utxo', function() { if (v.disabled) { console.log(v.addr + ' => disabled in JSON'); } else { - it('Address utxo for: ' + v.addr, function(done) { + it('Address unspent for: ' + v.addr, function(done) { this.timeout(2000); var a = new Address(v.addr, txDb); - a.getUtxo(function(err, utxo) { + a.update(function(err) { if (err) done(err); assert.equal(v.addr, a.addrStr); - if (v.length) utxo.length.should.equal(v.length, 'Unspent count'); + if (v.length) a.unspent.length.should.equal(v.length, 'Unspent count'); if (v.tx0id) { - var x=utxo.filter(function(x){ + var x=a.unspent.filter(function(x){ return x.txid === v.tx0id; }); assert(x,'found output'); @@ -113,17 +113,17 @@ describe('Address utxo', function() { x[0].amount.should.equal(v.tx0amount,'amount'); } done(); - }); + }, {onlyUnspent:1}); }); - it('Address utxo (cached) for: ' + v.addr, function(done) { + it('Address unspent (cached) for: ' + v.addr, function(done) { this.timeout(2000); var a = new Address(v.addr, txDb); - a.getUtxo(function(err, utxo) { + a.update(function(err) { if (err) done(err); assert.equal(v.addr, a.addrStr); - if (v.length) utxo.length.should.equal(v.length, 'Unspent count'); + if (v.length) a.unspent.length.should.equal(v.length, 'Unspent count'); if (v.tx0id) { - var x=utxo.filter(function(x){ + var x=a.unspent.filter(function(x){ return x.txid === v.tx0id; }); assert(x,'found output'); @@ -132,7 +132,7 @@ describe('Address utxo', function() { x[0].amount.should.equal(v.tx0amount,'amount'); } done(); - }); + }, {onlyUnspent:1}); }); } });