From e61045f553f42207d50426a726f6fe08423ff99a Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 20 May 2014 18:07:25 -0300 Subject: [PATCH] add block height to level DB --- app/models/Address.js | 4 ++-- lib/BlockDb.js | 38 ++++++++++++++++++++++---------------- lib/HistoricSync.js | 9 ++------- lib/PeerSync.js | 1 + lib/Sync.js | 38 ++++++++++++++++++++++---------------- lib/TransactionDb.js | 28 ++++++++++++++-------------- test/integration/addr.js | 1 + test/integration/utxo.json | 7 +++++++ 8 files changed, 71 insertions(+), 55 deletions(-) diff --git a/app/models/Address.js b/app/models/Address.js index 722607f2..114fdb59 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -107,7 +107,7 @@ Address.prototype.getUtxo = function(next) { ts: txItem.ts, scriptPubKey: scriptPubKey, amount: txItem.value_sat / BitcoreUtil.COIN, - confirmations: txItem.isConfirmed ? info.confirmations : 0, + confirmations: txItem.height ? info.confirmations : 0, }); } return a_c(err); @@ -149,7 +149,7 @@ Address.prototype.update = function(next, notxlist) { addSpend=1; } - if (txItem.isConfirmed) { + if (txItem.height) { self.txApperances += add; self.totalReceivedSat += v; if (! txItem.spentTxId ) { diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 0ce250e5..db9817d5 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -1,11 +1,11 @@ 'use strict'; var imports = require('soop').imports(); var ThisParent = imports.parent || require('events').EventEmitter; -var TIMESTAMP_PREFIX = 'bts-'; // b-ts- => -var PREV_PREFIX = 'bpr-'; // b-prev- => -var NEXT_PREFIX = 'bne-'; // b-next- => -var MAIN_PREFIX = 'bma-'; // b-main- => 1/0 -var TIP = 'bti-'; // last block on the chain +var TIMESTAMP_PREFIX = 'bts-'; // bts- => +var PREV_PREFIX = 'bpr-'; // bpr- => +var NEXT_PREFIX = 'bne-'; // bne- => +var MAIN_PREFIX = 'bma-'; // bma- => (0 is unconnected) +var TIP = 'bti-'; // bti = : last block on the chain var LAST_FILE_INDEX = 'file-'; // last processed file index var MAX_OPEN_FILES = 500; @@ -45,14 +45,14 @@ BlockDb.prototype.drop = function(cb) { // adds a block. Does not update Next pointer in // the block prev to the new block, nor TIP pointer // -BlockDb.prototype.add = function(b, cb) { +BlockDb.prototype.add = function(b, height, cb) { var self = this; var time_key = TIMESTAMP_PREFIX + ( b.time || Math.round(new Date().getTime() / 1000) ); return db.batch() .put(time_key, b.hash) - .put(MAIN_PREFIX + b.hash, 1) + .put(MAIN_PREFIX + b.hash, height) .put(PREV_PREFIX + b.hash, b.previousblockhash) .write(function(err){ if (!err) { @@ -64,12 +64,16 @@ BlockDb.prototype.add = function(b, cb) { BlockDb.prototype.getTip = function(cb) { db.get(TIP, function(err, val) { - return cb(err,val); + if (!val) return cb(); + + var v = val.split(':'); + return cb(err,v[0], parseInt(v[1])); }); }; -BlockDb.prototype.setTip = function(hash, cb) { - db.put(TIP, hash, function(err) { +BlockDb.prototype.setTip = function(hash, height, cb) { +//console.log('[BlockDb.js.75] TIP', hash, height); //TODO + db.put(TIP, hash + ':' + height, function(err) { return cb(err); }); }; @@ -113,19 +117,20 @@ BlockDb.prototype.getNext = function(hash, cb) { }); }; -BlockDb.prototype.isMain = function(hash, cb) { +BlockDb.prototype.getHeight = function(hash, cb) { db.get(MAIN_PREFIX + hash, function(err, val) { if (err && err.notFound) { err = null; val = 0;} return cb(err,parseInt(val)); }); }; -BlockDb.prototype.setMain = function(hash, isMain, cb) { - if (!isMain) console.log('\tNew orphan: %s',hash); - db.put(MAIN_PREFIX + hash, isMain?1:0, function(err) { +BlockDb.prototype.setHeight = function(hash, height, cb) { + if (!height) console.log('\tNew orphan: %s',hash); + db.put(MAIN_PREFIX + hash, height, function(err) { return cb(err); }); }; + BlockDb.prototype.setNext = function(hash, nextHash, cb) { db.put(NEXT_PREFIX + hash, nextHash, function(err) { return cb(err); @@ -184,10 +189,11 @@ BlockDb.prototype.fromHashWithInfo = function(hash, cb) { Rpc.getBlock(hash, function(err, info) { if (err || !info) return cb(err); - self.isMain(hash, function(err, val) { + //TODO can we get this from RPC .height? + self.getHeight(hash, function(err, height) { if (err) return cb(err); - info.isMainChain = val ? true : false; + info.isMainChain = height ? true : false; return cb(null, { hash: hash, diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 07196566..42355e8a 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -408,13 +408,8 @@ HistoricSync.prototype.start = function(opts, next) { self.syncedBlocks++; self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err) { if (err) return w_cb(self.setError(err)); - - self.sync.bDb.setTip(blockInfo.hash, function(err) { - if (err) return w_cb(self.setError(err)); - - setImmediate(function(){ - return w_cb(err); - }); + setImmediate(function(){ + return w_cb(err); }); }); } diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 41289b9b..8c24de1b 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -76,6 +76,7 @@ PeerSync.prototype.handleTx = function(info) { PeerSync.prototype.handleBlock = function(info) { var self = this; var block = info.message.block; +console.log('[PeerSync.js.78:block:]',block); //TODO var blockHash = bitcoreUtil.formatHashFull(block.calcHash()); console.log('[p2p_sync] Handle block: %s (allowReorgs: %s)', blockHash, self.allowReorgs); diff --git a/lib/Sync.js b/lib/Sync.js index c3b1ccd7..0c453f3e 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -77,7 +77,7 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { if (!b) return cb(); var self = this; - var oldTip, oldNext, needReorg = false; + var oldTip, oldNext, height, needReorg = false; var newPrev = b.previousblockhash; async.series([ @@ -104,9 +104,14 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { }, function(c) { if (!allowReorgs) return c(); - self.bDb.getTip(function(err, val) { - oldTip = val; - if (oldTip && newPrev !== oldTip) needReorg = true; + self.bDb.getTip(function(err, hash, h) { + oldTip = hash; + if (!hash) height = -1 + else height = h || 0; + if (oldTip && newPrev !== oldTip) { + needReorg = true; + console.log('## REORG Triggered, tip mismatch'); + } return c(); }); }, @@ -119,16 +124,16 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { }); }, function(c) { - self.bDb.add(b, c); + self.bDb.add(b, height + 1, c); }, function(c) { if (!needReorg) return c(); - console.log('NEW TIP: %s NEED REORG (old tip: %s)', b.hash, oldTip); - self.processReorg(oldTip, oldNext, newPrev, c); + console.log('NEW TIP: %s #%d NEED REORG (old tip: %s)', b.hash, height + 1, oldTip); + self.processReorg(oldTip, oldNext, newPrev, height + 1, c); }, function(c) { if (!allowReorgs) return c(); - self.bDb.setTip(b.hash, function(err) { + self.bDb.setTip(b.hash, height + 1, function(err) { return c(err); }); }, @@ -149,7 +154,7 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { -Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, cb) { +Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, newHeight, cb) { var self = this; var orphanizeFrom; @@ -170,7 +175,7 @@ Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, cb) { if (orphanizeFrom) return c(); console.log('# Reorg Case 2)'); - self.setBranchConnectedBackwards(newPrev, function(err, yHash, newYHashNext) { + self.setBranchConnectedBackwards(newPrev, newHeight-1, function(err, yHash, newYHashNext) { if (err) return c(err); self.bDb.getNext(yHash, function(err, yHashNext) { orphanizeFrom = yHashNext; @@ -192,11 +197,12 @@ Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, cb) { }); }; -Sync.prototype.setBlockMain = function(hash, isMain, cb) { +//height = false => unconnnected +Sync.prototype.setBlockHeight = function(hash, height, cb) { var self = this; - self.bDb.setMain(hash, isMain, function(err) { + self.bDb.setHeight(hash, height, function(err) { if (err) return cb(err); - return self.txDb.handleBlockChange(hash, isMain, cb); + return self.txDb.handleBlockChange(hash, height, cb); }); }; @@ -209,7 +215,7 @@ Sync.prototype.setBranchOrphan = function(fromHash, cb) { return hashInterator; }, function(c) { - self.setBlockMain(hashInterator, false, function(err) { + self.setBlockHeight(hashInterator, false, function(err) { if (err) return cb(err); self.bDb.getNext(hashInterator, function(err, val) { hashInterator = val; @@ -219,7 +225,7 @@ Sync.prototype.setBranchOrphan = function(fromHash, cb) { }, cb); }; -Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { +Sync.prototype.setBranchConnectedBackwards = function(fromHash, initialHeight, cb) { var self = this, hashInterator = fromHash, lastHash = fromHash, @@ -227,7 +233,7 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { async.doWhilst( function(c) { - self.setBlockMain(hashInterator, true, function(err) { + self.setBlockMain(hashInterator, initialHeight--, function(err) { if (err) return c(err); self.bDb.getPrev(hashInterator, function(err, val) { if (err) return c(err); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index b063011e..9a7f93d0 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -3,7 +3,7 @@ var imports = require('soop').imports(); // blockHash -> txid mapping -var IN_BLK_PREFIX = 'txb-'; //txb-- => 1/0 (connected or not) +var IN_BLK_PREFIX = 'txb-'; //txb-- => height/0 (connected:height or not connected:0) // Only for orphan blocks var FROM_BLK_PREFIX = 'tx-'; //tx-- => 1 @@ -306,7 +306,7 @@ TransactionDb.prototype.fromTxIdN = function(txid, n, confirmations, cb) { /* * If this TxID comes from an RPC request * the .confirmations value from bitcoind is available - * so we could avoid checking if the input were double spented + * so we could avoid checking if the input was double spent * * This speed up address calculations by ~30% * @@ -337,30 +337,30 @@ TransactionDb.prototype.fromTxIdN = function(txid, n, confirmations, cb) { TransactionDb.prototype.fillConfirmations = function(o, cb) { var self = this; - self.isConfirmed(o.txid, function(err, is) { + self.getHeight(o.txid, function(err, height) { if (err) return cb(err); - o.isConfirmed = is; + o.height = height; if (!o.spentTxId) return cb(); if (o.multipleSpentAttempts) { async.eachLimit(o.multipleSpentAttempts, CONCURRENCY, function(oi, e_c) { - self.isConfirmed(oi.spentTxId, function(err, is) { + self.getHeight(oi.spentTxId, function(err, height) { if (err) return; - if (is) { + if (height) { o.spentTxId = oi.spentTxId; o.index = oi.index; - o.spentIsConfirmed = 1; + o.spentHeight = height; } return e_c(); }); }, cb); } else { - self.isConfirmed(o.spentTxId, function(err, is) { + self.getHeight(o.spentTxId, function(err, height) { if (err) return cb(err); - o.spentIsConfirmed = is; + o.spentHeight = height; return cb(); }); } @@ -598,7 +598,7 @@ TransactionDb.prototype.setConfirmation = function(txId, blockHash, confirmed, c // This slowdown addr balance calculation by 100% -TransactionDb.prototype.isConfirmed = function(txId, c) { +TransactionDb.prototype.getHeight = function(txId, c) { var k = IN_BLK_PREFIX + txId; var ret = false; @@ -607,7 +607,7 @@ TransactionDb.prototype.isConfirmed = function(txId, c) { end: k + '~' }) .on('data', function(data) { - if (data.value === '1') ret = true; + if (parseInt(data.value)>0) ret = true; }) .on('error', function(err) { return c(err); @@ -617,7 +617,7 @@ TransactionDb.prototype.isConfirmed = function(txId, c) { }); }; -TransactionDb.prototype.handleBlockChange = function(hash, isMain, cb) { +TransactionDb.prototype.handleBlockChange = function(hash, height, cb) { var toChange = []; console.log('\tSearching Txs from block:' + hash); @@ -633,7 +633,7 @@ TransactionDb.prototype.handleBlockChange = function(hash, isMain, cb) { toChange.push({ key: k2 + ks[2] + '-' + ks[1], type: 'put', - value: isMain ? 1 : 0, + value: height, }); }) .on('error', function(err) { @@ -641,7 +641,7 @@ TransactionDb.prototype.handleBlockChange = function(hash, isMain, cb) { }) .on('end', function(err) { if (err) return cb(err); - console.log('\t%s %d Txs', isMain ? 'Confirming' : 'Invalidating', toChange.length); + console.log('\t%s %d Txs', height ? 'Confirming' : 'Unconfirming', toChange.length); db.batch(toChange, cb); }); }; diff --git a/test/integration/addr.js b/test/integration/addr.js index 7cc20199..76529dd1 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -66,6 +66,7 @@ describe('Address utxo', function() { var a = new Address(v.addr, txDb); a.getUtxo(function(err, utxo) { +console.log('[addr.js.68:utxo:]',utxo); //TODO if (err) done(err); assert.equal(v.addr, a.addrStr); if (v.length) assert.equal(v.length, utxo.length, 'length: ' + utxo.length); diff --git a/test/integration/utxo.json b/test/integration/utxo.json index 7216b9df..1d4d2312 100644 --- a/test/integration/utxo.json +++ b/test/integration/utxo.json @@ -5,5 +5,12 @@ "tx0id": "eeabc70063d3f266e190e8735bc4599c811d3a79d138da1364e88502069b029c", "tx0scriptPubKey": "76a9149e9f6515c70db535abdbbc983c7d8d1bff6c20cd88ac", "tx0amount": 0.38571339 + }, + { + "addr": "2N1pLkosf6o8Ciqs573iwwgVpuFS6NbNKx5", + "length": 1, + "tx0id": "eeabc70063d3f266e190e8735bc4599c811d3a79d138da1364e88502069b029c", + "tx0scriptPubKey": "76a9149e9f6515c70db535abdbbc983c7d8d1bff6c20cd88ac", + "tx0amount": 0.38571339 } ]