rm tdb dependency from Block DB

lib/BlockDb.js

@@ -10,7 +10,6 @@ var LAST_FILE_INDEX   = 'file-';     // last processed file index
 
 var MAX_OPEN_FILES    = 500;
 
-
 /**
 * Module dependencies.
 */
@@ -18,13 +17,11 @@ var levelup     = require('levelup'),
     config      = require('../config/config');
 var db  = imports.db || levelup(config.leveldb + '/blocks',{maxOpenFiles: MAX_OPEN_FILES} );
 var Rpc = imports.rpc || require('./Rpc');
-var PoolMatch = imports.poolMatch || require('soop').load('./PoolMatch',config);
 
 var tDb = require('./TransactionDb.js').default();
 
 var BlockDb = function() {
   BlockDb.super(this, arguments);
-  this.poolMatch = new PoolMatch();
 };
 BlockDb.parent = ThisParent;
 
@@ -53,7 +50,7 @@ BlockDb.prototype.add = function(b, height, cb) {
   return db.batch()
     .put(time_key, b.hash)
     .put(MAIN_PREFIX + b.hash, height)
-  .put(PREV_PREFIX + b.hash, b.previousblockhash)
+    .put(PREV_PREFIX + b.hash, b.previousblockhash)
     .write(function(err){
       if (!err) {
         self.emit('new_block', {blockid: b.hash});
@@ -72,7 +69,7 @@ BlockDb.prototype.getTip = function(cb) {
 };
 
 BlockDb.prototype.setTip = function(hash, height, cb) {
-//console.log('[BlockDb.js.75] TIP', hash, height); //TODO
+console.log('[BlockDb.js.75] TIP', hash, height); //TODO
   db.put(TIP, hash + ':' + height, function(err) {
     return cb(err);
   });
@@ -165,24 +162,6 @@ BlockDb.prototype.has = function(hash, cb) {
   });
 };
 
-BlockDb.prototype.getPoolInfo = function(tx, cb) {
-  var self = this;
-
-  tDb._getInfo(tx, function(e, a) {
-    if (e) return cb(false);
-
-    if (a && a.isCoinBase) {
-      var coinbaseHexBuffer = new Buffer(a.vin[0].coinbase, 'hex');
-      var aa = self.poolMatch.match(coinbaseHexBuffer);
-      
-      return cb(aa);
-    }
-    else {
-      return cb();
-    }
-  });
-};
-
 BlockDb.prototype.fromHashWithInfo = function(hash, cb) {
   var self = this;
 

lib/HistoricSync.js

@@ -235,7 +235,7 @@ HistoricSync.prototype.updateStartBlock = function(next) {
         self.sync.bDb.fromHashWithInfo(tip, function(err, bi) {
           blockInfo = bi ? bi.info : {};
           if (oldtip)
-            self.sync.setBlockMain(oldtip, false, cb);
+            self.sync.setBlockHeight(oldtip, false, cb);
           else
             return cb();
         });
@@ -312,8 +312,8 @@ HistoricSync.prototype.prepareRpcSync = function(opts, next) {
 
   if (self.blockExtractor) return next();
   self.getFn = self.getBlockFromRPC;
+  self.allowReorgs = true;
   self.currentRpcHash  = self.startBlock;
-  self.allowReorgs = false;
   return next();
 };
 
@@ -404,7 +404,9 @@ HistoricSync.prototype.start = function(opts, next) {
       function (w_cb) {
         self.getFn(function(err,blockInfo) {
           if (err) return w_cb(self.setError(err));
-          if (blockInfo && blockInfo.hash) {
+          if (blockInfo && blockInfo.hash
+              && (!opts.stopAt  || opts.stopAt !== blockInfo.hash)
+             ) {
             self.syncedBlocks++;
             self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err) {
               if (err) return w_cb(self.setError(err));

lib/Sync.js

@@ -97,11 +97,6 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) {
             (!val ? new Error('NEED_SYNC Ignoring block with non existing prev:' + b.hash) : null));
         });
       },
-      function(c) {
-        self.txDb.createFromBlock(b, function(err) {
-          return c(err);
-        });
-      },
       function(c) {
         if (!allowReorgs) return c();
         self.bDb.getTip(function(err, hash, h) {
@@ -115,6 +110,11 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) {
           return c();
         });
       },
+      function(c) {
+        self.txDb.createFromBlock(b, height, function(err) {
+          return c(err);
+        });
+      },
       function(c) {
         if (!needReorg) return c();
         self.bDb.getNext(newPrev, function(err, val) {
@@ -162,8 +162,8 @@ Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, newHeight, cb)
   async.series([
 
       function(c) {
-        self.bDb.isMain(newPrev, function(err, val) {
+        self.bDb.getHeight(newPrev, function(err, height) {
-          if (!val) return c();
+          if (!height) return c();
 
           console.log('# Reorg Case 1)');
           // case 1
@@ -202,7 +202,7 @@ Sync.prototype.setBlockHeight = function(hash, height, cb) {
   var self = this;
   self.bDb.setHeight(hash, height, function(err) {
     if (err) return cb(err);
-    return self.txDb.handleBlockChange(hash, height, cb);
+    return self.txDb.handleBlockChange(hash, height>-1? true : false, cb);
   });
 };
 
@@ -233,14 +233,14 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, initialHeight, c
 
   async.doWhilst(
     function(c) {
-      self.setBlockMain(hashInterator, initialHeight--, function(err) {
+      self.setBlockHeight(hashInterator, initialHeight--, function(err) {
         if (err) return c(err);
         self.bDb.getPrev(hashInterator, function(err, val) {
           if (err) return c(err);
           lastHash = hashInterator;
           hashInterator = val;
-          self.bDb.isMain(hashInterator, function(err, val) {
+          self.bDb.getHeight(hashInterator, function(err, height) {
-            isMain = val;
+            isMain = height ? 1 : 0;
             return c();
           });
         });
@@ -257,9 +257,10 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, initialHeight, c
 };
 
 
+//Store unconfirmed TXs
 Sync.prototype.storeTxs = function(txs, cb) {
   var self = this;
-  self.txDb.createFromArray(txs, null, function(err) {
+  self.txDb.createFromArray(txs, null, null, function(err) {
     if (err) return cb(err);
     return cb(err);
   });

lib/TransactionDb.js

@@ -1,12 +1,15 @@
 'use strict';
 
 var imports       = require('soop').imports();
+var PoolMatch = imports.poolMatch || require('soop').load('./PoolMatch',config);
 
-// blockHash -> txid mapping 
+// txid - blockhash mapping  (only for confirmed txs)
-var IN_BLK_PREFIX = 'txb-'; //txb-<txid>-<block> => height/0 (connected:height or not connected:0)
+var IN_BLK_PREFIX = 'txb-'; //txb-<txid> = <block> 
 
-// Only for orphan blocks
+var RECENT_ = 'tx-'; //tx-<block> = <txid> 
-var FROM_BLK_PREFIX = 'tx-'; //tx-<block>-<txid> => 1 
+
+
+pre-<height>- =txid
 
 // to show tx outs
 var OUTS_PREFIX = 'txo-'; //txo-<txid>-<n> => [addr, btc_sat]
@@ -51,6 +54,7 @@ var Address = require('soop').load('bitcore/lib/Address',{
 var TransactionDb = function() {
   TransactionDb.super(this, arguments);
   this.network = config.network === 'testnet' ? networks.testnet : networks.livenet;
+  this.poolMatch = new PoolMatch();
 };
 
 TransactionDb.prototype.close = function(cb) {
@@ -345,6 +349,7 @@ TransactionDb.prototype.fillConfirmations = function(o, cb) {
 
     if (o.multipleSpentAttempts) {
 
+      //TODO save it for later is height > 6
       async.eachLimit(o.multipleSpentAttempts, CONCURRENCY,
         function(oi, e_c) {
           self.getHeight(oi.spentTxId, function(err, height) {
@@ -392,6 +397,8 @@ TransactionDb.prototype.fromAddr = function(addr, cb) {
     })
     .on('end', function() {
 
+      //TODO is spent, and conf > 6, save it on ADDR_PREFIX for later
+      //and skip all the rest
       async.eachLimit(ret, CONCURRENCY, function(o, e_c) {
           var k = SPENT_PREFIX + o.txid + '-' + o.index + '-';
           db.createReadStream({
@@ -506,10 +513,13 @@ TransactionDb.prototype.adaptTxObject = function(txInfo) {
 
 
 
-TransactionDb.prototype.add = function(tx, blockhash, cb) {
+TransactionDb.prototype.add = function(tx, blockhash, height, cb) {
   var self = this;
   var addrs = [];
 
+  if (typeof height === 'undefined')
+    throw new Error('add should received height');
+
   if (tx.hash) self.adaptTxObject(tx);
 
   var ts = tx.time;
@@ -572,7 +582,7 @@ TransactionDb.prototype.add = function(tx, blockhash, cb) {
       if (!blockhash) {
         return p_c();
       }
-      return self.setConfirmation(tx.txid, blockhash, true, p_c);
+      return self.setBlock(tx.txid, blockhash, p_c);
     },
   ], function(err) {
     if (addrs.length > 0 && !blockhash) {
@@ -585,45 +595,24 @@ TransactionDb.prototype.add = function(tx, blockhash, cb) {
 
 
 
-TransactionDb.prototype.setConfirmation = function(txId, blockHash, confirmed, c) {
+TransactionDb.prototype.setBlock = function(txId, blockHash, c) {
   if (!blockHash) return c();
-
-  confirmed = confirmed ? 1 : 0;
-
   db.batch()
-    .put(IN_BLK_PREFIX + txId + '-' + blockHash, confirmed)
+    .put(IN_BLK_PREFIX + txId, blockHash)
-    .put(FROM_BLK_PREFIX + blockHash + '-' + txId, 1)
     .write(c);
 };
 
-
+TransactionDb.prototype.getBlock = function(txId, c) {
-// This slowdown addr balance calculation by 100%
+  db.get(IN_BLK_PREFIX + txId,c);
-TransactionDb.prototype.getHeight = function(txId, c) {
-  var k = IN_BLK_PREFIX + txId;
-  var ret = false;
-
-  db.createReadStream({
-    start: k,
-    end: k + '~'
-  })
-    .on('data', function(data) {
-      if (parseInt(data.value)>0) ret = true;
-    })
-    .on('error', function(err) {
-      return c(err);
-    })
-    .on('end', function(err) {
-      return c(err, ret);
-    });
 };
 
-TransactionDb.prototype.handleBlockChange = function(hash, height, cb) {
+TransactionDb.prototype.handleBlockChange = function(hash, isConnected, cb) {
   var toChange = [];
   console.log('\tSearching Txs from block:' + hash);
 
   var k = FROM_BLK_PREFIX + hash;
   var k2 = IN_BLK_PREFIX;
-  // This is slow, but prevent us to create a new block->tx index.
+
   db.createReadStream({
     start: k,
     end: k + '~'
@@ -647,10 +636,11 @@ TransactionDb.prototype.handleBlockChange = function(hash, height, cb) {
 };
 
 // txs can be a [hashes] or [txObjects]
-TransactionDb.prototype.createFromArray = function(txs, blockHash, next) {
+TransactionDb.prototype.createFromArray = function(txs, blockHash, height, next) {
   var self = this;
   if (!txs) return next();
 
+  
   async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) {
       if (typeof t === 'string') {
         // TODO: parse it from networks.genesisTX?
@@ -658,10 +648,10 @@ TransactionDb.prototype.createFromArray = function(txs, blockHash, next) {
 
         Rpc.getTxInfo(t, function(err, inInfo) {
           if (!inInfo) return each_cb(err);
-          return self.add(inInfo, blockHash, each_cb);
+          return self.add(inInfo, blockHash, height, each_cb);
         });
       } else {
-        return self.add(t, blockHash, each_cb);
+        return self.add(t, blockHash, height, each_cb);
       }
     },
     function(err) {
@@ -670,11 +660,33 @@ TransactionDb.prototype.createFromArray = function(txs, blockHash, next) {
 };
 
 
-TransactionDb.prototype.createFromBlock = function(b, next) {
+TransactionDb.prototype.createFromBlock = function(b, height, next) {
   var self = this;
   if (!b || !b.tx) return next();
 
-  return self.createFromArray(b.tx, b.hash, next);
+  if (typeof height === 'undefined')
+    throw new Error('createFromBlock should received height');
+
+  return self.createFromArray(b.tx, b.hash, height, next);
 };
 
+
+TransactionDb.prototype.getPoolInfo = function(tx, cb) {
+  var self = this;
+  self._getInfo(tx, function(e, a) {
+    if (e) return cb(false);
+
+    if (a && a.isCoinBase) {
+      var coinbaseHexBuffer = new Buffer(a.vin[0].coinbase, 'hex');
+      var aa = self.poolMatch.match(coinbaseHexBuffer);
+      
+      return cb(aa);
+    }
+    else {
+      return cb();
+    }
+  });
+};
+
+
 module.exports = require('soop')(TransactionDb);