more fixes on sync

app/models/Transaction.js

@@ -7,17 +7,25 @@ function spec() {
   
   var util            = require('bitcore/util/util'),
       TransactionRpc  = require('../../lib/TransactionRpc').class(),
-      TransactionOut  = require('./TransactionOut'),
+      TransactionDb   = require('../../lib/TransactionDb').class(),
       async           = require('async');
 
   var CONCURRENCY = 20;
 
-  function Transaction() {
+  function Transaction(tdb) {
     this.txid = null;
+    this.tdb = tdb || new TransactionDb();
   }
 
-  Transaction.fromIdWithInfo = function (txid,cb) {
+  Transaction.fromIdWithInfo = function (txid, tdb, cb) {
-    var tx = new Transaction();
+    if (typeof tdb === 'function') {
+      cb = tdb;
+      tdb = null;
+    }
+    var tx = new Transaction(tdb);
+
+
+console.log('[Transaction.js.27]',tx.tdb); //TODO
     tx.txid = txid;
 
     tx._fillInfo(function(err) {
@@ -33,21 +41,22 @@ function spec() {
 
     TransactionRpc.getRpcInfo(self.txid, function(err, info) {
       if (err) return next(err);
-      Transaction._fillOutpoints(info, function() {
+      self._fillOutpoints(info, function() {
         self.info = info;
         return next();
       });
     });
   };
 
-  Transaction._fillOutpoints = function(info, cb) {
+  Transaction.prototype._fillOutpoints = function(info, cb) {
+    var self  = this;
 
     if (!info || info.isCoinBase) return cb();
 
     var valueIn = 0;
     var incompleteInputs = 0;
     async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) {
-      TransactionOut.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) {
+      self.tdb.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) {
 
 
         if (err || !addr || !valueSat ) {

lib/BlockDb.js

@@ -44,7 +44,7 @@ function spec() {
 
     self.db.batch()
       .put(time_key, b.hash)
-      .put(PREV_ROOT + b.hash, b.prev_block)
+      .put(PREV_ROOT + b.hash, b.previousblockhash)
       .write(cb);
   };
 
@@ -82,6 +82,7 @@ function spec() {
 
   BlockDb.prototype.countNotOrphan = function(cb) {
     var c = 0;
+    console.log('Counting connected blocks. This could take some minutes'); 
     this.db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' })
       .on('data', function (data) {
         if (data.value !== 0) c++;
@@ -90,7 +91,7 @@ function spec() {
         return cb(err);
       })
       .on('end', function () {
-        return cb(null);
+        return cb(null, c);
       });
   };
 

lib/HistoricSync.js

@@ -9,7 +9,6 @@ function spec() {
   var RpcClient = require('bitcore/RpcClient').class();
   var bitutil = require('bitcore/util/util');
   var Address = require('bitcore/Address').class();
-	var Deserialize = require('bitcore/Deserialize');
   var Script = require('bitcore/Script').class();
   var networks = require('bitcore/networks');
   var async = require('async');
@@ -17,6 +16,7 @@ function spec() {
   var Sync = require('./Sync').class();
   var sockets = require('../app/controllers/socket.js');
   var BlockExtractor = require('./BlockExtractor.js').class();
+	//var Deserialize = require('bitcore/Deserialize');
 
 
   var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:';
@@ -37,6 +37,7 @@ function spec() {
     this.syncedBlocks = 0;
     this.skippedBlocks = 0;
     this.orphanBlocks = 0;
+    this.type ='';
   }
 
   function p() {
@@ -97,6 +98,7 @@ function spec() {
       syncedBlocks: this.syncedBlocks,
       orphanBlocks: this.orphanBlocks,
       error: this.error,
+      type: this.type,
     };
   };
 
@@ -273,7 +275,9 @@ function spec() {
         if (err || ! b) return c(err);
 
         blockInfo = b.getStandardizedObject(b.txs, self.network);
-        blockInfo.curWork = Deserialize.intFromCompact(b.bits);
+        // blockInfo.curWork = Deserialize.intFromCompact(b.bits);
+        // We keep the RPC field names
+        blockInfo.previousblockhash = blockInfo.prev_block;
 
         var ti=0;
         // Get TX Address
@@ -297,31 +301,40 @@ function spec() {
         return c();
       });
     },
-    //store it
+    //check prev
     function(c) {
-      if (blockInfo && self.prevHash && blockInfo.prev_block !== self.prevHash) {
+      if (blockInfo && self.prevHash && blockInfo.previousblockhash !== self.prevHash) {
 
         console.log('Hole found @%s', blockInfo.hash);
-        console.log('From: %s To: %s', self.prevHash, blockInfo.prev_block);
+        console.log('From: %s To: %s', self.prevHash, blockInfo.previousblockhash);
-        self.sync.checkOrphan(self.prevHash, blockInfo.prev_block, c);
+        self.sync.checkOrphan(self.prevHash, blockInfo.previousblockhash, c);
       }
       else return c();
     },
-    //store it
+    //check it
     function(c) {
       if (!blockInfo) return c();
 
-      self.sync.storeBlock(blockInfo, function(err) {
+      self.sync.blockDb.has(blockInfo.hash, function(err, had) {
-        existed = err && err.toString().match(/E11000/);
+        existed = had;
+        return c(err);
+      });
+    },
+    //store it
+    function(c) {
+      if (!blockInfo || existed) return c();
 
-        if (err && ! existed) return c(err);
+      self.sync.storeBlock(blockInfo, function(err) {
-        return c();
+        return c(err);
       });
     },
     function(c) {
       if (blockInfo && blockInfo.hash) {
         self.prevHash = blockInfo.hash;
-        self.syncedBlocks++;
+        if (existed)
+          self.skippedBlocks++;
+        else
+          self.syncedBlocks++;
       } else
         self.status = 'finished';
 
@@ -424,6 +437,7 @@ function spec() {
 
       if (scanOpts.fromFiles) {
         self.status = 'syncing';
+        self.type = 'from .dat Files';
         async.whilst(function() {
           return self.status === 'syncing';
         }, function (w_cb) {
@@ -437,6 +451,7 @@ function spec() {
         });
       }
       else {
+        self.type = 'from RPC calls';
         self.getPrevNextBlock(start, end, scanOpts, function(err) {
           return next(err);
         });
@@ -475,6 +490,7 @@ function spec() {
             }
             else {
               p('Genesis block found. Syncing upto known blocks.');
+              p('Got %d out of %d blocks', count, self.blockChainHeight);
               scanOpts.reverse = true;
               scanOpts.upToExisting = true;
             }

lib/TransactionDb.js

@@ -5,8 +5,8 @@ require('classtool');
 
 function spec() {
 
-  // blockHash -> txid mapping (to orphanize )/
+  // blockHash -> txid mapping (to reorgs)
-  var ROOT         = 'tx-b-';      //tx-b-<block> => txid
+  var ROOT         = 'tx-b-';      //tx-b-<txid>-<block> => 1
 
   // to show tx outs
   var OUTS_ROOT    = 'txouts-';      //txouts-<txid>-<n> => [addr, btc_sat]
@@ -107,6 +107,65 @@ function spec() {
       });
   };
 
+  TransactionDb.prototype._fillOutpoints = function(info, cb) {
+    var self  = this;
+
+    if (!info || info.isCoinBase) return cb();
+
+    var valueIn = 0;
+    var incompleteInputs = 0;
+    async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) {
+      self.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) {
+
+
+        if (err || !addr || !valueSat ) {
+          console.log('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid);
+          incompleteInputs = 1;
+          return c_in(); // error not scaled
+        }
+        i.addr     = addr;
+        i.valueSat = valueSat;
+        i.value    = valueSat / util.COIN;
+
+        valueIn += i.valueSat;
+        return c_in();
+      });
+    },
+    function () {
+      if (! incompleteInputs ) {
+        info.valueIn = valueIn / util.COIN;
+        info.fees    = (valueIn - parseInt(info.valueOut * util.COIN))  / util.COIN ;
+      }
+      else {
+        info.incompleteInputs = 1;
+      }
+      return cb();
+    });
+  };
+
+  TransactionDb.prototype._getInfo = function(txid, next) {
+    var self  = this;
+
+    TransactionRpc.getRpcInfo(txid, function(err, info) {
+      if (err) return next(err);
+
+      self._fillOutpoints(info, function() {
+        return next(null, info);
+      });
+    });
+  };
+
+
+  TransactionDb.prototype.fromIdWithInfo = function (txid, cb) {
+    var self = this;
+
+    self._getInfo(txid, function(err, info) {
+      if (err) return cb(err);
+      if (!info ) return cb();
+
+      return cb(err, {txid: txid, info: info} );
+    });
+  };
 
   TransactionDb.prototype.fromTxIdN = function(txid, n, cb) {
     var self = this;
@@ -117,8 +176,8 @@ function spec() {
       if (err && err.notFound) {
         err = null;
       }
-      var a = val.split('-');
+      var a = val.split(':');
-      return cb(err, val, a[0], a[1]);
+      return cb(err, a[0], a[1]);
     });
   };
 
@@ -343,7 +402,7 @@ console.log('[TransactionDb.js.165]', ret); //TODO
           self.add(inInfo, function(err) {
             if (err || !blockHash) return each_cb(err);
 
-            self.db.put(ROOT + blockHash, t, function(err) {
+            self.db.put(ROOT + t + '-' + blockHash, 1, function(err) {
               return each_cb(err);
             });
           });
@@ -353,7 +412,7 @@ console.log('[TransactionDb.js.165]', ret); //TODO
         self.add(t, function(err) {
           if (err) return each_cb(err);
 
-          self.db.put(ROOT + blockHash, t.txid, function(err) {
+          self.db.put(ROOT + t.txid + '-' + blockHash, 1, function(err) {
             return each_cb(err);
           });
         });

test/integration/01-transactionouts.js

@@ -5,16 +5,117 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development';
 
 
 
-var 
+var assert          = require('assert'),
-  assert          = require('assert'),
   fs              = require('fs'),
   util            = require('util'),
-  config          = require('../../config/config'),
   TransactionDb = require('../../lib/TransactionDb').class();
 
 var txItemsValid = JSON.parse(fs.readFileSync('test/integration/txitems.json'));
+var tdb;
 
-describe('TransactionDb', function(){
+describe('TransactionDb fromIdWithInfo', function(){
+
+  before(function(c) {
+    tdb = new TransactionDb();
+    return c();
+  });
+
+  var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c';
+  it('tx info ' + txid, function(done) {
+    tdb.fromIdWithInfo(txid, function(err, tx) {
+
+      if (err) done(err);
+      assert.equal(tx.txid, txid);
+      assert(!tx.info.isCoinBase);
+
+      for(var i=0; i<20; i++)
+        assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i);
+      assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0');
+      assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1');
+      done();
+    });
+  });
+
+  it('tx info', function(done) {
+    var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237';
+    tdb.fromIdWithInfo(txid, function(err, tx) {
+      if (err) done(err);
+      assert.equal(tx.txid, txid);
+      assert(!tx.info.isCoinBase);
+      done();
+    });
+  });
+
+  it('should pool tx\'s info from bitcoind', function(done) {
+    var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237';
+    tdb.fromIdWithInfo(txid, function(err, tx) {
+      if (err) done(err);
+      assert.equal(tx.info.txid, txid);
+      assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74');
+      assert.equal(tx.info.valueOut, 1.66174);
+      assert.equal(tx.info.fees, 0.0005 );
+      assert.equal(tx.info.size, 226 );
+      assert(!tx.info.isCoinBase);
+      done();
+    });
+  });
+
+  var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399';
+  it('test a coinbase TX ' + txid1, function(done) {
+    tdb.fromIdWithInfo(txid1, function(err, tx) {
+      if (err) done(err);
+      assert(tx.info.isCoinBase);
+      assert.equal(tx.info.txid, txid1);
+      assert(!tx.info.feeds);
+      done();
+    });
+  });
+  var txid22 = '666';
+  it('test invalid TX ' + txid22, function(done) {
+    tdb.fromIdWithInfo(txid22, function(err, tx) {
+      if (err && err.message.match(/must.be.hexadecimal/))  {
+          return done();
+      }
+      else {
+        return done(err);
+      }
+    });
+  });
+
+  var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227';
+  it('test unexisting TX ' + txid23, function(done) {
+
+    tdb.fromIdWithInfo(txid23, function(err, tx) {
+      assert(!err);
+      assert(!tx);
+      return done();
+    });
+  });
+
+
+
+  var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608';
+  it('create TX on the fly ' + txid2, function(done) {
+    tdb.fromIdWithInfo(txid2, function(err, tx) {
+      if (err) return done(err);
+      assert.equal(tx.info.txid, txid2);
+      done();
+    });
+  });
+
+  txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608';
+  it('test a broken TX ' + txid2, function(done) {
+    tdb.fromIdWithInfo(txid2, function(err, tx) {
+      if (err) return done(err);
+      assert.equal(tx.info.txid, txid2);
+      assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU');
+      done();
+    });
+  });
+});
+/*
+
+describe('TransactionDb Outs', function(){
 
   var tdb = new TransactionDb();
 
@@ -63,5 +164,5 @@ describe('TransactionDb', function(){
   });
 });
 
-
+*/
 

test/integration/transaction.js

@@ -1,120 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-
-process.env.NODE_ENV = process.env.NODE_ENV || 'development';
-
-
-
-var mongoose= require('mongoose'),
-  assert  = require('assert'),
-  config       = require('../../config/config'),
-  Transaction  = require('../../app/models/Transaction').class();
-
-
-mongoose.connection.on('error', function(err) { console.log(err); });
-
-describe('Transaction', function(){
-
-  before(function(done) {
-    mongoose.connect(config.db);
-    done();
-  });
-
-  after(function(done) {
-    mongoose.connection.close();
-    done();
-  });
-  var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c';
-  it('txid ' + txid, function(done) {
-    Transaction.fromIdWithInfo(txid, function(err, tx) {
-      if (err) done(err);
-      assert.equal(tx.txid, txid);
-      assert(!tx.info.isCoinBase);
-
-      for(var i=0; i<20; i++)
-        assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i);
-      assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0');
-      assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1');
-      done();
-    });
-  });
-
-  it('should pool tx\'s object from mongoose', function(done) {
-    var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237';
-    Transaction.fromIdWithInfo(txid, function(err, tx) {
-      if (err) done(err);
-      assert.equal(tx.txid, txid);
-      assert(!tx.info.isCoinBase);
-      done();
-    });
-  });
-
-  it('should pool tx\'s info from bitcoind', function(done) {
-    var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237';
-    Transaction.fromIdWithInfo(txid, function(err, tx) {
-      if (err) done(err);
-      assert.equal(tx.info.txid, txid);
-      assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74');
-      assert.equal(tx.info.valueOut, 1.66174);
-      assert.equal(tx.info.fees, 0.0005 );
-      assert.equal(tx.info.size, 226 );
-      assert(!tx.info.isCoinBase);
-      done();
-    });
-  });
-
-  var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399';
-  it('test a coinbase TX ' + txid1, function(done) {
-    Transaction.fromIdWithInfo(txid1, function(err, tx) {
-      if (err) done(err);
-      assert(tx.info.isCoinBase);
-      assert.equal(tx.info.txid, txid1);
-      assert(!tx.info.feeds);
-      done();
-    });
-  });
-  var txid22 = '666';
-  it('test invalid TX ' + txid22, function(done) {
-    Transaction.fromIdWithInfo(txid22, function(err, tx) {
-      if (err && err.message.match(/must.be.hexadecimal/))  {
-          return done();
-      }
-      else {
-        return done(err);
-      }
-    });
-  });
-
-  var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227';
-  it('test unexisting TX ' + txid23, function(done) {
-
-    Transaction.fromIdWithInfo(txid23, function(err, tx) {
-      assert(!err);
-      assert(!tx);
-      return done();
-    });
-  });
-
-
-
-  var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608';
-  it('create TX on the fly ' + txid2, function(done) {
-    Transaction.fromIdWithInfo(txid2, function(err, tx) {
-      if (err) return done(err);
-      assert.equal(tx.info.txid, txid2);
-      done();
-    });
-  });
-
-  var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608';
-  it('test a broken TX ' + txid2, function(done) {
-    Transaction.fromIdWithInfo(txid2, function(err, tx) {
-      if (err) return done(err);
-      assert.equal(tx.info.txid, txid2);
-      assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU');
-      done();
-    });
-  });
-
-});
-