Merge branch 'master' of github.com:bitpay/mystery into test/test-p2p-sync

Conflicts:
	lib/Sync.js

Gruntfile.js

@@ -48,7 +48,7 @@ module.exports = function(grunt) {
     },
     jshint: {
       all: {
-        src: ['Gruntfile.js', 'server.js', 'app/**/*.js', 'public/js/**'],
+        src: ['Gruntfile.js', 'server.js', 'app/**/*.js', 'public/js/**','lib/*.js'],
         options: {
           jshintrc: true
         }
@@ -66,7 +66,7 @@ module.exports = function(grunt) {
         options: {
           file: 'server.js',
           args: [],
-          ignoredFiles: ['public/**', 'test/**','util/**','lib/**'],
+          ignoredFiles: ['public/**', 'test/**','util/**'],
           watchedExtensions: ['js'],
           // nodeArgs: ['--debug'],
           delayTime: 1,

app/models/Transaction.js

@@ -141,7 +141,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time,  cb) {
       }
     },
     function (err) {
-      if (err) console.log (err);
+      if (err && !err.message.match(/E11000/) ) console.log (err);
       async.forEachLimit(info.vout, CONCURRENCY, function(o, next_out) {
 
         /*

lib/HistoricSync.js

@@ -0,0 +1,248 @@
+'use strict';
+
+require('classtool');
+
+
+function spec() {
+  var mongoose        = require('mongoose');
+  var util            = require('util');
+  var RpcClient       = require('bitcore/RpcClient').class();
+  var networks        = require('bitcore/networks');
+  var async           = require('async');
+  var config          = require('../config/config');
+  var Block           = require('../app/models/Block');
+  var Transaction     = require('../app/models/Transaction');
+  var TransactionItem = require('../app/models/TransactionItem');
+  var Sync            = require('./Sync').class();
+  var sockets         = require('../app/views/sockets/main.js');
+  var CONCURRENCY     = 5;
+
+
+  function HistoricSync(opts) {
+    this.block_count= 0;
+    this.block_total= 0;
+    this.network    = config.network === 'testnet' ? networks.testnet: networks.livenet;
+    this.sync       = new Sync(opts);
+  }
+
+  function p() {
+    var params = Array.prototype.slice.call(arguments);
+    
+    params.unshift('[historic_sync]');
+    console.log.apply(this,params);
+  }
+
+  var progress_bar = function(string, current, total) {
+    p(util.format('%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total)));
+  };
+
+  HistoricSync.prototype.init = function(opts,cb) {
+    this.rpc = new RpcClient(config.bitcoind);
+    this.opts = opts;
+    this.sync.init(opts, cb);
+  };
+
+  HistoricSync.prototype.close = function() {
+    this.sync.close();
+  };
+
+  HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
+
+    var that = this;
+
+    // recursion end.
+    if (!blockHash || (blockEnd && blockEnd === blockHash) ) {
+      return cb();
+    }
+
+    var existed = 0;
+    var blockInfo;
+    var blockObj;
+
+    async.series([ 
+      // Already got it?
+      function(c) {
+        Block.findOne({hash:blockHash}, function(err,block){
+          if (err) { p(err); return c(err); };
+          if (block) {
+            existed  = 1;
+            blockObj = block;
+          }
+
+          return c();
+        });
+      },
+      //show some (inacurate) status
+      function(c) {
+        if (that.block_count++ % 1000 === 0) {
+          progress_bar('sync status:', that.block_count, that.block_total);
+        }
+        return c();
+      },
+      //get Info from RPC
+      function(c) {
+
+        // TODO: if we store prev/next, no need to go to RPC
+        // if (blockObj && blockObj.nextBlockHash) return c();
+
+        that.rpc.getBlock(blockHash, function(err, ret) {
+          if (err) return c(err);
+
+          blockInfo = ret;
+          return c();
+        });
+      },
+      //store it
+      function(c) {
+        if (existed) return c();
+
+        that.sync.storeBlock(blockInfo.result, function(err, block) {
+          existed = err && err.toString().match(/E11000/);
+          if (err && ! existed)  return c(err);
+          return c();
+        });
+      },
+      /* TODO: Should Start to sync backwards? (this is for partial syncs)
+      function(c) {
+
+        if (blockInfo.result.prevblockhash != current.blockHash) {
+          p("reorg?");
+          opts.prev = 1;
+        }
+        return c();
+        }
+      */
+      ],
+      function (err){
+
+        if (err)
+          p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count);
+
+        if (blockInfo && blockInfo.result) {
+          if (opts.prev && blockInfo.result.previousblockhash) {
+            return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
+          }
+
+          if (opts.next && blockInfo.result.nextblockhash)
+            return that.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
+        }
+        return cb(err);
+    });
+  };
+
+  HistoricSync.prototype.syncBlocks = function(start, end, isForward, cb) {
+    var that = this;
+
+    p('Syncing Blocks, starting from: %s end: %s isForward:',
+                  start, end, isForward);
+
+
+    return that.getPrevNextBlock( start, end,
+                  isForward ? { next: 1 } : { prev: 1}, cb);
+  };
+
+  HistoricSync.prototype.do_import_history = function(opts, next) {
+    var that = this;
+
+    var retry_attemps = 100;
+    var retry_secs    = 2;
+
+    var block_best;
+    var block_height;
+
+    async.series([
+      function(cb) {
+        if (opts.destroy) {
+          p('Deleting Blocks...');
+          that.db.collections.blocks.drop(cb);
+        } else {
+          return cb();
+        }
+      },
+      function(cb) {
+        if (opts.destroy) {
+          p('Deleting TXs...');
+          that.db.collections.transactions.drop(cb);
+        } else {
+          return cb();
+        }
+      },
+      function(cb) {
+        if (opts.destroy) {
+          p('Deleting TXItems...');
+          that.db.collections.transactionitems.drop(cb);
+        } else {
+          return cb();
+        }
+      },
+      function(cb) {
+        that.rpc.getInfo(function(err, res) {
+          if (err) cb(err);
+
+          that.block_total = res.result.blocks;
+          return cb();
+        });
+      },
+      // We are not using getBestBlockHash, because is not available in all clients
+      function(cb) {
+        if (!opts.reverse) return cb();
+
+        that.rpc.getBlockCount(function(err, res) {
+          if (err) cb(err);
+          block_height = res.result;
+          return cb();
+        });
+      },
+      function(cb) {
+        if (!opts.reverse) return cb();
+
+        that.rpc.getBlockHash(block_height, function(err, res) {
+          if (err) cb(err);
+
+          block_best = res.result;
+          return cb();
+        });
+      },
+      ], 
+      function(err) {
+
+        function sync() {
+          var start, end, isForward;
+
+          if (opts.reverse) {
+            start     = block_best;
+            end       = that.network.genesisBlock.hash.reverse().toString('hex');
+            isForward = false;
+          }
+          else {
+            start     = that.network.genesisBlock.hash.reverse().toString('hex');
+            end       = null;
+            isForward = true;
+          }
+
+          that.syncBlocks(start, end, isForward, function(err) {
+
+            if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){
+              setTimeout(function() {
+                p("Retrying in %d secs ", retry_secs);
+                sync();
+              }, retry_secs * 1000);
+            }
+            else
+              return next(err, that.block_count);
+          });
+        }
+        sync();
+    });
+  }
+
+  HistoricSync.prototype.import_history = function(opts, next) {
+    var that = this;
+    that.do_import_history(opts, next);
+  };
+
+
+  return HistoricSync;
+}
+module.defineClass(spec);
+

lib/PeerSync.js

@@ -11,23 +11,27 @@ function spec() {
   var peerdb_fn = 'peerdb.json';
 
   function PeerSync() {}
-  PeerSync.prototype.init = function(config) {
+
+
+  PeerSync.prototype.init = function(config, cb) {
+
+    var that = this;
 
     var network = config && (config.network || 'testnet');
 
-    this.peerdb = undefined;
-    this.sync = new Sync({
+    that.peerdb = undefined;
+    that.sync = new Sync({
       networkName: network
     });
-    this.sync.init(config);
-
-    this.PeerManager = require('bitcore/PeerManager').createClass({
-      config: {
-        network: network
-      }
+    that.sync.init(config, function() { 
+      that.PeerManager = require('bitcore/PeerManager').createClass({
+        config: {
+          network: network
+        }
+      });
+      that.load_peers();
+      return cb();
     });
-    this.load_peers();
-
   };
 
   PeerSync.prototype.load_peers = function() {
@@ -51,6 +55,7 @@ function spec() {
       console.log('[p2p_sync] Handle inv for a ' + CoinConst.MSG.to_str(inv.type));
     });
     // TODO: should limit the invs to objects we haven't seen yet
+    console.log('getdata');
     info.conn.sendGetData(invs);
   };
 

public/views/transaction.html

@@ -23,10 +23,13 @@
       <td width="45%"> 
         <ul class="list-unstyled" data-ng-repeat="vin in tx.vin" data-ng-show="!tx.isCoinBase">
           <li>
-          <span data-ng-show="!vin.addr"><a href="/#!/tx/{{vin.txid}}">Address could not be parsed, {{vin.vout}}</a></span>  
+          <span data-ng-show="!vin.addr">Address could not be parsed</span>  
           <a data-ng-show="vin.addr" href="/#!/address/{{vin.addr}}">{{vin.addr}}</a>  
           <span class="pull-right badge">{{vin.value}} BTC</span>
           </li>
+
+          <li>
+          Outpoint: <a href="/#!/tx/{{vin.txid}}">{{vin.txid}},{{vin.vout}}</a>
         </ul>
         <div data-ng-show="tx.isCoinBase">
           <ul class="list-unstyled" data-ng-repeat="vinn in tx.vin">

server.js

@@ -10,6 +10,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development';
 var express = require('express'),
   fs = require('fs'),
   PeerSync = require('./lib/PeerSync').class(),
+  HistoricSync = require('./lib/HistoricSync').class(),
   mongoose = require('mongoose');
 
 /**
@@ -40,14 +41,27 @@ var walk = function(path) {
 };
 walk(models_path);
 
+// historic_sync process
+var hs = new HistoricSync();
+hs.init({
+  skip_db_connection: true,
+  networkName: config.network
+}, function() {
+  hs.import_history({
+    reverse: 1,
+  });
+});
+
+
 // p2p_sync process
 var ps = new PeerSync();
 ps.init({
   skip_db_connection: true,
   broadcast_txs: true,
   broadcast_blocks: true
+}, function() {
+  ps.run();
 });
-ps.run();
 
 // express app
 var app = express();

test/lib/PeerSync.js

@@ -5,7 +5,8 @@ var chai = require('chai'),
 
 var PeerSync = require('../../lib/PeerSync.js').class();
 describe('PeerSync', function() {
-  var ps, inv_info;
+  var ps;
+
   beforeEach(function() {
     ps = new PeerSync();
     ps.init();
@@ -13,6 +14,7 @@ describe('PeerSync', function() {
   afterEach(function(){
     ps.close();
   });
+
   describe('#init()', function() {
     it('should return with no errors', function() {
       var other_ps = new PeerSync();
@@ -20,8 +22,9 @@ describe('PeerSync', function() {
       other_ps.close();
     });
   });
+
   describe('#handle_inv()', function() {
-    inv_info = {
+    var inv_info = {
       message: {invs: []},
       conn: {sendGetData: sinon.spy()}
     };
@@ -32,16 +35,27 @@ describe('PeerSync', function() {
     });
     it('should call sendGetData', function() {
       ps.handle_inv(inv_info);
-      expect(inv_info.conn.calledOnce);
+      expect(inv_info.conn.sendGetData.calledOnce).to.be.ok;
     });
   });
+
   describe('#handle_tx()', function() {
-    it('should call storeTxs');
+    var tx_info = {
+      message: { tx: {getStandardizedObject: function(){
+        return {hash: '00000000e3fe5b3b5416374d8d65560a0792a6da71546d67b00c9d37e8a4cf59'};}}}
+    };
+    it('should call storeTxs', function(){
+      var spy = sinon.spy(ps.sync, 'storeTxs');
+      ps.handle_tx(tx_info);
+      expect(spy.calledOnce);
+    });
   });
+
   describe('#handle_block()', function() {
     it('should call storeBlock');
     it('should call storeTxs for each transaction');
   });
+
   describe('#run()', function() {
     it('should setup peerman');
   });

util/sync.js

@@ -8,7 +8,7 @@ require('buffertools').extend();
 
 var SYNC_VERSION = '0.1';
 var program = require('commander');
-var Sync = require('../lib/Sync').class();
+var HistoricSync = require('../lib/HistoricSync').class();
 var async = require('async');
 
 program
@@ -18,7 +18,7 @@ program
   .option('-R --reverse', 'Sync backwards', 0)
   .parse(process.argv);
 
-var sync = new Sync({
+var historicSync = new HistoricSync({
   networkName: program.network
 });
 
@@ -27,23 +27,22 @@ if (program.remove) {
 }
 
 async.series([
-function(cb) {
-  sync.init(program);
-  cb();
-},
-function(cb) {
-  sync.import_history(program, function(err, count) {
-    if (err) {
-      console.log('CRITICAL ERROR: ', err);
-    }
-    else {
-      console.log('Done! [%d blocks]', count, err);
-    }
+  function(cb) {
+    historicSync.init(program, cb);
+  },
+  function(cb) {
+    historicSync.import_history(program, function(err, count) {
+      if (err) {
+        console.log('CRITICAL ERROR: ', err);
+      }
+      else {
+        console.log('Done! [%d blocks]', count, err);
+      }
+      cb();
+    });
+  },
+  function(cb) {
+    historicSync.close();
     cb();
-  });
-},
-function(cb) {
-  sync.close();
-  cb();
 }]);