smart sync. Skips genesisTX

app/models/Block.js

@@ -55,6 +55,7 @@ BlockSchema.statics.customCreate = function(block, cb) {
   newBlock.hash = block.hash;
   newBlock.nextBlockHash = block.nextBlockHash;
 
+
   Transaction.createFromArray(block.tx, newBlock.time, function(err, inserted_txs) {
     if (err) return cb(err);
 

app/models/Transaction.js

@@ -19,6 +19,9 @@ var mongoose    = require('mongoose'),
 
 var CONCURRENCY = 5;
 
+// TODO: use bitcore networks module
+var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b';
+
 /**
  */
 var TransactionSchema = new Schema({
@@ -113,6 +116,10 @@ TransactionSchema.statics.createFromArray = function(txs, time, next) {
 
 TransactionSchema.statics.explodeTransactionItems = function(txid, time,  cb) {
 
+  // Is it from genesis block? (testnet==livenet)
+  // TODO: parse it from networks.genesisTX
+  if (txid === genesisTXID) return cb();
+
   this.queryInfo(txid, function(err, info) {
     if (err || !info) return cb(err);
 

lib/HistoricSync.js

@@ -16,6 +16,10 @@ function spec() {
     this.block_count= 0;
     this.block_total= 0;
     this.network    = config.network === 'testnet' ? networks.testnet: networks.livenet;
+
+    var genesisHashReversed = new Buffer(32);
+    this.network.genesisBlock.hash.copy(genesisHashReversed);
+    this.genesis = genesisHashReversed.reverse().toString('hex');
     this.sync       = new Sync(opts);
   }
 
@@ -44,12 +48,10 @@ function spec() {
 
   HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
 
-    var that = this;
+    var self = this;
 
     // recursion end.
-    if (!blockHash || (blockEnd && blockEnd === blockHash) ) {
+    if (!blockHash ) return cb();
-      return cb();
-    }
 
     var existed = 0;
     var blockInfo;
@@ -61,17 +63,16 @@ function spec() {
         Block.findOne({hash:blockHash}, function(err,block){
           if (err) { p(err); return c(err); }
           if (block) {
-            existed  = 1;
+            existed  =1;
-            blockObj = block;
+            blockObj =block;
           }
-
           return c();
         });
       },
       //show some (inacurate) status
       function(c) {
-        if (that.block_count % 1000 === 1) {
+        if (self.block_count % 1000 === 1) {
-          progress_bar('sync status:', that.block_count, that.block_total);
+          progress_bar('sync status:', self.block_count, self.block_total);
         }
         return c();
       },
@@ -81,7 +82,7 @@ function spec() {
         // TODO: if we store prev/next, no need to go to RPC
         // if (blockObj && blockObj.nextBlockHash) return c();
 
-        that.rpc.getBlock(blockHash, function(err, ret) {
+        self.rpc.getBlock(blockHash, function(err, ret) {
           if (err) return c(err);
 
           blockInfo = ret;
@@ -91,8 +92,10 @@ function spec() {
       //store it
       function(c) {
         if (existed) return c();
-        that.sync.storeBlock(blockInfo.result, function(err) {
+        self.sync.storeBlock(blockInfo.result, function(err) {
+
           existed = err && err.toString().match(/E11000/);
+
           if (err && ! existed)  return c(err);
           return c();
         });
@@ -110,39 +113,36 @@ function spec() {
       ],
       function (err){
 
+        if (err)
+          p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, self.block_count);
+
         if (opts.uptoexisting && existed) {
           p('DONE. Found existing block: ', blockHash);
           return cb(err);
         }
 
-        if (err)
+        if (blockEnd && blockEnd === blockHash) {
-          p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count);
+          p('DONE. Found END block: ', blockHash);
+          return cb(err);
+        }
 
+
+        // Continue
         if (blockInfo && blockInfo.result) {
-          block_total++;
+          self.block_count++;
           if (opts.prev && blockInfo.result.previousblockhash) {
-            return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
+            return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
           }
 
           if (opts.next && blockInfo.result.nextblockhash)
-            return that.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
+            return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
         }
         return cb(err);
     });
   };
 
-  HistoricSync.prototype.syncBlocks = function(start, end, opts, cb) {
-    var that = this;
-
-    p('Starting from: ', start);
-    p('         to  : ', end);
-    p('         opts: ', JSON.stringify(opts));
-
-    return that.getPrevNextBlock( start, end, opts , cb);
-  };
-
   HistoricSync.prototype.import_history = function(opts, next) {
-    var that = this;
+    var self = this;
 
     var retry_attemps = 100;
     var retry_secs    = 2;
@@ -154,7 +154,7 @@ function spec() {
       function(cb) {
         if (opts.destroy) {
           p('Deleting Blocks...');
-          that.db.collections.blocks.drop(cb);
+          self.db.collections.blocks.drop(cb);
         } else {
           return cb();
         }
@@ -162,7 +162,7 @@ function spec() {
       function(cb) {
         if (opts.destroy) {
           p('Deleting TXs...');
-          that.db.collections.transactions.drop(cb);
+          self.db.collections.transactions.drop(cb);
         } else {
           return cb();
         }
@@ -170,16 +170,16 @@ function spec() {
       function(cb) {
         if (opts.destroy) {
           p('Deleting TXItems...');
-          that.db.collections.transactionitems.drop(cb);
+          self.db.collections.transactionitems.drop(cb);
         } else {
           return cb();
         }
       },
       function(cb) {
-        that.rpc.getInfo(function(err, res) {
+        self.rpc.getInfo(function(err, res) {
           if (err) return cb(err);
 
-          that.block_total = res.result.blocks;
+          self.block_total = res.result.blocks;
           return cb();
         });
       },
@@ -187,7 +187,7 @@ function spec() {
       function(cb) {
         if (!opts.reverse) return cb();
 
-        that.rpc.getBlockCount(function(err, res) {
+        self.rpc.getBlockCount(function(err, res) {
           if (err) return cb(err);
           block_height = res.result;
           return cb();
@@ -196,7 +196,7 @@ function spec() {
       function(cb) {
         if (!opts.reverse) return cb();
 
-        that.rpc.getBlockHash(block_height, function(err, res) {
+        self.rpc.getBlockHash(block_height, function(err, res) {
           if (err) return cb(err);
 
           block_best = res.result;
@@ -210,17 +210,20 @@ function spec() {
         function sync() {
           if (opts.reverse) {
             start     = block_best;
-            end       = that.network.genesisBlock.hash.reverse().toString('hex');
+            end       = self.genesis;
             opts.prev      = true;
           }
           else {
-            start     = that.network.genesisBlock.hash.reverse().toString('hex');
+            start     = self.genesis;
             end       = null;
             opts.next      = true;
           }
 
-          that.syncBlocks(start, end, opts, function(err) {
+          p('Starting from: ', start);
+          p('         to  : ', end);
+          p('         opts: ', JSON.stringify(opts));
 
+          self.getPrevNextBlock( start, end, opts , function(err) {
             if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){
               setTimeout(function() {
                 p('Retrying in %d secs', retry_secs);
@@ -228,7 +231,7 @@ function spec() {
               }, retry_secs * 1000);
             }
             else
-              return next(err, that.block_count);
+              return next(err, self.block_count);
           });
         }
         if (!err)
@@ -238,13 +241,28 @@ function spec() {
     });
   };
 
-  // Reverse Imports (upto if we have genesis block?)
+  // upto if we have genesis block?
   HistoricSync.prototype.smart_import = function(next) {
-    var that = this;
+    var self = this;
-    var opts = {
+
-      prev: 1,
+    Block.findOne({hash:self.genesis}, function(err, b){
-    };
+      if (err) return next(err);
-    that.import_history(opts, next);
+
+
+      if (!b) {
+        p('Could not find Genesis block. Running FULL SYNC');
+      }
+      else {
+        p('Genesis block found. Syncing upto know blocks.');
+      }
+
+      var opts = {
+        reverse: 1,
+        uptoexisting: b ? true: false,
+      };
+
+      return self.import_history(opts, next);
+    });
   };
 
 

lib/Sync.js

@@ -19,6 +19,7 @@ function spec() {
     var that = this;
 
     Block.customCreate(block, function(err, block, inserted_txs){
+      if (err) return cb(err);
 
       if (block && that.opts.broadcast_blocks) {
         sockets.broadcast_block(block);
@@ -56,17 +57,6 @@ function spec() {
   };
 
 
-  Sync.prototype.syncBlocks = function(start, end, isForward, cb) {
-    var that = this;
-
-    console.log('Syncing Blocks, starting \n\tfrom: %s \n\tend: %s \n\tisForward:',
-                  start, end, isForward);
-
-
-    return that.getPrevNextBlock( start, end,
-                  isForward ? { next: 1 } : { prev: 1}, cb);
-  };
-
   Sync.prototype.init = function(opts, cb) {
     var that = this;
 

server.js

@@ -50,9 +50,7 @@ if (!config.disableHistoricSync) {
     skip_db_connection: true,
     networkName: config.network
   }, function() {
-    hs.import_history({
+    hs.smart_import(function(){
-      reverse: 1,
-    }, function(){
       console.log('historic_sync finished!');
     });
   });

util/sync.js

@@ -24,33 +24,40 @@ var historicSync = new HistoricSync({
   networkName: program.network
 });
 
+/*  TODO: Sure?
 if (program.remove) {
-  // TODO: Sure?
+
 }
+*/
 
 async.series([
   function(cb) {
     historicSync.init(program, cb);
   },
   function(cb) {
-    historicSync.import_history({
+    if (program.smart) {
-      network: program.network,
+      historicSync.smart_import(cb);
-      destroy: program.destroy,
+    }
-      reverse: program.reverse,
+    else {
-      uptoexisting: program.uptoexisting,
+      historicSync.import_history({
-      smart: program.smart,
+        destroy: program.destroy,
-    }, function(err, count) {
+        reverse: program.reverse,
-      if (err) {
+        uptoexisting: program.uptoexisting,
-        console.log('CRITICAL ERROR: ', err);
+      }, cb);
-      }
+    }
-      else {
-        console.log('Finished. [%d blocks]', count);
-      }
-      cb();
-    });
   },
   function(cb) {
     historicSync.close();
-    cb();
+    return cb();
-}]);
+  },
+  ],
+  function(err, count) {
+    if (err) {
+      console.log('CRITICAL ERROR: ', err);
+    }
+    else {
+      console.log('Finished. [%d blocks synced]', count[1]);
+    }
+    return;
+});