fromFiles WIP

app/models/TransactionOut.js

@@ -196,6 +196,8 @@ TransactionOutSchema.statics.createFromTxs = function(txs, next) {
         });
       },
       function(a_cb) {
+        if (txid === genesisTXID) return a_cb();
+
         Self.storeTransactionOuts(txInfo, function(err, addrs) {
           if (err) return a_cb(err);
 

lib/BlockExtractor.js

@@ -80,11 +80,9 @@ function spec() {
 
     var fd = fs.openSync(fname, 'r');
 
-//    if (status) return cb(new Error(status.message));
-
     var buffer = new Buffer(size);
 
-    var num = fs.readSync(fd, buffer, 0, size, 0);
+    fs.readSync(fd, buffer, 0, size, 0);
 
     self.currentBuffer = buffer;
     self.currentParser = new Parser(buffer);

lib/HistoricSync.js

@@ -13,6 +13,7 @@ function spec() {
   var Block = require('../app/models/Block');
   var Sync = require('./Sync').class();
   var sockets = require('../app/controllers/socket.js');
+  var BlockExtractor = require('./BlockExtractor.js').class();
 
 
   var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:';
@@ -125,7 +126,6 @@ if (self.syncPercentage>5) {
 
     var existed = false;
     var blockInfo;
-    var blockObj;
 
     async.series([
     // Already got it?
@@ -137,7 +137,6 @@ if (self.syncPercentage>5) {
         }
         if (block) {
           existed = true;
-          blockObj = block;
         }
         return c();
       });
@@ -150,22 +149,20 @@ if (self.syncPercentage>5) {
 
       return c();
     },
-    //get Info from RPC
-    function(c) {
 
-      // TODO: if we store prev/next, no need to go to RPC
-      // if (blockObj && blockObj.nextBlockHash) return c();
+    function(c) {
       self.rpc.getBlock(blockHash, function(err, ret) {
         if (err) return c(err);
 
-        blockInfo = ret;
+        blockInfo = ret ? ret.result : null;
         return c();
       });
     },
     //store it
     function(c) {
       if (existed) return c();
-      self.sync.storeBlock(blockInfo.result, function(err) {
+
+      self.sync.storeBlock(blockInfo, function(err) {
 
         existed = err && err.toString().match(/E11000/);
 
@@ -190,6 +187,7 @@ if (self.syncPercentage>5) {
         self.status = 'aborted';
         self.showProgress();
         p(self.err);
+        return cb(err);
       }
       else {
         self.err = null;
@@ -205,7 +203,7 @@ if (self.syncPercentage>5) {
       }
 
       // Continue
-      if (blockInfo && blockInfo.result) {
+      if (blockInfo) {
 
         if (existed)
           self.skippedBlocks++;
@@ -213,16 +211,98 @@ if (self.syncPercentage>5) {
           self.syncedBlocks++;
 
         // recursion
-        if (scanOpts.prev && blockInfo.result.previousblockhash)
-          return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, scanOpts, cb);
+        if (scanOpts.prev && blockInfo.previousblockhash)
+          return self.getPrevNextBlock(blockInfo.previousblockhash, blockEnd, scanOpts, cb);
         
-        if (scanOpts.next && blockInfo.result.nextblockhash)
-          return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, scanOpts, cb);
+        if (scanOpts.next && blockInfo.nextblockhash)
+          return self.getPrevNextBlock(blockInfo.nextblockhash, blockEnd, scanOpts, cb);
       }
       return cb(err);
     });
   };
 
+
+  HistoricSync.prototype.getBlockFromFile = function(height, scanOpts, cb) {
+    var self = this;
+
+    var nextHash;
+    var blockInfo;
+    var isMainChain;
+    var existed;
+
+    async.series([
+    // Is it in mainchain?
+    function(c) {
+      self.rpc.getBlockHash(height, function(err, res) {
+        if (err) return cb(err);
+
+        nextHash  = res.result;
+//console.log('[HistoricSync.js.235:nextHash:]',nextHash); //TODO
+        return c();
+      });
+    },
+    //show some (inacurate) status
+    function(c) {
+      if ( ( self.syncedBlocks + self.skippedBlocks)  % self.step === 1) {
+        self.showProgress();
+      }
+
+      return c();
+    },
+    //get Info
+    function(c) {
+      self.blockExtractor.getNextBlock(function(err, b) {
+        if (err || ! b) return c(err);
+
+        blockInfo = b.getStandardizedObject(b.txs);
+        return c();
+      });
+    },
+    //store it
+    function(c) {
+
+      isMainChain = blockInfo.hash === nextHash;
+
+      //TODO
+      blockInfo.isOrphan = !isMainChain;
+
+      self.sync.storeBlock(blockInfo, function(err) {
+        existed = err && err.toString().match(/E11000/);
+
+        if (err && ! existed) return c(err);
+        return c();
+      });
+    },
+    ], function(err) {
+
+      if (err) {
+        self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo.hash, err, self.syncedBlocks);
+        self.status = 'aborted';
+        self.showProgress();
+        p(err);
+        return cb(err);
+      }
+      else {
+        self.err = null;
+        self.status = 'syncing';
+      }
+
+      // Continue
+      if (blockInfo) {
+
+        // mainchain
+        if (isMainChain) height++;
+
+        self.syncedBlocks++;
+
+        return self.getBlockFromFile(height, scanOpts, cb);
+      }
+      return cb(err);
+    });
+  };
+
+
+
   HistoricSync.prototype.importHistory = function(scanOpts, next) {
     var self = this;
 
@@ -290,16 +370,23 @@ if (self.syncPercentage>5) {
         p('         to  : ', end);
         p('         scanOpts: ', JSON.stringify(scanOpts));
 
-        self.getPrevNextBlock(start, end, scanOpts, function(err) {
-          if (err && err.message.match(/ECONNREFUSED/)) {
-            setTimeout(function() {
-              p('Retrying in %d secs', retry_secs);
-              sync();
-            },
-            retry_secs * 1000);
-          }
-          else return next(err);
-        });
+        if (scanOpts.fromFiles) {
+          self.getBlockFromFile(0, scanOpts, function(err) {
+            return next(err);
+          });
+        }
+        else {
+          self.getPrevNextBlock(start, end, scanOpts, function(err) {
+            if (err && err.message.match(/ECONNREFUSED/)) {
+              setTimeout(function() {
+                p('Retrying in %d secs', retry_secs);
+                sync();
+              },
+              retry_secs * 1000);
+            }
+            else return next(err);
+          });
+        }
       }
 
 
@@ -326,7 +413,7 @@ if (self.syncPercentage>5) {
   };
 
   // upto if we have genesis block?
-  HistoricSync.prototype.smartImport = function(next) {
+  HistoricSync.prototype.smartImport = function(scanOpts, next) {
     var self = this;
 
     Block.fromHash(self.genesis, function(err, b) {
@@ -334,13 +421,12 @@ if (self.syncPercentage>5) {
       if (err) return next(err);
 
 
-      var scanOpts = {};
-
-      if (!b) {
+      if (!b || scanOpts.destroy) {
         p('Could not find Genesis block. Running FULL SYNC');
         if (config.bitcoind.dataDir) {
           p('bitcoind dataDir configured...importing blocks from .dat files');
           scanOpts.fromFiles = true;
+          self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
         }
         else {
           scanOpts.reverse = true;

lib/Sync.js

@@ -65,6 +65,7 @@ function spec() {
       function(b) { try {self.db.collections.transactionouts.drop(b);} catch (e) { return b(); } },
     ], next);
   };
+
   Sync.prototype.storeBlock = function(block, cb) {
     var self = this;
 

util/sync.js

@@ -16,6 +16,7 @@ program
   .option('-D --destroy', 'Remove current DB (and start from there)', 0)
   .option('-R --reverse', 'Sync backwards', 0)
   .option('-U --uptoexisting', 'Sync only until an existing block is found', 0)
+  .option('-F --fromfiles', 'Sync using bitcoind .dat block files (faster)', 0)
   .parse(process.argv);
 
 var historicSync = new HistoricSync();
@@ -32,13 +33,16 @@ async.series([
   function(cb) {
 
     if (program.smart) {
-      historicSync.smartImport(cb);
+      historicSync.smartImport({
+        destroy: program.destroy,
+      },cb);
     }
     else {
       historicSync.importHistory({
         destroy: program.destroy,
         reverse: program.reverse,
         upToExisting: program.uptoexisting,
+        fromFiles: program.fromfiles,
       }, cb);
     }
   },