remove recursion from HistoricSync for speed

lib/HistoricSync.js

@@ -260,6 +260,7 @@ function spec() {
   };
 
 
+
   HistoricSync.prototype.getBlockFromFile = function(height, scanOpts, cb) {
     var self = this;
 
@@ -288,6 +289,7 @@ function spec() {
     },
     //get Info
     function(c) {
+
       self.blockExtractor.getNextBlock(function(err, b) {
         if (err || ! b) return c(err);
 
@@ -348,16 +350,12 @@ function spec() {
       else {
 
         // Continue
-        if (blockInfo) {
+        if (blockInfo && blockInfo.hash) {
-
-          // mainchain
-          if (isMainChain) height++;
-
           self.syncedBlocks++;
           self.err = null;
           self.status = 'syncing';
 
-          return self.getBlockFromFile(height, scanOpts, cb);
+          return cb(null, true, isMainChain);
         }
         else {
           self.err = null;
@@ -421,44 +419,16 @@ function spec() {
       }
     },
     ], function(err) {
-      var start, end;
+      if (err) {
-      function sync() {
+        self.setError(err);
-        if (scanOpts.reverse) {
+        return next(err, 0);
-          start = lastBlock;
-          end = self.genesis;
-          scanOpts.prev = true;
-        }
-        else {
-          start = self.genesis;
-          end = null;
-          scanOpts.next = true;
-        }
-        p('Starting from: ', start);
-        p('         to  : ', end);
-        p('         scanOpts: ', JSON.stringify(scanOpts));
-
-        if (scanOpts.fromFiles) {
-          self.getBlockFromFile(0, scanOpts, function(err) {
-            return next(err);
-          });
-        }
-        else {
-          self.getPrevNextBlock(start, end, scanOpts, function(err) {
-            if (err && err.message.match(/ECONNREFUSED/)) {
-              setTimeout(function() {
-                p('Retrying in %d secs', retry_secs);
-                sync();
-              },
-              retry_secs * 1000);
-            }
-            else return next(err);
-          });
-        }
       }
 
 
-      if (!self.step) {
+      // SETUP Sync params
+      var start, end;
 
+      if (!self.step) {
         var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000);
 
         if (self.opts.progressStep) {
@@ -469,13 +439,49 @@ function spec() {
         self.step = step;
       }
 
-      if (err) {
+      if (scanOpts.reverse) {
-        self.setError(err);
+        start = lastBlock;
-        return next(err, 0);
+        end = self.genesis;
+        scanOpts.prev = true;
       }
       else {
-        sync();
+        start = self.genesis;
+        end = null;
+        scanOpts.next = true;
       }
+      p('Starting from: ', start);
+      p('         to  : ', end);
+      p('         scanOpts: ', JSON.stringify(scanOpts));
+
+      if (scanOpts.fromFiles) {
+
+        var keepGoing = true;
+        var height    = 0;
+
+        async.whilst(function() {
+          return keepGoing;
+        }, function (w_cb) {
+          self.getBlockFromFile(height, scanOpts, function(err, inKeepGoing, wasMainChain) {
+            keepGoing = inKeepGoing;
+            if (wasMainChain) height++;  
+            //Black magic from http://stackoverflow.com/questions/20936486/node-js-maximum-call-stack-size-exceeded
+            setImmediate(function(){
+
+              return w_cb(err);
+            });
+          })
+        }, function(err) {
+
+console.log('[HistoricSync.js.468]'); //TODO
+          return next();
+        });
+      }
+      else {
+        self.getPrevNextBlock(start, end, scanOpts, function(err) {
+          return next(err);
+        });
+      }
+
     });
   };
 

util/sync.js

@@ -1,4 +1,5 @@
-#! /usr/bin/env node
+#!/usr/bin/env node 
+
 
 'use strict';