sync fixes

lib/BlockDb.js

@@ -5,8 +5,8 @@ require('classtool');
 
 function spec() {
 
-  var TIMESTAMP_ROOT   = 'b-ts-';
-  var PREV_ROOT = 'b-prev-';      // b-prev-<hash> => <prev_hash> (0 if orphan)
+  var TIMESTAMP_ROOT  = 'b-ts-'; // b-ts-<ts> => <hash>
+  var PREV_ROOT       = 'b-prev-';      // b-prev-<hash> => <prev_hash> (0 if orphan)
 
 
   /**
@@ -49,6 +49,7 @@ function spec() {
   };
 
 
+
   BlockDb.prototype.setOrphan = function(hash, cb) {
     var self = this;
     
@@ -80,9 +81,8 @@ function spec() {
 
 
   BlockDb.prototype.countNotOrphan = function(cb) {
-    var self = this;
     var c = 0;
-    this.db.createReadStream({start: PREV_ROOT})
+    this.db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' })
       .on('data', function (data) {
         if (data.value !== 0) c++;
       })

lib/BlockExtractor.js

@@ -113,7 +113,7 @@ function spec() {
             magic = self.currentParser ? self.currentParser.buffer(4).toString('hex')
                         : null ;
 
-            if (!self.currentParser || self.currentParser.eof()) {
+            if (!self.currentParser || self.currentParser.eof() || magic === '00000000') {
               magic = null;
               if (self.nextFile()) {
                 console.log('Moving forward to file:' + self.currentFile() );
@@ -121,7 +121,8 @@ function spec() {
               }
               else {
                 console.log('Finished all files');
-                return cb();
+                magic = null;
+                return w_cb();
               }
             }
             else {
@@ -130,6 +131,8 @@ function spec() {
           }, a_cb);
       },
       function (a_cb) {
+        if (!magic) return a_cb();
+
         if (magic !== self.magic) {
           var e = new Error('CRITICAL ERROR: Magic number mismatch: ' +
                     magic + '!=' + self.magic);
@@ -141,6 +144,8 @@ function spec() {
         return a_cb();
       },
       function (a_cb) {
+        if (!magic) return a_cb();
+
         b = new Block();
         b.parse(self.currentParser);
         b.getHash();

lib/HistoricSync.js

@@ -115,13 +115,6 @@ function spec() {
     if (self.opts.shouldBroadcast) {
       sockets.broadcastSyncInfo(self.info());
     }
-
-//TODO
-  /*
-if (self.syncPercentage>10) {
-  console.log(self.info());
-  process.exit(1);
-}*/
   };
 
   HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) {
@@ -247,9 +240,7 @@ if (self.syncPercentage>10) {
         addrStrs = [ addr.toString() ];
         break;
       case Script.TX_MULTISIG:
-        var addrs = [];
         var chunks = s.capture();
-
         chunks.forEach(function(chunk) {
           var a = new Address(self.network.addressPubkey,  bitutil.sha256ripe160(chunk));
           addrStrs.push(a.toString());
@@ -262,15 +253,10 @@ if (self.syncPercentage>10) {
     return addrStrs;
   };
 
-var kk=0;
-
   HistoricSync.prototype.getBlockFromFile = function(scanOpts, cb) {
     var self = this;
-
-    var nextHash;
     var blockInfo;
     var existed;
-
     async.series([
     //show some (inacurate) status
     function(c) {
@@ -313,18 +299,18 @@ var kk=0;
     },
     //store it
     function(c) {
-      if (self.prevHash && blockInfo.prev_block !== self.prevHash) {
-
-        console.log('Orphans found:  @%s', blockInfo.hash);
+      if (blockInfo && self.prevHash && blockInfo.prev_block !== self.prevHash) {
 
+        console.log('Hole found @%s', blockInfo.hash);
         console.log('From: %s To: %s', self.prevHash, blockInfo.prev_block);
-process.exit(1);
-        self.sync.setOrphan(self.prevHash, blockInfo.prev_block, c);
+        self.sync.checkOrphan(self.prevHash, blockInfo.prev_block, c);
       }
       else return c();
     },
     //store it
     function(c) {
+      if (!blockInfo) return c();
+
       self.sync.storeBlock(blockInfo, function(err) {
         existed = err && err.toString().match(/E11000/);
 
@@ -343,7 +329,7 @@ process.exit(1);
     },
     ], function(err) {
       if (err) {
-        self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo.hash, err, self.syncedBlocks));
+        self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks));
       }
       return cb(err);
     });
@@ -437,9 +423,6 @@ process.exit(1);
       p('         scanOpts: ', JSON.stringify(scanOpts));
 
       if (scanOpts.fromFiles) {
-
-        var height    = 0;
-
         self.status = 'syncing';
         async.whilst(function() {
           return self.status === 'syncing';

lib/Sync.js

@@ -94,23 +94,41 @@ function spec() {
     });
   };
 
-  Sync.prototype.setOrphan = function(fromBlock, toBlock, c) {
+  Sync.prototype.checkOrphan = function(fromBlock, toBlock, c) {
     var self = this;
 
     var hash = fromBlock;
 
+    var co = 0;
+    var limit = 10;
+    var cont = 1;
+
     async.whilst(
       function () {
-        return hash && hash !== toBlock;
+        if (++co > limit) {
+console.log('[Sync.js.109] WARN: Reach reog depth limit'); //TODO
+        }
+        return cont && hash && hash !== toBlock && co < limit;
       },
       function (w_c) {
-        self.txDb.setOrphan(hash, function(err, insertedTxs, updateAddrs) {
-          if (err) return w_c(err);
-          self.blockDb.setOrphan(hash, function(err, prevHash){
+        //check with RPC if the block is mainchain
+        self.blockDb.fromHashWithInfo(hash, function (err, info) {
 
-            hash = prevHash;
-            return w_c(err);
-          });
+          if (!info) {
+            console.log('[Sync.js.107:hash:ORPHAN]',hash); //TODO
+            self.txDb.setOrphan(hash, function(err) {
+              if (err) return w_c(err);
+              self.blockDb.setOrphan(hash, function(err, prevHash){
+                hash = prevHash;
+                return w_c(err);
+              });
+            });
+          }
+          else {
+            console.log('[Sync.js.107:hash:NOT ORPHAN]',hash); //TODO
+            cont = 0;
+            return w_c();
+          }
         });
       },
       function (err) {

lib/TransactionDb.js

@@ -368,7 +368,7 @@ console.log('[TransactionDb.js.165]', ret); //TODO
 // txs can be a [hashes] or [txObjects]
   TransactionDb.prototype.createFromBlock = function(b, next) {
     var self = this;
-    if (!b.tx) return next();
+    if (!b || !b.tx) return next();
 
     return self.createFromArray(b.tx, b.hash, next);
   };

test/integration/sync.js

@@ -42,8 +42,25 @@ var test = function(s,cb) {
 
 
 
+var testNo = function(s,cb) {
+  async.each([2,3,4], function(i,c) {
+    s.blockDb.getPrev(b[i], function(err, p) {
+      assert.equal(p,b[i-1]);
+      return c();
+    });
+  }, function() {
+    s.blockDb.getPrev(b[1], function(err, p) {
+      assert.equal(p,b[0]);
+      return cb();
+    });
+  });
+};
+
+
+
+
 var s;
-describe('Sync setOrphan', function(){
+describe('Sync checkOrphan', function(){
 
   before(function(done) {
     s = new Sync();
@@ -54,15 +71,15 @@ describe('Sync setOrphan', function(){
     fix(s,done);
   });
 
-  it('setOrphan', function(done) {
+  it('checkOrphan', function(done) {
     this.timeout(100000);
 
     s.blockDb.has(b[0], function(err, has) {
       assert(has);
        s.blockDb.has(b[1], function(err, has) {
         assert(has);
-        s.setOrphan(b[4],b[1], function() {
-          test(s,done);
+        s.checkOrphan(b[4],b[1], function() {
+          testNo(s,done);
         });
       });
     });