simple test for sync

lib/BlockDb.js

@@ -16,8 +16,7 @@ function spec() {
       util        = require('bitcore/util/util'),
       levelup     = require('levelup'),
       BitcoreBlock= require('bitcore/Block').class(),
-      config      = require('../config/config'),
+      config      = require('../config/config');
-      fs          = require('fs');
 
   var BlockDb = function() {
     this.db = levelup(config.leveldb + '/blocks');
@@ -26,11 +25,11 @@ function spec() {
   BlockDb.prototype.drop = function(cb) {
     var self = this;
     var path = config.leveldb + '/blocks';
-    require('leveldown').destroy(path, function () {
+    self.db.close(function() {
-      fs.mkdirSync(config.leveldb);
+      require('leveldown').destroy(path, function () {
-      fs.mkdirSync(path);
+        self.db = levelup(path);
-      self.db = levelup(path);
+        return cb();
-      return cb();
+      });
     });
   };
 
@@ -52,21 +51,36 @@ function spec() {
 
   BlockDb.prototype.setOrphan = function(hash, cb) {
     var self = this;
-
     
     var k = PREV_ROOT + hash;
+
     self.db.get(k, function (err,oldPrevHash) {
       if (err || !oldPrevHash) return cb(err);
-     
       self.db.put(PREV_ROOT + hash, 0, function() {
         return cb(err, oldPrevHash);
       });
     });
-
     // We keep the block in TIMESTAMP_ROOT
   };
 
-  BlockDb.prototype.countNotOrphan = function(hash, cb) {
+  //mainly for testing
+  BlockDb.prototype.setPrev = function(hash, prevHash, cb) {
+    this.db.put(PREV_ROOT + hash, prevHash, function(err) {
+      return cb(err);
+    });
+  };
+
+  //mainly for testing
+  BlockDb.prototype.getPrev = function(hash, cb) {
+    this.db.get(PREV_ROOT + hash, function(err,val) {
+      return cb(err,val);
+    });
+  };
+
+
+
+  BlockDb.prototype.countNotOrphan = function(cb) {
+    var self = this;
     var c = 0;
     this.db.createReadStream({start: PREV_ROOT})
       .on('data', function (data) {
@@ -75,9 +89,6 @@ function spec() {
       .on('error', function (err) {
         return cb(err);
       })
-      .on('close', function () {
-        return cb(null);
-      })
       .on('end', function () {
         return cb(null);
       });

lib/HistoricSync.js

@@ -313,11 +313,12 @@ var kk=0;
     },
     //store it
     function(c) {
-      if (self.prevHash && blockInfo.prev_block !== self.prevHash) {
+      if (self.prevHash && blockinfo.prev_block !== self.prevHash) {
 
-        console.log('Orphans found: %s vs %s @%s',
+        console.log('Orphans found:  @%s', blockInfo.hash);
-          self.prevHash + ' vs. ' + blockInfo.prev_block, blockInfo.hash);
 
+        console.log('From: %s To: %s', self.prevHash, blockInfo.prev_block);
+process.exit(1);
         self.sync.setOrphan(self.prevHash, blockInfo.prev_block, c);
       }
       else return c();
@@ -349,6 +350,17 @@ var kk=0;
   };
 
 
+  HistoricSync.prototype.getBlockCount = function(cb) {
+    var self = this;
+
+    if (self.blockChainHeight) return cb();
+
+    self.rpc.getBlockCount(function(err, res) {
+      if (err) return cb(err);
+      self.blockChainHeight = res.result;
+      return cb();
+    });
+  };
 
   HistoricSync.prototype.importHistory = function(scanOpts, next) {
     var self = this;
@@ -364,13 +376,7 @@ var kk=0;
       return cb();
     },
     // We are not using getBestBlockHash, because is not available in all clients
-    function(cb) {
+    function (cb) { return self.getBlockCount(cb); },
-      self.rpc.getBlockCount(function(err, res) {
-        if (err) return cb(err);
-        self.blockChainHeight = res.result;
-        return cb();
-      });
-    },
     function(cb) {
       if (!scanOpts.reverse) return cb();
 
@@ -461,27 +467,37 @@ var kk=0;
     var self = this;
 
     self.sync.hasBlock(self.genesis, function(err, b) {
-
       if (err) return next(err);
+      self.sync.countNotOrphan(function(err, count) {
+        if (err) return next(err);
+          self.getBlockCount(function(err) {
+            if (err) return next(err);
 
-      if (!b || scanOpts.destroy) {
+            if (!b || scanOpts.destroy || self.blockChainHeight > count * 0.7 ) {
-        p('Could not find Genesis block. Running FULL SYNC');
-        if (config.bitcoind.dataDir) {
-          p('bitcoind dataDir configured...importing blocks from .dat files');
-          scanOpts.fromFiles = true;
-          self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
-        }
-        else {
-          scanOpts.reverse = true;
-        }
-      }
-      else {
-        p('Genesis block found. Syncing upto known blocks.');
-        scanOpts.reverse = true;
-        scanOpts.upToExisting = true;
-      }
 
-      return self.importHistory(scanOpts, next);
+              if (!b)
+                p('Could not find Genesis block. Running FULL SYNC');
+              else
+                p('Less that 70% of current blockchain is stored. Running FULL SYNC',
+                  parseInt(count/self.blockChainHeight*100));
+
+              if (config.bitcoind.dataDir) {
+                p('bitcoind dataDir configured...importing blocks from .dat files');
+                scanOpts.fromFiles = true;
+                self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
+              }
+              else {
+                scanOpts.reverse = true;
+              }
+            }
+            else {
+              p('Genesis block found. Syncing upto known blocks.');
+              scanOpts.reverse = true;
+              scanOpts.upToExisting = true;
+            }
+            return self.importHistory(scanOpts, next);
+        });
+      });
     });
   };
 

lib/Sync.js

@@ -63,8 +63,7 @@ function spec() {
     var self = this;
     async.series([
       function(b) { try {self.blockDb.drop(b);} catch (e) { return b(); } },
-      function(b) { try {self.db.collections.transactionitems.drop(b);} catch (e) { return b(); } },
+      function(b) { try {self.TransactionDb.drop(b);} catch (e) { return b(); } },
-      function(b) { try {self.db.collections.transactionouts.drop(b);} catch (e) { return b(); } },
     ], next);
   };
 
@@ -81,7 +80,6 @@ function spec() {
   };
  
 
-
   Sync.prototype.storeBlock = function(block, cb) {
     var self = this;
 
@@ -99,26 +97,24 @@ function spec() {
   Sync.prototype.setOrphan = function(fromBlock, toBlock, c) {
     var self = this;
 
-    var c = fromBlock;
+    var hash = fromBlock;
 
     async.whilst(
       function () {
-        return c !== toBlock;
+        return hash && hash !== toBlock;
       },
-      function () {
+      function (w_c) {
-        console.log('[Sync.js.113]: setOrphan', c); //TODO
         self.txDb.setOrphan(c, function(err, insertedTxs, updateAddrs) {
-          if (err) return cb(err);
+          if (err) return w_c(err);
+          self.blockDb.setOrphan(hash, function(err, prevHash){
 
-          self.blockDb.setOrphan(c, function(err, prevHash){
+            hash = prevHash;
-
+            return w_c(err);
-            c = prevHash;
-            return cb(err);
           });
         });
       },
-      function (err) { 
+      function (err) {
-        return c(err); 
+        return c(err);
       }
     );
   };

lib/TransactionDb.js

@@ -26,8 +26,7 @@ function spec() {
       util        = require('bitcore/util/util'),
       levelup     = require('levelup'),
       async       = require('async'),
-      config      = require('../config/config'),
+      config      = require('../config/config');
-      fs          = require('fs');
 
   var TransactionDb = function() {
     this.db = levelup(config.leveldb + '/txs');
@@ -35,12 +34,12 @@ function spec() {
 
   TransactionDb.prototype.drop = function(cb) {
     var self = this;
-    var path = config.leveldb + '/blocks';
+    var path = config.leveldb + '/txs';
-    require('leveldown').destroy(path, function () {
+    self.db.close(function() {
-      fs.mkdirSync(config.leveldb);
+      require('leveldown').destroy(path, function () {
-      fs.mkdirSync(path);
+        self.db = levelup(path);
-      self.db = levelup(path);
+        return cb();
-      return cb();
+      });
     });
   };
 
@@ -129,9 +128,6 @@ function spec() {
     var k = ADDR_ROOT + addr;
     var ret=[];
 
-  //var ADDR_ROOT    = 'txouts-addr-'; //txouts-addr-<addr>-<ts>-<txid>-<n> => + btc_sat
-  //var SPEND_ROOT   = 'txouts-spend-';//txouts-spend-<txid(out)>-<n(out)> => [txid(in),n(in),ts]
-    //
     //
     self.db.createReadStream({start: k, end: k + '~'})
       .on('data', function (data) {
@@ -149,7 +145,7 @@ function spec() {
       })
       .on('end', function () {
         async.each(ret, function(o, e_c) {
-          var k = SPEND_ROOT + '-' + o.txid + '-' + o.index; //TODO ---
+          var k = SPEND_ROOT + o.txid + '-' + o.index;
           self.db.get(k, function(err, val) {
             if (err && err.notFound) err=null;
             if (err || !val) return e_c(err);

package.json

@@ -60,7 +60,6 @@
     "bignum": "*",
     "express": "~3.4.7",
     "jade": "~1.0.2",
-    "mongoose": "~3.8.3",
     "lodash": "~2.4.1",
     "bower": "~1.2.8",
     "buffertools": "*",

test/integration/block.js

@@ -27,18 +27,25 @@ describe('BlockDb fromHashWithInfo', function(){
   it('return true in has', function(done) {
     bdb.has(TESTING_BLOCK, function(err, has) {
       assert.equal(has, true);
-console.log('[block.js.29:has:]',has); //TODO
       done();
     });
   });
-  it('return false in has', function(done) {
+  it('setOrphan', function(done) {
-    bdb.has('111', function(err, has) {
+    var b16 = '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b';
-      assert.equal(has, false);
+    var b17 = '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743';
-      done();
+
+    bdb.has(b17, function(err, has) {
+      assert(has);
+      bdb.setOrphan(b17, function(err, oldPrev) {
+        assert.equal(oldPrev, b16);
+        bdb.setPrev(b17, b16, function(err, oldPrev) {
+          bdb.getPrev(b17, function(err, p) {
+            assert.equal(p, b16);
+            done();
+          });
+        });
+      });
     });
   });
-
-
-
 });
 

test/integration/sync.js

@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+'use strict';
+
+process.env.NODE_ENV = process.env.NODE_ENV || 'development';
+
+
+var
+  assert  = require('assert'),
+  async   = require('async'),
+  Sync     = require('../../lib/Sync').class();
+
+
+var b = [
+  '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b', //B#16
+  '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743',
+  '000000008d55c3e978639f70af1d2bf1fe6f09cb3143e104405a599215c89a48',
+  '000000009b3bca4909f38313f2746120129cce4a699a1f552390955da470c5a9',
+  '00000000ede57f31cc598dc241d129ccb4d8168ef112afbdc870dc60a85f5dd3', //B#20
+];
+
+var fix = function(s,cb) {
+  async.each([1,2,3,4], function(i,c) {
+    s.blockDb.setPrev(b[i],b[i-1], function() {
+      return c();
+    });
+  }, cb);
+};
+
+var test = function(s,cb) {
+  async.each([2,3,4], function(i,c) {
+    s.blockDb.getPrev(b[i], function(err, p) {
+      assert.equal(p,0);
+      return c();
+    });
+  }, function() {
+    s.blockDb.getPrev(b[1], function(err, p) {
+      assert.equal(p,b[0]);
+      return cb();
+    });
+  });
+};
+
+
+
+var s;
+describe('Sync setOrphan', function(){
+
+  before(function(done) {
+    s = new Sync();
+    fix(s,done);
+  });
+
+  after(function(done) {
+    fix(s,done);
+  });
+
+  it('setOrphan', function(done) {
+    this.timeout(100000);
+
+    s.blockDb.has(b[0], function(err, has) {
+      assert(has);
+       s.blockDb.has(b[1], function(err, has) {
+        assert(has);
+        s.setOrphan(b[4],b[1], function() {
+          test(s,done);
+        });
+      });
+    });
+  });
+});
+