Merge pull request #95 from matiu/feature/scripts

Feature/scripts

README.md

@@ -9,6 +9,19 @@ require certain information from the blockchain that bitcoind does not provide.
 A blockchain explorer front-end have been developed to top of *Insight API*, it can
 be downloaded at [Github Insight Repository](https://github.com/bitpay/insight).
 
+## IMPORTANT: Upgrading from  v0.1 to v0.2
+In order to optimize some queries, the key-value layout in Level DB has been changed.
+An update script need to be run when updating from version v0.1.x to 0.2.x
+
+The script is located at:
+```
+  $ util/updateToV0.2.js
+```
+
+The script takes ~3minutes in testnet. While the script is running,  *Insight API*
+must be stopped.
+
+Alternatively, a total resync can be made, running `$ util/sync.js -D`
 
 ## Prerequisites
 

lib/BlockDb.js

@@ -250,6 +250,7 @@ BlockDb.prototype.setNext = function(hash, nextHash, cb) {
   });
 };
 
+// Unused
 BlockDb.prototype.countConnected = function(cb) {
   var c = 0;
   console.log('Counting connected blocks. This could take some minutes');
@@ -368,14 +369,10 @@ BlockDb.prototype.fillConfirmations = function(txouts, cb) {
       return !x.spentIsConfirmedCached // not 100%cached
         && !(x.isConfirmedCached && !x.spentTxId);  // and not 50%cached but not spent 
     });
-//console.log('[BlockDb.js.360:txouts:]',txs.length); //TODO
-var i=0;
     async.eachLimit(txs, CONCURRENCY, function(txout, e_c) {
       if(txout.isConfirmedCached) {
-//console.log('[BlockDb.js.378]', i++); //TODO
         self._fillConfirmationsOneSpent(txout,height, e_c);
       } else {
-//console.log('[BlockDb.js.3782]', i++); //TODO
         self._fillConfirmationsOne(txout,height, e_c);
       }
 
@@ -383,5 +380,45 @@ var i=0;
   });
 };
 
+/* this is only for migration scripts */
+BlockDb.prototype._runScript = function(script, cb) {
+  db.batch(script,cb);
+};
+
+BlockDb.prototype.migrateV02 = function(cb) {
+  var k = 'txb-';
+  this.txDb._db.createReadStream({
+    start: k,
+    end: k + '~'
+  }).pipe(db.createWriteStream()).on('close', cb);
+};
+
+BlockDb.prototype.migrateV02cleanup = function(cb) {
+
+  var self = this;
+  console.log('## deleting txb- from txs db'); //todo
+
+  var k = 'txb-';
+  var d = this.txDb._db;
+  d.createReadStream({
+    start: k,
+    end: k + '~'
+  })
+  .pipe(d.createWriteStream({type:'del'}))
+    .on('close', function(err){
+      if (err) return cb(err);
+      console.log('## deleting tx- from txs db'); //todo
+
+      var k = 'tx-';
+      var d = self.txDb._db;
+      d.createReadStream({
+        start: k,
+        end: k + '~'
+      })
+      .pipe(d.createWriteStream({type:'del'}))
+      .on('close',cb);
+    });
+};
+
 
 module.exports = require('soop')(BlockDb);

lib/HistoricSync.js

@@ -329,13 +329,25 @@ HistoricSync.prototype.setupSyncStatus = function() {
   this.syncPercentage = 0;
 };
 
+HistoricSync.prototype.checkDBVersion = function(cb) {
+  this.sync.txDb.checkVersion02(function(isOk){
+    if (!isOk) {
+      console.log('\n#############################\n\n ## Insight API DB is older that v0.2. Please resync using:\n $ util/sync.js -D\n More information at Insight API\'s Readme.md');
+      process.exit(1);
+    }
+    // Add more test here in future changes.
+    return cb();
+  });
+};
+
+
 HistoricSync.prototype.prepareToSync = function(opts, next) {
   var self = this;
 
   self.status = 'starting';
   async.series([
     function(s_c) {
-      self.sync.txDb.checkVersion02(s_c);
+      self.checkDBVersion(s_c);
     },
     function(s_c) {
       self.checkNetworkSettings(s_c);

lib/TransactionDb.js

@@ -59,6 +59,8 @@ var TransactionDb = function() {
   this.network = config.network === 'testnet' ? networks.testnet : networks.livenet;
   this.poolMatch = new PoolMatch();
   this.safeConfirmations = config.safeConfirmations || DEFAULT_SAFE_CONFIRMATIONS;
+
+  this._db = db; // this is only exposed for migration script
 };
 
 TransactionDb.prototype.close = function(cb) {
@@ -664,11 +666,8 @@ TransactionDb.prototype.getPoolInfo = function(txid, cb) {
 TransactionDb.prototype.checkVersion02 = function(cb) {
   var k = 'txb-f0315ffc38709d70ad5647e22048358dd3745f3ce3874223c80a7c92fab0c8ba-00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206';
   db.get(k, function(err, val) {
-    if (val) {
-      console.log('\n#############################\n\n ## Insight API DB is older that v0.2. Please resync using:\n $ util/sync.js -D\n More information at Insight API\'s Readme.md');
-      process.exit(1);
-    }
-    return cb();
+
+    return cb(!val);
   });
 };
 

util/updateToV0.2.js

@@ -0,0 +1,64 @@
+#!/usr/bin/env node 
+
+'use strict';
+
+var HistoricSync = require('../lib/HistoricSync');
+var async = require('async');
+
+
+var historicSync = new HistoricSync({ shouldBroadcastSync: false });
+var txDb=historicSync.sync.txDb;
+var bDb=historicSync.sync.bDb;
+
+var height  = 0;
+var hash    = historicSync.genesis;
+var tipHash;
+
+async.series([
+  function(c){
+    txDb.checkVersion02(function(isV2){
+      var err;
+      if(isV2) err='Already in v0.2!';
+      return c(err);
+    });
+  },
+  function(c){
+    var script=[];
+    async.whilst(
+      function() {
+        return hash;
+      },
+      function (w_cb) {
+        script=script.concat(bDb._setHeightScript(hash,height));
+        bDb.getNext(hash,function(err,val){
+          if (err) return w_cb(err);
+          tipHash = hash;
+          hash = val;
+          if (hash) height++;
+          if (!(height%1000) || !hash) {
+            console.log('*update 1/2\t%d blocks processed', height);
+            bDb._runScript(script, function(err) {
+              script=[];
+              return w_cb(err);
+            });
+          }
+          else return w_cb();
+        });
+      }, c);
+  },
+  function(c){
+    console.log('Migrating txs... (this will take some minutes...)'); //TODO
+    bDb.migrateV02(c);
+  },
+  function(c){
+    bDb.setTip(tipHash, height, c);
+  },
+  function(c){
+    bDb.migrateV02cleanup(c);
+  },
+  ],function(err){
+    if (err)
+      console.log('## '+err);
+    else
+      console.log('Finished OK.');
+});