diff --git a/README.md b/README.md index 02dd20e2..eecd3489 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,19 @@ require certain information from the blockchain that bitcoind does not provide. A blockchain explorer front-end have been developed to top of *Insight API*, it can be downloaded at [Github Insight Repository](https://github.com/bitpay/insight). +## IMPORTANT: Upgrading from v0.1 to v0.2 +In order to optimize some queries, the key-value layout in Level DB has been changed. +An update script need to be run when updating from version v0.1.x to 0.2.x + +The script is located at: +``` + $ util/updateToV0.2.js +``` + +The script takes ~3minutes in testnet. While the script is running, *Insight API* +must be stopped. + +Alternatively, a total resync can be made, running `$ util/sync.js -D` ## Prerequisites diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 93cb4d4c..b8e67b04 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -250,6 +250,7 @@ BlockDb.prototype.setNext = function(hash, nextHash, cb) { }); }; +// Unused BlockDb.prototype.countConnected = function(cb) { var c = 0; console.log('Counting connected blocks. This could take some minutes'); @@ -368,14 +369,10 @@ BlockDb.prototype.fillConfirmations = function(txouts, cb) { return !x.spentIsConfirmedCached // not 100%cached && !(x.isConfirmedCached && !x.spentTxId); // and not 50%cached but not spent }); -//console.log('[BlockDb.js.360:txouts:]',txs.length); //TODO -var i=0; async.eachLimit(txs, CONCURRENCY, function(txout, e_c) { if(txout.isConfirmedCached) { -//console.log('[BlockDb.js.378]', i++); //TODO self._fillConfirmationsOneSpent(txout,height, e_c); } else { -//console.log('[BlockDb.js.3782]', i++); //TODO self._fillConfirmationsOne(txout,height, e_c); } @@ -383,5 +380,45 @@ var i=0; }); }; +/* this is only for migration scripts */ +BlockDb.prototype._runScript = function(script, cb) { + db.batch(script,cb); +}; + +BlockDb.prototype.migrateV02 = function(cb) { + var k = 'txb-'; + this.txDb._db.createReadStream({ + start: k, + end: k + '~' + }).pipe(db.createWriteStream()).on('close', cb); +}; + +BlockDb.prototype.migrateV02cleanup = function(cb) { + + var self = this; + console.log('## deleting txb- from txs db'); //todo + + var k = 'txb-'; + var d = this.txDb._db; + d.createReadStream({ + start: k, + end: k + '~' + }) + .pipe(d.createWriteStream({type:'del'})) + .on('close', function(err){ + if (err) return cb(err); + console.log('## deleting tx- from txs db'); //todo + + var k = 'tx-'; + var d = self.txDb._db; + d.createReadStream({ + start: k, + end: k + '~' + }) + .pipe(d.createWriteStream({type:'del'})) + .on('close',cb); + }); +}; + module.exports = require('soop')(BlockDb); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index b0b65b52..29a79f02 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -329,13 +329,25 @@ HistoricSync.prototype.setupSyncStatus = function() { this.syncPercentage = 0; }; +HistoricSync.prototype.checkDBVersion = function(cb) { + this.sync.txDb.checkVersion02(function(isOk){ + if (!isOk) { + console.log('\n#############################\n\n ## Insight API DB is older that v0.2. Please resync using:\n $ util/sync.js -D\n More information at Insight API\'s Readme.md'); + process.exit(1); + } + // Add more test here in future changes. + return cb(); + }); +}; + + HistoricSync.prototype.prepareToSync = function(opts, next) { var self = this; self.status = 'starting'; async.series([ function(s_c) { - self.sync.txDb.checkVersion02(s_c); + self.checkDBVersion(s_c); }, function(s_c) { self.checkNetworkSettings(s_c); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index e58df2e0..e7a203e5 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -59,6 +59,8 @@ var TransactionDb = function() { this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; this.poolMatch = new PoolMatch(); this.safeConfirmations = config.safeConfirmations || DEFAULT_SAFE_CONFIRMATIONS; + + this._db = db; // this is only exposed for migration script }; TransactionDb.prototype.close = function(cb) { @@ -664,11 +666,8 @@ TransactionDb.prototype.getPoolInfo = function(txid, cb) { TransactionDb.prototype.checkVersion02 = function(cb) { var k = 'txb-f0315ffc38709d70ad5647e22048358dd3745f3ce3874223c80a7c92fab0c8ba-00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; db.get(k, function(err, val) { - if (val) { - console.log('\n#############################\n\n ## Insight API DB is older that v0.2. Please resync using:\n $ util/sync.js -D\n More information at Insight API\'s Readme.md'); - process.exit(1); - } - return cb(); + + return cb(!val); }); }; diff --git a/util/updateToV0.2.js b/util/updateToV0.2.js new file mode 100755 index 00000000..2e79ff39 --- /dev/null +++ b/util/updateToV0.2.js @@ -0,0 +1,64 @@ +#!/usr/bin/env node + +'use strict'; + +var HistoricSync = require('../lib/HistoricSync'); +var async = require('async'); + + +var historicSync = new HistoricSync({ shouldBroadcastSync: false }); +var txDb=historicSync.sync.txDb; +var bDb=historicSync.sync.bDb; + +var height = 0; +var hash = historicSync.genesis; +var tipHash; + +async.series([ + function(c){ + txDb.checkVersion02(function(isV2){ + var err; + if(isV2) err='Already in v0.2!'; + return c(err); + }); + }, + function(c){ + var script=[]; + async.whilst( + function() { + return hash; + }, + function (w_cb) { + script=script.concat(bDb._setHeightScript(hash,height)); + bDb.getNext(hash,function(err,val){ + if (err) return w_cb(err); + tipHash = hash; + hash = val; + if (hash) height++; + if (!(height%1000) || !hash) { + console.log('*update 1/2\t%d blocks processed', height); + bDb._runScript(script, function(err) { + script=[]; + return w_cb(err); + }); + } + else return w_cb(); + }); + }, c); + }, + function(c){ + console.log('Migrating txs... (this will take some minutes...)'); //TODO + bDb.migrateV02(c); + }, + function(c){ + bDb.setTip(tipHash, height, c); + }, + function(c){ + bDb.migrateV02cleanup(c); + }, + ],function(err){ + if (err) + console.log('## '+err); + else + console.log('Finished OK.'); +});