From c595d49a4e4c4c8f8990e59923a418678ac7c9b6 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 26 May 2014 18:49:03 -0300 Subject: [PATCH 1/3] db update script --- lib/BlockDb.js | 9 +++--- lib/HistoricSync.js | 14 ++++++++- lib/TransactionDb.js | 7 ++--- util/updateToV0.2.js | 68 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 10 deletions(-) create mode 100755 util/updateToV0.2.js diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 93cb4d4c..891ac4aa 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -250,6 +250,7 @@ BlockDb.prototype.setNext = function(hash, nextHash, cb) { }); }; +// Unused BlockDb.prototype.countConnected = function(cb) { var c = 0; console.log('Counting connected blocks. This could take some minutes'); @@ -368,14 +369,10 @@ BlockDb.prototype.fillConfirmations = function(txouts, cb) { return !x.spentIsConfirmedCached // not 100%cached && !(x.isConfirmedCached && !x.spentTxId); // and not 50%cached but not spent }); -//console.log('[BlockDb.js.360:txouts:]',txs.length); //TODO -var i=0; async.eachLimit(txs, CONCURRENCY, function(txout, e_c) { if(txout.isConfirmedCached) { -//console.log('[BlockDb.js.378]', i++); //TODO self._fillConfirmationsOneSpent(txout,height, e_c); } else { -//console.log('[BlockDb.js.3782]', i++); //TODO self._fillConfirmationsOne(txout,height, e_c); } @@ -383,5 +380,9 @@ var i=0; }); }; +// This is for DB upgrades mainly +BlockDb.prototype._runScript = function(script, cb) { + db.batch(script,cb); +}; module.exports = require('soop')(BlockDb); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index b0b65b52..29a79f02 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -329,13 +329,25 @@ HistoricSync.prototype.setupSyncStatus = function() { this.syncPercentage = 0; }; +HistoricSync.prototype.checkDBVersion = function(cb) { + this.sync.txDb.checkVersion02(function(isOk){ + if (!isOk) { + console.log('\n#############################\n\n ## Insight API DB is older that v0.2. Please resync using:\n $ util/sync.js -D\n More information at Insight API\'s Readme.md'); + process.exit(1); + } + // Add more test here in future changes. + return cb(); + }); +}; + + HistoricSync.prototype.prepareToSync = function(opts, next) { var self = this; self.status = 'starting'; async.series([ function(s_c) { - self.sync.txDb.checkVersion02(s_c); + self.checkDBVersion(s_c); }, function(s_c) { self.checkNetworkSettings(s_c); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index e58df2e0..ffd19b5d 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -664,11 +664,8 @@ TransactionDb.prototype.getPoolInfo = function(txid, cb) { TransactionDb.prototype.checkVersion02 = function(cb) { var k = 'txb-f0315ffc38709d70ad5647e22048358dd3745f3ce3874223c80a7c92fab0c8ba-00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; db.get(k, function(err, val) { - if (val) { - console.log('\n#############################\n\n ## Insight API DB is older that v0.2. Please resync using:\n $ util/sync.js -D\n More information at Insight API\'s Readme.md'); - process.exit(1); - } - return cb(); + + return cb(!val); }); }; diff --git a/util/updateToV0.2.js b/util/updateToV0.2.js new file mode 100755 index 00000000..5f165009 --- /dev/null +++ b/util/updateToV0.2.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node + +'use strict'; + +var HistoricSync = require('../lib/HistoricSync'); +var async = require('async'); + +// +// 1) +// var MAIN_PREFIX = 'bma-'; // bma- => (0 is unconnected) +// var TIP = 'bti-'; // bti = : last block on the chain +// +// var IN_BLK_PREFIX = 'btx-'; //btx- = +// v +// 2) DELETE txs/tx- +// 3) DELETE txs/txb +// + + +var historicSync = new HistoricSync({ shouldBroadcastSync: false }); +var txDb=historicSync.sync.txDb; +var bDb=historicSync.sync.bDb; + +var height = 0; +var hash = historicSync.genesis; +var tipHash; + +async.series([ + function(c){ + txDb.checkVersion02(function(isV2){ + var err; + if(isV2) err='Already in v0.2!'; + return c(err); + }); + }, + function(c){ + var script=[]; + async.whilst( + function() { + return hash; + }, + function (w_cb) { + script=script.concat(bDb._setHeightScript(hash,height)); + bDb.getNext(hash,function(err,val){ + if (err) return w_cb(err); + tipHash = hash; + hash = val; + height++; + if (!(height%1000) || !hash) { + console.log('\t%d blocks processed (set height 1/2)', height); + bDb._runScript(script, function(err) { + script=[]; + return w_cb(err); + }); + } + else return w_cb(); + }); + }, c); + }, + function(c){ + bDb.setTip(tipHash, height-1, c); + }, + ],function(err){ + if (err) + console.log('## '+err); + else + console.log('Finished OK.'); +}); From ef7ec2328dce7cce7d7d84bfd240245241106653 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 27 May 2014 16:14:44 -0300 Subject: [PATCH 2/3] upgrade script --- lib/BlockDb.js | 38 +++++++++++++++++++++++++++++++++++++- lib/TransactionDb.js | 2 ++ util/updateToV0.2.js | 24 ++++++++++-------------- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 891ac4aa..b8e67b04 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -380,9 +380,45 @@ BlockDb.prototype.fillConfirmations = function(txouts, cb) { }); }; -// This is for DB upgrades mainly +/* this is only for migration scripts */ BlockDb.prototype._runScript = function(script, cb) { db.batch(script,cb); }; +BlockDb.prototype.migrateV02 = function(cb) { + var k = 'txb-'; + this.txDb._db.createReadStream({ + start: k, + end: k + '~' + }).pipe(db.createWriteStream()).on('close', cb); +}; + +BlockDb.prototype.migrateV02cleanup = function(cb) { + + var self = this; + console.log('## deleting txb- from txs db'); //todo + + var k = 'txb-'; + var d = this.txDb._db; + d.createReadStream({ + start: k, + end: k + '~' + }) + .pipe(d.createWriteStream({type:'del'})) + .on('close', function(err){ + if (err) return cb(err); + console.log('## deleting tx- from txs db'); //todo + + var k = 'tx-'; + var d = self.txDb._db; + d.createReadStream({ + start: k, + end: k + '~' + }) + .pipe(d.createWriteStream({type:'del'})) + .on('close',cb); + }); +}; + + module.exports = require('soop')(BlockDb); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index ffd19b5d..e7a203e5 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -59,6 +59,8 @@ var TransactionDb = function() { this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; this.poolMatch = new PoolMatch(); this.safeConfirmations = config.safeConfirmations || DEFAULT_SAFE_CONFIRMATIONS; + + this._db = db; // this is only exposed for migration script }; TransactionDb.prototype.close = function(cb) { diff --git a/util/updateToV0.2.js b/util/updateToV0.2.js index 5f165009..2e79ff39 100755 --- a/util/updateToV0.2.js +++ b/util/updateToV0.2.js @@ -5,17 +5,6 @@ var HistoricSync = require('../lib/HistoricSync'); var async = require('async'); -// -// 1) -// var MAIN_PREFIX = 'bma-'; // bma- => (0 is unconnected) -// var TIP = 'bti-'; // bti = : last block on the chain -// -// var IN_BLK_PREFIX = 'btx-'; //btx- = -// v -// 2) DELETE txs/tx- -// 3) DELETE txs/txb -// - var historicSync = new HistoricSync({ shouldBroadcastSync: false }); var txDb=historicSync.sync.txDb; @@ -45,9 +34,9 @@ async.series([ if (err) return w_cb(err); tipHash = hash; hash = val; - height++; + if (hash) height++; if (!(height%1000) || !hash) { - console.log('\t%d blocks processed (set height 1/2)', height); + console.log('*update 1/2\t%d blocks processed', height); bDb._runScript(script, function(err) { script=[]; return w_cb(err); @@ -58,7 +47,14 @@ async.series([ }, c); }, function(c){ - bDb.setTip(tipHash, height-1, c); + console.log('Migrating txs... (this will take some minutes...)'); //TODO + bDb.migrateV02(c); + }, + function(c){ + bDb.setTip(tipHash, height, c); + }, + function(c){ + bDb.migrateV02cleanup(c); }, ],function(err){ if (err) From c90e7cef8ac0f71fe1cd2434969c532c8099aef5 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 27 May 2014 16:19:21 -0300 Subject: [PATCH 3/3] add info about update script in readme --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 02dd20e2..eecd3489 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,19 @@ require certain information from the blockchain that bitcoind does not provide. A blockchain explorer front-end have been developed to top of *Insight API*, it can be downloaded at [Github Insight Repository](https://github.com/bitpay/insight). +## IMPORTANT: Upgrading from v0.1 to v0.2 +In order to optimize some queries, the key-value layout in Level DB has been changed. +An update script need to be run when updating from version v0.1.x to 0.2.x + +The script is located at: +``` + $ util/updateToV0.2.js +``` + +The script takes ~3minutes in testnet. While the script is running, *Insight API* +must be stopped. + +Alternatively, a total resync can be made, running `$ util/sync.js -D` ## Prerequisites