From ab808e5c2b7cfcb85fea9fc17cf5d699bf8c8596 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 13 Feb 2014 16:14:22 -0300 Subject: [PATCH] refactor Rpc classes. add file index in DB --- lib/BlockDb.js | 48 +++++++++-------- lib/HistoricSync.js | 123 +++++++++++++++++++++++++----------------- lib/Rpc.js | 103 +++++++++++++++++++++++++++++++++++ lib/Sync.js | 2 +- lib/TransactionDb.js | 6 +-- lib/TransactionRpc.js | 62 --------------------- 6 files changed, 209 insertions(+), 135 deletions(-) create mode 100644 lib/Rpc.js delete mode 100644 lib/TransactionRpc.js diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 4cb79573..b664bc6b 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -5,23 +5,21 @@ require('classtool'); function spec(b) { - var TIMESTAMP_PREFIX = 'bts-'; // b-ts- => + var TIMESTAMP_PREFIX = 'bts-'; // b-ts- => var PREV_PREFIX = 'bpr-'; // b-prev- => var NEXT_PREFIX = 'bne-'; // b-next- => var MAIN_PREFIX = 'bma-'; // b-main- => 1/0 - var TIP = 'bti-'; // last block on the chain + var TIP = 'bti-'; // last block on the chain + var LAST_FILE_INDEX = 'file-'; // last processed file index /** * Module dependencies. */ - var RpcClient = require('bitcore/RpcClient').class(), - util = require('bitcore/util/util'), - levelup = require('levelup'), - BitcoreBlock= require('bitcore/Block').class(), + var levelup = require('levelup'), config = require('../config/config'); - var db = b.db || levelup(config.leveldb + '/blocks'); - var rpc = b.rpc || new RpcClient(config.bitcoind); + var db = b.db || levelup(config.leveldb + '/blocks'); + var Rpc = b.rpc || require('./Rpc').class(); var BlockDb = function() { }; @@ -80,6 +78,24 @@ function spec(b) { }); }; + + BlockDb.prototype.setLastFileIndex = function(idx, cb) { + var self = this; + if (this.lastFileIndexSaved === idx) return cb(); + + db.put(LAST_FILE_INDEX, idx, function(err) { + self.lastFileIndexSaved = idx; + return cb(err); + }); + }; + + BlockDb.prototype.getLastFileIndex = function(cb) { + db.get(LAST_FILE_INDEX, function(err,val) { + if (err && err.notFound) { err = null; val = null;} + return cb(err,val); + }); + }; + BlockDb.prototype.getNext = function(hash, cb) { db.get(NEXT_PREFIX + hash, function(err,val) { if (err && err.notFound) { err = null; val = null;} @@ -138,13 +154,8 @@ function spec(b) { BlockDb.prototype.fromHashWithInfo = function(hash, cb) { var self = this; - rpc.getBlock(hash, function(err, info) { - // Not found? - if (err && err.code === -5) return cb(); - if (err) return cb(err); - - if (info.result.height) - info.result.reward = BitcoreBlock.getBlockValue(info.result.height) / util.COIN ; + Rpc.getBlock(hash, function(err, info) { + if (err || !info) return cb(err); self.isMain(hash, function(err, val) { if (err) return cb(err); @@ -182,12 +193,7 @@ function spec(b) { }; BlockDb.prototype.blockIndex = function(height, cb) { - var rpc = new RpcClient(config.bitcoind); - rpc.getBlockHash(height, function(err, bh){ - if (err) return cb(err); - - cb(null, { blockHash: bh.result }); - }); + return Rpc.blockIndex(height,cb); }; return BlockDb; diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 753e2de0..effee3d0 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -123,10 +123,9 @@ function spec() { sockets.broadcastSyncInfo(self.info()); } -//TODO -// if (self.syncPercentage > 10) { -// process.exit(-1); -// } + // if (self.syncPercentage > 10) { + // process.exit(-1); + // } }; HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) { @@ -277,19 +276,24 @@ function spec() { } self.sync.storeTipBlock(blockInfo, function(err) { - if (blockInfo && blockInfo.hash) { - self.syncedBlocks++; - } else - self.status = 'finished'; - if (err) { self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); + return cb(err); } - return cb(err); + + self.sync.bDb.setLastFileIndex(self.blockExtractor.currentFileIndex, function(err) { + if (err) return cb(err); + + if (blockInfo && blockInfo.hash) { + self.syncedBlocks++; + } else + self.status = 'finished'; + + return cb(err); + }); }); }); - }; @@ -307,7 +311,7 @@ function spec() { }; - HistoricSync.prototype.getBlockCount = function(cb) { + HistoricSync.prototype.updateBlockCount = function(cb) { var self = this; if (self.blockChainHeight) return cb(); @@ -333,8 +337,9 @@ function spec() { } return cb(); }, - // We are not using getBestBlockHash, because is not available in all clients - function (cb) { return self.getBlockCount(cb); }, + function (cb) { + return self.updateBlockCount(cb); + }, function(cb) { if (!scanOpts.reverse) return cb(); self.rpc.getBlockHash(self.blockChainHeight, function(err, res) { @@ -372,8 +377,6 @@ function spec() { self.setError(err); return next(err, 0); } - - // SETUP Sync params var start, end; @@ -432,42 +435,66 @@ function spec() { }); }; - // upto if we have genesis block? + HistoricSync.prototype.smartImport = function(scanOpts, next) { var self = this; - self.sync.bDb.has(self.genesis, function(err, b) { - if (err) return next(err); - self.countNotOrphan(function(err, count) { - if (err) return next(err); - self.getBlockCount(function(err) { - if (err) return next(err); - - if (!b || scanOpts.destroy || count < self.blockChainHeight * 0.8 ) { - - if (!b) - p('Could not find Genesis block. Running FULL SYNC'); - else - p('Less that 80% of current blockchain is stored. Running FULL SYNC', - parseInt(count/self.blockChainHeight*100)); - - if (config.bitcoind.dataDir) { - p('bitcoind dataDir configured...importing blocks from .dat files'); - scanOpts.fromFiles = true; - self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); - } - else { - scanOpts.reverse = true; - } - } - else { - p('Genesis block found. Syncing upto old TIP.'); - p('Got ' + count + ' out of ' + self.blockChainHeight + ' blocks'); - scanOpts.reverse = true; - } - return self.importHistory(scanOpts, next); + var genesis, count; + async.series([ + function(s_c) { + self.sync.bDb.has(self.genesis, function(err, b) { + genesis = b; + return s_c(err); }); - }); + }, + function(s_c) { + if (scanOpts.destroy) return s_c(); + + self.countNotOrphan(function(err, c) { + count = c; + return s_c(err); + }); + }, + function(s_c) { + if (!config.bitcoind.dataDir) return s_c(); + + self.sync.bDb.getLastFileIndex(function(err, idx) { + self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); + if (idx) self.blockExtractor.currentFileIndex = idx; + return s_c(err); + }); + }, + function(s_c) { + self.updateBlockCount(s_c); + }], + function(err) { + if (err) return next(err); + + if (!genesis || scanOpts.destroy || count < self.blockChainHeight * 0.9 ) { + + // Full sync. + + if (!genesis) + p('Could not find Genesis block. Running FULL SYNC'); + else + p('Less that 90% of current blockchain is stored. Running FULL SYNC', + parseInt(count/self.blockChainHeight*100)); + + if (config.bitcoind.dataDir) { + p('bitcoind dataDir configured...importing blocks from .dat files'); + p('Starting from file: ' + self.blockExtractor.currentFileIndex); + scanOpts.fromFiles = true; + } + else { + scanOpts.reverse = true; + } + } + else { + p('Genesis block found. Syncing upto old TIP.'); + p('Got ' + count + ' out of ' + self.blockChainHeight + ' blocks'); + scanOpts.reverse = true; + } + return self.importHistory(scanOpts, next); }); }; diff --git a/lib/Rpc.js b/lib/Rpc.js new file mode 100644 index 00000000..fe8735b9 --- /dev/null +++ b/lib/Rpc.js @@ -0,0 +1,103 @@ +'use strict'; + +require('classtool'); + + +function spec(b) { + var RpcClient = require('bitcore/RpcClient').class(), + BitcoreTransaction = require('bitcore/Transaction').class(), + BitcoreBlock = require('bitcore/Block').class(), + bitcoreUtil = require('bitcore/util/util'), + util = require('util'), + config = require('../config/config'); + + var bitcoreRpc = b.bitcoreRpc || new RpcClient(config.bitcoind); + + function Rpc() { + } + + Rpc._parseRpcResult = function(info) { + var b = new Buffer(info.hex,'hex'); + var tx = new BitcoreTransaction(); + tx.parse(b); + + // Inputs + if (tx.isCoinBase()) { + info.isCoinBase = true; + } + + var n =0; + info.vin.forEach(function(i) { + i.n = n++; + }); + + // Outputs + var valueOutSat = 0; + info.vout.forEach( function(o) { + valueOutSat += o.value * bitcoreUtil.COIN; + }); + info.valueOut = parseInt(valueOutSat) / bitcoreUtil.COIN; + info.size = b.length; + + return info; + }; + + + Rpc.errMsg = function(err) { + var e = err; + e.message += util.format(' [Host: %s:%d User:%s Using password:%s]', + bitcoreRpc.host, + bitcoreRpc.port, + bitcoreRpc.user, + bitcoreRpc.pass?'yes':'no' + ); + return e; + }; + + Rpc.getRpcInfo = function(txid, cb) { + var self = this; + + bitcoreRpc.getRawTransaction(txid, 1, function(err, txInfo) { + + // Not found? + if (err && err.code === -5) return cb(); + if (err) return cb(self.errMsg(err)); + + var info = self._parseRpcResult(txInfo.result); + + return cb(null,info); + }); + }; + + + Rpc.blockIndex = function(height, cb) { + var self = this; + + bitcoreRpc.getBlockHash(height, function(err, bh){ + if (err) return cb(self.errMsg(err)); + cb(null, { blockHash: bh.result }); + }); + }; + + Rpc.getBlock = function(hash, cb) { + var self = this; + + bitcoreRpc.getBlock(hash, function(err,info) { + // Not found? + if (err && err.code === -5) return cb(); + if (err) return cb(self.errMsg(err)); + + + if (info.result.height) + info.result.reward = BitcoreBlock.getBlockValue(info.result.height) / util.COIN ; + + return cb(err,info); + }); + }; + + + return Rpc; +} +module.defineClass(spec); + + diff --git a/lib/Sync.js b/lib/Sync.js index d343fd0d..a6008c7f 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -143,7 +143,7 @@ function spec() { function(err) { if (!err) self._handleBroadcast(b.hash, null, updatedAddrs); if (err && err.toString().match(/WARN/) ) { - console.log(err); + //console.log(err); err=null; } return cb(err); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 607d10c9..720e62fc 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -25,7 +25,7 @@ function spec(b) { /** * Module dependencies. */ - var TransactionRpc = require('./TransactionRpc').class(), + var Rpc = b.rpc || require('./Rpc').class(), util = require('bitcore/util/util'), levelup = require('levelup'), async = require('async'), @@ -224,7 +224,7 @@ function spec(b) { TransactionDb.prototype._getInfo = function(txid, next) { var self = this; - TransactionRpc.getRpcInfo(txid, function(err, info) { + Rpc.getRpcInfo(txid, function(err, info) { if (err) return next(err); self._fillOutpoints(info, function() { @@ -573,7 +573,7 @@ function spec(b) { // TODO: parse it from networks.genesisTX? if (t === genesisTXID) return each_cb(); - TransactionRpc.getRpcInfo(t, function(err, inInfo) { + Rpc.getRpcInfo(t, function(err, inInfo) { if (!inInfo) return each_cb(err); return self.add(inInfo, blockHash, each_cb); diff --git a/lib/TransactionRpc.js b/lib/TransactionRpc.js deleted file mode 100644 index f249a7f9..00000000 --- a/lib/TransactionRpc.js +++ /dev/null @@ -1,62 +0,0 @@ -'use strict'; - -require('classtool'); - - -function spec(b) { - var RpcClient = require('bitcore/RpcClient').class(), - BitcoreTransaction = require('bitcore/Transaction').class(), - util = require('bitcore/util/util'), - config = require('../config/config'); - - var rpc = b.rpc || new RpcClient(config.bitcoind); - - function TransactionRpc() { - } - - TransactionRpc._parseRpcResult = function(info) { - var b = new Buffer(info.hex,'hex'); - var tx = new BitcoreTransaction(); - tx.parse(b); - - // Inputs - if (tx.isCoinBase()) { - info.isCoinBase = true; - } - - var n =0; - info.vin.forEach(function(i) { - i.n = n++; - }); - - // Outputs - var valueOutSat = 0; - info.vout.forEach( function(o) { - valueOutSat += o.value * util.COIN; - }); - info.valueOut = parseInt(valueOutSat) / util.COIN; - info.size = b.length; - - return info; - }; - - TransactionRpc.getRpcInfo = function(txid, cb) { - var Self = this; - - rpc.getRawTransaction(txid, 1, function(err, txInfo) { - - // Not found? - if (err && err.code === -5) return cb(); - if (err) return cb(err); - - var info = Self._parseRpcResult(txInfo.result); - - return cb(null,info); - }); - }; - - return TransactionRpc; -} -module.defineClass(spec); - -