From e19cd9f2ba552c13331c0d378899a229bf867523 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 5 Mar 2014 23:03:56 -0300 Subject: [PATCH 1/4] insight with soop. WIP --- app/controllers/transactions.js | 6 +- app/models/Address.js | 329 +++--- app/models/Status.js | 208 ++-- dev-util/get_block.js | 2 +- dev-util/get_tx.js | 2 +- lib/BlockDb.js | 403 ++++---- lib/BlockExtractor.js | 239 +++-- lib/HistoricSync.js | 735 +++++++------- lib/PeerSync.js | 240 +++-- lib/PoolMatch.js | 52 +- lib/Rpc.js | 182 ++-- lib/Sync.js | 525 +++++----- lib/TransactionDb.js | 1291 ++++++++++++------------ package.json | 2 +- test/integration/02-transactionouts.js | 2 +- 15 files changed, 2080 insertions(+), 2138 deletions(-) diff --git a/app/controllers/transactions.js b/app/controllers/transactions.js index 4b1ac985..2f501ead 100644 --- a/app/controllers/transactions.js +++ b/app/controllers/transactions.js @@ -7,9 +7,9 @@ var Address = require('../models/Address'); var async = require('async'); var common = require('./common'); -var TransactionDb = require('../../lib/TransactionDb').class(); -var BlockDb = require('../../lib/BlockDb').class(); -var Rpc = require('../../lib/Rpc').class(); +var Rpc = require('../../lib/Rpc'); +var TransactionDb = require('../../lib/TransactionDb').default(); +var BlockDb = require('../../lib/BlockDb').default(); var tDb = new TransactionDb(); var bdb = new BlockDb(); diff --git a/app/models/Address.js b/app/models/Address.js index aef9c255..f67ab1ca 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -1,194 +1,189 @@ 'use strict'; -require('classtool'); +//var imports = require('soop').imports(); +var async = require('async'); +var BitcoreAddress = require('bitcore/Address'); +var BitcoreTransaction = require('bitcore/Transaction'); +var BitcoreUtil = require('bitcore/util/util'); +var Parser = require('bitcore/util/BinaryParser'); +var Buffer = require('buffer').Buffer; +var TransactionDb = require('../../lib/TransactionDb').default(); +var CONCURRENCY = 5; + +function Address(addrStr) { + this.balanceSat = 0; + this.totalReceivedSat = 0; + this.totalSentSat = 0; + + this.unconfirmedBalanceSat = 0; + + this.txApperances = 0; + this.unconfirmedTxApperances= 0; + + // TODO store only txids? +index? +all? + this.transactions = []; + + var a = new BitcoreAddress(addrStr); + a.validate(); + this.addrStr = addrStr; + + Object.defineProperty(this, 'totalSent', { + get: function() { + return parseFloat(this.totalSentSat) / parseFloat(BitcoreUtil.COIN); + }, + set: function(i) { + this.totalSentSat = i * BitcoreUtil.COIN; + }, + enumerable: 1, + }); + + Object.defineProperty(this, 'balance', { + get: function() { + return parseFloat(this.balanceSat) / parseFloat(BitcoreUtil.COIN); + }, + set: function(i) { + this.balance = i * BitcoreUtil.COIN; + }, + enumerable: 1, + }); + + Object.defineProperty(this, 'totalReceived', { + get: function() { + return parseFloat(this.totalReceivedSat) / parseFloat(BitcoreUtil.COIN); + }, + set: function(i) { + this.totalReceived = i * BitcoreUtil.COIN; + }, + enumerable: 1, + }); -function spec() { - var async = require('async'); - var BitcoreAddress = require('bitcore/Address').class(); - var BitcoreUtil = require('bitcore/util/util'); - var TransactionDb = require('../../lib/TransactionDb').class(); - var BitcoreTransaction = require('bitcore/Transaction').class(); - var Parser = require('bitcore/util/BinaryParser').class(); - var Buffer = require('buffer').Buffer; - var CONCURRENCY = 5; + Object.defineProperty(this, 'unconfirmedBalance', { + get: function() { + return parseFloat(this.unconfirmedBalanceSat) / parseFloat(BitcoreUtil.COIN); + }, + set: function(i) { + this.unconfirmedBalanceSat = i * BitcoreUtil.COIN; + }, + enumerable: 1, + }); - function Address(addrStr) { - this.balanceSat = 0; - this.totalReceivedSat = 0; - this.totalSentSat = 0; +} - this.unconfirmedBalanceSat = 0; +Address.prototype._getScriptPubKey = function(hex,n) { + // ScriptPubKey is not provided by bitcoind RPC, so we parse it from tx hex. - this.txApperances = 0; - this.unconfirmedTxApperances= 0; + var parser = new Parser(new Buffer(hex,'hex')); + var tx = new BitcoreTransaction(); + tx.parse(parser); + return (tx.outs[n].s.toString('hex')); +}; - // TODO store only txids? +index? +all? - this.transactions = []; +Address.prototype.getUtxo = function(next) { + var self = this; + if (!self.addrStr) return next(); - var a = new BitcoreAddress(addrStr); - a.validate(); - this.addrStr = addrStr; - - Object.defineProperty(this, 'totalSent', { - get: function() { - return parseFloat(this.totalSentSat) / parseFloat(BitcoreUtil.COIN); - }, - set: function(i) { - this.totalSentSat = i * BitcoreUtil.COIN; - }, - enumerable: 1, - }); + var ret = []; + var db = new TransactionDb(); - Object.defineProperty(this, 'balance', { - get: function() { - return parseFloat(this.balanceSat) / parseFloat(BitcoreUtil.COIN); - }, - set: function(i) { - this.balance = i * BitcoreUtil.COIN; - }, - enumerable: 1, - }); + db.fromAddr(self.addrStr, function(err,txOut){ + if (err) return next(err); - Object.defineProperty(this, 'totalReceived', { - get: function() { - return parseFloat(this.totalReceivedSat) / parseFloat(BitcoreUtil.COIN); - }, - set: function(i) { - this.totalReceived = i * BitcoreUtil.COIN; - }, - enumerable: 1, - }); + // Complete utxo info + async.eachLimit(txOut,CONCURRENCY,function (txItem, a_c) { + db.fromIdInfoSimple(txItem.txid, function(err, info) { + var scriptPubKey = self._getScriptPubKey(info.hex, txItem.index); - Object.defineProperty(this, 'unconfirmedBalance', { - get: function() { - return parseFloat(this.unconfirmedBalanceSat) / parseFloat(BitcoreUtil.COIN); - }, - set: function(i) { - this.unconfirmedBalanceSat = i * BitcoreUtil.COIN; - }, - enumerable: 1, - }); - - } - - Address.prototype._getScriptPubKey = function(hex,n) { - // ScriptPubKey is not provided by bitcoind RPC, so we parse it from tx hex. - - var parser = new Parser(new Buffer(hex,'hex')); - var tx = new BitcoreTransaction(); - tx.parse(parser); - return (tx.outs[n].s.toString('hex')); - }; - - Address.prototype.getUtxo = function(next) { - var self = this; - if (!self.addrStr) return next(); - - var ret = []; - var db = new TransactionDb(); - - db.fromAddr(self.addrStr, function(err,txOut){ - if (err) return next(err); - - // Complete utxo info - async.eachLimit(txOut,CONCURRENCY,function (txItem, a_c) { - db.fromIdInfoSimple(txItem.txid, function(err, info) { - - var scriptPubKey = self._getScriptPubKey(info.hex, txItem.index); - - // we are filtering out even unconfirmed spents! - // add || !txItem.spentIsConfirmed - if (!txItem.spentTxId) { - ret.push({ - address: self.addrStr, - txid: txItem.txid, - vout: txItem.index, - ts: txItem.ts, - scriptPubKey: scriptPubKey, - amount: txItem.value_sat / BitcoreUtil.COIN, - confirmations: txItem.isConfirmed ? info.confirmations : 0, - }); - } - return a_c(err); - }); - }, function(err) { - return next(err,ret); + // we are filtering out even unconfirmed spents! + // add || !txItem.spentIsConfirmed + if (!txItem.spentTxId) { + ret.push({ + address: self.addrStr, + txid: txItem.txid, + vout: txItem.index, + ts: txItem.ts, + scriptPubKey: scriptPubKey, + amount: txItem.value_sat / BitcoreUtil.COIN, + confirmations: txItem.isConfirmed ? info.confirmations : 0, + }); + } + return a_c(err); }); + }, function(err) { + return next(err,ret); }); - }; + }); +}; - Address.prototype.update = function(next) { - var self = this; - if (!self.addrStr) return next(); +Address.prototype.update = function(next) { + var self = this; + if (!self.addrStr) return next(); - var txs = []; - var db = new TransactionDb(); - async.series([ - function (cb) { - var seen={}; - db.fromAddr(self.addrStr, function(err,txOut){ - if (err) return cb(err); - txOut.forEach(function(txItem){ - var add=0, addSpend=0; - var v = txItem.value_sat; + var txs = []; + var db = new TransactionDb(); + async.series([ + function (cb) { + var seen={}; + db.fromAddr(self.addrStr, function(err,txOut){ + if (err) return cb(err); + txOut.forEach(function(txItem){ + var add=0, addSpend=0; + var v = txItem.value_sat; - if ( !seen[txItem.txid] ) { - txs.push({txid: txItem.txid, ts: txItem.ts}); - seen[txItem.txid]=1; - add=1; + if ( !seen[txItem.txid] ) { + txs.push({txid: txItem.txid, ts: txItem.ts}); + seen[txItem.txid]=1; + add=1; + } + + if (txItem.spentTxId && !seen[txItem.spentTxId] ) { + txs.push({txid: txItem.spentTxId, ts: txItem.spentTs}); + seen[txItem.spentTxId]=1; + addSpend=1; + } + + if (txItem.isConfirmed) { + self.txApperances += add; + self.totalReceivedSat += v; + if (! txItem.spentTxId ) { + //unspent + self.balanceSat += v; } - - if (txItem.spentTxId && !seen[txItem.spentTxId] ) { - txs.push({txid: txItem.spentTxId, ts: txItem.spentTs}); - seen[txItem.spentTxId]=1; - addSpend=1; - } - - if (txItem.isConfirmed) { - self.txApperances += add; - self.totalReceivedSat += v; - if (! txItem.spentTxId ) { - //unspent - self.balanceSat += v; - } - else if(!txItem.spentIsConfirmed) { - // unspent - self.balanceSat += v; - self.unconfirmedBalanceSat -= v; - self.unconfirmedTxApperances += addSpend; - } - else { - // spent - self.totalSentSat += v; - self.txApperances += addSpend; - } + else if(!txItem.spentIsConfirmed) { + // unspent + self.balanceSat += v; + self.unconfirmedBalanceSat -= v; + self.unconfirmedTxApperances += addSpend; } else { - self.unconfirmedBalanceSat += v; - self.unconfirmedTxApperances += add; + // spent + self.totalSentSat += v; + self.txApperances += addSpend; } - }); - return cb(); + } + else { + self.unconfirmedBalanceSat += v; + self.unconfirmedTxApperances += add; + } }); - }, - ], function (err) { + return cb(); + }); + }, + ], function (err) { - // sort input and outputs togheter - txs.sort( - function compare(a,b) { - if (a.ts < b.ts) return 1; - if (a.ts > b.ts) return -1; - return 0; - }); + // sort input and outputs togheter + txs.sort( + function compare(a,b) { + if (a.ts < b.ts) return 1; + if (a.ts > b.ts) return -1; + return 0; + }); - self.transactions = txs.map(function(i) { return i.txid; } ); - return next(err); - }); - }; + self.transactions = txs.map(function(i) { return i.txid; } ); + return next(err); + }); +}; - return Address; -} -module.defineClass(spec); +module.exports = require('soop')(Address); diff --git a/app/models/Status.js b/app/models/Status.js index 106c3093..548974f3 100644 --- a/app/models/Status.js +++ b/app/models/Status.js @@ -1,112 +1,106 @@ 'use strict'; +//var imports = require('soop').imports(); -require('classtool'); - -function spec() { - var async = require('async'); - var RpcClient = require('bitcore/RpcClient').class(); - var BlockDb = require('../../lib/BlockDb').class(); - var config = require('../../config/config'); - var rpc = new RpcClient(config.bitcoind); - - function Status() { - this.bDb = new BlockDb(); - } - - Status.prototype.getInfo = function(next) { - var that = this; - async.series([ - function (cb) { - rpc.getInfo(function(err, info){ - if (err) return cb(err); - - that.info = info.result; - return cb(); - }); - }, - ], function (err) { - return next(err); - }); - }; - - Status.prototype.getDifficulty = function(next) { - var that = this; - async.series([ - function (cb) { - rpc.getDifficulty(function(err, df){ - if (err) return cb(err); - - that.difficulty = df.result; - return cb(); - }); - } - ], function (err) { - return next(err); - }); - }; - - Status.prototype.getTxOutSetInfo = function(next) { - var that = this; - async.series([ - function (cb) { - rpc.getTxOutSetInfo(function(err, txout){ - if (err) return cb(err); - - that.txoutsetinfo = txout.result; - return cb(); - }); - } - ], function (err) { - return next(err); - }); - }; - - Status.prototype.getBestBlockHash = function(next) { - var that = this; - async.series([ - function (cb) { - rpc.getBestBlockHash(function(err, bbh){ - if (err) return cb(err); - - that.bestblockhash = bbh.result; - return cb(); - }); - }, - - ], function (err) { - return next(err); - }); - }; - - Status.prototype.getLastBlockHash = function(next) { - var that = this; - that.bDb.getTip(function(err,tip) { - that.syncTipHash = tip; - async.waterfall( - [ - function(callback){ - rpc.getBlockCount(function(err, bc){ - if (err) return callback(err); - callback(null, bc.result); - }); - }, - function(bc, callback){ - rpc.getBlockHash(bc, function(err, bh){ - if (err) return callback(err); - callback(null, bh.result); - }); - } - ], - function (err, result) { - that.lastblockhash = result; - return next(); - } - ); - }); - }; - - return Status; +var async = require('async'); +var RpcClient = require('bitcore/RpcClient'); +var BlockDb = require('../../lib/BlockDb'); +var config = require('../../config/config'); +var rpc = new RpcClient(config.bitcoind); +function Status() { + this.bDb = new BlockDb(); } -module.defineClass(spec); +Status.prototype.getInfo = function(next) { + var that = this; + async.series([ + function (cb) { + rpc.getInfo(function(err, info){ + if (err) return cb(err); + + that.info = info.result; + return cb(); + }); + }, + ], function (err) { + return next(err); + }); +}; + +Status.prototype.getDifficulty = function(next) { + var that = this; + async.series([ + function (cb) { + rpc.getDifficulty(function(err, df){ + if (err) return cb(err); + + that.difficulty = df.result; + return cb(); + }); + } + ], function (err) { + return next(err); + }); +}; + +Status.prototype.getTxOutSetInfo = function(next) { + var that = this; + async.series([ + function (cb) { + rpc.getTxOutSetInfo(function(err, txout){ + if (err) return cb(err); + + that.txoutsetinfo = txout.result; + return cb(); + }); + } + ], function (err) { + return next(err); + }); +}; + +Status.prototype.getBestBlockHash = function(next) { + var that = this; + async.series([ + function (cb) { + rpc.getBestBlockHash(function(err, bbh){ + if (err) return cb(err); + + that.bestblockhash = bbh.result; + return cb(); + }); + }, + + ], function (err) { + return next(err); + }); +}; + +Status.prototype.getLastBlockHash = function(next) { + var that = this; + that.bDb.getTip(function(err,tip) { + that.syncTipHash = tip; + async.waterfall( + [ + function(callback){ + rpc.getBlockCount(function(err, bc){ + if (err) return callback(err); + callback(null, bc.result); + }); + }, + function(bc, callback){ + rpc.getBlockHash(bc, function(err, bh){ + if (err) return callback(err); + callback(null, bh.result); + }); + } + ], + function (err, result) { + that.lastblockhash = result; + return next(); + } + ); + }); +}; + +module.exports = require('soop')(Status); diff --git a/dev-util/get_block.js b/dev-util/get_block.js index f65d8c36..34da1b1c 100755 --- a/dev-util/get_block.js +++ b/dev-util/get_block.js @@ -5,7 +5,7 @@ var util = require('util'); process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var RpcClient = require('../node_modules/bitcore/RpcClient').class(); +var RpcClient = require('../node_modules/bitcore/RpcClient'); var config = require('../config/config'); diff --git a/dev-util/get_tx.js b/dev-util/get_tx.js index c30f0f60..39174722 100755 --- a/dev-util/get_tx.js +++ b/dev-util/get_tx.js @@ -2,7 +2,7 @@ 'use strict'; var util = require('util'); -var T = require('../lib/TransactionDb').class(); +var T = require('../lib/TransactionDb'); process.env.NODE_ENV = process.env.NODE_ENV || 'development'; diff --git a/lib/BlockDb.js b/lib/BlockDb.js index e06e307b..e0ef4d12 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -1,234 +1,225 @@ 'use strict'; +var imports = require('soop').imports(); +var parent = imports.parent || require('events').EventEmitter; +var TIMESTAMP_PREFIX = 'bts-'; // b-ts- => +var PREV_PREFIX = 'bpr-'; // b-prev- => +var NEXT_PREFIX = 'bne-'; // b-next- => +var MAIN_PREFIX = 'bma-'; // b-main- => 1/0 +var TIP = 'bti-'; // last block on the chain +var LAST_FILE_INDEX = 'file-'; // last processed file index -require('classtool'); +var MAX_OPEN_FILES = 500; -function spec(b) { +/** +* Module dependencies. +*/ +var levelup = require('levelup'), + config = require('../config/config'); +var db = imports.db || levelup(config.leveldb + '/blocks',{maxOpenFiles: MAX_OPEN_FILES} ); +var Rpc = imports.rpc || require('./Rpc'); +var PoolMatch = imports.poolMatch || require('soop').load('./PoolMatch',config); - var superclass = b.superclass || require('events').EventEmitter; - var TIMESTAMP_PREFIX = 'bts-'; // b-ts- => - var PREV_PREFIX = 'bpr-'; // b-prev- => - var NEXT_PREFIX = 'bne-'; // b-next- => - var MAIN_PREFIX = 'bma-'; // b-main- => 1/0 - var TIP = 'bti-'; // last block on the chain - var LAST_FILE_INDEX = 'file-'; // last processed file index +var TransactionDb = require('./TransactionDb.js').default(); - var MAX_OPEN_FILES = 500; +var BlockDb = function() { + BlockDb.super(this, arguments); + this.poolMatch = new PoolMatch(); +}; +BlockDb.parent = parent; - /** - * Module dependencies. - */ - var levelup = require('levelup'), - config = require('../config/config'); - var db = b.db || levelup(config.leveldb + '/blocks',{maxOpenFiles: MAX_OPEN_FILES} ); - var Rpc = b.rpc || require('./Rpc').class(); - var PoolMatch = b.poolMatch || require('./PoolMatch').class(config); +BlockDb.prototype.close = function(cb) { + db.close(cb); +}; - var TransactionDb = require('./TransactionDb.js').class(); - - var BlockDb = function() { - BlockDb.super(this, arguments); - this.poolMatch = new PoolMatch(); - }; - - BlockDb.superclass = superclass; - - BlockDb.prototype.close = function(cb) { - db.close(cb); - }; - - BlockDb.prototype.drop = function(cb) { - var path = config.leveldb + '/blocks'; - db.close(function() { - require('leveldown').destroy(path, function () { - db = levelup(path,{maxOpenFiles: MAX_OPEN_FILES} ); - return cb(); - }); +BlockDb.prototype.drop = function(cb) { + var path = config.leveldb + '/blocks'; + db.close(function() { + require('leveldown').destroy(path, function () { + db = levelup(path,{maxOpenFiles: MAX_OPEN_FILES} ); + return cb(); }); - }; + }); +}; - // adds a block. Does not update Next pointer in - // the block prev to the new block, nor TIP pointer - // - BlockDb.prototype.add = function(b, cb) { - var self = this; - var time_key = TIMESTAMP_PREFIX + - ( b.time || Math.round(new Date().getTime() / 1000) ); +// adds a block. Does not update Next pointer in +// the block prev to the new block, nor TIP pointer +// +BlockDb.prototype.add = function(b, cb) { + var self = this; + var time_key = TIMESTAMP_PREFIX + + ( b.time || Math.round(new Date().getTime() / 1000) ); - return db.batch() - .put(time_key, b.hash) - .put(MAIN_PREFIX + b.hash, 1) - .put(PREV_PREFIX + b.hash, b.previousblockhash) - .write(function(err){ - if (!err) { - self.emit('new_block', {blockid: b.hash}); - } - cb(err); - }); - }; - - BlockDb.prototype.getTip = function(cb) { - db.get(TIP, function(err, val) { - return cb(err,val); - }); - }; - - BlockDb.prototype.setTip = function(hash, cb) { - db.put(TIP, hash, function(err) { - return cb(err); - }); - }; - - //mainly for testing - BlockDb.prototype.setPrev = function(hash, prevHash, cb) { - db.put(PREV_PREFIX + hash, prevHash, function(err) { - return cb(err); - }); - }; - - BlockDb.prototype.getPrev = function(hash, cb) { - db.get(PREV_PREFIX + hash, function(err,val) { - if (err && err.notFound) { err = null; val = null;} - return cb(err,val); - }); - }; - - - BlockDb.prototype.setLastFileIndex = function(idx, cb) { - var self = this; - if (this.lastFileIndexSaved === idx) return cb(); - - db.put(LAST_FILE_INDEX, idx, function(err) { - self.lastFileIndexSaved = idx; - return cb(err); - }); - }; - - BlockDb.prototype.getLastFileIndex = function(cb) { - db.get(LAST_FILE_INDEX, function(err,val) { - if (err && err.notFound) { err = null; val = null;} - return cb(err,val); - }); - }; - - BlockDb.prototype.getNext = function(hash, cb) { - db.get(NEXT_PREFIX + hash, function(err,val) { - if (err && err.notFound) { err = null; val = null;} - return cb(err,val); - }); - }; - - BlockDb.prototype.isMain = function(hash, cb) { - db.get(MAIN_PREFIX + hash, function(err, val) { - if (err && err.notFound) { err = null; val = 0;} - return cb(err,parseInt(val)); - }); - }; - - BlockDb.prototype.setMain = function(hash, isMain, cb) { - if (!isMain) console.log('\tNew orphan: %s',hash); - db.put(MAIN_PREFIX + hash, isMain?1:0, function(err) { - return cb(err); - }); - }; - BlockDb.prototype.setNext = function(hash, nextHash, cb) { - db.put(NEXT_PREFIX + hash, nextHash, function(err) { - return cb(err); - }); - }; - - BlockDb.prototype.countConnected = function(cb) { - var c = 0; - console.log('Counting connected blocks. This could take some minutes'); - db.createReadStream({start: MAIN_PREFIX, end: MAIN_PREFIX + '~' }) - .on('data', function (data) { - if (data.value !== 0) c++; - }) - .on('error', function (err) { - return cb(err); - }) - .on('end', function () { - return cb(null, c); - }); - }; - - // .has() return true orphans also - BlockDb.prototype.has = function(hash, cb) { - var k = PREV_PREFIX + hash; - db.get(k, function (err) { - var ret = true; - if (err && err.notFound) { - err = null; - ret = false; + return db.batch() + .put(time_key, b.hash) + .put(MAIN_PREFIX + b.hash, 1) + .put(PREV_PREFIX + b.hash, b.previousblockhash) + .write(function(err){ + if (!err) { + self.emit('new_block', {blockid: b.hash}); } - return cb(err, ret); + cb(err); }); - }; +}; - BlockDb.prototype.getPoolInfo = function(tx, cb) { - var tr = new TransactionDb(); - var self = this; - - tr._getInfo(tx, function(e, a) { - if (e) return cb(false); +BlockDb.prototype.getTip = function(cb) { + db.get(TIP, function(err, val) { + return cb(err,val); + }); +}; - if (a.isCoinBase) { - var coinbaseHexBuffer = new Buffer(a.vin[0].coinbase, 'hex'); - var aa = self.poolMatch.match(coinbaseHexBuffer); - - return cb(aa); - } +BlockDb.prototype.setTip = function(hash, cb) { + db.put(TIP, hash, function(err) { + return cb(err); + }); +}; + +//mainly for testing +BlockDb.prototype.setPrev = function(hash, prevHash, cb) { + db.put(PREV_PREFIX + hash, prevHash, function(err) { + return cb(err); + }); +}; + +BlockDb.prototype.getPrev = function(hash, cb) { + db.get(PREV_PREFIX + hash, function(err,val) { + if (err && err.notFound) { err = null; val = null;} + return cb(err,val); + }); +}; + + +BlockDb.prototype.setLastFileIndex = function(idx, cb) { + var self = this; + if (this.lastFileIndexSaved === idx) return cb(); + + db.put(LAST_FILE_INDEX, idx, function(err) { + self.lastFileIndexSaved = idx; + return cb(err); + }); +}; + +BlockDb.prototype.getLastFileIndex = function(cb) { + db.get(LAST_FILE_INDEX, function(err,val) { + if (err && err.notFound) { err = null; val = null;} + return cb(err,val); + }); +}; + +BlockDb.prototype.getNext = function(hash, cb) { + db.get(NEXT_PREFIX + hash, function(err,val) { + if (err && err.notFound) { err = null; val = null;} + return cb(err,val); + }); +}; + +BlockDb.prototype.isMain = function(hash, cb) { + db.get(MAIN_PREFIX + hash, function(err, val) { + if (err && err.notFound) { err = null; val = 0;} + return cb(err,parseInt(val)); + }); +}; + +BlockDb.prototype.setMain = function(hash, isMain, cb) { + if (!isMain) console.log('\tNew orphan: %s',hash); + db.put(MAIN_PREFIX + hash, isMain?1:0, function(err) { + return cb(err); + }); +}; +BlockDb.prototype.setNext = function(hash, nextHash, cb) { + db.put(NEXT_PREFIX + hash, nextHash, function(err) { + return cb(err); + }); +}; + +BlockDb.prototype.countConnected = function(cb) { + var c = 0; + console.log('Counting connected blocks. This could take some minutes'); + db.createReadStream({start: MAIN_PREFIX, end: MAIN_PREFIX + '~' }) + .on('data', function (data) { + if (data.value !== 0) c++; + }) + .on('error', function (err) { + return cb(err); + }) + .on('end', function () { + return cb(null, c); }); - }; +}; - BlockDb.prototype.fromHashWithInfo = function(hash, cb) { - var self = this; +// .has() return true orphans also +BlockDb.prototype.has = function(hash, cb) { + var k = PREV_PREFIX + hash; + db.get(k, function (err) { + var ret = true; + if (err && err.notFound) { + err = null; + ret = false; + } + return cb(err, ret); + }); +}; - Rpc.getBlock(hash, function(err, info) { - if (err || !info) return cb(err); +BlockDb.prototype.getPoolInfo = function(tx, cb) { + var tr = new TransactionDb(); + var self = this; - self.isMain(hash, function(err, val) { - if (err) return cb(err); + tr._getInfo(tx, function(e, a) { + if (e) return cb(false); - info.isMainChain = val ? true : false; + if (a.isCoinBase) { + var coinbaseHexBuffer = new Buffer(a.vin[0].coinbase, 'hex'); + var aa = self.poolMatch.match(coinbaseHexBuffer); + + return cb(aa); + } + }); +}; - return cb(null, { - hash: hash, - info: info, - }); +BlockDb.prototype.fromHashWithInfo = function(hash, cb) { + var self = this; + + Rpc.getBlock(hash, function(err, info) { + if (err || !info) return cb(err); + + self.isMain(hash, function(err, val) { + if (err) return cb(err); + + info.isMainChain = val ? true : false; + + return cb(null, { + hash: hash, + info: info, }); }); - }; + }); +}; - BlockDb.prototype.getBlocksByDate = function(start_ts, end_ts, cb) { - var list = []; - db.createReadStream({ - start: TIMESTAMP_PREFIX + start_ts, - end: TIMESTAMP_PREFIX + end_ts, - fillCache: true - }) - .on('data', function (data) { - var k = data.key.split('-'); - list.push({ - ts: k[1], - hash: data.value, - }); - }) - .on('error', function (err) { - return cb(err); - }) - .on('end', function () { - return cb(null, list.reverse()); +BlockDb.prototype.getBlocksByDate = function(start_ts, end_ts, cb) { + var list = []; + db.createReadStream({ + start: TIMESTAMP_PREFIX + start_ts, + end: TIMESTAMP_PREFIX + end_ts, + fillCache: true + }) + .on('data', function (data) { + var k = data.key.split('-'); + list.push({ + ts: k[1], + hash: data.value, }); - }; - - BlockDb.prototype.blockIndex = function(height, cb) { - return Rpc.blockIndex(height,cb); - }; - - return BlockDb; -} -module.defineClass(spec); + }) + .on('error', function (err) { + return cb(err); + }) + .on('end', function () { + return cb(null, list.reverse()); + }); +}; +BlockDb.prototype.blockIndex = function(height, cb) { + return Rpc.blockIndex(height,cb); +}; +module.exports = require('soop')(BlockDb); diff --git a/lib/BlockExtractor.js b/lib/BlockExtractor.js index 92c6740e..967492dc 100644 --- a/lib/BlockExtractor.js +++ b/lib/BlockExtractor.js @@ -1,162 +1,155 @@ 'use strict'; +var Block = require('bitcore/Block'), + networks = require('bitcore/networks'), + Parser = require('bitcore/util/BinaryParser'), + fs = require('fs'), + Buffer = require('buffer').Buffer, + glob = require('glob'), + async = require('async'); -require('classtool'); +function BlockExtractor(dataDir, network) { -function spec() { + var self = this; + var path = dataDir + '/blocks/blk*.dat'; - var Block = require('bitcore/Block').class(), - networks = require('bitcore/networks'), - Parser = require('bitcore/util/BinaryParser').class(), - fs = require('fs'), - Buffer = require('buffer').Buffer, - glob = require('glob'), - async = require('async'); + self.dataDir = dataDir; + self.files = glob.sync(path); + self.nfiles = self.files.length; - function BlockExtractor(dataDir, network) { + if (self.nfiles === 0) + throw new Error('Could not find block files at: ' + path); - var self = this; - var path = dataDir + '/blocks/blk*.dat'; + self.currentFileIndex = 0; + self.isCurrentRead = false; + self.currentBuffer = null; + self.currentParser = null; + self.network = network === 'testnet' ? networks.testnet: networks.livenet; + self.magic = self.network.magic.toString('hex'); +} - self.dataDir = dataDir; - self.files = glob.sync(path); - self.nfiles = self.files.length; +BlockExtractor.prototype.currentFile = function() { + var self = this; - if (self.nfiles === 0) - throw new Error('Could not find block files at: ' + path); + return self.files[self.currentFileIndex]; +}; - self.currentFileIndex = 0; - self.isCurrentRead = false; - self.currentBuffer = null; - self.currentParser = null; - self.network = network === 'testnet' ? networks.testnet: networks.livenet; - self.magic = self.network.magic.toString('hex'); + +BlockExtractor.prototype.nextFile = function() { + var self = this; + + if (self.currentFileIndex < 0) return false; + + var ret = true; + + self.isCurrentRead = false; + self.currentBuffer = null; + self.currentParser = null; + + if (self.currentFileIndex < self.nfiles - 1) { + self.currentFileIndex++; } + else { + self.currentFileIndex=-1; + ret = false; + } + return ret; +}; - BlockExtractor.prototype.currentFile = function() { - var self = this; +BlockExtractor.prototype.readCurrentFileSync = function() { + var self = this; - return self.files[self.currentFileIndex]; - }; + if (self.currentFileIndex < 0 || self.isCurrentRead) return; - BlockExtractor.prototype.nextFile = function() { - var self = this; + self.isCurrentRead = true; - if (self.currentFileIndex < 0) return false; - - var ret = true; - - self.isCurrentRead = false; - self.currentBuffer = null; - self.currentParser = null; - - if (self.currentFileIndex < self.nfiles - 1) { - self.currentFileIndex++; - } - else { - self.currentFileIndex=-1; - ret = false; - } - return ret; - }; - - BlockExtractor.prototype.readCurrentFileSync = function() { - var self = this; - - if (self.currentFileIndex < 0 || self.isCurrentRead) return; + var fname = self.currentFile(); + if (!fname) return; - self.isCurrentRead = true; + var stats = fs.statSync(fname); - var fname = self.currentFile(); - if (!fname) return; + var size = stats.size; + console.log('Reading Blockfile %s [%d MB]', + fname, parseInt(size/1024/1024)); - var stats = fs.statSync(fname); + var fd = fs.openSync(fname, 'r'); - var size = stats.size; + var buffer = new Buffer(size); - console.log('Reading Blockfile %s [%d MB]', - fname, parseInt(size/1024/1024)); + fs.readSync(fd, buffer, 0, size, 0); - var fd = fs.openSync(fname, 'r'); - - var buffer = new Buffer(size); - - fs.readSync(fd, buffer, 0, size, 0); - - self.currentBuffer = buffer; - self.currentParser = new Parser(buffer); - }; + self.currentBuffer = buffer; + self.currentParser = new Parser(buffer); +}; - BlockExtractor.prototype.getNextBlock = function(cb) { - var self = this; +BlockExtractor.prototype.getNextBlock = function(cb) { + var self = this; - var b; - var magic; - async.series([ - function (a_cb) { - - async.whilst( - function() { - return (!magic); - }, - function(w_cb) { + var b; + var magic; + async.series([ + function (a_cb) { - self.readCurrentFileSync(); + async.whilst( + function() { + return (!magic); + }, + function(w_cb) { - if (self.currentFileIndex < 0) return cb(); + self.readCurrentFileSync(); + + if (self.currentFileIndex < 0) return cb(); - magic = self.currentParser ? self.currentParser.buffer(4).toString('hex') - : null ; + magic = self.currentParser ? self.currentParser.buffer(4).toString('hex') + : null ; - if (!self.currentParser || self.currentParser.eof() || magic === '00000000') { - magic = null; - if (self.nextFile()) { - console.log('Moving forward to file:' + self.currentFile() ); - return w_cb(); - } - else { - console.log('Finished all files'); - magic = null; - return w_cb(); - } - } - else { + if (!self.currentParser || self.currentParser.eof() || magic === '00000000') { + magic = null; + if (self.nextFile()) { + console.log('Moving forward to file:' + self.currentFile() ); return w_cb(); } - }, a_cb); - }, - function (a_cb) { - if (!magic) return a_cb(); + else { + console.log('Finished all files'); + magic = null; + return w_cb(); + } + } + else { + return w_cb(); + } + }, a_cb); + }, + function (a_cb) { + if (!magic) return a_cb(); - if (magic !== self.magic) { - var e = new Error('CRITICAL ERROR: Magic number mismatch: ' + - magic + '!=' + self.magic); - return a_cb(e); - } + if (magic !== self.magic) { + var e = new Error('CRITICAL ERROR: Magic number mismatch: ' + + magic + '!=' + self.magic); + return a_cb(e); + } - // spacer? - self.currentParser.word32le(); - return a_cb(); - }, - function (a_cb) { - if (!magic) return a_cb(); + // spacer? + self.currentParser.word32le(); + return a_cb(); + }, + function (a_cb) { + if (!magic) return a_cb(); - b = new Block(); - b.parse(self.currentParser); - b.getHash(); - return a_cb(); - }, - ], function(err) { - return cb(err,b); - }); - }; + b = new Block(); + b.parse(self.currentParser); + b.getHash(); + return a_cb(); + }, + ], function(err) { + return cb(err,b); + }); +}; - return BlockExtractor; -} -module.defineClass(spec); +module.exports = require('soop')(BlockExtractor); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index b87f54a1..9f732bcf 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -1,429 +1,424 @@ 'use strict'; -require('classtool'); +var imports = require('soop').imports(); + +var util = require('util'); +var assert = require('assert'); +var RpcClient = require('bitcore/RpcClient'); +var Script = require('bitcore/Script'); +var networks = require('bitcore/networks'); +var async = require('async'); +var config = require('../config/config'); +var Sync = require('./Sync'); +var sockets = require('../app/controllers/socket.js'); +var BlockExtractor = require('./BlockExtractor.js'); +var buffertools = require('buffertools'); + +// var bitcoreUtil = require('bitcore/util/util'); +// var Deserialize = require('bitcore/Deserialize'); +var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:'; -function spec() { - var util = require('util'); - var assert = require('assert'); - var RpcClient = require('bitcore/RpcClient').class(); - var Script = require('bitcore/Script').class(); - var networks = require('bitcore/networks'); - var async = require('async'); - var config = require('../config/config'); - var Sync = require('./Sync').class(); - var sockets = require('../app/controllers/socket.js'); - var BlockExtractor = require('./BlockExtractor.js').class(); - var buffertools = require('buffertools'); - - // var bitcoreUtil = require('bitcore/util/util'); - // var Deserialize = require('bitcore/Deserialize'); +var BAD_GEN_ERROR_DB = 'Bad genesis block. Network mismatch between Insight and levelDB? Insight is configured for:'; +function HistoricSync(opts) { + opts = opts || {}; + this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; - var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:'; + var genesisHashReversed = new Buffer(32); + this.network.genesisBlock.hash.copy(genesisHashReversed); + buffertools.reverse(genesisHashReversed); + this.genesis = genesisHashReversed.toString('hex'); - var BAD_GEN_ERROR_DB = 'Bad genesis block. Network mismatch between Insight and levelDB? Insight is configured for:'; - function HistoricSync(opts) { - opts = opts || {}; + this.rpc = new RpcClient(config.bitcoind); + this.shouldBroadcast = opts.shouldBroadcastSync; + this.sync = new Sync(opts); +} - this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; +function p() { + var args = []; + Array.prototype.push.apply(args, arguments); - var genesisHashReversed = new Buffer(32); - this.network.genesisBlock.hash.copy(genesisHashReversed); - buffertools.reverse(genesisHashReversed); - this.genesis = genesisHashReversed.toString('hex'); + args.unshift('[historic_sync]'); + /*jshint validthis:true */ + console.log.apply(this, args); +} - this.rpc = new RpcClient(config.bitcoind); - this.shouldBroadcast = opts.shouldBroadcastSync; - this.sync = new Sync(opts); +HistoricSync.prototype.showProgress = function() { + var self = this; + + if ( self.status ==='syncing' && + ( self.syncedBlocks ) % self.step !== 1) return; + + if (self.error) { + p('ERROR: ' + self.error); + } + else { + self.updatePercentage(); + p(util.format('status: [%d%%]', self.syncPercentage)); + } + if (self.shouldBroadcast) { + sockets.broadcastSyncInfo(self.info()); } - function p() { - var args = []; - Array.prototype.push.apply(args, arguments); +// if (self.syncPercentage > 10) { +// process.exit(-1); +// } +}; - args.unshift('[historic_sync]'); - /*jshint validthis:true */ - console.log.apply(this, args); - } - HistoricSync.prototype.showProgress = function() { - var self = this; +HistoricSync.prototype.setError = function(err) { + var self = this; + self.error = err.message?err.message:err.toString(); + self.status='error'; + self.showProgress(); + return err; +}; - if ( self.status ==='syncing' && - ( self.syncedBlocks ) % self.step !== 1) return; - if (self.error) { - p('ERROR: ' + self.error); + +HistoricSync.prototype.close = function() { + this.sync.close(); +}; + + +HistoricSync.prototype.info = function() { + this.updatePercentage(); + return { + status: this.status, + blockChainHeight: this.blockChainHeight, + syncPercentage: this.syncPercentage, + syncedBlocks: this.syncedBlocks, + syncTipHash: this.sync.tip, + error: this.error, + type: this.type, + startTs: this.startTs, + endTs: this.endTs, + }; +}; + +HistoricSync.prototype.updatePercentage = function() { + var r = this.syncedBlocks / this.blockChainHeight; + this.syncPercentage = parseFloat(100 * r).toFixed(3); + if (this.syncPercentage > 100) this.syncPercentage = 100; +}; + +HistoricSync.prototype.getBlockFromRPC = function(cb) { + var self = this; + + if (!self.currentRpcHash) return cb(); + + var blockInfo; + self.rpc.getBlock(self.currentRpcHash, function(err, ret) { + if (err) return cb(err); + if (ret) { + blockInfo = ret.result; + // this is to match block retreived from file + if (blockInfo.hash === self.genesis) + blockInfo.previousblockhash = + self.network.genesisBlock.prev_hash.toString('hex'); + + self.currentRpcHash = blockInfo.nextblockhash; } else { - self.updatePercentage(); - p(util.format('status: [%d%%]', self.syncPercentage)); - } - if (self.shouldBroadcast) { - sockets.broadcastSyncInfo(self.info()); + blockInfo = null; } + return cb(null, blockInfo); + }); +}; - // if (self.syncPercentage > 10) { - // process.exit(-1); - // } - }; +HistoricSync.prototype.getBlockFromFile = function(cb) { + var self = this; + + var blockInfo; + + //get Info + self.blockExtractor.getNextBlock(function(err, b) { + if (err || ! b) return cb(err); + blockInfo = b.getStandardizedObject(b.txs, self.network); + blockInfo.previousblockhash = blockInfo.prev_block; + + var ti=0; + // Get TX Address + b.txs.forEach(function(t) { - HistoricSync.prototype.setError = function(err) { - var self = this; - self.error = err.message?err.message:err.toString(); - self.status='error'; - self.showProgress(); - return err; - }; + var objTx = blockInfo.tx[ti++]; + + //add time from block + objTx.time = blockInfo.time; + + var to=0; + t.outs.forEach( function(o) { + var s = new Script(o.s); + var addrs = self.sync.txDb.getAddrStr(s); - HistoricSync.prototype.close = function() { - this.sync.close(); - }; - - - HistoricSync.prototype.info = function() { - this.updatePercentage(); - return { - status: this.status, - blockChainHeight: this.blockChainHeight, - syncPercentage: this.syncPercentage, - syncedBlocks: this.syncedBlocks, - syncTipHash: this.sync.tip, - error: this.error, - type: this.type, - startTs: this.startTs, - endTs: this.endTs, - }; - }; - - HistoricSync.prototype.updatePercentage = function() { - var r = this.syncedBlocks / this.blockChainHeight; - this.syncPercentage = parseFloat(100 * r).toFixed(3); - if (this.syncPercentage > 100) this.syncPercentage = 100; - }; - - HistoricSync.prototype.getBlockFromRPC = function(cb) { - var self = this; - - if (!self.currentRpcHash) return cb(); - - var blockInfo; - self.rpc.getBlock(self.currentRpcHash, function(err, ret) { - if (err) return cb(err); - if (ret) { - blockInfo = ret.result; - // this is to match block retreived from file - if (blockInfo.hash === self.genesis) - blockInfo.previousblockhash = - self.network.genesisBlock.prev_hash.toString('hex'); - - self.currentRpcHash = blockInfo.nextblockhash; - } - else { - blockInfo = null; - } - return cb(null, blockInfo); - }); - }; - - HistoricSync.prototype.getBlockFromFile = function(cb) { - var self = this; - - var blockInfo; - - //get Info - self.blockExtractor.getNextBlock(function(err, b) { - if (err || ! b) return cb(err); - blockInfo = b.getStandardizedObject(b.txs, self.network); - blockInfo.previousblockhash = blockInfo.prev_block; - - var ti=0; - // Get TX Address - b.txs.forEach(function(t) { - - - var objTx = blockInfo.tx[ti++]; - - //add time from block - objTx.time = blockInfo.time; - - var to=0; - t.outs.forEach( function(o) { - - - var s = new Script(o.s); - var addrs = self.sync.txDb.getAddrStr(s); - - // support only for p2pubkey p2pubkeyhash and p2sh - if (addrs.length === 1) { - objTx.out[to].addrStr = addrs[0]; - } - to++; - }); - }); - self.sync.bDb.setLastFileIndex(self.blockExtractor.currentFileIndex, function(err) { - return cb(err,blockInfo); + // support only for p2pubkey p2pubkeyhash and p2sh + if (addrs.length === 1) { + objTx.out[to].addrStr = addrs[0]; + } + to++; }); }); - }; - - HistoricSync.prototype.updateConnectedCountDB = function(cb) { - var self = this; - self.sync.bDb.countConnected(function(err, count) { - self.connectedCountDB = count || 0; - self.syncedBlocks = count || 0; - return cb(err); + self.sync.bDb.setLastFileIndex(self.blockExtractor.currentFileIndex, function(err) { + return cb(err,blockInfo); }); - }; + }); +}; + +HistoricSync.prototype.updateConnectedCountDB = function(cb) { + var self = this; + self.sync.bDb.countConnected(function(err, count) { + self.connectedCountDB = count || 0; + self.syncedBlocks = count || 0; + return cb(err); + }); +}; - HistoricSync.prototype.updateBlockChainHeight = function(cb) { - var self = this; +HistoricSync.prototype.updateBlockChainHeight = function(cb) { + var self = this; - self.rpc.getBlockCount(function(err, res) { - self.blockChainHeight = res.result; - return cb(err); - }); - }; + self.rpc.getBlockCount(function(err, res) { + self.blockChainHeight = res.result; + return cb(err); + }); +}; - HistoricSync.prototype.checkNetworkSettings = function(next) { - var self = this; +HistoricSync.prototype.checkNetworkSettings = function(next) { + var self = this; - self.hasGenesis = false; + self.hasGenesis = false; - // check network config - self.rpc.getBlockHash(0, function(err, res){ + // check network config + self.rpc.getBlockHash(0, function(err, res){ + if (!err && ( res && res.result !== self.genesis)) { + err = new Error(BAD_GEN_ERROR + config.network); + } + if (err) return next(err); + self.sync.bDb.has(self.genesis, function(err, b) { if (!err && ( res && res.result !== self.genesis)) { - err = new Error(BAD_GEN_ERROR + config.network); + err = new Error(BAD_GEN_ERROR_DB + config.network); } - if (err) return next(err); - self.sync.bDb.has(self.genesis, function(err, b) { - if (!err && ( res && res.result !== self.genesis)) { - err = new Error(BAD_GEN_ERROR_DB + config.network); + self.hasGenesis = b?true:false; + return next(err); + }); + }); +}; + +HistoricSync.prototype.updateStartBlock = function(next) { + var self = this; + + self.startBlock = self.genesis; + + self.sync.bDb.getTip(function(err,tip) { + if (!tip) return next(); + + var blockInfo; + var oldtip; + + //check that the tip is still on the mainchain + async.doWhilst( + function(cb) { + self.sync.bDb.fromHashWithInfo(tip, function(err, bi) { + blockInfo = bi ? bi.info : {}; + if (oldtip) + self.sync.setBlockMain(oldtip, false, cb); + else + return cb(); + }); + }, + function(err) { + if (err) return next(err); + var ret = false; + if ( self.blockChainHeight === blockInfo.height || + blockInfo.confirmations > 0) { + ret = false; } - self.hasGenesis = b?true:false; + else { + oldtip = tip; + tip = blockInfo.previousblockhash; + assert(tip); + p('Previous TIP is now orphan. Back to:' + tip); + ret = true; + } + return ret; + }, + function(err) { + self.startBlock = tip; + p('Resuming sync from block:'+tip); return next(err); - }); - }); - }; + } + ); + }); +}; - HistoricSync.prototype.updateStartBlock = function(next) { - var self = this; +HistoricSync.prototype.prepareFileSync = function(opts, next) { + var self = this; - self.startBlock = self.genesis; - - self.sync.bDb.getTip(function(err,tip) { - if (!tip) return next(); - - var blockInfo; - var oldtip; - - //check that the tip is still on the mainchain - async.doWhilst( - function(cb) { - self.sync.bDb.fromHashWithInfo(tip, function(err, bi) { - blockInfo = bi ? bi.info : {}; - if (oldtip) - self.sync.setBlockMain(oldtip, false, cb); - else - return cb(); - }); - }, - function(err) { - if (err) return next(err); - var ret = false; - if ( self.blockChainHeight === blockInfo.height || - blockInfo.confirmations > 0) { - ret = false; - } - else { - oldtip = tip; - tip = blockInfo.previousblockhash; - assert(tip); - p('Previous TIP is now orphan. Back to:' + tip); - ret = true; - } - return ret; - }, - function(err) { - self.startBlock = tip; - p('Resuming sync from block:'+tip); - return next(err); - } - ); - }); - }; - - HistoricSync.prototype.prepareFileSync = function(opts, next) { - var self = this; - - if ( opts.forceRPC || !config.bitcoind.dataDir || - self.connectedCountDB > self.blockChainHeight * 0.9) return next(); + if ( opts.forceRPC || !config.bitcoind.dataDir || + self.connectedCountDB > self.blockChainHeight * 0.9) return next(); - try { - self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); - } catch (e) { - p(e.message + '. Disabling file sync.'); - return next(); - } - - self.getFn = self.getBlockFromFile; - self.allowReorgs = true; - self.sync.bDb.getLastFileIndex(function(err, idx) { - if (opts.forceStartFile) - self.blockExtractor.currentFileIndex = opts.forceStartFile; - else if (idx) self.blockExtractor.currentFileIndex = idx; - - var h = self.genesis; - - p('Seeking file to:' + self.startBlock); - //forward till startBlock - async.whilst( - function() { - return h !== self.startBlock; - }, - function (w_cb) { - self.getBlockFromFile(function(err,b) { - if (!b) return w_cb('Could not find block ' + self.startBlock); - h=b.hash; - setImmediate(function(){ - return w_cb(err); - }); - }); - }, next); - }); - }; - - //NOP - HistoricSync.prototype.prepareRpcSync = function(opts, next) { - var self = this; - - if (self.blockExtractor) return next(); - self.getFn = self.getBlockFromRPC; - self.currentRpcHash = self.startBlock; - self.allowReorgs = false; + try { + self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); + } catch (e) { + p(e.message + '. Disabling file sync.'); return next(); - }; + } - HistoricSync.prototype.showSyncStartMessage = function() { - var self = this; + self.getFn = self.getBlockFromFile; + self.allowReorgs = true; + self.sync.bDb.getLastFileIndex(function(err, idx) { + if (opts.forceStartFile) + self.blockExtractor.currentFileIndex = opts.forceStartFile; + else if (idx) self.blockExtractor.currentFileIndex = idx; - p('Got ' + self.connectedCountDB + - ' blocks in current DB, out of ' + self.blockChainHeight + ' block at bitcoind'); + var h = self.genesis; - if (self.blockExtractor) { - p('bitcoind dataDir configured...importing blocks from .dat files'); - p('First file index: ' + self.blockExtractor.currentFileIndex); - } - else { - p('syncing from RPC (slow)'); - } - - p('Starting from: ', self.startBlock); - self.showProgress(); - }; - - - HistoricSync.prototype.setupSyncStatus = function() { - var self = this; - - var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000); - if (step < 10) step = 10; - - self.step = step; - self.type = self.blockExtractor?'from .dat Files':'from RPC calls'; - self.status = 'syncing'; - self.startTs = Date.now(); - self.endTs = null; - this.error = null; - this.syncPercentage = 0; - }; - - HistoricSync.prototype.prepareToSync = function(opts, next) { - var self = this; - - self.status = 'starting'; - async.series([ - function(s_c) { - self.checkNetworkSettings(s_c); + p('Seeking file to:' + self.startBlock); + //forward till startBlock + async.whilst( + function() { + return h !== self.startBlock; }, - function(s_c) { - self.updateConnectedCountDB(s_c); - }, - function(s_c) { - self.updateBlockChainHeight(s_c); - }, - function(s_c) { - self.updateStartBlock(s_c); - }, - function(s_c) { - self.prepareFileSync(opts, s_c); - }, - function(s_c) { - self.prepareRpcSync(opts, s_c); - }, - ], - function(err) { - if (err) return(self.setError(err)); + function (w_cb) { + self.getBlockFromFile(function(err,b) { + if (!b) return w_cb('Could not find block ' + self.startBlock); + h=b.hash; + setImmediate(function(){ + return w_cb(err); + }); + }); + }, next); + }); +}; - self.showSyncStartMessage(); - self.setupSyncStatus(); - return next(); - }); - }; +//NOP +HistoricSync.prototype.prepareRpcSync = function(opts, next) { + var self = this; - - HistoricSync.prototype.start = function(opts, next) { - var self = this; + if (self.blockExtractor) return next(); + self.getFn = self.getBlockFromRPC; + self.currentRpcHash = self.startBlock; + self.allowReorgs = false; + return next(); +}; - if (self.status==='starting' || self.status==='syncing') { - p('## Wont start to sync while status is %s', self.status); - return next(); - } +HistoricSync.prototype.showSyncStartMessage = function() { + var self = this; - self.prepareToSync(opts, function(err) { - if (err) return next(self.setError(err)); + p('Got ' + self.connectedCountDB + + ' blocks in current DB, out of ' + self.blockChainHeight + ' block at bitcoind'); - async.whilst( - function() { - self.showProgress(); - return self.status === 'syncing'; - }, - function (w_cb) { - self.getFn(function(err,blockInfo) { - if (err) return w_cb(self.setError(err)); - if (blockInfo && blockInfo.hash) { - self.syncedBlocks++; - self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err) { + if (self.blockExtractor) { + p('bitcoind dataDir configured...importing blocks from .dat files'); + p('First file index: ' + self.blockExtractor.currentFileIndex); + } + else { + p('syncing from RPC (slow)'); + } + + p('Starting from: ', self.startBlock); + self.showProgress(); +}; + + +HistoricSync.prototype.setupSyncStatus = function() { + var self = this; + + var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000); + if (step < 10) step = 10; + + self.step = step; + self.type = self.blockExtractor?'from .dat Files':'from RPC calls'; + self.status = 'syncing'; + self.startTs = Date.now(); + self.endTs = null; + this.error = null; + this.syncPercentage = 0; +}; + +HistoricSync.prototype.prepareToSync = function(opts, next) { + var self = this; + + self.status = 'starting'; + async.series([ + function(s_c) { + self.checkNetworkSettings(s_c); + }, + function(s_c) { + self.updateConnectedCountDB(s_c); + }, + function(s_c) { + self.updateBlockChainHeight(s_c); + }, + function(s_c) { + self.updateStartBlock(s_c); + }, + function(s_c) { + self.prepareFileSync(opts, s_c); + }, + function(s_c) { + self.prepareRpcSync(opts, s_c); + }, + ], + function(err) { + if (err) return(self.setError(err)); + + self.showSyncStartMessage(); + self.setupSyncStatus(); + return next(); + }); +}; + + +HistoricSync.prototype.start = function(opts, next) { + var self = this; + + if (self.status==='starting' || self.status==='syncing') { + p('## Wont start to sync while status is %s', self.status); + return next(); + } + + self.prepareToSync(opts, function(err) { + if (err) return next(self.setError(err)); + + async.whilst( + function() { + self.showProgress(); + return self.status === 'syncing'; + }, + function (w_cb) { + self.getFn(function(err,blockInfo) { + if (err) return w_cb(self.setError(err)); + if (blockInfo && blockInfo.hash) { + self.syncedBlocks++; + self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err) { + if (err) return w_cb(self.setError(err)); + + self.sync.bDb.setTip(blockInfo.hash, function(err) { if (err) return w_cb(self.setError(err)); - self.sync.bDb.setTip(blockInfo.hash, function(err) { - if (err) return w_cb(self.setError(err)); - - setImmediate(function(){ - return w_cb(err); - }); + setImmediate(function(){ + return w_cb(err); }); }); - } - else { - self.endTs = Date.now(); - self.status = 'finished'; - console.log('Done Syncing', self.info()); - return w_cb(err); - } - }); - }, next); - }); - }; - return HistoricSync; -} -module.defineClass(spec); + }); + } + else { + self.endTs = Date.now(); + self.status = 'finished'; + console.log('Done Syncing', self.info()); + return w_cb(err); + } + }); + }, next); + }); +}; +module.exports = require('soop')(HistoricSync); diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 7122fe4c..16678438 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -1,127 +1,121 @@ 'use strict'; -require('classtool'); +var fs = require('fs'); +var bitcoreUtil = require('bitcore/util/util'); +var Sync = require('./Sync'); +var Peer = require('bitcore/Peer'); +var config = require('../config/config'); +var networks = require('bitcore/networks'); -function spec() { - var fs = require('fs'); - var bitcoreUtil = require('bitcore/util/util'); - var Sync = require('./Sync').class(); - var Peer = require('bitcore/Peer').class(); - var config = require('../config/config'); - var networks = require('bitcore/networks'); - - var peerdb_fn = 'peerdb.json'; - - function PeerSync(opts) { - this.connected = false; - this.peerdb = undefined; - this.allowReorgs = false; - this.PeerManager = require('bitcore/PeerManager').createClass({ - network: (config.network === 'testnet' ? networks.testnet : networks.livenet) - }); - this.peerman = new this.PeerManager(); - this.load_peers(); - this.sync = new Sync(opts); - } - - PeerSync.prototype.load_peers = function() { - this.peerdb = [{ - ipv4: config.bitcoind.host, - port: config.bitcoind.p2pPort - }]; - - fs.writeFileSync(peerdb_fn, JSON.stringify(this.peerdb)); - }; - - PeerSync.prototype.info = function() { - return { - connected: this.connected, - host: this.peerdb[0].ipv4, - port: this.peerdb[0].port - }; - }; - - PeerSync.prototype.handleInv = function(info) { - var invs = info.message.invs; - info.conn.sendGetData(invs); - }; - - PeerSync.prototype.handleTx = function(info) { - var tx = info.message.tx.getStandardizedObject(); - tx.outs = info.message.tx.outs; - tx.ins = info.message.tx.ins; - console.log('[p2p_sync] Handle tx: ' + tx.hash); - tx.time = tx.time || Math.round(new Date().getTime() / 1000); - - this.sync.storeTxs([tx], function(err) { - if (err) { - console.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); - } - }); - }; - - PeerSync.prototype.handleBlock = function(info) { - var self = this; - var block = info.message.block; - var blockHash = bitcoreUtil.formatHashFull(block.calcHash()); - - console.log('[p2p_sync] Handle block: %s (allowReorgs: %s)', blockHash, self.allowReorgs); - - var tx_hashes = block.txs.map(function(tx) { - return bitcoreUtil.formatHashFull(tx.hash); - }); - - this.sync.storeTipBlock({ - 'hash': blockHash, - 'tx': tx_hashes, - 'previousblockhash': bitcoreUtil.formatHashFull(block.prev_hash), - }, self.allowReorgs, function(err) { - if (err && err.message.match(/NEED_SYNC/) && self.historicSync) { - console.log('[p2p_sync] Orphan block received. Triggering sync'); - self.historicSync.start({}, function(){ - console.log('[p2p_sync] Done resync.'); - }); - } - else if (err) { - console.log('[p2p_sync] Error in handle Block: ' + err); - } - }); - }; - - PeerSync.prototype.handle_connected = function(data) { - var peerman = data.pm; - var peers_n = peerman.peers.length; - console.log('[p2p_sync] Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's' : '')); - }; - - PeerSync.prototype.run = function() { - var self = this; - - this.peerdb.forEach(function(datum) { - var peer = new Peer(datum.ipv4, datum.port); - self.peerman.addPeer(peer); - }); - - this.peerman.on('connection', function(conn) { - self.connected = true; - conn.on('inv', self.handleInv.bind(self)); - conn.on('block', self.handleBlock.bind(self)); - conn.on('tx', self.handleTx.bind(self)); - }); - this.peerman.on('connect', self.handle_connected.bind(self)); - - this.peerman.on('netDisconnected', function() { - self.connected = false; - }); - - this.peerman.start(); - }; - - PeerSync.prototype.close = function() { - this.sync.close(); - }; - - - return PeerSync; +var peerdb_fn = 'peerdb.json'; +function PeerSync(opts) { + this.connected = false; + this.peerdb = undefined; + this.allowReorgs = false; + this.PeerManager = require('bitcore/PeerManager').createClass({ + network: (config.network === 'testnet' ? networks.testnet : networks.livenet) + }); + this.peerman = new this.PeerManager(); + this.load_peers(); + this.sync = new Sync(opts); } -module.defineClass(spec); + +PeerSync.prototype.load_peers = function() { + this.peerdb = [{ + ipv4: config.bitcoind.host, + port: config.bitcoind.p2pPort + }]; + + fs.writeFileSync(peerdb_fn, JSON.stringify(this.peerdb)); +}; + +PeerSync.prototype.info = function() { + return { + connected: this.connected, + host: this.peerdb[0].ipv4, + port: this.peerdb[0].port + }; +}; + +PeerSync.prototype.handleInv = function(info) { + var invs = info.message.invs; + info.conn.sendGetData(invs); +}; + +PeerSync.prototype.handleTx = function(info) { + var tx = info.message.tx.getStandardizedObject(); + tx.outs = info.message.tx.outs; + tx.ins = info.message.tx.ins; + console.log('[p2p_sync] Handle tx: ' + tx.hash); + tx.time = tx.time || Math.round(new Date().getTime() / 1000); + + this.sync.storeTxs([tx], function(err) { + if (err) { + console.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); + } + }); +}; + +PeerSync.prototype.handleBlock = function(info) { + var self = this; + var block = info.message.block; + var blockHash = bitcoreUtil.formatHashFull(block.calcHash()); + + console.log('[p2p_sync] Handle block: %s (allowReorgs: %s)', blockHash, self.allowReorgs); + + var tx_hashes = block.txs.map(function(tx) { + return bitcoreUtil.formatHashFull(tx.hash); + }); + + this.sync.storeTipBlock({ + 'hash': blockHash, + 'tx': tx_hashes, + 'previousblockhash': bitcoreUtil.formatHashFull(block.prev_hash), + }, self.allowReorgs, function(err) { + if (err && err.message.match(/NEED_SYNC/) && self.historicSync) { + console.log('[p2p_sync] Orphan block received. Triggering sync'); + self.historicSync.start({}, function(){ + console.log('[p2p_sync] Done resync.'); + }); + } + else if (err) { + console.log('[p2p_sync] Error in handle Block: ' + err); + } + }); +}; + +PeerSync.prototype.handle_connected = function(data) { + var peerman = data.pm; + var peers_n = peerman.peers.length; + console.log('[p2p_sync] Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's' : '')); +}; + +PeerSync.prototype.run = function() { + var self = this; + + this.peerdb.forEach(function(datum) { + var peer = new Peer(datum.ipv4, datum.port); + self.peerman.addPeer(peer); + }); + + this.peerman.on('connection', function(conn) { + self.connected = true; + conn.on('inv', self.handleInv.bind(self)); + conn.on('block', self.handleBlock.bind(self)); + conn.on('tx', self.handleTx.bind(self)); + }); + this.peerman.on('connect', self.handle_connected.bind(self)); + + this.peerman.on('netDisconnected', function() { + self.connected = false; + }); + + this.peerman.start(); +}; + +PeerSync.prototype.close = function() { + this.sync.close(); +}; + + +module.exports = require('soop')(PeerSync); diff --git a/lib/PoolMatch.js b/lib/PoolMatch.js index 574ad98b..7c526909 100644 --- a/lib/PoolMatch.js +++ b/lib/PoolMatch.js @@ -1,38 +1,32 @@ 'use strict'; -require('classtool'); +var imports = require('soop').imports(); +var fs = require('fs'); +var buffertools = require('buffertools'); +var db = imports.db || JSON.parse( fs.readFileSync(imports.poolMatchFile || './poolMatchFile.json')); -function spec(b) { +var PoolMatch = function() { + var self = this; - var fs = require('fs'); - var buffertools = require('buffertools'); - var db = b.db || JSON.parse( fs.readFileSync(b.poolMatchFile || './poolMatchFile.json')); - - var PoolMatch = function() { - var self = this; - - self.strings = {}; - db.forEach(function(pool) { - pool.searchStrings.forEach(function(s) { - self.strings[s] = { - poolName: pool.poolName, - url: pool.url - }; - }); + self.strings = {}; + db.forEach(function(pool) { + pool.searchStrings.forEach(function(s) { + self.strings[s] = { + poolName: pool.poolName, + url: pool.url + }; }); - }; + }); +}; - PoolMatch.prototype.match = function(buffer) { - var self = this; - for(var k in self.strings) { - if (buffertools.indexOf(buffer, k) >= 0) { - return self.strings[k]; - } +PoolMatch.prototype.match = function(buffer) { + var self = this; + for(var k in self.strings) { + if (buffertools.indexOf(buffer, k) >= 0) { + return self.strings[k]; } - }; + } +}; - return PoolMatch; -} -module.defineClass(spec); - +module.exports = require('soop')(PoolMatch); diff --git a/lib/Rpc.js b/lib/Rpc.js index 466690bd..5af328f9 100644 --- a/lib/Rpc.js +++ b/lib/Rpc.js @@ -1,114 +1,110 @@ 'use strict'; -require('classtool'); +var imports = require('soop').imports(); + +var RpcClient = require('bitcore/RpcClient'), + BitcoreBlock = require('bitcore/Block'), + bitcoreUtil = require('bitcore/util/util'), + util = require('util'), + config = require('../config/config'); + +var bitcoreRpc = imports.bitcoreRpc || new RpcClient(config.bitcoind); + +function Rpc() { +} + +Rpc._parseTxResult = function(info) { + var b = new Buffer(info.hex,'hex'); + + // remove fields we dont need, to speed and adapt the information + delete info.hex; + + // Inputs => add index + coinBase flag + var n =0; + info.vin.forEach(function(i) { + i.n = n++; + if (i.coinbase) info.isCoinBase = true; + if (i.scriptSig) delete i.scriptSig.hex; + }); + + // Outputs => add total + var valueOutSat = 0; + info.vout.forEach( function(o) { + valueOutSat += o.value * bitcoreUtil.COIN; + delete o.scriptPubKey.hex; + }); + info.valueOut = parseInt(valueOutSat) / bitcoreUtil.COIN; + info.size = b.length; + + return info; +}; -function spec(b) { - var RpcClient = require('bitcore/RpcClient').class(), - BitcoreBlock = require('bitcore/Block').class(), - bitcoreUtil = require('bitcore/util/util'), - util = require('util'), - config = require('../config/config'); +Rpc.errMsg = function(err) { + var e = err; + e.message += util.format(' [Host: %s:%d User:%s Using password:%s]', + bitcoreRpc.host, + bitcoreRpc.port, + bitcoreRpc.user, + bitcoreRpc.pass?'yes':'no' + ); + return e; +}; - var bitcoreRpc = b.bitcoreRpc || new RpcClient(config.bitcoind); +Rpc.getTxInfo = function(txid, doNotParse, cb) { + var self = this; - function Rpc() { + if (typeof doNotParse === 'function') { + cb = doNotParse; + doNotParse = false; } - Rpc._parseTxResult = function(info) { - var b = new Buffer(info.hex,'hex'); + bitcoreRpc.getRawTransaction(txid, 1, function(err, txInfo) { + // Not found? + if (err && err.code === -5) return cb(); + if (err) return cb(self.errMsg(err)); - // remove fields we dont need, to speed and adapt the information - delete info.hex; - - // Inputs => add index + coinBase flag - var n =0; - info.vin.forEach(function(i) { - i.n = n++; - if (i.coinbase) info.isCoinBase = true; - if (i.scriptSig) delete i.scriptSig.hex; - }); - - // Outputs => add total - var valueOutSat = 0; - info.vout.forEach( function(o) { - valueOutSat += o.value * bitcoreUtil.COIN; - delete o.scriptPubKey.hex; - }); - info.valueOut = parseInt(valueOutSat) / bitcoreUtil.COIN; - info.size = b.length; - - return info; - }; + var info = doNotParse ? txInfo.result : self._parseTxResult(txInfo.result); + return cb(null,info); + }); +}; - Rpc.errMsg = function(err) { - var e = err; - e.message += util.format(' [Host: %s:%d User:%s Using password:%s]', - bitcoreRpc.host, - bitcoreRpc.port, - bitcoreRpc.user, - bitcoreRpc.pass?'yes':'no' - ); - return e; - }; +Rpc.blockIndex = function(height, cb) { + var self = this; - Rpc.getTxInfo = function(txid, doNotParse, cb) { - var self = this; + bitcoreRpc.getBlockHash(height, function(err, bh){ + if (err) return cb(self.errMsg(err)); + cb(null, { blockHash: bh.result }); + }); +}; - if (typeof doNotParse === 'function') { - cb = doNotParse; - doNotParse = false; - } +Rpc.getBlock = function(hash, cb) { + var self = this; - bitcoreRpc.getRawTransaction(txid, 1, function(err, txInfo) { - // Not found? - if (err && err.code === -5) return cb(); - if (err) return cb(self.errMsg(err)); - - var info = doNotParse ? txInfo.result : self._parseTxResult(txInfo.result); - return cb(null,info); - }); - }; + bitcoreRpc.getBlock(hash, function(err,info) { + // Not found? + if (err && err.code === -5) return cb(); + if (err) return cb(self.errMsg(err)); - Rpc.blockIndex = function(height, cb) { - var self = this; + if (info.result.height) + info.result.reward = BitcoreBlock.getBlockValue(info.result.height) / bitcoreUtil.COIN ; - bitcoreRpc.getBlockHash(height, function(err, bh){ - if (err) return cb(self.errMsg(err)); - cb(null, { blockHash: bh.result }); - }); - }; + return cb(err,info.result); + }); +}; - Rpc.getBlock = function(hash, cb) { - var self = this; +Rpc.sendRawTransaction = function(rawtx, cb) { + var self = this; + bitcoreRpc.sendRawTransaction(rawtx, function(err, txid) { + if (err && err.code === -5) return cb(err); // transaction already in block chain + if (err) return cb(self.errMsg(err)); - bitcoreRpc.getBlock(hash, function(err,info) { - // Not found? - if (err && err.code === -5) return cb(); - if (err) return cb(self.errMsg(err)); - - - if (info.result.height) - info.result.reward = BitcoreBlock.getBlockValue(info.result.height) / bitcoreUtil.COIN ; - - return cb(err,info.result); - }); - }; - - Rpc.sendRawTransaction = function(rawtx, cb) { - var self = this; - bitcoreRpc.sendRawTransaction(rawtx, function(err, txid) { - if (err && err.code === -5) return cb(err); // transaction already in block chain - if (err) return cb(self.errMsg(err)); - - return cb(err, txid.result); - }); - }; - - return Rpc; -} -module.defineClass(spec); + return cb(err, txid.result); + }); +}; + +module.exports = require('soop')(Rpc); diff --git a/lib/Sync.js b/lib/Sync.js index a712304d..568a6a0f 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -1,289 +1,284 @@ 'use strict'; -require('classtool'); +var imports = require('soop').imports(); +var sockets = require('../app/controllers/socket.js'); + +var BlockDb = require('./BlockDb').default(); +var TransactionDb = require('./TransactionDb').default(); +var config = require('../config/config'); +var networks = require('bitcore/networks'); +var async = require('async'); -function spec() { - var sockets = require('../app/controllers/socket.js'); - var BlockDb = require('./BlockDb').class(); +function Sync(opts) { + this.opts = opts || {}; + this.bDb = new BlockDb(opts); + this.txDb = new TransactionDb(opts); + this.txDb.on('tx_for_address', this.handleTxForAddress.bind(this)); + this.txDb.on('new_tx', this.handleNewTx.bind(this)); + this.bDb.on('new_block', this.handleNewBlock.bind(this)); + this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; +} - var TransactionDb = require('./TransactionDb').class(); - var config = require('../config/config'); - var networks = require('bitcore/networks'); - var async = require('async'); +Sync.prototype.close = function(cb) { + var self = this; + self.txDb.close(function() { + self.bDb.close(cb); + }); +}; - function Sync(opts) { - this.opts = opts || {}; - this.bDb = new BlockDb(opts); - this.txDb = new TransactionDb(opts); - this.txDb.on('tx_for_address', this.handleTxForAddress.bind(this)); - this.txDb.on('new_tx', this.handleNewTx.bind(this)); - this.bDb.on('new_block', this.handleNewBlock.bind(this)); - this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; +Sync.prototype.destroy = function(next) { + var self = this; + async.series([ + + function(b) { + self.bDb.drop(b); + }, + function(b) { + self.txDb.drop(b); + }, + ], next); +}; + +/* + * Arrives a NEW block, which is the new TIP + * + * Case 0) Simple case + * A-B-C-D-E(TIP)-NEW + * + * Case 1) + * A-B-C-D-E(TIP) + * \ + * NEW + * + * 1) Declare D-E orphans (and possible invalidate TXs on them) + * + * Case 2) + * A-B-C-D-E(TIP) + * \ + * F-G-NEW + * 1) Set F-G as connected (mark TXs as valid) + * 2) Declare D-E orphans (and possible invalidate TXs on them) + * + * + * Case 3) + * + * A-B-C-D-E(TIP) ... NEW + * + * NEW is ignored (if allowReorgs is false) + * + * + */ + +Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { + + if (typeof allowReorgs === 'function') { + cb = allowReorgs; + allowReorgs = true; } + if (!b) return cb(); - Sync.prototype.close = function(cb) { - var self = this; - self.txDb.close(function() { - self.bDb.close(cb); - }); - }; + var self = this; + var oldTip, oldNext, needReorg = false; + var newPrev = b.previousblockhash; + async.series([ - Sync.prototype.destroy = function(next) { - var self = this; - async.series([ - - function(b) { - self.bDb.drop(b); - }, - function(b) { - self.txDb.drop(b); - }, - ], next); - }; - - /* - * Arrives a NEW block, which is the new TIP - * - * Case 0) Simple case - * A-B-C-D-E(TIP)-NEW - * - * Case 1) - * A-B-C-D-E(TIP) - * \ - * NEW - * - * 1) Declare D-E orphans (and possible invalidate TXs on them) - * - * Case 2) - * A-B-C-D-E(TIP) - * \ - * F-G-NEW - * 1) Set F-G as connected (mark TXs as valid) - * 2) Declare D-E orphans (and possible invalidate TXs on them) - * - * - * Case 3) - * - * A-B-C-D-E(TIP) ... NEW - * - * NEW is ignored (if allowReorgs is false) - * - * - */ - - Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { - - if (typeof allowReorgs === 'function') { - cb = allowReorgs; - allowReorgs = true; - } - if (!b) return cb(); - - var self = this; - var oldTip, oldNext, needReorg = false; - var newPrev = b.previousblockhash; - - async.series([ - - function(c) { - self.bDb.has(b.hash, function(err, val) { - return c(err || - (val ? new Error('WARN: Ignoring already existing block:' + b.hash) : null)); - }); - }, - function(c) { - if (!allowReorgs) return c(); - self.bDb.has(newPrev, function(err, val) { - if (!val && newPrev.match(/^0+$/)) return c(); - - return c(err || - (!val ? new Error('NEED_SYNC Ignoring block with non existing prev:' + b.hash) : null)); - }); - }, - function(c) { - self.txDb.createFromBlock(b, function(err) { - return c(err); - }); - }, - function(c) { - if (!allowReorgs) return c(); - self.bDb.getTip(function(err, val) { - oldTip = val; - if (oldTip && newPrev !== oldTip) needReorg = true; - return c(); - }); - }, - function(c) { - if (!needReorg) return c(); - self.bDb.getNext(newPrev, function(err, val) { - if (err) return c(err); - oldNext = val; - return c(); - }); - }, - function(c) { - self.bDb.add(b, c); - }, - function(c) { - if (!needReorg) return c(); - console.log('NEW TIP: %s NEED REORG (old tip: %s)', b.hash, oldTip); - self.processReorg(oldTip, oldNext, newPrev, c); - }, - function(c) { - if (!allowReorgs) return c(); - self.bDb.setTip(b.hash, function(err) { - return c(err); - }); - }, - function(c) { - self.bDb.setNext(newPrev, b.hash, function(err) { - return c(err); - }); - } - - ], - function(err) { - if (err && err.toString().match(/WARN/)) { - err = null; - } - return cb(err); - }); - }; - - - - Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, cb) { - var self = this; - - var orphanizeFrom; - - async.series([ - - function(c) { - self.bDb.isMain(newPrev, function(err, val) { - if (!val) return c(); - - console.log('# Reorg Case 1)'); - // case 1 - orphanizeFrom = oldNext; - return c(err); - }); - }, - function(c) { - if (orphanizeFrom) return c(); - - console.log('# Reorg Case 2)'); - self.setBranchConnectedBackwards(newPrev, function(err, yHash, newYHashNext) { - if (err) return c(err); - self.bDb.getNext(yHash, function(err, yHashNext) { - orphanizeFrom = yHashNext; - self.bDb.setNext(yHash, newYHashNext, function(err) { - return c(err); - }); - }); - }); - }, - function(c) { - if (!orphanizeFrom) return c(); - self.setBranchOrphan(orphanizeFrom, function(err) { - return c(err); - }); - }, - ], - function(err) { - return cb(err); - }); - }; - - Sync.prototype.setBlockMain = function(hash, isMain, cb) { - var self = this; - self.bDb.setMain(hash, isMain, function(err) { - if (err) return cb(err); - return self.txDb.handleBlockChange(hash, isMain, cb); - }); - }; - - Sync.prototype.setBranchOrphan = function(fromHash, cb) { - var self = this, - hashInterator = fromHash; - - async.whilst( - function() { - return hashInterator; - }, function(c) { - self.setBlockMain(hashInterator, false, function(err) { - if (err) return cb(err); - self.bDb.getNext(hashInterator, function(err, val) { - hashInterator = val; - return c(err); - }); + self.bDb.has(b.hash, function(err, val) { + return c(err || + (val ? new Error('WARN: Ignoring already existing block:' + b.hash) : null)); }); - }, cb); - }; - - Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { - var self = this, - hashInterator = fromHash, - lastHash = fromHash, - isMain; - - async.doWhilst( + }, function(c) { - self.setBlockMain(hashInterator, true, function(err) { + if (!allowReorgs) return c(); + self.bDb.has(newPrev, function(err, val) { + if (!val && newPrev.match(/^0+$/)) return c(); + + return c(err || + (!val ? new Error('NEED_SYNC Ignoring block with non existing prev:' + b.hash) : null)); + }); + }, + function(c) { + self.txDb.createFromBlock(b, function(err) { + return c(err); + }); + }, + function(c) { + if (!allowReorgs) return c(); + self.bDb.getTip(function(err, val) { + oldTip = val; + if (oldTip && newPrev !== oldTip) needReorg = true; + return c(); + }); + }, + function(c) { + if (!needReorg) return c(); + self.bDb.getNext(newPrev, function(err, val) { if (err) return c(err); - self.bDb.getPrev(hashInterator, function(err, val) { - if (err) return c(err); - lastHash = hashInterator; - hashInterator = val; - self.bDb.isMain(hashInterator, function(err, val) { - isMain = val; - return c(); - }); - }); + oldNext = val; + return c(); }); }, - function() { - return hashInterator && !isMain; + function(c) { + self.bDb.add(b, c); }, - function(err) { - console.log('\tFound yBlock:', hashInterator); - return cb(err, hashInterator, lastHash); + function(c) { + if (!needReorg) return c(); + console.log('NEW TIP: %s NEED REORG (old tip: %s)', b.hash, oldTip); + self.processReorg(oldTip, oldNext, newPrev, c); + }, + function(c) { + if (!allowReorgs) return c(); + self.bDb.setTip(b.hash, function(err) { + return c(err); + }); + }, + function(c) { + self.bDb.setNext(newPrev, b.hash, function(err) { + return c(err); + }); } - ); - }; - - Sync.prototype.handleTxForAddress = function(data) { - if (this.opts.shouldBroadcast) { - sockets.broadcastAddressTx(data.address, data.txid); - } - }; - - Sync.prototype.handleNewTx = function(data) { - if (this.opts.shouldBroadcast) { - sockets.broadcastTx(data.tx); - } - }; - - Sync.prototype.handleNewBlock = function(data) { - if (this.opts.shouldBroadcast) { - sockets.broadcastBlock(data.blockid); - } - }; - - Sync.prototype.storeTxs = function(txs, cb) { - var self = this; - self.txDb.createFromArray(txs, null, function(err) { - if (err) return cb(err); + ], + function(err) { + if (err && err.toString().match(/WARN/)) { + err = null; + } return cb(err); }); - }; +}; - return Sync; -} -module.defineClass(spec); + +Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, cb) { + var self = this; + + var orphanizeFrom; + + async.series([ + + function(c) { + self.bDb.isMain(newPrev, function(err, val) { + if (!val) return c(); + + console.log('# Reorg Case 1)'); + // case 1 + orphanizeFrom = oldNext; + return c(err); + }); + }, + function(c) { + if (orphanizeFrom) return c(); + + console.log('# Reorg Case 2)'); + self.setBranchConnectedBackwards(newPrev, function(err, yHash, newYHashNext) { + if (err) return c(err); + self.bDb.getNext(yHash, function(err, yHashNext) { + orphanizeFrom = yHashNext; + self.bDb.setNext(yHash, newYHashNext, function(err) { + return c(err); + }); + }); + }); + }, + function(c) { + if (!orphanizeFrom) return c(); + self.setBranchOrphan(orphanizeFrom, function(err) { + return c(err); + }); + }, + ], + function(err) { + return cb(err); + }); +}; + +Sync.prototype.setBlockMain = function(hash, isMain, cb) { + var self = this; + self.bDb.setMain(hash, isMain, function(err) { + if (err) return cb(err); + return self.txDb.handleBlockChange(hash, isMain, cb); + }); +}; + +Sync.prototype.setBranchOrphan = function(fromHash, cb) { + var self = this, + hashInterator = fromHash; + + async.whilst( + function() { + return hashInterator; + }, + function(c) { + self.setBlockMain(hashInterator, false, function(err) { + if (err) return cb(err); + self.bDb.getNext(hashInterator, function(err, val) { + hashInterator = val; + return c(err); + }); + }); + }, cb); +}; + +Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { + var self = this, + hashInterator = fromHash, + lastHash = fromHash, + isMain; + + async.doWhilst( + function(c) { + self.setBlockMain(hashInterator, true, function(err) { + if (err) return c(err); + self.bDb.getPrev(hashInterator, function(err, val) { + if (err) return c(err); + lastHash = hashInterator; + hashInterator = val; + self.bDb.isMain(hashInterator, function(err, val) { + isMain = val; + return c(); + }); + }); + }); + }, + function() { + return hashInterator && !isMain; + }, + function(err) { + console.log('\tFound yBlock:', hashInterator); + return cb(err, hashInterator, lastHash); + } + ); +}; + + +Sync.prototype.handleTxForAddress = function(data) { + if (this.opts.shouldBroadcast) { + sockets.broadcastAddressTx(data.address, data.txid); + } +}; + +Sync.prototype.handleNewTx = function(data) { + if (this.opts.shouldBroadcast) { + sockets.broadcastTx(data.tx); + } +}; + +Sync.prototype.handleNewBlock = function(data) { + if (this.opts.shouldBroadcast) { + sockets.broadcastBlock(data.blockid); + } +}; + +Sync.prototype.storeTxs = function(txs, cb) { + var self = this; + self.txDb.createFromArray(txs, null, function(err) { + if (err) return cb(err); + return cb(err); + }); +}; + + +module.exports = require('soop')(Sync); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 03415402..dfb4fd91 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -1,733 +1,728 @@ 'use strict'; -require('classtool'); +var imports = require('soop').imports(); +var parent = imports.parent || require('events').EventEmitter; +// blockHash -> txid mapping +var IN_BLK_PREFIX = 'txb-'; //txb-- => 1/0 (connected or not) -function spec(b) { +// Only for orphan blocks +var FROM_BLK_PREFIX = 'tx-'; //tx-- => 1 - var superclass = b.superclass || require('events').EventEmitter; - // blockHash -> txid mapping - var IN_BLK_PREFIX = 'txb-'; //txb-- => 1/0 (connected or not) +// to show tx outs +var OUTS_PREFIX = 'txo-'; //txo-- => [addr, btc_sat] +var SPENT_PREFIX = 'txs-'; //txs---- = ts - // Only for orphan blocks - var FROM_BLK_PREFIX = 'tx-'; //tx-- => 1 +// to sum up addr balance (only outs, spents are gotten later) +var ADDR_PREFIX = 'txa-'; //txa--- => + btc_sat:ts - // to show tx outs - var OUTS_PREFIX = 'txo-'; //txo-- => [addr, btc_sat] - var SPENT_PREFIX = 'txs-'; //txs---- = ts +// TODO: use bitcore networks module +var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; +var CONCURRENCY = 10; - // to sum up addr balance (only outs, spents are gotten later) - var ADDR_PREFIX = 'txa-'; //txa--- => + btc_sat:ts - - // TODO: use bitcore networks module - var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; - var CONCURRENCY = 10; - - var MAX_OPEN_FILES = 500; +var MAX_OPEN_FILES = 500; // var CONFIRMATION_NR_TO_NOT_CHECK = 10; //Spend - /** - * Module dependencies. - */ - var Rpc = b.rpc || require('./Rpc').class(), - util = require('bitcore/util/util'), - levelup = require('levelup'), - async = require('async'), - config = require('../config/config'), - assert = require('assert'); - var db = b.db || levelup(config.leveldb + '/txs',{maxOpenFiles: MAX_OPEN_FILES} ); - var Script = require('bitcore/Script').class(); - // This is 0.1.2 => c++ version of base57-native - var base58 = require('base58-native').base58Check; - var encodedData = require('bitcore/util/EncodedData').class({ - base58: base58 - }); - var versionedData = require('bitcore/util/VersionedData').class({ - superclass: encodedData - }); - var Address = require('bitcore/Address').class({ - superclass: versionedData - }); - var bitutil = require('bitcore/util/util'); - var networks = require('bitcore/networks'); +/** + * Module dependencies. + */ +var Rpc = imports.rpc || require('./Rpc'), + util = require('bitcore/util/util'), + levelup = require('levelup'), + async = require('async'), + config = require('../config/config'), + assert = require('assert'); +var db = imports.db || levelup(config.leveldb + '/txs',{maxOpenFiles: MAX_OPEN_FILES} ); +var Script = require('bitcore/Script'); +// This is 0.1.2 = > c++ version of base57-native +var base58 = require('base58-native').base58Check; +var encodedData = require('soop').load('bitcore/util/EncodedData',{ + base58: base58 +}); +var versionedData= require('soop').load('bitcore/util/VersionedData',{ + patent: encodedData +}); +var Address = require('soop').load('bitcore/Address',{ + parent: versionedData +}); +var bitutil = require('bitcore/util/util'); +var networks = require('bitcore/networks'); - var TransactionDb = function() { - TransactionDb.super(this, arguments); - this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; - }; - TransactionDb.superclass = superclass; +var TransactionDb = function() { + TransactionDb.super(this, arguments); + this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; +}; +TransactionDb.parent = parent; - TransactionDb.prototype.close = function(cb) { - db.close(cb); - }; +TransactionDb.prototype.close = function(cb) { + db.close(cb); +}; - TransactionDb.prototype.drop = function(cb) { - var path = config.leveldb + '/txs'; - db.close(function() { - require('leveldown').destroy(path, function() { - db = levelup(path, {maxOpenFiles: 500}); - return cb(); - }); +TransactionDb.prototype.drop = function(cb) { + var path = config.leveldb + '/txs'; + db.close(function() { + require('leveldown').destroy(path, function() { + db = levelup(path, {maxOpenFiles: 500}); + return cb(); }); - }; + }); +}; - TransactionDb.prototype.has = function(txid, cb) { +TransactionDb.prototype.has = function(txid, cb) { - var k = OUTS_PREFIX + txid; - db.get(k, function(err, val) { + var k = OUTS_PREFIX + txid; + db.get(k, function(err, val) { - var ret; + var ret; - if (err && err.notFound) { - err = null; - ret = false; - } - if (typeof val !== undefined) { - ret = true; - } - return cb(err, ret); - }); - }; - - TransactionDb.prototype._addSpentInfo = function(r, txid, index, ts) { - if (r.spentTxId) { - if (!r.multipleSpentAttempts) { - r.multipleSpentAttempts = [{ - txid: r.spentTxId, - index: r.index, - }]; - } - r.multipleSpentAttempts.push({ - txid: txid, - index: parseInt(index), - }); - } else { - r.spentTxId = txid; - r.spentIndex = parseInt(index); - r.spentTs = parseInt(ts); + if (err && err.notFound) { + err = null; + ret = false; } - }; + if (typeof val !== undefined) { + ret = true; + } + return cb(err, ret); + }); +}; + +TransactionDb.prototype._addSpentInfo = function(r, txid, index, ts) { + if (r.spentTxId) { + if (!r.multipleSpentAttempts) { + r.multipleSpentAttempts = [{ + txid: r.spentTxId, + index: r.index, + }]; + } + r.multipleSpentAttempts.push({ + txid: txid, + index: parseInt(index), + }); + } else { + r.spentTxId = txid; + r.spentIndex = parseInt(index); + r.spentTs = parseInt(ts); + } +}; - // This is not used now - TransactionDb.prototype.fromTxId = function(txid, cb) { - var self = this; - var k = OUTS_PREFIX + txid; - var ret = []; - var idx = {}; - var i = 0; +// This is not used now +TransactionDb.prototype.fromTxId = function(txid, cb) { + var self = this; + var k = OUTS_PREFIX + txid; + var ret = []; + var idx = {}; + var i = 0; - // outs. - db.createReadStream({ - start: k, - end: k + '~' + // outs. + db.createReadStream({ + start: k, + end: k + '~' + }) + .on('data', function(data) { + var k = data.key.split('-'); + var v = data.value.split(':'); + ret.push({ + addr: v[0], + value_sat: parseInt(v[1]), + index: parseInt(k[2]), + }); + idx[parseInt(k[2])] = i++; }) - .on('data', function(data) { - var k = data.key.split('-'); - var v = data.value.split(':'); - ret.push({ - addr: v[0], - value_sat: parseInt(v[1]), - index: parseInt(k[2]), - }); - idx[parseInt(k[2])] = i++; - }) - .on('error', function(err) { - return cb(err); - }) - .on('end', function() { - - var k = SPENT_PREFIX + txid + '-'; - db.createReadStream({ - start: k, - end: k + '~' - }) - .on('data', function(data) { - var k = data.key.split('-'); - var j = idx[parseInt(k[2])]; - - assert(typeof j !== 'undefined', 'Spent could not be stored: tx ' + txid + - 'spent in TX:' + k[1] + ',' + k[2] + ' j:' + j); - - self._addSpentInfo(ret[j], k[3], k[4], data.value); - }) - .on('error', function(err) { - return cb(err); - }) - .on('end', function(err) { - return cb(err, ret); - }); - }); - }; - - - TransactionDb.prototype._fillSpent = function(info, cb) { - var self = this; - - if (!info) return cb(); - - var k = SPENT_PREFIX + info.txid + '-'; - db.createReadStream({ - start: k, - end: k + '~' + .on('error', function(err) { + return cb(err); }) - .on('data', function(data) { - var k = data.key.split('-'); - self._addSpentInfo(info.vout[k[2]], k[3], k[4], data.value); - }) - .on('error', function(err) { - return cb(err); - }) - .on('end', function(err) { - return cb(err); - }); - }; + .on('end', function() { - - TransactionDb.prototype._fillOutpoints = function(info, cb) { - var self = this; - - if (!info || info.isCoinBase) return cb(); - - var valueIn = 0; - var incompleteInputs = 0; - - async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) { - self.fromTxIdN(i.txid, i.vout, info.confirmations, function(err, ret) { - //console.log('[TransactionDb.js.154:ret:]',ret); //TODO - if (!ret || !ret.addr || !ret.valueSat) { - console.log('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid); - if (ret) i.unconfirmedInput = ret.unconfirmedInput; - incompleteInputs = 1; - return c_in(); // error not scalated - } - - info.firstSeenTs = ret.spentTs; - i.unconfirmedInput = i.unconfirmedInput; - i.addr = ret.addr; - i.valueSat = ret.valueSat; - i.value = ret.valueSat / util.COIN; - valueIn += i.valueSat; - -/* - * If confirmed by bitcoind, we could not check for double spents - * but we prefer to keep the flag of double spent attempt - * - if (info.confirmations - && info.confirmations >= CONFIRMATION_NR_TO_NOT_CHECK) - return c_in(); -isspent -*/ - // Double spent? - if (ret.multipleSpentAttempt || !ret.spentTxId || - (ret.spentTxId && ret.spentTxId !== info.txid) - ) { - if (ret.multipleSpentAttempts) { - ret.multipleSpentAttempts.each(function(mul) { - if (mul.spentTxId !== info.txid) { - i.doubleSpentTxID = ret.spentTxId; - i.doubleSpentIndex = ret.spentIndex; - } - }); - } else if (!ret.spentTxId) { - i.dbError = 'Input spent not registered'; - } else { - i.doubleSpentTxID = ret.spentTxId; - i.doubleSpentIndex = ret.spentIndex; - } - } else { - i.doubleSpentTxID = null; - } - return c_in(); - }); - }, - function() { - if (!incompleteInputs) { - info.valueIn = valueIn / util.COIN; - info.fees = (valueIn - parseInt(info.valueOut * util.COIN)) / util.COIN; - } else { - info.incompleteInputs = 1; - } - return cb(); - }); - }; - - TransactionDb.prototype._getInfo = function(txid, next) { - var self = this; - - Rpc.getTxInfo(txid, function(err, info) { - if (err) return next(err); - - self._fillOutpoints(info, function() { - self._fillSpent(info, function() { - return next(null, info); - }); - }); - }); - }; - - - // Simplified / faster Info version: No spent / outpoints info. - TransactionDb.prototype.fromIdInfoSimple = function(txid, cb) { - Rpc.getTxInfo(txid, true, function(err, info) { - if (err) return cb(err); - if (!info) return cb(); - return cb(err, info); - }); - }; - - TransactionDb.prototype.fromIdWithInfo = function(txid, cb) { - var self = this; - - self._getInfo(txid, function(err, info) { - if (err) return cb(err); - if (!info) return cb(); - return cb(err, { - txid: txid, - info: info - }); - }); - }; - - TransactionDb.prototype.fromTxIdN = function(txid, n, confirmations, cb) { - var self = this; - var k = OUTS_PREFIX + txid + '-' + n; - - db.get(k, function(err, val) { - if (!val || (err && err.notFound)) { - return cb(null, { - unconfirmedInput: 1 - }); - } - - var a = val.split(':'); - var ret = { - addr: a[0], - valueSat: parseInt(a[1]), - }; - - /* - * If this TxID comes from an RPC request - * the .confirmations value from bitcoind is available - * so we could avoid checking if the input were double spented - * - * This speed up address calculations by ~30% - * - if (confirmations >= CONFIRMATION_NR_TO_NOT_CHECK) { - return cb(null, ret); - } - */ - - // spent? - var k = SPENT_PREFIX + txid + '-' + n + '-'; + var k = SPENT_PREFIX + txid + '-'; db.createReadStream({ start: k, end: k + '~' }) .on('data', function(data) { var k = data.key.split('-'); - self._addSpentInfo(ret, k[3], k[4], data.value); + var j = idx[parseInt(k[2])]; + + assert(typeof j !== 'undefined', 'Spent could not be stored: tx ' + txid + + 'spent in TX:' + k[1] + ',' + k[2] + ' j:' + j); + + self._addSpentInfo(ret[j], k[3], k[4], data.value); }) - .on('error', function(error) { - return cb(error); + .on('error', function(err) { + return cb(err); }) - .on('end', function() { - return cb(null, ret); + .on('end', function(err) { + return cb(err, ret); }); }); - }; +}; - TransactionDb.prototype.fillConfirmations = function(o, cb) { - var self = this; - self.isConfirmed(o.txid, function(err, is) { - if (err) return cb(err); +TransactionDb.prototype._fillSpent = function(info, cb) { + var self = this; - o.isConfirmed = is; - if (!o.spentTxId) return cb(); + if (!info) return cb(); - if (o.multipleSpentAttempts) { + var k = SPENT_PREFIX + info.txid + '-'; + db.createReadStream({ + start: k, + end: k + '~' + }) + .on('data', function(data) { + var k = data.key.split('-'); + self._addSpentInfo(info.vout[k[2]], k[3], k[4], data.value); + }) + .on('error', function(err) { + return cb(err); + }) + .on('end', function(err) { + return cb(err); + }); +}; - async.eachLimit(o.multipleSpentAttempts, CONCURRENCY, - function(oi, e_c) { - self.isConfirmed(oi.spentTxId, function(err, is) { - if (err) return; - if (is) { - o.spentTxId = oi.spentTxId; - o.index = oi.index; - o.spentIsConfirmed = 1; + +TransactionDb.prototype._fillOutpoints = function(info, cb) { + var self = this; + + if (!info || info.isCoinBase) return cb(); + + var valueIn = 0; + var incompleteInputs = 0; + + async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) { + self.fromTxIdN(i.txid, i.vout, info.confirmations, function(err, ret) { + //console.log('[TransactionDb.js.154:ret:]',ret); //TODO + if (!ret || !ret.addr || !ret.valueSat) { + console.log('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid); + if (ret) i.unconfirmedInput = ret.unconfirmedInput; + incompleteInputs = 1; + return c_in(); // error not scalated + } + + info.firstSeenTs = ret.spentTs; + i.unconfirmedInput = i.unconfirmedInput; + i.addr = ret.addr; + i.valueSat = ret.valueSat; + i.value = ret.valueSat / util.COIN; + valueIn += i.valueSat; + +/* +* If confirmed by bitcoind, we could not check for double spents +* but we prefer to keep the flag of double spent attempt +* + if (info.confirmations + && info.confirmations >= CONFIRMATION_NR_TO_NOT_CHECK) + return c_in(); +isspent +*/ + // Double spent? + if (ret.multipleSpentAttempt || !ret.spentTxId || + (ret.spentTxId && ret.spentTxId !== info.txid) + ) { + if (ret.multipleSpentAttempts) { + ret.multipleSpentAttempts.each(function(mul) { + if (mul.spentTxId !== info.txid) { + i.doubleSpentTxID = ret.spentTxId; + i.doubleSpentIndex = ret.spentIndex; } - return e_c(); }); - }, cb); + } else if (!ret.spentTxId) { + i.dbError = 'Input spent not registered'; + } else { + i.doubleSpentTxID = ret.spentTxId; + i.doubleSpentIndex = ret.spentIndex; + } + } else { + i.doubleSpentTxID = null; + } + return c_in(); + }); + }, + function() { + if (!incompleteInputs) { + info.valueIn = valueIn / util.COIN; + info.fees = (valueIn - parseInt(info.valueOut * util.COIN)) / util.COIN; } else { - self.isConfirmed(o.spentTxId, function(err, is) { - if (err) return cb(err); - o.spentIsConfirmed = is; - return cb(); - }); + info.incompleteInputs = 1; } + return cb(); }); - }; +}; - TransactionDb.prototype.fromAddr = function(addr, cb) { - var self = this; +TransactionDb.prototype._getInfo = function(txid, next) { + var self = this; - var k = ADDR_PREFIX + addr + '-'; - var ret = []; + Rpc.getTxInfo(txid, function(err, info) { + if (err) return next(err); + self._fillOutpoints(info, function() { + self._fillSpent(info, function() { + return next(null, info); + }); + }); + }); +}; + + +// Simplified / faster Info version: No spent / outpoints info. +TransactionDb.prototype.fromIdInfoSimple = function(txid, cb) { + Rpc.getTxInfo(txid, true, function(err, info) { + if (err) return cb(err); + if (!info) return cb(); + return cb(err, info); + }); +}; + +TransactionDb.prototype.fromIdWithInfo = function(txid, cb) { + var self = this; + + self._getInfo(txid, function(err, info) { + if (err) return cb(err); + if (!info) return cb(); + return cb(err, { + txid: txid, + info: info + }); + }); +}; + +TransactionDb.prototype.fromTxIdN = function(txid, n, confirmations, cb) { + var self = this; + var k = OUTS_PREFIX + txid + '-' + n; + + db.get(k, function(err, val) { + if (!val || (err && err.notFound)) { + return cb(null, { + unconfirmedInput: 1 + }); + } + + var a = val.split(':'); + var ret = { + addr: a[0], + valueSat: parseInt(a[1]), + }; + + /* + * If this TxID comes from an RPC request + * the .confirmations value from bitcoind is available + * so we could avoid checking if the input were double spented + * + * This speed up address calculations by ~30% + * + if (confirmations >= CONFIRMATION_NR_TO_NOT_CHECK) { + return cb(null, ret); + } + */ + + // spent? + var k = SPENT_PREFIX + txid + '-' + n + '-'; db.createReadStream({ start: k, end: k + '~' }) .on('data', function(data) { var k = data.key.split('-'); - var v = data.value.split(':'); - ret.push({ - txid: k[2], - index: parseInt(k[3]), - value_sat: parseInt(v[0]), - ts: parseInt(v[1]), - }); + self._addSpentInfo(ret, k[3], k[4], data.value); }) - .on('error', function(err) { - return cb(err); + .on('error', function(error) { + return cb(error); }) .on('end', function() { - - async.eachLimit(ret, CONCURRENCY, function(o, e_c) { - var k = SPENT_PREFIX + o.txid + '-' + o.index + '-'; - db.createReadStream({ - start: k, - end: k + '~' - }) - .on('data', function(data) { - var k = data.key.split('-'); - self._addSpentInfo(o, k[3], k[4], data.value); - }) - .on('error', function(err) { - return e_c(err); - }) - .on('end', function(err) { - return e_c(err); - }); - }, - function() { - async.eachLimit(ret, CONCURRENCY, function(o, e_c) { - self.fillConfirmations(o, e_c); - }, function(err) { - return cb(err, ret); - }); - }); + return cb(null, ret); }); - }; + }); +}; +TransactionDb.prototype.fillConfirmations = function(o, cb) { + var self = this; - TransactionDb.prototype.removeFromTxId = function(txid, cb) { + self.isConfirmed(o.txid, function(err, is) { + if (err) return cb(err); - async.series([ + o.isConfirmed = is; + if (!o.spentTxId) return cb(); - function(c) { + if (o.multipleSpentAttempts) { + + async.eachLimit(o.multipleSpentAttempts, CONCURRENCY, + function(oi, e_c) { + self.isConfirmed(oi.spentTxId, function(err, is) { + if (err) return; + if (is) { + o.spentTxId = oi.spentTxId; + o.index = oi.index; + o.spentIsConfirmed = 1; + } + return e_c(); + }); + }, cb); + } else { + self.isConfirmed(o.spentTxId, function(err, is) { + if (err) return cb(err); + o.spentIsConfirmed = is; + return cb(); + }); + } + }); +}; + +TransactionDb.prototype.fromAddr = function(addr, cb) { + var self = this; + + var k = ADDR_PREFIX + addr + '-'; + var ret = []; + + db.createReadStream({ + start: k, + end: k + '~' + }) + .on('data', function(data) { + var k = data.key.split('-'); + var v = data.value.split(':'); + ret.push({ + txid: k[2], + index: parseInt(k[3]), + value_sat: parseInt(v[0]), + ts: parseInt(v[1]), + }); + }) + .on('error', function(err) { + return cb(err); + }) + .on('end', function() { + + async.eachLimit(ret, CONCURRENCY, function(o, e_c) { + var k = SPENT_PREFIX + o.txid + '-' + o.index + '-'; db.createReadStream({ - start: OUTS_PREFIX + txid + '-', - end: OUTS_PREFIX + txid + '~', - }).pipe( + start: k, + end: k + '~' + }) + .on('data', function(data) { + var k = data.key.split('-'); + self._addSpentInfo(o, k[3], k[4], data.value); + }) + .on('error', function(err) { + return e_c(err); + }) + .on('end', function(err) { + return e_c(err); + }); + }, + function() { + async.eachLimit(ret, CONCURRENCY, function(o, e_c) { + self.fillConfirmations(o, e_c); + }, function(err) { + return cb(err, ret); + }); + }); + }); +}; + + +TransactionDb.prototype.removeFromTxId = function(txid, cb) { + + async.series([ + + function(c) { + db.createReadStream({ + start: OUTS_PREFIX + txid + '-', + end: OUTS_PREFIX + txid + '~', + }).pipe( + db.createWriteStream({ + type: 'del' + }) + ).on('close', c); + }, + function(c) { + db.createReadStream({ + start: SPENT_PREFIX + txid + '-', + end: SPENT_PREFIX + txid + '~' + }) + .pipe( db.createWriteStream({ type: 'del' }) - ).on('close', c); + ).on('close', c); + } + ], + function(err) { + cb(err); + }); + +}; + + +// TODO. replace with +// Script.prototype.getAddrStrs if that one get merged in bitcore +TransactionDb.prototype.getAddrStr = function(s) { + var self = this; + + var addrStrs = []; + var type = s.classify(); + var addr; + + switch (type) { + case Script.TX_PUBKEY: + var chunk = s.captureOne(); + addr = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk)); + addrStrs.push(addr.toString()); + break; + case Script.TX_PUBKEYHASH: + addr = new Address(self.network.addressPubkey, s.captureOne()); + addrStrs.push(addr.toString()); + break; + case Script.TX_SCRIPTHASH: + addr = new Address(self.network.addressScript, s.captureOne()); + addrStrs.push(addr.toString()); + break; + case Script.TX_MULTISIG: + var chunks = s.capture(); + chunks.forEach(function(chunk) { + var a = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk)); + addrStrs.push(a.toString()); + }); + break; + case Script.TX_UNKNOWN: + break; + } + + return addrStrs; +}; + +TransactionDb.prototype.adaptTxObject = function(txInfo) { + var self = this; + // adapt bitcore TX object to bitcoind JSON response + txInfo.txid = txInfo.hash; + + + var to = 0; + var tx = txInfo; + if (tx.outs) { + tx.outs.forEach(function(o) { + var s = new Script(o.s); + var addrs = self.getAddrStr(s); + + // support only for p2pubkey p2pubkeyhash and p2sh + if (addrs.length === 1) { + tx.out[to].addrStr = addrs[0]; + tx.out[to].n = to; + } + to++; + }); + } + + var count = 0; + txInfo.vin = txInfo. in .map(function(txin) { + var i = {}; + + if (txin.coinbase) { + txInfo.isCoinBase = true; + } else { + i.txid = txin.prev_out.hash; + i.vout = txin.prev_out.n; + } + i.n = count++; + return i; + }); + + + count = 0; + txInfo.vout = txInfo.out.map(function(txout) { + var o = {}; + + o.value = txout.value; + o.n = count++; + + if (txout.addrStr) { + o.scriptPubKey = {}; + o.scriptPubKey.addresses = [txout.addrStr]; + } + return o; + }); +}; + + + +TransactionDb.prototype.add = function(tx, blockhash, cb) { + var self = this; + var addrs = []; + + if (tx.hash) self.adaptTxObject(tx); + + var ts = tx.time; + + async.series([ + // Input Outpoints (mark them as spent) + function(p_c) { + if (tx.isCoinBase) return p_c(); + async.forEachLimit(tx.vin, CONCURRENCY, + function(i, next_out) { + db.batch() + .put(SPENT_PREFIX + i.txid + '-' + i.vout + '-' + tx.txid + '-' + i.n, + ts || 0) + .write(next_out); }, - function(c) { - db.createReadStream({ - start: SPENT_PREFIX + txid + '-', - end: SPENT_PREFIX + txid + '~' - }) - .pipe( - db.createWriteStream({ - type: 'del' - }) - ).on('close', c); - } - ], - function(err) { - cb(err); - }); - - }; - - - // TODO. replace with - // Script.prototype.getAddrStrs if that one get merged in bitcore - TransactionDb.prototype.getAddrStr = function(s) { - var self = this; - - var addrStrs = []; - var type = s.classify(); - var addr; - - switch (type) { - case Script.TX_PUBKEY: - var chunk = s.captureOne(); - addr = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk)); - addrStrs.push(addr.toString()); - break; - case Script.TX_PUBKEYHASH: - addr = new Address(self.network.addressPubkey, s.captureOne()); - addrStrs.push(addr.toString()); - break; - case Script.TX_SCRIPTHASH: - addr = new Address(self.network.addressScript, s.captureOne()); - addrStrs.push(addr.toString()); - break; - case Script.TX_MULTISIG: - var chunks = s.capture(); - chunks.forEach(function(chunk) { - var a = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk)); - addrStrs.push(a.toString()); + function(err) { + return p_c(err); }); - break; - case Script.TX_UNKNOWN: - break; - } + }, + // Parse Outputs + function(p_c) { + async.forEachLimit(tx.vout, CONCURRENCY, + function(o, next_out) { + if (o.value && o.scriptPubKey && + o.scriptPubKey.addresses && + o.scriptPubKey.addresses[0] && !o.scriptPubKey.addresses[1] // TODO : not supported + ) { + var addr = o.scriptPubKey.addresses[0]; + var sat = Math.round(o.value * util.COIN); - return addrStrs; - }; + if (addrs.indexOf(addr) === -1) { + addrs.push(addr); + } - TransactionDb.prototype.adaptTxObject = function(txInfo) { - var self = this; - // adapt bitcore TX object to bitcoind JSON response - txInfo.txid = txInfo.hash; - - - var to = 0; - var tx = txInfo; - if (tx.outs) { - tx.outs.forEach(function(o) { - var s = new Script(o.s); - var addrs = self.getAddrStr(s); - - // support only for p2pubkey p2pubkeyhash and p2sh - if (addrs.length === 1) { - tx.out[to].addrStr = addrs[0]; - tx.out[to].n = to; - } - to++; - }); - } - - var count = 0; - txInfo.vin = txInfo. in .map(function(txin) { - var i = {}; - - if (txin.coinbase) { - txInfo.isCoinBase = true; - } else { - i.txid = txin.prev_out.hash; - i.vout = txin.prev_out.n; - } - i.n = count++; - return i; - }); - - - count = 0; - txInfo.vout = txInfo.out.map(function(txout) { - var o = {}; - - o.value = txout.value; - o.n = count++; - - if (txout.addrStr) { - o.scriptPubKey = {}; - o.scriptPubKey.addresses = [txout.addrStr]; - } - return o; - }); - }; - - - - TransactionDb.prototype.add = function(tx, blockhash, cb) { - var self = this; - var addrs = []; - - if (tx.hash) self.adaptTxObject(tx); - - var ts = tx.time; - - async.series([ - // Input Outpoints (mark them as spent) - function(p_c) { - if (tx.isCoinBase) return p_c(); - async.forEachLimit(tx.vin, CONCURRENCY, - function(i, next_out) { - db.batch() - .put(SPENT_PREFIX + i.txid + '-' + i.vout + '-' + tx.txid + '-' + i.n, - ts || 0) - .write(next_out); - }, - function(err) { - return p_c(err); - }); - }, - // Parse Outputs - function(p_c) { - async.forEachLimit(tx.vout, CONCURRENCY, - function(o, next_out) { - if (o.value && o.scriptPubKey && - o.scriptPubKey.addresses && - o.scriptPubKey.addresses[0] && !o.scriptPubKey.addresses[1] // TODO : not supported - ) { - var addr = o.scriptPubKey.addresses[0]; - var sat = Math.round(o.value * util.COIN); - - if (addrs.indexOf(addr) === -1) { - addrs.push(addr); + // existed? + var k = OUTS_PREFIX + tx.txid + '-' + o.n; + db.get(k, function(err) { + if (err && err.notFound) { + db.batch() + .put(k, addr + ':' + sat) + .put(ADDR_PREFIX + addr + '-' + tx.txid + '-' + o.n, sat + ':' + ts) + .write(next_out); + } else { + return next_out(); } - - // existed? - var k = OUTS_PREFIX + tx.txid + '-' + o.n; - db.get(k, function(err) { - if (err && err.notFound) { - db.batch() - .put(k, addr + ':' + sat) - .put(ADDR_PREFIX + addr + '-' + tx.txid + '-' + o.n, sat + ':' + ts) - .write(next_out); - } else { - return next_out(); - } - }); - } else { - return next_out(); - } - }, - function(err) { - if (err) { - console.log('ERR at TX %s: %s', tx.txid, err); - return cb(err); - } - return p_c(); - }); - }, - function(p_c) { - if (!blockhash) { + }); + } else { + return next_out(); + } + }, + function(err) { + if (err) { + console.log('ERR at TX %s: %s', tx.txid, err); + return cb(err); + } return p_c(); - } - return self.setConfirmation(tx.txid, blockhash, true, p_c); - }, - ], function(err) { - if (addrs.length > 0 && !blockhash) { - // only emit if we are processing a single tx (not from a block) - addrs.forEach(function(addr) { - self.emit('tx_for_address', { - address: addr, - txid: tx.txid - }); }); + }, + function(p_c) { + if (!blockhash) { + return p_c(); } - self.emit('new_tx', { - tx: tx - }); - - return cb(err); - }); - }; - - - - TransactionDb.prototype.setConfirmation = function(txId, blockHash, confirmed, c) { - if (!blockHash) return c(); - - confirmed = confirmed ? 1 : 0; - - db.batch() - .put(IN_BLK_PREFIX + txId + '-' + blockHash, confirmed) - .put(FROM_BLK_PREFIX + blockHash + '-' + txId, 1) - .write(c); - }; - - - // This slowdown addr balance calculation by 100% - TransactionDb.prototype.isConfirmed = function(txId, c) { - var k = IN_BLK_PREFIX + txId; - var ret = false; - - db.createReadStream({ - start: k, - end: k + '~' - }) - .on('data', function(data) { - if (data.value === '1') ret = true; - }) - .on('error', function(err) { - return c(err); - }) - .on('end', function(err) { - return c(err, ret); - }); - }; - - TransactionDb.prototype.handleBlockChange = function(hash, isMain, cb) { - var toChange = []; - console.log('\tSearching Txs from block:' + hash); - - var k = FROM_BLK_PREFIX + hash; - var k2 = IN_BLK_PREFIX; - // This is slow, but prevent us to create a new block->tx index. - db.createReadStream({ - start: k, - end: k + '~' - }) - .on('data', function(data) { - var ks = data.key.split('-'); - toChange.push({ - key: k2 + ks[2] + '-' + ks[1], - type: 'put', - value: isMain ? 1 : 0, + return self.setConfirmation(tx.txid, blockhash, true, p_c); + }, + ], function(err) { + if (addrs.length > 0 && !blockhash) { + // only emit if we are processing a single tx (not from a block) + addrs.forEach(function(addr) { + self.emit('tx_for_address', { + address: addr, + txid: tx.txid }); - }) - .on('error', function(err) { - return cb(err); - }) - .on('end', function(err) { - if (err) return cb(err); - console.log('\t%s %d Txs', isMain ? 'Confirming' : 'Invalidating', toChange.length); - db.batch(toChange, cb); }); - }; + } + self.emit('new_tx', { + tx: tx + }); - // txs can be a [hashes] or [txObjects] - TransactionDb.prototype.createFromArray = function(txs, blockHash, next) { - var self = this; - if (!txs) return next(); + return cb(err); + }); +}; - async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) { - if (typeof t === 'string') { - // TODO: parse it from networks.genesisTX? - if (t === genesisTXID) return each_cb(); - Rpc.getTxInfo(t, function(err, inInfo) { - if (!inInfo) return each_cb(err); - return self.add(inInfo, blockHash, each_cb); - }); - } else { - return self.add(t, blockHash, each_cb); - } - }, - function(err) { - return next(err); +TransactionDb.prototype.setConfirmation = function(txId, blockHash, confirmed, c) { + if (!blockHash) return c(); + + confirmed = confirmed ? 1 : 0; + + db.batch() + .put(IN_BLK_PREFIX + txId + '-' + blockHash, confirmed) + .put(FROM_BLK_PREFIX + blockHash + '-' + txId, 1) + .write(c); +}; + + +// This slowdown addr balance calculation by 100% +TransactionDb.prototype.isConfirmed = function(txId, c) { + var k = IN_BLK_PREFIX + txId; + var ret = false; + + db.createReadStream({ + start: k, + end: k + '~' + }) + .on('data', function(data) { + if (data.value === '1') ret = true; + }) + .on('error', function(err) { + return c(err); + }) + .on('end', function(err) { + return c(err, ret); + }); +}; + +TransactionDb.prototype.handleBlockChange = function(hash, isMain, cb) { + var toChange = []; + console.log('\tSearching Txs from block:' + hash); + + var k = FROM_BLK_PREFIX + hash; + var k2 = IN_BLK_PREFIX; + // This is slow, but prevent us to create a new block->tx index. + db.createReadStream({ + start: k, + end: k + '~' + }) + .on('data', function(data) { + var ks = data.key.split('-'); + toChange.push({ + key: k2 + ks[2] + '-' + ks[1], + type: 'put', + value: isMain ? 1 : 0, }); - }; + }) + .on('error', function(err) { + return cb(err); + }) + .on('end', function(err) { + if (err) return cb(err); + console.log('\t%s %d Txs', isMain ? 'Confirming' : 'Invalidating', toChange.length); + db.batch(toChange, cb); + }); +}; + +// txs can be a [hashes] or [txObjects] +TransactionDb.prototype.createFromArray = function(txs, blockHash, next) { + var self = this; + if (!txs) return next(); + + async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) { + if (typeof t === 'string') { + // TODO: parse it from networks.genesisTX? + if (t === genesisTXID) return each_cb(); + + Rpc.getTxInfo(t, function(err, inInfo) { + if (!inInfo) return each_cb(err); + + return self.add(inInfo, blockHash, each_cb); + }); + } else { + return self.add(t, blockHash, each_cb); + } + }, + function(err) { + return next(err); + }); +}; - TransactionDb.prototype.createFromBlock = function(b, next) { - var self = this; - if (!b || !b.tx) return next(); +TransactionDb.prototype.createFromBlock = function(b, next) { + var self = this; + if (!b || !b.tx) return next(); - return self.createFromArray(b.tx, b.hash, next); - }; + return self.createFromArray(b.tx, b.hash, next); +}; - return TransactionDb; -} -module.defineClass(spec); +module.exports = require('soop')(TransactionDb); diff --git a/package.json b/package.json index 4df57439..4188f83c 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "leveldown": "*", "levelup": "*", "glob": "*", - "classtool": "*", + "soop": "git://github.com/gasteve/node-soop.git", "commander": "*", "bignum": "*", "express": "~3.4.7", diff --git a/test/integration/02-transactionouts.js b/test/integration/02-transactionouts.js index 5b18fd2a..e61635fd 100644 --- a/test/integration/02-transactionouts.js +++ b/test/integration/02-transactionouts.js @@ -11,7 +11,7 @@ var util = require('util'), async = require('async'), config = require('../../config/config'), - TransactionDb = require('../../lib/TransactionDb').class(); + TransactionDb = require('../../lib/TransactionDb').default(); var spentValid = JSON.parse(fs.readFileSync('test/integration/spent.json')); From 040ea6bfb4dcadf34c8787e69fc3f3e877a729b3 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 10 Mar 2014 12:46:48 -0300 Subject: [PATCH 2/4] adapt insight-api to work with bitcores soop version --- app/controllers/blocks.js | 2 +- app/controllers/transactions.js | 6 ++---- dev-util/stats | 6 +++++- insight.js | 4 ++-- lib/BlockDb.js | 8 +++----- lib/HistoricSync.js | 5 ++--- lib/PeerSync.js | 2 +- lib/Sync.js | 8 +++----- lib/TransactionDb.js | 6 +++--- util/p2p.js | 2 +- util/sync.js | 2 +- 11 files changed, 24 insertions(+), 27 deletions(-) diff --git a/app/controllers/blocks.js b/app/controllers/blocks.js index def4e359..984fef82 100644 --- a/app/controllers/blocks.js +++ b/app/controllers/blocks.js @@ -5,7 +5,7 @@ */ var common = require('./common'), async = require('async'), - BlockDb = require('../../lib/BlockDb').class(); + BlockDb = require('../../lib/BlockDb'); var bdb = new BlockDb(); diff --git a/app/controllers/transactions.js b/app/controllers/transactions.js index 2f501ead..6179244c 100644 --- a/app/controllers/transactions.js +++ b/app/controllers/transactions.js @@ -8,11 +8,9 @@ var async = require('async'); var common = require('./common'); var Rpc = require('../../lib/Rpc'); -var TransactionDb = require('../../lib/TransactionDb').default(); -var BlockDb = require('../../lib/BlockDb').default(); -var tDb = new TransactionDb(); -var bdb = new BlockDb(); +var tDb = require('../../lib/TransactionDb').default(); +var bdb = require('../../lib/BlockDb').default(); exports.send = function(req, res) { Rpc.sendRawTransaction(req.body.rawtx, function(err, txid) { diff --git a/dev-util/stats b/dev-util/stats index f68bf9e1..142758ff 100644 --- a/dev-util/stats +++ b/dev-util/stats @@ -41,7 +41,7 @@ first 10% => sacando los contenidos adentro de getblock from file de => 4.5s!! => con base58 cpp => 21s - => toda la testnet => 17m + => toda la testnet => 17m !! 10% de blk2 => 50s con base58cpp @@ -54,3 +54,7 @@ first 10% => 15s comentando desde b.getStandardizedObject() => 39s comentando dps b.getStandardizedObject() + +Mon Mar 10 11:59:25 ART 2014 +10% de blk 0 (testnet) + => 37s diff --git a/insight.js b/insight.js index 040df23b..1cb74c08 100644 --- a/insight.js +++ b/insight.js @@ -9,8 +9,8 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; */ var express = require('express'), fs = require('fs'), - PeerSync = require('./lib/PeerSync').class(), - HistoricSync = require('./lib/HistoricSync').class(); + PeerSync = require('./lib/PeerSync'), + HistoricSync = require('./lib/HistoricSync'); //Initializing system variables var config = require('./config/config'); diff --git a/lib/BlockDb.js b/lib/BlockDb.js index e0ef4d12..bab3d53f 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -1,6 +1,6 @@ 'use strict'; -var imports = require('soop').imports(); -var parent = imports.parent || require('events').EventEmitter; +var imports = require('soop').imports(); +var ThisParent = imports.parent || require('events').EventEmitter; var TIMESTAMP_PREFIX = 'bts-'; // b-ts- => var PREV_PREFIX = 'bpr-'; // b-prev- => var NEXT_PREFIX = 'bne-'; // b-next- => @@ -26,8 +26,7 @@ var BlockDb = function() { BlockDb.super(this, arguments); this.poolMatch = new PoolMatch(); }; - -BlockDb.parent = parent; +BlockDb.parent = ThisParent; BlockDb.prototype.close = function(cb) { db.close(cb); @@ -57,7 +56,6 @@ BlockDb.prototype.add = function(b, cb) { .put(PREV_PREFIX + b.hash, b.previousblockhash) .write(function(err){ if (!err) { - self.emit('new_block', {blockid: b.hash}); } cb(err); }); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 9f732bcf..ef80c239 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -1,14 +1,13 @@ 'use strict'; var imports = require('soop').imports(); - var util = require('util'); var assert = require('assert'); +var async = require('async'); var RpcClient = require('bitcore/RpcClient'); var Script = require('bitcore/Script'); var networks = require('bitcore/networks'); -var async = require('async'); -var config = require('../config/config'); +var config = imports.config || require('../config/config'); var Sync = require('./Sync'); var sockets = require('../app/controllers/socket.js'); var BlockExtractor = require('./BlockExtractor.js'); diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 16678438..50e678f1 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -12,7 +12,7 @@ function PeerSync(opts) { this.connected = false; this.peerdb = undefined; this.allowReorgs = false; - this.PeerManager = require('bitcore/PeerManager').createClass({ + this.PeerManager = require('soop').load('../node_modules/bitcore/PeerManager',{ network: (config.network === 'testnet' ? networks.testnet : networks.livenet) }); this.peerman = new this.PeerManager(); diff --git a/lib/Sync.js b/lib/Sync.js index 568a6a0f..933d9d7a 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -3,17 +3,15 @@ var imports = require('soop').imports(); var sockets = require('../app/controllers/socket.js'); -var BlockDb = require('./BlockDb').default(); -var TransactionDb = require('./TransactionDb').default(); -var config = require('../config/config'); +var config = imports.config || require('../config/config'); var networks = require('bitcore/networks'); var async = require('async'); function Sync(opts) { this.opts = opts || {}; - this.bDb = new BlockDb(opts); - this.txDb = new TransactionDb(opts); + this.bDb = require('./BlockDb').default(); + this.txDb = require('./TransactionDb').default(); this.txDb.on('tx_for_address', this.handleTxForAddress.bind(this)); this.txDb.on('new_tx', this.handleNewTx.bind(this)); this.bDb.on('new_block', this.handleNewBlock.bind(this)); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index dfb4fd91..42d64fcd 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -2,7 +2,7 @@ var imports = require('soop').imports(); -var parent = imports.parent || require('events').EventEmitter; +var ThisParent = imports.parent || require('events').EventEmitter; // blockHash -> txid mapping var IN_BLK_PREFIX = 'txb-'; //txb-- => 1/0 (connected or not) @@ -39,7 +39,7 @@ var encodedData = require('soop').load('bitcore/util/EncodedData',{ base58: base58 }); var versionedData= require('soop').load('bitcore/util/VersionedData',{ - patent: encodedData + parent: encodedData }); var Address = require('soop').load('bitcore/Address',{ parent: versionedData @@ -51,7 +51,7 @@ var TransactionDb = function() { TransactionDb.super(this, arguments); this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; }; -TransactionDb.parent = parent; +TransactionDb.parent = ThisParent; TransactionDb.prototype.close = function(cb) { db.close(cb); diff --git a/util/p2p.js b/util/p2p.js index c7a11b5e..0d1d885b 100755 --- a/util/p2p.js +++ b/util/p2p.js @@ -3,7 +3,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var PeerSync = require('../lib/PeerSync').class(); +var PeerSync = require('../lib/PeerSync'); var PROGRAM_VERSION = '0.1'; var program = require('commander'); diff --git a/util/sync.js b/util/sync.js index ee29e65a..e3fe31d4 100755 --- a/util/sync.js +++ b/util/sync.js @@ -7,7 +7,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var SYNC_VERSION = '0.1'; var program = require('commander'); -var HistoricSync = require('../lib/HistoricSync').class(); +var HistoricSync = require('../lib/HistoricSync'); var async = require('async'); program From 645da13bee9d5802c12cdfa4eb5f812dfc8392de Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 10 Mar 2014 13:03:59 -0300 Subject: [PATCH 3/4] update package json --- package.json | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 4188f83c..e2f7ce5a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "insight-bitcore-api", "description": "An open-source bitcoin blockchain API. The Insight API provides you with a convenient, powerful and simple way to query and broadcast data on the bitcoin network and build your own services with it.", - "version": "0.1.3", + "version": "0.1.4", "author": { "name": "Ryan X Charles", "email": "ryan@bitpay.com" @@ -36,13 +36,11 @@ "license": "MIT", "keywords": [ "insight", - "secret", - "enigma", - "riddle", - "mystification", - "puzzle", - "conundrum", - "api", + "insight api", + "blockchain", + "bitcoin api", + "blockchain api", + "json", "bitcore" ], "engines": { @@ -52,6 +50,7 @@ "start": "node node_modules/grunt-cli/bin/grunt" }, "dependencies": { + "bitcore": "~0.1.6", "base58-native": "0.1.2", "async": "*", "leveldown": "*", @@ -67,7 +66,6 @@ "moment": "~2.5.0", "sinon": "~1.7.3", "chai": "~1.8.1", - "bitcore": "git://github.com/bitpay/bitcore.git", "bufferput": "git://github.com/bitpay/node-bufferput.git", "xmlhttprequest": "~1.6.0" }, From ae4769555b78e15034e7a7b7fddf03a626ea62b4 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 10 Mar 2014 15:06:10 -0300 Subject: [PATCH 4/4] support for new bitcore version ~0.1.6 --- app/controllers/status.js | 2 +- app/models/Address.js | 8 ++++---- app/models/Status.js | 3 +-- lib/BlockDb.js | 6 +++--- test/integration/01-transactionouts.js | 6 +++--- test/integration/02-transactionouts.js | 2 +- test/integration/addr.js | 6 +++--- test/integration/block.js | 4 ++-- test/integration/blockExtractor.js | 2 +- test/integration/blocklist.js | 4 ++-- test/integration/nodecheck.js | 4 ++-- test/integration/status.js | 2 +- test/lib/PeerSync.js | 2 +- 13 files changed, 25 insertions(+), 26 deletions(-) diff --git a/app/controllers/status.js b/app/controllers/status.js index 0d9194bb..c64eb6aa 100644 --- a/app/controllers/status.js +++ b/app/controllers/status.js @@ -17,7 +17,7 @@ exports.show = function(req, res) { } else { var option = req.query.q; - var statusObject = Status.new(); + var statusObject = new Status(); var returnJsonp = function (err) { if (err || ! statusObject) diff --git a/app/models/Address.js b/app/models/Address.js index f67ab1ca..aff16114 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -1,13 +1,13 @@ 'use strict'; -//var imports = require('soop').imports(); +var imports = require('soop').imports(); var async = require('async'); var BitcoreAddress = require('bitcore/Address'); var BitcoreTransaction = require('bitcore/Transaction'); var BitcoreUtil = require('bitcore/util/util'); var Parser = require('bitcore/util/BinaryParser'); var Buffer = require('buffer').Buffer; -var TransactionDb = require('../../lib/TransactionDb').default(); +var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb').default(); var CONCURRENCY = 5; function Address(addrStr) { @@ -84,7 +84,7 @@ Address.prototype.getUtxo = function(next) { if (!self.addrStr) return next(); var ret = []; - var db = new TransactionDb(); + var db = TransactionDb; db.fromAddr(self.addrStr, function(err,txOut){ if (err) return next(err); @@ -121,7 +121,7 @@ Address.prototype.update = function(next) { if (!self.addrStr) return next(); var txs = []; - var db = new TransactionDb(); + var db = TransactionDb; async.series([ function (cb) { var seen={}; diff --git a/app/models/Status.js b/app/models/Status.js index 548974f3..39e52476 100644 --- a/app/models/Status.js +++ b/app/models/Status.js @@ -3,12 +3,11 @@ var async = require('async'); var RpcClient = require('bitcore/RpcClient'); -var BlockDb = require('../../lib/BlockDb'); var config = require('../../config/config'); var rpc = new RpcClient(config.bitcoind); function Status() { - this.bDb = new BlockDb(); + this.bDb = require('../../lib/BlockDb').default(); } Status.prototype.getInfo = function(next) { diff --git a/lib/BlockDb.js b/lib/BlockDb.js index bab3d53f..0da84e30 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -20,7 +20,7 @@ var db = imports.db || levelup(config.leveldb + '/blocks',{maxOpenFiles: MAX_OP var Rpc = imports.rpc || require('./Rpc'); var PoolMatch = imports.poolMatch || require('soop').load('./PoolMatch',config); -var TransactionDb = require('./TransactionDb.js').default(); +var tDb = require('./TransactionDb.js').default(); var BlockDb = function() { BlockDb.super(this, arguments); @@ -56,6 +56,7 @@ BlockDb.prototype.add = function(b, cb) { .put(PREV_PREFIX + b.hash, b.previousblockhash) .write(function(err){ if (!err) { + self.emit('new_block', {blockid: b.hash}); } cb(err); }); @@ -160,10 +161,9 @@ BlockDb.prototype.has = function(hash, cb) { }; BlockDb.prototype.getPoolInfo = function(tx, cb) { - var tr = new TransactionDb(); var self = this; - tr._getInfo(tx, function(e, a) { + tDb._getInfo(tx, function(e, a) { if (e) return cb(false); if (a.isCoinBase) { diff --git a/test/integration/01-transactionouts.js b/test/integration/01-transactionouts.js index 0fc7e0ff..bb41fd25 100644 --- a/test/integration/01-transactionouts.js +++ b/test/integration/01-transactionouts.js @@ -8,7 +8,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var assert = require('assert'), fs = require('fs'), util = require('util'), - TransactionDb = require('../../lib/TransactionDb').class(); + TransactionDb = require('../../lib/TransactionDb').default(); var txItemsValid = JSON.parse(fs.readFileSync('test/integration/txitems.json')); var txDb; @@ -16,7 +16,7 @@ var txDb; describe('TransactionDb fromIdWithInfo', function(){ before(function(c) { - txDb = new TransactionDb(); + txDb = TransactionDb; return c(); }); @@ -118,7 +118,7 @@ describe('TransactionDb fromIdWithInfo', function(){ describe('TransactionDb Outs', function(){ before(function(c) { - txDb = new TransactionDb(); + txDb = TransactionDb; return c(); }); diff --git a/test/integration/02-transactionouts.js b/test/integration/02-transactionouts.js index e61635fd..6dc1f851 100644 --- a/test/integration/02-transactionouts.js +++ b/test/integration/02-transactionouts.js @@ -20,7 +20,7 @@ var txDb; describe('TransactionDb Expenses', function(){ before(function(c) { - txDb = new TransactionDb(); + txDb = TransactionDb; // lets spend! async.each(Object.keys(spentValid), diff --git a/test/integration/addr.js b/test/integration/addr.js index fecc7360..2bce598b 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -6,8 +6,8 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var assert = require('assert'), fs = require('fs'), - Address = require('../../app/models/Address').class(), - TransactionDb = require('../../lib/TransactionDb').class(), + Address = require('../../app/models/Address'), + TransactionDb = require('../../lib/TransactionDb').default(), addrValid = JSON.parse(fs.readFileSync('test/integration/addr.json')), utxoValid = JSON.parse(fs.readFileSync('test/integration/utxo.json')); @@ -15,7 +15,7 @@ var txDb; describe('Address balances', function() { before(function(c) { - txDb = new TransactionDb(); + txDb = TransactionDb; return c(); }); diff --git a/test/integration/block.js b/test/integration/block.js index 037f7094..0d6e5e4e 100644 --- a/test/integration/block.js +++ b/test/integration/block.js @@ -10,14 +10,14 @@ var TESTING_BLOCK = '000000000185678d3d7ecc9962c96418174431f93fe20bf216d55652724 var assert = require('assert'), // config = require('../../config/config'), - BlockDb = require('../../lib/BlockDb').class(); + BlockDb = require('../../lib/BlockDb').default(); var bDb; describe('BlockDb fromHashWithInfo', function() { before(function(c) { - bDb = new BlockDb(); + bDb = BlockDb; return c(); }); diff --git a/test/integration/blockExtractor.js b/test/integration/blockExtractor.js index 13c7e8d2..0e222f83 100644 --- a/test/integration/blockExtractor.js +++ b/test/integration/blockExtractor.js @@ -7,7 +7,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var assert = require('assert'), config = require('../../config/config'), - BlockExtractor = require('../../lib/BlockExtractor').class(), + BlockExtractor = require('../../lib/BlockExtractor'), networks = require('bitcore/networks'), util = require('bitcore/util/util'); diff --git a/test/integration/blocklist.js b/test/integration/blocklist.js index 628ef0d9..6491c89c 100644 --- a/test/integration/blocklist.js +++ b/test/integration/blocklist.js @@ -9,7 +9,7 @@ var START_TS = 1; var END_TS = '1296688928~'; // 2/2/2011 23:23PM var assert = require('assert'), - BlockDb = require('../../lib/BlockDb').class(); + BlockDb = require('../../lib/BlockDb').default(); var bDb; @@ -17,7 +17,7 @@ describe('BlockDb getBlocksByDate', function(){ before(function(c) { - bDb = new BlockDb(); + bDb = BlockDb; return c(); }); diff --git a/test/integration/nodecheck.js b/test/integration/nodecheck.js index 1cfef438..2e2973cd 100644 --- a/test/integration/nodecheck.js +++ b/test/integration/nodecheck.js @@ -1,8 +1,8 @@ 'use strict'; -var BlockDb = require('../../lib/BlockDb').class(); +var BlockDb = require('../../lib/BlockDb').default(); var height_needed = 180000; -var bDb = new BlockDb(); +var bDb = BlockDb; var expect = require('chai').expect; diff --git a/test/integration/status.js b/test/integration/status.js index 82dea3a4..d912fa02 100644 --- a/test/integration/status.js +++ b/test/integration/status.js @@ -4,7 +4,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var assert = require('assert'), - Status = require('../../app/models/Status').class(); + Status = require('../../app/models/Status'); describe('Status', function(){ diff --git a/test/lib/PeerSync.js b/test/lib/PeerSync.js index c4dc9734..092c9bae 100644 --- a/test/lib/PeerSync.js +++ b/test/lib/PeerSync.js @@ -3,7 +3,7 @@ var chai = require('chai'), expect = chai.expect, sinon = require('sinon'); -var PeerSync = require('../../lib/PeerSync.js').class(); +var PeerSync = require('../../lib/PeerSync.js'); describe('PeerSync', function() { var ps;