From fc509f6e7e74b2ab060215f6ced87811e4a72bd0 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 3 Feb 2014 14:53:37 -0300 Subject: [PATCH 01/25] remove recursion from HistoricSync for speed --- lib/HistoricSync.js | 94 ++++++++++++++++++++++++--------------------- util/sync.js | 3 +- 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 3ab8f60..9d22e1b 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -260,6 +260,7 @@ function spec() { }; + HistoricSync.prototype.getBlockFromFile = function(height, scanOpts, cb) { var self = this; @@ -288,6 +289,7 @@ function spec() { }, //get Info function(c) { + self.blockExtractor.getNextBlock(function(err, b) { if (err || ! b) return c(err); @@ -348,16 +350,12 @@ function spec() { else { // Continue - if (blockInfo) { - - // mainchain - if (isMainChain) height++; - + if (blockInfo && blockInfo.hash) { self.syncedBlocks++; self.err = null; self.status = 'syncing'; - return self.getBlockFromFile(height, scanOpts, cb); + return cb(null, true, isMainChain); } else { self.err = null; @@ -421,44 +419,16 @@ function spec() { } }, ], function(err) { - var start, end; - function sync() { - if (scanOpts.reverse) { - start = lastBlock; - end = self.genesis; - scanOpts.prev = true; - } - else { - start = self.genesis; - end = null; - scanOpts.next = true; - } - p('Starting from: ', start); - p(' to : ', end); - p(' scanOpts: ', JSON.stringify(scanOpts)); - - if (scanOpts.fromFiles) { - self.getBlockFromFile(0, scanOpts, function(err) { - return next(err); - }); - } - else { - self.getPrevNextBlock(start, end, scanOpts, function(err) { - if (err && err.message.match(/ECONNREFUSED/)) { - setTimeout(function() { - p('Retrying in %d secs', retry_secs); - sync(); - }, - retry_secs * 1000); - } - else return next(err); - }); - } + if (err) { + self.setError(err); + return next(err, 0); } - if (!self.step) { + // SETUP Sync params + var start, end; + if (!self.step) { var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000); if (self.opts.progressStep) { @@ -469,13 +439,49 @@ function spec() { self.step = step; } - if (err) { - self.setError(err); - return next(err, 0); + if (scanOpts.reverse) { + start = lastBlock; + end = self.genesis; + scanOpts.prev = true; } else { - sync(); + start = self.genesis; + end = null; + scanOpts.next = true; } + p('Starting from: ', start); + p(' to : ', end); + p(' scanOpts: ', JSON.stringify(scanOpts)); + + if (scanOpts.fromFiles) { + + var keepGoing = true; + var height = 0; + + async.whilst(function() { + return keepGoing; + }, function (w_cb) { + self.getBlockFromFile(height, scanOpts, function(err, inKeepGoing, wasMainChain) { + keepGoing = inKeepGoing; + if (wasMainChain) height++; + //Black magic from http://stackoverflow.com/questions/20936486/node-js-maximum-call-stack-size-exceeded + setImmediate(function(){ + + return w_cb(err); + }); + }) + }, function(err) { + +console.log('[HistoricSync.js.468]'); //TODO + return next(); + }); + } + else { + self.getPrevNextBlock(start, end, scanOpts, function(err) { + return next(err); + }); + } + }); }; diff --git a/util/sync.js b/util/sync.js index 3c1af7d..7a9a094 100755 --- a/util/sync.js +++ b/util/sync.js @@ -1,4 +1,5 @@ -#! /usr/bin/env node +#!/usr/bin/env node + 'use strict'; From 5734890f104345ec1281ac604fb66939c5376c5d Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 3 Feb 2014 23:30:46 -0300 Subject: [PATCH 02/25] sync with levelDb working! --- app/models/Address.js | 51 ------- app/models/Block.js | 186 ----------------------- app/models/Transaction.js | 16 +- app/models/TransactionOut.js | 270 --------------------------------- config/config.js | 1 + dev-util/block-level.js | 25 ++++ lib/BlockDb.js | 123 +++++++++++++++ lib/HistoricSync.js | 103 +++++-------- lib/Sync.js | 41 +++-- lib/TransactionDb.js | 282 +++++++++++++++++++++++++++++++++++ package.json | 2 + 11 files changed, 507 insertions(+), 593 deletions(-) delete mode 100644 app/models/Block.js delete mode 100644 app/models/TransactionOut.js create mode 100755 dev-util/block-level.js create mode 100644 lib/BlockDb.js create mode 100644 lib/TransactionDb.js diff --git a/app/models/Address.js b/app/models/Address.js index fd0f8d0..be918b3 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -102,54 +102,3 @@ function spec() { } module.defineClass(spec); - -/** - * Addr Schema Idea for moogose. Not used now. - * -var AddressSchema = new Schema({ - - // For now we keep this as short as possible - // More fields will be propably added as we move - // forward with the UX - addr: { - type: String, - index: true, - unique: true, - }, - inputs: [{ - type: mongoose.Schema.Types.ObjectId, - ref: 'TransactionItem' //Edit: I'd put the schema. Silly me. - }], - output: [{ - type: mongoose.Schema.Types.ObjectId, - ref: 'TransactionItem' //Edit: I'd put the schema. Silly me. - }], -}); - - -AddressSchema.statics.load = function(id, cb) { - this.findOne({ - _id: id - }).exec(cb); -}; - - -AddressSchema.statics.fromAddr = function(hash, cb) { - this.findOne({ - hash: hash, - }).exec(cb); -}; - - -AddressSchema.statics.fromAddrWithInfo = function(hash, cb) { - this.fromHash(hash, function(err, addr) { - if (err) return cb(err); - if (!addr) { return cb(new Error('Addr not found')); } -// TODO -// addr.getInfo(function(err) { return cb(err,addr); } ); - }); -}; - -module.exports = mongoose.model('Address', AddressSchema); -*/ - diff --git a/app/models/Block.js b/app/models/Block.js deleted file mode 100644 index 8a01ca5..0000000 --- a/app/models/Block.js +++ /dev/null @@ -1,186 +0,0 @@ -'use strict'; - -/** - * Module dependencies. - */ -var mongoose = require('mongoose'), - Schema = mongoose.Schema, - RpcClient = require('bitcore/RpcClient').class(), - util = require('bitcore/util/util'), - async = require('async'), - BitcoreBlock= require('bitcore/Block').class(), - TransactionOut = require('./TransactionOut'), - config = require('../../config/config') - ; - -/** - * Block Schema - */ -var BlockSchema = new Schema({ - - // For now we keep this as short as possible - // More fields will be propably added as we move - // forward with the UX - _id: { - type: Buffer, - index: true, - unique: true, - required: true, - }, - time: Number, - nextBlockHash: Buffer, - isOrphan: Boolean, -}); - -BlockSchema.virtual('hash').get(function () { - return this._id; -}); - - -BlockSchema.virtual('hash').set(function (hash) { - this._id = hash; -}); - - -BlockSchema.virtual('hashStr').get(function () { - return this._id.toString('hex'); -}); - - -BlockSchema.virtual('hashStr').set(function (hashStr) { - if (hashStr) - this._id = new Buffer(hashStr,'hex'); - else - this._id = null; -}); - - - -BlockSchema.virtual('nextBlockHashStr').get(function () { - return this.nextBlockHash.toString('hex'); -}); - -BlockSchema.virtual('nextBlockHashStr').set(function (hashStr) { - if (hashStr) - this.nextBlockHash = new Buffer(hashStr,'hex'); - else - this.nextBlockHash = null; -}); - -/* -BlockSchema.path('title').validate(function(title) { - return title.length; -},'Title cannot be blank'); -*/ - -/** - * Statics - */ - -BlockSchema.statics.customCreate = function(block, cb) { - var Self= this; - - var BlockSchema = mongoose.model('Block', BlockSchema); - - var newBlock = new Self(); - - newBlock.time = block.time ? block.time : Math.round(new Date().getTime() / 1000); - newBlock.hashStr = block.hash; - newBlock.isOrphan = block.isOrphan; - newBlock.nextBlockHashStr = block.nextBlockHash; - - var insertedTxs, updateAddrs; - - async.series([ - function(a_cb) { - TransactionOut.createFromTxs(block.tx, block.isOrphan, - function(err, inInsertedTxs, inUpdateAddrs) { - insertedTxs = inInsertedTxs; - updateAddrs = inUpdateAddrs; - return a_cb(err); - }); - }, function(a_cb) { - newBlock.save(function(err) { - return a_cb(err); - }); - }], - function (err) { - return cb(err, newBlock, insertedTxs, updateAddrs); - }); -}; - - -BlockSchema.statics.blockIndex = function(height, cb) { - var rpc = new RpcClient(config.bitcoind); - var hashStr = {}; - rpc.getBlockHash(height, function(err, bh){ - if (err) return cb(err); - hashStr.blockHash = bh.result; - cb(null, hashStr); - }); -}; - -BlockSchema.statics.fromHash = function(hashStr, cb) { - var hash = new Buffer(hashStr, 'hex'); - - this.findOne({ - _id: hash, - }).exec(cb); -}; - - -BlockSchema.statics.fromHashWithInfo = function(hashStr, cb) { - var That = this; - - That.fromHash(hashStr, function(err, block) { - if (err) return cb(err); - - if (!block) { - // No in mongo...but maybe in bitcoind... lets query it - block = new That(); - - block.hashStr = hashStr; - block.getInfo(function(err, blockInfo) { - if (err) return cb(err); - if (!blockInfo) return cb(); - - block.save(function(err) { - return cb(err,block); - }); - }); - } - else { - block.getInfo(function(err) { - return cb(err,block); - }); - } - }); -}; - -// TODO: Can we store the rpc instance in the Block object? -BlockSchema.methods.getInfo = function (next) { - - var self = this; - var rpc = new RpcClient(config.bitcoind); - - rpc.getBlock(self.hashStr, function(err, blockInfo) { - // Not found? - if (err && err.code === -5) return next(); - - if (err) return next(err); - - /* - * Not sure this is the right way to do it. - * Any other way to lazy load a property in a mongoose object? - */ - - self.info = blockInfo.result; - self.info.reward = BitcoreBlock.getBlockValue(self.info.height) / util.COIN ; - - return next(null, self.info); - }); -}; - - - -module.exports = mongoose.model('Block', BlockSchema); diff --git a/app/models/Transaction.js b/app/models/Transaction.js index 58a39e8..f3b20dc 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -28,8 +28,6 @@ function spec() { }); }; - - Transaction.prototype._fillInfo = function(next) { var self = this; @@ -42,8 +40,6 @@ function spec() { }); }; - - Transaction._fillOutpoints = function(info, cb) { if (!info || info.isCoinBase) return cb(); @@ -51,17 +47,17 @@ function spec() { var valueIn = 0; var incompleteInputs = 0; async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) { - TransactionOut.fromTxIdN(i.txid, i.vout, function(err, out) { + TransactionOut.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) { - if (err || !out || ! out.addr) { + + if (err || !addr || !valueSat ) { console.log('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid); incompleteInputs = 1; return c_in(); // error not scaled } - - i.addr = out.addr; - i.valueSat = out.value_sat; - i.value = out.value_sat / util.COIN; + i.addr = addr; + i.valueSat = valueSat; + i.value = valueSat / util.COIN; valueIn += i.valueSat; return c_in(); diff --git a/app/models/TransactionOut.js b/app/models/TransactionOut.js deleted file mode 100644 index a0fec41..0000000 --- a/app/models/TransactionOut.js +++ /dev/null @@ -1,270 +0,0 @@ -'use strict'; - -/** - * Module dependencies. - */ -var mongoose = require('mongoose'), - async = require('async'), - util = require('bitcore/util/util'), - TransactionRpc = require('../../lib/TransactionRpc').class(), - Schema = mongoose.Schema; - -var CONCURRENCY = 15; -// TODO: use bitcore networks module -var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; - -var TransactionOutSchema = new Schema({ - txidBuf: { - type: Buffer, - index: true, - }, - index: Number, - addr: { - type: String, - index: true, - }, - value_sat: Number, - fromOrphan: Boolean, - - spendTxIdBuf: Buffer, - spendIndex: Number, - spendFromOrphan: Boolean, -}); - - -// Compound index - -TransactionOutSchema.index({txidBuf: 1, index: 1}, {unique: true, sparse: true}); -TransactionOutSchema.index({spendTxIdBuf: 1, spendIndex: 1}, {unique: true, sparse: true}); - -TransactionOutSchema.virtual('txid').get(function () { - return this.txidBuf.toString('hex'); -}); - - -TransactionOutSchema.virtual('spendTxid').get(function () { - if (!this.spendTxIdBuf) return (null); - return this.spendTxIdBuf.toString('hex'); -}); - - -TransactionOutSchema.virtual('txid').set(function (txidStr) { - if (txidStr) - this.txidBuf = new Buffer(txidStr,'hex'); - else - this.txidBuf = null; -}); - -TransactionOutSchema.statics.fromTxId = function(txid, cb) { - var txidBuf = new Buffer(txid, 'hex'); - - this.find({ - txidBuf: txidBuf, - }).exec(function (err,items) { - - // sort by index - return cb(err,items.sort(function(a,b){ - return a.index - b.index; - })); - }); -}; - -TransactionOutSchema.statics.fromTxIdOne = function(txid, cb) { - var txidBuf = new Buffer(txid, 'hex'); - - this.find({ - txidBuf: txidBuf, - }).exec(function (err,item) { - return cb(err, item[0]); - }); -}; - - -TransactionOutSchema.statics.fromTxIdN = function(txid, n, cb) { - var txidBuf = new Buffer(txid, 'hex'); - this.findOne({ - txidBuf: txidBuf, index: n - }).exec(cb); -}; - -TransactionOutSchema.statics.removeFromTxId = function(txid, cb) { - var txidBuf = new Buffer(txid, 'hex'); - this.remove({ txidBuf: txidBuf }).exec(cb); -}; - - - -TransactionOutSchema.statics.storeTransactionOuts = function(txInfo, fromOrphan, cb) { - - var Self = this; - var addrs = []; - var is_new = true; - - if (txInfo.hash) { - - // adapt bitcore TX object to bitcoind JSON response - txInfo.txid = txInfo.hash; - - var count = 0; - txInfo.vin = txInfo.in.map(function (txin) { - var i = {}; - - if (txin.coinbase) { - txInfo.isCoinBase = true; - } - else { - i.txid= txin.prev_out.hash; - i.vout= txin.prev_out.n; - }; - i.n = count++; - return i; - }); - - - count = 0; - txInfo.vout = txInfo.out.map(function (txout) { - var o = {}; - - o.value = txout.value; - o.n = count++; - - if (txout.addrStr){ - o.scriptPubKey = {}; - o.scriptPubKey.addresses = [txout.addrStr]; - } - return o; - }); - - } - - var bTxId = new Buffer(txInfo.txid,'hex'); - - - async.series([ - // Input Outpoints (mark them as spended) - function(p_c) { - if (txInfo.isCoinBase) return p_c(); - async.forEachLimit(txInfo.vin, CONCURRENCY, - function(i, next_out) { - var b = new Buffer(i.txid,'hex'); - var data = { - txidBuf: b, - index: i.vout, - - spendTxIdBuf: bTxId, - spendIndex: i.n, - }; - if (fromOrphan) data.spendFromOrphan = true; - Self.update({txidBuf: b, index: i.vout}, data, {upsert: true}, next_out); - }, - function (err) { - if (err) { - if (!err.message.match(/E11000/)) { - console.log('ERR at TX %s: %s', txInfo.txid, err); - return cb(err); - } - } - return p_c(); - }); - }, - // Parse Outputs - function(p_c) { - async.forEachLimit(txInfo.vout, CONCURRENCY, - function(o, next_out) { - if (o.value && o.scriptPubKey && - o.scriptPubKey.addresses && - o.scriptPubKey.addresses[0] && - ! o.scriptPubKey.addresses[1] // TODO : not supported - ){ - - // This is only to broadcast (WIP) -// if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { -// addrs.push(o.scriptPubKey.addresses[0]); -// } - - var data = { - txidBuf: bTxId, - index : o.n, - - value_sat : o.value * util.COIN, - addr : o.scriptPubKey.addresses[0], - }; - if (fromOrphan) data.fromOrphan = true; - Self.update({txidBuf: bTxId, index: o.n}, data, {upsert: true}, next_out); - } - else { - console.log ('WARN in TX: %s could not parse OUTPUT %d', txInfo.txid, o.n); - return next_out(); - } - }, - function (err) { - if (err) { - if (err.message.match(/E11000/)) { - is_new = false; - } - else { - console.log('ERR at TX %s: %s', txInfo.txid, err); - return cb(err); - } - } - return p_c(); - }); - }], function(err) { - return cb(err, addrs, is_new); - }); -}; - - -// txs can be a [hashes] or [txObjects] -TransactionOutSchema.statics.createFromTxs = function(txs, fromOrphan, next) { - var Self = this; - - if (typeof fromOrphan === 'function') { - next = fromOrphan; - fromOrphan = false; - } - - if (!txs) return next(); - - var inserted_txs = []; - var updated_addrs = {}; - - async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) { - - var txInfo; - - async.series([ - function(a_cb) { - if (typeof t !== 'string') { - txInfo = t; - return a_cb(); - } - - // Is it from genesis block? (testnet==livenet) - // TODO: parse it from networks.genesisTX? - if (t === genesisTXID) return a_cb(); - - TransactionRpc.getRpcInfo(t, function(err, inInfo) { - txInfo =inInfo; - return a_cb(err); - }); - }, - function(a_cb) { - if (!txInfo) return a_cb(); - - Self.storeTransactionOuts(txInfo, fromOrphan, function(err, addrs) { - if (err) return a_cb(err); - return a_cb(); - }); - }], - function(err) { - return each_cb(err); - }); - }, - function(err) { - return next(err, inserted_txs, updated_addrs); - }); -}; - - -module.exports = mongoose.model('TransactionOut', TransactionOutSchema); diff --git a/config/config.js b/config/config.js index f28964c..7ab059b 100644 --- a/config/config.js +++ b/config/config.js @@ -24,6 +24,7 @@ module.exports = { appName: 'Insight ' + env, port: process.env.PORT || 3000, db: 'mongodb://localhost/insight-' + env, + leveldb: './db', bitcoind: { protocol: process.env.BITCOIND_PROTO || 'http', user: process.env.BITCOIND_USER || 'user', diff --git a/dev-util/block-level.js b/dev-util/block-level.js new file mode 100755 index 0000000..559717f --- /dev/null +++ b/dev-util/block-level.js @@ -0,0 +1,25 @@ +#!/usr/bin/env node + +var + config = require('../config/config'), + levelup = require('levelup'); + + +db = levelup(config.leveldb + '/blocks'); + +db.createReadStream({start: 'b-'}) + .on('data', function (data) { +console.log('[block-level.js.11:data:]',data); //TODO + if (data==false) c++; + }) + .on('error', function (err) { + return cb(err); + }) + .on('close', function () { + return cb(null); + }) + .on('end', function () { + return cb(null); + }); + + diff --git a/lib/BlockDb.js b/lib/BlockDb.js new file mode 100644 index 0000000..59ba778 --- /dev/null +++ b/lib/BlockDb.js @@ -0,0 +1,123 @@ +'use strict'; + +require('classtool'); + + +function spec() { + + var TIMESTAMP_ROOT = 'b-ts-'; + var ORPHAN_FLAG_ROOT = 'b-orphan-'; + + + /** + * Module dependencies. + */ + var RpcClient = require('bitcore/RpcClient').class(), + util = require('bitcore/util/util'), + levelup = require('levelup'), + BitcoreBlock= require('bitcore/Block').class(), + TransactionDb = require('.//TransactionDb'), + config = require('../config/config'), + fs = require('fs'); + + var BlockDb = function() { + this.db = levelup(config.leveldb + '/blocks'); + }; + + BlockDb.prototype.drop = function(cb) { + var self = this; + var path = config.leveldb + '/blocks'; + require('leveldown').destroy(path, function () { + fs.mkdirSync(config.leveldb); + fs.mkdirSync(path); + self.db = levelup(path); + return cb(); + }); + }; + + BlockDb.prototype.add = function(b, cb) { + var self = this; + + if (!b.hash) return cb(new Error('no Hash at Block.save')); + + + var time_key = TIMESTAMP_ROOT + + ( b.timestamp || Math.round(new Date().getTime() / 1000) ); + + + self.db.batch() + .put(time_key, b.hash) + .put(ORPHAN_FLAG_ROOT + b.hash, b.isOrphan || 0) + .write(cb); + }; + + BlockDb.prototype.countNotOrphan = function(hash, cb) { + var c = 0; + this.db.createReadStream({start: ORPHAN_FLAG_ROOT}) + .on('data', function (data) { + if (data === false) c++; + }) + .on('error', function (err) { + return cb(err); + }) + .on('close', function () { + return cb(null); + }) + .on('end', function () { + return cb(null); + }); + }; + + BlockDb.prototype.has = function(hash, cb) { + var self = this; + + var k = ORPHAN_FLAG_ROOT + hash; + self.db.get(k, function (err,val) { + + var ret; + + if (err && err.notFound) { + err = null; + ret = false; + } + if (typeof val !== 'undefined') { + ret = true; + } + return cb(err, ret); + }); + }; + + + BlockDb.prototype.fromHashWithInfo = function(hash, cb) { + var rpc = new RpcClient(config.bitcoind); + + rpc.getBlock(hash, function(err, info) { + // Not found? + if (err && err.code === -5) return cb(); + if (err) return cb(err); + + info.reward = BitcoreBlock.getBlockValue(info.height) / util.COIN ; + + return cb(null, { + hash: hash, + info: info.result, + }); + }); + }; + + BlockDb.blockIndex = function(height, cb) { + var rpc = new RpcClient(config.bitcoind); + rpc.getBlockHash(height, function(err, bh){ + if (err) return cb(err); + + cb(null, { blockHash: bh.result }); + }); + }; + + + + return BlockDb; +} +module.defineClass(spec); + + diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 9d22e1b..6bf14d8 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -9,11 +9,11 @@ function spec() { var RpcClient = require('bitcore/RpcClient').class(); var bitutil = require('bitcore/util/util'); var Address = require('bitcore/Address').class(); + var Deserialize = require('bitcore/Deserialize'); var Script = require('bitcore/Script').class(); var networks = require('bitcore/networks'); var async = require('async'); var config = require('../config/config'); - var Block = require('../app/models/Block'); var Sync = require('./Sync').class(); var sockets = require('../app/controllers/socket.js'); var BlockExtractor = require('./BlockExtractor.js').class(); @@ -36,7 +36,7 @@ function spec() { this.syncPercentage = 0; this.syncedBlocks = 0; this.skippedBlocks = 0; - + this.orphanBlocks = 0; } function p() { @@ -95,11 +95,12 @@ function spec() { syncPercentage: this.syncPercentage, skippedBlocks: this.skippedBlocks, syncedBlocks: this.syncedBlocks, + orphanBlocks: this.orphanBlocks, error: this.error, }; }; - HistoricSync.prototype.showProgress = function(height) { + HistoricSync.prototype.showProgress = function() { var self = this; if (self.error) { @@ -109,11 +110,17 @@ function spec() { self.syncPercentage = parseFloat(100 * (self.syncedBlocks + self.skippedBlocks) / self.blockChainHeight).toFixed(3); if (self.syncPercentage > 100) self.syncPercentage = 100; - p(util.format('status: [%d%%] skipped: %d ', self.syncPercentage, self.skippedBlocks, height)); + p(util.format('status: [%d%%] skipped: %d ', self.syncPercentage, self.skippedBlocks)); } if (self.opts.shouldBroadcast) { sockets.broadcastSyncInfo(self.info()); } + +//TODO +if (self.syncPercentage>10) { + console.log(self.info()); + process.exit(1); +} }; HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) { @@ -128,14 +135,13 @@ function spec() { async.series([ // Already got it? function(c) { - Block.fromHash(blockHash, function(err, block) { + self.sync.hasBlock(blockHash, function(err, ret) { if (err) { p(err); return c(err); } - if (block) { - existed = true; - } + + if (ret) existed = true; return c(); }); }, @@ -181,14 +187,10 @@ function spec() { ], function(err) { if (err) { - self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncedBlocks); - self.status = 'aborted'; - self.showProgress(); - p(self.err); + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncedBlocks)); return cb(err); } else { - self.err = null; self.status = 'syncing'; } @@ -259,30 +261,20 @@ function spec() { return addrStrs; }; +var kk=0; - - HistoricSync.prototype.getBlockFromFile = function(height, scanOpts, cb) { + HistoricSync.prototype.getBlockFromFile = function(scanOpts, cb) { var self = this; var nextHash; var blockInfo; - var isMainChain; var existed; async.series([ - // Is it in mainchain? - function(c) { - self.rpc.getBlockHash(height, function(err, res) { - if (err) return cb(err); - - nextHash = res.result; - return c(); - }); - }, //show some (inacurate) status function(c) { if ( ( self.syncedBlocks + self.skippedBlocks) % self.step === 1) { - self.showProgress(height); + self.showProgress(); } return c(); @@ -294,6 +286,7 @@ function spec() { if (err || ! b) return c(err); blockInfo = b.getStandardizedObject(b.txs, self.network); + blockInfo.curWork = Deserialize.intFromCompact(b.bits); var ti=0; // Get TX Address @@ -320,10 +313,6 @@ function spec() { //store it function(c) { - isMainChain = blockInfo.hash === nextHash; - - blockInfo.isOrphan = !isMainChain; - /* * In file sync, orphan blocks are just ignored. * This is to simplify our schema and the @@ -338,29 +327,25 @@ function spec() { return c(); }); }, - ], function(err) { + function(c) { - if (err) { - self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo.hash, err, self.syncedBlocks); - self.status = 'aborted'; - self.showProgress(); - p(err); - return cb(err); + if (self.prevHash && blockInfo.prev_block !== self.prevHash) { + self.setError('found orphan:' + self.prevHash + ' vs. ' + blockInfo.prev_block); } else { - // Continue if (blockInfo && blockInfo.hash) { + self.prevHash = blockInfo.hash; self.syncedBlocks++; - self.err = null; - self.status = 'syncing'; - - return cb(null, true, isMainChain); - } - else { - self.err = null; + } else self.status = 'finished'; - } + } + + return c(); + }, + ], function(err) { + if (err) { + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo.hash, err, self.syncedBlocks)); } return cb(err); }); @@ -371,8 +356,6 @@ function spec() { HistoricSync.prototype.importHistory = function(scanOpts, next) { var self = this; - var retry_secs = 2; - var lastBlock; async.series([ @@ -404,10 +387,7 @@ function spec() { function(cb) { if (scanOpts.upToExisting) { // should be isOrphan = true or null to be more accurate. - Block.count({ - isOrphan: null - }, - function(err, count) { + self.sync.countNotOrphan(function(err, count) { if (err) return cb(err); self.syncedBlocks = count || 0; @@ -455,25 +435,19 @@ function spec() { if (scanOpts.fromFiles) { - var keepGoing = true; var height = 0; + self.status = 'syncing'; async.whilst(function() { - return keepGoing; + return self.status === 'syncing'; }, function (w_cb) { - self.getBlockFromFile(height, scanOpts, function(err, inKeepGoing, wasMainChain) { - keepGoing = inKeepGoing; - if (wasMainChain) height++; - //Black magic from http://stackoverflow.com/questions/20936486/node-js-maximum-call-stack-size-exceeded + self.getBlockFromFile(scanOpts, function(err) { setImmediate(function(){ - return w_cb(err); }); - }) + }); }, function(err) { - -console.log('[HistoricSync.js.468]'); //TODO - return next(); + return next(err); }); } else { @@ -489,11 +463,10 @@ console.log('[HistoricSync.js.468]'); //TODO HistoricSync.prototype.smartImport = function(scanOpts, next) { var self = this; - Block.fromHash(self.genesis, function(err, b) { + self.sync.hasBlock(self.genesis, function(err, b) { if (err) return next(err); - if (!b || scanOpts.destroy) { p('Could not find Genesis block. Running FULL SYNC'); if (config.bitcoind.dataDir) { diff --git a/lib/Sync.js b/lib/Sync.js index bb445fa..39b4348 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -6,13 +6,15 @@ require('classtool'); function spec() { var mongoose = require('mongoose'); var config = require('../config/config'); - var Block = require('../app/models/Block'); - var TransactionOut = require('../app/models/TransactionOut'); var sockets = require('../app/controllers/socket.js'); + var BlockDb = require('./BlockDb').class(); + var TransactionDb = require('./TransactionDb').class(); var async = require('async'); function Sync() { + this.blockDb = new BlockDb(); + this.txDb = new TransactionDb(); } Sync.prototype.init = function(opts, cb) { @@ -60,30 +62,46 @@ function spec() { Sync.prototype.destroy = function(next) { var self = this; async.series([ - function(b) { try {self.db.collections.blocks.drop(b);} catch (e) { return b(); } }, + function(b) { try {self.blockDb.drop(b);} catch (e) { return b(); } }, function(b) { try {self.db.collections.transactionitems.drop(b);} catch (e) { return b(); } }, function(b) { try {self.db.collections.transactionouts.drop(b);} catch (e) { return b(); } }, ], next); }; + + Sync.prototype.hasBlock = function(hash, cb) { + var self = this; + + return self.blockDb.has(hash, cb); + }; + + Sync.prototype.countNotOrphan = function(hash, cb) { + var self = this; + return self.blockDb.countNotOrphan(hash, cb); + }; + + + Sync.prototype.storeBlock = function(block, cb) { var self = this; - Block.customCreate(block, function(err, block, inserted_txs, updated_addrs){ + self.txDb.createFromBlock(block, function(err, insertedTxs, updateAddrs) { if (err) return cb(err); - self._handleBroadcast(block, inserted_txs, updated_addrs); - - return cb(); + self.blockDb.add(block, function(err){ + if (err) return cb(err); + self._handleBroadcast(block, insertedTxs, updateAddrs); + return cb(); + }); }); }; - Sync.prototype._handleBroadcast = function(block, inserted_txs, updated_addrs) { + Sync.prototype._handleBroadcast = function(hash, inserted_txs, updated_addrs) { var self = this; - if (block && self.opts.broadcast_blocks) { - sockets.broadcast_block(block); + if (hash && self.opts.broadcast_blocks) { + sockets.broadcast_block({hash: hash}); } if (inserted_txs && self.opts.broadcast_txs) { @@ -107,7 +125,8 @@ function spec() { Sync.prototype.storeTxs = function(txs, cb) { var self = this; - TransactionOut.createFromTxs(txs, function(err, inserted_txs, updated_addrs) { + // TODO + self.txDb.createFromTxs(txs, function(err, inserted_txs, updated_addrs) { if (err) return cb(err); self._handleBroadcast(null, inserted_txs, updated_addrs); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js new file mode 100644 index 0000000..4ca43e6 --- /dev/null +++ b/lib/TransactionDb.js @@ -0,0 +1,282 @@ +'use strict'; + +require('classtool'); + + +function spec() { + var ROOT = 'tx-'; //tx-- => [addr, btc_sat] + var OUTS_ROOT = 'txouts-'; //txouts-- => [addr, btc_sat] + var ADDR_ROOT = 'txouts-addr-'; //txouts-addr---- => (+/-) btc_sat + + // TODO: use bitcore networks module + var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; + var CONCURRENCY = 100; + + /** + * Module dependencies. + */ + var TransactionRpc = require('./TransactionRpc').class(), + util = require('bitcore/util/util'), + levelup = require('levelup'), + async = require('async'), + config = require('../config/config'), + fs = require('fs'); + + var TransactionDb = function() { + this.db = levelup(config.leveldb + '/txs'); + }; + + TransactionDb.prototype.drop = function(cb) { + var self = this; + var path = config.leveldb + '/blocks'; + require('leveldown').destroy(path, function () { + fs.mkdirSync(config.leveldb); + fs.mkdirSync(path); + self.db = levelup(path); + return cb(); + }); + }; + +/* + txidBuf: { + type: Buffer, + index: true, + }, + index: Number, + addr: { + type: String, + index: true, + }, + value_sat: Number, + fromOrphan: Boolean, + + spendTxIdBuf: Buffer, + spendIndex: Number, + spendFromOrphan: Boolean, + */ + +// TransactionDb.prototype.fromTxIdOne = function(txid, cb) { TODO + TransactionDb.prototype.has = function(txid, cb) { + var self = this; + + var k = ROOT + txid; + self.db.get(k, function (err,val) { + + var ret; + + if (err && err.notFound) { + err = null; + ret = false; + } + if (typeof val !== undefined) { + ret = true; + } + return cb(err, ret); + }); + }; + + + TransactionDb.prototype.fromTxIdN = function(txid, n, cb) { + var self = this; + + var k = OUTS_ROOT + txid + '-' + n; + + self.db.get(k, function (err,val) { + if (err && err.notFound) { + err = null; + } + + var a = val.split('-'); + + return cb(err, val, a[0], a[1]); + }); + }; + + TransactionDb.prototype.adaptTxObject = function(txInfo) { + + // adapt bitcore TX object to bitcoind JSON response + txInfo.txid = txInfo.hash; + + var count = 0; + txInfo.vin = txInfo.in.map(function (txin) { + var i = {}; + + if (txin.coinbase) { + txInfo.isCoinBase = true; + } + else { + i.txid= txin.prev_out.hash; + i.vout= txin.prev_out.n; + } + i.n = count++; + return i; + }); + + + count = 0; + txInfo.vout = txInfo.out.map(function (txout) { + var o = {}; + + o.value = txout.value; + o.n = count++; + + if (txout.addrStr){ + o.scriptPubKey = {}; + o.scriptPubKey.addresses = [txout.addrStr]; + } + return o; + }); + }; + + + TransactionDb.prototype.add = function(tx, fromOrphan, cb) { + var self = this; + var addrs = []; + var is_new = true; + + if (tx.hash) self.adaptTxObject(tx); + + //TODO + var ts = 1; + + + //TODO + if (fromOrphan) return cb(); + + async.series([ + // Input Outpoints (mark them as spended) + function(p_c) { + if (tx.isCoinBase) return p_c(); + async.forEachLimit(tx.vin, CONCURRENCY, + function(i, next_out) { + + // TODO +return next_out(); + +/* self.db.batch() + .put() + var data = { + txidBuf: b, + index: i.vout, + + spendTxIdBuf: bTxId, + spendIndex: i.n, + }; + if (fromOrphan) data.spendFromOrphan = true; + Self.update({txidBuf: b, index: i.vout}, data, {upsert: true}, next_out); +*/ + }, + function (err) { + if (err) { + if (!err.message.match(/E11000/)) { + console.log('ERR at TX %s: %s', tx.txid, err); + return cb(err); + } + } + return p_c(); + }); + }, + // Parse Outputs + function(p_c) { + async.forEachLimit(tx.vout, CONCURRENCY, + function(o, next_out) { + if (o.value && o.scriptPubKey && + o.scriptPubKey.addresses && + o.scriptPubKey.addresses[0] && + ! o.scriptPubKey.addresses[1] // TODO : not supported + ){ + + // This is only to broadcast (WIP) + // if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { + // addrs.push(o.scriptPubKey.addresses[0]); + // } + + //if (fromOrphan) data.fromOrphan = true; // TODO + + var addr = o.scriptPubKey.addresses[0]; + var sat = o.value * util.COIN; + self.db.batch() + .put( OUTS_ROOT + tx.txid + o.n, addr + ':' + sat) + .put( ADDR_ROOT + addr + '-' + ts + '-' + tx.txid + + '-' + o.n, sat) + .write(next_out); + + } + else { + console.log ('WARN in TX: %s could not parse OUTPUT %d', tx.txid, o.n); + return next_out(); + } + }, + function (err) { + if (err) { + if (err.message.match(/E11000/)) { + is_new = false; + } + else { + console.log('ERR at TX %s: %s', tx.txid, err); + return cb(err); + } + } + return p_c(); + }); + }], function(err) { + return cb(err, addrs, is_new); + }); + }; + + TransactionDb.prototype.createFromArray = function(txs, fromOrphan, blockHash, next) { + var self = this; + + if (!txs) return next(); + + // TODO + var insertedTxs = []; + var updatedAddrs = {}; + + async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) { + if (typeof t === 'string') { + // Is it from genesis block? (testnet==livenet) + // TODO: parse it from networks.genesisTX? + if (t === genesisTXID) return each_cb(); + + TransactionRpc.getRpcInfo(t, function(err, inInfo) { + if (!inInfo) return each_cb(err); + + self.add(inInfo, fromOrphan, function(err) { + if (err) return each_cb(err); + + self.db.put(ROOT + t, blockHash, function(err) { + return each_cb(err); + }); + }); + }); + } + else { + self.add(t, fromOrphan, function(err) { + if (err) return each_cb(err); + + self.db.put(ROOT + t.txid, blockHash, function(err) { + return each_cb(err); + }); + }); + } + }, + function(err) { + + + return next(err, insertedTxs, updatedAddrs); + }); +}; + + +// txs can be a [hashes] or [txObjects] + TransactionDb.prototype.createFromBlock = function(b, next) { + var self = this; + if (!b.tx) return next(); + + return self.createFromArray(b.tx, b.isOrphan, b.hash, next); + }; + + return TransactionDb; +} +module.defineClass(spec); diff --git a/package.json b/package.json index faf621f..6860d54 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,8 @@ }, "dependencies": { "async": "*", + "leveldown": "*", + "levelup": "*", "glob": "*", "classtool": "*", "commander": "*", From fb300c2561688685bea9c82b644ef8a180e6dc45 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 3 Feb 2014 23:31:37 -0300 Subject: [PATCH 03/25] sync with levelDb working! --- lib/HistoricSync.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 6bf14d8..9f1b306 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -117,10 +117,11 @@ function spec() { } //TODO + /* if (self.syncPercentage>10) { console.log(self.info()); process.exit(1); -} +}*/ }; HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) { From 339f6f79fc8e151284d71fdf3a3dd78d70d81258 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 4 Feb 2014 03:22:58 -0300 Subject: [PATCH 04/25] mocha 01-txout passing --- lib/BlockDb.js | 31 +++-- lib/HistoricSync.js | 32 +++-- lib/Sync.js | 26 ++++ lib/TransactionDb.js | 162 ++++++++++++++++--------- test/integration/01-transactionouts.js | 71 ++++++----- test/integration/block.js | 64 +++------- test/integration/txitems.json | 9 ++ 7 files changed, 229 insertions(+), 166 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 59ba778..44831ae 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -6,7 +6,7 @@ require('classtool'); function spec() { var TIMESTAMP_ROOT = 'b-ts-'; - var ORPHAN_FLAG_ROOT = 'b-orphan-'; + var PREV_ROOT = 'b-prev-'; // b-prev- => (0 if orphan) /** @@ -16,7 +16,6 @@ function spec() { util = require('bitcore/util/util'), levelup = require('levelup'), BitcoreBlock= require('bitcore/Block').class(), - TransactionDb = require('.//TransactionDb'), config = require('../config/config'), fs = require('fs'); @@ -44,18 +43,34 @@ function spec() { var time_key = TIMESTAMP_ROOT + ( b.timestamp || Math.round(new Date().getTime() / 1000) ); - self.db.batch() .put(time_key, b.hash) - .put(ORPHAN_FLAG_ROOT + b.hash, b.isOrphan || 0) + .put(PREV_ROOT + b.hash, b.prev_block) .write(cb); }; + + BlockDb.prototype.setOrphan = function(hash, cb) { + var self = this; + + + var k = PREV_ROOT + hash; + self.db.get(k, function (err,oldPrevHash) { + if (err || !oldPrevHash) return cb(err); + + self.db.put(PREV_ROOT + hash, 0, function() { + return cb(err, oldPrevHash); + }); + }); + + // We keep the block in TIMESTAMP_ROOT + }; + BlockDb.prototype.countNotOrphan = function(hash, cb) { var c = 0; - this.db.createReadStream({start: ORPHAN_FLAG_ROOT}) + this.db.createReadStream({start: PREV_ROOT}) .on('data', function (data) { - if (data === false) c++; + if (data.value !== 0) c++; }) .on('error', function (err) { return cb(err); @@ -71,11 +86,9 @@ function spec() { BlockDb.prototype.has = function(hash, cb) { var self = this; - var k = ORPHAN_FLAG_ROOT + hash; + var k = PREV_ROOT + hash; self.db.get(k, function (err,val) { - var ret; - if (err && err.notFound) { err = null; ret = false; diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 9f1b306..69ac2ea 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -313,14 +313,17 @@ var kk=0; }, //store it function(c) { + if (self.prevHash && blockInfo.prev_block !== self.prevHash) { - /* - * In file sync, orphan blocks are just ignored. - * This is to simplify our schema and the - * sync process - */ - if (blockInfo.isOrphan) return c(); + console.log('Orphans found: %s vs %s @%s', + self.prevHash + ' vs. ' + blockInfo.prev_block, blockInfo.hash); + self.sync.setOrphan(self.prevHash, blockInfo.prev_block, c); + } + else return c(); + }, + //store it + function(c) { self.sync.storeBlock(blockInfo, function(err) { existed = err && err.toString().match(/E11000/); @@ -329,18 +332,11 @@ var kk=0; }); }, function(c) { - - if (self.prevHash && blockInfo.prev_block !== self.prevHash) { - self.setError('found orphan:' + self.prevHash + ' vs. ' + blockInfo.prev_block); - } - else { - - if (blockInfo && blockInfo.hash) { - self.prevHash = blockInfo.hash; - self.syncedBlocks++; - } else - self.status = 'finished'; - } + if (blockInfo && blockInfo.hash) { + self.prevHash = blockInfo.hash; + self.syncedBlocks++; + } else + self.status = 'finished'; return c(); }, diff --git a/lib/Sync.js b/lib/Sync.js index 39b4348..bfa2d55 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -96,6 +96,32 @@ function spec() { }); }; + Sync.prototype.setOrphan = function(fromBlock, toBlock, c) { + var self = this; + + var c = fromBlock; + + async.whilst( + function () { + return c !== toBlock; + }, + function () { + console.log('[Sync.js.113]: setOrphan', c); //TODO + self.txDb.setOrphan(c, function(err, insertedTxs, updateAddrs) { + if (err) return cb(err); + + self.blockDb.setOrphan(c, function(err, prevHash){ + + c = prevHash; + return cb(err); + }); + }); + }, + function (err) { + return c(err); + } + ); + }; Sync.prototype._handleBroadcast = function(hash, inserted_txs, updated_addrs) { var self = this; diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 4ca43e6..918fe7a 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -4,9 +4,16 @@ require('classtool'); function spec() { - var ROOT = 'tx-'; //tx-- => [addr, btc_sat] + + // blockHash -> txid mapping (to orphanize )/ + var ROOT = 'tx-b-'; //tx-b- => txid + + // to show tx outs var OUTS_ROOT = 'txouts-'; //txouts-- => [addr, btc_sat] - var ADDR_ROOT = 'txouts-addr-'; //txouts-addr---- => (+/-) btc_sat + + // to sum up addr balance + var ADDR_ROOT = 'txouts-addr-'; //txouts-addr---- => + btc_sat + var SPEND_ROOT = 'txouts-spend-';//txouts-spend-- => [txid(in),n(in),ts] // TODO: use bitcore networks module var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; @@ -37,29 +44,12 @@ function spec() { }); }; -/* - txidBuf: { - type: Buffer, - index: true, - }, - index: Number, - addr: { - type: String, - index: true, - }, - value_sat: Number, - fromOrphan: Boolean, - - spendTxIdBuf: Buffer, - spendIndex: Number, - spendFromOrphan: Boolean, - */ // TransactionDb.prototype.fromTxIdOne = function(txid, cb) { TODO TransactionDb.prototype.has = function(txid, cb) { var self = this; - var k = ROOT + txid; + var k = OUTS_ROOT + txid; self.db.get(k, function (err,val) { var ret; @@ -75,6 +65,31 @@ function spec() { }); }; + TransactionDb.prototype.fromTxId = function(txid, cb) { + var self = this; + + var k = OUTS_ROOT + txid; + var ret=[]; + + self.db.createReadStream({start: k, end: k + '~'}) + .on('data', function (data) { + var k = data.key.split('-'); + var v = data.value.split(':'); + ret.push({ + addr: v[0], + value_sat: v[1], + index: k[2], + }); + }) + .on('error', function (err) { + return cb(err); + }) + .on('end', function (err) { + return cb(err, ret); + }); + }; + + TransactionDb.prototype.fromTxIdN = function(txid, n, cb) { var self = this; @@ -92,6 +107,49 @@ function spec() { }); }; + // Only for testing. Very slow (toRm outs to rm, only to speedup) + TransactionDb.prototype.removeFromTxId = function(txid, toRm, cb) { + var self = this; + + async.series([ + function(c) { + self.db.createReadStream({ + start: OUTS_ROOT + txid, + end: OUTS_ROOT + txid + '~', + }).pipe( + self.db.createWriteStream({type:'del'}) + ).on('close', c); + }, + function(s_c) { + if (toRm && toRm.length) return s_c(); + + toRm = []; + self.db.createReadStream({ + start: SPEND_ROOT + }) + .on('data', function(data) { + if (data.value.indexOf(txid) >= 0) { + toRm.push(data.key); + console.log('To Remove Found', data.key); //TODO + } + }) + .on('end', function() { + return s_c(); + }); + }, + function(s_c) { + async.each(toRm, function(k,e_c) { + self.db.del(k,e_c); + }, s_c); + }], + function(err) { + cb(err); + }); + + }; + + + TransactionDb.prototype.adaptTxObject = function(txInfo) { // adapt bitcore TX object to bitcoind JSON response @@ -129,42 +187,26 @@ function spec() { }; - TransactionDb.prototype.add = function(tx, fromOrphan, cb) { + TransactionDb.prototype.add = function(tx, cb) { var self = this; var addrs = []; var is_new = true; if (tx.hash) self.adaptTxObject(tx); - //TODO - var ts = 1; + var ts = tx.timestamp; - //TODO - if (fromOrphan) return cb(); - async.series([ // Input Outpoints (mark them as spended) function(p_c) { if (tx.isCoinBase) return p_c(); async.forEachLimit(tx.vin, CONCURRENCY, function(i, next_out) { - - // TODO -return next_out(); - -/* self.db.batch() - .put() - var data = { - txidBuf: b, - index: i.vout, - - spendTxIdBuf: bTxId, - spendIndex: i.n, - }; - if (fromOrphan) data.spendFromOrphan = true; - Self.update({txidBuf: b, index: i.vout}, data, {upsert: true}, next_out); -*/ + self.db.batch() + .put( SPEND_ROOT + i.txid + '-' + i.vout , + tx.txid + ':' + i.n + ':' + ts) + .write(next_out); }, function (err) { if (err) { @@ -185,18 +227,15 @@ return next_out(); o.scriptPubKey.addresses[0] && ! o.scriptPubKey.addresses[1] // TODO : not supported ){ - // This is only to broadcast (WIP) // if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { // addrs.push(o.scriptPubKey.addresses[0]); // } - //if (fromOrphan) data.fromOrphan = true; // TODO - var addr = o.scriptPubKey.addresses[0]; var sat = o.value * util.COIN; self.db.batch() - .put( OUTS_ROOT + tx.txid + o.n, addr + ':' + sat) + .put( OUTS_ROOT + tx.txid + '-' + o.n, addr + ':' + sat) .put( ADDR_ROOT + addr + '-' + ts + '-' + tx.txid + '-' + o.n, sat) .write(next_out); @@ -224,7 +263,7 @@ return next_out(); }); }; - TransactionDb.prototype.createFromArray = function(txs, fromOrphan, blockHash, next) { + TransactionDb.prototype.createFromArray = function(txs, blockHash, next) { var self = this; if (!txs) return next(); @@ -235,6 +274,7 @@ return next_out(); async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) { if (typeof t === 'string') { + // Is it from genesis block? (testnet==livenet) // TODO: parse it from networks.genesisTX? if (t === genesisTXID) return each_cb(); @@ -242,28 +282,27 @@ return next_out(); TransactionRpc.getRpcInfo(t, function(err, inInfo) { if (!inInfo) return each_cb(err); - self.add(inInfo, fromOrphan, function(err) { - if (err) return each_cb(err); + self.add(inInfo, function(err) { + if (err || !blockHash) return each_cb(err); - self.db.put(ROOT + t, blockHash, function(err) { + self.db.put(ROOT + blockHash, t, function(err) { return each_cb(err); }); }); }); } else { - self.add(t, fromOrphan, function(err) { + self.add(t, function(err) { if (err) return each_cb(err); - self.db.put(ROOT + t.txid, blockHash, function(err) { + self.db.put(ROOT + blockHash, t.txid, function(err) { return each_cb(err); }); }); } }, function(err) { - - +console.log('[TransactionDb.js.308]'); //TODO return next(err, insertedTxs, updatedAddrs); }); }; @@ -274,9 +313,22 @@ return next_out(); var self = this; if (!b.tx) return next(); - return self.createFromArray(b.tx, b.isOrphan, b.hash, next); + return self.createFromArray(b.tx, b.hash, next); }; + + TransactionDb.prototype.setOrphan = function(blockHash, next) { +// var self = this; + + //Get Txs +// TODO + + //Mark Tx's output as fromOrphan + //Mark Tx's outpoiunt as fromOrphan. Undo spents + return next(); + }; + + return TransactionDb; } module.defineClass(spec); diff --git a/test/integration/01-transactionouts.js b/test/integration/01-transactionouts.js index 57a7f2d..fc5a316 100644 --- a/test/integration/01-transactionouts.js +++ b/test/integration/01-transactionouts.js @@ -5,60 +5,57 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var mongoose = require('mongoose'), +var assert = require('assert'), fs = require('fs'), util = require('util'), config = require('../../config/config'), - TransactionOut = require('../../app/models/TransactionOut'); + TransactionDb = require('../../lib/TransactionDb').class(); -var txItemsValid = JSON.parse(fs.readFileSync('test/model/txitems.json')); - -mongoose.connection.on('error', function(err) { console.log(err); }); +var txItemsValid = JSON.parse(fs.readFileSync('test/integration/txitems.json')); describe('TransactionOut', function(){ - before(function(done) { - mongoose.connect(config.db); - done(); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); + var tdb = new TransactionDb(); txItemsValid.forEach( function(v) { if (v.disabled) return; - it('test a exploding tx ' + v.txid, function(done) { + it('test a processing tx ' + v.txid, function(done) { + this.timeout(60000); // Remove first - TransactionOut.removeFromTxId(v.txid, function(err) { - TransactionOut._explodeTransactionOuts(v.txid, function(err, tx) { - if (err) done(err); + tdb.removeFromTxId(v.txid, v.toRm, function() { - TransactionOut - .fromTxId( v.txid, function(err, readItems) { + tdb.fromTxId( v.txid, function(err, readItems) { + assert.equal(readItems.length,0); - var unmatch={}; + var unmatch=[]; + tdb.createFromArray([v.txid], null, function(err) { + if (err) return done(err); + + tdb.fromTxId( v.txid, function(err, readItems) { + + v.items.forEach(function(validItem){ + unmatch[validItem.addr] =1; + }); + assert.equal(readItems.length,v.items.length); + + v.items.forEach(function(validItem){ + var readItem = readItems.shift(); + + assert.equal(readItem.addr,validItem.addr); + assert.equal(readItem.value_sat,validItem.value_sat); + assert.equal(readItem.index,validItem.index); + assert.equal(readItem.spendIndex, null); + assert.equal(readItem.spendTxIdBuf, null); + delete unmatch[validItem.addr]; + }); + + var valid = util.inspect(v.items, { depth: null }); + assert(!Object.keys(unmatch).length,'\n\tUnmatchs:' + Object.keys(unmatch) + "\n\n" +valid + '\nvs.\n' + readItems); + return done(); - v.items.forEach(function(validItem){ - unmatch[validItem.addr] =1; }); - v.items.forEach(function(validItem){ - var readItem = readItems.shift(); - assert.equal(readItem.addr,validItem.addr); - assert.equal(readItem.value_sat,validItem.value_sat); - assert.equal(readItem.index,validItem.index); - assert.equal(readItem.spendIndex, null); - assert.equal(readItem.spendTxIdBuf, null); - delete unmatch[validItem.addr]; - }); - - var valid = util.inspect(v.items, { depth: null }); - assert(!Object.keys(unmatch).length,'\n\tUnmatchs:' + Object.keys(unmatch) + "\n\n" +valid + '\nvs.\n' + readItems); - done(); - }); }); }); diff --git a/test/integration/block.js b/test/integration/block.js index 9ca09c0..1b6ea50 100644 --- a/test/integration/block.js +++ b/test/integration/block.js @@ -7,68 +7,38 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var TESTING_BLOCK = '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74'; var - mongoose= require('mongoose'), assert = require('assert'), config = require('../../config/config'), - Block = require('../../app/models/Block'); + BlockDb = require('../../lib/BlockDb').class(); -mongoose.connection.on('error', function(err) { console.log(err); }); - -describe('Block fromHashWithInfo', function(){ - - before(function(done) { - mongoose.connect(config.db); - done(); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); - - - it('should poll block\'s info from mongoose', function(done) { - Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) { - if (err) done(err); - - - var h = new Buffer(TESTING_BLOCK,'hex'); - assert(b2.hashStr === TESTING_BLOCK); - assert.equal(b2.hashStr, TESTING_BLOCK); - done(); - }); - }); +describe('BlockDb fromHashWithInfo', function(){ + var bdb = new BlockDb(); it('should poll block\'s info from bitcoind', function(done) { - Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) { + bdb.fromHashWithInfo(TESTING_BLOCK, function(err, b2) { if (err) done(err); + assert.equal(b2.hash, TESTING_BLOCK); assert.equal(b2.info.hash, TESTING_BLOCK); assert.equal(b2.info.chainwork, '000000000000000000000000000000000000000000000000001b6dc969ffe847'); done(); }); }); - - - it('hash Virtuals SET', function(done) { - var b = new Block(); - b.hashStr = 'a1a2'; - assert.equal(b.hash.toString('hex'),'a1a2'); - b.nextBlockHashStr = 'a1a3'; - assert.equal(b.nextBlockHash.toString('hex'),'a1a3'); - done(); + it('return true in has', function(done) { + bdb.has(TESTING_BLOCK, function(err, has) { + assert.equal(has, true); +console.log('[block.js.29:has:]',has); //TODO + done(); + }); + }); + it('return false in has', function(done) { + bdb.has('111', function(err, has) { + assert.equal(has, false); + done(); + }); }); - it('hash Virtuals GET', function(done) { - var b = new Block(); - b.hash = new Buffer('a1a2','hex'); - assert.equal(b.hashStr,'a1a2'); - - b.nextBlockHash = new Buffer('b2b1','hex'); - assert.equal(b.nextBlockHashStr,'b2b1'); - done(); - }); }); diff --git a/test/integration/txitems.json b/test/integration/txitems.json index 72ff969..eae6b11 100644 --- a/test/integration/txitems.json +++ b/test/integration/txitems.json @@ -5,6 +5,9 @@ }, { "txid": "21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237", + "toRm": [ + "txouts-spend-86a03cac7d87f596008c6d5a8d3fd8b88842932ea6f0337673eda16f6b472f7f-0" + ], "items": [ { "addr": "mzjLe62faUqCSjkwQkwPAL5nYyR8K132fA", @@ -20,6 +23,9 @@ }, { "txid": "b633a6249d4a2bc123e7f8a151cae2d4afd17aa94840009f8697270c7818ceee", + "toRm": [ + "txouts-spend-01621403689cb4a95699a3dbae029d7031c5667678ef14e2054793954fb27917-0" + ], "items": [ { "addr": "mhfQJUSissP6nLM5pz6DxHfctukrrLct2T", @@ -35,6 +41,9 @@ }, { "txid": "ca2f42e44455b8a84434de139efea1fe2c7d71414a8939e0a20f518849085c3b", + "toRm": [ + "txouts-spend-2d7b680fb06e4d7eeb65ca49ac7522276586e0090b7fe662fc708129429c5e6a-0" + ], "items": [ { "addr": "mhqyL1nDQDo1WLH9qH8sjRjx2WwrnmAaXE", From 65aad6d0ad4e8799e94d93cc846350955d3fbc48 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 4 Feb 2014 03:41:32 -0300 Subject: [PATCH 05/25] 02 transaction tests passing --- lib/TransactionDb.js | 27 +++++++++++++++++++++----- test/integration/02-transactionouts.js | 26 ++++++++----------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 918fe7a..e76eeb3 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -71,6 +71,7 @@ function spec() { var k = OUTS_ROOT + txid; var ret=[]; + // outs. self.db.createReadStream({start: k, end: k + '~'}) .on('data', function (data) { var k = data.key.split('-'); @@ -84,8 +85,26 @@ function spec() { .on('error', function (err) { return cb(err); }) - .on('end', function (err) { - return cb(err, ret); + .on('end', function () { + var k = SPEND_ROOT + txid; + var l = ret.length; + self.db.createReadStream({start: k, end: k + '~'}) + .on('data', function (data) { + var k = data.key.split('-'); + var v = data.value.split(':'); + for(var i=0; i Date: Tue, 4 Feb 2014 04:06:03 -0300 Subject: [PATCH 06/25] addr test working ~50% --- app/models/Address.js | 7 ++-- lib/TransactionDb.js | 50 ++++++++++++++++++++++++-- test/integration/01-transactionouts.js | 2 +- test/integration/addr.js | 13 +------ test/integration/addr.json | 8 +++++ 5 files changed, 61 insertions(+), 19 deletions(-) diff --git a/app/models/Address.js b/app/models/Address.js index be918b3..e6717e7 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -5,11 +5,12 @@ require('classtool'); function spec() { var async = require('async'); - var TransactionOut = require('./TransactionOut'); var BitcoreAddress = require('bitcore/Address').class(); var BitcoreUtil = require('bitcore/util/util'); + var TransactionDb = require('../../lib/TransactionDb').class(); function Address(addrStr) { + this.tdb = new TransactionDb(); this.balanceSat = 0; this.totalReceivedSat = 0; this.totalSentSat = 0; @@ -71,14 +72,14 @@ function spec() { }, */ function (cb) { - TransactionOut.find({addr:self.addrStr}).exec(function(err,txOut){ + self.tdb.fromAddr(self.addrStr, function(err,txOut){ if (err) return cb(err); txOut.forEach(function(txItem){ self.totalReceivedSat += txItem.value_sat; self.transactions.push(txItem.txid); - if (! txItem.spendTxIdBuf) { + if (! txItem.spendTxId) { // unspent self.balanceSat += txItem.value_sat; self.txApperances +=1; diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index e76eeb3..b4c3ecb 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -109,7 +109,6 @@ function spec() { }; - TransactionDb.prototype.fromTxIdN = function(txid, n, cb) { var self = this; @@ -119,13 +118,58 @@ function spec() { if (err && err.notFound) { err = null; } - var a = val.split('-'); - return cb(err, val, a[0], a[1]); }); }; + TransactionDb.prototype.fromAddr = function(addr, cb) { + var self = this; + + var k = ADDR_ROOT + addr; + var ret=[]; + + //var ADDR_ROOT = 'txouts-addr-'; //txouts-addr---- => + btc_sat + //var SPEND_ROOT = 'txouts-spend-';//txouts-spend-- => [txid(in),n(in),ts] + // + // + self.db.createReadStream({start: k, end: k + '~'}) + .on('data', function (data) { + var k = data.key.split('-'); + var v = data.value.split(':'); + ret.push({ + value_sat: v[0], + ts: k[3], + txid: k[4], + index: k[5], + }); + }) + .on('error', function (err) { + return cb(err); + }) + .on('end', function () { + async.each(ret, function(o, e_c) { + var k = SPEND_ROOT + '-' + o.txid + '-' + o.index; //TODO --- + self.db.get(k, function(err, val) { + if (err && err.notFound) err=null; + if (err || !val) return e_c(err); + + var v = val.split(':'); + o.spendTxId= v[0]; + o.spendIndex=v[1]; + return e_c(); + }); + }, + function(err) { + +console.log('[TransactionDb.js.165]', ret); //TODO + return cb(err,ret); + }); + }); + }; + + + // Only for testing. Very slow (toRm outs to rm, only to speedup) TransactionDb.prototype.removeFromTxId = function(txid, toRm, cb) { var self = this; diff --git a/test/integration/01-transactionouts.js b/test/integration/01-transactionouts.js index fc5a316..b9e5ef5 100644 --- a/test/integration/01-transactionouts.js +++ b/test/integration/01-transactionouts.js @@ -14,7 +14,7 @@ var var txItemsValid = JSON.parse(fs.readFileSync('test/integration/txitems.json')); -describe('TransactionOut', function(){ +describe('TransactionDb', function(){ var tdb = new TransactionDb(); diff --git a/test/integration/addr.js b/test/integration/addr.js index 362bcac..1239614 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -5,23 +5,12 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var assert = require('assert'), fs = require('fs'), - mongoose= require('mongoose'), config = require('../../config/config'), Address = require('../../app/models/Address').class(); - addrValid = JSON.parse(fs.readFileSync('test/model/addr.json')); + addrValid = JSON.parse(fs.readFileSync('test/integration/addr.json')); describe('Address balances', function(){ - before(function(done) { - mongoose.connect(config.db); - done(); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); - addrValid.forEach( function(v) { if (v.disabled) { console.log(v.addr + " => disabled in JSON"); diff --git a/test/integration/addr.json b/test/integration/addr.json index 1812aab..fd174e6 100644 --- a/test/integration/addr.json +++ b/test/integration/addr.json @@ -1,10 +1,12 @@ [ { +"disabled":1, "addr": "mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ", "balance": 43.1, "txApperances": 19 }, { +"disabled":1, "addr": "mp3Rzxx9s1A21SY3sjJ3CQoa2Xjph7e5eS", "balance": 0, "totalReceived": 50, @@ -12,6 +14,7 @@ "txApperances": 2 }, { +"disabled":1, "addr": "muyg1K5WsHkfMVCkUXU2y7Xp5ZD6RGzCeH", "balance": 0.38571339, "totalReceived": 0.38571339, @@ -26,6 +29,7 @@ "txApperances": 13 }, { +"disabled":1, "addr": "n47CfqnKWdNwqY1UWxTmNJAqYutFxdH3zY", "balance": 0, "totalReceived":26.4245, @@ -33,6 +37,7 @@ "txApperances": 4 }, { +"disabled":1, "addr": "mzSyyXgofoBxpr6gYcU3cV345G8hJpixRd", "balance": 0, "totalReceived":3.9775, @@ -40,6 +45,7 @@ "txApperances": 2 }, { +"disabled":1, "addr": "mzW2hdZN2um7WBvTDerdahKqRgj3md9C29", "txApperances": 2034, "balance": 1049.69744099, @@ -47,6 +53,7 @@ "totalSent": 0 }, { +"disabled":1, "addr": "mjRmkmYzvZN3cA3aBKJgYJ65epn3WCG84H", "txApperances": 13343, "balance": 46413.0, @@ -54,6 +61,7 @@ "totalSent": 310717.17644359 }, { +"disabled":1, "addr": "mgKY35SXqxFpcKK3Dq9mW9919N7wYXvcFM", "txApperances": 1, "balance": 0.01979459, From fd4f6e82cc26e2a5be1d2ad306bb4a1dcff608c9 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 4 Feb 2014 13:06:05 -0300 Subject: [PATCH 07/25] simple test for sync --- lib/BlockDb.js | 39 +++++++++++++-------- lib/HistoricSync.js | 72 ++++++++++++++++++++++++--------------- lib/Sync.js | 24 ++++++------- lib/TransactionDb.js | 20 +++++------ package.json | 1 - test/integration/block.js | 23 ++++++++----- test/integration/sync.js | 71 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 173 insertions(+), 77 deletions(-) create mode 100644 test/integration/sync.js diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 44831ae..13dd2e8 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -16,8 +16,7 @@ function spec() { util = require('bitcore/util/util'), levelup = require('levelup'), BitcoreBlock= require('bitcore/Block').class(), - config = require('../config/config'), - fs = require('fs'); + config = require('../config/config'); var BlockDb = function() { this.db = levelup(config.leveldb + '/blocks'); @@ -26,11 +25,11 @@ function spec() { BlockDb.prototype.drop = function(cb) { var self = this; var path = config.leveldb + '/blocks'; - require('leveldown').destroy(path, function () { - fs.mkdirSync(config.leveldb); - fs.mkdirSync(path); - self.db = levelup(path); - return cb(); + self.db.close(function() { + require('leveldown').destroy(path, function () { + self.db = levelup(path); + return cb(); + }); }); }; @@ -52,21 +51,36 @@ function spec() { BlockDb.prototype.setOrphan = function(hash, cb) { var self = this; - var k = PREV_ROOT + hash; + self.db.get(k, function (err,oldPrevHash) { if (err || !oldPrevHash) return cb(err); - self.db.put(PREV_ROOT + hash, 0, function() { return cb(err, oldPrevHash); }); }); - // We keep the block in TIMESTAMP_ROOT }; - BlockDb.prototype.countNotOrphan = function(hash, cb) { + //mainly for testing + BlockDb.prototype.setPrev = function(hash, prevHash, cb) { + this.db.put(PREV_ROOT + hash, prevHash, function(err) { + return cb(err); + }); + }; + + //mainly for testing + BlockDb.prototype.getPrev = function(hash, cb) { + this.db.get(PREV_ROOT + hash, function(err,val) { + return cb(err,val); + }); + }; + + + + BlockDb.prototype.countNotOrphan = function(cb) { + var self = this; var c = 0; this.db.createReadStream({start: PREV_ROOT}) .on('data', function (data) { @@ -75,9 +89,6 @@ function spec() { .on('error', function (err) { return cb(err); }) - .on('close', function () { - return cb(null); - }) .on('end', function () { return cb(null); }); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 69ac2ea..1c2c3ee 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -313,11 +313,12 @@ var kk=0; }, //store it function(c) { - if (self.prevHash && blockInfo.prev_block !== self.prevHash) { + if (self.prevHash && blockinfo.prev_block !== self.prevHash) { - console.log('Orphans found: %s vs %s @%s', - self.prevHash + ' vs. ' + blockInfo.prev_block, blockInfo.hash); + console.log('Orphans found: @%s', blockInfo.hash); + console.log('From: %s To: %s', self.prevHash, blockInfo.prev_block); +process.exit(1); self.sync.setOrphan(self.prevHash, blockInfo.prev_block, c); } else return c(); @@ -349,6 +350,17 @@ var kk=0; }; + HistoricSync.prototype.getBlockCount = function(cb) { + var self = this; + + if (self.blockChainHeight) return cb(); + + self.rpc.getBlockCount(function(err, res) { + if (err) return cb(err); + self.blockChainHeight = res.result; + return cb(); + }); + }; HistoricSync.prototype.importHistory = function(scanOpts, next) { var self = this; @@ -364,13 +376,7 @@ var kk=0; return cb(); }, // We are not using getBestBlockHash, because is not available in all clients - function(cb) { - self.rpc.getBlockCount(function(err, res) { - if (err) return cb(err); - self.blockChainHeight = res.result; - return cb(); - }); - }, + function (cb) { return self.getBlockCount(cb); }, function(cb) { if (!scanOpts.reverse) return cb(); @@ -461,27 +467,37 @@ var kk=0; var self = this; self.sync.hasBlock(self.genesis, function(err, b) { - if (err) return next(err); + self.sync.countNotOrphan(function(err, count) { + if (err) return next(err); + self.getBlockCount(function(err) { + if (err) return next(err); - if (!b || scanOpts.destroy) { - p('Could not find Genesis block. Running FULL SYNC'); - if (config.bitcoind.dataDir) { - p('bitcoind dataDir configured...importing blocks from .dat files'); - scanOpts.fromFiles = true; - self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); - } - else { - scanOpts.reverse = true; - } - } - else { - p('Genesis block found. Syncing upto known blocks.'); - scanOpts.reverse = true; - scanOpts.upToExisting = true; - } + if (!b || scanOpts.destroy || self.blockChainHeight > count * 0.7 ) { - return self.importHistory(scanOpts, next); + if (!b) + p('Could not find Genesis block. Running FULL SYNC'); + else + p('Less that 70% of current blockchain is stored. Running FULL SYNC', + parseInt(count/self.blockChainHeight*100)); + + if (config.bitcoind.dataDir) { + p('bitcoind dataDir configured...importing blocks from .dat files'); + scanOpts.fromFiles = true; + self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); + } + else { + scanOpts.reverse = true; + } + } + else { + p('Genesis block found. Syncing upto known blocks.'); + scanOpts.reverse = true; + scanOpts.upToExisting = true; + } + return self.importHistory(scanOpts, next); + }); + }); }); }; diff --git a/lib/Sync.js b/lib/Sync.js index bfa2d55..27362de 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -63,8 +63,7 @@ function spec() { var self = this; async.series([ function(b) { try {self.blockDb.drop(b);} catch (e) { return b(); } }, - function(b) { try {self.db.collections.transactionitems.drop(b);} catch (e) { return b(); } }, - function(b) { try {self.db.collections.transactionouts.drop(b);} catch (e) { return b(); } }, + function(b) { try {self.TransactionDb.drop(b);} catch (e) { return b(); } }, ], next); }; @@ -81,7 +80,6 @@ function spec() { }; - Sync.prototype.storeBlock = function(block, cb) { var self = this; @@ -99,26 +97,24 @@ function spec() { Sync.prototype.setOrphan = function(fromBlock, toBlock, c) { var self = this; - var c = fromBlock; + var hash = fromBlock; async.whilst( function () { - return c !== toBlock; + return hash && hash !== toBlock; }, - function () { - console.log('[Sync.js.113]: setOrphan', c); //TODO + function (w_c) { self.txDb.setOrphan(c, function(err, insertedTxs, updateAddrs) { - if (err) return cb(err); + if (err) return w_c(err); + self.blockDb.setOrphan(hash, function(err, prevHash){ - self.blockDb.setOrphan(c, function(err, prevHash){ - - c = prevHash; - return cb(err); + hash = prevHash; + return w_c(err); }); }); }, - function (err) { - return c(err); + function (err) { + return c(err); } ); }; diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index b4c3ecb..f3ab69b 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -26,8 +26,7 @@ function spec() { util = require('bitcore/util/util'), levelup = require('levelup'), async = require('async'), - config = require('../config/config'), - fs = require('fs'); + config = require('../config/config'); var TransactionDb = function() { this.db = levelup(config.leveldb + '/txs'); @@ -35,12 +34,12 @@ function spec() { TransactionDb.prototype.drop = function(cb) { var self = this; - var path = config.leveldb + '/blocks'; - require('leveldown').destroy(path, function () { - fs.mkdirSync(config.leveldb); - fs.mkdirSync(path); - self.db = levelup(path); - return cb(); + var path = config.leveldb + '/txs'; + self.db.close(function() { + require('leveldown').destroy(path, function () { + self.db = levelup(path); + return cb(); + }); }); }; @@ -129,9 +128,6 @@ function spec() { var k = ADDR_ROOT + addr; var ret=[]; - //var ADDR_ROOT = 'txouts-addr-'; //txouts-addr---- => + btc_sat - //var SPEND_ROOT = 'txouts-spend-';//txouts-spend-- => [txid(in),n(in),ts] - // // self.db.createReadStream({start: k, end: k + '~'}) .on('data', function (data) { @@ -149,7 +145,7 @@ function spec() { }) .on('end', function () { async.each(ret, function(o, e_c) { - var k = SPEND_ROOT + '-' + o.txid + '-' + o.index; //TODO --- + var k = SPEND_ROOT + o.txid + '-' + o.index; self.db.get(k, function(err, val) { if (err && err.notFound) err=null; if (err || !val) return e_c(err); diff --git a/package.json b/package.json index 6860d54..4a6e18d 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,6 @@ "bignum": "*", "express": "~3.4.7", "jade": "~1.0.2", - "mongoose": "~3.8.3", "lodash": "~2.4.1", "bower": "~1.2.8", "buffertools": "*", diff --git a/test/integration/block.js b/test/integration/block.js index 1b6ea50..3650313 100644 --- a/test/integration/block.js +++ b/test/integration/block.js @@ -27,18 +27,25 @@ describe('BlockDb fromHashWithInfo', function(){ it('return true in has', function(done) { bdb.has(TESTING_BLOCK, function(err, has) { assert.equal(has, true); -console.log('[block.js.29:has:]',has); //TODO done(); }); }); - it('return false in has', function(done) { - bdb.has('111', function(err, has) { - assert.equal(has, false); - done(); + it('setOrphan', function(done) { + var b16 = '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b'; + var b17 = '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743'; + + bdb.has(b17, function(err, has) { + assert(has); + bdb.setOrphan(b17, function(err, oldPrev) { + assert.equal(oldPrev, b16); + bdb.setPrev(b17, b16, function(err, oldPrev) { + bdb.getPrev(b17, function(err, p) { + assert.equal(p, b16); + done(); + }); + }); + }); }); }); - - - }); diff --git a/test/integration/sync.js b/test/integration/sync.js new file mode 100644 index 0000000..f0e2589 --- /dev/null +++ b/test/integration/sync.js @@ -0,0 +1,71 @@ +#!/usr/bin/env node +'use strict'; + +process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + + +var + assert = require('assert'), + async = require('async'), + Sync = require('../../lib/Sync').class(); + + +var b = [ + '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b', //B#16 + '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743', + '000000008d55c3e978639f70af1d2bf1fe6f09cb3143e104405a599215c89a48', + '000000009b3bca4909f38313f2746120129cce4a699a1f552390955da470c5a9', + '00000000ede57f31cc598dc241d129ccb4d8168ef112afbdc870dc60a85f5dd3', //B#20 +]; + +var fix = function(s,cb) { + async.each([1,2,3,4], function(i,c) { + s.blockDb.setPrev(b[i],b[i-1], function() { + return c(); + }); + }, cb); +}; + +var test = function(s,cb) { + async.each([2,3,4], function(i,c) { + s.blockDb.getPrev(b[i], function(err, p) { + assert.equal(p,0); + return c(); + }); + }, function() { + s.blockDb.getPrev(b[1], function(err, p) { + assert.equal(p,b[0]); + return cb(); + }); + }); +}; + + + +var s; +describe('Sync setOrphan', function(){ + + before(function(done) { + s = new Sync(); + fix(s,done); + }); + + after(function(done) { + fix(s,done); + }); + + it('setOrphan', function(done) { + this.timeout(100000); + + s.blockDb.has(b[0], function(err, has) { + assert(has); + s.blockDb.has(b[1], function(err, has) { + assert(has); + s.setOrphan(b[4],b[1], function() { + test(s,done); + }); + }); + }); + }); +}); + From 0881b8e5380eb9bab1bfc032b935fa59d07461c2 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 4 Feb 2014 14:14:09 -0300 Subject: [PATCH 08/25] fix orphanazer --- lib/HistoricSync.js | 2 +- lib/Sync.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 1c2c3ee..b84ca9e 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -313,7 +313,7 @@ var kk=0; }, //store it function(c) { - if (self.prevHash && blockinfo.prev_block !== self.prevHash) { + if (self.prevHash && blockInfo.prev_block !== self.prevHash) { console.log('Orphans found: @%s', blockInfo.hash); diff --git a/lib/Sync.js b/lib/Sync.js index 27362de..17c524e 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -104,7 +104,7 @@ function spec() { return hash && hash !== toBlock; }, function (w_c) { - self.txDb.setOrphan(c, function(err, insertedTxs, updateAddrs) { + self.txDb.setOrphan(hash, function(err, insertedTxs, updateAddrs) { if (err) return w_c(err); self.blockDb.setOrphan(hash, function(err, prevHash){ From fe8bb95e28e1542e5bfdb35adaa45f49aa17090d Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Tue, 4 Feb 2014 17:22:51 -0300 Subject: [PATCH 09/25] block test: list of hashes by date --- README.md | 11 ++++++----- lib/BlockDb.js | 22 ++++++++++++++++++++++ test/integration/blocklist.js | 28 ++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 test/integration/blocklist.js diff --git a/README.md b/README.md index 4ee92d8..9b8d691 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,6 @@ $ npm install -g bower ## Additional Packages * Express - Defined as npm module in the [package.json](package.json) file. -* Mongoose - Defined as npm module in the [package.json](package.json) file. * AngularJS - Defined as bower module in the [bower.json](bower.json) file. * Twitter Bootstrap - Defined as bower module in the [bower.json](bower.json) file. * UI Bootstrap - Defined as bower module in the [bower.json](bower.json) file. @@ -55,13 +54,15 @@ $ npm install -g bower http://localhost:3000 - If you get an error, please check the next section "Post-install" - ## Syncing old blockchain data - Run sync from insight repository (to save old blocks and transactions in MongoDB): + Run sync from insight repository (to save old blocks and transactions in + LevelDB): + + Create folders: - $ utils/sync.js + $ mkdir -p db/blocks + $ utils/sync.js -S Check utils/sync.js --help for options. diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 13dd2e8..35f3adb 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -129,6 +129,28 @@ function spec() { }); }; + BlockDb.prototype.getBlocksByDate = function(start_ts, end_ts, cb) { + var self = this; + var list = []; + this.db.createReadStream({ + start: TIMESTAMP_ROOT + start_ts, + end: TIMESTAMP_ROOT + end_ts + }) + .on('data', function (data) { + list.push({ + ts: data.key.replace(TIMESTAMP_ROOT, ''), + hash: data.value, + info: {} + }); + }) + .on('error', function (err) { + return cb(err); + }) + .on('end', function () { + return cb(null, list); + }); + }; + BlockDb.blockIndex = function(height, cb) { var rpc = new RpcClient(config.bitcoind); rpc.getBlockHash(height, function(err, bh){ diff --git a/test/integration/blocklist.js b/test/integration/blocklist.js new file mode 100644 index 0000000..6b50952 --- /dev/null +++ b/test/integration/blocklist.js @@ -0,0 +1,28 @@ +#!/usr/bin/env node +'use strict'; + +process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + +var TESTING_BLOCK = '00000000b7cc12abe8a9a604813aab1f2c4f3a242a021065be52393a147a1a86'; +var START_TS = '1391538611'; +var END_TS = '1391538638'; + +var + assert = require('assert'), + config = require('../../config/config'), + BlockDb = require('../../lib/BlockDb').class(); + +describe('BlockDb getHashes', function(){ + + var bdb = new BlockDb(); + it('Get Hash by Date', function(done) { + + bdb.getBlocksByDate(START_TS, END_TS, function(err, list) { + if (err) done(err); + assert.equal(list[0].ts, START_TS); + assert.equal(list[0].hash, TESTING_BLOCK); + done(); + }); + }); +}); + From d9f6de4e0caf7789de7f340aa01d51b9ee154a9f Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 4 Feb 2014 19:55:58 -0300 Subject: [PATCH 10/25] sync fixes --- lib/BlockDb.js | 8 ++++---- lib/BlockExtractor.js | 9 +++++++-- lib/HistoricSync.js | 29 ++++++----------------------- lib/Sync.js | 34 ++++++++++++++++++++++++++-------- lib/TransactionDb.js | 2 +- test/integration/sync.js | 25 +++++++++++++++++++++---- 6 files changed, 65 insertions(+), 42 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 13dd2e8..551d8df 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -5,8 +5,8 @@ require('classtool'); function spec() { - var TIMESTAMP_ROOT = 'b-ts-'; - var PREV_ROOT = 'b-prev-'; // b-prev- => (0 if orphan) + var TIMESTAMP_ROOT = 'b-ts-'; // b-ts- => + var PREV_ROOT = 'b-prev-'; // b-prev- => (0 if orphan) /** @@ -49,6 +49,7 @@ function spec() { }; + BlockDb.prototype.setOrphan = function(hash, cb) { var self = this; @@ -80,9 +81,8 @@ function spec() { BlockDb.prototype.countNotOrphan = function(cb) { - var self = this; var c = 0; - this.db.createReadStream({start: PREV_ROOT}) + this.db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' }) .on('data', function (data) { if (data.value !== 0) c++; }) diff --git a/lib/BlockExtractor.js b/lib/BlockExtractor.js index 857ed71..92c6740 100644 --- a/lib/BlockExtractor.js +++ b/lib/BlockExtractor.js @@ -113,7 +113,7 @@ function spec() { magic = self.currentParser ? self.currentParser.buffer(4).toString('hex') : null ; - if (!self.currentParser || self.currentParser.eof()) { + if (!self.currentParser || self.currentParser.eof() || magic === '00000000') { magic = null; if (self.nextFile()) { console.log('Moving forward to file:' + self.currentFile() ); @@ -121,7 +121,8 @@ function spec() { } else { console.log('Finished all files'); - return cb(); + magic = null; + return w_cb(); } } else { @@ -130,6 +131,8 @@ function spec() { }, a_cb); }, function (a_cb) { + if (!magic) return a_cb(); + if (magic !== self.magic) { var e = new Error('CRITICAL ERROR: Magic number mismatch: ' + magic + '!=' + self.magic); @@ -141,6 +144,8 @@ function spec() { return a_cb(); }, function (a_cb) { + if (!magic) return a_cb(); + b = new Block(); b.parse(self.currentParser); b.getHash(); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index b84ca9e..95c6b14 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -115,13 +115,6 @@ function spec() { if (self.opts.shouldBroadcast) { sockets.broadcastSyncInfo(self.info()); } - -//TODO - /* -if (self.syncPercentage>10) { - console.log(self.info()); - process.exit(1); -}*/ }; HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) { @@ -247,9 +240,7 @@ if (self.syncPercentage>10) { addrStrs = [ addr.toString() ]; break; case Script.TX_MULTISIG: - var addrs = []; var chunks = s.capture(); - chunks.forEach(function(chunk) { var a = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk)); addrStrs.push(a.toString()); @@ -262,15 +253,10 @@ if (self.syncPercentage>10) { return addrStrs; }; -var kk=0; - HistoricSync.prototype.getBlockFromFile = function(scanOpts, cb) { var self = this; - - var nextHash; var blockInfo; var existed; - async.series([ //show some (inacurate) status function(c) { @@ -313,18 +299,18 @@ var kk=0; }, //store it function(c) { - if (self.prevHash && blockInfo.prev_block !== self.prevHash) { - - console.log('Orphans found: @%s', blockInfo.hash); + if (blockInfo && self.prevHash && blockInfo.prev_block !== self.prevHash) { + console.log('Hole found @%s', blockInfo.hash); console.log('From: %s To: %s', self.prevHash, blockInfo.prev_block); -process.exit(1); - self.sync.setOrphan(self.prevHash, blockInfo.prev_block, c); + self.sync.checkOrphan(self.prevHash, blockInfo.prev_block, c); } else return c(); }, //store it function(c) { + if (!blockInfo) return c(); + self.sync.storeBlock(blockInfo, function(err) { existed = err && err.toString().match(/E11000/); @@ -343,7 +329,7 @@ process.exit(1); }, ], function(err) { if (err) { - self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo.hash, err, self.syncedBlocks)); + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); } return cb(err); }); @@ -437,9 +423,6 @@ process.exit(1); p(' scanOpts: ', JSON.stringify(scanOpts)); if (scanOpts.fromFiles) { - - var height = 0; - self.status = 'syncing'; async.whilst(function() { return self.status === 'syncing'; diff --git a/lib/Sync.js b/lib/Sync.js index 17c524e..ffd9f06 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -94,23 +94,41 @@ function spec() { }); }; - Sync.prototype.setOrphan = function(fromBlock, toBlock, c) { + Sync.prototype.checkOrphan = function(fromBlock, toBlock, c) { var self = this; var hash = fromBlock; + var co = 0; + var limit = 10; + var cont = 1; + async.whilst( function () { - return hash && hash !== toBlock; + if (++co > limit) { +console.log('[Sync.js.109] WARN: Reach reog depth limit'); //TODO + } + return cont && hash && hash !== toBlock && co < limit; }, function (w_c) { - self.txDb.setOrphan(hash, function(err, insertedTxs, updateAddrs) { - if (err) return w_c(err); - self.blockDb.setOrphan(hash, function(err, prevHash){ + //check with RPC if the block is mainchain + self.blockDb.fromHashWithInfo(hash, function (err, info) { - hash = prevHash; - return w_c(err); - }); + if (!info) { + console.log('[Sync.js.107:hash:ORPHAN]',hash); //TODO + self.txDb.setOrphan(hash, function(err) { + if (err) return w_c(err); + self.blockDb.setOrphan(hash, function(err, prevHash){ + hash = prevHash; + return w_c(err); + }); + }); + } + else { + console.log('[Sync.js.107:hash:NOT ORPHAN]',hash); //TODO + cont = 0; + return w_c(); + } }); }, function (err) { diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index f3ab69b..38924e9 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -368,7 +368,7 @@ console.log('[TransactionDb.js.165]', ret); //TODO // txs can be a [hashes] or [txObjects] TransactionDb.prototype.createFromBlock = function(b, next) { var self = this; - if (!b.tx) return next(); + if (!b || !b.tx) return next(); return self.createFromArray(b.tx, b.hash, next); }; diff --git a/test/integration/sync.js b/test/integration/sync.js index f0e2589..be6a078 100644 --- a/test/integration/sync.js +++ b/test/integration/sync.js @@ -42,8 +42,25 @@ var test = function(s,cb) { +var testNo = function(s,cb) { + async.each([2,3,4], function(i,c) { + s.blockDb.getPrev(b[i], function(err, p) { + assert.equal(p,b[i-1]); + return c(); + }); + }, function() { + s.blockDb.getPrev(b[1], function(err, p) { + assert.equal(p,b[0]); + return cb(); + }); + }); +}; + + + + var s; -describe('Sync setOrphan', function(){ +describe('Sync checkOrphan', function(){ before(function(done) { s = new Sync(); @@ -54,15 +71,15 @@ describe('Sync setOrphan', function(){ fix(s,done); }); - it('setOrphan', function(done) { + it('checkOrphan', function(done) { this.timeout(100000); s.blockDb.has(b[0], function(err, has) { assert(has); s.blockDb.has(b[1], function(err, has) { assert(has); - s.setOrphan(b[4],b[1], function() { - test(s,done); + s.checkOrphan(b[4],b[1], function() { + testNo(s,done); }); }); }); From c516c113cb66c92a75078d117fdfcdadf17a6782 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 4 Feb 2014 23:50:18 -0300 Subject: [PATCH 11/25] more fixes on sync --- app/models/Transaction.js | 23 +++-- lib/BlockDb.js | 5 +- lib/HistoricSync.js | 40 ++++++--- lib/TransactionDb.js | 71 +++++++++++++-- test/integration/01-transactionouts.js | 111 +++++++++++++++++++++-- test/integration/transaction.js | 120 ------------------------- 6 files changed, 218 insertions(+), 152 deletions(-) delete mode 100644 test/integration/transaction.js diff --git a/app/models/Transaction.js b/app/models/Transaction.js index f3b20dc..678f870 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -7,17 +7,25 @@ function spec() { var util = require('bitcore/util/util'), TransactionRpc = require('../../lib/TransactionRpc').class(), - TransactionOut = require('./TransactionOut'), + TransactionDb = require('../../lib/TransactionDb').class(), async = require('async'); var CONCURRENCY = 20; - function Transaction() { + function Transaction(tdb) { this.txid = null; + this.tdb = tdb || new TransactionDb(); } - Transaction.fromIdWithInfo = function (txid,cb) { - var tx = new Transaction(); + Transaction.fromIdWithInfo = function (txid, tdb, cb) { + if (typeof tdb === 'function') { + cb = tdb; + tdb = null; + } + var tx = new Transaction(tdb); + + +console.log('[Transaction.js.27]',tx.tdb); //TODO tx.txid = txid; tx._fillInfo(function(err) { @@ -33,21 +41,22 @@ function spec() { TransactionRpc.getRpcInfo(self.txid, function(err, info) { if (err) return next(err); - Transaction._fillOutpoints(info, function() { + self._fillOutpoints(info, function() { self.info = info; return next(); }); }); }; - Transaction._fillOutpoints = function(info, cb) { + Transaction.prototype._fillOutpoints = function(info, cb) { + var self = this; if (!info || info.isCoinBase) return cb(); var valueIn = 0; var incompleteInputs = 0; async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) { - TransactionOut.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) { + self.tdb.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) { if (err || !addr || !valueSat ) { diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 551d8df..51dd7ea 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -44,7 +44,7 @@ function spec() { self.db.batch() .put(time_key, b.hash) - .put(PREV_ROOT + b.hash, b.prev_block) + .put(PREV_ROOT + b.hash, b.previousblockhash) .write(cb); }; @@ -82,6 +82,7 @@ function spec() { BlockDb.prototype.countNotOrphan = function(cb) { var c = 0; + console.log('Counting connected blocks. This could take some minutes'); this.db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' }) .on('data', function (data) { if (data.value !== 0) c++; @@ -90,7 +91,7 @@ function spec() { return cb(err); }) .on('end', function () { - return cb(null); + return cb(null, c); }); }; diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 95c6b14..eab593e 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -9,7 +9,6 @@ function spec() { var RpcClient = require('bitcore/RpcClient').class(); var bitutil = require('bitcore/util/util'); var Address = require('bitcore/Address').class(); - var Deserialize = require('bitcore/Deserialize'); var Script = require('bitcore/Script').class(); var networks = require('bitcore/networks'); var async = require('async'); @@ -17,6 +16,7 @@ function spec() { var Sync = require('./Sync').class(); var sockets = require('../app/controllers/socket.js'); var BlockExtractor = require('./BlockExtractor.js').class(); + //var Deserialize = require('bitcore/Deserialize'); var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:'; @@ -37,6 +37,7 @@ function spec() { this.syncedBlocks = 0; this.skippedBlocks = 0; this.orphanBlocks = 0; + this.type =''; } function p() { @@ -97,6 +98,7 @@ function spec() { syncedBlocks: this.syncedBlocks, orphanBlocks: this.orphanBlocks, error: this.error, + type: this.type, }; }; @@ -273,7 +275,9 @@ function spec() { if (err || ! b) return c(err); blockInfo = b.getStandardizedObject(b.txs, self.network); - blockInfo.curWork = Deserialize.intFromCompact(b.bits); + // blockInfo.curWork = Deserialize.intFromCompact(b.bits); + // We keep the RPC field names + blockInfo.previousblockhash = blockInfo.prev_block; var ti=0; // Get TX Address @@ -297,31 +301,40 @@ function spec() { return c(); }); }, - //store it + //check prev function(c) { - if (blockInfo && self.prevHash && blockInfo.prev_block !== self.prevHash) { + if (blockInfo && self.prevHash && blockInfo.previousblockhash !== self.prevHash) { console.log('Hole found @%s', blockInfo.hash); - console.log('From: %s To: %s', self.prevHash, blockInfo.prev_block); - self.sync.checkOrphan(self.prevHash, blockInfo.prev_block, c); + console.log('From: %s To: %s', self.prevHash, blockInfo.previousblockhash); + self.sync.checkOrphan(self.prevHash, blockInfo.previousblockhash, c); } else return c(); }, - //store it + //check it function(c) { if (!blockInfo) return c(); - self.sync.storeBlock(blockInfo, function(err) { - existed = err && err.toString().match(/E11000/); + self.sync.blockDb.has(blockInfo.hash, function(err, had) { + existed = had; + return c(err); + }); + }, + //store it + function(c) { + if (!blockInfo || existed) return c(); - if (err && ! existed) return c(err); - return c(); + self.sync.storeBlock(blockInfo, function(err) { + return c(err); }); }, function(c) { if (blockInfo && blockInfo.hash) { self.prevHash = blockInfo.hash; - self.syncedBlocks++; + if (existed) + self.skippedBlocks++; + else + self.syncedBlocks++; } else self.status = 'finished'; @@ -424,6 +437,7 @@ function spec() { if (scanOpts.fromFiles) { self.status = 'syncing'; + self.type = 'from .dat Files'; async.whilst(function() { return self.status === 'syncing'; }, function (w_cb) { @@ -437,6 +451,7 @@ function spec() { }); } else { + self.type = 'from RPC calls'; self.getPrevNextBlock(start, end, scanOpts, function(err) { return next(err); }); @@ -475,6 +490,7 @@ function spec() { } else { p('Genesis block found. Syncing upto known blocks.'); + p('Got %d out of %d blocks', count, self.blockChainHeight); scanOpts.reverse = true; scanOpts.upToExisting = true; } diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 38924e9..3f8394c 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -5,8 +5,8 @@ require('classtool'); function spec() { - // blockHash -> txid mapping (to orphanize )/ - var ROOT = 'tx-b-'; //tx-b- => txid + // blockHash -> txid mapping (to reorgs) + var ROOT = 'tx-b-'; //tx-b-- => 1 // to show tx outs var OUTS_ROOT = 'txouts-'; //txouts-- => [addr, btc_sat] @@ -107,6 +107,65 @@ function spec() { }); }; + TransactionDb.prototype._fillOutpoints = function(info, cb) { + var self = this; + + if (!info || info.isCoinBase) return cb(); + + var valueIn = 0; + var incompleteInputs = 0; + async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) { + self.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) { + + + if (err || !addr || !valueSat ) { + console.log('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid); + incompleteInputs = 1; + return c_in(); // error not scaled + } + i.addr = addr; + i.valueSat = valueSat; + i.value = valueSat / util.COIN; + + valueIn += i.valueSat; + return c_in(); + }); + }, + function () { + if (! incompleteInputs ) { + info.valueIn = valueIn / util.COIN; + info.fees = (valueIn - parseInt(info.valueOut * util.COIN)) / util.COIN ; + } + else { + info.incompleteInputs = 1; + } + return cb(); + }); + }; + + TransactionDb.prototype._getInfo = function(txid, next) { + var self = this; + + TransactionRpc.getRpcInfo(txid, function(err, info) { + if (err) return next(err); + + self._fillOutpoints(info, function() { + return next(null, info); + }); + }); + }; + + + TransactionDb.prototype.fromIdWithInfo = function (txid, cb) { + var self = this; + + self._getInfo(txid, function(err, info) { + if (err) return cb(err); + if (!info ) return cb(); + + return cb(err, {txid: txid, info: info} ); + }); + }; TransactionDb.prototype.fromTxIdN = function(txid, n, cb) { var self = this; @@ -117,8 +176,8 @@ function spec() { if (err && err.notFound) { err = null; } - var a = val.split('-'); - return cb(err, val, a[0], a[1]); + var a = val.split(':'); + return cb(err, a[0], a[1]); }); }; @@ -343,7 +402,7 @@ console.log('[TransactionDb.js.165]', ret); //TODO self.add(inInfo, function(err) { if (err || !blockHash) return each_cb(err); - self.db.put(ROOT + blockHash, t, function(err) { + self.db.put(ROOT + t + '-' + blockHash, 1, function(err) { return each_cb(err); }); }); @@ -353,7 +412,7 @@ console.log('[TransactionDb.js.165]', ret); //TODO self.add(t, function(err) { if (err) return each_cb(err); - self.db.put(ROOT + blockHash, t.txid, function(err) { + self.db.put(ROOT + t.txid + '-' + blockHash, 1, function(err) { return each_cb(err); }); }); diff --git a/test/integration/01-transactionouts.js b/test/integration/01-transactionouts.js index b9e5ef5..12705fd 100644 --- a/test/integration/01-transactionouts.js +++ b/test/integration/01-transactionouts.js @@ -5,16 +5,117 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var - assert = require('assert'), +var assert = require('assert'), fs = require('fs'), util = require('util'), - config = require('../../config/config'), TransactionDb = require('../../lib/TransactionDb').class(); var txItemsValid = JSON.parse(fs.readFileSync('test/integration/txitems.json')); +var tdb; -describe('TransactionDb', function(){ +describe('TransactionDb fromIdWithInfo', function(){ + + before(function(c) { + tdb = new TransactionDb(); + return c(); + }); + + var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c'; + it('tx info ' + txid, function(done) { + tdb.fromIdWithInfo(txid, function(err, tx) { + + if (err) done(err); + assert.equal(tx.txid, txid); + assert(!tx.info.isCoinBase); + + for(var i=0; i<20; i++) + assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i); + assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0'); + assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1'); + done(); + }); + }); + + it('tx info', function(done) { + var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; + tdb.fromIdWithInfo(txid, function(err, tx) { + if (err) done(err); + assert.equal(tx.txid, txid); + assert(!tx.info.isCoinBase); + done(); + }); + }); + + it('should pool tx\'s info from bitcoind', function(done) { + var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; + tdb.fromIdWithInfo(txid, function(err, tx) { + if (err) done(err); + assert.equal(tx.info.txid, txid); + assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74'); + assert.equal(tx.info.valueOut, 1.66174); + assert.equal(tx.info.fees, 0.0005 ); + assert.equal(tx.info.size, 226 ); + assert(!tx.info.isCoinBase); + done(); + }); + }); + + var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399'; + it('test a coinbase TX ' + txid1, function(done) { + tdb.fromIdWithInfo(txid1, function(err, tx) { + if (err) done(err); + assert(tx.info.isCoinBase); + assert.equal(tx.info.txid, txid1); + assert(!tx.info.feeds); + done(); + }); + }); + var txid22 = '666'; + it('test invalid TX ' + txid22, function(done) { + tdb.fromIdWithInfo(txid22, function(err, tx) { + if (err && err.message.match(/must.be.hexadecimal/)) { + return done(); + } + else { + return done(err); + } + }); + }); + + var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227'; + it('test unexisting TX ' + txid23, function(done) { + + tdb.fromIdWithInfo(txid23, function(err, tx) { + assert(!err); + assert(!tx); + return done(); + }); + }); + + + + var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; + it('create TX on the fly ' + txid2, function(done) { + tdb.fromIdWithInfo(txid2, function(err, tx) { + if (err) return done(err); + assert.equal(tx.info.txid, txid2); + done(); + }); + }); + + txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; + it('test a broken TX ' + txid2, function(done) { + tdb.fromIdWithInfo(txid2, function(err, tx) { + if (err) return done(err); + assert.equal(tx.info.txid, txid2); + assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU'); + done(); + }); + }); +}); +/* + +describe('TransactionDb Outs', function(){ var tdb = new TransactionDb(); @@ -63,5 +164,5 @@ describe('TransactionDb', function(){ }); }); - +*/ diff --git a/test/integration/transaction.js b/test/integration/transaction.js deleted file mode 100644 index d5a87c0..0000000 --- a/test/integration/transaction.js +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -process.env.NODE_ENV = process.env.NODE_ENV || 'development'; - - - -var mongoose= require('mongoose'), - assert = require('assert'), - config = require('../../config/config'), - Transaction = require('../../app/models/Transaction').class(); - - -mongoose.connection.on('error', function(err) { console.log(err); }); - -describe('Transaction', function(){ - - before(function(done) { - mongoose.connect(config.db); - done(); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); - var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c'; - it('txid ' + txid, function(done) { - Transaction.fromIdWithInfo(txid, function(err, tx) { - if (err) done(err); - assert.equal(tx.txid, txid); - assert(!tx.info.isCoinBase); - - for(var i=0; i<20; i++) - assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i); - assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0'); - assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1'); - done(); - }); - }); - - it('should pool tx\'s object from mongoose', function(done) { - var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; - Transaction.fromIdWithInfo(txid, function(err, tx) { - if (err) done(err); - assert.equal(tx.txid, txid); - assert(!tx.info.isCoinBase); - done(); - }); - }); - - it('should pool tx\'s info from bitcoind', function(done) { - var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; - Transaction.fromIdWithInfo(txid, function(err, tx) { - if (err) done(err); - assert.equal(tx.info.txid, txid); - assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74'); - assert.equal(tx.info.valueOut, 1.66174); - assert.equal(tx.info.fees, 0.0005 ); - assert.equal(tx.info.size, 226 ); - assert(!tx.info.isCoinBase); - done(); - }); - }); - - var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399'; - it('test a coinbase TX ' + txid1, function(done) { - Transaction.fromIdWithInfo(txid1, function(err, tx) { - if (err) done(err); - assert(tx.info.isCoinBase); - assert.equal(tx.info.txid, txid1); - assert(!tx.info.feeds); - done(); - }); - }); - var txid22 = '666'; - it('test invalid TX ' + txid22, function(done) { - Transaction.fromIdWithInfo(txid22, function(err, tx) { - if (err && err.message.match(/must.be.hexadecimal/)) { - return done(); - } - else { - return done(err); - } - }); - }); - - var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227'; - it('test unexisting TX ' + txid23, function(done) { - - Transaction.fromIdWithInfo(txid23, function(err, tx) { - assert(!err); - assert(!tx); - return done(); - }); - }); - - - - var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; - it('create TX on the fly ' + txid2, function(done) { - Transaction.fromIdWithInfo(txid2, function(err, tx) { - if (err) return done(err); - assert.equal(tx.info.txid, txid2); - done(); - }); - }); - - var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; - it('test a broken TX ' + txid2, function(done) { - Transaction.fromIdWithInfo(txid2, function(err, tx) { - if (err) return done(err); - assert.equal(tx.info.txid, txid2); - assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU'); - done(); - }); - }); - -}); - From 71b27aede00676d72129dceb6d2b865170e48556 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 5 Feb 2014 02:48:54 -0300 Subject: [PATCH 12/25] better remove in txDB --- app/models/Transaction.js | 91 -------------------- lib/HistoricSync.js | 32 ++++--- lib/Sync.js | 16 +--- lib/TransactionDb.js | 35 +++----- test/integration/01-transactionouts.js | 32 +++---- test/integration/transactionouts-expenses.js | 67 -------------- test/integration/txitems.json | 3 +- 7 files changed, 52 insertions(+), 224 deletions(-) delete mode 100644 app/models/Transaction.js delete mode 100644 test/integration/transactionouts-expenses.js diff --git a/app/models/Transaction.js b/app/models/Transaction.js deleted file mode 100644 index 678f870..0000000 --- a/app/models/Transaction.js +++ /dev/null @@ -1,91 +0,0 @@ -'use strict'; - -require('classtool'); - - -function spec() { - - var util = require('bitcore/util/util'), - TransactionRpc = require('../../lib/TransactionRpc').class(), - TransactionDb = require('../../lib/TransactionDb').class(), - async = require('async'); - - var CONCURRENCY = 20; - - function Transaction(tdb) { - this.txid = null; - this.tdb = tdb || new TransactionDb(); - } - - Transaction.fromIdWithInfo = function (txid, tdb, cb) { - if (typeof tdb === 'function') { - cb = tdb; - tdb = null; - } - var tx = new Transaction(tdb); - - -console.log('[Transaction.js.27]',tx.tdb); //TODO - tx.txid = txid; - - tx._fillInfo(function(err) { - if (err) return cb(err); - if (! tx.info ) return cb(); - - return cb(err,tx); - }); - }; - - Transaction.prototype._fillInfo = function(next) { - var self = this; - - TransactionRpc.getRpcInfo(self.txid, function(err, info) { - if (err) return next(err); - self._fillOutpoints(info, function() { - self.info = info; - return next(); - }); - }); - }; - - Transaction.prototype._fillOutpoints = function(info, cb) { - var self = this; - - if (!info || info.isCoinBase) return cb(); - - var valueIn = 0; - var incompleteInputs = 0; - async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) { - self.tdb.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) { - - - if (err || !addr || !valueSat ) { - console.log('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid); - incompleteInputs = 1; - return c_in(); // error not scaled - } - i.addr = addr; - i.valueSat = valueSat; - i.value = valueSat / util.COIN; - - valueIn += i.valueSat; - return c_in(); - }); - }, - function () { - if (! incompleteInputs ) { - info.valueIn = valueIn / util.COIN; - info.fees = (valueIn - parseInt(info.valueOut * util.COIN)) / util.COIN ; - } - else { - info.incompleteInputs = 1; - } - return cb(); - }); - }; - - return Transaction; -} -module.defineClass(spec); - - diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index eab593e..3747fe9 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -131,7 +131,7 @@ function spec() { async.series([ // Already got it? function(c) { - self.sync.hasBlock(blockHash, function(err, ret) { + self.sync.blockDb.has(blockHash, function(err, ret) { if (err) { p(err); return c(err); @@ -163,11 +163,7 @@ function spec() { if (existed) return c(); self.sync.storeBlock(blockInfo, function(err) { - - existed = err && err.toString().match(/E11000/); - - if (err && ! existed) return c(err); - return c(); + return c(err); }); }, /* TODO: Should Start to sync backwards? (this is for partial syncs) @@ -349,6 +345,20 @@ function spec() { }; + HistoricSync.prototype.countNotOrphan = function(cb) { + var self = this; + + if (self.notOrphanCount) return cb(null, self.notOrphanCount); + + + self.sync.blockDb.countNotOrphan(function(err, count) { + if (err) return cb(err); + self.notOrphanCount = count; + return cb(null, self.notOrphanCount); + }); + }; + + HistoricSync.prototype.getBlockCount = function(cb) { var self = this; @@ -389,7 +399,7 @@ function spec() { function(cb) { if (scanOpts.upToExisting) { // should be isOrphan = true or null to be more accurate. - self.sync.countNotOrphan(function(err, count) { + self.countNotOrphan(function(err, count) { if (err) return cb(err); self.syncedBlocks = count || 0; @@ -464,19 +474,19 @@ function spec() { HistoricSync.prototype.smartImport = function(scanOpts, next) { var self = this; - self.sync.hasBlock(self.genesis, function(err, b) { + self.sync.blockDb.has(self.genesis, function(err, b) { if (err) return next(err); - self.sync.countNotOrphan(function(err, count) { + self.countNotOrphan(function(err, count) { if (err) return next(err); self.getBlockCount(function(err) { if (err) return next(err); - if (!b || scanOpts.destroy || self.blockChainHeight > count * 0.7 ) { + if (!b || scanOpts.destroy || count < self.blockChainHeight * 0.8 ) { if (!b) p('Could not find Genesis block. Running FULL SYNC'); else - p('Less that 70% of current blockchain is stored. Running FULL SYNC', + p('Less that 80% of current blockchain is stored. Running FULL SYNC', parseInt(count/self.blockChainHeight*100)); if (config.bitcoind.dataDir) { diff --git a/lib/Sync.js b/lib/Sync.js index ffd9f06..61ac309 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -62,24 +62,12 @@ function spec() { Sync.prototype.destroy = function(next) { var self = this; async.series([ - function(b) { try {self.blockDb.drop(b);} catch (e) { return b(); } }, - function(b) { try {self.TransactionDb.drop(b);} catch (e) { return b(); } }, + function(b) { self.blockDb.drop(b); }, + function(b) { self.txDb.drop(b); }, ], next); }; - Sync.prototype.hasBlock = function(hash, cb) { - var self = this; - - return self.blockDb.has(hash, cb); - }; - - Sync.prototype.countNotOrphan = function(hash, cb) { - var self = this; - return self.blockDb.countNotOrphan(hash, cb); - }; - - Sync.prototype.storeBlock = function(block, cb) { var self = this; diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 3f8394c..47d9f9b 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -26,7 +26,8 @@ function spec() { util = require('bitcore/util/util'), levelup = require('levelup'), async = require('async'), - config = require('../config/config'); + config = require('../config/config'), + assert = require('assert'); var TransactionDb = function() { this.db = levelup(config.leveldb + '/txs'); @@ -91,12 +92,16 @@ function spec() { .on('data', function (data) { var k = data.key.split('-'); var v = data.value.split(':'); + var set=0; for(var i=0; i= 0) { - toRm.push(data.key); - console.log('To Remove Found', data.key); //TODO - } - }) - .on('end', function() { - return s_c(); - }); - }, - function(s_c) { - async.each(toRm, function(k,e_c) { - self.db.del(k,e_c); - }, s_c); + .pipe( + self.db.createWriteStream({type:'del'}) + ).on('close', c); }], function(err) { cb(err); diff --git a/test/integration/01-transactionouts.js b/test/integration/01-transactionouts.js index 12705fd..c59de6b 100644 --- a/test/integration/01-transactionouts.js +++ b/test/integration/01-transactionouts.js @@ -11,18 +11,18 @@ var assert = require('assert'), TransactionDb = require('../../lib/TransactionDb').class(); var txItemsValid = JSON.parse(fs.readFileSync('test/integration/txitems.json')); -var tdb; +var txDb; describe('TransactionDb fromIdWithInfo', function(){ before(function(c) { - tdb = new TransactionDb(); + txDb = new TransactionDb(); return c(); }); var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c'; it('tx info ' + txid, function(done) { - tdb.fromIdWithInfo(txid, function(err, tx) { + txDb.fromIdWithInfo(txid, function(err, tx) { if (err) done(err); assert.equal(tx.txid, txid); @@ -38,7 +38,7 @@ describe('TransactionDb fromIdWithInfo', function(){ it('tx info', function(done) { var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; - tdb.fromIdWithInfo(txid, function(err, tx) { + txDb.fromIdWithInfo(txid, function(err, tx) { if (err) done(err); assert.equal(tx.txid, txid); assert(!tx.info.isCoinBase); @@ -48,7 +48,7 @@ describe('TransactionDb fromIdWithInfo', function(){ it('should pool tx\'s info from bitcoind', function(done) { var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; - tdb.fromIdWithInfo(txid, function(err, tx) { + txDb.fromIdWithInfo(txid, function(err, tx) { if (err) done(err); assert.equal(tx.info.txid, txid); assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74'); @@ -62,7 +62,7 @@ describe('TransactionDb fromIdWithInfo', function(){ var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399'; it('test a coinbase TX ' + txid1, function(done) { - tdb.fromIdWithInfo(txid1, function(err, tx) { + txDb.fromIdWithInfo(txid1, function(err, tx) { if (err) done(err); assert(tx.info.isCoinBase); assert.equal(tx.info.txid, txid1); @@ -72,7 +72,7 @@ describe('TransactionDb fromIdWithInfo', function(){ }); var txid22 = '666'; it('test invalid TX ' + txid22, function(done) { - tdb.fromIdWithInfo(txid22, function(err, tx) { + txDb.fromIdWithInfo(txid22, function(err, tx) { if (err && err.message.match(/must.be.hexadecimal/)) { return done(); } @@ -85,7 +85,7 @@ describe('TransactionDb fromIdWithInfo', function(){ var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227'; it('test unexisting TX ' + txid23, function(done) { - tdb.fromIdWithInfo(txid23, function(err, tx) { + txDb.fromIdWithInfo(txid23, function(err, tx) { assert(!err); assert(!tx); return done(); @@ -96,7 +96,7 @@ describe('TransactionDb fromIdWithInfo', function(){ var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; it('create TX on the fly ' + txid2, function(done) { - tdb.fromIdWithInfo(txid2, function(err, tx) { + txDb.fromIdWithInfo(txid2, function(err, tx) { if (err) return done(err); assert.equal(tx.info.txid, txid2); done(); @@ -105,7 +105,7 @@ describe('TransactionDb fromIdWithInfo', function(){ txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; it('test a broken TX ' + txid2, function(done) { - tdb.fromIdWithInfo(txid2, function(err, tx) { + txDb.fromIdWithInfo(txid2, function(err, tx) { if (err) return done(err); assert.equal(tx.info.txid, txid2); assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU'); @@ -113,28 +113,25 @@ describe('TransactionDb fromIdWithInfo', function(){ }); }); }); -/* describe('TransactionDb Outs', function(){ - var tdb = new TransactionDb(); - txItemsValid.forEach( function(v) { if (v.disabled) return; it('test a processing tx ' + v.txid, function(done) { this.timeout(60000); // Remove first - tdb.removeFromTxId(v.txid, v.toRm, function() { + txDb.removeFromTxId(v.txid, function() { - tdb.fromTxId( v.txid, function(err, readItems) { + txDb.fromTxId( v.txid, function(err, readItems) { assert.equal(readItems.length,0); var unmatch=[]; - tdb.createFromArray([v.txid], null, function(err) { + txDb.createFromArray([v.txid], null, function(err) { if (err) return done(err); - tdb.fromTxId( v.txid, function(err, readItems) { + txDb.fromTxId( v.txid, function(err, readItems) { v.items.forEach(function(validItem){ unmatch[validItem.addr] =1; @@ -164,5 +161,4 @@ describe('TransactionDb Outs', function(){ }); }); -*/ diff --git a/test/integration/transactionouts-expenses.js b/test/integration/transactionouts-expenses.js deleted file mode 100644 index b8b04be..0000000 --- a/test/integration/transactionouts-expenses.js +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -process.env.NODE_ENV = process.env.NODE_ENV || 'development'; - - - -var mongoose = require('mongoose'), - assert = require('assert'), - fs = require('fs'), - util = require('util'), - async = require('async'), - config = require('../../config/config'), - TransactionOut = require('../../app/models/TransactionOut'); - -var spentValid = JSON.parse(fs.readFileSync('test/model/spent.json')); - -mongoose.connection.on('error', function(err) { console.log(err); }); - - - -describe('TransactionOut Expenses', function(){ - - before(function(done) { - mongoose.connect(config.db); - - // lets spend! - async.each(Object.keys(spentValid), - function(txid,c_out) { - async.each(spentValid[txid], - function(i,c_in) { - TransactionOut._explodeTransactionOuts(i.txid, function(err) { - return c_in(); - }); - }, - function(err) { - return c_out(); - } - ); - }, - function(err) { - console.log('[transactionouts.js.88]'); //TODO - return done(); - } - ); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); - - Object.keys(spentValid).forEach( function(txid) { - it('test result of spending tx ' + txid, function(done) { - var s = spentValid[txid]; - var c=0; - TransactionOut.fromTxId( txid, function(err, readItems) { - s.forEach( function(v) { - assert.equal(readItems[c].spendTxIdBuf.toString('hex'),v.txid); - assert.equal(readItems[c].spendIndex,v.n); - c++; - }); - done(); - }); - }); - }); -}); diff --git a/test/integration/txitems.json b/test/integration/txitems.json index eae6b11..bee3b43 100644 --- a/test/integration/txitems.json +++ b/test/integration/txitems.json @@ -6,7 +6,8 @@ { "txid": "21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237", "toRm": [ - "txouts-spend-86a03cac7d87f596008c6d5a8d3fd8b88842932ea6f0337673eda16f6b472f7f-0" + "txouts-spend-86a03cac7d87f596008c6d5a8d3fd8b88842932ea6f0337673eda16f6b472f7f-0", + "txouts-spend-bcd8da8ee847da377f8aaca92502c05e5f914c6a2452753146013b0e642a25a0-0" ], "items": [ { From e4095cbf8eab442e83a55ed57fe444e25d815b5a Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 5 Feb 2014 03:34:46 -0300 Subject: [PATCH 13/25] fix timestamp and rounding in TXs --- app/models/Address.js | 27 ++++++++------------------- dev-util/get_tx.js | 22 +++++++--------------- dev-util/sync-level.js | 17 +++++++++++++++++ dev-util/tx-level.js | 19 +++++++++++++++++++ lib/HistoricSync.js | 6 ++++++ lib/TransactionDb.js | 3 +-- test/integration/addr.js | 17 ++++++++++++----- test/integration/addr.json | 20 ++++++-------------- 8 files changed, 76 insertions(+), 55 deletions(-) create mode 100644 dev-util/sync-level.js create mode 100755 dev-util/tx-level.js diff --git a/app/models/Address.js b/app/models/Address.js index e6717e7..9c5651b 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -9,8 +9,8 @@ function spec() { var BitcoreUtil = require('bitcore/util/util'); var TransactionDb = require('../../lib/TransactionDb').class(); - function Address(addrStr) { - this.tdb = new TransactionDb(); + function Address(addrStr, txDb) { + this.txDb = txDb || new TransactionDb(); this.balanceSat = 0; this.totalReceivedSat = 0; this.totalSentSat = 0; @@ -58,35 +58,24 @@ function spec() { Address.prototype.update = function(next) { var self = this; async.series([ -/* function (cb) { - TransactionIn.find({addr:self.addrStr}).exec(function(err,txIn){ - if (err) return cb(err); - - txIn.forEach(function(txItem){ - - self.balanceSat += txItem.value_sat; - self.totalReceivedSat += txItem.value_sat; - }); - return cb(); - }); - }, -*/ function (cb) { - self.tdb.fromAddr(self.addrStr, function(err,txOut){ + self.txDb.fromAddr(self.addrStr, function(err,txOut){ if (err) return cb(err); txOut.forEach(function(txItem){ - self.totalReceivedSat += txItem.value_sat; + var v = parseInt(txItem.value_sat); + + self.totalReceivedSat += v; self.transactions.push(txItem.txid); if (! txItem.spendTxId) { // unspent - self.balanceSat += txItem.value_sat; + self.balanceSat += v; self.txApperances +=1; } else { // spent - self.totalSentSat += txItem.value_sat; + self.totalSentSat += v; self.transactions.push(txItem.spendTxid); self.txApperances +=2; } diff --git a/dev-util/get_tx.js b/dev-util/get_tx.js index a937c7f..c30f0f6 100755 --- a/dev-util/get_tx.js +++ b/dev-util/get_tx.js @@ -2,9 +2,7 @@ 'use strict'; var util = require('util'); -var T = require('../app/models/Transaction').class(); -var mongoose= require('mongoose'), - config = require('../config/config'); +var T = require('../lib/TransactionDb').class(); process.env.NODE_ENV = process.env.NODE_ENV || 'development'; @@ -14,21 +12,15 @@ var hash = process.argv[2] || 'e2253359458db3e732c82a43fc62f56979ff59928f25a2df3 +var t = new T(); +t.fromIdWithInfo(hash, function(err, ret) { -mongoose.connect(config.db); -mongoose.connection.on('error', function(err) { console.log('error!', err); }); -mongoose.connection.on('open', function() { - - T.fromIdWithInfo(hash, function(err, ret) { - - console.log('Err:'); - console.log(err); + console.log('Err:'); + console.log(err); - console.log('Ret:'); - console.log(util.inspect(ret,{depth:null})); - mongoose.connection.close(); - }); + console.log('Ret:'); + console.log(util.inspect(ret,{depth:null})); }); diff --git a/dev-util/sync-level.js b/dev-util/sync-level.js new file mode 100644 index 0000000..2f2f95b --- /dev/null +++ b/dev-util/sync-level.js @@ -0,0 +1,17 @@ +#!/usr/bin/env node +'use strict'; + +var Sync = require('../lib/Sync').class(); + + +var s = new Sync(); + + +s.setOrphan( + '0000000000c2b1e8dab92a72741289e5ef0d4f375fd1b26f729da2ba979c028a', + '000000000228f9d02654459e09998c7557afa9082784c11226853f5feb805df9', + function (err) { + console.log('[sync-level.js.15]',err); //TODO + }); + + diff --git a/dev-util/tx-level.js b/dev-util/tx-level.js new file mode 100755 index 0000000..a90cc2e --- /dev/null +++ b/dev-util/tx-level.js @@ -0,0 +1,19 @@ +#!/usr/bin/env node + +var + config = require('../config/config'), + levelup = require('levelup'); + + +db = levelup(config.leveldb + '/txs'); + +var s = 'txouts-addr-mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ'; +db.createReadStream({start: s, end: s+'~'}) + .on('data', function (data) { +console.log('[block-level.js.11:data:]',data); //TODO + if (data==false) c++; + }) + .on('end', function () { + }); + + diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 3747fe9..b0b541d 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -278,7 +278,13 @@ function spec() { var ti=0; // Get TX Address b.txs.forEach(function(t) { + + var objTx = blockInfo.tx[ti++]; + + //add time from block + objTx.time = blockInfo.time; + var to=0; t.outs.forEach( function(o) { diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 47d9f9b..e5a8753 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -221,7 +221,6 @@ function spec() { }); }, function(err) { - console.log('[TransactionDb.js.165]', ret); //TODO return cb(err,ret); }); @@ -341,7 +340,7 @@ console.log('[TransactionDb.js.165]', ret); //TODO // } var addr = o.scriptPubKey.addresses[0]; - var sat = o.value * util.COIN; + var sat = Math.round(o.value * util.COIN); self.db.batch() .put( OUTS_ROOT + tx.txid + '-' + o.n, addr + ':' + sat) .put( ADDR_ROOT + addr + '-' + ts + '-' + tx.txid + diff --git a/test/integration/addr.js b/test/integration/addr.js index 1239614..3f31bc6 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -1,16 +1,23 @@ #!/usr/bin/env node +'use strict'; process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var - assert = require('assert'), +var assert = require('assert'), fs = require('fs'), - config = require('../../config/config'), - Address = require('../../app/models/Address').class(); + Address = require('../../app/models/Address').class(), + TransactionDb = require('../../lib/TransactionDb').class(), addrValid = JSON.parse(fs.readFileSync('test/integration/addr.json')); + var txDb; describe('Address balances', function(){ + before(function(c) { + txDb = new TransactionDb(); + return c(); + }); + + addrValid.forEach( function(v) { if (v.disabled) { console.log(v.addr + " => disabled in JSON"); @@ -19,7 +26,7 @@ describe('Address balances', function(){ it('Info for: ' + v.addr, function(done) { this.timeout(5000); - var a = new Address(v.addr); + var a = new Address(v.addr, txDb); a.update(function(err) { if (err) done(err); diff --git a/test/integration/addr.json b/test/integration/addr.json index fd174e6..efd513e 100644 --- a/test/integration/addr.json +++ b/test/integration/addr.json @@ -1,12 +1,6 @@ [ + { -"disabled":1, - "addr": "mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ", - "balance": 43.1, - "txApperances": 19 - }, - { -"disabled":1, "addr": "mp3Rzxx9s1A21SY3sjJ3CQoa2Xjph7e5eS", "balance": 0, "totalReceived": 50, @@ -14,7 +8,6 @@ "txApperances": 2 }, { -"disabled":1, "addr": "muyg1K5WsHkfMVCkUXU2y7Xp5ZD6RGzCeH", "balance": 0.38571339, "totalReceived": 0.38571339, @@ -29,7 +22,6 @@ "txApperances": 13 }, { -"disabled":1, "addr": "n47CfqnKWdNwqY1UWxTmNJAqYutFxdH3zY", "balance": 0, "totalReceived":26.4245, @@ -37,7 +29,6 @@ "txApperances": 4 }, { -"disabled":1, "addr": "mzSyyXgofoBxpr6gYcU3cV345G8hJpixRd", "balance": 0, "totalReceived":3.9775, @@ -45,7 +36,11 @@ "txApperances": 2 }, { -"disabled":1, + "addr": "mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ", + "balance": 43.1, + "txApperances": 19 + }, + { "addr": "mzW2hdZN2um7WBvTDerdahKqRgj3md9C29", "txApperances": 2034, "balance": 1049.69744099, @@ -53,7 +48,6 @@ "totalSent": 0 }, { -"disabled":1, "addr": "mjRmkmYzvZN3cA3aBKJgYJ65epn3WCG84H", "txApperances": 13343, "balance": 46413.0, @@ -61,7 +55,6 @@ "totalSent": 310717.17644359 }, { -"disabled":1, "addr": "mgKY35SXqxFpcKK3Dq9mW9919N7wYXvcFM", "txApperances": 1, "balance": 0.01979459, @@ -70,7 +63,6 @@ "transactions": [ "91800d80bb4c69b238c9bfd94eb5155ab821e6b25cae5c79903d12853bbb4ed5" ] }, { - "disabled":1, "addr": "mmvP3mTe53qxHdPqXEvdu8WdC7GfQ2vmx5", "balance": 10580.50027254, "totalReceived": 12157.65075053, From df3336627706ab48ca7da5eeec32e0c3f0ad7847 Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Wed, 5 Feb 2014 10:16:40 -0300 Subject: [PATCH 14/25] removed mongoose. method: block list by date --- app/controllers/blocks.js | 16 +++++++++------- app/controllers/transactions.js | 7 +++++-- app/models/Transaction.js | 2 +- config/config.js | 1 - insight.js | 20 +------------------- lib/BlockDb.js | 4 +--- lib/Sync.js | 30 +----------------------------- 7 files changed, 18 insertions(+), 62 deletions(-) diff --git a/app/controllers/blocks.js b/app/controllers/blocks.js index 726a086..4acc120 100644 --- a/app/controllers/blocks.js +++ b/app/controllers/blocks.js @@ -3,17 +3,17 @@ /** * Module dependencies. */ -var mongoose = require('mongoose'), - Block = mongoose.model('Block'), - common = require('./common'), - async = require('async'); +var common = require('./common'), + async = require('async'), + BlockDb = require('../../lib/BlockDb').class(); +var bdb = new BlockDb(); /** * Find block by hash ... */ exports.block = function(req, res, next, hash) { - Block.fromHashWithInfo(hash, function(err, block) { + bdb.fromHashWithInfo(hash, function(err, block) { if (err || ! block) return common.handleErrors(err, res, next); else { @@ -37,7 +37,7 @@ exports.show = function(req, res) { * Show block by Height */ exports.blockindex = function(req, res, next, height) { - Block.blockIndex(height, function(err, hashStr) { + bdb.blockIndex(height, function(err, hashStr) { if (err) { console.log(err); res.status(400).send('Bad Request'); // TODO @@ -49,7 +49,7 @@ exports.blockindex = function(req, res, next, height) { }; var getBlock = function(blockhash, cb) { - Block.fromHashWithInfo(blockhash, function(err, block) { + bdb.fromHashWithInfo(blockhash, function(err, block) { if (err) { console.log(err); return cb(err); @@ -103,6 +103,7 @@ exports.list = function(req, res) { var prev = formatTimestamp(new Date((gte - 86400) * 1000)); var next = formatTimestamp(new Date(lte * 1000)); + /* Block .find({ time: { @@ -134,4 +135,5 @@ exports.list = function(req, res) { }); } }); + */ }; diff --git a/app/controllers/transactions.js b/app/controllers/transactions.js index f7dd671..831025c 100644 --- a/app/controllers/transactions.js +++ b/app/controllers/transactions.js @@ -4,11 +4,14 @@ * Module dependencies. */ var Transaction = require('../models/Transaction').class(); -var Block = require('../models/Block'); var Address = require('../models/Address'); var async = require('async'); var common = require('./common'); +var BlockDb = require('../../lib/BlockDb').class(); + +var bdb = new BlockDb(); + /** * Find transaction by hash ... @@ -68,7 +71,7 @@ exports.list = function(req, res, next) { var txs; if (bId) { - Block.fromHashWithInfo(bId, function(err, block) { + bdb.fromHashWithInfo(bId, function(err, block) { if (err) { console.log(err); return res.status(500).send('Internal Server Error'); diff --git a/app/models/Transaction.js b/app/models/Transaction.js index f3b20dc..b752cfd 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -7,7 +7,7 @@ function spec() { var util = require('bitcore/util/util'), TransactionRpc = require('../../lib/TransactionRpc').class(), - TransactionOut = require('./TransactionOut'), + TransactionOut = require('../../lib/TransactionDb'), async = require('async'); var CONCURRENCY = 20; diff --git a/config/config.js b/config/config.js index 7ab059b..92fbeb6 100644 --- a/config/config.js +++ b/config/config.js @@ -23,7 +23,6 @@ module.exports = { root: rootPath, appName: 'Insight ' + env, port: process.env.PORT || 3000, - db: 'mongodb://localhost/insight-' + env, leveldb: './db', bitcoind: { protocol: process.env.BITCOIND_PROTO || 'http', diff --git a/insight.js b/insight.js index 9698509..29b0c4a 100644 --- a/insight.js +++ b/insight.js @@ -9,9 +9,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var express = require('express'), fs = require('fs'), PeerSync = require('./lib/PeerSync').class(), - HistoricSync = require('./lib/HistoricSync').class(), - mongoose = require('mongoose'); - + HistoricSync = require('./lib/HistoricSync').class(); //Initializing system variables var config = require('./config/config'); @@ -21,22 +19,6 @@ var config = require('./config/config'); */ var expressApp = express(); -/** - * Bootstrap db connection - */ -// If mongod is running -mongoose.connection.on('open', function() { - console.log('Connected to mongo server.'); -}); - -// If mongod is not running -mongoose.connection.on('error', function(err) { - console.log('Could not connect to mongo server!'); - console.log(err); -}); - -mongoose.connect(config.db); - /** * Bootstrap models */ diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 35f3adb..ca37c70 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -160,9 +160,7 @@ function spec() { }); }; - - - return BlockDb; + return BlockDb; } module.defineClass(spec); diff --git a/lib/Sync.js b/lib/Sync.js index 17c524e..f2aa966 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -4,7 +4,6 @@ require('classtool'); function spec() { - var mongoose = require('mongoose'); var config = require('../config/config'); var sockets = require('../app/controllers/socket.js'); var BlockDb = require('./BlockDb').class(); @@ -22,34 +21,7 @@ function spec() { self.opts = opts; - if (!(opts && opts.skipDbConnection)) { - - if (mongoose.connection.readyState !== 1) { - mongoose.connect(config.db, function(err) { - if (err) { - console.log('CRITICAL ERROR: connecting to mongoDB:',err); - return (err); - } - }); - } - - self.db = mongoose.connection; - - self.db.on('error', function(err) { - console.log('MongoDB ERROR:' + err); - return cb(err); - }); - - self.db.on('disconnect', function(err) { - console.log('MongoDB disconnect:' + err); - return cb(err); - }); - - return self.db.once('open', function(err) { - return cb(err); - }); - } - else return cb(); + return cb(); }; Sync.prototype.close = function() { From 8b1092e3ffec43cf3c5aa2800489891fda3004c3 Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Wed, 5 Feb 2014 10:30:41 -0300 Subject: [PATCH 15/25] fix controller transaction --- app/controllers/transactions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/transactions.js b/app/controllers/transactions.js index 831025c..9eeb5f1 100644 --- a/app/controllers/transactions.js +++ b/app/controllers/transactions.js @@ -3,7 +3,7 @@ /** * Module dependencies. */ -var Transaction = require('../models/Transaction').class(); +var Transaction = require('../../lib/TransactionDb').class(); var Address = require('../models/Address'); var async = require('async'); var common = require('./common'); From aebfdebfde21eb7a7440d1412d62312c5aaaf022 Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Wed, 5 Feb 2014 10:59:01 -0300 Subject: [PATCH 16/25] fix test --- lib/BlockDb.js | 2 +- test/integration/blocklist.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 81c57eb..8b9539d 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -133,7 +133,7 @@ function spec() { BlockDb.prototype.getBlocksByDate = function(start_ts, end_ts, cb) { var self = this; var list = []; - this.db.createReadStream({ + self.db.createReadStream({ start: TIMESTAMP_ROOT + start_ts, end: TIMESTAMP_ROOT + end_ts }) diff --git a/test/integration/blocklist.js b/test/integration/blocklist.js index 6b50952..8225a17 100644 --- a/test/integration/blocklist.js +++ b/test/integration/blocklist.js @@ -3,9 +3,9 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var TESTING_BLOCK = '00000000b7cc12abe8a9a604813aab1f2c4f3a242a021065be52393a147a1a86'; -var START_TS = '1391538611'; -var END_TS = '1391538638'; +var TESTING_BLOCK = '000000001f56660def9b5898ea8411d7b028854e78502e521f9ebd53e673751c'; +var START_TS = '1391607675'; +var END_TS = '1391607709'; var assert = require('assert'), From 548572bfa3e07e1db638b4c50b103ef065201ded Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 5 Feb 2014 11:23:41 -0300 Subject: [PATCH 17/25] all test working again with level DB --- app/models/Address.js | 3 +-- lib/BlockDb.js | 8 ++++++-- lib/Sync.js | 19 ++++++++++--------- lib/TransactionDb.js | 11 +++++++---- test/integration/01-transactionouts.js | 15 +++++++++++++++ test/integration/02-transactionouts.js | 20 +++++++++++++------- test/integration/addr.js | 7 ++++++- test/integration/addr.json | 11 ++++++----- test/integration/block.js | 25 ++++++++++++++++++------- test/integration/blockExtractor.js | 4 ++-- test/integration/status.js | 18 +++--------------- test/integration/sync.js | 20 ++++++++++++-------- 12 files changed, 99 insertions(+), 62 deletions(-) diff --git a/app/models/Address.js b/app/models/Address.js index 9c5651b..8dc6286 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -63,7 +63,6 @@ function spec() { if (err) return cb(err); txOut.forEach(function(txItem){ - var v = parseInt(txItem.value_sat); self.totalReceivedSat += v; @@ -76,7 +75,7 @@ function spec() { else { // spent self.totalSentSat += v; - self.transactions.push(txItem.spendTxid); + self.transactions.push(txItem.spendTxId); self.txApperances +=2; } }); diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 51dd7ea..7d5491a 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -18,8 +18,12 @@ function spec() { BitcoreBlock= require('bitcore/Block').class(), config = require('../config/config'); - var BlockDb = function() { - this.db = levelup(config.leveldb + '/blocks'); + var BlockDb = function(db) { + this.db = db || levelup(config.leveldb + '/blocks'); + }; + + BlockDb.prototype.close = function(cb) { + this.db.close(cb); }; BlockDb.prototype.drop = function(cb) { diff --git a/lib/Sync.js b/lib/Sync.js index 61ac309..a4f4c1c 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -13,7 +13,7 @@ function spec() { function Sync() { - this.blockDb = new BlockDb(); + this.bDb = new BlockDb(); this.txDb = new TransactionDb(); } @@ -52,17 +52,18 @@ function spec() { else return cb(); }; - Sync.prototype.close = function() { - if ( this.db && this.db.readyState ) { - this.db.close(); - } + Sync.prototype.close = function(cb) { + var self = this; + self.txDb.close(function() { + self.bDb.close(cb); + }); }; Sync.prototype.destroy = function(next) { var self = this; async.series([ - function(b) { self.blockDb.drop(b); }, + function(b) { self.bDb.drop(b); }, function(b) { self.txDb.drop(b); }, ], next); }; @@ -74,7 +75,7 @@ function spec() { self.txDb.createFromBlock(block, function(err, insertedTxs, updateAddrs) { if (err) return cb(err); - self.blockDb.add(block, function(err){ + self.bDb.add(block, function(err){ if (err) return cb(err); self._handleBroadcast(block, insertedTxs, updateAddrs); return cb(); @@ -100,13 +101,13 @@ console.log('[Sync.js.109] WARN: Reach reog depth limit'); //TODO }, function (w_c) { //check with RPC if the block is mainchain - self.blockDb.fromHashWithInfo(hash, function (err, info) { + self.bDb.fromHashWithInfo(hash, function (err, info) { if (!info) { console.log('[Sync.js.107:hash:ORPHAN]',hash); //TODO self.txDb.setOrphan(hash, function(err) { if (err) return w_c(err); - self.blockDb.setOrphan(hash, function(err, prevHash){ + self.bDb.setOrphan(hash, function(err, prevHash){ hash = prevHash; return w_c(err); }); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index e5a8753..8250901 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -6,7 +6,7 @@ require('classtool'); function spec() { // blockHash -> txid mapping (to reorgs) - var ROOT = 'tx-b-'; //tx-b-- => 1 + var ROOT = 'tx-b-'; //tx-b-- => 1/0 (connected or not) // to show tx outs var OUTS_ROOT = 'txouts-'; //txouts-- => [addr, btc_sat] @@ -29,8 +29,12 @@ function spec() { config = require('../config/config'), assert = require('assert'); - var TransactionDb = function() { - this.db = levelup(config.leveldb + '/txs'); + var TransactionDb = function(db) { + this.db = db || levelup(config.leveldb + '/txs'); + }; + + TransactionDb.prototype.close = function(cb) { + this.db.close(cb); }; TransactionDb.prototype.drop = function(cb) { @@ -221,7 +225,6 @@ function spec() { }); }, function(err) { -console.log('[TransactionDb.js.165]', ret); //TODO return cb(err,ret); }); }); diff --git a/test/integration/01-transactionouts.js b/test/integration/01-transactionouts.js index c59de6b..ce70a22 100644 --- a/test/integration/01-transactionouts.js +++ b/test/integration/01-transactionouts.js @@ -20,6 +20,11 @@ describe('TransactionDb fromIdWithInfo', function(){ return c(); }); + + after(function(c) { + txDb.close(c); + }); + var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c'; it('tx info ' + txid, function(done) { txDb.fromIdWithInfo(txid, function(err, tx) { @@ -116,6 +121,16 @@ describe('TransactionDb fromIdWithInfo', function(){ describe('TransactionDb Outs', function(){ + before(function(c) { + txDb = new TransactionDb(); + return c(); + }); + + + after(function(c) { + txDb.close(c); + }); + txItemsValid.forEach( function(v) { if (v.disabled) return; it('test a processing tx ' + v.txid, function(done) { diff --git a/test/integration/02-transactionouts.js b/test/integration/02-transactionouts.js index 84595b4..a5edce7 100644 --- a/test/integration/02-transactionouts.js +++ b/test/integration/02-transactionouts.js @@ -15,37 +15,43 @@ var var spentValid = JSON.parse(fs.readFileSync('test/integration/spent.json')); -describe('TransactionDb Expenses', function(){ - var tdb = new TransactionDb(); +var txDb; - before(function(done) { +describe('TransactionDb Expenses', function(){ + + before(function(c) { + txDb = new TransactionDb(); // lets spend! async.each(Object.keys(spentValid), function(txid,c_out) { async.each(spentValid[txid], function(i,c_in) { - tdb.createFromArray([i.txid], null, function(err) { + txDb.createFromArray([i.txid], null, function(err) { return c_in(); }); }, function(err) { - console.log('Done spending ', txid); //TODO return c_out(); } ); }, function(err) { - return done(); + return c(); } ); }); + after(function(c) { + txDb.close(c); + }); + + Object.keys(spentValid).forEach( function(txid) { it('test result of spending tx ' + txid, function(done) { var s = spentValid[txid]; var c=0; - tdb.fromTxId( txid, function(err, readItems) { + txDb.fromTxId( txid, function(err, readItems) { s.forEach( function(v) { assert.equal(readItems[c].spendTxId,v.txid); assert.equal(readItems[c].spendIndex,v.n); diff --git a/test/integration/addr.js b/test/integration/addr.js index 3f31bc6..763cd11 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -17,6 +17,11 @@ describe('Address balances', function(){ return c(); }); + after(function(c) { + txDb.close(c); + }); + + addrValid.forEach( function(v) { if (v.disabled) { @@ -43,7 +48,7 @@ describe('Address balances', function(){ if (v.transactions) { v.transactions.forEach( function(tx) { - assert(a.transactions.indexOf(tx)>-1); + assert(a.transactions.indexOf(tx)>-1,'have tx '+tx); }); } done(); diff --git a/test/integration/addr.json b/test/integration/addr.json index efd513e..ce08d21 100644 --- a/test/integration/addr.json +++ b/test/integration/addr.json @@ -37,14 +37,14 @@ }, { "addr": "mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ", - "balance": 43.1, - "txApperances": 19 + "txApperances": 27, + "balance": 5.1 }, { "addr": "mzW2hdZN2um7WBvTDerdahKqRgj3md9C29", - "txApperances": 2034, - "balance": 1049.69744099, - "totalReceived": 1049.69744099, + "txApperances": 6033, + "balance": 1049.69744101, + "totalReceived": 1049.69744101, "totalSent": 0 }, { @@ -67,6 +67,7 @@ "balance": 10580.50027254, "totalReceived": 12157.65075053, "totalSent": 1577.15047799, + "txApperances": 459, "transactions": [ "91800d80bb4c69b238c9bfd94eb5155ab821e6b25cae5c79903d12853bbb4ed5", "f6e80d4fd1a2377406856c67d0cee5ac7e5120993ff97e617ca9aac33b4c6b1e", diff --git a/test/integration/block.js b/test/integration/block.js index 3650313..b8c34ed 100644 --- a/test/integration/block.js +++ b/test/integration/block.js @@ -11,12 +11,23 @@ var config = require('../../config/config'), BlockDb = require('../../lib/BlockDb').class(); +var bDb; describe('BlockDb fromHashWithInfo', function(){ - var bdb = new BlockDb(); + + before(function(c) { + bDb = new BlockDb(); + return c(); + }); + + + after(function(c) { + bDb.close(c); + }); + it('should poll block\'s info from bitcoind', function(done) { - bdb.fromHashWithInfo(TESTING_BLOCK, function(err, b2) { + bDb.fromHashWithInfo(TESTING_BLOCK, function(err, b2) { if (err) done(err); assert.equal(b2.hash, TESTING_BLOCK); assert.equal(b2.info.hash, TESTING_BLOCK); @@ -25,7 +36,7 @@ describe('BlockDb fromHashWithInfo', function(){ }); }); it('return true in has', function(done) { - bdb.has(TESTING_BLOCK, function(err, has) { + bDb.has(TESTING_BLOCK, function(err, has) { assert.equal(has, true); done(); }); @@ -34,12 +45,12 @@ describe('BlockDb fromHashWithInfo', function(){ var b16 = '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b'; var b17 = '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743'; - bdb.has(b17, function(err, has) { + bDb.has(b17, function(err, has) { assert(has); - bdb.setOrphan(b17, function(err, oldPrev) { + bDb.setOrphan(b17, function(err, oldPrev) { assert.equal(oldPrev, b16); - bdb.setPrev(b17, b16, function(err, oldPrev) { - bdb.getPrev(b17, function(err, p) { + bDb.setPrev(b17, b16, function(err, oldPrev) { + bDb.getPrev(b17, function(err, p) { assert.equal(p, b16); done(); }); diff --git a/test/integration/blockExtractor.js b/test/integration/blockExtractor.js index e7ff291..13c7e8d 100644 --- a/test/integration/blockExtractor.js +++ b/test/integration/blockExtractor.js @@ -13,7 +13,7 @@ var assert = require('assert'), //var txItemsValid = JSON.parse(fs.readFileSync('test/model/txitems.json')); -describe('TransactionOut', function(){ +describe('BlockExtractor', function(){ var be = new BlockExtractor(config.bitcoind.dataDir, config.network); @@ -51,7 +51,7 @@ describe('TransactionOut', function(){ }); }); - it('should read 100000 blocks with no error ', function(done) { + it.skip('should read 100000 blocks with no error ', function(done) { var i=0; while(i++<100000) { diff --git a/test/integration/status.js b/test/integration/status.js index e7f078a..663e473 100644 --- a/test/integration/status.js +++ b/test/integration/status.js @@ -1,25 +1,13 @@ #!/usr/bin/env node +'use strict'; process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var - assert = require('assert'), - config = require('../../config/config'), - Status = require('../../app/models/Status').class(), - mongoose= require('mongoose'); +var assert = require('assert'), + Status = require('../../app/models/Status').class(); describe('Status', function(){ - before(function(done) { - mongoose.connect(config.db); - done(); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); - it('getInfo', function(done) { var d = new Status(); diff --git a/test/integration/sync.js b/test/integration/sync.js index be6a078..82b87c4 100644 --- a/test/integration/sync.js +++ b/test/integration/sync.js @@ -20,7 +20,7 @@ var b = [ var fix = function(s,cb) { async.each([1,2,3,4], function(i,c) { - s.blockDb.setPrev(b[i],b[i-1], function() { + s.bDb.setPrev(b[i],b[i-1], function() { return c(); }); }, cb); @@ -28,12 +28,12 @@ var fix = function(s,cb) { var test = function(s,cb) { async.each([2,3,4], function(i,c) { - s.blockDb.getPrev(b[i], function(err, p) { + s.bDb.getPrev(b[i], function(err, p) { assert.equal(p,0); return c(); }); }, function() { - s.blockDb.getPrev(b[1], function(err, p) { + s.bDb.getPrev(b[1], function(err, p) { assert.equal(p,b[0]); return cb(); }); @@ -44,12 +44,12 @@ var test = function(s,cb) { var testNo = function(s,cb) { async.each([2,3,4], function(i,c) { - s.blockDb.getPrev(b[i], function(err, p) { + s.bDb.getPrev(b[i], function(err, p) { assert.equal(p,b[i-1]); return c(); }); }, function() { - s.blockDb.getPrev(b[1], function(err, p) { + s.bDb.getPrev(b[1], function(err, p) { assert.equal(p,b[0]); return cb(); }); @@ -68,15 +68,19 @@ describe('Sync checkOrphan', function(){ }); after(function(done) { - fix(s,done); + + fix(s,function() { + s.close(done); + }); + }); it('checkOrphan', function(done) { this.timeout(100000); - s.blockDb.has(b[0], function(err, has) { + s.bDb.has(b[0], function(err, has) { assert(has); - s.blockDb.has(b[1], function(err, has) { + s.bDb.has(b[1], function(err, has) { assert(has); s.checkOrphan(b[4],b[1], function() { testNo(s,done); From a031941c62787d7038d2a7b4f3667af118b4fdec Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 5 Feb 2014 12:31:38 -0300 Subject: [PATCH 18/25] fix time at block --- lib/BlockDb.js | 2 +- lib/HistoricSync.js | 8 ++++---- test/integration/blocklist.js | 35 ++++++++++++++++++++++++----------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 67eada8..d53fa4b 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -44,7 +44,7 @@ function spec() { var time_key = TIMESTAMP_ROOT + - ( b.timestamp || Math.round(new Date().getTime() / 1000) ); + ( b.time || Math.round(new Date().getTime() / 1000) ); self.db.batch() .put(time_key, b.hash) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index b0b541d..6b547bc 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -131,7 +131,7 @@ function spec() { async.series([ // Already got it? function(c) { - self.sync.blockDb.has(blockHash, function(err, ret) { + self.sync.bDb.has(blockHash, function(err, ret) { if (err) { p(err); return c(err); @@ -317,7 +317,7 @@ function spec() { function(c) { if (!blockInfo) return c(); - self.sync.blockDb.has(blockInfo.hash, function(err, had) { + self.sync.bDb.has(blockInfo.hash, function(err, had) { existed = had; return c(err); }); @@ -357,7 +357,7 @@ function spec() { if (self.notOrphanCount) return cb(null, self.notOrphanCount); - self.sync.blockDb.countNotOrphan(function(err, count) { + self.sync.bDb.countNotOrphan(function(err, count) { if (err) return cb(err); self.notOrphanCount = count; return cb(null, self.notOrphanCount); @@ -480,7 +480,7 @@ function spec() { HistoricSync.prototype.smartImport = function(scanOpts, next) { var self = this; - self.sync.blockDb.has(self.genesis, function(err, b) { + self.sync.bDb.has(self.genesis, function(err, b) { if (err) return next(err); self.countNotOrphan(function(err, count) { if (err) return next(err); diff --git a/test/integration/blocklist.js b/test/integration/blocklist.js index 8225a17..bc18dba 100644 --- a/test/integration/blocklist.js +++ b/test/integration/blocklist.js @@ -3,24 +3,37 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var TESTING_BLOCK = '000000001f56660def9b5898ea8411d7b028854e78502e521f9ebd53e673751c'; -var START_TS = '1391607675'; -var END_TS = '1391607709'; +var TESTING_BLOCK0 = '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943'; +var TESTING_BLOCK1 = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; +var START_TS = 1293895128; // 1/1/2011 +var END_TS = 1296688428; // 2/2/2011 23:23PM -var - assert = require('assert'), - config = require('../../config/config'), +var assert = require('assert'), BlockDb = require('../../lib/BlockDb').class(); -describe('BlockDb getHashes', function(){ +var bDb; + +describe('BlockDb getBlocksByDate', function(){ + + + before(function(c) { + bDb = new BlockDb(); + return c(); + }); + + + after(function(c) { + bDb.close(c); + }); - var bdb = new BlockDb(); it('Get Hash by Date', function(done) { - bdb.getBlocksByDate(START_TS, END_TS, function(err, list) { + bDb.getBlocksByDate(START_TS, END_TS, function(err, list) { if (err) done(err); - assert.equal(list[0].ts, START_TS); - assert.equal(list[0].hash, TESTING_BLOCK); + assert(list, 'returns list'); + assert.equal(list.length,2, 'list has 2 items'); + assert.equal(list[0].hash, TESTING_BLOCK0); + assert.equal(list[1].hash, TESTING_BLOCK1); done(); }); }); From 061e4e4b342b1214b8e5c0c5db5df2a4e543ea61 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 5 Feb 2014 13:02:51 -0300 Subject: [PATCH 19/25] leveldb connection using classtool --- lib/BlockDb.js | 42 +++++++++++--------------- lib/TransactionDb.js | 40 ++++++++++++------------ test/integration/01-transactionouts.js | 9 ------ test/integration/02-transactionouts.js | 7 +---- test/integration/addr.js | 6 ---- test/integration/block.js | 5 --- test/integration/blocklist.js | 9 ++---- 7 files changed, 40 insertions(+), 78 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index d53fa4b..39e220d 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -3,7 +3,7 @@ require('classtool'); -function spec() { +function spec(b) { var TIMESTAMP_ROOT = 'b-ts-'; // b-ts- => var PREV_ROOT = 'b-prev-'; // b-prev- => (0 if orphan) @@ -17,36 +17,34 @@ function spec() { levelup = require('levelup'), BitcoreBlock= require('bitcore/Block').class(), config = require('../config/config'); + var db = b.db || levelup(config.leveldb + '/blocks'); - var BlockDb = function(db) { - this.db = db || levelup(config.leveldb + '/blocks'); + + var BlockDb = function() { }; BlockDb.prototype.close = function(cb) { - this.db.close(cb); + db.close(cb); }; BlockDb.prototype.drop = function(cb) { - var self = this; var path = config.leveldb + '/blocks'; - self.db.close(function() { + db.close(function() { require('leveldown').destroy(path, function () { - self.db = levelup(path); + db = levelup(path); return cb(); }); }); }; BlockDb.prototype.add = function(b, cb) { - var self = this; - if (!b.hash) return cb(new Error('no Hash at Block.save')); var time_key = TIMESTAMP_ROOT + ( b.time || Math.round(new Date().getTime() / 1000) ); - self.db.batch() + db.batch() .put(time_key, b.hash) .put(PREV_ROOT + b.hash, b.previousblockhash) .write(cb); @@ -55,13 +53,11 @@ function spec() { BlockDb.prototype.setOrphan = function(hash, cb) { - var self = this; - var k = PREV_ROOT + hash; - self.db.get(k, function (err,oldPrevHash) { + db.get(k, function (err,oldPrevHash) { if (err || !oldPrevHash) return cb(err); - self.db.put(PREV_ROOT + hash, 0, function() { + db.put(PREV_ROOT + hash, 0, function() { return cb(err, oldPrevHash); }); }); @@ -70,14 +66,14 @@ function spec() { //mainly for testing BlockDb.prototype.setPrev = function(hash, prevHash, cb) { - this.db.put(PREV_ROOT + hash, prevHash, function(err) { + db.put(PREV_ROOT + hash, prevHash, function(err) { return cb(err); }); }; //mainly for testing BlockDb.prototype.getPrev = function(hash, cb) { - this.db.get(PREV_ROOT + hash, function(err,val) { + db.get(PREV_ROOT + hash, function(err,val) { return cb(err,val); }); }; @@ -86,8 +82,8 @@ function spec() { BlockDb.prototype.countNotOrphan = function(cb) { var c = 0; - console.log('Counting connected blocks. This could take some minutes'); - this.db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' }) + console.log('Counting connected blocks. This could take some minutes'); + db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' }) .on('data', function (data) { if (data.value !== 0) c++; }) @@ -100,10 +96,8 @@ function spec() { }; BlockDb.prototype.has = function(hash, cb) { - var self = this; - var k = PREV_ROOT + hash; - self.db.get(k, function (err,val) { + db.get(k, function (err,val) { var ret; if (err && err.notFound) { err = null; @@ -135,17 +129,15 @@ function spec() { }; BlockDb.prototype.getBlocksByDate = function(start_ts, end_ts, cb) { - var self = this; var list = []; - self.db.createReadStream({ + db.createReadStream({ start: TIMESTAMP_ROOT + start_ts, end: TIMESTAMP_ROOT + end_ts - }) + }) .on('data', function (data) { list.push({ ts: data.key.replace(TIMESTAMP_ROOT, ''), hash: data.value, - info: {} }); }) .on('error', function (err) { diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 8250901..ebc5669 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -3,7 +3,7 @@ require('classtool'); -function spec() { +function spec(b) { // blockHash -> txid mapping (to reorgs) var ROOT = 'tx-b-'; //tx-b-- => 1/0 (connected or not) @@ -28,21 +28,21 @@ function spec() { async = require('async'), config = require('../config/config'), assert = require('assert'); + var db = b.db || levelup(config.leveldb + '/txs'); - var TransactionDb = function(db) { - this.db = db || levelup(config.leveldb + '/txs'); + var TransactionDb = function() { }; TransactionDb.prototype.close = function(cb) { - this.db.close(cb); + db.close(cb); }; TransactionDb.prototype.drop = function(cb) { var self = this; var path = config.leveldb + '/txs'; - self.db.close(function() { + db.close(function() { require('leveldown').destroy(path, function () { - self.db = levelup(path); + db = levelup(path); return cb(); }); }); @@ -54,7 +54,7 @@ function spec() { var self = this; var k = OUTS_ROOT + txid; - self.db.get(k, function (err,val) { + db.get(k, function (err,val) { var ret; @@ -76,7 +76,7 @@ function spec() { var ret=[]; // outs. - self.db.createReadStream({start: k, end: k + '~'}) + db.createReadStream({start: k, end: k + '~'}) .on('data', function (data) { var k = data.key.split('-'); var v = data.value.split(':'); @@ -92,7 +92,7 @@ function spec() { .on('end', function () { var k = SPEND_ROOT + txid; var l = ret.length; - self.db.createReadStream({start: k, end: k + '~'}) + db.createReadStream({start: k, end: k + '~'}) .on('data', function (data) { var k = data.key.split('-'); var v = data.value.split(':'); @@ -181,7 +181,7 @@ function spec() { var k = OUTS_ROOT + txid + '-' + n; - self.db.get(k, function (err,val) { + db.get(k, function (err,val) { if (err && err.notFound) { err = null; } @@ -197,7 +197,7 @@ function spec() { var ret=[]; // - self.db.createReadStream({start: k, end: k + '~'}) + db.createReadStream({start: k, end: k + '~'}) .on('data', function (data) { var k = data.key.split('-'); var v = data.value.split(':'); @@ -214,7 +214,7 @@ function spec() { .on('end', function () { async.each(ret, function(o, e_c) { var k = SPEND_ROOT + o.txid + '-' + o.index; - self.db.get(k, function(err, val) { + db.get(k, function(err, val) { if (err && err.notFound) err=null; if (err || !val) return e_c(err); @@ -237,20 +237,20 @@ function spec() { async.series([ function(c) { - self.db.createReadStream({ + db.createReadStream({ start: OUTS_ROOT + txid, end: OUTS_ROOT + txid + '~', }).pipe( - self.db.createWriteStream({type:'del'}) + db.createWriteStream({type:'del'}) ).on('close', c); }, function(c) { - self.db.createReadStream({ + db.createReadStream({ start: SPEND_ROOT + txid, end: SPEND_ROOT + txid + '~' }) .pipe( - self.db.createWriteStream({type:'del'}) + db.createWriteStream({type:'del'}) ).on('close', c); }], function(err) { @@ -313,7 +313,7 @@ function spec() { if (tx.isCoinBase) return p_c(); async.forEachLimit(tx.vin, CONCURRENCY, function(i, next_out) { - self.db.batch() + db.batch() .put( SPEND_ROOT + i.txid + '-' + i.vout , tx.txid + ':' + i.n + ':' + ts) .write(next_out); @@ -344,7 +344,7 @@ function spec() { var addr = o.scriptPubKey.addresses[0]; var sat = Math.round(o.value * util.COIN); - self.db.batch() + db.batch() .put( OUTS_ROOT + tx.txid + '-' + o.n, addr + ':' + sat) .put( ADDR_ROOT + addr + '-' + ts + '-' + tx.txid + '-' + o.n, sat) @@ -395,7 +395,7 @@ function spec() { self.add(inInfo, function(err) { if (err || !blockHash) return each_cb(err); - self.db.put(ROOT + t + '-' + blockHash, 1, function(err) { + db.put(ROOT + t + '-' + blockHash, 1, function(err) { return each_cb(err); }); }); @@ -405,7 +405,7 @@ function spec() { self.add(t, function(err) { if (err) return each_cb(err); - self.db.put(ROOT + t.txid + '-' + blockHash, 1, function(err) { + db.put(ROOT + t.txid + '-' + blockHash, 1, function(err) { return each_cb(err); }); }); diff --git a/test/integration/01-transactionouts.js b/test/integration/01-transactionouts.js index ce70a22..0fc7e0f 100644 --- a/test/integration/01-transactionouts.js +++ b/test/integration/01-transactionouts.js @@ -21,10 +21,6 @@ describe('TransactionDb fromIdWithInfo', function(){ }); - after(function(c) { - txDb.close(c); - }); - var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c'; it('tx info ' + txid, function(done) { txDb.fromIdWithInfo(txid, function(err, tx) { @@ -126,11 +122,6 @@ describe('TransactionDb Outs', function(){ return c(); }); - - after(function(c) { - txDb.close(c); - }); - txItemsValid.forEach( function(v) { if (v.disabled) return; it('test a processing tx ' + v.txid, function(done) { diff --git a/test/integration/02-transactionouts.js b/test/integration/02-transactionouts.js index a5edce7..112a761 100644 --- a/test/integration/02-transactionouts.js +++ b/test/integration/02-transactionouts.js @@ -11,7 +11,7 @@ var util = require('util'), async = require('async'), config = require('../../config/config'), - TransactionDb = require('../../lib/TransactionDb').class(); + TransactionDb = require('../../lib/TransactionDb').class(); var spentValid = JSON.parse(fs.readFileSync('test/integration/spent.json')); @@ -42,11 +42,6 @@ describe('TransactionDb Expenses', function(){ ); }); - after(function(c) { - txDb.close(c); - }); - - Object.keys(spentValid).forEach( function(txid) { it('test result of spending tx ' + txid, function(done) { var s = spentValid[txid]; diff --git a/test/integration/addr.js b/test/integration/addr.js index 763cd11..00ec531 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -17,12 +17,6 @@ describe('Address balances', function(){ return c(); }); - after(function(c) { - txDb.close(c); - }); - - - addrValid.forEach( function(v) { if (v.disabled) { console.log(v.addr + " => disabled in JSON"); diff --git a/test/integration/block.js b/test/integration/block.js index b8c34ed..e61655b 100644 --- a/test/integration/block.js +++ b/test/integration/block.js @@ -21,11 +21,6 @@ describe('BlockDb fromHashWithInfo', function(){ return c(); }); - - after(function(c) { - bDb.close(c); - }); - it('should poll block\'s info from bitcoind', function(done) { bDb.fromHashWithInfo(TESTING_BLOCK, function(err, b2) { if (err) done(err); diff --git a/test/integration/blocklist.js b/test/integration/blocklist.js index bc18dba..26e5d41 100644 --- a/test/integration/blocklist.js +++ b/test/integration/blocklist.js @@ -5,8 +5,8 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var TESTING_BLOCK0 = '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943'; var TESTING_BLOCK1 = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; -var START_TS = 1293895128; // 1/1/2011 -var END_TS = 1296688428; // 2/2/2011 23:23PM +var START_TS = 1; +var END_TS = '1296688928~'; // 2/2/2011 23:23PM var assert = require('assert'), BlockDb = require('../../lib/BlockDb').class(); @@ -21,11 +21,6 @@ describe('BlockDb getBlocksByDate', function(){ return c(); }); - - after(function(c) { - bDb.close(c); - }); - it('Get Hash by Date', function(done) { bDb.getBlocksByDate(START_TS, END_TS, function(err, list) { From 11aa379bb402ded12af916451c2b272c3e79d28d Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 5 Feb 2014 13:38:39 -0300 Subject: [PATCH 20/25] fix sync params --- insight.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/insight.js b/insight.js index 29b0c4a..b330942 100644 --- a/insight.js +++ b/insight.js @@ -49,7 +49,6 @@ if (!config.disableHistoricSync) { historicSync = new HistoricSync(); historicSync.init({ - skipDbConnection: true, shouldBroadcast: true, networkName: config.network }, function(err) { @@ -58,7 +57,7 @@ if (!config.disableHistoricSync) { console.log('[historic_sync] ' + txt); } else { - historicSync.smartImport(function(err){ + historicSync.smartImport({}, function(err){ var txt = 'ended.'; if (err) txt = 'ABORTED with error: ' + err.message; console.log('[historic_sync] ' + txt, historicSync.info()); @@ -73,7 +72,6 @@ if (!config.disableHistoricSync) { if (!config.disableP2pSync) { var ps = new PeerSync(); ps.init({ - skipDbConnection: true, broadcast_txs: true, broadcast_address_tx: true, broadcast_blocks: true, From f1c52a97a0710dc734e7e245243020909daf5fcb Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Wed, 5 Feb 2014 14:17:19 -0300 Subject: [PATCH 21/25] added limit to method blocks by date --- lib/BlockDb.js | 5 +++-- test/integration/blocklist.js | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 39e220d..fa654c1 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -128,11 +128,12 @@ function spec(b) { }); }; - BlockDb.prototype.getBlocksByDate = function(start_ts, end_ts, cb) { + BlockDb.prototype.getBlocksByDate = function(start_ts, end_ts, limit, cb) { var list = []; db.createReadStream({ start: TIMESTAMP_ROOT + start_ts, - end: TIMESTAMP_ROOT + end_ts + end: TIMESTAMP_ROOT + end_ts, + limit: limit }) .on('data', function (data) { list.push({ diff --git a/test/integration/blocklist.js b/test/integration/blocklist.js index 26e5d41..d1ed33d 100644 --- a/test/integration/blocklist.js +++ b/test/integration/blocklist.js @@ -7,6 +7,7 @@ var TESTING_BLOCK0 = '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d7 var TESTING_BLOCK1 = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; var START_TS = 1; var END_TS = '1296688928~'; // 2/2/2011 23:23PM +var LIMIT = 2; var assert = require('assert'), BlockDb = require('../../lib/BlockDb').class(); @@ -23,7 +24,7 @@ describe('BlockDb getBlocksByDate', function(){ it('Get Hash by Date', function(done) { - bDb.getBlocksByDate(START_TS, END_TS, function(err, list) { + bDb.getBlocksByDate(START_TS, END_TS, LIMIT, function(err, list) { if (err) done(err); assert(list, 'returns list'); assert.equal(list.length,2, 'list has 2 items'); From a66b7e54ceccc6a335cd3f3b1f54182abd94ac69 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 5 Feb 2014 14:25:52 -0300 Subject: [PATCH 22/25] adapt REST API for leveldb --- app/controllers/transactions.js | 10 +++++++--- app/models/Address.js | 6 +++--- lib/BlockDb.js | 1 + 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/controllers/transactions.js b/app/controllers/transactions.js index 9eeb5f1..e69b76d 100644 --- a/app/controllers/transactions.js +++ b/app/controllers/transactions.js @@ -3,11 +3,11 @@ /** * Module dependencies. */ -var Transaction = require('../../lib/TransactionDb').class(); var Address = require('../models/Address'); var async = require('async'); var common = require('./common'); +var TransactionDb = require('../../lib/TransactionDb').class(); var BlockDb = require('../../lib/BlockDb').class(); var bdb = new BlockDb(); @@ -17,7 +17,9 @@ var bdb = new BlockDb(); * Find transaction by hash ... */ exports.transaction = function(req, res, next, txid) { - Transaction.fromIdWithInfo(txid, function(err, tx) { + var tDb = new TransactionDb(); + + tDb.fromIdWithInfo(txid, function(err, tx) { if (err || ! tx) return common.handleErrors(err, res); else { @@ -40,7 +42,9 @@ exports.show = function(req, res) { var getTransaction = function(txid, cb) { - Transaction.fromIdWithInfo(txid, function(err, tx) { + var tDb = new TransactionDb(); + + tDb.fromIdWithInfo(txid, function(err, tx) { if (err) { console.log(err); } diff --git a/app/models/Address.js b/app/models/Address.js index 8dc6286..97e0527 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -9,8 +9,7 @@ function spec() { var BitcoreUtil = require('bitcore/util/util'); var TransactionDb = require('../../lib/TransactionDb').class(); - function Address(addrStr, txDb) { - this.txDb = txDb || new TransactionDb(); + function Address(addrStr) { this.balanceSat = 0; this.totalReceivedSat = 0; this.totalSentSat = 0; @@ -57,9 +56,10 @@ function spec() { Address.prototype.update = function(next) { var self = this; + var db = new TransactionDb(); async.series([ function (cb) { - self.txDb.fromAddr(self.addrStr, function(err,txOut){ + db.fromAddr(self.addrStr, function(err,txOut){ if (err) return cb(err); txOut.forEach(function(txItem){ diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 39e220d..430c4b5 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -135,6 +135,7 @@ function spec(b) { end: TIMESTAMP_ROOT + end_ts }) .on('data', function (data) { +console.log('[BlockDb.js.137:data:]',data); //TODO list.push({ ts: data.key.replace(TIMESTAMP_ROOT, ''), hash: data.value, From 2f98fe35b335157c4e62f1be9c33c15cd1d43291 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Wed, 5 Feb 2014 15:08:39 -0300 Subject: [PATCH 23/25] rpc on classtool --- app/models/Address.js | 3 +-- lib/BlockDb.js | 9 +++------ lib/TransactionDb.js | 23 ++++++++++++----------- lib/TransactionRpc.js | 11 +++++------ test/integration/addr.js | 8 +++----- test/integration/addr.json | 6 +++--- 6 files changed, 27 insertions(+), 33 deletions(-) diff --git a/app/models/Address.js b/app/models/Address.js index 97e0527..d377cb1 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -61,9 +61,8 @@ function spec() { function (cb) { db.fromAddr(self.addrStr, function(err,txOut){ if (err) return cb(err); - txOut.forEach(function(txItem){ - var v = parseInt(txItem.value_sat); + var v = txItem.value_sat; self.totalReceivedSat += v; self.transactions.push(txItem.txid); diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 430c4b5..95f9159 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -17,7 +17,8 @@ function spec(b) { levelup = require('levelup'), BitcoreBlock= require('bitcore/Block').class(), config = require('../config/config'); - var db = b.db || levelup(config.leveldb + '/blocks'); + var db = b.db || levelup(config.leveldb + '/blocks'); + var rpc = b.rpc || new RpcClient(config.bitcoind); var BlockDb = function() { @@ -112,8 +113,6 @@ function spec(b) { BlockDb.prototype.fromHashWithInfo = function(hash, cb) { - var rpc = new RpcClient(config.bitcoind); - rpc.getBlock(hash, function(err, info) { // Not found? if (err && err.code === -5) return cb(); @@ -135,7 +134,6 @@ function spec(b) { end: TIMESTAMP_ROOT + end_ts }) .on('data', function (data) { -console.log('[BlockDb.js.137:data:]',data); //TODO list.push({ ts: data.key.replace(TIMESTAMP_ROOT, ''), hash: data.value, @@ -149,8 +147,7 @@ console.log('[BlockDb.js.137:data:]',data); //TODO }); }; - BlockDb.blockIndex = function(height, cb) { - var rpc = new RpcClient(config.bitcoind); + BlockDb.prototype.blockIndex = function(height, cb) { rpc.getBlockHash(height, function(err, bh){ if (err) return cb(err); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index ebc5669..7a45d74 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -82,8 +82,8 @@ function spec(b) { var v = data.value.split(':'); ret.push({ addr: v[0], - value_sat: v[1], - index: k[2], + value_sat: parseInt(v[1]), + index: parseInt(k[2]), }); }) .on('error', function (err) { @@ -98,9 +98,9 @@ function spec(b) { var v = data.value.split(':'); var set=0; for(var i=0; i Date: Wed, 5 Feb 2014 15:11:39 -0300 Subject: [PATCH 24/25] front-end of block list by date is ready. fillCache for getting blocks. --- app/controllers/blocks.js | 54 ++++++++++++++++----------------------- lib/BlockDb.js | 3 ++- public/js/config.js | 4 +-- 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/app/controllers/blocks.js b/app/controllers/blocks.js index 4acc120..468bc13 100644 --- a/app/controllers/blocks.js +++ b/app/controllers/blocks.js @@ -72,7 +72,7 @@ console.log('[blocks.js.60]: could not get %s from RPC. Orphan? Error?', blockha * List of blocks by date */ exports.list = function(req, res) { - var limit = req.query.limit || 0; + var limit = req.query.limit || -1; var isToday = false; //helper to convert timestamps to yyyy-mm-dd format @@ -103,37 +103,27 @@ exports.list = function(req, res) { var prev = formatTimestamp(new Date((gte - 86400) * 1000)); var next = formatTimestamp(new Date(lte * 1000)); - /* - Block - .find({ - time: { - '$gte': gte, - '$lte': lte + bdb.getBlocksByDate(gte, lte, limit, function(err, blocks) { + if (err) { + res.status(500).send(err); + } + else { + var blockshashList = []; + for(var i=0;i Date: Wed, 5 Feb 2014 15:27:50 -0300 Subject: [PATCH 25/25] fixed block by height --- lib/BlockDb.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index a64ae66..426f4c9 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -151,7 +151,7 @@ console.log('[BlockDb.js.137:data:]',data); //TODO }); }; - BlockDb.blockIndex = function(height, cb) { + BlockDb.prototype.blockIndex = function(height, cb) { var rpc = new RpcClient(config.bitcoind); rpc.getBlockHash(height, function(err, bh){ if (err) return cb(err);