From c516c113cb66c92a75078d117fdfcdadf17a6782 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 4 Feb 2014 23:50:18 -0300 Subject: [PATCH] more fixes on sync --- app/models/Transaction.js | 23 +++-- lib/BlockDb.js | 5 +- lib/HistoricSync.js | 40 ++++++--- lib/TransactionDb.js | 71 +++++++++++++-- test/integration/01-transactionouts.js | 111 +++++++++++++++++++++-- test/integration/transaction.js | 120 ------------------------- 6 files changed, 218 insertions(+), 152 deletions(-) delete mode 100644 test/integration/transaction.js diff --git a/app/models/Transaction.js b/app/models/Transaction.js index f3b20dc..678f870 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -7,17 +7,25 @@ function spec() { var util = require('bitcore/util/util'), TransactionRpc = require('../../lib/TransactionRpc').class(), - TransactionOut = require('./TransactionOut'), + TransactionDb = require('../../lib/TransactionDb').class(), async = require('async'); var CONCURRENCY = 20; - function Transaction() { + function Transaction(tdb) { this.txid = null; + this.tdb = tdb || new TransactionDb(); } - Transaction.fromIdWithInfo = function (txid,cb) { - var tx = new Transaction(); + Transaction.fromIdWithInfo = function (txid, tdb, cb) { + if (typeof tdb === 'function') { + cb = tdb; + tdb = null; + } + var tx = new Transaction(tdb); + + +console.log('[Transaction.js.27]',tx.tdb); //TODO tx.txid = txid; tx._fillInfo(function(err) { @@ -33,21 +41,22 @@ function spec() { TransactionRpc.getRpcInfo(self.txid, function(err, info) { if (err) return next(err); - Transaction._fillOutpoints(info, function() { + self._fillOutpoints(info, function() { self.info = info; return next(); }); }); }; - Transaction._fillOutpoints = function(info, cb) { + Transaction.prototype._fillOutpoints = function(info, cb) { + var self = this; if (!info || info.isCoinBase) return cb(); var valueIn = 0; var incompleteInputs = 0; async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) { - TransactionOut.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) { + self.tdb.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) { if (err || !addr || !valueSat ) { diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 551d8df..51dd7ea 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -44,7 +44,7 @@ function spec() { self.db.batch() .put(time_key, b.hash) - .put(PREV_ROOT + b.hash, b.prev_block) + .put(PREV_ROOT + b.hash, b.previousblockhash) .write(cb); }; @@ -82,6 +82,7 @@ function spec() { BlockDb.prototype.countNotOrphan = function(cb) { var c = 0; + console.log('Counting connected blocks. This could take some minutes'); this.db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' }) .on('data', function (data) { if (data.value !== 0) c++; @@ -90,7 +91,7 @@ function spec() { return cb(err); }) .on('end', function () { - return cb(null); + return cb(null, c); }); }; diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 95c6b14..eab593e 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -9,7 +9,6 @@ function spec() { var RpcClient = require('bitcore/RpcClient').class(); var bitutil = require('bitcore/util/util'); var Address = require('bitcore/Address').class(); - var Deserialize = require('bitcore/Deserialize'); var Script = require('bitcore/Script').class(); var networks = require('bitcore/networks'); var async = require('async'); @@ -17,6 +16,7 @@ function spec() { var Sync = require('./Sync').class(); var sockets = require('../app/controllers/socket.js'); var BlockExtractor = require('./BlockExtractor.js').class(); + //var Deserialize = require('bitcore/Deserialize'); var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:'; @@ -37,6 +37,7 @@ function spec() { this.syncedBlocks = 0; this.skippedBlocks = 0; this.orphanBlocks = 0; + this.type =''; } function p() { @@ -97,6 +98,7 @@ function spec() { syncedBlocks: this.syncedBlocks, orphanBlocks: this.orphanBlocks, error: this.error, + type: this.type, }; }; @@ -273,7 +275,9 @@ function spec() { if (err || ! b) return c(err); blockInfo = b.getStandardizedObject(b.txs, self.network); - blockInfo.curWork = Deserialize.intFromCompact(b.bits); + // blockInfo.curWork = Deserialize.intFromCompact(b.bits); + // We keep the RPC field names + blockInfo.previousblockhash = blockInfo.prev_block; var ti=0; // Get TX Address @@ -297,31 +301,40 @@ function spec() { return c(); }); }, - //store it + //check prev function(c) { - if (blockInfo && self.prevHash && blockInfo.prev_block !== self.prevHash) { + if (blockInfo && self.prevHash && blockInfo.previousblockhash !== self.prevHash) { console.log('Hole found @%s', blockInfo.hash); - console.log('From: %s To: %s', self.prevHash, blockInfo.prev_block); - self.sync.checkOrphan(self.prevHash, blockInfo.prev_block, c); + console.log('From: %s To: %s', self.prevHash, blockInfo.previousblockhash); + self.sync.checkOrphan(self.prevHash, blockInfo.previousblockhash, c); } else return c(); }, - //store it + //check it function(c) { if (!blockInfo) return c(); - self.sync.storeBlock(blockInfo, function(err) { - existed = err && err.toString().match(/E11000/); + self.sync.blockDb.has(blockInfo.hash, function(err, had) { + existed = had; + return c(err); + }); + }, + //store it + function(c) { + if (!blockInfo || existed) return c(); - if (err && ! existed) return c(err); - return c(); + self.sync.storeBlock(blockInfo, function(err) { + return c(err); }); }, function(c) { if (blockInfo && blockInfo.hash) { self.prevHash = blockInfo.hash; - self.syncedBlocks++; + if (existed) + self.skippedBlocks++; + else + self.syncedBlocks++; } else self.status = 'finished'; @@ -424,6 +437,7 @@ function spec() { if (scanOpts.fromFiles) { self.status = 'syncing'; + self.type = 'from .dat Files'; async.whilst(function() { return self.status === 'syncing'; }, function (w_cb) { @@ -437,6 +451,7 @@ function spec() { }); } else { + self.type = 'from RPC calls'; self.getPrevNextBlock(start, end, scanOpts, function(err) { return next(err); }); @@ -475,6 +490,7 @@ function spec() { } else { p('Genesis block found. Syncing upto known blocks.'); + p('Got %d out of %d blocks', count, self.blockChainHeight); scanOpts.reverse = true; scanOpts.upToExisting = true; } diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 38924e9..3f8394c 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -5,8 +5,8 @@ require('classtool'); function spec() { - // blockHash -> txid mapping (to orphanize )/ - var ROOT = 'tx-b-'; //tx-b- => txid + // blockHash -> txid mapping (to reorgs) + var ROOT = 'tx-b-'; //tx-b-- => 1 // to show tx outs var OUTS_ROOT = 'txouts-'; //txouts-- => [addr, btc_sat] @@ -107,6 +107,65 @@ function spec() { }); }; + TransactionDb.prototype._fillOutpoints = function(info, cb) { + var self = this; + + if (!info || info.isCoinBase) return cb(); + + var valueIn = 0; + var incompleteInputs = 0; + async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) { + self.fromTxIdN(i.txid, i.vout, function(err, addr, valueSat) { + + + if (err || !addr || !valueSat ) { + console.log('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid); + incompleteInputs = 1; + return c_in(); // error not scaled + } + i.addr = addr; + i.valueSat = valueSat; + i.value = valueSat / util.COIN; + + valueIn += i.valueSat; + return c_in(); + }); + }, + function () { + if (! incompleteInputs ) { + info.valueIn = valueIn / util.COIN; + info.fees = (valueIn - parseInt(info.valueOut * util.COIN)) / util.COIN ; + } + else { + info.incompleteInputs = 1; + } + return cb(); + }); + }; + + TransactionDb.prototype._getInfo = function(txid, next) { + var self = this; + + TransactionRpc.getRpcInfo(txid, function(err, info) { + if (err) return next(err); + + self._fillOutpoints(info, function() { + return next(null, info); + }); + }); + }; + + + TransactionDb.prototype.fromIdWithInfo = function (txid, cb) { + var self = this; + + self._getInfo(txid, function(err, info) { + if (err) return cb(err); + if (!info ) return cb(); + + return cb(err, {txid: txid, info: info} ); + }); + }; TransactionDb.prototype.fromTxIdN = function(txid, n, cb) { var self = this; @@ -117,8 +176,8 @@ function spec() { if (err && err.notFound) { err = null; } - var a = val.split('-'); - return cb(err, val, a[0], a[1]); + var a = val.split(':'); + return cb(err, a[0], a[1]); }); }; @@ -343,7 +402,7 @@ console.log('[TransactionDb.js.165]', ret); //TODO self.add(inInfo, function(err) { if (err || !blockHash) return each_cb(err); - self.db.put(ROOT + blockHash, t, function(err) { + self.db.put(ROOT + t + '-' + blockHash, 1, function(err) { return each_cb(err); }); }); @@ -353,7 +412,7 @@ console.log('[TransactionDb.js.165]', ret); //TODO self.add(t, function(err) { if (err) return each_cb(err); - self.db.put(ROOT + blockHash, t.txid, function(err) { + self.db.put(ROOT + t.txid + '-' + blockHash, 1, function(err) { return each_cb(err); }); }); diff --git a/test/integration/01-transactionouts.js b/test/integration/01-transactionouts.js index b9e5ef5..12705fd 100644 --- a/test/integration/01-transactionouts.js +++ b/test/integration/01-transactionouts.js @@ -5,16 +5,117 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var - assert = require('assert'), +var assert = require('assert'), fs = require('fs'), util = require('util'), - config = require('../../config/config'), TransactionDb = require('../../lib/TransactionDb').class(); var txItemsValid = JSON.parse(fs.readFileSync('test/integration/txitems.json')); +var tdb; -describe('TransactionDb', function(){ +describe('TransactionDb fromIdWithInfo', function(){ + + before(function(c) { + tdb = new TransactionDb(); + return c(); + }); + + var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c'; + it('tx info ' + txid, function(done) { + tdb.fromIdWithInfo(txid, function(err, tx) { + + if (err) done(err); + assert.equal(tx.txid, txid); + assert(!tx.info.isCoinBase); + + for(var i=0; i<20; i++) + assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i); + assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0'); + assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1'); + done(); + }); + }); + + it('tx info', function(done) { + var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; + tdb.fromIdWithInfo(txid, function(err, tx) { + if (err) done(err); + assert.equal(tx.txid, txid); + assert(!tx.info.isCoinBase); + done(); + }); + }); + + it('should pool tx\'s info from bitcoind', function(done) { + var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; + tdb.fromIdWithInfo(txid, function(err, tx) { + if (err) done(err); + assert.equal(tx.info.txid, txid); + assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74'); + assert.equal(tx.info.valueOut, 1.66174); + assert.equal(tx.info.fees, 0.0005 ); + assert.equal(tx.info.size, 226 ); + assert(!tx.info.isCoinBase); + done(); + }); + }); + + var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399'; + it('test a coinbase TX ' + txid1, function(done) { + tdb.fromIdWithInfo(txid1, function(err, tx) { + if (err) done(err); + assert(tx.info.isCoinBase); + assert.equal(tx.info.txid, txid1); + assert(!tx.info.feeds); + done(); + }); + }); + var txid22 = '666'; + it('test invalid TX ' + txid22, function(done) { + tdb.fromIdWithInfo(txid22, function(err, tx) { + if (err && err.message.match(/must.be.hexadecimal/)) { + return done(); + } + else { + return done(err); + } + }); + }); + + var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227'; + it('test unexisting TX ' + txid23, function(done) { + + tdb.fromIdWithInfo(txid23, function(err, tx) { + assert(!err); + assert(!tx); + return done(); + }); + }); + + + + var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; + it('create TX on the fly ' + txid2, function(done) { + tdb.fromIdWithInfo(txid2, function(err, tx) { + if (err) return done(err); + assert.equal(tx.info.txid, txid2); + done(); + }); + }); + + txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; + it('test a broken TX ' + txid2, function(done) { + tdb.fromIdWithInfo(txid2, function(err, tx) { + if (err) return done(err); + assert.equal(tx.info.txid, txid2); + assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU'); + done(); + }); + }); +}); +/* + +describe('TransactionDb Outs', function(){ var tdb = new TransactionDb(); @@ -63,5 +164,5 @@ describe('TransactionDb', function(){ }); }); - +*/ diff --git a/test/integration/transaction.js b/test/integration/transaction.js deleted file mode 100644 index d5a87c0..0000000 --- a/test/integration/transaction.js +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -process.env.NODE_ENV = process.env.NODE_ENV || 'development'; - - - -var mongoose= require('mongoose'), - assert = require('assert'), - config = require('../../config/config'), - Transaction = require('../../app/models/Transaction').class(); - - -mongoose.connection.on('error', function(err) { console.log(err); }); - -describe('Transaction', function(){ - - before(function(done) { - mongoose.connect(config.db); - done(); - }); - - after(function(done) { - mongoose.connection.close(); - done(); - }); - var txid = '7e621eeb02874ab039a8566fd36f4591e65eca65313875221842c53de6907d6c'; - it('txid ' + txid, function(done) { - Transaction.fromIdWithInfo(txid, function(err, tx) { - if (err) done(err); - assert.equal(tx.txid, txid); - assert(!tx.info.isCoinBase); - - for(var i=0; i<20; i++) - assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i); - assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0'); - assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1'); - done(); - }); - }); - - it('should pool tx\'s object from mongoose', function(done) { - var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; - Transaction.fromIdWithInfo(txid, function(err, tx) { - if (err) done(err); - assert.equal(tx.txid, txid); - assert(!tx.info.isCoinBase); - done(); - }); - }); - - it('should pool tx\'s info from bitcoind', function(done) { - var txid = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca237'; - Transaction.fromIdWithInfo(txid, function(err, tx) { - if (err) done(err); - assert.equal(tx.info.txid, txid); - assert.equal(tx.info.blockhash, '000000000185678d3d7ecc9962c96418174431f93fe20bf216d5565272423f74'); - assert.equal(tx.info.valueOut, 1.66174); - assert.equal(tx.info.fees, 0.0005 ); - assert.equal(tx.info.size, 226 ); - assert(!tx.info.isCoinBase); - done(); - }); - }); - - var txid1 = '2a104bab1782e9b6445583296d4a0ecc8af304e4769ceb64b890e8219c562399'; - it('test a coinbase TX ' + txid1, function(done) { - Transaction.fromIdWithInfo(txid1, function(err, tx) { - if (err) done(err); - assert(tx.info.isCoinBase); - assert.equal(tx.info.txid, txid1); - assert(!tx.info.feeds); - done(); - }); - }); - var txid22 = '666'; - it('test invalid TX ' + txid22, function(done) { - Transaction.fromIdWithInfo(txid22, function(err, tx) { - if (err && err.message.match(/must.be.hexadecimal/)) { - return done(); - } - else { - return done(err); - } - }); - }); - - var txid23 = '21798ddc9664ac0ef618f52b151dda82dafaf2e26d2bbef6cdaf55a6957ca227'; - it('test unexisting TX ' + txid23, function(done) { - - Transaction.fromIdWithInfo(txid23, function(err, tx) { - assert(!err); - assert(!tx); - return done(); - }); - }); - - - - var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; - it('create TX on the fly ' + txid2, function(done) { - Transaction.fromIdWithInfo(txid2, function(err, tx) { - if (err) return done(err); - assert.equal(tx.info.txid, txid2); - done(); - }); - }); - - var txid2 = '64496d005faee77ac5a18866f50af6b8dd1f60107d6795df34c402747af98608'; - it('test a broken TX ' + txid2, function(done) { - Transaction.fromIdWithInfo(txid2, function(err, tx) { - if (err) return done(err); - assert.equal(tx.info.txid, txid2); - assert.equal(tx.info.vin[0].addr, 'n1JagbRWBDi6VMvG7HfZmXX74dB9eiHJzU'); - done(); - }); - }); - -}); -