diff --git a/app/models/Transaction.js b/app/models/Transaction.js index 12b4698..245978a 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -1,361 +1,200 @@ 'use strict'; -/** - * Module dependencies. - */ - -var mongoose = require('mongoose'), - Schema = mongoose.Schema, - async = require('async'), - RpcClient = require('bitcore/RpcClient').class(), - Transaction = require('bitcore/Transaction').class(), - Address = require('bitcore/Address').class(), - BitcoreBlock = require('bitcore/Block').class(), - networks = require('bitcore/networks'), - util = require('bitcore/util/util'), - bignum = require('bignum'), - config = require('../../config/config'), - sockets = require('../controllers/socket.js'), - TransactionItem = require('./TransactionItem'); - -var CONCURRENCY = 5; - -// TODO: use bitcore networks module -var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; - -/** - */ -var TransactionSchema = new Schema({ - // For now we keep this as short as possible - // More fields will be propably added as we move - // forward with the UX - txid: { - type: String, - index: true, - unique: true, - }, -/* TODO? - orphaned: { - type: Boolean, - default: false, - }, - */ - time: Number, -}); - -/** - * Statics - */ - -TransactionSchema.statics.load = function(id, cb) { - this.findOne({ - _id: id - }).exec(cb); -}; +require('classtool'); -TransactionSchema.statics.fromId = function(txid, cb) { - this.findOne({ - txid: txid, - }).exec(cb); -}; +function spec() { + + var async = require('async'), + RpcClient = require('bitcore/RpcClient').class(), + BitcoreTransaction = require('bitcore/Transaction').class(), + Address = require('bitcore/Address').class(), + BitcoreBlock = require('bitcore/Block').class(), + networks = require('bitcore/networks'), + util = require('bitcore/util/util'), + bignum = require('bignum'), + config = require('../../config/config'), + sockets = require('../controllers/socket.js'), + TransactionItem = require('./TransactionItem'); + + var CONCURRENCY = 15; -TransactionSchema.statics.fromIdWithInfo = function(txid, cb) { - var That = this; - - this.fromId(txid, function(err, tx) { - if (err) return cb(err); - - if (!tx) { - // No in mongo...but maybe in bitcoind... lets query it - tx = new That(); - - tx.txid = txid; - tx.fillInfo(function(err, txInfo) { - - if (err) return cb(err); - if (!txInfo) return cb(); - - tx.save(function(err) { - return cb(err,tx); - }); - }); - } - else { - tx.fillInfo(function(err) { - return cb(err,tx); - }); - } - }); -}; + function Transaction() { + this.txid = null; + } -TransactionSchema.statics.createFromArray = function(txs, time, next) { - var that = this; - if (!txs) return next(); - var mongo_txs = []; - - async.forEachLimit(txs, CONCURRENCY, function(txid, cb) { - - that.explodeTransactionItems( txid, time, function(err, addrs) { - if (err) return next(err); - if (addrs) { - async.each(addrs, function(addr){ - sockets.broadcast_address_tx(addr, {'txid': txid}); - }); - - } - - that.create({txid: txid, time: time}, function(err, new_tx) { - if (err && ! err.toString().match(/E11000/)) return cb(err); - - if (new_tx) { - mongo_txs.push(new_tx); - } - - return cb(); - }); - }); - }, - function(err) { - return next(err, mongo_txs); - }); -}; - - -TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) { - var addrs = []; - - // Is it from genesis block? (testnet==livenet) - // TODO: parse it from networks.genesisTX - if (txid === genesisTXID) return cb(); - - this.queryInfo(txid, function(err, info) { - if (err || !info) return cb(err); - - var index = 0; - info.vin.forEach( function(i){ - i.n = index++; - }); - - async.forEachLimit(info.vin, CONCURRENCY, function(i, next_in) { - if (i.addr && i.value) { - - TransactionItem.create({ - txid : txid, - value_sat : -1 * i.valueSat, - addr : i.addr, - index : i.n, - ts : time, - }, next_in); - if (addrs.indexOf(i.addr) === -1) { - addrs.push(i.addr); - } - } - else { - if ( !i.coinbase ) { - console.log ('WARN in TX: %s: could not parse INPUT %d', txid, i.n); - } - return next_in(); - } - }, - function (err) { - if (err && !err.message.match(/E11000/) ) console.log (err); - async.forEachLimit(info.vout, CONCURRENCY, function(o, next_out) { - - /* - * TODO Support multisigs - */ - if (o.value && o.scriptPubKey && o.scriptPubKey.addresses && o.scriptPubKey.addresses[0]) { - TransactionItem.create({ - txid : txid, - value_sat : o.valueSat, - addr : o.scriptPubKey.addresses[0], // TODO: only address 0? - index : o.n, - ts : time, - }, next_out); - if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { - addrs.push(o.scriptPubKey.addresses[0]); - } - } - else { - console.log ('WARN in TX: %s could not parse OUTPUT %d', txid, o.n); - return next_out(); - } - }, - function (err) { - if (err && ! err.toString().match(/E11000/)) return cb(err); - return cb(null, addrs); - }); - }); - }); -}; - - - -TransactionSchema.statics.getOutpoints = function (tx, next) { - - if (tx.isCoinBase()) return next(); - - var rpc = new RpcClient(config.bitcoind); - var network = ( config.network === 'testnet') ? networks.testnet : networks.livenet ; - - async.forEachLimit(tx.ins, CONCURRENCY, function(i, cb) { - - var outHash = i.getOutpointHash(); - var outIndex = i.getOutpointIndex(); - var outHashBase64 = outHash.reverse().toString('hex'); - - var c=0; - rpc.getRawTransaction(outHashBase64, function(err, txdata) { - var txin = new Transaction(); - if (err || ! txdata.result) return cb( new Error('Input TX '+outHashBase64+' not found')); - - var b = new Buffer(txdata.result,'hex'); - txin.parse(b); - - /* - *We have to parse it anyways. It will have outputs even it is a coinbase tx - if ( txin.isCoinBase() ) { - return cb(); - } - */ - - txin.outs.forEach( function(j) { - // console.log( c + ': ' + util.formatValue(j.v) ); - if (c === outIndex) { - i.value = j.v; - - // This is used for pay-to-pubkey transaction in which - // the pubkey is not provided on the input - var scriptPubKey = j.getScript(); - var hash = scriptPubKey.simpleOutHash(); - if (hash) { - var addr = new Address(network.addressPubkey, hash); - i.addrFromOutput = addr.toString(); - } - } - c++; - }); - return cb(); - }); - }, - function(err) { - return next(err); - } - ); -}; - - -TransactionSchema.statics.queryInfo = function(txid, cb) { - var that = this; - var network = ( config.network === 'testnet') ? networks.testnet : networks.livenet ; - var rpc = new RpcClient(config.bitcoind); - - rpc.getRawTransaction(txid, 1, function(err, txInfo) { - - // Not found? - if (err && err.code === -5) return cb(); - - if (err) return cb(err); - - var info = txInfo.result; - - // Transaction parsing - var b = new Buffer(txInfo.result.hex,'hex'); + Transaction.fromIdWithInfo = function (txid,cb) { var tx = new Transaction(); - tx.parse(b); + tx.txid = txid; + + tx.fillInfo(function(err) { + if (err) return cb(err); + if (! tx.info ) return cb(); + + return cb(err,tx); + }); + }; + + + + Transaction.prototype.fillInfo = function(next) { + var self = this; + + Transaction.queryInfo(self.txid, function(err, info) { + if (err) return next(err); + + self.info = info; + return next(); + }); + }; + + + Transaction.getOutpoints = function (tx, next) { + + if (tx.isCoinBase()) return next(); + + var rpc = new RpcClient(config.bitcoind); + var network = ( config.network === 'testnet') ? networks.testnet : networks.livenet ; + + async.forEachLimit(tx.ins, CONCURRENCY, function(i, cb) { + + var outHash = i.getOutpointHash(); + var outIndex = i.getOutpointIndex(); + var outHashBase64 = outHash.reverse().toString('hex'); + + var c=0; + rpc.getRawTransaction(outHashBase64, function(err, txdata) { + var txin = new BitcoreTransaction(); + if (err || ! txdata.result) return cb( new Error('Input TX '+outHashBase64+' not found')); + + var b = new Buffer(txdata.result,'hex'); + txin.parse(b); + + /* + *We have to parse it anyways. It will have outputs even it is a coinbase tx + if ( txin.isCoinBase() ) { + return cb(); + } + */ + + txin.outs.forEach( function(j) { + // console.log( c + ': ' + util.formatValue(j.v) ); + if (c === outIndex) { + i.value = j.v; + + // This is used for pay-to-pubkey transaction in which + // the pubkey is not provided on the input + var scriptPubKey = j.getScript(); + var hash = scriptPubKey.simpleOutHash(); + if (hash) { + var addr = new Address(network.addressPubkey, hash); + i.addrFromOutput = addr.toString(); + } + } + c++; + }); + return cb(); + }); + }, + function(err) { + return next(err); + } + ); + }; + + + Transaction.queryInfo = function(txid, cb) { + var self = this; + var network = ( config.network === 'testnet') ? networks.testnet : networks.livenet ; + var rpc = new RpcClient(config.bitcoind); + + rpc.getRawTransaction(txid, 1, function(err, txInfo) { + + // Not found? + if (err && err.code === -5) return cb(); - that.getOutpoints(tx, function(err) { if (err) return cb(err); - // Copy TX relevant values to .info + var info = txInfo.result; - var c = 0; - var valueIn = bignum(0); - var valueOut = bignum(0); + // Transaction parsing + var b = new Buffer(txInfo.result.hex,'hex'); + var tx = new BitcoreTransaction(); + tx.parse(b); - if ( tx.isCoinBase() ) { - info.isCoinBase = true; - } - else { - tx.ins.forEach(function(i) { - if (i.value) { - info.vin[c].value = parseFloat(util.formatValue(i.value)); - var n = util.valueToBigInt(i.value).toNumber(); - info.vin[c].valueSat = n; - valueIn = valueIn.add( n ); + self.getOutpoints(tx, function(err) { + if (err) return cb(err); - var scriptSig = i.getScript(); - var pubKey = scriptSig.simpleInPubKey(); + // Copy TX relevant values to .info - // We check for pubKey in case a broken / strange TX. - if (pubKey) { - var pubKeyHash = util.sha256ripe160(pubKey); - var addr = new Address(network.addressPubkey, pubKeyHash); - var addrStr = addr.toString(); - info.vin[c].addr = addrStr; + var c = 0; + var valueIn = bignum(0); + var valueOut = bignum(0); + + if ( tx.isCoinBase() ) { + info.isCoinBase = true; + } + else { + tx.ins.forEach(function(i) { + if (i.value) { + info.vin[c].value = parseFloat(util.formatValue(i.value)); + var n = util.valueToBigInt(i.value).toNumber(); + info.vin[c].valueSat = n; + valueIn = valueIn.add( n ); + + var scriptSig = i.getScript(); + var pubKey = scriptSig.simpleInPubKey(); + + // We check for pubKey in case a broken / strange TX. + if (pubKey) { + var pubKeyHash = util.sha256ripe160(pubKey); + var addr = new Address(network.addressPubkey, pubKeyHash); + var addrStr = addr.toString(); + info.vin[c].addr = addrStr; + } + else { + if (i.addrFromOutput) + info.vin[c].addr = i.addrFromOutput; + } } else { - if (i.addrFromOutput) - info.vin[c].addr = i.addrFromOutput; + console.log('TX could not be parsed: %s,%d' ,txInfo.result.txid, c); } - } - else { - console.log('TX could not be parsed: %s,%d' ,txInfo.result.txid, c); - } + c++; + }); + } + + c=0; + tx.outs.forEach( function(i) { + var n = util.valueToBigInt(i.v).toNumber(); + valueOut = valueOut.add(n); + + info.vout[c].valueSat = n; c++; }); - } - c=0; - tx.outs.forEach( function(i) { - var n = util.valueToBigInt(i.v).toNumber(); - valueOut = valueOut.add(n); + info.valueOut = valueOut / util.COIN; - info.vout[c].valueSat = n; - c++; + if ( !tx.isCoinBase() ) { + info.valueIn = valueIn / util.COIN; + info.fees = (valueIn - valueOut) / util.COIN; + } + else { + var reward = BitcoreBlock.getBlockValue(info.height) / util.COIN; + info.vin[0].reward = reward; + info.valueIn = reward; + } + + info.size = b.length; + return cb(null, info); }); - - info.valueOut = valueOut / util.COIN; - - if ( !tx.isCoinBase() ) { - info.valueIn = valueIn / util.COIN; - info.fees = (valueIn - valueOut) / util.COIN; - } - else { - var reward = BitcoreBlock.getBlockValue(info.height) / util.COIN; - info.vin[0].reward = reward; - info.valueIn = reward; - } - - info.size = b.length; - - return cb(null, info); }); - }); -}; + }; + + return Transaction; +} +module.defineClass(spec); - -TransactionSchema.methods.fillInfo = function(next) { - var that = this; - - mongoose.model('Transaction', TransactionSchema).queryInfo(that.txid, function(err, info) { - if (err) return next(err); - - that.info = info; - if (! that.info) { - return next(); - } - else { - that.info.time = that.time; - return next(); - } - }); -}; - - - -module.exports = mongoose.model('Transaction', TransactionSchema); diff --git a/app/models/TransactionItem.js b/app/models/TransactionItem.js index 5a04f84..d56f8f4 100644 --- a/app/models/TransactionItem.js +++ b/app/models/TransactionItem.js @@ -4,8 +4,12 @@ * Module dependencies. */ var mongoose = require('mongoose'), + async = require('async'), + Transaction = require('./Transaction').class(), Schema = mongoose.Schema; +var CONCURRENCY = 15; + var TransactionItemSchema = new Schema({ txid: String, index: Number, @@ -20,6 +24,10 @@ var TransactionItemSchema = new Schema({ }); +// TODO: use bitcore networks module +var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; + + // Compound index TransactionItemSchema.index({txid: 1, index: 1, value_sat: 1}, {unique: true, dropDups: true}); @@ -51,4 +59,125 @@ TransactionItemSchema.statics.fromTxId = function(txid, cb) { }); }; + +TransactionItemSchema.statics.explodeTransactionItems = function(txid, cb) { + + var Self = this; + var addrs = []; + var is_new = true; + + // Is it from genesis block? (testnet==livenet) + // TODO: parse it from networks.genesisTX + if (txid === genesisTXID) return cb(); + + Transaction.queryInfo(txid, function(err, info) { + if (err || !info) return cb(err); + + var index = 0; + info.vin.forEach( function(i){ + i.n = index++; + }); + + async.forEachLimit(info.vin, CONCURRENCY, function(i, next_in) { + if (i.addr && i.value) { + + Self.create({ + txid : txid, + value_sat : -1 * i.valueSat, + addr : i.addr, + index : i.n, + ts : info.time, + }, next_in); + if (addrs.indexOf(i.addr) === -1) { + addrs.push(i.addr); + } + } + else { + if ( !i.coinbase ) { + console.log ('WARN in TX: %s: could not parse INPUT %d', txid, i.n); + } + return next_in(); + } + }, + function (err) { + + if (err) { + if (err.message.match(/E11000/)) { + is_new = false; + } + else { + console.log('ERR at TX %s: %s', txid, err); + return cb(err); + } + } + + // Parse Outputs + async.forEachLimit(info.vout, CONCURRENCY, function(o, next_out) { + + /* + * TODO Support multisigs + */ + if (o.value && o.scriptPubKey && o.scriptPubKey.addresses && o.scriptPubKey.addresses[0]) { + Self.create({ + txid : txid, + value_sat : o.valueSat, + addr : o.scriptPubKey.addresses[0], // TODO: only address 0? + index : o.n, + ts : info.time, + }, next_out); + if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { + addrs.push(o.scriptPubKey.addresses[0]); + } + } + else { + console.log ('WARN in TX: %s could not parse OUTPUT %d', txid, o.n); + return next_out(); + } + }, + function (err) { + if (err) { + if (err.message.match(/E11000/)) { + is_new = false; + } + else { + console.log('ERR at TX %s: %s', txid, err); + return cb(err); + } + } + return cb(null, addrs, is_new); + }); + }); + }); +}; + + + + +TransactionItemSchema.statics.createFromArray = function(txs, next) { + if (!txs) return next(); + + var inserted_txs = []; + + async.forEachLimit(txs, CONCURRENCY, function(txid, cb, was_new) { + + TransactionItemSchema.explodeTransactionItems( txid, function(err, addrs) { + if (err) return next(err); + if (addrs) { + async.each(addrs, function(addr){ + sockets.broadcast_address_tx(addr, {'txid': txid}); + }); + } + if (was_new) { + inserted_txs.push(txid); + } + + return cb(); + }); + }, + function(err) { + return next(err, inserted_txs); + }); +}; + + module.exports = mongoose.model('TransactionItem', TransactionItemSchema); diff --git a/lib/PeerSync.js b/lib/PeerSync.js index d4c861c..996b8fd 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -62,7 +62,7 @@ function spec() { if (this.verbose) { console.log('[p2p_sync] Handle tx: ' + tx.hash); } - this.sync.storeTxs([tx.hash], null, function(err) { + this.sync.storeTxs([tx.hash], function(err) { if (err) { console.log('[PeerSync.js.71:err:]',err); //TODO console.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); diff --git a/lib/Sync.js b/lib/Sync.js index 3ff5554..19e08ca 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -91,12 +91,10 @@ function spec() { }; - Sync.prototype.storeTxs = function(txs, inTime, cb) { + Sync.prototype.storeTxs = function(txs, cb) { var self = this; - var time = inTime ? inTime : Math.round(new Date().getTime() / 1000); - - Transaction.createFromArray(txs, time, function(err, inserted_txs) { + Transaction.createFromArray(txs, function(err, inserted_txs) { if (!err && inserted_txs && self.opts.broadcast_txs) { inserted_txs.forEach(function(tx) { diff --git a/test/model/transaction.js b/test/model/transaction.js index a9e8adb..489182d 100644 --- a/test/model/transaction.js +++ b/test/model/transaction.js @@ -1,20 +1,17 @@ #!/usr/bin/env node +'use strict'; process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var - mongoose= require('mongoose'), +var mongoose= require('mongoose'), assert = require('assert'), config = require('../../config/config'), - Transaction = require('../../app/models/Transaction'), - TransactionItem = require('../../app/models/TransactionItem'), - fs = require('fs'), - util = require('util'); + Transaction = require('../../app/models/Transaction').class(), + TransactionItem = require('../../app/models/TransactionItem'); -var txItemsValid = JSON.parse(fs.readFileSync('test/model/txitems.json')); mongoose.connection.on('error', function(err) { console.log(err); }); describe('Transaction', function(){ @@ -122,43 +119,5 @@ describe('Transaction', function(){ }); }); - - txItemsValid.forEach( function(v) { - if (v.disabled) return; - it('test a exploding TX ' + v.txid, function(done) { - - // Remove first - TransactionItem.remove({txid: v.txid}, function(err) { - - var now = Math.round(new Date().getTime() / 1000); - Transaction.explodeTransactionItems(v.txid, now, function(err, tx) { - if (err) done(err); - - TransactionItem - .fromTxId( v.txid, function(err, readItems) { - - var unmatch={}; - - v.items.forEach(function(validItem){ - unmatch[validItem.addr] =1; - }); - v.items.forEach(function(validItem){ - var readItem = readItems.shift(); - assert.equal(readItem.addr,validItem.addr); - assert.equal(readItem.value_sat,validItem.value_sat); - assert.equal(readItem.index,validItem.index); - delete unmatch[validItem.addr]; - }); - - var valid = util.inspect(v.items, { depth: null }); - assert(!Object.keys(unmatch).length, - '\n\tUnmatchs:' + Object.keys(unmatch) + "\n\n" +valid + '\nvs.\n' + readItems); - done(); - }); - }); - }); - }); - }); - }); diff --git a/test/model/transactionitems.js b/test/model/transactionitems.js new file mode 100644 index 0000000..146476a --- /dev/null +++ b/test/model/transactionitems.js @@ -0,0 +1,69 @@ +#!/usr/bin/env node +'use strict'; + +process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + + + +var mongoose = require('mongoose'), + assert = require('assert'), + fs = require('fs'), + util = require('util'), + config = require('../../config/config'), + Transaction = require('../../app/models/Transaction').class(), + TransactionItem = require('../../app/models/TransactionItem'); + +var txItemsValid = JSON.parse(fs.readFileSync('test/model/txitems.json')); + + +mongoose.connection.on('error', function(err) { console.log(err); }); + +describe('TransactionItem', function(){ + + before(function(done) { + mongoose.connect(config.db); + done(); + }); + + after(function(done) { + mongoose.connection.close(); + done(); + }); + + txItemsValid.forEach( function(v) { + if (v.disabled) return; + it('test a exploding TX ' + v.txid, function(done) { + + // Remove first + TransactionItem.remove({txid: v.txid}, function(err) { + + TransactionItem.explodeTransactionItems(v.txid, function(err, tx) { + if (err) done(err); + + TransactionItem + .fromTxId( v.txid, function(err, readItems) { + + var unmatch={}; + + v.items.forEach(function(validItem){ + unmatch[validItem.addr] =1; + }); + v.items.forEach(function(validItem){ + var readItem = readItems.shift(); + assert.equal(readItem.addr,validItem.addr); + assert.equal(readItem.value_sat,validItem.value_sat); + assert.equal(readItem.index,validItem.index); + delete unmatch[validItem.addr]; + }); + + var valid = util.inspect(v.items, { depth: null }); + assert(!Object.keys(unmatch).length,'\n\tUnmatchs:' + Object.keys(unmatch) + "\n\n" +valid + '\nvs.\n' + readItems); + done(); + }); + }); + }); + }); + }); + +}); +