From 5ea85584c46e4dbb0247f10b45559118c4204e46 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 30 Jan 2014 11:50:05 -0300 Subject: [PATCH] git fixes --- app/models/Block.js | 21 +++- app/models/TransactionOut.js | 179 ++++++++++++++++-------------- config/env/development.js | 21 ++++ config/env/production.js | 22 ++++ config/env/test.js | 22 ++++ dev-util/explode_tx.js | 2 +- {util => dev-util}/status_info.js | 0 lib/HistoricSync.js | 22 +++- lib/Sync.js | 2 +- package.json | 1 + 10 files changed, 194 insertions(+), 98 deletions(-) create mode 100755 config/env/development.js create mode 100755 config/env/production.js create mode 100755 config/env/test.js rename {util => dev-util}/status_info.js (100%) diff --git a/app/models/Block.js b/app/models/Block.js index 030131d..9301669 100644 --- a/app/models/Block.js +++ b/app/models/Block.js @@ -7,6 +7,7 @@ var mongoose = require('mongoose'), Schema = mongoose.Schema, RpcClient = require('bitcore/RpcClient').class(), util = require('bitcore/util/util'), + async = require('async'), BitcoreBlock= require('bitcore/Block').class(), TransactionOut = require('./TransactionOut'), config = require('../../config/config') @@ -87,12 +88,22 @@ BlockSchema.statics.customCreate = function(block, cb) { newBlock.hashStr = block.hash; newBlock.nextBlockHashStr = block.nextBlockHash; - TransactionOut.createFromArray(block.tx, function(err, inserted_txs, update_addrs) { - if (err) return cb(err); + var insertedTxs, updateAddrs; - newBlock.save(function(err) { - return cb(err, newBlock, inserted_txs, update_addrs); - }); + async.series([ + function(a_cb) { + TransactionOut.createFromTxs(block.tx, function(err, inInsertedTxs, inUpdateAddrs) { + insertedTxs = inInsertedTxs; + updateAddrs = inUpdateAddrs; + return a_cb(err); + }); + }, function(a_cb) { + newBlock.save(function(err) { + return a_cb(err); + }); + }], + function (err) { + return cb(err, newBlock, insertedTxs, updateAddrs); }); }; diff --git a/app/models/TransactionOut.js b/app/models/TransactionOut.js index 4bf4c61..56548dc 100644 --- a/app/models/TransactionOut.js +++ b/app/models/TransactionOut.js @@ -91,97 +91,90 @@ TransactionOutSchema.statics.removeFromTxId = function(txid, cb) { -TransactionOutSchema.statics._explodeTransactionOuts = function(txid, cb) { +TransactionOutSchema.statics.storeTransactionOuts = function(txInfo, cb) { var Self = this; var addrs = []; var is_new = true; - // Is it from genesis block? (testnet==livenet) - // TODO: parse it from networks.genesisTX - if (txid === genesisTXID) return cb(); - TransactionRpc.getRpcInfo(txid, function(err, info) { + var bTxId = new Buffer(txInfo.txid,'hex'); - if (err || !info) return cb(err); + async.series([ + // Input Outpoints (mark them as spended) + function(p_c) { + if (txInfo.isCoinBase) return p_c(); + async.forEachLimit(txInfo.vin, CONCURRENCY, + function(i, next_out) { + var b = new Buffer(i.txid,'hex'); + var data = { + txidBuf: b, + index: i.vout, - var bTxId = new Buffer(txid,'hex'); - - async.series([ - // Input Outputs (mark them as spended) - function(p_c) { - if (info.isCoinBase) return p_c(); - async.forEachLimit(info.vin, CONCURRENCY, - function(i, next_out) { - var b = new Buffer(i.txid,'hex'); - var data = { - txidBuf: b, - index: i.vout, - - spendTxIdBuf: bTxId, - spendIndex: i.n, - }; - Self.update({txidBuf: b, index: i.vout}, data, {upsert: true}, next_out); - }, - function (err) { - if (err) { - if (!err.message.match(/E11000/)) { - console.log('ERR at TX %s: %s', txid, err); - return cb(err); - } + spendTxIdBuf: bTxId, + spendIndex: i.n, + }; + Self.update({txidBuf: b, index: i.vout}, data, {upsert: true}, next_out); + }, + function (err) { + if (err) { + if (!err.message.match(/E11000/)) { + console.log('ERR at TX %s: %s', txInfo.txid, err); + return cb(err); } - return p_c(); - }); - }, - // Parse Outputs - function(p_c) { - async.forEachLimit(info.vout, CONCURRENCY, - function(o, next_out) { - if (o.value && o.scriptPubKey && - o.scriptPubKey.addresses && - o.scriptPubKey.addresses[0] && - ! o.scriptPubKey.addresses[1] // TODO : not supported - ){ + } + return p_c(); + }); + }, + // Parse Outputs + function(p_c) { + async.forEachLimit(txInfo.vout, CONCURRENCY, + function(o, next_out) { + if (o.value && o.scriptPubKey && + o.scriptPubKey.addresses && + o.scriptPubKey.addresses[0] && + ! o.scriptPubKey.addresses[1] // TODO : not supported + ){ - // This is only to broadcast - if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { - addrs.push(o.scriptPubKey.addresses[0]); - } + // This is only to broadcast + if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { + addrs.push(o.scriptPubKey.addresses[0]); + } - var data = { - txidBuf: bTxId, - index : o.n, + var data = { + txidBuf: bTxId, + index : o.n, - value_sat : o.value * util.COIN, - addr : o.scriptPubKey.addresses[0], - }; - Self.update({txidBuf: bTxId, index: o.n}, data, {upsert: true}, next_out); + value_sat : o.value * util.COIN, + addr : o.scriptPubKey.addresses[0], + }; + Self.update({txidBuf: bTxId, index: o.n}, data, {upsert: true}, next_out); + } + else { + console.log ('WARN in TX: %s could not parse OUTPUT %d', txInfo.txid, o.n); + return next_out(); + } + }, + function (err) { + if (err) { + if (err.message.match(/E11000/)) { + is_new = false; } else { - console.log ('WARN in TX: %s could not parse OUTPUT %d', txid, o.n); - return next_out(); + console.log('ERR at TX %s: %s', txInfo.txid, err); + return cb(err); } - }, - function (err) { - if (err) { - if (err.message.match(/E11000/)) { - is_new = false; - } - else { - console.log('ERR at TX %s: %s', txid, err); - return cb(err); - } - } - return p_c(); - }); - }], function() { - return cb(null, addrs, is_new); + } + return p_c(); }); + }], function(err) { + return cb(err, addrs, is_new); }); }; -TransactionOutSchema.statics.createFromArray = function(txs, next) { +// txs can be a [hashes] or [txObjects] +TransactionOutSchema.statics.createFromTxs = function(txs, next) { var Self = this; if (!txs) return next(); @@ -191,24 +184,38 @@ TransactionOutSchema.statics.createFromArray = function(txs, next) { async.forEachLimit(txs, CONCURRENCY, function(txid, cb, was_new) { - Self._explodeTransactionOuts( txid, function(err, addrs) { - - if (err) return next(err); - - if (was_new) { - inserted_txs.push(txid); - addrs.each(function(a) { - if ( !updated_addrs[a]) updated_addrs[a] = []; - updated_addrs[a].push(txid); + var txInfo; + async.series([ + function(a_cb) { + // Is it from genesis block? (testnet==livenet) + // TODO: parse it from networks.genesisTX? + if (txid === genesisTXID) return a_cb(); + TransactionRpc.getRpcInfo(txid, function(err, inInfo) { + txInfo =inInfo; + return a_cb(err); }); - } + }, + function(a_cb) { + Self.storeTransactionOuts(txInfo, function(err, addrs) { + if (err) return a_cb(err); - return cb(); + if (was_new) { + inserted_txs.push(txid); + addrs.each(function(a) { + if ( !updated_addrs[a]) updated_addrs[a] = []; + updated_addrs[a].push(txid); + }); + } + return a_cb(); + }); + }], + function(err) { + return cb(err); + }); + }, + function(err) { + return next(err, inserted_txs, updated_addrs); }); - }, - function(err) { - return next(err, inserted_txs, updated_addrs); - }); }; diff --git a/config/env/development.js b/config/env/development.js new file mode 100755 index 0000000..2e41110 --- /dev/null +++ b/config/env/development.js @@ -0,0 +1,21 @@ +'use strict'; + +module.exports = { + db: 'mongodb://localhost/insight-dev', + app: { + name: 'Insight - Development' + }, + bitcoind: { + protocol: process.env.BITCOIND_PROTO || 'http', + user: process.env.BITCOIND_USER || 'user', + pass: process.env.BITCOIND_PASS || 'pass', + host: process.env.BITCOIND_HOST || '127.0.0.1', + port: process.env.BITCOIND_PORT || '18332', + p2pPort: process.env.BITCOIND_P2P_PORT || '18333', + disableAgent: true, + dataDir: process.env.BITCOIND_DATADIR || './testnet3', + }, + network: process.env.INSIGHT_NETWORK || 'testnet', + disableP2pSync: false, + disableHistoricSync: false, +}; diff --git a/config/env/production.js b/config/env/production.js new file mode 100755 index 0000000..c679713 --- /dev/null +++ b/config/env/production.js @@ -0,0 +1,22 @@ +'use strict'; + +module.exports = { + db: 'mongodb://localhost/insight-test', + app: { + name: 'Insight - Prod' + }, + port: '3301', + bitcoind: { + protocol: process.env.BITCOIND_PROTO || 'http', + user: process.env.BITCOIND_USER || 'user', + pass: process.env.BITCOIND_PASS || 'pass', + host: process.env.BITCOIND_HOST || '127.0.0.1', + port: process.env.BITCOIND_PORT || '18332', + p2pPort: process.env.BITCOIND_P2P_PORT || '18333', + disableAgent: true, + dataDir: process.env.BITCOIND_DATADIR || './testnet3', + }, + network: process.env.INSIGHT_NETWORK || 'testnet', + disableP2pSync: false, + disableHistoricSync: false, +}; diff --git a/config/env/test.js b/config/env/test.js new file mode 100755 index 0000000..3faf05b --- /dev/null +++ b/config/env/test.js @@ -0,0 +1,22 @@ +'use strict'; + +module.exports = { + db: 'mongodb://localhost/insight-test', + app: { + name: 'Insight - Test' + }, + port: '3301', + bitcoind: { + protocol: process.env.BITCOIND_PROTO || 'http', + user: process.env.BITCOIND_USER || 'user', + pass: process.env.BITCOIND_PASS || 'pass', + host: process.env.BITCOIND_HOST || '127.0.0.1', + port: process.env.BITCOIND_PORT || '18332', + p2pPort: process.env.BITCOIND_P2P_PORT || '18333', + disableAgent: true, + dataDir: process.env.BITCOIND_DATADIR || './testnet3', + }, + network: process.env.INSIGHT_NETWORK || 'testnet', + disableP2pSync: false, + disableHistoricSync: false, +}; diff --git a/dev-util/explode_tx.js b/dev-util/explode_tx.js index 814b33b..f9849c2 100755 --- a/dev-util/explode_tx.js +++ b/dev-util/explode_tx.js @@ -25,7 +25,7 @@ mongoose.connection.on('open', function() { var b = new Buffer(hash,'hex'); - T.createFromArray([hash], function(err, ret) { + T.createFromTxs([hash], function(err, ret) { console.log('Err:'); console.log(err); diff --git a/util/status_info.js b/dev-util/status_info.js similarity index 100% rename from util/status_info.js rename to dev-util/status_info.js diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 55065c7..0fcf69b 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -106,6 +106,11 @@ function spec() { if (self.syncPercentage > 100) self.syncPercentage = 100; p(util.format('status: [%d%%] skipped: %d', self.syncPercentage, self.skippedBlocks)); +//TODO +if (self.syncPercentage>5) { + process.exit(1); +} + // } if (self.opts.shouldBroadcast) { sockets.broadcastSyncInfo(self.info()); @@ -328,18 +333,25 @@ function spec() { if (err) return next(err); + + var scanOpts = {}; + if (!b) { p('Could not find Genesis block. Running FULL SYNC'); + if (config.bitcoind.dataDir) { + p('bitcoind dataDir configured...importing blocks from .dat files'); + scanOpts.fromFiles = true; + } + else { + scanOpts.reverse = true; + } } else { p('Genesis block found. Syncing upto known blocks.'); + scanOpts.reverse = true; + scanOpts.upToExisting = true; } - var scanOpts = { - reverse: true, - upToExisting: b ? true: false, - }; - return self.importHistory(scanOpts, next); }); }; diff --git a/lib/Sync.js b/lib/Sync.js index ea4794b..ab70e4d 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -106,7 +106,7 @@ function spec() { Sync.prototype.storeTxs = function(txs, cb) { var self = this; - TransactionOut.createFromArray(txs, function(err, inserted_txs, updated_addrs) { + TransactionOut.createFromTxs(txs, function(err, inserted_txs, updated_addrs) { if (err) return cb(err); self._handleBroadcast(null, inserted_txs, updated_addrs); diff --git a/package.json b/package.json index c0fa2e7..faf621f 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ }, "dependencies": { "async": "*", + "glob": "*", "classtool": "*", "commander": "*", "bignum": "*",