Merge branch 'master' of github.com:bitpay/mystery into test/test-p2p-sync

This commit is contained in:
Manuel Araoz 2014-01-16 13:03:16 -03:00
commit 3ab958eab2
21 changed files with 518 additions and 47 deletions

51
app/controllers/status.js Normal file
View File

@ -0,0 +1,51 @@
'use strict';
/**
* Module dependencies.
*/
var Status = require('../models/Status');
var async = require('async');
/**
* Status
*/
exports.show = function(req, res) {
if (! req.query.q) {
res.status(400).send('Bad Request');
}
else {
var s = req.query.q;
var d = Status.new();
if (s == 'getInfo') {
d.getInfo(function(err) {
if (err) next(err);
res.jsonp(d);
});
}
else if (s == 'getDifficulty') {
d.getDifficulty(function(err) {
if (err) next(err);
res.jsonp(d);
});
}
else if (s == 'getTxOutSetInfo') {
d.getTxOutSetInfo(function(err) {
if (err) next(err);
res.jsonp(d);
});
}
else if (s == 'getBestBlockHash') {
d.getBestBlockHash(function(err) {
if (err) next(err);
res.jsonp(d);
});
}
else {
res.status(400).send('Bad Request');
}
}
};

View File

@ -26,6 +26,7 @@ var BlockSchema = new Schema({
unique: true,
},
time: Number,
nextBlockHash: String,
});
/**
@ -42,19 +43,23 @@ BlockSchema.path('title').validate(function(title) {
* Statics
*/
BlockSchema.statics.createTimestamped = function(block, cb) {
BlockSchema.statics.customCreate = function(block, cb) {
var That= this;
var now = Math.round(new Date().getTime() / 1000);
var BlockSchema = mongoose.model('Block', BlockSchema);
var newBlock = new That();
newBlock.time = now;
Transaction.createFromArray(block.tx, function(err, inserted_txs) {
var newBlock = new That();
newBlock.time = block.time ? block.time : Math.round(new Date().getTime() / 1000);
newBlock.hash = block.hash;
newBlock.nextBlockHash = block.nextBlockHash;
Transaction.createFromArray(block.tx, newBlock.time, function(err, inserted_txs) {
if (err) return cb(err);
newBlock.save(function(err) {
return cb(err, inserted_txs);
return cb(err, newBlock, inserted_txs);
});
});
};

86
app/models/Status.js Normal file
View File

@ -0,0 +1,86 @@
'use strict';
require('classtool');
function spec() {
var async = require('async');
var RpcClient = require('bitcore/RpcClient').class();
var config = require('../../config/config');
var rpc = new RpcClient(config.bitcoind);
function Status() {
this.info = {};
this.difficulty = {};
this.txoutsetinfo = {};
this.bestblockhash = {};
}
Status.prototype.getInfo = function(next) {
var that = this;
async.series([
function (cb) {
rpc.getInfo(function(err, info){
if (err) return cb(err);
that.info = info.result;
return cb();
});
}
], function (err) {
return next(err);
});
};
Status.prototype.getDifficulty = function(next) {
var that = this;
async.series([
function (cb) {
rpc.getDifficulty(function(err, df){
if (err) return cb(err);
that.difficulty = df.result;
return cb();
});
}
], function (err) {
return next(err);
});
};
Status.prototype.getTxOutSetInfo = function(next) {
var that = this;
async.series([
function (cb) {
rpc.getTxOutSetInfo(function(err, txout){
if (err) return cb(err);
that.txoutsetinfo = txout.result;
return cb();
});
}
], function (err) {
return next(err);
});
};
Status.prototype.getBestBlockHash = function(next) {
var that = this;
async.series([
function (cb) {
rpc.getBestBlockHash(function(err, bbh){
if (err) return cb(err);
that.bestblockhash = bbh.result;
return cb();
});
}
], function (err) {
return next(err);
});
};
return Status;
}
module.defineClass(spec);

View File

@ -87,18 +87,17 @@ TransactionSchema.statics.fromIdWithInfo = function(txid, cb) {
};
TransactionSchema.statics.createFromArray = function(txs, next) {
TransactionSchema.statics.createFromArray = function(txs, time, next) {
var that = this;
if (!txs) return next();
var mongo_txs = [];
var now = Math.round(new Date().getTime() / 1000);
async.forEachLimit(txs, CONCURRENCY, function(txid, cb) {
that.explodeTransactionItems( txid, function(err) {
that.explodeTransactionItems( txid, time, function(err) {
if (err) return next(err);
that.create({txid: txid, time: now}, function(err, new_tx) {
that.create({txid: txid, time: time}, function(err, new_tx) {
if (err && ! err.toString().match(/E11000/)) return cb(err);
if (new_tx) mongo_txs.push(new_tx);
@ -112,7 +111,7 @@ TransactionSchema.statics.createFromArray = function(txs, next) {
};
TransactionSchema.statics.explodeTransactionItems = function(txid, cb) {
TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) {
this.queryInfo(txid, function(err, info) {
if (err || !info) return cb(err);
@ -131,7 +130,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, cb) {
value_sat : -1 * i.valueSat,
addr : i.addr,
index : i.n,
ts : info.time,
ts : time,
}, next_in);
}
else {
@ -155,7 +154,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, cb) {
value_sat : o.valueSat,
addr : o.scriptPubKey.addresses[0],
index : o.n,
ts : info.time,
ts : time,
}, next_out);
}
else {

View File

@ -27,6 +27,7 @@ script(type='text/javascript', src='/js/directives.js')
script(type='text/javascript', src='/js/filters.js')
//Application Services
script(type='text/javascript', src='/js/services/status.js')
script(type='text/javascript', src='/js/services/address.js')
script(type='text/javascript', src='/js/services/transactions.js')
script(type='text/javascript', src='/js/services/blocks.js')
@ -40,4 +41,5 @@ script(type='text/javascript', src='/js/controllers/blocks.js')
script(type='text/javascript', src='/js/controllers/transactions.js')
script(type='text/javascript', src='/js/controllers/address.js')
script(type='text/javascript', src='/js/controllers/search.js')
script(type='text/javascript', src='/js/controllers/status.js')
script(type='text/javascript', src='/js/init.js')

View File

@ -11,6 +11,7 @@ module.exports = {
pass: process.env.BITCOIND_PASS || 'real_mystery',
host: process.env.BITCOIND_HOST || '127.0.0.1',
port: process.env.BITCOIND_PORT || '18332',
disableAgent: true,
},
network: 'testnet',
}

2
config/env/test.js vendored
View File

@ -12,7 +12,7 @@ module.exports = {
pass: process.env.BITCOIND_PASS || 'real_mystery',
host: process.env.BITCOIND_HOST || '127.0.0.1',
port: process.env.BITCOIND_PORT || '18332',
keepConnectionAlive: false,
disableAgent: true,
},
network: 'testnet',
}

View File

@ -25,4 +25,8 @@ module.exports = function(app) {
app.get('/api/addr/:addr', addresses.show);
app.param('addr', addresses.address);
// Status route
var st = require('../app/controllers/status');
app.get('/api/status', st.show);
};

View File

@ -57,7 +57,7 @@ function spec() {
PeerSync.prototype.handle_tx = function(info) {
var tx = info.message.tx.getStandardizedObject();
console.log('[p2p_sync] Handle tx: ' + tx.hash);
this.sync.storeTxs([tx.hash], function(err) {
this.sync.storeTxs([tx.hash], null, function(err) {
if (err) {
console.log('[p2p_sync] Error in handle TX: ' + err);
}
@ -78,6 +78,7 @@ function spec() {
this.sync.storeBlock({
'hash': blockHash,
'tx': tx_hashes,
// TODO NEXT BLOCK / PREV BLOCK?
},
function(err) {
if (err) {

View File

@ -29,10 +29,12 @@ function spec() {
};
Sync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
var that = this;
// recursion end.
if (!blockHash || (blockEnd && blockEnd == blockHash) ) {
console.log("Reach end:", blockHash, blockEnd);
return cb();
}
@ -100,8 +102,9 @@ function spec() {
console.log("ERROR: @%s: %s [count: block_count: %d]", blockHash, err, that.block_count);
if (blockInfo && blockInfo.result) {
if (opts.prev && blockInfo.result.prevblockhash)
return that.getPrevNextBlock(blockInfo.result.prevblockhash, blockEnd, opts, cb);
if (opts.prev && blockInfo.result.previousblockhash) {
return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
}
if (opts.next && blockInfo.result.nextblockhash)
return that.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
@ -113,30 +116,53 @@ function spec() {
Sync.prototype.storeBlock = function(block, cb) {
var that = this;
Block.createTimestamped(block, function(err, b){
Block.customCreate(block, function(err, block, inserted_txs){
if (b && that.opts.broadcast_blocks) {
sockets.broadcast_block(b);
if (block && that.opts.broadcast_blocks) {
sockets.broadcast_block(block);
}
if (that.opts.broadcast_txs) {
block.tx.each(function(tx) {
sockets.broadcast_tx(new_tx);
if (inserted_txs && that.opts.broadcast_txs) {
inserted_txs.forEach(function(tx) {
sockets.broadcast_tx(tx);
});
}
that.tx_count += block.tx.length;
if (inserted_txs)
that.tx_count += inserted_txs.length;
return cb();
});
};
Sync.prototype.syncBlocks = function(start, end, cb) {
Sync.prototype.storeTxs = function(txs, inTime, cb) {
var that = this;
console.log('Syncing Blocks, starting from: %s end: %s ',start, end);
var time = inTime ? inTime : Math.round(new Date().getTime() / 1000);
return that.getPrevNextBlock(start, end, { next: 1 }, cb);
Transaction.createFromArray(txs, time, function(err, inserted_txs) {
if (!err && inserted_txs && that.opts.broadcast_txs) {
inserted_txs.forEach(function(tx) {
sockets.broadcast_tx(tx);
});
}
return cb(err);
});
};
Sync.prototype.syncBlocks = function(start, end, isForward, cb) {
var that = this;
console.log('Syncing Blocks, starting \n\tfrom: %s \n\tend: %s \n\tisForward:',
start, end, isForward);
return that.getPrevNextBlock( start, end,
isForward ? { next: 1 } : { prev: 1}, cb);
};
// This is not currently used. Transactions are represented by txid only
@ -227,7 +253,7 @@ function spec() {
}
// This will trigger an RPC call
Transaction.explodeTransactionItems( tx.txid, function(err) {
Transaction.explodeTransactionItems( tx.txid, tx.time, function(err) {
if (proc++ % 1000 === 0) progress_bar('\tproc', pull, total);
next(err);
});
@ -266,6 +292,8 @@ function spec() {
var retry_attemps = 100;
var retry_secs = 2;
var block_best;
this.db.once('open', function() {
async.series([
function(cb) {
@ -273,7 +301,7 @@ function spec() {
console.log('Deleting Blocks...');
that.db.collections.blocks.drop(cb);
} else {
cb();
return cb();
}
},
function(cb) {
@ -281,7 +309,7 @@ function spec() {
console.log('Deleting TXs...');
that.db.collections.transactions.drop(cb);
} else {
cb();
return cb();
}
},
function(cb) {
@ -289,7 +317,7 @@ function spec() {
console.log('Deleting TXItems...');
that.db.collections.transactionitems.drop(cb);
} else {
cb();
return cb();
}
},
function(cb) {
@ -301,11 +329,34 @@ function spec() {
});
},
function(cb) {
if (!opts.reverse) return cb();
that.rpc.getBestBlockHash(function(err, res) {
if (err) cb(err);
block_best = res.result;
return cb();
});
},
], function(err) {
function sync() {
var startingBlockHash = that.network.genesisBlock.hash.reverse().toString('hex');
var start, end, isForward;
that.syncBlocks( startingBlockHash, null, function(err) {
if (opts.reverse) {
start = block_best;
end = that.network.genesisBlock.hash.reverse().toString('hex');
isForward = false;
}
else {
start = that.network.genesisBlock.hash.reverse().toString('hex');
end = null;
isForward = true;
}
that.syncBlocks(start, end, isForward, function(err) {
if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){
setTimeout(function() {
@ -321,9 +372,6 @@ function spec() {
if (!opts.skip_blocks) {
sync();
}
},
], function(err) {
return next(err, that.block_count);
});
});
};

View File

@ -13,7 +13,8 @@ var app = angular.module('mystery',
'mystery.transactions',
'monospaced.qrcode',
'mystery.address',
'mystery.search'
'mystery.search',
'mystery.status'
]);
angular.module('mystery.system', []);
@ -22,3 +23,4 @@ angular.module('mystery.blocks', []);
angular.module('mystery.transactions', []);
angular.module('mystery.address', []);
angular.module('mystery.search', []);
angular.module('mystery.status', []);

View File

@ -22,6 +22,9 @@ angular.module('mystery').config(['$routeProvider',
when('/address/:addrStr', {
templateUrl: 'views/address.html'
}).
when('/status', {
templateUrl: 'views/status.html'
}).
otherwise({
redirectTo: '/'
});

View File

@ -3,10 +3,16 @@
angular.module('mystery.system').controller('HeaderController', ['$scope', 'Global', function ($scope, Global) {
$scope.global = Global;
$scope.menu = [{
'title': 'Blocks',
'link': 'blocks'
}];
$scope.menu = [
{
'title': 'Blocks',
'link': 'blocks'
},
{
'title': 'Status',
'link': 'status'
}
];
$scope.isCollapsed = false;
}]);

View File

@ -0,0 +1,26 @@
'use strict';
angular.module('mystery.status').controller('StatusController', ['$scope', '$routeParams', '$location', 'Global', 'Status', function ($scope, $routeParams, $location, Global, Status) {
$scope.global = Global;
$scope.getData = function(q) {
Status.get({
q: q
}, function(d) {
if (q == 'getInfo') {
$scope.info = d.info;
}
if (q == 'getDifficulty') {
$scope.difficulty = d.difficulty;
}
if (q == 'getTxOutSetInfo') {
$scope.txoutsetinfo = d.txoutsetinfo;
}
if (q == 'getBestBlockHash') {
$scope.bestblockhash = d.bestblockhash;
}
});
};
}]);

View File

@ -0,0 +1,8 @@
'use strict';
angular.module('mystery.status').factory('Status', ['$resource', function($resource) {
return $resource('/api/status', {
q: '@q'
});
}]);

144
public/views/status.html Normal file
View File

@ -0,0 +1,144 @@
<section data-ng-controller="StatusController">
<div class="page-header">
<h1>
Status
</h1>
</div>
<div class="row">
<div class="col-lg-6">
<h3>getInfo</h3>
<table class="table table-striped" data-ng-init="getData('getInfo')">
<tbody>
<tr data-ng-show="!info">
<td colspan="2" class="text-center">Loading...</td>
</tr>
<tr>
<td>Version</td>
<td>{{info.version}}</td>
</tr>
<tr>
<td>protocolversion</td>
<td>{{info.protocolversion}}</td>
</tr>
<tr>
<td>walletversion</td>
<td>{{info.walletversion}}</td>
</tr>
<tr>
<td>balance</td>
<td>{{info.balance}}</td>
</tr>
<tr>
<td>blocks</td>
<td>{{info.blocks}}</td>
</tr>
<tr>
<td>timeoffset</td>
<td>{{info.timeoffset}}</td>
</tr>
<tr>
<td>connections</td>
<td>{{info.connections}}</td>
</tr>
<tr>
<td>proxy</td>
<td>{{info.proxy}}</td>
</tr>
<tr>
<td>difficulty</td>
<td>{{info.difficulty}}</td>
</tr>
<tr>
<td>testnet</td>
<td>{{info.testnet}}</td>
</tr>
<tr>
<td>keypoololdest</td>
<td>{{info.keypoololdest}}</td>
</tr>
<tr>
<td>keypoolsize</td>
<td>{{info.keypoolsize}}</td>
</tr>
<tr>
<td>paytxfee</td>
<td>{{info.paytxfee}}</td>
</tr>
<tr>
<td>errors</td>
<td>{{info.errors}}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-lg-6">
<h3>getTxOutSetInfo</h3>
<table class="table table-striped" data-ng-init="getData('getTxOutSetInfo')">
<tbody>
<tr data-ng-show="!txoutsetinfo">
<td colspan="2" class="text-center">Loading...</td>
</tr>
<tr>
<td>Height</td>
<td>{{txoutsetinfo.height}}</td>
</tr>
<tr>
<td>bestblock</td>
<td>{{txoutsetinfo.bestblock}}</td>
</tr>
<tr>
<td>transactions</td>
<td>{{txoutsetinfo.transactions}}</td>
</tr>
<tr>
<td>txouts</td>
<td>{{txoutsetinfo.txouts}}</td>
</tr>
<tr>
<td>bytes_serialized</td>
<td>{{txoutsetinfo.bytes_serialized}}</td>
</tr>
<tr>
<td>hash_serialized</td>
<td>{{txoutsetinfo.hash_serialized}}</td>
</tr>
<tr>
<td>total_amount</td>
<td>{{txoutsetinfo.total_amount}}</td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="row">
<div class="col-lg-6">
<h3>getDifficulty</h3>
<table class="table table-striped" data-ng-init="getData('getDifficulty')">
<tbody>
<tr data-ng-show="!difficulty">
<td colspan="2" class="text-center">Loading...</td>
</tr>
<tr>
<td>Difficulty</td>
<td>{{difficulty}}</td>
</tr>
</tbody>
</table>
</div>
<div class="col-lg-6">
<h3>getBestBlockHash</h3>
<table class="table table-striped" data-ng-init="getData('getBestBlockHash')">
<tbody>
<tr data-ng-show="!bestblockhash">
<td colspan="1" class="text-center">Loading...</td>
</tr>
<tr>
<td>{{bestblockhash}}</td>
</tr>
</tbody>
</table>
</div>
</div>
</section>

65
test/model/status.js Normal file
View File

@ -0,0 +1,65 @@
#!/usr/bin/env node
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var
assert = require('assert'),
config = require('../../config/config'),
Status = require('../../app/models/Status').class(),
mongoose= require('mongoose');
describe('Status', function(){
before(function(done) {
mongoose.connect(config.db);
done();
});
after(function(done) {
mongoose.connection.close();
done();
});
it('getInfo', function(done) {
var d = new Status();
d.getInfo(function(err) {
if (err) done(err);
assert.equal('number', typeof d.info.difficulty);
done();
});
});
it('getDifficulty', function(done) {
var d = new Status();
d.getDifficulty(function(err) {
if (err) done(err);
assert.equal('number', typeof d.difficulty);
done();
});
});
it('getTxOutSetInfo', function(done) {
var d = new Status();
d.getTxOutSetInfo(function(err) {
if (err) done(err);
assert.equal('number', typeof d.txoutsetinfo.txouts);
done();
});
});
it('getBestBlockHash', function(done) {
var d = new Status();
d.getBestBlockHash(function(err) {
if (err) done(err);
assert.equal('string', typeof d.bestblockhash);
done();
});
});
});

View File

@ -118,7 +118,8 @@ describe('Transaction', function(){
// Remove first
TransactionItem.remove({txid: v.txid}, function(err) {
Transaction.explodeTransactionItems(v.txid, function(err, tx) {
var now = Math.round(new Date().getTime() / 1000);
Transaction.explodeTransactionItems(v.txid, now, function(err, tx) {
if (err) done(err);
TransactionItem

View File

@ -11,7 +11,7 @@ var block_hash = process.argv[2] || '0000000000b6288775bbd326bedf324ca8717a15191
var rpc = new RpcClient(config.bitcoind);
var block = rpc.getBlock(block_hash, function(err, block) {
var block = rpc.getBestBlockHash( function(err, block) {
console.log("Err:");
console.log(err);

20
util/status_info.js Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env node
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var RpcClient = require('../node_modules/bitcore/RpcClient').class();
var config = require('../config/config');
var rpc = new RpcClient(config.bitcoind);
var block = rpc.getInfo(function(err, block) {
if (err) {
console.log("Err:");
console.log(err);
}
console.log("Block info:");
console.log(block);
});

View File

@ -14,9 +14,8 @@ var async = require('async');
program
.version(SYNC_VERSION)
.option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet')
.option('-D --destroy', 'Remove current DB (and start from there)', '0')
.option('--skip_blocks', 'Sync blocks')
.option('--skip_txs', 'Sync transactions')
.option('-D --destroy', 'Remove current DB (and start from there)', 0)
.option('-R --reverse', 'Sync backwards', 0)
.parse(process.argv);
var sync = new Sync({
@ -38,7 +37,7 @@ function(cb) {
console.log('CRITICAL ERROR: ', err);
}
else {
console.log('Done! [%d blocks]', count);
console.log('Done! [%d blocks]', count, err);
}
cb();
});