Merge branch 'master' of github.com:bitpay/mystery into test/test-p2p-sync
Conflicts: lib/Sync.js
This commit is contained in:
commit
0a94f2514b
|
@ -48,7 +48,7 @@ module.exports = function(grunt) {
|
|||
},
|
||||
jshint: {
|
||||
all: {
|
||||
src: ['Gruntfile.js', 'server.js', 'app/**/*.js', 'public/js/**'],
|
||||
src: ['Gruntfile.js', 'server.js', 'app/**/*.js', 'public/js/**','lib/*.js'],
|
||||
options: {
|
||||
jshintrc: true
|
||||
}
|
||||
|
@ -66,7 +66,7 @@ module.exports = function(grunt) {
|
|||
options: {
|
||||
file: 'server.js',
|
||||
args: [],
|
||||
ignoredFiles: ['public/**', 'test/**','util/**','lib/**'],
|
||||
ignoredFiles: ['public/**', 'test/**','util/**'],
|
||||
watchedExtensions: ['js'],
|
||||
// nodeArgs: ['--debug'],
|
||||
delayTime: 1,
|
||||
|
|
|
@ -141,7 +141,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) {
|
|||
}
|
||||
},
|
||||
function (err) {
|
||||
if (err) console.log (err);
|
||||
if (err && !err.message.match(/E11000/) ) console.log (err);
|
||||
async.forEachLimit(info.vout, CONCURRENCY, function(o, next_out) {
|
||||
|
||||
/*
|
||||
|
|
|
@ -0,0 +1,248 @@
|
|||
'use strict';
|
||||
|
||||
require('classtool');
|
||||
|
||||
|
||||
function spec() {
|
||||
var mongoose = require('mongoose');
|
||||
var util = require('util');
|
||||
var RpcClient = require('bitcore/RpcClient').class();
|
||||
var networks = require('bitcore/networks');
|
||||
var async = require('async');
|
||||
var config = require('../config/config');
|
||||
var Block = require('../app/models/Block');
|
||||
var Transaction = require('../app/models/Transaction');
|
||||
var TransactionItem = require('../app/models/TransactionItem');
|
||||
var Sync = require('./Sync').class();
|
||||
var sockets = require('../app/views/sockets/main.js');
|
||||
var CONCURRENCY = 5;
|
||||
|
||||
|
||||
function HistoricSync(opts) {
|
||||
this.block_count= 0;
|
||||
this.block_total= 0;
|
||||
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;
|
||||
this.sync = new Sync(opts);
|
||||
}
|
||||
|
||||
function p() {
|
||||
var params = Array.prototype.slice.call(arguments);
|
||||
|
||||
params.unshift('[historic_sync]');
|
||||
console.log.apply(this,params);
|
||||
}
|
||||
|
||||
var progress_bar = function(string, current, total) {
|
||||
p(util.format('%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total)));
|
||||
};
|
||||
|
||||
HistoricSync.prototype.init = function(opts,cb) {
|
||||
this.rpc = new RpcClient(config.bitcoind);
|
||||
this.opts = opts;
|
||||
this.sync.init(opts, cb);
|
||||
};
|
||||
|
||||
HistoricSync.prototype.close = function() {
|
||||
this.sync.close();
|
||||
};
|
||||
|
||||
HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
|
||||
|
||||
var that = this;
|
||||
|
||||
// recursion end.
|
||||
if (!blockHash || (blockEnd && blockEnd === blockHash) ) {
|
||||
return cb();
|
||||
}
|
||||
|
||||
var existed = 0;
|
||||
var blockInfo;
|
||||
var blockObj;
|
||||
|
||||
async.series([
|
||||
// Already got it?
|
||||
function(c) {
|
||||
Block.findOne({hash:blockHash}, function(err,block){
|
||||
if (err) { p(err); return c(err); };
|
||||
if (block) {
|
||||
existed = 1;
|
||||
blockObj = block;
|
||||
}
|
||||
|
||||
return c();
|
||||
});
|
||||
},
|
||||
//show some (inacurate) status
|
||||
function(c) {
|
||||
if (that.block_count++ % 1000 === 0) {
|
||||
progress_bar('sync status:', that.block_count, that.block_total);
|
||||
}
|
||||
return c();
|
||||
},
|
||||
//get Info from RPC
|
||||
function(c) {
|
||||
|
||||
// TODO: if we store prev/next, no need to go to RPC
|
||||
// if (blockObj && blockObj.nextBlockHash) return c();
|
||||
|
||||
that.rpc.getBlock(blockHash, function(err, ret) {
|
||||
if (err) return c(err);
|
||||
|
||||
blockInfo = ret;
|
||||
return c();
|
||||
});
|
||||
},
|
||||
//store it
|
||||
function(c) {
|
||||
if (existed) return c();
|
||||
|
||||
that.sync.storeBlock(blockInfo.result, function(err, block) {
|
||||
existed = err && err.toString().match(/E11000/);
|
||||
if (err && ! existed) return c(err);
|
||||
return c();
|
||||
});
|
||||
},
|
||||
/* TODO: Should Start to sync backwards? (this is for partial syncs)
|
||||
function(c) {
|
||||
|
||||
if (blockInfo.result.prevblockhash != current.blockHash) {
|
||||
p("reorg?");
|
||||
opts.prev = 1;
|
||||
}
|
||||
return c();
|
||||
}
|
||||
*/
|
||||
],
|
||||
function (err){
|
||||
|
||||
if (err)
|
||||
p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, that.block_count);
|
||||
|
||||
if (blockInfo && blockInfo.result) {
|
||||
if (opts.prev && blockInfo.result.previousblockhash) {
|
||||
return that.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
|
||||
}
|
||||
|
||||
if (opts.next && blockInfo.result.nextblockhash)
|
||||
return that.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
|
||||
}
|
||||
return cb(err);
|
||||
});
|
||||
};
|
||||
|
||||
HistoricSync.prototype.syncBlocks = function(start, end, isForward, cb) {
|
||||
var that = this;
|
||||
|
||||
p('Syncing Blocks, starting from: %s end: %s isForward:',
|
||||
start, end, isForward);
|
||||
|
||||
|
||||
return that.getPrevNextBlock( start, end,
|
||||
isForward ? { next: 1 } : { prev: 1}, cb);
|
||||
};
|
||||
|
||||
HistoricSync.prototype.do_import_history = function(opts, next) {
|
||||
var that = this;
|
||||
|
||||
var retry_attemps = 100;
|
||||
var retry_secs = 2;
|
||||
|
||||
var block_best;
|
||||
var block_height;
|
||||
|
||||
async.series([
|
||||
function(cb) {
|
||||
if (opts.destroy) {
|
||||
p('Deleting Blocks...');
|
||||
that.db.collections.blocks.drop(cb);
|
||||
} else {
|
||||
return cb();
|
||||
}
|
||||
},
|
||||
function(cb) {
|
||||
if (opts.destroy) {
|
||||
p('Deleting TXs...');
|
||||
that.db.collections.transactions.drop(cb);
|
||||
} else {
|
||||
return cb();
|
||||
}
|
||||
},
|
||||
function(cb) {
|
||||
if (opts.destroy) {
|
||||
p('Deleting TXItems...');
|
||||
that.db.collections.transactionitems.drop(cb);
|
||||
} else {
|
||||
return cb();
|
||||
}
|
||||
},
|
||||
function(cb) {
|
||||
that.rpc.getInfo(function(err, res) {
|
||||
if (err) cb(err);
|
||||
|
||||
that.block_total = res.result.blocks;
|
||||
return cb();
|
||||
});
|
||||
},
|
||||
// We are not using getBestBlockHash, because is not available in all clients
|
||||
function(cb) {
|
||||
if (!opts.reverse) return cb();
|
||||
|
||||
that.rpc.getBlockCount(function(err, res) {
|
||||
if (err) cb(err);
|
||||
block_height = res.result;
|
||||
return cb();
|
||||
});
|
||||
},
|
||||
function(cb) {
|
||||
if (!opts.reverse) return cb();
|
||||
|
||||
that.rpc.getBlockHash(block_height, function(err, res) {
|
||||
if (err) cb(err);
|
||||
|
||||
block_best = res.result;
|
||||
return cb();
|
||||
});
|
||||
},
|
||||
],
|
||||
function(err) {
|
||||
|
||||
function sync() {
|
||||
var start, end, isForward;
|
||||
|
||||
if (opts.reverse) {
|
||||
start = block_best;
|
||||
end = that.network.genesisBlock.hash.reverse().toString('hex');
|
||||
isForward = false;
|
||||
}
|
||||
else {
|
||||
start = that.network.genesisBlock.hash.reverse().toString('hex');
|
||||
end = null;
|
||||
isForward = true;
|
||||
}
|
||||
|
||||
that.syncBlocks(start, end, isForward, function(err) {
|
||||
|
||||
if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){
|
||||
setTimeout(function() {
|
||||
p("Retrying in %d secs ", retry_secs);
|
||||
sync();
|
||||
}, retry_secs * 1000);
|
||||
}
|
||||
else
|
||||
return next(err, that.block_count);
|
||||
});
|
||||
}
|
||||
sync();
|
||||
});
|
||||
}
|
||||
|
||||
HistoricSync.prototype.import_history = function(opts, next) {
|
||||
var that = this;
|
||||
that.do_import_history(opts, next);
|
||||
};
|
||||
|
||||
|
||||
return HistoricSync;
|
||||
}
|
||||
module.defineClass(spec);
|
||||
|
|
@ -11,23 +11,27 @@ function spec() {
|
|||
var peerdb_fn = 'peerdb.json';
|
||||
|
||||
function PeerSync() {}
|
||||
PeerSync.prototype.init = function(config) {
|
||||
|
||||
|
||||
PeerSync.prototype.init = function(config, cb) {
|
||||
|
||||
var that = this;
|
||||
|
||||
var network = config && (config.network || 'testnet');
|
||||
|
||||
this.peerdb = undefined;
|
||||
this.sync = new Sync({
|
||||
that.peerdb = undefined;
|
||||
that.sync = new Sync({
|
||||
networkName: network
|
||||
});
|
||||
this.sync.init(config);
|
||||
|
||||
this.PeerManager = require('bitcore/PeerManager').createClass({
|
||||
that.sync.init(config, function() {
|
||||
that.PeerManager = require('bitcore/PeerManager').createClass({
|
||||
config: {
|
||||
network: network
|
||||
}
|
||||
});
|
||||
this.load_peers();
|
||||
|
||||
that.load_peers();
|
||||
return cb();
|
||||
});
|
||||
};
|
||||
|
||||
PeerSync.prototype.load_peers = function() {
|
||||
|
@ -51,6 +55,7 @@ function spec() {
|
|||
console.log('[p2p_sync] Handle inv for a ' + CoinConst.MSG.to_str(inv.type));
|
||||
});
|
||||
// TODO: should limit the invs to objects we haven't seen yet
|
||||
console.log('getdata');
|
||||
info.conn.sendGetData(invs);
|
||||
};
|
||||
|
||||
|
|
|
@ -23,10 +23,13 @@
|
|||
<td width="45%">
|
||||
<ul class="list-unstyled" data-ng-repeat="vin in tx.vin" data-ng-show="!tx.isCoinBase">
|
||||
<li>
|
||||
<span data-ng-show="!vin.addr"><a href="/#!/tx/{{vin.txid}}">Address could not be parsed, {{vin.vout}}</a></span>
|
||||
<span data-ng-show="!vin.addr">Address could not be parsed</span>
|
||||
<a data-ng-show="vin.addr" href="/#!/address/{{vin.addr}}">{{vin.addr}}</a>
|
||||
<span class="pull-right badge">{{vin.value}} BTC</span>
|
||||
</li>
|
||||
|
||||
<li>
|
||||
Outpoint: <a href="/#!/tx/{{vin.txid}}">{{vin.txid}},{{vin.vout}}</a>
|
||||
</ul>
|
||||
<div data-ng-show="tx.isCoinBase">
|
||||
<ul class="list-unstyled" data-ng-repeat="vinn in tx.vin">
|
||||
|
|
16
server.js
16
server.js
|
@ -10,6 +10,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development';
|
|||
var express = require('express'),
|
||||
fs = require('fs'),
|
||||
PeerSync = require('./lib/PeerSync').class(),
|
||||
HistoricSync = require('./lib/HistoricSync').class(),
|
||||
mongoose = require('mongoose');
|
||||
|
||||
/**
|
||||
|
@ -40,14 +41,27 @@ var walk = function(path) {
|
|||
};
|
||||
walk(models_path);
|
||||
|
||||
// historic_sync process
|
||||
var hs = new HistoricSync();
|
||||
hs.init({
|
||||
skip_db_connection: true,
|
||||
networkName: config.network
|
||||
}, function() {
|
||||
hs.import_history({
|
||||
reverse: 1,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
// p2p_sync process
|
||||
var ps = new PeerSync();
|
||||
ps.init({
|
||||
skip_db_connection: true,
|
||||
broadcast_txs: true,
|
||||
broadcast_blocks: true
|
||||
}, function() {
|
||||
ps.run();
|
||||
});
|
||||
ps.run();
|
||||
|
||||
// express app
|
||||
var app = express();
|
||||
|
|
|
@ -5,7 +5,8 @@ var chai = require('chai'),
|
|||
|
||||
var PeerSync = require('../../lib/PeerSync.js').class();
|
||||
describe('PeerSync', function() {
|
||||
var ps, inv_info;
|
||||
var ps;
|
||||
|
||||
beforeEach(function() {
|
||||
ps = new PeerSync();
|
||||
ps.init();
|
||||
|
@ -13,6 +14,7 @@ describe('PeerSync', function() {
|
|||
afterEach(function(){
|
||||
ps.close();
|
||||
});
|
||||
|
||||
describe('#init()', function() {
|
||||
it('should return with no errors', function() {
|
||||
var other_ps = new PeerSync();
|
||||
|
@ -20,8 +22,9 @@ describe('PeerSync', function() {
|
|||
other_ps.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('#handle_inv()', function() {
|
||||
inv_info = {
|
||||
var inv_info = {
|
||||
message: {invs: []},
|
||||
conn: {sendGetData: sinon.spy()}
|
||||
};
|
||||
|
@ -32,16 +35,27 @@ describe('PeerSync', function() {
|
|||
});
|
||||
it('should call sendGetData', function() {
|
||||
ps.handle_inv(inv_info);
|
||||
expect(inv_info.conn.calledOnce);
|
||||
expect(inv_info.conn.sendGetData.calledOnce).to.be.ok;
|
||||
});
|
||||
});
|
||||
|
||||
describe('#handle_tx()', function() {
|
||||
it('should call storeTxs');
|
||||
var tx_info = {
|
||||
message: { tx: {getStandardizedObject: function(){
|
||||
return {hash: '00000000e3fe5b3b5416374d8d65560a0792a6da71546d67b00c9d37e8a4cf59'};}}}
|
||||
};
|
||||
it('should call storeTxs', function(){
|
||||
var spy = sinon.spy(ps.sync, 'storeTxs');
|
||||
ps.handle_tx(tx_info);
|
||||
expect(spy.calledOnce);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#handle_block()', function() {
|
||||
it('should call storeBlock');
|
||||
it('should call storeTxs for each transaction');
|
||||
});
|
||||
|
||||
describe('#run()', function() {
|
||||
it('should setup peerman');
|
||||
});
|
||||
|
|
21
util/sync.js
21
util/sync.js
|
@ -8,7 +8,7 @@ require('buffertools').extend();
|
|||
|
||||
var SYNC_VERSION = '0.1';
|
||||
var program = require('commander');
|
||||
var Sync = require('../lib/Sync').class();
|
||||
var HistoricSync = require('../lib/HistoricSync').class();
|
||||
var async = require('async');
|
||||
|
||||
program
|
||||
|
@ -18,7 +18,7 @@ program
|
|||
.option('-R --reverse', 'Sync backwards', 0)
|
||||
.parse(process.argv);
|
||||
|
||||
var sync = new Sync({
|
||||
var historicSync = new HistoricSync({
|
||||
networkName: program.network
|
||||
});
|
||||
|
||||
|
@ -27,12 +27,11 @@ if (program.remove) {
|
|||
}
|
||||
|
||||
async.series([
|
||||
function(cb) {
|
||||
sync.init(program);
|
||||
cb();
|
||||
},
|
||||
function(cb) {
|
||||
sync.import_history(program, function(err, count) {
|
||||
function(cb) {
|
||||
historicSync.init(program, cb);
|
||||
},
|
||||
function(cb) {
|
||||
historicSync.import_history(program, function(err, count) {
|
||||
if (err) {
|
||||
console.log('CRITICAL ERROR: ', err);
|
||||
}
|
||||
|
@ -41,9 +40,9 @@ function(cb) {
|
|||
}
|
||||
cb();
|
||||
});
|
||||
},
|
||||
function(cb) {
|
||||
sync.close();
|
||||
},
|
||||
function(cb) {
|
||||
historicSync.close();
|
||||
cb();
|
||||
}]);
|
||||
|
||||
|
|
Loading…
Reference in New Issue