stream sync status
This commit is contained in:
Manuel Araoz
parent
21f03749c8
commit
9c1906641a
|
|
@ -27,6 +27,5 @@ module.exports.broadcast_address_tx = function(address, tx) {
|
|||
};
|
||||
|
||||
module.exports.broadcastSyncInfo = function(syncInfo) {
|
||||
console.log('broadcastSyncInfo');
|
||||
ios.sockets.in('sync').emit('status', syncInfo);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,43 +2,39 @@
|
|||
|
||||
require('classtool');
|
||||
|
||||
|
||||
function spec() {
|
||||
var util = require('util');
|
||||
var RpcClient = require('bitcore/RpcClient').class();
|
||||
var networks = require('bitcore/networks');
|
||||
var async = require('async');
|
||||
var config = require('../config/config');
|
||||
var Block = require('../app/models/Block');
|
||||
var Sync = require('./Sync').class();
|
||||
var sockets = require('../app/controllers/socket.js');
|
||||
var util = require('util');
|
||||
var RpcClient = require('bitcore/RpcClient').class();
|
||||
var networks = require('bitcore/networks');
|
||||
var async = require('async');
|
||||
var config = require('../config/config');
|
||||
var Block = require('../app/models/Block');
|
||||
var Sync = require('./Sync').class();
|
||||
var sockets = require('../app/controllers/socket.js');
|
||||
|
||||
function HistoricSync(opts) {
|
||||
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;
|
||||
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;
|
||||
|
||||
var genesisHashReversed = new Buffer(32);
|
||||
this.network.genesisBlock.hash.copy(genesisHashReversed);
|
||||
this.genesis = genesisHashReversed.reverse().toString('hex');
|
||||
this.sync = new Sync(opts);
|
||||
|
||||
this.sync = new Sync(opts);
|
||||
|
||||
//available status: new / syncing / finished / aborted
|
||||
this.status = 'new';
|
||||
this.status = 'new';
|
||||
this.syncInfo = {};
|
||||
}
|
||||
|
||||
function p() {
|
||||
var args = [];
|
||||
Array.prototype.push.apply( args, arguments );
|
||||
Array.prototype.push.apply(args, arguments);
|
||||
|
||||
|
||||
|
||||
args.unshift('[historic_sync]');
|
||||
/*jshint validthis:true */
|
||||
console.log.apply(this, args);
|
||||
}
|
||||
|
||||
HistoricSync.prototype.init = function(opts,cb) {
|
||||
HistoricSync.prototype.init = function(opts, cb) {
|
||||
this.rpc = new RpcClient(config.bitcoind);
|
||||
this.opts = opts;
|
||||
this.sync.init(opts, cb);
|
||||
|
|
@ -48,76 +44,84 @@ function spec() {
|
|||
this.sync.close();
|
||||
};
|
||||
|
||||
|
||||
HistoricSync.prototype.showProgress = function() {
|
||||
var self = this;
|
||||
|
||||
var i = self.syncInfo;
|
||||
var per = parseInt(100 * i.syncedBlocks / i.blocksToSync);
|
||||
p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per));
|
||||
if (self.opts.broadcast) {
|
||||
if (self.opts.shouldBroadcast) {
|
||||
sockets.broadcastSyncInfo(self.syncInfo);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
|
||||
var self = this;
|
||||
|
||||
// recursion end.
|
||||
if (!blockHash ) return cb();
|
||||
if (!blockHash) return cb();
|
||||
|
||||
var existed = 0;
|
||||
var existed = false;
|
||||
var blockInfo;
|
||||
var blockObj;
|
||||
|
||||
async.series([
|
||||
// Already got it?
|
||||
function(c) {
|
||||
Block.findOne({hash:blockHash}, function(err,block){
|
||||
if (err) { p(err); return c(err); }
|
||||
if (block) {
|
||||
existed =1;
|
||||
blockObj =block;
|
||||
}
|
||||
return c();
|
||||
});
|
||||
// Already got it?
|
||||
function(c) {
|
||||
Block.findOne({
|
||||
hash: blockHash
|
||||
},
|
||||
//show some (inacurate) status
|
||||
function(c) {
|
||||
var step = parseInt(self.syncInfo.blocksToSync / 100);
|
||||
if (step < 10) step = 10;
|
||||
|
||||
if (self.syncInfo.syncedBlocks % step === 1) {
|
||||
self.showProgress();
|
||||
function(err, block) {
|
||||
if (err) {
|
||||
p(err);
|
||||
return c(err);
|
||||
}
|
||||
if (block) {
|
||||
existed = true;
|
||||
blockObj = block;
|
||||
}
|
||||
return c();
|
||||
},
|
||||
//get Info from RPC
|
||||
function(c) {
|
||||
});
|
||||
},
|
||||
//show some (inacurate) status
|
||||
function(c) {
|
||||
if (!self.step) {
|
||||
var step = parseInt(self.syncInfo.blocksToSync / 100);
|
||||
if (self.opts.progressStep) {
|
||||
step = self.opts.progressStep;
|
||||
}
|
||||
if (step < 2) step = 2;
|
||||
self.step = step;
|
||||
}
|
||||
if (self.syncInfo.syncedBlocks % self.step === 1) {
|
||||
self.showProgress();
|
||||
}
|
||||
return c();
|
||||
},
|
||||
//get Info from RPC
|
||||
function(c) {
|
||||
|
||||
// TODO: if we store prev/next, no need to go to RPC
|
||||
// if (blockObj && blockObj.nextBlockHash) return c();
|
||||
// TODO: if we store prev/next, no need to go to RPC
|
||||
// if (blockObj && blockObj.nextBlockHash) return c();
|
||||
self.rpc.getBlock(blockHash, function(err, ret) {
|
||||
if (err) return c(err);
|
||||
|
||||
self.rpc.getBlock(blockHash, function(err, ret) {
|
||||
if (err) return c(err);
|
||||
blockInfo = ret;
|
||||
return c();
|
||||
});
|
||||
},
|
||||
//store it
|
||||
function(c) {
|
||||
if (existed) return c();
|
||||
self.sync.storeBlock(blockInfo.result, function(err) {
|
||||
|
||||
blockInfo = ret;
|
||||
return c();
|
||||
});
|
||||
},
|
||||
//store it
|
||||
function(c) {
|
||||
if (existed) return c();
|
||||
self.sync.storeBlock(blockInfo.result, function(err) {
|
||||
existed = err && err.toString().match(/E11000/);
|
||||
|
||||
existed = err && err.toString().match(/E11000/);
|
||||
|
||||
if (err && ! existed) return c(err);
|
||||
return c();
|
||||
});
|
||||
},
|
||||
/* TODO: Should Start to sync backwards? (this is for partial syncs)
|
||||
if (err && ! existed) return c(err);
|
||||
return c();
|
||||
});
|
||||
},
|
||||
/* TODO: Should Start to sync backwards? (this is for partial syncs)
|
||||
function(c) {
|
||||
|
||||
if (blockInfo.result.prevblockhash != current.blockHash) {
|
||||
|
|
@ -127,164 +131,164 @@ function spec() {
|
|||
return c();
|
||||
}
|
||||
*/
|
||||
],
|
||||
function (err){
|
||||
], function(err) {
|
||||
|
||||
if (err) {
|
||||
self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks);
|
||||
self.status = 'aborted';
|
||||
p(self.err);
|
||||
}
|
||||
if (err) {
|
||||
self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks);
|
||||
self.status = 'aborted';
|
||||
p(self.err);
|
||||
}
|
||||
|
||||
else {
|
||||
self.err = null;
|
||||
self.status = 'syncing';
|
||||
}
|
||||
else {
|
||||
self.err = null;
|
||||
self.status = 'syncing';
|
||||
}
|
||||
|
||||
if (opts.upToExisting && existed ) {
|
||||
var diff = self.syncInfo.blocksToSync - self.syncInfo.syncedBlocks;
|
||||
if (diff <= 0) {
|
||||
self.status = 'finished';
|
||||
p('DONE. Found existing block: ', blockHash);
|
||||
return cb(err);
|
||||
}
|
||||
else {
|
||||
self.syncInfo.skipped_blocks = self.syncInfo.skipped_blocks || 1;
|
||||
if ((self.syncInfo.skipped_blocks++ % 1000) === 1 ) {
|
||||
p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks, self.syncInfo.blocksToSync, self.syncInfo.skipped_blocks);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (blockEnd && blockEnd === blockHash) {
|
||||
if (opts.upToExisting && existed) {
|
||||
var diff = self.syncInfo.blocksToSync - self.syncInfo.syncedBlocks;
|
||||
if (diff <= 0) {
|
||||
self.status = 'finished';
|
||||
p('DONE. Found END block: ', blockHash);
|
||||
p('DONE. Found existing block: ', blockHash);
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
|
||||
// Continue
|
||||
if (blockInfo && blockInfo.result) {
|
||||
if (! existed) self.syncInfo.syncedBlocks++;
|
||||
if (opts.prev && blockInfo.result.previousblockhash) {
|
||||
return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
|
||||
else {
|
||||
self.syncInfo.skipped_blocks = self.syncInfo.skipped_blocks || 1;
|
||||
if ((self.syncInfo.skipped_blocks++ % 1000) === 1) {
|
||||
p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks, self.syncInfo.blocksToSync, self.syncInfo.skipped_blocks);
|
||||
}
|
||||
|
||||
if (opts.next && blockInfo.result.nextblockhash)
|
||||
return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
|
||||
}
|
||||
}
|
||||
|
||||
if (blockEnd && blockEnd === blockHash) {
|
||||
self.status = 'finished';
|
||||
p('DONE. Found END block: ', blockHash);
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
// Continue
|
||||
if (blockInfo && blockInfo.result) {
|
||||
if (!existed) self.syncInfo.syncedBlocks++;
|
||||
if (opts.prev && blockInfo.result.previousblockhash) {
|
||||
return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
|
||||
}
|
||||
|
||||
if (opts.next && blockInfo.result.nextblockhash) return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
|
||||
}
|
||||
return cb(err);
|
||||
});
|
||||
};
|
||||
|
||||
HistoricSync.prototype.import_history = function(opts, next) {
|
||||
var self = this;
|
||||
|
||||
var retry_secs = 2;
|
||||
var retry_secs = 2;
|
||||
|
||||
var bestBlock;
|
||||
var blockChainHeight;
|
||||
|
||||
async.series([
|
||||
function(cb) {
|
||||
if (opts.destroy) {
|
||||
p('Deleting DB...');
|
||||
return self.sync.destroy(cb);
|
||||
}
|
||||
function(cb) {
|
||||
if (opts.destroy) {
|
||||
p('Deleting DB...');
|
||||
return self.sync.destroy(cb);
|
||||
}
|
||||
return cb();
|
||||
},
|
||||
// We are not using getBestBlockHash, because is not available in all clients
|
||||
function(cb) {
|
||||
if (!opts.reverse) return cb();
|
||||
|
||||
self.rpc.getBlockCount(function(err, res) {
|
||||
if (err) return cb(err);
|
||||
blockChainHeight = res.result;
|
||||
return cb();
|
||||
},
|
||||
// We are not using getBestBlockHash, because is not available in all clients
|
||||
function(cb) {
|
||||
if (!opts.reverse) return cb();
|
||||
});
|
||||
},
|
||||
function(cb) {
|
||||
if (!opts.reverse) return cb();
|
||||
|
||||
self.rpc.getBlockCount(function(err, res) {
|
||||
self.rpc.getBlockHash(blockChainHeight, function(err, res) {
|
||||
if (err) return cb(err);
|
||||
|
||||
bestBlock = res.result;
|
||||
|
||||
return cb();
|
||||
});
|
||||
},
|
||||
function(cb) {
|
||||
// This is only to inform progress.
|
||||
if (!opts.upToExisting) {
|
||||
self.rpc.getInfo(function(err, res) {
|
||||
if (err) return cb(err);
|
||||
blockChainHeight = res.result;
|
||||
self.syncInfo.blocksToSync = res.result.blocks;
|
||||
return cb();
|
||||
});
|
||||
},
|
||||
function(cb) {
|
||||
if (!opts.reverse) return cb();
|
||||
|
||||
self.rpc.getBlockHash(blockChainHeight, function(err, res) {
|
||||
}
|
||||
else {
|
||||
// should be isOrphan = true or null to be more accurate.
|
||||
Block.count({
|
||||
isOrphan: null
|
||||
},
|
||||
function(err, count) {
|
||||
if (err) return cb(err);
|
||||
|
||||
bestBlock = res.result;
|
||||
|
||||
self.syncInfo.blocksToSync = blockChainHeight - count;
|
||||
if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1;
|
||||
return cb();
|
||||
});
|
||||
},
|
||||
function(cb) {
|
||||
// This is only to inform progress.
|
||||
if (!opts.upToExisting) {
|
||||
self.rpc.getInfo(function(err, res) {
|
||||
if (err) return cb(err);
|
||||
self.syncInfo.blocksToSync = res.result.blocks;
|
||||
return cb();
|
||||
});
|
||||
}
|
||||
},
|
||||
], function(err) {
|
||||
|
||||
var start, end;
|
||||
function sync() {
|
||||
if (opts.reverse) {
|
||||
start = bestBlock;
|
||||
end = self.genesis;
|
||||
opts.prev = true;
|
||||
}
|
||||
else {
|
||||
// should be isOrphan = true or null to be more accurate.
|
||||
Block.count({ isOrphan: null}, function(err, count) {
|
||||
if (err) return cb(err);
|
||||
start = self.genesis;
|
||||
end = null;
|
||||
opts.next = true;
|
||||
}
|
||||
|
||||
self.syncInfo.blocksToSync = blockChainHeight - count;
|
||||
if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1;
|
||||
return cb();
|
||||
});
|
||||
}
|
||||
},
|
||||
],
|
||||
function(err) {
|
||||
self.syncInfo = util._extend(self.syncInfo, {
|
||||
start: start,
|
||||
isStartGenesis: start === self.genesis,
|
||||
end: end,
|
||||
isEndGenesis: end === self.genesis,
|
||||
scanningForward: opts.next,
|
||||
scanningBackward: opts.prev,
|
||||
upToExisting: opts.upToExisting,
|
||||
syncedBlocks: 0,
|
||||
});
|
||||
|
||||
p('Starting from: ', start);
|
||||
p(' to : ', end);
|
||||
p(' opts: ', JSON.stringify(opts));
|
||||
|
||||
var start, end;
|
||||
function sync() {
|
||||
if (opts.reverse) {
|
||||
start = bestBlock;
|
||||
end = self.genesis;
|
||||
opts.prev = true;
|
||||
}
|
||||
else {
|
||||
start = self.genesis;
|
||||
end = null;
|
||||
opts.next = true;
|
||||
self.getPrevNextBlock(start, end, opts, function(err) {
|
||||
if (err && err.message.match(/ECONNREFUSED/)) {
|
||||
setTimeout(function() {
|
||||
p('Retrying in %d secs', retry_secs);
|
||||
sync();
|
||||
},
|
||||
retry_secs * 1000);
|
||||
}
|
||||
else return next(err);
|
||||
});
|
||||
}
|
||||
|
||||
self.syncInfo = util._extend(self.syncInfo, {
|
||||
start: start,
|
||||
isStartGenesis: start === self.genesis,
|
||||
end: end,
|
||||
isEndGenesis: end === self.genesis,
|
||||
scanningForward: opts.next,
|
||||
scanningBackward: opts.prev,
|
||||
upToExisting: opts.upToExisting,
|
||||
syncedBlocks: 0,
|
||||
});
|
||||
|
||||
p('Starting from: ', start);
|
||||
p(' to : ', end);
|
||||
p(' opts: ', JSON.stringify(opts));
|
||||
|
||||
self.getPrevNextBlock( start, end, opts , function(err) {
|
||||
if (err && err.message.match(/ECONNREFUSED/)){
|
||||
setTimeout(function() {
|
||||
p('Retrying in %d secs', retry_secs);
|
||||
sync();
|
||||
}, retry_secs * 1000);
|
||||
}
|
||||
else
|
||||
return next(err);
|
||||
});
|
||||
}
|
||||
|
||||
if (err) {
|
||||
self.syncInfo = util._extend(self.syncInfo, { error: err.message });
|
||||
return next(err, 0);
|
||||
}
|
||||
else {
|
||||
sync();
|
||||
}
|
||||
if (err) {
|
||||
self.syncInfo = util._extend(self.syncInfo, {
|
||||
error: err.message
|
||||
});
|
||||
return next(err, 0);
|
||||
}
|
||||
else {
|
||||
sync();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
|
|
@ -292,11 +296,13 @@ function spec() {
|
|||
HistoricSync.prototype.smart_import = function(next) {
|
||||
var self = this;
|
||||
|
||||
Block.findOne({hash:self.genesis}, function(err, b){
|
||||
Block.findOne({
|
||||
hash: self.genesis
|
||||
},
|
||||
function(err, b) {
|
||||
|
||||
if (err) return next(err);
|
||||
|
||||
|
||||
if (!b) {
|
||||
p('Could not find Genesis block. Running FULL SYNC');
|
||||
}
|
||||
|
|
@ -305,7 +311,7 @@ function spec() {
|
|||
}
|
||||
|
||||
var opts = {
|
||||
reverse: 1,
|
||||
reverse: true,
|
||||
upToExisting: b ? true: false,
|
||||
};
|
||||
|
||||
|
|
@ -313,7 +319,6 @@ function spec() {
|
|||
});
|
||||
};
|
||||
|
||||
|
||||
return HistoricSync;
|
||||
}
|
||||
module.defineClass(spec);
|
||||
|
|
|
|||
|
|
@ -1,16 +1,18 @@
|
|||
'use strict';
|
||||
|
||||
angular.module('insight.status').controller('StatusController',
|
||||
function ($scope, $routeParams, $location, $rootScope, Global, Status, Sync, get_socket) {
|
||||
function($scope, $routeParams, $location, $rootScope, Global, Status, Sync, get_socket) {
|
||||
$scope.global = Global;
|
||||
|
||||
$scope.getStatus = function(q) {
|
||||
Status.get({
|
||||
q: 'get' + q
|
||||
}, function(d) {
|
||||
q: 'get' + q
|
||||
},
|
||||
function(d) {
|
||||
$rootScope.infoError = null;
|
||||
angular.extend($scope, d);
|
||||
}, function(e) {
|
||||
},
|
||||
function(e) {
|
||||
if (e.status === 503) {
|
||||
$rootScope.infoError = 'Backend Error. ' + e.data;
|
||||
}
|
||||
|
|
@ -20,30 +22,35 @@ angular.module('insight.status').controller('StatusController',
|
|||
});
|
||||
};
|
||||
|
||||
var on_sync_update = function(sync) {
|
||||
if (sync.blocksToSync > sync.syncedBlocks) {
|
||||
var p = parseInt(100*(sync.syncedBlocks) / sync.blocksToSync);
|
||||
var delta = sync.blocksToSync - sync.syncedBlocks;
|
||||
sync.message = 'Sync ' + p + '% ['+delta+' blocks remaining]';
|
||||
sync.style = 'warn';
|
||||
} else {
|
||||
sync.message = 'On sync';
|
||||
sync.style = 'success';
|
||||
}
|
||||
sync.tooltip = 'Synced blocks: '+sync.syncedBlocks;
|
||||
$scope.sync = sync;
|
||||
};
|
||||
|
||||
$scope.getSync = function() {
|
||||
Sync.get({}, function(sync) {
|
||||
Sync.get({},
|
||||
function(sync) {
|
||||
$rootScope.syncError = null;
|
||||
|
||||
if (sync.blocksToSync > sync.syncedBlocks ) {
|
||||
sync.message = 'Blocks to sync: ' + (sync.blocksToSync - sync.syncedBlocks);
|
||||
sync.tooltip = 'Skipped blocks:' + sync.skipped_blocks;
|
||||
}
|
||||
else {
|
||||
sync.message = 'On sync';
|
||||
sync.tooltip = '';
|
||||
}
|
||||
|
||||
|
||||
$scope.sync = sync;
|
||||
}, function(e) {
|
||||
on_sync_update(sync);
|
||||
},
|
||||
function(e) {
|
||||
$rootScope.syncError = 'Could not get sync information' + e;
|
||||
});
|
||||
};
|
||||
|
||||
var socket = get_socket($scope);
|
||||
socket.emit('subscribe', 'sync');
|
||||
socket.on('status', function(info) {
|
||||
console.log('info '+JSON.stringify(info));
|
||||
socket.on('status', function(sync) {
|
||||
on_sync_update(sync);
|
||||
});
|
||||
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<div data-ng-controller="FooterController">
|
||||
<div>
|
||||
<div class="container">
|
||||
<p class="text-center text-muted"><a href="/">Insight</a></p>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -26,12 +26,12 @@
|
|||
<div class="status" data-ng-controller="StatusController">
|
||||
<span data-ng-init="getSync()">
|
||||
<span class="small" tooltip="{{sync.tooltip}}" tooltip-placement="down">
|
||||
<i><strong> {{sync.message}} </strong></i>
|
||||
<i class="{{sync.style}}"><strong> {{sync.message}} </strong></i>
|
||||
</span>
|
||||
</span>
|
||||
<span data-ng-init="getStatus('Info')">
|
||||
<i class="small">
|
||||
<strong> Connections: </strong> {{info.connections}}
|
||||
<strong> Conn: </strong> {{info.connections}}
|
||||
</i>
|
||||
<i class="small">
|
||||
<strong>Height:</strong> {{info.blocks}}
|
||||
|
|
|
|||
Loading…
Reference in New Issue