zcash-hackworks
/
insight-ui-zcash
mirror of https://github.com/zcash-hackworks/insight-ui-zcash.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #108 from maraoz/feature/realtime-sync-info 

Feature/realtime sync info
This commit is contained in:
Manuel Aráoz 2014-01-21 08:31:42 -08:00
parent 852e390dd8 2d176a6431
commit 590acff915
5 changed files with 227 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/socket.js
View File

@ -27,5 +27,5 @@ module.exports.broadcast_address_tx = function(address, tx) {
}; };
module.exports.broadcastSyncInfo = function(syncInfo) { module.exports.broadcastSyncInfo = function(syncInfo) {
ios.sockets.emit('sync', syncInfo); ios.sockets.in('sync').emit('status', syncInfo);
}; };

379
lib/HistoricSync.js
View File

@ -2,43 +2,39 @@
require('classtool'); require('classtool');
function spec() { function spec() {
var util = require('util'); var util = require('util');
var RpcClient = require('bitcore/RpcClient').class(); var RpcClient = require('bitcore/RpcClient').class();
var networks = require('bitcore/networks'); var networks = require('bitcore/networks');
var async = require('async'); var async = require('async');
var config = require('../config/config'); var config = require('../config/config');
var Block = require('../app/models/Block'); var Block = require('../app/models/Block');
var Sync = require('./Sync').class(); var Sync = require('./Sync').class();
var sockets = require('../app/controllers/socket.js'); var sockets = require('../app/controllers/socket.js');
function HistoricSync(opts) { function HistoricSync(opts) {
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;
var genesisHashReversed = new Buffer(32); var genesisHashReversed = new Buffer(32);
this.network.genesisBlock.hash.copy(genesisHashReversed); this.network.genesisBlock.hash.copy(genesisHashReversed);
this.genesis = genesisHashReversed.reverse().toString('hex'); this.genesis = genesisHashReversed.reverse().toString('hex');
this.sync = new Sync(opts); this.sync = new Sync(opts);
//available status: new / syncing / finished / aborted //available status: new / syncing / finished / aborted
this.status = 'new'; this.status = 'new';
this.syncInfo = {}; this.syncInfo = {};
} }
function p() { function p() {
var args = []; var args = [];
Array.prototype.push.apply( args, arguments ); Array.prototype.push.apply(args, arguments);
args.unshift('[historic_sync]'); args.unshift('[historic_sync]');
/*jshint validthis:true */ /*jshint validthis:true */
console.log.apply(this, args); console.log.apply(this, args);
} }
HistoricSync.prototype.init = function(opts,cb) { HistoricSync.prototype.init = function(opts, cb) {
this.rpc = new RpcClient(config.bitcoind); this.rpc = new RpcClient(config.bitcoind);
this.opts = opts; this.opts = opts;
this.sync.init(opts, cb); this.sync.init(opts, cb);
@ -48,76 +44,84 @@ function spec() {
this.sync.close(); this.sync.close();
}; };
HistoricSync.prototype.showProgress = function() { HistoricSync.prototype.showProgress = function() {
var self = this; var self = this;
var i = self.syncInfo; var i = self.syncInfo;
var per = parseInt(100 * i.syncedBlocks / i.blocksToSync); var per = parseInt(100 * i.syncedBlocks / i.blocksToSync);
p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per)); p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per));
if (self.opts.broadcast) { if (self.opts.shouldBroadcast) {
sockets.broadcastSyncInfo(self.syncInfo); sockets.broadcastSyncInfo(self.syncInfo);
} }
}; };
HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
var self = this; var self = this;
// recursion end. // recursion end.
if (!blockHash ) return cb(); if (!blockHash) return cb();
var existed = 0; var existed = false;
var blockInfo; var blockInfo;
var blockObj; var blockObj;
async.series([ async.series([
// Already got it? // Already got it?
function(c) { function(c) {
Block.findOne({hash:blockHash}, function(err,block){ Block.findOne({
if (err) { p(err); return c(err); } hash: blockHash
if (block) {
existed =1;
blockObj =block;
}
return c();
});
}, },
//show some (inacurate) status function(err, block) {
function(c) { if (err) {
var step = parseInt(self.syncInfo.blocksToSync / 100); p(err);
if (step < 10) step = 10; return c(err);
}
if (self.syncInfo.syncedBlocks % step === 1) { if (block) {
self.showProgress(); existed = true;
blockObj = block;
} }
return c(); return c();
}, });
//get Info from RPC },
function(c) { //show some (inacurate) status
function(c) {
if (!self.step) {
var step = parseInt(self.syncInfo.blocksToSync / 100);
if (self.opts.progressStep) {
step = self.opts.progressStep;
}
if (step < 2) step = 2;
self.step = step;
}
if (self.syncInfo.syncedBlocks % self.step === 1) {
self.showProgress();
}
return c();
},
//get Info from RPC
function(c) {
// TODO: if we store prev/next, no need to go to RPC // TODO: if we store prev/next, no need to go to RPC
// if (blockObj && blockObj.nextBlockHash) return c(); // if (blockObj && blockObj.nextBlockHash) return c();
self.rpc.getBlock(blockHash, function(err, ret) {
if (err) return c(err);
self.rpc.getBlock(blockHash, function(err, ret) { blockInfo = ret;
if (err) return c(err); return c();
});
},
//store it
function(c) {
if (existed) return c();
self.sync.storeBlock(blockInfo.result, function(err) {
blockInfo = ret; existed = err && err.toString().match(/E11000/);
return c();
});
},
//store it
function(c) {
if (existed) return c();
self.sync.storeBlock(blockInfo.result, function(err) {
existed = err && err.toString().match(/E11000/); if (err && ! existed) return c(err);
return c();
if (err && ! existed) return c(err); });
return c(); },
}); /* TODO: Should Start to sync backwards? (this is for partial syncs)
},
/* TODO: Should Start to sync backwards? (this is for partial syncs)
function(c) { function(c) {
if (blockInfo.result.prevblockhash != current.blockHash) { if (blockInfo.result.prevblockhash != current.blockHash) {
@ -127,164 +131,164 @@ function spec() {
return c(); return c();
} }
*/ */
], ], function(err) {
function (err){
if (err) { if (err) {
self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks); self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks);
self.status = 'aborted'; self.status = 'aborted';
p(self.err); p(self.err);
} }
else { else {
self.err = null; self.err = null;
self.status = 'syncing'; self.status = 'syncing';
} }
if (opts.upToExisting && existed ) { if (opts.upToExisting && existed) {
var diff = self.syncInfo.blocksToSync - self.syncInfo.syncedBlocks; var diff = self.syncInfo.blocksToSync - self.syncInfo.syncedBlocks;
if (diff <= 0) { if (diff <= 0) {
self.status = 'finished';
p('DONE. Found existing block: ', blockHash);
return cb(err);
}
else {
self.syncInfo.skipped_blocks = self.syncInfo.skipped_blocks || 1;
if ((self.syncInfo.skipped_blocks++ % 1000) === 1 ) {
p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks, self.syncInfo.blocksToSync, self.syncInfo.skipped_blocks);
}
}
}
if (blockEnd && blockEnd === blockHash) {
self.status = 'finished'; self.status = 'finished';
p('DONE. Found END block: ', blockHash); p('DONE. Found existing block: ', blockHash);
return cb(err); return cb(err);
} }
else {
self.syncInfo.skipped_blocks = self.syncInfo.skipped_blocks || 1;
// Continue if ((self.syncInfo.skipped_blocks++ % 1000) === 1) {
if (blockInfo && blockInfo.result) { p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks, self.syncInfo.blocksToSync, self.syncInfo.skipped_blocks);
if (! existed) self.syncInfo.syncedBlocks++;
if (opts.prev && blockInfo.result.previousblockhash) {
return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
} }
if (opts.next && blockInfo.result.nextblockhash)
return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
} }
}
if (blockEnd && blockEnd === blockHash) {
self.status = 'finished';
p('DONE. Found END block: ', blockHash);
return cb(err); return cb(err);
}
// Continue
if (blockInfo && blockInfo.result) {
if (!existed) self.syncInfo.syncedBlocks++;
if (opts.prev && blockInfo.result.previousblockhash) {
return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
}
if (opts.next && blockInfo.result.nextblockhash) return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb);
}
return cb(err);
}); });
}; };
HistoricSync.prototype.import_history = function(opts, next) { HistoricSync.prototype.import_history = function(opts, next) {
var self = this; var self = this;
var retry_secs = 2; var retry_secs = 2;
var bestBlock; var bestBlock;
var blockChainHeight; var blockChainHeight;
async.series([ async.series([
function(cb) { function(cb) {
if (opts.destroy) { if (opts.destroy) {
p('Deleting DB...'); p('Deleting DB...');
return self.sync.destroy(cb); return self.sync.destroy(cb);
} }
return cb();
},
// We are not using getBestBlockHash, because is not available in all clients
function(cb) {
if (!opts.reverse) return cb();
self.rpc.getBlockCount(function(err, res) {
if (err) return cb(err);
blockChainHeight = res.result;
return cb(); return cb();
}, });
// We are not using getBestBlockHash, because is not available in all clients },
function(cb) { function(cb) {
if (!opts.reverse) return cb(); if (!opts.reverse) return cb();
self.rpc.getBlockCount(function(err, res) { self.rpc.getBlockHash(blockChainHeight, function(err, res) {
if (err) return cb(err);
bestBlock = res.result;
return cb();
});
},
function(cb) {
// This is only to inform progress.
if (!opts.upToExisting) {
self.rpc.getInfo(function(err, res) {
if (err) return cb(err); if (err) return cb(err);
blockChainHeight = res.result; self.syncInfo.blocksToSync = res.result.blocks;
return cb(); return cb();
}); });
}, }
function(cb) { else {
if (!opts.reverse) return cb(); // should be isOrphan = true or null to be more accurate.
Block.count({
self.rpc.getBlockHash(blockChainHeight, function(err, res) { isOrphan: null
},
function(err, count) {
if (err) return cb(err); if (err) return cb(err);
bestBlock = res.result; self.syncInfo.blocksToSync = blockChainHeight - count;
if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1;
return cb(); return cb();
}); });
}, }
function(cb) { },
// This is only to inform progress. ], function(err) {
if (!opts.upToExisting) {
self.rpc.getInfo(function(err, res) { var start, end;
if (err) return cb(err); function sync() {
self.syncInfo.blocksToSync = res.result.blocks; if (opts.reverse) {
return cb(); start = bestBlock;
}); end = self.genesis;
opts.prev = true;
} }
else { else {
// should be isOrphan = true or null to be more accurate. start = self.genesis;
Block.count({ isOrphan: null}, function(err, count) { end = null;
if (err) return cb(err); opts.next = true;
}
self.syncInfo.blocksToSync = blockChainHeight - count; self.syncInfo = util._extend(self.syncInfo, {
if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1; start: start,
return cb(); isStartGenesis: start === self.genesis,
}); end: end,
} isEndGenesis: end === self.genesis,
}, scanningForward: opts.next,
], scanningBackward: opts.prev,
function(err) { upToExisting: opts.upToExisting,
syncedBlocks: 0,
});
p('Starting from: ', start);
p(' to : ', end);
p(' opts: ', JSON.stringify(opts));
var start, end; self.getPrevNextBlock(start, end, opts, function(err) {
function sync() { if (err && err.message.match(/ECONNREFUSED/)) {
if (opts.reverse) { setTimeout(function() {
start = bestBlock; p('Retrying in %d secs', retry_secs);
end = self.genesis; sync();
opts.prev = true; },
} retry_secs * 1000);
else {
start = self.genesis;
end = null;
opts.next = true;
} }
else return next(err);
});
}
self.syncInfo = util._extend(self.syncInfo, { if (err) {
start: start, self.syncInfo = util._extend(self.syncInfo, {
isStartGenesis: start === self.genesis, error: err.message
end: end, });
isEndGenesis: end === self.genesis, return next(err, 0);
scanningForward: opts.next, }
scanningBackward: opts.prev, else {
upToExisting: opts.upToExisting, sync();
syncedBlocks: 0, }
});
p('Starting from: ', start);
p(' to : ', end);
p(' opts: ', JSON.stringify(opts));
self.getPrevNextBlock( start, end, opts , function(err) {
if (err && err.message.match(/ECONNREFUSED/)){
setTimeout(function() {
p('Retrying in %d secs', retry_secs);
sync();
}, retry_secs * 1000);
}
else
return next(err);
});
}
if (err) {
self.syncInfo = util._extend(self.syncInfo, { error: err.message });
return next(err, 0);
}
else {
sync();
}
}); });
}; };
@ -292,11 +296,13 @@ function spec() {
HistoricSync.prototype.smart_import = function(next) { HistoricSync.prototype.smart_import = function(next) {
var self = this; var self = this;
Block.findOne({hash:self.genesis}, function(err, b){ Block.findOne({
hash: self.genesis
},
function(err, b) {
if (err) return next(err); if (err) return next(err);
if (!b) { if (!b) {
p('Could not find Genesis block. Running FULL SYNC'); p('Could not find Genesis block. Running FULL SYNC');
} }
@ -305,7 +311,7 @@ function spec() {
} }
var opts = { var opts = {
reverse: 1, reverse: true,
upToExisting: b ? true: false, upToExisting: b ? true: false,
}; };
@ -313,7 +319,6 @@ function spec() {
}); });
}; };
return HistoricSync; return HistoricSync;
} }
module.defineClass(spec); module.defineClass(spec);

50
public/js/controllers/status.js
View File

@ -1,16 +1,18 @@
'use strict'; 'use strict';
angular.module('insight.status').controller('StatusController', angular.module('insight.status').controller('StatusController',
function ($scope, $routeParams, $location, $rootScope, Global, Status, Sync) { function($scope, $routeParams, $location, $rootScope, Global, Status, Sync, get_socket) {
$scope.global = Global; $scope.global = Global;
$scope.getStatus = function(q) { $scope.getStatus = function(q) {
Status.get({ Status.get({
q: 'get' + q q: 'get' + q
}, function(d) { },
function(d) {
$rootScope.infoError = null; $rootScope.infoError = null;
angular.extend($scope, d); angular.extend($scope, d);
}, function(e) { },
function(e) {
if (e.status === 503) { if (e.status === 503) {
$rootScope.infoError = 'Backend Error. ' + e.data; $rootScope.infoError = 'Backend Error. ' + e.data;
} }
@ -20,24 +22,36 @@ angular.module('insight.status').controller('StatusController',
}); });
}; };
var on_sync_update = function(sync) {
if (sync.blocksToSync > sync.syncedBlocks) {
var p = parseInt(100*(sync.syncedBlocks) / sync.blocksToSync);
var delta = sync.blocksToSync - sync.syncedBlocks;
sync.message = 'Sync ' + p + '% ['+delta+' blocks remaining]';
sync.style = 'warn';
} else {
sync.message = 'On sync';
sync.style = 'success';
}
sync.tooltip = 'Synced blocks: '+sync.syncedBlocks;
$scope.sync = sync;
};
$scope.getSync = function() { $scope.getSync = function() {
Sync.get({}, function(sync) { Sync.get({},
function(sync) {
$rootScope.syncError = null; $rootScope.syncError = null;
on_sync_update(sync);
if (sync.blocksToSync > sync.syncedBlocks ) { },
sync.message = 'Blocks to sync: ' + (sync.blocksToSync - sync.syncedBlocks); function(e) {
sync.tooltip = 'Skipped blocks:' + sync.skipped_blocks;
}
else {
sync.message = 'On sync';
sync.tooltip = '';
}
$scope.sync = sync;
}, function(e) {
$rootScope.syncError = 'Could not get sync information' + e; $rootScope.syncError = 'Could not get sync information' + e;
}); });
}; };
var socket = get_socket($scope);
socket.emit('subscribe', 'sync');
socket.on('status', function(sync) {
on_sync_update(sync);
});
}); });

2
public/views/header.html
View File

@ -26,7 +26,7 @@
<div class="status" data-ng-controller="StatusController"> <div class="status" data-ng-controller="StatusController">
<span data-ng-init="getSync()"> <span data-ng-init="getSync()">
<span class="small" tooltip="{{sync.tooltip}}" tooltip-placement="down"> <span class="small" tooltip="{{sync.tooltip}}" tooltip-placement="down">
<i><strong> {{sync.message}} </strong></i> <i class="{{sync.style}}"><strong> {{sync.message}} </strong></i>
</span> </span>
</span> </span>
<span data-ng-init="getStatus('Info')"> <span data-ng-init="getStatus('Info')">

1
server.js
View File

@ -60,6 +60,7 @@ if (!config.disableHistoricSync) {
historicSync.init({ historicSync.init({
skipDbConnection: true, skipDbConnection: true,
shouldBroadcast: true, shouldBroadcast: true,
progressStep: 2,
networkName: config.network networkName: config.network
}, function() { }, function() {
historicSync.smart_import(function(err){ historicSync.smart_import(function(err){