diff --git a/app/controllers/socket.js b/app/controllers/socket.js index acfa2c9..ace6315 100644 --- a/app/controllers/socket.js +++ b/app/controllers/socket.js @@ -27,5 +27,5 @@ module.exports.broadcast_address_tx = function(address, tx) { }; module.exports.broadcastSyncInfo = function(syncInfo) { - ios.sockets.emit('sync', syncInfo); + ios.sockets.in('sync').emit('status', syncInfo); }; diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index b33b4bd..f6a29e2 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -2,43 +2,39 @@ require('classtool'); - function spec() { - var util = require('util'); - var RpcClient = require('bitcore/RpcClient').class(); - var networks = require('bitcore/networks'); - var async = require('async'); - var config = require('../config/config'); - var Block = require('../app/models/Block'); - var Sync = require('./Sync').class(); - var sockets = require('../app/controllers/socket.js'); + var util = require('util'); + var RpcClient = require('bitcore/RpcClient').class(); + var networks = require('bitcore/networks'); + var async = require('async'); + var config = require('../config/config'); + var Block = require('../app/models/Block'); + var Sync = require('./Sync').class(); + var sockets = require('../app/controllers/socket.js'); function HistoricSync(opts) { - this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; + this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; var genesisHashReversed = new Buffer(32); this.network.genesisBlock.hash.copy(genesisHashReversed); this.genesis = genesisHashReversed.reverse().toString('hex'); - this.sync = new Sync(opts); - + this.sync = new Sync(opts); //available status: new / syncing / finished / aborted - this.status = 'new'; + this.status = 'new'; this.syncInfo = {}; } function p() { var args = []; - Array.prototype.push.apply( args, arguments ); + Array.prototype.push.apply(args, arguments); - - args.unshift('[historic_sync]'); /*jshint validthis:true */ console.log.apply(this, args); } - HistoricSync.prototype.init = function(opts,cb) { + HistoricSync.prototype.init = function(opts, cb) { this.rpc = new RpcClient(config.bitcoind); this.opts = opts; this.sync.init(opts, cb); @@ -48,76 +44,84 @@ function spec() { this.sync.close(); }; - HistoricSync.prototype.showProgress = function() { var self = this; var i = self.syncInfo; var per = parseInt(100 * i.syncedBlocks / i.blocksToSync); p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per)); - if (self.opts.broadcast) { + if (self.opts.shouldBroadcast) { sockets.broadcastSyncInfo(self.syncInfo); } }; - HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { var self = this; // recursion end. - if (!blockHash ) return cb(); + if (!blockHash) return cb(); - var existed = 0; + var existed = false; var blockInfo; var blockObj; async.series([ - // Already got it? - function(c) { - Block.findOne({hash:blockHash}, function(err,block){ - if (err) { p(err); return c(err); } - if (block) { - existed =1; - blockObj =block; - } - return c(); - }); + // Already got it? + function(c) { + Block.findOne({ + hash: blockHash }, - //show some (inacurate) status - function(c) { - var step = parseInt(self.syncInfo.blocksToSync / 100); - if (step < 10) step = 10; - - if (self.syncInfo.syncedBlocks % step === 1) { - self.showProgress(); + function(err, block) { + if (err) { + p(err); + return c(err); + } + if (block) { + existed = true; + blockObj = block; } return c(); - }, - //get Info from RPC - function(c) { + }); + }, + //show some (inacurate) status + function(c) { + if (!self.step) { + var step = parseInt(self.syncInfo.blocksToSync / 100); + if (self.opts.progressStep) { + step = self.opts.progressStep; + } + if (step < 2) step = 2; + self.step = step; + } + if (self.syncInfo.syncedBlocks % self.step === 1) { + self.showProgress(); + } + return c(); + }, + //get Info from RPC + function(c) { - // TODO: if we store prev/next, no need to go to RPC - // if (blockObj && blockObj.nextBlockHash) return c(); + // TODO: if we store prev/next, no need to go to RPC + // if (blockObj && blockObj.nextBlockHash) return c(); + self.rpc.getBlock(blockHash, function(err, ret) { + if (err) return c(err); - self.rpc.getBlock(blockHash, function(err, ret) { - if (err) return c(err); + blockInfo = ret; + return c(); + }); + }, + //store it + function(c) { + if (existed) return c(); + self.sync.storeBlock(blockInfo.result, function(err) { - blockInfo = ret; - return c(); - }); - }, - //store it - function(c) { - if (existed) return c(); - self.sync.storeBlock(blockInfo.result, function(err) { + existed = err && err.toString().match(/E11000/); - existed = err && err.toString().match(/E11000/); - - if (err && ! existed) return c(err); - return c(); - }); - }, - /* TODO: Should Start to sync backwards? (this is for partial syncs) + if (err && ! existed) return c(err); + return c(); + }); + }, + /* TODO: Should Start to sync backwards? (this is for partial syncs) function(c) { if (blockInfo.result.prevblockhash != current.blockHash) { @@ -127,164 +131,164 @@ function spec() { return c(); } */ - ], - function (err){ + ], function(err) { - if (err) { - self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks); - self.status = 'aborted'; - p(self.err); - } + if (err) { + self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks); + self.status = 'aborted'; + p(self.err); + } - else { - self.err = null; - self.status = 'syncing'; - } + else { + self.err = null; + self.status = 'syncing'; + } - if (opts.upToExisting && existed ) { - var diff = self.syncInfo.blocksToSync - self.syncInfo.syncedBlocks; - if (diff <= 0) { - self.status = 'finished'; - p('DONE. Found existing block: ', blockHash); - return cb(err); - } - else { - self.syncInfo.skipped_blocks = self.syncInfo.skipped_blocks || 1; - if ((self.syncInfo.skipped_blocks++ % 1000) === 1 ) { - p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks, self.syncInfo.blocksToSync, self.syncInfo.skipped_blocks); - } - } - } - - if (blockEnd && blockEnd === blockHash) { + if (opts.upToExisting && existed) { + var diff = self.syncInfo.blocksToSync - self.syncInfo.syncedBlocks; + if (diff <= 0) { self.status = 'finished'; - p('DONE. Found END block: ', blockHash); + p('DONE. Found existing block: ', blockHash); return cb(err); } - - - // Continue - if (blockInfo && blockInfo.result) { - if (! existed) self.syncInfo.syncedBlocks++; - if (opts.prev && blockInfo.result.previousblockhash) { - return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); + else { + self.syncInfo.skipped_blocks = self.syncInfo.skipped_blocks || 1; + if ((self.syncInfo.skipped_blocks++ % 1000) === 1) { + p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks, self.syncInfo.blocksToSync, self.syncInfo.skipped_blocks); } - - if (opts.next && blockInfo.result.nextblockhash) - return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb); } + } + + if (blockEnd && blockEnd === blockHash) { + self.status = 'finished'; + p('DONE. Found END block: ', blockHash); return cb(err); + } + + // Continue + if (blockInfo && blockInfo.result) { + if (!existed) self.syncInfo.syncedBlocks++; + if (opts.prev && blockInfo.result.previousblockhash) { + return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); + } + + if (opts.next && blockInfo.result.nextblockhash) return self.getPrevNextBlock(blockInfo.result.nextblockhash, blockEnd, opts, cb); + } + return cb(err); }); }; HistoricSync.prototype.import_history = function(opts, next) { var self = this; - var retry_secs = 2; + var retry_secs = 2; var bestBlock; var blockChainHeight; async.series([ - function(cb) { - if (opts.destroy) { - p('Deleting DB...'); - return self.sync.destroy(cb); - } + function(cb) { + if (opts.destroy) { + p('Deleting DB...'); + return self.sync.destroy(cb); + } + return cb(); + }, + // We are not using getBestBlockHash, because is not available in all clients + function(cb) { + if (!opts.reverse) return cb(); + + self.rpc.getBlockCount(function(err, res) { + if (err) return cb(err); + blockChainHeight = res.result; return cb(); - }, - // We are not using getBestBlockHash, because is not available in all clients - function(cb) { - if (!opts.reverse) return cb(); + }); + }, + function(cb) { + if (!opts.reverse) return cb(); - self.rpc.getBlockCount(function(err, res) { + self.rpc.getBlockHash(blockChainHeight, function(err, res) { + if (err) return cb(err); + + bestBlock = res.result; + + return cb(); + }); + }, + function(cb) { + // This is only to inform progress. + if (!opts.upToExisting) { + self.rpc.getInfo(function(err, res) { if (err) return cb(err); - blockChainHeight = res.result; + self.syncInfo.blocksToSync = res.result.blocks; return cb(); }); - }, - function(cb) { - if (!opts.reverse) return cb(); - - self.rpc.getBlockHash(blockChainHeight, function(err, res) { + } + else { + // should be isOrphan = true or null to be more accurate. + Block.count({ + isOrphan: null + }, + function(err, count) { if (err) return cb(err); - bestBlock = res.result; - + self.syncInfo.blocksToSync = blockChainHeight - count; + if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1; return cb(); }); - }, - function(cb) { - // This is only to inform progress. - if (!opts.upToExisting) { - self.rpc.getInfo(function(err, res) { - if (err) return cb(err); - self.syncInfo.blocksToSync = res.result.blocks; - return cb(); - }); + } + }, + ], function(err) { + + var start, end; + function sync() { + if (opts.reverse) { + start = bestBlock; + end = self.genesis; + opts.prev = true; } else { - // should be isOrphan = true or null to be more accurate. - Block.count({ isOrphan: null}, function(err, count) { - if (err) return cb(err); + start = self.genesis; + end = null; + opts.next = true; + } - self.syncInfo.blocksToSync = blockChainHeight - count; - if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1; - return cb(); - }); - } - }, - ], - function(err) { + self.syncInfo = util._extend(self.syncInfo, { + start: start, + isStartGenesis: start === self.genesis, + end: end, + isEndGenesis: end === self.genesis, + scanningForward: opts.next, + scanningBackward: opts.prev, + upToExisting: opts.upToExisting, + syncedBlocks: 0, + }); + p('Starting from: ', start); + p(' to : ', end); + p(' opts: ', JSON.stringify(opts)); - var start, end; - function sync() { - if (opts.reverse) { - start = bestBlock; - end = self.genesis; - opts.prev = true; - } - else { - start = self.genesis; - end = null; - opts.next = true; + self.getPrevNextBlock(start, end, opts, function(err) { + if (err && err.message.match(/ECONNREFUSED/)) { + setTimeout(function() { + p('Retrying in %d secs', retry_secs); + sync(); + }, + retry_secs * 1000); } + else return next(err); + }); + } - self.syncInfo = util._extend(self.syncInfo, { - start: start, - isStartGenesis: start === self.genesis, - end: end, - isEndGenesis: end === self.genesis, - scanningForward: opts.next, - scanningBackward: opts.prev, - upToExisting: opts.upToExisting, - syncedBlocks: 0, - }); - - p('Starting from: ', start); - p(' to : ', end); - p(' opts: ', JSON.stringify(opts)); - - self.getPrevNextBlock( start, end, opts , function(err) { - if (err && err.message.match(/ECONNREFUSED/)){ - setTimeout(function() { - p('Retrying in %d secs', retry_secs); - sync(); - }, retry_secs * 1000); - } - else - return next(err); - }); - } - - if (err) { - self.syncInfo = util._extend(self.syncInfo, { error: err.message }); - return next(err, 0); - } - else { - sync(); - } + if (err) { + self.syncInfo = util._extend(self.syncInfo, { + error: err.message + }); + return next(err, 0); + } + else { + sync(); + } }); }; @@ -292,11 +296,13 @@ function spec() { HistoricSync.prototype.smart_import = function(next) { var self = this; - Block.findOne({hash:self.genesis}, function(err, b){ + Block.findOne({ + hash: self.genesis + }, + function(err, b) { if (err) return next(err); - if (!b) { p('Could not find Genesis block. Running FULL SYNC'); } @@ -305,7 +311,7 @@ function spec() { } var opts = { - reverse: 1, + reverse: true, upToExisting: b ? true: false, }; @@ -313,7 +319,6 @@ function spec() { }); }; - return HistoricSync; } module.defineClass(spec); diff --git a/public/js/controllers/status.js b/public/js/controllers/status.js index f1fdded..d4d4111 100644 --- a/public/js/controllers/status.js +++ b/public/js/controllers/status.js @@ -1,16 +1,18 @@ 'use strict'; angular.module('insight.status').controller('StatusController', - function ($scope, $routeParams, $location, $rootScope, Global, Status, Sync) { +function($scope, $routeParams, $location, $rootScope, Global, Status, Sync, get_socket) { $scope.global = Global; $scope.getStatus = function(q) { Status.get({ - q: 'get' + q - }, function(d) { + q: 'get' + q + }, + function(d) { $rootScope.infoError = null; angular.extend($scope, d); - }, function(e) { + }, + function(e) { if (e.status === 503) { $rootScope.infoError = 'Backend Error. ' + e.data; } @@ -20,24 +22,36 @@ angular.module('insight.status').controller('StatusController', }); }; + var on_sync_update = function(sync) { + if (sync.blocksToSync > sync.syncedBlocks) { + var p = parseInt(100*(sync.syncedBlocks) / sync.blocksToSync); + var delta = sync.blocksToSync - sync.syncedBlocks; + sync.message = 'Sync ' + p + '% ['+delta+' blocks remaining]'; + sync.style = 'warn'; + } else { + sync.message = 'On sync'; + sync.style = 'success'; + } + sync.tooltip = 'Synced blocks: '+sync.syncedBlocks; + $scope.sync = sync; + }; + $scope.getSync = function() { - Sync.get({}, function(sync) { + Sync.get({}, + function(sync) { $rootScope.syncError = null; - - if (sync.blocksToSync > sync.syncedBlocks ) { - sync.message = 'Blocks to sync: ' + (sync.blocksToSync - sync.syncedBlocks); - sync.tooltip = 'Skipped blocks:' + sync.skipped_blocks; - } - else { - sync.message = 'On sync'; - sync.tooltip = ''; - } - - - $scope.sync = sync; - }, function(e) { + on_sync_update(sync); + }, + function(e) { $rootScope.syncError = 'Could not get sync information' + e; }); }; + + var socket = get_socket($scope); + socket.emit('subscribe', 'sync'); + socket.on('status', function(sync) { + on_sync_update(sync); + }); + }); diff --git a/public/views/header.html b/public/views/header.html index 7144912..47917e1 100755 --- a/public/views/header.html +++ b/public/views/header.html @@ -26,7 +26,7 @@
- {{sync.message}} + {{sync.message}} diff --git a/server.js b/server.js index 365ff28..c2fde5f 100644 --- a/server.js +++ b/server.js @@ -60,6 +60,7 @@ if (!config.disableHistoricSync) { historicSync.init({ skipDbConnection: true, shouldBroadcast: true, + progressStep: 2, networkName: config.network }, function() { historicSync.smart_import(function(err){