diff --git a/.gitignore b/.gitignore index 888b5ce..aabec27 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ node_modules # extras *.swp +*.swo *~ .project peerdb.json diff --git a/README.md b/README.md index dc344c6..8c1ed48 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,12 @@ A REST API is provided at /api. The entry points are: /api/txs/?address=ADDR /api/txs/?address=mmhmMNfBiZZ37g1tgg2t8DDbNoEdqKVxAL + +### Sync status +``` + /api/sync +``` + ## Web Socket API The web socket API is served using [socket.io](http://socket.io) at: ``` @@ -160,6 +166,22 @@ Sample output: } ``` +'sync': every 1% increment on the sync task, this event will be triggered. + +Sample output: +``` +{ +blocksToSync: 164141, +syncedBlocks: 475, +upToExisting: true, +scanningBackward: true, +isEndGenesis: true, +end: "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943", +isStartGenesis: false, +start: "000000009f929800556a8f3cfdbe57c187f2f679e351b12f7011bfc276c41b6d" +} +``` + ## Troubleshooting If you did not get all library during grunt command, please use the follow command: diff --git a/app/controllers/socket.js b/app/controllers/socket.js index 7bcad79..fa78f55 100644 --- a/app/controllers/socket.js +++ b/app/controllers/socket.js @@ -21,3 +21,7 @@ module.exports.broadcast_tx = function(tx) { module.exports.broadcast_block = function(block) { ios.sockets.emit('block', block); }; + +module.exports.broadcastSyncInfo = function(syncInfo) { + ios.sockets.emit('block', syncInfo); +}; diff --git a/app/controllers/status.js b/app/controllers/status.js index 22970a9..01b51b9 100644 --- a/app/controllers/status.js +++ b/app/controllers/status.js @@ -45,4 +45,8 @@ exports.show = function(req, res, next) { } }; - +exports.sync = function(req, res, next) { + if (req.syncInfo) + res.jsonp(req.syncInfo); + next(); +}; diff --git a/app/models/Block.js b/app/models/Block.js index 709fbcf..e4d84d3 100644 --- a/app/models/Block.js +++ b/app/models/Block.js @@ -27,6 +27,7 @@ var BlockSchema = new Schema({ }, time: Number, nextBlockHash: String, + isOrphan: Boolean, }); /** diff --git a/app/models/Transaction.js b/app/models/Transaction.js index 84c8614..fdad11e 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -142,7 +142,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) { } else { if ( !i.coinbase ) { - console.log ('TX: %s,%d could not parse INPUT', txid, i.n); + console.log ('WARN in TX: %s: could not parse INPUT %d', txid, i.n); } return next_in(); } @@ -165,7 +165,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) { }, next_out); } else { - console.log ('TX: %s,%d could not parse OUTPUT', txid, o.n); + console.log ('WARN in TX: %s could not parse OUTPUT %d', txid, o.n); return next_out(); } }, diff --git a/config/express.js b/config/express.js index a25db3b..0e5c58d 100644 --- a/config/express.js +++ b/config/express.js @@ -7,7 +7,8 @@ var express = require('express'), helpers = require('view-helpers'), config = require('./config'); -module.exports = function(app, passport, db) { +module.exports = function(app, historicSync) { + app.set('showStackError', true); //Prettify HTML @@ -26,9 +27,17 @@ module.exports = function(app, passport, db) { app.set('view engine', 'jade'); //Enable jsonp - app.enable("jsonp callback"); + app.enable('jsonp callback'); + //custom middleware + function setHistoric(req, res, next) { + req.syncInfo = historicSync.syncInfo; + next(); + } + app.use('/api/sync', setHistoric); + app.configure(function() { + //cookieParser should be above session app.use(express.cookieParser()); @@ -43,6 +52,7 @@ module.exports = function(app, passport, db) { //routes should be at the last app.use(app.router); + //Setting the fav icon and static folder app.use(express.favicon()); app.use(express.static(config.root + '/public')); diff --git a/config/routes.js b/config/routes.js index 2f26dc5..fc27b78 100644 --- a/config/routes.js +++ b/config/routes.js @@ -1,6 +1,6 @@ 'use strict'; -module.exports = function(app) { +module.exports = function(app, historicSync) { //Home route var index = require('../app/controllers/index'); @@ -29,4 +29,6 @@ module.exports = function(app) { var st = require('../app/controllers/status'); app.get('/api/status', st.show); + app.get('/api/sync', st.sync); + }; diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index fdadfcc..2735f44 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -11,31 +11,33 @@ function spec() { var config = require('../config/config'); var Block = require('../app/models/Block'); var Sync = require('./Sync').class(); + var sockets = require('../app/controllers/socket.js'); function HistoricSync(opts) { - this.block_count= 0; - this.block_total= 0; this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; var genesisHashReversed = new Buffer(32); this.network.genesisBlock.hash.copy(genesisHashReversed); this.genesis = genesisHashReversed.reverse().toString('hex'); this.sync = new Sync(opts); + + + //available status: new / syncing / finished / aborted + this.status = 'new'; + this.syncInfo = {}; } function p() { var args = []; Array.prototype.push.apply( args, arguments ); + + args.unshift('[historic_sync]'); /*jshint validthis:true */ console.log.apply(this, args); } - var progress_bar = function(string, current, total) { - p(util.format('%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total))); - }; - HistoricSync.prototype.init = function(opts,cb) { this.rpc = new RpcClient(config.bitcoind); this.opts = opts; @@ -46,8 +48,20 @@ function spec() { this.sync.close(); }; - HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { + HistoricSync.prototype.showProgress = function() { + var self = this; + + var i = self.syncInfo; + var per = parseInt(100 * i.syncedBlocks / i.blocksToSync); + p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per)); + if (self.opts.broadcast) { + sockets.broadcastSyncInfo(self.syncInfo); + } + }; + + + HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { var self = this; // recursion end. @@ -71,8 +85,11 @@ function spec() { }, //show some (inacurate) status function(c) { - if (self.block_count % 1000 === 1) { - progress_bar('sync status:', self.block_count, self.block_total); + var step = parseInt(self.syncInfo.blocksToSync / 100); + if (step < 10) step = 10; + + if (self.syncInfo.syncedBlocks % step === 1) { + self.showProgress(); } return c(); }, @@ -113,15 +130,30 @@ function spec() { ], function (err){ - if (err) - p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, self.block_count); + if (err) { + self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks); + self.status = 'aborted'; + p(self.err); + } - if (opts.uptoexisting && existed) { - p('DONE. Found existing block: ', blockHash); - return cb(err); + else { + self.err = null; + self.status = 'syncing'; + } + + if (opts.upToExisting && existed ) { + if (self.syncInfo.blocksToSync <= self.syncInfo.syncedBlocks) { + self.status = 'finished'; + p('DONE. Found existing block: ', blockHash); + return cb(err); + } + else { + p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks); + } } if (blockEnd && blockEnd === blockHash) { + self.status = 'finished'; p('DONE. Found END block: ', blockHash); return cb(err); } @@ -129,7 +161,7 @@ function spec() { // Continue if (blockInfo && blockInfo.result) { - self.block_count++; + self.syncInfo.syncedBlocks++; if (opts.prev && blockInfo.result.previousblockhash) { return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); } @@ -144,44 +176,18 @@ function spec() { HistoricSync.prototype.import_history = function(opts, next) { var self = this; - var retry_attemps = 100; var retry_secs = 2; - var block_best; - var block_height; + var bestBlock; + var blockChainHeight; async.series([ function(cb) { if (opts.destroy) { - p('Deleting Blocks...'); - self.db.collections.blocks.drop(cb); - } else { - return cb(); + p('Deleting DB...'); + return self.sync.destroy(cb); } - }, - function(cb) { - if (opts.destroy) { - p('Deleting TXs...'); - self.db.collections.transactions.drop(cb); - } else { - return cb(); - } - }, - function(cb) { - if (opts.destroy) { - p('Deleting TXItems...'); - self.db.collections.transactionitems.drop(cb); - } else { - return cb(); - } - }, - function(cb) { - self.rpc.getInfo(function(err, res) { - if (err) return cb(err); - - self.block_total = res.result.blocks; - return cb(); - }); + return cb(); }, // We are not using getBestBlockHash, because is not available in all clients function(cb) { @@ -189,27 +195,49 @@ function spec() { self.rpc.getBlockCount(function(err, res) { if (err) return cb(err); - block_height = res.result; + blockChainHeight = res.result; return cb(); }); }, function(cb) { if (!opts.reverse) return cb(); - self.rpc.getBlockHash(block_height, function(err, res) { + self.rpc.getBlockHash(blockChainHeight, function(err, res) { if (err) return cb(err); - block_best = res.result; + bestBlock = res.result; + return cb(); }); }, + function(cb) { + // This is only to inform progress. + if (!opts.upToExisting) { + self.rpc.getInfo(function(err, res) { + if (err) return cb(err); + self.syncInfo.blocksToSync = res.result.blocks; + return cb(); + }); + } + else { + // should be isOrphan = true or null to be more accurate. + Block.count({ isOrphan: null}, function(err, count) { + if (err) return cb(err); + + self.syncInfo.blocksToSync = blockChainHeight - count; + if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1; + return cb(); + }); + } + }, ], function(err) { + var start, end; function sync() { if (opts.reverse) { - start = block_best; + start = bestBlock; end = self.genesis; opts.prev = true; } @@ -219,25 +247,38 @@ function spec() { opts.next = true; } + self.syncInfo = util._extend(self.syncInfo, { + start: start, + isStartGenesis: start === self.genesis, + end: end, + isEndGenesis: end === self.genesis, + scanningForward: opts.next, + scanningBackward: opts.prev, + upToExisting: opts.upToExisting, + syncedBlocks: 0, + }); + p('Starting from: ', start); p(' to : ', end); p(' opts: ', JSON.stringify(opts)); self.getPrevNextBlock( start, end, opts , function(err) { - if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){ + if (err && err.message.match(/ECONNREFUSED/)){ setTimeout(function() { p('Retrying in %d secs', retry_secs); sync(); }, retry_secs * 1000); } else - return next(err, self.block_count); + return next(err); }); } + if (!err) sync(); - else + else { return next(err, 0); + } }); }; @@ -246,6 +287,7 @@ function spec() { var self = this; Block.findOne({hash:self.genesis}, function(err, b){ + if (err) return next(err); @@ -253,12 +295,12 @@ function spec() { p('Could not find Genesis block. Running FULL SYNC'); } else { - p('Genesis block found. Syncing upto know blocks.'); + p('Genesis block found. Syncing upto known blocks.'); } var opts = { reverse: 1, - uptoexisting: b ? true: false, + upToExisting: b ? true: false, }; return self.import_history(opts, next); diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 49526cf..3d7e89c 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -15,8 +15,6 @@ function spec() { PeerSync.prototype.init = function(config, cb) { if (!config) config = {}; - var that = this; - var network = config && (config.network || 'testnet'); this.verbose = config.verbose; @@ -71,6 +69,8 @@ function spec() { } this.sync.storeTxs([tx.hash], null, function(err) { if (err) { +console.log('[PeerSync.js.71:err:]',err); //TODO + console.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); } }); diff --git a/lib/Sync.js b/lib/Sync.js index ff2ffb8..3ff5554 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -9,30 +9,82 @@ function spec() { var Block = require('../app/models/Block'); var Transaction = require('../app/models/Transaction'); var sockets = require('../app/controllers/socket.js'); + var async = require('async'); function Sync() { this.tx_count = 0; } + Sync.prototype.init = function(opts, cb) { + var self = this; + + self.opts = opts; + + if (!(opts && opts.skipDbConnection)) { + + if (mongoose.connection.readyState !== 1) { + mongoose.connect(config.db, function(err) { + if (err) { + console.log('CRITICAL ERROR: connecting to mongoDB:',err); + return (err); + } + }); + } + + self.db = mongoose.connection; + + self.db.on('error', function(err) { + console.log('MongoDB ERROR:' + err); + return cb(err); + }); + + self.db.on('disconnect', function(err) { + console.log('MongoDB disconnect:' + err); + return cb(err); + }); + + return self.db.once('open', function(err) { + return cb(err); + }); + } + else return cb(); + }; + + Sync.prototype.close = function() { + if ( this.db && this.db.readyState ) { + this.db.close(); + } + }; + + + Sync.prototype.destroy = function(next) { + var self = this; + async.series([ + function(b) { return self.db.collections.blocks.drop(b);}, + function(b) { return self.db.collections.transactions.drop(b);}, + function(b) { return self.db.collections.transactionitems.drop(b);}, + ], next); + }; + Sync.prototype.storeBlock = function(block, cb) { - var that = this; + var self = this; Block.customCreate(block, function(err, block, inserted_txs){ if (err) return cb(err); - if (block && that.opts.broadcast_blocks) { + if (block && self.opts.broadcast_blocks) { sockets.broadcast_block(block); } - if (inserted_txs && that.opts.broadcast_txs) { + if (inserted_txs && self.opts.broadcast_txs) { inserted_txs.forEach(function(tx) { sockets.broadcast_tx(tx); }); } if (inserted_txs) - that.tx_count += inserted_txs.length; + self.tx_count += inserted_txs.length; return cb(); }); @@ -40,12 +92,12 @@ function spec() { Sync.prototype.storeTxs = function(txs, inTime, cb) { - var that = this; + var self = this; var time = inTime ? inTime : Math.round(new Date().getTime() / 1000); Transaction.createFromArray(txs, time, function(err, inserted_txs) { - if (!err && inserted_txs && that.opts.broadcast_txs) { + if (!err && inserted_txs && self.opts.broadcast_txs) { inserted_txs.forEach(function(tx) { sockets.broadcast_tx(tx); @@ -55,40 +107,6 @@ function spec() { return cb(err); }); }; - - - Sync.prototype.init = function(opts, cb) { - var that = this; - - that.opts = opts; - - if (!(opts && opts.skip_db_connection)) { - if (!mongoose.connection) { - mongoose.connect(config.db, {server: {auto_reconnect: true}} ); - } - - this.db = mongoose.connection; - - this.db.on('error', function(err) { - console.log('connection error:' + err); - mongoose.disconnect(); - }); - - this.db.on('disconnect', function(err) { - console.log('disconnect:' + err); - mongoose.connect(config.db, {server: {auto_reconnect: true}} ); - }); - - return that.db.once('open', cb); - } - else return cb(); - }; - - Sync.prototype.close = function() { - if (!(this.opts && this.opts.skip_db_connection)) { - this.db.close(); - } - }; return Sync; } module.defineClass(spec); diff --git a/server.js b/server.js index 84e659a..1f49503 100644 --- a/server.js +++ b/server.js @@ -24,7 +24,7 @@ var express = require('express'), var config = require('./config/config'); //Bootstrap db connection -var db = mongoose.connect(config.db); +mongoose.connect(config.db); //Bootstrap models var models_path = __dirname + '/app/models'; @@ -44,14 +44,19 @@ var walk = function(path) { walk(models_path); // historic_sync process +var historicSync = {}; if (!config.disableHistoricSync) { - var hs = new HistoricSync(); - hs.init({ - skip_db_connection: true, + historicSync = new HistoricSync(); + historicSync.init({ + skipDbConnection: true, + shouldBroadcast: true, networkName: config.network }, function() { - hs.smart_import(function(){ - console.log('[historic_sync] finished!'); + historicSync.smart_import(function(err){ + var txt= 'ended.'; + if (err) txt = 'ABORTED with error: ' + err.message; + + console.log('[historic_sync] ' + txt, historicSync.syncInfo); }); }); } @@ -61,7 +66,7 @@ if (!config.disableHistoricSync) { if (!config.disableP2pSync) { var ps = new PeerSync(); ps.init({ - skip_db_connection: true, + skipDbConnection: true, broadcast_txs: true, broadcast_blocks: true }, function() { @@ -74,7 +79,7 @@ if (!config.disableP2pSync) { var app = express(); //express settings -require('./config/express')(app, db); +require('./config/express')(app, historicSync); //Bootstrap routes require('./config/routes')(app); diff --git a/util/sync.js b/util/sync.js index beccf0f..d399a1e 100755 --- a/util/sync.js +++ b/util/sync.js @@ -29,7 +29,6 @@ if (program.remove) { } */ - async.series([ function(cb) { historicSync.init(program, cb); @@ -42,22 +41,18 @@ async.series([ historicSync.import_history({ destroy: program.destroy, reverse: program.reverse, - uptoexisting: program.uptoexisting, + upToExisting: program.uptoexisting, }, cb); } }, - function(cb) { - historicSync.close(); - return cb(); - }, ], - function(err, count) { + function(err) { + historicSync.close(); if (err) { console.log('CRITICAL ERROR: ', err); } else { - console.log('Finished. [%d blocks synced]', count[1]); + console.log('Finished.\n Status:\n', historicSync.syncInfo); } - return; });