From 8d672de5bd3039437ea6dbd2c435a7fbd00ac21f Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 12:05:27 -0400 Subject: [PATCH 01/21] refactoring message broker into plugin --- app/controllers/socket.js | 54 +++----------------------------------- config/config.js | 6 +++-- insight.js | 53 ++++++++++++++++++++++++++++++++++--- plugins/mailbox.js | 2 ++ test/test.socket-server.js | 2 +- 5 files changed, 61 insertions(+), 56 deletions(-) create mode 100644 plugins/mailbox.js diff --git a/app/controllers/socket.js b/app/controllers/socket.js index 9220cf2e..47952f94 100644 --- a/app/controllers/socket.js +++ b/app/controllers/socket.js @@ -3,9 +3,6 @@ // server-side socket behaviour var ios = null; // io is already taken in express var util = require('bitcore').util; -var mdb = require('../../lib/MessageDb').default(); -var microtime = require('microtime'); -var enableMessageBroker; var verbose = false; var log = function() { @@ -15,64 +12,21 @@ var log = function() { } module.exports.init = function(io_ext, config) { - enableMessageBroker = config ? config.enableMessageBroker : false; ios = io_ext; + verbose = config.verbose; if (ios) { // when a new socket connects ios.sockets.on('connection', function(socket) { log('New connection from ' + socket.id); // when it subscribes, make it join the according room socket.on('subscribe', function(topic) { - if (socket.rooms.length === 1) { - log('subscribe to ' + topic); - socket.join(topic); - } + log('subscribe to ' + topic); + socket.join(topic); }); - if (enableMessageBroker) { - // when it requests sync, send him all pending messages - socket.on('sync', function(ts) { - log('Sync requested by ' + socket.id); - log(' from timestamp '+ts); - var rooms = socket.rooms; - if (rooms.length !== 2) { - socket.emit('insight-error', 'Must subscribe with public key before syncing'); - return; - } - var to = rooms[1]; - var upper_ts = Math.round(microtime.now()); - log(' to timestamp '+upper_ts); - mdb.getMessages(to, ts, upper_ts, function(err, messages) { - if (err) { - throw new Error('Couldn\'t get messages on sync request: ' + err); - } - log('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); - for (var i = 0; i < messages.length; i++) { - broadcastMessage(messages[i], socket); - } - }); - }); - - // when it sends a message, add it to db - socket.on('message', function(m) { - log('Message sent from ' + m.pubkey + ' to ' + m.to); - mdb.addMessage(m, function(err) { - if (err) { - throw new Error('Couldn\'t add message to database: ' + err); - } - }); - }); - - - // disconnect handler - socket.on('disconnect', function() { - log('disconnected ' + socket.id); - }); - } }); - if (enableMessageBroker) - mdb.on('message', broadcastMessage); } + return ios; }; var simpleTx = function(tx) { diff --git a/config/config.js b/config/config.js index 8d7c02be..35b3cf37 100644 --- a/config/config.js +++ b/config/config.js @@ -61,6 +61,7 @@ dataDir += network === 'testnet' ? 'testnet3' : ''; var safeConfirmations = process.env.INSIGHT_SAFE_CONFIRMATIONS || 6; var ignoreCache = process.env.INSIGHT_IGNORE_CACHE || 0; +var verbose = process.env.VERBOSE === 'true'; var bitcoindConf = { @@ -76,7 +77,7 @@ var bitcoindConf = { disableAgent: true }; -var enableMessageBroker = process.env.ENABLE_MESSAGE_BROKER === 'true'; +var enableMailbox = process.env.ENABLE_MAILBOX === 'true'; if (!fs.existsSync(db)) { var err = fs.mkdirSync(db); @@ -89,7 +90,8 @@ if (!fs.existsSync(db)) { } module.exports = { - enableMessageBroker: enableMessageBroker, + enableMailbox: enableMailbox, + verbose: verbose, version: version, root: rootPath, publicPath: process.env.INSIGHT_PUBLIC_PATH || false, diff --git a/insight.js b/insight.js index a55703b5..0b7a268e 100755 --- a/insight.js +++ b/insight.js @@ -111,16 +111,63 @@ if (!config.disableHistoricSync) { if (peerSync) peerSync.allowReorgs = true; -//express settings +// express settings require('./config/express')(expressApp, historicSync, peerSync); -//Bootstrap routes +// routes require('./config/routes')(expressApp); // socket.io var server = require('http').createServer(expressApp); var ios = require('socket.io')(server); -require('./app/controllers/socket.js').init(ios, config); + +// plugins +require('./app/controllers/socket.js').init(ios); +if (config.enableMessageBroker) { + var mdb = require('../../lib/MessageDb').default(); + + // when it requests sync, send him all pending messages + socket.on('sync', function(ts) { + log('Sync requested by ' + socket.id); + log(' from timestamp ' + ts); + var rooms = socket.rooms; + if (rooms.length !== 2) { + socket.emit('insight-error', 'Must subscribe with public key before syncing'); + return; + } + var to = rooms[1]; + var upper_ts = Math.round(microtime.now()); + log(' to timestamp ' + upper_ts); + mdb.getMessages(to, ts, upper_ts, function(err, messages) { + if (err) { + throw new Error('Couldn\'t get messages on sync request: ' + err); + } + log('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); + for (var i = 0; i < messages.length; i++) { + broadcastMessage(messages[i], socket); + } + }); + }); + + // when it sends a message, add it to db + socket.on('message', function(m) { + log('Message sent from ' + m.pubkey + ' to ' + m.to); + mdb.addMessage(m, function(err) { + if (err) { + throw new Error('Couldn\'t add message to database: ' + err); + } + }); + }); + + + // disconnect handler + socket.on('disconnect', function() { + log('disconnected ' + socket.id); + }); + + mdb.on('message', broadcastMessage); + +} //Start the app by listening on server.listen(config.port, function() { diff --git a/plugins/mailbox.js b/plugins/mailbox.js new file mode 100644 index 00000000..441c5a1d --- /dev/null +++ b/plugins/mailbox.js @@ -0,0 +1,2 @@ + +var microtime = require('microtime'); diff --git a/test/test.socket-server.js b/test/test.socket-server.js index f0744729..6920889c 100644 --- a/test/test.socket-server.js +++ b/test/test.socket-server.js @@ -15,7 +15,7 @@ describe('socket server', function() { }); it('should register socket handlers', function() { var io = { - sockets: new EventEmitter() + sockets: new EventEmitter(), } socket.init(io); From a5fda98cdc5142e702876cacff5b7df50ed4f31c Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 12:21:58 -0400 Subject: [PATCH 02/21] use winston for logging --- README.md | 2 +- app/controllers/socket.js | 17 +++----- config/config.js | 2 - insight.js | 50 ++-------------------- lib/logger.js | 2 + plugins/mailbox.js | 90 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 101 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index abf3c08f..a470313a 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ INSIGHT_NETWORK [= 'livenet' | 'testnet'] INSIGHT_DB # Path where to store insight's internal DB. (defaults to $HOME/.insight) INSIGHT_SAFE_CONFIRMATIONS=6 # Nr. of confirmation needed to start caching transaction information INSIGHT_IGNORE_CACHE # True to ignore cache of spents in transaction, with more than INSIGHT_SAFE_CONFIRMATIONS confirmations. This is useful for tracking double spents for old transactions. -ENABLE_MESSAGE_BROKER # if "true" will enable message broker module +ENABLE_MAILBOX # if "true" will enable message broker module ``` diff --git a/app/controllers/socket.js b/app/controllers/socket.js index 47952f94..1c82ea77 100644 --- a/app/controllers/socket.js +++ b/app/controllers/socket.js @@ -3,24 +3,17 @@ // server-side socket behaviour var ios = null; // io is already taken in express var util = require('bitcore').util; +var logger = require('../../lib/logger'); -var verbose = false; -var log = function() { - if (verbose) { - console.log(arguments); - } -} - -module.exports.init = function(io_ext, config) { +module.exports.init = function(io_ext) { ios = io_ext; - verbose = config.verbose; if (ios) { // when a new socket connects ios.sockets.on('connection', function(socket) { - log('New connection from ' + socket.id); + logger.info('New connection from ' + socket.id); // when it subscribes, make it join the according room socket.on('subscribe', function(topic) { - log('subscribe to ' + topic); + logger.info('subscribe to ' + topic); socket.join(topic); }); @@ -76,7 +69,7 @@ module.exports.broadcastSyncInfo = function(historicSync) { var broadcastMessage = module.exports.broadcastMessage = function(message, socket) { if (ios) { var s = socket || ios.sockets.in(message.to); - log('sending message to ' + message.to); + logger.info('sending message to ' + message.to); s.emit('message', message); } diff --git a/config/config.js b/config/config.js index 35b3cf37..ca1fe4cc 100644 --- a/config/config.js +++ b/config/config.js @@ -61,7 +61,6 @@ dataDir += network === 'testnet' ? 'testnet3' : ''; var safeConfirmations = process.env.INSIGHT_SAFE_CONFIRMATIONS || 6; var ignoreCache = process.env.INSIGHT_IGNORE_CACHE || 0; -var verbose = process.env.VERBOSE === 'true'; var bitcoindConf = { @@ -91,7 +90,6 @@ if (!fs.existsSync(db)) { module.exports = { enableMailbox: enableMailbox, - verbose: verbose, version: version, root: rootPath, publicPath: process.env.INSIGHT_PUBLIC_PATH || false, diff --git a/insight.js b/insight.js index 0b7a268e..9dbbc8f1 100755 --- a/insight.js +++ b/insight.js @@ -119,54 +119,12 @@ require('./config/routes')(expressApp); // socket.io var server = require('http').createServer(expressApp); -var ios = require('socket.io')(server); +var ios = require('socket.io')(server, config); +require('./app/controllers/socket.js').init(ios); // plugins -require('./app/controllers/socket.js').init(ios); -if (config.enableMessageBroker) { - var mdb = require('../../lib/MessageDb').default(); - - // when it requests sync, send him all pending messages - socket.on('sync', function(ts) { - log('Sync requested by ' + socket.id); - log(' from timestamp ' + ts); - var rooms = socket.rooms; - if (rooms.length !== 2) { - socket.emit('insight-error', 'Must subscribe with public key before syncing'); - return; - } - var to = rooms[1]; - var upper_ts = Math.round(microtime.now()); - log(' to timestamp ' + upper_ts); - mdb.getMessages(to, ts, upper_ts, function(err, messages) { - if (err) { - throw new Error('Couldn\'t get messages on sync request: ' + err); - } - log('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); - for (var i = 0; i < messages.length; i++) { - broadcastMessage(messages[i], socket); - } - }); - }); - - // when it sends a message, add it to db - socket.on('message', function(m) { - log('Message sent from ' + m.pubkey + ' to ' + m.to); - mdb.addMessage(m, function(err) { - if (err) { - throw new Error('Couldn\'t add message to database: ' + err); - } - }); - }); - - - // disconnect handler - socket.on('disconnect', function() { - log('disconnected ' + socket.id); - }); - - mdb.on('message', broadcastMessage); - +if (config.enableMailbox) { + require('./plugins/mailbox').init(ios, config.mailbox); } //Start the app by listening on diff --git a/lib/logger.js b/lib/logger.js index 779b46de..3946aceb 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,5 +1,7 @@ var winston = require('winston'); + +winston.transports.console.level = 'debug'; winston.info('starting...') module.exports.logger=winston; diff --git a/plugins/mailbox.js b/plugins/mailbox.js index 441c5a1d..3145f793 100644 --- a/plugins/mailbox.js +++ b/plugins/mailbox.js @@ -1,2 +1,90 @@ - var microtime = require('microtime'); +var mdb = require('../../lib/MessageDb').default(); + + +module.exports.init = function(ios, config) { + + ios.sockets.on('connection', function(socket) { + // when it requests sync, send him all pending messages + socket.on('sync', function(ts) { + log('Sync requested by ' + socket.id); + log(' from timestamp ' + ts); + var rooms = socket.rooms; + if (rooms.length !== 2) { + socket.emit('insight-error', 'Must subscribe with public key before syncing'); + return; + } + var to = rooms[1]; + var upper_ts = Math.round(microtime.now()); + log(' to timestamp ' + upper_ts); + mdb.getMessages(to, ts, upper_ts, function(err, messages) { + if (err) { + throw new Error('Couldn\'t get messages on sync request: ' + err); + } + log('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); + for (var i = 0; i < messages.length; i++) { + broadcastMessage(messages[i], socket); + } + }); + }); + + // when it sends a message, add it to db + socket.on('message', function(m) { + log('Message sent from ' + m.pubkey + ' to ' + m.to); + mdb.addMessage(m, function(err) { + if (err) { + throw new Error('Couldn\'t add message to database: ' + err); + } + }); + }); + + + // disconnect handler + socket.on('disconnect', function() { + log('disconnected ' + socket.id); + }); + + mdb.on('message', broadcastMessage); + // when it requests sync, send him all pending messages + socket.on('sync', function(ts) { + log('Sync requested by ' + socket.id); + log(' from timestamp ' + ts); + var rooms = socket.rooms; + if (rooms.length !== 2) { + socket.emit('insight-error', 'Must subscribe with public key before syncing'); + return; + } + var to = rooms[1]; + var upper_ts = Math.round(microtime.now()); + log(' to timestamp ' + upper_ts); + mdb.getMessages(to, ts, upper_ts, function(err, messages) { + if (err) { + throw new Error('Couldn\'t get messages on sync request: ' + err); + } + log('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); + for (var i = 0; i < messages.length; i++) { + broadcastMessage(messages[i], socket); + } + }); + }); + + // when it sends a message, add it to db + socket.on('message', function(m) { + log('Message sent from ' + m.pubkey + ' to ' + m.to); + mdb.addMessage(m, function(err) { + if (err) { + throw new Error('Couldn\'t add message to database: ' + err); + } + }); + }); + + + // disconnect handler + socket.on('disconnect', function() { + log('disconnected ' + socket.id); + }); + }); + + mdb.on('message', broadcastMessage); + +}; From f1c3d7c4983b8fc8a9162090927948b7960bbc7c Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 12:49:57 -0400 Subject: [PATCH 03/21] fix logging and plugin --- app/controllers/socket.js | 14 +++--------- lib/MessageDb.js | 3 --- lib/TransactionDb.js | 2 +- lib/logger.js | 8 +++---- package.json | 6 ++--- plugins/mailbox.js | 46 ++++++++++++++++++++++++--------------- 6 files changed, 40 insertions(+), 39 deletions(-) diff --git a/app/controllers/socket.js b/app/controllers/socket.js index 1c82ea77..59f4a186 100644 --- a/app/controllers/socket.js +++ b/app/controllers/socket.js @@ -3,17 +3,17 @@ // server-side socket behaviour var ios = null; // io is already taken in express var util = require('bitcore').util; -var logger = require('../../lib/logger'); +var logger = require('../../lib/logger').logger; module.exports.init = function(io_ext) { ios = io_ext; if (ios) { // when a new socket connects ios.sockets.on('connection', function(socket) { - logger.info('New connection from ' + socket.id); + logger.debug('New connection from ' + socket.id); // when it subscribes, make it join the according room socket.on('subscribe', function(topic) { - logger.info('subscribe to ' + topic); + logger.debug('subscribe to ' + topic); socket.join(topic); }); @@ -66,11 +66,3 @@ module.exports.broadcastSyncInfo = function(historicSync) { ios.sockets.in('sync').emit('status', historicSync); }; -var broadcastMessage = module.exports.broadcastMessage = function(message, socket) { - if (ios) { - var s = socket || ios.sockets.in(message.to); - logger.info('sending message to ' + message.to); - s.emit('message', message); - } - -} diff --git a/lib/MessageDb.js b/lib/MessageDb.js index 0742820c..a7afe885 100644 --- a/lib/MessageDb.js +++ b/lib/MessageDb.js @@ -19,10 +19,7 @@ var MAX_OPEN_FILES = 500; var CONCURRENCY = 5; -var d = logger.log; -var info = logger.info; var db; - var MessageDb = function(opts) { opts = opts || {}; this.path = config.leveldb + '/messages' + (opts.name ? ('-' + opts.name) : ''); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index cc1ff7de..ef501150 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -39,7 +39,7 @@ var bitcore = require('bitcore'), bitcoreUtil = bitcore.util, buffertools = require('buffertools'); -var logger = require('./logger').logger; +var logger = require('./logger'); var inf = logger.info; var db = imports.db || levelup(config.leveldb + '/txs', { diff --git a/lib/logger.js b/lib/logger.js index 3946aceb..13533688 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,7 +1,7 @@ var winston = require('winston'); +var logger = winston; +logger.transports.Console.level = 'debug'; +logger.info('starting...'); -winston.transports.console.level = 'debug'; -winston.info('starting...') - -module.exports.logger=winston; +module.exports.logger = logger; diff --git a/package.json b/package.json index 52893590..e7644776 100644 --- a/package.json +++ b/package.json @@ -66,11 +66,11 @@ "preconditions": "^1.0.7", "should": "~2.1.1", "sinon": "~1.7.3", + "socket.io": "1.0.6", + "socket.io-client": "1.0.6", "soop": "=0.1.5", "winston": "*", - "xmlhttprequest": "~1.6.0", - "socket.io": "1.0.6", - "socket.io-client": "1.0.6" + "xmlhttprequest": "~1.6.0" }, "devDependencies": { "chai": "*", diff --git a/plugins/mailbox.js b/plugins/mailbox.js index 3145f793..a9eaf9f7 100644 --- a/plugins/mailbox.js +++ b/plugins/mailbox.js @@ -1,14 +1,17 @@ var microtime = require('microtime'); -var mdb = require('../../lib/MessageDb').default(); +var mdb = require('../lib/MessageDb').default(); +var logger = require('../lib/logger').logger; +var preconditions = require('preconditions').singleton(); - -module.exports.init = function(ios, config) { - - ios.sockets.on('connection', function(socket) { +var io; +module.exports.init = function(ext_io, config) { + preconditions.checkArgument(ext_io); + io = ext_io; + io.sockets.on('connection', function(socket) { // when it requests sync, send him all pending messages socket.on('sync', function(ts) { - log('Sync requested by ' + socket.id); - log(' from timestamp ' + ts); + logger.debug('Sync requested by ' + socket.id); + logger.debug(' from timestamp ' + ts); var rooms = socket.rooms; if (rooms.length !== 2) { socket.emit('insight-error', 'Must subscribe with public key before syncing'); @@ -16,12 +19,12 @@ module.exports.init = function(ios, config) { } var to = rooms[1]; var upper_ts = Math.round(microtime.now()); - log(' to timestamp ' + upper_ts); + logger.debug(' to timestamp ' + upper_ts); mdb.getMessages(to, ts, upper_ts, function(err, messages) { if (err) { throw new Error('Couldn\'t get messages on sync request: ' + err); } - log('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); + logger.debug('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); for (var i = 0; i < messages.length; i++) { broadcastMessage(messages[i], socket); } @@ -30,7 +33,7 @@ module.exports.init = function(ios, config) { // when it sends a message, add it to db socket.on('message', function(m) { - log('Message sent from ' + m.pubkey + ' to ' + m.to); + logger.debug('Message sent from ' + m.pubkey + ' to ' + m.to); mdb.addMessage(m, function(err) { if (err) { throw new Error('Couldn\'t add message to database: ' + err); @@ -41,14 +44,14 @@ module.exports.init = function(ios, config) { // disconnect handler socket.on('disconnect', function() { - log('disconnected ' + socket.id); + logger.debug('disconnected ' + socket.id); }); mdb.on('message', broadcastMessage); // when it requests sync, send him all pending messages socket.on('sync', function(ts) { - log('Sync requested by ' + socket.id); - log(' from timestamp ' + ts); + logger.debug('Sync requested by ' + socket.id); + logger.debug(' from timestamp ' + ts); var rooms = socket.rooms; if (rooms.length !== 2) { socket.emit('insight-error', 'Must subscribe with public key before syncing'); @@ -56,12 +59,12 @@ module.exports.init = function(ios, config) { } var to = rooms[1]; var upper_ts = Math.round(microtime.now()); - log(' to timestamp ' + upper_ts); + logger.debug(' to timestamp ' + upper_ts); mdb.getMessages(to, ts, upper_ts, function(err, messages) { if (err) { throw new Error('Couldn\'t get messages on sync request: ' + err); } - log('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); + logger.debug('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); for (var i = 0; i < messages.length; i++) { broadcastMessage(messages[i], socket); } @@ -70,7 +73,7 @@ module.exports.init = function(ios, config) { // when it sends a message, add it to db socket.on('message', function(m) { - log('Message sent from ' + m.pubkey + ' to ' + m.to); + logger.debug('Message sent from ' + m.pubkey + ' to ' + m.to); mdb.addMessage(m, function(err) { if (err) { throw new Error('Couldn\'t add message to database: ' + err); @@ -81,10 +84,19 @@ module.exports.init = function(ios, config) { // disconnect handler socket.on('disconnect', function() { - log('disconnected ' + socket.id); + logger.debug('disconnected ' + socket.id); }); }); mdb.on('message', broadcastMessage); }; + + + +var broadcastMessage = module.exports.broadcastMessage = function(message, socket) { + preconditions.checkState(io); + var s = socket || io.sockets.in(message.to); + logger.debug('sending message to ' + message.to); + s.emit('message', message); +} From 4628f20f9f9bc75d0830b39bba184782e535f01f Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 13:02:45 -0400 Subject: [PATCH 04/21] fix weird logging bug --- lib/BlockDb.js | 9 ++++----- lib/logger.js | 10 ++++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 8c8a8f81..ed8e62d5 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -26,7 +26,6 @@ var async = require('async'); var logger = require('./logger').logger; -var d = logger.log; var info = logger.info; var BlockDb = function(opts) { @@ -113,7 +112,7 @@ BlockDb.prototype._changeBlockHeight = function(hash, height, cb) { var self = this; var dbScript1 = this._setHeightScript(hash,height); - d('Getting TXS FROM %s to set it Main', hash); + logger.log('Getting TXS FROM %s to set it Main', hash); this.fromHashWithInfo(hash, function(err, bi) { if (!bi || !bi.info || !bi.info.tx) throw new Error('unable to get info for block:'+ hash); @@ -121,10 +120,10 @@ BlockDb.prototype._changeBlockHeight = function(hash, height, cb) { var dbScript2; if (height>=0) { dbScript2 = self._addTxsScript(bi.info.tx, hash, height); - info('\t%s %d Txs', 'Confirming', bi.info.tx.length); + logger.info('\t%s %d Txs', 'Confirming', bi.info.tx.length); } else { dbScript2 = self._delTxsScript(bi.info.tx); - info('\t%s %d Txs', 'Unconfirming', bi.info.tx.length); + logger.info('\t%s %d Txs', 'Unconfirming', bi.info.tx.length); } db.batch(dbScript2.concat(dbScript1),cb); }); @@ -230,7 +229,7 @@ BlockDb.prototype.getHeight = function(hash, cb) { }; BlockDb.prototype._setHeightScript = function(hash, height) { - d('setHeight: %s #%d', hash,height); + logger.log('setHeight: %s #%d', hash,height); return ([{ type: 'put', key: MAIN_PREFIX + hash, diff --git a/lib/logger.js b/lib/logger.js index 13533688..61140eeb 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,7 +1,13 @@ var winston = require('winston'); -var logger = winston; -logger.transports.Console.level = 'debug'; +var logger = new winston.Logger({ + transports: [ + new winston.transports.Console({ + level: 'error' + }), + ] +}); +logger.transports.console.level = 'debug'; logger.info('starting...'); module.exports.logger = logger; From 4f613ab4d08dcc96cf5c1839b88be010dc5bc664 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 13:05:52 -0400 Subject: [PATCH 05/21] fix plugin bug --- plugins/mailbox.js | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/plugins/mailbox.js b/plugins/mailbox.js index a9eaf9f7..5d6cab76 100644 --- a/plugins/mailbox.js +++ b/plugins/mailbox.js @@ -9,45 +9,6 @@ module.exports.init = function(ext_io, config) { io = ext_io; io.sockets.on('connection', function(socket) { // when it requests sync, send him all pending messages - socket.on('sync', function(ts) { - logger.debug('Sync requested by ' + socket.id); - logger.debug(' from timestamp ' + ts); - var rooms = socket.rooms; - if (rooms.length !== 2) { - socket.emit('insight-error', 'Must subscribe with public key before syncing'); - return; - } - var to = rooms[1]; - var upper_ts = Math.round(microtime.now()); - logger.debug(' to timestamp ' + upper_ts); - mdb.getMessages(to, ts, upper_ts, function(err, messages) { - if (err) { - throw new Error('Couldn\'t get messages on sync request: ' + err); - } - logger.debug('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); - for (var i = 0; i < messages.length; i++) { - broadcastMessage(messages[i], socket); - } - }); - }); - - // when it sends a message, add it to db - socket.on('message', function(m) { - logger.debug('Message sent from ' + m.pubkey + ' to ' + m.to); - mdb.addMessage(m, function(err) { - if (err) { - throw new Error('Couldn\'t add message to database: ' + err); - } - }); - }); - - - // disconnect handler - socket.on('disconnect', function() { - logger.debug('disconnected ' + socket.id); - }); - - mdb.on('message', broadcastMessage); // when it requests sync, send him all pending messages socket.on('sync', function(ts) { logger.debug('Sync requested by ' + socket.id); @@ -81,7 +42,6 @@ module.exports.init = function(ext_io, config) { }); }); - // disconnect handler socket.on('disconnect', function() { logger.debug('disconnected ' + socket.id); From 0ff490dbf31af3718511860155a530825725dec6 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 13:10:38 -0400 Subject: [PATCH 06/21] make logging level configurable via env vars --- README.md | 3 ++- config/config.js | 2 ++ lib/logger.js | 3 ++- plugins/mailbox.js | 1 - 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a470313a..c949c887 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,8 @@ INSIGHT_NETWORK [= 'livenet' | 'testnet'] INSIGHT_DB # Path where to store insight's internal DB. (defaults to $HOME/.insight) INSIGHT_SAFE_CONFIRMATIONS=6 # Nr. of confirmation needed to start caching transaction information INSIGHT_IGNORE_CACHE # True to ignore cache of spents in transaction, with more than INSIGHT_SAFE_CONFIRMATIONS confirmations. This is useful for tracking double spents for old transactions. -ENABLE_MAILBOX # if "true" will enable message broker module +ENABLE_MAILBOX # if "true" will enable mailbox plugin +LOGGER_LEVEL # defaults to 'info', can be 'debug','verbose','error', etc. ``` diff --git a/config/config.js b/config/config.js index ca1fe4cc..f3941acc 100644 --- a/config/config.js +++ b/config/config.js @@ -77,6 +77,7 @@ var bitcoindConf = { }; var enableMailbox = process.env.ENABLE_MAILBOX === 'true'; +var loggerLevel = process.env.LOGGER_LEVEL || 'info'; if (!fs.existsSync(db)) { var err = fs.mkdirSync(db); @@ -90,6 +91,7 @@ if (!fs.existsSync(db)) { module.exports = { enableMailbox: enableMailbox, + loggerLevel: loggerLevel, version: version, root: rootPath, publicPath: process.env.INSIGHT_PUBLIC_PATH || false, diff --git a/lib/logger.js b/lib/logger.js index 61140eeb..2884ec72 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -1,4 +1,5 @@ var winston = require('winston'); +var config = require('../config/config'); var logger = new winston.Logger({ transports: [ @@ -7,7 +8,7 @@ var logger = new winston.Logger({ }), ] }); -logger.transports.console.level = 'debug'; +logger.transports.console.level = config.loggerLevel; logger.info('starting...'); module.exports.logger = logger; diff --git a/plugins/mailbox.js b/plugins/mailbox.js index 5d6cab76..75cfdb9a 100644 --- a/plugins/mailbox.js +++ b/plugins/mailbox.js @@ -8,7 +8,6 @@ module.exports.init = function(ext_io, config) { preconditions.checkArgument(ext_io); io = ext_io; io.sockets.on('connection', function(socket) { - // when it requests sync, send him all pending messages // when it requests sync, send him all pending messages socket.on('sync', function(ts) { logger.debug('Sync requested by ' + socket.id); From 461fda09be6eca7de21036c4c5bf90f6dbd270fa Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 14:40:56 -0400 Subject: [PATCH 07/21] create ratelimiter plugin --- README.md | 1 + config/config.js | 2 ++ insight.js | 28 ++++++++++++++-------------- package.json | 2 ++ plugins/mailbox.js | 1 + plugins/ratelimiter.js | 10 ++++++++++ 6 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 plugins/ratelimiter.js diff --git a/README.md b/README.md index c949c887..d31a9b92 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,7 @@ INSIGHT_DB # Path where to store insight's internal DB. (defaults to INSIGHT_SAFE_CONFIRMATIONS=6 # Nr. of confirmation needed to start caching transaction information INSIGHT_IGNORE_CACHE # True to ignore cache of spents in transaction, with more than INSIGHT_SAFE_CONFIRMATIONS confirmations. This is useful for tracking double spents for old transactions. ENABLE_MAILBOX # if "true" will enable mailbox plugin +ENABLE_RATELIMITER # if "true" will enable the ratelimiter plugin LOGGER_LEVEL # defaults to 'info', can be 'debug','verbose','error', etc. ``` diff --git a/config/config.js b/config/config.js index f3941acc..8e1fbbeb 100644 --- a/config/config.js +++ b/config/config.js @@ -77,6 +77,7 @@ var bitcoindConf = { }; var enableMailbox = process.env.ENABLE_MAILBOX === 'true'; +var enableRatelimiter = process.env.ENABLE_RATELIMITER === 'true'; var loggerLevel = process.env.LOGGER_LEVEL || 'info'; if (!fs.existsSync(db)) { @@ -91,6 +92,7 @@ if (!fs.existsSync(db)) { module.exports = { enableMailbox: enableMailbox, + enableRatelimiter: enableRatelimiter, loggerLevel: loggerLevel, version: version, root: rootPath, diff --git a/insight.js b/insight.js index 9dbbc8f1..d278e801 100755 --- a/insight.js +++ b/insight.js @@ -4,15 +4,13 @@ //Set the node enviornment variable if not set before process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -/** - * Module dependencies. - */ -var express = require('express'), - fs = require('fs'), - PeerSync = require('./lib/PeerSync'), - HistoricSync = require('./lib/HistoricSync'); +var fs = require('fs'); +var PeerSync = require('./lib/PeerSync'); +var HistoricSync = require('./lib/HistoricSync'); + +var express = require('express'); +var connect = require('connect'); -//Initializing system variables var config = require('./config/config'); // text title @@ -54,14 +52,12 @@ console.log( config.bitcoind.dataDir + (config.network === 'testnet' ? '*' : ''), (config.network === 'testnet' ? '* (/testnet3 is added automatically)' : '') ); -/** - * express app - */ + + +// create express app var expressApp = express(); -/** - * Bootstrap models - */ +// Bootstrap models var models_path = __dirname + '/app/models'; var walk = function(path) { fs.readdirSync(path).forEach(function(file) { @@ -127,6 +123,10 @@ if (config.enableMailbox) { require('./plugins/mailbox').init(ios, config.mailbox); } +if (config.enableRatelimiter) { + require('./plugins/ratelimiter').init(expressApp, config.ratelimiter); +} + //Start the app by listening on server.listen(config.port, function() { console.log('insight server listening on port %d in %s mode', server.address().port, process.env.NODE_ENV); diff --git a/package.json b/package.json index e7644776..409826d1 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,8 @@ "bufferput": "git://github.com/bitpay/node-bufferput.git", "buffertools": "*", "commander": "*", + "connect": "^2.25.7", + "connect-ratelimit": "0.0.6", "express": "~3.4.7", "glob": "*", "leveldown": "*", diff --git a/plugins/mailbox.js b/plugins/mailbox.js index 75cfdb9a..74d62f52 100644 --- a/plugins/mailbox.js +++ b/plugins/mailbox.js @@ -5,6 +5,7 @@ var preconditions = require('preconditions').singleton(); var io; module.exports.init = function(ext_io, config) { + logger.info('Using mailbox plugin'); preconditions.checkArgument(ext_io); io = ext_io; io.sockets.on('connection', function(socket) { diff --git a/plugins/ratelimiter.js b/plugins/ratelimiter.js new file mode 100644 index 00000000..5628b8de --- /dev/null +++ b/plugins/ratelimiter.js @@ -0,0 +1,10 @@ +var logger = require('../lib/logger').logger; +var preconditions = require('preconditions').singleton(); + +var limiter = require('connect-ratelimit'); + +module.exports.init = function(app, config) { + preconditions.checkArgument(app); + logger.info('Using ratelimiter plugin'); + +}; From 4dcc246ea55bf4bfbab0709ff994f1c7d7eb64ed Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 14:41:48 -0400 Subject: [PATCH 08/21] remove starting --- lib/logger.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/logger.js b/lib/logger.js index 2884ec72..23680925 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -9,6 +9,5 @@ var logger = new winston.Logger({ ] }); logger.transports.console.level = config.loggerLevel; -logger.info('starting...'); module.exports.logger = logger; From b7f25852d5afa5d73627eb23c656160e3c23ef6b Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 15:21:05 -0400 Subject: [PATCH 09/21] move help text to --help --- insight.js | 32 ++++++++++++++++++-------------- package.json | 2 +- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/insight.js b/insight.js index d278e801..acb3547f 100755 --- a/insight.js +++ b/insight.js @@ -10,11 +10,14 @@ var HistoricSync = require('./lib/HistoricSync'); var express = require('express'); var connect = require('connect'); +var program = require('commander'); var config = require('./config/config'); +var logger = require('./lib/logger').logger; +program + .version(config.version); // text title -/*jshint multistr: true */ console.log( '\n\ ____ _ __ __ ___ _ \n\ @@ -23,8 +26,9 @@ console.log( _/ // / / (__ ) / /_/ / / / / /_ / ___ |/ /_/ / / \n\ /___/_/ /_/____/_/\\__, /_/ /_/\\__/ /_/ |_/ .___/_/ \n\ /____/ /_/ \n\ -\n\t\t\t\t\t\tv%s\n\ -# Configuration:\n\ +\n\t\t\t\t\t\tv%s\n', config.version); +program.on('--help', function() { + logger.info('\n# Configuration:\n\ \tINSIGHT_NETWORK (Network): %s\n\ \tINSIGHT_DB (Database Path): %s\n\ \tINSIGHT_SAFE_CONFIRMATIONS (Safe Confirmations): %s\n\ @@ -41,18 +45,18 @@ console.log( \nChange setting by assigning the enviroment variables above. Example:\n\ $ INSIGHT_NETWORK="testnet" BITCOIND_HOST="123.123.123.123" ./insight.js\ \n\n', - config.version, - config.network, config.leveldb, config.safeConfirmations, config.ignoreCache ? 'yes' : 'no', - config.bitcoind.user, - config.bitcoind.pass ? 'Yes(hidden)' : 'No', - config.bitcoind.protocol, - config.bitcoind.host, - config.bitcoind.port, - config.bitcoind.p2pPort, - config.bitcoind.dataDir + (config.network === 'testnet' ? '*' : ''), (config.network === 'testnet' ? '* (/testnet3 is added automatically)' : '') -); - + config.network, config.leveldb, config.safeConfirmations, config.ignoreCache ? 'yes' : 'no', + config.bitcoind.user, + config.bitcoind.pass ? 'Yes(hidden)' : 'No', + config.bitcoind.protocol, + config.bitcoind.host, + config.bitcoind.port, + config.bitcoind.p2pPort, + config.bitcoind.dataDir + (config.network === 'testnet' ? '*' : ''), (config.network === 'testnet' ? '* (/testnet3 is added automatically)' : '') + ); +}); +program.parse(process.argv); // create express app var expressApp = express(); diff --git a/package.json b/package.json index 409826d1..f006af6e 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "bitcore": "git://github.com/bitpay/bitcore.git#aa41c70cff2583d810664c073a324376c39c8b36", "bufferput": "git://github.com/bitpay/node-bufferput.git", "buffertools": "*", - "commander": "*", + "commander": "^2.3.0", "connect": "^2.25.7", "connect-ratelimit": "0.0.6", "express": "~3.4.7", From 52671cf093589ab8c1cc4bfb0b7b3570fd442c85 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 15:37:25 -0400 Subject: [PATCH 10/21] add simple rate limiter --- insight.js | 10 ++++++---- plugins/ratelimiter.js | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/insight.js b/insight.js index acb3547f..8fa23981 100755 --- a/insight.js +++ b/insight.js @@ -111,9 +111,6 @@ if (!config.disableHistoricSync) { if (peerSync) peerSync.allowReorgs = true; -// express settings -require('./config/express')(expressApp, historicSync, peerSync); - // routes require('./config/routes')(expressApp); @@ -131,9 +128,14 @@ if (config.enableRatelimiter) { require('./plugins/ratelimiter').init(expressApp, config.ratelimiter); } + +// express settings +require('./config/express')(expressApp, historicSync, peerSync); + + //Start the app by listening on server.listen(config.port, function() { - console.log('insight server listening on port %d in %s mode', server.address().port, process.env.NODE_ENV); + logger.info('insight server listening on port %d in %s mode', server.address().port, process.env.NODE_ENV); }); //expose app diff --git a/plugins/ratelimiter.js b/plugins/ratelimiter.js index 5628b8de..73748345 100644 --- a/plugins/ratelimiter.js +++ b/plugins/ratelimiter.js @@ -7,4 +7,28 @@ module.exports.init = function(app, config) { preconditions.checkArgument(app); logger.info('Using ratelimiter plugin'); + config = config || {}; + config.whitelistRPH = config.whitelistRPH || 5000; + config.normalRPH = config.normalRPH || 1; + + console.log('asdasdasd'); + app.use(limiter({ + whitelist: [], + blacklist: ['localhost'], // 'example.com' + categories: { + whitelist: { + totalRequests: config.whitelistRPH, + every: 60 * 60 * 1000 + }, + blacklist: { + totalRequests: 0, + every: 0 + }, + normal: { + totalRequests: config.normalRPH, + every: 60 * 60 * 1000 + } + } + })); + }; From 7a24d3bcd85ee86b71e8d1e8965f4531d4966b0a Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 15:56:45 -0400 Subject: [PATCH 11/21] improve logging on HistoricSync --- lib/HistoricSync.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 6c0e6a60..636bb11f 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -427,7 +427,8 @@ HistoricSync.prototype.start = function(opts, next) { else { self.endTs = Date.now(); self.status = 'finished'; - console.log('Done Syncing', self.info()); + var info = self.info(); + logger.debug('Done Syncing blockchain', info.type, 'to height', info.height); return w_cb(err); } }); From 8de655eb2b2072899cc17ba1100841964819cb50 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 15:58:21 -0400 Subject: [PATCH 12/21] improve logging on Sync --- lib/Sync.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/Sync.js b/lib/Sync.js index 60f160fb..36b0fa7d 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -86,10 +86,10 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { var self = this; if ( self.storingBlock ) { - console.log('## Storing a block already. Delaying storeTipBlock with:' + + logger.debug('Storing a block already. Delaying storeTipBlock with:' + b.hash); return setTimeout( function() { - console.log('## Retrying storeTipBlock with: ' + b.hash); + logger.debug('Retrying storeTipBlock with: ' + b.hash); self.storeTipBlock(b,allowReorgs,cb); }, 1000); } @@ -123,7 +123,7 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { oldHeight = hash ? (h || 0) : -1 if (oldTip && newPrev !== oldTip) { needReorg = true; - console.log('## REORG Triggered, tip mismatch'); + logger.debug('REORG Triggered, tip mismatch'); } return c(); }); @@ -195,7 +195,7 @@ Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, oldHeight, cb) if (height<0) return c(); newHeight = height + 1; - info('# Reorg Case 1) OldNext: %s NewHeight: %d', oldNext, newHeight); + info('Reorg Case 1) OldNext: %s NewHeight: %d', oldNext, newHeight); orphanizeFrom = oldNext; return c(err); }); @@ -203,7 +203,7 @@ Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, oldHeight, cb) function(c) { if (orphanizeFrom) return c(); - info('# Reorg Case 2)'); + info('Reorg Case 2)'); self.setBranchConnectedBackwards(newPrev, function(err, yHash, newYHashNext, height) { if (err) return c(err); newHeight = height; From 38f160c213d546fee8cdbcfc135967609037fdce Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 17:42:21 -0400 Subject: [PATCH 13/21] making rate limiter work --- config/express.js | 1 - insight.js | 4 +--- package.json | 3 +-- plugins/ratelimiter.js | 17 ++++++++++------- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/config/express.js b/config/express.js index 2e70ade4..af916819 100644 --- a/config/express.js +++ b/config/express.js @@ -28,7 +28,6 @@ module.exports = function(app, historicSync, peerSync) { //Enable jsonp app.enable('jsonp callback'); - app.use(config.apiPrefix + '/sync', setHistoric); app.use(config.apiPrefix + '/peer', setPeer); app.use(express.logger('dev')); diff --git a/insight.js b/insight.js index 8fa23981..4735545b 100755 --- a/insight.js +++ b/insight.js @@ -9,7 +9,6 @@ var PeerSync = require('./lib/PeerSync'); var HistoricSync = require('./lib/HistoricSync'); var express = require('express'); -var connect = require('connect'); var program = require('commander'); var config = require('./config/config'); @@ -111,8 +110,6 @@ if (!config.disableHistoricSync) { if (peerSync) peerSync.allowReorgs = true; -// routes -require('./config/routes')(expressApp); // socket.io var server = require('http').createServer(expressApp); @@ -130,6 +127,7 @@ if (config.enableRatelimiter) { // express settings +require('./config/routes')(expressApp); require('./config/express')(expressApp, historicSync, peerSync); diff --git a/package.json b/package.json index f006af6e..27cb0adf 100644 --- a/package.json +++ b/package.json @@ -58,8 +58,7 @@ "bufferput": "git://github.com/bitpay/node-bufferput.git", "buffertools": "*", "commander": "^2.3.0", - "connect": "^2.25.7", - "connect-ratelimit": "0.0.6", + "connect-ratelimit": "git://github.com/dharmafly/connect-ratelimit.git#0550eff209c54f35078f46445000797fa942ab97", "express": "~3.4.7", "glob": "*", "leveldown": "*", diff --git a/plugins/ratelimiter.js b/plugins/ratelimiter.js index 73748345..31150356 100644 --- a/plugins/ratelimiter.js +++ b/plugins/ratelimiter.js @@ -2,6 +2,7 @@ var logger = require('../lib/logger').logger; var preconditions = require('preconditions').singleton(); var limiter = require('connect-ratelimit'); +var ONE_HOUR = 60 * 60 * 1000; module.exports.init = function(app, config) { preconditions.checkArgument(app); @@ -9,24 +10,26 @@ module.exports.init = function(app, config) { config = config || {}; config.whitelistRPH = config.whitelistRPH || 5000; - config.normalRPH = config.normalRPH || 1; + config.normalRPH = config.normalRPH || 2; + config.blacklistRPH = config.blacklistRPH || 0; + console.log(config); - console.log('asdasdasd'); app.use(limiter({ whitelist: [], - blacklist: ['localhost'], // 'example.com' + end: true, + blacklist: [], // 'example.com' categories: { whitelist: { totalRequests: config.whitelistRPH, - every: 60 * 60 * 1000 + every: ONE_HOUR }, blacklist: { - totalRequests: 0, - every: 0 + totalRequests: config.blacklistRPH, + every: ONE_HOUR }, normal: { totalRequests: config.normalRPH, - every: 60 * 60 * 1000 + every: ONE_HOUR } } })); From 6e9edca86d6074c8cbe1d00deae5c4df60f291ba Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 17:48:23 -0400 Subject: [PATCH 14/21] making rate limiter work 2 --- insight.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/insight.js b/insight.js index 4735545b..bb7c05b8 100755 --- a/insight.js +++ b/insight.js @@ -117,13 +117,14 @@ var ios = require('socket.io')(server, config); require('./app/controllers/socket.js').init(ios); // plugins +if (config.enableRatelimiter) { + require('./plugins/ratelimiter').init(expressApp, config.ratelimiter); +} + if (config.enableMailbox) { require('./plugins/mailbox').init(ios, config.mailbox); } -if (config.enableRatelimiter) { - require('./plugins/ratelimiter').init(expressApp, config.ratelimiter); -} // express settings From 5465f321113de3cf1b1f0e12d71bee6d7393031f Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 20 Aug 2014 18:04:58 -0400 Subject: [PATCH 15/21] making rate limiter work 3 --- app/controllers/socket.js | 8 ++++++-- plugins/mailbox.js | 8 ++------ plugins/ratelimiter.js | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/app/controllers/socket.js b/app/controllers/socket.js index 59f4a186..8c573b5b 100644 --- a/app/controllers/socket.js +++ b/app/controllers/socket.js @@ -10,13 +10,18 @@ module.exports.init = function(io_ext) { if (ios) { // when a new socket connects ios.sockets.on('connection', function(socket) { - logger.debug('New connection from ' + socket.id); + logger.verbose('New connection from ' + socket.id); // when it subscribes, make it join the according room socket.on('subscribe', function(topic) { logger.debug('subscribe to ' + topic); socket.join(topic); }); + // disconnect handler + socket.on('disconnect', function() { + logger.verbose('disconnected ' + socket.id); + }); + }); } return ios; @@ -65,4 +70,3 @@ module.exports.broadcastSyncInfo = function(historicSync) { if (ios) ios.sockets.in('sync').emit('status', historicSync); }; - diff --git a/plugins/mailbox.js b/plugins/mailbox.js index 74d62f52..a32da98c 100644 --- a/plugins/mailbox.js +++ b/plugins/mailbox.js @@ -11,7 +11,7 @@ module.exports.init = function(ext_io, config) { io.sockets.on('connection', function(socket) { // when it requests sync, send him all pending messages socket.on('sync', function(ts) { - logger.debug('Sync requested by ' + socket.id); + logger.verbose('Sync requested by ' + socket.id); logger.debug(' from timestamp ' + ts); var rooms = socket.rooms; if (rooms.length !== 2) { @@ -25,7 +25,7 @@ module.exports.init = function(ext_io, config) { if (err) { throw new Error('Couldn\'t get messages on sync request: ' + err); } - logger.debug('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); + logger.verbose('\tFound ' + messages.length + ' message' + (messages.length !== 1 ? 's' : '')); for (var i = 0; i < messages.length; i++) { broadcastMessage(messages[i], socket); } @@ -42,10 +42,6 @@ module.exports.init = function(ext_io, config) { }); }); - // disconnect handler - socket.on('disconnect', function() { - logger.debug('disconnected ' + socket.id); - }); }); mdb.on('message', broadcastMessage); diff --git a/plugins/ratelimiter.js b/plugins/ratelimiter.js index 31150356..6f634039 100644 --- a/plugins/ratelimiter.js +++ b/plugins/ratelimiter.js @@ -10,9 +10,8 @@ module.exports.init = function(app, config) { config = config || {}; config.whitelistRPH = config.whitelistRPH || 5000; - config.normalRPH = config.normalRPH || 2; + config.normalRPH = config.normalRPH || 500; config.blacklistRPH = config.blacklistRPH || 0; - console.log(config); app.use(limiter({ whitelist: [], From e3de0efdd2352a6c5d2fcdd0672ff2f51bb3df3b Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 21 Aug 2014 13:45:32 -0400 Subject: [PATCH 16/21] add https support --- config/config.js | 2 ++ config/express.js | 1 + insight.js | 14 +++++++++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/config/config.js b/config/config.js index 8e1fbbeb..34967233 100644 --- a/config/config.js +++ b/config/config.js @@ -79,6 +79,7 @@ var bitcoindConf = { var enableMailbox = process.env.ENABLE_MAILBOX === 'true'; var enableRatelimiter = process.env.ENABLE_RATELIMITER === 'true'; var loggerLevel = process.env.LOGGER_LEVEL || 'info'; +var enableHTTPS = process.env.ENABLE_HTTPS === 'true'; if (!fs.existsSync(db)) { var err = fs.mkdirSync(db); @@ -94,6 +95,7 @@ module.exports = { enableMailbox: enableMailbox, enableRatelimiter: enableRatelimiter, loggerLevel: loggerLevel, + enableHTTPS: enableHTTPS, version: version, root: rootPath, publicPath: process.env.INSIGHT_PUBLIC_PATH || false, diff --git a/config/express.js b/config/express.js index af916819..1d6fb62f 100644 --- a/config/express.js +++ b/config/express.js @@ -37,6 +37,7 @@ module.exports = function(app, historicSync, peerSync) { app.use(express.compress()); app.use(function(req, res, next) { + console.log('asdasdasdasd'); res.setHeader('Access-Control-Allow-Origin', '*'); res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE'); res.setHeader('Access-Control-Allow-Headers', 'X-Requested-With,content-type'); diff --git a/insight.js b/insight.js index bb7c05b8..77dbf934 100755 --- a/insight.js +++ b/insight.js @@ -8,6 +8,8 @@ var fs = require('fs'); var PeerSync = require('./lib/PeerSync'); var HistoricSync = require('./lib/HistoricSync'); +var http = require('http'); +var https = require('https'); var express = require('express'); var program = require('commander'); @@ -60,6 +62,17 @@ program.parse(process.argv); // create express app var expressApp = express(); + +// setup http/https base server +var protocol = config.enableHTTPS ? https : http; +var serverOpts = {}; +if (config.enableHTTPS) { + serverOpts.key = fs.readFileSync('./etc/test-key.pem'); + serverOpts.cert = fs.readFileSync('./etc/test-cert.pem'); +} +var server = protocol.createServer(serverOpts, expressApp); +console.log(config.enableHTTPS); + // Bootstrap models var models_path = __dirname + '/app/models'; var walk = function(path) { @@ -112,7 +125,6 @@ if (peerSync) peerSync.allowReorgs = true; // socket.io -var server = require('http').createServer(expressApp); var ios = require('socket.io')(server, config); require('./app/controllers/socket.js').init(ios); From 506a267ed0a853a64dcfab340036381c05d18863 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 21 Aug 2014 13:47:51 -0400 Subject: [PATCH 17/21] add https support 2 --- README.md | 1 + insight.js | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d31a9b92..78fae9a1 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ INSIGHT_IGNORE_CACHE # True to ignore cache of spents in transaction, with more ENABLE_MAILBOX # if "true" will enable mailbox plugin ENABLE_RATELIMITER # if "true" will enable the ratelimiter plugin LOGGER_LEVEL # defaults to 'info', can be 'debug','verbose','error', etc. +ENABLE_HTTPS # if "true" it will server using SSL/HTTPS ``` diff --git a/insight.js b/insight.js index 77dbf934..4806581a 100755 --- a/insight.js +++ b/insight.js @@ -64,14 +64,15 @@ var expressApp = express(); // setup http/https base server -var protocol = config.enableHTTPS ? https : http; -var serverOpts = {}; +var server; if (config.enableHTTPS) { + var serverOpts = {}; serverOpts.key = fs.readFileSync('./etc/test-key.pem'); serverOpts.cert = fs.readFileSync('./etc/test-cert.pem'); + server = https.createServer(serverOpts, expressApp); +} else { + server = http.createServer(expressApp); } -var server = protocol.createServer(serverOpts, expressApp); -console.log(config.enableHTTPS); // Bootstrap models var models_path = __dirname + '/app/models'; From 5d18b3038dc71379f2592f571dfaab6c97a1f99a Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 21 Aug 2014 13:48:18 -0400 Subject: [PATCH 18/21] add test key and cert --- etc/test-cert.pem | 14 ++++++++++++++ etc/test-key.pem | 15 +++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 etc/test-cert.pem create mode 100644 etc/test-key.pem diff --git a/etc/test-cert.pem b/etc/test-cert.pem new file mode 100644 index 00000000..4e95885d --- /dev/null +++ b/etc/test-cert.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICMjCCAZugAwIBAgIJAK9dKmjfxq+BMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAkFSMRMwEQYDVQQIDApTb21lLVN0YXRlMQ4wDAYDVQQKDAVDb3BheTAeFw0x +NDA4MjExNzQyMTBaFw0xNDA5MjAxNzQyMTBaMDIxCzAJBgNVBAYTAkFSMRMwEQYD +VQQIDApTb21lLVN0YXRlMQ4wDAYDVQQKDAVDb3BheTCBnzANBgkqhkiG9w0BAQEF +AAOBjQAwgYkCgYEA1BbMI6V06LKoBrcf5bJ8LH7EjwqbEacIOpiY7B+8W3sAM1bB +6hA2IlPvKL3qTdhMMKFZGZMYypmlAQTI1N+VNSwJHNjyepFbtkdNytSC8qw8bhak +yt4TByYEw1NMYx7I0OOdjh/DKsS+EOIgQDT9zSB+NgErKb0mKrginwgk5XkCAwEA +AaNQME4wHQYDVR0OBBYEFM0G1agUfY4zRNfxJ+0sHV3EsoGKMB8GA1UdIwQYMBaA +FM0G1agUfY4zRNfxJ+0sHV3EsoGKMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADgYEAOg7n1RCyB1BJ6TuF99i25H7kpGUSL57ajNyyCKDciTPmpxVJ5knAjPYa +hbXX+dlq2B8QEnfkE5FMDLkO3RS3xU8YfekIDHofDuXR9boD/4rRlsN8md2Jmkr6 +MyRtYPtsPWVeoz0WmG5f1yobHmh7mYf17oN+uRJKX68s8G6b/SQ= +-----END CERTIFICATE----- diff --git a/etc/test-key.pem b/etc/test-key.pem new file mode 100644 index 00000000..7bb97c26 --- /dev/null +++ b/etc/test-key.pem @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDUFswjpXTosqgGtx/lsnwsfsSPCpsRpwg6mJjsH7xbewAzVsHq +EDYiU+8ovepN2EwwoVkZkxjKmaUBBMjU35U1LAkc2PJ6kVu2R03K1ILyrDxuFqTK +3hMHJgTDU0xjHsjQ452OH8MqxL4Q4iBANP3NIH42ASspvSYquCKfCCTleQIDAQAB +AoGAMUzDUx3o2RZ+XGFA9uHQX39wLVfnx+itzwEduvV9kT48Q7LNDJ2MF9qu4yeS +SVoYC83Vqk45Gw8v/dag4GrAgdk1NHZZ56Z/G55m06Y45xS6ZarBdbe0N1jdZEab +RG3FgxyPSUiZ5aLIMxMMtgt/DRv9BPpIeLNDMgyQRjVWlMkCQQDzlLwkp4bo+CAY +UMcsSN+KGurEMsuF0qc/+TLqpKDoOaLtd1F+Ntn20tQqeH0YLWktFvzAgY7wYXrb +lhMuAxa7AkEA3ucGEXNqwu1qVP4fXfEN1E0Y5X/euXMsfgNG8IK82hF3h83hnqNM +3FcGFOyKnL7E5TfRlJfxhAGqUfCe+2zjWwJBAKA6CID8CkyZW1NjX4EL9q+8AQ5K +c4J2DTqRzCJ5ZLcdosUeJecmYb5w9MtzMqaCyJq2clCXaNVK6iwjzj4IHh0CQQCY +sgwvIjCtrfQcmyUjtoExwUrf1LPfuK1u+ZG8KuNyQ2rtxjTb9qQtgRPye4QNEoZR +O+a/c0MImhdyIHLYa+RnAkEAwfLD4q+FDx4eX0ANO7/PI/XiJGqi6x1cYUwyRg9o +2S6hN5RnUD/nf2HKHU0esp34UMY/UWMrodCRDZj/ijg4UA== +-----END RSA PRIVATE KEY----- From ff9ccf5ca45ce3eb5f52a2aaf22009e79252ffb4 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 21 Aug 2014 16:08:14 -0400 Subject: [PATCH 19/21] change limits --- plugins/ratelimiter.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/ratelimiter.js b/plugins/ratelimiter.js index 6f634039..0da5e2dc 100644 --- a/plugins/ratelimiter.js +++ b/plugins/ratelimiter.js @@ -9,8 +9,8 @@ module.exports.init = function(app, config) { logger.info('Using ratelimiter plugin'); config = config || {}; - config.whitelistRPH = config.whitelistRPH || 5000; - config.normalRPH = config.normalRPH || 500; + config.whitelistRPH = config.whitelistRPH || 50000; + config.normalRPH = config.normalRPH || 1000; config.blacklistRPH = config.blacklistRPH || 0; app.use(limiter({ From 0e5e20c09ceb2678f13e15b94450247bffa88dac Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 21 Aug 2014 16:36:52 -0400 Subject: [PATCH 20/21] fixing insight for copay --- config/express.js | 8 -------- config/headers.js | 13 +++++++++++++ insight.js | 2 ++ 3 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 config/headers.js diff --git a/config/express.js b/config/express.js index 1d6fb62f..d61bf614 100644 --- a/config/express.js +++ b/config/express.js @@ -36,14 +36,6 @@ module.exports = function(app, historicSync, peerSync) { app.use(express.methodOverride()); app.use(express.compress()); - app.use(function(req, res, next) { - console.log('asdasdasdasd'); - res.setHeader('Access-Control-Allow-Origin', '*'); - res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE'); - res.setHeader('Access-Control-Allow-Headers', 'X-Requested-With,content-type'); - next(); - }); - if (config.publicPath) { var staticPath = path.normalize(config.rootPath + '/../' + config.publicPath); diff --git a/config/headers.js b/config/headers.js new file mode 100644 index 00000000..9614ceff --- /dev/null +++ b/config/headers.js @@ -0,0 +1,13 @@ +'use strict'; + +var logger = require('../lib/logger').logger; + +module.exports = function(app) { + + app.use(function(req, res, next) { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE'); + res.setHeader('Access-Control-Allow-Headers', 'X-Requested-With,content-type'); + next(); + }); +}; diff --git a/insight.js b/insight.js index 4806581a..d9ee63f8 100755 --- a/insight.js +++ b/insight.js @@ -62,6 +62,8 @@ program.parse(process.argv); // create express app var expressApp = express(); +// setup headers +require('./config/headers')(expressApp); // setup http/https base server var server; From 28aeac6f9b101c21f1069b9f1f669ccfbce58180 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 21 Aug 2014 17:34:15 -0400 Subject: [PATCH 21/21] fix logging and config --- config/express.js | 19 +++++-------------- insight.js | 11 +++-------- plugins/ratelimiter.js | 1 - 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/config/express.js b/config/express.js index d61bf614..20da68f7 100644 --- a/config/express.js +++ b/config/express.js @@ -3,9 +3,10 @@ /** * Module dependencies. */ -var express = require('express'), - config = require('./config'), - path = require('path'); +var express = require('express'); +var config = require('./config'); +var path = require('path'); +var logger = require('../lib/logger').logger; module.exports = function(app, historicSync, peerSync) { @@ -22,11 +23,8 @@ module.exports = function(app, historicSync, peerSync) { }; app.set('showStackError', true); - - // Compress JSON outputs app.set('json spaces', 0); - //Enable jsonp app.enable('jsonp callback'); app.use(config.apiPrefix + '/sync', setHistoric); app.use(config.apiPrefix + '/peer', setPeer); @@ -38,12 +36,10 @@ module.exports = function(app, historicSync, peerSync) { if (config.publicPath) { var staticPath = path.normalize(config.rootPath + '/../' + config.publicPath); - //IMPORTANT: for html5mode, this line must to be before app.router app.use(express.static(staticPath)); } - // manual helpers app.use(function(req, res, next) { app.locals.config = config; next(); @@ -52,15 +48,10 @@ module.exports = function(app, historicSync, peerSync) { //routes should be at the last app.use(app.router); - //Assume "not found" in the error msgs is a 404. this is somewhat silly, but valid, you can do whatever you like, set properties, use instanceof etc. + //Assume "not found" in the error msgs is a 404 app.use(function(err, req, res, next) { - //Treat as 404 if (~err.message.indexOf('not found')) return next(); - - //Log it console.error(err.stack); - - //Error page res.status(500).jsonp({ status: 500, error: err.stack diff --git a/insight.js b/insight.js index d9ee63f8..5fd90118 100755 --- a/insight.js +++ b/insight.js @@ -94,10 +94,7 @@ var walk = function(path) { walk(models_path); -/** - * p2pSync process - */ - +// p2pSync process var peerSync = new PeerSync({ shouldBroadcast: true }); @@ -106,9 +103,7 @@ if (!config.disableP2pSync) { peerSync.run(); } -/** - * historic_sync process - */ +// historic_sync process var historicSync = new HistoricSync({ shouldBroadcastSync: true }); @@ -143,8 +138,8 @@ if (config.enableMailbox) { // express settings -require('./config/routes')(expressApp); require('./config/express')(expressApp, historicSync, peerSync); +require('./config/routes')(expressApp); //Start the app by listening on diff --git a/plugins/ratelimiter.js b/plugins/ratelimiter.js index 0da5e2dc..6523673a 100644 --- a/plugins/ratelimiter.js +++ b/plugins/ratelimiter.js @@ -32,5 +32,4 @@ module.exports.init = function(app, config) { } } })); - };