From 544f916f4f9419534147215a2a39722181c4e479 Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Fri, 22 May 2015 17:24:52 -0300 Subject: [PATCH 1/3] refactor statistics to filter at the db level --- lib/stats.js | 262 +++++++++++++++++++++++-------------------------- lib/storage.js | 1 + 2 files changed, 126 insertions(+), 137 deletions(-) diff --git a/lib/stats.js b/lib/stats.js index 2ab4a6d..da13bac 100644 --- a/lib/stats.js +++ b/lib/stats.js @@ -3,155 +3,143 @@ 'use strict'; var _ = require('lodash'); +var $ = require('preconditions').singleton(); +var async = require('async'); +var log = require('npmlog'); +log.debug = log.verbose; +log.disableColor(); var mongodb = require('mongodb'); var moment = require('moment'); -var async = require('async'); + var config = require('../config'); +var storage = require('./storage'); +function Stats(opts) { + opts = opts || {}; -var c = config.storageOpts.mongoDb; -var url = c.uri; -var startDate = moment(); -var endDate = moment(); - -var stats = {}; -var wallets = {}; -var bwsStats = {}; - -bwsStats.cleanUp = function() { - stats = { - 'livenet': {}, - 'testnet': {} - }; + this.network = opts.network || 'livenet'; + this.from = moment(opts.from || '2015-01-01'); + this.to = moment(opts.to); + this.fromTs = Math.floor(this.from.startOf('day').valueOf() / 1000); + this.toTs = Math.floor(this.to.endOf('day').valueOf() / 1000); }; +Stats.prototype.run = function(cb) { + var self = this; -bwsStats.AddingWalletToCache = function(data) { - if (!data) return; - wallets[data.id] = data.network; -}; - -bwsStats.TotalNewWallets = function(data) { - if (!data) return; - var day = moment(data.createdOn * 1000).format('YYYYMMDD'); - if (!stats[data.network][day]) { - stats[data.network][day] = { - totalTx: 0, - totalAmount: 0, - totalNewWallets: 0 - }; - } - stats[data.network][day].totalNewWallets++; -}; - -bwsStats.TotalTxps = function(data) { - if (!data) return; - var day = moment(data.createdOn * 1000).format('YYYYMMDD'); - var network = wallets[data.walletId]; - if (!stats[network][day]) { - stats[network][day] = { - totalTx: 0, - totalAmount: 0, - totalNewWallets: 0 - }; - } - stats[network][day].totalTx++; - stats[network][day].totalAmount += data.amount; -}; - -bwsStats.ProcessData = function(DB, cb) { - bwsStats.ProccesWallets(DB, function() { - bwsStats.ProccesNewWallets(DB, function() { - bwsStats.ProccesTxs(DB, function() { - DB.close(); - cb(); - }); - }); - }); -}; - -bwsStats.ProccesWallets = function(DB, cb) { - var collection = DB.collection('wallets'); - collection.find({}).toArray(function(err, items) { + var uri = config.storageOpts.mongoDb.uri; + mongodb.MongoClient.connect(uri, function(err, db) { if (err) { - console.log('Error.', err); - return cb(err); - } - - items.forEach(function(it) { - bwsStats.AddingWalletToCache(it); - }); - cb(); - }); -}; - -bwsStats.ProccesNewWallets = function(DB, cb) { - var collection = DB.collection('wallets'); - var start = Math.floor(startDate.startOf('day').valueOf() / 1000); - var end = Math.floor(endDate.endOf('day').valueOf() / 1000); - - collection.find({ - createdOn: { - $gt: start, - $lt: end - } - }).toArray(function(err, items) { - if (err) { - console.log('Error.', err); - return cb(err); - } - items.forEach(function(it) { - bwsStats.TotalNewWallets(it); - }); - cb(); - }); -}; - -bwsStats.ProccesTxs = function(DB, cb) { - var collection = DB.collection('txs'); - var start = Math.floor(startDate.startOf('day').valueOf() / 1000); - var end = Math.floor(endDate.endOf('day').valueOf() / 1000); - - collection.find({ - createdOn: { - $gt: start, - $lt: end - } - }).toArray( - function(err, items) { - if (err) { - console.log('Error.', err); - return cb(err); - } else { - items.forEach(function(it) { - bwsStats.TotalTxps(it); - }); - } - cb(); - }); -}; - -bwsStats.getStats = function(opts, cb) { - if (opts) { - startDate = moment(opts.from); - endDate = moment(opts.to); - } - bwsStats.cleanUp(); - - mongodb.MongoClient.connect(url, function(err, db) { - if (err) { - console.log('Unable to connect to the mongoDB server. Error:', err); + log.error('Unable to connect to the mongoDB', err); return cb(err, null); } - console.log('Connection established to ', url); - bwsStats.ProcessData(db, function(err) { - if (err) { - console.log('Error.', err); - return cb(err, null); - } - cb(null, stats); + log.info('Connection established to ' + uri); + self.db = db; + self._getStats(function(err, stats) { + if (err) return cb(err); + return cb(null, stats); }); }); }; -module.exports = bwsStats; +Stats.prototype._getStats = function(cb) { + var self = this; + var result = {}; + async.parallel([ + + function(next) { + self._getNewWallets(next); + }, + function(next) { + self._getTxProposals(next); + }, + ], function(err, results) { + if (err) return cb(err); + + result.newWallets = results[0]; + return cb(null, result); + }); +}; + +Stats.prototype._countBy = function(data, key) { + return _.map(_.groupBy(data, key), function(v, k) { + var item = {}; + item[key] = k; + item['count'] = v.length; + return item; + }); +}; + +Stats.prototype._sumBy = function(data, key, attr) { + return _.map(_.groupBy(data, key), function(v, k) { + var item = {}; + item[key] = k; + item[attr] = _.reduce(v, function(memo, x) { + return memo + x[attr]; + }, 0); + return item; + }); +}; + +Stats.prototype._getNewWallets = function(cb) { + var self = this; + + self.db.collection(storage.collections.WALLETS) + .find({ + network: self.network, + createdOn: { + $gte: self.fromTs, + $lte: self.toTs, + }, + }) + .toArray(function(err, wallets) { + if (err) return cb(err); + + var data = _.map(wallets, function(wallet) { + return { + day: moment(wallet.createdOn * 1000).format('YYYYMMDD'), + type: wallet.m + '-of-' + wallet.n, + }; + }); + + var stats = { + byDay: self._countBy(data, 'day'), + byType: self._countBy(data, 'type'), + }; + + return cb(null, stats); + }); +}; + +Stats.prototype._getTxProposals = function(cb) { + var self = this; + + self.db.collection(storage.collections.TXS) + .find({ + network: self.network, + status: 'broadcasted', + createdOn: { + $gte: self.fromTs, + $lte: self.toTs, + }, + }) + .toArray(function(err, txps) { + if (err) return cb(err); + + var data = _.map(txps, function(txp) { + return { + day: moment(txp.createdOn * 1000).format('YYYYMMDD'), + amount: txp.amount, + }; + }); + + var stats = { + nbByDay: self._countBy(data, 'day'), + amountByDay: self._sumBy(data, 'day', 'amount'), + }; + + return cb(null, stats); + }); +}; + +module.exports = Stats; diff --git a/lib/storage.js b/lib/storage.js index 15a512a..ead5b29 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -426,4 +426,5 @@ Storage.prototype._dump = function(cb, fn) { }); }; +Storage.collections = collections; module.exports = Storage; From 7e4a5f966a9b4c8e8ac5b8993c6b277ec931fd7e Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Tue, 26 May 2015 14:59:55 -0300 Subject: [PATCH 2/3] refactor REST endpoint --- lib/expressapp.js | 14 ++++++++------ lib/stats.js | 11 +++++++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/lib/expressapp.js b/lib/expressapp.js index 01a1c6b..cc6bcc2 100644 --- a/lib/expressapp.js +++ b/lib/expressapp.js @@ -9,7 +9,7 @@ var querystring = require('querystring'); var bodyParser = require('body-parser') var WalletService = require('./server'); -var stats = require('./stats'); +var Stats = require('./stats'); log.disableColor(); log.debug = log.verbose; @@ -340,18 +340,20 @@ ExpressApp.prototype.start = function(opts, cb) { }); }); - router.get('/v1/stats/:from/:to/', function(req, res) { + router.get('/v1/stats/', function(req, res) { var opts = {}; - opts.from = req.params['from']; - opts.to = req.params['to']; - stats.getStats(opts, function(err, data) { + if (req.query.network) opts.network = req.query.network; + if (req.query.from) opts.from = req.query.from; + if (req.query.to) opts.to = req.query.to; + + var stats = new Stats(opts); + stats.run(function(err, data) { if (err) return returnError(err, res, req); res.json(data); res.end(); }); }); - this.app.use(opts.basePath || '/bws/api', router); WalletService.initialize(opts, cb); diff --git a/lib/stats.js b/lib/stats.js index da13bac..f96063b 100644 --- a/lib/stats.js +++ b/lib/stats.js @@ -57,6 +57,7 @@ Stats.prototype._getStats = function(cb) { if (err) return cb(err); result.newWallets = results[0]; + result.txProposals = results[1]; return cb(null, result); }); }; @@ -98,15 +99,21 @@ Stats.prototype._getNewWallets = function(cb) { var data = _.map(wallets, function(wallet) { return { day: moment(wallet.createdOn * 1000).format('YYYYMMDD'), - type: wallet.m + '-of-' + wallet.n, + type: (wallet.m == 1 && wallet.n == 1) ? 'personal' : 'shared', + config: wallet.m + '-of-' + wallet.n, }; }); var stats = { byDay: self._countBy(data, 'day'), - byType: self._countBy(data, 'type'), + byConfig: self._countBy(data, 'config'), }; + stats.byTypeThenDay = _.groupBy(data, 'type'); + _.each(stats.byTypeThenDay, function(v, k) { + stats.byTypeThenDay[k] = self._countBy(v, 'day'); + }); + return cb(null, stats); }); }; From b23eca316727f7c95712452f662a5a14adf29e7e Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Tue, 26 May 2015 16:36:44 -0300 Subject: [PATCH 3/3] v0.0.32 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index efab764..b17a8bd 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "bitcore-wallet-service", "description": "A service for Mutisig HD Bitcoin Wallets", "author": "BitPay Inc", - "version": "0.0.31", + "version": "0.0.32", "keywords": [ "bitcoin", "copay",