added currency rates plugin + integration tests

config/config.js

@@ -88,6 +88,7 @@ var enableEmailstore = process.env.ENABLE_EMAILSTORE === 'true';
 var enablePublicInfo = process.env.ENABLE_PUBLICINFO === 'true';
 var loggerLevel = process.env.LOGGER_LEVEL || 'info';
 var enableHTTPS = process.env.ENABLE_HTTPS === 'true';
+var enableCurrencyRates = process.env.ENABLE_CURRENCYRATES === 'true';
 
 if (!fs.existsSync(db)) {
   mkdirp.sync(db);
@@ -106,6 +107,8 @@ module.exports = {
   credentialstore: require('../plugins/config-credentialstore'),
   enableEmailstore: enableEmailstore,
   emailstore: require('../plugins/config-emailstore'),
+  enableCurrencyRates: enableCurrencyRates,
+  currencyrates: require('../plugins/config-currencyrates'),
   enablePublicInfo: enablePublicInfo,
   publicInfo: require('../plugins/publicInfo/config'),
   loggerLevel: loggerLevel,

config/routes.js

@@ -70,6 +70,12 @@ module.exports = function(app) {
     app.post(apiPrefix + '/email/delete/item/:key', emailPlugin.erase);
   }
 
+  // Currency rates plugin
+  if (config.enableCurrencyRates) {
+    var currencyRatesPlugin = require('../plugins/currencyrates');
+    app.get(apiPrefix + '/rates/:code', currencyRatesPlugin.getRate);
+  }
+
   // Address routes
   var messages = require('../app/controllers/messages');
   app.get(apiPrefix + '/messages/verify', messages.verify);

insight.js

@@ -147,6 +147,10 @@ if (config.enableEmailstore) {
   require('./plugins/emailstore').init(config.emailstore);
 }
 
+if (config.enableCurrencyRates) {
+  require('./plugins/currencyrates').init(config.currencyrates);
+}
+
 // express settings
 require('./config/express')(expressApp, historicSync, peerSync);
 require('./config/routes')(expressApp);

package.json

@@ -27,6 +27,10 @@
     {
       "name": "Juan Ignacio Sosa Lopez",
       "email": "bechilandia@gmail.com"
+    },
+    {
+      "name": "Ivan Socolsky",
+      "email": "jungans@gmail.com"
     }
   ],
   "bugs": {
@@ -71,6 +75,7 @@
     "moment": "~2.5.0",
     "nodemailer": "^1.3.0",
     "preconditions": "^1.0.7",
+    "request": "^2.48.0",
     "socket.io": "1.0.6",
     "socket.io-client": "1.0.6",
     "soop": "=0.1.5",

plugins/config-currencyrates.js

@@ -0,0 +1,4 @@
+module.exports = {
+	fetchIntervalInMinutes: 60,
+	defaultSource: 'BitPay',
+};

plugins/currencyRates/bitpay.js

@@ -0,0 +1,15 @@
+var _ = require('lodash');
+
+module.exports.id = 'BitPay';
+module.exports.url = 'https://bitpay.com/api/rates/';
+
+module.exports.parseFn = function(raw) {
+  var rates = _.compact(_.map(raw, function(d) {
+    if (!d.code || !d.rate) return null;
+    return {
+      code: d.code,
+      rate: d.rate,
+    };
+  }));
+  return rates;
+};

plugins/currencyRates/bitstamp.js

@@ -0,0 +1,11 @@
+var _ = require('lodash');
+
+module.exports.id = 'Bitstamp';
+module.exports.url = 'https://www.bitstamp.net/api/ticker/';
+
+module.exports.parseFn = function(raw) {
+  return [{
+    code: 'USD',
+    rate: parseFloat(raw.last)
+  }];
+};

plugins/currencyrates.js

@@ -0,0 +1,198 @@
+(function() {
+  'use strict';
+
+  var _ = require('lodash');
+  var async = require('async');
+  var levelup = require('levelup');
+  var request = require('request');
+  var preconditions = require('preconditions').singleton();
+
+  var logger = require('../lib/logger').logger;
+  var globalConfig = require('../config/config');
+
+  var currencyRatesPlugin = {};
+
+  function getCurrentTs() {
+    return Math.floor(new Date() / 1000);
+  };
+
+  function getKey(sourceId, code, ts) {
+    var key = sourceId + '-' + code.toUpperCase();
+    if (ts) {
+      key += '-' + ts;
+    }
+    return key;
+  };
+
+  function returnError(error, res) {
+    res.status(error.code).json({
+      error: error.message,
+    }).end();
+  };
+
+  currencyRatesPlugin.init = function(config) {
+    logger.info('Using currencyrates plugin');
+    
+    config = config || {};
+
+    var path = globalConfig.leveldb + '/currencyRates' + (globalConfig.name ? ('-' + globalConfig.name) : '');
+    currencyRatesPlugin.db = config.db || globalConfig.db || levelup(path);
+    
+    if (_.isArray(config.sources)) {
+      currencyRatesPlugin.sources = config.sources;
+    } else {
+      currencyRatesPlugin.sources = [
+        require('./currencyRates/bitpay'),
+        require('./currencyRates/bitstamp'),
+      ];
+    }
+    currencyRatesPlugin.request = config.request || request;
+    currencyRatesPlugin.defaultSource = config.defaultSource || globalConfig.defaultSource;
+
+    var interval = config.fetchIntervalInMinutes || globalConfig.fetchIntervalInMinutes;
+    if (interval) {
+      currencyRatesPlugin._fetch();
+      setInterval(function() {
+        currencyRatesPlugin._fetch();
+      }, interval * 60 * 1000);
+    }
+    currencyRatesPlugin.initialized = true;
+  };
+
+  currencyRatesPlugin._retrieve = function(source, cb) {
+    logger.debug('Fetching data for ' + source.id);
+    currencyRatesPlugin.request.get({
+      url: source.url,
+      json: true
+    }, function(err, res, body) {
+      if (err || !body) {
+        logger.warn('Error fetching data for ' + source.id, err);
+        return cb(err);
+      }
+
+      logger.debug('Data for ' + source.id + ' fetched successfully');
+
+      if (!source.parseFn) {
+        return cb('No parse function for source ' + source.id);
+      }
+      var rates = source.parseFn(body);
+
+      return cb(null, rates);
+    });
+  };
+
+  currencyRatesPlugin._store = function(source, rates, cb) {
+    logger.debug('Storing data for ' + source.id);
+    var ts = getCurrentTs();
+    var ops = _.map(rates, function(r) {
+      return {
+        type: 'put',
+        key: getKey(source.id, r.code, ts),
+        value: r.rate,
+      };
+    });
+
+    currencyRatesPlugin.db.batch(ops, function(err) {
+      if (err) {
+        logger.warn('Error storing data for ' + source.id, err);
+        return cb(err);
+      }
+      logger.debug('Data for ' + source.id + ' stored successfully');
+      return cb();
+    });
+  };
+
+  currencyRatesPlugin._dump = function(opts) {
+    var all = [];
+    currencyRatesPlugin.db.readStream(opts)
+      .on('data', console.log);
+  };
+
+  currencyRatesPlugin._fetch = function(cb) {
+    cb = cb || function() {};
+
+    preconditions.shouldNotBeFalsey(currencyRatesPlugin.initialized);
+
+    async.each(currencyRatesPlugin.sources, function(source, cb) {
+      currencyRatesPlugin._retrieve(source, function(err, res) {
+        if (err) {
+          logger.warn(err);
+          return cb();
+        }
+        currencyRatesPlugin._store(source, res, function(err, res) {
+          return cb();
+        });
+      });
+    }, function(err) {
+      return cb(err);
+    });
+  };
+
+  currencyRatesPlugin._getOneRate = function(sourceId, code, ts, cb) {
+    var result = null;
+
+    currencyRatesPlugin.db.createValueStream({
+      lte: getKey(sourceId, code, ts),
+      gte: getKey(sourceId, code) + '!',
+      reverse: true,
+      limit: 1,
+    })
+      .on('data', function(data) {
+        var num = parseFloat(data);
+        result = _.isNumber(num) && !_.isNaN(num) ? num : null;
+      })
+      .on('error', function(err) {
+        return cb(err);
+      })
+      .on('end', function() {
+        return cb(null, result);
+      });
+  };
+
+  currencyRatesPlugin._getRate = function(sourceId, code, ts, cb) {
+    preconditions.shouldNotBeFalsey(currencyRatesPlugin.initialized);
+    preconditions.shouldNotBeEmpty(code);
+    preconditions.shouldBeFunction(cb);
+
+    ts = ts || getCurrentTs();
+
+    if (!_.isArray(ts)) {
+      return currencyRatesPlugin._getOneRate(sourceId, code, ts, function(err, rate) {
+        if (err) return cb(err);
+        return cb(null, {
+          rate: rate
+        });
+      });
+    }
+
+    async.map(ts, function(ts, cb) {
+      currencyRatesPlugin._getOneRate(sourceId, code, ts, function(err, rate) {
+        if (err) return cb(err);
+        return cb(null, {
+          ts: parseInt(ts),
+          rate: rate
+        });
+      });
+    }, function(err, res) {
+      if (err) return cb(err);
+      return cb(null, res);
+    });
+  };
+
+  currencyRatesPlugin.getRate = function(req, res) {
+    var source = req.param('source') || currencyRatesPlugin.defaultSource;
+    var ts = req.param('ts');
+    if (_.isString(ts) && ts.indexOf(',') !== -1) {
+      ts = ts.split(',');
+    }
+    currencyRatesPlugin._getRate(source, req.param('code'), ts, function(err, result) {
+      if (err) returnError({
+        code: 500,
+        message: err,
+      });
+      res.json(result);
+    });
+  };
+
+  module.exports = currencyRatesPlugin;
+})();

test/test.CurrencyRates.js

@@ -0,0 +1,265 @@
+'use strict';
+
+var chai = require('chai');
+var assert = require('assert');
+var sinon = require('sinon');
+var should = chai.should;
+var expect = chai.expect;
+
+var levelup = require('levelup');
+var memdown = require('memdown');
+var logger = require('../lib/logger').logger;
+logger.transports.console.level = 'non';
+
+var rates = require('../plugins/currencyrates');
+
+var db;
+
+describe('Rates service', function() {
+  beforeEach(function() {
+    db = levelup(memdown);
+  });
+
+  describe('#getRate', function() {
+    beforeEach(function() {
+      rates.init({
+        db: db,
+      });
+    });
+    it('should get rate with exact ts', function(done) {
+      db.batch([{
+        type: 'put',
+        key: 'bitpay-USD-10',
+        value: 123.45
+      }, ]);
+      rates._getRate('bitpay', 'USD', 10, function(err, res) {
+        expect(err).to.not.exist;
+        res.rate.should.equal(123.45);
+        done();
+      });
+    });
+    it('should get rate with approximate ts', function(done) {
+      db.batch([{
+        type: 'put',
+        key: 'bitpay-USD-10',
+        value: 123.45,
+      }, {
+        type: 'put',
+        key: 'bitpay-USD-20',
+        value: 200.00,
+      }]);
+      rates._getRate('bitpay', 'USD', 25, function(err, res) {
+        res.rate.should.equal(200.00);
+        done();
+      });
+    });
+    it('should return null when no rate found', function(done) {
+      db.batch([{
+        type: 'put',
+        key: 'bitpay-USD-20',
+        value: 123.45,
+      }, {
+        type: 'put',
+        key: 'bitpay-USD-30',
+        value: 200.00,
+      }]);
+      rates._getRate('bitpay', 'USD', 10, function(err, res) {
+        expect(res.rate).to.be.null;
+        done();
+      });
+    });
+    it('should get rate from specified source', function(done) {
+      db.batch([{
+        type: 'put',
+        key: 'bitpay-USD-10',
+        value: 123.45,
+      }, {
+        type: 'put',
+        key: 'bitstamp-USD-10',
+        value: 200.00,
+      }]);
+      rates._getRate('bitpay', 'USD', 12, function(err, res) {
+        res.rate.should.equal(123.45);
+        done();
+      });
+    });
+    it('should get rate for specified currency', function(done) {
+      db.batch([{
+        type: 'put',
+        key: 'bitpay-USD-10',
+        value: 123.45,
+      }, {
+        type: 'put',
+        key: 'bitpay-EUR-10',
+        value: 200.00,
+      }]);
+      rates._getRate('bitpay', 'EUR', 12, function(err, res) {
+        res.rate.should.equal(200.00);
+        done();
+      });
+    });
+    it('should get multiple rates', function(done) {
+      db.batch([{
+        type: 'put',
+        key: 'bitpay-USD-10',
+        value: 100.00,
+      }, {
+        type: 'put',
+        key: 'bitpay-USD-20',
+        value: 200.00,
+      }, {
+        type: 'put',
+        key: 'bitstamp-USD-30',
+        value: 300.00,
+      }, {
+        type: 'put',
+        key: 'bitpay-USD-30',
+        value: 400.00,
+      }]);
+      rates._getRate('bitpay', 'USD', [10, 20, 35], function(err, res) {
+        expect(err).to.not.exist;
+        res.length.should.equal(3);
+        res[0].ts.should.equal(10);
+        res[1].ts.should.equal(20);
+        res[2].ts.should.equal(35);
+        res[0].rate.should.equal(100.00);
+        res[1].rate.should.equal(200.00);
+        res[2].rate.should.equal(400.00);
+        done();
+      });
+    });
+  });
+
+  describe('#fetch', function() {
+    it('should fetch from all sources', function(done) {
+      var sources = [];
+      sources.push({
+        id: 'id1',
+        url: 'http://dummy1',
+        parseFn: function(raw) {
+          return raw;
+        },
+      });
+      sources.push({
+        id: 'id2',
+        url: 'http://dummy2',
+        parseFn: function(raw) {
+          return raw;
+        },
+      });
+
+      var ds1 = [{
+        code: 'USD',
+        rate: 123.45,
+      }, {
+        code: 'EUR',
+        rate: 200.00,
+      }];
+      var ds2 = [{
+        code: 'USD',
+        rate: 126.39,
+      }];
+
+      var request = sinon.stub();
+      request.get = sinon.stub();
+      request.get.withArgs({
+        url: 'http://dummy1',
+        json: true
+      }).yields(null, null, ds1);
+      request.get.withArgs({
+        url: 'http://dummy2',
+        json: true
+      }).yields(null, null, ds2);
+
+      rates.init({
+        db: db,
+        sources: sources,
+        request: request,
+      });
+
+      var clock = sinon.useFakeTimers(1400000000 * 1000);
+
+      rates._fetch(function(err, res) {
+        clock.restore();
+
+        expect(err).to.not.exist;
+
+        var result = [];
+        db.readStream()
+          .on('data', function(data) {
+            result.push(data);
+          })
+          .on('close', function() {
+            result.length.should.equal(3);
+            result[0].key.should.equal('id1-EUR-1400000000');
+            result[1].key.should.equal('id1-USD-1400000000');
+            result[2].key.should.equal('id2-USD-1400000000');
+            parseFloat(result[0].value).should.equal(200.00);
+            parseFloat(result[1].value).should.equal(123.45);
+            parseFloat(result[2].value).should.equal(126.39);
+            done();
+          });
+      });
+    });
+
+    it('should not stop when failing to fetch source', function(done) {
+      var sources = [];
+      sources.push({
+        id: 'id1',
+        url: 'http://dummy1',
+        parseFn: function(raw) {
+          return raw;
+        },
+      });
+      sources.push({
+        id: 'id2',
+        url: 'http://dummy2',
+        parseFn: function(raw) {
+          return raw;
+        },
+      });
+
+      var ds2 = [{
+        code: 'USD',
+        rate: 126.39,
+      }];
+
+      var request = sinon.stub();
+      request.get = sinon.stub();
+      request.get.withArgs({
+        url: 'http://dummy1',
+        json: true
+      }).yields('dummy error', null, null);
+      request.get.withArgs({
+        url: 'http://dummy2',
+        json: true
+      }).yields(null, null, ds2);
+
+      rates.init({
+        db: db,
+        sources: sources,
+        request: request,
+      });
+
+      var clock = sinon.useFakeTimers(1400000000 * 1000);
+
+      rates._fetch(function(err, res) {
+        clock.restore();
+
+        expect(err).to.not.exist;
+
+        var result = [];
+        db.readStream()
+          .on('data', function(data) {
+            result.push(data);
+          })
+          .on('close', function() {
+            result.length.should.equal(1);
+            result[0].key.should.equal('id2-USD-1400000000');
+            parseFloat(result[0].value).should.equal(126.39);
+            done();
+          });
+      });
+    });
+  });
+});