Merge pull request #432 from isocolsky/feat/rate-service

Fiat exchange rate service

config.js

@@ -52,6 +52,10 @@ var config = {
     subjectPrefix: '',
     pushServerUrl: 'http://localhost:8000/send',
   },
+  fiatRateServiceOpts: {
+    defaultProvider: 'BitPay',
+    fetchInterval: 10, // in minutes
+  },
   // To use email notifications uncomment this:
   // emailOpts: {
   //  host: 'localhost',

fiatrateservice/fiatrateservice.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+'use strict';
+
+var config = require('../config');
+var FiatRateService = require('../lib/fiatrateservice');
+
+var service = new FiatRateService();
+service.init(config, function(err) {
+  if (err) throw err;
+  service.startCron(config, function(err) {
+    if (err) throw err;
+
+    console.log('Fiat rate service started');
+  });
+});

lib/common/defaults.js

@@ -40,4 +40,8 @@ Defaults.FEE_LEVELS = [{
 // Minimum nb of addresses a wallet must have to start using 2-step balance optimization
 Defaults.TWO_STEP_BALANCE_THRESHOLD = 100;
 
+Defaults.FIAT_RATE_PROVIDER = 'BitPay';
+Defaults.FIAT_RATE_FETCH_INTERVAL = 10; // In minutes
+Defaults.FIAT_RATE_MAX_LOOK_BACK_TIME = 120; // In minutes
+
 module.exports = Defaults;

lib/expressapp.js

@@ -5,7 +5,6 @@ var async = require('async');
 var log = require('npmlog');
 
 var express = require('express');
-var querystring = require('querystring');
 var bodyParser = require('body-parser')
 
 var WalletService = require('./server');
@@ -114,7 +113,7 @@ ExpressApp.prototype.start = function(opts, cb) {
     };
   };
 
-  function getServer(req, res, cb) {
+  function getServer(req, res) {
     var opts = {
       clientVersion: req.header('x-client-version'),
     };
@@ -501,6 +500,25 @@ ExpressApp.prototype.start = function(opts, cb) {
     });
   });
 
+  router.get('/v1/fiatrates/:code/', function(req, res) {
+    getServerWithAuth(req, res, function(server) {
+      var opts = {
+        code: req.params['code'],
+        source: req.query.source,
+        ts: +req.query.ts,
+      };
+      server.getFiatRate(opts, function(err, rates) {
+        if (err) returnError({
+          code: 500,
+          message: err,
+        });
+        res.json(rates);
+        res.end();
+      });
+    });
+  });
+
+
   this.app.use(opts.basePath || '/bws/api', router);
 
   WalletService.initialize(opts, cb);

lib/fiatrateproviders/bitpay.js

@@ -0,0 +1,18 @@
+var _ = require('lodash');
+
+var provider = {
+  name: 'BitPay',
+  url: 'https://bitpay.com/api/rates/',
+  parseFn: function(raw) {
+    var rates = _.compact(_.map(raw, function(d) {
+      if (!d.code || !d.rate) return null;
+      return {
+        code: d.code,
+        value: d.rate,
+      };
+    }));
+    return rates;
+  },
+};
+
+module.exports = provider;

lib/fiatrateproviders/bitstamp.js

@@ -0,0 +1,12 @@
+var provider = {
+  name: 'Bitstamp',
+  url: 'https://www.bitstamp.net/api/ticker/',
+  parseFn: function(raw) {
+    return [{
+      code: 'USD',
+      value: parseFloat(raw.last)
+    }];
+  }
+};
+
+module.exports = provider;

lib/fiatrateproviders/index.js

@@ -0,0 +1,6 @@
+var Providers = {
+  BitPay: require('./bitpay'),
+  Bitstamp: require('./bitstamp'),
+}
+
+module.exports = Providers;

lib/fiatrateservice.js

@@ -0,0 +1,138 @@
+'use strict';
+
+var _ = require('lodash');
+var $ = require('preconditions').singleton();
+var async = require('async');
+var log = require('npmlog');
+log.debug = log.verbose;
+var request = require('request');
+
+var Common = require('./common');
+var Defaults = Common.Defaults;
+
+var Storage = require('./storage');
+var Model = require('./model');
+
+function FiatRateService() {};
+
+FiatRateService.prototype.init = function(opts, cb) {
+  var self = this;
+
+  opts = opts || {};
+
+  self.request = opts.request || request;
+  self.defaultProvider = opts.defaultProvider || Defaults.FIAT_RATE_PROVIDER;
+
+  async.parallel([
+
+    function(done) {
+      if (opts.storage) {
+        self.storage = opts.storage;
+        done();
+      } else {
+        self.storage = new Storage();
+        self.storage.connect(opts.storageOpts, done);
+      }
+    },
+  ], function(err) {
+    if (err) {
+      log.error(err);
+    }
+    return cb(err);
+  });
+};
+
+FiatRateService.prototype.startCron = function(opts, cb) {
+  var self = this;
+
+  opts = opts || {};
+
+  self.providers = _.values(require('./fiatrateproviders'));
+
+  var interval = opts.fetchInterval || Defaults.FIAT_RATE_FETCH_INTERVAL;
+  if (interval) {
+    self._fetch();
+    setInterval(function() {
+      self._fetch();
+    }, interval * 60 * 1000);
+  }
+
+  return cb();
+};
+
+FiatRateService.prototype._fetch = function(cb) {
+  var self = this;
+
+  cb = cb || function() {};
+
+  async.each(self.providers, function(provider, next) {
+    self._retrieve(provider, function(err, res) {
+      if (err) {
+        log.warn('Error retrieving data for ' + provider.name, err);
+        return next();
+      }
+      self.storage.storeFiatRate(provider.name, res, function(err) {
+        if (err) {
+          log.warn('Error storing data for ' + provider.name, err);
+        }
+        return next();
+      });
+    });
+  }, cb);
+};
+
+FiatRateService.prototype._retrieve = function(provider, cb) {
+  var self = this;
+
+  log.debug('Fetching data for ' + provider.name);
+  self.request.get({
+    url: provider.url,
+    json: true,
+  }, function(err, res, body) {
+    if (err || !body) {
+      return cb(err);
+    }
+
+    log.debug('Data for ' + provider.name + ' fetched successfully');
+
+    if (!provider.parseFn) {
+      return cb(new Error('No parse function for provider ' + provider.name));
+    }
+    var rates = provider.parseFn(body);
+
+    return cb(null, rates);
+  });
+};
+
+
+FiatRateService.prototype.getRate = function(opts, cb) {
+  var self = this;
+
+  $.shouldBeFunction(cb);
+
+  opts = opts || {};
+
+  var now = Date.now();
+  var provider = opts.provider || self.defaultProvider;
+  var ts = (_.isNumber(opts.ts) || _.isArray(opts.ts)) ? opts.ts : now;
+
+  async.map([].concat(ts), function(ts, cb) {
+    self.storage.fetchFiatRate(provider, opts.code, ts, function(err, rate) {
+      if (err) return cb(err);
+      if (rate && (now - rate.ts) > Defaults.FIAT_RATE_MAX_LOOK_BACK_TIME * 60 * 1000) rate = null;
+
+      return cb(null, {
+        ts: +ts,
+        rate: rate ? rate.value : undefined,
+        fetchedOn: rate ? rate.ts : undefined,
+      });
+    });
+  }, function(err, res) {
+    if (err) return cb(err);
+    if (!_.isArray(ts)) res = res[0];
+    return cb(null, res);
+  });
+};
+
+
+module.exports = FiatRateService;

lib/server.js

@@ -22,6 +22,7 @@ var Lock = require('./lock');
 var Storage = require('./storage');
 var MessageBroker = require('./messagebroker');
 var BlockchainExplorer = require('./blockchainexplorer');
+var FiatRateService = require('./fiatrateservice');
 
 var Model = require('./model');
 var Wallet = Model.Wallet;
@@ -33,6 +34,7 @@ var storage;
 var blockchainExplorer;
 var blockchainExplorerOpts;
 var messageBroker;
+var fiatRateService;
 var serviceVersion;
 
 var HISTORY_LIMIT = 10;
@@ -50,6 +52,7 @@ function WalletService() {
   this.blockchainExplorer = blockchainExplorer;
   this.blockchainExplorerOpts = blockchainExplorerOpts;
   this.messageBroker = messageBroker;
+  this.fiatRateService = fiatRateService;
   this.notifyTicker = 0;
 };
 
@@ -101,6 +104,22 @@ WalletService.initialize = function(opts, cb) {
     return cb();
   };
 
+  function initFiatRateService(cb) {
+    if (opts.fiatRateService) {
+      fiatRateService = opts.fiatRateService;
+      return cb();
+    } else {
+      var newFiatRateService = new FiatRateService();
+      var opts2 = opts.fiatRateServiceOpts || {};
+      opts2.storage = storage;
+      newFiatRateService.init(opts2, function(err) {
+        if (err) return cb(err);
+        fiatRateService = newFiatRateService;
+        return cb();
+      });
+    }
+  };
+
   async.series([
 
     function(next) {
@@ -109,6 +128,9 @@ WalletService.initialize = function(opts, cb) {
     function(next) {
       initMessageBroker(next);
     },
+    function(next) {
+      initFiatRateService(next);
+    },
   ], function(err) {
     if (err) {
       log.error('Could not initialize', err);
@@ -2365,6 +2387,26 @@ WalletService.prototype.startScan = function(opts, cb) {
   });
 };
 
+/**
+ * Returns exchange rate for the specified currency & timestamp.
+ * @param {Object} opts
+ * @param {string} opts.code - Currency ISO code.
+ * @param {Date} [opts.ts] - A timestamp to base the rate on (default Date.now()).
+ * @param {String} [opts.provider] - A provider of exchange rates (default 'BitPay').
+ * @returns {Object} rates - The exchange rate.
+ */
+WalletService.prototype.getFiatRate = function(opts, cb) {
+  var self = this;
+
+  if (!Utils.checkRequired(opts, ['code']))
+    return cb(new ClientError('Required argument missing'));
+
+  self.fiatRateService.getRate(opts, function(err, rate) {
+    if (err) return cb(err);
+    return cb(null, rate);
+  });
+};
+
 
 module.exports = WalletService;
 module.exports.ClientError = ClientError;

lib/storage.js

@@ -21,6 +21,7 @@ var collections = {
   PREFERENCES: 'preferences',
   EMAIL_QUEUE: 'email_queue',
   CACHE: 'cache',
+  FIAT_RATES: 'fiat_rates',
 };
 
 var Storage = function(opts) {
@@ -572,6 +573,38 @@ Storage.prototype.fetchActiveAddresses = function(walletId, cb) {
   });
 };
 
+Storage.prototype.storeFiatRate = function(providerName, rates, cb) {
+  var self = this;
+
+  var now = Date.now();
+  async.each(rates, function(rate, next) {
+    self.db.collection(collections.FIAT_RATES).insert({
+      provider: providerName,
+      ts: now,
+      code: rate.code,
+      value: rate.value,
+    }, {
+      w: 1
+    }, next);
+  }, cb);
+};
+
+Storage.prototype.fetchFiatRate = function(providerName, code, ts, cb) {
+  var self = this;
+  self.db.collection(collections.FIAT_RATES).find({
+    provider: providerName,
+    code: code,
+    ts: {
+      $lte: ts
+    },
+  }).sort({
+    ts: -1
+  }).limit(1).toArray(function(err, result) {
+    if (err || _.isEmpty(result)) return cb(err);
+    return cb(null, result[0]);
+  });
+};
+
 Storage.prototype._dump = function(cb, fn) {
   fn = fn || console.log;
   cb = cb || function() {};

start.sh

@@ -34,5 +34,6 @@ run_program messagebroker/messagebroker.js pids/messagebroker.pid logs/messagebr
 run_program bcmonitor/bcmonitor.js pids/bcmonitor.pid logs/bcmonitor.log
 run_program emailservice/emailservice.js pids/emailservice.pid logs/emailservice.log
 run_program pushnotificationsservice/pushnotificationsservice.js pids/pushnotificationsservice.pid logs/pushnotificationsservice.log
+run_program fiatrateservice/fiatrateservice.js pids/fiatrateservice.pid logs/fiatrateservice.log
 run_program bws.js pids/bws.pid logs/bws.log
 

stop.sh

@@ -11,6 +11,7 @@ stop_program ()
 }
 
 stop_program pids/bws.pid
+stop_program pids/fiatrateservice.pid
 stop_program pids/emailservice.pid
 stop_program pids/bcmonitor.pid
 stop_program pids/pushnotificationsservice.pid

test/integration/fiatrateservice.js

@@ -0,0 +1,296 @@
+'use strict';
+
+var _ = require('lodash');
+var async = require('async');
+
+var chai = require('chai');
+var sinon = require('sinon');
+var should = chai.should();
+var log = require('npmlog');
+log.debug = log.verbose;
+log.level = 'info';
+
+var helpers = require('./helpers');
+
+var FiatRateService = require('../../lib/fiatrateservice');
+
+describe('Fiat rate service', function() {
+  var service, request;
+
+  before(function(done) {
+    helpers.before(done);
+  });
+  after(function(done) {
+    helpers.after(done);
+  });
+  beforeEach(function(done) {
+    helpers.beforeEach(function() {
+      service = new FiatRateService();
+      request = sinon.stub();
+      request.get = sinon.stub();
+      service.init({
+        storage: helpers.getStorage(),
+        request: request,
+      }, function(err) {
+        should.not.exist(err);
+        service.startCron({}, done);
+      });
+    });
+  });
+  describe('#getRate', function() {
+    it('should get current rate', function(done) {
+      service.storage.storeFiatRate('BitPay', [{
+        code: 'USD',
+        value: 123.45,
+      }], function(err) {
+        should.not.exist(err);
+        service.getRate({
+          code: 'USD'
+        }, function(err, res) {
+          should.not.exist(err);
+          res.rate.should.equal(123.45);
+          done();
+        });
+      });
+    });
+    it('should get current rate for different currency', function(done) {
+      service.storage.storeFiatRate('BitPay', [{
+        code: 'USD',
+        value: 123.45,
+      }], function(err) {
+        should.not.exist(err);
+        service.storage.storeFiatRate('BitPay', [{
+          code: 'EUR',
+          value: 345.67,
+        }], function(err) {
+          should.not.exist(err);
+          service.getRate({
+            code: 'EUR'
+          }, function(err, res) {
+            should.not.exist(err);
+            res.rate.should.equal(345.67);
+            done();
+          });
+        });
+      });
+    });
+
+    it('should get current rate for different provider', function(done) {
+      service.storage.storeFiatRate('BitPay', [{
+        code: 'USD',
+        value: 100.00,
+      }], function(err) {
+        should.not.exist(err);
+        service.storage.storeFiatRate('Bitstamp', [{
+          code: 'USD',
+          value: 200.00,
+        }], function(err) {
+          should.not.exist(err);
+          service.getRate({
+            code: 'USD'
+          }, function(err, res) {
+            should.not.exist(err);
+            res.rate.should.equal(100.00, 'Should use default provider');
+            service.getRate({
+              code: 'USD',
+              provider: 'Bitstamp',
+            }, function(err, res) {
+              should.not.exist(err);
+              res.rate.should.equal(200.00);
+              done();
+            });
+          });
+        });
+      });
+    });
+
+    it('should get rate for specific ts', function(done) {
+      var clock = sinon.useFakeTimers(0, 'Date');
+      clock.tick(20);
+      service.storage.storeFiatRate('BitPay', [{
+        code: 'USD',
+        value: 123.45,
+      }], function(err) {
+        should.not.exist(err);
+        clock.tick(100);
+        service.storage.storeFiatRate('BitPay', [{
+          code: 'USD',
+          value: 345.67,
+        }], function(err) {
+          should.not.exist(err);
+          service.getRate({
+            code: 'USD',
+            ts: 50,
+          }, function(err, res) {
+            should.not.exist(err);
+            res.ts.should.equal(50);
+            res.rate.should.equal(123.45);
+            res.fetchedOn.should.equal(20);
+            clock.restore();
+            done();
+          });
+        });
+      });
+    });
+
+    it('should get rates for a series of ts', function(done) {
+      var clock = sinon.useFakeTimers(0, 'Date');
+      async.each([1.00, 2.00, 3.00, 4.00], function(value, next) {
+        clock.tick(100);
+        service.storage.storeFiatRate('BitPay', [{
+          code: 'USD',
+          value: value,
+        }, {
+          code: 'EUR',
+          value: value,
+        }], next);
+      }, function(err) {
+        should.not.exist(err);
+        service.getRate({
+          code: 'USD',
+          ts: [50, 100, 199, 500],
+        }, function(err, res) {
+          should.not.exist(err);
+          res.length.should.equal(4);
+
+          res[0].ts.should.equal(50);
+          should.not.exist(res[0].rate);
+          should.not.exist(res[0].fetchedOn);
+
+          res[1].ts.should.equal(100);
+          res[1].rate.should.equal(1.00);
+          res[1].fetchedOn.should.equal(100);
+
+          res[2].ts.should.equal(199);
+          res[2].rate.should.equal(1.00);
+          res[2].fetchedOn.should.equal(100);
+
+          res[3].ts.should.equal(500);
+          res[3].rate.should.equal(4.00);
+          res[3].fetchedOn.should.equal(400);
+
+          clock.restore();
+          done();
+        });
+      });
+    });
+
+    it('should not get rate older than 2hs', function(done) {
+      var clock = sinon.useFakeTimers(0, 'Date');
+      service.storage.storeFiatRate('BitPay', [{
+        code: 'USD',
+        value: 123.45,
+      }], function(err) {
+        should.not.exist(err);
+        clock.tick((120 * 60 - 1) * 1000); // Almost 2 hours
+        service.getRate({
+          code: 'USD',
+        }, function(err, res) {
+          should.not.exist(err);
+          res.rate.should.equal(123.45);
+          res.fetchedOn.should.equal(0);
+          clock.restore();
+          clock.tick(2 * 1000); // 2 seconds later...
+          service.getRate({
+            code: 'USD',
+          }, function(err, res) {
+            should.not.exist(err);
+            should.not.exist(res.rate);
+            clock.restore();
+            done();
+          });
+        });
+      });
+    });
+
+  });
+
+  describe('#fetch', function() {
+    it('should fetch rates from all providers', function(done) {
+      var clock = sinon.useFakeTimers(100, 'Date');
+      var bitpay = [{
+        code: 'USD',
+        rate: 123.45,
+      }, {
+        code: 'EUR',
+        rate: 234.56,
+      }];
+      var bitstamp = {
+        last: 120.00,
+      };
+      request.get.withArgs({
+        url: 'https://bitpay.com/api/rates/',
+        json: true
+      }).yields(null, null, bitpay);
+      request.get.withArgs({
+        url: 'https://www.bitstamp.net/api/ticker/',
+        json: true
+      }).yields(null, null, bitstamp);
+
+      service._fetch(function(err) {
+        should.not.exist(err);
+        service.getRate({
+          code: 'USD'
+        }, function(err, res) {
+          should.not.exist(err);
+          res.fetchedOn.should.equal(100);
+          res.rate.should.equal(123.45);
+          service.getRate({
+            code: 'USD',
+            provider: 'Bitstamp',
+          }, function(err, res) {
+            should.not.exist(err);
+            res.fetchedOn.should.equal(100);
+            res.rate.should.equal(120.00);
+            service.getRate({
+              code: 'EUR'
+            }, function(err, res) {
+              should.not.exist(err);
+              res.fetchedOn.should.equal(100);
+              res.rate.should.equal(234.56);
+              clock.restore();
+              done();
+            });
+          });
+        });
+      });
+    });
+
+    it('should not stop when failing to fetch provider', function(done) {
+      var clock = sinon.useFakeTimers(100, 'Date');
+      var bitstamp = {
+        last: 120.00,
+      };
+      request.get.withArgs({
+        url: 'https://bitpay.com/api/rates/',
+        json: true
+      }).yields('dummy error', null, null);
+      request.get.withArgs({
+        url: 'https://www.bitstamp.net/api/ticker/',
+        json: true
+      }).yields(null, null, bitstamp);
+
+      service._fetch(function(err) {
+        should.not.exist(err);
+        service.getRate({
+          code: 'USD'
+        }, function(err, res) {
+          should.not.exist(err);
+          res.ts.should.equal(100);
+          should.not.exist(res.rate)
+          should.not.exist(res.fetchedOn)
+          service.getRate({
+            code: 'USD',
+            provider: 'Bitstamp'
+          }, function(err, res) {
+            should.not.exist(err);
+            res.fetchedOn.should.equal(100);
+            res.rate.should.equal(120.00);
+            clock.restore();
+            done();
+          });
+        });
+      });
+    });
+  });
+});