From 450da4ecdcd7089c12ed64a020c0771729a18ac7 Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Thu, 7 Jan 2016 12:29:47 -0300 Subject: [PATCH 01/11] add fiat rate service --- lib/expressapp.js | 21 ++++- lib/fiatrateproviders/bitpay.js | 17 ++++ lib/fiatrateproviders/bitstamp.js | 11 +++ lib/fiatrateproviders/index.js | 6 ++ lib/fiatrateservice.js | 132 ++++++++++++++++++++++++++++ lib/storage.js | 34 +++++++ test/integration/fiatrateservice.js | 52 +++++++++++ 7 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 lib/fiatrateproviders/bitpay.js create mode 100644 lib/fiatrateproviders/bitstamp.js create mode 100644 lib/fiatrateproviders/index.js create mode 100644 lib/fiatrateservice.js create mode 100644 test/integration/fiatrateservice.js diff --git a/lib/expressapp.js b/lib/expressapp.js index c78b1d2..599230a 100644 --- a/lib/expressapp.js +++ b/lib/expressapp.js @@ -5,7 +5,6 @@ var async = require('async'); var log = require('npmlog'); var express = require('express'); -var querystring = require('querystring'); var bodyParser = require('body-parser') var WalletService = require('./server'); @@ -501,6 +500,26 @@ ExpressApp.prototype.start = function(opts, cb) { }); }); + router.get('/v1/fiatrates/:code/', function(req, res) { + var opts = { + code: req.params['code'], + source: req.query.source, + ts: req.query.ts, + } + // if (_.isString(ts) && ts.indexOf(',') !== -1) { + // ts = ts.split(','); + // } + server.getFiatRate(opts, function(err, rates) { + if (err) returnError({ + code: 500, + message: err, + }); + res.json(rates); + res.end(); + }); + }); + + this.app.use(opts.basePath || '/bws/api', router); WalletService.initialize(opts, cb); diff --git a/lib/fiatrateproviders/bitpay.js b/lib/fiatrateproviders/bitpay.js new file mode 100644 index 0000000..ba58f01 --- /dev/null +++ b/lib/fiatrateproviders/bitpay.js @@ -0,0 +1,17 @@ +var _ = require('lodash'); + +var provider = { + url: 'https://bitpay.com/api/rates/', + parseFn: function(raw) { + var rates = _.compact(_.map(raw, function(d) { + if (!d.code || !d.rate) return null; + return { + code: d.code, + value: d.rate, + }; + })); + return rates; + }, +}; + +module.exports = provider; diff --git a/lib/fiatrateproviders/bitstamp.js b/lib/fiatrateproviders/bitstamp.js new file mode 100644 index 0000000..3c34ed5 --- /dev/null +++ b/lib/fiatrateproviders/bitstamp.js @@ -0,0 +1,11 @@ +var provider = { + url: 'https://www.bitstamp.net/api/ticker/', + parseFn: function(raw) { + return [{ + code: 'USD', + value: parseFloat(raw.last) + }]; + } +}; + +module.exports = provider; diff --git a/lib/fiatrateproviders/index.js b/lib/fiatrateproviders/index.js new file mode 100644 index 0000000..d361b21 --- /dev/null +++ b/lib/fiatrateproviders/index.js @@ -0,0 +1,6 @@ +var Providers = { + BitPay: require('./bitpay'), + Bitstamp: require('./bitstamp'), +} + +module.exports = Providers; diff --git a/lib/fiatrateservice.js b/lib/fiatrateservice.js new file mode 100644 index 0000000..dc57514 --- /dev/null +++ b/lib/fiatrateservice.js @@ -0,0 +1,132 @@ +'use strict'; + +var _ = require('lodash'); +var $ = require('preconditions').singleton(); +var async = require('async'); +var log = require('npmlog'); +log.debug = log.verbose; +var request = require('request'); + +var Utils = require('./common/utils'); +var Storage = require('./storage'); + +var Model = require('./model'); + + +var DEFAULT_PROVIDER = 'BitPay'; +var FETCH_INTERVAL = 15; // In minutes + +function FiatRateService() {}; + +FiatRateService.prototype.start = function(opts, cb) { + var self = this; + + opts = opts || {}; + + if (_.isArray(opts.providers)) { + self.providers = opts.providers; + } else { + self.providers = require('./fiatrateproviders'); + } + self.request = opts.request || request; + self.defaultProvider = opts.defaultProvider || DEFAULT_PROVIDER; + + async.parallel([ + + function(done) { + if (opts.storage) { + self.storage = opts.storage; + done(); + } else { + self.storage = new Storage(); + self.storage.connect(opts.storageOpts, done); + } + }, + ], function(err) { + if (err) { + log.error(err); + return cb(err); + } + + var interval = opts.fetchInterval || FETCH_INTERVAL; + if (interval) { + self._fetch(); + setInterval(function() { + self._fetch(); + }, interval * 60 * 1000); + } + + return cb(); + }); +}; + +FiatRateService.prototype._fetch = function(cb) { + var self = this; + + cb = cb || function() {}; + + async.each(_.values(self.providers), function(provider, next) { + self._retrieve(provider, function(err, res) { + if (err) { + log.warn(err); + return next(); + } + self.storage.storeFiatRate(provider.name, res, function(err) { + return next(); + }); + }); + }, cb); +}; + +FiatRateService.prototype._retrieve = function(provider, cb) { + var self = this; + + log.debug('Fetching data for ' + provider.name); + self.request.get({ + url: provider.url, + json: true, + }, function(err, res, body) { + if (err || !body) { + log.warn('Error fetching data for ' + provider.name, err); + return cb(err); + } + + log.debug('Data for ' + provider.name + ' fetched successfully'); + + if (!provider.parseFn) { + return cb('No parse function for provider ' + provider.name); + } + var rates = provider.parseFn(body); + + return cb(null, rates); + }); +}; + + +FiatRateService.prototype.getRate = function(code, opts, cb) { + var self = this; + + $.shouldBeFunction(cb); + + opts = opts || {}; + + var providerName = opts.providerName || DEFAULT_PROVIDER; + var ts = opts.ts || Date.now(); + + async.map([].concat(ts), function(ts, cb) { + self.storage.fetchFiatRate(providerName, code, ts, function(err, rate) { + if (err) return cb(err); + return cb(null, { + ts: +ts, + rate: rate + }); + }); + }, function(err, res) { + if (err) return cb(err); + if (!_.isArray(ts)) res = res[0]; + return cb(null, res); + }); +}; + + +module.exports = FiatRateService; diff --git a/lib/storage.js b/lib/storage.js index d159ece..dc2d00a 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -21,6 +21,7 @@ var collections = { PREFERENCES: 'preferences', EMAIL_QUEUE: 'email_queue', CACHE: 'cache', + FIAT_RATES: 'fiat_rates', }; var Storage = function(opts) { @@ -572,6 +573,39 @@ Storage.prototype.fetchActiveAddresses = function(walletId, cb) { }); }; +Storage.prototype.storeFiatRate = function(providerName, rates, cb) { + var self = this; + + var now = Date.now(); + async.each(rates, function(rate, next) { + self.db.collection(collections.FIAT_RATES).insert({ + provider: providerName, + ts: now, + code: rate.code, + value: rate.value, + }, { + w: 1 + }, next); + }, cb); +}; + +Storage.prototype.fetchFiatRate = function(providerName, code, ts, cb) { + var self = this; + + self.db.collection(collections.FIAT_RATES).find({ + provider: providerName, + code: code, + ts: { + $lte: ts + }, + }).sort({ + ts: -1 + }).limit(1).toArray(function(err, result) { + if (err || _.isEmpty(result)) return cb(err); + return cb(null, result[0]); + }); +}; + Storage.prototype._dump = function(cb, fn) { fn = fn || console.log; cb = cb || function() {}; diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js new file mode 100644 index 0000000..460da7c --- /dev/null +++ b/test/integration/fiatrateservice.js @@ -0,0 +1,52 @@ +'use strict'; + +var _ = require('lodash'); +var async = require('async'); + +var chai = require('chai'); +var sinon = require('sinon'); +var should = chai.should(); +var log = require('npmlog'); +log.debug = log.verbose; +log.level = 'info'; + +var helpers = require('./helpers'); + +var FiatRateService = require('../../lib/fiatrateservice'); + +describe('Fiat rate service', function() { + var service, request; + + before(function(done) { + helpers.before(done); + }); + after(function(done) { + helpers.after(done); + }); + beforeEach(function(done) { + helpers.beforeEach(function() { + service = new FiatRateService(); + request = sinon.stub(); + request.get = sinon.stub(); + service.start({ + storage: helpers.getStorage(), + request: request, + }, done); + }); + }); + describe.only('#getRate', function() { + it('should get current rate', function(done) { + service.storage.storeFiatRate('BitPay', [{ + code: 'USD', + value: 123.45, + }], function(err) { + should.not.exist(err); + service.getRate('USD', {}, function(err, res) { + should.not.exist(err); + res.rate.value.should.equal(123.45); + done(); + }); + }); + }); + }); +}); From 0922a5fa6e348142e7bef38e2812da876009058c Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Thu, 7 Jan 2016 15:58:09 -0300 Subject: [PATCH 02/11] integration tests --- lib/fiatrateservice.js | 2 +- lib/storage.js | 1 - test/integration/fiatrateservice.js | 80 +++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/lib/fiatrateservice.js b/lib/fiatrateservice.js index dc57514..a653fbf 100644 --- a/lib/fiatrateservice.js +++ b/lib/fiatrateservice.js @@ -118,7 +118,7 @@ FiatRateService.prototype.getRate = function(code, opts, cb) { if (err) return cb(err); return cb(null, { ts: +ts, - rate: rate + rate: rate ? rate.value : undefined, }); }); }, function(err, res) { diff --git a/lib/storage.js b/lib/storage.js index dc2d00a..c77e3a4 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -591,7 +591,6 @@ Storage.prototype.storeFiatRate = function(providerName, rates, cb) { Storage.prototype.fetchFiatRate = function(providerName, code, ts, cb) { var self = this; - self.db.collection(collections.FIAT_RATES).find({ provider: providerName, code: code, diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js index 460da7c..0228f40 100644 --- a/test/integration/fiatrateservice.js +++ b/test/integration/fiatrateservice.js @@ -14,7 +14,7 @@ var helpers = require('./helpers'); var FiatRateService = require('../../lib/fiatrateservice'); -describe('Fiat rate service', function() { +describe.only('Fiat rate service', function() { var service, request; before(function(done) { @@ -34,7 +34,7 @@ describe('Fiat rate service', function() { }, done); }); }); - describe.only('#getRate', function() { + describe('#getRate', function() { it('should get current rate', function(done) { service.storage.storeFiatRate('BitPay', [{ code: 'USD', @@ -43,7 +43,81 @@ describe('Fiat rate service', function() { should.not.exist(err); service.getRate('USD', {}, function(err, res) { should.not.exist(err); - res.rate.value.should.equal(123.45); + res.rate.should.equal(123.45); + done(); + }); + }); + }); + it('should get current for different currency', function(done) { + service.storage.storeFiatRate('BitPay', [{ + code: 'USD', + value: 123.45, + }], function(err) { + should.not.exist(err); + service.storage.storeFiatRate('BitPay', [{ + code: 'EUR', + value: 345.67, + }], function(err) { + should.not.exist(err); + service.getRate('EUR', {}, function(err, res) { + should.not.exist(err); + res.rate.should.equal(345.67); + done(); + }); + }); + }); + }); + it('should get rate for specific ts', function(done) { + var clock = sinon.useFakeTimers(0, 'Date'); + service.storage.storeFiatRate('BitPay', [{ + code: 'USD', + value: 123.45, + }], function(err) { + should.not.exist(err); + clock.tick(100); + service.storage.storeFiatRate('BitPay', [{ + code: 'USD', + value: 345.67, + }], function(err) { + should.not.exist(err); + service.getRate('USD', { + ts: 50, + }, function(err, res) { + should.not.exist(err); + res.rate.should.equal(123.45); + clock.restore(); + done(); + }); + }); + }); + }); + + it('should get rates for a series of ts', function(done) { + var clock = sinon.useFakeTimers(0, 'Date'); + async.each([1.00, 2.00, 3.00, 4.00], function(value, next) { + clock.tick(100); + service.storage.storeFiatRate('BitPay', [{ + code: 'USD', + value: value, + }, { + code: 'EUR', + value: value, + }], next); + }, function(err) { + should.not.exist(err); + service.getRate('USD', { + ts: [50, 100, 500], + }, function(err, res) { + should.not.exist(err); + res.length.should.equal(3); + + res[0].ts.should.equal(50); + should.not.exist(res[0].rate); + res[1].ts.should.equal(100); + res[1].rate.should.equal(1.00); + res[2].ts.should.equal(500); + res[2].rate.should.equal(4.00); + clock.restore(); done(); }); }); From e62e5b032d100c1078e88022b872759e63f13465 Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Mon, 11 Jan 2016 15:25:13 -0300 Subject: [PATCH 03/11] add fetchedOn to returned rates --- lib/fiatrateservice.js | 1 + test/integration/fiatrateservice.js | 23 ++++++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/fiatrateservice.js b/lib/fiatrateservice.js index a653fbf..0b6849d 100644 --- a/lib/fiatrateservice.js +++ b/lib/fiatrateservice.js @@ -119,6 +119,7 @@ FiatRateService.prototype.getRate = function(code, opts, cb) { return cb(null, { ts: +ts, rate: rate ? rate.value : undefined, + fetchedOn: rate ? rate.ts : undefined, }); }); }, function(err, res) { diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js index 0228f40..304dd48 100644 --- a/test/integration/fiatrateservice.js +++ b/test/integration/fiatrateservice.js @@ -48,7 +48,7 @@ describe.only('Fiat rate service', function() { }); }); }); - it('should get current for different currency', function(done) { + it('should get current rate for different currency', function(done) { service.storage.storeFiatRate('BitPay', [{ code: 'USD', value: 123.45, @@ -69,6 +69,7 @@ describe.only('Fiat rate service', function() { }); it('should get rate for specific ts', function(done) { var clock = sinon.useFakeTimers(0, 'Date'); + clock.tick(20); service.storage.storeFiatRate('BitPay', [{ code: 'USD', value: 123.45, @@ -84,7 +85,9 @@ describe.only('Fiat rate service', function() { ts: 50, }, function(err, res) { should.not.exist(err); + res.ts.should.equal(50); res.rate.should.equal(123.45); + res.fetchedOn.should.equal(20); clock.restore(); done(); }); @@ -106,17 +109,27 @@ describe.only('Fiat rate service', function() { }, function(err) { should.not.exist(err); service.getRate('USD', { - ts: [50, 100, 500], + ts: [50, 100, 199, 500], }, function(err, res) { should.not.exist(err); - res.length.should.equal(3); + res.length.should.equal(4); res[0].ts.should.equal(50); should.not.exist(res[0].rate); + should.not.exist(res[0].fetchedOn); + res[1].ts.should.equal(100); res[1].rate.should.equal(1.00); - res[2].ts.should.equal(500); - res[2].rate.should.equal(4.00); + res[1].fetchedOn.should.equal(100); + + res[2].ts.should.equal(199); + res[2].rate.should.equal(1.00); + res[2].fetchedOn.should.equal(100); + + res[3].ts.should.equal(500); + res[3].rate.should.equal(4.00); + res[3].fetchedOn.should.equal(400); + clock.restore(); done(); }); From 5d868d57c457fa22e33c0720f4dce6af0de85131 Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Mon, 11 Jan 2016 15:28:39 -0300 Subject: [PATCH 04/11] test specific provider --- lib/fiatrateservice.js | 4 ++-- test/integration/fiatrateservice.js | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/lib/fiatrateservice.js b/lib/fiatrateservice.js index 0b6849d..a0f06e7 100644 --- a/lib/fiatrateservice.js +++ b/lib/fiatrateservice.js @@ -110,11 +110,11 @@ FiatRateService.prototype.getRate = function(code, opts, cb) { opts = opts || {}; - var providerName = opts.providerName || DEFAULT_PROVIDER; + var provider = opts.provider || DEFAULT_PROVIDER; var ts = opts.ts || Date.now(); async.map([].concat(ts), function(ts, cb) { - self.storage.fetchFiatRate(providerName, code, ts, function(err, rate) { + self.storage.fetchFiatRate(provider, code, ts, function(err, rate) { if (err) return cb(err); return cb(null, { ts: +ts, diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js index 304dd48..ea6b6bb 100644 --- a/test/integration/fiatrateservice.js +++ b/test/integration/fiatrateservice.js @@ -67,6 +67,33 @@ describe.only('Fiat rate service', function() { }); }); }); + + it('should get current rate for different provider', function(done) { + service.storage.storeFiatRate('BitPay', [{ + code: 'USD', + value: 100.00, + }], function(err) { + should.not.exist(err); + service.storage.storeFiatRate('Bitstamp', [{ + code: 'USD', + value: 200.00, + }], function(err) { + should.not.exist(err); + service.getRate('USD', {}, function(err, res) { + should.not.exist(err); + res.rate.should.equal(100.00, 'Should use default provider'); + service.getRate('USD', { + provider: 'Bitstamp' + }, function(err, res) { + should.not.exist(err); + res.rate.should.equal(200.00); + done(); + }); + }); + }); + }); + }); + it('should get rate for specific ts', function(done) { var clock = sinon.useFakeTimers(0, 'Date'); clock.tick(20); From aa415268fca0ed7cd58070026ed66035b82d1b3c Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Mon, 11 Jan 2016 16:27:30 -0300 Subject: [PATCH 05/11] test fetch --- lib/fiatrateproviders/bitpay.js | 1 + lib/fiatrateproviders/bitstamp.js | 1 + lib/fiatrateservice.js | 5 +-- test/integration/fiatrateservice.js | 49 ++++++++++++++++++++++++++++- 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/lib/fiatrateproviders/bitpay.js b/lib/fiatrateproviders/bitpay.js index ba58f01..6c22874 100644 --- a/lib/fiatrateproviders/bitpay.js +++ b/lib/fiatrateproviders/bitpay.js @@ -1,6 +1,7 @@ var _ = require('lodash'); var provider = { + name: 'BitPay', url: 'https://bitpay.com/api/rates/', parseFn: function(raw) { var rates = _.compact(_.map(raw, function(d) { diff --git a/lib/fiatrateproviders/bitstamp.js b/lib/fiatrateproviders/bitstamp.js index 3c34ed5..dab4775 100644 --- a/lib/fiatrateproviders/bitstamp.js +++ b/lib/fiatrateproviders/bitstamp.js @@ -1,4 +1,5 @@ var provider = { + name: 'Bitstamp', url: 'https://www.bitstamp.net/api/ticker/', parseFn: function(raw) { return [{ diff --git a/lib/fiatrateservice.js b/lib/fiatrateservice.js index a0f06e7..9048d85 100644 --- a/lib/fiatrateservice.js +++ b/lib/fiatrateservice.js @@ -26,7 +26,7 @@ FiatRateService.prototype.start = function(opts, cb) { if (_.isArray(opts.providers)) { self.providers = opts.providers; } else { - self.providers = require('./fiatrateproviders'); + self.providers = _.values(require('./fiatrateproviders')); } self.request = opts.request || request; self.defaultProvider = opts.defaultProvider || DEFAULT_PROVIDER; @@ -65,13 +65,14 @@ FiatRateService.prototype._fetch = function(cb) { cb = cb || function() {}; - async.each(_.values(self.providers), function(provider, next) { + async.each(self.providers, function(provider, next) { self._retrieve(provider, function(err, res) { if (err) { log.warn(err); return next(); } self.storage.storeFiatRate(provider.name, res, function(err) { + if (err) log.warn(err); return next(); }); }); diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js index ea6b6bb..53b92a9 100644 --- a/test/integration/fiatrateservice.js +++ b/test/integration/fiatrateservice.js @@ -14,7 +14,7 @@ var helpers = require('./helpers'); var FiatRateService = require('../../lib/fiatrateservice'); -describe.only('Fiat rate service', function() { +describe('Fiat rate service', function() { var service, request; before(function(done) { @@ -163,4 +163,51 @@ describe.only('Fiat rate service', function() { }); }); }); + + describe('#fetch', function() { + it('should fetch rates from all providers', function(done) { + var clock = sinon.useFakeTimers(100, 'Date'); + var bitpay = [{ + code: 'USD', + rate: 123.45, + }, { + code: 'EUR', + rate: 234.56, + }]; + var bitstamp = { + last: 120.00, + }; + request.get.withArgs({ + url: 'https://bitpay.com/api/rates/', + json: true + }).yields(null, null, bitpay); + request.get.withArgs({ + url: 'https://www.bitstamp.net/api/ticker/', + json: true + }).yields(null, null, bitstamp); + + service._fetch(function(err) { + should.not.exist(err); + service.getRate('USD', {}, function(err, res) { + should.not.exist(err); + res.fetchedOn.should.equal(100); + res.rate.should.equal(123.45); + service.getRate('USD', { + provider: 'Bitstamp' + }, function(err, res) { + should.not.exist(err); + res.fetchedOn.should.equal(100); + res.rate.should.equal(120.00); + service.getRate('EUR', {}, function(err, res) { + should.not.exist(err); + res.fetchedOn.should.equal(100); + res.rate.should.equal(234.56); + clock.restore(); + done(); + }); + }); + }); + }); + }); + }); }); From 8d6d545139c167491b582025287b927ed8020347 Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Mon, 11 Jan 2016 16:36:51 -0300 Subject: [PATCH 06/11] test error handling in fetching process --- lib/fiatrateservice.js | 9 ++++---- test/integration/fiatrateservice.js | 34 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/lib/fiatrateservice.js b/lib/fiatrateservice.js index 9048d85..7cc6e52 100644 --- a/lib/fiatrateservice.js +++ b/lib/fiatrateservice.js @@ -68,11 +68,13 @@ FiatRateService.prototype._fetch = function(cb) { async.each(self.providers, function(provider, next) { self._retrieve(provider, function(err, res) { if (err) { - log.warn(err); + log.warn('Error retrieving data for ' + provider.name, err); return next(); } self.storage.storeFiatRate(provider.name, res, function(err) { - if (err) log.warn(err); + if (err) { + log.warn('Error storing data for ' + provider.name, err); + } return next(); }); }); @@ -88,14 +90,13 @@ FiatRateService.prototype._retrieve = function(provider, cb) { json: true, }, function(err, res, body) { if (err || !body) { - log.warn('Error fetching data for ' + provider.name, err); return cb(err); } log.debug('Data for ' + provider.name + ' fetched successfully'); if (!provider.parseFn) { - return cb('No parse function for provider ' + provider.name); + return cb(new Error('No parse function for provider ' + provider.name)); } var rates = provider.parseFn(body); diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js index 53b92a9..98420f0 100644 --- a/test/integration/fiatrateservice.js +++ b/test/integration/fiatrateservice.js @@ -209,5 +209,39 @@ describe('Fiat rate service', function() { }); }); }); + + it('should not stop when failing to fetch provider', function(done) { + var clock = sinon.useFakeTimers(100, 'Date'); + var bitstamp = { + last: 120.00, + }; + request.get.withArgs({ + url: 'https://bitpay.com/api/rates/', + json: true + }).yields('dummy error', null, null); + request.get.withArgs({ + url: 'https://www.bitstamp.net/api/ticker/', + json: true + }).yields(null, null, bitstamp); + + service._fetch(function(err) { + should.not.exist(err); + service.getRate('USD', {}, function(err, res) { + should.not.exist(err); + res.ts.should.equal(100); + should.not.exist(res.rate) + should.not.exist(res.fetchedOn) + service.getRate('USD', { + provider: 'Bitstamp' + }, function(err, res) { + should.not.exist(err); + res.fetchedOn.should.equal(100); + res.rate.should.equal(120.00); + clock.restore(); + done(); + }); + }); + }); + }); }); }); From aac0b70df4825a7bc703af33c9b83e4d523e7a1b Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Mon, 11 Jan 2016 17:10:24 -0300 Subject: [PATCH 07/11] separate initialization from cron job --- lib/fiatrateservice.js | 39 ++++++++++++++++------------- lib/server.js | 17 +++++++++++++ test/integration/fiatrateservice.js | 9 ++++--- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/lib/fiatrateservice.js b/lib/fiatrateservice.js index 7cc6e52..5b9f4ae 100644 --- a/lib/fiatrateservice.js +++ b/lib/fiatrateservice.js @@ -18,16 +18,11 @@ var FETCH_INTERVAL = 15; // In minutes function FiatRateService() {}; -FiatRateService.prototype.start = function(opts, cb) { +FiatRateService.prototype.init = function(opts, cb) { var self = this; opts = opts || {}; - if (_.isArray(opts.providers)) { - self.providers = opts.providers; - } else { - self.providers = _.values(require('./fiatrateproviders')); - } self.request = opts.request || request; self.defaultProvider = opts.defaultProvider || DEFAULT_PROVIDER; @@ -45,21 +40,29 @@ FiatRateService.prototype.start = function(opts, cb) { ], function(err) { if (err) { log.error(err); - return cb(err); } - - var interval = opts.fetchInterval || FETCH_INTERVAL; - if (interval) { - self._fetch(); - setInterval(function() { - self._fetch(); - }, interval * 60 * 1000); - } - - return cb(); + return cb(err); }); }; +FiatRateService.prototype.startCron = function(opts, cb) { + var self = this; + + opts = opts || {}; + + self.providers = _.values(require('./fiatrateproviders')); + + var interval = opts.fetchInterval || FETCH_INTERVAL; + if (interval) { + self._fetch(); + setInterval(function() { + self._fetch(); + }, interval * 60 * 1000); + } + + return cb(); +}; + FiatRateService.prototype._fetch = function(cb) { var self = this; @@ -112,7 +115,7 @@ FiatRateService.prototype.getRate = function(code, opts, cb) { opts = opts || {}; - var provider = opts.provider || DEFAULT_PROVIDER; + var provider = opts.provider || self.defaultProvider; var ts = opts.ts || Date.now(); async.map([].concat(ts), function(ts, cb) { diff --git a/lib/server.js b/lib/server.js index 0ab90cf..b0133c5 100644 --- a/lib/server.js +++ b/lib/server.js @@ -2365,6 +2365,23 @@ WalletService.prototype.startScan = function(opts, cb) { }); }; +/** + * Returns exchange rate for the specified currency & timestamp. + * @param {Object} opts + * @param {string} opts.code - Currency ISO code. + * @param {Date} [opts.ts] - A timestamp to base the rate on (default Date.now()). + * @param {String} [opts.provider] - A provider of exchange rates (default 'BitPay'). + * @returns {Object} rates - The exchange rate. + */ +WalletService.prototype.getRate = function(opts, cb) { + var self = this; + + if (!Utils.checkRequired(opts, ['code'])) + return cb(new ClientError('Required argument missing')); + + +}; + module.exports = WalletService; module.exports.ClientError = ClientError; diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js index 98420f0..5fbd1db 100644 --- a/test/integration/fiatrateservice.js +++ b/test/integration/fiatrateservice.js @@ -14,7 +14,7 @@ var helpers = require('./helpers'); var FiatRateService = require('../../lib/fiatrateservice'); -describe('Fiat rate service', function() { +describe.only('Fiat rate service', function() { var service, request; before(function(done) { @@ -28,10 +28,13 @@ describe('Fiat rate service', function() { service = new FiatRateService(); request = sinon.stub(); request.get = sinon.stub(); - service.start({ + service.init({ storage: helpers.getStorage(), request: request, - }, done); + }, function(err) { + should.not.exist(err); + service.startCron({}, done); + }); }); }); describe('#getRate', function() { From 10ac3a4d65ff006000564f480322cea39715679c Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Mon, 11 Jan 2016 17:46:36 -0300 Subject: [PATCH 08/11] init fiat rate service from within bws --- config.js | 4 ++++ lib/expressapp.js | 11 ++++------- lib/server.js | 29 +++++++++++++++++++++++++++-- test/integration/fiatrateservice.js | 2 +- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/config.js b/config.js index c3887e4..25c502f 100644 --- a/config.js +++ b/config.js @@ -52,6 +52,10 @@ var config = { subjectPrefix: '', pushServerUrl: 'http://localhost:8000/send', }, + fiatRateServiceOpts: { + defaultProvider: 'BitPay', + fetchInterval: 15, // in minutes + }, // To use email notifications uncomment this: // emailOpts: { // host: 'localhost', diff --git a/lib/expressapp.js b/lib/expressapp.js index 599230a..30899f8 100644 --- a/lib/expressapp.js +++ b/lib/expressapp.js @@ -502,13 +502,10 @@ ExpressApp.prototype.start = function(opts, cb) { router.get('/v1/fiatrates/:code/', function(req, res) { var opts = { - code: req.params['code'], - source: req.query.source, - ts: req.query.ts, - } - // if (_.isString(ts) && ts.indexOf(',') !== -1) { - // ts = ts.split(','); - // } + code: req.params['code'], + source: req.query.source, + ts: req.query.ts, + } server.getFiatRate(opts, function(err, rates) { if (err) returnError({ code: 500, diff --git a/lib/server.js b/lib/server.js index b0133c5..68f0578 100644 --- a/lib/server.js +++ b/lib/server.js @@ -22,6 +22,7 @@ var Lock = require('./lock'); var Storage = require('./storage'); var MessageBroker = require('./messagebroker'); var BlockchainExplorer = require('./blockchainexplorer'); +var FiatRateService = require('./fiatrateservice'); var Model = require('./model'); var Wallet = Model.Wallet; @@ -33,6 +34,7 @@ var storage; var blockchainExplorer; var blockchainExplorerOpts; var messageBroker; +var fiatRateService; var serviceVersion; var HISTORY_LIMIT = 10; @@ -50,6 +52,7 @@ function WalletService() { this.blockchainExplorer = blockchainExplorer; this.blockchainExplorerOpts = blockchainExplorerOpts; this.messageBroker = messageBroker; + this.fiatRateService = fiatRateService; this.notifyTicker = 0; }; @@ -101,6 +104,22 @@ WalletService.initialize = function(opts, cb) { return cb(); }; + function initFiatRateService(cb) { + if (opts.fiatRateService) { + fiatRateService = opts.fiatRateService; + return cb(); + } else { + var newFiatRateService = new FiatRateService(); + var opts2 = opts.fiatRateServiceOpts || {}; + opts2.storage = storage; + newFiatRateService.init(opts2, function(err) { + if (err) return cb(err); + fiatRateService = newFiatRateService; + return cb(); + }); + } + }; + async.series([ function(next) { @@ -109,6 +128,9 @@ WalletService.initialize = function(opts, cb) { function(next) { initMessageBroker(next); }, + function(next) { + initFiatRateService(next); + }, ], function(err) { if (err) { log.error('Could not initialize', err); @@ -2373,13 +2395,16 @@ WalletService.prototype.startScan = function(opts, cb) { * @param {String} [opts.provider] - A provider of exchange rates (default 'BitPay'). * @returns {Object} rates - The exchange rate. */ -WalletService.prototype.getRate = function(opts, cb) { +WalletService.prototype.getFiatRate = function(opts, cb) { var self = this; if (!Utils.checkRequired(opts, ['code'])) return cb(new ClientError('Required argument missing')); - + self.fiatRateService.getRate(opts, function(err, rate) { + if (err) return cb(err); + return cb(null, rate); + }); }; diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js index 5fbd1db..169c7cf 100644 --- a/test/integration/fiatrateservice.js +++ b/test/integration/fiatrateservice.js @@ -14,7 +14,7 @@ var helpers = require('./helpers'); var FiatRateService = require('../../lib/fiatrateservice'); -describe.only('Fiat rate service', function() { +describe('Fiat rate service', function() { var service, request; before(function(done) { From b9b1bddea8a73654a26ffd7f50082b63e61f312f Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Tue, 12 Jan 2016 11:45:46 -0300 Subject: [PATCH 09/11] REST endpoint + added to shell command --- config.js | 2 +- fiatrateservice/fiatrateservice.js | 16 ++++++++++++ lib/expressapp.js | 26 ++++++++++--------- lib/fiatrateservice.js | 6 ++--- start.sh | 1 + stop.sh | 1 + test/integration/fiatrateservice.js | 40 ++++++++++++++++++++--------- 7 files changed, 64 insertions(+), 28 deletions(-) create mode 100644 fiatrateservice/fiatrateservice.js diff --git a/config.js b/config.js index 25c502f..4ba24b7 100644 --- a/config.js +++ b/config.js @@ -54,7 +54,7 @@ var config = { }, fiatRateServiceOpts: { defaultProvider: 'BitPay', - fetchInterval: 15, // in minutes + fetchInterval: 10, // in minutes }, // To use email notifications uncomment this: // emailOpts: { diff --git a/fiatrateservice/fiatrateservice.js b/fiatrateservice/fiatrateservice.js new file mode 100644 index 0000000..6cc5a99 --- /dev/null +++ b/fiatrateservice/fiatrateservice.js @@ -0,0 +1,16 @@ +#!/usr/bin/env node + +'use strict'; + +var config = require('../config'); +var FiatRateService = require('../lib/fiatrateservice'); + +var service = new FiatRateService(); +service.init(config, function(err) { + if (err) throw err; + service.startCron(config, function(err) { + if (err) throw err; + + console.log('Fiat rate service started'); + }); +}); diff --git a/lib/expressapp.js b/lib/expressapp.js index 30899f8..c197374 100644 --- a/lib/expressapp.js +++ b/lib/expressapp.js @@ -113,7 +113,7 @@ ExpressApp.prototype.start = function(opts, cb) { }; }; - function getServer(req, res, cb) { + function getServer(req, res) { var opts = { clientVersion: req.header('x-client-version'), }; @@ -501,18 +501,20 @@ ExpressApp.prototype.start = function(opts, cb) { }); router.get('/v1/fiatrates/:code/', function(req, res) { - var opts = { - code: req.params['code'], - source: req.query.source, - ts: req.query.ts, - } - server.getFiatRate(opts, function(err, rates) { - if (err) returnError({ - code: 500, - message: err, + getServerWithAuth(req, res, function(server) { + var opts = { + code: req.params['code'], + source: req.query.source, + ts: +req.query.ts, + }; + server.getFiatRate(opts, function(err, rates) { + if (err) returnError({ + code: 500, + message: err, + }); + res.json(rates); + res.end(); }); - res.json(rates); - res.end(); }); }); diff --git a/lib/fiatrateservice.js b/lib/fiatrateservice.js index 5b9f4ae..9a3306a 100644 --- a/lib/fiatrateservice.js +++ b/lib/fiatrateservice.js @@ -108,7 +108,7 @@ FiatRateService.prototype._retrieve = function(provider, cb) { }; -FiatRateService.prototype.getRate = function(code, opts, cb) { +FiatRateService.prototype.getRate = function(opts, cb) { var self = this; $.shouldBeFunction(cb); @@ -116,10 +116,10 @@ FiatRateService.prototype.getRate = function(code, opts, cb) { opts = opts || {}; var provider = opts.provider || self.defaultProvider; - var ts = opts.ts || Date.now(); + var ts = _.isNumber(opts.ts) ? opts.ts : Date.now(); async.map([].concat(ts), function(ts, cb) { - self.storage.fetchFiatRate(provider, code, ts, function(err, rate) { + self.storage.fetchFiatRate(provider, opts.code, ts, function(err, rate) { if (err) return cb(err); return cb(null, { ts: +ts, diff --git a/start.sh b/start.sh index bfe1e55..3cd1e27 100755 --- a/start.sh +++ b/start.sh @@ -34,5 +34,6 @@ run_program messagebroker/messagebroker.js pids/messagebroker.pid logs/messagebr run_program bcmonitor/bcmonitor.js pids/bcmonitor.pid logs/bcmonitor.log run_program emailservice/emailservice.js pids/emailservice.pid logs/emailservice.log run_program pushnotificationsservice/pushnotificationsservice.js pids/pushnotificationsservice.pid logs/pushnotificationsservice.log +run_program fiatrateservice/fiatrateservice.js pids/fiatrateservice.pid logs/fiatrateservice.log run_program bws.js pids/bws.pid logs/bws.log diff --git a/stop.sh b/stop.sh index b15de9d..7ad5071 100755 --- a/stop.sh +++ b/stop.sh @@ -11,6 +11,7 @@ stop_program () } stop_program pids/bws.pid +stop_program pids/fiatrateservice.pid stop_program pids/emailservice.pid stop_program pids/bcmonitor.pid stop_program pids/pushnotificationsservice.pid diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js index 169c7cf..58d4d06 100644 --- a/test/integration/fiatrateservice.js +++ b/test/integration/fiatrateservice.js @@ -44,7 +44,9 @@ describe('Fiat rate service', function() { value: 123.45, }], function(err) { should.not.exist(err); - service.getRate('USD', {}, function(err, res) { + service.getRate({ + code: 'USD' + }, function(err, res) { should.not.exist(err); res.rate.should.equal(123.45); done(); @@ -62,7 +64,9 @@ describe('Fiat rate service', function() { value: 345.67, }], function(err) { should.not.exist(err); - service.getRate('EUR', {}, function(err, res) { + service.getRate({ + code: 'EUR' + }, function(err, res) { should.not.exist(err); res.rate.should.equal(345.67); done(); @@ -82,11 +86,14 @@ describe('Fiat rate service', function() { value: 200.00, }], function(err) { should.not.exist(err); - service.getRate('USD', {}, function(err, res) { + service.getRate({ + code: 'USD' + }, function(err, res) { should.not.exist(err); res.rate.should.equal(100.00, 'Should use default provider'); - service.getRate('USD', { - provider: 'Bitstamp' + service.getRate({ + code: 'USD', + provider: 'Bitstamp', }, function(err, res) { should.not.exist(err); res.rate.should.equal(200.00); @@ -111,7 +118,8 @@ describe('Fiat rate service', function() { value: 345.67, }], function(err) { should.not.exist(err); - service.getRate('USD', { + service.getRate({ + code: 'USD', ts: 50, }, function(err, res) { should.not.exist(err); @@ -138,7 +146,8 @@ describe('Fiat rate service', function() { }], next); }, function(err) { should.not.exist(err); - service.getRate('USD', { + service.getRate({ + code: 'USD', ts: [50, 100, 199, 500], }, function(err, res) { should.not.exist(err); @@ -191,17 +200,22 @@ describe('Fiat rate service', function() { service._fetch(function(err) { should.not.exist(err); - service.getRate('USD', {}, function(err, res) { + service.getRate({ + code: 'USD' + }, function(err, res) { should.not.exist(err); res.fetchedOn.should.equal(100); res.rate.should.equal(123.45); - service.getRate('USD', { - provider: 'Bitstamp' + service.getRate({ + code: 'USD', + provider: 'Bitstamp', }, function(err, res) { should.not.exist(err); res.fetchedOn.should.equal(100); res.rate.should.equal(120.00); - service.getRate('EUR', {}, function(err, res) { + service.getRate({ + code: 'EUR' + }, function(err, res) { should.not.exist(err); res.fetchedOn.should.equal(100); res.rate.should.equal(234.56); @@ -229,7 +243,9 @@ describe('Fiat rate service', function() { service._fetch(function(err) { should.not.exist(err); - service.getRate('USD', {}, function(err, res) { + service.getRate({ + code: 'USD' + }, function(err, res) { should.not.exist(err); res.ts.should.equal(100); should.not.exist(res.rate) From 57912a31804e69cebb903765fa98c94dda499751 Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Tue, 12 Jan 2016 11:53:35 -0300 Subject: [PATCH 10/11] define constants in defaults.js --- lib/common/defaults.js | 4 ++++ lib/fiatrateservice.js | 15 ++++++--------- test/integration/fiatrateservice.js | 3 ++- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/common/defaults.js b/lib/common/defaults.js index 52e6d5a..43e6845 100644 --- a/lib/common/defaults.js +++ b/lib/common/defaults.js @@ -40,4 +40,8 @@ Defaults.FEE_LEVELS = [{ // Minimum nb of addresses a wallet must have to start using 2-step balance optimization Defaults.TWO_STEP_BALANCE_THRESHOLD = 100; +Defaults.FIAT_RATE_PROVIDER = 'BitPay'; +Defaults.FIAT_RATE_FETCH_INTERVAL = 10; // In minutes + + module.exports = Defaults; diff --git a/lib/fiatrateservice.js b/lib/fiatrateservice.js index 9a3306a..5fca814 100644 --- a/lib/fiatrateservice.js +++ b/lib/fiatrateservice.js @@ -7,15 +7,12 @@ var log = require('npmlog'); log.debug = log.verbose; var request = require('request'); -var Utils = require('./common/utils'); +var Common = require('./common'); +var Defaults = Common.Defaults; + var Storage = require('./storage'); - var Model = require('./model'); - -var DEFAULT_PROVIDER = 'BitPay'; -var FETCH_INTERVAL = 15; // In minutes - function FiatRateService() {}; FiatRateService.prototype.init = function(opts, cb) { @@ -24,7 +21,7 @@ FiatRateService.prototype.init = function(opts, cb) { opts = opts || {}; self.request = opts.request || request; - self.defaultProvider = opts.defaultProvider || DEFAULT_PROVIDER; + self.defaultProvider = opts.defaultProvider || Defaults.FIAT_RATE_PROVIDER; async.parallel([ @@ -52,7 +49,7 @@ FiatRateService.prototype.startCron = function(opts, cb) { self.providers = _.values(require('./fiatrateproviders')); - var interval = opts.fetchInterval || FETCH_INTERVAL; + var interval = opts.fetchInterval || Defaults.FIAT_RATE_FETCH_INTERVAL; if (interval) { self._fetch(); setInterval(function() { @@ -116,7 +113,7 @@ FiatRateService.prototype.getRate = function(opts, cb) { opts = opts || {}; var provider = opts.provider || self.defaultProvider; - var ts = _.isNumber(opts.ts) ? opts.ts : Date.now(); + var ts = (_.isNumber(opts.ts) || _.isArray(opts.ts)) ? opts.ts : Date.now(); async.map([].concat(ts), function(ts, cb) { self.storage.fetchFiatRate(provider, opts.code, ts, function(err, rate) { diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js index 58d4d06..d9bf8f3 100644 --- a/test/integration/fiatrateservice.js +++ b/test/integration/fiatrateservice.js @@ -250,7 +250,8 @@ describe('Fiat rate service', function() { res.ts.should.equal(100); should.not.exist(res.rate) should.not.exist(res.fetchedOn) - service.getRate('USD', { + service.getRate({ + code: 'USD', provider: 'Bitstamp' }, function(err, res) { should.not.exist(err); From 8a6c5dfc13eaa3f63e59a5add22cae6db66c1ddc Mon Sep 17 00:00:00 2001 From: Ivan Socolsky Date: Wed, 13 Jan 2016 16:29:04 -0300 Subject: [PATCH 11/11] establish a limit on look back time for a valid rate --- lib/common/defaults.js | 2 +- lib/fiatrateservice.js | 5 ++++- test/integration/fiatrateservice.js | 29 +++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/lib/common/defaults.js b/lib/common/defaults.js index 43e6845..b5b366a 100644 --- a/lib/common/defaults.js +++ b/lib/common/defaults.js @@ -42,6 +42,6 @@ Defaults.TWO_STEP_BALANCE_THRESHOLD = 100; Defaults.FIAT_RATE_PROVIDER = 'BitPay'; Defaults.FIAT_RATE_FETCH_INTERVAL = 10; // In minutes - +Defaults.FIAT_RATE_MAX_LOOK_BACK_TIME = 120; // In minutes module.exports = Defaults; diff --git a/lib/fiatrateservice.js b/lib/fiatrateservice.js index 5fca814..018e69f 100644 --- a/lib/fiatrateservice.js +++ b/lib/fiatrateservice.js @@ -112,12 +112,15 @@ FiatRateService.prototype.getRate = function(opts, cb) { opts = opts || {}; + var now = Date.now(); var provider = opts.provider || self.defaultProvider; - var ts = (_.isNumber(opts.ts) || _.isArray(opts.ts)) ? opts.ts : Date.now(); + var ts = (_.isNumber(opts.ts) || _.isArray(opts.ts)) ? opts.ts : now; async.map([].concat(ts), function(ts, cb) { self.storage.fetchFiatRate(provider, opts.code, ts, function(err, rate) { if (err) return cb(err); + if (rate && (now - rate.ts) > Defaults.FIAT_RATE_MAX_LOOK_BACK_TIME * 60 * 1000) rate = null; + return cb(null, { ts: +ts, rate: rate ? rate.value : undefined, diff --git a/test/integration/fiatrateservice.js b/test/integration/fiatrateservice.js index d9bf8f3..3efbc46 100644 --- a/test/integration/fiatrateservice.js +++ b/test/integration/fiatrateservice.js @@ -174,6 +174,35 @@ describe('Fiat rate service', function() { }); }); }); + + it('should not get rate older than 2hs', function(done) { + var clock = sinon.useFakeTimers(0, 'Date'); + service.storage.storeFiatRate('BitPay', [{ + code: 'USD', + value: 123.45, + }], function(err) { + should.not.exist(err); + clock.tick((120 * 60 - 1) * 1000); // Almost 2 hours + service.getRate({ + code: 'USD', + }, function(err, res) { + should.not.exist(err); + res.rate.should.equal(123.45); + res.fetchedOn.should.equal(0); + clock.restore(); + clock.tick(2 * 1000); // 2 seconds later... + service.getRate({ + code: 'USD', + }, function(err, res) { + should.not.exist(err); + should.not.exist(res.rate); + clock.restore(); + done(); + }); + }); + }); + }); + }); describe('#fetch', function() {