From af21ae6c1c964cd40e48e974af56ea2e1361304f Mon Sep 17 00:00:00 2001 From: David Frank Date: Tue, 27 Jan 2015 21:11:26 +0800 Subject: [PATCH] basic refactor done --- LIMITS.md | 14 ++--- README.md | 14 ++--- index.js | 105 +++++++++++++++++++++++--------- lib/headers.js | 114 +++++++++++++++++++++++++++++++++++ lib/response.js | 156 ++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 3 + test/test.js | 12 +++- 7 files changed, 372 insertions(+), 46 deletions(-) create mode 100644 lib/headers.js create mode 100644 lib/response.js diff --git a/LIMITS.md b/LIMITS.md index 18e9b1b..78a654d 100644 --- a/LIMITS.md +++ b/LIMITS.md @@ -2,18 +2,16 @@ Known limits ============ -**As of 1.x release** +*As of 1.x release* -- Topics such as cross-origin, CSP, mixed content are ignored, given our server-side context. +- Topics such as cross-origin, content security policy, mixed content, service workers are ignored, given our server-side context. - Url input must be an absolute url, using either `http` or `https` as scheme. -- Doesn't export `Headers`, `Body`, `Request`, `Response` classes yet, as we currenly use a much simpler implementation. +- On the upside, there are no forbidden headers, and `res.url` contains the final url when following redirects. -- For convenience, `res.body()` is a transform stream instead of byte stream, so decoding can be handled independently. +- For convenience, `res.body` is a transform stream, so decoding can be handled independently. -- Similarly, `options.body` can either be a string or a readable stream. +- Similarly, `req.body` can either be a string or a readable stream. -- For convenience, maximum redirect count (`options.follow`) and request timeout (`options.timeout`) are adjustable. - -- There is currently no built-in caching support, as server-side requirement varies greatly between use-cases. +- There is currently no built-in caching, as server-side caching varies by use-cases. diff --git a/README.md b/README.md index e4363b5..271e1eb 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,11 @@ A light-weight module that brings `window.fetch` to node.js I really like the notion of Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch): it bridges the API gap between client-side and server-side http requests, so developers have less to worry about. -But I think the term [isomorphic](http://isomorphic.net/) is generally misleading: it gives developers a false sense of security that their javascript code will run happily on both controlled server environment as well as uncontrollable user browsers. When the latter is only true for a small subset of modern browsers, not to mention quirks in native implementation. +But I think the term [isomorphic](http://isomorphic.net/) is generally misleading: it gives developers a false sense of security that their javascript code will run happily on both controlled server environment as well as uncontrollable user browsers. When the latter is only true for a subset of modern browsers, not to mention quirks in native implementation. Instead of implementing `XMLHttpRequest` in node to run browser-specific [fetch polyfill](https://github.com/github/fetch), why not go from node's `http` to `fetch` API directly? Node has native stream support, browserify build targets (browsers) don't, so underneath they are going to be vastly different anyway. -IMHO, it's safer to be aware of javascript runtime's strength and weakness, than to assume they are a unified platform under a singular spec. - -Hence `node-fetch`, minimal code for a `window.fetch` compatible API. +Hence `node-fetch`, minimal code for a `window.fetch` compatible API on node.js runtime. # Features @@ -28,11 +26,11 @@ Hence `node-fetch`, minimal code for a `window.fetch` compatible API. - Use native promise, but allow substituting it with [insert your favorite promise library]. -# Difference to client-side fetch +# Difference from client-side fetch -- This module is WIP, see [Known limits](https://github.com/bitinn/node-fetch/blob/master/LIMITS.md) for details. - -(If you spot a missing feature that `window.fetch` offers, feel free to open an issue. Pull requests are welcomed too!) +- See [Known limits](https://github.com/bitinn/node-fetch/blob/master/LIMITS.md) for details. +- If you happen to use a missing feature that `window.fetch` offers, feel free to open an issue. +- Pull requests are welcomed too! # Install diff --git a/index.js b/index.js index 2d8e311..47e1861 100644 --- a/index.js +++ b/index.js @@ -2,7 +2,7 @@ /** * index.js * - * export fetch + * a request API compatible with window.fetch */ var parse = require('url').parse; @@ -12,10 +12,13 @@ var https = require('https'); var zlib = require('zlib'); var stream = require('stream'); +var Response = require('./lib/response'); +var Headers = require('./lib/headers'); + module.exports = Fetch; /** - * Create an instance of Decent + * Fetch class * * @param String url Absolute url * @param Object opts Fetch options @@ -23,18 +26,21 @@ module.exports = Fetch; */ function Fetch(url, opts) { + // allow call as function if (!(this instanceof Fetch)) return new Fetch(url, opts); + // allow custom promise if (!Fetch.Promise) { throw new Error('native promise missing, set Fetch.Promise to your favorite alternative'); } + Response.Promise = Fetch.Promise; + var self = this; + // wrap http.request into fetch return new Fetch.Promise(function(resolve, reject) { - opts = opts || {}; - var uri = parse(url); if (!uri.protocol || !uri.hostname) { @@ -47,8 +53,6 @@ function Fetch(url, opts) { return; } - // TODO: detect type and decode data - var request; if (uri.protocol === 'https:') { request = https.request; @@ -56,28 +60,66 @@ function Fetch(url, opts) { request = http.request; } - // avoid side-effect on input + opts = opts || {}; + + // avoid side-effect on input options var options = { hostname: uri.hostname , port: uri.port - , method: opts.method - , path: uri.path - , headers: opts.headers || {} + , path: uri.path || '/' , auth: uri.auth + , method: opts.method || 'GET' + , headers: opts.headers || {} , follow: opts.follow || 20 , counter: opts.counter || 0 - , agent: opts.agent + , timeout: opts.timeout || 0 + , compress: opts.compress || true + , size: opts.size || 0 , body: opts.body - , timeout: opts.timeout + , agent: opts.agent }; + // normalize headers + var headers = new Headers(options.headers); + + if (options.compress) { + headers.set('accept-encoding', 'gzip,deflate'); + } + + if (!headers.has('user-agent')) { + headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); + } + + if (!headers.has('connection')) { + headers.set('connection', 'close'); + } + + if (!headers.has('accept')) { + headers.set('accept', '*/*'); + } + + options.headers = headers.raw(); + + // send request var req = request(options); + var started = false; + + req.on('socket', function(socket) { + if (!started && options.timeout) { + started = true; + setTimeout(function() { + req.abort(); + reject(new Error('network timeout at: ' + uri.href)); + }, options.timeout); + } + }); req.on('error', function(err) { reject(new Error('request to ' + uri.href + ' failed, reason: ' + err.message)); }); req.on('response', function(res) { + // handle redirect if (self.isRedirect(res.statusCode)) { if (options.counter >= options.follow) { reject(Error('maximum redirect reached at: ' + uri.href)); @@ -87,19 +129,35 @@ function Fetch(url, opts) { reject(Error('redirect location header missing at: ' + uri.href)); } - return Fetch(resolve(uri.href, res.headers.location), options); + resolve(Fetch(resolve(uri.href, res.headers.location), options)); + return; } - var output = { - status: res.statusCode - , headers: res.headers - , body: res.pipe(new stream.PassThrough()) - , url: uri.href - }; + // handle compression + var body = res.pipe(new stream.PassThrough()); + var headers = new Headers(res.headers); + + if (headers.has('content-encoding')) { + var name = headers.get('content-encoding'); + + if (name == 'gzip' || name == 'x-gzip') { + body = body.pipe(zlib.createGunzip()); + } else if (name == 'deflate' || name == 'x-deflate') { + body = body.pipe(zlib.createInflate()); + } + } + + // response object + var output = new Response(body, { + url: uri.href + , status: res.statusCode + , headers: headers + }); resolve(output); }); + // accept string or readable stream as body if (typeof options.body === 'string') { req.write(options.body); req.end(); @@ -108,19 +166,12 @@ function Fetch(url, opts) { } else { req.end(); } - - if (options.timeout) { - setTimeout(function() { - req.abort(); - reject(new Error('network timeout at: ' + uri.href)); - }, options.timeout); - } }); }; /** - * Create an instance of Decent + * Redirect code matching * * @param Number code Status code * @return Boolean diff --git a/lib/headers.js b/lib/headers.js new file mode 100644 index 0000000..95492d0 --- /dev/null +++ b/lib/headers.js @@ -0,0 +1,114 @@ + +/** + * headers.js + * + * Headers class offers convenient helpers + */ + +module.exports = Headers; + +/** + * Headers class + * + * @param Object headers Response headers + * @return Void + */ +function Headers(headers) { + + var self = this; + this._headers = {}; + + for (var prop in headers) { + if (headers.hasOwnProperty(prop)) { + if (typeof headers[prop] === 'string') { + this.set(prop, headers[prop]); + } else if (headers[prop].length > 0) { + headers[prop].forEach(function(item) { + self.append(prop, item); + }); + } + } + } + +} + +/** + * Return first header value given name + * + * @param String name Header name + * @return Mixed + */ +Headers.prototype.get = function(name) { + var list = this._headers[name.toLowerCase()]; + return list ? list[0] : null; +}; + +/** + * Return all header values given name + * + * @param String name Header name + * @return Array + */ +Headers.prototype.getAll = function(name) { + if (!this.has(name)) { + return []; + } + + return this._headers[name.toLowerCase()]; +}; + +/** + * Overwrite header values given name + * + * @param String name Header name + * @param String value Header value + * @return Void + */ +Headers.prototype.set = function(name, value) { + this._headers[name.toLowerCase()] = [value]; +}; + +/** + * Append a value onto existing header + * + * @param String name Header name + * @param String value Header value + * @return Void + */ +Headers.prototype.append = function(name, value) { + if (!this.has(name)) { + this.set(name, value); + return; + } + + this._headers[name.toLowerCase()].push(value); +}; + +/** + * Check for header name existence + * + * @param String name Header name + * @return Boolean + */ +Headers.prototype.has = function(name) { + return this._headers.hasOwnProperty(name.toLowerCase()); +}; + +/** + * Delete all header values given name + * + * @param String name Header name + * @return Void + */ +Headers.prototype['delete'] = function(name) { + delete this._headers[name.toLowerCase()]; +}; + +/** + * Return raw headers (non-spec api) + * + * @return Object + */ +Headers.prototype.raw = function() { + return this._headers; +}; diff --git a/lib/response.js b/lib/response.js new file mode 100644 index 0000000..aedcc02 --- /dev/null +++ b/lib/response.js @@ -0,0 +1,156 @@ + +/** + * response.js + * + * Response class provides content decoding + */ + +var http = require('http'); +var stream = require('stream'); +var convert = require('encoding').convert; + +module.exports = Response; + +/** + * Response class + * + * @param Stream body Readable stream + * @param Object opts Response options + * @return Void + */ +function Response(body, opts) { + + this.url = opts.url; + this.status = opts.status; + this.statusText = http.STATUS_CODES[this.status]; + this.headers = opts.headers; + this.body = body.pipe(new stream.PassThrough()); + this.bodyUsed = false; + +} + +/** + * Decode response as json + * + * @return Promise + */ +Response.prototype.json = function() { + + return this._decode().then(function(text) { + return JSON.parse(text); + }); + +} + +/** + * Decode response as text + * + * @return Promise + */ +Response.prototype.text = function() { + + return this._decode(); + +} + +/** + * Decode buffers into utf-8 string + * + * @return Promise + */ +Response.prototype._decode = function() { + + var self = this; + + if (this.bodyUsed) { + return Response.Promise.reject(new Error('body used already for: ' + this.url)); + } + + this.bodyUsed = true; + this._bytes = 0; + this._abort = false; + this._raw = []; + + return new Response.Promise(function(resolve, reject) { + self.body.on('data', function(chunk) { + if (chunk === null) { + return; + } + + if (self.size && self._bytes > self.size) { + self._abort = true; + reject(new Error('content size at ' + self.url + ' over limit: ' + self.size)); + self.body.abort(); + return; + } + + self._bytes += chunk.length; + self._raw.push(chunk); + }); + + self.body.on('end', function() { + if (self._abort) { + return; + } + + resolve(self._convert()); + }); + }); + +}; + +/** + * Detect buffer encoding and convert to target encoding + * ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding + * + * @param String encoding Target encoding + * @return String + */ +Response.prototype._convert = function(encoding) { + + encoding = encoding || 'utf-8'; + + var charset = 'utf-8'; + var res; + + // header + if (this.headers.has('content-type')) { + res = /charset=(.*)/i.exec(this.headers.get('content-type')); + } + + // html5 + if (!res && this._raw.length > 0) { + res = / 0) { + res = /