diff --git a/CHANGELOG.md b/CHANGELOG.md index 06be686..b8f05a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,16 @@ Changelog # 1.x release -## v1.5.3 (master) +## v1.6.0 (master) + +- Enhance: added res.buffer() api for convenience, it returns body as a Node.js buffer +- Enhance: better old server support by handling raw deflate response +- Enhance: skip encoding detection for non-HTML/XML response +- Enhance: minor document update +- Fix: HEAD request doesn't need decompression, as body is empty +- Fix: req.body now accepts a Node.js buffer + +## v1.5.3 - Fix: handles 204 and 304 responses when body is empty but content-encoding is gzip/deflate - Fix: allow resolving response and cloned response in any order diff --git a/LIMITS.md b/LIMITS.md index 18b1bfb..5b33489 100644 --- a/LIMITS.md +++ b/LIMITS.md @@ -12,10 +12,10 @@ Known differences - For convenience, `res.body` is a transform stream, so decoding can be handled independently. -- Similarly, `req.body` can either be a string or a readable stream. +- Similarly, `req.body` can either be a string, a buffer or a readable stream. - Also, you can handle rejected fetch requests through checking `err.type` and `err.code`. -- Only support `res.text()` and `res.json()` at the moment, until there are good use-cases for blob. +- Only support `res.text()`, `res.json()`, `res.buffer()` at the moment, until there are good use-cases for blob/arrayBuffer. - There is currently no built-in caching, as server-side caching varies by use-cases. diff --git a/README.md b/README.md index 0176c42..8f55e21 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,9 @@ A light-weight module that brings `window.fetch` to Node.js # Motivation -I really like the notion of Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch): it bridges the API gap between client-side and server-side http requests, so developers have less to worry about. +Instead of implementing `XMLHttpRequest` in Node.js to run browser-specific [Fetch polyfill](https://github.com/github/fetch), why not go from native `http` to `Fetch` API directly? Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js runtime. -Instead of implementing `XMLHttpRequest` in node to run browser-specific [fetch polyfill](https://github.com/github/fetch), why not go from node's `http` to `fetch` API directly? Node has native stream support, browserify build targets (browsers) don't, so underneath they are going to be vastly different anyway. - -Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js runtime. +See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch) for isomorphic usage (exports `node-fetch` for server-side, `whatwg-fetch` for client-side). # Features @@ -24,8 +22,8 @@ Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js - Make conscious trade-off when following [whatwg fetch spec](https://fetch.spec.whatwg.org/) and [stream spec](https://streams.spec.whatwg.org/) implementation details, document known difference. - Use native promise, but allow substituting it with [insert your favorite promise library]. - Use native stream for body, on both request and response. -- Decode content encoding (gzip/deflate) properly, and convert string output (such as `res.text()` and `res.json()`) to utf-8 automatically. -- Useful extensions such as timeout, redirect limit, response size limit, explicit reject errors. +- Decode content encoding (gzip/deflate) properly, and convert string output (such as `res.text()` and `res.json()`) to UTF-8 automatically. +- Useful extensions such as timeout, redirect limit, response size limit, explicit errors for troubleshooting. # Difference from client-side fetch @@ -45,7 +43,7 @@ Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js ```javascript var fetch = require('node-fetch'); -// If you are not on node v0.12, set a Promise library first, eg. +// if you are on node v0.10, set a Promise library first, eg. // fetch.Promise = require('bluebird'); // plain text or html @@ -66,6 +64,8 @@ fetch('https://api.github.com/users/github') console.log(json); }); +// since v1.6.0, there is also res.buffer() for convenience + // meta fetch('https://github.com/') @@ -110,6 +110,7 @@ fetch('http://httpbin.org/post', { method: 'POST', body: form }) }); // post with form-data (custom headers) +// note that getHeaders() is non-standard api var FormData = require('form-data'); var form = new FormData(); @@ -121,7 +122,7 @@ fetch('http://httpbin.org/post', { method: 'POST', body: form, headers: form.get console.log(json); }); -// node 0.11+, yield with co +// node 0.12+, yield with co var co = require('co'); co(function *() { diff --git a/index.js b/index.js index 10fb8ff..e233c2f 100644 --- a/index.js +++ b/index.js @@ -162,45 +162,79 @@ function Fetch(url, opts) { return; } - // handle compression - var body = res.pipe(new stream.PassThrough()); - var headers = new Headers(res.headers); - - if (options.compress && headers.has('content-encoding')) { - var name = headers.get('content-encoding'); - - // no need to pipe no content and not modified response body - if (res.statusCode !== 204 && res.statusCode !== 304) { - if (name == 'gzip' || name == 'x-gzip') { - body = body.pipe(zlib.createGunzip()); - } else if (name == 'deflate' || name == 'x-deflate') { - body = body.pipe(zlib.createInflate()); - } - } - } - // normalize location header for manual redirect mode + var headers = new Headers(res.headers); if (options.redirect === 'manual' && headers.has('location')) { headers.set('location', resolve_url(options.url, headers.get('location'))); } - // response object - var output = new Response(body, { + // prepare response + var body = res.pipe(new stream.PassThrough()); + var response_options = { url: options.url , status: res.statusCode , statusText: res.statusMessage , headers: headers , size: options.size , timeout: options.timeout - }); + }; + // response object + var output; + + // in following scenarios we ignore compression support + // 1. compression support is disabled + // 2. HEAD request + // 3. no content-encoding header + // 4. no content response (204) + // 5. content not modified response (304) + if (!options.compress || options.method === 'HEAD' || !headers.has('content-encoding') || res.statusCode === 204 || res.statusCode === 304) { + output = new Response(body, response_options); + resolve(output); + return; + } + + // otherwise, check for gzip or deflate + var name = headers.get('content-encoding'); + + // for gzip + if (name == 'gzip' || name == 'x-gzip') { + body = body.pipe(zlib.createGunzip()); + output = new Response(body, response_options); + resolve(output); + return; + + // for deflate + } else if (name == 'deflate' || name == 'x-deflate') { + // handle the infamous raw deflate response from old servers + // a hack for old IIS and Apache servers + var raw = res.pipe(new stream.PassThrough()); + raw.once('data', function(chunk) { + // see http://stackoverflow.com/questions/37519828 + if ((chunk[0] & 0x0F) === 0x08) { + body = body.pipe(zlib.createInflate()); + } else { + body = body.pipe(zlib.createInflateRaw()); + } + output = new Response(body, response_options); + resolve(output); + }); + return; + } + + // otherwise, use response as-is + output = new Response(body, response_options); resolve(output); + return; }); - // accept string or readable stream as body + // accept string, buffer or readable stream as body if (typeof options.body === 'string') { req.write(options.body); req.end(); + } else if (options.body instanceof Buffer) { + req.write(options.body); + req.end() } else if (typeof options.body === 'object' && options.body.pipe) { options.body.pipe(req); } else { diff --git a/lib/body.js b/lib/body.js index 8e69b5d..6477da0 100644 --- a/lib/body.js +++ b/lib/body.js @@ -39,8 +39,8 @@ function Body(body, opts) { */ Body.prototype.json = function() { - return this._decode().then(function(text) { - return JSON.parse(text); + return this._decode().then(function(buffer) { + return JSON.parse(buffer.toString()); }); }; @@ -52,6 +52,19 @@ Body.prototype.json = function() { */ Body.prototype.text = function() { + return this._decode().then(function(buffer) { + return buffer.toString(); + }); + +}; + +/** + * Decode response as buffer (non-spec api) + * + * @return Promise + */ +Body.prototype.buffer = function() { + return this._decode(); }; @@ -77,12 +90,14 @@ Body.prototype._decode = function() { return new Body.Promise(function(resolve, reject) { var resTimeout; + // body is string if (typeof self.body === 'string') { self._bytes = self.body.length; self._raw = [new Buffer(self.body)]; return resolve(self._convert()); } + // body is buffer if (self.body instanceof Buffer) { self._bytes = self.body.length; self._raw = [self.body]; @@ -102,6 +117,7 @@ Body.prototype._decode = function() { reject(new FetchError('invalid response body at: ' + self.url + ' reason: ' + err.message, 'system', err)); }); + // body is stream self.body.on('data', function(chunk) { if (self._abort || chunk === null) { return; @@ -140,12 +156,18 @@ Body.prototype._convert = function(encoding) { encoding = encoding || 'utf-8'; + var ct = this.headers.get('content-type'); var charset = 'utf-8'; var res, str; // header - if (this.headers.has('content-type')) { - res = /charset=([^;]*)/i.exec(this.headers.get('content-type')); + if (ct) { + // skip encoding detection altogether if not html/xml/plain text + if (!/text\/html|text\/plain|\+xml|\/xml/i.test(ct)) { + return Buffer.concat(this._raw); + } + + res = /charset=([^;]*)/i.exec(ct); } // no charset in content type, peek at response body for at most 1024 bytes @@ -189,12 +211,12 @@ Body.prototype._convert = function(encoding) { } } - // turn raw buffers into utf-8 string + // turn raw buffers into a single utf-8 buffer return convert( Buffer.concat(this._raw) , encoding , charset - ).toString(); + ); }; diff --git a/test/server.js b/test/server.js index c57cfb0..08e582d 100644 --- a/test/server.js +++ b/test/server.js @@ -82,6 +82,15 @@ TestServer.prototype.router = function(req, res) { }); } + if (p === '/deflate-raw') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Content-Encoding', 'deflate'); + zlib.deflateRaw('hello world', function(err, buffer) { + res.end(buffer); + }); + } + if (p === '/sdch') { res.statusCode = 200; res.setHeader('Content-Type', 'text/plain'); @@ -248,6 +257,12 @@ TestServer.prototype.router = function(req, res) { res.end('client error'); } + if (p === '/error/404') { + res.statusCode = 404; + res.setHeader('Content-Encoding', 'gzip'); + res.end(); + } + if (p === '/error/500') { res.statusCode = 500; res.setHeader('Content-Type', 'text/plain'); diff --git a/test/test.js b/test/test.js index 2251023..84d20af 100644 --- a/test/test.js +++ b/test/test.js @@ -493,6 +493,17 @@ describe('node-fetch', function() { }); }); + it('should decompress deflate raw response from old apache server', function() { + url = base + '/deflate-raw'; + return fetch(url).then(function(res) { + expect(res.headers.get('content-type')).to.equal('text/plain'); + return res.text().then(function(result) { + expect(result).to.be.a('string'); + expect(result).to.equal('hello world'); + }); + }); + }); + it('should skip decompression if unsupported', function() { url = base + '/sdch'; return fetch(url).then(function(res) { @@ -607,6 +618,22 @@ describe('node-fetch', function() { }); }); + it('should allow POST request with buffer body', function() { + url = base + '/inspect'; + opts = { + method: 'POST' + , body: new Buffer('a=1', 'utf-8') + }; + return fetch(url, opts).then(function(res) { + return res.json(); + }).then(function(res) { + expect(res.method).to.equal('POST'); + expect(res.body).to.equal('a=1'); + expect(res.headers['transfer-encoding']).to.equal('chunked'); + expect(res.headers['content-length']).to.be.undefined; + }); + }); + it('should allow POST request with readable stream as body', function() { var body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); @@ -771,6 +798,23 @@ describe('node-fetch', function() { expect(res.statusText).to.equal('OK'); expect(res.headers.get('content-type')).to.equal('text/plain'); expect(res.body).to.be.an.instanceof(stream.Transform); + return res.text(); + }).then(function(text) { + expect(text).to.equal(''); + }); + }); + + it('should allow HEAD request with content-encoding header', function() { + url = base + '/error/404'; + opts = { + method: 'HEAD' + }; + return fetch(url, opts).then(function(res) { + expect(res.status).to.equal(404); + expect(res.headers.get('content-encoding')).to.equal('gzip'); + return res.text(); + }).then(function(text) { + expect(text).to.equal(''); }); }); @@ -1227,6 +1271,13 @@ describe('node-fetch', function() { }); }); + it('should support buffer() method in Response constructor', function() { + var res = new Response('a=1'); + return res.buffer().then(function(result) { + expect(result.toString()).to.equal('a=1'); + }); + }); + it('should support clone() method in Response constructor', function() { var body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); @@ -1307,6 +1358,17 @@ describe('node-fetch', function() { }); });
 + it('should support buffer() method in Request constructor', function() { + url = base; + var req = new Request(url, { + body: 'a=1' + }); + expect(req.url).to.equal(url); + return req.buffer().then(function(result) { + expect(result.toString()).to.equal('a=1'); + }); + });
 + it('should support arbitrary url in Request constructor', function() { url = 'anything'; var req = new Request(url); @@ -1347,10 +1409,11 @@ describe('node-fetch', function() { }); }); - it('should support text() and json() method in Body constructor', function() { + it('should support text(), json() and buffer() method in Body constructor', function() { var body = new Body('a=1'); expect(body).to.have.property('text'); expect(body).to.have.property('json'); + expect(body).to.have.property('buffer'); });
 it('should create custom FetchError', function() {