blockworks-foundation
/
node-fetch
mirror of https://github.com/blockworks-foundation/node-fetch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #141 from bitinn/august-bugfix 

August bugfix, see commit for details
This commit is contained in:
David Frank 2016-08-03 18:07:51 +08:00 committed by GitHub
parent 952aa2a135 8cce714b48
commit ed9a72d84e
7 changed files with 183 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
CHANGELOG.md
View File

@ -5,7 +5,16 @@ Changelog
# 1.x release
## v1.5.3 (master)
## v1.6.0 (master)
- Enhance: added res.buffer() api for convenience, it returns body as a Node.js buffer
- Enhance: better old server support by handling raw deflate response
- Enhance: skip encoding detection for non-HTML/XML response
- Enhance: minor document update
- Fix: HEAD request doesn't need decompression, as body is empty
- Fix: req.body now accepts a Node.js buffer
## v1.5.3
- Fix: handles 204 and 304 responses when body is empty but content-encoding is gzip/deflate
- Fix: allow resolving response and cloned response in any order

4
LIMITS.md
View File

@ -12,10 +12,10 @@ Known differences
- For convenience, `res.body` is a transform stream, so decoding can be handled independently.
- Similarly, `req.body` can either be a string or a readable stream.
- Similarly, `req.body` can either be a string, a buffer or a readable stream.
- Also, you can handle rejected fetch requests through checking `err.type` and `err.code`.
- Only support `res.text()` and `res.json()` at the moment, until there are good use-cases for blob.
- Only support `res.text()`, `res.json()`, `res.buffer()` at the moment, until there are good use-cases for blob/arrayBuffer.
- There is currently no built-in caching, as server-side caching varies by use-cases.

17
README.md
View File

@ -11,11 +11,9 @@ A light-weight module that brings `window.fetch` to Node.js
# Motivation
I really like the notion of Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch): it bridges the API gap between client-side and server-side http requests, so developers have less to worry about.
Instead of implementing `XMLHttpRequest` in Node.js to run browser-specific [Fetch polyfill](https://github.com/github/fetch), why not go from native `http` to `Fetch` API directly? Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js runtime.
Instead of implementing `XMLHttpRequest` in node to run browser-specific [fetch polyfill](https://github.com/github/fetch), why not go from node's `http` to `fetch` API directly? Node has native stream support, browserify build targets (browsers) don't, so underneath they are going to be vastly different anyway.
Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js runtime.
See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch) for isomorphic usage (exports `node-fetch` for server-side, `whatwg-fetch` for client-side).
# Features
@ -24,8 +22,8 @@ Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js
- Make conscious trade-off when following [whatwg fetch spec](https://fetch.spec.whatwg.org/) and [stream spec](https://streams.spec.whatwg.org/) implementation details, document known difference.
- Use native promise, but allow substituting it with [insert your favorite promise library].
- Use native stream for body, on both request and response.
- Decode content encoding (gzip/deflate) properly, and convert string output (such as `res.text()` and `res.json()`) to utf-8 automatically.
- Useful extensions such as timeout, redirect limit, response size limit, explicit reject errors.
- Decode content encoding (gzip/deflate) properly, and convert string output (such as `res.text()` and `res.json()`) to UTF-8 automatically.
- Useful extensions such as timeout, redirect limit, response size limit, explicit errors for troubleshooting.
# Difference from client-side fetch
@ -45,7 +43,7 @@ Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js
```javascript
var fetch = require('node-fetch');
// If you are not on node v0.12, set a Promise library first, eg.
// if you are on node v0.10, set a Promise library first, eg.
// fetch.Promise = require('bluebird');
// plain text or html
@ -66,6 +64,8 @@ fetch('https://api.github.com/users/github')
console.log(json);
});
// since v1.6.0, there is also res.buffer() for convenience
// meta
fetch('https://github.com/')
@ -110,6 +110,7 @@ fetch('http://httpbin.org/post', { method: 'POST', body: form })
});
// post with form-data (custom headers)
// note that getHeaders() is non-standard api
var FormData = require('form-data');
var form = new FormData();
@ -121,7 +122,7 @@ fetch('http://httpbin.org/post', { method: 'POST', body: form, headers: form.get
console.log(json);
});
// node 0.11+, yield with co
// node 0.12+, yield with co
var co = require('co');
co(function *() {

76
index.js
View File

@ -162,45 +162,79 @@ function Fetch(url, opts) {
return;
}
// handle compression
var body = res.pipe(new stream.PassThrough());
var headers = new Headers(res.headers);
if (options.compress && headers.has('content-encoding')) {
var name = headers.get('content-encoding');
// no need to pipe no content and not modified response body
if (res.statusCode !== 204 && res.statusCode !== 304) {
if (name == 'gzip' || name == 'x-gzip') {
body = body.pipe(zlib.createGunzip());
} else if (name == 'deflate' || name == 'x-deflate') {
body = body.pipe(zlib.createInflate());
}
}
}
// normalize location header for manual redirect mode
var headers = new Headers(res.headers);
if (options.redirect === 'manual' && headers.has('location')) {
headers.set('location', resolve_url(options.url, headers.get('location')));
}
// response object
var output = new Response(body, {
// prepare response
var body = res.pipe(new stream.PassThrough());
var response_options = {
url: options.url
, status: res.statusCode
, statusText: res.statusMessage
, headers: headers
, size: options.size
, timeout: options.timeout
});
};
// response object
var output;
// in following scenarios we ignore compression support
// 1. compression support is disabled
// 2. HEAD request
// 3. no content-encoding header
// 4. no content response (204)
// 5. content not modified response (304)
if (!options.compress || options.method === 'HEAD' || !headers.has('content-encoding') || res.statusCode === 204 || res.statusCode === 304) {
output = new Response(body, response_options);
resolve(output);
return;
}
// otherwise, check for gzip or deflate
var name = headers.get('content-encoding');
// for gzip
if (name == 'gzip' || name == 'x-gzip') {
body = body.pipe(zlib.createGunzip());
output = new Response(body, response_options);
resolve(output);
return;
// for deflate
} else if (name == 'deflate' || name == 'x-deflate') {
// handle the infamous raw deflate response from old servers
// a hack for old IIS and Apache servers
var raw = res.pipe(new stream.PassThrough());
raw.once('data', function(chunk) {
// see http://stackoverflow.com/questions/37519828
if ((chunk[0] & 0x0F) === 0x08) {
body = body.pipe(zlib.createInflate());
} else {
body = body.pipe(zlib.createInflateRaw());
}
output = new Response(body, response_options);
resolve(output);
});
return;
}
// otherwise, use response as-is
output = new Response(body, response_options);
resolve(output);
return;
});
// accept string or readable stream as body
// accept string, buffer or readable stream as body
if (typeof options.body === 'string') {
req.write(options.body);
req.end();
} else if (options.body instanceof Buffer) {
req.write(options.body);
req.end()
} else if (typeof options.body === 'object' && options.body.pipe) {
options.body.pipe(req);
} else {

34
lib/body.js
View File

@ -39,8 +39,8 @@ function Body(body, opts) {
*/
Body.prototype.json = function() {
return this._decode().then(function(text) {
return JSON.parse(text);
return this._decode().then(function(buffer) {
return JSON.parse(buffer.toString());
});
};
@ -52,6 +52,19 @@ Body.prototype.json = function() {
*/
Body.prototype.text = function() {
return this._decode().then(function(buffer) {
return buffer.toString();
});
};
/**
* Decode response as buffer (non-spec api)
*
* @return Promise
*/
Body.prototype.buffer = function() {
return this._decode();
};
@ -77,12 +90,14 @@ Body.prototype._decode = function() {
return new Body.Promise(function(resolve, reject) {
var resTimeout;
// body is string
if (typeof self.body === 'string') {
self._bytes = self.body.length;
self._raw = [new Buffer(self.body)];
return resolve(self._convert());
}
// body is buffer
if (self.body instanceof Buffer) {
self._bytes = self.body.length;
self._raw = [self.body];
@ -102,6 +117,7 @@ Body.prototype._decode = function() {
reject(new FetchError('invalid response body at: ' + self.url + ' reason: ' + err.message, 'system', err));
});
// body is stream
self.body.on('data', function(chunk) {
if (self._abort || chunk === null) {
return;
@ -140,12 +156,18 @@ Body.prototype._convert = function(encoding) {
encoding = encoding || 'utf-8';
var ct = this.headers.get('content-type');
var charset = 'utf-8';
var res, str;
// header
if (this.headers.has('content-type')) {
res = /charset=([^;]*)/i.exec(this.headers.get('content-type'));
if (ct) {
// skip encoding detection altogether if not html/xml/plain text
if (!/text\/html|text\/plain|\+xml|\/xml/i.test(ct)) {
return Buffer.concat(this._raw);
}
res = /charset=([^;]*)/i.exec(ct);
}
// no charset in content type, peek at response body for at most 1024 bytes
@ -189,12 +211,12 @@ Body.prototype._convert = function(encoding) {
}
}
// turn raw buffers into utf-8 string
// turn raw buffers into a single utf-8 buffer
return convert(
Buffer.concat(this._raw)
, encoding
, charset
).toString();
);
};

15
test/server.js
View File

@ -82,6 +82,15 @@ TestServer.prototype.router = function(req, res) {
});
}
if (p === '/deflate-raw') {
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
res.setHeader('Content-Encoding', 'deflate');
zlib.deflateRaw('hello world', function(err, buffer) {
res.end(buffer);
});
}
if (p === '/sdch') {
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain');
@ -248,6 +257,12 @@ TestServer.prototype.router = function(req, res) {
res.end('client error');
}
if (p === '/error/404') {
res.statusCode = 404;
res.setHeader('Content-Encoding', 'gzip');
res.end();
}
if (p === '/error/500') {
res.statusCode = 500;
res.setHeader('Content-Type', 'text/plain');

65
test/test.js
View File

@ -493,6 +493,17 @@ describe('node-fetch', function() {
});
});
it('should decompress deflate raw response from old apache server', function() {
url = base + '/deflate-raw';
return fetch(url).then(function(res) {
expect(res.headers.get('content-type')).to.equal('text/plain');
return res.text().then(function(result) {
expect(result).to.be.a('string');
expect(result).to.equal('hello world');
});
});
});
it('should skip decompression if unsupported', function() {
url = base + '/sdch';
return fetch(url).then(function(res) {
@ -607,6 +618,22 @@ describe('node-fetch', function() {
});
});
it('should allow POST request with buffer body', function() {
url = base + '/inspect';
opts = {
method: 'POST'
, body: new Buffer('a=1', 'utf-8')
};
return fetch(url, opts).then(function(res) {
return res.json();
}).then(function(res) {
expect(res.method).to.equal('POST');
expect(res.body).to.equal('a=1');
expect(res.headers['transfer-encoding']).to.equal('chunked');
expect(res.headers['content-length']).to.be.undefined;
});
});
it('should allow POST request with readable stream as body', function() {
var body = resumer().queue('a=1').end();
body = body.pipe(new stream.PassThrough());
@ -771,6 +798,23 @@ describe('node-fetch', function() {
expect(res.statusText).to.equal('OK');
expect(res.headers.get('content-type')).to.equal('text/plain');
expect(res.body).to.be.an.instanceof(stream.Transform);
return res.text();
}).then(function(text) {
expect(text).to.equal('');
});
});
it('should allow HEAD request with content-encoding header', function() {
url = base + '/error/404';
opts = {
method: 'HEAD'
};
return fetch(url, opts).then(function(res) {
expect(res.status).to.equal(404);
expect(res.headers.get('content-encoding')).to.equal('gzip');
return res.text();
}).then(function(text) {
expect(text).to.equal('');
});
});
@ -1227,6 +1271,13 @@ describe('node-fetch', function() {
});
});
it('should support buffer() method in Response constructor', function() {
var res = new Response('a=1');
return res.buffer().then(function(result) {
expect(result.toString()).to.equal('a=1');
});
});
it('should support clone() method in Response constructor', function() {
var body = resumer().queue('a=1').end();
body = body.pipe(new stream.PassThrough());
@ -1307,6 +1358,17 @@ describe('node-fetch', function() {
});
});

it('should support buffer() method in Request constructor', function() {
url = base;
var req = new Request(url, {
body: 'a=1'
});
expect(req.url).to.equal(url);
return req.buffer().then(function(result) {
expect(result.toString()).to.equal('a=1');
});
});

it('should support arbitrary url in Request constructor', function() {
url = 'anything';
var req = new Request(url);
@ -1347,10 +1409,11 @@ describe('node-fetch', function() {
});
});
it('should support text() and json() method in Body constructor', function() {
it('should support text(), json() and buffer() method in Body constructor', function() {
var body = new Body('a=1');
expect(body).to.have.property('text');
expect(body).to.have.property('json');
expect(body).to.have.property('buffer');
});

it('should create custom FetchError', function() {