Merge pull request #90 from bitinn/prepare-new-release

Prepare new release v1.4.0
This commit is contained in:
David Frank 2016-03-19 19:01:10 +08:00
commit 083c31f126
10 changed files with 441 additions and 176 deletions

View File

@ -2,7 +2,6 @@ language: node_js
node_js:
- "0.10"
- "0.12"
- "iojs"
- "node"
before_install: npm install -g npm
script: npm run coverage

View File

@ -5,7 +5,17 @@ Changelog
# 1.x release
## v1.3.3 (master)
## v1.4.0 (master)
- Enhance: Request and Response now have `clone` method (thx to @kirill-konshin for the initial PR)
- Enhance: Request and Response now have proper string and buffer body support (thx to @kirill-konshin)
- Enhance: Body constructor has been refactored out (thx to @kirill-konshin)
- Enhance: Headers now has `forEach` method (thx to @tricoder42)
- Enhance: back to 100% code coverage
- Fix: better form-data support (thx to @item4)
- Fix: better character encoding detection under chunked encoding (thx to @dsuket for the initial PR)
## v1.3.3
- Fix: make sure `Content-Length` header is set when body is string for POST/PUT/PATCH requests
- Fix: handle body stream error, for cases such as incorrect `Content-Encoding` header

View File

@ -17,7 +17,9 @@ var Response = require('./lib/response');
var Headers = require('./lib/headers');
var Request = require('./lib/request');
// commonjs
module.exports = Fetch;
// es6 default export compatibility
module.exports.default = module.exports;
/**
@ -170,6 +172,7 @@ function Fetch(url, opts) {
var output = new Response(body, {
url: options.url
, status: res.statusCode
, statusText: res.statusMessage
, headers: headers
, size: options.size
, timeout: options.timeout

View File

@ -5,6 +5,8 @@
*/
var convert = require('encoding').convert;
var bodyStream = require('is-stream');
var PassThrough = require('stream').PassThrough;
module.exports = Body;
@ -17,14 +19,14 @@ module.exports = Body;
*/
function Body(body, opts) {
opts = opts || {};
opts = opts || {};
this.body = body;
this.bodyUsed = false;
this.size = opts.size || 0;
this.timeout = opts.timeout || 0;
this._raw = [];
this._abort = false;
this.body = body;
this.bodyUsed = false;
this.size = opts.size || 0;
this.timeout = opts.timeout || 0;
this._raw = [];
this._abort = false;
}
@ -35,9 +37,9 @@ function Body(body, opts) {
*/
Body.prototype.json = function() {
return this._decode().then(function(text) {
return JSON.parse(text);
});
return this._decode().then(function(text) {
return JSON.parse(text);
});
};
@ -48,7 +50,7 @@ Body.prototype.json = function() {
*/
Body.prototype.text = function() {
return this._decode();
return this._decode();
};
@ -59,69 +61,69 @@ Body.prototype.text = function() {
*/
Body.prototype._decode = function() {
var self = this;
var self = this;
if (this.bodyUsed) {
return Body.Promise.reject(new Error('body used already for: ' + this.url));
}
if (this.bodyUsed) {
return Body.Promise.reject(new Error('body used already for: ' + this.url));
}
this.bodyUsed = true;
this._bytes = 0;
this._abort = false;
this._raw = [];
this.bodyUsed = true;
this._bytes = 0;
this._abort = false;
this._raw = [];
return new Body.Promise(function(resolve, reject) {
var resTimeout;
return new Body.Promise(function(resolve, reject) {
var resTimeout;
if (typeof self.body === 'string') {
self._bytes = self.body.length;
self._raw = [new Buffer(self.body)];
return resolve(self._convert());
}
if (typeof self.body === 'string') {
self._bytes = self.body.length;
self._raw = [new Buffer(self.body)];
return resolve(self._convert());
}
if (self.body instanceof Buffer) {
self._bytes = self.body.length;
self._raw = [self.body];
return resolve(self._convert());
}
if (self.body instanceof Buffer) {
self._bytes = self.body.length;
self._raw = [self.body];
return resolve(self._convert());
}
// allow timeout on slow response body
if (self.timeout) {
resTimeout = setTimeout(function() {
self._abort = true;
reject(new Error('response timeout at ' + self.url + ' over limit: ' + self.timeout));
}, self.timeout);
}
// allow timeout on slow response body
if (self.timeout) {
resTimeout = setTimeout(function() {
self._abort = true;
reject(new Error('response timeout at ' + self.url + ' over limit: ' + self.timeout));
}, self.timeout);
}
// handle stream error, such as incorrect content-encoding
self.body.on('error', function(err) {
reject(new Error('invalid response body at: ' + self.url + ' reason: ' + err.message));
});
// handle stream error, such as incorrect content-encoding
self.body.on('error', function(err) {
reject(new Error('invalid response body at: ' + self.url + ' reason: ' + err.message));
});
self.body.on('data', function(chunk) {
if (self._abort || chunk === null) {
return;
}
self.body.on('data', function(chunk) {
if (self._abort || chunk === null) {
return;
}
if (self.size && self._bytes + chunk.length > self.size) {
self._abort = true;
reject(new Error('content size at ' + self.url + ' over limit: ' + self.size));
return;
}
if (self.size && self._bytes + chunk.length > self.size) {
self._abort = true;
reject(new Error('content size at ' + self.url + ' over limit: ' + self.size));
return;
}
self._bytes += chunk.length;
self._raw.push(chunk);
});
self._bytes += chunk.length;
self._raw.push(chunk);
});
self.body.on('end', function() {
if (self._abort) {
return;
}
self.body.on('end', function() {
if (self._abort) {
return;
}
clearTimeout(resTimeout);
resolve(self._convert());
});
});
clearTimeout(resTimeout);
resolve(self._convert());
});
});
};
@ -134,59 +136,88 @@ Body.prototype._decode = function() {
*/
Body.prototype._convert = function(encoding) {
encoding = encoding || 'utf-8';
encoding = encoding || 'utf-8';
var charset = 'utf-8';
var res, str;
var charset = 'utf-8';
var res, str;
// header
if (this.headers.has('content-type')) {
res = /charset=([^;]*)/i.exec(this.headers.get('content-type'));
}
// header
if (this.headers.has('content-type')) {
res = /charset=([^;]*)/i.exec(this.headers.get('content-type'));
}
// no charset in content type, peek at response body
if (!res && this._raw.length > 0) {
str = this._raw[0].toString().substr(0, 1024);
}
// no charset in content type, peek at response body for at most 1024 bytes
if (!res && this._raw.length > 0) {
for (var i = 0; i < this._raw.length; i++) {
str += this._raw[i].toString()
if (str.length > 1024) {
break;
}
}
str = str.substr(0, 1024);
}
// html5
if (!res && str) {
res = /<meta.+?charset=(['"])(.+?)\1/i.exec(str);
}
// html5
if (!res && str) {
res = /<meta.+?charset=(['"])(.+?)\1/i.exec(str);
}
// html4
if (!res && str) {
res = /<meta[\s]+?http-equiv=(['"])content-type\1[\s]+?content=(['"])(.+?)\2/i.exec(str);
// html4
if (!res && str) {
res = /<meta[\s]+?http-equiv=(['"])content-type\1[\s]+?content=(['"])(.+?)\2/i.exec(str);
if (res) {
res = /charset=(.*)/i.exec(res.pop());
}
}
if (res) {
res = /charset=(.*)/i.exec(res.pop());
}
}
// xml
if (!res && str) {
res = /<\?xml.+?encoding=(['"])(.+?)\1/i.exec(str);
}
// xml
if (!res && str) {
res = /<\?xml.+?encoding=(['"])(.+?)\1/i.exec(str);
}
// found charset
if (res) {
charset = res.pop();
// found charset
if (res) {
charset = res.pop();
// prevent decode issues when sites use incorrect encoding
// ref: https://hsivonen.fi/encoding-menu/
if (charset === 'gb2312' || charset === 'gbk') {
charset = 'gb18030';
}
}
// prevent decode issues when sites use incorrect encoding
// ref: https://hsivonen.fi/encoding-menu/
if (charset === 'gb2312' || charset === 'gbk') {
charset = 'gb18030';
}
}
// turn raw buffers into utf-8 string
return convert(
Buffer.concat(this._raw)
, encoding
, charset
).toString();
// turn raw buffers into utf-8 string
return convert(
Buffer.concat(this._raw)
, encoding
, charset
).toString();
};
/**
* Clone body given Res/Req instance
*
* @param Mixed instance Response or Request instance
* @return Mixed
*/
Body.prototype._clone = function(instance) {
var pass;
var body = instance.body;
if (instance.bodyUsed) {
throw new Error('cannot clone body after it is used');
}
if (bodyStream(body)) {
pass = new PassThrough();
body.pipe(pass);
body = pass;
}
return body;
}
// expose Promise
Body.Promise = global.Promise;
Body.Promise = global.Promise;

View File

@ -70,15 +70,16 @@ Headers.prototype.getAll = function(name) {
/**
* Iterate over all headers
* @param Function callback Executed for each item with parameters (value, name, thisArg)
* @param Boolean thisArg `this` context for callback function
* @return Void
*
* @param Function callback Executed for each item with parameters (value, name, thisArg)
* @param Boolean thisArg `this` context for callback function
* @return Void
*/
Headers.prototype.forEach = function(callback, thisArg) {
Object.getOwnPropertyNames(this._headers).forEach(function(name) {
this._headers[name].forEach(function(value) {
callback.call(thisArg, value, name, this)
}, this)
callback.call(thisArg, value, name, this)
}, this)
}, this)
}
@ -136,4 +137,4 @@ Headers.prototype['delete'] = function(name) {
*/
Headers.prototype.raw = function() {
return this._headers;
};
};

View File

@ -50,13 +50,13 @@ function Request(input, init) {
this.follow = init.follow !== undefined ?
init.follow : input.follow !== undefined ?
input.follow : 20;
this.counter = init.counter || input.follow || 0;
this.compress = init.compress !== undefined ?
init.compress : input.compress !== undefined ?
input.compress : true;
this.agent = init.agent || input.agent;
this.counter = init.counter || input.counter || input.follow || 0;
this.agent = init.agent || input.agent || input.agent;
Body.call(this, init.body || input.body, {
Body.call(this, init.body || this._clone(input), {
timeout: init.timeout || input.timeout || 0,
size: init.size || input.size || 0
});
@ -70,3 +70,12 @@ function Request(input, init) {
}
Request.prototype = Object.create(Body.prototype);
/**
* Clone this request
*
* @return Request
*/
Request.prototype.clone = function() {
return new Request(this);
};

View File

@ -5,7 +5,6 @@
*/
var http = require('http');
var convert = require('encoding').convert;
var Headers = require('./headers');
var Body = require('./body');
@ -24,7 +23,7 @@ function Response(body, opts) {
this.url = opts.url;
this.status = opts.status;
this.statusText = http.STATUS_CODES[this.status];
this.statusText = opts.statusText || http.STATUS_CODES[this.status];
this.headers = new Headers(opts.headers);
this.ok = this.status >= 200 && this.status < 300;
@ -34,5 +33,17 @@ function Response(body, opts) {
Response.prototype = Object.create(Body.prototype);
// expose Promise
Response.Promise = global.Promise;
/**
* Clone this response
*
* @return Response
*/
Response.prototype.clone = function() {
return new Response(this._clone(this), {
url: this.url
, status: this.status
, statusText: this.statusText
, headers: this.headers
, ok: this.ok
});
};

View File

@ -24,18 +24,19 @@
},
"homepage": "https://github.com/bitinn/node-fetch",
"devDependencies": {
"bluebird": "^2.9.1",
"chai": "^1.10.0",
"chai-as-promised": "^4.1.1",
"bluebird": "^3.3.4",
"chai": "^3.5.0",
"chai-as-promised": "^5.2.0",
"coveralls": "^2.11.2",
"form-data": "^1.0.0-rc1",
"istanbul": "^0.3.5",
"istanbul": "^0.4.2",
"mocha": "^2.1.0",
"parted": "^0.1.1",
"promise": "^6.1.0",
"promise": "^7.1.1",
"resumer": "0.0.0"
},
"dependencies": {
"encoding": "^0.1.11"
"encoding": "^0.1.11",
"is-stream": "^1.0.1"
}
}

View File

@ -41,6 +41,12 @@ TestServer.prototype.router = function(req, res) {
res.end('text');
}
if (p === '/options') {
res.statusCode = 200;
res.setHeader('Allow', 'GET, HEAD, OPTIONS');
res.end('hello world');
}
if (p === '/html') {
res.statusCode = 200;
res.setHeader('Content-Type', 'text/html');
@ -168,6 +174,29 @@ TestServer.prototype.router = function(req, res) {
res.end(convert('中文', 'gbk'));
}
if (p === '/encoding/chunked') {
res.statusCode = 200;
res.setHeader('Content-Type', 'text/html');
res.setHeader('Transfer-Encoding', 'chunked');
var padding = 'a';
for (var i = 0; i < 10; i++) {
res.write(padding);
}
res.end(convert('<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS" /><div>日本語</div>', 'Shift_JIS'));
}
if (p === '/encoding/invalid') {
res.statusCode = 200;
res.setHeader('Content-Type', 'text/html');
res.setHeader('Transfer-Encoding', 'chunked');
// because node v0.12 doesn't have str.repeat
var padding = new Array(120 + 1).join('a');
for (var i = 0; i < 10; i++) {
res.write(padding);
}
res.end(convert('中文', 'gbk'));
}
if (p === '/redirect/301') {
res.statusCode = 301;
res.setHeader('Location', '/inspect');

View File

@ -10,6 +10,7 @@ var spawn = require('child_process').spawn;
var stream = require('stream');
var resumer = require('resumer');
var FormData = require('form-data');
var http = require('http');
var TestServer = require('./server');
@ -18,6 +19,7 @@ var fetch = require('../index.js');
var Headers = require('../lib/headers.js');
var Response = require('../lib/response.js');
var Request = require('../lib/request.js');
var Body = require('../lib/body.js');
// test with native promise on node 0.11, and bluebird for node 0.10
fetch.Promise = fetch.Promise || bluebird;
@ -523,6 +525,7 @@ describe('node-fetch', function() {
}).then(function(res) {
expect(res.method).to.equal('POST');
expect(res.headers['content-type']).to.contain('multipart/form-data');
expect(res.headers['content-length']).to.be.a('string');
expect(res.body).to.equal('a=1');
});
});
@ -545,6 +548,7 @@ describe('node-fetch', function() {
}).then(function(res) {
expect(res.method).to.equal('POST');
expect(res.headers['content-type']).to.contain('multipart/form-data');
expect(res.headers['content-length']).to.be.a('string');
expect(res.headers.b).to.equal('2');
expect(res.body).to.equal('a=1');
});
@ -603,6 +607,19 @@ describe('node-fetch', function() {
});
});
it('should allow OPTIONS request', function() {
url = base + '/options';
opts = {
method: 'OPTIONS'
};
return fetch(url, opts).then(function(res) {
expect(res.status).to.equal(200);
expect(res.statusText).to.equal('OK');
expect(res.headers.get('allow')).to.equal('GET, HEAD, OPTIONS');
expect(res.body).to.be.an.instanceof(stream.Transform);
});
});
it('should reject decoding body twice', function() {
url = base + '/plain';
return fetch(url).then(function(res) {
@ -709,6 +726,30 @@ describe('node-fetch', function() {
});
});
it('should support chunked encoding, html4 detect', function() {
url = base + '/encoding/chunked';
return fetch(url).then(function(res) {
expect(res.status).to.equal(200);
// because node v0.12 doesn't have str.repeat
var padding = new Array(10 + 1).join('a');
return res.text().then(function(result) {
expect(result).to.equal(padding + '<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS" /><div>日本語</div>');
});
});
});
it('should only do encoding detection up to 1024 bytes', function() {
url = base + '/encoding/invalid';
return fetch(url).then(function(res) {
expect(res.status).to.equal(200);
// because node v0.12 doesn't have str.repeat
var padding = new Array(1200 + 1).join('a');
return res.text().then(function(result) {
expect(result).to.not.equal(padding + '中文');
});
});
});
it('should allow piping response body as stream', function(done) {
url = base + '/hello';
fetch(url).then(function(res) {
@ -725,6 +766,62 @@ describe('node-fetch', function() {
});
});
it('should allow cloning a response, and use both as stream', function(done) {
url = base + '/hello';
return fetch(url).then(function(res) {
var counter = 0;
var r1 = res.clone();
expect(res.body).to.be.an.instanceof(stream.Transform);
expect(r1.body).to.be.an.instanceof(stream.Transform);
res.body.on('data', function(chunk) {
if (chunk === null) {
return;
}
expect(chunk.toString()).to.equal('world');
});
res.body.on('end', function() {
counter++;
if (counter == 2) {
done();
}
});
r1.body.on('data', function(chunk) {
if (chunk === null) {
return;
}
expect(chunk.toString()).to.equal('world');
});
r1.body.on('end', function() {
counter++;
if (counter == 2) {
done();
}
});
});
});
it('should allow cloning a json response, and log it as text response', function() {
url = base + '/json';
return fetch(url).then(function(res) {
var r1 = res.clone();
return fetch.Promise.all([r1.text(), res.json()]).then(function(results) {
expect(results[0]).to.equal('{"name":"value"}');
expect(results[1]).to.deep.equal({name: 'value'});
});
});
});
it('should not allow cloning a response after its been used', function() {
url = base + '/hello';
return fetch(url).then(function(res) {
return res.text().then(function(result) {
expect(function() {
var r1 = res.clone();
}).to.throw(Error);
});
})
});
it('should allow get all responses of a header', function() {
url = base + '/cookie';
return fetch(url).then(function(res) {
@ -737,25 +834,24 @@ describe('node-fetch', function() {
it('should allow iterating through all headers', function() {
var headers = new Headers({
a: 1,
b: [2, 3],
a: 1
, b: [2, 3]
, c: [4]
});
expect(headers).to.have.property('forEach');
var result = [];
headers.forEach(function(val, key) {
result.push([key, val]);
});
var myHeaders = [];
function callback(value, name) {
myHeaders.push([name, value]);
}
expected = [
["a", "1"],
["b", "2"],
["b", "3"]
["a", "1"]
, ["b", "2"]
, ["b", "3"]
, ["c", "4"]
];
expect(headers.forEach).to.be.defined;
headers.forEach(callback, headers);
expect(myHeaders).to.be.deep.equal(expected);
expect(result).to.deep.equal(expected);
});
it('should allow deleting header', function() {
@ -824,7 +920,6 @@ describe('node-fetch', function() {
expect(h1._headers['a']).to.include('1');
expect(h1._headers['a']).to.not.include('2');
expect(h1._headers['b']).to.not.include('1');
expect(h2._headers['a']).to.include('1');
expect(h2._headers['a']).to.not.include('2');
@ -891,14 +986,66 @@ describe('node-fetch', function() {
});
it('should support parsing headers in Response constructor', function() {
var body = resumer().queue('a=1').end();
body = body.pipe(new stream.PassThrough());
var res = new Response(body, {
var res = new Response(null, {
headers: {
a: '1'
}
});
expect(res.headers.get('a')).to.equal('1');
});
it('should support text() method in Response constructor', function() {
var res = new Response('a=1');
return res.text().then(function(result) {
expect(result).to.equal('a=1');
});
});
it('should support json() method in Response constructor', function() {
var res = new Response('{"a":1}');
return res.json().then(function(result) {
expect(result.a).to.equal(1);
});
});
it('should support clone() method in Response constructor', function() {
var res = new Response('a=1', {
headers: {
a: '1'
}
, url: base
, status: 346
, statusText: 'production'
});
var cl = res.clone();
expect(cl.headers.get('a')).to.equal('1');
expect(cl.url).to.equal(base);
expect(cl.status).to.equal(346);
expect(cl.statusText).to.equal('production');
expect(cl.ok).to.be.false;
return cl.text().then(function(result) {
expect(result).to.equal('a=1');
});
});
it('should support stream as body in Response constructor', function() {
var body = resumer().queue('a=1').end();
body = body.pipe(new stream.PassThrough());
var res = new Response(body);
return res.text().then(function(result) {
expect(result).to.equal('a=1');
});
});
it('should support string as body in Response constructor', function() {
var res = new Response('a=1');
return res.text().then(function(result) {
expect(result).to.equal('a=1');
});
});
it('should support buffer as body in Response constructor', function() {
var res = new Response(new Buffer('a=1'));
return res.text().then(function(result) {
expect(result).to.equal('a=1');
});
@ -915,6 +1062,62 @@ describe('node-fetch', function() {
expect(req.headers.get('a')).to.equal('1');
});
it('should support text() method in Request constructor', function() {
url = base;
var req = new Request(url, {
body: 'a=1'
});
expect(req.url).to.equal(url);
return req.text().then(function(result) {
expect(result).to.equal('a=1');
});
});
it('should support json() method in Request constructor', function() {
url = base;
var req = new Request(url, {
body: '{"a":1}'
});
expect(req.url).to.equal(url);
return req.json().then(function(result) {
expect(result.a).to.equal(1);
});
});
it('should support clone() method in Request constructor', function() {
url = base;
var agent = new http.Agent();
var req = new Request(url, {
body: 'a=1'
, method: 'POST'
, headers: {
b: '2'
}
, follow: 3
, compress: false
, agent: agent
});
var cl = req.clone();
expect(cl.url).to.equal(url);
expect(cl.method).to.equal('POST');
expect(cl.headers.get('b')).to.equal('2');
expect(cl.follow).to.equal(3);
expect(cl.compress).to.equal(false);
expect(cl.method).to.equal('POST');
expect(cl.counter).to.equal(3);
expect(cl.agent).to.equal(agent);
return fetch.Promise.all([cl.text(), req.text()]).then(function(results) {
expect(results[0]).to.equal('a=1');
expect(results[1]).to.equal('a=1');
});
});
it('should support text() and json() method in Body constructor', function() {
var body = new Body('a=1');
expect(body).to.have.property('text');
expect(body).to.have.property('json');
});
it('should support https request', function() {
this.timeout(5000);
url = 'https://github.com/';
@ -927,36 +1130,4 @@ describe('node-fetch', function() {
});
});
it('should support parsing headers in Response constructor', function(){
var r = new Response(null, {headers: {'foo': 'bar'}});
expect(r.headers.get('foo')).to.equal('bar');
});
it('should support parsing headers in Request constructor', function(){
var r = new Request('http://foo', {headers: {'foo': 'bar'}});
expect(r.headers.get('foo')).to.equal('bar');
});
it('should support string bodies in Response constructor', function() {
return (new Response('foo')).text().then(function(text) {
expect(text).to.equal('foo');
});
});
it('should support text() method in Request class', function() {
return (new Request('http://foo', {body: 'foo'})).text().then(function(text) {
expect(text).to.equal('foo');
});
});
it('should support json() method in Request class', function() {
return (new Request('http://foo', {body: '{"foo":"bar"}'})).json().then(function(json) {
expect(json.foo).to.equal('bar');
});
});
});