diff --git a/.travis.yml b/.travis.yml index 692c2d6..1faa00a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: node_js node_js: - "0.10" - "0.12" - - "iojs" - "node" before_install: npm install -g npm script: npm run coverage \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 337bd7b..dc12bee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,17 @@ Changelog # 1.x release -## v1.3.3 (master) +## v1.4.0 (master) + +- Enhance: Request and Response now have `clone` method (thx to @kirill-konshin for the initial PR) +- Enhance: Request and Response now have proper string and buffer body support (thx to @kirill-konshin) +- Enhance: Body constructor has been refactored out (thx to @kirill-konshin) +- Enhance: Headers now has `forEach` method (thx to @tricoder42) +- Enhance: back to 100% code coverage +- Fix: better form-data support (thx to @item4) +- Fix: better character encoding detection under chunked encoding (thx to @dsuket for the initial PR) + +## v1.3.3 - Fix: make sure `Content-Length` header is set when body is string for POST/PUT/PATCH requests - Fix: handle body stream error, for cases such as incorrect `Content-Encoding` header diff --git a/index.js b/index.js index 2512b31..02072dc 100644 --- a/index.js +++ b/index.js @@ -17,7 +17,9 @@ var Response = require('./lib/response'); var Headers = require('./lib/headers'); var Request = require('./lib/request'); +// commonjs module.exports = Fetch; +// es6 default export compatibility module.exports.default = module.exports; /** @@ -170,6 +172,7 @@ function Fetch(url, opts) { var output = new Response(body, { url: options.url , status: res.statusCode + , statusText: res.statusMessage , headers: headers , size: options.size , timeout: options.timeout diff --git a/lib/body.js b/lib/body.js index 7c15f91..fdc1139 100644 --- a/lib/body.js +++ b/lib/body.js @@ -5,6 +5,8 @@ */ var convert = require('encoding').convert; +var bodyStream = require('is-stream'); +var PassThrough = require('stream').PassThrough; module.exports = Body; @@ -17,14 +19,14 @@ module.exports = Body; */ function Body(body, opts) { - opts = opts || {}; + opts = opts || {}; - this.body = body; - this.bodyUsed = false; - this.size = opts.size || 0; - this.timeout = opts.timeout || 0; - this._raw = []; - this._abort = false; + this.body = body; + this.bodyUsed = false; + this.size = opts.size || 0; + this.timeout = opts.timeout || 0; + this._raw = []; + this._abort = false; } @@ -35,9 +37,9 @@ function Body(body, opts) { */ Body.prototype.json = function() { - return this._decode().then(function(text) { - return JSON.parse(text); - }); + return this._decode().then(function(text) { + return JSON.parse(text); + }); }; @@ -48,7 +50,7 @@ Body.prototype.json = function() { */ Body.prototype.text = function() { - return this._decode(); + return this._decode(); }; @@ -59,69 +61,69 @@ Body.prototype.text = function() { */ Body.prototype._decode = function() { - var self = this; + var self = this; - if (this.bodyUsed) { - return Body.Promise.reject(new Error('body used already for: ' + this.url)); - } + if (this.bodyUsed) { + return Body.Promise.reject(new Error('body used already for: ' + this.url)); + } - this.bodyUsed = true; - this._bytes = 0; - this._abort = false; - this._raw = []; + this.bodyUsed = true; + this._bytes = 0; + this._abort = false; + this._raw = []; - return new Body.Promise(function(resolve, reject) { - var resTimeout; + return new Body.Promise(function(resolve, reject) { + var resTimeout; - if (typeof self.body === 'string') { - self._bytes = self.body.length; - self._raw = [new Buffer(self.body)]; - return resolve(self._convert()); - } + if (typeof self.body === 'string') { + self._bytes = self.body.length; + self._raw = [new Buffer(self.body)]; + return resolve(self._convert()); + } - if (self.body instanceof Buffer) { - self._bytes = self.body.length; - self._raw = [self.body]; - return resolve(self._convert()); - } + if (self.body instanceof Buffer) { + self._bytes = self.body.length; + self._raw = [self.body]; + return resolve(self._convert()); + } - // allow timeout on slow response body - if (self.timeout) { - resTimeout = setTimeout(function() { - self._abort = true; - reject(new Error('response timeout at ' + self.url + ' over limit: ' + self.timeout)); - }, self.timeout); - } + // allow timeout on slow response body + if (self.timeout) { + resTimeout = setTimeout(function() { + self._abort = true; + reject(new Error('response timeout at ' + self.url + ' over limit: ' + self.timeout)); + }, self.timeout); + } - // handle stream error, such as incorrect content-encoding - self.body.on('error', function(err) { - reject(new Error('invalid response body at: ' + self.url + ' reason: ' + err.message)); - }); + // handle stream error, such as incorrect content-encoding + self.body.on('error', function(err) { + reject(new Error('invalid response body at: ' + self.url + ' reason: ' + err.message)); + }); - self.body.on('data', function(chunk) { - if (self._abort || chunk === null) { - return; - } + self.body.on('data', function(chunk) { + if (self._abort || chunk === null) { + return; + } - if (self.size && self._bytes + chunk.length > self.size) { - self._abort = true; - reject(new Error('content size at ' + self.url + ' over limit: ' + self.size)); - return; - } + if (self.size && self._bytes + chunk.length > self.size) { + self._abort = true; + reject(new Error('content size at ' + self.url + ' over limit: ' + self.size)); + return; + } - self._bytes += chunk.length; - self._raw.push(chunk); - }); + self._bytes += chunk.length; + self._raw.push(chunk); + }); - self.body.on('end', function() { - if (self._abort) { - return; - } + self.body.on('end', function() { + if (self._abort) { + return; + } - clearTimeout(resTimeout); - resolve(self._convert()); - }); - }); + clearTimeout(resTimeout); + resolve(self._convert()); + }); + }); }; @@ -134,59 +136,88 @@ Body.prototype._decode = function() { */ Body.prototype._convert = function(encoding) { - encoding = encoding || 'utf-8'; + encoding = encoding || 'utf-8'; - var charset = 'utf-8'; - var res, str; + var charset = 'utf-8'; + var res, str; - // header - if (this.headers.has('content-type')) { - res = /charset=([^;]*)/i.exec(this.headers.get('content-type')); - } + // header + if (this.headers.has('content-type')) { + res = /charset=([^;]*)/i.exec(this.headers.get('content-type')); + } - // no charset in content type, peek at response body - if (!res && this._raw.length > 0) { - str = this._raw[0].toString().substr(0, 1024); - } + // no charset in content type, peek at response body for at most 1024 bytes + if (!res && this._raw.length > 0) { + for (var i = 0; i < this._raw.length; i++) { + str += this._raw[i].toString() + if (str.length > 1024) { + break; + } + } + str = str.substr(0, 1024); + } - // html5 - if (!res && str) { - res = /= 200 && this.status < 300; @@ -34,5 +33,17 @@ function Response(body, opts) { Response.prototype = Object.create(Body.prototype); -// expose Promise -Response.Promise = global.Promise; +/** + * Clone this response + * + * @return Response + */ +Response.prototype.clone = function() { + return new Response(this._clone(this), { + url: this.url + , status: this.status + , statusText: this.statusText + , headers: this.headers + , ok: this.ok + }); +}; diff --git a/package.json b/package.json index b58358d..a861bb1 100644 --- a/package.json +++ b/package.json @@ -24,18 +24,19 @@ }, "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { - "bluebird": "^2.9.1", - "chai": "^1.10.0", - "chai-as-promised": "^4.1.1", + "bluebird": "^3.3.4", + "chai": "^3.5.0", + "chai-as-promised": "^5.2.0", "coveralls": "^2.11.2", "form-data": "^1.0.0-rc1", - "istanbul": "^0.3.5", + "istanbul": "^0.4.2", "mocha": "^2.1.0", "parted": "^0.1.1", - "promise": "^6.1.0", + "promise": "^7.1.1", "resumer": "0.0.0" }, "dependencies": { - "encoding": "^0.1.11" + "encoding": "^0.1.11", + "is-stream": "^1.0.1" } } diff --git a/test/server.js b/test/server.js index 7403922..89732ec 100644 --- a/test/server.js +++ b/test/server.js @@ -41,6 +41,12 @@ TestServer.prototype.router = function(req, res) { res.end('text'); } + if (p === '/options') { + res.statusCode = 200; + res.setHeader('Allow', 'GET, HEAD, OPTIONS'); + res.end('hello world'); + } + if (p === '/html') { res.statusCode = 200; res.setHeader('Content-Type', 'text/html'); @@ -168,6 +174,29 @@ TestServer.prototype.router = function(req, res) { res.end(convert('中文', 'gbk')); } + if (p === '/encoding/chunked') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/html'); + res.setHeader('Transfer-Encoding', 'chunked'); + var padding = 'a'; + for (var i = 0; i < 10; i++) { + res.write(padding); + } + res.end(convert('
日本語
', 'Shift_JIS')); + } + + if (p === '/encoding/invalid') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/html'); + res.setHeader('Transfer-Encoding', 'chunked'); + // because node v0.12 doesn't have str.repeat + var padding = new Array(120 + 1).join('a'); + for (var i = 0; i < 10; i++) { + res.write(padding); + } + res.end(convert('中文', 'gbk')); + } + if (p === '/redirect/301') { res.statusCode = 301; res.setHeader('Location', '/inspect'); diff --git a/test/test.js b/test/test.js index 813e8c9..1a7fe08 100644 --- a/test/test.js +++ b/test/test.js @@ -10,6 +10,7 @@ var spawn = require('child_process').spawn; var stream = require('stream'); var resumer = require('resumer'); var FormData = require('form-data'); +var http = require('http'); var TestServer = require('./server'); @@ -18,6 +19,7 @@ var fetch = require('../index.js'); var Headers = require('../lib/headers.js'); var Response = require('../lib/response.js'); var Request = require('../lib/request.js'); +var Body = require('../lib/body.js'); // test with native promise on node 0.11, and bluebird for node 0.10 fetch.Promise = fetch.Promise || bluebird; @@ -523,6 +525,7 @@ describe('node-fetch', function() { }).then(function(res) { expect(res.method).to.equal('POST'); expect(res.headers['content-type']).to.contain('multipart/form-data'); + expect(res.headers['content-length']).to.be.a('string'); expect(res.body).to.equal('a=1'); }); }); @@ -545,6 +548,7 @@ describe('node-fetch', function() { }).then(function(res) { expect(res.method).to.equal('POST'); expect(res.headers['content-type']).to.contain('multipart/form-data'); + expect(res.headers['content-length']).to.be.a('string'); expect(res.headers.b).to.equal('2'); expect(res.body).to.equal('a=1'); }); @@ -603,6 +607,19 @@ describe('node-fetch', function() { }); }); + it('should allow OPTIONS request', function() { + url = base + '/options'; + opts = { + method: 'OPTIONS' + }; + return fetch(url, opts).then(function(res) { + expect(res.status).to.equal(200); + expect(res.statusText).to.equal('OK'); + expect(res.headers.get('allow')).to.equal('GET, HEAD, OPTIONS'); + expect(res.body).to.be.an.instanceof(stream.Transform); + }); + }); + it('should reject decoding body twice', function() { url = base + '/plain'; return fetch(url).then(function(res) { @@ -709,6 +726,30 @@ describe('node-fetch', function() { }); }); + it('should support chunked encoding, html4 detect', function() { + url = base + '/encoding/chunked'; + return fetch(url).then(function(res) { + expect(res.status).to.equal(200); + // because node v0.12 doesn't have str.repeat + var padding = new Array(10 + 1).join('a'); + return res.text().then(function(result) { + expect(result).to.equal(padding + '
日本語
'); + }); + }); + }); + + it('should only do encoding detection up to 1024 bytes', function() { + url = base + '/encoding/invalid'; + return fetch(url).then(function(res) { + expect(res.status).to.equal(200); + // because node v0.12 doesn't have str.repeat + var padding = new Array(1200 + 1).join('a'); + return res.text().then(function(result) { + expect(result).to.not.equal(padding + '中文'); + }); + }); + }); + it('should allow piping response body as stream', function(done) { url = base + '/hello'; fetch(url).then(function(res) { @@ -725,6 +766,62 @@ describe('node-fetch', function() { }); }); + it('should allow cloning a response, and use both as stream', function(done) { + url = base + '/hello'; + return fetch(url).then(function(res) { + var counter = 0; + var r1 = res.clone(); + expect(res.body).to.be.an.instanceof(stream.Transform); + expect(r1.body).to.be.an.instanceof(stream.Transform); + res.body.on('data', function(chunk) { + if (chunk === null) { + return; + } + expect(chunk.toString()).to.equal('world'); + }); + res.body.on('end', function() { + counter++; + if (counter == 2) { + done(); + } + }); + r1.body.on('data', function(chunk) { + if (chunk === null) { + return; + } + expect(chunk.toString()).to.equal('world'); + }); + r1.body.on('end', function() { + counter++; + if (counter == 2) { + done(); + } + }); + }); + }); + + it('should allow cloning a json response, and log it as text response', function() { + url = base + '/json'; + return fetch(url).then(function(res) { + var r1 = res.clone(); + return fetch.Promise.all([r1.text(), res.json()]).then(function(results) { + expect(results[0]).to.equal('{"name":"value"}'); + expect(results[1]).to.deep.equal({name: 'value'}); + }); + }); + }); + + it('should not allow cloning a response after its been used', function() { + url = base + '/hello'; + return fetch(url).then(function(res) { + return res.text().then(function(result) { + expect(function() { + var r1 = res.clone(); + }).to.throw(Error); + }); + }) + }); + it('should allow get all responses of a header', function() { url = base + '/cookie'; return fetch(url).then(function(res) { @@ -737,25 +834,24 @@ describe('node-fetch', function() { it('should allow iterating through all headers', function() { var headers = new Headers({ - a: 1, - b: [2, 3], + a: 1 + , b: [2, 3] + , c: [4] + }); + expect(headers).to.have.property('forEach'); + + var result = []; + headers.forEach(function(val, key) { + result.push([key, val]); }); - var myHeaders = []; - function callback(value, name) { - myHeaders.push([name, value]); - } - expected = [ - ["a", "1"], - ["b", "2"], - ["b", "3"] + ["a", "1"] + , ["b", "2"] + , ["b", "3"] + , ["c", "4"] ]; - - expect(headers.forEach).to.be.defined; - - headers.forEach(callback, headers); - expect(myHeaders).to.be.deep.equal(expected); + expect(result).to.deep.equal(expected); }); it('should allow deleting header', function() { @@ -824,7 +920,6 @@ describe('node-fetch', function() { expect(h1._headers['a']).to.include('1'); expect(h1._headers['a']).to.not.include('2'); - expect(h1._headers['b']).to.not.include('1'); expect(h2._headers['a']).to.include('1'); expect(h2._headers['a']).to.not.include('2'); @@ -891,14 +986,66 @@ describe('node-fetch', function() { }); it('should support parsing headers in Response constructor', function() { - var body = resumer().queue('a=1').end(); - body = body.pipe(new stream.PassThrough()); - var res = new Response(body, { + var res = new Response(null, { headers: { a: '1' } }); expect(res.headers.get('a')).to.equal('1'); + }); + + it('should support text() method in Response constructor', function() { + var res = new Response('a=1'); + return res.text().then(function(result) { + expect(result).to.equal('a=1'); + }); + }); + + it('should support json() method in Response constructor', function() { + var res = new Response('{"a":1}'); + return res.json().then(function(result) { + expect(result.a).to.equal(1); + }); + }); + + it('should support clone() method in Response constructor', function() { + var res = new Response('a=1', { + headers: { + a: '1' + } + , url: base + , status: 346 + , statusText: 'production' + }); + var cl = res.clone(); + expect(cl.headers.get('a')).to.equal('1'); + expect(cl.url).to.equal(base); + expect(cl.status).to.equal(346); + expect(cl.statusText).to.equal('production'); + expect(cl.ok).to.be.false; + return cl.text().then(function(result) { + expect(result).to.equal('a=1'); + }); + }); + + it('should support stream as body in Response constructor', function() { + var body = resumer().queue('a=1').end(); + body = body.pipe(new stream.PassThrough()); + var res = new Response(body); + return res.text().then(function(result) { + expect(result).to.equal('a=1'); + }); + }); + + it('should support string as body in Response constructor', function() { + var res = new Response('a=1'); + return res.text().then(function(result) { + expect(result).to.equal('a=1'); + }); + }); + + it('should support buffer as body in Response constructor', function() { + var res = new Response(new Buffer('a=1')); return res.text().then(function(result) { expect(result).to.equal('a=1'); }); @@ -915,6 +1062,62 @@ describe('node-fetch', function() { expect(req.headers.get('a')).to.equal('1'); }); + it('should support text() method in Request constructor', function() { + url = base; + var req = new Request(url, { + body: 'a=1' + }); + expect(req.url).to.equal(url); + return req.text().then(function(result) { + expect(result).to.equal('a=1'); + }); + });
 + + it('should support json() method in Request constructor', function() { + url = base; + var req = new Request(url, { + body: '{"a":1}' + }); + expect(req.url).to.equal(url); + return req.json().then(function(result) { + expect(result.a).to.equal(1); + }); + });
 + + it('should support clone() method in Request constructor', function() { + url = base; + var agent = new http.Agent(); + var req = new Request(url, { + body: 'a=1' + , method: 'POST' + , headers: { + b: '2' + } + , follow: 3 + , compress: false + , agent: agent + }); + var cl = req.clone(); + expect(cl.url).to.equal(url); + expect(cl.method).to.equal('POST'); + expect(cl.headers.get('b')).to.equal('2'); + expect(cl.follow).to.equal(3); + expect(cl.compress).to.equal(false); + expect(cl.method).to.equal('POST'); + expect(cl.counter).to.equal(3); + expect(cl.agent).to.equal(agent); + return fetch.Promise.all([cl.text(), req.text()]).then(function(results) { + expect(results[0]).to.equal('a=1'); + expect(results[1]).to.equal('a=1'); + }); + }); + + it('should support text() and json() method in Body constructor', function() { + var body = new Body('a=1'); + expect(body).to.have.property('text'); + expect(body).to.have.property('json'); + });
 + it('should support https request', function() { this.timeout(5000); url = 'https://github.com/'; @@ -927,36 +1130,4 @@ describe('node-fetch', function() { }); }); - it('should support parsing headers in Response constructor', function(){ - - var r = new Response(null, {headers: {'foo': 'bar'}}); - expect(r.headers.get('foo')).to.equal('bar'); - - }); - - it('should support parsing headers in Request constructor', function(){ - - var r = new Request('http://foo', {headers: {'foo': 'bar'}}); - expect(r.headers.get('foo')).to.equal('bar'); - - }); - - it('should support string bodies in Response constructor', function() { - return (new Response('foo')).text().then(function(text) { - expect(text).to.equal('foo'); - }); - }); - - it('should support text() method in Request class', function() { - return (new Request('http://foo', {body: 'foo'})).text().then(function(text) { - expect(text).to.equal('foo'); - }); - });
 - - it('should support json() method in Request class', function() { - return (new Request('http://foo', {body: '{"foo":"bar"}'})).json().then(function(json) { - expect(json.foo).to.equal('bar'); - }); - });
 - });