xml encoding detection, response size limit

CHANGELOG.md

@@ -5,7 +5,12 @@ Changelog
 
 # 1.x release
 
-## v1.0.2 (master)
+## v1.0.3 (master)
+
+- Fix: response size limit should reject large chunk
+- Enhance: added character encoding detection for xml, such as rss/atom feed (encoding in DTD)
+
+## v1.0.2
 
 - Fix: added res.ok per spec change
 

lib/response.js

@@ -74,15 +74,11 @@ Response.prototype._decode = function() {
 
 	return new Response.Promise(function(resolve, reject) {
 		self.body.on('data', function(chunk) {
-			if (self._abort) {
+			if (self._abort || chunk === null) {
 				return;
 			}
 
-			if (chunk === null) {
-				return;
-			}
-
-			if (self.size && self._bytes > self.size) {
+			if (self.size && self._bytes + chunk.length > self.size) {
 				self._abort = true;
 				reject(new Error('content size at ' + self.url + ' over limit: ' + self.size));
 				return;
@@ -115,27 +111,37 @@ Response.prototype._convert = function(encoding) {
 	encoding = encoding || 'utf-8';
 
 	var charset = 'utf-8';
-	var res;
+	var res, str;
 
 	// header
 	if (this.headers.has('content-type')) {
 		res = /charset=(.*)/i.exec(this.headers.get('content-type'));
 	}
 
-	// html5
+	// no charset in content type, peek at response body
 	if (!res && this._raw.length > 0) {
-		res = /<meta.+?charset=(['"])(.+?)\1/i.exec(this._raw[0].toString());
+		str = this._raw[0].toString().substr(0, 1024);
+	}
+
+	// html5
+	if (!res && str) {
+		res = /<meta.+?charset=(['"])(.+?)\1/i.exec(str);
 	}
 
 	// html4
-	if (!res && this._raw.length > 0) {
-		res = /<meta[\s]+?http-equiv=(['"])content-type\1[\s]+?content=(['"])(.+?)\2/i.exec(this._raw[0].toString());
+	if (!res && str) {
+		res = /<meta[\s]+?http-equiv=(['"])content-type\1[\s]+?content=(['"])(.+?)\2/i.exec(str);
 
 		if (res) {
 			res = /charset=(.*)/i.exec(res.pop());
 		}
 	}
 
+	// xml
+	if (!res && str) {
+		res = /<\?xml.+?encoding=(['"])(.+?)\1/i.exec(str);
+	}
+
 	// found charset
 	if (res) {
 		charset = res.pop();

test/server.js

@@ -54,17 +54,6 @@ TestServer.prototype.router = function(req, res) {
 		}));
 	}
 
-	if (p === '/long') {
-		res.statusCode = 200;
-		res.setHeader('Content-Type', 'text/plain');
-		setTimeout(function() {
-			res.write('test');
-		}, 50);
-		setTimeout(function() {
-			res.end('test');
-		}, 100);
-	}
-
 	if (p === '/gzip') {
 		res.statusCode = 200;
 		res.setHeader('Content-Type', 'text/plain');
@@ -104,6 +93,23 @@ TestServer.prototype.router = function(req, res) {
 		res.end('cookie');
 	}
 
+	if (p === '/size/chunk') {
+		res.statusCode = 200;
+		res.setHeader('Content-Type', 'text/plain');
+		setTimeout(function() {
+			res.write('test');
+		}, 50);
+		setTimeout(function() {
+			res.end('test');
+		}, 100);
+	}
+
+	if (p === '/size/long') {
+		res.statusCode = 200;
+		res.setHeader('Content-Type', 'text/plain');
+		res.end('testtest');
+	}
+
 	if (p === '/encoding/gbk') {
 		res.statusCode = 200;
 		res.setHeader('Content-Type', 'text/html');
@@ -122,6 +128,12 @@ TestServer.prototype.router = function(req, res) {
 		res.end(convert('<div>日本語</div>', 'Shift_JIS'));
 	}
 
+	if (p === '/encoding/euc-jp') {
+		res.statusCode = 200;
+		res.setHeader('Content-Type', 'text/xml');
+		res.end(convert('<?xml version="1.0" encoding="EUC-JP"?><title>日本語</title>', 'EUC-JP'));
+	}
+
 	if (p === '/encoding/utf8') {
 		res.statusCode = 200;
 		res.end('中文');

test/test.js

@@ -428,10 +428,10 @@ describe('node-fetch', function() {
 		});
 	});
 
-	it('should support maximum response size', function() {
-		url = base + '/long';
+	it('should support maximum response size, multiple chunk', function() {
+		url = base + '/size/chunk';
 		opts = {
-			size: 1
+			size: 5
 		};
 		return fetch(url, opts).then(function(res) {
 			expect(res.status).to.equal(200);
@@ -440,7 +440,29 @@ describe('node-fetch', function() {
 		});
 	});
 
-	it('should support encoding decode, conte-type detect', function() {
+	it('should support maximum response size, single chunk', function() {
+		url = base + '/size/long';
+		opts = {
+			size: 5
+		};
+		return fetch(url, opts).then(function(res) {
+			expect(res.status).to.equal(200);
+			expect(res.headers.get('content-type')).to.equal('text/plain');
+			return expect(res.text()).to.eventually.be.rejectedWith(Error);
+		});
+	});
+
+	it('should support encoding decode, xml dtd detect', function() {
+		url = base + '/encoding/euc-jp';
+		return fetch(url).then(function(res) {
+			expect(res.status).to.equal(200);
+			return res.text().then(function(result) {
+				expect(result).to.equal('<?xml version="1.0" encoding="EUC-JP"?><title>日本語</title>');
+			});
+		});
+	});
+
+	it('should support encoding decode, content-type detect', function() {
 		url = base + '/encoding/shift-jis';
 		return fetch(url).then(function(res) {
 			expect(res.status).to.equal(200);