basic refactor done
This commit is contained in:
parent
93a983d815
commit
af21ae6c1c
14
LIMITS.md
14
LIMITS.md
|
@ -2,18 +2,16 @@
|
|||
Known limits
|
||||
============
|
||||
|
||||
**As of 1.x release**
|
||||
*As of 1.x release*
|
||||
|
||||
- Topics such as cross-origin, CSP, mixed content are ignored, given our server-side context.
|
||||
- Topics such as cross-origin, content security policy, mixed content, service workers are ignored, given our server-side context.
|
||||
|
||||
- Url input must be an absolute url, using either `http` or `https` as scheme.
|
||||
|
||||
- Doesn't export `Headers`, `Body`, `Request`, `Response` classes yet, as we currenly use a much simpler implementation.
|
||||
- On the upside, there are no forbidden headers, and `res.url` contains the final url when following redirects.
|
||||
|
||||
- For convenience, `res.body()` is a transform stream instead of byte stream, so decoding can be handled independently.
|
||||
- For convenience, `res.body` is a transform stream, so decoding can be handled independently.
|
||||
|
||||
- Similarly, `options.body` can either be a string or a readable stream.
|
||||
- Similarly, `req.body` can either be a string or a readable stream.
|
||||
|
||||
- For convenience, maximum redirect count (`options.follow`) and request timeout (`options.timeout`) are adjustable.
|
||||
|
||||
- There is currently no built-in caching support, as server-side requirement varies greatly between use-cases.
|
||||
- There is currently no built-in caching, as server-side caching varies by use-cases.
|
||||
|
|
14
README.md
14
README.md
|
@ -12,13 +12,11 @@ A light-weight module that brings `window.fetch` to node.js
|
|||
|
||||
I really like the notion of Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch): it bridges the API gap between client-side and server-side http requests, so developers have less to worry about.
|
||||
|
||||
But I think the term [isomorphic](http://isomorphic.net/) is generally misleading: it gives developers a false sense of security that their javascript code will run happily on both controlled server environment as well as uncontrollable user browsers. When the latter is only true for a small subset of modern browsers, not to mention quirks in native implementation.
|
||||
But I think the term [isomorphic](http://isomorphic.net/) is generally misleading: it gives developers a false sense of security that their javascript code will run happily on both controlled server environment as well as uncontrollable user browsers. When the latter is only true for a subset of modern browsers, not to mention quirks in native implementation.
|
||||
|
||||
Instead of implementing `XMLHttpRequest` in node to run browser-specific [fetch polyfill](https://github.com/github/fetch), why not go from node's `http` to `fetch` API directly? Node has native stream support, browserify build targets (browsers) don't, so underneath they are going to be vastly different anyway.
|
||||
|
||||
IMHO, it's safer to be aware of javascript runtime's strength and weakness, than to assume they are a unified platform under a singular spec.
|
||||
|
||||
Hence `node-fetch`, minimal code for a `window.fetch` compatible API.
|
||||
Hence `node-fetch`, minimal code for a `window.fetch` compatible API on node.js runtime.
|
||||
|
||||
|
||||
# Features
|
||||
|
@ -28,11 +26,11 @@ Hence `node-fetch`, minimal code for a `window.fetch` compatible API.
|
|||
- Use native promise, but allow substituting it with [insert your favorite promise library].
|
||||
|
||||
|
||||
# Difference to client-side fetch
|
||||
# Difference from client-side fetch
|
||||
|
||||
- This module is WIP, see [Known limits](https://github.com/bitinn/node-fetch/blob/master/LIMITS.md) for details.
|
||||
|
||||
(If you spot a missing feature that `window.fetch` offers, feel free to open an issue. Pull requests are welcomed too!)
|
||||
- See [Known limits](https://github.com/bitinn/node-fetch/blob/master/LIMITS.md) for details.
|
||||
- If you happen to use a missing feature that `window.fetch` offers, feel free to open an issue.
|
||||
- Pull requests are welcomed too!
|
||||
|
||||
|
||||
# Install
|
||||
|
|
105
index.js
105
index.js
|
@ -2,7 +2,7 @@
|
|||
/**
|
||||
* index.js
|
||||
*
|
||||
* export fetch
|
||||
* a request API compatible with window.fetch
|
||||
*/
|
||||
|
||||
var parse = require('url').parse;
|
||||
|
@ -12,10 +12,13 @@ var https = require('https');
|
|||
var zlib = require('zlib');
|
||||
var stream = require('stream');
|
||||
|
||||
var Response = require('./lib/response');
|
||||
var Headers = require('./lib/headers');
|
||||
|
||||
module.exports = Fetch;
|
||||
|
||||
/**
|
||||
* Create an instance of Decent
|
||||
* Fetch class
|
||||
*
|
||||
* @param String url Absolute url
|
||||
* @param Object opts Fetch options
|
||||
|
@ -23,18 +26,21 @@ module.exports = Fetch;
|
|||
*/
|
||||
function Fetch(url, opts) {
|
||||
|
||||
// allow call as function
|
||||
if (!(this instanceof Fetch))
|
||||
return new Fetch(url, opts);
|
||||
|
||||
// allow custom promise
|
||||
if (!Fetch.Promise) {
|
||||
throw new Error('native promise missing, set Fetch.Promise to your favorite alternative');
|
||||
}
|
||||
|
||||
Response.Promise = Fetch.Promise;
|
||||
|
||||
var self = this;
|
||||
|
||||
// wrap http.request into fetch
|
||||
return new Fetch.Promise(function(resolve, reject) {
|
||||
opts = opts || {};
|
||||
|
||||
var uri = parse(url);
|
||||
|
||||
if (!uri.protocol || !uri.hostname) {
|
||||
|
@ -47,8 +53,6 @@ function Fetch(url, opts) {
|
|||
return;
|
||||
}
|
||||
|
||||
// TODO: detect type and decode data
|
||||
|
||||
var request;
|
||||
if (uri.protocol === 'https:') {
|
||||
request = https.request;
|
||||
|
@ -56,28 +60,66 @@ function Fetch(url, opts) {
|
|||
request = http.request;
|
||||
}
|
||||
|
||||
// avoid side-effect on input
|
||||
opts = opts || {};
|
||||
|
||||
// avoid side-effect on input options
|
||||
var options = {
|
||||
hostname: uri.hostname
|
||||
, port: uri.port
|
||||
, method: opts.method
|
||||
, path: uri.path
|
||||
, headers: opts.headers || {}
|
||||
, path: uri.path || '/'
|
||||
, auth: uri.auth
|
||||
, method: opts.method || 'GET'
|
||||
, headers: opts.headers || {}
|
||||
, follow: opts.follow || 20
|
||||
, counter: opts.counter || 0
|
||||
, agent: opts.agent
|
||||
, timeout: opts.timeout || 0
|
||||
, compress: opts.compress || true
|
||||
, size: opts.size || 0
|
||||
, body: opts.body
|
||||
, timeout: opts.timeout
|
||||
, agent: opts.agent
|
||||
};
|
||||
|
||||
// normalize headers
|
||||
var headers = new Headers(options.headers);
|
||||
|
||||
if (options.compress) {
|
||||
headers.set('accept-encoding', 'gzip,deflate');
|
||||
}
|
||||
|
||||
if (!headers.has('user-agent')) {
|
||||
headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)');
|
||||
}
|
||||
|
||||
if (!headers.has('connection')) {
|
||||
headers.set('connection', 'close');
|
||||
}
|
||||
|
||||
if (!headers.has('accept')) {
|
||||
headers.set('accept', '*/*');
|
||||
}
|
||||
|
||||
options.headers = headers.raw();
|
||||
|
||||
// send request
|
||||
var req = request(options);
|
||||
var started = false;
|
||||
|
||||
req.on('socket', function(socket) {
|
||||
if (!started && options.timeout) {
|
||||
started = true;
|
||||
setTimeout(function() {
|
||||
req.abort();
|
||||
reject(new Error('network timeout at: ' + uri.href));
|
||||
}, options.timeout);
|
||||
}
|
||||
});
|
||||
|
||||
req.on('error', function(err) {
|
||||
reject(new Error('request to ' + uri.href + ' failed, reason: ' + err.message));
|
||||
});
|
||||
|
||||
req.on('response', function(res) {
|
||||
// handle redirect
|
||||
if (self.isRedirect(res.statusCode)) {
|
||||
if (options.counter >= options.follow) {
|
||||
reject(Error('maximum redirect reached at: ' + uri.href));
|
||||
|
@ -87,19 +129,35 @@ function Fetch(url, opts) {
|
|||
reject(Error('redirect location header missing at: ' + uri.href));
|
||||
}
|
||||
|
||||
return Fetch(resolve(uri.href, res.headers.location), options);
|
||||
resolve(Fetch(resolve(uri.href, res.headers.location), options));
|
||||
return;
|
||||
}
|
||||
|
||||
var output = {
|
||||
status: res.statusCode
|
||||
, headers: res.headers
|
||||
, body: res.pipe(new stream.PassThrough())
|
||||
, url: uri.href
|
||||
};
|
||||
// handle compression
|
||||
var body = res.pipe(new stream.PassThrough());
|
||||
var headers = new Headers(res.headers);
|
||||
|
||||
if (headers.has('content-encoding')) {
|
||||
var name = headers.get('content-encoding');
|
||||
|
||||
if (name == 'gzip' || name == 'x-gzip') {
|
||||
body = body.pipe(zlib.createGunzip());
|
||||
} else if (name == 'deflate' || name == 'x-deflate') {
|
||||
body = body.pipe(zlib.createInflate());
|
||||
}
|
||||
}
|
||||
|
||||
// response object
|
||||
var output = new Response(body, {
|
||||
url: uri.href
|
||||
, status: res.statusCode
|
||||
, headers: headers
|
||||
});
|
||||
|
||||
resolve(output);
|
||||
});
|
||||
|
||||
// accept string or readable stream as body
|
||||
if (typeof options.body === 'string') {
|
||||
req.write(options.body);
|
||||
req.end();
|
||||
|
@ -108,19 +166,12 @@ function Fetch(url, opts) {
|
|||
} else {
|
||||
req.end();
|
||||
}
|
||||
|
||||
if (options.timeout) {
|
||||
setTimeout(function() {
|
||||
req.abort();
|
||||
reject(new Error('network timeout at: ' + uri.href));
|
||||
}, options.timeout);
|
||||
}
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* Create an instance of Decent
|
||||
* Redirect code matching
|
||||
*
|
||||
* @param Number code Status code
|
||||
* @return Boolean
|
||||
|
|
|
@ -0,0 +1,114 @@
|
|||
|
||||
/**
|
||||
* headers.js
|
||||
*
|
||||
* Headers class offers convenient helpers
|
||||
*/
|
||||
|
||||
module.exports = Headers;
|
||||
|
||||
/**
|
||||
* Headers class
|
||||
*
|
||||
* @param Object headers Response headers
|
||||
* @return Void
|
||||
*/
|
||||
function Headers(headers) {
|
||||
|
||||
var self = this;
|
||||
this._headers = {};
|
||||
|
||||
for (var prop in headers) {
|
||||
if (headers.hasOwnProperty(prop)) {
|
||||
if (typeof headers[prop] === 'string') {
|
||||
this.set(prop, headers[prop]);
|
||||
} else if (headers[prop].length > 0) {
|
||||
headers[prop].forEach(function(item) {
|
||||
self.append(prop, item);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Return first header value given name
|
||||
*
|
||||
* @param String name Header name
|
||||
* @return Mixed
|
||||
*/
|
||||
Headers.prototype.get = function(name) {
|
||||
var list = this._headers[name.toLowerCase()];
|
||||
return list ? list[0] : null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Return all header values given name
|
||||
*
|
||||
* @param String name Header name
|
||||
* @return Array
|
||||
*/
|
||||
Headers.prototype.getAll = function(name) {
|
||||
if (!this.has(name)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this._headers[name.toLowerCase()];
|
||||
};
|
||||
|
||||
/**
|
||||
* Overwrite header values given name
|
||||
*
|
||||
* @param String name Header name
|
||||
* @param String value Header value
|
||||
* @return Void
|
||||
*/
|
||||
Headers.prototype.set = function(name, value) {
|
||||
this._headers[name.toLowerCase()] = [value];
|
||||
};
|
||||
|
||||
/**
|
||||
* Append a value onto existing header
|
||||
*
|
||||
* @param String name Header name
|
||||
* @param String value Header value
|
||||
* @return Void
|
||||
*/
|
||||
Headers.prototype.append = function(name, value) {
|
||||
if (!this.has(name)) {
|
||||
this.set(name, value);
|
||||
return;
|
||||
}
|
||||
|
||||
this._headers[name.toLowerCase()].push(value);
|
||||
};
|
||||
|
||||
/**
|
||||
* Check for header name existence
|
||||
*
|
||||
* @param String name Header name
|
||||
* @return Boolean
|
||||
*/
|
||||
Headers.prototype.has = function(name) {
|
||||
return this._headers.hasOwnProperty(name.toLowerCase());
|
||||
};
|
||||
|
||||
/**
|
||||
* Delete all header values given name
|
||||
*
|
||||
* @param String name Header name
|
||||
* @return Void
|
||||
*/
|
||||
Headers.prototype['delete'] = function(name) {
|
||||
delete this._headers[name.toLowerCase()];
|
||||
};
|
||||
|
||||
/**
|
||||
* Return raw headers (non-spec api)
|
||||
*
|
||||
* @return Object
|
||||
*/
|
||||
Headers.prototype.raw = function() {
|
||||
return this._headers;
|
||||
};
|
|
@ -0,0 +1,156 @@
|
|||
|
||||
/**
|
||||
* response.js
|
||||
*
|
||||
* Response class provides content decoding
|
||||
*/
|
||||
|
||||
var http = require('http');
|
||||
var stream = require('stream');
|
||||
var convert = require('encoding').convert;
|
||||
|
||||
module.exports = Response;
|
||||
|
||||
/**
|
||||
* Response class
|
||||
*
|
||||
* @param Stream body Readable stream
|
||||
* @param Object opts Response options
|
||||
* @return Void
|
||||
*/
|
||||
function Response(body, opts) {
|
||||
|
||||
this.url = opts.url;
|
||||
this.status = opts.status;
|
||||
this.statusText = http.STATUS_CODES[this.status];
|
||||
this.headers = opts.headers;
|
||||
this.body = body.pipe(new stream.PassThrough());
|
||||
this.bodyUsed = false;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode response as json
|
||||
*
|
||||
* @return Promise
|
||||
*/
|
||||
Response.prototype.json = function() {
|
||||
|
||||
return this._decode().then(function(text) {
|
||||
return JSON.parse(text);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode response as text
|
||||
*
|
||||
* @return Promise
|
||||
*/
|
||||
Response.prototype.text = function() {
|
||||
|
||||
return this._decode();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode buffers into utf-8 string
|
||||
*
|
||||
* @return Promise
|
||||
*/
|
||||
Response.prototype._decode = function() {
|
||||
|
||||
var self = this;
|
||||
|
||||
if (this.bodyUsed) {
|
||||
return Response.Promise.reject(new Error('body used already for: ' + this.url));
|
||||
}
|
||||
|
||||
this.bodyUsed = true;
|
||||
this._bytes = 0;
|
||||
this._abort = false;
|
||||
this._raw = [];
|
||||
|
||||
return new Response.Promise(function(resolve, reject) {
|
||||
self.body.on('data', function(chunk) {
|
||||
if (chunk === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (self.size && self._bytes > self.size) {
|
||||
self._abort = true;
|
||||
reject(new Error('content size at ' + self.url + ' over limit: ' + self.size));
|
||||
self.body.abort();
|
||||
return;
|
||||
}
|
||||
|
||||
self._bytes += chunk.length;
|
||||
self._raw.push(chunk);
|
||||
});
|
||||
|
||||
self.body.on('end', function() {
|
||||
if (self._abort) {
|
||||
return;
|
||||
}
|
||||
|
||||
resolve(self._convert());
|
||||
});
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* Detect buffer encoding and convert to target encoding
|
||||
* ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding
|
||||
*
|
||||
* @param String encoding Target encoding
|
||||
* @return String
|
||||
*/
|
||||
Response.prototype._convert = function(encoding) {
|
||||
|
||||
encoding = encoding || 'utf-8';
|
||||
|
||||
var charset = 'utf-8';
|
||||
var res;
|
||||
|
||||
// header
|
||||
if (this.headers.has('content-type')) {
|
||||
res = /charset=(.*)/i.exec(this.headers.get('content-type'));
|
||||
}
|
||||
|
||||
// html5
|
||||
if (!res && this._raw.length > 0) {
|
||||
res = /<meta.+?charset=(['"])(.+?)\1/i.exec(this._raw[0].toString());
|
||||
}
|
||||
|
||||
// html4
|
||||
if (!res && this._raw.length > 0) {
|
||||
res = /<meta[\s]+?http-equiv=(['"])content-type\1[\s]+?content=(['"])(.+?)\2/i.exec(this._raw[0].toString());
|
||||
|
||||
if (res) {
|
||||
res = /charset=(.*)/i.exec(res.pop());
|
||||
}
|
||||
}
|
||||
|
||||
// found charset
|
||||
if (res) {
|
||||
charset = res.pop();
|
||||
|
||||
// prevent decode issues when sites use incorrect encoding
|
||||
// see: https://hsivonen.fi/encoding-menu/
|
||||
if (charset === 'gb2312' || charset === 'gbk') {
|
||||
charset = 'gb18030';
|
||||
}
|
||||
}
|
||||
|
||||
// turn raw buffers into utf-8 string
|
||||
return convert(
|
||||
Buffer.concat(this._raw)
|
||||
, encoding
|
||||
, charset
|
||||
).toString();
|
||||
|
||||
}
|
||||
|
||||
// expose Promise
|
||||
Response.Promise = global.Promise;
|
|
@ -27,5 +27,8 @@
|
|||
"chai-as-promised": "^4.1.1",
|
||||
"mocha": "^2.1.0",
|
||||
"promise": "^6.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"encoding": "^0.1.11"
|
||||
}
|
||||
}
|
||||
|
|
12
test/test.js
12
test/test.js
|
@ -11,6 +11,8 @@ var TestServer = require('./server');
|
|||
|
||||
// test subjects
|
||||
var fetch = require('../index.js');
|
||||
var Headers = require('../lib/headers.js');
|
||||
var Response = require('../lib/response.js');
|
||||
// test with native promise on node 0.11, and bluebird for node 0.10
|
||||
fetch.Promise = fetch.Promise || bluebird;
|
||||
|
||||
|
@ -73,13 +75,17 @@ describe('Fetch', function() {
|
|||
return expect(fetch(url)).to.eventually.be.rejectedWith(Error);
|
||||
});
|
||||
|
||||
it('should resolve status code, headers, body correctly', function() {
|
||||
it('should resolve into response', function() {
|
||||
url = base + '/hello';
|
||||
return fetch(url).then(function(res) {
|
||||
expect(res).to.be.an.instanceof(Response);
|
||||
expect(res.headers).to.be.an.instanceof(Headers);
|
||||
expect(res.headers.get('content-type')).to.equal('text/plain');
|
||||
expect(res.status).to.equal(200);
|
||||
expect(res.headers).to.include({ 'content-type': 'text/plain' });
|
||||
expect(res.body).to.be.an.instanceof(stream.Transform);
|
||||
expect(res.statusText).to.equal('OK');
|
||||
expect(res.url).to.equal(url);
|
||||
expect(res.body).to.be.an.instanceof(stream.Transform);
|
||||
expect(res.bodyUsed).to.be.false;
|
||||
});
|
||||
});
|
||||
|
||||
|
|
Loading…
Reference in New Issue