blockworks-foundation
/
node-fetch
mirror of https://github.com/blockworks-foundation/node-fetch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

basic refactor done

This commit is contained in:
David Frank 2015-01-27 21:11:26 +08:00
parent 93a983d815
commit af21ae6c1c
7 changed files with 372 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
LIMITS.md
View File

@ -2,18 +2,16 @@
Known limits
============
**As of 1.x release**
*As of 1.x release*
- Topics such as cross-origin, CSP, mixed content are ignored, given our server-side context.
- Topics such as cross-origin, content security policy, mixed content, service workers are ignored, given our server-side context.
- Url input must be an absolute url, using either `http` or `https` as scheme.
- Doesn't export `Headers`, `Body`, `Request`, `Response` classes yet, as we currenly use a much simpler implementation.
- On the upside, there are no forbidden headers, and `res.url` contains the final url when following redirects.
- For convenience, `res.body()` is a transform stream instead of byte stream, so decoding can be handled independently.
- For convenience, `res.body` is a transform stream, so decoding can be handled independently.
- Similarly, `options.body` can either be a string or a readable stream.
- Similarly, `req.body` can either be a string or a readable stream.
- For convenience, maximum redirect count (`options.follow`) and request timeout (`options.timeout`) are adjustable.
- There is currently no built-in caching support, as server-side requirement varies greatly between use-cases.
- There is currently no built-in caching, as server-side caching varies by use-cases.

14
README.md
View File

@ -12,13 +12,11 @@ A light-weight module that brings `window.fetch` to node.js
I really like the notion of Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch): it bridges the API gap between client-side and server-side http requests, so developers have less to worry about.
But I think the term [isomorphic](http://isomorphic.net/) is generally misleading: it gives developers a false sense of security that their javascript code will run happily on both controlled server environment as well as uncontrollable user browsers. When the latter is only true for a small subset of modern browsers, not to mention quirks in native implementation.
But I think the term [isomorphic](http://isomorphic.net/) is generally misleading: it gives developers a false sense of security that their javascript code will run happily on both controlled server environment as well as uncontrollable user browsers. When the latter is only true for a subset of modern browsers, not to mention quirks in native implementation.
Instead of implementing `XMLHttpRequest` in node to run browser-specific [fetch polyfill](https://github.com/github/fetch), why not go from node's `http` to `fetch` API directly? Node has native stream support, browserify build targets (browsers) don't, so underneath they are going to be vastly different anyway.
IMHO, it's safer to be aware of javascript runtime's strength and weakness, than to assume they are a unified platform under a singular spec.
Hence `node-fetch`, minimal code for a `window.fetch` compatible API.
Hence `node-fetch`, minimal code for a `window.fetch` compatible API on node.js runtime.
# Features
@ -28,11 +26,11 @@ Hence `node-fetch`, minimal code for a `window.fetch` compatible API.
- Use native promise, but allow substituting it with [insert your favorite promise library].
# Difference to client-side fetch
# Difference from client-side fetch
- This module is WIP, see [Known limits](https://github.com/bitinn/node-fetch/blob/master/LIMITS.md) for details.
(If you spot a missing feature that `window.fetch` offers, feel free to open an issue. Pull requests are welcomed too!)
- See [Known limits](https://github.com/bitinn/node-fetch/blob/master/LIMITS.md) for details.
- If you happen to use a missing feature that `window.fetch` offers, feel free to open an issue.
- Pull requests are welcomed too!
# Install

105
index.js
View File

@ -2,7 +2,7 @@
/**
* index.js
*
* export fetch
* a request API compatible with window.fetch
*/
var parse = require('url').parse;
@ -12,10 +12,13 @@ var https = require('https');
var zlib = require('zlib');
var stream = require('stream');
var Response = require('./lib/response');
var Headers = require('./lib/headers');
module.exports = Fetch;
/**
* Create an instance of Decent
* Fetch class
*
* @param String url Absolute url
* @param Object opts Fetch options
@ -23,18 +26,21 @@ module.exports = Fetch;
*/
function Fetch(url, opts) {
// allow call as function
if (!(this instanceof Fetch))
return new Fetch(url, opts);
// allow custom promise
if (!Fetch.Promise) {
throw new Error('native promise missing, set Fetch.Promise to your favorite alternative');
}
Response.Promise = Fetch.Promise;
var self = this;
// wrap http.request into fetch
return new Fetch.Promise(function(resolve, reject) {
opts = opts || {};
var uri = parse(url);
if (!uri.protocol || !uri.hostname) {
@ -47,8 +53,6 @@ function Fetch(url, opts) {
return;
}
// TODO: detect type and decode data
var request;
if (uri.protocol === 'https:') {
request = https.request;
@ -56,28 +60,66 @@ function Fetch(url, opts) {
request = http.request;
}
// avoid side-effect on input
opts = opts || {};
// avoid side-effect on input options
var options = {
hostname: uri.hostname
, port: uri.port
, method: opts.method
, path: uri.path
, headers: opts.headers || {}
, path: uri.path || '/'
, auth: uri.auth
, method: opts.method || 'GET'
, headers: opts.headers || {}
, follow: opts.follow || 20
, counter: opts.counter || 0
, agent: opts.agent
, timeout: opts.timeout || 0
, compress: opts.compress || true
, size: opts.size || 0
, body: opts.body
, timeout: opts.timeout
, agent: opts.agent
};
// normalize headers
var headers = new Headers(options.headers);
if (options.compress) {
headers.set('accept-encoding', 'gzip,deflate');
}
if (!headers.has('user-agent')) {
headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)');
}
if (!headers.has('connection')) {
headers.set('connection', 'close');
}
if (!headers.has('accept')) {
headers.set('accept', '*/*');
}
options.headers = headers.raw();
// send request
var req = request(options);
var started = false;
req.on('socket', function(socket) {
if (!started && options.timeout) {
started = true;
setTimeout(function() {
req.abort();
reject(new Error('network timeout at: ' + uri.href));
}, options.timeout);
}
});
req.on('error', function(err) {
reject(new Error('request to ' + uri.href + ' failed, reason: ' + err.message));
});
req.on('response', function(res) {
// handle redirect
if (self.isRedirect(res.statusCode)) {
if (options.counter >= options.follow) {
reject(Error('maximum redirect reached at: ' + uri.href));
@ -87,19 +129,35 @@ function Fetch(url, opts) {
reject(Error('redirect location header missing at: ' + uri.href));
}
return Fetch(resolve(uri.href, res.headers.location), options);
resolve(Fetch(resolve(uri.href, res.headers.location), options));
return;
}
var output = {
status: res.statusCode
, headers: res.headers
, body: res.pipe(new stream.PassThrough())
, url: uri.href
};
// handle compression
var body = res.pipe(new stream.PassThrough());
var headers = new Headers(res.headers);
if (headers.has('content-encoding')) {
var name = headers.get('content-encoding');
if (name == 'gzip' || name == 'x-gzip') {
body = body.pipe(zlib.createGunzip());
} else if (name == 'deflate' || name == 'x-deflate') {
body = body.pipe(zlib.createInflate());
}
}
// response object
var output = new Response(body, {
url: uri.href
, status: res.statusCode
, headers: headers
});
resolve(output);
});
// accept string or readable stream as body
if (typeof options.body === 'string') {
req.write(options.body);
req.end();
@ -108,19 +166,12 @@ function Fetch(url, opts) {
} else {
req.end();
}
if (options.timeout) {
setTimeout(function() {
req.abort();
reject(new Error('network timeout at: ' + uri.href));
}, options.timeout);
}
});
};
/**
* Create an instance of Decent
* Redirect code matching
*
* @param Number code Status code
* @return Boolean

114
lib/headers.js Normal file
View File

@ -0,0 +1,114 @@
/**
* headers.js
*
* Headers class offers convenient helpers
*/
module.exports = Headers;
/**
* Headers class
*
* @param Object headers Response headers
* @return Void
*/
function Headers(headers) {
var self = this;
this._headers = {};
for (var prop in headers) {
if (headers.hasOwnProperty(prop)) {
if (typeof headers[prop] === 'string') {
this.set(prop, headers[prop]);
} else if (headers[prop].length > 0) {
headers[prop].forEach(function(item) {
self.append(prop, item);
});
}
}
}
}
/**
* Return first header value given name
*
* @param String name Header name
* @return Mixed
*/
Headers.prototype.get = function(name) {
var list = this._headers[name.toLowerCase()];
return list ? list[0] : null;
};
/**
* Return all header values given name
*
* @param String name Header name
* @return Array
*/
Headers.prototype.getAll = function(name) {
if (!this.has(name)) {
return [];
}
return this._headers[name.toLowerCase()];
};
/**
* Overwrite header values given name
*
* @param String name Header name
* @param String value Header value
* @return Void
*/
Headers.prototype.set = function(name, value) {
this._headers[name.toLowerCase()] = [value];
};
/**
* Append a value onto existing header
*
* @param String name Header name
* @param String value Header value
* @return Void
*/
Headers.prototype.append = function(name, value) {
if (!this.has(name)) {
this.set(name, value);
return;
}
this._headers[name.toLowerCase()].push(value);
};
/**
* Check for header name existence
*
* @param String name Header name
* @return Boolean
*/
Headers.prototype.has = function(name) {
return this._headers.hasOwnProperty(name.toLowerCase());
};
/**
* Delete all header values given name
*
* @param String name Header name
* @return Void
*/
Headers.prototype['delete'] = function(name) {
delete this._headers[name.toLowerCase()];
};
/**
* Return raw headers (non-spec api)
*
* @return Object
*/
Headers.prototype.raw = function() {
return this._headers;
};

156
lib/response.js Normal file
View File

@ -0,0 +1,156 @@
/**
* response.js
*
* Response class provides content decoding
*/
var http = require('http');
var stream = require('stream');
var convert = require('encoding').convert;
module.exports = Response;
/**
* Response class
*
* @param Stream body Readable stream
* @param Object opts Response options
* @return Void
*/
function Response(body, opts) {
this.url = opts.url;
this.status = opts.status;
this.statusText = http.STATUS_CODES[this.status];
this.headers = opts.headers;
this.body = body.pipe(new stream.PassThrough());
this.bodyUsed = false;
}
/**
* Decode response as json
*
* @return Promise
*/
Response.prototype.json = function() {
return this._decode().then(function(text) {
return JSON.parse(text);
});
}
/**
* Decode response as text
*
* @return Promise
*/
Response.prototype.text = function() {
return this._decode();
}
/**
* Decode buffers into utf-8 string
*
* @return Promise
*/
Response.prototype._decode = function() {
var self = this;
if (this.bodyUsed) {
return Response.Promise.reject(new Error('body used already for: ' + this.url));
}
this.bodyUsed = true;
this._bytes = 0;
this._abort = false;
this._raw = [];
return new Response.Promise(function(resolve, reject) {
self.body.on('data', function(chunk) {
if (chunk === null) {
return;
}
if (self.size && self._bytes > self.size) {
self._abort = true;
reject(new Error('content size at ' + self.url + ' over limit: ' + self.size));
self.body.abort();
return;
}
self._bytes += chunk.length;
self._raw.push(chunk);
});
self.body.on('end', function() {
if (self._abort) {
return;
}
resolve(self._convert());
});
});
};
/**
* Detect buffer encoding and convert to target encoding
* ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding
*
* @param String encoding Target encoding
* @return String
*/
Response.prototype._convert = function(encoding) {
encoding = encoding || 'utf-8';
var charset = 'utf-8';
var res;
// header
if (this.headers.has('content-type')) {
res = /charset=(.*)/i.exec(this.headers.get('content-type'));
}
// html5
if (!res && this._raw.length > 0) {
res = /<meta.+?charset=(['"])(.+?)\1/i.exec(this._raw[0].toString());
}
// html4
if (!res && this._raw.length > 0) {
res = /<meta[\s]+?http-equiv=(['"])content-type\1[\s]+?content=(['"])(.+?)\2/i.exec(this._raw[0].toString());
if (res) {
res = /charset=(.*)/i.exec(res.pop());
}
}
// found charset
if (res) {
charset = res.pop();
// prevent decode issues when sites use incorrect encoding
// see: https://hsivonen.fi/encoding-menu/
if (charset === 'gb2312' || charset === 'gbk') {
charset = 'gb18030';
}
}
// turn raw buffers into utf-8 string
return convert(
Buffer.concat(this._raw)
, encoding
, charset
).toString();
}
// expose Promise
Response.Promise = global.Promise;

3
package.json
View File

@ -27,5 +27,8 @@
"chai-as-promised": "^4.1.1",
"mocha": "^2.1.0",
"promise": "^6.1.0"
},
"dependencies": {
"encoding": "^0.1.11"
}
}

12
test/test.js
View File

@ -11,6 +11,8 @@ var TestServer = require('./server');
// test subjects
var fetch = require('../index.js');
var Headers = require('../lib/headers.js');
var Response = require('../lib/response.js');
// test with native promise on node 0.11, and bluebird for node 0.10
fetch.Promise = fetch.Promise || bluebird;
@ -73,13 +75,17 @@ describe('Fetch', function() {
return expect(fetch(url)).to.eventually.be.rejectedWith(Error);
});
it('should resolve status code, headers, body correctly', function() {
it('should resolve into response', function() {
url = base + '/hello';
return fetch(url).then(function(res) {
expect(res).to.be.an.instanceof(Response);
expect(res.headers).to.be.an.instanceof(Headers);
expect(res.headers.get('content-type')).to.equal('text/plain');
expect(res.status).to.equal(200);
expect(res.headers).to.include({ 'content-type': 'text/plain' });
expect(res.body).to.be.an.instanceof(stream.Transform);
expect(res.statusText).to.equal('OK');
expect(res.url).to.equal(url);
expect(res.body).to.be.an.instanceof(stream.Transform);
expect(res.bodyUsed).to.be.false;
});
});