2015-01-26 01:02:34 -08:00
|
|
|
/**
|
2020-03-13 08:06:25 -07:00
|
|
|
* Index.js
|
2015-01-26 01:02:34 -08:00
|
|
|
*
|
2015-01-27 05:11:26 -08:00
|
|
|
* a request API compatible with window.fetch
|
2018-03-04 19:38:57 -08:00
|
|
|
*
|
|
|
|
* All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/.
|
2015-01-26 01:02:34 -08:00
|
|
|
*/
|
|
|
|
|
2018-07-22 09:58:03 -07:00
|
|
|
import http from 'http';
|
|
|
|
import https from 'https';
|
|
|
|
import zlib from 'zlib';
|
2020-03-13 08:06:25 -07:00
|
|
|
import Stream, {PassThrough, pipeline as pump} from 'stream';
|
2020-05-31 08:15:27 -07:00
|
|
|
import dataUriToBuffer from 'data-uri-to-buffer';
|
2018-07-22 09:58:03 -07:00
|
|
|
|
2020-06-10 04:16:51 -07:00
|
|
|
import {writeToStream} from './body.js';
|
2020-05-20 23:50:31 -07:00
|
|
|
import Response from './response.js';
|
2020-05-25 07:43:10 -07:00
|
|
|
import Headers, {fromRawHeaders} from './headers.js';
|
2020-05-20 23:50:31 -07:00
|
|
|
import Request, {getNodeRequestOptions} from './request.js';
|
2020-06-10 04:17:35 -07:00
|
|
|
import {FetchError} from './errors/fetch-error.js';
|
|
|
|
import {AbortError} from './errors/abort-error.js';
|
2020-05-22 21:00:02 -07:00
|
|
|
import {isRedirect} from './utils/is-redirect.js';
|
2020-05-20 23:50:31 -07:00
|
|
|
|
2020-05-23 10:32:10 -07:00
|
|
|
export {Headers, Request, Response, FetchError, AbortError, isRedirect};
|
2018-11-05 01:42:51 -08:00
|
|
|
|
2020-05-31 08:15:27 -07:00
|
|
|
const supportedSchemas = new Set(['data:', 'http:', 'https:']);
|
|
|
|
|
2015-01-26 01:02:34 -08:00
|
|
|
/**
|
2016-10-10 11:50:04 -07:00
|
|
|
* Fetch function
|
2015-01-26 01:02:34 -08:00
|
|
|
*
|
2020-05-31 08:15:27 -07:00
|
|
|
* @param {string | URL | import('./request').default} url - Absolute url or Request instance
|
|
|
|
* @param {*} [options_] - Fetch options
|
|
|
|
* @return {Promise<import('./response').default>}
|
2015-01-26 01:02:34 -08:00
|
|
|
*/
|
2020-05-28 14:57:57 -07:00
|
|
|
export default async function fetch(url, options_) {
|
|
|
|
return new Promise((resolve, reject) => {
|
2020-03-13 08:06:25 -07:00
|
|
|
// Build request object
|
|
|
|
const request = new Request(url, options_);
|
2016-11-23 13:39:35 -08:00
|
|
|
const options = getNodeRequestOptions(request);
|
2020-05-31 08:15:27 -07:00
|
|
|
if (!supportedSchemas.has(options.protocol)) {
|
|
|
|
throw new TypeError(`node-fetch cannot load ${url}. URL scheme "${options.protocol.replace(/:$/, '')}" is not supported.`);
|
|
|
|
}
|
2015-01-26 02:15:07 -08:00
|
|
|
|
2020-05-31 08:15:27 -07:00
|
|
|
if (options.protocol === 'data:') {
|
|
|
|
const data = dataUriToBuffer(request.url);
|
|
|
|
const response = new Response(data, {headers: {'Content-Type': data.typeFull}});
|
|
|
|
resolve(response);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Wrap http.request into fetch
|
2017-04-08 18:33:46 -07:00
|
|
|
const send = (options.protocol === 'https:' ? https : http).request;
|
2020-03-13 08:06:25 -07:00
|
|
|
const {signal} = request;
|
2018-11-12 20:40:11 -08:00
|
|
|
let response = null;
|
|
|
|
|
2020-03-13 08:06:25 -07:00
|
|
|
const abort = () => {
|
|
|
|
const error = new AbortError('The operation was aborted.');
|
2018-11-12 20:40:11 -08:00
|
|
|
reject(error);
|
|
|
|
if (request.body && request.body instanceof Stream.Readable) {
|
|
|
|
request.body.destroy(error);
|
|
|
|
}
|
2020-03-13 08:06:25 -07:00
|
|
|
|
|
|
|
if (!response || !response.body) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-11-12 20:40:11 -08:00
|
|
|
response.body.emit('error', error);
|
2020-03-13 08:06:25 -07:00
|
|
|
};
|
2018-11-12 20:40:11 -08:00
|
|
|
|
|
|
|
if (signal && signal.aborted) {
|
|
|
|
abort();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
const abortAndFinalize = () => {
|
|
|
|
abort();
|
|
|
|
finalize();
|
2020-03-13 08:06:25 -07:00
|
|
|
};
|
2015-01-26 02:15:07 -08:00
|
|
|
|
2020-03-13 08:06:25 -07:00
|
|
|
// Send request
|
|
|
|
const request_ = send(options);
|
2015-01-27 05:11:26 -08:00
|
|
|
|
2018-11-12 20:40:11 -08:00
|
|
|
if (signal) {
|
|
|
|
signal.addEventListener('abort', abortAndFinalize);
|
|
|
|
}
|
|
|
|
|
2020-05-25 08:11:56 -07:00
|
|
|
const finalize = () => {
|
2020-03-13 08:06:25 -07:00
|
|
|
request_.abort();
|
|
|
|
if (signal) {
|
|
|
|
signal.removeEventListener('abort', abortAndFinalize);
|
|
|
|
}
|
2020-05-25 08:11:56 -07:00
|
|
|
};
|
2018-03-24 20:48:33 -07:00
|
|
|
|
2021-07-18 13:15:19 -07:00
|
|
|
request_.on('error', error => {
|
|
|
|
reject(new FetchError(`request to ${request.url} failed, reason: ${error.message}`, 'system', error));
|
2018-03-24 20:48:33 -07:00
|
|
|
finalize();
|
2015-01-26 05:28:23 -08:00
|
|
|
});
|
|
|
|
|
2021-07-18 13:15:19 -07:00
|
|
|
fixResponseChunkedTransferBadEnding(request_, error => {
|
|
|
|
response.body.destroy(error);
|
2021-02-22 23:14:09 -08:00
|
|
|
});
|
|
|
|
|
|
|
|
/* c8 ignore next 18 */
|
|
|
|
if (process.version < 'v14') {
|
|
|
|
// Before Node.js 14, pipeline() does not fully support async iterators and does not always
|
|
|
|
// properly handle when the socket close/end events are out of order.
|
|
|
|
request_.on('socket', s => {
|
|
|
|
let endedWithEventsCount;
|
|
|
|
s.prependListener('end', () => {
|
|
|
|
endedWithEventsCount = s._eventsCount;
|
|
|
|
});
|
|
|
|
s.prependListener('close', hadError => {
|
|
|
|
// if end happened before close but the socket didn't emit an error, do it now
|
|
|
|
if (response && endedWithEventsCount < s._eventsCount && !hadError) {
|
2021-07-18 13:15:19 -07:00
|
|
|
const error = new Error('Premature close');
|
|
|
|
error.code = 'ERR_STREAM_PREMATURE_CLOSE';
|
|
|
|
response.body.emit('error', error);
|
2021-02-22 23:14:09 -08:00
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2020-05-25 08:11:56 -07:00
|
|
|
request_.on('response', response_ => {
|
2020-05-22 12:19:05 -07:00
|
|
|
request_.setTimeout(0);
|
2020-05-25 08:11:56 -07:00
|
|
|
const headers = fromRawHeaders(response_.rawHeaders);
|
2015-01-26 09:46:32 -08:00
|
|
|
|
2018-03-04 19:29:59 -08:00
|
|
|
// HTTP fetch step 5
|
2020-05-25 08:11:56 -07:00
|
|
|
if (isRedirect(response_.statusCode)) {
|
2018-03-04 19:29:59 -08:00
|
|
|
// HTTP fetch step 5.2
|
|
|
|
const location = headers.get('Location');
|
|
|
|
|
|
|
|
// HTTP fetch step 5.3
|
2020-03-13 08:06:25 -07:00
|
|
|
const locationURL = location === null ? null : new URL(location, request.url);
|
2018-03-04 19:29:59 -08:00
|
|
|
|
|
|
|
// HTTP fetch step 5.5
|
|
|
|
switch (request.redirect) {
|
|
|
|
case 'error':
|
2019-10-10 13:26:58 -07:00
|
|
|
reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${request.url}`, 'no-redirect'));
|
2018-03-24 20:48:33 -07:00
|
|
|
finalize();
|
2018-03-04 19:29:59 -08:00
|
|
|
return;
|
|
|
|
case 'manual':
|
2020-03-13 08:06:25 -07:00
|
|
|
// Node-fetch-specific step: make manual redirect a bit easier to use by setting the Location header value to the resolved URL.
|
2018-03-04 19:29:59 -08:00
|
|
|
if (locationURL !== null) {
|
2020-12-31 21:38:11 -08:00
|
|
|
headers.set('Location', locationURL);
|
2018-03-04 19:29:59 -08:00
|
|
|
}
|
2020-03-13 08:06:25 -07:00
|
|
|
|
2018-03-04 19:29:59 -08:00
|
|
|
break;
|
2020-03-13 08:06:25 -07:00
|
|
|
case 'follow': {
|
2018-03-04 19:29:59 -08:00
|
|
|
// HTTP-redirect fetch step 2
|
|
|
|
if (locationURL === null) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
// HTTP-redirect fetch step 5
|
|
|
|
if (request.counter >= request.follow) {
|
|
|
|
reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect'));
|
2018-03-24 20:48:33 -07:00
|
|
|
finalize();
|
2018-03-04 19:29:59 -08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// HTTP-redirect fetch step 6 (counter increment)
|
|
|
|
// Create a new Request object.
|
2020-03-13 08:06:25 -07:00
|
|
|
const requestOptions = {
|
2018-03-04 19:29:59 -08:00
|
|
|
headers: new Headers(request.headers),
|
|
|
|
follow: request.follow,
|
|
|
|
counter: request.counter + 1,
|
|
|
|
agent: request.agent,
|
|
|
|
compress: request.compress,
|
|
|
|
method: request.method,
|
2018-11-12 20:40:11 -08:00
|
|
|
body: request.body,
|
2020-09-05 05:44:41 -07:00
|
|
|
signal: request.signal,
|
|
|
|
size: request.size
|
2018-03-04 19:29:59 -08:00
|
|
|
};
|
|
|
|
|
|
|
|
// HTTP-redirect fetch step 9
|
2020-06-10 04:16:51 -07:00
|
|
|
if (response_.statusCode !== 303 && request.body && options_.body instanceof Stream.Readable) {
|
2018-03-24 20:48:33 -07:00
|
|
|
reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect'));
|
|
|
|
finalize();
|
|
|
|
return;
|
2018-03-04 19:29:59 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// HTTP-redirect fetch step 11
|
2020-05-25 08:11:56 -07:00
|
|
|
if (response_.statusCode === 303 || ((response_.statusCode === 301 || response_.statusCode === 302) && request.method === 'POST')) {
|
2020-03-13 08:06:25 -07:00
|
|
|
requestOptions.method = 'GET';
|
|
|
|
requestOptions.body = undefined;
|
|
|
|
requestOptions.headers.delete('content-length');
|
2018-03-04 19:29:59 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
// HTTP-redirect fetch step 15
|
2020-03-13 08:06:25 -07:00
|
|
|
resolve(fetch(new Request(locationURL, requestOptions)));
|
2018-03-24 20:48:33 -07:00
|
|
|
finalize();
|
2018-03-04 19:29:59 -08:00
|
|
|
return;
|
2020-03-13 08:06:25 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
default:
|
2020-12-31 21:38:44 -08:00
|
|
|
return reject(new TypeError(`Redirect option '${request.redirect}' is not a valid value of RequestRedirect`));
|
2015-09-28 07:30:41 -07:00
|
|
|
}
|
2016-04-05 11:47:23 -07:00
|
|
|
}
|
|
|
|
|
2020-03-13 08:06:25 -07:00
|
|
|
// Prepare response
|
2020-09-18 03:33:15 -07:00
|
|
|
if (signal) {
|
|
|
|
response_.once('end', () => {
|
2020-03-13 08:06:25 -07:00
|
|
|
signal.removeEventListener('abort', abortAndFinalize);
|
2020-09-18 03:33:15 -07:00
|
|
|
});
|
|
|
|
}
|
2020-03-13 08:06:25 -07:00
|
|
|
|
2020-09-18 03:33:15 -07:00
|
|
|
let body = pump(response_, new PassThrough(), reject);
|
2020-06-09 17:26:24 -07:00
|
|
|
// see https://github.com/nodejs/node/pull/29376
|
|
|
|
if (process.version < 'v12.10') {
|
|
|
|
response_.on('aborted', abortAndFinalize);
|
|
|
|
}
|
2018-11-12 20:40:11 -08:00
|
|
|
|
2020-03-13 08:06:25 -07:00
|
|
|
const responseOptions = {
|
2018-03-04 20:29:12 -08:00
|
|
|
url: request.url,
|
2020-05-25 08:11:56 -07:00
|
|
|
status: response_.statusCode,
|
|
|
|
statusText: response_.statusMessage,
|
2020-03-13 08:06:25 -07:00
|
|
|
headers,
|
2018-03-04 20:29:12 -08:00
|
|
|
size: request.size,
|
2020-03-13 08:06:25 -07:00
|
|
|
counter: request.counter,
|
|
|
|
highWaterMark: request.highWaterMark
|
2016-08-03 02:31:46 -07:00
|
|
|
};
|
|
|
|
|
2018-03-04 19:38:57 -08:00
|
|
|
// HTTP-network fetch step 12.1.1.3
|
2017-02-26 21:27:27 -08:00
|
|
|
const codings = headers.get('Content-Encoding');
|
|
|
|
|
2018-03-04 19:38:57 -08:00
|
|
|
// HTTP-network fetch step 12.1.1.4: handle content codings
|
2016-08-03 02:31:46 -07:00
|
|
|
|
|
|
|
// in following scenarios we ignore compression support
|
|
|
|
// 1. compression support is disabled
|
|
|
|
// 2. HEAD request
|
2017-02-26 21:27:27 -08:00
|
|
|
// 3. no Content-Encoding header
|
2016-08-03 02:31:46 -07:00
|
|
|
// 4. no content response (204)
|
|
|
|
// 5. content not modified response (304)
|
2020-05-25 08:11:56 -07:00
|
|
|
if (!request.compress || request.method === 'HEAD' || codings === null || response_.statusCode === 204 || response_.statusCode === 304) {
|
2020-03-13 08:06:25 -07:00
|
|
|
response = new Response(body, responseOptions);
|
2018-11-12 20:40:11 -08:00
|
|
|
resolve(response);
|
2016-08-03 02:31:46 -07:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2017-05-15 04:45:04 -07:00
|
|
|
// For Node v6+
|
2017-02-22 00:05:55 -08:00
|
|
|
// Be less strict when decoding compressed responses, since sometimes
|
|
|
|
// servers send slightly invalid responses that are still accepted
|
|
|
|
// by common browsers.
|
|
|
|
// Always using Z_SYNC_FLUSH is what cURL does.
|
|
|
|
const zlibOptions = {
|
|
|
|
flush: zlib.Z_SYNC_FLUSH,
|
|
|
|
finishFlush: zlib.Z_SYNC_FLUSH
|
|
|
|
};
|
|
|
|
|
2020-03-13 08:06:25 -07:00
|
|
|
// For gzip
|
|
|
|
if (codings === 'gzip' || codings === 'x-gzip') {
|
2020-09-18 03:33:15 -07:00
|
|
|
body = pump(body, zlib.createGunzip(zlibOptions), reject);
|
2020-03-13 08:06:25 -07:00
|
|
|
response = new Response(body, responseOptions);
|
2018-11-12 20:40:11 -08:00
|
|
|
resolve(response);
|
2016-08-03 02:31:46 -07:00
|
|
|
return;
|
2017-02-26 21:27:27 -08:00
|
|
|
}
|
2016-08-03 02:31:46 -07:00
|
|
|
|
2020-03-13 08:06:25 -07:00
|
|
|
// For deflate
|
|
|
|
if (codings === 'deflate' || codings === 'x-deflate') {
|
|
|
|
// Handle the infamous raw deflate response from old servers
|
2016-08-03 02:31:46 -07:00
|
|
|
// a hack for old IIS and Apache servers
|
2020-09-18 03:33:15 -07:00
|
|
|
const raw = pump(response_, new PassThrough(), reject);
|
2016-10-10 11:50:04 -07:00
|
|
|
raw.once('data', chunk => {
|
2020-03-13 08:06:25 -07:00
|
|
|
// See http://stackoverflow.com/questions/37519828
|
2021-07-18 13:15:19 -07:00
|
|
|
body = (chunk[0] & 0x0F) === 0x08 ? pump(body, zlib.createInflate(), reject) : pump(body, zlib.createInflateRaw(), reject);
|
2020-03-13 08:06:25 -07:00
|
|
|
|
|
|
|
response = new Response(body, responseOptions);
|
2018-11-12 20:40:11 -08:00
|
|
|
resolve(response);
|
2016-08-03 02:31:46 -07:00
|
|
|
});
|
|
|
|
return;
|
|
|
|
}
|
2015-01-26 02:15:07 -08:00
|
|
|
|
2020-03-13 08:06:25 -07:00
|
|
|
// For br
|
2020-05-22 19:48:14 -07:00
|
|
|
if (codings === 'br') {
|
2020-09-18 03:33:15 -07:00
|
|
|
body = pump(body, zlib.createBrotliDecompress(), reject);
|
2020-03-13 08:06:25 -07:00
|
|
|
response = new Response(body, responseOptions);
|
2019-04-26 09:20:15 -07:00
|
|
|
resolve(response);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-03-13 08:06:25 -07:00
|
|
|
// Otherwise, use response as-is
|
|
|
|
response = new Response(body, responseOptions);
|
2018-11-12 20:40:11 -08:00
|
|
|
resolve(response);
|
2015-01-26 02:15:07 -08:00
|
|
|
});
|
|
|
|
|
2020-03-13 08:06:25 -07:00
|
|
|
writeToStream(request_, request);
|
2015-01-26 02:15:07 -08:00
|
|
|
});
|
2020-05-25 10:20:30 -07:00
|
|
|
}
|
2021-02-22 23:14:09 -08:00
|
|
|
|
|
|
|
function fixResponseChunkedTransferBadEnding(request, errorCallback) {
|
2021-08-12 09:37:22 -07:00
|
|
|
const LAST_CHUNK = Buffer.from('0\r\n\r\n');
|
2021-02-22 23:14:09 -08:00
|
|
|
|
2021-08-12 09:37:22 -07:00
|
|
|
let isChunkedTransfer = false;
|
|
|
|
let properLastChunkReceived = false;
|
|
|
|
let previousChunk;
|
2021-02-22 23:14:09 -08:00
|
|
|
|
|
|
|
request.on('response', response => {
|
|
|
|
const {headers} = response;
|
2021-08-12 09:37:22 -07:00
|
|
|
isChunkedTransfer = headers['transfer-encoding'] === 'chunked' && !headers['content-length'];
|
|
|
|
});
|
2021-02-22 23:14:09 -08:00
|
|
|
|
2021-08-12 09:37:22 -07:00
|
|
|
request.on('socket', socket => {
|
|
|
|
const onSocketClose = () => {
|
|
|
|
if (isChunkedTransfer && !properLastChunkReceived) {
|
|
|
|
const error = new Error('Premature close');
|
|
|
|
error.code = 'ERR_STREAM_PREMATURE_CLOSE';
|
|
|
|
errorCallback(error);
|
|
|
|
}
|
|
|
|
};
|
2021-02-22 23:14:09 -08:00
|
|
|
|
2021-08-12 09:37:22 -07:00
|
|
|
socket.prependListener('close', onSocketClose);
|
|
|
|
|
|
|
|
request.on('abort', () => {
|
|
|
|
socket.removeListener('close', onSocketClose);
|
|
|
|
});
|
|
|
|
|
|
|
|
socket.on('data', buf => {
|
|
|
|
properLastChunkReceived = Buffer.compare(buf.slice(-5), LAST_CHUNK) === 0;
|
|
|
|
|
|
|
|
// Sometimes final 0-length chunk and end of message code are in separate packets
|
|
|
|
if (!properLastChunkReceived && previousChunk) {
|
|
|
|
properLastChunkReceived = (
|
|
|
|
Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 &&
|
|
|
|
Buffer.compare(buf.slice(-2), LAST_CHUNK.slice(3)) === 0
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
previousChunk = buf;
|
|
|
|
});
|
2021-02-22 23:14:09 -08:00
|
|
|
});
|
|
|
|
}
|