/** * Index.js * * a request API compatible with window.fetch * * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. */ import http from 'node:http'; import https from 'node:https'; import zlib from 'node:zlib'; import Stream, {PassThrough, pipeline as pump} from 'node:stream'; import {Buffer} from 'node:buffer'; import dataUriToBuffer from 'data-uri-to-buffer'; import {writeToStream, clone} from './body.js'; import Response from './response.js'; import Headers, {fromRawHeaders} from './headers.js'; import Request, {getNodeRequestOptions} from './request.js'; import {FetchError} from './errors/fetch-error.js'; import {AbortError} from './errors/abort-error.js'; import {isRedirect} from './utils/is-redirect.js'; import {FormData} from 'formdata-polyfill/esm.min.js'; import {isDomainOrSubdomain} from './utils/is.js'; import {parseReferrerPolicyFromHeader} from './utils/referrer.js'; import { Blob, File, fileFromSync, fileFrom, blobFromSync, blobFrom } from 'fetch-blob/from.js'; export {FormData, Headers, Request, Response, FetchError, AbortError, isRedirect}; export {Blob, File, fileFromSync, fileFrom, blobFromSync, blobFrom}; const supportedSchemas = new Set(['data:', 'http:', 'https:']); /** * Fetch function * * @param {string | URL | import('./request').default} url - Absolute url or Request instance * @param {*} [options_] - Fetch options * @return {Promise} */ export default async function fetch(url, options_) { return new Promise((resolve, reject) => { // Build request object const request = new Request(url, options_); const {parsedURL, options} = getNodeRequestOptions(request); if (!supportedSchemas.has(parsedURL.protocol)) { throw new TypeError(`node-fetch cannot load ${url}. URL scheme "${parsedURL.protocol.replace(/:$/, '')}" is not supported.`); } if (parsedURL.protocol === 'data:') { const data = dataUriToBuffer(request.url); const response = new Response(data, {headers: {'Content-Type': data.typeFull}}); resolve(response); return; } // Wrap http.request into fetch const send = (parsedURL.protocol === 'https:' ? https : http).request; const {signal} = request; let response = null; const abort = () => { const error = new AbortError('The operation was aborted.'); reject(error); if (request.body && request.body instanceof Stream.Readable) { request.body.destroy(error); } if (!response || !response.body) { return; } response.body.emit('error', error); }; if (signal && signal.aborted) { abort(); return; } const abortAndFinalize = () => { abort(); finalize(); }; // Send request const request_ = send(parsedURL.toString(), options); if (signal) { signal.addEventListener('abort', abortAndFinalize); } const finalize = () => { request_.abort(); if (signal) { signal.removeEventListener('abort', abortAndFinalize); } }; request_.on('error', error => { reject(new FetchError(`request to ${request.url} failed, reason: ${error.message}`, 'system', error)); finalize(); }); fixResponseChunkedTransferBadEnding(request_, error => { response.body.destroy(error); }); /* c8 ignore next 18 */ if (process.version < 'v14') { // Before Node.js 14, pipeline() does not fully support async iterators and does not always // properly handle when the socket close/end events are out of order. request_.on('socket', s => { let endedWithEventsCount; s.prependListener('end', () => { endedWithEventsCount = s._eventsCount; }); s.prependListener('close', hadError => { // if end happened before close but the socket didn't emit an error, do it now if (response && endedWithEventsCount < s._eventsCount && !hadError) { const error = new Error('Premature close'); error.code = 'ERR_STREAM_PREMATURE_CLOSE'; response.body.emit('error', error); } }); }); } request_.on('response', response_ => { request_.setTimeout(0); const headers = fromRawHeaders(response_.rawHeaders); // HTTP fetch step 5 if (isRedirect(response_.statusCode)) { // HTTP fetch step 5.2 const location = headers.get('Location'); // HTTP fetch step 5.3 let locationURL = null; try { locationURL = location === null ? null : new URL(location, request.url); } catch { // error here can only be invalid URL in Location: header // do not throw when options.redirect == manual // let the user extract the errorneous redirect URL if (request.redirect !== 'manual') { reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect')); finalize(); return; } } // HTTP fetch step 5.5 switch (request.redirect) { case 'error': reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${request.url}`, 'no-redirect')); finalize(); return; case 'manual': // Nothing to do break; case 'follow': { // HTTP-redirect fetch step 2 if (locationURL === null) { break; } // HTTP-redirect fetch step 5 if (request.counter >= request.follow) { reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); finalize(); return; } // HTTP-redirect fetch step 6 (counter increment) // Create a new Request object. const requestOptions = { headers: new Headers(request.headers), follow: request.follow, counter: request.counter + 1, agent: request.agent, compress: request.compress, method: request.method, body: clone(request), signal: request.signal, size: request.size, referrer: request.referrer, referrerPolicy: request.referrerPolicy }; // when forwarding sensitive headers like "Authorization", // "WWW-Authenticate", and "Cookie" to untrusted targets, // headers will be ignored when following a redirect to a domain // that is not a subdomain match or exact match of the initial domain. // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" // will forward the sensitive headers, but a redirect to "bar.com" will not. if (!isDomainOrSubdomain(request.url, locationURL)) { for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) { requestOptions.headers.delete(name); } } // HTTP-redirect fetch step 9 if (response_.statusCode !== 303 && request.body && options_.body instanceof Stream.Readable) { reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')); finalize(); return; } // HTTP-redirect fetch step 11 if (response_.statusCode === 303 || ((response_.statusCode === 301 || response_.statusCode === 302) && request.method === 'POST')) { requestOptions.method = 'GET'; requestOptions.body = undefined; requestOptions.headers.delete('content-length'); } // HTTP-redirect fetch step 14 const responseReferrerPolicy = parseReferrerPolicyFromHeader(headers); if (responseReferrerPolicy) { requestOptions.referrerPolicy = responseReferrerPolicy; } // HTTP-redirect fetch step 15 resolve(fetch(new Request(locationURL, requestOptions))); finalize(); return; } default: return reject(new TypeError(`Redirect option '${request.redirect}' is not a valid value of RequestRedirect`)); } } // Prepare response if (signal) { response_.once('end', () => { signal.removeEventListener('abort', abortAndFinalize); }); } let body = pump(response_, new PassThrough(), error => { if (error) { reject(error); } }); // see https://github.com/nodejs/node/pull/29376 /* c8 ignore next 3 */ if (process.version < 'v12.10') { response_.on('aborted', abortAndFinalize); } const responseOptions = { url: request.url, status: response_.statusCode, statusText: response_.statusMessage, headers, size: request.size, counter: request.counter, highWaterMark: request.highWaterMark }; // HTTP-network fetch step 12.1.1.3 const codings = headers.get('Content-Encoding'); // HTTP-network fetch step 12.1.1.4: handle content codings // in following scenarios we ignore compression support // 1. compression support is disabled // 2. HEAD request // 3. no Content-Encoding header // 4. no content response (204) // 5. content not modified response (304) if (!request.compress || request.method === 'HEAD' || codings === null || response_.statusCode === 204 || response_.statusCode === 304) { response = new Response(body, responseOptions); resolve(response); return; } // For Node v6+ // Be less strict when decoding compressed responses, since sometimes // servers send slightly invalid responses that are still accepted // by common browsers. // Always using Z_SYNC_FLUSH is what cURL does. const zlibOptions = { flush: zlib.Z_SYNC_FLUSH, finishFlush: zlib.Z_SYNC_FLUSH }; // For gzip if (codings === 'gzip' || codings === 'x-gzip') { body = pump(body, zlib.createGunzip(zlibOptions), error => { if (error) { reject(error); } }); response = new Response(body, responseOptions); resolve(response); return; } // For deflate if (codings === 'deflate' || codings === 'x-deflate') { // Handle the infamous raw deflate response from old servers // a hack for old IIS and Apache servers const raw = pump(response_, new PassThrough(), error => { if (error) { reject(error); } }); raw.once('data', chunk => { // See http://stackoverflow.com/questions/37519828 if ((chunk[0] & 0x0F) === 0x08) { body = pump(body, zlib.createInflate(), error => { if (error) { reject(error); } }); } else { body = pump(body, zlib.createInflateRaw(), error => { if (error) { reject(error); } }); } response = new Response(body, responseOptions); resolve(response); }); raw.once('end', () => { // Some old IIS servers return zero-length OK deflate responses, so // 'data' is never emitted. See https://github.com/node-fetch/node-fetch/pull/903 if (!response) { response = new Response(body, responseOptions); resolve(response); } }); return; } // For br if (codings === 'br') { body = pump(body, zlib.createBrotliDecompress(), error => { if (error) { reject(error); } }); response = new Response(body, responseOptions); resolve(response); return; } // Otherwise, use response as-is response = new Response(body, responseOptions); resolve(response); }); // eslint-disable-next-line promise/prefer-await-to-then writeToStream(request_, request).catch(reject); }); } function fixResponseChunkedTransferBadEnding(request, errorCallback) { const LAST_CHUNK = Buffer.from('0\r\n\r\n'); let isChunkedTransfer = false; let properLastChunkReceived = false; let previousChunk; request.on('response', response => { const {headers} = response; isChunkedTransfer = headers['transfer-encoding'] === 'chunked' && !headers['content-length']; }); request.on('socket', socket => { const onSocketClose = () => { if (isChunkedTransfer && !properLastChunkReceived) { const error = new Error('Premature close'); error.code = 'ERR_STREAM_PREMATURE_CLOSE'; errorCallback(error); } }; socket.prependListener('close', onSocketClose); request.on('abort', () => { socket.removeListener('close', onSocketClose); }); socket.on('data', buf => { properLastChunkReceived = Buffer.compare(buf.slice(-5), LAST_CHUNK) === 0; // Sometimes final 0-length chunk and end of message code are in separate packets if (!properLastChunkReceived && previousChunk) { properLastChunkReceived = ( Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 && Buffer.compare(buf.slice(-2), LAST_CHUNK.slice(3)) === 0 ); } previousChunk = buf; }); }); }