feat: Implement form-data encoding (#603)
Co-authored-by: Steve Moser <contact@stevemoser.org> Co-authored-by: Antoni Kepinski <xxczaki@pm.me> Co-authored-by: Richie Bendall <richiebendall@gmail.com> Co-authored-by: Konstantin Vyatkin <tino@vtkn.io> Co-authored-by: aeb-sia <50743092+aeb-sia@users.noreply.github.com> Co-authored-by: Nazar Mokrynskyi <nazar@mokrynskyi.com> Co-authored-by: Erick Calder <e@arix.com> Co-authored-by: Yaacov Rydzinski <yaacovCR@gmail.com> Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com>
This commit is contained in:
parent
2d796bde76
commit
a38b533ad6
14
README.md
14
README.md
|
@ -381,6 +381,20 @@ const options = {
|
|||
})();
|
||||
```
|
||||
|
||||
node-fetch also supports spec-compliant FormData implementations such as [formdata-node](https://github.com/octet-stream/form-data):
|
||||
|
||||
```js
|
||||
const fetch = require('node-fetch');
|
||||
const FormData = require('formdata-node');
|
||||
|
||||
const form = new FormData();
|
||||
form.set('greeting', 'Hello, world!');
|
||||
|
||||
fetch('https://httpbin.org/post', {method: 'POST', body: form})
|
||||
.then(res => res.json())
|
||||
.then(json => console.log(json));
|
||||
```
|
||||
|
||||
### Request cancellation with AbortSignal
|
||||
|
||||
You may cancel requests with `AbortController`. A suggested implementation is [`abort-controller`](https://www.npmjs.com/package/abort-controller).
|
||||
|
|
12
package.json
12
package.json
|
@ -49,6 +49,7 @@
|
|||
"devDependencies": {
|
||||
"abort-controller": "^3.0.0",
|
||||
"abortcontroller-polyfill": "^1.4.0",
|
||||
"busboy": "^0.3.1",
|
||||
"c8": "^7.1.2",
|
||||
"chai": "^4.2.0",
|
||||
"chai-as-promised": "^7.1.1",
|
||||
|
@ -57,6 +58,7 @@
|
|||
"coveralls": "^3.1.0",
|
||||
"delay": "^4.3.0",
|
||||
"form-data": "^3.0.0",
|
||||
"formdata-node": "^2.0.0",
|
||||
"mocha": "^7.2.0",
|
||||
"p-timeout": "^3.2.0",
|
||||
"parted": "^0.1.1",
|
||||
|
@ -94,7 +96,15 @@
|
|||
"import/extensions": 0,
|
||||
"import/no-useless-path-segments": 0,
|
||||
"unicorn/import-index": 0,
|
||||
"capitalized-comments": 0
|
||||
"capitalized-comments": 0,
|
||||
"node/no-unsupported-features/node-builtins": [
|
||||
"error",
|
||||
{
|
||||
"ignores": [
|
||||
"stream.Readable.from"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"ignores": [
|
||||
"dist",
|
||||
|
|
26
src/body.js
26
src/body.js
|
@ -9,9 +9,11 @@ import Stream, {PassThrough} from 'stream';
|
|||
import {types} from 'util';
|
||||
|
||||
import Blob from 'fetch-blob';
|
||||
|
||||
import {FetchError} from './errors/fetch-error.js';
|
||||
import {FetchBaseError} from './errors/base.js';
|
||||
import {isBlob, isURLSearchParameters} from './utils/is.js';
|
||||
import {formDataIterator, getBoundary, getFormDataLength} from './utils/form-data.js';
|
||||
import {isBlob, isURLSearchParameters, isFormData} from './utils/is.js';
|
||||
|
||||
const INTERNALS = Symbol('Body internals');
|
||||
|
||||
|
@ -28,6 +30,8 @@ export default class Body {
|
|||
constructor(body, {
|
||||
size = 0
|
||||
} = {}) {
|
||||
let boundary = null;
|
||||
|
||||
if (body === null) {
|
||||
// Body is undefined or null
|
||||
body = null;
|
||||
|
@ -46,6 +50,10 @@ export default class Body {
|
|||
body = Buffer.from(body.buffer, body.byteOffset, body.byteLength);
|
||||
} else if (body instanceof Stream) {
|
||||
// Body is stream
|
||||
} else if (isFormData(body)) {
|
||||
// Body is an instance of formdata-node
|
||||
boundary = `NodeFetchFormDataBoundary${getBoundary()}`;
|
||||
body = Stream.Readable.from(formDataIterator(body, boundary));
|
||||
} else {
|
||||
// None of the above
|
||||
// coerce to string then buffer
|
||||
|
@ -54,6 +62,7 @@ export default class Body {
|
|||
|
||||
this[INTERNALS] = {
|
||||
body,
|
||||
boundary,
|
||||
disturbed: false,
|
||||
error: null
|
||||
};
|
||||
|
@ -264,7 +273,7 @@ export const clone = (instance, highWaterMark) => {
|
|||
* @param {any} body Any options.body input
|
||||
* @returns {string | null}
|
||||
*/
|
||||
export const extractContentType = body => {
|
||||
export const extractContentType = (body, request) => {
|
||||
// Body is null or undefined
|
||||
if (body === null) {
|
||||
return null;
|
||||
|
@ -295,6 +304,10 @@ export const extractContentType = body => {
|
|||
return `multipart/form-data;boundary=${body.getBoundary()}`;
|
||||
}
|
||||
|
||||
if (isFormData(body)) {
|
||||
return `multipart/form-data; boundary=${request[INTERNALS].boundary}`;
|
||||
}
|
||||
|
||||
// Body is stream - can't really do much about this
|
||||
if (body instanceof Stream) {
|
||||
return null;
|
||||
|
@ -313,7 +326,9 @@ export const extractContentType = body => {
|
|||
* @param {any} obj.body Body object from the Body instance.
|
||||
* @returns {number | null}
|
||||
*/
|
||||
export const getTotalBytes = ({body}) => {
|
||||
export const getTotalBytes = request => {
|
||||
const {body} = request;
|
||||
|
||||
// Body is null or undefined
|
||||
if (body === null) {
|
||||
return 0;
|
||||
|
@ -334,6 +349,11 @@ export const getTotalBytes = ({body}) => {
|
|||
return body.hasKnownLength && body.hasKnownLength() ? body.getLengthSync() : null;
|
||||
}
|
||||
|
||||
// Body is a spec-compliant form-data
|
||||
if (isFormData(body)) {
|
||||
return getFormDataLength(request[INTERNALS].boundary);
|
||||
}
|
||||
|
||||
// Body is stream
|
||||
return null;
|
||||
};
|
||||
|
|
|
@ -69,7 +69,7 @@ export default class Request extends Body {
|
|||
const headers = new Headers(init.headers || input.headers || {});
|
||||
|
||||
if (inputBody !== null && !headers.has('Content-Type')) {
|
||||
const contentType = extractContentType(inputBody);
|
||||
const contentType = extractContentType(inputBody, this);
|
||||
if (contentType) {
|
||||
headers.append('Content-Type', contentType);
|
||||
}
|
||||
|
@ -169,7 +169,8 @@ export const getNodeRequestOptions = request => {
|
|||
|
||||
if (request.body !== null) {
|
||||
const totalBytes = getTotalBytes(request);
|
||||
if (typeof totalBytes === 'number') {
|
||||
// Set Content-Length if totalBytes is a number (that is not NaN)
|
||||
if (typeof totalBytes === 'number' && !Number.isNaN(totalBytes)) {
|
||||
contentLengthValue = String(totalBytes);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,82 @@
|
|||
import {randomBytes} from 'crypto';
|
||||
|
||||
import {isBlob} from './is.js';
|
||||
|
||||
const carriage = '\r\n';
|
||||
const dashes = '-'.repeat(2);
|
||||
const carriageLength = Buffer.byteLength(carriage);
|
||||
|
||||
/**
|
||||
* @param {string} boundary
|
||||
*/
|
||||
const getFooter = boundary => `${dashes}${boundary}${dashes}${carriage.repeat(2)}`;
|
||||
|
||||
/**
|
||||
* @param {string} boundary
|
||||
* @param {string} name
|
||||
* @param {*} field
|
||||
*
|
||||
* @return {string}
|
||||
*/
|
||||
function getHeader(boundary, name, field) {
|
||||
let header = '';
|
||||
|
||||
header += `${dashes}${boundary}${carriage}`;
|
||||
header += `Content-Disposition: form-data; name="${name}"`;
|
||||
|
||||
if (isBlob(field)) {
|
||||
header += `; filename="${field.name}"${carriage}`;
|
||||
header += `Content-Type: ${field.type || 'application/octet-stream'}`;
|
||||
}
|
||||
|
||||
return `${header}${carriage.repeat(2)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return {string}
|
||||
*/
|
||||
export const getBoundary = () => randomBytes(8).toString('hex');
|
||||
|
||||
/**
|
||||
* @param {FormData} form
|
||||
* @param {string} boundary
|
||||
*/
|
||||
export async function * formDataIterator(form, boundary) {
|
||||
for (const [name, value] of form) {
|
||||
yield getHeader(boundary, name, value);
|
||||
|
||||
if (isBlob(value)) {
|
||||
yield * value.stream();
|
||||
} else {
|
||||
yield value;
|
||||
}
|
||||
|
||||
yield carriage;
|
||||
}
|
||||
|
||||
yield getFooter(boundary);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {FormData} form
|
||||
* @param {string} boundary
|
||||
*/
|
||||
export function getFormDataLength(form, boundary) {
|
||||
let length = 0;
|
||||
|
||||
for (const [name, value] of form) {
|
||||
length += Buffer.byteLength(getHeader(boundary, name, value));
|
||||
|
||||
if (isBlob(value)) {
|
||||
length += value.size;
|
||||
} else {
|
||||
length += Buffer.byteLength(String(value));
|
||||
}
|
||||
|
||||
length += carriageLength;
|
||||
}
|
||||
|
||||
length += Buffer.byteLength(getFooter(boundary));
|
||||
|
||||
return length;
|
||||
}
|
|
@ -28,7 +28,7 @@ export const isURLSearchParameters = object => {
|
|||
};
|
||||
|
||||
/**
|
||||
* Check if `obj` is a W3C `Blob` object (which `File` inherits from)
|
||||
* Check if `object` is a W3C `Blob` object (which `File` inherits from)
|
||||
*
|
||||
* @param {*} obj
|
||||
* @return {boolean}
|
||||
|
@ -44,6 +44,28 @@ export const isBlob = object => {
|
|||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if `obj` is a spec-compliant `FormData` object
|
||||
*
|
||||
* @param {*} object
|
||||
* @return {boolean}
|
||||
*/
|
||||
export function isFormData(object) {
|
||||
return (
|
||||
typeof object === 'object' &&
|
||||
typeof object.append === 'function' &&
|
||||
typeof object.set === 'function' &&
|
||||
typeof object.get === 'function' &&
|
||||
typeof object.getAll === 'function' &&
|
||||
typeof object.delete === 'function' &&
|
||||
typeof object.keys === 'function' &&
|
||||
typeof object.values === 'function' &&
|
||||
typeof object.entries === 'function' &&
|
||||
typeof object.constructor === 'function' &&
|
||||
object[NAME] === 'FormData'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if `obj` is an instance of AbortSignal.
|
||||
*
|
||||
|
|
|
@ -0,0 +1,104 @@
|
|||
import FormData from 'formdata-node';
|
||||
import Blob from 'fetch-blob';
|
||||
|
||||
import chai from 'chai';
|
||||
|
||||
import read from './utils/read-stream.js';
|
||||
|
||||
import {getFormDataLength, getBoundary, formDataIterator} from '../src/utils/form-data.js';
|
||||
|
||||
const {expect} = chai;
|
||||
|
||||
const carriage = '\r\n';
|
||||
|
||||
const getFooter = boundary => `--${boundary}--${carriage.repeat(2)}`;
|
||||
|
||||
describe('FormData', () => {
|
||||
it('should return a length for empty form-data', () => {
|
||||
const form = new FormData();
|
||||
const boundary = getBoundary();
|
||||
|
||||
expect(getFormDataLength(form, boundary)).to.be.equal(Buffer.byteLength(getFooter(boundary)));
|
||||
});
|
||||
|
||||
it('should add a Blob field\'s size to the FormData length', () => {
|
||||
const form = new FormData();
|
||||
const boundary = getBoundary();
|
||||
|
||||
const string = 'Hello, world!';
|
||||
const expected = Buffer.byteLength(
|
||||
`--${boundary}${carriage}` +
|
||||
`Content-Disposition: form-data; name="field"${carriage.repeat(2)}` +
|
||||
string +
|
||||
`${carriage}${getFooter(boundary)}`
|
||||
);
|
||||
|
||||
form.set('field', string);
|
||||
|
||||
expect(getFormDataLength(form, boundary)).to.be.equal(expected);
|
||||
});
|
||||
|
||||
it('should return a length for a Blob field', () => {
|
||||
const form = new FormData();
|
||||
const boundary = getBoundary();
|
||||
|
||||
const blob = new Blob(['Hello, world!'], {type: 'text/plain'});
|
||||
|
||||
form.set('blob', blob);
|
||||
|
||||
const expected = blob.size + Buffer.byteLength(
|
||||
`--${boundary}${carriage}` +
|
||||
'Content-Disposition: form-data; name="blob"; ' +
|
||||
`filename="blob"${carriage}Content-Type: text/plain` +
|
||||
`${carriage.repeat(3)}${getFooter(boundary)}`
|
||||
);
|
||||
|
||||
expect(getFormDataLength(form, boundary)).to.be.equal(expected);
|
||||
});
|
||||
|
||||
it('should create a body from empty form-data', async () => {
|
||||
const form = new FormData();
|
||||
const boundary = getBoundary();
|
||||
|
||||
expect(String(await read(formDataIterator(form, boundary)))).to.be.equal(getFooter(boundary));
|
||||
});
|
||||
|
||||
it('should set default content-type', async () => {
|
||||
const form = new FormData();
|
||||
const boundary = getBoundary();
|
||||
|
||||
form.set('blob', new Blob([]));
|
||||
|
||||
expect(String(await read(formDataIterator(form, boundary)))).to.contain('Content-Type: application/octet-stream');
|
||||
});
|
||||
|
||||
it('should create a body with a FormData field', async () => {
|
||||
const form = new FormData();
|
||||
const boundary = getBoundary();
|
||||
const string = 'Hello, World!';
|
||||
|
||||
form.set('field', string);
|
||||
|
||||
const expected = `--${boundary}${carriage}` +
|
||||
`Content-Disposition: form-data; name="field"${carriage.repeat(2)}` +
|
||||
string +
|
||||
`${carriage}${getFooter(boundary)}`;
|
||||
|
||||
expect(String(await read(formDataIterator(form, boundary)))).to.be.equal(expected);
|
||||
});
|
||||
|
||||
it('should create a body with a FormData Blob field', async () => {
|
||||
const form = new FormData();
|
||||
const boundary = getBoundary();
|
||||
|
||||
const expected = `--${boundary}${carriage}` +
|
||||
'Content-Disposition: form-data; name="blob"; ' +
|
||||
`filename="blob"${carriage}Content-Type: text/plain${carriage.repeat(2)}` +
|
||||
'Hello, World!' +
|
||||
`${carriage}${getFooter(boundary)}`;
|
||||
|
||||
form.set('blob', new Blob(['Hello, World!'], {type: 'text/plain'}));
|
||||
|
||||
expect(String(await read(formDataIterator(form, boundary)))).to.be.equal(expected);
|
||||
});
|
||||
});
|
37
test/main.js
37
test/main.js
|
@ -1,10 +1,10 @@
|
|||
// Test tools
|
||||
/* eslint-disable node/no-unsupported-features/node-builtins */
|
||||
import zlib from 'zlib';
|
||||
import crypto from 'crypto';
|
||||
import http from 'http';
|
||||
import fs from 'fs';
|
||||
import stream from 'stream';
|
||||
import path from 'path';
|
||||
import {lookup} from 'dns';
|
||||
import vm from 'vm';
|
||||
import chai from 'chai';
|
||||
|
@ -12,6 +12,7 @@ import chaiPromised from 'chai-as-promised';
|
|||
import chaiIterator from 'chai-iterator';
|
||||
import chaiString from 'chai-string';
|
||||
import FormData from 'form-data';
|
||||
import FormDataNode from 'formdata-node';
|
||||
import stringToArrayBuffer from 'string-to-arraybuffer';
|
||||
import delay from 'delay';
|
||||
import AbortControllerPolyfill from 'abortcontroller-polyfill/dist/abortcontroller.js';
|
||||
|
@ -58,8 +59,6 @@ after(done => {
|
|||
local.stop(done);
|
||||
});
|
||||
|
||||
const itIf = value => value ? it : it.skip;
|
||||
|
||||
function streamToPromise(stream, dataHandler) {
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (...args) => {
|
||||
|
@ -108,7 +107,8 @@ describe('node-fetch', () => {
|
|||
return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, /URL scheme "ftp" is not supported/);
|
||||
});
|
||||
|
||||
itIf(process.platform !== 'win32')('should reject with error on network failure', () => {
|
||||
it('should reject with error on network failure', function () {
|
||||
this.timeout(5000);
|
||||
const url = 'http://localhost:50000/';
|
||||
return expect(fetch(url)).to.eventually.be.rejected
|
||||
.and.be.an.instanceOf(FetchError)
|
||||
|
@ -125,7 +125,8 @@ describe('node-fetch', () => {
|
|||
return expect(err).to.not.have.property('erroredSysCall');
|
||||
});
|
||||
|
||||
itIf(process.platform !== 'win32')('system error is extracted from failed requests', () => {
|
||||
it('system error is extracted from failed requests', function () {
|
||||
this.timeout(5000);
|
||||
const url = 'http://localhost:50000/';
|
||||
return expect(fetch(url)).to.eventually.be.rejected
|
||||
.and.be.an.instanceOf(FetchError)
|
||||
|
@ -1285,7 +1286,7 @@ describe('node-fetch', () => {
|
|||
});
|
||||
});
|
||||
|
||||
itIf(process.platform !== 'win32')('should allow POST request with form-data using stream as body', () => {
|
||||
it('should allow POST request with form-data using stream as body', () => {
|
||||
const form = new FormData();
|
||||
form.append('my_field', fs.createReadStream('test/utils/dummy.txt'));
|
||||
|
||||
|
@ -1329,6 +1330,30 @@ describe('node-fetch', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('should support spec-compliant form-data as POST body', () => {
|
||||
const form = new FormDataNode();
|
||||
|
||||
const filename = path.join('test', 'utils', 'dummy.txt');
|
||||
|
||||
form.set('field', 'some text');
|
||||
form.set('file', fs.createReadStream(filename), {
|
||||
size: fs.statSync(filename).size
|
||||
});
|
||||
|
||||
const url = `${base}multipart`;
|
||||
const options = {
|
||||
method: 'POST',
|
||||
body: form
|
||||
};
|
||||
|
||||
return fetch(url, options).then(res => res.json()).then(res => {
|
||||
expect(res.method).to.equal('POST');
|
||||
expect(res.headers['content-type']).to.startWith('multipart/form-data');
|
||||
expect(res.body).to.contain('field=');
|
||||
expect(res.body).to.contain('file=');
|
||||
});
|
||||
});
|
||||
|
||||
it('should allow POST request with object body', () => {
|
||||
const url = `${base}inspect`;
|
||||
// Note that fetch simply calls tostring on an object
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
/* eslint-disable node/no-unsupported-features/node-builtins */
|
||||
|
||||
import stream from 'stream';
|
||||
import http from 'http';
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
/* eslint-disable node/no-unsupported-features/node-builtins */
|
||||
|
||||
import * as stream from 'stream';
|
||||
import chai from 'chai';
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
export default async function readStream(stream) {
|
||||
const chunks = [];
|
||||
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk instanceof Buffer ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
|
||||
return Buffer.concat(chunks);
|
||||
}
|
|
@ -1,7 +1,6 @@
|
|||
import http from 'http';
|
||||
import zlib from 'zlib';
|
||||
import parted from 'parted';
|
||||
const {multipart: Multipart} = parted;
|
||||
import Busboy from 'busboy';
|
||||
|
||||
export default class TestServer {
|
||||
constructor() {
|
||||
|
@ -364,12 +363,19 @@ export default class TestServer {
|
|||
if (p === '/multipart') {
|
||||
res.statusCode = 200;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
const parser = new Multipart(request.headers['content-type']);
|
||||
const busboy = new Busboy({headers: request.headers});
|
||||
let body = '';
|
||||
parser.on('part', (field, part) => {
|
||||
body += field + '=' + part;
|
||||
busboy.on('file', async (fieldName, file, fileName) => {
|
||||
body += `${fieldName}=${fileName}`;
|
||||
// consume file data
|
||||
// eslint-disable-next-line no-empty, no-unused-vars
|
||||
for await (const c of file) { }
|
||||
});
|
||||
parser.on('end', () => {
|
||||
|
||||
busboy.on('field', (fieldName, value) => {
|
||||
body += `${fieldName}=${value}`;
|
||||
});
|
||||
busboy.on('finish', () => {
|
||||
res.end(JSON.stringify({
|
||||
method: request.method,
|
||||
url: request.url,
|
||||
|
@ -377,7 +383,7 @@ export default class TestServer {
|
|||
body
|
||||
}));
|
||||
});
|
||||
request.pipe(parser);
|
||||
request.pipe(busboy);
|
||||
}
|
||||
|
||||
if (p === '/m%C3%B6bius') {
|
||||
|
@ -387,4 +393,3 @@ export default class TestServer {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue