feat: Implement form-data encoding (#603)

Co-authored-by: Steve Moser <contact@stevemoser.org>
Co-authored-by: Antoni Kepinski <xxczaki@pm.me>
Co-authored-by: Richie Bendall <richiebendall@gmail.com>
Co-authored-by: Konstantin Vyatkin <tino@vtkn.io>
Co-authored-by: aeb-sia <50743092+aeb-sia@users.noreply.github.com>
Co-authored-by: Nazar Mokrynskyi <nazar@mokrynskyi.com>
Co-authored-by: Erick Calder <e@arix.com>
Co-authored-by: Yaacov Rydzinski <yaacovCR@gmail.com>
Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com>

README.md

@@ -381,6 +381,20 @@ const options = {
 })();
 ```
 
+node-fetch also supports spec-compliant FormData implementations such as [formdata-node](https://github.com/octet-stream/form-data):
+
+```js
+const fetch = require('node-fetch');
+const FormData = require('formdata-node');
+
+const form = new FormData();
+form.set('greeting', 'Hello, world!');
+
+fetch('https://httpbin.org/post', {method: 'POST', body: form})
+	.then(res => res.json())
+	.then(json => console.log(json));
+```
+
 ### Request cancellation with AbortSignal
 
 You may cancel requests with `AbortController`. A suggested implementation is [`abort-controller`](https://www.npmjs.com/package/abort-controller).

docs/CHANGELOG.md

@@ -33,7 +33,7 @@ Changelog
 
 ## v3.0.0-beta.5
 
-> NOTE: Since the previous beta version included serious issues, such as [#749](https://github.com/node-fetch/node-fetch/issues/749), they will now be deprecated. 
+> NOTE: Since the previous beta version included serious issues, such as [#749](https://github.com/node-fetch/node-fetch/issues/749), they will now be deprecated.
 
 - Enhance: use built-in AbortSignal for typings.
 - Enhance: compile CJS modules as a seperate set of files.

package.json

@@ -49,6 +49,7 @@
 	"devDependencies": {
 		"abort-controller": "^3.0.0",
 		"abortcontroller-polyfill": "^1.4.0",
+		"busboy": "^0.3.1",
 		"c8": "^7.1.2",
 		"chai": "^4.2.0",
 		"chai-as-promised": "^7.1.1",
@@ -57,6 +58,7 @@
 		"coveralls": "^3.1.0",
 		"delay": "^4.3.0",
 		"form-data": "^3.0.0",
+		"formdata-node": "^2.0.0",
 		"mocha": "^7.2.0",
 		"p-timeout": "^3.2.0",
 		"parted": "^0.1.1",
@@ -94,7 +96,15 @@
 			"import/extensions": 0,
 			"import/no-useless-path-segments": 0,
 			"unicorn/import-index": 0,
-			"capitalized-comments": 0
+			"capitalized-comments": 0,
+			"node/no-unsupported-features/node-builtins": [
+				"error",
+				{
+					"ignores": [
+						"stream.Readable.from"
+					]
+				}
+			]
 		},
 		"ignores": [
 			"dist",

src/body.js

@@ -9,9 +9,11 @@ import Stream, {PassThrough} from 'stream';
 import {types} from 'util';
 
 import Blob from 'fetch-blob';
+
 import {FetchError} from './errors/fetch-error.js';
 import {FetchBaseError} from './errors/base.js';
-import {isBlob, isURLSearchParameters} from './utils/is.js';
+import {formDataIterator, getBoundary, getFormDataLength} from './utils/form-data.js';
+import {isBlob, isURLSearchParameters, isFormData} from './utils/is.js';
 
 const INTERNALS = Symbol('Body internals');
 
@@ -28,6 +30,8 @@ export default class Body {
 	constructor(body, {
 		size = 0
 	} = {}) {
+		let boundary = null;
+
 		if (body === null) {
 			// Body is undefined or null
 			body = null;
@@ -46,6 +50,10 @@ export default class Body {
 			body = Buffer.from(body.buffer, body.byteOffset, body.byteLength);
 		} else if (body instanceof Stream) {
 			// Body is stream
+		} else if (isFormData(body)) {
+			// Body is an instance of formdata-node
+			boundary = `NodeFetchFormDataBoundary${getBoundary()}`;
+			body = Stream.Readable.from(formDataIterator(body, boundary));
 		} else {
 			// None of the above
 			// coerce to string then buffer
@@ -54,6 +62,7 @@ export default class Body {
 
 		this[INTERNALS] = {
 			body,
+			boundary,
 			disturbed: false,
 			error: null
 		};
@@ -146,7 +155,7 @@ Object.defineProperties(Body.prototype, {
  *
  * Ref: https://fetch.spec.whatwg.org/#concept-body-consume-body
  *
- * @return  Promise
+ * @return Promise
  */
 async function consumeBody(data) {
 	if (data[INTERNALS].disturbed) {
@@ -264,7 +273,7 @@ export const clone = (instance, highWaterMark) => {
  * @param {any} body Any options.body input
  * @returns {string | null}
  */
-export const extractContentType = body => {
+export const extractContentType = (body, request) => {
 	// Body is null or undefined
 	if (body === null) {
 		return null;
@@ -295,6 +304,10 @@ export const extractContentType = body => {
 		return `multipart/form-data;boundary=${body.getBoundary()}`;
 	}
 
+	if (isFormData(body)) {
+		return `multipart/form-data; boundary=${request[INTERNALS].boundary}`;
+	}
+
 	// Body is stream - can't really do much about this
 	if (body instanceof Stream) {
 		return null;
@@ -313,7 +326,9 @@ export const extractContentType = body => {
  * @param {any} obj.body Body object from the Body instance.
  * @returns {number | null}
  */
-export const getTotalBytes = ({body}) => {
+export const getTotalBytes = request => {
+	const {body} = request;
+
 	// Body is null or undefined
 	if (body === null) {
 		return 0;
@@ -334,6 +349,11 @@ export const getTotalBytes = ({body}) => {
 		return body.hasKnownLength && body.hasKnownLength() ? body.getLengthSync() : null;
 	}
 
+	// Body is a spec-compliant form-data
+	if (isFormData(body)) {
+		return getFormDataLength(request[INTERNALS].boundary);
+	}
+
 	// Body is stream
 	return null;
 };

src/request.js

@@ -69,7 +69,7 @@ export default class Request extends Body {
 		const headers = new Headers(init.headers || input.headers || {});
 
 		if (inputBody !== null && !headers.has('Content-Type')) {
-			const contentType = extractContentType(inputBody);
+			const contentType = extractContentType(inputBody, this);
 			if (contentType) {
 				headers.append('Content-Type', contentType);
 			}
@@ -169,7 +169,8 @@ export const getNodeRequestOptions = request => {
 
 	if (request.body !== null) {
 		const totalBytes = getTotalBytes(request);
-		if (typeof totalBytes === 'number') {
+		// Set Content-Length if totalBytes is a number (that is not NaN)
+		if (typeof totalBytes === 'number' && !Number.isNaN(totalBytes)) {
 			contentLengthValue = String(totalBytes);
 		}
 	}

src/utils/form-data.js

@@ -0,0 +1,82 @@
+import {randomBytes} from 'crypto';
+
+import {isBlob} from './is.js';
+
+const carriage = '\r\n';
+const dashes = '-'.repeat(2);
+const carriageLength = Buffer.byteLength(carriage);
+
+/**
+ * @param {string} boundary
+ */
+const getFooter = boundary => `${dashes}${boundary}${dashes}${carriage.repeat(2)}`;
+
+/**
+ * @param {string} boundary
+ * @param {string} name
+ * @param {*} field
+ *
+ * @return {string}
+ */
+function getHeader(boundary, name, field) {
+	let header = '';
+
+	header += `${dashes}${boundary}${carriage}`;
+	header += `Content-Disposition: form-data; name="${name}"`;
+
+	if (isBlob(field)) {
+		header += `; filename="${field.name}"${carriage}`;
+		header += `Content-Type: ${field.type || 'application/octet-stream'}`;
+	}
+
+	return `${header}${carriage.repeat(2)}`;
+}
+
+/**
+ * @return {string}
+ */
+export const getBoundary = () => randomBytes(8).toString('hex');
+
+/**
+ * @param {FormData} form
+ * @param {string} boundary
+ */
+export async function * formDataIterator(form, boundary) {
+	for (const [name, value] of form) {
+		yield getHeader(boundary, name, value);
+
+		if (isBlob(value)) {
+			yield * value.stream();
+		} else {
+			yield value;
+		}
+
+		yield carriage;
+	}
+
+	yield getFooter(boundary);
+}
+
+/**
+ * @param {FormData} form
+ * @param {string} boundary
+ */
+export function getFormDataLength(form, boundary) {
+	let length = 0;
+
+	for (const [name, value] of form) {
+		length += Buffer.byteLength(getHeader(boundary, name, value));
+
+		if (isBlob(value)) {
+			length += value.size;
+		} else {
+			length += Buffer.byteLength(String(value));
+		}
+
+		length += carriageLength;
+	}
+
+	length += Buffer.byteLength(getFooter(boundary));
+
+	return length;
+}

src/utils/is.js

@@ -28,7 +28,7 @@ export const isURLSearchParameters = object => {
 };
 
 /**
- * Check if `obj` is a W3C `Blob` object (which `File` inherits from)
+ * Check if `object` is a W3C `Blob` object (which `File` inherits from)
  *
  * @param  {*} obj
  * @return {boolean}
@@ -44,6 +44,28 @@ export const isBlob = object => {
 	);
 };
 
+/**
+ * Check if `obj` is a spec-compliant `FormData` object
+ *
+ * @param {*} object
+ * @return {boolean}
+ */
+export function isFormData(object) {
+	return (
+		typeof object === 'object' &&
+		typeof object.append === 'function' &&
+		typeof object.set === 'function' &&
+		typeof object.get === 'function' &&
+		typeof object.getAll === 'function' &&
+		typeof object.delete === 'function' &&
+		typeof object.keys === 'function' &&
+		typeof object.values === 'function' &&
+		typeof object.entries === 'function' &&
+		typeof object.constructor === 'function' &&
+		object[NAME] === 'FormData'
+	);
+}
+
 /**
  * Check if `obj` is an instance of AbortSignal.
  *

test/form-data.js

@@ -0,0 +1,104 @@
+import FormData from 'formdata-node';
+import Blob from 'fetch-blob';
+
+import chai from 'chai';
+
+import read from './utils/read-stream.js';
+
+import {getFormDataLength, getBoundary, formDataIterator} from '../src/utils/form-data.js';
+
+const {expect} = chai;
+
+const carriage = '\r\n';
+
+const getFooter = boundary => `--${boundary}--${carriage.repeat(2)}`;
+
+describe('FormData', () => {
+	it('should return a length for empty form-data', () => {
+		const form = new FormData();
+		const boundary = getBoundary();
+
+		expect(getFormDataLength(form, boundary)).to.be.equal(Buffer.byteLength(getFooter(boundary)));
+	});
+
+	it('should add a Blob field\'s size to the FormData length', () => {
+		const form = new FormData();
+		const boundary = getBoundary();
+
+		const string = 'Hello, world!';
+		const expected = Buffer.byteLength(
+			`--${boundary}${carriage}` +
+			`Content-Disposition: form-data; name="field"${carriage.repeat(2)}` +
+			string +
+			`${carriage}${getFooter(boundary)}`
+		);
+
+		form.set('field', string);
+
+		expect(getFormDataLength(form, boundary)).to.be.equal(expected);
+	});
+
+	it('should return a length for a Blob field', () => {
+		const form = new FormData();
+		const boundary = getBoundary();
+
+		const blob = new Blob(['Hello, world!'], {type: 'text/plain'});
+
+		form.set('blob', blob);
+
+		const expected = blob.size + Buffer.byteLength(
+			`--${boundary}${carriage}` +
+			'Content-Disposition: form-data; name="blob"; ' +
+			`filename="blob"${carriage}Content-Type: text/plain` +
+			`${carriage.repeat(3)}${getFooter(boundary)}`
+		);
+
+		expect(getFormDataLength(form, boundary)).to.be.equal(expected);
+	});
+
+	it('should create a body from empty form-data', async () => {
+		const form = new FormData();
+		const boundary = getBoundary();
+
+		expect(String(await read(formDataIterator(form, boundary)))).to.be.equal(getFooter(boundary));
+	});
+
+	it('should set default content-type', async () => {
+		const form = new FormData();
+		const boundary = getBoundary();
+
+		form.set('blob', new Blob([]));
+
+		expect(String(await read(formDataIterator(form, boundary)))).to.contain('Content-Type: application/octet-stream');
+	});
+
+	it('should create a body with a FormData field', async () => {
+		const form = new FormData();
+		const boundary = getBoundary();
+		const string = 'Hello, World!';
+
+		form.set('field', string);
+
+		const expected = `--${boundary}${carriage}` +
+		`Content-Disposition: form-data; name="field"${carriage.repeat(2)}` +
+		string +
+		`${carriage}${getFooter(boundary)}`;
+
+		expect(String(await read(formDataIterator(form, boundary)))).to.be.equal(expected);
+	});
+
+	it('should create a body with a FormData Blob field', async () => {
+		const form = new FormData();
+		const boundary = getBoundary();
+
+		const expected = `--${boundary}${carriage}` +
+		'Content-Disposition: form-data; name="blob"; ' +
+		`filename="blob"${carriage}Content-Type: text/plain${carriage.repeat(2)}` +
+		'Hello, World!' +
+		`${carriage}${getFooter(boundary)}`;
+
+		form.set('blob', new Blob(['Hello, World!'], {type: 'text/plain'}));
+
+		expect(String(await read(formDataIterator(form, boundary)))).to.be.equal(expected);
+	});
+});

test/main.js

@@ -1,10 +1,10 @@
 // Test tools
-/* eslint-disable node/no-unsupported-features/node-builtins */
 import zlib from 'zlib';
 import crypto from 'crypto';
 import http from 'http';
 import fs from 'fs';
 import stream from 'stream';
+import path from 'path';
 import {lookup} from 'dns';
 import vm from 'vm';
 import chai from 'chai';
@@ -12,6 +12,7 @@ import chaiPromised from 'chai-as-promised';
 import chaiIterator from 'chai-iterator';
 import chaiString from 'chai-string';
 import FormData from 'form-data';
+import FormDataNode from 'formdata-node';
 import stringToArrayBuffer from 'string-to-arraybuffer';
 import delay from 'delay';
 import AbortControllerPolyfill from 'abortcontroller-polyfill/dist/abortcontroller.js';
@@ -58,8 +59,6 @@ after(done => {
 	local.stop(done);
 });
 
-const itIf = value => value ? it : it.skip;
-
 function streamToPromise(stream, dataHandler) {
 	return new Promise((resolve, reject) => {
 		stream.on('data', (...args) => {
@@ -108,7 +107,8 @@ describe('node-fetch', () => {
 		return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, /URL scheme "ftp" is not supported/);
 	});
 
-	itIf(process.platform !== 'win32')('should reject with error on network failure', () => {
+	it('should reject with error on network failure', function () {
+		this.timeout(5000);
 		const url = 'http://localhost:50000/';
 		return expect(fetch(url)).to.eventually.be.rejected
 			.and.be.an.instanceOf(FetchError)
@@ -125,7 +125,8 @@ describe('node-fetch', () => {
 		return expect(err).to.not.have.property('erroredSysCall');
 	});
 
-	itIf(process.platform !== 'win32')('system error is extracted from failed requests', () => {
+	it('system error is extracted from failed requests', function () {
+		this.timeout(5000);
 		const url = 'http://localhost:50000/';
 		return expect(fetch(url)).to.eventually.be.rejected
 			.and.be.an.instanceOf(FetchError)
@@ -1285,7 +1286,7 @@ describe('node-fetch', () => {
 		});
 	});
 
-	itIf(process.platform !== 'win32')('should allow POST request with form-data using stream as body', () => {
+	it('should allow POST request with form-data using stream as body', () => {
 		const form = new FormData();
 		form.append('my_field', fs.createReadStream('test/utils/dummy.txt'));
 
@@ -1329,6 +1330,30 @@ describe('node-fetch', () => {
 		});
 	});
 
+	it('should support spec-compliant form-data as POST body', () => {
+		const form = new FormDataNode();
+
+		const filename = path.join('test', 'utils', 'dummy.txt');
+
+		form.set('field', 'some text');
+		form.set('file', fs.createReadStream(filename), {
+			size: fs.statSync(filename).size
+		});
+
+		const url = `${base}multipart`;
+		const options = {
+			method: 'POST',
+			body: form
+		};
+
+		return fetch(url, options).then(res => res.json()).then(res => {
+			expect(res.method).to.equal('POST');
+			expect(res.headers['content-type']).to.startWith('multipart/form-data');
+			expect(res.body).to.contain('field=');
+			expect(res.body).to.contain('file=');
+		});
+	});
+
 	it('should allow POST request with object body', () => {
 		const url = `${base}inspect`;
 		// Note that fetch simply calls tostring on an object

test/request.js

@@ -1,4 +1,3 @@
-/* eslint-disable node/no-unsupported-features/node-builtins */
 
 import stream from 'stream';
 import http from 'http';

test/response.js

@@ -1,4 +1,3 @@
-/* eslint-disable node/no-unsupported-features/node-builtins */
 
 import * as stream from 'stream';
 import chai from 'chai';

test/utils/read-stream.js

@@ -0,0 +1,9 @@
+export default async function readStream(stream) {
+	const chunks = [];
+
+	for await (const chunk of stream) {
+		chunks.push(chunk instanceof Buffer ? chunk : Buffer.from(chunk));
+	}
+
+	return Buffer.concat(chunks);
+}

test/utils/server.js

@@ -1,7 +1,6 @@
 import http from 'http';
 import zlib from 'zlib';
-import parted from 'parted';
+import Busboy from 'busboy';
-const {multipart: Multipart} = parted;
 
 export default class TestServer {
 	constructor() {
@@ -364,12 +363,19 @@ export default class TestServer {
 		if (p === '/multipart') {
 			res.statusCode = 200;
 			res.setHeader('Content-Type', 'application/json');
-			const parser = new Multipart(request.headers['content-type']);
+			const busboy = new Busboy({headers: request.headers});
 			let body = '';
-			parser.on('part', (field, part) => {
+			busboy.on('file', async (fieldName, file, fileName) => {
-				body += field + '=' + part;
+				body += `${fieldName}=${fileName}`;
+				// consume file data
+				// eslint-disable-next-line no-empty, no-unused-vars
+				for await (const c of file) { }
 			});
-			parser.on('end', () => {
+
+			busboy.on('field', (fieldName, value) => {
+				body += `${fieldName}=${value}`;
+			});
+			busboy.on('finish', () => {
 				res.end(JSON.stringify({
 					method: request.method,
 					url: request.url,
@@ -377,7 +383,7 @@ export default class TestServer {
 					body
 				}));
 			});
-			request.pipe(parser);
+			request.pipe(busboy);
 		}
 
 		if (p === '/m%C3%B6bius') {
@@ -387,4 +393,3 @@ export default class TestServer {
 		}
 	}
 }
-