feat: you can now supply your own HTTP agent to a web3.js Connection (#29125)

* You can now supply your own `https?.Agent` when creating a `Connection` object

* Don't use HTTP agents in test mode

* Tests that assert the behaviour of the `agentOverride` config of `Connection`

* s/agentOverride/httpAgent/ is less confusing when the value is `false`

web3.js/package.json

@@ -51,7 +51,7 @@
     "pretty": "prettier --check '{,{src,test}/**/}*.{j,t}s'",
     "pretty:fix": "prettier --write '{,{src,test}/**/}*.{j,t}s'",
     "re": "semantic-release --repository-url git@github.com:solana-labs/solana-web3.js.git",
-    "test": "cross-env TS_NODE_COMPILER_OPTIONS='{ \"module\": \"commonjs\", \"target\": \"es2019\" }' ts-mocha --require esm './test/**/*.test.ts'",
+    "test": "cross-env NODE_ENV=test TS_NODE_COMPILER_OPTIONS='{ \"module\": \"commonjs\", \"target\": \"es2019\" }' ts-mocha --require esm './test/**/*.test.ts'",
     "test:cover": "nyc --reporter=lcov npm run test",
     "test:live": "TEST_LIVE=1 npm run test",
     "test:live-with-test-validator": "start-server-and-test 'solana-test-validator --reset --quiet' http://localhost:8899/health test:live"

web3.js/src/connection.ts

@@ -2,6 +2,8 @@ import bs58 from 'bs58';
 import {Buffer} from 'buffer';
 // @ts-ignore
 import fastStableStringify from 'fast-stable-stringify';
+import type {Agent as HttpAgent} from 'http';
+import {Agent as HttpsAgent} from 'https';
 import {
   type as pick,
   number,
@@ -1450,11 +1452,47 @@ function createRpcClient(
   customFetch?: FetchFn,
   fetchMiddleware?: FetchMiddleware,
   disableRetryOnRateLimit?: boolean,
+  httpAgent?: HttpAgent | HttpsAgent | false,
 ): RpcClient {
   const fetch = customFetch ? customFetch : fetchImpl;
-  let agentManager: AgentManager | undefined;
-  if (!process.env.BROWSER) {
-    agentManager = new AgentManager(url.startsWith('https:') /* useHttps */);
+  let agentManager:
+    | {requestEnd(): void; requestStart(): HttpAgent | HttpsAgent}
+    | undefined;
+  if (process.env.BROWSER) {
+    if (httpAgent != null) {
+      console.warn(
+        'You have supplied an `httpAgent` when creating a `Connection` in a browser environment.' +
+          'It has been ignored; `httpAgent` is only used in Node environments.',
+      );
+    }
+  } else {
+    if (httpAgent == null) {
+      if (process.env.NODE_ENV !== 'test') {
+        agentManager = new AgentManager(
+          url.startsWith('https:') /* useHttps */,
+        );
+      }
+    } else {
+      if (httpAgent !== false) {
+        const isHttps = url.startsWith('https:');
+        if (isHttps && !(httpAgent instanceof HttpsAgent)) {
+          throw new Error(
+            'The endpoint `' +
+              url +
+              '` can only be paired with an `https.Agent`. You have, instead, supplied an ' +
+              '`http.Agent` through `httpAgent`.',
+          );
+        } else if (!isHttps && httpAgent instanceof HttpsAgent) {
+          throw new Error(
+            'The endpoint `' +
+              url +
+              '` can only be paired with an `http.Agent`. You have, instead, supplied an ' +
+              '`https.Agent` through `httpAgent`.',
+          );
+        }
+        agentManager = {requestEnd() {}, requestStart: () => httpAgent};
+      }
+    }
   }
 
   let fetchWithMiddleware: FetchFn | undefined;
@@ -2855,6 +2893,12 @@ export type FetchMiddleware = (
  * Configuration for instantiating a Connection
  */
 export type ConnectionConfig = {
+  /**
+   * An `http.Agent` that will be used to manage socket connections (eg. to implement connection
+   * persistence). Set this to `false` to create a connection that uses no agent. This applies to
+   * Node environments only.
+   */
+  httpAgent?: HttpAgent | HttpsAgent | false;
   /** Optional commitment level */
   commitment?: Commitment;
   /** Optional endpoint URL to the fullnode JSON RPC PubSub WebSocket Endpoint */
@@ -2972,6 +3016,7 @@ export class Connection {
     let fetch;
     let fetchMiddleware;
     let disableRetryOnRateLimit;
+    let httpAgent;
     if (commitmentOrConfig && typeof commitmentOrConfig === 'string') {
       this._commitment = commitmentOrConfig;
     } else if (commitmentOrConfig) {
@@ -2983,6 +3028,7 @@ export class Connection {
       fetch = commitmentOrConfig.fetch;
       fetchMiddleware = commitmentOrConfig.fetchMiddleware;
       disableRetryOnRateLimit = commitmentOrConfig.disableRetryOnRateLimit;
+      httpAgent = commitmentOrConfig.httpAgent;
     }
 
     this._rpcEndpoint = assertEndpointUrl(endpoint);
@@ -2994,6 +3040,7 @@ export class Connection {
       fetch,
       fetchMiddleware,
       disableRetryOnRateLimit,
+      httpAgent,
     );
     this._rpcRequest = createRpcRequest(this._rpcClient);
     this._rpcBatchRequest = createRpcBatchRequest(this._rpcClient);

web3.js/test/connection.test.ts

@@ -3,8 +3,10 @@ import {Buffer} from 'buffer';
 import * as splToken from '@solana/spl-token';
 import {expect, use} from 'chai';
 import chaiAsPromised from 'chai-as-promised';
+import {Agent as HttpAgent} from 'http';
+import {Agent as HttpsAgent} from 'https';
 import {AbortController} from 'node-abort-controller';
-import {mock, useFakeTimers, SinonFakeTimers} from 'sinon';
+import {match, mock, spy, useFakeTimers, SinonFakeTimers} from 'sinon';
 import sinonChai from 'sinon-chai';
 
 import {
@@ -188,6 +190,50 @@ describe('Connection', function () {
     });
   }
 
+  describe('override HTTP agent', () => {
+    let previousBrowserEnv;
+    beforeEach(() => {
+      previousBrowserEnv = process.env.BROWSER;
+      delete process.env.BROWSER;
+    });
+    afterEach(() => {
+      process.env.BROWSER = previousBrowserEnv;
+    });
+
+    it('uses no agent with fetch when `overrideAgent` is `false`', () => {
+      const fetch = spy();
+      const c = new Connection(url, {httpAgent: false, fetch});
+      c.getBlock(0);
+      expect(fetch).to.have.been.calledWith(
+        match.any,
+        match({agent: undefined}),
+      );
+    });
+
+    it('uses the supplied `overrideAgent` with fetch', () => {
+      const fetch = spy();
+      const httpAgent = new HttpsAgent();
+      const c = new Connection('https://example.com', {httpAgent, fetch});
+      c.getBlock(0);
+      expect(fetch).to.have.been.calledWith(
+        match.any,
+        match({agent: httpAgent}),
+      );
+    });
+
+    it('throws when the supplied `overrideAgent` is http but the endpoint is https', () => {
+      expect(() => {
+        new Connection('https://example.com', {httpAgent: new HttpAgent()});
+      }).to.throw;
+    });
+
+    it('throws when the supplied `overrideAgent` is https but the endpoint is http', () => {
+      expect(() => {
+        new Connection('http://example.com', {httpAgent: new HttpsAgent()});
+      }).to.throw;
+    });
+  });
+
   it('should attribute middleware fatals to the middleware', async () => {
     let connection = new Connection(url, {
       // eslint-disable-next-line @typescript-eslint/no-unused-vars