http: add http.setGlobalProxyFromEnv()

This adds an API to dynamically enable built-in proxy support
for all of fetch() and http.request()/https.request(), so
that users do not have to be aware of them all and configure them
one by one.

PR-URL: #60953
Reviewed-By: Matteo Collina <[email protected]>
Reviewed-By: Tim Perry <[email protected]>

Author: joyeecheung

doc/api/http.md

@@ -4361,6 +4361,32 @@ added:
 
 Set the maximum number of idle HTTP parsers.
 
+## `http.setGlobalProxyFromEnv([proxyEnv])`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+* `proxyEnv` {Object} An object containing proxy configuration. This accepts the
+  same options as the `proxyEnv` option accepted by [`Agent`][]. **Default:**
+  `process.env`.
+* Returns: {Function} A function that restores the original agent and dispatcher
+  settings to the state before this `http.setGlobalProxyFromEnv()` is invoked.
+
+Dynamically resets the global configurations to enable built-in proxy support for
+`fetch()` and `http.request()`/`https.request()` at runtime, as an alternative
+to using the `--use-env-proxy` flag or `NODE_USE_ENV_PROXY` environment variable.
+It can also be used to override settings configured from the environment variables.
+
+As this function resets the global configurations, any previously configured
+`http.globalAgent`, `https.globalAgent` or undici global dispatcher would be
+overridden after this function is invoked. It's recommended to invoke it before any
+requests are made and avoid invoking it in the middle of any requests.
+
+See [Built-in Proxy Support][] for details on proxy URL formats and `NO_PROXY`
+syntax.
+
 ## Class: `WebSocket`
 
 <!-- YAML
@@ -4383,6 +4409,9 @@ added:
 When Node.js creates the global agent, if the `NODE_USE_ENV_PROXY` environment variable is
 set to `1` or `--use-env-proxy` is enabled, the global agent will be constructed
 with `proxyEnv: process.env`, enabling proxy support based on the environment variables.
+
+To enable proxy support dynamically and globally, use [`http.setGlobalProxyFromEnv()`][].
+
 Custom agents can also be created with proxy support by passing a
 `proxyEnv` option when constructing the agent. The value can be `process.env`
 if they just want to inherit the configuration from the environment variables,
@@ -4438,6 +4467,86 @@ Or the `--use-env-proxy` flag.
 HTTP_PROXY=http://proxy.example.com:8080 NO_PROXY=localhost,127.0.0.1 node --use-env-proxy client.js
 ```
 
+To enable proxy support dynamically and globally with `process.env` (the default option of `http.setGlobalProxyFromEnv()`):
+
+```cjs
+const http = require('node:http');
+
+// Reads proxy-related environment variables from process.env
+const restore = http.setGlobalProxyFromEnv();
+
+// Subsequent requests will use the configured proxies from environment variables
+http.get('http://www.example.com', (res) => {
+  // This request will be proxied if HTTP_PROXY or http_proxy is set
+});
+
+fetch('https://www.example.com', (res) => {
+  // This request will be proxied if HTTPS_PROXY or https_proxy is set
+});
+
+// To restore the original global agent and dispatcher settings, call the returned function.
+// restore();
+```
+
+```mjs
+import http from 'node:http';
+
+// Reads proxy-related environment variables from process.env
+http.setGlobalProxyFromEnv();
+
+// Subsequent requests will use the configured proxies from environment variables
+http.get('http://www.example.com', (res) => {
+  // This request will be proxied if HTTP_PROXY or http_proxy is set
+});
+
+fetch('https://www.example.com', (res) => {
+  // This request will be proxied if HTTPS_PROXY or https_proxy is set
+});
+
+// To restore the original global agent and dispatcher settings, call the returned function.
+// restore();
+```
+
+To enable proxy support dynamically and globally with custom settings:
+
+```cjs
+const http = require('node:http');
+
+const restore = http.setGlobalProxyFromEnv({
+  http_proxy: 'http://proxy.example.com:8080',
+  https_proxy: 'https://proxy.example.com:8443',
+  no_proxy: 'localhost,127.0.0.1,.internal.example.com',
+});
+
+// Subsequent requests will use the configured proxies
+http.get('http://www.example.com', (res) => {
+  // This request will be proxied through proxy.example.com:8080
+});
+
+fetch('https://www.example.com', (res) => {
+  // This request will be proxied through proxy.example.com:8443
+});
+```
+
+```mjs
+import http from 'node:http';
+
+http.setGlobalProxyFromEnv({
+  http_proxy: 'http://proxy.example.com:8080',
+  https_proxy: 'https://proxy.example.com:8443',
+  no_proxy: 'localhost,127.0.0.1,.internal.example.com',
+});
+
+// Subsequent requests will use the configured proxies
+http.get('http://www.example.com', (res) => {
+  // This request will be proxied through proxy.example.com:8080
+});
+
+fetch('https://www.example.com', (res) => {
+  // This request will be proxied through proxy.example.com:8443
+});
+```
+
 To create a custom agent with built-in proxy support:
 
 ```cjs
@@ -4501,6 +4610,7 @@ const agent2 = new http.Agent({ proxyEnv: process.env });
 [`http.get()`]: #httpgetoptions-callback
 [`http.globalAgent`]: #httpglobalagent
 [`http.request()`]: #httprequestoptions-callback
+[`http.setGlobalProxyFromEnv()`]: #httpsetglobalproxyfromenvproxyenv
 [`message.headers`]: #messageheaders
 [`message.rawHeaders`]: #messagerawheaders
 [`message.socket`]: #messagesocket

lib/_http_agent.js

@@ -40,7 +40,7 @@ const {
   kProxyConfig,
   checkShouldUseProxy,
   kWaitForProxyTunnel,
-  filterEnvForProxies,
+  getGlobalAgent,
 } = require('internal/http');
 const { AsyncResource } = require('async_hooks');
 const { async_id_symbol } = require('internal/async_hooks').symbols;
@@ -627,9 +627,5 @@ function asyncResetHandle(socket) {
 
 module.exports = {
   Agent,
-  globalAgent: new Agent({
-    keepAlive: true, scheduling: 'lifo', timeout: 5000,
-    // This normalized from both --use-env-proxy and NODE_USE_ENV_PROXY settings.
-    proxyEnv: getOptionValue('--use-env-proxy') ? filterEnvForProxies(process.env) : undefined,
-  }),
+  globalAgent: getGlobalAgent(getOptionValue('--use-env-proxy') ? process.env : undefined, Agent),
 };

lib/http.js

@@ -25,11 +25,12 @@ const {
   ObjectDefineProperty,
 } = primordials;
 
-const { validateInteger } = require('internal/validators');
+const { validateInteger, validateObject } = require('internal/validators');
 const httpAgent = require('_http_agent');
 const { ClientRequest } = require('_http_client');
 const { methods, parsers } = require('_http_common');
 const { IncomingMessage } = require('_http_incoming');
+const { ERR_PROXY_INVALID_CONFIG } = require('internal/errors').codes;
 const {
   validateHeaderName,
   validateHeaderValue,
@@ -41,6 +42,11 @@ const {
   Server,
   ServerResponse,
 } = require('_http_server');
+const {
+  parseProxyUrl,
+  getGlobalAgent,
+} = require('internal/http');
+const { URL } = require('internal/url');
 let maxHeaderSize;
 let undici;
 
@@ -123,6 +129,58 @@ function lazyUndici() {
   return undici ??= require('internal/deps/undici/undici');
 }
 
+function setGlobalProxyFromEnv(env = process.env) {
+  validateObject(env, 'proxyEnv');
+  const httpProxy = parseProxyUrl(env, 'http:');
+  const httpsProxy = parseProxyUrl(env, 'https:');
+  const noProxy = env.no_proxy || env.NO_PROXY;
+
+  if (!httpProxy && !httpsProxy) {
+    return () => {};
+  }
+
+  if (httpProxy && !URL.canParse(httpProxy)) {
+    throw new ERR_PROXY_INVALID_CONFIG(httpProxy);
+  }
+  if (httpsProxy && !URL.canParse(httpsProxy)) {
+    throw new ERR_PROXY_INVALID_CONFIG(httpsProxy);
+  }
+
+  let originalDispatcher, originalHttpsAgent, originalHttpAgent;
+  if (httpProxy || httpsProxy) {
+    // Set it for fetch.
+    const { setGlobalDispatcher, getGlobalDispatcher, EnvHttpProxyAgent } = lazyUndici();
+    const envHttpProxyAgent = new EnvHttpProxyAgent({
+      __proto__: null, httpProxy, httpsProxy, noProxy,
+    });
+    originalDispatcher = getGlobalDispatcher();
+    setGlobalDispatcher(envHttpProxyAgent);
+  }
+
+  if (httpProxy) {
+    originalHttpAgent = module.exports.globalAgent;
+    module.exports.globalAgent = getGlobalAgent(env, httpAgent.Agent);
+  }
+  if (httpsProxy && !!process.versions.openssl) {
+    const https = require('https');
+    originalHttpsAgent = https.globalAgent;
+    https.globalAgent = getGlobalAgent(env, https.Agent);
+  }
+
+  return function restore() {
+    if (originalDispatcher) {
+      const { setGlobalDispatcher } = lazyUndici();
+      setGlobalDispatcher(originalDispatcher);
+    }
+    if (originalHttpAgent) {
+      module.exports.globalAgent = originalHttpAgent;
+    }
+    if (originalHttpsAgent) {
+      require('https').globalAgent = originalHttpsAgent;
+    }
+  };
+}
+
 module.exports = {
   _connectionListener,
   METHODS: methods.toSorted(),
@@ -142,6 +200,7 @@ module.exports = {
     validateInteger(max, 'max', 1);
     parsers.max = max;
   },
+  setGlobalProxyFromEnv,
 };
 
 ObjectDefineProperty(module.exports, 'maxHeaderSize', {

lib/https.js

@@ -50,8 +50,8 @@ const tls = require('tls');
 const {
   kProxyConfig,
   checkShouldUseProxy,
-  filterEnvForProxies,
   kWaitForProxyTunnel,
+  getGlobalAgent,
 } = require('internal/http');
 const { Agent: HttpAgent } = require('_http_agent');
 const {
@@ -602,11 +602,7 @@ Agent.prototype._evictSession = function _evictSession(key) {
   delete this._sessionCache.map[key];
 };
 
-const globalAgent = new Agent({
-  keepAlive: true, scheduling: 'lifo', timeout: 5000,
-  // This normalized from both --use-env-proxy and NODE_USE_ENV_PROXY settings.
-  proxyEnv: getOptionValue('--use-env-proxy') ? filterEnvForProxies(process.env) : undefined,
-});
+const globalAgent = getGlobalAgent(getOptionValue('--use-env-proxy') ? process.env : undefined, Agent);
 
 /**
  * Makes a request to a secure web server.

lib/internal/http.js

@@ -186,11 +186,7 @@ class ProxyConfig {
   }
 }
 
-function parseProxyConfigFromEnv(env, protocol, keepAlive) {
-  // We only support proxying for HTTP and HTTPS requests.
-  if (protocol !== 'http:' && protocol !== 'https:') {
-    return null;
-  }
+function parseProxyUrl(env, protocol) {
   // Get the proxy url - following the most popular convention, lower case takes precedence.
   // See https://about.gitlab.com/blog/we-need-to-talk-no-proxy/#http_proxy-and-https_proxy
   const proxyUrl = (protocol === 'https:') ?
@@ -204,6 +200,20 @@ function parseProxyConfigFromEnv(env, protocol, keepAlive) {
     throw new ERR_PROXY_INVALID_CONFIG(`Invalid proxy URL: ${proxyUrl}`);
   }
 
+  return proxyUrl;
+}
+
+function parseProxyConfigFromEnv(env, protocol, keepAlive) {
+  // We only support proxying for HTTP and HTTPS requests.
+  if (protocol !== 'http:' && protocol !== 'https:') {
+    return null;
+  }
+
+  const proxyUrl = parseProxyUrl(env, protocol);
+  if (proxyUrl === null) {
+    return null;
+  }
+
   // Only http:// and https:// proxies are supported.
   // Ignore instead of throw, in case other protocols are supposed to be
   // handled by the user land.
@@ -244,6 +254,13 @@ function filterEnvForProxies(env) {
   };
 }
 
+function getGlobalAgent(proxyEnv, Agent) {
+  return new Agent({
+    keepAlive: true, scheduling: 'lifo', timeout: 5000,
+    proxyEnv,
+  });
+}
+
 module.exports = {
   kOutHeaders: Symbol('kOutHeaders'),
   kNeedDrain: Symbol('kNeedDrain'),
@@ -257,4 +274,6 @@ module.exports = {
   getNextTraceEventId,
   isTraceHTTPEnabled,
   filterEnvForProxies,
+  getGlobalAgent,
+  parseProxyUrl,
 };

test/client-proxy/test-http-proxy-fetch.mjs

@@ -20,9 +20,7 @@ await once(proxy, 'listening');
 const serverHost = `localhost:${server.address().port}`;
 
 // FIXME(undici:4083): undici currently always tunnels the request over
-// CONNECT if proxyTunnel is not explicitly set to false, but what we
-// need is for it to be automatically false for HTTP requests to be
-// consistent with curl.
+// CONNECT if proxyTunnel is not explicitly set to false.
 const expectedLogs = [{
   method: 'CONNECT',
   url: serverHost,

test/client-proxy/test-http-set-global-proxy-from-env-fetch-default.mjs

@@ -0,0 +1,35 @@
+// Tests that http.setGlobalProxyFromEnv() without arguments uses process.env.
+
+import '../common/index.mjs';
+import assert from 'node:assert';
+import { startTestServers, checkProxiedFetch } from '../common/proxy-server.js';
+
+const { proxyLogs, proxyUrl, shutdown, httpEndpoint: { serverHost, requestUrl } } = await startTestServers({
+  httpEndpoint: true,
+});
+
+// Test that calling setGlobalProxyFromEnv() without arguments uses process.env
+await checkProxiedFetch({
+  FETCH_URL: requestUrl,
+  // Set the proxy in the environment instead of passing it to SET_GLOBAL_PROXY
+  http_proxy: proxyUrl,
+  SET_GLOBAL_PROXY_DEFAULT: '1',  // Signal to call without arguments
+}, {
+  stdout: 'Hello world',
+});
+
+shutdown();
+
+// FIXME(undici:4083): undici currently always tunnels the request over
+// CONNECT if proxyTunnel is not explicitly set to false.
+const expectedLogs = [{
+  method: 'CONNECT',
+  url: serverHost,
+  headers: {
+    'connection': 'close',
+    'host': serverHost,
+    'proxy-connection': 'keep-alive',
+  },
+}];
+
+assert.deepStrictEqual(proxyLogs, expectedLogs);

test/client-proxy/test-http-set-global-proxy-from-env-fetch-empty.mjs

@@ -0,0 +1,21 @@
+// Tests that http.setGlobalProxyFromEnv() is a no-op when no proxy is configured.
+
+import '../common/index.mjs';
+import assert from 'node:assert';
+import { startTestServers, checkProxiedFetch } from '../common/proxy-server.js';
+
+const { proxyLogs, shutdown, httpEndpoint: { requestUrl } } = await startTestServers({
+  httpEndpoint: true,
+});
+
+await checkProxiedFetch({
+  FETCH_URL: requestUrl,
+  SET_GLOBAL_PROXY: JSON.stringify({}),
+}, {
+  stdout: 'Hello world',
+});
+
+shutdown();
+
+// Verify request did NOT go through proxy.
+assert.deepStrictEqual(proxyLogs, []);