http: support http proxy for fetch under NODE_USE_ENV_PROXY

When enabled, Node.js parses the `HTTP_PROXY`, `HTTPS_PROXY` and
`NO_PROXY` environment variables during startup, and tunnels requests
over the specified proxy.

This currently only affects requests sent over `fetch()`. Support for
other built-in `http` and `https` methods is under way.

PR-URL: https://github.com/nodejs/node/pull/57165
Refs: https://github.com/nodejs/undici/issues/1650
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>

doc/api/cli.md

@@ -3629,6 +3629,21 @@ If `value` equals `'0'`, certificate validation is disabled for TLS connections.
 This makes TLS, and HTTPS by extension, insecure. The use of this environment
 variable is strongly discouraged.
 
+### `NODE_USE_ENV_PROXY=1`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1.1 - Active Development
+
+When enabled, Node.js parses the `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY`
+environment variables during startup, and tunnels requests over the
+specified proxy.
+
+This currently only affects requests sent over `fetch()`. Support for other
+built-in `http` and `https` methods is under way.
+
 ### `NODE_USE_SYSTEM_CA=1`
 
 <!-- YAML

doc/node.1

@@ -842,6 +842,19 @@ When set to
 .Ar 0 ,
 TLS certificate validation is disabled.
 .
+.It Ev NODE_USE_ENV_PROXY
+When enabled, Node.js parses the
+.Ar HTTP_PROXY
+,
+.Ar HTTPS_PROXY
+and
+.Ar NO_PROXY
+environment variables during startup, and tunnels requests over the specified proxy.
+.Pp
+This currently only affects requests sent over
+.Ar fetch() .
+Support for other built-in http and https methods is under way.
+.
 .It Ev NODE_USE_SYSTEM_CA
 Similar to
 .Fl -use-system-ca .

lib/internal/process/pre_execution.js

@@ -127,6 +127,7 @@ function prepareExecution(options) {
   initializeConfigFileSupport();
 
   require('internal/dns/utils').initializeDns();
+  setupHttpProxy();
 
   setupSymbolDisposePolyfill();
 
@@ -189,6 +190,21 @@ function setupSymbolDisposePolyfill() {
   }
 }
 
+function setupHttpProxy() {
+  if (process.env.NODE_USE_ENV_PROXY &&
+      (process.env.HTTP_PROXY || process.env.HTTPS_PROXY ||
+       process.env.http_proxy || process.env.https_proxy)) {
+    const { setGlobalDispatcher, EnvHttpProxyAgent } = require('internal/deps/undici/undici');
+    const envHttpProxyAgent = new EnvHttpProxyAgent();
+    setGlobalDispatcher(envHttpProxyAgent);
+    // TODO(joyeecheung): This currently only affects fetch. Implement handling in the
+    // http/https Agent constructor too.
+    // TODO(joyeecheung): This is currently guarded with NODE_USE_ENV_PROXY. Investigate whether
+    // it's possible to enable it by default without stepping on other existing libraries that
+    // sets the global dispatcher or monkey patches the global agent.
+  }
+}
+
 function setupUserModules(forceDefaultLoader = false) {
   initializeCJSLoader();
   initializeESMLoader(forceDefaultLoader);

test/common/proxy-server.js

@@ -0,0 +1,100 @@
+'use strict';
+
+const net = require('net');
+const http = require('http');
+const assert = require('assert');
+
+function logRequest(logs, req) {
+  logs.push({
+    method: req.method,
+    url: req.url,
+    headers: { ...req.headers },
+  });
+}
+
+// This creates a minimal proxy server that logs the requests it gets
+// to an array before performing proxying.
+exports.createProxyServer = function() {
+  const logs = [];
+
+  const proxy = http.createServer();
+  proxy.on('request', (req, res) => {
+    logRequest(logs, req);
+    const [hostname, port] = req.headers.host.split(':');
+    const targetPort = port || 80;
+
+    const options = {
+      hostname: hostname,
+      port: targetPort,
+      path: req.url,
+      method: req.method,
+      headers: req.headers,
+    };
+
+    const proxyReq = http.request(options, (proxyRes) => {
+      res.writeHead(proxyRes.statusCode, proxyRes.headers);
+      proxyRes.pipe(res, { end: true });
+    });
+
+    proxyReq.on('error', (err) => {
+      logs.push({ error: err, source: 'proxy request' });
+      res.writeHead(500);
+      res.end('Proxy error: ' + err.message);
+    });
+
+    req.pipe(proxyReq, { end: true });
+  });
+
+  proxy.on('connect', (req, res, head) => {
+    logRequest(logs, req);
+
+    const [hostname, port] = req.url.split(':');
+    const proxyReq = net.connect(port, hostname, () => {
+      res.write(
+        'HTTP/1.1 200 Connection Established\r\n' +
+        'Proxy-agent: Node.js-Proxy\r\n' +
+        '\r\n',
+      );
+      proxyReq.write(head);
+      res.pipe(proxyReq);
+      proxyReq.pipe(res);
+    });
+
+    proxyReq.on('error', (err) => {
+      logs.push({ error: err, source: 'proxy request' });
+      res.write('HTTP/1.1 500 Connection Error\r\n\r\n');
+      res.end('Proxy error: ' + err.message);
+    });
+  });
+
+  proxy.on('error', (err) => {
+    logs.push({ error: err, source: 'proxy server' });
+  });
+
+  return { proxy, logs };
+};
+
+exports.checkProxiedRequest = async function(envExtension, expectation) {
+  const { spawnPromisified } = require('./');
+  const fixtures = require('./fixtures');
+  const { code, signal, stdout, stderr } = await spawnPromisified(
+    process.execPath,
+    ['--no-warnings', fixtures.path('fetch-and-log.mjs')], {
+      env: {
+        ...process.env,
+        ...envExtension,
+      },
+    });
+
+  assert.deepStrictEqual({
+    stderr: stderr.trim(),
+    stdout: stdout.trim(),
+    code,
+    signal,
+  }, {
+    stderr: '',
+    code: 0,
+    signal: null,
+    ...expectation,
+  });
+};

test/fixtures/fetch-and-log.mjs

@@ -0,0 +1,3 @@
+const response = await fetch(process.env.FETCH_URL);
+const body = await response.text();
+console.log(body);

test/parallel/test-http-proxy-fetch.js

@@ -0,0 +1,61 @@
+'use strict';
+
+const common = require('../common');
+const assert = require('assert');
+const { once } = require('events');
+const http = require('http');
+const { createProxyServer, checkProxiedRequest } = require('../common/proxy-server');
+
+(async () => {
+  // Start a server to process the final request.
+  const server = http.createServer(common.mustCall((req, res) => {
+    res.end('Hello world');
+  }, 2));
+  server.on('error', common.mustNotCall((err) => { console.error('Server error', err); }));
+  server.listen(0);
+  await once(server, 'listening');
+
+  // Start a minimal proxy server.
+  const { proxy, logs } = createProxyServer();
+  proxy.listen(0);
+  await once(proxy, 'listening');
+
+  const serverHost = `localhost:${server.address().port}`;
+
+  // FIXME(undici:4083): undici currently always tunnels the request over
+  // CONNECT, no matter it's HTTP traffic or not, which is different from e.g.
+  // how curl behaves.
+  const expectedLogs = [{
+    method: 'CONNECT',
+    url: serverHost,
+    headers: {
+      // FIXME(undici:4086): this should be keep-alive.
+      connection: 'close',
+      host: serverHost
+    }
+  }];
+
+  // Check upper-cased HTTPS_PROXY environment variable.
+  await checkProxiedRequest({
+    NODE_USE_ENV_PROXY: 1,
+    FETCH_URL: `http://${serverHost}/test`,
+    HTTP_PROXY: `http://localhost:${proxy.address().port}`,
+  }, {
+    stdout: 'Hello world',
+  });
+  assert.deepStrictEqual(logs, expectedLogs);
+
+  // Check lower-cased https_proxy environment variable.
+  logs.splice(0, logs.length);
+  await checkProxiedRequest({
+    NODE_USE_ENV_PROXY: 1,
+    FETCH_URL: `http://${serverHost}/test`,
+    http_proxy: `http://localhost:${proxy.address().port}`,
+  }, {
+    stdout: 'Hello world',
+  });
+  assert.deepStrictEqual(logs, expectedLogs);
+
+  proxy.close();
+  server.close();
+})().then(common.mustCall());

test/parallel/test-https-proxy-fetch.js

@@ -0,0 +1,67 @@
+'use strict';
+
+const common = require('../common');
+if (!common.hasCrypto)
+  common.skip('missing crypto');
+
+const fixtures = require('../common/fixtures');
+const assert = require('assert');
+const https = require('https');
+const { once } = require('events');
+const { createProxyServer, checkProxiedRequest } = require('../common/proxy-server');
+
+(async () => {
+  // Start a server to process the final request.
+  const server = https.createServer({
+    cert: fixtures.readKey('agent8-cert.pem'),
+    key: fixtures.readKey('agent8-key.pem'),
+  }, common.mustCall((req, res) => {
+    res.end('Hello world');
+  }, 2));
+  server.on('error', common.mustNotCall((err) => { console.error('Server error', err); }));
+  server.listen(0);
+  await once(server, 'listening');
+
+  // Start a minimal proxy server.
+  const { proxy, logs } = createProxyServer();
+  proxy.listen(0);
+  await once(proxy, 'listening');
+
+  const serverHost = `localhost:${server.address().port}`;
+
+  const expectedLogs = [{
+    method: 'CONNECT',
+    url: serverHost,
+    headers: {
+      // FIXME(undici:4086): this should be keep-alive.
+      connection: 'close',
+      host: serverHost
+    }
+  }];
+
+  // Check upper-cased HTTPS_PROXY environment variable.
+  await checkProxiedRequest({
+    NODE_USE_ENV_PROXY: 1,
+    FETCH_URL: `https://${serverHost}/test`,
+    HTTPS_PROXY: `http://localhost:${proxy.address().port}`,
+    NODE_EXTRA_CA_CERTS: fixtures.path('keys', 'fake-startcom-root-cert.pem'),
+  }, {
+    stdout: 'Hello world',
+  });
+  assert.deepStrictEqual(logs, expectedLogs);
+
+  // Check lower-cased https_proxy environment variable.
+  logs.splice(0, logs.length);
+  await checkProxiedRequest({
+    NODE_USE_ENV_PROXY: 1,
+    FETCH_URL: `https://${serverHost}/test`,
+    https_proxy: `http://localhost:${proxy.address().port}`,
+    NODE_EXTRA_CA_CERTS: fixtures.path('keys', 'fake-startcom-root-cert.pem'),
+  }, {
+    stdout: 'Hello world',
+  });
+  assert.deepStrictEqual(logs, expectedLogs);
+
+  proxy.close();
+  server.close();
+})().then(common.mustCall());