Fetch: support ipv6first and ipv4first priorities for individual requests

lib/fetch.js

@@ -1,8 +1,85 @@
 import { URL } from 'url';
+import dns from 'node:dns';
 import { h1NoCache, noCache, createUrl, AbortController, AbortError, FetchError } from '@adobe/fetch';
 import { serialize as serializeCookie, parse as parseCookie } from 'cookie';
 import log from '../logging.js';
 
+/*
+import diagnostics_channel from 'node:diagnostics_channel';
+
+// Log the actual IP each outgoing connection is made to.
+diagnostics_channel.subscribe('net.client.socket', ({ socket }) => {
+    socket.once('connect', () => {
+        // Skip non-http connections (redis, etc.).
+        if (socket.remotePort !== 443 && socket.remotePort !== 80) return;
+        const host = socket.servername || socket._host || '';
+        log(`   -- connected`, host, `${socket.remoteAddress} (${socket.remoteFamily}):${socket.remotePort}`);
+    });
+});
+*/
+
+const makeLookup = (order) => (hostname, options, callback) =>
+    dns.lookup(hostname, { ...options, order }, callback);
+
+// Debug lookup.
+/*
+const makeLookup = (order) => (hostname, options, callback) =>
+    dns.lookup(hostname, { ...options, order }, (error, address, family) => {
+        if (error) {
+            log(`   -- lookup ${order} error`, hostname, error.code, error.message);
+        } else if (Array.isArray(address)) {
+            // With autoSelectFamily lookup is called with `all: true`.
+            log(`   -- lookup ${order}`, hostname, address.map(a => `${a.address} (v${a.family})`).join(', '));
+        } else {
+            log(`   -- lookup ${order}`, hostname, `${address} (v${family})`);
+        }
+        callback(error, address, family);
+    });
+*/
+
+// `ipv6first` order requires Node >= 20.13. autoSelectFamily falls back to the other family on connect failure.
+const ORDER_DEFAULT = 'default';
+const ORDER_IPV6_FIRST = 'ipv6first';
+const ORDER_IPV4_FIRST = 'ipv4first';
+
+const LOOKUPS = {
+    [ORDER_DEFAULT]: undefined,     // Node's global resolver order.
+    [ORDER_IPV6_FIRST]: { lookup: makeLookup(ORDER_IPV6_FIRST) },
+    [ORDER_IPV4_FIRST]: { lookup: makeLookup(ORDER_IPV4_FIRST) }
+};
+
+const PROFILE_DEFAULT = 'default';
+const PROFILE_KEEP_ALIVE = 'keepAlive';
+const PROFILE_AUTHORIZED = 'authorized';
+
+const PROFILES = {
+    [PROFILE_DEFAULT]:    {
+        rejectUnauthorized: false,  // By default skip auth check for all.
+        h2: { enablePush: false }
+    },
+    [PROFILE_KEEP_ALIVE]: {
+        rejectUnauthorized: false,
+        h1: { keepAlive: true },
+        h2: { enablePush: false }
+    },
+    [PROFILE_AUTHORIZED]: {         // `rejectUnauthorized: true` - by `fetch` default.
+        h2: { enablePush: false }
+    }
+};
+
+const H1 = 'h1';
+const H2 = 'h2';
+
+const FETCH_BY_PROTOCOL = {
+    [H1]: h1NoCache,
+    [H2]: noCache
+};
+
+// Lazily created fetch contexts, memoized per profile/protocol/dns order combination.
+const fetchFuncs = {};
+/*
+Generates fetches like:
+
 const fetchKeepAlive = noCache({
     h1: {
         keepAlive: true
@@ -24,19 +101,36 @@ const fetchAuthorized = noCache({
     h2: {
         enablePush: false
     }
-}).fetch;    // `rejectUnauthorized: true` - by `fetch` default.
-const fetchH1Authorized = h1NoCache().fetch;      // `rejectUnauthorized: true` - by `fetch` default.
+}).fetch;                       // `rejectUnauthorized: true` - by `fetch` default.
+
+const fetchH1Authorized = h1NoCache().fetch;      
+                                // `rejectUnauthorized: true` - by `fetch` default.
 
-const { fetch } = noCache({
+const fetch = noCache({
     h2: {
         enablePush: false
     },
     rejectUnauthorized: false   // By default skip auth check for all.
-});
+}).fetch;
 
 const fetchH1 = h1NoCache({
     rejectUnauthorized: false   // By default skip auth check for all.
 }).fetch;
+*/
+
+function getFetchFunc(profile, protocol, order) {
+    const key = `${profile}:${protocol}:${order}`;
+    if (!fetchFuncs[key]) {
+        const profileOpts = PROFILES[profile];
+        const lookup = LOOKUPS[order];
+        const h1 = { ...profileOpts.h1, ...lookup };
+        const h2 = { ...profileOpts.h2, ...lookup };
+        const fetchOptions = { ...profileOpts, h1, h2 };
+        const fetchFunc = FETCH_BY_PROTOCOL[protocol];
+        fetchFuncs[key] = fetchFunc(fetchOptions).fetch;
+    }
+    return fetchFuncs[key];
+}
 
 export function skipLoggingFetchError(error) {
     return error?.name === 'AbortError' 
@@ -49,7 +143,17 @@ export function skipLoggingFetchError(error) {
         || error?.code === 'ETIMEDOUT';
 }
 
-function doFetch(fetch_func, h1_fetch_func, options) {
+function selectFetchFunc(options, profile = PROFILE_DEFAULT) {
+    let order = ORDER_DEFAULT;
+    if (options.ipv6first) {
+        order = ORDER_IPV6_FIRST;
+    } else if (options.ipv4first) {
+        order = ORDER_IPV4_FIRST;
+    }
+    return getFetchFunc(profile, options.disable_http2 ? H1 : H2, order);
+}
+
+function doFetch(profile, options) {
 
     const fetch_options = Object.assign({}, options);
     const is_head_request = fetch_options.method === 'HEAD';
@@ -73,7 +177,7 @@ function doFetch(fetch_func, h1_fetch_func, options) {
         abortController.abort();
     }, options.timeout || CONFIG.RESPONSE_TIMEOUT);
 
-    const a_fetch_func = options.disable_http2 ? h1_fetch_func: fetch_func;
+    const a_fetch_func = selectFetchFunc(options, profile);
     return new Promise((resolve, reject) => {
         a_fetch_func(uri, fetch_options)
             .then(response => {
@@ -99,7 +203,7 @@ function doFetch(fetch_func, h1_fetch_func, options) {
                     && CONFIG.DISABLE_HTTP2_CHECKS?.some(check => typeof check === 'function'
                                                                 && check(response.status, headers))) {
                     log('   -- doFetch check disabled h2', uri);
-                    resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true})));
+                    resolve(doFetch(profile, Object.assign({}, options, {disable_http2: true})));
                 } else {
                     stream.status = response.status;
                     if (response.url && response.url !== uri) { // Set as final destination url if Fetch follows 301/302 re-directs
@@ -120,7 +224,7 @@ function doFetch(fetch_func, h1_fetch_func, options) {
                 if (!options.disable_http2 && error.code && /^ERR_HTTP2/.test(error.code)) {
 
                     log('   -- doFetch http2 error', error.code, uri);
-                    resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true})));
+                    resolve(doFetch(profile, Object.assign({}, options, {disable_http2: true})));
 
                 } else if (!options.disable_http2 && error.code && error instanceof FetchError && error.code === 'ABORT_ERR') {
 
@@ -130,12 +234,12 @@ function doFetch(fetch_func, h1_fetch_func, options) {
                      * https://app.everviz.com/show/O0Cy7Dyt
                      */
                     log('   -- doFetch h2 aborted error', uri);
-                    resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true})));
+                    resolve(doFetch(profile, Object.assign({}, options, {disable_http2: true})));
 
                 } else if (!options.stopRecursion && CONFIG.ERRORS_TO_RETRY?.some(code => error.code?.indexOf(code) > -1)) {
 
                     log('   -- doFetch ECONNRESET retry', error.code, uri);
-                    resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {stopRecursion: true, disable_http2: true})));
+                    resolve(doFetch(profile, Object.assign({}, options, {stopRecursion: true, disable_http2: true})));
 
                 } else {
                     if (error instanceof AbortError) {
@@ -149,15 +253,15 @@ function doFetch(fetch_func, h1_fetch_func, options) {
 }
 
 export function fetchStreamKeepAlive(options) {
-    return doFetch(fetchKeepAlive, fetchH1KeepAlive, options);
+    return doFetch(PROFILE_KEEP_ALIVE, options);
 }
 
 export function fetchStream(options) {
-    return doFetch(fetch, fetchH1, options);
+    return doFetch(PROFILE_DEFAULT, options);
 };
 
 export function fetchStreamAuthorized(options) {
-    return doFetch(fetchAuthorized, fetchH1Authorized, options);
+    return doFetch(PROFILE_AUTHORIZED, options);
 };
 
 export function fetchData(options) {
@@ -184,7 +288,7 @@ export function fetchData(options) {
         abortController.abort();
     }, options.timeout || CONFIG.RESPONSE_TIMEOUT);
 
-    const a_fetch_func = options.disable_http2 ? fetchH1: fetch;
+    const a_fetch_func = selectFetchFunc(options);
     return new Promise((resolve, reject) => {
         a_fetch_func(uri, fetch_options)
             .then(response => {

lib/utils.js

@@ -126,6 +126,15 @@ export function prepareRequestOptions(request_options, options) {
                     request_options.headers[key] = value;
                 }
             }
+
+            // Lowest priority for `ipv6first/ipv4first` from `providerOptions`.
+            if (options?.getProviderOptions) {
+                if (options.getProviderOptions('app.ipv6first')) {
+                    request_options.ipv6first = true;
+                } else if (options.getProviderOptions('app.ipv4first')) {
+                    request_options.ipv4first = true;
+                }
+            }
 
             if (CONFIG.PROXY || options?.proxy) {
 
@@ -153,6 +162,12 @@ export function prepareRequestOptions(request_options, options) {
                     if (proxy.disable_http2) {
                         request_options.disable_http2 = true;
                     }
+                    // Middle priority for `ipv6first/ipv4first` from `proxy`.
+                    if (proxy.ipv6first) {
+                        request_options.ipv6first = true;
+                    } else if (proxy.ipv4first) {
+                        request_options.ipv4first = true;
+                    }
                     if (proxy.disable_language) {
                         disable_language = true;
                     }
@@ -168,6 +183,13 @@ export function prepareRequestOptions(request_options, options) {
                 }
             }
 
+            // Highest priority for `ipv6first/ipv4first` from `options`.
+            if (options?.ipv6first) {
+                request_options.ipv6first = true;
+            } else if (options?.ipv4first) {
+                request_options.ipv4first = true;
+            }
+
             if (options?.user_agent) {
                 // Let `options.user_agent` be prioritized over `proxy.user_agent`.
                 request_options.headers['User-Agent'] = options?.user_agent;

patches/@adobe__fetch.patch

@@ -0,0 +1,21 @@
+diff --git a/src/core/request.js b/src/core/request.js
+index 6f7120e3e383ed44b4fe63861109f365579d7425..19de7a26bc712e172b78efb4de4ef97ad8f269e3 100644
+--- a/src/core/request.js
++++ b/src/core/request.js
+@@ -160,7 +160,16 @@ const determineProtocol = async (ctx, url, signal) => {
+   const rejectUnauthorized = !((_rejectUnauthorized === false
+     || h1Opts.rejectUnauthorized === false
+     || h2Opts.rejectUnauthorized === false));
++  // PATCH: forward DNS/socket options from context to the ALPN negotiation socket
++  const socketOptions = {};
++  ['lookup', 'family', 'autoSelectFamily', 'autoSelectFamilyAttemptTimeout'].forEach((name) => {
++    const value = h2Opts[name] !== undefined ? h2Opts[name] : h1Opts[name];
++    if (value !== undefined) {
++      socketOptions[name] = value;
++    }
++  });
+   const connectOptions = {
++    ...socketOptions,
+     servername: url.hostname, // enable SNI (Server Name Indication) extension
+     ALPNProtocols: ctx.alpnProtocols,
+     signal, // optional abort signal

pnpm-workspace.yaml

@@ -25,4 +25,5 @@ overrides:
   probe-image-size@<=7.3.0: ^7.4.0
   proxy-addr@>=1.1.0 <2.0.8: ^2.0.8
 patchedDependencies:
+  '@adobe/fetch': patches/@adobe__fetch.patch
   readabilitySAX@1.6.1: patches/readabilitySAX@1.6.1.patch