diff --git a/lib/fetch.js b/lib/fetch.js index b7f29f8dd..c19d85349 100644 --- a/lib/fetch.js +++ b/lib/fetch.js @@ -1,8 +1,85 @@ import { URL } from 'url'; +import dns from 'node:dns'; import { h1NoCache, noCache, createUrl, AbortController, AbortError, FetchError } from '@adobe/fetch'; import { serialize as serializeCookie, parse as parseCookie } from 'cookie'; import log from '../logging.js'; +/* +import diagnostics_channel from 'node:diagnostics_channel'; + +// Log the actual IP each outgoing connection is made to. +diagnostics_channel.subscribe('net.client.socket', ({ socket }) => { + socket.once('connect', () => { + // Skip non-http connections (redis, etc.). + if (socket.remotePort !== 443 && socket.remotePort !== 80) return; + const host = socket.servername || socket._host || ''; + log(` -- connected`, host, `${socket.remoteAddress} (${socket.remoteFamily}):${socket.remotePort}`); + }); +}); +*/ + +const makeLookup = (order) => (hostname, options, callback) => + dns.lookup(hostname, { ...options, order }, callback); + +// Debug lookup. +/* +const makeLookup = (order) => (hostname, options, callback) => + dns.lookup(hostname, { ...options, order }, (error, address, family) => { + if (error) { + log(` -- lookup ${order} error`, hostname, error.code, error.message); + } else if (Array.isArray(address)) { + // With autoSelectFamily lookup is called with `all: true`. + log(` -- lookup ${order}`, hostname, address.map(a => `${a.address} (v${a.family})`).join(', ')); + } else { + log(` -- lookup ${order}`, hostname, `${address} (v${family})`); + } + callback(error, address, family); + }); +*/ + +// `ipv6first` order requires Node >= 20.13. autoSelectFamily falls back to the other family on connect failure. +const ORDER_DEFAULT = 'default'; +const ORDER_IPV6_FIRST = 'ipv6first'; +const ORDER_IPV4_FIRST = 'ipv4first'; + +const LOOKUPS = { + [ORDER_DEFAULT]: undefined, // Node's global resolver order. + [ORDER_IPV6_FIRST]: { lookup: makeLookup(ORDER_IPV6_FIRST) }, + [ORDER_IPV4_FIRST]: { lookup: makeLookup(ORDER_IPV4_FIRST) } +}; + +const PROFILE_DEFAULT = 'default'; +const PROFILE_KEEP_ALIVE = 'keepAlive'; +const PROFILE_AUTHORIZED = 'authorized'; + +const PROFILES = { + [PROFILE_DEFAULT]: { + rejectUnauthorized: false, // By default skip auth check for all. + h2: { enablePush: false } + }, + [PROFILE_KEEP_ALIVE]: { + rejectUnauthorized: false, + h1: { keepAlive: true }, + h2: { enablePush: false } + }, + [PROFILE_AUTHORIZED]: { // `rejectUnauthorized: true` - by `fetch` default. + h2: { enablePush: false } + } +}; + +const H1 = 'h1'; +const H2 = 'h2'; + +const FETCH_BY_PROTOCOL = { + [H1]: h1NoCache, + [H2]: noCache +}; + +// Lazily created fetch contexts, memoized per profile/protocol/dns order combination. +const fetchFuncs = {}; +/* +Generates fetches like: + const fetchKeepAlive = noCache({ h1: { keepAlive: true @@ -24,19 +101,36 @@ const fetchAuthorized = noCache({ h2: { enablePush: false } -}).fetch; // `rejectUnauthorized: true` - by `fetch` default. -const fetchH1Authorized = h1NoCache().fetch; // `rejectUnauthorized: true` - by `fetch` default. +}).fetch; // `rejectUnauthorized: true` - by `fetch` default. + +const fetchH1Authorized = h1NoCache().fetch; + // `rejectUnauthorized: true` - by `fetch` default. -const { fetch } = noCache({ +const fetch = noCache({ h2: { enablePush: false }, rejectUnauthorized: false // By default skip auth check for all. -}); +}).fetch; const fetchH1 = h1NoCache({ rejectUnauthorized: false // By default skip auth check for all. }).fetch; +*/ + +function getFetchFunc(profile, protocol, order) { + const key = `${profile}:${protocol}:${order}`; + if (!fetchFuncs[key]) { + const profileOpts = PROFILES[profile]; + const lookup = LOOKUPS[order]; + const h1 = { ...profileOpts.h1, ...lookup }; + const h2 = { ...profileOpts.h2, ...lookup }; + const fetchOptions = { ...profileOpts, h1, h2 }; + const fetchFunc = FETCH_BY_PROTOCOL[protocol]; + fetchFuncs[key] = fetchFunc(fetchOptions).fetch; + } + return fetchFuncs[key]; +} export function skipLoggingFetchError(error) { return error?.name === 'AbortError' @@ -49,7 +143,17 @@ export function skipLoggingFetchError(error) { || error?.code === 'ETIMEDOUT'; } -function doFetch(fetch_func, h1_fetch_func, options) { +function selectFetchFunc(options, profile = PROFILE_DEFAULT) { + let order = ORDER_DEFAULT; + if (options.ipv6first) { + order = ORDER_IPV6_FIRST; + } else if (options.ipv4first) { + order = ORDER_IPV4_FIRST; + } + return getFetchFunc(profile, options.disable_http2 ? H1 : H2, order); +} + +function doFetch(profile, options) { const fetch_options = Object.assign({}, options); const is_head_request = fetch_options.method === 'HEAD'; @@ -73,7 +177,7 @@ function doFetch(fetch_func, h1_fetch_func, options) { abortController.abort(); }, options.timeout || CONFIG.RESPONSE_TIMEOUT); - const a_fetch_func = options.disable_http2 ? h1_fetch_func: fetch_func; + const a_fetch_func = selectFetchFunc(options, profile); return new Promise((resolve, reject) => { a_fetch_func(uri, fetch_options) .then(response => { @@ -99,7 +203,7 @@ function doFetch(fetch_func, h1_fetch_func, options) { && CONFIG.DISABLE_HTTP2_CHECKS?.some(check => typeof check === 'function' && check(response.status, headers))) { log(' -- doFetch check disabled h2', uri); - resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true}))); + resolve(doFetch(profile, Object.assign({}, options, {disable_http2: true}))); } else { stream.status = response.status; if (response.url && response.url !== uri) { // Set as final destination url if Fetch follows 301/302 re-directs @@ -120,7 +224,7 @@ function doFetch(fetch_func, h1_fetch_func, options) { if (!options.disable_http2 && error.code && /^ERR_HTTP2/.test(error.code)) { log(' -- doFetch http2 error', error.code, uri); - resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true}))); + resolve(doFetch(profile, Object.assign({}, options, {disable_http2: true}))); } else if (!options.disable_http2 && error.code && error instanceof FetchError && error.code === 'ABORT_ERR') { @@ -130,12 +234,12 @@ function doFetch(fetch_func, h1_fetch_func, options) { * https://app.everviz.com/show/O0Cy7Dyt */ log(' -- doFetch h2 aborted error', uri); - resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true}))); + resolve(doFetch(profile, Object.assign({}, options, {disable_http2: true}))); } else if (!options.stopRecursion && CONFIG.ERRORS_TO_RETRY?.some(code => error.code?.indexOf(code) > -1)) { log(' -- doFetch ECONNRESET retry', error.code, uri); - resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {stopRecursion: true, disable_http2: true}))); + resolve(doFetch(profile, Object.assign({}, options, {stopRecursion: true, disable_http2: true}))); } else { if (error instanceof AbortError) { @@ -149,15 +253,15 @@ function doFetch(fetch_func, h1_fetch_func, options) { } export function fetchStreamKeepAlive(options) { - return doFetch(fetchKeepAlive, fetchH1KeepAlive, options); + return doFetch(PROFILE_KEEP_ALIVE, options); } export function fetchStream(options) { - return doFetch(fetch, fetchH1, options); + return doFetch(PROFILE_DEFAULT, options); }; export function fetchStreamAuthorized(options) { - return doFetch(fetchAuthorized, fetchH1Authorized, options); + return doFetch(PROFILE_AUTHORIZED, options); }; export function fetchData(options) { @@ -184,7 +288,7 @@ export function fetchData(options) { abortController.abort(); }, options.timeout || CONFIG.RESPONSE_TIMEOUT); - const a_fetch_func = options.disable_http2 ? fetchH1: fetch; + const a_fetch_func = selectFetchFunc(options); return new Promise((resolve, reject) => { a_fetch_func(uri, fetch_options) .then(response => { diff --git a/lib/utils.js b/lib/utils.js index 6a80e9ade..3e5735d72 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -126,6 +126,15 @@ export function prepareRequestOptions(request_options, options) { request_options.headers[key] = value; } } + + // Lowest priority for `ipv6first/ipv4first` from `providerOptions`. + if (options?.getProviderOptions) { + if (options.getProviderOptions('app.ipv6first')) { + request_options.ipv6first = true; + } else if (options.getProviderOptions('app.ipv4first')) { + request_options.ipv4first = true; + } + } if (CONFIG.PROXY || options?.proxy) { @@ -153,6 +162,12 @@ export function prepareRequestOptions(request_options, options) { if (proxy.disable_http2) { request_options.disable_http2 = true; } + // Middle priority for `ipv6first/ipv4first` from `proxy`. + if (proxy.ipv6first) { + request_options.ipv6first = true; + } else if (proxy.ipv4first) { + request_options.ipv4first = true; + } if (proxy.disable_language) { disable_language = true; } @@ -168,6 +183,13 @@ export function prepareRequestOptions(request_options, options) { } } + // Highest priority for `ipv6first/ipv4first` from `options`. + if (options?.ipv6first) { + request_options.ipv6first = true; + } else if (options?.ipv4first) { + request_options.ipv4first = true; + } + if (options?.user_agent) { // Let `options.user_agent` be prioritized over `proxy.user_agent`. request_options.headers['User-Agent'] = options?.user_agent; diff --git a/patches/@adobe__fetch.patch b/patches/@adobe__fetch.patch new file mode 100644 index 000000000..a57987b1c --- /dev/null +++ b/patches/@adobe__fetch.patch @@ -0,0 +1,21 @@ +diff --git a/src/core/request.js b/src/core/request.js +index 6f7120e3e383ed44b4fe63861109f365579d7425..19de7a26bc712e172b78efb4de4ef97ad8f269e3 100644 +--- a/src/core/request.js ++++ b/src/core/request.js +@@ -160,7 +160,16 @@ const determineProtocol = async (ctx, url, signal) => { + const rejectUnauthorized = !((_rejectUnauthorized === false + || h1Opts.rejectUnauthorized === false + || h2Opts.rejectUnauthorized === false)); ++ // PATCH: forward DNS/socket options from context to the ALPN negotiation socket ++ const socketOptions = {}; ++ ['lookup', 'family', 'autoSelectFamily', 'autoSelectFamilyAttemptTimeout'].forEach((name) => { ++ const value = h2Opts[name] !== undefined ? h2Opts[name] : h1Opts[name]; ++ if (value !== undefined) { ++ socketOptions[name] = value; ++ } ++ }); + const connectOptions = { ++ ...socketOptions, + servername: url.hostname, // enable SNI (Server Name Indication) extension + ALPNProtocols: ctx.alpnProtocols, + signal, // optional abort signal diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5acafc28a..d3dce785f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,7 @@ overrides: proxy-addr@>=1.1.0 <2.0.8: ^2.0.8 patchedDependencies: + '@adobe/fetch': fe94317880319fddf09a3d0f8eda551d5a2a205441bf5a7d634005239702b38d readabilitySAX@1.6.1: 13d40f78c0537e2538fd3a338f5ead833d968e2a80882e92f1eb3580b599c757 importers: @@ -40,7 +41,7 @@ importers: dependencies: '@adobe/fetch': specifier: ^4.1.11 - version: 4.1.11(supports-color@8.1.1) + version: 4.1.11(patch_hash=fe94317880319fddf09a3d0f8eda551d5a2a205441bf5a7d634005239702b38d)(supports-color@8.1.1) async: specifier: ^3.2.4 version: 3.2.4 @@ -1376,7 +1377,7 @@ packages: snapshots: - '@adobe/fetch@4.1.11(supports-color@8.1.1)': + '@adobe/fetch@4.1.11(patch_hash=fe94317880319fddf09a3d0f8eda551d5a2a205441bf5a7d634005239702b38d)(supports-color@8.1.1)': dependencies: debug: 4.4.0(supports-color@8.1.1) http-cache-semantics: 4.3.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c385fbc06..66711fa27 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -25,4 +25,5 @@ overrides: probe-image-size@<=7.3.0: ^7.4.0 proxy-addr@>=1.1.0 <2.0.8: ^2.0.8 patchedDependencies: + '@adobe/fetch': patches/@adobe__fetch.patch readabilitySAX@1.6.1: patches/readabilitySAX@1.6.1.patch