feat: harden request reliability and runtime telemetry

# Conflicts:
#	index.ts
#	lib/codex-manager/commands/status.ts
#	lib/runtime/metrics.ts
#	test/runtime-metrics.test.ts

Author: ndycode

README.md

@@ -19,6 +19,7 @@ Codex CLI-first multi-account OAuth manager for the official `@openai/codex` CLI
 - Interactive dashboard for account actions and settings
 - Experimental settings tab for staged sync, backup, and refresh-guard controls
 - Forecast, report, fix, and doctor commands for operational safety
+- Runtime request budget, cooldown, and traffic observability in `codex auth status` / `codex auth report`
 - Flagged account verification and restore flow
 - Session affinity and live account sync controls
 - Proactive refresh and preemptive quota deferral controls
@@ -156,6 +157,14 @@ If browser launch is blocked, use the alternate login paths in [docs/getting-sta
 | `codex auth report --live --json` | How do I get the full machine-readable health report? |
 | `codex auth fix --live --model gpt-5-codex` | How do I run live repair probes with a chosen model? |
 
+### Reliability behavior
+
+- whole-pool replay is disabled by default when every account is rate-limited
+- active requests use a bounded outbound request budget so one prompt cannot walk the full pool indefinitely
+- repeated cross-account 5xx bursts trigger a short cooldown instead of continuing aggressive rotation
+- proactive refresh is staggered to reduce background refresh bursts
+- `codex auth status` and `codex auth report --json` surface recent runtime request metrics and active cooldown windows
+
 ---
 
 ## Dashboard Hotkeys
@@ -265,6 +274,8 @@ codex auth login
 
 - `codex auth` unrecognized: run `where codex`, then follow `docs/troubleshooting.md` for routing fallback commands
 - Switch succeeds but wrong account appears active: run `codex auth switch <index>`, then restart session
+- Requests fail fast with a pool cooldown message: wait for the cooldown window or inspect `codex auth status`
+- Requests fail fast after repeated upstream 5xx errors: inspect `codex auth report --json` for runtime traffic and cooldown details
 - OAuth callback on port `1455` fails: free the port and re-run `codex auth login`
 - Browser launch is blocked or you are in a headless shell: re-run `codex auth login --manual` or set `CODEX_AUTH_NO_BROWSER=1`
 - `missing field id_token` / `token_expired` / `refresh_token_reused`: re-login affected account

index.ts

@@ -152,10 +152,18 @@ import {
 	parseFailoverMode,
 } from "./lib/request/failover-config.js";
 import {
-	buildStreamFailoverCandidateOrder,
 	capStreamFailoverMax,
 	computeOutboundRequestAttemptBudget,
 } from "./lib/request/request-attempt-budget.js";
+import {
+	armPoolExhaustionCooldown,
+	buildAdaptiveStreamFailoverCandidateOrder,
+	clearPoolExhaustionCooldown,
+	clearServerBurstCooldown,
+	getPoolExhaustionCooldownRemaining,
+	getServerBurstCooldownRemaining,
+	recordServerBurstFailure,
+} from "./lib/request/request-resilience.js";
 import {
 	evaluateFailurePolicy,
 	type FailoverMode,
@@ -239,6 +247,7 @@ import {
 	ensureRefreshGuardianState,
 	ensureSessionAffinityState,
 } from "./lib/runtime/runtime-services.js";
+import { mutateRuntimeObservabilitySnapshot } from "./lib/runtime/runtime-observability.js";
 import { applyAccountStorageScopeFromConfig } from "./lib/runtime/storage-scope.js";
 import { showRuntimeToast } from "./lib/runtime/toast.js";
 import { createRuntimeSessionRecoveryHook } from "./lib/runtime/session-recovery.js";
@@ -353,9 +362,14 @@ let sessionAffinityWriteVersion = 0;
 	totalRequests: number;
 	successfulRequests: number;
 	failedRequests: number;
+	responsesRequests: number;
+	authRefreshRequests: number;
+	diagnosticProbeRequests: number;
 	outboundRequestAttemptBudget: number | null;
 	outboundRequestAttemptsConsumed: number;
 	requestAttemptBudgetExhaustions: number;
+	poolExhaustionFastFails: number;
+	serverBurstFastFails: number;
 	rateLimitedResponses: number;
 	serverErrors: number;
 	networkErrors: number;
@@ -379,9 +393,14 @@ let sessionAffinityWriteVersion = 0;
 		totalRequests: 0,
 		successfulRequests: 0,
 		failedRequests: 0,
+		responsesRequests: 0,
+		authRefreshRequests: 0,
+		diagnosticProbeRequests: 0,
 		outboundRequestAttemptBudget: null,
 		outboundRequestAttemptsConsumed: 0,
 		requestAttemptBudgetExhaustions: 0,
+		poolExhaustionFastFails: 0,
+		serverBurstFastFails: 0,
 		rateLimitedResponses: 0,
 		serverErrors: 0,
 		networkErrors: 0,
@@ -422,6 +441,23 @@ let sessionAffinityWriteVersion = 0;
 		findMatchingAccountIndex,
 		modelFamilies: MODEL_FAMILIES,
 	});
+	const syncRuntimeObservability = (requestId: string | null): void => {
+		mutateRuntimeObservabilitySnapshot((snapshot) => {
+			snapshot.currentRequestId = requestId;
+			snapshot.poolExhaustionCooldownUntil =
+				getPoolExhaustionCooldownRemaining() > 0
+					? Date.now() + getPoolExhaustionCooldownRemaining()
+					: null;
+			snapshot.serverBurstCooldownUntil =
+				getServerBurstCooldownRemaining() > 0
+					? Date.now() + getServerBurstCooldownRemaining()
+					: null;
+			snapshot.responsesRequests = runtimeMetrics.responsesRequests;
+			snapshot.authRefreshRequests = runtimeMetrics.authRefreshRequests;
+			snapshot.diagnosticProbeRequests = runtimeMetrics.diagnosticProbeRequests;
+			snapshot.runtimeMetrics = { ...runtimeMetrics };
+		});
+	};
 	const persistAccountPoolAndFlagged = async (
 		results: TokenSuccessWithAccount[],
 		flaggedStorage: Parameters<typeof saveFlaggedAccounts>[0],
@@ -929,31 +965,65 @@ let sessionAffinityWriteVersion = 0;
 										sessionAffinityKey,
 									);
 								sessionAffinityStore?.prune();
-								const requestCorrelationId = setCorrelationId(
-									threadIdCandidate
-										? `${threadIdCandidate}:${Date.now()}`
-										: undefined,
-								);
-								runtimeMetrics.lastRequestAt = Date.now();
+							const requestCorrelationId = setCorrelationId(
+								threadIdCandidate
+									? `${threadIdCandidate}:${Date.now()}`
+									: undefined,
+							);
+							const requestTraceId = requestCorrelationId;
+							runtimeMetrics.lastRequestAt = Date.now();
+							syncRuntimeObservability(requestTraceId);
 
 							const abortSignal = requestInit?.signal ?? init?.signal ?? null;
 							const sleep = createAbortableSleep(abortSignal);
+							const poolCooldownRemainingMs = getPoolExhaustionCooldownRemaining();
+							if (poolCooldownRemainingMs > 0) {
+								runtimeMetrics.failedRequests += 1;
+								runtimeMetrics.poolExhaustionFastFails += 1;
+								runtimeMetrics.lastError = "Pool exhaustion cooldown active";
+								syncRuntimeObservability(requestTraceId);
+								return new Response(
+									JSON.stringify({
+										error: {
+											message: `The account pool is cooling down after recent rate-limit exhaustion. Try again in ${formatWaitTime(poolCooldownRemainingMs)} or inspect \`codex auth status\`.`,
+										},
+									}),
+									{ status: 429, headers: { "content-type": "application/json; charset=utf-8" } },
+								);
+							}
+							const serverBurstCooldownRemainingMs = getServerBurstCooldownRemaining();
+							if (serverBurstCooldownRemainingMs > 0) {
+								runtimeMetrics.failedRequests += 1;
+								runtimeMetrics.serverBurstFastFails += 1;
+								runtimeMetrics.lastError = "Server burst cooldown active";
+								syncRuntimeObservability(requestTraceId);
+								return new Response(
+									JSON.stringify({
+										error: {
+											message: `Multiple accounts recently failed with upstream server errors. Try again in ${formatWaitTime(serverBurstCooldownRemainingMs)} or inspect \`codex auth report --json\`.`,
+										},
+									}),
+									{ status: 503, headers: { "content-type": "application/json; charset=utf-8" } },
+								);
+							}
 							const maxOutboundRequestAttempts =
 								computeOutboundRequestAttemptBudget({
 									accountCount: accountManager.getAccountCount(),
 									maxSameAccountRetries,
 									emptyResponseMaxRetries,
 									streamFailoverMax,
 								});
-							runtimeMetrics.outboundRequestAttemptBudget =
-								maxOutboundRequestAttempts;
+								runtimeMetrics.outboundRequestAttemptBudget ??=
+									maxOutboundRequestAttempts;
 							logDebug("Configured outbound request attempt budget.", {
+								requestTraceId,
 								budget: maxOutboundRequestAttempts,
 								accountCount: accountManager.getAccountCount(),
 								maxSameAccountRetries,
 								emptyResponseMaxRetries,
 								streamFailoverMax,
 							});
+							syncRuntimeObservability(requestTraceId);
 							let outboundRequestAttemptsRemaining =
 								maxOutboundRequestAttempts;
 							const tryConsumeOutboundRequestAttempt = (
@@ -964,17 +1034,18 @@ let sessionAffinityWriteVersion = 0;
 									runtimeMetrics.requestAttemptBudgetExhaustions += 1;
 									runtimeMetrics.lastError =
 										`Request attempt budget exhausted after ${maxOutboundRequestAttempts} outbound request(s)`;
-									logWarn(
-										"Request attempt budget exhausted.",
-										{
-											reason,
-											accountIndex,
-											budget: maxOutboundRequestAttempts,
-											consumed:
-												runtimeMetrics.outboundRequestAttemptsConsumed,
-										},
-									);
-									return false;
+											logWarn(
+												"Request attempt budget exhausted.",
+												{
+													requestTraceId,
+													reason,
+													accountIndex,
+													budget: maxOutboundRequestAttempts,
+											consumed: maxOutboundRequestAttempts,
+												},
+											);
+											syncRuntimeObservability(requestTraceId);
+											return false;
 								}
 
 								runtimeMetrics.outboundRequestAttemptsConsumed += 1;
@@ -1096,6 +1167,8 @@ let sessionAffinityWriteVersion = 0;
 										) as OAuthAuthDetails;
 										try {
 											if (shouldRefreshToken(accountAuth, tokenRefreshSkewMs)) {
+												runtimeMetrics.authRefreshRequests += 1;
+												syncRuntimeObservability(requestTraceId);
 												accountAuth = (await refreshAndUpdateToken(
 													accountAuth,
 													client,
@@ -1409,6 +1482,8 @@ let sessionAffinityWriteVersion = 0;
 
 											try {
 												runtimeMetrics.totalRequests++;
+												runtimeMetrics.responsesRequests++;
+												syncRuntimeObservability(requestTraceId);
 												response = await fetch(
 													url,
 													applyProxyCompatibleInit(url, {
@@ -1978,7 +2053,7 @@ let sessionAffinityWriteVersion = 0;
 													break;
 												}
 
-												if (errorResponse.status === 429 && rateLimit) {
+										if (errorResponse.status === 429 && rateLimit) {
 													runtimeMetrics.rateLimitedResponses++;
 													const retryAfterMs =
 														rateLimit?.retryAfterMs ?? 60_000;
@@ -2085,11 +2160,14 @@ let sessionAffinityWriteVersion = 0;
 											let responseForSuccess = response;
 											if (isStreaming) {
 												const streamFallbackCandidateOrder =
-													buildStreamFailoverCandidateOrder(
+													buildAdaptiveStreamFailoverCandidateOrder(
 														account.index,
-														accountSnapshotList.map(
-															(candidate) => candidate.index,
-														),
+														accountSnapshotList as Array<
+															Pick<
+																import("./lib/accounts.js").ManagedAccount,
+																"index" | "lastUsed" | "enabled" | "coolingDownUntil" | "rateLimitResetTimes"
+															>
+														>,
 													);
 												runtimeMetrics.lastStreamFailoverCandidateCount =
 													streamFallbackCandidateOrder.length;
@@ -2098,12 +2176,14 @@ let sessionAffinityWriteVersion = 0;
 												logDebug(
 													"Prepared stream failover candidates.",
 													{
+														requestTraceId,
 														primaryAccountIndex: account.index,
 														candidateCount:
 															streamFallbackCandidateOrder.length,
 														candidateIndices: streamFallbackCandidateOrder,
 													},
 												);
+												syncRuntimeObservability(requestTraceId);
 												responseForSuccess = withStreamingFailover(
 													response,
 													async (failoverAttempt, emittedBytes) => {
@@ -2136,13 +2216,15 @@ let sessionAffinityWriteVersion = 0;
 																fallbackAccount,
 															) as OAuthAuthDetails;
 															try {
-																if (
-																	shouldRefreshToken(
-																		fallbackAuth,
-																		tokenRefreshSkewMs,
-																	)
-																) {
-																	fallbackAuth = (await refreshAndUpdateToken(
+															if (
+																shouldRefreshToken(
+																	fallbackAuth,
+																	tokenRefreshSkewMs,
+																)
+															) {
+																runtimeMetrics.authRefreshRequests += 1;
+																syncRuntimeObservability(requestTraceId);
+																fallbackAuth = (await refreshAndUpdateToken(
 																		fallbackAuth,
 																		client,
 																	)) as OAuthAuthDetails;
@@ -2278,6 +2360,8 @@ let sessionAffinityWriteVersion = 0;
 
 															try {
 																runtimeMetrics.totalRequests++;
+																runtimeMetrics.responsesRequests++;
+																syncRuntimeObservability(requestTraceId);
 																const fallbackResponse = await fetch(
 																	url,
 																	applyProxyCompatibleInit(url, {
@@ -2507,6 +2591,24 @@ let sessionAffinityWriteVersion = 0;
 															);
 															break;
 														}
+														if (response.status >= 500) {
+															const serverBurstCooldownUntil =
+																recordServerBurstFailure(account.index);
+															if (serverBurstCooldownUntil > 0) {
+																runtimeMetrics.serverBurstFastFails += 1;
+																runtimeMetrics.lastError =
+																	"Repeated upstream server errors across the account pool";
+																syncRuntimeObservability(requestTraceId);
+																return new Response(
+																	JSON.stringify({
+																		error: {
+																			message: `Upstream server failures were observed across multiple accounts. Pausing retries for ${formatWaitTime(serverBurstCooldownUntil - Date.now())}. Check \`codex auth report --json\` for runtime metrics.`,
+																		},
+																	}),
+																	{ status: 503, headers: { "content-type": "application/json; charset=utf-8" } },
+																);
+															}
+														}
 														logWarn(
 															`Empty response after ${emptyResponseMaxRetries} retries. Returning as-is.`,
 														);
@@ -2558,6 +2660,9 @@ let sessionAffinityWriteVersion = 0;
 												lastCodexCliActiveSyncIndex =
 													successAccountForResponse.index;
 											}
+											clearPoolExhaustionCooldown();
+											clearServerBurstCooldown();
+											syncRuntimeObservability(requestTraceId);
 											return successResponse;
 										}
 										if (retryNextAccountBeforeFallback) {
@@ -2603,26 +2708,31 @@ let sessionAffinityWriteVersion = 0;
 										continue;
 									}
 
-									const waitLabel =
-										waitMs > 0 ? formatWaitTime(waitMs) : "a bit";
-									const message =
-										count === 0
-											? "No Codex accounts configured. Run `codex login`."
-											: waitMs > 0
-												? `All ${count} account(s) are rate-limited. Try again in ${waitLabel} or add another account with \`codex login\`.`
-												: `All ${count} account(s) failed (server errors or auth issues). Check account health with \`codex-health\`.`;
-									runtimeMetrics.failedRequests++;
-									runtimeMetrics.lastError = message;
-									return new Response(JSON.stringify({ error: { message } }), {
+											const waitLabel =
+												waitMs > 0 ? formatWaitTime(waitMs) : "a bit";
+											if (count > 0 && waitMs > 0) {
+												armPoolExhaustionCooldown(waitMs);
+											}
+											const message =
+												count === 0
+													? "No Codex accounts configured. Run `codex login`."
+													: waitMs > 0
+														? `All ${count} account(s) are rate-limited. A short pool cooldown is now active for ${waitLabel}. Try again later or inspect \`codex auth status\`.`
+														: `All ${count} account(s) failed (server errors or auth issues). Check account health with \`codex auth report --json\`.`;
+											runtimeMetrics.failedRequests++;
+											runtimeMetrics.lastError = message;
+											syncRuntimeObservability(requestTraceId);
+											return new Response(JSON.stringify({ error: { message } }), {
 										status: waitMs > 0 ? 429 : 503,
 										headers: {
 											"content-type": "application/json; charset=utf-8",
 										},
 									});
 								}
-							} finally {
-								clearCorrelationId();
-							}
+								} finally {
+									syncRuntimeObservability(null);
+									clearCorrelationId();
+								}
 						},
 					};
 				} finally {