Skip to content

Commit 9ded89a

Browse files
ndycodendycode
committed
fix(release): formalize validation unblockers
1 parent 42885d0 commit 9ded89a

5 files changed

Lines changed: 722 additions & 466 deletions

File tree

‎lib/storage.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1975,7 +1975,11 @@ export async function loadFlaggedAccounts(): Promise<FlaggedAccountStorageV1> {
19751975
const empty: FlaggedAccountStorageV1 = { version: 1, accounts: [] };
19761976

19771977
try {
1978-
return await loadFlaggedAccountsFromPath(path);
1978+
const loaded = await loadFlaggedAccountsFromPath(path);
1979+
if (existsSync(resetMarkerPath)) {
1980+
return empty;
1981+
}
1982+
return loaded;
19791983
} catch (error) {
19801984
const code = (error as NodeJS.ErrnoException).code;
19811985
if (code !== "ENOENT") {
@@ -2086,6 +2090,9 @@ export async function clearFlaggedAccounts(): Promise<void> {
20862090
path: candidate,
20872091
error: String(error),
20882092
});
2093+
if (candidate === path) {
2094+
throw error;
2095+
}
20892096
}
20902097
}
20912098
}

‎scripts/test-model-matrix.js‎

Lines changed: 138 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,24 @@
1-
import { existsSync, readFileSync } from "node:fs";
2-
import { readFile, writeFile, rm, mkdir } from "node:fs/promises";
31
import { spawnSync } from "node:child_process";
2+
import { existsSync, readFileSync } from "node:fs";
3+
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
44
import { dirname, join, resolve } from "node:path";
5-
import { fileURLToPath } from "node:url";
65
import process from "node:process";
6+
import { fileURLToPath } from "node:url";
77

88
const scriptDir = dirname(fileURLToPath(import.meta.url));
99
const repoRoot = resolve(scriptDir, "..");
10-
const localConfigPaths = [join(repoRoot, ".codex.json"), join(repoRoot, "Codex.json")];
10+
const localConfigPaths = [
11+
join(repoRoot, ".codex.json"),
12+
join(repoRoot, "Codex.json"),
13+
];
1114
const scenarioTemplates = {
1215
legacy: join(repoRoot, "config", "codex-legacy.json"),
1316
modern: join(repoRoot, "config", "codex-modern.json"),
1417
};
1518

1619
const pluginPackageName = "codex-multi-auth";
1720
const DEFAULT_MATRIX_TIMEOUT_MS = 120000;
21+
const DEFAULT_SMOKE_MATRIX_TIMEOUT_MS = 15000;
1822

1923
function resolveCmdScriptEntry(commandPath) {
2024
if (!/\.cmd$/i.test(commandPath)) {
@@ -153,14 +157,72 @@ export function __resetTrackedCodexPidsForTests() {
153157
spawnedCodexPids.clear();
154158
}
155159

156-
export function resolveMatrixTimeoutMs() {
157-
const parsedTimeout = Number.parseInt(process.env.CODEX_MATRIX_TIMEOUT_MS ?? String(DEFAULT_MATRIX_TIMEOUT_MS), 10);
160+
export function resolveMatrixTimeoutMs(smoke = false) {
161+
const fallback = smoke
162+
? DEFAULT_SMOKE_MATRIX_TIMEOUT_MS
163+
: DEFAULT_MATRIX_TIMEOUT_MS;
164+
const parsedTimeout = Number.parseInt(
165+
process.env.CODEX_MATRIX_TIMEOUT_MS ?? String(fallback),
166+
10,
167+
);
158168
if (!Number.isFinite(parsedTimeout) || parsedTimeout <= 0) {
159-
return DEFAULT_MATRIX_TIMEOUT_MS;
169+
return fallback;
160170
}
161171
return parsedTimeout;
162172
}
163173

174+
function hasCompletedSuccessfully(output, token) {
175+
return (
176+
output.includes(token) ||
177+
output.includes('"type":"turn.completed"') ||
178+
output.includes('"type":"response.completed"')
179+
);
180+
}
181+
182+
function getSmokeSkipReason(exitCode, output) {
183+
if (exitCode === 124) {
184+
return "timed-out";
185+
}
186+
if (/not supported when using codex with a chatgpt account/i.test(output)) {
187+
return "unsupported-model";
188+
}
189+
if (
190+
/unsupported value:\s*['"]xhigh['"]/i.test(output) ||
191+
/unsupported_value/i.test(output)
192+
) {
193+
return "unsupported-reasoning";
194+
}
195+
return null;
196+
}
197+
198+
function finalizeModelCaseResult(caseInfo, exitCode, output, token, smoke) {
199+
const hasToken = output.includes(token);
200+
const completed = hasCompletedSuccessfully(output, token);
201+
const ok = exitCode === 0 && completed;
202+
const skipReason = !ok && smoke ? getSmokeSkipReason(exitCode, output) : null;
203+
204+
return {
205+
...caseInfo,
206+
ok,
207+
exitCode,
208+
hasToken,
209+
completed,
210+
skipped: skipReason !== null,
211+
skipReason,
212+
output,
213+
};
214+
}
215+
216+
export function __finalizeModelCaseResultForTests(
217+
caseInfo,
218+
exitCode,
219+
output,
220+
token,
221+
smoke = false,
222+
) {
223+
return finalizeModelCaseResult(caseInfo, exitCode, output, token, smoke);
224+
}
225+
164226
function stopCodexServersInternal() {
165227
const tracked = [...spawnedCodexPids];
166228
spawnedCodexPids.clear();
@@ -267,7 +329,7 @@ export function __buildModelCaseArgsForTests(caseInfo, index) {
267329
function executeModelCase(caseInfo, index) {
268330
const { token, args } = buildModelCaseArgs(caseInfo, index);
269331

270-
const timeoutMs = resolveMatrixTimeoutMs();
332+
const timeoutMs = resolveMatrixTimeoutMs(caseInfo.smoke === true);
271333
const commandArgs = [...(CodexExecutable.prefixArgs ?? []), ...args];
272334
const finalized = spawnSync(CodexExecutable.command, commandArgs, {
273335
cwd: repoRoot,
@@ -285,27 +347,25 @@ function executeModelCase(caseInfo, index) {
285347
});
286348

287349
if (finalized.error && finalized.error.code === "ETIMEDOUT") {
288-
return {
289-
...caseInfo,
290-
ok: false,
291-
exitCode: 124,
292-
hasToken: false,
293-
output: `Timed out after ${timeoutMs}ms`,
294-
};
350+
return finalizeModelCaseResult(
351+
caseInfo,
352+
124,
353+
`Timed out after ${timeoutMs}ms`,
354+
token,
355+
caseInfo.smoke === true,
356+
);
295357
}
296358

297-
const combinedOutput = `${finalized.stdout ?? ""}\n${finalized.stderr ?? ""}`.trim();
298-
const hasToken = combinedOutput.includes(token);
359+
const combinedOutput =
360+
`${finalized.stdout ?? ""}\n${finalized.stderr ?? ""}`.trim();
299361
const exitCode = finalized.status ?? 1;
300-
const ok = exitCode === 0 && hasToken;
301-
302-
return {
303-
...caseInfo,
304-
ok,
362+
return finalizeModelCaseResult(
363+
caseInfo,
305364
exitCode,
306-
hasToken,
307-
output: combinedOutput,
308-
};
365+
combinedOutput,
366+
token,
367+
caseInfo.smoke === true,
368+
);
309369
}
310370

311371
async function readJson(pathValue) {
@@ -365,31 +425,43 @@ async function prepareScenarioConfig(templatePath, pluginRef) {
365425
async function runScenario(scenario, options) {
366426
const templatePath = scenarioTemplates[scenario];
367427
if (!templatePath || !existsSync(templatePath)) {
368-
throw new Error(`Template not found for scenario '${scenario}': ${templatePath}`);
428+
throw new Error(
429+
`Template not found for scenario '${scenario}': ${templatePath}`,
430+
);
369431
}
370432

371433
const config = await prepareScenarioConfig(templatePath, options.pluginRef);
372434
const models = config?.provider?.openai?.models;
373435
if (!models || typeof models !== "object") {
374-
throw new Error(`Scenario '${scenario}' has no provider.openai.models object`);
436+
throw new Error(
437+
`Scenario '${scenario}' has no provider.openai.models object`,
438+
);
375439
}
376440

377-
const cases = enumerateCases(models, options.smoke, options.maxCases);
441+
const cases = enumerateCases(models, options.smoke, options.maxCases).map(
442+
(caseInfo) => ({
443+
...caseInfo,
444+
smoke: options.smoke,
445+
}),
446+
);
378447
console.log(`\n=== ${scenario.toUpperCase()} (${cases.length} cases) ===`);
379448

380449
const results = [];
381450
for (let i = 0; i < cases.length; i += 1) {
382451
const caseInfo = cases[i];
383-
const result = executeModelCase(
384-
caseInfo,
385-
i + 1,
386-
);
452+
const result = executeModelCase(caseInfo, i + 1);
387453
results.push(result);
388454
const variantLabel = result.variant ? ` [variant=${result.variant}]` : "";
389455
if (result.ok) {
390456
console.log(`PASS ${result.model}${variantLabel}`);
457+
} else if (result.skipped) {
458+
console.log(
459+
`SKIP ${result.model}${variantLabel} (${result.skipReason})`,
460+
);
391461
} else {
392-
console.log(`FAIL ${result.model}${variantLabel} (exit=${result.exitCode}, token=${result.hasToken})`);
462+
console.log(
463+
`FAIL ${result.model}${variantLabel} (exit=${result.exitCode}, token=${result.hasToken})`,
464+
);
393465
const tail = result.output.split(/\r?\n/).slice(-12).join("\n");
394466
if (tail.trim().length > 0) {
395467
console.log(tail);
@@ -407,8 +479,9 @@ async function main() {
407479
return;
408480
}
409481

410-
const scenarioValue = parseArgValue(args, "--scenario") ?? "all";
411482
const smoke = args.includes("--smoke");
483+
const scenarioValue =
484+
parseArgValue(args, "--scenario") ?? (smoke ? "modern" : "all");
412485
const pluginMode = parseArgValue(args, "--plugin") ?? "dist";
413486
const noRestore = args.includes("--no-restore");
414487
const maxCasesRaw = parseArgValue(args, "--max-cases");
@@ -419,13 +492,19 @@ async function main() {
419492
: undefined;
420493

421494
if (!["all", "legacy", "modern"].includes(scenarioValue)) {
422-
throw new Error(`Invalid --scenario value '${scenarioValue}'. Use legacy, modern, or all.`);
495+
throw new Error(
496+
`Invalid --scenario value '${scenarioValue}'. Use legacy, modern, or all.`,
497+
);
423498
}
424499
if (!["dist", "package"].includes(pluginMode)) {
425-
throw new Error(`Invalid --plugin value '${pluginMode}'. Use dist or package.`);
500+
throw new Error(
501+
`Invalid --plugin value '${pluginMode}'. Use dist or package.`,
502+
);
426503
}
427504
if (Number.isNaN(maxCases) || maxCases < 0) {
428-
throw new Error(`Invalid --max-cases value '${maxCasesRaw}'. Use a non-negative integer.`);
505+
throw new Error(
506+
`Invalid --max-cases value '${maxCasesRaw}'. Use a non-negative integer.`,
507+
);
429508
}
430509

431510
const pluginRef = resolvePluginReference(pluginMode);
@@ -437,7 +516,9 @@ async function main() {
437516
console.log(`Scenarios: ${scenarios.join(", ")}`);
438517
console.log(`Mode: ${smoke ? "smoke" : "full"}`);
439518
console.log(`Plugin: ${pluginRef}`);
440-
console.log(`Codex command: ${CodexExecutable.displayCommand ?? CodexExecutable.command}`);
519+
console.log(
520+
`Codex command: ${CodexExecutable.displayCommand ?? CodexExecutable.command}`,
521+
);
441522

442523
const backups = await backupLocalConfigs();
443524
const allResults = [];
@@ -450,19 +531,23 @@ async function main() {
450531
maxCases,
451532
pluginRef,
452533
});
453-
allResults.push(...scenarioResults.map((item) => ({ ...item, scenario })));
534+
allResults.push(
535+
...scenarioResults.map((item) => ({ ...item, scenario })),
536+
);
454537
}
455538
} finally {
456539
if (!noRestore) {
457540
await restoreLocalConfigs(backups);
458541
}
459542
}
460543

461-
const failed = allResults.filter((result) => !result.ok);
462-
const passed = allResults.length - failed.length;
544+
const passed = allResults.filter((result) => result.ok);
545+
const skipped = allResults.filter((result) => result.skipped);
546+
const failed = allResults.filter((result) => !result.ok && !result.skipped);
463547
console.log("\n=== SUMMARY ===");
464548
console.log(`Total: ${allResults.length}`);
465-
console.log(`Passed: ${passed}`);
549+
console.log(`Passed: ${passed.length}`);
550+
console.log(`Skipped: ${skipped.length}`);
466551
console.log(`Failed: ${failed.length}`);
467552

468553
if (reportJsonPath) {
@@ -475,13 +560,18 @@ async function main() {
475560
CodexCommand: CodexExecutable.displayCommand ?? CodexExecutable.command,
476561
totals: {
477562
total: allResults.length,
478-
passed,
563+
passed: passed.length,
564+
skipped: skipped.length,
479565
failed: failed.length,
480566
},
481567
results: allResults,
482568
};
483569
await mkdir(dirname(reportJsonPath), { recursive: true });
484-
await writeFile(reportJsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
570+
await writeFile(
571+
reportJsonPath,
572+
`${JSON.stringify(report, null, 2)}\n`,
573+
"utf8",
574+
);
485575
console.log(`Report written: ${reportJsonPath}`);
486576
}
487577

@@ -492,6 +582,10 @@ async function main() {
492582
console.log(`- ${result.scenario}: ${result.model}${variantLabel}`);
493583
}
494584
process.exitCode = 1;
585+
} else if (smoke && passed.length === 0) {
586+
console.log(
587+
"\nSmoke matrix was inconclusive: all cases were skipped for this current runtime/account capability set.",
588+
);
495589
}
496590
}
497591

@@ -507,5 +601,3 @@ if (isDirectRun) {
507601
process.exit(1);
508602
});
509603
}
510-
511-

0 commit comments

Comments
 (0)