diff --git a/.changeset/fix-copilot-cli-model-selection-persistence.md b/.changeset/fix-copilot-cli-model-selection-persistence.md new file mode 100644 index 00000000000..91e77f988dd --- /dev/null +++ b/.changeset/fix-copilot-cli-model-selection-persistence.md @@ -0,0 +1,5 @@ +--- +"gh-aw": patch +--- + +Fix routed and fallback Copilot CLI runs recording an empty configured model in the model-selection log, and keep routing attribution, token-usage accounting and the runner metadata snapshot running when model-selection persistence fails. diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 050b8bd5b66..c23c0573485 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -1294,13 +1294,17 @@ async function main() { return; } modelRoutingSelection = routingResult.selection; + // Model-selection evidence records the requested identifier: `auto` for routed runs, + // otherwise the configured model or alias (or the fallback) before alias resolution. + let selectionConfiguredModel = configuredCopilotModel; if (modelRoutingSelection) { + selectionConfiguredModel = "auto"; if (!copilotSDKMode) { resolvedArgs = applyCopilotRoutingArgs(resolvedArgs, modelRoutingSelection); if (configuredWireAPI) process.env.COPILOT_PROVIDER_WIRE_API = configuredWireAPI; } } else { - applyModelFallback(process.env, "COPILOT_MODEL", log); + selectionConfiguredModel = applyModelFallback(process.env, "COPILOT_MODEL", log) || configuredCopilotModel; await applyCopilotModelAliasResolution({ awfReflectData, logger: log, @@ -1523,7 +1527,7 @@ async function main() { // Redact --prompt / -p value from logs to avoid leaking prompt content const safeArgs = currentArgs.map((arg, i) => (currentArgs[i - 1] === "--prompt" || currentArgs[i - 1] === "-p" ? "" : arg)); - const modelObserver = !verifyModel ? { check: () => "" } : createCopilotModelObserver({ configuredModel: configuredCopilotModel, resolvedModel: process.env.COPILOT_MODEL || "", sessionId, logger: log }); + const modelObserver = !verifyModel ? { check: () => "" } : createCopilotModelObserver({ configuredModel: selectionConfiguredModel, resolvedModel: process.env.COPILOT_MODEL || "", sessionId, logger: log }); // Driver mode: run copilot_sdk_driver.cjs as a normal subprocess. The harness has // already started the sidecar; the driver only opens an SDK client connection. const result = await runProcess({ diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 3d4de80bc91..32efcb0d306 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -4259,3 +4259,124 @@ process.exit(1);` }); }); }); + +describe("Copilot CLI model-selection evidence through harness startup", () => { + // Derived from the agent artifact of run 39147795806: awf-reflect.json keeps the copilot endpoint and routing selection + // (unconfigured provider endpoints and unrelated top-level reflect metadata are omitted). + const fixtureDir = path.join(path.dirname(require.resolve("./copilot_harness.cjs")), "fixtures", "copilot-routed-run-39147795806"); + const { persistCopilotModelSelection } = require("./copilot_model_selection.cjs"); + const tempDirs = []; + + afterEach(() => { + for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); + }); + + function runHarness(envOverrides, { routed, providerPort = 0, sdk = false }) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "copilot-selection-startup-")); + tempDirs.push(tmpDir); + const preloadPath = path.join(tmpDir, "reflect-preload.cjs"); + const stubPath = path.join(tmpDir, "copilot-stub.cjs"); + const argsPath = path.join(tmpDir, "stub-args.json"); + const reflect = JSON.parse(fs.readFileSync(path.join(fixtureDir, "awf-reflect.json"), "utf8")); + if (!routed) delete reflect.routing; + if (providerPort) Object.assign(reflect.endpoints[0], { base_url: `http://127.0.0.1:${providerPort}`, models_url: `http://127.0.0.1:${providerPort}/models` }); + fs.writeFileSync(path.join(tmpDir, "reflect.json"), JSON.stringify(reflect)); + fs.writeFileSync( + preloadPath, + `const body = require("fs").readFileSync(${JSON.stringify(path.join(tmpDir, "reflect.json"))}, "utf8"); +require(${JSON.stringify(require.resolve("./awf_reflect.cjs"))}).fetchAWFReflect = async () => ({ ok: true, reflectData: JSON.parse(body) }); +if (${providerPort} && process.argv[1].endsWith("copilot_harness.cjs")) require("net").createServer(socket => socket.end()).listen(${providerPort}, "127.0.0.1").unref();` + ); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const path = require("path"); +const args = process.argv.slice(2); +fs.writeFileSync(${JSON.stringify(argsPath)}, JSON.stringify(args)); +if (args.includes("--session-id")) { + const model = args.includes("--model") ? args[args.lastIndexOf("--model") + 1] : process.env.COPILOT_MODEL; + const dir = path.join(process.env.COPILOT_HOME, "session-state", args[args.indexOf("--session-id") + 1]); + fs.mkdirSync(dir, { recursive: true }); + fs.appendFileSync(path.join(dir, "events.jsonl"), JSON.stringify({ type: "assistant.message", data: { model, content: "done" } }) + "\\n"); +} +process.stdout.write("done\\n");` + ); + // SDK sidecar double: listens on the requested --port until the harness stops it. + const sidecarPath = path.join(tmpDir, "copilot-sidecar.cjs"); + fs.writeFileSync(sidecarPath, `#!/usr/bin/env node\nconst args = process.argv.slice(2);\nrequire("net").createServer(socket => socket.end()).listen(Number(args[args.indexOf("--port") + 1]), "127.0.0.1");\n`, { mode: 0o755 }); + const env = { ...harnessChildEnv, GH_AW_TMP_DIR: tmpDir, COPILOT_HOME: path.join(tmpDir, "copilot-home"), GH_AW_HARNESS_MAX_RETRIES: "0", GH_AW_HARNESS_STALL_WARNING_MS: "0", NODE_OPTIONS: `--require ${preloadPath}` }; + for (const key of ["COPILOT_MODEL", "GH_AW_MODEL_FALLBACK", "GH_AW_MODEL_ROUTING", "AWF_REFLECT_ENABLED", "GH_AW_SKIP_REFLECT", "COPILOT_SDK_URI", "GH_AW_PHASE", "GH_AW_SAFE_OUTPUTS", "COPILOT_PROVIDER_WIRE_API", "GH_AW_MODEL_ALIASES"]) + delete env[key]; + Object.assign(env, envOverrides); + const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, ...(sdk ? [stubPath, sidecarPath] : [stubPath, "--prompt", "Hello"])], { + cwd: path.dirname(require.resolve("./copilot_harness.cjs")), + env, + encoding: "utf8", + timeout: 20000, + }); + const selectionPath = path.join(tmpDir, "sandbox/agent/logs/copilot-model-selection.json"); + return { + tmpDir, + result, + args: fs.existsSync(argsPath) ? JSON.parse(fs.readFileSync(argsPath, "utf8")) : null, + selection: fs.existsSync(selectionPath) ? JSON.parse(fs.readFileSync(selectionPath, "utf8")) : null, + }; + } + + async function freePort() { + const server = require("net").createServer(); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address(); + await new Promise(resolve => server.close(resolve)); + return port; + } + + function persistFor(tmpDir) { + fs.copyFileSync(path.join(fixtureDir, "aw_info.json"), path.join(tmpDir, "aw_info.json")); + persistCopilotModelSelection({ GH_AW_TMP_DIR: tmpDir, GH_AW_ENGINE_ID: "copilot" }); + return JSON.parse(fs.readFileSync(path.join(tmpDir, "aw_info.json"), "utf8")); + } + + it("records configured_model auto for a routed CLI run with COPILOT_MODEL unset", () => { + const { tmpDir, result, args, selection } = runHarness({ AWF_REFLECT_ENABLED: "1", GH_AW_MODEL_ROUTING: "1" }, { routed: true }); + + expect(result.status, result.stderr).toBe(0); + expect(args.slice(args.indexOf("--model"), args.indexOf("--model") + 2)).toEqual(["--model", "gpt-5.6-luna"]); + expect(selection).toEqual({ status: "observed", configured_model: "auto", resolved_model: "gpt-5.6-luna", effective_model: "gpt-5.6-luna" }); + expect(persistFor(tmpDir)).toMatchObject({ + model: "gpt-5.6-luna", + requested_model: "auto", + model_selection: { status: "observed", configured_model: "auto", resolved_model: "gpt-5.6-luna", effective_model: "gpt-5.6-luna" }, + }); + }); + + it("records the fallback configured model when COPILOT_MODEL is unset", () => { + const { tmpDir, result, args, selection } = runHarness({ GH_AW_SKIP_REFLECT: "true", GH_AW_MODEL_FALLBACK: "gpt-5.6-luna" }, { routed: false }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toContain("applied GH_AW_MODEL_FALLBACK to COPILOT_MODEL"); + expect(args).not.toContain("--reasoning-effort"); + expect(selection).toEqual({ status: "observed", configured_model: "gpt-5.6-luna", resolved_model: "gpt-5.6-luna", effective_model: "gpt-5.6-luna" }); + expect(persistFor(tmpDir)).toMatchObject({ model: "gpt-5.6-luna", model_selection: { configured_model: "gpt-5.6-luna", effective_model: "gpt-5.6-luna" } }); + }); + + it("keeps an explicit configured model for a non-routed CLI run", () => { + const { result, selection } = runHarness({ GH_AW_SKIP_REFLECT: "true", COPILOT_MODEL: "gpt-5.6-luna", GH_AW_MODEL_FALLBACK: "claude-sonnet-5" }, { routed: false }); + + expect(result.status, result.stderr).toBe(0); + expect(selection).toEqual({ status: "observed", configured_model: "gpt-5.6-luna", resolved_model: "gpt-5.6-luna", effective_model: "gpt-5.6-luna" }); + }); + + it.each([true, false])("does not create a CLI selection record in SDK mode (routed=%s)", async routed => { + const [providerPort, sdkPort] = [await freePort(), await freePort()]; + const { result, args, selection } = runHarness( + { AWF_REFLECT_ENABLED: "1", ...(routed ? { GH_AW_MODEL_ROUTING: "1" } : { COPILOT_MODEL: "gpt-5.6-luna" }), COPILOT_SDK_URI: `http://127.0.0.1:${sdkPort}` }, + { routed, providerPort, sdk: true } + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toContain("copilot-sdk driver mode: starting sidecar"); + expect(args).not.toContain("--session-id"); + expect(selection).toBeNull(); + }); +}); diff --git a/actions/setup/js/fixtures/copilot-routed-run-39147795806/aw_info.json b/actions/setup/js/fixtures/copilot-routed-run-39147795806/aw_info.json new file mode 100644 index 00000000000..de20aae23fd --- /dev/null +++ b/actions/setup/js/fixtures/copilot-routed-run-39147795806/aw_info.json @@ -0,0 +1,32 @@ +{ + "engine_id": "copilot", + "engine_name": "GitHub Copilot CLI", + "model": "auto", + "version": "1.0.98", + "agent_version": "1.0.98", + "workflow_name": "Smoke Copilot Routed", + "experimental": false, + "supports_tools_allowlist": true, + "cache_memory": false, + "run_id": 39147795806, + "run_number": 46, + "run_attempt": "1", + "repository": "github/gh-aw", + "ref": "refs/heads/main", + "sha": "eb42b26807ac37c20f0a43ba3beba68375ab43e1", + "actor": "SivaKesava1", + "event_name": "workflow_dispatch", + "target_repo": "", + "staged": false, + "dry_run": false, + "allowed_domains": ["defaults"], + "firewall_enabled": true, + "awf_version": "v0.28.62", + "awmg_version": "", + "agent_runtime": "", + "steps": { + "firewall": "squid" + }, + "created_at": "2026-10-18T17:04:03.230Z", + "frontmatter_emoji": "🧭" +} diff --git a/actions/setup/js/fixtures/copilot-routed-run-39147795806/awf-reflect.json b/actions/setup/js/fixtures/copilot-routed-run-39147795806/awf-reflect.json new file mode 100644 index 00000000000..73830fd18f3 --- /dev/null +++ b/actions/setup/js/fixtures/copilot-routed-run-39147795806/awf-reflect.json @@ -0,0 +1,554 @@ +{ + "endpoints": [ + { + "provider": "copilot", + "port": 10002, + "base_url": "http://api-proxy:10002", + "configured": true, + "models": [ + "claude-haiku-4.5", + "claude-haiku-5.5", + "claude-opus-5.5", + "claude-sonnet-5", + "claude-sonnet-5.5", + "gpt-3.5-turbo", + "gpt-3.5-turbo-0613", + "gpt-4", + "gpt-4-0613", + "gpt-4-o-preview", + "gpt-4.1", + "gpt-4.1-2025-04-14", + "gpt-41-copilot", + "gpt-4o", + "gpt-4o-2024-05-13", + "gpt-4o-2024-08-06", + "gpt-4o-2024-11-20", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "gpt-5-mini", + "gpt-5.3-codex", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.5", + "gpt-5.6-luna", + "gpt-5.6-luna-utility", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "grok-4.5", + "grok-4.6", + "grok-4.7", + "mai-code-1.1-flash", + "text-embedding-3-small", + "text-embedding-3-small-inference", + "text-embedding-ada-002", + "trajectory-compaction" + ], + "model_metadata": [ + { "id": "claude-haiku-4.5", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01", "pricing": { "default": { "input": 1, "output": 5, "cachedInput": 0.1, "cacheWrite": 1.25 } } }, + { + "id": "claude-haiku-5.5", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 0.1, "output": 0.5, "cachedInput": 0.01, "cacheWrite": 0.12, "threshold": 100000 }, "long_context": { "input": 0.5, "output": 2.5, "cachedInput": 0.05, "cacheWrite": 0.62, "threshold": 936000 } } + }, + { + "id": "claude-opus-5.5", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 4, "output": 20, "cachedInput": 0.2, "cacheWrite": 5, "threshold": 200000 }, "long_context": { "input": 4, "output": 20, "cachedInput": 0.2, "cacheWrite": 5, "threshold": 1000000 } } + }, + { + "id": "claude-sonnet-5", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 2, "output": 10, "cachedInput": 0.2, "cacheWrite": 2.5, "threshold": 200000 }, "long_context": { "input": 2, "output": 10, "cachedInput": 0.2, "cacheWrite": 2.5, "threshold": 936000 } } + }, + { + "id": "claude-sonnet-5.5", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 2, "output": 10, "cachedInput": 0.1, "cacheWrite": 2.5, "threshold": 200000 }, "long_context": { "input": 2, "output": 10, "cachedInput": 0.1, "cacheWrite": 2.5, "threshold": 936000 } } + }, + { "id": "gpt-3.5-turbo", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-3.5-turbo-0613", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-4", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01", "pricing": { "default": { "input": 2.5, "output": 10, "cachedInput": 1.25, "cacheWrite": 0 } } }, + { "id": "gpt-4-0613", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01", "pricing": { "default": { "input": 2.5, "output": 10, "cachedInput": 1.25, "cacheWrite": 0 } } }, + { "id": "gpt-4-o-preview", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-4.1", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-4.1-2025-04-14", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-41-copilot", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01", "pricing": { "default": { "input": 2, "output": 8, "cachedInput": 0.5, "cacheWrite": 0 } } }, + { "id": "gpt-4o", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-4o-2024-05-13", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-4o-2024-08-06", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-4o-2024-11-20", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-4o-mini", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-4o-mini-2024-07-18", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "gpt-5-mini", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01", "pricing": { "default": { "input": 0.25, "output": 2, "cachedInput": 0.02, "cacheWrite": 0 } } }, + { + "id": "gpt-5.3-codex", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 1.75, "output": 14, "cachedInput": 0.17, "cacheWrite": 0, "threshold": 272000 } } + }, + { + "id": "gpt-5.4", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 2.5, "output": 15, "cachedInput": 0.25, "cacheWrite": 0, "threshold": 272000 }, "long_context": { "input": 5, "output": 22.5, "cachedInput": 0.5, "cacheWrite": 0, "threshold": 922000 } } + }, + { + "id": "gpt-5.4-mini", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 0.75, "output": 4.5, "cachedInput": 0.07, "cacheWrite": 0, "threshold": 272000 } } + }, + { + "id": "gpt-5.5", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 5, "output": 30, "cachedInput": 0.5, "cacheWrite": 0, "threshold": 272000 }, "long_context": { "input": 10, "output": 45, "cachedInput": 1, "cacheWrite": 0, "threshold": 922000 } } + }, + { + "id": "gpt-5.6-luna", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 0.2, "output": 1.2, "cachedInput": 0.02, "cacheWrite": 0.25, "threshold": 200000 }, "long_context": { "input": 0.4, "output": 1.8, "cachedInput": 0.04, "cacheWrite": 0.5, "threshold": 922000 } } + }, + { "id": "gpt-5.6-luna-utility", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { + "id": "gpt-5.6-sol", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 4, "output": 20, "cachedInput": 0.4, "cacheWrite": 5, "threshold": 272000 }, "long_context": { "input": 8, "output": 30, "cachedInput": 0.8, "cacheWrite": 10, "threshold": 922000 } } + }, + { + "id": "gpt-5.6-terra", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 2, "output": 12, "cachedInput": 0.2, "cacheWrite": 2.5, "threshold": 272000 }, "long_context": { "input": 4, "output": 18, "cachedInput": 0.4, "cacheWrite": 5, "threshold": 922000 } } + }, + { + "id": "gpt-6-astra", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 10, "output": 50, "cachedInput": 1, "cacheWrite": 12.5, "threshold": 272000 }, "long_context": { "input": 20, "output": 75, "cachedInput": 2, "cacheWrite": 25, "threshold": 1050000 } } + }, + { + "id": "gpt-6-luna", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { + "default": { "input": 0.1, "output": 0.5, "cachedInput": 0.01, "cacheWrite": 0.12, "threshold": 272000 }, + "long_context": { "input": 0.2, "output": 0.75, "cachedInput": 0.02, "cacheWrite": 0.25, "threshold": 872000 } + } + }, + { + "id": "gpt-6-sol", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 2, "output": 10, "cachedInput": 0.2, "cacheWrite": 2.5, "threshold": 272000 }, "long_context": { "input": 4, "output": 15, "cachedInput": 0.4, "cacheWrite": 5, "threshold": 872000 } } + }, + { + "id": "gpt-6.1-sol", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 2, "output": 10, "cachedInput": 0.1, "cacheWrite": 2.5, "threshold": 272000 }, "long_context": { "input": 4, "output": 15, "cachedInput": 0.2, "cacheWrite": 5, "threshold": 922000 } } + }, + { + "id": "grok-4.5", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "wire_api": "responses", + "pricing": { "default": { "input": 2, "output": 6, "cachedInput": 0.5, "cacheWrite": 0, "threshold": 200000 }, "long_context": { "input": 4, "output": 12, "cachedInput": 1, "cacheWrite": 0, "threshold": 500000 } } + }, + { + "id": "grok-4.6", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "wire_api": "responses", + "pricing": { "default": { "input": 2, "output": 6, "cachedInput": 0.5, "cacheWrite": 0, "threshold": 200000 }, "long_context": { "input": 4, "output": 12, "cachedInput": 1, "cacheWrite": 0, "threshold": 500000 } } + }, + { + "id": "grok-4.7", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "wire_api": "responses", + "pricing": { "default": { "input": 2, "output": 6, "cachedInput": 0.5, "cacheWrite": 0, "threshold": 200000 }, "long_context": { "input": 4, "output": 12, "cachedInput": 1, "cacheWrite": 0, "threshold": 500000 } } + }, + { + "id": "mai-code-1.1-flash", + "source": "provider", + "observed_at": "2026-10-18T17:05:38.926Z", + "api_version": "2026-07-01", + "pricing": { "default": { "input": 0.2, "output": 1.2, "cachedInput": 0.02, "cacheWrite": 0.25, "threshold": 128000 } } + }, + { "id": "text-embedding-3-small", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "text-embedding-3-small-inference", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "text-embedding-ada-002", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01" }, + { "id": "trajectory-compaction", "source": "provider", "observed_at": "2026-10-18T17:05:38.926Z", "api_version": "2026-07-01", "wire_api": "completions" } + ], + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": ["claude-sonnet-5.5", "claude-sonnet-5", "claude-opus-5.5", "claude-haiku-5.5", "claude-haiku-4.5"], + "chat/completions": ["claude-sonnet-5.5", "claude-sonnet-5", "claude-opus-5.5", "claude-haiku-5.5", "claude-haiku-4.5"], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": ["gpt-5.4", "gpt-5-mini"], + "responses": ["gpt-6.1-sol", "gpt-6-astra", "gpt-6-luna", "gpt-6-sol", "gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.5", "gpt-5.4", "gpt-5.4-mini", "gpt-5.3-codex", "gpt-5-mini"] + }, + "google": { "messages": [], "chat/completions": [], "responses": [] } + } + }, + "routing_models": [ + { "model_id": "claude-haiku-4.5", "source": "provider", "supported_endpoints": ["/chat/completions", "/v1/messages"], "supported_reasoning_efforts": [], "context_window_tokens": 200000, "candidate_metadata_complete": true }, + { + "model_id": "claude-haiku-5.5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-opus-5.5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-sonnet-5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-sonnet-5.5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-3.5-turbo", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 16384, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-3.5-turbo-0613", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 16384, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 32768, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4-0613", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 32768, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4-o-preview", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4.1", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4.1-2025-04-14", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-41-copilot", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-2024-05-13", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-2024-08-06", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-2024-11-20", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-mini", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-mini-2024-07-18", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-5-mini", + "source": "provider", + "supported_endpoints": ["/chat/completions", "/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high"], + "context_window_tokens": 264000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.3-codex", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], + "context_window_tokens": 400000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.4", + "source": "provider", + "supported_endpoints": ["/responses", "/chat/completions", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.4-mini", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh"], + "context_window_tokens": 400000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.5", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.6-luna", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.6-luna-utility", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-5.6-sol", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.6-terra", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6-astra", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6-luna", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6-sol", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6.1-sol", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { "model_id": "grok-4.5", "source": "provider", "supported_endpoints": ["/responses"], "supported_reasoning_efforts": ["low", "medium", "high"], "context_window_tokens": 500000, "candidate_metadata_complete": true }, + { "model_id": "grok-4.6", "source": "provider", "supported_endpoints": ["/responses"], "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], "context_window_tokens": 500000, "candidate_metadata_complete": true }, + { "model_id": "grok-4.7", "source": "provider", "supported_endpoints": ["/responses"], "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], "context_window_tokens": 500000, "candidate_metadata_complete": true }, + { + "model_id": "mai-code-1.1-flash", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high"], + "context_window_tokens": 256000, + "candidate_metadata_complete": true + }, + { + "model_id": "text-embedding-3-small", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "text-embedding-3-small-inference", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "text-embedding-ada-002", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "trajectory-compaction", + "source": "provider", + "supported_endpoints": ["/chat/completions"], + "supported_reasoning_efforts": [], + "context_window_tokens": 262144, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + } + ], + "models_url": "http://api-proxy:10002/models", + "credential_kind": "github_token", + "selected_scheme": "Bearer", + "inference_credential_source": "github_token", + "inference_selected_scheme": "Bearer", + "integration_id_source": "github_copilot_integration_id" + } + ], + "routing": { "status": "selected", "selection": { "provider": "copilot", "model": "github-copilot/gpt-5.6-luna", "wire_model": "gpt-5.6-luna", "effort": "max", "endpoint": "/responses" } } +} diff --git a/actions/setup/js/fixtures/copilot-routed-run-39147795806/awf-routing-outcome.json b/actions/setup/js/fixtures/copilot-routed-run-39147795806/awf-routing-outcome.json new file mode 100644 index 00000000000..82a35c9179e --- /dev/null +++ b/actions/setup/js/fixtures/copilot-routed-run-39147795806/awf-routing-outcome.json @@ -0,0 +1 @@ +{ "status": "selected", "wire_model": "gpt-5.6-luna", "endpoint": "/responses", "selected_endpoint": "/responses", "effort": "max", "applied_effort": "max" } diff --git a/actions/setup/js/fixtures/copilot-routed-run-39147795806/copilot-model-selection.json b/actions/setup/js/fixtures/copilot-routed-run-39147795806/copilot-model-selection.json new file mode 100644 index 00000000000..61036ed323b --- /dev/null +++ b/actions/setup/js/fixtures/copilot-routed-run-39147795806/copilot-model-selection.json @@ -0,0 +1 @@ +{ "status": "observed", "configured_model": "", "resolved_model": "gpt-5.6-luna", "effective_model": "gpt-5.6-luna" } diff --git a/actions/setup/js/fixtures/copilot-routed-run-39147795806/model-routing.jsonl b/actions/setup/js/fixtures/copilot-routed-run-39147795806/model-routing.jsonl new file mode 100644 index 00000000000..dd5674c8a00 --- /dev/null +++ b/actions/setup/js/fixtures/copilot-routed-run-39147795806/model-routing.jsonl @@ -0,0 +1,6 @@ +{"_schema":"model-routing/v0.28.62","timestamp":"2026-10-18T17:05:38.963Z","event":"model_routing","stage":"classification","purpose":"routing_classification","attempt":1,"classifier_model":"github-copilot/gpt-5.6-luna","classifier_effort":"high"} +{"_schema":"model-routing/v0.28.62","timestamp":"2026-10-18T17:05:45.104Z","event":"model_routing","stage":"selection","objective":{"goal":"cost","mode":"economy"},"provider":"copilot","selected_provider":"copilot","wire_model":"gpt-5.6-luna","endpoint":"/responses","labels":{"task_type":"chore","scope":"cross_system","task_complexity":"trivial"},"mode":"economy","classifier_model":"github-copilot/gpt-5.6-luna","classifier_effort":"high","router":{"name":"gh-aw-router","version":"0.1.3"},"ranked_choices":[{"model":"github-copilot/gpt-5.6-luna","effort":"max"},{"model":"github-copilot/gpt-5.6-luna","effort":"xhigh"},{"model":"github-copilot/gpt-5.6-luna","effort":"high"},{"model":"github-copilot/gpt-5.6-luna","effort":"medium"},{"model":"github-copilot/gpt-5.4-mini","effort":"xhigh"},{"model":"github-copilot/gpt-5.4-mini","effort":"high"},{"model":"github-copilot/gpt-5.4-mini","effort":"medium"},{"model":"github-copilot/gpt-5.6-luna","effort":"low"},{"model":"github-copilot/gpt-5.4-mini","effort":"low"},{"model":"github-copilot/claude-haiku-4.5","effort":null},{"model":"github-copilot/gpt-5.6-luna","effort":"none"},{"model":"github-copilot/gpt-5.4-mini","effort":"none"}],"conversation_sha256":"7af299df550a5bfe2ae9f80ccae3e5cb3a68c7d60b5b0c159f727ef364241498","selected_id":"choice-0009","selected_model":"github-copilot/gpt-5.6-luna","selected_effort":"max","degraded_classification":false,"degraded_reason":null,"classifier_attempts":1,"eligible_choices":12,"catalogue_overlap":12,"latency_ms":6156,"interaction_id":"39147795806-1","github_repository":"github/gh-aw","github_workflow_ref":"github/gh-aw/.github/workflows/smoke-copilot-routed.lock.yml@refs/heads/main"} +{"_schema":"model-routing/v0.28.62","timestamp":"2026-10-18T17:05:56.986Z","event":"model_routing","stage":"request","routed":"as_selected","deviations":[],"unavailable":[],"method":"POST","pathname":"/responses","provider":"copilot","requested_model":"gpt-5.6-luna","requested_effort":"max","selected_provider":"copilot","selected_model":"gpt-5.6-luna","selected_effort":"max","selected_endpoint":"/responses","requested_endpoint":"/responses","upstream_endpoint":"/responses","request_id":"350154af-d6ed-4d8c-b176-b93da7144bdd","outcome":"completed","status":200} +{"_schema":"model-routing/v0.28.62","timestamp":"2026-10-18T17:05:58.589Z","event":"model_routing","stage":"request","routed":"as_selected","deviations":[],"unavailable":[],"method":"POST","pathname":"/responses","provider":"copilot","requested_model":"gpt-5.6-luna","requested_effort":"max","selected_provider":"copilot","selected_model":"gpt-5.6-luna","selected_effort":"max","selected_endpoint":"/responses","requested_endpoint":"/responses","upstream_endpoint":"/responses","request_id":"79b45ef3-a365-4e5e-9e39-89c48698f37e","outcome":"completed","status":200} +{"_schema":"model-routing/v0.28.62","timestamp":"2026-10-18T17:06:01.270Z","event":"model_routing","stage":"request","routed":"as_selected","deviations":[],"unavailable":[],"method":"POST","pathname":"/responses","provider":"copilot","requested_model":"gpt-5.6-luna","requested_effort":"max","selected_provider":"copilot","selected_model":"gpt-5.6-luna","selected_effort":"max","selected_endpoint":"/responses","requested_endpoint":"/responses","upstream_endpoint":"/responses","request_id":"c2453852-d56d-4964-b514-3536a525afda","outcome":"completed","status":200} +{"_schema":"model-routing/v0.28.62","timestamp":"2026-10-18T17:06:05.354Z","event":"model_routing","stage":"request","routed":"as_selected","deviations":[],"unavailable":[],"method":"POST","pathname":"/responses","provider":"copilot","requested_model":"gpt-5.6-luna","requested_effort":"max","selected_provider":"copilot","selected_model":"gpt-5.6-luna","selected_effort":"max","selected_endpoint":"/responses","requested_endpoint":"/responses","upstream_endpoint":"/responses","request_id":"8fcd367d-53d6-48ac-b56f-620c1a1a6a80","outcome":"completed","status":200} diff --git a/actions/setup/js/fixtures/copilot-routed-run-39147795806/token-usage.jsonl b/actions/setup/js/fixtures/copilot-routed-run-39147795806/token-usage.jsonl new file mode 100644 index 00000000000..4f7166c6bb2 --- /dev/null +++ b/actions/setup/js/fixtures/copilot-routed-run-39147795806/token-usage.jsonl @@ -0,0 +1,5 @@ +{"_schema":"token-usage/v0.28.62","timestamp":"2026-10-18T17:05:45.096Z","event":"token_usage","request_id":"78a3c387-fe05-4ad8-9e8a-ccd5806dc19c","provider":"copilot","model":"gpt-5.6-luna","path":"/responses","status":200,"streaming":false,"input_tokens":941,"output_tokens":549,"cache_read_tokens":0,"cache_write_tokens":0,"duration_ms":6129,"response_bytes":20133,"purpose":"routing_classification","x_initiator":"agent","ai_credits_this_response":0.0847,"ai_credits_total":0.0847,"ai_credits_pricing_source":"provider","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false,"ai_credits_pricing_observed_at":"2026-10-18T17:05:38.926Z","ai_credits_pricing_api_version":"2026-07-01"} +{"_schema":"token-usage/v0.28.62","timestamp":"2026-10-18T17:05:56.985Z","event":"token_usage","request_id":"350154af-d6ed-4d8c-b176-b93da7144bdd","provider":"copilot","model":"gpt-5.6-luna","path":"/responses","status":200,"streaming":true,"input_tokens":23197,"output_tokens":400,"cache_read_tokens":0,"cache_write_tokens":0,"duration_ms":6725,"response_bytes":237349,"x_initiator":"user","ai_credits_this_response":0.51194,"ai_credits_total":0.59664,"ai_credits_pricing_source":"provider","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false,"ai_credits_pricing_observed_at":"2026-10-18T17:05:38.926Z","ai_credits_pricing_api_version":"2026-07-01"} +{"_schema":"token-usage/v0.28.62","timestamp":"2026-10-18T17:05:58.589Z","event":"token_usage","request_id":"79b45ef3-a365-4e5e-9e39-89c48698f37e","provider":"copilot","model":"gpt-5.6-luna","path":"/responses","status":200,"streaming":true,"input_tokens":23799,"output_tokens":45,"cache_read_tokens":23194,"cache_write_tokens":0,"duration_ms":1536,"response_bytes":225482,"x_initiator":"agent","ai_credits_this_response":0.063888,"ai_credits_total":0.660528,"ai_credits_pricing_source":"provider","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false,"ai_credits_pricing_observed_at":"2026-10-18T17:05:38.926Z","ai_credits_pricing_api_version":"2026-07-01"} +{"_schema":"token-usage/v0.28.62","timestamp":"2026-10-18T17:06:01.270Z","event":"token_usage","request_id":"c2453852-d56d-4964-b514-3536a525afda","provider":"copilot","model":"gpt-5.6-luna","path":"/responses","status":200,"streaming":true,"input_tokens":23870,"output_tokens":196,"cache_read_tokens":23796,"cache_write_tokens":0,"duration_ms":2614,"response_bytes":261689,"x_initiator":"agent","ai_credits_this_response":0.072592,"ai_credits_total":0.73312,"ai_credits_pricing_source":"provider","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false,"ai_credits_pricing_observed_at":"2026-10-18T17:05:38.926Z","ai_credits_pricing_api_version":"2026-07-01"} +{"_schema":"token-usage/v0.28.62","timestamp":"2026-10-18T17:06:05.354Z","event":"token_usage","request_id":"8fcd367d-53d6-48ac-b56f-620c1a1a6a80","provider":"copilot","model":"gpt-5.6-luna","path":"/responses","status":200,"streaming":true,"input_tokens":24084,"output_tokens":109,"cache_read_tokens":23867,"cache_write_tokens":0,"duration_ms":3844,"response_bytes":298553,"x_initiator":"agent","ai_credits_this_response":0.065154,"ai_credits_total":0.798274,"ai_credits_pricing_source":"provider","ai_credits_pricing_tier":"default","ai_credits_accounting_policy":"concrete_model","ai_credits_fallback_pricing_used":false,"ai_credits_pricing_observed_at":"2026-10-18T17:05:38.926Z","ai_credits_pricing_api_version":"2026-07-01"} diff --git a/actions/setup/js/parse_token_usage.cjs b/actions/setup/js/parse_token_usage.cjs index 6f72d6b03e5..31346f2ef34 100644 --- a/actions/setup/js/parse_token_usage.cjs +++ b/actions/setup/js/parse_token_usage.cjs @@ -550,8 +550,14 @@ async function appendStepSummarySection(title, markdown, workingSet = null) { async function main(copilotSessionStateDir = getGhAwPath("sandbox/agent/logs/copilot-session-state")) { /** @type {{routing: any, effective: any}|null} */ let routedAttribution = null; + // Invalid selection evidence is not persisted; its error is reported after routing attribution and usage accounting. + let selectionError = ""; try { persistCopilotModelSelection(); + } catch (error) { + selectionError = getErrorMessage(error); + } + try { routedAttribution = recordModelRoutingFromArtifacts(); const tokenUsagePaths = getReadableTokenUsagePaths(getTokenUsagePaths()); if (tokenUsagePaths.length === 0) { @@ -644,6 +650,7 @@ async function main(copilotSessionStateDir = getGhAwPath("sandbox/agent/logs/cop core.setFailed(`${ERR_PARSE}: ${getErrorMessage(error)}`); } finally { persistAgentAwInfoCopy(); + if (selectionError) core.setFailed(`${ERR_PARSE}: ${selectionError}`); } } diff --git a/actions/setup/js/parse_token_usage.test.cjs b/actions/setup/js/parse_token_usage.test.cjs index 823d740dc9b..143130ea178 100644 --- a/actions/setup/js/parse_token_usage.test.cjs +++ b/actions/setup/js/parse_token_usage.test.cjs @@ -475,6 +475,86 @@ describe("parse_token_usage", () => { } }); + describe("routed Copilot CLI run 39147795806", () => { + // Unmodified records from the run's agent artifact (JSON reformatted only). + const fixtureDir = path.join(__dirname, "fixtures", "copilot-routed-run-39147795806"); + const realSelection = fs.readFileSync(path.join(fixtureDir, "copilot-model-selection.json"), "utf8"); + + function setupRoutedRun(selectionContent) { + process.env.GH_AW_TMP_DIR = tmpDir; + process.env.GH_AW_MODEL_ROUTING_ENABLED = "true"; + process.env.GH_AW_ENGINE_ID = "copilot"; + process.env.GITHUB_STEP_SUMMARY = path.join(tmpDir, "step-summary.md"); + const proxyLogs = path.join(tmpDir, "sandbox/firewall/logs/api-proxy-logs"); + fs.mkdirSync(proxyLogs, { recursive: true }); + fs.mkdirSync(path.join(tmpDir, "agent"), { recursive: true }); + fs.copyFileSync(path.join(fixtureDir, "aw_info.json"), path.join(tmpDir, "aw_info.json")); + fs.copyFileSync(path.join(fixtureDir, "awf-routing-outcome.json"), path.join(tmpDir, "agent/awf-routing-outcome.json")); + fs.copyFileSync(path.join(fixtureDir, "model-routing.jsonl"), path.join(proxyLogs, "model-routing.jsonl")); + fs.copyFileSync(path.join(fixtureDir, "token-usage.jsonl"), path.join(proxyLogs, "token-usage.jsonl")); + const selectionPath = path.join(tmpDir, "sandbox/agent/logs/copilot-model-selection.json"); + fs.mkdirSync(path.dirname(selectionPath), { recursive: true }); + fs.writeFileSync(selectionPath, selectionContent); + } + + function expectRoutingAndUsage() { + for (const relative of ["aw_info.json", "agent/aw_info.json", "runtime/aw_info.json"]) { + expect(JSON.parse(originalReadFileSync(path.join(tmpDir, relative), "utf8"))).toMatchObject({ + engine_id: "copilot", + model: "gpt-5.6-luna", + requested_model: "auto", + model_routing: { status: "selected", wire_model: "gpt-5.6-luna", effort: "max", endpoint: "/responses" }, + }); + } + const summaryText = originalReadFileSync(path.join(tmpDir, "step-summary.md"), "utf8"); + expect(summaryText).toContain(DEFAULT_SUMMARY_TITLE); + expect(summaryText).toContain("| **Total** | | **95,891** | **1,299** |"); + expect(JSON.parse(originalReadFileSync(path.join(tmpDir, "agent_usage.json"), "utf8"))).toMatchObject({ + primary_model: "gpt-5.6-luna", + model_routing: { status: "selected", wire_model: "gpt-5.6-luna" }, + }); + expect(mockCore.setOutput).toHaveBeenCalledWith("model", "gpt-5.6-luna"); + expect(mockCore.setOutput).toHaveBeenCalledWith("model_routing_status", "selected"); + expect(mockCore.setOutput).toHaveBeenCalledWith("primary_model", "gpt-5.6-luna"); + expect(mockCore.setOutput).toHaveBeenCalledWith("aic", expect.any(String)); + } + + test.each([ + ["the real empty configured_model record", realSelection, "Invalid Copilot model selection"], + ["an empty file", "", "Unexpected end of JSON input"], + ["malformed JSON", "{", "JSON"], + ])("records routing and usage when Copilot model selection persistence fails (%s)", async (_name, selectionContent, diagnostic) => { + setupRoutedRun(selectionContent); + + await main(); + + expect(mockCore.setFailed).toHaveBeenCalledTimes(1); + expect(mockCore.setFailed).toHaveBeenCalledWith(expect.stringMatching(/^ERR_PARSE: Cannot record Copilot model selection: /)); + expect(mockCore.setFailed).toHaveBeenCalledWith(expect.stringContaining(diagnostic)); + expectRoutingAndUsage(); + for (const relative of ["aw_info.json", "agent/aw_info.json", "runtime/aw_info.json"]) { + expect(JSON.parse(originalReadFileSync(path.join(tmpDir, relative), "utf8"))).not.toHaveProperty("model_selection"); + } + }); + + test("persists the corrected routed selection with routing and usage", async () => { + setupRoutedRun(JSON.stringify({ ...JSON.parse(realSelection), configured_model: "auto" })); + + await main(); + + expect(mockCore.setFailed).not.toHaveBeenCalled(); + expectRoutingAndUsage(); + for (const relative of ["aw_info.json", "agent/aw_info.json", "runtime/aw_info.json"]) { + expect(JSON.parse(originalReadFileSync(path.join(tmpDir, relative), "utf8")).model_selection).toEqual({ + status: "observed", + configured_model: "auto", + resolved_model: "gpt-5.6-luna", + effective_model: "gpt-5.6-luna", + }); + } + }); + }); + test("does not trust selected routing from sandbox-writable reflect data", async () => { process.env.GH_AW_TMP_DIR = tmpDir; process.env.GH_AW_MODEL_ROUTING_ENABLED = "true"; diff --git a/actions/setup/js/unified_session.test.cjs b/actions/setup/js/unified_session.test.cjs index bc51d2a5f6e..7da946726a1 100644 --- a/actions/setup/js/unified_session.test.cjs +++ b/actions/setup/js/unified_session.test.cjs @@ -1072,6 +1072,39 @@ describe("Unified conclusion session", () => { expect(fs.readFileSync(path.join(root, "usage/aw_session.jsonl"), "utf8")).toBe(original); }); + it("normalizes the runner snapshot of routed Copilot CLI run 39147795806 into workflow.info", async () => { + // Records from the run's agent artifact, with the harness selection carrying the routed request `auto`. + const fixtureDir = path.join(import.meta.dirname, "fixtures", "copilot-routed-run-39147795806"); + const fixture = name => fs.readFileSync(path.join(fixtureDir, name), "utf8"); + write("aw_info.json", fixture("aw_info.json")); + write("agent/awf-routing-outcome.json", fixture("awf-routing-outcome.json")); + write("sandbox/firewall/logs/api-proxy-logs/model-routing.jsonl", fixture("model-routing.jsonl")); + write("sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl", fixture("token-usage.jsonl")); + write("sandbox/agent/logs/copilot-model-selection.json", { ...JSON.parse(fixture("copilot-model-selection.json")), configured_model: "auto" }); + process.env.GH_AW_TMP_DIR = root; + process.env.GH_AW_MODEL_ROUTING_ENABLED = "true"; + process.env.GH_AW_ENGINE_ID = "copilot"; + process.env.GITHUB_STEP_SUMMARY = path.join(root, "step-summary.md"); + for (const key of ["GH_AW_PHASE", "GH_AW_AGENT_USAGE_PATH", "GH_AW_AGENT_USAGE_JSONL_PATH", "GH_AW_ENGINE_MODEL"]) delete process.env[key]; + global.core = { info: vi.fn(), warning: vi.fn(), debug: vi.fn(), setFailed: vi.fn(), setOutput: vi.fn(), exportVariable: vi.fn() }; + await req("./parse_token_usage.cjs").main(); + expect(global.core.setFailed).not.toHaveBeenCalled(); + // The uploaded artifact carries the runner snapshot, not the runner-root metadata. + fs.rmSync(path.join(root, "aw_info.json")); + + const { events } = collectUnifiedSession({ rootDir: root, warn: vi.fn() }); + expect(events.find(event => event.type === "workflow.info")).toMatchObject({ + data: { + engineId: "copilot", + model: "gpt-5.6-luna", + requestedModel: "auto", + modelRouting: { status: "selected", wireModel: "gpt-5.6-luna", effort: "max", effectiveEndpoint: "/responses" }, + modelSelection: { status: "observed", configuredModel: "auto", resolvedModel: "gpt-5.6-luna", effectiveModel: "gpt-5.6-luna" }, + }, + provenance: { component: "workflow", path: "runtime/aw_info.json" }, + }); + }); + it("does not promote agent-only aw_info metadata to workflow.info", () => { write("agent/aw_info.json", { engine_id: "claude", model_routing: { status: "selected", wire_model: "forged-model" }, context: { episode_id: "forged" } }); const { events } = collectUnifiedSession({ rootDir: root, warn: vi.fn() });