From fc47b7e036295d1b9dd214a9e7bc2b2a0c04f94a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:23:46 +0000 Subject: [PATCH 1/5] Initial plan From f3718dcae7ffd096693ae1e88b36b8e24ac56e98 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:43:37 +0000 Subject: [PATCH 2/5] Resolve Copilot sub-agent models in the harness and fail closed Co-authored-by: SivaKesava1 <11771739+SivaKesava1@users.noreply.github.com> --- actions/setup/js/copilot_harness.cjs | 229 ++- actions/setup/js/copilot_harness.test.cjs | 337 ++++ actions/setup/js/copilot_sdk_driver.cjs | 1 + actions/setup/js/copilot_sdk_driver.test.cjs | 53 +- actions/setup/js/copilot_sdk_session.cjs | 31 +- actions/setup/js/copilot_sdk_session.test.cjs | 2 + ...models-copilot-sub-agents-39304573724.json | 67 + ...eflect-copilot-sub-agents-39304573724.json | 1575 +++++++++++++++++ docs/src/content/docs/reference/engines.md | 17 +- 9 files changed, 2281 insertions(+), 31 deletions(-) create mode 100644 actions/setup/js/fixtures/awf-config-models-copilot-sub-agents-39304573724.json create mode 100644 actions/setup/js/fixtures/awf-reflect-copilot-sub-agents-39304573724.json diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index e3b24127f76..580af9e6688 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -55,6 +55,8 @@ require("./shim.cjs"); const { getErrorMessage } = require("./error_helpers.cjs"); const { maskSecret } = require("./actions_secret_masking.cjs"); const fs = require("fs"); +const os = require("os"); +const path = require("path"); const crypto = require("crypto"); const { getPromptPath, renderTemplateFromFile } = require("./messages_core.cjs"); const { @@ -103,6 +105,8 @@ const { loadModelsJson } = require("./model_costs.cjs"); const { applyCopilotWireAPI: configureCopilotWireAPI, warnCopilotSubagentEndpoints, recordModelEndpointMismatch, getSupportedEndpoints, MODEL_ENDPOINT_FIX } = require("./copilot_wire_api.cjs"); const { loadModelAliasMap, resolveEngineModel, normalizeForCopilotCLI, ModelAliasResolutionError, recordModelResolution, splitModelIdentifier } = require("./resolve_model_alias.cjs"); const { createCopilotModelObserver, recordCopilotModelSelection } = require("./copilot_model_selection.cjs"); +const { loadCopilotSDKCustomAgents } = require("./extract_inline_sub_agents.cjs"); +const { resolveCopilotSDKWorkingDirectory } = require("./copilot_sdk_session.cjs"); const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported|No model available\b[^\n]*policy enablement/i; const { parseAICreditsErrorInfoFromAuditLog, parseMaxAICreditsFromAuditLog, parseMaxAICreditsExceededFromAuditLog, parseAPIProxyGuardRejectionFromEventLog, formatAPIProxyGuardRejection } = require("./ai_credits_context.cjs"); @@ -650,6 +654,205 @@ async function resolveCopilotSDKMultiProvider(options) { return null; // unreachable, keeps TypeScript control-flow analysis happy } +/** + * Read the Copilot custom agent files in one directory, one entry per regular `.md` file. + * The name is the frontmatter `name` or the file name without `.agent.md` / `.md`; + * `model` is the trimmed frontmatter `model` string, when declared. + * @param {string} directory + * @param {(msg: string) => void} [logger] + * @returns {Array<{ name: string, model?: string, path: string }>} + */ +function listCopilotAgentFiles(directory, logger = log) { + const { extractFrontmatterAndBody } = require("./frontmatter_hash_pure.cjs"); + const { parseDocument } = require("./work_queue_yaml.cjs"); + let entries; + try { + entries = fs.readdirSync(directory, { withFileTypes: true }); + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) logger(`warning: cannot read custom agents in ${directory}: ${getErrorMessage(error)}`); + return []; + } + const agents = []; + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (!entry.isFile() || !entry.name.endsWith(".md")) continue; + const agentPath = path.join(directory, entry.name); + try { + const { frontmatterText } = extractFrontmatterAndBody(fs.readFileSync(agentPath, "utf8")); + /** @type {any} */ + const document = parseDocument(frontmatterText); + if (!document || document.errors.length) throw new Error("cannot parse agent frontmatter"); + const frontmatter = document.toJS() || {}; + if (typeof frontmatter !== "object" || Array.isArray(frontmatter)) throw new Error("agent frontmatter must be a mapping"); + const name = typeof frontmatter.name === "string" && frontmatter.name.trim() ? frontmatter.name.trim() : entry.name.replace(/(?:\.agent)?\.md$/, ""); + agents.push({ name, path: agentPath, ...(typeof frontmatter.model === "string" ? { model: frontmatter.model.trim() } : {}) }); + } catch (error) { + logger(`warning: cannot read custom agent ${agentPath}: ${getErrorMessage(error)}`); + } + } + return agents; +} + +/** + * Replace the top-level frontmatter `model:` of a Copilot agent file. Only the `model:` line + * changes; the rest of the file is preserved byte for byte. + * @param {string} agentPath + * @param {string} model + */ +function rewriteCopilotAgentFileModel(agentPath, model) { + const content = fs.readFileSync(agentPath, "utf8"); + const lines = content.split("\n"); + const end = lines.findIndex((line, index) => index > 0 && line.trim() === "---"); + const modelLines = lines.map((line, index) => (index > 0 && index < end && /^model[ \t]*:/.test(line) ? index : -1)).filter(index => index >= 0); + if (lines[0]?.trim() !== "---" || end < 0 || modelLines.length !== 1) { + throw new Error(`cannot rewrite model in ${agentPath}: expected exactly one top-level 'model:' line in the frontmatter`); + } + const index = modelLines[0]; + lines[index] = `model: ${JSON.stringify(model)}${lines[index].endsWith("\r") ? "\r" : ""}`; + fs.writeFileSync(agentPath, lines.join("\n")); +} + +/** + * Resolve the declared models of Copilot sub-agents before Copilot starts, in CLI and SDK + * mode and on routed runs. Each `model:` goes through the shared resolver (Copilot provider + * scope, the workflow alias map, `normalizeForCopilotCLI`) and the live-catalogue check that + * the main SDK model uses. With `providerConfig` (SDK mode), the resolved id must also qualify + * against the SDK provider models. A sub-agent without `model:` inherits the main model and is + * not recorded. + * + * Every sub-agent with a model gets a `model.resolution` record (role = agent name), so + * several sub-agents sharing one dead model each get a `failed` record. When any sub-agent + * fails, the first failure is thrown as a ModelAliasResolutionError after all records are + * written; callers must not start Copilot. `/reflect` is refreshed at most once. + * + * @param {{ + * agents: Array<{ name: string, model?: string }>, + * reflectData: any, + * refetchReflectData?: () => Promise, + * providerConfig?: { providers: Array<{name: string}>, models: Array<{id: string, provider: string}> } | null, + * aliasMap?: Record | null, + * env?: NodeJS.ProcessEnv, + * logger?: (msg: string) => void, + * }} options + * @returns {Promise>} + * `resolvedModel` is the resolved Copilot id; `model` is the id to hand to Copilot (qualified in SDK mode). + */ +async function resolveCopilotSubagentModels(options) { + const logger = options.logger || log; + const env = options.env || process.env; + const declared = options.agents.filter(agent => typeof agent.model === "string" && agent.model.trim()); + if (declared.length === 0) return []; + const aliasMap = options.aliasMap !== undefined ? options.aliasMap : (loadAwfConfigData()?.apiProxy?.models ?? loadModelAliasMap({ reflectData: options.reflectData, env })); + let reflectData = options.reflectData; + /** @type {Promise | null} */ + let refresh = null; + const refreshOnce = () => { + refresh ??= (options.refetchReflectData ? options.refetchReflectData() : Promise.resolve(null)).then(refreshed => { + if (refreshed) reflectData = refreshed; + return refreshed || null; + }); + return refresh; + }; + const liveCheck = env.AWF_REFLECT_ENABLED === "1" || Boolean(options.reflectData); + /** @type {InstanceType[]} */ + const failures = []; + const resolutions = []; + for (const agent of declared) { + const configuredModel = String(agent.model).trim(); + const scope = scopeModelToReflectProviders(configuredModel); + const fail = (/** @type {InstanceType} */ failure, /** @type {string} */ detail) => { + logger(`copilot sub-agent '${agent.name}': ${failure.message}${detail ? ` ${detail}` : ""} Refusing to start Copilot with a different sub-agent model.`); + recordModelResolution({ engine: "copilot", role: agent.name, configured_model: configuredModel, resolved_model: null, provider: failure.provider || null, status: "failed", cause: failure.reason }, env); + failures.push(failure); + }; + let result; + try { + result = await resolveEngineModel({ + engine: "copilot", + role: agent.name, + configuredModel, + aliasMap, + reflectData, + provider: "copilot", + refetchReflectData: refreshOnce, + logPrefix: `copilot sub-agent '${agent.name}' model alias resolution`, + normalize: r => normalizeForCopilotCLI(r.resolvedModel), + env, + logger, + }); + } catch (failure) { + if (!(failure instanceof ModelAliasResolutionError)) throw failure; + fail(failure, ""); + continue; + } + const resolvedModel = result.engineModel; + const resolvedNote = resolvedModel !== configuredModel ? ` (resolved to '${resolvedModel}')` : ""; + const failureProvider = scope.providers && scope.providers !== REFLECT_PROVIDER_ALIASES.github ? scope.prefix : "copilot"; + if (liveCheck && !isModelListedOnConfiguredReflectEndpoint(resolvedModel, reflectData)) { + const initialReason = classifyUnlistedReflectModel(resolvedModel, reflectData); + const refreshed = await refreshOnce(); + if (!refreshed || !isModelListedOnConfiguredReflectEndpoint(resolvedModel, refreshed)) { + const reason = initialReason === "model_unavailable" || !refreshed ? initialReason : classifyUnlistedReflectModel(resolvedModel, refreshed); + fail(new ModelAliasResolutionError(configuredModel, reason, { provider: failureProvider }), `Sub-agent model '${configuredModel}'${resolvedNote} is not listed on the configured awf-reflect endpoints.`); + continue; + } + } + let model = resolvedModel; + if (options.providerConfig) { + const qualified = qualifyModelForMultiProvider(resolvedModel, options.providerConfig); + if (!qualified) { + fail( + new ModelAliasResolutionError(configuredModel, "model_unavailable", { provider: failureProvider }), + `Sub-agent model '${configuredModel}'${resolvedNote} is not in the Copilot SDK provider models (models without a supported wire API are skipped).` + ); + continue; + } + model = qualified; + } + logger(`copilot sub-agent '${agent.name}': model '${configuredModel}' -> '${model}'`); + resolutions.push({ name: agent.name, configuredModel, resolvedModel, model }); + } + if (failures.length > 0) throw failures[0]; + return resolutions; +} + +/** + * CLI mode: resolve the sub-agent models of the agent files Copilot CLI loads from + * `/.github/agents` (repository agents and restored inline sub-agents) and write + * each resolved id back into the file's `model:` when it differs from the declared value. + * Stops the process before Copilot starts when a sub-agent model cannot be resolved. + * + * @param {{ + * reflectData: any, + * refetchReflectData?: () => Promise, + * agentsDir?: string, + * env?: NodeJS.ProcessEnv, + * logger?: (msg: string) => void, + * }} options + * @returns {Promise>} + */ +async function applyCopilotCLISubagentModels(options) { + const logger = options.logger || log; + const env = options.env || process.env; + const agentsDir = options.agentsDir || path.join(env.GITHUB_WORKSPACE || process.cwd(), ".github", "agents"); + const files = listCopilotAgentFiles(agentsDir, logger); + let resolutions; + try { + resolutions = await resolveCopilotSubagentModels({ agents: files, reflectData: options.reflectData, refetchReflectData: options.refetchReflectData, env, logger }); + } catch (failure) { + if (!(failure instanceof ModelAliasResolutionError)) throw failure; + process.exit(1); + return []; // unreachable, keeps TypeScript control-flow analysis happy + } + for (const resolution of resolutions) { + if (resolution.model === resolution.configuredModel) continue; + for (const file of files.filter(file => file.name === resolution.name && file.model === resolution.configuredModel)) { + rewriteCopilotAgentFileModel(file.path, resolution.model); + logger(`copilot sub-agent '${resolution.name}': wrote model '${resolution.model}' to ${file.path}`); + } + } + return resolutions; +} + /** * Configure and validate COPILOT_PROVIDER_WIRE_API using AWF endpoint metadata. * @@ -1112,10 +1315,11 @@ function detectCopilotErrors(output) { * resolvedModel: string, * routingEffort?: string|null, * multiProviderJson?: string, - * }} options + * subagentModelsJson?: string, + * }} options `subagentModelsJson` maps custom agent names to their resolved, qualified models. * @returns {NodeJS.ProcessEnv} */ -function buildCopilotSDKChildEnv({ sdkEnv, copilotSDKMode, copilotConnectionToken, providerBaseUrl, providerType, providerWireApi, resolvedModel, routingEffort, multiProviderJson }) { +function buildCopilotSDKChildEnv({ sdkEnv, copilotSDKMode, copilotConnectionToken, providerBaseUrl, providerType, providerWireApi, resolvedModel, routingEffort, multiProviderJson, subagentModelsJson }) { if (!copilotSDKMode) { return sdkEnv; } @@ -1128,6 +1332,7 @@ function buildCopilotSDKChildEnv({ sdkEnv, copilotSDKMode, copilotConnectionToke ...sdkEnv, COPILOT_CONNECTION_TOKEN: copilotConnectionToken, ...(multiProviderJson ? { GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON: multiProviderJson } : {}), + ...(subagentModelsJson ? { GH_AW_COPILOT_SDK_SUBAGENT_MODELS: subagentModelsJson } : {}), COPILOT_MODEL: qualifiedModel || resolvedModel, ...(routingEffort ? { COPILOT_REASONING_EFFORT: routingEffort } : {}), // Native Copilot CLI BYOK env vars — consumed by the headless sidecar for all sessions. @@ -1451,6 +1656,10 @@ async function main() { }); if (invocationModel?.hasOverride) resolvedArgs = applyCopilotInvocationModel(resolvedArgs, process.env.COPILOT_MODEL || ""); } + // Sub-agent models are resolved on routed runs too. + if (!copilotSDKMode) { + await applyCopilotCLISubagentModels({ reflectData: awfReflectData, refetchReflectData, logger: log }); + } if (!copilotSDKMode) { try { applyCopilotWireAPI({ @@ -1488,6 +1697,7 @@ async function main() { let providerWireApi = ""; let resolvedModel = ""; let multiProviderJson = ""; + let subagentModelsJson = ""; let primaryProviderName = ""; if (copilotSDKMode) { const configuredModel = modelRoutingSelection?.wire_model || process.env.COPILOT_MODEL || ""; @@ -1531,6 +1741,16 @@ async function main() { models: multiProvider.models, }); + // The same custom agents the SDK driver loads: workflow definitions override repository ones. + const agents = loadCopilotSDKCustomAgents(resolveCopilotSDKWorkingDirectory(), path.join(os.tmpdir(), "gh-aw"), log); + try { + const resolutions = await resolveCopilotSubagentModels({ agents, reflectData: awfReflectData, refetchReflectData, providerConfig: multiProvider, logger: log }); + subagentModelsJson = JSON.stringify(Object.fromEntries(resolutions.map(resolution => [resolution.name, resolution.model]))); + } catch (failure) { + if (!(failure instanceof ModelAliasResolutionError)) throw failure; + process.exit(1); + } + const uniqueProviderBaseUrls = [...new Set(multiProvider.providers.map(provider => String(provider.baseUrl || "").trim()).filter(Boolean))]; if (uniqueProviderBaseUrls.length === 0) { log("copilot-sdk driver mode: no provider baseUrls to probe — skipping listener readiness check"); @@ -1580,6 +1800,7 @@ async function main() { resolvedModel, routingEffort: modelRoutingSelection?.effort ?? null, multiProviderJson, + subagentModelsJson, }); const childEnv = Object.keys(sdkChildEnv).length > 0 ? { ...process.env, ...sdkChildEnv } : undefined; @@ -2165,6 +2386,10 @@ if (typeof module !== "undefined" && module.exports) { applyModelFallback, applyCopilotModelAliasResolution, resolveCopilotSDKMultiProvider, + resolveCopilotSubagentModels, + applyCopilotCLISubagentModels, + listCopilotAgentFiles, + rewriteCopilotAgentFileModel, applyCopilotWireAPI, warnCopilotSubagentEndpoints, resolveAWFModelRoutingSelection, diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index dd52c2ff04c..24e8d64c92f 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -74,6 +74,8 @@ const { warnCopilotSubagentEndpoints, applyCopilotModelAliasResolution, resolveCopilotSDKMultiProvider, + resolveCopilotSubagentModels, + applyCopilotCLISubagentModels, formatInferenceEndpointForLog, logCopilotInferenceConfiguration, resolveLongRunTokenThreshold, @@ -4653,3 +4655,338 @@ require(${JSON.stringify(require.resolve("./awf_reflect.cjs"))}).fetchAWFReflect }); }); }); + +describe("Copilot sub-agent model resolution (recorded /reflect from run 39304573724)", () => { + // Agent artifact of https://github.com/github/gh-aw/actions/runs/39304573724 (smoke-copilot-sub-agents, SDK mode): + // sandbox/firewall/awf-reflect.json (models_fetch_complete true, one configured Copilot endpoint with 38 models, + // no gpt-5.4-mini) and awf-config.json apiProxy.models (the workflow's alias map). That run silently served the + // mini54-whoami sub-agent (gpt-5.4-mini) on the main model gpt-5.3-codex (gh-aw#70388). + const fixtureDir = path.join(path.dirname(require.resolve("./copilot_harness.cjs")), "fixtures"); + const reflectPath = path.join(fixtureDir, "awf-reflect-copilot-sub-agents-39304573724.json"); + const awfConfigFixturePath = path.join(fixtureDir, "awf-config-models-copilot-sub-agents-39304573724.json"); + const recordedReflect = () => JSON.parse(fs.readFileSync(reflectPath, "utf8")); + const aliasMap = () => JSON.parse(fs.readFileSync(awfConfigFixturePath, "utf8")).apiProxy.models; + const { ModelAliasResolutionError } = require("./resolve_model_alias.cjs"); + const { resolveMultiProviderFromReflect } = require("./awf_reflect.cjs"); + const { qualifyModelForMultiProvider } = require("./copilot_sdk_multi_provider.cjs"); + const tempDirs = []; + + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); + }); + + function tempDir(prefix) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; + } + + const readRecords = dir => { + const file = path.join(dir, "agent/model-resolution.json"); + return fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : null; + }; + + /** In-process: the harness sub-agent resolution step against the recorded catalogue. */ + async function resolveAgents(agents, { reflect = recordedReflect(), refreshed = [], providerConfig = null } = {}) { + const dir = tempDir("copilot-subagent-models-"); + const env = { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1" }; + const logs = []; + const refetch = vi.fn().mockResolvedValue(null); + for (const data of refreshed) refetch.mockResolvedValueOnce(data); + let resolutions = null; + let error = null; + try { + resolutions = await resolveCopilotSubagentModels({ agents, reflectData: reflect, refetchReflectData: refetch, providerConfig, aliasMap: aliasMap(), env, logger: message => logs.push(message) }); + } catch (err) { + error = err; + } + return { resolutions, error, refetch, logs, records: readRecords(dir) }; + } + + const sdkProviderConfig = () => resolveMultiProviderFromReflect({ model: "gpt-5.3-codex", reflectData: recordedReflect() }); + + it("case 1: model small resolves through the gh-aw alias to claude-haiku-5.5, not the main model", async () => { + const run = await resolveAgents([{ name: "haiku-small", model: "small" }]); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([{ name: "haiku-small", configuredModel: "small", resolvedModel: "claude-haiku-5.5", model: "claude-haiku-5.5" }]); + expect(run.records).toEqual([{ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }]); + }); + + it("case 1b: model large resolves through the gh-aw alias to claude-sonnet-5.5", async () => { + const run = await resolveAgents([{ name: "proposer", model: "large" }]); + expect(run.error).toBeNull(); + expect(run.resolutions[0].model).toBe("claude-sonnet-5.5"); + }); + + it("case 2: a live exact id claude-haiku-4.5 is unchanged", async () => { + const run = await resolveAgents([{ name: "haiku-whoami", model: "claude-haiku-4.5" }]); + expect(run.error).toBeNull(); + expect(run.refetch).not.toHaveBeenCalled(); + expect(run.resolutions).toEqual([{ name: "haiku-whoami", configuredModel: "claude-haiku-4.5", resolvedModel: "claude-haiku-4.5", model: "claude-haiku-4.5" }]); + expect(run.records).toEqual([{ engine: "copilot", role: "haiku-whoami", configured_model: "claude-haiku-4.5", resolved_model: "claude-haiku-4.5", provider: "copilot", status: "exact", cause: null }]); + }); + + it("case 3: a dead id gpt-5.4-mini fails closed with model_unavailable after one refresh", async () => { + const run = await resolveAgents( + [ + { name: "haiku-whoami", model: "claude-haiku-4.5" }, + { name: "mini54-whoami", model: "gpt-5.4-mini" }, + { name: "other-mini54", model: "gpt-5.4-mini" }, + ], + { refreshed: [recordedReflect()] } + ); + expect(run.error).toBeInstanceOf(ModelAliasResolutionError); + expect(run.error.reason).toBe("model_unavailable"); + expect(run.error.message).toContain("The requested model is not supported: 'gpt-5.4-mini' is not available in the workflow's live model catalog"); + expect(run.refetch).toHaveBeenCalledTimes(1); + expect(run.resolutions).toBeNull(); + const failure = run.logs.find(message => message.startsWith("copilot sub-agent 'mini54-whoami':")); + expect(failure).toContain("Refusing to start Copilot with a different sub-agent model."); + expect(detectErrors(failure).modelNotSupportedError).toBe(true); + // Every sub-agent sharing the dead model gets its own failed record. + expect(run.records).toEqual([ + { engine: "copilot", role: "haiku-whoami", configured_model: "claude-haiku-4.5", resolved_model: "claude-haiku-4.5", provider: "copilot", status: "exact", cause: null }, + { engine: "copilot", role: "mini54-whoami", configured_model: "gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }, + { engine: "copilot", role: "other-mini54", configured_model: "gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }, + ]); + }); + + it("case 3b: medium is not a gh-aw alias and fails closed as a literal id", async () => { + const run = await resolveAgents([{ name: "workflow-agent", model: "medium" }], { refreshed: [recordedReflect()] }); + expect(run.error).toBeInstanceOf(ModelAliasResolutionError); + expect(run.error.reason).toBe("model_unavailable"); + expect(run.records).toEqual([{ engine: "copilot", role: "workflow-agent", configured_model: "medium", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }]); + }); + + it("case 3c: a dead id starts when the bounded refresh lists it", async () => { + const refreshed = recordedReflect(); + refreshed.endpoints.find(endpoint => endpoint.provider === "copilot").models.push("gpt-5.4-mini"); + const run = await resolveAgents([{ name: "mini54-whoami", model: "gpt-5.4-mini" }], { refreshed: [refreshed] }); + expect(run.error).toBeNull(); + expect(run.refetch).toHaveBeenCalledTimes(1); + expect(run.resolutions[0].model).toBe("gpt-5.4-mini"); + }); + + it("case 4: the qualified alias copilot/opus resolves to the newest live Opus", async () => { + const run = await resolveAgents([{ name: "reviewer", model: "copilot/opus" }]); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([{ name: "reviewer", configuredModel: "copilot/opus", resolvedModel: "claude-opus-5.5", model: "claude-opus-5.5" }]); + expect(run.records).toEqual([{ engine: "copilot", role: "reviewer", configured_model: "copilot/opus", resolved_model: "claude-opus-5.5", provider: "copilot", status: "resolved", cause: null }]); + }); + + it("case 5: a sub-agent without model inherits the main model and gets no record", async () => { + const run = await resolveAgents([{ name: "helper" }]); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([]); + expect(run.records).toBeNull(); + }); + + it("case 6 (SDK): customAgents[].model is the qualified resolved model, not the session model", async () => { + const providerConfig = sdkProviderConfig(); + const expected = qualifyModelForMultiProvider("claude-haiku-5.5", providerConfig); + expect(expected).toMatch(/^copilot-(?:responses|completions)\/claude-haiku-5\.5$/); + const run = await resolveAgents([{ name: "haiku-small", model: "small" }, { name: "helper" }], { providerConfig }); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([{ name: "haiku-small", configuredModel: "small", resolvedModel: "claude-haiku-5.5", model: expected }]); + + // The harness hands the resolved models to the SDK driver, which passes them to the session. + const { runWithCopilotSDK } = require("./copilot_sdk_session.cjs"); + const base = tempDir("copilot-subagent-sdk-session-"); + fs.mkdirSync(path.join(base, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(base, ".github/agents/haiku-small.agent.md"), "---\nmodel: small\n---\nWho are you?"); + fs.writeFileSync(path.join(base, ".github/agents/helper.agent.md"), "---\ndescription: helper\n---\nHelp."); + vi.stubEnv("GH_AW_SESSION_STATE_BASE_DIR", base); + vi.stubEnv("COPILOT_MODEL", ""); + vi.stubEnv("GH_AW_MODEL_FALLBACK", ""); + vi.stubEnv("GH_AW_MODEL_ROUTING", ""); + vi.stubEnv("GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON", JSON.stringify(providerConfig)); + vi.stubEnv("GH_AW_COPILOT_SDK_SUBAGENT_MODELS", JSON.stringify(Object.fromEntries(run.resolutions.map(resolution => [resolution.name, resolution.model])))); + const createSession = vi.fn().mockResolvedValue({ sessionId: "subagent-models", on: () => {}, sendAndWait: async () => ({ data: { content: "done" } }), disconnect: async () => {} }); + class FakeCopilotClient { + start = async () => {}; + stop = async () => {}; + createSession = createSession; + } + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test", + logger: () => {}, + workingDirectory: base, + agentsBaseDir: base, + bare: true, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: () => ({}) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(0); + const session = createSession.mock.calls[0][0]; + expect(session.model).toBe(providerConfig.model); + expect(session.customAgents.find(agent => agent.name === "haiku-small").model).toBe(expected); + expect(session.customAgents.find(agent => agent.name === "haiku-small").model).not.toBe(session.model); + expect(session.customAgents.find(agent => agent.name === "helper").model).toBe(session.model); + }); + + it("case 6 (CLI): the agent file Copilot CLI loads gets the resolved model; exact ids are left untouched", async () => { + const dir = tempDir("copilot-subagent-cli-files-"); + const agentsDir = path.join(dir, ".github/agents"); + fs.mkdirSync(agentsDir, { recursive: true }); + const small = "---\nname: haiku-small\ndescription: Small model\nmodel: small # alias\ntools: [read]\n---\nWho are you?\nmodel: small\n"; + const exact = "---\nmodel: claude-haiku-4.5\n---\nExact.\n"; + fs.writeFileSync(path.join(agentsDir, "haiku-small.agent.md"), small); + fs.writeFileSync(path.join(agentsDir, "haiku-whoami.agent.md"), exact); + const awfConfigPath = path.join(dir, "awf-config.json"); + fs.copyFileSync(awfConfigFixturePath, awfConfigPath); + const resolutions = await applyCopilotCLISubagentModels({ reflectData: recordedReflect(), agentsDir, env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, logger: () => {} }); + expect(resolutions.map(resolution => [resolution.name, resolution.model])).toEqual([ + ["haiku-small", "claude-haiku-5.5"], + ["haiku-whoami", "claude-haiku-4.5"], + ]); + expect(fs.readFileSync(path.join(agentsDir, "haiku-small.agent.md"), "utf8")).toBe(small.replace("model: small # alias", 'model: "claude-haiku-5.5"')); + expect(fs.readFileSync(path.join(agentsDir, "haiku-whoami.agent.md"), "utf8")).toBe(exact); + }); + + /** Spawn the harness against the recorded catalogue with the given workspace agent files. */ + async function runHarness({ sdk = false, agents, reflect = recordedReflect(), env: envOverrides = {} }) { + const dir = tempDir("copilot-subagent-harness-"); + const workspace = path.join(dir, "workspace"); + fs.mkdirSync(path.join(workspace, ".github/agents"), { recursive: true }); + for (const [name, content] of Object.entries(agents)) fs.writeFileSync(path.join(workspace, ".github/agents", name), content); + const ostmp = path.join(dir, "ostmp"); + fs.mkdirSync(ostmp); + const startedPath = path.join(dir, "copilot-started.json"); + let providerPort = 0; + let sdkPort = 0; + if (sdk) { + const freePort = async () => { + const server = require("net").createServer(); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address(); + await new Promise(resolve => server.close(resolve)); + return port; + }; + [providerPort, sdkPort] = [await freePort(), await freePort()]; + for (const endpoint of reflect.endpoints.filter(endpoint => endpoint.configured)) Object.assign(endpoint, { base_url: `http://127.0.0.1:${providerPort}`, models_url: `http://127.0.0.1:${providerPort}/models` }); + } + fs.writeFileSync(path.join(dir, "reflect.json"), JSON.stringify(reflect)); + const preloadPath = path.join(dir, "reflect-preload.cjs"); + fs.writeFileSync( + preloadPath, + `const body = require("fs").readFileSync(${JSON.stringify(path.join(dir, "reflect.json"))}, "utf8"); +require(${JSON.stringify(require.resolve("./awf_reflect.cjs"))}).fetchAWFReflect = async () => ({ ok: true, reflectData: JSON.parse(body) }); +if (${providerPort} && process.argv[1].endsWith("copilot_harness.cjs")) require("net").createServer(socket => socket.end()).listen(${providerPort}, "127.0.0.1").unref();` + ); + const stubPath = path.join(dir, "copilot-stub.cjs"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const path = require("path"); +const args = process.argv.slice(2); +fs.writeFileSync(${JSON.stringify(startedPath)}, JSON.stringify({ args, subagentModels: process.env.GH_AW_COPILOT_SDK_SUBAGENT_MODELS || null })); +if (args.includes("--session-id")) { + const model = args.includes("--model") ? args[args.lastIndexOf("--model") + 1] : process.env.COPILOT_MODEL; + const sessionDir = path.join(process.env.COPILOT_HOME, "session-state", args[args.indexOf("--session-id") + 1]); + fs.mkdirSync(sessionDir, { recursive: true }); + fs.appendFileSync(path.join(sessionDir, "events.jsonl"), JSON.stringify({ type: "assistant.message", data: { model, content: "done" } }) + "\\n"); +} +process.stdout.write("done\\n");` + ); + const sidecarPath = path.join(dir, "copilot-sidecar.cjs"); + fs.writeFileSync(sidecarPath, `#!/usr/bin/env node\nconst args = process.argv.slice(2);\nrequire("net").createServer(socket => socket.end()).listen(Number(args[args.indexOf("--port") + 1]), "127.0.0.1");\n`, { mode: 0o755 }); + const awfConfigPath = path.join(dir, "awf-config.json"); + fs.copyFileSync(awfConfigFixturePath, awfConfigPath); + const env = { + ...harnessChildEnv, + GH_AW_TMP_DIR: dir, + TMPDIR: ostmp, + GITHUB_WORKSPACE: workspace, + COPILOT_HOME: path.join(dir, "copilot-home"), + GH_AW_HARNESS_MAX_RETRIES: "0", + GH_AW_HARNESS_STALL_WARNING_MS: "0", + NODE_OPTIONS: `--require ${preloadPath}`, + }; + for (const key of [ + "COPILOT_MODEL", + "GH_AW_MODEL_FALLBACK", + "GH_AW_MODEL_ROUTING", + "GH_AW_SKIP_REFLECT", + "COPILOT_SDK_URI", + "GH_AW_PHASE", + "GH_AW_SAFE_OUTPUTS", + "COPILOT_PROVIDER_WIRE_API", + "GH_AW_MODEL_ALIASES", + "GH_AW_ENGINE_CWD", + "GH_AW_SUB_AGENT_DIR", + "GH_AW_SUB_AGENT_EXT", + "GH_AW_COPILOT_SDK_SUBAGENT_MODELS", + ]) + delete env[key]; + Object.assign(env, { AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath, COPILOT_MODEL: "gpt-5.3-codex", ...(sdk ? { COPILOT_SDK_URI: `http://127.0.0.1:${sdkPort}` } : {}) }, envOverrides); + const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, ...(sdk ? [stubPath, sidecarPath] : [stubPath, "--prompt", "Hello"])], { + cwd: path.dirname(require.resolve("./copilot_harness.cjs")), + env, + encoding: "utf8", + timeout: 20000, + }); + return { + result, + workspace, + started: fs.existsSync(startedPath) ? JSON.parse(fs.readFileSync(startedPath, "utf8")) : null, + records: readRecords(dir), + agentFile: name => fs.readFileSync(path.join(workspace, ".github/agents", name), "utf8"), + }; + } + + const smokeAgents = { + "haiku-whoami.agent.md": "---\nname: haiku-whoami\nmodel: claude-haiku-4.5\n---\nWho are you?\n", + "mini54-whoami.agent.md": "---\nname: mini54-whoami\nmodel: gpt-5.4-mini\n---\nWho are you?\n", + }; + + it.each([false, true])("case 3 end to end: a dead sub-agent model stops the harness before Copilot starts (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, agents: smokeAgents }); + expect(run.result.status, run.result.stderr).toBe(1); + expect(run.started).toBeNull(); + expect(run.result.stderr).not.toContain("starting sidecar"); + expect(run.result.stderr).toContain("copilot sub-agent 'mini54-whoami': The requested model is not supported: 'gpt-5.4-mini' is not available in the workflow's live model catalog"); + expect(detectErrors(run.result.stderr).modelNotSupportedError).toBe(true); + expect(run.records).toEqual([ + { engine: "copilot", role: "main", configured_model: "gpt-5.3-codex", resolved_model: "gpt-5.3-codex", provider: null, status: "exact", cause: null }, + { engine: "copilot", role: "haiku-whoami", configured_model: "claude-haiku-4.5", resolved_model: "claude-haiku-4.5", provider: "copilot", status: "exact", cause: null }, + { engine: "copilot", role: "mini54-whoami", configured_model: "gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }, + ]); + expect(run.agentFile("mini54-whoami.agent.md")).toBe(smokeAgents["mini54-whoami.agent.md"]); + }); + + it("case 6 end to end (CLI): Copilot CLI starts with the resolved model in the agent file", async () => { + const run = await runHarness({ agents: { "haiku-small.agent.md": "---\nname: haiku-small\nmodel: small\n---\nWho are you?\n" } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started).not.toBeNull(); + expect(run.agentFile("haiku-small.agent.md")).toBe('---\nname: haiku-small\nmodel: "claude-haiku-5.5"\n---\nWho are you?\n'); + expect(run.records).toContainEqual({ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }); + }); + + it("case 6 end to end (SDK): the driver receives the qualified resolved sub-agent model", async () => { + const run = await runHarness({ sdk: true, agents: { "haiku-small.agent.md": "---\nname: haiku-small\nmodel: small\n---\nWho are you?\n" } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(JSON.parse(run.started.subagentModels)).toEqual({ "haiku-small": qualifyModelForMultiProvider("claude-haiku-5.5", sdkProviderConfig()) }); + expect(run.agentFile("haiku-small.agent.md")).toBe("---\nname: haiku-small\nmodel: small\n---\nWho are you?\n"); + }); + + describe("case 7: routed runs (recorded /reflect routing from run 39147795806)", () => { + // fixtures/copilot-routed-run-39147795806/awf-reflect.json: the router selected gpt-5.6-luna; its Copilot + // catalogue lists claude-haiku-5.5 but no claude-opus-4.8. + const routedReflect = () => JSON.parse(fs.readFileSync(path.join(fixtureDir, "copilot-routed-run-39147795806", "awf-reflect.json"), "utf8")); + + it.each([false, true])("still resolves and records sub-agent models without a main-role record (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, reflect: routedReflect(), agents: { "haiku-small.agent.md": "---\nmodel: small\n---\nWho are you?\n" }, env: { GH_AW_MODEL_ROUTING: "1", COPILOT_MODEL: "" } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.records).toEqual([{ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }]); + }); + + it.each([false, true])("still fails closed for a dead sub-agent model (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, reflect: routedReflect(), agents: { "opus.agent.md": "---\nmodel: claude-opus-4.8\n---\nWho are you?\n" }, env: { GH_AW_MODEL_ROUTING: "1", COPILOT_MODEL: "" } }); + expect(run.result.status, run.result.stderr).toBe(1); + expect(run.started).toBeNull(); + expect(run.records).toEqual([{ engine: "copilot", role: "opus", configured_model: "claude-opus-4.8", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }]); + }); + }); +}); diff --git a/actions/setup/js/copilot_sdk_driver.cjs b/actions/setup/js/copilot_sdk_driver.cjs index f8f6a631b80..700f4d99b1e 100644 --- a/actions/setup/js/copilot_sdk_driver.cjs +++ b/actions/setup/js/copilot_sdk_driver.cjs @@ -13,6 +13,7 @@ * COPILOT_MODEL — model override (optional) * GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON — JSON-encoded multi-provider config (required). * Shape: { model, providers: NamedProviderConfig[], models: ProviderModelConfig[] } + * GH_AW_COPILOT_SDK_SUBAGENT_MODELS — JSON map of custom agent name → model resolved by the harness (optional) * GH_AW_COPILOT_SDK_SERVER_ARGS — JSON-encoded allow-tool sidecar args (set by the engine) * * The sidecar is started and stopped by the harness; the driver only opens a diff --git a/actions/setup/js/copilot_sdk_driver.test.cjs b/actions/setup/js/copilot_sdk_driver.test.cjs index a7be1d46a6f..bf210d622a0 100644 --- a/actions/setup/js/copilot_sdk_driver.test.cjs +++ b/actions/setup/js/copilot_sdk_driver.test.cjs @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, vi, beforeAll, afterAll, afterEach } from "vitest"; import { createRequire } from "module"; import * as fs from "fs"; import * as path from "path"; @@ -31,6 +31,10 @@ describe("copilot_sdk_driver.cjs", () => { }); describe("runWithCopilotSDK", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it.each([ ["gpt-5.4", "claude-sonnet-4.6", "copilot-responses", "copilot-completions"], ["claude-sonnet-4.6", "gpt-5.4", "copilot-completions", "copilot-responses"], @@ -43,9 +47,10 @@ describe("copilot_sdk_driver.cjs", () => { path.join(workspace, ".github/agents/researcher.agent.md"), `---\nname: researcher\ndescription: >-\n Research across\n providers\ntools:\n - read\n - grep\nmodel: copilot/${agentModel}\n---\nResearch carefully.\n` ); - fs.writeFileSync(path.join(workflow, ".github/agents/workflow.agent.md"), "---\nname: workflow-agent\nmodel: medium\ntools: read, grep\n---\nWorkflow prompt."); + fs.writeFileSync(path.join(workflow, ".github/agents/workflow.agent.md"), "---\nname: workflow-agent\ntools: read, grep\n---\nWorkflow prompt."); for (const alias of ["small", "large"]) fs.writeFileSync(path.join(workflow, `.github/agents/${alias}.agent.md`), `---\nmodel: ${alias}\n---\nAlias prompt.`); - fs.writeFileSync(path.join(workflow, ".github/agents/unknown.agent.md"), "---\nname: unknown-agent\nmodel: unavailable-model\n---\nKeep this agent."); + // The harness resolves aliases before the driver starts; the driver uses those models, not the session model. + vi.stubEnv("GH_AW_COPILOT_SDK_SUBAGENT_MODELS", JSON.stringify({ small: `${agentProvider}/${agentModel}`, large: agentModel })); const createSession = vi.fn().mockResolvedValue({ sessionId: `${sessionProvider}-agent-session`, on: () => {}, @@ -90,21 +95,39 @@ describe("copilot_sdk_driver.cjs", () => { customAgents: expect.arrayContaining([ { name: "researcher", displayName: "researcher", description: "Research across providers", tools: ["read", "grep"], prompt: "Research carefully.", model: `${agentProvider}/${agentModel}` }, { name: "workflow-agent", displayName: "workflow-agent", tools: ["read", "grep"], prompt: "Workflow prompt.", model: `${sessionProvider}/${sessionModel}` }, - { name: "unknown-agent", displayName: "unknown-agent", prompt: "Keep this agent.", model: `${sessionProvider}/${sessionModel}` }, - { name: "small", displayName: "small", prompt: "Alias prompt.", model: `${sessionProvider}/${sessionModel}` }, - { name: "large", displayName: "large", prompt: "Alias prompt.", model: `${sessionProvider}/${sessionModel}` }, + { name: "small", displayName: "small", prompt: "Alias prompt.", model: `${agentProvider}/${agentModel}` }, + { name: "large", displayName: "large", prompt: "Alias prompt.", model: `${agentProvider}/${agentModel}` }, ]), }) ); - expect(logger).toHaveBeenCalledWith(expect.stringMatching(/warning:.*unknown-agent.*unavailable-model.*session model/i)); - const captured = fs - .readFileSync(path.join(testSessionStateDir, `${sessionProvider}-agent-session/events.jsonl`), "utf8") - .trim() - .split("\n") - .map(JSON.parse); - expect(captured.filter(event => event.type === "subagent.model_unavailable")).toEqual([ - expect.objectContaining({ data: { agentName: "unknown-agent", declaredModel: "unavailable-model", model: `${sessionProvider}/${sessionModel}` } }), - ]); + }); + + it.each(["medium", "unavailable-model"])("fails closed for an unresolved custom agent model %s instead of using the session model", async agentModel => { + const workflow = path.join(testSessionStateDir, `unresolved-${agentModel}-workflow`); + fs.mkdirSync(path.join(workflow, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(workflow, ".github/agents/unknown.agent.md"), `---\nname: unknown-agent\nmodel: ${agentModel}\n---\nUnknown model.`); + const createSession = vi.fn(); + class FakeCopilotClient { + start = vi.fn().mockResolvedValue(undefined); + createSession = createSession; + stop = vi.fn().mockResolvedValue(undefined); + } + const logger = vi.fn(); + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test prompt", + model: "copilot/gpt-5.4", + models: [{ id: "gpt-5.4", provider: "copilot-responses" }], + providers: [{ name: "copilot-responses", type: "openai", baseUrl: "http://api-proxy:10002", wireApi: "responses" }], + workingDirectory: workflow, + agentsBaseDir: workflow, + bare: true, + logger, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: vi.fn(() => ({})) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`Cannot qualify custom agent "unknown-agent" model "${agentModel}"`); + expect(createSession).not.toHaveBeenCalled(); }); it("captures ephemeral selection and final model results including BYOK and subagent context", async () => { diff --git a/actions/setup/js/copilot_sdk_session.cjs b/actions/setup/js/copilot_sdk_session.cjs index 133ae62c406..aabbbd641cb 100644 --- a/actions/setup/js/copilot_sdk_session.cjs +++ b/actions/setup/js/copilot_sdk_session.cjs @@ -111,6 +111,20 @@ function resolveCopilotSDKWorkingDirectory() { return path.resolve(process.env.GITHUB_WORKSPACE || process.cwd(), process.env.GH_AW_ENGINE_CWD || "."); } +/** + * Parse GH_AW_COPILOT_SDK_SUBAGENT_MODELS: custom agent name → model resolved by the harness. + * @param {string | undefined} value + * @returns {Record} + */ +function parseSubagentModelsJson(value) { + if (!value) return {}; + const parsed = JSON.parse(value); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || !Object.values(parsed).every(model => typeof model === "string")) { + throw new Error("GH_AW_COPILOT_SDK_SUBAGENT_MODELS must map custom agent names to models"); + } + return parsed; +} + /** * Run a Copilot agentic session using the @github/copilot-sdk. * @@ -369,16 +383,14 @@ async function runWithCopilotSDK({ const providerConfig = providers && providerModels ? { providers, models: providerModels } : null; const sessionModel = providerConfig ? qualifyModelForMultiProvider(requestedModel, providerConfig) : requestedModel; if (providerConfig && !sessionModel) throw new Error(`Cannot qualify session model "${requestedModel || "(none)"}" against the configured provider models`); - const unavailableModels = []; + // The harness resolves declared sub-agent models (aliases included) before the driver starts. + const resolvedSubagentModels = parseSubagentModelsJson(process.env.GH_AW_COPILOT_SDK_SUBAGENT_MODELS); const customAgents = loadCopilotSDKCustomAgents(workingDirectory, agentsBaseDir, log).map(agent => { - const requested = agent.model; - const alias = !requested || ["small", "medium", "large"].includes(requested.toLowerCase()); - const qualified = alias ? sessionModel : providerConfig ? qualifyModelForMultiProvider(requested, providerConfig) : requested; - if (!alias && !qualified) { - log(`warning: custom agent "${agent.name}" model "${requested}" is not in the configured provider models; using session model "${sessionModel || "(none)"}"`); - unavailableModels.push({ agentName: agent.name, declaredModel: requested, model: sessionModel || undefined }); - } - return { ...agent, model: qualified || sessionModel || undefined }; + if (!agent.model) return { ...agent, model: sessionModel || undefined }; + const requested = Object.prototype.hasOwnProperty.call(resolvedSubagentModels, agent.name) ? resolvedSubagentModels[agent.name] : agent.model; + const qualified = providerConfig ? qualifyModelForMultiProvider(requested, providerConfig) : requested; + if (!qualified) throw new Error(`Cannot qualify custom agent "${agent.name}" model "${agent.model}" against the configured provider models`); + return { ...agent, model: qualified }; }); const skillsDirectory = path.resolve(agentsBaseDir, ".github", "skills"); const skillsStat = fs.lstatSync(skillsDirectory, { throwIfNoEntry: false }); @@ -410,7 +422,6 @@ async function runWithCopilotSDK({ fs.mkdirSync(sessionDir, { recursive: true }); const eventsPath = path.join(sessionDir, "events.jsonl"); eventsStream = fs.createWriteStream(eventsPath, { flags: "a" }); - for (const unavailable of unavailableModels) writeDriverEvent("subagent.model_unavailable", unavailable); // Snapshot to a non-null local for closure-safe writes (JSDoc nullability narrowing). const stream = eventsStream; const assistantEvents = []; diff --git a/actions/setup/js/copilot_sdk_session.test.cjs b/actions/setup/js/copilot_sdk_session.test.cjs index 30d231d6e7f..40576768802 100644 --- a/actions/setup/js/copilot_sdk_session.test.cjs +++ b/actions/setup/js/copilot_sdk_session.test.cjs @@ -141,6 +141,8 @@ describe("copilot_sdk_session runtime configuration", () => { const app = path.join(base, "packages/app"); fs.mkdirSync(path.join(app, ".github/agents"), { recursive: true }); fs.writeFileSync(path.join(app, ".github/agents/local.agent.md"), "---\nmodel: small\n---\nApp agent."); + // The harness resolved `small` before the driver started. + vi.stubEnv("GH_AW_COPILOT_SDK_SUBAGENT_MODELS", JSON.stringify({ local: "claude-sonnet-4.6" })); vi.stubEnv("GITHUB_WORKSPACE", base); vi.spyOn(process, "cwd").mockReturnValue(app); const createSession = vi.fn().mockResolvedValue({ diff --git a/actions/setup/js/fixtures/awf-config-models-copilot-sub-agents-39304573724.json b/actions/setup/js/fixtures/awf-config-models-copilot-sub-agents-39304573724.json new file mode 100644 index 00000000000..953d043176d --- /dev/null +++ b/actions/setup/js/fixtures/awf-config-models-copilot-sub-agents-39304573724.json @@ -0,0 +1,67 @@ +{ + "apiProxy": { + "models": { + "agent": ["sonnet-6x", "gpt-6", "gpt-5.4", "gpt-5.5", "gpt-5.6", "gpt-5.3", "gemini-pro", "any"], + "antigravity": ["copilot/antigravity*", "google/antigravity*", "gemini/antigravity*"], + "any": ["copilot/*", "anthropic/*", "openai/*", "google/*", "gemini/*"], + "auto": ["copilot/auto", "large"], + "claude": ["agent"], + "codex": ["agent"], + "coding": ["copilot/gpt-5*codex*", "openai/gpt-5*codex*", "gpt-5-codex", "kimi"], + "computer-use": ["copilot/*computer-use*", "google/*computer-use*", "gemini/*computer-use*", "openai/*computer-use*"], + "copilot": ["agent"], + "deep-research": ["copilot/deep-research*", "copilot/o3-deep-research*", "copilot/o4-mini-deep-research*", "google/deep-research*", "gemini/deep-research*", "openai/o3-deep-research*", "openai/o4-mini-deep-research*"], + "detection": ["small"], + "evals": ["small"], + "fable": ["copilot/*fable*", "anthropic/*fable*"], + "gemini": ["agent"], + "gemini-3-flash": ["copilot/gemini-3*flash*", "google/gemini-3*flash*", "gemini/gemini-3*flash*"], + "gemini-3-pro": ["copilot/gemini-3*pro*", "google/gemini-3*pro*", "google/nano-banana*", "gemini/gemini-3*pro*"], + "gemini-3.1-flash": ["copilot/gemini-3.1*flash*", "google/gemini-3.1*flash*", "gemini/gemini-3.1*flash*"], + "gemini-3.1-pro": ["copilot/gemini-3.1*pro*", "google/gemini-3.1*pro*", "gemini/gemini-3.1*pro*"], + "gemini-3.5-flash": ["copilot/gemini-3.5*flash*", "google/gemini-3.5*flash*", "gemini/gemini-3.5*flash*"], + "gemini-3.6-flash": ["copilot/gemini-3.6*flash*", "google/gemini-3.6*flash*", "gemini/gemini-3.6*flash*"], + "gemini-3.7-flash": ["copilot/gemini-3.7*flash*", "google/gemini-3.7*flash*", "gemini/gemini-3.7*flash*"], + "gemini-3.8-flash": ["copilot/gemini-3.8*flash*", "google/gemini-3.8*flash*", "gemini/gemini-3.8*flash*"], + "gemini-flash": ["copilot/gemini-*flash*", "google/gemini-*flash*", "gemini/gemini-*flash*"], + "gemini-flash-lite": ["copilot/gemini-*flash*lite*", "google/gemini-*flash*lite*", "gemini/gemini-*flash*lite*"], + "gemini-omni": ["copilot/gemini-omni*", "google/gemini-omni*", "gemini/gemini-omni*"], + "gemini-pro": ["copilot/gemini-*pro*", "google/gemini-*pro*", "gemini/gemini-*pro*"], + "gemma": ["copilot/gemma*", "google/gemma*", "gemini/gemma*"], + "gpt-5": ["copilot/gpt-5*", "openai/gpt-5*"], + "gpt-5-codex": ["copilot/gpt-5*codex*", "openai/gpt-5*codex*"], + "gpt-5-mini": ["copilot/gpt-5*mini*", "openai/gpt-5*mini*"], + "gpt-5-nano": ["copilot/gpt-5*nano*", "openai/gpt-5*nano*"], + "gpt-5-pro": ["copilot/gpt-5*pro*", "openai/gpt-5*pro*"], + "gpt-5.1": ["copilot/gpt-5.1*", "openai/gpt-5.1*"], + "gpt-5.2": ["copilot/gpt-5.2*", "openai/gpt-5.2*"], + "gpt-5.3": ["copilot/gpt-5.3*", "openai/gpt-5.3*"], + "gpt-5.4": ["copilot/gpt-5.4*", "openai/gpt-5.4*"], + "gpt-5.5": ["copilot/gpt-5.5*", "openai/gpt-5.5*"], + "gpt-5.6": ["copilot/gpt-5.6*", "openai/gpt-5.6*"], + "gpt-6": ["copilot/gpt-6*", "openai/gpt-6*"], + "grok": ["copilot/*grok*", "openai/*grok*"], + "haiku": ["copilot/*haiku*", "anthropic/*haiku*"], + "image-generation": ["copilot/gpt-image*", "openai/gpt-image*", "openai/chatgpt-image*", "copilot/gemini-*image*", "google/gemini-*image*", "gemini/gemini-*image*", "google/imagen*"], + "kimi": ["copilot/kimi*", "openai/kimi*"], + "kiwi": ["copilot/kiwi*", "openai/kiwi*"], + "large": ["sonnet", "gpt-6", "gpt-5-pro", "gpt-5", "gemini-pro"], + "lyria": ["google/lyria*", "gemini/lyria*", "copilot/lyria*"], + "mai-code": ["copilot/MAI-Code*", "copilot/mai-code*", "openai/MAI-Code*"], + "mini": ["haiku", "gpt-5-mini", "gpt-5-nano", "gemini-flash-lite"], + "nano-banana": ["copilot/nano-banana*", "google/nano-banana*", "gemini/nano-banana*"], + "opus": ["copilot/*opus*", "anthropic/*opus*"], + "opusplan": ["opus?effort=high"], + "raptor-mini": ["copilot/raptor*", "openai/raptor*"], + "reasoning": ["copilot/o1*", "copilot/o3*", "copilot/o4*", "openai/o1*", "openai/o3*", "openai/o4*"], + "robotics": ["copilot/*robotics*", "google/*robotics*", "gemini/*robotics*"], + "small": ["mini"], + "small-agent": ["haiku", "gpt-5-mini", "gemini-flash"], + "sonnet": ["copilot/*sonnet*", "anthropic/*sonnet*"], + "sonnet-6x": ["copilot/*sonnet-4.5*", "copilot/*sonnet-4.6*", "copilot/*sonnet-5*", "copilot/*sonnet-4-5-*", "anthropic/*sonnet-4-5-*", "copilot/*sonnet-4-6*", "anthropic/*sonnet-4-6*", "anthropic/*sonnet-5*"], + "summarization": ["haiku", "gpt-5-mini", "gemini-flash-lite", "mini"], + "veo": ["google/veo*", "gemini/veo*"], + "vision": ["copilot/gemini-*image*", "google/gemini-*image*", "gemini/gemini-*image*", "copilot/gemini-*flash*", "google/gemini-*flash*", "gemini/gemini-*flash*"] + } + } +} diff --git a/actions/setup/js/fixtures/awf-reflect-copilot-sub-agents-39304573724.json b/actions/setup/js/fixtures/awf-reflect-copilot-sub-agents-39304573724.json new file mode 100644 index 00000000000..d292923a894 --- /dev/null +++ b/actions/setup/js/fixtures/awf-reflect-copilot-sub-agents-39304573724.json @@ -0,0 +1,1575 @@ +{ + "endpoints": [ + { + "provider": "openai", + "port": 10000, + "base_url": "http://api-proxy:10000", + "configured": false, + "models": null, + "model_metadata": null, + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "google": { + "messages": [], + "chat/completions": [], + "responses": [] + } + } + }, + "routing_models": null, + "models_url": "http://api-proxy:10000/v1/models" + }, + { + "provider": "anthropic", + "port": 10001, + "base_url": "http://api-proxy:10001", + "configured": false, + "models": null, + "model_metadata": null, + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "google": { + "messages": [], + "chat/completions": [], + "responses": [] + } + } + }, + "routing_models": null, + "models_url": "http://api-proxy:10001/v1/models" + }, + { + "provider": "copilot", + "port": 10002, + "base_url": "http://api-proxy:10002", + "configured": true, + "models": [ + "claude-haiku-4.5", + "claude-haiku-5.5", + "claude-opus-5", + "claude-opus-5.5", + "claude-sonnet-5", + "claude-sonnet-5.5", + "gpt-3.5-turbo", + "gpt-3.5-turbo-0613", + "gpt-4", + "gpt-4-0613", + "gpt-4-o-preview", + "gpt-4.1", + "gpt-4.1-2025-04-14", + "gpt-41-copilot", + "gpt-4o", + "gpt-4o-2024-05-13", + "gpt-4o-2024-08-06", + "gpt-4o-2024-11-20", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "gpt-5-mini", + "gpt-5.3-codex", + "gpt-5.6-luna", + "gpt-5.6-luna-utility", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "grok-4.5", + "grok-4.6", + "grok-4.7", + "mai-code-1.1-flash", + "text-embedding-3-small", + "text-embedding-3-small-inference", + "text-embedding-ada-002", + "trajectory-compaction" + ], + "model_metadata": [ + { + "id": "claude-haiku-4.5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 1, + "output": 5, + "cachedInput": 0.1, + "cacheWrite": 1.25 + } + } + }, + { + "id": "claude-haiku-5.5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 0.1, + "output": 0.5, + "cachedInput": 0.01, + "cacheWrite": 0.12, + "threshold": 100000 + }, + "long_context": { + "input": 0.5, + "output": 2.5, + "cachedInput": 0.05, + "cacheWrite": 0.62, + "threshold": 936000 + } + } + }, + { + "id": "claude-opus-5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 5, + "output": 25, + "cachedInput": 0.5, + "cacheWrite": 6.25, + "threshold": 200000 + }, + "long_context": { + "input": 5, + "output": 25, + "cachedInput": 0.5, + "cacheWrite": 6.25, + "threshold": 936000 + } + } + }, + { + "id": "claude-opus-5.5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 4, + "output": 20, + "cachedInput": 0.2, + "cacheWrite": 5, + "threshold": 200000 + }, + "long_context": { + "input": 4, + "output": 20, + "cachedInput": 0.2, + "cacheWrite": 5, + "threshold": 1000000 + } + } + }, + { + "id": "claude-sonnet-5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 10, + "cachedInput": 0.2, + "cacheWrite": 2.5, + "threshold": 200000 + }, + "long_context": { + "input": 2, + "output": 10, + "cachedInput": 0.2, + "cacheWrite": 2.5, + "threshold": 936000 + } + } + }, + { + "id": "claude-sonnet-5.5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 10, + "cachedInput": 0.1, + "cacheWrite": 2.5, + "threshold": 200000 + }, + "long_context": { + "input": 2, + "output": 10, + "cachedInput": 0.1, + "cacheWrite": 2.5, + "threshold": 936000 + } + } + }, + { + "id": "gpt-3.5-turbo", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-3.5-turbo-0613", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2.5, + "output": 10, + "cachedInput": 1.25, + "cacheWrite": 0 + } + } + }, + { + "id": "gpt-4-0613", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2.5, + "output": 10, + "cachedInput": 1.25, + "cacheWrite": 0 + } + } + }, + { + "id": "gpt-4-o-preview", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4.1", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4.1-2025-04-14", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-41-copilot", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 8, + "cachedInput": 0.5, + "cacheWrite": 0 + } + } + }, + { + "id": "gpt-4o", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4o-2024-05-13", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4o-2024-08-06", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4o-2024-11-20", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4o-mini", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4o-mini-2024-07-18", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-5-mini", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 0.25, + "output": 2, + "cachedInput": 0.02, + "cacheWrite": 0 + } + } + }, + { + "id": "gpt-5.3-codex", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 1.75, + "output": 14, + "cachedInput": 0.17, + "cacheWrite": 0, + "threshold": 272000 + } + } + }, + { + "id": "gpt-5.6-luna", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 0.2, + "output": 1.2, + "cachedInput": 0.02, + "cacheWrite": 0.25, + "threshold": 200000 + }, + "long_context": { + "input": 0.4, + "output": 1.8, + "cachedInput": 0.04, + "cacheWrite": 0.5, + "threshold": 922000 + } + } + }, + { + "id": "gpt-5.6-luna-utility", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-5.6-sol", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 4, + "output": 20, + "cachedInput": 0.4, + "cacheWrite": 5, + "threshold": 272000 + }, + "long_context": { + "input": 8, + "output": 30, + "cachedInput": 0.8, + "cacheWrite": 10, + "threshold": 922000 + } + } + }, + { + "id": "gpt-5.6-terra", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 12, + "cachedInput": 0.2, + "cacheWrite": 2.5, + "threshold": 272000 + }, + "long_context": { + "input": 4, + "output": 18, + "cachedInput": 0.4, + "cacheWrite": 5, + "threshold": 922000 + } + } + }, + { + "id": "gpt-6-astra", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 10, + "output": 50, + "cachedInput": 1, + "cacheWrite": 12.5, + "threshold": 272000 + }, + "long_context": { + "input": 20, + "output": 75, + "cachedInput": 2, + "cacheWrite": 25, + "threshold": 1050000 + } + } + }, + { + "id": "gpt-6-luna", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 0.1, + "output": 0.5, + "cachedInput": 0.01, + "cacheWrite": 0.12, + "threshold": 272000 + }, + "long_context": { + "input": 0.2, + "output": 0.75, + "cachedInput": 0.02, + "cacheWrite": 0.25, + "threshold": 872000 + } + } + }, + { + "id": "gpt-6-sol", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 10, + "cachedInput": 0.2, + "cacheWrite": 2.5, + "threshold": 272000 + }, + "long_context": { + "input": 4, + "output": 15, + "cachedInput": 0.4, + "cacheWrite": 5, + "threshold": 872000 + } + } + }, + { + "id": "gpt-6.1-sol", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 10, + "cachedInput": 0.1, + "cacheWrite": 2.5, + "threshold": 272000 + }, + "long_context": { + "input": 4, + "output": 15, + "cachedInput": 0.2, + "cacheWrite": 5, + "threshold": 922000 + } + } + }, + { + "id": "grok-4.5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "wire_api": "responses", + "pricing": { + "default": { + "input": 2, + "output": 6, + "cachedInput": 0.5, + "cacheWrite": 0, + "threshold": 200000 + }, + "long_context": { + "input": 4, + "output": 12, + "cachedInput": 1, + "cacheWrite": 0, + "threshold": 500000 + } + } + }, + { + "id": "grok-4.6", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "wire_api": "responses", + "pricing": { + "default": { + "input": 2, + "output": 6, + "cachedInput": 0.5, + "cacheWrite": 0, + "threshold": 200000 + }, + "long_context": { + "input": 4, + "output": 12, + "cachedInput": 1, + "cacheWrite": 0, + "threshold": 500000 + } + } + }, + { + "id": "grok-4.7", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "wire_api": "responses", + "pricing": { + "default": { + "input": 2, + "output": 6, + "cachedInput": 0.5, + "cacheWrite": 0, + "threshold": 200000 + }, + "long_context": { + "input": 4, + "output": 12, + "cachedInput": 1, + "cacheWrite": 0, + "threshold": 500000 + } + } + }, + { + "id": "mai-code-1.1-flash", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 0.2, + "output": 1.2, + "cachedInput": 0.02, + "cacheWrite": 0.25, + "threshold": 128000 + } + } + }, + { + "id": "text-embedding-3-small", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "text-embedding-3-small-inference", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "text-embedding-ada-002", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "trajectory-compaction", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "wire_api": "completions" + } + ], + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": ["claude-sonnet-5.5", "claude-sonnet-5", "claude-opus-5.5", "claude-opus-5", "claude-haiku-5.5", "claude-haiku-4.5"], + "chat/completions": ["claude-sonnet-5.5", "claude-sonnet-5", "claude-opus-5.5", "claude-opus-5", "claude-haiku-5.5", "claude-haiku-4.5"], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": ["gpt-5-mini"], + "responses": ["gpt-6.1-sol", "gpt-6-astra", "gpt-6-luna", "gpt-6-sol", "gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.3-codex", "gpt-5-mini"] + }, + "google": { + "messages": [], + "chat/completions": [], + "responses": [] + } + } + }, + "routing_models": [ + { + "model_id": "claude-haiku-4.5", + "source": "provider", + "supported_endpoints": ["/chat/completions", "/v1/messages"], + "supported_reasoning_efforts": [], + "context_window_tokens": 200000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-haiku-5.5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-opus-5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-opus-5.5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-sonnet-5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-sonnet-5.5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-3.5-turbo", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 16384, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-3.5-turbo-0613", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 16384, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 32768, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4-0613", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 32768, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4-o-preview", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4.1", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4.1-2025-04-14", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-41-copilot", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-2024-05-13", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-2024-08-06", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-2024-11-20", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-mini", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-mini-2024-07-18", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-5-mini", + "source": "provider", + "supported_endpoints": ["/chat/completions", "/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high"], + "context_window_tokens": 264000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.3-codex", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], + "context_window_tokens": 400000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.6-luna", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.6-luna-utility", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-5.6-sol", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.6-terra", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6-astra", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6-luna", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6-sol", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6.1-sol", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "grok-4.5", + "source": "provider", + "supported_endpoints": ["/responses"], + "supported_reasoning_efforts": ["low", "medium", "high"], + "context_window_tokens": 500000, + "candidate_metadata_complete": true + }, + { + "model_id": "grok-4.6", + "source": "provider", + "supported_endpoints": ["/responses"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], + "context_window_tokens": 500000, + "candidate_metadata_complete": true + }, + { + "model_id": "grok-4.7", + "source": "provider", + "supported_endpoints": ["/responses"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], + "context_window_tokens": 500000, + "candidate_metadata_complete": true + }, + { + "model_id": "mai-code-1.1-flash", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high"], + "context_window_tokens": 256000, + "candidate_metadata_complete": true + }, + { + "model_id": "text-embedding-3-small", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "text-embedding-3-small-inference", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "text-embedding-ada-002", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "trajectory-compaction", + "source": "provider", + "supported_endpoints": ["/chat/completions"], + "supported_reasoning_efforts": [], + "context_window_tokens": 262144, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + } + ], + "models_url": "http://api-proxy:10002/models", + "credential_kind": "github_token", + "selected_scheme": "Bearer", + "inference_credential_source": "github_token", + "inference_selected_scheme": "Bearer", + "integration_id_source": "github_copilot_integration_id" + }, + { + "provider": "gemini", + "port": 10003, + "base_url": "http://api-proxy:10003", + "configured": false, + "models": null, + "model_metadata": null, + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "google": { + "messages": [], + "chat/completions": [], + "responses": [] + } + } + }, + "routing_models": null, + "models_url": "http://api-proxy:10003/v1beta/models" + }, + { + "provider": "vertex", + "port": 10004, + "base_url": "http://api-proxy:10004", + "configured": false, + "models": null, + "model_metadata": null, + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "google": { + "messages": [], + "chat/completions": [], + "responses": [] + } + } + }, + "routing_models": null, + "models_url": null + } + ], + "models_fetch_complete": true, + "model_aliases": { + "agent": ["sonnet-6x", "gpt-6", "gpt-5.4", "gpt-5.5", "gpt-5.6", "gpt-5.3", "gemini-pro", "any"], + "any": ["copilot/*", "anthropic/*", "openai/*", "google/*", "gemini/*"], + "auto": ["copilot/auto", "large"], + "claude": ["agent"], + "codex": ["agent"], + "coding": ["copilot/gpt-5*codex*", "openai/gpt-5*codex*", "gpt-5-codex", "kimi"], + "copilot": ["agent"], + "detection": ["small"], + "evals": ["small"], + "gemini": ["agent"], + "gpt-5": ["copilot/gpt-5*", "openai/gpt-5*"], + "gpt-5-codex": ["copilot/gpt-5*codex*", "openai/gpt-5*codex*"], + "gpt-5-mini": ["copilot/gpt-5*mini*", "openai/gpt-5*mini*"], + "gpt-5.3": ["copilot/gpt-5.3*", "openai/gpt-5.3*"], + "gpt-5.6": ["copilot/gpt-5.6*", "openai/gpt-5.6*"], + "gpt-6": ["copilot/gpt-6*", "openai/gpt-6*"], + "grok": ["copilot/*grok*", "openai/*grok*"], + "haiku": ["copilot/*haiku*", "anthropic/*haiku*"], + "large": ["sonnet", "gpt-6", "gpt-5-pro", "gpt-5", "gemini-pro"], + "mai-code": ["copilot/MAI-Code*", "copilot/mai-code*", "openai/MAI-Code*"], + "mini": ["haiku", "gpt-5-mini", "gpt-5-nano", "gemini-flash-lite"], + "opus": ["copilot/*opus*", "anthropic/*opus*"], + "opusplan": ["opus?effort=high"], + "small": ["mini"], + "small-agent": ["haiku", "gpt-5-mini", "gemini-flash"], + "sonnet": ["copilot/*sonnet*", "anthropic/*sonnet*"], + "sonnet-6x": ["copilot/*sonnet-4.5*", "copilot/*sonnet-4.6*", "copilot/*sonnet-5*", "copilot/*sonnet-4-5-*", "anthropic/*sonnet-4-5-*", "copilot/*sonnet-4-6*", "anthropic/*sonnet-4-6*", "anthropic/*sonnet-5*"], + "summarization": ["haiku", "gpt-5-mini", "gemini-flash-lite", "mini"] + }, + "model_fallback": { + "enabled": true, + "strategy": "middle_power", + "excludeEngines": [] + }, + "model_fallback_effective": { + "openai": { + "enabled": true, + "strategy": "middle_power", + "excludeEngines": [], + "suppressed": false + }, + "anthropic": { + "enabled": true, + "strategy": "middle_power", + "excludeEngines": [], + "suppressed": false + }, + "copilot": { + "enabled": false, + "strategy": "middle_power", + "excludeEngines": [], + "suppressed": true, + "suppression_reason": "copilot_standard_authoritative" + }, + "gemini": { + "enabled": true, + "strategy": "middle_power", + "excludeEngines": [], + "suppressed": false + }, + "vertex": { + "enabled": true, + "strategy": "middle_power", + "excludeEngines": [], + "suppressed": false + } + }, + "ai_credits": { + "total": 12.401028, + "max_ai_credits": 1000, + "effective_max_ai_credits": 1000, + "percent_used": 1.24, + "thresholds_crossed": [], + "thresholds_pending": [], + "by_model": { + "gpt-5.3-codex": { + "input_credits": 6.3672, + "cached_input_credits": 1.63968, + "cache_write_credits": 0, + "output_credits": 1.4882, + "total": 9.49508, + "pricing_source": "operator", + "pricing_tier": "default", + "accounting_policy": "concrete_model", + "fallback_pricing_used": false, + "dynamic_selector": null + }, + "claude-haiku-4.5": { + "input_credits": 0.0378, + "cached_input_credits": 0, + "cache_write_credits": 2.493, + "output_credits": 0.0055, + "total": 2.5363, + "pricing_source": "provider", + "pricing_tier": "default", + "accounting_policy": "concrete_model", + "fallback_pricing_used": false, + "dynamic_selector": null + }, + "gpt-5-mini": { + "input_credits": 0.3354, + "cached_input_credits": 0.002048, + "cache_write_credits": 0, + "output_credits": 0.0322, + "total": 0.369648, + "pricing_source": "provider", + "pricing_tier": "default", + "accounting_policy": "concrete_model", + "fallback_pricing_used": false, + "dynamic_selector": null + } + } + }, + "time_steering": { + "enabled": true, + "timeout_minutes": 10, + "started_at_ms": 1792435452871, + "percent_elapsed": 5.51, + "thresholds_crossed": [], + "thresholds_pending": [] + }, + "runs": { + "enabled": true, + "max_runs": 500, + "invocation_count": 9, + "remaining_runs": 491 + }, + "cache_misses": { + "enabled": true, + "max_cache_misses": 5, + "consecutive_cache_misses": 0, + "remaining_cache_misses": 5 + }, + "permission_denied": { + "enabled": false, + "max_permission_denied": null, + "denied_count": 0 + }, + "model_api_mapping": { + "available": true, + "last_updated": "2026-10-08T07:12:00Z", + "providers": ["openai", "anthropic"], + "models": { + "openai": { + "endpoints": { + "responses": { + "path": "/v1/responses", + "description": "Newer unified API for agentic, tool-using, and multimodal workflows. Required for GPT-5+ models; some older models (o1, o3, gpt-4o, gpt-4.1) support it alongside chat/completions." + }, + "chat_completions": { + "path": "/v1/chat/completions", + "description": "Legacy conversational API. Being deprecated for newer models by end of 2026." + } + }, + "models": [ + { + "family": "gpt-6-astra", + "patterns": ["gpt-6-astra*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Newest flagship model (Trusted Access Program rollout). Supports Chat Completions and Responses APIs." + }, + { + "family": "gpt-6-sol", + "patterns": ["gpt-6-sol*"], + "endpoints": ["chat_completions", "responses"], + "notes": "GPT-6 tier balancing intelligence and cost. Supports both Chat Completions and Responses APIs per official model page." + }, + { + "family": "gpt-6.1-sol", + "patterns": ["gpt-6.1-sol*"], + "endpoints": ["chat_completions", "responses"], + "notes": "GPT-6.1 tier balancing intelligence and cost (successor to gpt-6-sol). Supports both Chat Completions and Responses APIs, consistent with other GPT-6 tiers." + }, + { + "family": "gpt-6-luna", + "patterns": ["gpt-6-luna*"], + "endpoints": ["chat_completions", "responses"], + "notes": "GPT-6 tier for cost-sensitive, high-volume use. Supports both Chat Completions and Responses APIs per official model page." + }, + { + "family": "gpt-daybreak", + "patterns": ["gpt-daybreak-blue-latest", "gpt-daybreak-red-latest"], + "endpoints": ["responses"], + "notes": "Responses API only (ResponsesModel-only Literal). Experimental/preview aliases not present in ChatModel." + }, + { + "family": "gpt-rosalind-research", + "patterns": ["gpt-rosalind-research*"], + "endpoints": ["responses"], + "notes": "Life sciences reasoning model, Trusted Access Program (approved organizations only). Responses API only, consistent with other restricted-access specialized models (gpt-daybreak-*, gpt-5.6-cyber). Billing begins October 5, 2026." + }, + { + "family": "gpt-5.6-cyber", + "patterns": ["gpt-5.6-cyber*"], + "endpoints": ["responses"], + "notes": "Responses API only (ResponsesModel-only Literal, not in ChatModel). More specific than gpt-5.6* pattern below." + }, + { + "family": "gpt-5.6", + "patterns": ["gpt-5.6*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints. Includes gpt-5.6-sol, gpt-5.6-terra, gpt-5.6-luna per OpenAI SDK ChatModel. gpt-5.6-cyber is responses-only (see separate entry above)." + }, + { + "family": "gpt-5.5-pro", + "patterns": ["gpt-5.5-pro", "gpt-5.5-pro-*"], + "endpoints": ["responses"], + "notes": "Responses API only (ResponsesOnlyModel per OpenAI OpenAPI spec). More specific than gpt-5.5* pattern below. Includes dated variants like gpt-5.5-pro-2026-04-23." + }, + { + "family": "gpt-5.5", + "patterns": ["gpt-5.5*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints. Listed on the official GPT-5.5 model page. gpt-5.5-pro is responses-only (see separate entry above)." + }, + { + "family": "gpt-5.4", + "patterns": ["gpt-5.4*"], + "routingPatterns": ["gpt-5.4", "gpt-5.4-2026-03-05"], + "endpoints": ["chat_completions", "responses"], + "routing": { + "reasoningEfforts": ["none", "low", "medium", "high", "xhigh"], + "contextWindowTokens": 1050000 + }, + "notes": "Supports both endpoints on OpenAI. GPT-5.4 routing metadata is exact-model only; mini/nano variants need separately verified data. The model page lists none, low, medium, high, and xhigh reasoning effort and a 1,050,000-token context window." + }, + { + "family": "gpt-5.3-codex", + "patterns": ["gpt-5.3-codex", "gpt-5.3-codex-*"], + "endpoints": ["responses"], + "notes": "Responses API only per official model page (developers.openai.com/api/docs/models/gpt-5.3-codex). More specific than gpt-5.3* pattern below." + }, + { + "family": "gpt-5.3", + "patterns": ["gpt-5.3*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints. Only gpt-5.3-chat-latest confirmed for both endpoints; gpt-5.3-codex is responses-only (see separate entry above). No bare gpt-5.3 base model exists." + }, + { + "family": "gpt-5.2-codex", + "patterns": ["gpt-5.2-codex", "gpt-5.2-codex-*"], + "endpoints": ["responses"], + "notes": "Responses API only per official model page (developers.openai.com/api/docs/models/gpt-5.2-codex). More specific than gpt-5.2* pattern below." + }, + { + "family": "gpt-5.2-pro", + "patterns": ["gpt-5.2-pro", "gpt-5.2-pro-*"], + "endpoints": ["responses"], + "notes": "Responses API only per official model page (developers.openai.com/api/docs/models/gpt-5.2-pro). More specific than gpt-5.2* pattern below." + }, + { + "family": "gpt-5.2", + "patterns": ["gpt-5.2*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints. gpt-5.2-pro and gpt-5.2-codex are responses-only (see separate entries above)." + }, + { + "family": "gpt-5.1-codex-max", + "patterns": ["gpt-5.1-codex-max"], + "endpoints": ["responses"], + "notes": "Responses API only (ResponsesOnlyModel)." + }, + { + "family": "gpt-5.1-codex", + "patterns": ["gpt-5.1-codex", "gpt-5.1-codex-*"], + "endpoints": ["responses"], + "notes": "Responses API only per official model pages (developers.openai.com/api/docs/models/gpt-5.1-codex, /gpt-5.1-codex-mini). More specific than gpt-5.1* pattern below. gpt-5.1-codex-max is covered by a dedicated entry above." + }, + { + "family": "gpt-5.1", + "patterns": ["gpt-5.1*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints for gpt-5.1 and gpt-5.1-mini. gpt-5.1-codex, gpt-5.1-codex-mini, and gpt-5.1-codex-max are responses-only (see separate entries above)." + }, + { + "family": "gpt-5-codex/pro", + "patterns": ["gpt-5-codex", "gpt-5-pro", "gpt-5-pro-*"], + "endpoints": ["responses"], + "notes": "Responses API only for gpt-5-codex and gpt-5-pro (ResponsesOnlyModel). Includes dated variants like gpt-5-pro-2025-10-06." + }, + { + "family": "gpt-5", + "patterns": ["gpt-5", "gpt-5-*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints for gpt-5, gpt-5-mini, gpt-5-nano." + }, + { + "family": "o4-mini-deep-research", + "patterns": ["o4-mini-deep-research", "o4-mini-deep-research-*"], + "endpoints": ["responses"], + "notes": "Responses API only (ResponsesOnlyModel). Includes dated variants like o4-mini-deep-research-2025-06-26." + }, + { + "family": "o4", + "patterns": ["o4*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Reasoning model. o4-mini supports both endpoints (ModelIdsShared)." + }, + { + "family": "o3-pro/deep-research", + "patterns": ["o3-pro", "o3-pro-*", "o3-deep-research", "o3-deep-research-*"], + "endpoints": ["responses"], + "notes": "Responses API only. o3-pro and o3-deep-research are in ResponsesModel but not ChatModel." + }, + { + "family": "o3", + "patterns": ["o3*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Reasoning model. o3, o3-mini support both endpoints. o3-pro/deep-research use responses only (covered by separate entry above)." + }, + { + "family": "o1-pro", + "patterns": ["o1-pro", "o1-pro-*"], + "endpoints": ["responses"], + "notes": "Responses API only. o1-pro is in ResponsesModel but not ChatModel." + }, + { + "family": "o1-mini/preview", + "patterns": ["o1-mini*", "o1-preview*"], + "endpoints": ["chat_completions"], + "notes": "Chat Completions API only per official model pages (developers.openai.com/api/docs/models/o1-mini, /o1-preview). Not available via Responses API. More specific than o1* pattern below." + }, + { + "family": "o1", + "patterns": ["o1*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Reasoning model. o1 (base) supports both endpoints. o1-pro is responses-only and o1-mini/o1-preview are chat_completions-only (see separate entries above)." + }, + { + "family": "computer-use-preview", + "patterns": ["computer-use-preview*"], + "endpoints": ["responses"], + "notes": "Responses API only. Specialized model for computer use tasks." + }, + { + "family": "codex-mini", + "patterns": ["codex-mini*"], + "endpoints": ["responses"], + "notes": "Responses API only per official model page (developers.openai.com/api/docs/models/codex-mini-latest). Chat Completions is marked unsupported on the model page." + }, + { + "family": "chatgpt-4o-latest", + "patterns": ["chatgpt-4o-latest"], + "endpoints": ["chat_completions", "responses"], + "notes": "Rolling alias for the latest ChatGPT-optimized GPT-4o model. Supports both endpoints (ModelIdsShared). Not matched by gpt-4o* pattern due to chatgpt- prefix." + }, + { + "family": "gpt-4o", + "patterns": ["gpt-4o*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Multimodal. Supports both endpoints. Includes audio and search preview variants." + }, + { + "family": "gpt-4.1", + "patterns": ["gpt-4.1*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints. Includes gpt-4.1-mini, gpt-4.1-nano." + }, + { + "family": "gpt-4.5", + "patterns": ["gpt-4.5*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Preview model. Now supports both endpoints per official model page (developers.openai.com/api/docs/models/gpt-4.5-preview); Responses API support was added since the previous mapping update." + }, + { + "family": "gpt-4-turbo", + "patterns": ["gpt-4-turbo*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Legacy. In ModelIdsShared per OpenAI spec v2.3.0; supports both endpoints." + }, + { + "family": "gpt-4", + "patterns": ["gpt-4", "gpt-4-*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Legacy. In ModelIdsShared per OpenAI spec v2.3.0; supports both endpoints." + }, + { + "family": "gpt-3.5-turbo", + "patterns": ["gpt-3.5-turbo*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Legacy. In ModelIdsShared per OpenAI spec v2.3.0; supports both endpoints. Deprecated." + } + ] + }, + "anthropic": { + "endpoints": { + "messages": { + "path": "/v1/messages", + "description": "Unified Messages API for all Claude models. Supports tool use, streaming, extended thinking, and vision." + } + }, + "models": [ + { + "family": "claude-opus-5", + "patterns": ["claude-opus-5*"], + "routingPatterns": ["claude-opus-5-5", "claude-opus-5"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Flagship Opus generation 5 models with verified 1M-token context windows and the Anthropic effort parameter." + }, + { + "family": "claude-fable-5", + "patterns": ["claude-fable-5*"], + "routingPatterns": ["claude-fable-5-1", "claude-fable-5"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Latest flagship family with verified 1M-token context windows and the Anthropic effort parameter." + }, + { + "family": "claude-mythos-5", + "patterns": ["claude-mythos-5*", "claude-mythos-preview*"], + "routingPatterns": ["claude-mythos-5-1", "claude-mythos-5", "claude-mythos-preview*"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Specialized models, invitation-only, with verified 1M-token context windows and the Anthropic effort parameter." + }, + { + "family": "claude-sonnet-5", + "patterns": ["claude-sonnet-5*"], + "routingPatterns": ["claude-sonnet-5-5", "claude-sonnet-5"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Best combination of speed and intelligence with verified 1M-token context windows and the Anthropic effort parameter." + }, + { + "family": "claude-opus-4", + "patterns": ["claude-opus-4*"], + "routingPatterns": ["claude-opus-4-8", "claude-opus-4-7", "claude-opus-4-6"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Opus 4.6 and newer models have verified 1M-token context windows and support the Anthropic effort parameter." + }, + { + "family": "claude-sonnet-4", + "patterns": ["claude-sonnet-4*"], + "routingPatterns": ["claude-sonnet-4-6"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Sonnet 4.6 has a verified 1M-token context window and supports the Anthropic effort parameter." + }, + { + "family": "claude-haiku-5", + "patterns": ["claude-haiku-5*"], + "endpoints": ["messages"], + "notes": "Fast/cheap Haiku generation 5 (claude-haiku-5-5). 1M token context window per official models overview; effort levels not verified, so no routing metadata." + }, + { + "family": "claude-haiku-4", + "patterns": ["claude-haiku-4*"], + "endpoints": ["messages"], + "notes": "Includes claude-haiku-4-5. Fast/cheap. 200K token context." + }, + { + "family": "claude-3.5-sonnet", + "patterns": ["claude-3-5-sonnet*", "claude-3.5-sonnet*"], + "endpoints": ["messages"], + "notes": "Previous generation. 200K token context." + }, + { + "family": "claude-3.5-haiku", + "patterns": ["claude-3-5-haiku*", "claude-3.5-haiku*"], + "endpoints": ["messages"], + "notes": "Previous generation fast model." + }, + { + "family": "claude-3.7-sonnet", + "patterns": ["claude-3-7-sonnet*", "claude-3.7-sonnet*"], + "endpoints": ["messages"], + "notes": "Previous generation extended thinking model. 200K token context." + }, + { + "family": "claude-3-opus", + "patterns": ["claude-3-opus*"], + "endpoints": ["messages"], + "notes": "Previous generation. 200K token context." + } + ] + } + }, + "error": null + }, + "routing": null +} diff --git a/docs/src/content/docs/reference/engines.md b/docs/src/content/docs/reference/engines.md index 230591a1a2c..c29ae817b53 100644 --- a/docs/src/content/docs/reference/engines.md +++ b/docs/src/content/docs/reference/engines.md @@ -610,10 +610,19 @@ The built-in SDK driver loads inline sub-agents and repository sub-agent models against the AWF model catalog. Copilot models use separate Responses and Chat Completions providers according to their supported endpoints, so GPT and Claude agents can use different wire APIs in the same session. -The aliases `small`, `medium`, and `large` inherit the session model; an unknown -sub-agent model logs a warning and inherits it. Sub-agent activity and final model -results, including BYOK status, are retained in `usage/aw_session.jsonl`. -This behavior applies to SDK mode, not the default CLI mode. +Sub-agent activity and final model results, including BYOK status, are retained +in `usage/aw_session.jsonl`. + +In both SDK and CLI mode, the harness resolves each sub-agent's `model:` before +Copilot starts, including on routed runs. It uses gh-aw model aliases (for +example `small` and `large`) and checks the live AWF model catalog. Each result +is recorded as a `model.resolution` event whose role is the agent name. In CLI +mode, the resolved id is written to the workspace agent file that Copilot CLI +loads; in SDK mode, it is qualified against the SDK provider models. `medium` is +not an alias and is treated as a literal model id. If a sub-agent model cannot be +resolved, is not in the live catalog, or does not qualify in SDK mode, the run +fails before Copilot starts instead of falling back to the main model. A +sub-agent without `model:` inherits the main model and has no resolution record. Use top-level `max-tool-denials` to stop SDK inference when tool requests are repeatedly denied. The default is `5`. From 7f35145678b6d2106ea19acf9c91c1a5f60bd8fa Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:49:55 +0000 Subject: [PATCH 3/5] Keep CLI rewrites of tracked agent files out of commits (skip-worktree) Co-authored-by: SivaKesava1 <11771739+SivaKesava1@users.noreply.github.com> --- actions/setup/js/copilot_harness.cjs | 29 +++++++++++++++++++++++ actions/setup/js/copilot_harness.test.cjs | 19 +++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 580af9e6688..ca9c48cd025 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -846,13 +846,42 @@ async function applyCopilotCLISubagentModels(options) { for (const resolution of resolutions) { if (resolution.model === resolution.configuredModel) continue; for (const file of files.filter(file => file.name === resolution.name && file.model === resolution.configuredModel)) { + const original = fs.readFileSync(file.path); rewriteCopilotAgentFileModel(file.path, resolution.model); + if (!hideTrackedAgentFileRewrite(file.path, logger)) { + fs.writeFileSync(file.path, original); + logger(`copilot sub-agent '${resolution.name}': cannot keep the resolved model out of commits for tracked file ${file.path}. Refusing to start Copilot with an unresolved sub-agent model.`); + process.exit(1); + return []; + } logger(`copilot sub-agent '${resolution.name}': wrote model '${resolution.model}' to ${file.path}`); } } return resolutions; } +/** + * Keep a harness rewrite of a git-tracked agent file out of the agent's commits and + * safe-output patches by marking it skip-worktree. Untracked files (restored inline + * sub-agents) need nothing. Returns false when a tracked file cannot be hidden. + * @param {string} agentPath + * @param {(msg: string) => void} logger + * @returns {boolean} + */ +function hideTrackedAgentFileRewrite(agentPath, logger) { + const { spawnSync } = require("child_process"); + const cwd = path.dirname(agentPath); + const file = path.basename(agentPath); + const tracked = spawnSync("git", ["ls-files", "--error-unmatch", "--", file], { cwd, stdio: "ignore" }); + if (tracked.error || tracked.status !== 0) return true; + const hidden = spawnSync("git", ["update-index", "--skip-worktree", "--", file], { cwd, encoding: "utf8" }); + if (hidden.error || hidden.status !== 0) { + logger(`warning: git update-index --skip-worktree failed for ${agentPath}: ${hidden.error ? getErrorMessage(hidden.error) : String(hidden.stderr || "").trim()}`); + return false; + } + return true; +} + /** * Configure and validate COPILOT_PROVIDER_WIRE_API using AWF endpoint metadata. * diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 24e8d64c92f..37412ba1bfc 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -4845,6 +4845,25 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 expect(fs.readFileSync(path.join(agentsDir, "haiku-whoami.agent.md"), "utf8")).toBe(exact); }); + it("case 6 (CLI, tracked file): a rewritten repository agent file stays out of git status and commits", async () => { + const { execFileSync } = require("child_process"); + const dir = tempDir("copilot-subagent-cli-tracked-"); + const git = (/** @type {string[]} */ ...args) => execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], { cwd: dir, encoding: "utf8" }); + git("init", "-q"); + const agentsDir = path.join(dir, ".github/agents"); + fs.mkdirSync(agentsDir, { recursive: true }); + fs.writeFileSync(path.join(agentsDir, "repo-small.agent.md"), "---\nmodel: small\n---\nRepository agent.\n"); + git("add", "-A"); + git("commit", "-q", "-m", "init"); + const awfConfigPath = path.join(dir, "awf-config.json"); + fs.copyFileSync(awfConfigFixturePath, awfConfigPath); + await applyCopilotCLISubagentModels({ reflectData: recordedReflect(), agentsDir, env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, logger: () => {} }); + expect(fs.readFileSync(path.join(agentsDir, "repo-small.agent.md"), "utf8")).toBe('---\nmodel: "claude-haiku-5.5"\n---\nRepository agent.\n'); + git("add", "-A"); + expect(git("status", "--porcelain", "--", ".github")).toBe(""); + expect(git("diff", "--cached", "--name-only", "--", ".github")).toBe(""); + }); + /** Spawn the harness against the recorded catalogue with the given workspace agent files. */ async function runHarness({ sdk = false, agents, reflect = recordedReflect(), env: envOverrides = {} }) { const dir = tempDir("copilot-subagent-harness-"); From b81266d7fc80240424039b107a51d2e534b0b370 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 02:03:31 +0000 Subject: [PATCH 4/5] Address Copilot sub-agent model review feedback Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/copilot_harness.cjs | 247 ++++++++++++++++++---- actions/setup/js/copilot_harness.test.cjs | 237 +++++++++++++++++++-- actions/setup/js/copilot_sdk_session.cjs | 2 +- 3 files changed, 429 insertions(+), 57 deletions(-) diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index ca9c48cd025..4d6771c5017 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -106,7 +106,6 @@ const { applyCopilotWireAPI: configureCopilotWireAPI, warnCopilotSubagentEndpoin const { loadModelAliasMap, resolveEngineModel, normalizeForCopilotCLI, ModelAliasResolutionError, recordModelResolution, splitModelIdentifier } = require("./resolve_model_alias.cjs"); const { createCopilotModelObserver, recordCopilotModelSelection } = require("./copilot_model_selection.cjs"); const { loadCopilotSDKCustomAgents } = require("./extract_inline_sub_agents.cjs"); -const { resolveCopilotSDKWorkingDirectory } = require("./copilot_sdk_session.cjs"); const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported|No model available\b[^\n]*policy enablement/i; const { parseAICreditsErrorInfoFromAuditLog, parseMaxAICreditsFromAuditLog, parseMaxAICreditsExceededFromAuditLog, parseAPIProxyGuardRejectionFromEventLog, formatAPIProxyGuardRejection } = require("./ai_credits_context.cjs"); @@ -655,8 +654,8 @@ async function resolveCopilotSDKMultiProvider(options) { } /** - * Read the Copilot custom agent files in one directory, one entry per regular `.md` file. - * The name is the frontmatter `name` or the file name without `.agent.md` / `.md`; + * Read the Copilot custom agent files in one directory, one entry per regular `.agent.md` file. + * The name is the frontmatter `name` or the file name without `.agent.md`; * `model` is the trimmed frontmatter `model` string, when declared. * @param {string} directory * @param {(msg: string) => void} [logger] @@ -674,7 +673,7 @@ function listCopilotAgentFiles(directory, logger = log) { } const agents = []; for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { - if (!entry.isFile() || !entry.name.endsWith(".md")) continue; + if (!entry.isFile() || !entry.name.endsWith(".agent.md")) continue; const agentPath = path.join(directory, entry.name); try { const { frontmatterText } = extractFrontmatterAndBody(fs.readFileSync(agentPath, "utf8")); @@ -683,7 +682,7 @@ function listCopilotAgentFiles(directory, logger = log) { if (!document || document.errors.length) throw new Error("cannot parse agent frontmatter"); const frontmatter = document.toJS() || {}; if (typeof frontmatter !== "object" || Array.isArray(frontmatter)) throw new Error("agent frontmatter must be a mapping"); - const name = typeof frontmatter.name === "string" && frontmatter.name.trim() ? frontmatter.name.trim() : entry.name.replace(/(?:\.agent)?\.md$/, ""); + const name = typeof frontmatter.name === "string" && frontmatter.name.trim() ? frontmatter.name.trim() : entry.name.slice(0, -".agent.md".length); agents.push({ name, path: agentPath, ...(typeof frontmatter.model === "string" ? { model: frontmatter.model.trim() } : {}) }); } catch (error) { logger(`warning: cannot read custom agent ${agentPath}: ${getErrorMessage(error)}`); @@ -693,22 +692,60 @@ function listCopilotAgentFiles(directory, logger = log) { } /** - * Replace the top-level frontmatter `model:` of a Copilot agent file. Only the `model:` line - * changes; the rest of the file is preserved byte for byte. + * Locate the top-level YAML model scalar in a Copilot agent file. + * @param {string} content + * @returns {{ value: string, source: string, start: number, end: number }} + */ +function getCopilotAgentModelScalar(content) { + const { extractFrontmatterAndBody } = require("./frontmatter_hash_pure.cjs"); + const { parseDocument } = require("./work_queue_yaml.cjs"); + const { frontmatterText } = extractFrontmatterAndBody(content); + /** @type {any} */ + const document = parseDocument(frontmatterText); + if (!document || document.errors.length || !document.contents || !Array.isArray(document.contents.items)) { + throw new Error("cannot parse agent frontmatter"); + } + const pairs = document.contents.items.filter(pair => pair?.key?.value === "model"); + if (pairs.length !== 1 || typeof pairs[0].value?.value !== "string" || !Array.isArray(pairs[0].value.range)) { + throw new Error("expected exactly one top-level string 'model:' scalar in the frontmatter"); + } + const frontmatterStart = content.indexOf("\n") + 1; + const start = frontmatterStart + pairs[0].value.range[0]; + const end = frontmatterStart + pairs[0].value.range[1]; + return { value: pairs[0].value.value, source: content.slice(start, end), start, end }; +} + +/** + * Replace the complete YAML model scalar, including any block-scalar continuation lines. + * @param {string} content + * @param {string} model + * @returns {string} + */ +function replaceCopilotAgentFileModelContent(content, model) { + const scalar = getCopilotAgentModelScalar(content); + const lineEnding = scalar.source.match(/(?:\r?\n)+$/)?.[0] || ""; + return content.slice(0, scalar.start) + JSON.stringify(model) + lineEnding + content.slice(scalar.end); +} + +/** + * Replace the top-level frontmatter `model:` scalar of a Copilot agent file. The rest of the + * file, including comments and the body, is preserved byte for byte. * @param {string} agentPath * @param {string} model */ function rewriteCopilotAgentFileModel(agentPath, model) { - const content = fs.readFileSync(agentPath, "utf8"); - const lines = content.split("\n"); - const end = lines.findIndex((line, index) => index > 0 && line.trim() === "---"); - const modelLines = lines.map((line, index) => (index > 0 && index < end && /^model[ \t]*:/.test(line) ? index : -1)).filter(index => index >= 0); - if (lines[0]?.trim() !== "---" || end < 0 || modelLines.length !== 1) { - throw new Error(`cannot rewrite model in ${agentPath}: expected exactly one top-level 'model:' line in the frontmatter`); + let content; + try { + content = fs.readFileSync(agentPath, "utf8"); + } catch (error) { + throw new Error(`cannot read agent file ${agentPath}: ${getErrorMessage(error)}`, { cause: error }); + } + const updated = replaceCopilotAgentFileModelContent(content, model); + try { + fs.writeFileSync(agentPath, updated); + } catch (error) { + throw new Error(`cannot rewrite model in ${agentPath}: ${getErrorMessage(error)}`, { cause: error }); } - const index = modelLines[0]; - lines[index] = `model: ${JSON.stringify(model)}${lines[index].endsWith("\r") ? "\r" : ""}`; - fs.writeFileSync(agentPath, lines.join("\n")); } /** @@ -729,6 +766,7 @@ function rewriteCopilotAgentFileModel(agentPath, model) { * reflectData: any, * refetchReflectData?: () => Promise, * providerConfig?: { providers: Array<{name: string}>, models: Array<{id: string, provider: string}> } | null, + * refreshProviderConfig?: (reflectData: any) => Promise<{ providers: Array<{name: string}>, models: Array<{id: string, provider: string}> } | null>, * aliasMap?: Record | null, * env?: NodeJS.ProcessEnv, * logger?: (msg: string) => void, @@ -739,15 +777,24 @@ function rewriteCopilotAgentFileModel(agentPath, model) { async function resolveCopilotSubagentModels(options) { const logger = options.logger || log; const env = options.env || process.env; - const declared = options.agents.filter(agent => typeof agent.model === "string" && agent.model.trim()); + const declared = options.agents.filter(agent => typeof agent.model === "string" && agent.model.trim() && agent.model.trim().toLowerCase() !== "inherited"); if (declared.length === 0) return []; const aliasMap = options.aliasMap !== undefined ? options.aliasMap : (loadAwfConfigData()?.apiProxy?.models ?? loadModelAliasMap({ reflectData: options.reflectData, env })); let reflectData = options.reflectData; + let providerConfig = options.providerConfig; /** @type {Promise | null} */ let refresh = null; const refreshOnce = () => { refresh ??= (options.refetchReflectData ? options.refetchReflectData() : Promise.resolve(null)).then(refreshed => { - if (refreshed) reflectData = refreshed; + if (refreshed) { + reflectData = refreshed; + if (options.refreshProviderConfig) { + return options.refreshProviderConfig(refreshed).then(config => { + if (config) providerConfig = config; + return refreshed; + }); + } + } return refreshed || null; }); return refresh; @@ -787,18 +834,29 @@ async function resolveCopilotSubagentModels(options) { const resolvedModel = result.engineModel; const resolvedNote = resolvedModel !== configuredModel ? ` (resolved to '${resolvedModel}')` : ""; const failureProvider = scope.providers && scope.providers !== REFLECT_PROVIDER_ALIASES.github ? scope.prefix : "copilot"; - if (liveCheck && !isModelListedOnConfiguredReflectEndpoint(resolvedModel, reflectData)) { - const initialReason = classifyUnlistedReflectModel(resolvedModel, reflectData); + const scopedResolvedModel = scope.prefix ? `${scope.prefix}/${resolvedModel}` : resolvedModel; + if (liveCheck && !isModelListedOnConfiguredReflectEndpoint(scopedResolvedModel, reflectData)) { + const initialReason = classifyUnlistedReflectModel(scopedResolvedModel, reflectData); const refreshed = await refreshOnce(); - if (!refreshed || !isModelListedOnConfiguredReflectEndpoint(resolvedModel, refreshed)) { - const reason = initialReason === "model_unavailable" || !refreshed ? initialReason : classifyUnlistedReflectModel(resolvedModel, refreshed); + if (!refreshed || !isModelListedOnConfiguredReflectEndpoint(scopedResolvedModel, refreshed)) { + const reason = initialReason === "model_unavailable" || !refreshed ? initialReason : classifyUnlistedReflectModel(scopedResolvedModel, refreshed); fail(new ModelAliasResolutionError(configuredModel, reason, { provider: failureProvider }), `Sub-agent model '${configuredModel}'${resolvedNote} is not listed on the configured awf-reflect endpoints.`); continue; } } let model = resolvedModel; - if (options.providerConfig) { - const qualified = qualifyModelForMultiProvider(resolvedModel, options.providerConfig); + if (providerConfig) { + let qualified; + try { + const qualificationModel = scope.providers === REFLECT_PROVIDER_ALIASES.github ? `copilot/${resolvedModel}` : scope.prefix ? `${scope.prefix}/${resolvedModel}` : resolvedModel; + qualified = qualifyModelForMultiProvider(qualificationModel, providerConfig); + } catch (error) { + fail( + new ModelAliasResolutionError(configuredModel, "model_unavailable", { provider: failureProvider }), + `Sub-agent model '${configuredModel}'${resolvedNote} could not be qualified against the Copilot SDK provider models: ${getErrorMessage(error)}` + ); + continue; + } if (!qualified) { fail( new ModelAliasResolutionError(configuredModel, "model_unavailable", { provider: failureProvider }), @@ -825,6 +883,7 @@ async function resolveCopilotSubagentModels(options) { * reflectData: any, * refetchReflectData?: () => Promise, * agentsDir?: string, + * rewrittenFiles?: Array<{ path: string, configuredModel: string, resolvedModel: string, modelSource: string }>, * env?: NodeJS.ProcessEnv, * logger?: (msg: string) => void, * }} options @@ -833,7 +892,7 @@ async function resolveCopilotSubagentModels(options) { async function applyCopilotCLISubagentModels(options) { const logger = options.logger || log; const env = options.env || process.env; - const agentsDir = options.agentsDir || path.join(env.GITHUB_WORKSPACE || process.cwd(), ".github", "agents"); + const agentsDir = options.agentsDir || path.join(env.GH_AW_ENGINE_CWD || env.GITHUB_WORKSPACE || process.cwd(), ".github", "agents"); const files = listCopilotAgentFiles(agentsDir, logger); let resolutions; try { @@ -846,14 +905,36 @@ async function applyCopilotCLISubagentModels(options) { for (const resolution of resolutions) { if (resolution.model === resolution.configuredModel) continue; for (const file of files.filter(file => file.name === resolution.name && file.model === resolution.configuredModel)) { - const original = fs.readFileSync(file.path); - rewriteCopilotAgentFileModel(file.path, resolution.model); - if (!hideTrackedAgentFileRewrite(file.path, logger)) { + let original; + let modelScalar; + try { + original = fs.readFileSync(file.path, "utf8"); + modelScalar = getCopilotAgentModelScalar(original); + rewriteCopilotAgentFileModel(file.path, resolution.model); + } catch (error) { + if (typeof original === "string") { + try { + fs.writeFileSync(file.path, original); + } catch (restoreError) { + logger(`warning: cannot restore ${file.path} after a rewrite failure: ${getErrorMessage(restoreError)}`); + } + } + if (options.rewrittenFiles) restoreCopilotCLIAgentFiles(options.rewrittenFiles, logger); + logger(`copilot sub-agent '${resolution.name}': cannot rewrite ${file.path}: ${getErrorMessage(error)}. Refusing to start Copilot.`); + process.exit(1); + return []; + } + const trackingState = hideTrackedAgentFileRewrite(file.path, logger); + if (!trackingState) { fs.writeFileSync(file.path, original); + if (options.rewrittenFiles) restoreCopilotCLIAgentFiles(options.rewrittenFiles, logger); logger(`copilot sub-agent '${resolution.name}': cannot keep the resolved model out of commits for tracked file ${file.path}. Refusing to start Copilot with an unresolved sub-agent model.`); process.exit(1); return []; } + if (trackingState === true) { + options.rewrittenFiles?.push({ path: file.path, configuredModel: resolution.configuredModel, resolvedModel: resolution.model, modelSource: modelScalar.source }); + } logger(`copilot sub-agent '${resolution.name}': wrote model '${resolution.model}' to ${file.path}`); } } @@ -861,27 +942,70 @@ async function applyCopilotCLISubagentModels(options) { } /** - * Keep a harness rewrite of a git-tracked agent file out of the agent's commits and - * safe-output patches by marking it skip-worktree. Untracked files (restored inline - * sub-agents) need nothing. Returns false when a tracked file cannot be hidden. + * Keep a harness rewrite of a git-tracked agent file out of the agent's commits by marking it + * skip-worktree until the harness restores the model and clears the flag. Untracked files need + * nothing. Returns false when tracking state or the index flag cannot be verified. * @param {string} agentPath * @param {(msg: string) => void} logger - * @returns {boolean} + * @returns {boolean|"untracked"} */ function hideTrackedAgentFileRewrite(agentPath, logger) { const { spawnSync } = require("child_process"); const cwd = path.dirname(agentPath); const file = path.basename(agentPath); - const tracked = spawnSync("git", ["ls-files", "--error-unmatch", "--", file], { cwd, stdio: "ignore" }); - if (tracked.error || tracked.status !== 0) return true; - const hidden = spawnSync("git", ["update-index", "--skip-worktree", "--", file], { cwd, encoding: "utf8" }); + const tracked = spawnSync("git", ["ls-files", "--error-unmatch", "--", file], { cwd, encoding: "utf8", timeout: 5000 }); + if (tracked.error || (tracked.status !== 0 && tracked.status !== 1)) { + logger(`warning: git ls-files could not determine tracking state for ${agentPath}: ${tracked.error ? getErrorMessage(tracked.error) : String(tracked.stderr || "").trim()}`); + return false; + } + if (tracked.status === 1) return "untracked"; + const hidden = spawnSync("git", ["update-index", "--skip-worktree", "--", file], { cwd, encoding: "utf8", timeout: 5000 }); if (hidden.error || hidden.status !== 0) { logger(`warning: git update-index --skip-worktree failed for ${agentPath}: ${hidden.error ? getErrorMessage(hidden.error) : String(hidden.stderr || "").trim()}`); return false; } + const verified = spawnSync("git", ["ls-files", "-t", "--", file], { cwd, encoding: "utf8", timeout: 5000 }); + if (verified.error || verified.status !== 0 || !/^S\s/.test(verified.stdout || "")) { + logger(`warning: git did not retain skip-worktree for ${agentPath}: ${verified.error ? getErrorMessage(verified.error) : String(verified.stderr || verified.stdout || "").trim()}`); + spawnSync("git", ["update-index", "--no-skip-worktree", "--", file], { cwd, encoding: "utf8", timeout: 5000 }); + return false; + } return true; } +/** + * Restore harness-only model substitutions while preserving any other agent-file edits. + * @param {Array<{ path: string, configuredModel: string, resolvedModel: string, modelSource: string }>} rewrittenFiles + * @param {(msg: string) => void} logger + * @returns {boolean} + */ +function restoreCopilotCLIAgentFiles(rewrittenFiles, logger = log) { + const { spawnSync } = require("child_process"); + let success = true; + for (const file of rewrittenFiles) { + const cwd = path.dirname(file.path); + const name = path.basename(file.path); + const cleared = spawnSync("git", ["update-index", "--no-skip-worktree", "--", name], { cwd, encoding: "utf8", timeout: 5000 }); + const verified = cleared.error || cleared.status !== 0 ? null : spawnSync("git", ["ls-files", "-t", "--", name], { cwd, encoding: "utf8", timeout: 5000 }); + if (!verified || verified.error || verified.status !== 0 || /^S\s/.test(verified.stdout || "")) { + logger(`warning: cannot clear skip-worktree for ${file.path}: ${cleared.error ? getErrorMessage(cleared.error) : String(cleared.stderr || verified?.stderr || "").trim()}`); + success = false; + continue; + } + try { + const content = fs.readFileSync(file.path, "utf8"); + const current = getCopilotAgentModelScalar(content); + if (current.value === file.resolvedModel && file.resolvedModel !== file.configuredModel) { + fs.writeFileSync(file.path, content.slice(0, current.start) + file.modelSource + content.slice(current.end)); + } + } catch (error) { + logger(`warning: cannot restore harness model substitution in ${file.path}: ${getErrorMessage(error)}`); + success = false; + } + } + return success; +} + /** * Configure and validate COPILOT_PROVIDER_WIRE_API using AWF endpoint metadata. * @@ -1588,6 +1712,7 @@ function resolvePromptFileInput(args) { */ async function main() { const [, , command, ...args] = process.argv; + const cliAgentFileRestorations = []; const retryConfig = resolveRetryConfig(process.env, log); const { maxRetries, initialDelayMs, backoffMultiplier, maxDelayMs } = retryConfig; @@ -1685,10 +1810,6 @@ async function main() { }); if (invocationModel?.hasOverride) resolvedArgs = applyCopilotInvocationModel(resolvedArgs, process.env.COPILOT_MODEL || ""); } - // Sub-agent models are resolved on routed runs too. - if (!copilotSDKMode) { - await applyCopilotCLISubagentModels({ reflectData: awfReflectData, refetchReflectData, logger: log }); - } if (!copilotSDKMode) { try { applyCopilotWireAPI({ @@ -1717,6 +1838,17 @@ async function main() { process.exit(0); } + // Sub-agent models are resolved on routed runs too, but a pre-existing noop must not + // make startup depend on sub-agent model availability. + if (!copilotSDKMode) { + await applyCopilotCLISubagentModels({ + reflectData: awfReflectData, + refetchReflectData, + rewrittenFiles: cliAgentFileRestorations, + logger: log, + }); + } + // Resolve BYOK provider from live reflect data (SDK mode only). // Multi-provider BYOK is the only supported mode — fail immediately if the // provider cannot be resolved so retries are not wasted on a misconfigured environment. @@ -1732,7 +1864,7 @@ async function main() { const configuredModel = modelRoutingSelection?.wire_model || process.env.COPILOT_MODEL || ""; const modelsJson = loadModelsJson(); - const multiProvider = await resolveCopilotSDKMultiProvider({ + let multiProvider = await resolveCopilotSDKMultiProvider({ configuredModel, requestedModel: selectionConfiguredModel, routed: Boolean(modelRoutingSelection), @@ -1771,15 +1903,41 @@ async function main() { }); // The same custom agents the SDK driver loads: workflow definitions override repository ones. - const agents = loadCopilotSDKCustomAgents(resolveCopilotSDKWorkingDirectory(), path.join(os.tmpdir(), "gh-aw"), log); + const sdkWorkingDirectory = path.resolve(process.env.GITHUB_WORKSPACE || process.cwd(), process.env.GH_AW_ENGINE_CWD || "."); + const agents = loadCopilotSDKCustomAgents(sdkWorkingDirectory, path.join(os.tmpdir(), "gh-aw"), log); try { - const resolutions = await resolveCopilotSubagentModels({ agents, reflectData: awfReflectData, refetchReflectData, providerConfig: multiProvider, logger: log }); + const resolutions = await resolveCopilotSubagentModels({ + agents, + reflectData: awfReflectData, + refetchReflectData, + providerConfig: multiProvider, + refreshProviderConfig: async reflectData => { + multiProvider = resolveMultiProviderFromReflect({ + model: configuredModel, + wireApi: process.env.COPILOT_PROVIDER_WIRE_API, + reflectData, + modelsJson, + logger: log, + }) || { model: "", providers: [], models: [] }; + return multiProvider; + }, + logger: log, + }); subagentModelsJson = JSON.stringify(Object.fromEntries(resolutions.map(resolution => [resolution.name, resolution.model]))); } catch (failure) { if (!(failure instanceof ModelAliasResolutionError)) throw failure; process.exit(1); } + resolvedModel = multiProvider.model; + primaryProviderName = multiProvider.models.find(m => `${m.provider}/${m.id}` === resolvedModel)?.provider ?? multiProvider.providers[0]?.name ?? ""; + const refreshedPrimaryProvider = multiProvider.providers.find(p => p.name === primaryProviderName) ?? multiProvider.providers[0]; + providerBaseUrl = refreshedPrimaryProvider?.baseUrl ?? ""; + providerType = refreshedPrimaryProvider?.type ?? "openai"; + const configuredProviderWireApi = process.env.COPILOT_PROVIDER_WIRE_API; + providerWireApi = configuredProviderWireApi === "responses" || configuredProviderWireApi === "completions" ? configuredProviderWireApi : refreshedPrimaryProvider?.wireApi || ""; + multiProviderJson = JSON.stringify({ model: multiProvider.model, providers: multiProvider.providers, models: multiProvider.models }); + const uniqueProviderBaseUrls = [...new Set(multiProvider.providers.map(provider => String(provider.baseUrl || "").trim()).filter(Boolean))]; if (uniqueProviderBaseUrls.length === 0) { log("copilot-sdk driver mode: no provider baseUrls to probe — skipping listener readiness check"); @@ -2337,6 +2495,9 @@ async function main() { await fetchAWFReflect({ logger: log }); } } finally { + if (!restoreCopilotCLIAgentFiles(cliAgentFileRestorations, log)) { + lastExitCode = 1; + } await stopCopilotSDKServer(copilotSDKServer, { logger: log }); } log(`done: exitCode=${lastExitCode} totalDuration=${formatDuration(Date.now() - driverStartTime)}`); @@ -2419,6 +2580,8 @@ if (typeof module !== "undefined" && module.exports) { applyCopilotCLISubagentModels, listCopilotAgentFiles, rewriteCopilotAgentFileModel, + hideTrackedAgentFileRewrite, + restoreCopilotCLIAgentFiles, applyCopilotWireAPI, warnCopilotSubagentEndpoints, resolveAWFModelRoutingSelection, diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 37412ba1bfc..2f8dbb73c40 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -76,6 +76,8 @@ const { resolveCopilotSDKMultiProvider, resolveCopilotSubagentModels, applyCopilotCLISubagentModels, + listCopilotAgentFiles, + restoreCopilotCLIAgentFiles, formatInferenceEndpointForLog, logCopilotInferenceConfiguration, resolveLongRunTokenThreshold, @@ -4683,13 +4685,21 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 return dir; } + function initGitRepo(dir, files = []) { + const { execFileSync } = require("child_process"); + execFileSync("git", ["init", "-q"], { cwd: dir }); + execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", "add", "-A"], { cwd: dir }); + if (files.length > 0) execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", "add", ...files], { cwd: dir }); + execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-q", "--allow-empty", "-m", "init"], { cwd: dir }); + } + const readRecords = dir => { const file = path.join(dir, "agent/model-resolution.json"); return fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : null; }; /** In-process: the harness sub-agent resolution step against the recorded catalogue. */ - async function resolveAgents(agents, { reflect = recordedReflect(), refreshed = [], providerConfig = null } = {}) { + async function resolveAgents(agents, { reflect = recordedReflect(), refreshed = [], providerConfig = null, refreshProviderConfig } = {}) { const dir = tempDir("copilot-subagent-models-"); const env = { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1" }; const logs = []; @@ -4698,7 +4708,7 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 let resolutions = null; let error = null; try { - resolutions = await resolveCopilotSubagentModels({ agents, reflectData: reflect, refetchReflectData: refetch, providerConfig, aliasMap: aliasMap(), env, logger: message => logs.push(message) }); + resolutions = await resolveCopilotSubagentModels({ agents, reflectData: reflect, refetchReflectData: refetch, providerConfig, refreshProviderConfig, aliasMap: aliasMap(), env, logger: message => logs.push(message) }); } catch (err) { error = err; } @@ -4763,10 +4773,15 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 it("case 3c: a dead id starts when the bounded refresh lists it", async () => { const refreshed = recordedReflect(); refreshed.endpoints.find(endpoint => endpoint.provider === "copilot").models.push("gpt-5.4-mini"); - const run = await resolveAgents([{ name: "mini54-whoami", model: "gpt-5.4-mini" }], { refreshed: [refreshed] }); + const providerConfig = sdkProviderConfig(); + const run = await resolveAgents([{ name: "mini54-whoami", model: "gpt-5.4-mini" }], { + refreshed: [refreshed], + providerConfig, + refreshProviderConfig: data => Promise.resolve(resolveMultiProviderFromReflect({ model: "gpt-5.3-codex", reflectData: data })), + }); expect(run.error).toBeNull(); expect(run.refetch).toHaveBeenCalledTimes(1); - expect(run.resolutions[0].model).toBe("gpt-5.4-mini"); + expect(run.resolutions[0].model).toBe(qualifyModelForMultiProvider("gpt-5.4-mini", resolveMultiProviderFromReflect({ model: "gpt-5.3-codex", reflectData: refreshed }))); }); it("case 4: the qualified alias copilot/opus resolves to the newest live Opus", async () => { @@ -4776,8 +4791,43 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 expect(run.records).toEqual([{ engine: "copilot", role: "reviewer", configured_model: "copilot/opus", resolved_model: "claude-opus-5.5", provider: "copilot", status: "resolved", cause: null }]); }); - it("case 5: a sub-agent without model inherits the main model and gets no record", async () => { - const run = await resolveAgents([{ name: "helper" }]); + it("preserves explicit Copilot provider scope during liveness checks and SDK qualification", async () => { + const reflect = recordedReflect(); + reflect.endpoints.push({ provider: "openai", configured: true, models: ["gpt-5.4-mini"] }); + const run = await resolveAgents([{ name: "mini54-whoami", model: "copilot/gpt-5.4-mini" }], { + reflect, + refreshed: [reflect], + providerConfig: { + providers: [{ name: "copilot-responses" }, { name: "openai" }], + models: [{ id: "gpt-5.4-mini", provider: "openai" }], + }, + }); + expect(run.error).toBeInstanceOf(ModelAliasResolutionError); + expect(run.error.reason).toBe("model_unavailable"); + expect(run.records).toEqual([{ engine: "copilot", role: "mini54-whoami", configured_model: "copilot/gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }]); + }); + + it("records qualification ambiguity as a failed sub-agent and continues collecting failures", async () => { + const providerConfig = sdkProviderConfig(); + const duplicate = providerConfig.models.find(model => model.id === "claude-haiku-5.5"); + providerConfig.providers.push({ name: "copilot-duplicate" }); + providerConfig.models.push({ ...duplicate, provider: "copilot-duplicate" }); + const run = await resolveAgents( + [ + { name: "ambiguous", model: "claude-haiku-5.5" }, + { name: "also-dead", model: "gpt-5.4-mini" }, + ], + { providerConfig, refreshed: [recordedReflect()] } + ); + expect(run.error).toBeInstanceOf(ModelAliasResolutionError); + expect(run.records).toEqual([ + { engine: "copilot", role: "ambiguous", configured_model: "claude-haiku-5.5", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }, + { engine: "copilot", role: "also-dead", configured_model: "gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }, + ]); + }); + + it("case 5: an inherited sub-agent model inherits the main model and gets no record", async () => { + const run = await resolveAgents([{ name: "helper", model: "inherited" }]); expect(run.error).toBeNull(); expect(run.resolutions).toEqual([]); expect(run.records).toBeNull(); @@ -4787,7 +4837,13 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 const providerConfig = sdkProviderConfig(); const expected = qualifyModelForMultiProvider("claude-haiku-5.5", providerConfig); expect(expected).toMatch(/^copilot-(?:responses|completions)\/claude-haiku-5\.5$/); - const run = await resolveAgents([{ name: "haiku-small", model: "small" }, { name: "helper" }], { providerConfig }); + const run = await resolveAgents( + [ + { name: "haiku-small", model: "small" }, + { name: "helper", model: "inherited" }, + ], + { providerConfig } + ); expect(run.error).toBeNull(); expect(run.resolutions).toEqual([{ name: "haiku-small", configuredModel: "small", resolvedModel: "claude-haiku-5.5", model: expected }]); @@ -4796,7 +4852,7 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 const base = tempDir("copilot-subagent-sdk-session-"); fs.mkdirSync(path.join(base, ".github/agents"), { recursive: true }); fs.writeFileSync(path.join(base, ".github/agents/haiku-small.agent.md"), "---\nmodel: small\n---\nWho are you?"); - fs.writeFileSync(path.join(base, ".github/agents/helper.agent.md"), "---\ndescription: helper\n---\nHelp."); + fs.writeFileSync(path.join(base, ".github/agents/helper.agent.md"), "---\ndescription: helper\nmodel: inherited\n---\nHelp."); vi.stubEnv("GH_AW_SESSION_STATE_BASE_DIR", base); vi.stubEnv("COPILOT_MODEL", ""); vi.stubEnv("GH_AW_MODEL_FALLBACK", ""); @@ -4834,6 +4890,7 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 const exact = "---\nmodel: claude-haiku-4.5\n---\nExact.\n"; fs.writeFileSync(path.join(agentsDir, "haiku-small.agent.md"), small); fs.writeFileSync(path.join(agentsDir, "haiku-whoami.agent.md"), exact); + initGitRepo(dir); const awfConfigPath = path.join(dir, "awf-config.json"); fs.copyFileSync(awfConfigFixturePath, awfConfigPath); const resolutions = await applyCopilotCLISubagentModels({ reflectData: recordedReflect(), agentsDir, env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, logger: () => {} }); @@ -4841,10 +4898,56 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 ["haiku-small", "claude-haiku-5.5"], ["haiku-whoami", "claude-haiku-4.5"], ]); - expect(fs.readFileSync(path.join(agentsDir, "haiku-small.agent.md"), "utf8")).toBe(small.replace("model: small # alias", 'model: "claude-haiku-5.5"')); + expect(fs.readFileSync(path.join(agentsDir, "haiku-small.agent.md"), "utf8")).toBe(small.replace("model: small # alias", 'model: "claude-haiku-5.5" # alias')); expect(fs.readFileSync(path.join(agentsDir, "haiku-whoami.agent.md"), "utf8")).toBe(exact); }); + it("lists only .agent.md files and derives fallback names from that suffix", () => { + const dir = tempDir("copilot-subagent-list-"); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "notes.md"), "---\nmodel: gpt-5.4-mini\n---\nNot an agent."); + fs.writeFileSync(path.join(dir, "use-filename.agent.md"), "---\nmodel: claude-haiku-4.5\n---\nAgent."); + expect(listCopilotAgentFiles(dir, () => {})).toEqual([{ name: "use-filename", path: path.join(dir, "use-filename.agent.md"), model: "claude-haiku-4.5" }]); + }); + + it("loads the CLI harness without importing the SDK-only undici dependency", () => { + const dir = tempDir("copilot-harness-no-sdk-dependency-"); + const preloadPath = path.join(dir, "block-undici.cjs"); + fs.writeFileSync( + preloadPath, + `const Module = require("module"); +const load = Module._load; +Module._load = function(request, ...args) { + if (request === "undici") throw new Error("undici is unavailable"); + return load.call(this, request, ...args); +};` + ); + const result = spawnSync(process.execPath, ["--require", preloadPath, "-e", `require(${JSON.stringify(require.resolve("./copilot_harness.cjs"))})`], { + cwd: path.dirname(require.resolve("./copilot_harness.cjs")), + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + }); + + it("rewrites the complete multiline model scalar in CLI custom agents", async () => { + const dir = tempDir("copilot-subagent-cli-multiline-"); + const agentsDir = path.join(dir, ".github/agents"); + fs.mkdirSync(agentsDir, { recursive: true }); + const original = "---\nname: multiline\nmodel: >-\n small\ndescription: retained\n---\nPrompt retained.\n"; + fs.writeFileSync(path.join(agentsDir, "multiline.agent.md"), original); + initGitRepo(dir); + const awfConfigPath = path.join(dir, "awf-config.json"); + fs.copyFileSync(awfConfigFixturePath, awfConfigPath); + const resolutions = await applyCopilotCLISubagentModels({ + reflectData: recordedReflect(), + agentsDir, + env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, + logger: () => {}, + }); + expect(resolutions[0].model).toBe("claude-haiku-5.5"); + expect(fs.readFileSync(path.join(agentsDir, "multiline.agent.md"), "utf8")).toBe('---\nname: multiline\nmodel: "claude-haiku-5.5"\ndescription: retained\n---\nPrompt retained.\n'); + }); + it("case 6 (CLI, tracked file): a rewritten repository agent file stays out of git status and commits", async () => { const { execFileSync } = require("child_process"); const dir = tempDir("copilot-subagent-cli-tracked-"); @@ -4859,17 +4962,93 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 fs.copyFileSync(awfConfigFixturePath, awfConfigPath); await applyCopilotCLISubagentModels({ reflectData: recordedReflect(), agentsDir, env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, logger: () => {} }); expect(fs.readFileSync(path.join(agentsDir, "repo-small.agent.md"), "utf8")).toBe('---\nmodel: "claude-haiku-5.5"\n---\nRepository agent.\n'); + expect(git("ls-files", "-t", "--", ".github/agents/repo-small.agent.md").startsWith("S ")).toBe(true); git("add", "-A"); expect(git("status", "--porcelain", "--", ".github")).toBe(""); expect(git("diff", "--cached", "--name-only", "--", ".github")).toBe(""); }); + it("clears skip-worktree after CLI and preserves edits to a tracked agent file", async () => { + const { execFileSync } = require("child_process"); + const dir = tempDir("copilot-subagent-cli-restore-"); + const git = (/** @type {string[]} */ ...args) => execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], { cwd: dir, encoding: "utf8" }); + git("init", "-q"); + const agentsDir = path.join(dir, ".github/agents"); + fs.mkdirSync(agentsDir, { recursive: true }); + const agentPath = path.join(agentsDir, "repo-small.agent.md"); + fs.writeFileSync(agentPath, "---\nmodel: small\n---\nOriginal prompt.\n"); + git("add", "-A"); + git("commit", "-q", "-m", "init"); + const awfConfigPath = path.join(dir, "awf-config.json"); + fs.copyFileSync(awfConfigFixturePath, awfConfigPath); + const rewrittenFiles = []; + await applyCopilotCLISubagentModels({ + reflectData: recordedReflect(), + agentsDir, + rewrittenFiles, + env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, + logger: () => {}, + }); + fs.writeFileSync(agentPath, '---\nmodel: "claude-haiku-5.5"\n---\nEdited prompt.\n'); + expect(git("status", "--porcelain", "--", ".github")).toBe(""); + expect(restoreCopilotCLIAgentFiles(rewrittenFiles, () => {})).toBe(true); + expect(git("ls-files", "-t", "--", ".github/agents/repo-small.agent.md").startsWith("H ")).toBe(true); + expect(fs.readFileSync(agentPath, "utf8")).toBe("---\nmodel: small\n---\nEdited prompt.\n"); + expect(git("status", "--porcelain", "--", ".github")).toContain(".github/agents/repo-small.agent.md"); + }); + + it.each(["tracking-state-error", "skip-worktree-error", "skip-worktree-not-retained"])("restores the original tracked agent file when %s", async failureMode => { + const { execFileSync } = require("child_process"); + const childProcess = require("child_process"); + const dir = tempDir("copilot-subagent-cli-skip-failure-"); + const git = (...args) => execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], { cwd: dir, encoding: "utf8" }); + git("init", "-q"); + const agentsDir = path.join(dir, ".github/agents"); + fs.mkdirSync(agentsDir, { recursive: true }); + const agentPath = path.join(agentsDir, "repo-small.agent.md"); + const original = "---\nmodel: small\n---\nOriginal prompt.\n"; + fs.writeFileSync(agentPath, original); + git("add", "-A"); + git("commit", "-q", "-m", "init"); + const awfConfigPath = path.join(dir, "awf-config.json"); + fs.copyFileSync(awfConfigFixturePath, awfConfigPath); + const realSpawnSync = childProcess.spawnSync; + vi.spyOn(childProcess, "spawnSync").mockImplementation((command, args, options) => { + if (command === "git" && args[0] === "ls-files" && args.includes("--error-unmatch")) { + if (failureMode === "tracking-state-error") return { status: 128, stdout: "", stderr: "index unavailable" }; + return { status: 0, stdout: ".github/agents/repo-small.agent.md\n", stderr: "" }; + } + if (command === "git" && args[0] === "update-index" && args.includes("--skip-worktree")) { + return { status: failureMode === "skip-worktree-error" ? 1 : 0, stdout: "", stderr: "index unavailable" }; + } + if (command === "git" && args[0] === "ls-files" && args.includes("-t")) return { status: 0, stdout: "H .github/agents/repo-small.agent.md\n", stderr: "" }; + return realSpawnSync(command, args, options); + }); + vi.spyOn(process, "exit").mockImplementation(code => { + throw new Error(`exit ${code}`); + }); + await expect( + applyCopilotCLISubagentModels({ + reflectData: recordedReflect(), + agentsDir, + env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, + logger: () => {}, + }) + ).rejects.toThrow("exit 1"); + expect(fs.readFileSync(agentPath, "utf8")).toBe(original); + }); + /** Spawn the harness against the recorded catalogue with the given workspace agent files. */ - async function runHarness({ sdk = false, agents, reflect = recordedReflect(), env: envOverrides = {} }) { + async function runHarness({ sdk = false, agents, workspaceAgents = {}, noop = false, reflect = recordedReflect(), env: envOverrides = {} }) { const dir = tempDir("copilot-subagent-harness-"); const workspace = path.join(dir, "workspace"); + const engineCwd = envOverrides.GH_AW_ENGINE_CWD || workspace; + fs.mkdirSync(path.join(engineCwd, ".github/agents"), { recursive: true }); fs.mkdirSync(path.join(workspace, ".github/agents"), { recursive: true }); - for (const [name, content] of Object.entries(agents)) fs.writeFileSync(path.join(workspace, ".github/agents", name), content); + for (const [name, content] of Object.entries(agents)) fs.writeFileSync(path.join(engineCwd, ".github/agents", name), content); + for (const [name, content] of Object.entries(workspaceAgents)) fs.writeFileSync(path.join(workspace, ".github/agents", name), content); + initGitRepo(workspace); + if (engineCwd !== workspace) initGitRepo(engineCwd); const ostmp = path.join(dir, "ostmp"); fs.mkdirSync(ostmp); const startedPath = path.join(dir, "copilot-started.json"); @@ -4900,7 +5079,9 @@ if (${providerPort} && process.argv[1].endsWith("copilot_harness.cjs")) require( `const fs = require("fs"); const path = require("path"); const args = process.argv.slice(2); -fs.writeFileSync(${JSON.stringify(startedPath)}, JSON.stringify({ args, subagentModels: process.env.GH_AW_COPILOT_SDK_SUBAGENT_MODELS || null })); +const agentsDir = path.join(process.env.GH_AW_ENGINE_CWD || process.env.GITHUB_WORKSPACE || process.cwd(), ".github", "agents"); +const agentFiles = fs.existsSync(agentsDir) ? Object.fromEntries(fs.readdirSync(agentsDir).filter(name => name.endsWith(".agent.md")).map(name => [name, fs.readFileSync(path.join(agentsDir, name), "utf8")])) : {}; +fs.writeFileSync(${JSON.stringify(startedPath)}, JSON.stringify({ args, agentFiles, subagentModels: process.env.GH_AW_COPILOT_SDK_SUBAGENT_MODELS || null })); if (args.includes("--session-id")) { const model = args.includes("--model") ? args[args.lastIndexOf("--model") + 1] : process.env.COPILOT_MODEL; const sessionDir = path.join(process.env.COPILOT_HOME, "session-state", args[args.indexOf("--session-id") + 1]); @@ -4913,11 +5094,14 @@ process.stdout.write("done\\n");` fs.writeFileSync(sidecarPath, `#!/usr/bin/env node\nconst args = process.argv.slice(2);\nrequire("net").createServer(socket => socket.end()).listen(Number(args[args.indexOf("--port") + 1]), "127.0.0.1");\n`, { mode: 0o755 }); const awfConfigPath = path.join(dir, "awf-config.json"); fs.copyFileSync(awfConfigFixturePath, awfConfigPath); + const safeOutputsPath = path.join(dir, "safe-outputs.jsonl"); + if (noop) fs.writeFileSync(safeOutputsPath, '{"type":"noop","message":"nothing to do"}\n'); const env = { ...harnessChildEnv, GH_AW_TMP_DIR: dir, TMPDIR: ostmp, GITHUB_WORKSPACE: workspace, + ...(noop ? { GH_AW_SAFE_OUTPUTS: safeOutputsPath } : {}), COPILOT_HOME: path.join(dir, "copilot-home"), GH_AW_HARNESS_MAX_RETRIES: "0", GH_AW_HARNESS_STALL_WARNING_MS: "0", @@ -4939,6 +5123,7 @@ process.stdout.write("done\\n");` "GH_AW_COPILOT_SDK_SUBAGENT_MODELS", ]) delete env[key]; + if (noop) env.GH_AW_SAFE_OUTPUTS = safeOutputsPath; Object.assign(env, { AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath, COPILOT_MODEL: "gpt-5.3-codex", ...(sdk ? { COPILOT_SDK_URI: `http://127.0.0.1:${sdkPort}` } : {}) }, envOverrides); const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, ...(sdk ? [stubPath, sidecarPath] : [stubPath, "--prompt", "Hello"])], { cwd: path.dirname(require.resolve("./copilot_harness.cjs")), @@ -4949,9 +5134,10 @@ process.stdout.write("done\\n");` return { result, workspace, + engineCwd, started: fs.existsSync(startedPath) ? JSON.parse(fs.readFileSync(startedPath, "utf8")) : null, records: readRecords(dir), - agentFile: name => fs.readFileSync(path.join(workspace, ".github/agents", name), "utf8"), + agentFile: name => fs.readFileSync(path.join(engineCwd, ".github/agents", name), "utf8"), }; } @@ -4975,11 +5161,34 @@ process.stdout.write("done\\n");` expect(run.agentFile("mini54-whoami.agent.md")).toBe(smokeAgents["mini54-whoami.agent.md"]); }); + it("skips CLI sub-agent validation when a noop is already present", async () => { + const run = await runHarness({ agents: { "dead.agent.md": "---\nmodel: gpt-5.4-mini\n---\nUnused.\n" }, noop: true }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started).toBeNull(); + expect(run.result.stderr).toContain("pre-flight: noop message found in safe-outputs"); + expect(run.result.stderr).not.toContain("copilot sub-agent"); + expect(run.records).toEqual([{ engine: "copilot", role: "main", configured_model: "gpt-5.3-codex", resolved_model: "gpt-5.3-codex", provider: null, status: "exact", cause: null }]); + }); + + it("scans only custom agents in the CLI's effective engine working directory", async () => { + const dir = tempDir("copilot-subagent-engine-cwd-"); + const engineCwd = path.join(dir, "engine-checkout"); + const run = await runHarness({ + agents: { "effective.agent.md": "---\nmodel: small\n---\nEffective checkout.\n" }, + workspaceAgents: { "unrelated.agent.md": "---\nmodel: gpt-5.4-mini\n---\nNot loaded by this CLI.\n" }, + env: { GH_AW_ENGINE_CWD: engineCwd }, + }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started).not.toBeNull(); + expect(run.records).toContainEqual({ engine: "copilot", role: "effective", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }); + }); + it("case 6 end to end (CLI): Copilot CLI starts with the resolved model in the agent file", async () => { const run = await runHarness({ agents: { "haiku-small.agent.md": "---\nname: haiku-small\nmodel: small\n---\nWho are you?\n" } }); expect(run.result.status, run.result.stderr).toBe(0); expect(run.started).not.toBeNull(); - expect(run.agentFile("haiku-small.agent.md")).toBe('---\nname: haiku-small\nmodel: "claude-haiku-5.5"\n---\nWho are you?\n'); + expect(run.started.agentFiles["haiku-small.agent.md"]).toBe('---\nname: haiku-small\nmodel: "claude-haiku-5.5"\n---\nWho are you?\n'); + expect(run.agentFile("haiku-small.agent.md")).toBe("---\nname: haiku-small\nmodel: small\n---\nWho are you?\n"); expect(run.records).toContainEqual({ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }); }); diff --git a/actions/setup/js/copilot_sdk_session.cjs b/actions/setup/js/copilot_sdk_session.cjs index aabbbd641cb..66aebf2438e 100644 --- a/actions/setup/js/copilot_sdk_session.cjs +++ b/actions/setup/js/copilot_sdk_session.cjs @@ -386,7 +386,7 @@ async function runWithCopilotSDK({ // The harness resolves declared sub-agent models (aliases included) before the driver starts. const resolvedSubagentModels = parseSubagentModelsJson(process.env.GH_AW_COPILOT_SDK_SUBAGENT_MODELS); const customAgents = loadCopilotSDKCustomAgents(workingDirectory, agentsBaseDir, log).map(agent => { - if (!agent.model) return { ...agent, model: sessionModel || undefined }; + if (!agent.model || agent.model.trim().toLowerCase() === "inherited") return { ...agent, model: sessionModel || undefined }; const requested = Object.prototype.hasOwnProperty.call(resolvedSubagentModels, agent.name) ? resolvedSubagentModels[agent.name] : agent.model; const qualified = providerConfig ? qualifyModelForMultiProvider(requested, providerConfig) : requested; if (!qualified) throw new Error(`Cannot qualify custom agent "${agent.name}" model "${agent.model}" against the configured provider models`); From b9f7f48a3baa4e58e9509232f6eaaea674a942ff Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:14:50 +0000 Subject: [PATCH 5/5] Fail closed only for declared Copilot sub-agents, skip detection, write CLI personal-agent copies Co-authored-by: SivaKesava1 <11771739+SivaKesava1@users.noreply.github.com> --- actions/setup/js/copilot_harness.cjs | 229 ++++++------- actions/setup/js/copilot_harness.test.cjs | 329 ++++++++++++------- actions/setup/js/copilot_sdk_driver.test.cjs | 21 +- actions/setup/js/copilot_sdk_session.cjs | 21 +- docs/src/content/docs/reference/engines.md | 25 +- 5 files changed, 355 insertions(+), 270 deletions(-) diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 00b39ec624f..4e967a97015 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -105,7 +105,8 @@ const { loadModelsJson } = require("./model_costs.cjs"); const { applyCopilotWireAPI: configureCopilotWireAPI, warnCopilotSubagentEndpoints, recordModelEndpointMismatch, getSupportedEndpoints, MODEL_ENDPOINT_FIX } = require("./copilot_wire_api.cjs"); const { loadModelAliasMap, resolveEngineModel, normalizeForCopilotCLI, ModelAliasResolutionError, recordModelResolution, splitModelIdentifier } = require("./resolve_model_alias.cjs"); const { createCopilotModelObserver, recordCopilotModelSelection } = require("./copilot_model_selection.cjs"); -const { loadCopilotSDKCustomAgents } = require("./extract_inline_sub_agents.cjs"); +const { getEngineSubAgentTarget } = require("./extract_inline_sub_agents.cjs"); +const { inlineArtifactTarget } = require("./inline_artifact_target.cjs"); const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported|No model available\b[^\n]*policy enablement/i; const { parseAICreditsErrorInfoFromAuditLog, parseMaxAICreditsFromAuditLog, parseMaxAICreditsExceededFromAuditLog, parseAPIProxyGuardRejectionFromEventLog, formatAPIProxyGuardRejection } = require("./ai_credits_context.cjs"); @@ -691,6 +692,27 @@ function listCopilotAgentFiles(directory, logger = log) { return agents; } +/** + * List the Copilot sub-agents of a run. Repository agents come from + * `/.github/agents`; workflow-declared inline sub-agents come from the + * directory they are staged in under `agentsBaseDir` (the one `loadCopilotSDKCustomAgents` + * reads). An inline sub-agent overrides a repository agent with the same name and is declared. + * @param {string} workingDirectory + * @param {string} agentsBaseDir + * @param {(msg: string) => void} [logger] + * @returns {Array<{ name: string, model?: string, path: string, declared: boolean }>} + */ +function listCopilotSubagents(workingDirectory, agentsBaseDir, logger = log) { + const declaredDir = path.resolve(agentsBaseDir, inlineArtifactTarget("SUB_AGENT", getEngineSubAgentTarget("copilot")).dir); + const repositoryDir = path.resolve(workingDirectory, ".github", "agents"); + const agents = new Map(); + if (repositoryDir !== declaredDir) { + for (const agent of listCopilotAgentFiles(repositoryDir, logger)) agents.set(agent.name, { ...agent, declared: false }); + } + for (const agent of listCopilotAgentFiles(declaredDir, logger)) agents.set(agent.name, { ...agent, declared: true }); + return [...agents.values()]; +} + /** * Locate the top-level YAML model scalar in a Copilot agent file. * @param {string} content @@ -728,12 +750,16 @@ function replaceCopilotAgentFileModelContent(content, model) { } /** - * Replace the top-level frontmatter `model:` scalar of a Copilot agent file. The rest of the - * file, including comments and the body, is preserved byte for byte. + * Write a copy of a Copilot agent file to Copilot CLI's personal agents directory with only + * the frontmatter `model:` scalar replaced. The copy keeps the file name, which Copilot CLI + * uses as the agent ID, so it overrides the repository agent with the same ID. The source + * file is never modified. * @param {string} agentPath + * @param {string} personalAgentsDir * @param {string} model + * @returns {string} The path of the copy. */ -function rewriteCopilotAgentFileModel(agentPath, model) { +function writeCopilotPersonalAgentCopy(agentPath, personalAgentsDir, model) { let content; try { content = fs.readFileSync(agentPath, "utf8"); @@ -741,11 +767,14 @@ function rewriteCopilotAgentFileModel(agentPath, model) { throw new Error(`cannot read agent file ${agentPath}: ${getErrorMessage(error)}`, { cause: error }); } const updated = replaceCopilotAgentFileModelContent(content, model); + const target = path.join(personalAgentsDir, path.basename(agentPath)); try { - fs.writeFileSync(agentPath, updated); + fs.mkdirSync(personalAgentsDir, { recursive: true }); + fs.writeFileSync(target, updated); } catch (error) { - throw new Error(`cannot rewrite model in ${agentPath}: ${getErrorMessage(error)}`, { cause: error }); + throw new Error(`cannot write personal agent ${target}: ${getErrorMessage(error)}`, { cause: error }); } + return target; } /** @@ -757,20 +786,24 @@ function rewriteCopilotAgentFileModel(agentPath, model) { * not recorded. * * Every sub-agent with a model gets a `model.resolution` record (role = agent name), so - * several sub-agents sharing one dead model each get a `failed` record. When any sub-agent - * fails, the first failure is thrown as a ModelAliasResolutionError after all records are - * written; callers must not start Copilot. `/reflect` is refreshed at most once. + * several sub-agents sharing one dead model each get a `failed` record. Workflow-declared + * sub-agents (the default) fail closed: when any of them fails, the first failure is thrown + * as a ModelAliasResolutionError after all records are written; callers must not start + * Copilot. A repository agent (`declared: false`) that fails gets a warning and a `failed` + * record, is left out of the result, and does not stop the run. `/reflect` is refreshed at + * most once. * * @param {{ - * agents: Array<{ name: string, model?: string }>, + * agents: Array<{ name: string, model?: string, path?: string, declared?: boolean }>, * reflectData: any, * refetchReflectData?: () => Promise, * providerConfig?: { providers: Array<{name: string}>, models: Array<{id: string, provider: string}> } | null, * refreshProviderConfig?: (reflectData: any) => Promise<{ providers: Array<{name: string}>, models: Array<{id: string, provider: string}> } | null>, * aliasMap?: Record | null, + * repositoryFallback?: string, * env?: NodeJS.ProcessEnv, * logger?: (msg: string) => void, - * }} options + * }} options `repositoryFallback` describes what happens to a repository agent whose model fails. * @returns {Promise>} * `resolvedModel` is the resolved Copilot id; `model` is the id to hand to Copilot (qualified in SDK mode). */ @@ -805,10 +838,20 @@ async function resolveCopilotSubagentModels(options) { const resolutions = []; for (const agent of declared) { const configuredModel = String(agent.model).trim(); + const repositoryAgent = agent.declared === false; + // A repository agent the workflow may never use must not make the run look like an + // unsupported-model failure, so its resolver failure is reported by the warning below. + const agentLogger = repositoryAgent ? (/** @type {string} */ message) => (MODEL_NOT_SUPPORTED_PATTERN.test(message) ? undefined : logger(message)) : logger; const scope = scopeModelToReflectProviders(configuredModel); const fail = (/** @type {InstanceType} */ failure, /** @type {string} */ detail) => { - logger(`copilot sub-agent '${agent.name}': ${failure.message}${detail ? ` ${detail}` : ""} Refusing to start Copilot with a different sub-agent model.`); recordModelResolution({ engine: "copilot", role: agent.name, configured_model: configuredModel, resolved_model: null, provider: failure.provider || null, status: "failed", cause: failure.reason }, env); + if (repositoryAgent) { + logger( + `warning: repository agent '${agent.name}' (${agent.path || "unknown file"}) declares model '${configuredModel}', which could not be resolved (cause: ${failure.reason}); ${options.repositoryFallback || "the agent is left unchanged"}.` + ); + return; + } + logger(`copilot sub-agent '${agent.name}': ${failure.message}${detail ? ` ${detail}` : ""} Refusing to start Copilot with a different sub-agent model.`); failures.push(failure); }; let result; @@ -824,7 +867,7 @@ async function resolveCopilotSubagentModels(options) { logPrefix: `copilot sub-agent '${agent.name}' model alias resolution`, normalize: r => normalizeForCopilotCLI(r.resolvedModel), env, - logger, + logger: agentLogger, }); } catch (failure) { if (!(failure instanceof ModelAliasResolutionError)) throw failure; @@ -874,16 +917,20 @@ async function resolveCopilotSubagentModels(options) { } /** - * CLI mode: resolve the sub-agent models of the agent files Copilot CLI loads from - * `/.github/agents` (repository agents and restored inline sub-agents) and write - * each resolved id back into the file's `model:` when it differs from the declared value. - * Stops the process before Copilot starts when a sub-agent model cannot be resolved. + * CLI mode: resolve the sub-agent models of the agents Copilot CLI loads (repository agents + * in the engine working directory and workflow-declared inline sub-agents) and, for each + * resolved id that differs from the declared value, write a copy of the agent file with the + * resolved `model:` to Copilot CLI's personal agents directory + * (`${COPILOT_HOME:-$HOME/.copilot}/agents`). Files in the checkout are never modified. + * Stops the process before Copilot starts when a declared sub-agent model cannot be resolved + * or its copy cannot be written; repository agents only produce warnings. * * @param {{ * reflectData: any, * refetchReflectData?: () => Promise, - * agentsDir?: string, - * rewrittenFiles?: Array<{ path: string, configuredModel: string, resolvedModel: string, modelSource: string }>, + * workingDirectory?: string, + * agentsBaseDir?: string, + * personalAgentsDir?: string, * env?: NodeJS.ProcessEnv, * logger?: (msg: string) => void, * }} options @@ -892,11 +939,20 @@ async function resolveCopilotSubagentModels(options) { async function applyCopilotCLISubagentModels(options) { const logger = options.logger || log; const env = options.env || process.env; - const agentsDir = options.agentsDir || path.join(env.GH_AW_ENGINE_CWD || env.GITHUB_WORKSPACE || process.cwd(), ".github", "agents"); - const files = listCopilotAgentFiles(agentsDir, logger); + const workingDirectory = options.workingDirectory || path.resolve(env.GITHUB_WORKSPACE || process.cwd(), env.GH_AW_ENGINE_CWD || "."); + const agentsBaseDir = options.agentsBaseDir || path.join(os.tmpdir(), "gh-aw"); + const personalAgentsDir = options.personalAgentsDir || path.join(env.COPILOT_HOME || path.join(env.HOME || os.homedir(), ".copilot"), "agents"); + const agents = listCopilotSubagents(workingDirectory, agentsBaseDir, logger); let resolutions; try { - resolutions = await resolveCopilotSubagentModels({ agents: files, reflectData: options.reflectData, refetchReflectData: options.refetchReflectData, env, logger }); + resolutions = await resolveCopilotSubagentModels({ + agents, + reflectData: options.reflectData, + refetchReflectData: options.refetchReflectData, + repositoryFallback: "Copilot CLI uses the agent file unchanged", + env, + logger, + }); } catch (failure) { if (!(failure instanceof ModelAliasResolutionError)) throw failure; process.exit(1); @@ -904,106 +960,24 @@ async function applyCopilotCLISubagentModels(options) { } for (const resolution of resolutions) { if (resolution.model === resolution.configuredModel) continue; - for (const file of files.filter(file => file.name === resolution.name && file.model === resolution.configuredModel)) { - let original; - let modelScalar; - try { - original = fs.readFileSync(file.path, "utf8"); - modelScalar = getCopilotAgentModelScalar(original); - rewriteCopilotAgentFileModel(file.path, resolution.model); - } catch (error) { - if (typeof original === "string") { - try { - fs.writeFileSync(file.path, original); - } catch (restoreError) { - logger(`warning: cannot restore ${file.path} after a rewrite failure: ${getErrorMessage(restoreError)}`); - } - } - if (options.rewrittenFiles) restoreCopilotCLIAgentFiles(options.rewrittenFiles, logger); - logger(`copilot sub-agent '${resolution.name}': cannot rewrite ${file.path}: ${getErrorMessage(error)}. Refusing to start Copilot.`); - process.exit(1); - return []; - } - const trackingState = hideTrackedAgentFileRewrite(file.path, logger); - if (!trackingState) { - fs.writeFileSync(file.path, original); - if (options.rewrittenFiles) restoreCopilotCLIAgentFiles(options.rewrittenFiles, logger); - logger(`copilot sub-agent '${resolution.name}': cannot keep the resolved model out of commits for tracked file ${file.path}. Refusing to start Copilot with an unresolved sub-agent model.`); - process.exit(1); - return []; - } - if (trackingState === true) { - options.rewrittenFiles?.push({ path: file.path, configuredModel: resolution.configuredModel, resolvedModel: resolution.model, modelSource: modelScalar.source }); - } - logger(`copilot sub-agent '${resolution.name}': wrote model '${resolution.model}' to ${file.path}`); - } - } - return resolutions; -} - -/** - * Keep a harness rewrite of a git-tracked agent file out of the agent's commits by marking it - * skip-worktree until the harness restores the model and clears the flag. Untracked files need - * nothing. Returns false when tracking state or the index flag cannot be verified. - * @param {string} agentPath - * @param {(msg: string) => void} logger - * @returns {boolean|"untracked"} - */ -function hideTrackedAgentFileRewrite(agentPath, logger) { - const { spawnSync } = require("child_process"); - const cwd = path.dirname(agentPath); - const file = path.basename(agentPath); - const tracked = spawnSync("git", ["ls-files", "--error-unmatch", "--", file], { cwd, encoding: "utf8", timeout: 5000 }); - if (tracked.error || (tracked.status !== 0 && tracked.status !== 1)) { - logger(`warning: git ls-files could not determine tracking state for ${agentPath}: ${tracked.error ? getErrorMessage(tracked.error) : String(tracked.stderr || "").trim()}`); - return false; - } - if (tracked.status === 1) return "untracked"; - const hidden = spawnSync("git", ["update-index", "--skip-worktree", "--", file], { cwd, encoding: "utf8", timeout: 5000 }); - if (hidden.error || hidden.status !== 0) { - logger(`warning: git update-index --skip-worktree failed for ${agentPath}: ${hidden.error ? getErrorMessage(hidden.error) : String(hidden.stderr || "").trim()}`); - return false; - } - const verified = spawnSync("git", ["ls-files", "-t", "--", file], { cwd, encoding: "utf8", timeout: 5000 }); - if (verified.error || verified.status !== 0 || !/^S\s/.test(verified.stdout || "")) { - logger(`warning: git did not retain skip-worktree for ${agentPath}: ${verified.error ? getErrorMessage(verified.error) : String(verified.stderr || verified.stdout || "").trim()}`); - spawnSync("git", ["update-index", "--no-skip-worktree", "--", file], { cwd, encoding: "utf8", timeout: 5000 }); - return false; - } - return true; -} - -/** - * Restore harness-only model substitutions while preserving any other agent-file edits. - * @param {Array<{ path: string, configuredModel: string, resolvedModel: string, modelSource: string }>} rewrittenFiles - * @param {(msg: string) => void} logger - * @returns {boolean} - */ -function restoreCopilotCLIAgentFiles(rewrittenFiles, logger = log) { - const { spawnSync } = require("child_process"); - let success = true; - for (const file of rewrittenFiles) { - const cwd = path.dirname(file.path); - const name = path.basename(file.path); - const cleared = spawnSync("git", ["update-index", "--no-skip-worktree", "--", name], { cwd, encoding: "utf8", timeout: 5000 }); - const verified = cleared.error || cleared.status !== 0 ? null : spawnSync("git", ["ls-files", "-t", "--", name], { cwd, encoding: "utf8", timeout: 5000 }); - if (!verified || verified.error || verified.status !== 0 || /^S\s/.test(verified.stdout || "")) { - logger(`warning: cannot clear skip-worktree for ${file.path}: ${cleared.error ? getErrorMessage(cleared.error) : String(cleared.stderr || verified?.stderr || "").trim()}`); - success = false; - continue; - } + const agent = agents.find(candidate => candidate.name === resolution.name); + if (!agent) continue; try { - const content = fs.readFileSync(file.path, "utf8"); - const current = getCopilotAgentModelScalar(content); - if (current.value === file.resolvedModel && file.resolvedModel !== file.configuredModel) { - fs.writeFileSync(file.path, content.slice(0, current.start) + file.modelSource + content.slice(current.end)); - } + const target = writeCopilotPersonalAgentCopy(agent.path, personalAgentsDir, resolution.model); + logger(`copilot sub-agent '${resolution.name}': wrote model '${resolution.model}' to personal agent ${target}`); } catch (error) { - logger(`warning: cannot restore harness model substitution in ${file.path}: ${getErrorMessage(error)}`); - success = false; + if (!agent.declared) { + logger( + `warning: repository agent '${resolution.name}' (${agent.path}): cannot write personal agent copy with model '${resolution.model}' to ${personalAgentsDir}: ${getErrorMessage(error)}; Copilot CLI uses the agent file unchanged.` + ); + continue; + } + logger(`copilot sub-agent '${resolution.name}': cannot write personal agent copy of ${agent.path} to ${personalAgentsDir}: ${getErrorMessage(error)}. Refusing to start Copilot with an unresolved sub-agent model.`); + process.exit(1); + return []; // unreachable, keeps TypeScript control-flow analysis happy } } - return success; + return resolutions; } /** @@ -1723,7 +1697,6 @@ async function main() { const executionEvidence = captureExecutionEvidenceConfiguration(process.env); stripExecutionEvidenceEnv(process.env); const [, , command, ...args] = process.argv; - const cliAgentFileRestorations = []; const retryConfig = resolveRetryConfig(process.env, log); const { maxRetries, initialDelayMs, backoffMultiplier, maxDelayMs } = retryConfig; @@ -1850,12 +1823,13 @@ async function main() { } // Sub-agent models are resolved on routed runs too, but a pre-existing noop must not - // make startup depend on sub-agent model availability. - if (!copilotSDKMode) { + // make startup depend on sub-agent model availability. Threat detection uses no + // sub-agents, so it neither reads, resolves, records nor writes agent files. + const resolveSubagentModels = !isDetectionPhase(process.env.GH_AW_PHASE); + if (!copilotSDKMode && resolveSubagentModels) { await applyCopilotCLISubagentModels({ reflectData: awfReflectData, refetchReflectData, - rewrittenFiles: cliAgentFileRestorations, logger: log, }); } @@ -1915,10 +1889,11 @@ async function main() { // The same custom agents the SDK driver loads: workflow definitions override repository ones. const sdkWorkingDirectory = path.resolve(process.env.GITHUB_WORKSPACE || process.cwd(), process.env.GH_AW_ENGINE_CWD || "."); - const agents = loadCopilotSDKCustomAgents(sdkWorkingDirectory, path.join(os.tmpdir(), "gh-aw"), log); + const agents = resolveSubagentModels ? listCopilotSubagents(sdkWorkingDirectory, path.join(os.tmpdir(), "gh-aw"), log) : []; try { const resolutions = await resolveCopilotSubagentModels({ agents, + repositoryFallback: "the SDK session gives it the session model", reflectData: awfReflectData, refetchReflectData, providerConfig: multiProvider, @@ -2506,9 +2481,6 @@ async function main() { await fetchAWFReflect({ logger: log }); } } finally { - if (!restoreCopilotCLIAgentFiles(cliAgentFileRestorations, log)) { - lastExitCode = 1; - } await stopCopilotSDKServer(copilotSDKServer, { logger: log }); } log(`done: exitCode=${lastExitCode} totalDuration=${formatDuration(Date.now() - driverStartTime)}`); @@ -2591,9 +2563,8 @@ if (typeof module !== "undefined" && module.exports) { resolveCopilotSubagentModels, applyCopilotCLISubagentModels, listCopilotAgentFiles, - rewriteCopilotAgentFileModel, - hideTrackedAgentFileRewrite, - restoreCopilotCLIAgentFiles, + listCopilotSubagents, + writeCopilotPersonalAgentCopy, applyCopilotWireAPI, warnCopilotSubagentEndpoints, resolveAWFModelRoutingSelection, diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 0613b3cc7b8..3a0c7be383a 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -78,7 +78,7 @@ const { resolveCopilotSubagentModels, applyCopilotCLISubagentModels, listCopilotAgentFiles, - restoreCopilotCLIAgentFiles, + listCopilotSubagents, formatInferenceEndpointForLog, logCopilotInferenceConfiguration, resolveLongRunTokenThreshold, @@ -4904,24 +4904,110 @@ describe("Copilot sub-agent model resolution (recorded /reflect from run 3930457 expect(session.customAgents.find(agent => agent.name === "helper").model).toBe(session.model); }); - it("case 6 (CLI): the agent file Copilot CLI loads gets the resolved model; exact ids are left untouched", async () => { - const dir = tempDir("copilot-subagent-cli-files-"); - const agentsDir = path.join(dir, ".github/agents"); + const vscodePinnedAgent = "---\nname: planner\ndescription: Plans work\nmodel: GPT-5.4 (copilot)\n---\nPlan the work.\n"; + + it("a repository agent pinned to the VS Code name GPT-5.4 (copilot) records failed and warns without stopping the run", async () => { + // `model: GPT-5.4 (copilot)` is a real VS Code pin (adamgell/cmtraceopen planner.agent.md). + const agentPath = "/repo/.github/agents/planner.agent.md"; + const run = await resolveAgents([{ name: "planner", model: "GPT-5.4 (copilot)", declared: false, path: agentPath }], { refreshed: [recordedReflect()] }); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([]); + expect(run.records).toEqual([{ engine: "copilot", role: "planner", configured_model: "GPT-5.4 (copilot)", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }]); + const warning = run.logs.find(message => message.startsWith("warning: repository agent 'planner'")); + expect(warning).toContain(agentPath); + expect(warning).toContain("'GPT-5.4 (copilot)'"); + expect(warning).toContain("model_unavailable"); + expect(detectErrors(run.logs.join("\n")).modelNotSupportedError).toBe(false); + }); + + it("a repository agent with model small resolves to claude-haiku-5.5", async () => { + const run = await resolveAgents([{ name: "haiku-small", model: "small", declared: false, path: "/repo/.github/agents/haiku-small.agent.md" }]); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([{ name: "haiku-small", configuredModel: "small", resolvedModel: "claude-haiku-5.5", model: "claude-haiku-5.5" }]); + expect(run.records).toEqual([{ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }]); + }); + + it("SDK mode: a repository agent pinned to GPT-5.4 (copilot) gets the session model with a warning", async () => { + const providerConfig = sdkProviderConfig(); + const run = await resolveAgents( + [ + { name: "planner", model: "GPT-5.4 (copilot)", declared: false, path: "planner.agent.md" }, + { name: "haiku-small", model: "small", declared: false, path: "haiku-small.agent.md" }, + ], + { providerConfig, refreshed: [recordedReflect()] } + ); + expect(run.error).toBeNull(); + expect(run.records).toContainEqual({ engine: "copilot", role: "planner", configured_model: "GPT-5.4 (copilot)", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }); + expect(run.resolutions.map(resolution => resolution.name)).toEqual(["haiku-small"]); + + const { runWithCopilotSDK } = require("./copilot_sdk_session.cjs"); + const base = tempDir("copilot-subagent-sdk-repository-"); + fs.mkdirSync(path.join(base, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(base, ".github/agents/planner.agent.md"), vscodePinnedAgent); + fs.writeFileSync(path.join(base, ".github/agents/haiku-small.agent.md"), "---\nmodel: small\n---\nWho are you?"); + vi.stubEnv("GH_AW_SESSION_STATE_BASE_DIR", base); + vi.stubEnv("COPILOT_MODEL", ""); + vi.stubEnv("GH_AW_MODEL_FALLBACK", ""); + vi.stubEnv("GH_AW_MODEL_ROUTING", ""); + vi.stubEnv("GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON", JSON.stringify(providerConfig)); + vi.stubEnv("GH_AW_COPILOT_SDK_SUBAGENT_MODELS", JSON.stringify(Object.fromEntries(run.resolutions.map(resolution => [resolution.name, resolution.model])))); + const createSession = vi.fn().mockResolvedValue({ sessionId: "repository-agent-models", on: () => {}, sendAndWait: async () => ({ data: { content: "done" } }), disconnect: async () => {} }); + class FakeCopilotClient { + start = async () => {}; + stop = async () => {}; + createSession = createSession; + } + const logs = []; + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test", + logger: message => logs.push(message), + workingDirectory: base, + agentsBaseDir: base, + bare: true, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: () => ({}) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(0); + const session = createSession.mock.calls[0][0]; + expect(session.customAgents.find(agent => agent.name === "planner").model).toBe(session.model); + expect(session.customAgents.find(agent => agent.name === "haiku-small").model).toBe(qualifyModelForMultiProvider("claude-haiku-5.5", providerConfig)); + expect(logs).toContainEqual(expect.stringMatching(/warning: custom agent "planner" model "GPT-5\.4 \(copilot\)".*using session model/)); + }); + + it("CLI mode writes a personal agent copy with the same file name and the resolved model; the checkout stays untouched", async () => { + const { execFileSync } = require("child_process"); + const dir = tempDir("copilot-subagent-cli-personal-"); + const workspace = path.join(dir, "workspace"); + const agentsDir = path.join(workspace, ".github/agents"); fs.mkdirSync(agentsDir, { recursive: true }); - const small = "---\nname: haiku-small\ndescription: Small model\nmodel: small # alias\ntools: [read]\n---\nWho are you?\nmodel: small\n"; - const exact = "---\nmodel: claude-haiku-4.5\n---\nExact.\n"; - fs.writeFileSync(path.join(agentsDir, "haiku-small.agent.md"), small); - fs.writeFileSync(path.join(agentsDir, "haiku-whoami.agent.md"), exact); - initGitRepo(dir); + const files = { + "haiku-small.agent.md": "---\nname: haiku-small\ndescription: Small model\nmodel: small # alias\ntools: [read]\n---\nWho are you?\nmodel: small\n", + "haiku-whoami.agent.md": "---\nmodel: claude-haiku-4.5\n---\nExact.\n", + "multiline.agent.md": "---\nname: multiline\nmodel: >-\n small\ndescription: retained\n---\nPrompt retained.\n", + }; + for (const [name, content] of Object.entries(files)) fs.writeFileSync(path.join(agentsDir, name), content); + initGitRepo(workspace); const awfConfigPath = path.join(dir, "awf-config.json"); fs.copyFileSync(awfConfigFixturePath, awfConfigPath); - const resolutions = await applyCopilotCLISubagentModels({ reflectData: recordedReflect(), agentsDir, env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, logger: () => {} }); + const copilotHome = path.join(dir, "copilot-home"); + const resolutions = await applyCopilotCLISubagentModels({ + reflectData: recordedReflect(), + workingDirectory: workspace, + agentsBaseDir: path.join(dir, "gh-aw"), + env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath, COPILOT_HOME: copilotHome }, + logger: () => {}, + }); expect(resolutions.map(resolution => [resolution.name, resolution.model])).toEqual([ ["haiku-small", "claude-haiku-5.5"], ["haiku-whoami", "claude-haiku-4.5"], + ["multiline", "claude-haiku-5.5"], ]); - expect(fs.readFileSync(path.join(agentsDir, "haiku-small.agent.md"), "utf8")).toBe(small.replace("model: small # alias", 'model: "claude-haiku-5.5" # alias')); - expect(fs.readFileSync(path.join(agentsDir, "haiku-whoami.agent.md"), "utf8")).toBe(exact); + const personalAgentsDir = path.join(copilotHome, "agents"); + expect(fs.readdirSync(personalAgentsDir).sort()).toEqual(["haiku-small.agent.md", "multiline.agent.md"]); + expect(fs.readFileSync(path.join(personalAgentsDir, "haiku-small.agent.md"), "utf8")).toBe(files["haiku-small.agent.md"].replace("model: small # alias", 'model: "claude-haiku-5.5" # alias')); + expect(fs.readFileSync(path.join(personalAgentsDir, "multiline.agent.md"), "utf8")).toBe('---\nname: multiline\nmodel: "claude-haiku-5.5"\ndescription: retained\n---\nPrompt retained.\n'); + for (const [name, content] of Object.entries(files)) expect(fs.readFileSync(path.join(agentsDir, name), "utf8")).toBe(content); + expect(execFileSync("git", ["status", "--porcelain"], { cwd: workspace, encoding: "utf8" })).toBe(""); }); it("lists only .agent.md files and derives fallback names from that suffix", () => { @@ -4951,117 +5037,60 @@ Module._load = function(request, ...args) { expect(result.status, result.stderr).toBe(0); }); - it("rewrites the complete multiline model scalar in CLI custom agents", async () => { - const dir = tempDir("copilot-subagent-cli-multiline-"); - const agentsDir = path.join(dir, ".github/agents"); - fs.mkdirSync(agentsDir, { recursive: true }); - const original = "---\nname: multiline\nmodel: >-\n small\ndescription: retained\n---\nPrompt retained.\n"; - fs.writeFileSync(path.join(agentsDir, "multiline.agent.md"), original); - initGitRepo(dir); - const awfConfigPath = path.join(dir, "awf-config.json"); - fs.copyFileSync(awfConfigFixturePath, awfConfigPath); - const resolutions = await applyCopilotCLISubagentModels({ - reflectData: recordedReflect(), - agentsDir, - env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, - logger: () => {}, - }); - expect(resolutions[0].model).toBe("claude-haiku-5.5"); - expect(fs.readFileSync(path.join(agentsDir, "multiline.agent.md"), "utf8")).toBe('---\nname: multiline\nmodel: "claude-haiku-5.5"\ndescription: retained\n---\nPrompt retained.\n'); - }); - - it("case 6 (CLI, tracked file): a rewritten repository agent file stays out of git status and commits", async () => { - const { execFileSync } = require("child_process"); - const dir = tempDir("copilot-subagent-cli-tracked-"); - const git = (/** @type {string[]} */ ...args) => execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], { cwd: dir, encoding: "utf8" }); - git("init", "-q"); - const agentsDir = path.join(dir, ".github/agents"); - fs.mkdirSync(agentsDir, { recursive: true }); - fs.writeFileSync(path.join(agentsDir, "repo-small.agent.md"), "---\nmodel: small\n---\nRepository agent.\n"); - git("add", "-A"); - git("commit", "-q", "-m", "init"); - const awfConfigPath = path.join(dir, "awf-config.json"); - fs.copyFileSync(awfConfigFixturePath, awfConfigPath); - await applyCopilotCLISubagentModels({ reflectData: recordedReflect(), agentsDir, env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, logger: () => {} }); - expect(fs.readFileSync(path.join(agentsDir, "repo-small.agent.md"), "utf8")).toBe('---\nmodel: "claude-haiku-5.5"\n---\nRepository agent.\n'); - expect(git("ls-files", "-t", "--", ".github/agents/repo-small.agent.md").startsWith("S ")).toBe(true); - git("add", "-A"); - expect(git("status", "--porcelain", "--", ".github")).toBe(""); - expect(git("diff", "--cached", "--name-only", "--", ".github")).toBe(""); - }); - - it("clears skip-worktree after CLI and preserves edits to a tracked agent file", async () => { - const { execFileSync } = require("child_process"); - const dir = tempDir("copilot-subagent-cli-restore-"); - const git = (/** @type {string[]} */ ...args) => execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], { cwd: dir, encoding: "utf8" }); - git("init", "-q"); - const agentsDir = path.join(dir, ".github/agents"); - fs.mkdirSync(agentsDir, { recursive: true }); - const agentPath = path.join(agentsDir, "repo-small.agent.md"); - fs.writeFileSync(agentPath, "---\nmodel: small\n---\nOriginal prompt.\n"); - git("add", "-A"); - git("commit", "-q", "-m", "init"); - const awfConfigPath = path.join(dir, "awf-config.json"); - fs.copyFileSync(awfConfigFixturePath, awfConfigPath); - const rewrittenFiles = []; - await applyCopilotCLISubagentModels({ - reflectData: recordedReflect(), - agentsDir, - rewrittenFiles, - env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, - logger: () => {}, - }); - fs.writeFileSync(agentPath, '---\nmodel: "claude-haiku-5.5"\n---\nEdited prompt.\n'); - expect(git("status", "--porcelain", "--", ".github")).toBe(""); - expect(restoreCopilotCLIAgentFiles(rewrittenFiles, () => {})).toBe(true); - expect(git("ls-files", "-t", "--", ".github/agents/repo-small.agent.md").startsWith("H ")).toBe(true); - expect(fs.readFileSync(agentPath, "utf8")).toBe("---\nmodel: small\n---\nEdited prompt.\n"); - expect(git("status", "--porcelain", "--", ".github")).toContain(".github/agents/repo-small.agent.md"); + it("treats an inline sub-agent as declared and lets it override a repository agent with the same name", () => { + const dir = tempDir("copilot-subagent-declared-"); + const workspace = path.join(dir, "workspace"); + const base = path.join(dir, "gh-aw"); + fs.mkdirSync(path.join(workspace, ".github/agents"), { recursive: true }); + fs.mkdirSync(path.join(base, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(workspace, ".github/agents/planner.agent.md"), vscodePinnedAgent); + fs.writeFileSync(path.join(workspace, ".github/agents/shared.agent.md"), "---\nmodel: GPT-5.4 (copilot)\n---\nRepository.\n"); + fs.writeFileSync(path.join(base, ".github/agents/shared.agent.md"), "---\nmodel: small\n---\nInline.\n"); + expect(listCopilotSubagents(workspace, base, () => {})).toEqual([ + { name: "planner", path: path.join(workspace, ".github/agents/planner.agent.md"), model: "GPT-5.4 (copilot)", declared: false }, + { name: "shared", path: path.join(base, ".github/agents/shared.agent.md"), model: "small", declared: true }, + ]); }); - it.each(["tracking-state-error", "skip-worktree-error", "skip-worktree-not-retained"])("restores the original tracked agent file when %s", async failureMode => { - const { execFileSync } = require("child_process"); - const childProcess = require("child_process"); - const dir = tempDir("copilot-subagent-cli-skip-failure-"); - const git = (...args) => execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], { cwd: dir, encoding: "utf8" }); - git("init", "-q"); - const agentsDir = path.join(dir, ".github/agents"); + it.each([true, false])("CLI mode: a personal copy that cannot be written fails closed only for a declared sub-agent (declared=%s)", async declared => { + const dir = tempDir("copilot-subagent-cli-copy-failure-"); + const workspace = path.join(dir, "workspace"); + const base = path.join(dir, "gh-aw"); + const agentsDir = path.join(declared ? base : workspace, ".github/agents"); fs.mkdirSync(agentsDir, { recursive: true }); - const agentPath = path.join(agentsDir, "repo-small.agent.md"); - const original = "---\nmodel: small\n---\nOriginal prompt.\n"; - fs.writeFileSync(agentPath, original); - git("add", "-A"); - git("commit", "-q", "-m", "init"); + fs.writeFileSync(path.join(agentsDir, "haiku-small.agent.md"), "---\nmodel: small\n---\nWho are you?\n"); const awfConfigPath = path.join(dir, "awf-config.json"); fs.copyFileSync(awfConfigFixturePath, awfConfigPath); - const realSpawnSync = childProcess.spawnSync; - vi.spyOn(childProcess, "spawnSync").mockImplementation((command, args, options) => { - if (command === "git" && args[0] === "ls-files" && args.includes("--error-unmatch")) { - if (failureMode === "tracking-state-error") return { status: 128, stdout: "", stderr: "index unavailable" }; - return { status: 0, stdout: ".github/agents/repo-small.agent.md\n", stderr: "" }; - } - if (command === "git" && args[0] === "update-index" && args.includes("--skip-worktree")) { - return { status: failureMode === "skip-worktree-error" ? 1 : 0, stdout: "", stderr: "index unavailable" }; - } - if (command === "git" && args[0] === "ls-files" && args.includes("-t")) return { status: 0, stdout: "H .github/agents/repo-small.agent.md\n", stderr: "" }; - return realSpawnSync(command, args, options); - }); + // COPILOT_HOME is a file, so the personal agents directory cannot be created. + const copilotHome = path.join(dir, "copilot-home"); + fs.writeFileSync(copilotHome, "not a directory"); vi.spyOn(process, "exit").mockImplementation(code => { throw new Error(`exit ${code}`); }); - await expect( - applyCopilotCLISubagentModels({ - reflectData: recordedReflect(), - agentsDir, - env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath }, - logger: () => {}, - }) - ).rejects.toThrow("exit 1"); - expect(fs.readFileSync(agentPath, "utf8")).toBe(original); + const logs = []; + const apply = applyCopilotCLISubagentModels({ + reflectData: recordedReflect(), + workingDirectory: workspace, + agentsBaseDir: base, + env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath, COPILOT_HOME: copilotHome }, + logger: message => logs.push(message), + }); + if (declared) { + await expect(apply).rejects.toThrow("exit 1"); + expect(logs).toContainEqual(expect.stringContaining("copilot sub-agent 'haiku-small': cannot write personal agent copy")); + } else { + expect((await apply).map(resolution => resolution.model)).toEqual(["claude-haiku-5.5"]); + expect(logs).toContainEqual(expect.stringContaining("warning: repository agent 'haiku-small'")); + } + expect(fs.readFileSync(path.join(agentsDir, "haiku-small.agent.md"), "utf8")).toBe("---\nmodel: small\n---\nWho are you?\n"); }); - /** Spawn the harness against the recorded catalogue with the given workspace agent files. */ - async function runHarness({ sdk = false, agents, workspaceAgents = {}, noop = false, reflect = recordedReflect(), env: envOverrides = {} }) { + /** + * Spawn the harness against the recorded catalogue. `agents` are repository agent files in the engine + * working directory; `inlineAgents` are workflow-declared sub-agents, staged under `/gh-aw` + * and restored (untracked) into the workspace as `restore_inline_sub_agents.sh` does. + */ + async function runHarness({ sdk = false, agents = {}, inlineAgents = {}, workspaceAgents = {}, noop = false, reflect = recordedReflect(), env: envOverrides = {} }) { const dir = tempDir("copilot-subagent-harness-"); const workspace = path.join(dir, "workspace"); const engineCwd = envOverrides.GH_AW_ENGINE_CWD || workspace; @@ -5072,7 +5101,11 @@ Module._load = function(request, ...args) { initGitRepo(workspace); if (engineCwd !== workspace) initGitRepo(engineCwd); const ostmp = path.join(dir, "ostmp"); - fs.mkdirSync(ostmp); + fs.mkdirSync(path.join(ostmp, "gh-aw", ".github/agents"), { recursive: true }); + for (const [name, content] of Object.entries(inlineAgents)) { + fs.writeFileSync(path.join(ostmp, "gh-aw", ".github/agents", name), content); + fs.writeFileSync(path.join(workspace, ".github/agents", name), content); + } const startedPath = path.join(dir, "copilot-started.json"); let providerPort = 0; let sdkPort = 0; @@ -5102,8 +5135,10 @@ if (${providerPort} && process.argv[1].endsWith("copilot_harness.cjs")) require( const path = require("path"); const args = process.argv.slice(2); const agentsDir = path.join(process.env.GH_AW_ENGINE_CWD || process.env.GITHUB_WORKSPACE || process.cwd(), ".github", "agents"); -const agentFiles = fs.existsSync(agentsDir) ? Object.fromEntries(fs.readdirSync(agentsDir).filter(name => name.endsWith(".agent.md")).map(name => [name, fs.readFileSync(path.join(agentsDir, name), "utf8")])) : {}; -fs.writeFileSync(${JSON.stringify(startedPath)}, JSON.stringify({ args, agentFiles, subagentModels: process.env.GH_AW_COPILOT_SDK_SUBAGENT_MODELS || null })); +const readAgents = dir => (fs.existsSync(dir) ? Object.fromEntries(fs.readdirSync(dir).filter(name => name.endsWith(".agent.md")).map(name => [name, fs.readFileSync(path.join(dir, name), "utf8")])) : {}); +const agentFiles = readAgents(agentsDir); +const personalAgents = readAgents(path.join(process.env.COPILOT_HOME, "agents")); +fs.writeFileSync(${JSON.stringify(startedPath)}, JSON.stringify({ args, agentFiles, personalAgents, subagentModels: process.env.GH_AW_COPILOT_SDK_SUBAGENT_MODELS || null })); if (args.includes("--session-id")) { const model = args.includes("--model") ? args[args.lastIndexOf("--model") + 1] : process.env.COPILOT_MODEL; const sessionDir = path.join(process.env.COPILOT_HOME, "session-state", args[args.indexOf("--session-id") + 1]); @@ -5160,6 +5195,8 @@ process.stdout.write("done\\n");` started: fs.existsSync(startedPath) ? JSON.parse(fs.readFileSync(startedPath, "utf8")) : null, records: readRecords(dir), agentFile: name => fs.readFileSync(path.join(engineCwd, ".github/agents", name), "utf8"), + personalAgentsDir: path.join(dir, "copilot-home", "agents"), + gitStatus: () => require("child_process").execFileSync("git", ["status", "--porcelain"], { cwd: workspace, encoding: "utf8" }), }; } @@ -5168,8 +5205,8 @@ process.stdout.write("done\\n");` "mini54-whoami.agent.md": "---\nname: mini54-whoami\nmodel: gpt-5.4-mini\n---\nWho are you?\n", }; - it.each([false, true])("case 3 end to end: a dead sub-agent model stops the harness before Copilot starts (sdk=%s)", async sdk => { - const run = await runHarness({ sdk, agents: smokeAgents }); + it.each([false, true])("case 3 end to end: a dead declared sub-agent model stops the harness before Copilot starts (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, inlineAgents: smokeAgents }); expect(run.result.status, run.result.stderr).toBe(1); expect(run.started).toBeNull(); expect(run.result.stderr).not.toContain("starting sidecar"); @@ -5183,8 +5220,43 @@ process.stdout.write("done\\n");` expect(run.agentFile("mini54-whoami.agent.md")).toBe(smokeAgents["mini54-whoami.agent.md"]); }); + it.each([false, true])("a repository agent pinned to GPT-5.4 (copilot) does not stop the run (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, agents: { "planner.agent.md": vscodePinnedAgent } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started).not.toBeNull(); + expect(run.records).toContainEqual({ engine: "copilot", role: "planner", configured_model: "GPT-5.4 (copilot)", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }); + expect(run.result.stderr).toMatch(/warning: repository agent 'planner' \(.*planner\.agent\.md\) declares model 'GPT-5\.4 \(copilot\)'/); + expect(run.result.stderr).toContain(sdk ? "the SDK session gives it the session model" : "Copilot CLI uses the agent file unchanged"); + expect(detectErrors(run.result.stderr).modelNotSupportedError).toBe(false); + if (sdk) expect(JSON.parse(run.started.subagentModels)).toEqual({}); + expect(run.started.personalAgents).toEqual({}); + expect(run.agentFile("planner.agent.md")).toBe(vscodePinnedAgent); + }); + + it.each([false, true])("a declared inline sub-agent with gpt-5.4-mini still fails closed beside the same repository agent (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, agents: { "planner.agent.md": vscodePinnedAgent }, inlineAgents: { "mini54-whoami.agent.md": smokeAgents["mini54-whoami.agent.md"] } }); + expect(run.result.status, run.result.stderr).toBe(1); + expect(run.started).toBeNull(); + expect(run.result.stderr).toContain("copilot sub-agent 'mini54-whoami': The requested model is not supported: 'gpt-5.4-mini' is not available in the workflow's live model catalog"); + expect(run.result.stderr).toContain("warning: repository agent 'planner'"); + expect(run.records).toContainEqual({ engine: "copilot", role: "planner", configured_model: "GPT-5.4 (copilot)", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }); + expect(run.records).toContainEqual({ engine: "copilot", role: "mini54-whoami", configured_model: "gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }); + }); + + it("skips sub-agent resolution in the threat-detection phase", async () => { + const repositoryAgents = { "dead.agent.md": "---\nmodel: gpt-5.4-mini\n---\nDead.\n", "haiku-small.agent.md": "---\nmodel: small\n---\nWho are you?\n" }; + const run = await runHarness({ agents: repositoryAgents, inlineAgents: { "mini54-whoami.agent.md": smokeAgents["mini54-whoami.agent.md"] }, env: { GH_AW_PHASE: "detection" } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started).not.toBeNull(); + expect(run.result.stderr).not.toContain("copilot sub-agent"); + expect(run.result.stderr).not.toContain("repository agent"); + expect((run.records || []).filter(record => record.role !== "main")).toEqual([]); + expect(fs.existsSync(run.personalAgentsDir)).toBe(false); + for (const [name, content] of Object.entries(repositoryAgents)) expect(run.agentFile(name)).toBe(content); + }); + it("skips CLI sub-agent validation when a noop is already present", async () => { - const run = await runHarness({ agents: { "dead.agent.md": "---\nmodel: gpt-5.4-mini\n---\nUnused.\n" }, noop: true }); + const run = await runHarness({ inlineAgents: { "dead.agent.md": "---\nmodel: gpt-5.4-mini\n---\nUnused.\n" }, noop: true }); expect(run.result.status, run.result.stderr).toBe(0); expect(run.started).toBeNull(); expect(run.result.stderr).toContain("pre-flight: noop message found in safe-outputs"); @@ -5203,17 +5275,28 @@ process.stdout.write("done\\n");` expect(run.result.status, run.result.stderr).toBe(0); expect(run.started).not.toBeNull(); expect(run.records).toContainEqual({ engine: "copilot", role: "effective", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }); + expect(run.records.map(record => record.role)).not.toContain("unrelated"); }); - it("case 6 end to end (CLI): Copilot CLI starts with the resolved model in the agent file", async () => { + it("case 6 end to end (CLI): Copilot CLI starts with a personal copy of a repository agent carrying the resolved model", async () => { const run = await runHarness({ agents: { "haiku-small.agent.md": "---\nname: haiku-small\nmodel: small\n---\nWho are you?\n" } }); expect(run.result.status, run.result.stderr).toBe(0); expect(run.started).not.toBeNull(); - expect(run.started.agentFiles["haiku-small.agent.md"]).toBe('---\nname: haiku-small\nmodel: "claude-haiku-5.5"\n---\nWho are you?\n'); + expect(run.started.personalAgents).toEqual({ "haiku-small.agent.md": '---\nname: haiku-small\nmodel: "claude-haiku-5.5"\n---\nWho are you?\n' }); + expect(run.started.agentFiles["haiku-small.agent.md"]).toBe("---\nname: haiku-small\nmodel: small\n---\nWho are you?\n"); expect(run.agentFile("haiku-small.agent.md")).toBe("---\nname: haiku-small\nmodel: small\n---\nWho are you?\n"); + expect(run.gitStatus()).toBe(""); expect(run.records).toContainEqual({ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }); }); + it("case 6 end to end (CLI): a declared inline sub-agent gets a personal copy with the resolved model", async () => { + const inline = "---\nname: inline-large\nmodel: large\n---\nPropose.\n"; + const run = await runHarness({ inlineAgents: { "inline-large.agent.md": inline } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started.personalAgents).toEqual({ "inline-large.agent.md": '---\nname: inline-large\nmodel: "claude-sonnet-5.5"\n---\nPropose.\n' }); + expect(run.agentFile("inline-large.agent.md")).toBe(inline); + }); + it("case 6 end to end (SDK): the driver receives the qualified resolved sub-agent model", async () => { const run = await runHarness({ sdk: true, agents: { "haiku-small.agent.md": "---\nname: haiku-small\nmodel: small\n---\nWho are you?\n" } }); expect(run.result.status, run.result.stderr).toBe(0); @@ -5232,8 +5315,8 @@ process.stdout.write("done\\n");` expect(run.records).toEqual([{ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }]); }); - it.each([false, true])("still fails closed for a dead sub-agent model (sdk=%s)", async sdk => { - const run = await runHarness({ sdk, reflect: routedReflect(), agents: { "opus.agent.md": "---\nmodel: claude-opus-4.8\n---\nWho are you?\n" }, env: { GH_AW_MODEL_ROUTING: "1", COPILOT_MODEL: "" } }); + it.each([false, true])("still fails closed for a dead declared sub-agent model (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, reflect: routedReflect(), inlineAgents: { "opus.agent.md": "---\nmodel: claude-opus-4.8\n---\nWho are you?\n" }, env: { GH_AW_MODEL_ROUTING: "1", COPILOT_MODEL: "" } }); expect(run.result.status, run.result.stderr).toBe(1); expect(run.started).toBeNull(); expect(run.records).toEqual([{ engine: "copilot", role: "opus", configured_model: "claude-opus-4.8", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }]); diff --git a/actions/setup/js/copilot_sdk_driver.test.cjs b/actions/setup/js/copilot_sdk_driver.test.cjs index 6745bfb3677..89c28ed66b6 100644 --- a/actions/setup/js/copilot_sdk_driver.test.cjs +++ b/actions/setup/js/copilot_sdk_driver.test.cjs @@ -102,11 +102,14 @@ describe("copilot_sdk_driver.cjs", () => { ); }); - it.each(["medium", "unavailable-model"])("fails closed for an unresolved custom agent model %s instead of using the session model", async agentModel => { - const workflow = path.join(testSessionStateDir, `unresolved-${agentModel}-workflow`); + it.each(["medium", "GPT-5.4 (copilot)"])("gives a repository agent whose model %s the harness could not resolve the session model with a warning", async agentModel => { + const workflow = path.join(testSessionStateDir, `unresolved-${agentModel.replace(/\W+/g, "-")}-workflow`); fs.mkdirSync(path.join(workflow, ".github/agents"), { recursive: true }); fs.writeFileSync(path.join(workflow, ".github/agents/unknown.agent.md"), `---\nname: unknown-agent\nmodel: ${agentModel}\n---\nUnknown model.`); - const createSession = vi.fn(); + // The harness recorded the failure and left the repository agent out of the resolved map. + vi.stubEnv("GH_AW_COPILOT_SDK_SUBAGENT_MODELS", "{}"); + const sessionId = `unresolved-${agentModel.replace(/\W+/g, "-")}-session`; + const createSession = vi.fn().mockResolvedValue({ sessionId, on: () => {}, sendAndWait: vi.fn().mockResolvedValue({ data: { content: "done" } }), disconnect: vi.fn().mockResolvedValue(undefined) }); class FakeCopilotClient { start = vi.fn().mockResolvedValue(undefined); createSession = createSession; @@ -125,9 +128,15 @@ describe("copilot_sdk_driver.cjs", () => { logger, sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: vi.fn(() => ({})) }, approveAll: () => "allow" }, }); - expect(result.exitCode).toBe(1); - expect(result.output).toContain(`Cannot qualify custom agent "unknown-agent" model "${agentModel}"`); - expect(createSession).not.toHaveBeenCalled(); + expect(result.exitCode).toBe(0); + expect(createSession.mock.calls[0][0].customAgents).toEqual([{ name: "unknown-agent", displayName: "unknown-agent", prompt: "Unknown model.", model: "copilot-responses/gpt-5.4" }]); + expect(logger).toHaveBeenCalledWith(expect.stringMatching(/warning:.*unknown-agent.*session model/i)); + const captured = fs + .readFileSync(path.join(testSessionStateDir, `${sessionId}/events.jsonl`), "utf8") + .trim() + .split("\n") + .map(JSON.parse); + expect(captured.filter(event => event.type === "subagent.model_unavailable")).toEqual([expect.objectContaining({ data: { agentName: "unknown-agent", declaredModel: agentModel, model: "copilot-responses/gpt-5.4" } })]); }); it("captures ephemeral selection and final model results including BYOK and subagent context", async () => { diff --git a/actions/setup/js/copilot_sdk_session.cjs b/actions/setup/js/copilot_sdk_session.cjs index 66aebf2438e..af3079a9f63 100644 --- a/actions/setup/js/copilot_sdk_session.cjs +++ b/actions/setup/js/copilot_sdk_session.cjs @@ -383,14 +383,24 @@ async function runWithCopilotSDK({ const providerConfig = providers && providerModels ? { providers, models: providerModels } : null; const sessionModel = providerConfig ? qualifyModelForMultiProvider(requestedModel, providerConfig) : requestedModel; if (providerConfig && !sessionModel) throw new Error(`Cannot qualify session model "${requestedModel || "(none)"}" against the configured provider models`); - // The harness resolves declared sub-agent models (aliases included) before the driver starts. + // The harness resolves sub-agent models (aliases included) before the driver starts and + // stops the run when a workflow-declared sub-agent fails. A repository agent whose model + // the harness could not resolve is absent from the map and inherits the session model. const resolvedSubagentModels = parseSubagentModelsJson(process.env.GH_AW_COPILOT_SDK_SUBAGENT_MODELS); + const unavailableModels = []; const customAgents = loadCopilotSDKCustomAgents(workingDirectory, agentsBaseDir, log).map(agent => { if (!agent.model || agent.model.trim().toLowerCase() === "inherited") return { ...agent, model: sessionModel || undefined }; - const requested = Object.prototype.hasOwnProperty.call(resolvedSubagentModels, agent.name) ? resolvedSubagentModels[agent.name] : agent.model; - const qualified = providerConfig ? qualifyModelForMultiProvider(requested, providerConfig) : requested; - if (!qualified) throw new Error(`Cannot qualify custom agent "${agent.name}" model "${agent.model}" against the configured provider models`); - return { ...agent, model: qualified }; + if (Object.prototype.hasOwnProperty.call(resolvedSubagentModels, agent.name)) { + const resolved = resolvedSubagentModels[agent.name]; + const qualified = providerConfig ? qualifyModelForMultiProvider(resolved, providerConfig) : resolved; + if (!qualified) throw new Error(`Cannot qualify custom agent "${agent.name}" model "${resolved}" against the configured provider models`); + return { ...agent, model: qualified }; + } + const qualified = providerConfig ? qualifyModelForMultiProvider(agent.model, providerConfig) : agent.model; + if (qualified) return { ...agent, model: qualified }; + log(`warning: custom agent "${agent.name}" model "${agent.model}" is not in the configured provider models; using session model "${sessionModel || "(none)"}"`); + unavailableModels.push({ agentName: agent.name, declaredModel: agent.model, model: sessionModel || undefined }); + return { ...agent, model: sessionModel || undefined }; }); const skillsDirectory = path.resolve(agentsBaseDir, ".github", "skills"); const skillsStat = fs.lstatSync(skillsDirectory, { throwIfNoEntry: false }); @@ -422,6 +432,7 @@ async function runWithCopilotSDK({ fs.mkdirSync(sessionDir, { recursive: true }); const eventsPath = path.join(sessionDir, "events.jsonl"); eventsStream = fs.createWriteStream(eventsPath, { flags: "a" }); + for (const unavailable of unavailableModels) writeDriverEvent("subagent.model_unavailable", unavailable); // Snapshot to a non-null local for closure-safe writes (JSDoc nullability narrowing). const stream = eventsStream; const assistantEvents = []; diff --git a/docs/src/content/docs/reference/engines.md b/docs/src/content/docs/reference/engines.md index 25f2929af0c..a6ae113d75d 100644 --- a/docs/src/content/docs/reference/engines.md +++ b/docs/src/content/docs/reference/engines.md @@ -624,13 +624,24 @@ in `usage/aw_session.jsonl`. In both SDK and CLI mode, the harness resolves each sub-agent's `model:` before Copilot starts, including on routed runs. It uses gh-aw model aliases (for example `small` and `large`) and checks the live AWF model catalog. Each result -is recorded as a `model.resolution` event whose role is the agent name. In CLI -mode, the resolved id is written to the workspace agent file that Copilot CLI -loads; in SDK mode, it is qualified against the SDK provider models. `medium` is -not an alias and is treated as a literal model id. If a sub-agent model cannot be -resolved, is not in the live catalog, or does not qualify in SDK mode, the run -fails before Copilot starts instead of falling back to the main model. A -sub-agent without `model:` inherits the main model and has no resolution record. +is recorded as a `model.resolution` event whose role is the agent name. `medium` +is not an alias and is treated as a literal model id. A sub-agent without +`model:` inherits the main model and has no resolution record. + +Workflow-declared sub-agents (inline `## agent:` blocks) fail closed: if a model +cannot be resolved, is not in the live catalog, or does not qualify in SDK mode, +the run fails before Copilot starts. Other repository `.github/agents/*.agent.md` +files, which are often shared with VS Code, do not stop the run: a model that +cannot be resolved is recorded with `status: failed` and logged as a warning. In +CLI mode, that agent is used unchanged; in SDK mode, it inherits the session +model. An inline sub-agent overrides a repository agent with the same name. + +In CLI mode, when the resolved id differs from the declared `model:`, the harness +writes a copy of the agent file, with the same file name and only `model:` +changed, to Copilot CLI's personal agents directory +(`${COPILOT_HOME:-$HOME/.copilot}/agents/`). A personal agent overrides the +repository agent with the same ID, so files in the checkout are never modified. +Sub-agent resolution is skipped in the threat-detection phase. Use top-level `max-tool-denials` to stop SDK inference when tool requests are repeatedly denied. The default is `5`.