diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 2d76871c98b..4e967a97015 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -55,6 +55,8 @@ require("./shim.cjs"); const { getErrorMessage } = require("./error_helpers.cjs"); const { maskSecret } = require("./actions_secret_masking.cjs"); const fs = require("fs"); +const os = require("os"); +const path = require("path"); const crypto = require("crypto"); const { getPromptPath, renderTemplateFromFile } = require("./messages_core.cjs"); const { @@ -103,6 +105,8 @@ const { loadModelsJson } = require("./model_costs.cjs"); const { applyCopilotWireAPI: configureCopilotWireAPI, warnCopilotSubagentEndpoints, recordModelEndpointMismatch, getSupportedEndpoints, MODEL_ENDPOINT_FIX } = require("./copilot_wire_api.cjs"); const { loadModelAliasMap, resolveEngineModel, normalizeForCopilotCLI, ModelAliasResolutionError, recordModelResolution, splitModelIdentifier } = require("./resolve_model_alias.cjs"); const { createCopilotModelObserver, recordCopilotModelSelection } = require("./copilot_model_selection.cjs"); +const { getEngineSubAgentTarget } = require("./extract_inline_sub_agents.cjs"); +const { inlineArtifactTarget } = require("./inline_artifact_target.cjs"); const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported|No model available\b[^\n]*policy enablement/i; const { parseAICreditsErrorInfoFromAuditLog, parseMaxAICreditsFromAuditLog, parseMaxAICreditsExceededFromAuditLog, parseAPIProxyGuardRejectionFromEventLog, formatAPIProxyGuardRejection } = require("./ai_credits_context.cjs"); @@ -650,6 +654,332 @@ async function resolveCopilotSDKMultiProvider(options) { return null; // unreachable, keeps TypeScript control-flow analysis happy } +/** + * Read the Copilot custom agent files in one directory, one entry per regular `.agent.md` file. + * The name is the frontmatter `name` or the file name without `.agent.md`; + * `model` is the trimmed frontmatter `model` string, when declared. + * @param {string} directory + * @param {(msg: string) => void} [logger] + * @returns {Array<{ name: string, model?: string, path: string }>} + */ +function listCopilotAgentFiles(directory, logger = log) { + const { extractFrontmatterAndBody } = require("./frontmatter_hash_pure.cjs"); + const { parseDocument } = require("./work_queue_yaml.cjs"); + let entries; + try { + entries = fs.readdirSync(directory, { withFileTypes: true }); + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) logger(`warning: cannot read custom agents in ${directory}: ${getErrorMessage(error)}`); + return []; + } + const agents = []; + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (!entry.isFile() || !entry.name.endsWith(".agent.md")) continue; + const agentPath = path.join(directory, entry.name); + try { + const { frontmatterText } = extractFrontmatterAndBody(fs.readFileSync(agentPath, "utf8")); + /** @type {any} */ + const document = parseDocument(frontmatterText); + if (!document || document.errors.length) throw new Error("cannot parse agent frontmatter"); + const frontmatter = document.toJS() || {}; + if (typeof frontmatter !== "object" || Array.isArray(frontmatter)) throw new Error("agent frontmatter must be a mapping"); + const name = typeof frontmatter.name === "string" && frontmatter.name.trim() ? frontmatter.name.trim() : entry.name.slice(0, -".agent.md".length); + agents.push({ name, path: agentPath, ...(typeof frontmatter.model === "string" ? { model: frontmatter.model.trim() } : {}) }); + } catch (error) { + logger(`warning: cannot read custom agent ${agentPath}: ${getErrorMessage(error)}`); + } + } + return agents; +} + +/** + * List the Copilot sub-agents of a run. Repository agents come from + * `/.github/agents`; workflow-declared inline sub-agents come from the + * directory they are staged in under `agentsBaseDir` (the one `loadCopilotSDKCustomAgents` + * reads). An inline sub-agent overrides a repository agent with the same name and is declared. + * @param {string} workingDirectory + * @param {string} agentsBaseDir + * @param {(msg: string) => void} [logger] + * @returns {Array<{ name: string, model?: string, path: string, declared: boolean }>} + */ +function listCopilotSubagents(workingDirectory, agentsBaseDir, logger = log) { + const declaredDir = path.resolve(agentsBaseDir, inlineArtifactTarget("SUB_AGENT", getEngineSubAgentTarget("copilot")).dir); + const repositoryDir = path.resolve(workingDirectory, ".github", "agents"); + const agents = new Map(); + if (repositoryDir !== declaredDir) { + for (const agent of listCopilotAgentFiles(repositoryDir, logger)) agents.set(agent.name, { ...agent, declared: false }); + } + for (const agent of listCopilotAgentFiles(declaredDir, logger)) agents.set(agent.name, { ...agent, declared: true }); + return [...agents.values()]; +} + +/** + * Locate the top-level YAML model scalar in a Copilot agent file. + * @param {string} content + * @returns {{ value: string, source: string, start: number, end: number }} + */ +function getCopilotAgentModelScalar(content) { + const { extractFrontmatterAndBody } = require("./frontmatter_hash_pure.cjs"); + const { parseDocument } = require("./work_queue_yaml.cjs"); + const { frontmatterText } = extractFrontmatterAndBody(content); + /** @type {any} */ + const document = parseDocument(frontmatterText); + if (!document || document.errors.length || !document.contents || !Array.isArray(document.contents.items)) { + throw new Error("cannot parse agent frontmatter"); + } + const pairs = document.contents.items.filter(pair => pair?.key?.value === "model"); + if (pairs.length !== 1 || typeof pairs[0].value?.value !== "string" || !Array.isArray(pairs[0].value.range)) { + throw new Error("expected exactly one top-level string 'model:' scalar in the frontmatter"); + } + const frontmatterStart = content.indexOf("\n") + 1; + const start = frontmatterStart + pairs[0].value.range[0]; + const end = frontmatterStart + pairs[0].value.range[1]; + return { value: pairs[0].value.value, source: content.slice(start, end), start, end }; +} + +/** + * Replace the complete YAML model scalar, including any block-scalar continuation lines. + * @param {string} content + * @param {string} model + * @returns {string} + */ +function replaceCopilotAgentFileModelContent(content, model) { + const scalar = getCopilotAgentModelScalar(content); + const lineEnding = scalar.source.match(/(?:\r?\n)+$/)?.[0] || ""; + return content.slice(0, scalar.start) + JSON.stringify(model) + lineEnding + content.slice(scalar.end); +} + +/** + * Write a copy of a Copilot agent file to Copilot CLI's personal agents directory with only + * the frontmatter `model:` scalar replaced. The copy keeps the file name, which Copilot CLI + * uses as the agent ID, so it overrides the repository agent with the same ID. The source + * file is never modified. + * @param {string} agentPath + * @param {string} personalAgentsDir + * @param {string} model + * @returns {string} The path of the copy. + */ +function writeCopilotPersonalAgentCopy(agentPath, personalAgentsDir, model) { + let content; + try { + content = fs.readFileSync(agentPath, "utf8"); + } catch (error) { + throw new Error(`cannot read agent file ${agentPath}: ${getErrorMessage(error)}`, { cause: error }); + } + const updated = replaceCopilotAgentFileModelContent(content, model); + const target = path.join(personalAgentsDir, path.basename(agentPath)); + try { + fs.mkdirSync(personalAgentsDir, { recursive: true }); + fs.writeFileSync(target, updated); + } catch (error) { + throw new Error(`cannot write personal agent ${target}: ${getErrorMessage(error)}`, { cause: error }); + } + return target; +} + +/** + * Resolve the declared models of Copilot sub-agents before Copilot starts, in CLI and SDK + * mode and on routed runs. Each `model:` goes through the shared resolver (Copilot provider + * scope, the workflow alias map, `normalizeForCopilotCLI`) and the live-catalogue check that + * the main SDK model uses. With `providerConfig` (SDK mode), the resolved id must also qualify + * against the SDK provider models. A sub-agent without `model:` inherits the main model and is + * not recorded. + * + * Every sub-agent with a model gets a `model.resolution` record (role = agent name), so + * several sub-agents sharing one dead model each get a `failed` record. Workflow-declared + * sub-agents (the default) fail closed: when any of them fails, the first failure is thrown + * as a ModelAliasResolutionError after all records are written; callers must not start + * Copilot. A repository agent (`declared: false`) that fails gets a warning and a `failed` + * record, is left out of the result, and does not stop the run. `/reflect` is refreshed at + * most once. + * + * @param {{ + * agents: Array<{ name: string, model?: string, path?: string, declared?: boolean }>, + * reflectData: any, + * refetchReflectData?: () => Promise, + * providerConfig?: { providers: Array<{name: string}>, models: Array<{id: string, provider: string}> } | null, + * refreshProviderConfig?: (reflectData: any) => Promise<{ providers: Array<{name: string}>, models: Array<{id: string, provider: string}> } | null>, + * aliasMap?: Record | null, + * repositoryFallback?: string, + * env?: NodeJS.ProcessEnv, + * logger?: (msg: string) => void, + * }} options `repositoryFallback` describes what happens to a repository agent whose model fails. + * @returns {Promise>} + * `resolvedModel` is the resolved Copilot id; `model` is the id to hand to Copilot (qualified in SDK mode). + */ +async function resolveCopilotSubagentModels(options) { + const logger = options.logger || log; + const env = options.env || process.env; + const declared = options.agents.filter(agent => typeof agent.model === "string" && agent.model.trim() && agent.model.trim().toLowerCase() !== "inherited"); + if (declared.length === 0) return []; + const aliasMap = options.aliasMap !== undefined ? options.aliasMap : (loadAwfConfigData()?.apiProxy?.models ?? loadModelAliasMap({ reflectData: options.reflectData, env })); + let reflectData = options.reflectData; + let providerConfig = options.providerConfig; + /** @type {Promise | null} */ + let refresh = null; + const refreshOnce = () => { + refresh ??= (options.refetchReflectData ? options.refetchReflectData() : Promise.resolve(null)).then(refreshed => { + if (refreshed) { + reflectData = refreshed; + if (options.refreshProviderConfig) { + return options.refreshProviderConfig(refreshed).then(config => { + if (config) providerConfig = config; + return refreshed; + }); + } + } + return refreshed || null; + }); + return refresh; + }; + const liveCheck = env.AWF_REFLECT_ENABLED === "1" || Boolean(options.reflectData); + /** @type {InstanceType[]} */ + const failures = []; + const resolutions = []; + for (const agent of declared) { + const configuredModel = String(agent.model).trim(); + const repositoryAgent = agent.declared === false; + // A repository agent the workflow may never use must not make the run look like an + // unsupported-model failure, so its resolver failure is reported by the warning below. + const agentLogger = repositoryAgent ? (/** @type {string} */ message) => (MODEL_NOT_SUPPORTED_PATTERN.test(message) ? undefined : logger(message)) : logger; + const scope = scopeModelToReflectProviders(configuredModel); + const fail = (/** @type {InstanceType} */ failure, /** @type {string} */ detail) => { + recordModelResolution({ engine: "copilot", role: agent.name, configured_model: configuredModel, resolved_model: null, provider: failure.provider || null, status: "failed", cause: failure.reason }, env); + if (repositoryAgent) { + logger( + `warning: repository agent '${agent.name}' (${agent.path || "unknown file"}) declares model '${configuredModel}', which could not be resolved (cause: ${failure.reason}); ${options.repositoryFallback || "the agent is left unchanged"}.` + ); + return; + } + logger(`copilot sub-agent '${agent.name}': ${failure.message}${detail ? ` ${detail}` : ""} Refusing to start Copilot with a different sub-agent model.`); + failures.push(failure); + }; + let result; + try { + result = await resolveEngineModel({ + engine: "copilot", + role: agent.name, + configuredModel, + aliasMap, + reflectData, + provider: "copilot", + refetchReflectData: refreshOnce, + logPrefix: `copilot sub-agent '${agent.name}' model alias resolution`, + normalize: r => normalizeForCopilotCLI(r.resolvedModel), + env, + logger: agentLogger, + }); + } catch (failure) { + if (!(failure instanceof ModelAliasResolutionError)) throw failure; + fail(failure, ""); + continue; + } + const resolvedModel = result.engineModel; + const resolvedNote = resolvedModel !== configuredModel ? ` (resolved to '${resolvedModel}')` : ""; + const failureProvider = scope.providers && scope.providers !== REFLECT_PROVIDER_ALIASES.github ? scope.prefix : "copilot"; + const scopedResolvedModel = scope.prefix ? `${scope.prefix}/${resolvedModel}` : resolvedModel; + if (liveCheck && !isModelListedOnConfiguredReflectEndpoint(scopedResolvedModel, reflectData)) { + const initialReason = classifyUnlistedReflectModel(scopedResolvedModel, reflectData); + const refreshed = await refreshOnce(); + if (!refreshed || !isModelListedOnConfiguredReflectEndpoint(scopedResolvedModel, refreshed)) { + const reason = initialReason === "model_unavailable" || !refreshed ? initialReason : classifyUnlistedReflectModel(scopedResolvedModel, refreshed); + fail(new ModelAliasResolutionError(configuredModel, reason, { provider: failureProvider }), `Sub-agent model '${configuredModel}'${resolvedNote} is not listed on the configured awf-reflect endpoints.`); + continue; + } + } + let model = resolvedModel; + if (providerConfig) { + let qualified; + try { + const qualificationModel = scope.providers === REFLECT_PROVIDER_ALIASES.github ? `copilot/${resolvedModel}` : scope.prefix ? `${scope.prefix}/${resolvedModel}` : resolvedModel; + qualified = qualifyModelForMultiProvider(qualificationModel, providerConfig); + } catch (error) { + fail( + new ModelAliasResolutionError(configuredModel, "model_unavailable", { provider: failureProvider }), + `Sub-agent model '${configuredModel}'${resolvedNote} could not be qualified against the Copilot SDK provider models: ${getErrorMessage(error)}` + ); + continue; + } + if (!qualified) { + fail( + new ModelAliasResolutionError(configuredModel, "model_unavailable", { provider: failureProvider }), + `Sub-agent model '${configuredModel}'${resolvedNote} is not in the Copilot SDK provider models (models without a supported wire API are skipped).` + ); + continue; + } + model = qualified; + } + logger(`copilot sub-agent '${agent.name}': model '${configuredModel}' -> '${model}'`); + resolutions.push({ name: agent.name, configuredModel, resolvedModel, model }); + } + if (failures.length > 0) throw failures[0]; + return resolutions; +} + +/** + * CLI mode: resolve the sub-agent models of the agents Copilot CLI loads (repository agents + * in the engine working directory and workflow-declared inline sub-agents) and, for each + * resolved id that differs from the declared value, write a copy of the agent file with the + * resolved `model:` to Copilot CLI's personal agents directory + * (`${COPILOT_HOME:-$HOME/.copilot}/agents`). Files in the checkout are never modified. + * Stops the process before Copilot starts when a declared sub-agent model cannot be resolved + * or its copy cannot be written; repository agents only produce warnings. + * + * @param {{ + * reflectData: any, + * refetchReflectData?: () => Promise, + * workingDirectory?: string, + * agentsBaseDir?: string, + * personalAgentsDir?: string, + * env?: NodeJS.ProcessEnv, + * logger?: (msg: string) => void, + * }} options + * @returns {Promise>} + */ +async function applyCopilotCLISubagentModels(options) { + const logger = options.logger || log; + const env = options.env || process.env; + const workingDirectory = options.workingDirectory || path.resolve(env.GITHUB_WORKSPACE || process.cwd(), env.GH_AW_ENGINE_CWD || "."); + const agentsBaseDir = options.agentsBaseDir || path.join(os.tmpdir(), "gh-aw"); + const personalAgentsDir = options.personalAgentsDir || path.join(env.COPILOT_HOME || path.join(env.HOME || os.homedir(), ".copilot"), "agents"); + const agents = listCopilotSubagents(workingDirectory, agentsBaseDir, logger); + let resolutions; + try { + resolutions = await resolveCopilotSubagentModels({ + agents, + reflectData: options.reflectData, + refetchReflectData: options.refetchReflectData, + repositoryFallback: "Copilot CLI uses the agent file unchanged", + env, + logger, + }); + } catch (failure) { + if (!(failure instanceof ModelAliasResolutionError)) throw failure; + process.exit(1); + return []; // unreachable, keeps TypeScript control-flow analysis happy + } + for (const resolution of resolutions) { + if (resolution.model === resolution.configuredModel) continue; + const agent = agents.find(candidate => candidate.name === resolution.name); + if (!agent) continue; + try { + const target = writeCopilotPersonalAgentCopy(agent.path, personalAgentsDir, resolution.model); + logger(`copilot sub-agent '${resolution.name}': wrote model '${resolution.model}' to personal agent ${target}`); + } catch (error) { + if (!agent.declared) { + logger( + `warning: repository agent '${resolution.name}' (${agent.path}): cannot write personal agent copy with model '${resolution.model}' to ${personalAgentsDir}: ${getErrorMessage(error)}; Copilot CLI uses the agent file unchanged.` + ); + continue; + } + logger(`copilot sub-agent '${resolution.name}': cannot write personal agent copy of ${agent.path} to ${personalAgentsDir}: ${getErrorMessage(error)}. Refusing to start Copilot with an unresolved sub-agent model.`); + process.exit(1); + return []; // unreachable, keeps TypeScript control-flow analysis happy + } + } + return resolutions; +} + /** * Configure and validate COPILOT_PROVIDER_WIRE_API using AWF endpoint metadata. * @@ -1112,10 +1442,11 @@ function detectCopilotErrors(output) { * resolvedModel: string, * routingEffort?: string|null, * multiProviderJson?: string, - * }} options + * subagentModelsJson?: string, + * }} options `subagentModelsJson` maps custom agent names to their resolved, qualified models. * @returns {NodeJS.ProcessEnv} */ -function buildCopilotSDKChildEnv({ sdkEnv, copilotSDKMode, copilotConnectionToken, providerBaseUrl, providerType, providerWireApi, resolvedModel, routingEffort, multiProviderJson }) { +function buildCopilotSDKChildEnv({ sdkEnv, copilotSDKMode, copilotConnectionToken, providerBaseUrl, providerType, providerWireApi, resolvedModel, routingEffort, multiProviderJson, subagentModelsJson }) { if (!copilotSDKMode) { return sdkEnv; } @@ -1128,6 +1459,7 @@ function buildCopilotSDKChildEnv({ sdkEnv, copilotSDKMode, copilotConnectionToke ...sdkEnv, COPILOT_CONNECTION_TOKEN: copilotConnectionToken, ...(multiProviderJson ? { GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON: multiProviderJson } : {}), + ...(subagentModelsJson ? { GH_AW_COPILOT_SDK_SUBAGENT_MODELS: subagentModelsJson } : {}), COPILOT_MODEL: qualifiedModel || resolvedModel, ...(routingEffort ? { COPILOT_REASONING_EFFORT: routingEffort } : {}), // Native Copilot CLI BYOK env vars — consumed by the headless sidecar for all sessions. @@ -1490,6 +1822,18 @@ async function main() { process.exit(0); } + // Sub-agent models are resolved on routed runs too, but a pre-existing noop must not + // make startup depend on sub-agent model availability. Threat detection uses no + // sub-agents, so it neither reads, resolves, records nor writes agent files. + const resolveSubagentModels = !isDetectionPhase(process.env.GH_AW_PHASE); + if (!copilotSDKMode && resolveSubagentModels) { + await applyCopilotCLISubagentModels({ + reflectData: awfReflectData, + refetchReflectData, + logger: log, + }); + } + // Resolve BYOK provider from live reflect data (SDK mode only). // Multi-provider BYOK is the only supported mode — fail immediately if the // provider cannot be resolved so retries are not wasted on a misconfigured environment. @@ -1499,12 +1843,13 @@ async function main() { let providerWireApi = ""; let resolvedModel = ""; let multiProviderJson = ""; + let subagentModelsJson = ""; let primaryProviderName = ""; if (copilotSDKMode) { const configuredModel = modelRoutingSelection?.wire_model || process.env.COPILOT_MODEL || ""; const modelsJson = loadModelsJson(); - const multiProvider = await resolveCopilotSDKMultiProvider({ + let multiProvider = await resolveCopilotSDKMultiProvider({ configuredModel, requestedModel: selectionConfiguredModel, routed: Boolean(modelRoutingSelection), @@ -1542,6 +1887,43 @@ async function main() { models: multiProvider.models, }); + // The same custom agents the SDK driver loads: workflow definitions override repository ones. + const sdkWorkingDirectory = path.resolve(process.env.GITHUB_WORKSPACE || process.cwd(), process.env.GH_AW_ENGINE_CWD || "."); + const agents = resolveSubagentModels ? listCopilotSubagents(sdkWorkingDirectory, path.join(os.tmpdir(), "gh-aw"), log) : []; + try { + const resolutions = await resolveCopilotSubagentModels({ + agents, + repositoryFallback: "the SDK session gives it the session model", + reflectData: awfReflectData, + refetchReflectData, + providerConfig: multiProvider, + refreshProviderConfig: async reflectData => { + multiProvider = resolveMultiProviderFromReflect({ + model: configuredModel, + wireApi: process.env.COPILOT_PROVIDER_WIRE_API, + reflectData, + modelsJson, + logger: log, + }) || { model: "", providers: [], models: [] }; + return multiProvider; + }, + logger: log, + }); + subagentModelsJson = JSON.stringify(Object.fromEntries(resolutions.map(resolution => [resolution.name, resolution.model]))); + } catch (failure) { + if (!(failure instanceof ModelAliasResolutionError)) throw failure; + process.exit(1); + } + + resolvedModel = multiProvider.model; + primaryProviderName = multiProvider.models.find(m => `${m.provider}/${m.id}` === resolvedModel)?.provider ?? multiProvider.providers[0]?.name ?? ""; + const refreshedPrimaryProvider = multiProvider.providers.find(p => p.name === primaryProviderName) ?? multiProvider.providers[0]; + providerBaseUrl = refreshedPrimaryProvider?.baseUrl ?? ""; + providerType = refreshedPrimaryProvider?.type ?? "openai"; + const configuredProviderWireApi = process.env.COPILOT_PROVIDER_WIRE_API; + providerWireApi = configuredProviderWireApi === "responses" || configuredProviderWireApi === "completions" ? configuredProviderWireApi : refreshedPrimaryProvider?.wireApi || ""; + multiProviderJson = JSON.stringify({ model: multiProvider.model, providers: multiProvider.providers, models: multiProvider.models }); + const uniqueProviderBaseUrls = [...new Set(multiProvider.providers.map(provider => String(provider.baseUrl || "").trim()).filter(Boolean))]; if (uniqueProviderBaseUrls.length === 0) { log("copilot-sdk driver mode: no provider baseUrls to probe — skipping listener readiness check"); @@ -1591,6 +1973,7 @@ async function main() { resolvedModel, routingEffort: modelRoutingSelection?.effort ?? null, multiProviderJson, + subagentModelsJson, }); const childEnv = Object.keys(sdkChildEnv).length > 0 ? buildCopilotEngineChildEnv(process.env, sdkChildEnv) : undefined; @@ -2177,6 +2560,11 @@ if (typeof module !== "undefined" && module.exports) { applyModelFallback, applyCopilotModelAliasResolution, resolveCopilotSDKMultiProvider, + resolveCopilotSubagentModels, + applyCopilotCLISubagentModels, + listCopilotAgentFiles, + listCopilotSubagents, + writeCopilotPersonalAgentCopy, applyCopilotWireAPI, warnCopilotSubagentEndpoints, resolveAWFModelRoutingSelection, diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index a44893bab78..3a0c7be383a 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -75,6 +75,10 @@ const { warnCopilotSubagentEndpoints, applyCopilotModelAliasResolution, resolveCopilotSDKMultiProvider, + resolveCopilotSubagentModels, + applyCopilotCLISubagentModels, + listCopilotAgentFiles, + listCopilotSubagents, formatInferenceEndpointForLog, logCopilotInferenceConfiguration, resolveLongRunTokenThreshold, @@ -4675,3 +4679,647 @@ require(${JSON.stringify(require.resolve("./awf_reflect.cjs"))}).fetchAWFReflect }); }); }); + +describe("Copilot sub-agent model resolution (recorded /reflect from run 39304573724)", () => { + // Agent artifact of https://github.com/github/gh-aw/actions/runs/39304573724 (smoke-copilot-sub-agents, SDK mode): + // sandbox/firewall/awf-reflect.json (models_fetch_complete true, one configured Copilot endpoint with 38 models, + // no gpt-5.4-mini) and awf-config.json apiProxy.models (the workflow's alias map). That run silently served the + // mini54-whoami sub-agent (gpt-5.4-mini) on the main model gpt-5.3-codex (gh-aw#70388). + const fixtureDir = path.join(path.dirname(require.resolve("./copilot_harness.cjs")), "fixtures"); + const reflectPath = path.join(fixtureDir, "awf-reflect-copilot-sub-agents-39304573724.json"); + const awfConfigFixturePath = path.join(fixtureDir, "awf-config-models-copilot-sub-agents-39304573724.json"); + const recordedReflect = () => JSON.parse(fs.readFileSync(reflectPath, "utf8")); + const aliasMap = () => JSON.parse(fs.readFileSync(awfConfigFixturePath, "utf8")).apiProxy.models; + const { ModelAliasResolutionError } = require("./resolve_model_alias.cjs"); + const { resolveMultiProviderFromReflect } = require("./awf_reflect.cjs"); + const { qualifyModelForMultiProvider } = require("./copilot_sdk_multi_provider.cjs"); + const tempDirs = []; + + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); + }); + + function tempDir(prefix) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; + } + + function initGitRepo(dir, files = []) { + const { execFileSync } = require("child_process"); + execFileSync("git", ["init", "-q"], { cwd: dir }); + execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", "add", "-A"], { cwd: dir }); + if (files.length > 0) execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", "add", ...files], { cwd: dir }); + execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-q", "--allow-empty", "-m", "init"], { cwd: dir }); + } + + const readRecords = dir => { + const file = path.join(dir, "agent/model-resolution.json"); + return fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : null; + }; + + /** In-process: the harness sub-agent resolution step against the recorded catalogue. */ + async function resolveAgents(agents, { reflect = recordedReflect(), refreshed = [], providerConfig = null, refreshProviderConfig } = {}) { + const dir = tempDir("copilot-subagent-models-"); + const env = { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1" }; + const logs = []; + const refetch = vi.fn().mockResolvedValue(null); + for (const data of refreshed) refetch.mockResolvedValueOnce(data); + let resolutions = null; + let error = null; + try { + resolutions = await resolveCopilotSubagentModels({ agents, reflectData: reflect, refetchReflectData: refetch, providerConfig, refreshProviderConfig, aliasMap: aliasMap(), env, logger: message => logs.push(message) }); + } catch (err) { + error = err; + } + return { resolutions, error, refetch, logs, records: readRecords(dir) }; + } + + const sdkProviderConfig = () => resolveMultiProviderFromReflect({ model: "gpt-5.3-codex", reflectData: recordedReflect() }); + + it("case 1: model small resolves through the gh-aw alias to claude-haiku-5.5, not the main model", async () => { + const run = await resolveAgents([{ name: "haiku-small", model: "small" }]); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([{ name: "haiku-small", configuredModel: "small", resolvedModel: "claude-haiku-5.5", model: "claude-haiku-5.5" }]); + expect(run.records).toEqual([{ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }]); + }); + + it("case 1b: model large resolves through the gh-aw alias to claude-sonnet-5.5", async () => { + const run = await resolveAgents([{ name: "proposer", model: "large" }]); + expect(run.error).toBeNull(); + expect(run.resolutions[0].model).toBe("claude-sonnet-5.5"); + }); + + it("case 2: a live exact id claude-haiku-4.5 is unchanged", async () => { + const run = await resolveAgents([{ name: "haiku-whoami", model: "claude-haiku-4.5" }]); + expect(run.error).toBeNull(); + expect(run.refetch).not.toHaveBeenCalled(); + expect(run.resolutions).toEqual([{ name: "haiku-whoami", configuredModel: "claude-haiku-4.5", resolvedModel: "claude-haiku-4.5", model: "claude-haiku-4.5" }]); + expect(run.records).toEqual([{ engine: "copilot", role: "haiku-whoami", configured_model: "claude-haiku-4.5", resolved_model: "claude-haiku-4.5", provider: "copilot", status: "exact", cause: null }]); + }); + + it("case 3: a dead id gpt-5.4-mini fails closed with model_unavailable after one refresh", async () => { + const run = await resolveAgents( + [ + { name: "haiku-whoami", model: "claude-haiku-4.5" }, + { name: "mini54-whoami", model: "gpt-5.4-mini" }, + { name: "other-mini54", model: "gpt-5.4-mini" }, + ], + { refreshed: [recordedReflect()] } + ); + expect(run.error).toBeInstanceOf(ModelAliasResolutionError); + expect(run.error.reason).toBe("model_unavailable"); + expect(run.error.message).toContain("The requested model is not supported: 'gpt-5.4-mini' is not available in the workflow's live model catalog"); + expect(run.refetch).toHaveBeenCalledTimes(1); + expect(run.resolutions).toBeNull(); + const failure = run.logs.find(message => message.startsWith("copilot sub-agent 'mini54-whoami':")); + expect(failure).toContain("Refusing to start Copilot with a different sub-agent model."); + expect(detectErrors(failure).modelNotSupportedError).toBe(true); + // Every sub-agent sharing the dead model gets its own failed record. + expect(run.records).toEqual([ + { engine: "copilot", role: "haiku-whoami", configured_model: "claude-haiku-4.5", resolved_model: "claude-haiku-4.5", provider: "copilot", status: "exact", cause: null }, + { engine: "copilot", role: "mini54-whoami", configured_model: "gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }, + { engine: "copilot", role: "other-mini54", configured_model: "gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }, + ]); + }); + + it("case 3b: medium is not a gh-aw alias and fails closed as a literal id", async () => { + const run = await resolveAgents([{ name: "workflow-agent", model: "medium" }], { refreshed: [recordedReflect()] }); + expect(run.error).toBeInstanceOf(ModelAliasResolutionError); + expect(run.error.reason).toBe("model_unavailable"); + expect(run.records).toEqual([{ engine: "copilot", role: "workflow-agent", configured_model: "medium", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }]); + }); + + it("case 3c: a dead id starts when the bounded refresh lists it", async () => { + const refreshed = recordedReflect(); + refreshed.endpoints.find(endpoint => endpoint.provider === "copilot").models.push("gpt-5.4-mini"); + const providerConfig = sdkProviderConfig(); + const run = await resolveAgents([{ name: "mini54-whoami", model: "gpt-5.4-mini" }], { + refreshed: [refreshed], + providerConfig, + refreshProviderConfig: data => Promise.resolve(resolveMultiProviderFromReflect({ model: "gpt-5.3-codex", reflectData: data })), + }); + expect(run.error).toBeNull(); + expect(run.refetch).toHaveBeenCalledTimes(1); + expect(run.resolutions[0].model).toBe(qualifyModelForMultiProvider("gpt-5.4-mini", resolveMultiProviderFromReflect({ model: "gpt-5.3-codex", reflectData: refreshed }))); + }); + + it("case 4: the qualified alias copilot/opus resolves to the newest live Opus", async () => { + const run = await resolveAgents([{ name: "reviewer", model: "copilot/opus" }]); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([{ name: "reviewer", configuredModel: "copilot/opus", resolvedModel: "claude-opus-5.5", model: "claude-opus-5.5" }]); + expect(run.records).toEqual([{ engine: "copilot", role: "reviewer", configured_model: "copilot/opus", resolved_model: "claude-opus-5.5", provider: "copilot", status: "resolved", cause: null }]); + }); + + it("preserves explicit Copilot provider scope during liveness checks and SDK qualification", async () => { + const reflect = recordedReflect(); + reflect.endpoints.push({ provider: "openai", configured: true, models: ["gpt-5.4-mini"] }); + const run = await resolveAgents([{ name: "mini54-whoami", model: "copilot/gpt-5.4-mini" }], { + reflect, + refreshed: [reflect], + providerConfig: { + providers: [{ name: "copilot-responses" }, { name: "openai" }], + models: [{ id: "gpt-5.4-mini", provider: "openai" }], + }, + }); + expect(run.error).toBeInstanceOf(ModelAliasResolutionError); + expect(run.error.reason).toBe("model_unavailable"); + expect(run.records).toEqual([{ engine: "copilot", role: "mini54-whoami", configured_model: "copilot/gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }]); + }); + + it("records qualification ambiguity as a failed sub-agent and continues collecting failures", async () => { + const providerConfig = sdkProviderConfig(); + const duplicate = providerConfig.models.find(model => model.id === "claude-haiku-5.5"); + providerConfig.providers.push({ name: "copilot-duplicate" }); + providerConfig.models.push({ ...duplicate, provider: "copilot-duplicate" }); + const run = await resolveAgents( + [ + { name: "ambiguous", model: "claude-haiku-5.5" }, + { name: "also-dead", model: "gpt-5.4-mini" }, + ], + { providerConfig, refreshed: [recordedReflect()] } + ); + expect(run.error).toBeInstanceOf(ModelAliasResolutionError); + expect(run.records).toEqual([ + { engine: "copilot", role: "ambiguous", configured_model: "claude-haiku-5.5", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }, + { engine: "copilot", role: "also-dead", configured_model: "gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }, + ]); + }); + + it("case 5: an inherited sub-agent model inherits the main model and gets no record", async () => { + const run = await resolveAgents([{ name: "helper", model: "inherited" }]); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([]); + expect(run.records).toBeNull(); + }); + + it("case 6 (SDK): customAgents[].model is the qualified resolved model, not the session model", async () => { + const providerConfig = sdkProviderConfig(); + const expected = qualifyModelForMultiProvider("claude-haiku-5.5", providerConfig); + expect(expected).toMatch(/^copilot-(?:responses|completions)\/claude-haiku-5\.5$/); + const run = await resolveAgents( + [ + { name: "haiku-small", model: "small" }, + { name: "helper", model: "inherited" }, + ], + { providerConfig } + ); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([{ name: "haiku-small", configuredModel: "small", resolvedModel: "claude-haiku-5.5", model: expected }]); + + // The harness hands the resolved models to the SDK driver, which passes them to the session. + const { runWithCopilotSDK } = require("./copilot_sdk_session.cjs"); + const base = tempDir("copilot-subagent-sdk-session-"); + fs.mkdirSync(path.join(base, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(base, ".github/agents/haiku-small.agent.md"), "---\nmodel: small\n---\nWho are you?"); + fs.writeFileSync(path.join(base, ".github/agents/helper.agent.md"), "---\ndescription: helper\nmodel: inherited\n---\nHelp."); + vi.stubEnv("GH_AW_SESSION_STATE_BASE_DIR", base); + vi.stubEnv("COPILOT_MODEL", ""); + vi.stubEnv("GH_AW_MODEL_FALLBACK", ""); + vi.stubEnv("GH_AW_MODEL_ROUTING", ""); + vi.stubEnv("GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON", JSON.stringify(providerConfig)); + vi.stubEnv("GH_AW_COPILOT_SDK_SUBAGENT_MODELS", JSON.stringify(Object.fromEntries(run.resolutions.map(resolution => [resolution.name, resolution.model])))); + const createSession = vi.fn().mockResolvedValue({ sessionId: "subagent-models", on: () => {}, sendAndWait: async () => ({ data: { content: "done" } }), disconnect: async () => {} }); + class FakeCopilotClient { + start = async () => {}; + stop = async () => {}; + createSession = createSession; + } + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test", + logger: () => {}, + workingDirectory: base, + agentsBaseDir: base, + bare: true, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: () => ({}) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(0); + const session = createSession.mock.calls[0][0]; + expect(session.model).toBe(providerConfig.model); + expect(session.customAgents.find(agent => agent.name === "haiku-small").model).toBe(expected); + expect(session.customAgents.find(agent => agent.name === "haiku-small").model).not.toBe(session.model); + expect(session.customAgents.find(agent => agent.name === "helper").model).toBe(session.model); + }); + + const vscodePinnedAgent = "---\nname: planner\ndescription: Plans work\nmodel: GPT-5.4 (copilot)\n---\nPlan the work.\n"; + + it("a repository agent pinned to the VS Code name GPT-5.4 (copilot) records failed and warns without stopping the run", async () => { + // `model: GPT-5.4 (copilot)` is a real VS Code pin (adamgell/cmtraceopen planner.agent.md). + const agentPath = "/repo/.github/agents/planner.agent.md"; + const run = await resolveAgents([{ name: "planner", model: "GPT-5.4 (copilot)", declared: false, path: agentPath }], { refreshed: [recordedReflect()] }); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([]); + expect(run.records).toEqual([{ engine: "copilot", role: "planner", configured_model: "GPT-5.4 (copilot)", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }]); + const warning = run.logs.find(message => message.startsWith("warning: repository agent 'planner'")); + expect(warning).toContain(agentPath); + expect(warning).toContain("'GPT-5.4 (copilot)'"); + expect(warning).toContain("model_unavailable"); + expect(detectErrors(run.logs.join("\n")).modelNotSupportedError).toBe(false); + }); + + it("a repository agent with model small resolves to claude-haiku-5.5", async () => { + const run = await resolveAgents([{ name: "haiku-small", model: "small", declared: false, path: "/repo/.github/agents/haiku-small.agent.md" }]); + expect(run.error).toBeNull(); + expect(run.resolutions).toEqual([{ name: "haiku-small", configuredModel: "small", resolvedModel: "claude-haiku-5.5", model: "claude-haiku-5.5" }]); + expect(run.records).toEqual([{ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }]); + }); + + it("SDK mode: a repository agent pinned to GPT-5.4 (copilot) gets the session model with a warning", async () => { + const providerConfig = sdkProviderConfig(); + const run = await resolveAgents( + [ + { name: "planner", model: "GPT-5.4 (copilot)", declared: false, path: "planner.agent.md" }, + { name: "haiku-small", model: "small", declared: false, path: "haiku-small.agent.md" }, + ], + { providerConfig, refreshed: [recordedReflect()] } + ); + expect(run.error).toBeNull(); + expect(run.records).toContainEqual({ engine: "copilot", role: "planner", configured_model: "GPT-5.4 (copilot)", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }); + expect(run.resolutions.map(resolution => resolution.name)).toEqual(["haiku-small"]); + + const { runWithCopilotSDK } = require("./copilot_sdk_session.cjs"); + const base = tempDir("copilot-subagent-sdk-repository-"); + fs.mkdirSync(path.join(base, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(base, ".github/agents/planner.agent.md"), vscodePinnedAgent); + fs.writeFileSync(path.join(base, ".github/agents/haiku-small.agent.md"), "---\nmodel: small\n---\nWho are you?"); + vi.stubEnv("GH_AW_SESSION_STATE_BASE_DIR", base); + vi.stubEnv("COPILOT_MODEL", ""); + vi.stubEnv("GH_AW_MODEL_FALLBACK", ""); + vi.stubEnv("GH_AW_MODEL_ROUTING", ""); + vi.stubEnv("GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON", JSON.stringify(providerConfig)); + vi.stubEnv("GH_AW_COPILOT_SDK_SUBAGENT_MODELS", JSON.stringify(Object.fromEntries(run.resolutions.map(resolution => [resolution.name, resolution.model])))); + const createSession = vi.fn().mockResolvedValue({ sessionId: "repository-agent-models", on: () => {}, sendAndWait: async () => ({ data: { content: "done" } }), disconnect: async () => {} }); + class FakeCopilotClient { + start = async () => {}; + stop = async () => {}; + createSession = createSession; + } + const logs = []; + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test", + logger: message => logs.push(message), + workingDirectory: base, + agentsBaseDir: base, + bare: true, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: () => ({}) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(0); + const session = createSession.mock.calls[0][0]; + expect(session.customAgents.find(agent => agent.name === "planner").model).toBe(session.model); + expect(session.customAgents.find(agent => agent.name === "haiku-small").model).toBe(qualifyModelForMultiProvider("claude-haiku-5.5", providerConfig)); + expect(logs).toContainEqual(expect.stringMatching(/warning: custom agent "planner" model "GPT-5\.4 \(copilot\)".*using session model/)); + }); + + it("CLI mode writes a personal agent copy with the same file name and the resolved model; the checkout stays untouched", async () => { + const { execFileSync } = require("child_process"); + const dir = tempDir("copilot-subagent-cli-personal-"); + const workspace = path.join(dir, "workspace"); + const agentsDir = path.join(workspace, ".github/agents"); + fs.mkdirSync(agentsDir, { recursive: true }); + const files = { + "haiku-small.agent.md": "---\nname: haiku-small\ndescription: Small model\nmodel: small # alias\ntools: [read]\n---\nWho are you?\nmodel: small\n", + "haiku-whoami.agent.md": "---\nmodel: claude-haiku-4.5\n---\nExact.\n", + "multiline.agent.md": "---\nname: multiline\nmodel: >-\n small\ndescription: retained\n---\nPrompt retained.\n", + }; + for (const [name, content] of Object.entries(files)) fs.writeFileSync(path.join(agentsDir, name), content); + initGitRepo(workspace); + const awfConfigPath = path.join(dir, "awf-config.json"); + fs.copyFileSync(awfConfigFixturePath, awfConfigPath); + const copilotHome = path.join(dir, "copilot-home"); + const resolutions = await applyCopilotCLISubagentModels({ + reflectData: recordedReflect(), + workingDirectory: workspace, + agentsBaseDir: path.join(dir, "gh-aw"), + env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath, COPILOT_HOME: copilotHome }, + logger: () => {}, + }); + expect(resolutions.map(resolution => [resolution.name, resolution.model])).toEqual([ + ["haiku-small", "claude-haiku-5.5"], + ["haiku-whoami", "claude-haiku-4.5"], + ["multiline", "claude-haiku-5.5"], + ]); + const personalAgentsDir = path.join(copilotHome, "agents"); + expect(fs.readdirSync(personalAgentsDir).sort()).toEqual(["haiku-small.agent.md", "multiline.agent.md"]); + expect(fs.readFileSync(path.join(personalAgentsDir, "haiku-small.agent.md"), "utf8")).toBe(files["haiku-small.agent.md"].replace("model: small # alias", 'model: "claude-haiku-5.5" # alias')); + expect(fs.readFileSync(path.join(personalAgentsDir, "multiline.agent.md"), "utf8")).toBe('---\nname: multiline\nmodel: "claude-haiku-5.5"\ndescription: retained\n---\nPrompt retained.\n'); + for (const [name, content] of Object.entries(files)) expect(fs.readFileSync(path.join(agentsDir, name), "utf8")).toBe(content); + expect(execFileSync("git", ["status", "--porcelain"], { cwd: workspace, encoding: "utf8" })).toBe(""); + }); + + it("lists only .agent.md files and derives fallback names from that suffix", () => { + const dir = tempDir("copilot-subagent-list-"); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, "notes.md"), "---\nmodel: gpt-5.4-mini\n---\nNot an agent."); + fs.writeFileSync(path.join(dir, "use-filename.agent.md"), "---\nmodel: claude-haiku-4.5\n---\nAgent."); + expect(listCopilotAgentFiles(dir, () => {})).toEqual([{ name: "use-filename", path: path.join(dir, "use-filename.agent.md"), model: "claude-haiku-4.5" }]); + }); + + it("loads the CLI harness without importing the SDK-only undici dependency", () => { + const dir = tempDir("copilot-harness-no-sdk-dependency-"); + const preloadPath = path.join(dir, "block-undici.cjs"); + fs.writeFileSync( + preloadPath, + `const Module = require("module"); +const load = Module._load; +Module._load = function(request, ...args) { + if (request === "undici") throw new Error("undici is unavailable"); + return load.call(this, request, ...args); +};` + ); + const result = spawnSync(process.execPath, ["--require", preloadPath, "-e", `require(${JSON.stringify(require.resolve("./copilot_harness.cjs"))})`], { + cwd: path.dirname(require.resolve("./copilot_harness.cjs")), + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + }); + + it("treats an inline sub-agent as declared and lets it override a repository agent with the same name", () => { + const dir = tempDir("copilot-subagent-declared-"); + const workspace = path.join(dir, "workspace"); + const base = path.join(dir, "gh-aw"); + fs.mkdirSync(path.join(workspace, ".github/agents"), { recursive: true }); + fs.mkdirSync(path.join(base, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(workspace, ".github/agents/planner.agent.md"), vscodePinnedAgent); + fs.writeFileSync(path.join(workspace, ".github/agents/shared.agent.md"), "---\nmodel: GPT-5.4 (copilot)\n---\nRepository.\n"); + fs.writeFileSync(path.join(base, ".github/agents/shared.agent.md"), "---\nmodel: small\n---\nInline.\n"); + expect(listCopilotSubagents(workspace, base, () => {})).toEqual([ + { name: "planner", path: path.join(workspace, ".github/agents/planner.agent.md"), model: "GPT-5.4 (copilot)", declared: false }, + { name: "shared", path: path.join(base, ".github/agents/shared.agent.md"), model: "small", declared: true }, + ]); + }); + + it.each([true, false])("CLI mode: a personal copy that cannot be written fails closed only for a declared sub-agent (declared=%s)", async declared => { + const dir = tempDir("copilot-subagent-cli-copy-failure-"); + const workspace = path.join(dir, "workspace"); + const base = path.join(dir, "gh-aw"); + const agentsDir = path.join(declared ? base : workspace, ".github/agents"); + fs.mkdirSync(agentsDir, { recursive: true }); + fs.writeFileSync(path.join(agentsDir, "haiku-small.agent.md"), "---\nmodel: small\n---\nWho are you?\n"); + const awfConfigPath = path.join(dir, "awf-config.json"); + fs.copyFileSync(awfConfigFixturePath, awfConfigPath); + // COPILOT_HOME is a file, so the personal agents directory cannot be created. + const copilotHome = path.join(dir, "copilot-home"); + fs.writeFileSync(copilotHome, "not a directory"); + vi.spyOn(process, "exit").mockImplementation(code => { + throw new Error(`exit ${code}`); + }); + const logs = []; + const apply = applyCopilotCLISubagentModels({ + reflectData: recordedReflect(), + workingDirectory: workspace, + agentsBaseDir: base, + env: { GH_AW_TMP_DIR: dir, AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath, COPILOT_HOME: copilotHome }, + logger: message => logs.push(message), + }); + if (declared) { + await expect(apply).rejects.toThrow("exit 1"); + expect(logs).toContainEqual(expect.stringContaining("copilot sub-agent 'haiku-small': cannot write personal agent copy")); + } else { + expect((await apply).map(resolution => resolution.model)).toEqual(["claude-haiku-5.5"]); + expect(logs).toContainEqual(expect.stringContaining("warning: repository agent 'haiku-small'")); + } + expect(fs.readFileSync(path.join(agentsDir, "haiku-small.agent.md"), "utf8")).toBe("---\nmodel: small\n---\nWho are you?\n"); + }); + + /** + * Spawn the harness against the recorded catalogue. `agents` are repository agent files in the engine + * working directory; `inlineAgents` are workflow-declared sub-agents, staged under `/gh-aw` + * and restored (untracked) into the workspace as `restore_inline_sub_agents.sh` does. + */ + async function runHarness({ sdk = false, agents = {}, inlineAgents = {}, workspaceAgents = {}, noop = false, reflect = recordedReflect(), env: envOverrides = {} }) { + const dir = tempDir("copilot-subagent-harness-"); + const workspace = path.join(dir, "workspace"); + const engineCwd = envOverrides.GH_AW_ENGINE_CWD || workspace; + fs.mkdirSync(path.join(engineCwd, ".github/agents"), { recursive: true }); + fs.mkdirSync(path.join(workspace, ".github/agents"), { recursive: true }); + for (const [name, content] of Object.entries(agents)) fs.writeFileSync(path.join(engineCwd, ".github/agents", name), content); + for (const [name, content] of Object.entries(workspaceAgents)) fs.writeFileSync(path.join(workspace, ".github/agents", name), content); + initGitRepo(workspace); + if (engineCwd !== workspace) initGitRepo(engineCwd); + const ostmp = path.join(dir, "ostmp"); + fs.mkdirSync(path.join(ostmp, "gh-aw", ".github/agents"), { recursive: true }); + for (const [name, content] of Object.entries(inlineAgents)) { + fs.writeFileSync(path.join(ostmp, "gh-aw", ".github/agents", name), content); + fs.writeFileSync(path.join(workspace, ".github/agents", name), content); + } + const startedPath = path.join(dir, "copilot-started.json"); + let providerPort = 0; + let sdkPort = 0; + if (sdk) { + const freePort = async () => { + const server = require("net").createServer(); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address(); + await new Promise(resolve => server.close(resolve)); + return port; + }; + [providerPort, sdkPort] = [await freePort(), await freePort()]; + for (const endpoint of reflect.endpoints.filter(endpoint => endpoint.configured)) Object.assign(endpoint, { base_url: `http://127.0.0.1:${providerPort}`, models_url: `http://127.0.0.1:${providerPort}/models` }); + } + fs.writeFileSync(path.join(dir, "reflect.json"), JSON.stringify(reflect)); + const preloadPath = path.join(dir, "reflect-preload.cjs"); + fs.writeFileSync( + preloadPath, + `const body = require("fs").readFileSync(${JSON.stringify(path.join(dir, "reflect.json"))}, "utf8"); +require(${JSON.stringify(require.resolve("./awf_reflect.cjs"))}).fetchAWFReflect = async () => ({ ok: true, reflectData: JSON.parse(body) }); +if (${providerPort} && process.argv[1].endsWith("copilot_harness.cjs")) require("net").createServer(socket => socket.end()).listen(${providerPort}, "127.0.0.1").unref();` + ); + const stubPath = path.join(dir, "copilot-stub.cjs"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const path = require("path"); +const args = process.argv.slice(2); +const agentsDir = path.join(process.env.GH_AW_ENGINE_CWD || process.env.GITHUB_WORKSPACE || process.cwd(), ".github", "agents"); +const readAgents = dir => (fs.existsSync(dir) ? Object.fromEntries(fs.readdirSync(dir).filter(name => name.endsWith(".agent.md")).map(name => [name, fs.readFileSync(path.join(dir, name), "utf8")])) : {}); +const agentFiles = readAgents(agentsDir); +const personalAgents = readAgents(path.join(process.env.COPILOT_HOME, "agents")); +fs.writeFileSync(${JSON.stringify(startedPath)}, JSON.stringify({ args, agentFiles, personalAgents, subagentModels: process.env.GH_AW_COPILOT_SDK_SUBAGENT_MODELS || null })); +if (args.includes("--session-id")) { + const model = args.includes("--model") ? args[args.lastIndexOf("--model") + 1] : process.env.COPILOT_MODEL; + const sessionDir = path.join(process.env.COPILOT_HOME, "session-state", args[args.indexOf("--session-id") + 1]); + fs.mkdirSync(sessionDir, { recursive: true }); + fs.appendFileSync(path.join(sessionDir, "events.jsonl"), JSON.stringify({ type: "assistant.message", data: { model, content: "done" } }) + "\\n"); +} +process.stdout.write("done\\n");` + ); + const sidecarPath = path.join(dir, "copilot-sidecar.cjs"); + fs.writeFileSync(sidecarPath, `#!/usr/bin/env node\nconst args = process.argv.slice(2);\nrequire("net").createServer(socket => socket.end()).listen(Number(args[args.indexOf("--port") + 1]), "127.0.0.1");\n`, { mode: 0o755 }); + const awfConfigPath = path.join(dir, "awf-config.json"); + fs.copyFileSync(awfConfigFixturePath, awfConfigPath); + const safeOutputsPath = path.join(dir, "safe-outputs.jsonl"); + if (noop) fs.writeFileSync(safeOutputsPath, '{"type":"noop","message":"nothing to do"}\n'); + const env = { + ...harnessChildEnv, + GH_AW_TMP_DIR: dir, + TMPDIR: ostmp, + GITHUB_WORKSPACE: workspace, + ...(noop ? { GH_AW_SAFE_OUTPUTS: safeOutputsPath } : {}), + COPILOT_HOME: path.join(dir, "copilot-home"), + GH_AW_HARNESS_MAX_RETRIES: "0", + GH_AW_HARNESS_STALL_WARNING_MS: "0", + NODE_OPTIONS: `--require ${preloadPath}`, + }; + for (const key of [ + "COPILOT_MODEL", + "GH_AW_MODEL_FALLBACK", + "GH_AW_MODEL_ROUTING", + "GH_AW_SKIP_REFLECT", + "COPILOT_SDK_URI", + "GH_AW_PHASE", + "GH_AW_SAFE_OUTPUTS", + "COPILOT_PROVIDER_WIRE_API", + "GH_AW_MODEL_ALIASES", + "GH_AW_ENGINE_CWD", + "GH_AW_SUB_AGENT_DIR", + "GH_AW_SUB_AGENT_EXT", + "GH_AW_COPILOT_SDK_SUBAGENT_MODELS", + ]) + delete env[key]; + if (noop) env.GH_AW_SAFE_OUTPUTS = safeOutputsPath; + Object.assign(env, { AWF_REFLECT_ENABLED: "1", GH_AW_AWF_CONFIG_PATH: awfConfigPath, COPILOT_MODEL: "gpt-5.3-codex", ...(sdk ? { COPILOT_SDK_URI: `http://127.0.0.1:${sdkPort}` } : {}) }, envOverrides); + const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, ...(sdk ? [stubPath, sidecarPath] : [stubPath, "--prompt", "Hello"])], { + cwd: path.dirname(require.resolve("./copilot_harness.cjs")), + env, + encoding: "utf8", + timeout: 20000, + }); + return { + result, + workspace, + engineCwd, + started: fs.existsSync(startedPath) ? JSON.parse(fs.readFileSync(startedPath, "utf8")) : null, + records: readRecords(dir), + agentFile: name => fs.readFileSync(path.join(engineCwd, ".github/agents", name), "utf8"), + personalAgentsDir: path.join(dir, "copilot-home", "agents"), + gitStatus: () => require("child_process").execFileSync("git", ["status", "--porcelain"], { cwd: workspace, encoding: "utf8" }), + }; + } + + const smokeAgents = { + "haiku-whoami.agent.md": "---\nname: haiku-whoami\nmodel: claude-haiku-4.5\n---\nWho are you?\n", + "mini54-whoami.agent.md": "---\nname: mini54-whoami\nmodel: gpt-5.4-mini\n---\nWho are you?\n", + }; + + it.each([false, true])("case 3 end to end: a dead declared sub-agent model stops the harness before Copilot starts (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, inlineAgents: smokeAgents }); + expect(run.result.status, run.result.stderr).toBe(1); + expect(run.started).toBeNull(); + expect(run.result.stderr).not.toContain("starting sidecar"); + expect(run.result.stderr).toContain("copilot sub-agent 'mini54-whoami': The requested model is not supported: 'gpt-5.4-mini' is not available in the workflow's live model catalog"); + expect(detectErrors(run.result.stderr).modelNotSupportedError).toBe(true); + expect(run.records).toEqual([ + { engine: "copilot", role: "main", configured_model: "gpt-5.3-codex", resolved_model: "gpt-5.3-codex", provider: null, status: "exact", cause: null }, + { engine: "copilot", role: "haiku-whoami", configured_model: "claude-haiku-4.5", resolved_model: "claude-haiku-4.5", provider: "copilot", status: "exact", cause: null }, + { engine: "copilot", role: "mini54-whoami", configured_model: "gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }, + ]); + expect(run.agentFile("mini54-whoami.agent.md")).toBe(smokeAgents["mini54-whoami.agent.md"]); + }); + + it.each([false, true])("a repository agent pinned to GPT-5.4 (copilot) does not stop the run (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, agents: { "planner.agent.md": vscodePinnedAgent } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started).not.toBeNull(); + expect(run.records).toContainEqual({ engine: "copilot", role: "planner", configured_model: "GPT-5.4 (copilot)", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }); + expect(run.result.stderr).toMatch(/warning: repository agent 'planner' \(.*planner\.agent\.md\) declares model 'GPT-5\.4 \(copilot\)'/); + expect(run.result.stderr).toContain(sdk ? "the SDK session gives it the session model" : "Copilot CLI uses the agent file unchanged"); + expect(detectErrors(run.result.stderr).modelNotSupportedError).toBe(false); + if (sdk) expect(JSON.parse(run.started.subagentModels)).toEqual({}); + expect(run.started.personalAgents).toEqual({}); + expect(run.agentFile("planner.agent.md")).toBe(vscodePinnedAgent); + }); + + it.each([false, true])("a declared inline sub-agent with gpt-5.4-mini still fails closed beside the same repository agent (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, agents: { "planner.agent.md": vscodePinnedAgent }, inlineAgents: { "mini54-whoami.agent.md": smokeAgents["mini54-whoami.agent.md"] } }); + expect(run.result.status, run.result.stderr).toBe(1); + expect(run.started).toBeNull(); + expect(run.result.stderr).toContain("copilot sub-agent 'mini54-whoami': The requested model is not supported: 'gpt-5.4-mini' is not available in the workflow's live model catalog"); + expect(run.result.stderr).toContain("warning: repository agent 'planner'"); + expect(run.records).toContainEqual({ engine: "copilot", role: "planner", configured_model: "GPT-5.4 (copilot)", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }); + expect(run.records).toContainEqual({ engine: "copilot", role: "mini54-whoami", configured_model: "gpt-5.4-mini", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }); + }); + + it("skips sub-agent resolution in the threat-detection phase", async () => { + const repositoryAgents = { "dead.agent.md": "---\nmodel: gpt-5.4-mini\n---\nDead.\n", "haiku-small.agent.md": "---\nmodel: small\n---\nWho are you?\n" }; + const run = await runHarness({ agents: repositoryAgents, inlineAgents: { "mini54-whoami.agent.md": smokeAgents["mini54-whoami.agent.md"] }, env: { GH_AW_PHASE: "detection" } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started).not.toBeNull(); + expect(run.result.stderr).not.toContain("copilot sub-agent"); + expect(run.result.stderr).not.toContain("repository agent"); + expect((run.records || []).filter(record => record.role !== "main")).toEqual([]); + expect(fs.existsSync(run.personalAgentsDir)).toBe(false); + for (const [name, content] of Object.entries(repositoryAgents)) expect(run.agentFile(name)).toBe(content); + }); + + it("skips CLI sub-agent validation when a noop is already present", async () => { + const run = await runHarness({ inlineAgents: { "dead.agent.md": "---\nmodel: gpt-5.4-mini\n---\nUnused.\n" }, noop: true }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started).toBeNull(); + expect(run.result.stderr).toContain("pre-flight: noop message found in safe-outputs"); + expect(run.result.stderr).not.toContain("copilot sub-agent"); + expect(run.records).toEqual([{ engine: "copilot", role: "main", configured_model: "gpt-5.3-codex", resolved_model: "gpt-5.3-codex", provider: null, status: "exact", cause: null }]); + }); + + it("scans only custom agents in the CLI's effective engine working directory", async () => { + const dir = tempDir("copilot-subagent-engine-cwd-"); + const engineCwd = path.join(dir, "engine-checkout"); + const run = await runHarness({ + agents: { "effective.agent.md": "---\nmodel: small\n---\nEffective checkout.\n" }, + workspaceAgents: { "unrelated.agent.md": "---\nmodel: gpt-5.4-mini\n---\nNot loaded by this CLI.\n" }, + env: { GH_AW_ENGINE_CWD: engineCwd }, + }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started).not.toBeNull(); + expect(run.records).toContainEqual({ engine: "copilot", role: "effective", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }); + expect(run.records.map(record => record.role)).not.toContain("unrelated"); + }); + + it("case 6 end to end (CLI): Copilot CLI starts with a personal copy of a repository agent carrying the resolved model", async () => { + const run = await runHarness({ agents: { "haiku-small.agent.md": "---\nname: haiku-small\nmodel: small\n---\nWho are you?\n" } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started).not.toBeNull(); + expect(run.started.personalAgents).toEqual({ "haiku-small.agent.md": '---\nname: haiku-small\nmodel: "claude-haiku-5.5"\n---\nWho are you?\n' }); + expect(run.started.agentFiles["haiku-small.agent.md"]).toBe("---\nname: haiku-small\nmodel: small\n---\nWho are you?\n"); + expect(run.agentFile("haiku-small.agent.md")).toBe("---\nname: haiku-small\nmodel: small\n---\nWho are you?\n"); + expect(run.gitStatus()).toBe(""); + expect(run.records).toContainEqual({ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }); + }); + + it("case 6 end to end (CLI): a declared inline sub-agent gets a personal copy with the resolved model", async () => { + const inline = "---\nname: inline-large\nmodel: large\n---\nPropose.\n"; + const run = await runHarness({ inlineAgents: { "inline-large.agent.md": inline } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.started.personalAgents).toEqual({ "inline-large.agent.md": '---\nname: inline-large\nmodel: "claude-sonnet-5.5"\n---\nPropose.\n' }); + expect(run.agentFile("inline-large.agent.md")).toBe(inline); + }); + + it("case 6 end to end (SDK): the driver receives the qualified resolved sub-agent model", async () => { + const run = await runHarness({ sdk: true, agents: { "haiku-small.agent.md": "---\nname: haiku-small\nmodel: small\n---\nWho are you?\n" } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(JSON.parse(run.started.subagentModels)).toEqual({ "haiku-small": qualifyModelForMultiProvider("claude-haiku-5.5", sdkProviderConfig()) }); + expect(run.agentFile("haiku-small.agent.md")).toBe("---\nname: haiku-small\nmodel: small\n---\nWho are you?\n"); + }); + + describe("case 7: routed runs (recorded /reflect routing from run 39147795806)", () => { + // fixtures/copilot-routed-run-39147795806/awf-reflect.json: the router selected gpt-5.6-luna; its Copilot + // catalogue lists claude-haiku-5.5 but no claude-opus-4.8. + const routedReflect = () => JSON.parse(fs.readFileSync(path.join(fixtureDir, "copilot-routed-run-39147795806", "awf-reflect.json"), "utf8")); + + it.each([false, true])("still resolves and records sub-agent models without a main-role record (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, reflect: routedReflect(), agents: { "haiku-small.agent.md": "---\nmodel: small\n---\nWho are you?\n" }, env: { GH_AW_MODEL_ROUTING: "1", COPILOT_MODEL: "" } }); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.records).toEqual([{ engine: "copilot", role: "haiku-small", configured_model: "small", resolved_model: "claude-haiku-5.5", provider: "copilot", status: "resolved", cause: null }]); + }); + + it.each([false, true])("still fails closed for a dead declared sub-agent model (sdk=%s)", async sdk => { + const run = await runHarness({ sdk, reflect: routedReflect(), inlineAgents: { "opus.agent.md": "---\nmodel: claude-opus-4.8\n---\nWho are you?\n" }, env: { GH_AW_MODEL_ROUTING: "1", COPILOT_MODEL: "" } }); + expect(run.result.status, run.result.stderr).toBe(1); + expect(run.started).toBeNull(); + expect(run.records).toEqual([{ engine: "copilot", role: "opus", configured_model: "claude-opus-4.8", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" }]); + }); + }); +}); diff --git a/actions/setup/js/copilot_sdk_driver.cjs b/actions/setup/js/copilot_sdk_driver.cjs index f8f6a631b80..700f4d99b1e 100644 --- a/actions/setup/js/copilot_sdk_driver.cjs +++ b/actions/setup/js/copilot_sdk_driver.cjs @@ -13,6 +13,7 @@ * COPILOT_MODEL — model override (optional) * GH_AW_COPILOT_SDK_MULTI_PROVIDER_JSON — JSON-encoded multi-provider config (required). * Shape: { model, providers: NamedProviderConfig[], models: ProviderModelConfig[] } + * GH_AW_COPILOT_SDK_SUBAGENT_MODELS — JSON map of custom agent name → model resolved by the harness (optional) * GH_AW_COPILOT_SDK_SERVER_ARGS — JSON-encoded allow-tool sidecar args (set by the engine) * * The sidecar is started and stopped by the harness; the driver only opens a diff --git a/actions/setup/js/copilot_sdk_driver.test.cjs b/actions/setup/js/copilot_sdk_driver.test.cjs index d47ce27f143..89c28ed66b6 100644 --- a/actions/setup/js/copilot_sdk_driver.test.cjs +++ b/actions/setup/js/copilot_sdk_driver.test.cjs @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, vi, beforeAll, afterAll, afterEach } from "vitest"; import { createRequire } from "module"; import * as fs from "fs"; import * as path from "path"; @@ -31,6 +31,10 @@ describe("copilot_sdk_driver.cjs", () => { }); describe("runWithCopilotSDK", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it.each([ ["gpt-5.4", "claude-sonnet-4.6", "copilot-responses", "copilot-completions"], ["claude-sonnet-4.6", "gpt-5.4", "copilot-completions", "copilot-responses"], @@ -43,9 +47,10 @@ describe("copilot_sdk_driver.cjs", () => { path.join(workspace, ".github/agents/researcher.agent.md"), `---\nname: researcher\ndescription: >-\n Research across\n providers\ntools:\n - read\n - grep\nmodel: copilot/${agentModel}\n---\nResearch carefully.\n` ); - fs.writeFileSync(path.join(workflow, ".github/agents/workflow.agent.md"), "---\nname: workflow-agent\nmodel: medium\ntools: read, grep\n---\nWorkflow prompt."); + fs.writeFileSync(path.join(workflow, ".github/agents/workflow.agent.md"), "---\nname: workflow-agent\ntools: read, grep\n---\nWorkflow prompt."); for (const alias of ["small", "large"]) fs.writeFileSync(path.join(workflow, `.github/agents/${alias}.agent.md`), `---\nmodel: ${alias}\n---\nAlias prompt.`); - fs.writeFileSync(path.join(workflow, ".github/agents/unknown.agent.md"), "---\nname: unknown-agent\nmodel: unavailable-model\n---\nKeep this agent."); + // The harness resolves aliases before the driver starts; the driver uses those models, not the session model. + vi.stubEnv("GH_AW_COPILOT_SDK_SUBAGENT_MODELS", JSON.stringify({ small: `${agentProvider}/${agentModel}`, large: agentModel })); const createSession = vi.fn().mockResolvedValue({ sessionId: `${sessionProvider}-agent-session`, on: () => {}, @@ -90,21 +95,48 @@ describe("copilot_sdk_driver.cjs", () => { customAgents: expect.arrayContaining([ { name: "researcher", displayName: "researcher", description: "Research across providers", tools: ["read", "grep"], prompt: "Research carefully.", model: `${agentProvider}/${agentModel}` }, { name: "workflow-agent", displayName: "workflow-agent", tools: ["read", "grep"], prompt: "Workflow prompt.", model: `${sessionProvider}/${sessionModel}` }, - { name: "unknown-agent", displayName: "unknown-agent", prompt: "Keep this agent.", model: `${sessionProvider}/${sessionModel}` }, - { name: "small", displayName: "small", prompt: "Alias prompt.", model: `${sessionProvider}/${sessionModel}` }, - { name: "large", displayName: "large", prompt: "Alias prompt.", model: `${sessionProvider}/${sessionModel}` }, + { name: "small", displayName: "small", prompt: "Alias prompt.", model: `${agentProvider}/${agentModel}` }, + { name: "large", displayName: "large", prompt: "Alias prompt.", model: `${agentProvider}/${agentModel}` }, ]), }) ); - expect(logger).toHaveBeenCalledWith(expect.stringMatching(/warning:.*unknown-agent.*unavailable-model.*session model/i)); + }); + + it.each(["medium", "GPT-5.4 (copilot)"])("gives a repository agent whose model %s the harness could not resolve the session model with a warning", async agentModel => { + const workflow = path.join(testSessionStateDir, `unresolved-${agentModel.replace(/\W+/g, "-")}-workflow`); + fs.mkdirSync(path.join(workflow, ".github/agents"), { recursive: true }); + fs.writeFileSync(path.join(workflow, ".github/agents/unknown.agent.md"), `---\nname: unknown-agent\nmodel: ${agentModel}\n---\nUnknown model.`); + // The harness recorded the failure and left the repository agent out of the resolved map. + vi.stubEnv("GH_AW_COPILOT_SDK_SUBAGENT_MODELS", "{}"); + const sessionId = `unresolved-${agentModel.replace(/\W+/g, "-")}-session`; + const createSession = vi.fn().mockResolvedValue({ sessionId, on: () => {}, sendAndWait: vi.fn().mockResolvedValue({ data: { content: "done" } }), disconnect: vi.fn().mockResolvedValue(undefined) }); + class FakeCopilotClient { + start = vi.fn().mockResolvedValue(undefined); + createSession = createSession; + stop = vi.fn().mockResolvedValue(undefined); + } + const logger = vi.fn(); + const result = await runWithCopilotSDK({ + sdkUri: "http://127.0.0.1:3002", + prompt: "test prompt", + model: "copilot/gpt-5.4", + models: [{ id: "gpt-5.4", provider: "copilot-responses" }], + providers: [{ name: "copilot-responses", type: "openai", baseUrl: "http://api-proxy:10002", wireApi: "responses" }], + workingDirectory: workflow, + agentsBaseDir: workflow, + bare: true, + logger, + sdkModule: { CopilotClient: FakeCopilotClient, RuntimeConnection: { forUri: vi.fn(() => ({})) }, approveAll: () => "allow" }, + }); + expect(result.exitCode).toBe(0); + expect(createSession.mock.calls[0][0].customAgents).toEqual([{ name: "unknown-agent", displayName: "unknown-agent", prompt: "Unknown model.", model: "copilot-responses/gpt-5.4" }]); + expect(logger).toHaveBeenCalledWith(expect.stringMatching(/warning:.*unknown-agent.*session model/i)); const captured = fs - .readFileSync(path.join(testSessionStateDir, `${sessionProvider}-agent-session/events.jsonl`), "utf8") + .readFileSync(path.join(testSessionStateDir, `${sessionId}/events.jsonl`), "utf8") .trim() .split("\n") .map(JSON.parse); - expect(captured.filter(event => event.type === "subagent.model_unavailable")).toEqual([ - expect.objectContaining({ data: { agentName: "unknown-agent", declaredModel: "unavailable-model", model: `${sessionProvider}/${sessionModel}` } }), - ]); + expect(captured.filter(event => event.type === "subagent.model_unavailable")).toEqual([expect.objectContaining({ data: { agentName: "unknown-agent", declaredModel: agentModel, model: "copilot-responses/gpt-5.4" } })]); }); it("captures ephemeral selection and final model results including BYOK and subagent context", async () => { diff --git a/actions/setup/js/copilot_sdk_session.cjs b/actions/setup/js/copilot_sdk_session.cjs index 133ae62c406..af3079a9f63 100644 --- a/actions/setup/js/copilot_sdk_session.cjs +++ b/actions/setup/js/copilot_sdk_session.cjs @@ -111,6 +111,20 @@ function resolveCopilotSDKWorkingDirectory() { return path.resolve(process.env.GITHUB_WORKSPACE || process.cwd(), process.env.GH_AW_ENGINE_CWD || "."); } +/** + * Parse GH_AW_COPILOT_SDK_SUBAGENT_MODELS: custom agent name → model resolved by the harness. + * @param {string | undefined} value + * @returns {Record} + */ +function parseSubagentModelsJson(value) { + if (!value) return {}; + const parsed = JSON.parse(value); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || !Object.values(parsed).every(model => typeof model === "string")) { + throw new Error("GH_AW_COPILOT_SDK_SUBAGENT_MODELS must map custom agent names to models"); + } + return parsed; +} + /** * Run a Copilot agentic session using the @github/copilot-sdk. * @@ -369,16 +383,24 @@ async function runWithCopilotSDK({ const providerConfig = providers && providerModels ? { providers, models: providerModels } : null; const sessionModel = providerConfig ? qualifyModelForMultiProvider(requestedModel, providerConfig) : requestedModel; if (providerConfig && !sessionModel) throw new Error(`Cannot qualify session model "${requestedModel || "(none)"}" against the configured provider models`); + // The harness resolves sub-agent models (aliases included) before the driver starts and + // stops the run when a workflow-declared sub-agent fails. A repository agent whose model + // the harness could not resolve is absent from the map and inherits the session model. + const resolvedSubagentModels = parseSubagentModelsJson(process.env.GH_AW_COPILOT_SDK_SUBAGENT_MODELS); const unavailableModels = []; const customAgents = loadCopilotSDKCustomAgents(workingDirectory, agentsBaseDir, log).map(agent => { - const requested = agent.model; - const alias = !requested || ["small", "medium", "large"].includes(requested.toLowerCase()); - const qualified = alias ? sessionModel : providerConfig ? qualifyModelForMultiProvider(requested, providerConfig) : requested; - if (!alias && !qualified) { - log(`warning: custom agent "${agent.name}" model "${requested}" is not in the configured provider models; using session model "${sessionModel || "(none)"}"`); - unavailableModels.push({ agentName: agent.name, declaredModel: requested, model: sessionModel || undefined }); + if (!agent.model || agent.model.trim().toLowerCase() === "inherited") return { ...agent, model: sessionModel || undefined }; + if (Object.prototype.hasOwnProperty.call(resolvedSubagentModels, agent.name)) { + const resolved = resolvedSubagentModels[agent.name]; + const qualified = providerConfig ? qualifyModelForMultiProvider(resolved, providerConfig) : resolved; + if (!qualified) throw new Error(`Cannot qualify custom agent "${agent.name}" model "${resolved}" against the configured provider models`); + return { ...agent, model: qualified }; } - return { ...agent, model: qualified || sessionModel || undefined }; + const qualified = providerConfig ? qualifyModelForMultiProvider(agent.model, providerConfig) : agent.model; + if (qualified) return { ...agent, model: qualified }; + log(`warning: custom agent "${agent.name}" model "${agent.model}" is not in the configured provider models; using session model "${sessionModel || "(none)"}"`); + unavailableModels.push({ agentName: agent.name, declaredModel: agent.model, model: sessionModel || undefined }); + return { ...agent, model: sessionModel || undefined }; }); const skillsDirectory = path.resolve(agentsBaseDir, ".github", "skills"); const skillsStat = fs.lstatSync(skillsDirectory, { throwIfNoEntry: false }); diff --git a/actions/setup/js/copilot_sdk_session.test.cjs b/actions/setup/js/copilot_sdk_session.test.cjs index 30d231d6e7f..40576768802 100644 --- a/actions/setup/js/copilot_sdk_session.test.cjs +++ b/actions/setup/js/copilot_sdk_session.test.cjs @@ -141,6 +141,8 @@ describe("copilot_sdk_session runtime configuration", () => { const app = path.join(base, "packages/app"); fs.mkdirSync(path.join(app, ".github/agents"), { recursive: true }); fs.writeFileSync(path.join(app, ".github/agents/local.agent.md"), "---\nmodel: small\n---\nApp agent."); + // The harness resolved `small` before the driver started. + vi.stubEnv("GH_AW_COPILOT_SDK_SUBAGENT_MODELS", JSON.stringify({ local: "claude-sonnet-4.6" })); vi.stubEnv("GITHUB_WORKSPACE", base); vi.spyOn(process, "cwd").mockReturnValue(app); const createSession = vi.fn().mockResolvedValue({ diff --git a/actions/setup/js/fixtures/awf-config-models-copilot-sub-agents-39304573724.json b/actions/setup/js/fixtures/awf-config-models-copilot-sub-agents-39304573724.json new file mode 100644 index 00000000000..953d043176d --- /dev/null +++ b/actions/setup/js/fixtures/awf-config-models-copilot-sub-agents-39304573724.json @@ -0,0 +1,67 @@ +{ + "apiProxy": { + "models": { + "agent": ["sonnet-6x", "gpt-6", "gpt-5.4", "gpt-5.5", "gpt-5.6", "gpt-5.3", "gemini-pro", "any"], + "antigravity": ["copilot/antigravity*", "google/antigravity*", "gemini/antigravity*"], + "any": ["copilot/*", "anthropic/*", "openai/*", "google/*", "gemini/*"], + "auto": ["copilot/auto", "large"], + "claude": ["agent"], + "codex": ["agent"], + "coding": ["copilot/gpt-5*codex*", "openai/gpt-5*codex*", "gpt-5-codex", "kimi"], + "computer-use": ["copilot/*computer-use*", "google/*computer-use*", "gemini/*computer-use*", "openai/*computer-use*"], + "copilot": ["agent"], + "deep-research": ["copilot/deep-research*", "copilot/o3-deep-research*", "copilot/o4-mini-deep-research*", "google/deep-research*", "gemini/deep-research*", "openai/o3-deep-research*", "openai/o4-mini-deep-research*"], + "detection": ["small"], + "evals": ["small"], + "fable": ["copilot/*fable*", "anthropic/*fable*"], + "gemini": ["agent"], + "gemini-3-flash": ["copilot/gemini-3*flash*", "google/gemini-3*flash*", "gemini/gemini-3*flash*"], + "gemini-3-pro": ["copilot/gemini-3*pro*", "google/gemini-3*pro*", "google/nano-banana*", "gemini/gemini-3*pro*"], + "gemini-3.1-flash": ["copilot/gemini-3.1*flash*", "google/gemini-3.1*flash*", "gemini/gemini-3.1*flash*"], + "gemini-3.1-pro": ["copilot/gemini-3.1*pro*", "google/gemini-3.1*pro*", "gemini/gemini-3.1*pro*"], + "gemini-3.5-flash": ["copilot/gemini-3.5*flash*", "google/gemini-3.5*flash*", "gemini/gemini-3.5*flash*"], + "gemini-3.6-flash": ["copilot/gemini-3.6*flash*", "google/gemini-3.6*flash*", "gemini/gemini-3.6*flash*"], + "gemini-3.7-flash": ["copilot/gemini-3.7*flash*", "google/gemini-3.7*flash*", "gemini/gemini-3.7*flash*"], + "gemini-3.8-flash": ["copilot/gemini-3.8*flash*", "google/gemini-3.8*flash*", "gemini/gemini-3.8*flash*"], + "gemini-flash": ["copilot/gemini-*flash*", "google/gemini-*flash*", "gemini/gemini-*flash*"], + "gemini-flash-lite": ["copilot/gemini-*flash*lite*", "google/gemini-*flash*lite*", "gemini/gemini-*flash*lite*"], + "gemini-omni": ["copilot/gemini-omni*", "google/gemini-omni*", "gemini/gemini-omni*"], + "gemini-pro": ["copilot/gemini-*pro*", "google/gemini-*pro*", "gemini/gemini-*pro*"], + "gemma": ["copilot/gemma*", "google/gemma*", "gemini/gemma*"], + "gpt-5": ["copilot/gpt-5*", "openai/gpt-5*"], + "gpt-5-codex": ["copilot/gpt-5*codex*", "openai/gpt-5*codex*"], + "gpt-5-mini": ["copilot/gpt-5*mini*", "openai/gpt-5*mini*"], + "gpt-5-nano": ["copilot/gpt-5*nano*", "openai/gpt-5*nano*"], + "gpt-5-pro": ["copilot/gpt-5*pro*", "openai/gpt-5*pro*"], + "gpt-5.1": ["copilot/gpt-5.1*", "openai/gpt-5.1*"], + "gpt-5.2": ["copilot/gpt-5.2*", "openai/gpt-5.2*"], + "gpt-5.3": ["copilot/gpt-5.3*", "openai/gpt-5.3*"], + "gpt-5.4": ["copilot/gpt-5.4*", "openai/gpt-5.4*"], + "gpt-5.5": ["copilot/gpt-5.5*", "openai/gpt-5.5*"], + "gpt-5.6": ["copilot/gpt-5.6*", "openai/gpt-5.6*"], + "gpt-6": ["copilot/gpt-6*", "openai/gpt-6*"], + "grok": ["copilot/*grok*", "openai/*grok*"], + "haiku": ["copilot/*haiku*", "anthropic/*haiku*"], + "image-generation": ["copilot/gpt-image*", "openai/gpt-image*", "openai/chatgpt-image*", "copilot/gemini-*image*", "google/gemini-*image*", "gemini/gemini-*image*", "google/imagen*"], + "kimi": ["copilot/kimi*", "openai/kimi*"], + "kiwi": ["copilot/kiwi*", "openai/kiwi*"], + "large": ["sonnet", "gpt-6", "gpt-5-pro", "gpt-5", "gemini-pro"], + "lyria": ["google/lyria*", "gemini/lyria*", "copilot/lyria*"], + "mai-code": ["copilot/MAI-Code*", "copilot/mai-code*", "openai/MAI-Code*"], + "mini": ["haiku", "gpt-5-mini", "gpt-5-nano", "gemini-flash-lite"], + "nano-banana": ["copilot/nano-banana*", "google/nano-banana*", "gemini/nano-banana*"], + "opus": ["copilot/*opus*", "anthropic/*opus*"], + "opusplan": ["opus?effort=high"], + "raptor-mini": ["copilot/raptor*", "openai/raptor*"], + "reasoning": ["copilot/o1*", "copilot/o3*", "copilot/o4*", "openai/o1*", "openai/o3*", "openai/o4*"], + "robotics": ["copilot/*robotics*", "google/*robotics*", "gemini/*robotics*"], + "small": ["mini"], + "small-agent": ["haiku", "gpt-5-mini", "gemini-flash"], + "sonnet": ["copilot/*sonnet*", "anthropic/*sonnet*"], + "sonnet-6x": ["copilot/*sonnet-4.5*", "copilot/*sonnet-4.6*", "copilot/*sonnet-5*", "copilot/*sonnet-4-5-*", "anthropic/*sonnet-4-5-*", "copilot/*sonnet-4-6*", "anthropic/*sonnet-4-6*", "anthropic/*sonnet-5*"], + "summarization": ["haiku", "gpt-5-mini", "gemini-flash-lite", "mini"], + "veo": ["google/veo*", "gemini/veo*"], + "vision": ["copilot/gemini-*image*", "google/gemini-*image*", "gemini/gemini-*image*", "copilot/gemini-*flash*", "google/gemini-*flash*", "gemini/gemini-*flash*"] + } + } +} diff --git a/actions/setup/js/fixtures/awf-reflect-copilot-sub-agents-39304573724.json b/actions/setup/js/fixtures/awf-reflect-copilot-sub-agents-39304573724.json new file mode 100644 index 00000000000..d292923a894 --- /dev/null +++ b/actions/setup/js/fixtures/awf-reflect-copilot-sub-agents-39304573724.json @@ -0,0 +1,1575 @@ +{ + "endpoints": [ + { + "provider": "openai", + "port": 10000, + "base_url": "http://api-proxy:10000", + "configured": false, + "models": null, + "model_metadata": null, + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "google": { + "messages": [], + "chat/completions": [], + "responses": [] + } + } + }, + "routing_models": null, + "models_url": "http://api-proxy:10000/v1/models" + }, + { + "provider": "anthropic", + "port": 10001, + "base_url": "http://api-proxy:10001", + "configured": false, + "models": null, + "model_metadata": null, + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "google": { + "messages": [], + "chat/completions": [], + "responses": [] + } + } + }, + "routing_models": null, + "models_url": "http://api-proxy:10001/v1/models" + }, + { + "provider": "copilot", + "port": 10002, + "base_url": "http://api-proxy:10002", + "configured": true, + "models": [ + "claude-haiku-4.5", + "claude-haiku-5.5", + "claude-opus-5", + "claude-opus-5.5", + "claude-sonnet-5", + "claude-sonnet-5.5", + "gpt-3.5-turbo", + "gpt-3.5-turbo-0613", + "gpt-4", + "gpt-4-0613", + "gpt-4-o-preview", + "gpt-4.1", + "gpt-4.1-2025-04-14", + "gpt-41-copilot", + "gpt-4o", + "gpt-4o-2024-05-13", + "gpt-4o-2024-08-06", + "gpt-4o-2024-11-20", + "gpt-4o-mini", + "gpt-4o-mini-2024-07-18", + "gpt-5-mini", + "gpt-5.3-codex", + "gpt-5.6-luna", + "gpt-5.6-luna-utility", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "grok-4.5", + "grok-4.6", + "grok-4.7", + "mai-code-1.1-flash", + "text-embedding-3-small", + "text-embedding-3-small-inference", + "text-embedding-ada-002", + "trajectory-compaction" + ], + "model_metadata": [ + { + "id": "claude-haiku-4.5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 1, + "output": 5, + "cachedInput": 0.1, + "cacheWrite": 1.25 + } + } + }, + { + "id": "claude-haiku-5.5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 0.1, + "output": 0.5, + "cachedInput": 0.01, + "cacheWrite": 0.12, + "threshold": 100000 + }, + "long_context": { + "input": 0.5, + "output": 2.5, + "cachedInput": 0.05, + "cacheWrite": 0.62, + "threshold": 936000 + } + } + }, + { + "id": "claude-opus-5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 5, + "output": 25, + "cachedInput": 0.5, + "cacheWrite": 6.25, + "threshold": 200000 + }, + "long_context": { + "input": 5, + "output": 25, + "cachedInput": 0.5, + "cacheWrite": 6.25, + "threshold": 936000 + } + } + }, + { + "id": "claude-opus-5.5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 4, + "output": 20, + "cachedInput": 0.2, + "cacheWrite": 5, + "threshold": 200000 + }, + "long_context": { + "input": 4, + "output": 20, + "cachedInput": 0.2, + "cacheWrite": 5, + "threshold": 1000000 + } + } + }, + { + "id": "claude-sonnet-5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 10, + "cachedInput": 0.2, + "cacheWrite": 2.5, + "threshold": 200000 + }, + "long_context": { + "input": 2, + "output": 10, + "cachedInput": 0.2, + "cacheWrite": 2.5, + "threshold": 936000 + } + } + }, + { + "id": "claude-sonnet-5.5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 10, + "cachedInput": 0.1, + "cacheWrite": 2.5, + "threshold": 200000 + }, + "long_context": { + "input": 2, + "output": 10, + "cachedInput": 0.1, + "cacheWrite": 2.5, + "threshold": 936000 + } + } + }, + { + "id": "gpt-3.5-turbo", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-3.5-turbo-0613", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2.5, + "output": 10, + "cachedInput": 1.25, + "cacheWrite": 0 + } + } + }, + { + "id": "gpt-4-0613", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2.5, + "output": 10, + "cachedInput": 1.25, + "cacheWrite": 0 + } + } + }, + { + "id": "gpt-4-o-preview", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4.1", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4.1-2025-04-14", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-41-copilot", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 8, + "cachedInput": 0.5, + "cacheWrite": 0 + } + } + }, + { + "id": "gpt-4o", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4o-2024-05-13", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4o-2024-08-06", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4o-2024-11-20", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4o-mini", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-4o-mini-2024-07-18", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-5-mini", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 0.25, + "output": 2, + "cachedInput": 0.02, + "cacheWrite": 0 + } + } + }, + { + "id": "gpt-5.3-codex", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 1.75, + "output": 14, + "cachedInput": 0.17, + "cacheWrite": 0, + "threshold": 272000 + } + } + }, + { + "id": "gpt-5.6-luna", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 0.2, + "output": 1.2, + "cachedInput": 0.02, + "cacheWrite": 0.25, + "threshold": 200000 + }, + "long_context": { + "input": 0.4, + "output": 1.8, + "cachedInput": 0.04, + "cacheWrite": 0.5, + "threshold": 922000 + } + } + }, + { + "id": "gpt-5.6-luna-utility", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "gpt-5.6-sol", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 4, + "output": 20, + "cachedInput": 0.4, + "cacheWrite": 5, + "threshold": 272000 + }, + "long_context": { + "input": 8, + "output": 30, + "cachedInput": 0.8, + "cacheWrite": 10, + "threshold": 922000 + } + } + }, + { + "id": "gpt-5.6-terra", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 12, + "cachedInput": 0.2, + "cacheWrite": 2.5, + "threshold": 272000 + }, + "long_context": { + "input": 4, + "output": 18, + "cachedInput": 0.4, + "cacheWrite": 5, + "threshold": 922000 + } + } + }, + { + "id": "gpt-6-astra", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 10, + "output": 50, + "cachedInput": 1, + "cacheWrite": 12.5, + "threshold": 272000 + }, + "long_context": { + "input": 20, + "output": 75, + "cachedInput": 2, + "cacheWrite": 25, + "threshold": 1050000 + } + } + }, + { + "id": "gpt-6-luna", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 0.1, + "output": 0.5, + "cachedInput": 0.01, + "cacheWrite": 0.12, + "threshold": 272000 + }, + "long_context": { + "input": 0.2, + "output": 0.75, + "cachedInput": 0.02, + "cacheWrite": 0.25, + "threshold": 872000 + } + } + }, + { + "id": "gpt-6-sol", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 10, + "cachedInput": 0.2, + "cacheWrite": 2.5, + "threshold": 272000 + }, + "long_context": { + "input": 4, + "output": 15, + "cachedInput": 0.4, + "cacheWrite": 5, + "threshold": 872000 + } + } + }, + { + "id": "gpt-6.1-sol", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 2, + "output": 10, + "cachedInput": 0.1, + "cacheWrite": 2.5, + "threshold": 272000 + }, + "long_context": { + "input": 4, + "output": 15, + "cachedInput": 0.2, + "cacheWrite": 5, + "threshold": 922000 + } + } + }, + { + "id": "grok-4.5", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "wire_api": "responses", + "pricing": { + "default": { + "input": 2, + "output": 6, + "cachedInput": 0.5, + "cacheWrite": 0, + "threshold": 200000 + }, + "long_context": { + "input": 4, + "output": 12, + "cachedInput": 1, + "cacheWrite": 0, + "threshold": 500000 + } + } + }, + { + "id": "grok-4.6", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "wire_api": "responses", + "pricing": { + "default": { + "input": 2, + "output": 6, + "cachedInput": 0.5, + "cacheWrite": 0, + "threshold": 200000 + }, + "long_context": { + "input": 4, + "output": 12, + "cachedInput": 1, + "cacheWrite": 0, + "threshold": 500000 + } + } + }, + { + "id": "grok-4.7", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "wire_api": "responses", + "pricing": { + "default": { + "input": 2, + "output": 6, + "cachedInput": 0.5, + "cacheWrite": 0, + "threshold": 200000 + }, + "long_context": { + "input": 4, + "output": 12, + "cachedInput": 1, + "cacheWrite": 0, + "threshold": 500000 + } + } + }, + { + "id": "mai-code-1.1-flash", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "pricing": { + "default": { + "input": 0.2, + "output": 1.2, + "cachedInput": 0.02, + "cacheWrite": 0.25, + "threshold": 128000 + } + } + }, + { + "id": "text-embedding-3-small", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "text-embedding-3-small-inference", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "text-embedding-ada-002", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01" + }, + { + "id": "trajectory-compaction", + "source": "provider", + "observed_at": "2026-10-19T18:44:07.552Z", + "api_version": "2026-07-01", + "wire_api": "completions" + } + ], + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": ["claude-sonnet-5.5", "claude-sonnet-5", "claude-opus-5.5", "claude-opus-5", "claude-haiku-5.5", "claude-haiku-4.5"], + "chat/completions": ["claude-sonnet-5.5", "claude-sonnet-5", "claude-opus-5.5", "claude-opus-5", "claude-haiku-5.5", "claude-haiku-4.5"], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": ["gpt-5-mini"], + "responses": ["gpt-6.1-sol", "gpt-6-astra", "gpt-6-luna", "gpt-6-sol", "gpt-5.6-luna", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.3-codex", "gpt-5-mini"] + }, + "google": { + "messages": [], + "chat/completions": [], + "responses": [] + } + } + }, + "routing_models": [ + { + "model_id": "claude-haiku-4.5", + "source": "provider", + "supported_endpoints": ["/chat/completions", "/v1/messages"], + "supported_reasoning_efforts": [], + "context_window_tokens": 200000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-haiku-5.5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-opus-5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-opus-5.5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-sonnet-5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "claude-sonnet-5.5", + "source": "provider", + "supported_endpoints": ["/v1/messages", "/chat/completions"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-3.5-turbo", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 16384, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-3.5-turbo-0613", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 16384, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 32768, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4-0613", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 32768, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4-o-preview", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4.1", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4.1-2025-04-14", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-41-copilot", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-2024-05-13", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-2024-08-06", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-2024-11-20", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-mini", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-4o-mini-2024-07-18", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": 128000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-5-mini", + "source": "provider", + "supported_endpoints": ["/chat/completions", "/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high"], + "context_window_tokens": 264000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.3-codex", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], + "context_window_tokens": 400000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.6-luna", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.6-luna-utility", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "gpt-5.6-sol", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-5.6-terra", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6-astra", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6-luna", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6-sol", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1000000, + "candidate_metadata_complete": true + }, + { + "model_id": "gpt-6.1-sol", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "context_window_tokens": 1050000, + "candidate_metadata_complete": true + }, + { + "model_id": "grok-4.5", + "source": "provider", + "supported_endpoints": ["/responses"], + "supported_reasoning_efforts": ["low", "medium", "high"], + "context_window_tokens": 500000, + "candidate_metadata_complete": true + }, + { + "model_id": "grok-4.6", + "source": "provider", + "supported_endpoints": ["/responses"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], + "context_window_tokens": 500000, + "candidate_metadata_complete": true + }, + { + "model_id": "grok-4.7", + "source": "provider", + "supported_endpoints": ["/responses"], + "supported_reasoning_efforts": ["low", "medium", "high", "xhigh"], + "context_window_tokens": 500000, + "candidate_metadata_complete": true + }, + { + "model_id": "mai-code-1.1-flash", + "source": "provider", + "supported_endpoints": ["/responses", "ws:/responses"], + "supported_reasoning_efforts": ["low", "medium", "high"], + "context_window_tokens": 256000, + "candidate_metadata_complete": true + }, + { + "model_id": "text-embedding-3-small", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "text-embedding-3-small-inference", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "text-embedding-ada-002", + "source": "provider", + "supported_endpoints": [], + "supported_reasoning_efforts": [], + "context_window_tokens": null, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + }, + { + "model_id": "trajectory-compaction", + "source": "provider", + "supported_endpoints": ["/chat/completions"], + "supported_reasoning_efforts": [], + "context_window_tokens": 262144, + "candidate_metadata_complete": false, + "candidate_metadata_reason": "Model is not enabled in the Copilot model picker" + } + ], + "models_url": "http://api-proxy:10002/models", + "credential_kind": "github_token", + "selected_scheme": "Bearer", + "inference_credential_source": "github_token", + "inference_selected_scheme": "Bearer", + "integration_id_source": "github_copilot_integration_id" + }, + { + "provider": "gemini", + "port": 10003, + "base_url": "http://api-proxy:10003", + "configured": false, + "models": null, + "model_metadata": null, + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "google": { + "messages": [], + "chat/completions": [], + "responses": [] + } + } + }, + "routing_models": null, + "models_url": "http://api-proxy:10003/v1beta/models" + }, + { + "provider": "vertex", + "port": 10004, + "base_url": "http://api-proxy:10004", + "configured": false, + "models": null, + "model_metadata": null, + "automatic_model_selection": { + "model": "auto", + "constraint_field": "model_provider", + "model_providers": ["anthropic", "openai", "google"], + "preserves_backend": true, + "requires_advertised_endpoints": true, + "candidates": { + "anthropic": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "openai": { + "messages": [], + "chat/completions": [], + "responses": [] + }, + "google": { + "messages": [], + "chat/completions": [], + "responses": [] + } + } + }, + "routing_models": null, + "models_url": null + } + ], + "models_fetch_complete": true, + "model_aliases": { + "agent": ["sonnet-6x", "gpt-6", "gpt-5.4", "gpt-5.5", "gpt-5.6", "gpt-5.3", "gemini-pro", "any"], + "any": ["copilot/*", "anthropic/*", "openai/*", "google/*", "gemini/*"], + "auto": ["copilot/auto", "large"], + "claude": ["agent"], + "codex": ["agent"], + "coding": ["copilot/gpt-5*codex*", "openai/gpt-5*codex*", "gpt-5-codex", "kimi"], + "copilot": ["agent"], + "detection": ["small"], + "evals": ["small"], + "gemini": ["agent"], + "gpt-5": ["copilot/gpt-5*", "openai/gpt-5*"], + "gpt-5-codex": ["copilot/gpt-5*codex*", "openai/gpt-5*codex*"], + "gpt-5-mini": ["copilot/gpt-5*mini*", "openai/gpt-5*mini*"], + "gpt-5.3": ["copilot/gpt-5.3*", "openai/gpt-5.3*"], + "gpt-5.6": ["copilot/gpt-5.6*", "openai/gpt-5.6*"], + "gpt-6": ["copilot/gpt-6*", "openai/gpt-6*"], + "grok": ["copilot/*grok*", "openai/*grok*"], + "haiku": ["copilot/*haiku*", "anthropic/*haiku*"], + "large": ["sonnet", "gpt-6", "gpt-5-pro", "gpt-5", "gemini-pro"], + "mai-code": ["copilot/MAI-Code*", "copilot/mai-code*", "openai/MAI-Code*"], + "mini": ["haiku", "gpt-5-mini", "gpt-5-nano", "gemini-flash-lite"], + "opus": ["copilot/*opus*", "anthropic/*opus*"], + "opusplan": ["opus?effort=high"], + "small": ["mini"], + "small-agent": ["haiku", "gpt-5-mini", "gemini-flash"], + "sonnet": ["copilot/*sonnet*", "anthropic/*sonnet*"], + "sonnet-6x": ["copilot/*sonnet-4.5*", "copilot/*sonnet-4.6*", "copilot/*sonnet-5*", "copilot/*sonnet-4-5-*", "anthropic/*sonnet-4-5-*", "copilot/*sonnet-4-6*", "anthropic/*sonnet-4-6*", "anthropic/*sonnet-5*"], + "summarization": ["haiku", "gpt-5-mini", "gemini-flash-lite", "mini"] + }, + "model_fallback": { + "enabled": true, + "strategy": "middle_power", + "excludeEngines": [] + }, + "model_fallback_effective": { + "openai": { + "enabled": true, + "strategy": "middle_power", + "excludeEngines": [], + "suppressed": false + }, + "anthropic": { + "enabled": true, + "strategy": "middle_power", + "excludeEngines": [], + "suppressed": false + }, + "copilot": { + "enabled": false, + "strategy": "middle_power", + "excludeEngines": [], + "suppressed": true, + "suppression_reason": "copilot_standard_authoritative" + }, + "gemini": { + "enabled": true, + "strategy": "middle_power", + "excludeEngines": [], + "suppressed": false + }, + "vertex": { + "enabled": true, + "strategy": "middle_power", + "excludeEngines": [], + "suppressed": false + } + }, + "ai_credits": { + "total": 12.401028, + "max_ai_credits": 1000, + "effective_max_ai_credits": 1000, + "percent_used": 1.24, + "thresholds_crossed": [], + "thresholds_pending": [], + "by_model": { + "gpt-5.3-codex": { + "input_credits": 6.3672, + "cached_input_credits": 1.63968, + "cache_write_credits": 0, + "output_credits": 1.4882, + "total": 9.49508, + "pricing_source": "operator", + "pricing_tier": "default", + "accounting_policy": "concrete_model", + "fallback_pricing_used": false, + "dynamic_selector": null + }, + "claude-haiku-4.5": { + "input_credits": 0.0378, + "cached_input_credits": 0, + "cache_write_credits": 2.493, + "output_credits": 0.0055, + "total": 2.5363, + "pricing_source": "provider", + "pricing_tier": "default", + "accounting_policy": "concrete_model", + "fallback_pricing_used": false, + "dynamic_selector": null + }, + "gpt-5-mini": { + "input_credits": 0.3354, + "cached_input_credits": 0.002048, + "cache_write_credits": 0, + "output_credits": 0.0322, + "total": 0.369648, + "pricing_source": "provider", + "pricing_tier": "default", + "accounting_policy": "concrete_model", + "fallback_pricing_used": false, + "dynamic_selector": null + } + } + }, + "time_steering": { + "enabled": true, + "timeout_minutes": 10, + "started_at_ms": 1792435452871, + "percent_elapsed": 5.51, + "thresholds_crossed": [], + "thresholds_pending": [] + }, + "runs": { + "enabled": true, + "max_runs": 500, + "invocation_count": 9, + "remaining_runs": 491 + }, + "cache_misses": { + "enabled": true, + "max_cache_misses": 5, + "consecutive_cache_misses": 0, + "remaining_cache_misses": 5 + }, + "permission_denied": { + "enabled": false, + "max_permission_denied": null, + "denied_count": 0 + }, + "model_api_mapping": { + "available": true, + "last_updated": "2026-10-08T07:12:00Z", + "providers": ["openai", "anthropic"], + "models": { + "openai": { + "endpoints": { + "responses": { + "path": "/v1/responses", + "description": "Newer unified API for agentic, tool-using, and multimodal workflows. Required for GPT-5+ models; some older models (o1, o3, gpt-4o, gpt-4.1) support it alongside chat/completions." + }, + "chat_completions": { + "path": "/v1/chat/completions", + "description": "Legacy conversational API. Being deprecated for newer models by end of 2026." + } + }, + "models": [ + { + "family": "gpt-6-astra", + "patterns": ["gpt-6-astra*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Newest flagship model (Trusted Access Program rollout). Supports Chat Completions and Responses APIs." + }, + { + "family": "gpt-6-sol", + "patterns": ["gpt-6-sol*"], + "endpoints": ["chat_completions", "responses"], + "notes": "GPT-6 tier balancing intelligence and cost. Supports both Chat Completions and Responses APIs per official model page." + }, + { + "family": "gpt-6.1-sol", + "patterns": ["gpt-6.1-sol*"], + "endpoints": ["chat_completions", "responses"], + "notes": "GPT-6.1 tier balancing intelligence and cost (successor to gpt-6-sol). Supports both Chat Completions and Responses APIs, consistent with other GPT-6 tiers." + }, + { + "family": "gpt-6-luna", + "patterns": ["gpt-6-luna*"], + "endpoints": ["chat_completions", "responses"], + "notes": "GPT-6 tier for cost-sensitive, high-volume use. Supports both Chat Completions and Responses APIs per official model page." + }, + { + "family": "gpt-daybreak", + "patterns": ["gpt-daybreak-blue-latest", "gpt-daybreak-red-latest"], + "endpoints": ["responses"], + "notes": "Responses API only (ResponsesModel-only Literal). Experimental/preview aliases not present in ChatModel." + }, + { + "family": "gpt-rosalind-research", + "patterns": ["gpt-rosalind-research*"], + "endpoints": ["responses"], + "notes": "Life sciences reasoning model, Trusted Access Program (approved organizations only). Responses API only, consistent with other restricted-access specialized models (gpt-daybreak-*, gpt-5.6-cyber). Billing begins October 5, 2026." + }, + { + "family": "gpt-5.6-cyber", + "patterns": ["gpt-5.6-cyber*"], + "endpoints": ["responses"], + "notes": "Responses API only (ResponsesModel-only Literal, not in ChatModel). More specific than gpt-5.6* pattern below." + }, + { + "family": "gpt-5.6", + "patterns": ["gpt-5.6*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints. Includes gpt-5.6-sol, gpt-5.6-terra, gpt-5.6-luna per OpenAI SDK ChatModel. gpt-5.6-cyber is responses-only (see separate entry above)." + }, + { + "family": "gpt-5.5-pro", + "patterns": ["gpt-5.5-pro", "gpt-5.5-pro-*"], + "endpoints": ["responses"], + "notes": "Responses API only (ResponsesOnlyModel per OpenAI OpenAPI spec). More specific than gpt-5.5* pattern below. Includes dated variants like gpt-5.5-pro-2026-04-23." + }, + { + "family": "gpt-5.5", + "patterns": ["gpt-5.5*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints. Listed on the official GPT-5.5 model page. gpt-5.5-pro is responses-only (see separate entry above)." + }, + { + "family": "gpt-5.4", + "patterns": ["gpt-5.4*"], + "routingPatterns": ["gpt-5.4", "gpt-5.4-2026-03-05"], + "endpoints": ["chat_completions", "responses"], + "routing": { + "reasoningEfforts": ["none", "low", "medium", "high", "xhigh"], + "contextWindowTokens": 1050000 + }, + "notes": "Supports both endpoints on OpenAI. GPT-5.4 routing metadata is exact-model only; mini/nano variants need separately verified data. The model page lists none, low, medium, high, and xhigh reasoning effort and a 1,050,000-token context window." + }, + { + "family": "gpt-5.3-codex", + "patterns": ["gpt-5.3-codex", "gpt-5.3-codex-*"], + "endpoints": ["responses"], + "notes": "Responses API only per official model page (developers.openai.com/api/docs/models/gpt-5.3-codex). More specific than gpt-5.3* pattern below." + }, + { + "family": "gpt-5.3", + "patterns": ["gpt-5.3*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints. Only gpt-5.3-chat-latest confirmed for both endpoints; gpt-5.3-codex is responses-only (see separate entry above). No bare gpt-5.3 base model exists." + }, + { + "family": "gpt-5.2-codex", + "patterns": ["gpt-5.2-codex", "gpt-5.2-codex-*"], + "endpoints": ["responses"], + "notes": "Responses API only per official model page (developers.openai.com/api/docs/models/gpt-5.2-codex). More specific than gpt-5.2* pattern below." + }, + { + "family": "gpt-5.2-pro", + "patterns": ["gpt-5.2-pro", "gpt-5.2-pro-*"], + "endpoints": ["responses"], + "notes": "Responses API only per official model page (developers.openai.com/api/docs/models/gpt-5.2-pro). More specific than gpt-5.2* pattern below." + }, + { + "family": "gpt-5.2", + "patterns": ["gpt-5.2*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints. gpt-5.2-pro and gpt-5.2-codex are responses-only (see separate entries above)." + }, + { + "family": "gpt-5.1-codex-max", + "patterns": ["gpt-5.1-codex-max"], + "endpoints": ["responses"], + "notes": "Responses API only (ResponsesOnlyModel)." + }, + { + "family": "gpt-5.1-codex", + "patterns": ["gpt-5.1-codex", "gpt-5.1-codex-*"], + "endpoints": ["responses"], + "notes": "Responses API only per official model pages (developers.openai.com/api/docs/models/gpt-5.1-codex, /gpt-5.1-codex-mini). More specific than gpt-5.1* pattern below. gpt-5.1-codex-max is covered by a dedicated entry above." + }, + { + "family": "gpt-5.1", + "patterns": ["gpt-5.1*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints for gpt-5.1 and gpt-5.1-mini. gpt-5.1-codex, gpt-5.1-codex-mini, and gpt-5.1-codex-max are responses-only (see separate entries above)." + }, + { + "family": "gpt-5-codex/pro", + "patterns": ["gpt-5-codex", "gpt-5-pro", "gpt-5-pro-*"], + "endpoints": ["responses"], + "notes": "Responses API only for gpt-5-codex and gpt-5-pro (ResponsesOnlyModel). Includes dated variants like gpt-5-pro-2025-10-06." + }, + { + "family": "gpt-5", + "patterns": ["gpt-5", "gpt-5-*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints for gpt-5, gpt-5-mini, gpt-5-nano." + }, + { + "family": "o4-mini-deep-research", + "patterns": ["o4-mini-deep-research", "o4-mini-deep-research-*"], + "endpoints": ["responses"], + "notes": "Responses API only (ResponsesOnlyModel). Includes dated variants like o4-mini-deep-research-2025-06-26." + }, + { + "family": "o4", + "patterns": ["o4*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Reasoning model. o4-mini supports both endpoints (ModelIdsShared)." + }, + { + "family": "o3-pro/deep-research", + "patterns": ["o3-pro", "o3-pro-*", "o3-deep-research", "o3-deep-research-*"], + "endpoints": ["responses"], + "notes": "Responses API only. o3-pro and o3-deep-research are in ResponsesModel but not ChatModel." + }, + { + "family": "o3", + "patterns": ["o3*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Reasoning model. o3, o3-mini support both endpoints. o3-pro/deep-research use responses only (covered by separate entry above)." + }, + { + "family": "o1-pro", + "patterns": ["o1-pro", "o1-pro-*"], + "endpoints": ["responses"], + "notes": "Responses API only. o1-pro is in ResponsesModel but not ChatModel." + }, + { + "family": "o1-mini/preview", + "patterns": ["o1-mini*", "o1-preview*"], + "endpoints": ["chat_completions"], + "notes": "Chat Completions API only per official model pages (developers.openai.com/api/docs/models/o1-mini, /o1-preview). Not available via Responses API. More specific than o1* pattern below." + }, + { + "family": "o1", + "patterns": ["o1*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Reasoning model. o1 (base) supports both endpoints. o1-pro is responses-only and o1-mini/o1-preview are chat_completions-only (see separate entries above)." + }, + { + "family": "computer-use-preview", + "patterns": ["computer-use-preview*"], + "endpoints": ["responses"], + "notes": "Responses API only. Specialized model for computer use tasks." + }, + { + "family": "codex-mini", + "patterns": ["codex-mini*"], + "endpoints": ["responses"], + "notes": "Responses API only per official model page (developers.openai.com/api/docs/models/codex-mini-latest). Chat Completions is marked unsupported on the model page." + }, + { + "family": "chatgpt-4o-latest", + "patterns": ["chatgpt-4o-latest"], + "endpoints": ["chat_completions", "responses"], + "notes": "Rolling alias for the latest ChatGPT-optimized GPT-4o model. Supports both endpoints (ModelIdsShared). Not matched by gpt-4o* pattern due to chatgpt- prefix." + }, + { + "family": "gpt-4o", + "patterns": ["gpt-4o*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Multimodal. Supports both endpoints. Includes audio and search preview variants." + }, + { + "family": "gpt-4.1", + "patterns": ["gpt-4.1*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Supports both endpoints. Includes gpt-4.1-mini, gpt-4.1-nano." + }, + { + "family": "gpt-4.5", + "patterns": ["gpt-4.5*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Preview model. Now supports both endpoints per official model page (developers.openai.com/api/docs/models/gpt-4.5-preview); Responses API support was added since the previous mapping update." + }, + { + "family": "gpt-4-turbo", + "patterns": ["gpt-4-turbo*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Legacy. In ModelIdsShared per OpenAI spec v2.3.0; supports both endpoints." + }, + { + "family": "gpt-4", + "patterns": ["gpt-4", "gpt-4-*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Legacy. In ModelIdsShared per OpenAI spec v2.3.0; supports both endpoints." + }, + { + "family": "gpt-3.5-turbo", + "patterns": ["gpt-3.5-turbo*"], + "endpoints": ["chat_completions", "responses"], + "notes": "Legacy. In ModelIdsShared per OpenAI spec v2.3.0; supports both endpoints. Deprecated." + } + ] + }, + "anthropic": { + "endpoints": { + "messages": { + "path": "/v1/messages", + "description": "Unified Messages API for all Claude models. Supports tool use, streaming, extended thinking, and vision." + } + }, + "models": [ + { + "family": "claude-opus-5", + "patterns": ["claude-opus-5*"], + "routingPatterns": ["claude-opus-5-5", "claude-opus-5"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Flagship Opus generation 5 models with verified 1M-token context windows and the Anthropic effort parameter." + }, + { + "family": "claude-fable-5", + "patterns": ["claude-fable-5*"], + "routingPatterns": ["claude-fable-5-1", "claude-fable-5"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Latest flagship family with verified 1M-token context windows and the Anthropic effort parameter." + }, + { + "family": "claude-mythos-5", + "patterns": ["claude-mythos-5*", "claude-mythos-preview*"], + "routingPatterns": ["claude-mythos-5-1", "claude-mythos-5", "claude-mythos-preview*"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Specialized models, invitation-only, with verified 1M-token context windows and the Anthropic effort parameter." + }, + { + "family": "claude-sonnet-5", + "patterns": ["claude-sonnet-5*"], + "routingPatterns": ["claude-sonnet-5-5", "claude-sonnet-5"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Best combination of speed and intelligence with verified 1M-token context windows and the Anthropic effort parameter." + }, + { + "family": "claude-opus-4", + "patterns": ["claude-opus-4*"], + "routingPatterns": ["claude-opus-4-8", "claude-opus-4-7", "claude-opus-4-6"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Opus 4.6 and newer models have verified 1M-token context windows and support the Anthropic effort parameter." + }, + { + "family": "claude-sonnet-4", + "patterns": ["claude-sonnet-4*"], + "routingPatterns": ["claude-sonnet-4-6"], + "endpoints": ["messages"], + "routing": { + "reasoningEfforts": ["low", "medium", "high", "max"], + "contextWindowTokens": 1000000 + }, + "notes": "Sonnet 4.6 has a verified 1M-token context window and supports the Anthropic effort parameter." + }, + { + "family": "claude-haiku-5", + "patterns": ["claude-haiku-5*"], + "endpoints": ["messages"], + "notes": "Fast/cheap Haiku generation 5 (claude-haiku-5-5). 1M token context window per official models overview; effort levels not verified, so no routing metadata." + }, + { + "family": "claude-haiku-4", + "patterns": ["claude-haiku-4*"], + "endpoints": ["messages"], + "notes": "Includes claude-haiku-4-5. Fast/cheap. 200K token context." + }, + { + "family": "claude-3.5-sonnet", + "patterns": ["claude-3-5-sonnet*", "claude-3.5-sonnet*"], + "endpoints": ["messages"], + "notes": "Previous generation. 200K token context." + }, + { + "family": "claude-3.5-haiku", + "patterns": ["claude-3-5-haiku*", "claude-3.5-haiku*"], + "endpoints": ["messages"], + "notes": "Previous generation fast model." + }, + { + "family": "claude-3.7-sonnet", + "patterns": ["claude-3-7-sonnet*", "claude-3.7-sonnet*"], + "endpoints": ["messages"], + "notes": "Previous generation extended thinking model. 200K token context." + }, + { + "family": "claude-3-opus", + "patterns": ["claude-3-opus*"], + "endpoints": ["messages"], + "notes": "Previous generation. 200K token context." + } + ] + } + }, + "error": null + }, + "routing": null +} diff --git a/docs/src/content/docs/reference/engines.md b/docs/src/content/docs/reference/engines.md index d12195c788d..cc0e118480d 100644 --- a/docs/src/content/docs/reference/engines.md +++ b/docs/src/content/docs/reference/engines.md @@ -641,10 +641,30 @@ The built-in SDK driver loads inline sub-agents and repository sub-agent models against the AWF model catalog. Copilot models use separate Responses and Chat Completions providers according to their supported endpoints, so GPT and Claude agents can use different wire APIs in the same session. -The aliases `small`, `medium`, and `large` inherit the session model; an unknown -sub-agent model logs a warning and inherits it. Sub-agent activity and final model -results, including BYOK status, are retained in `usage/aw_session.jsonl`. -This behavior applies to SDK mode, not the default CLI mode. +Sub-agent activity and final model results, including BYOK status, are retained +in `usage/aw_session.jsonl`. + +In both SDK and CLI mode, the harness resolves each sub-agent's `model:` before +Copilot starts, including on routed runs. It uses gh-aw model aliases (for +example `small` and `large`) and checks the live AWF model catalog. Each result +is recorded as a `model.resolution` event whose role is the agent name. `medium` +is not an alias and is treated as a literal model id. A sub-agent without +`model:` inherits the main model and has no resolution record. + +Workflow-declared sub-agents (inline `## agent:` blocks) fail closed: if a model +cannot be resolved, is not in the live catalog, or does not qualify in SDK mode, +the run fails before Copilot starts. Other repository `.github/agents/*.agent.md` +files, which are often shared with VS Code, do not stop the run: a model that +cannot be resolved is recorded with `status: failed` and logged as a warning. In +CLI mode, that agent is used unchanged; in SDK mode, it inherits the session +model. An inline sub-agent overrides a repository agent with the same name. + +In CLI mode, when the resolved id differs from the declared `model:`, the harness +writes a copy of the agent file, with the same file name and only `model:` +changed, to Copilot CLI's personal agents directory +(`${COPILOT_HOME:-$HOME/.copilot}/agents/`). A personal agent overrides the +repository agent with the same ID, so files in the checkout are never modified. +Sub-agent resolution is skipped in the threat-detection phase. Use top-level `max-tool-denials` to stop SDK inference when tool requests are repeatedly denied. The default is `5`.