diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index 35f05e886d8..d817a257a31 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"87daa01645c01a6306e488cdefdde65a2a32cddd1b6dbb0306c6cb4d7975c193","body_hash":"cd71e47b9b57310c4a132adb44ac4d3e8fed2afba30f07b721f1ec349a1f2e11","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"b634f8a4621f1eda519893cb3bd1b95516d1dff9b4693f855ca7198c56f29f44","body_hash":"a8c8e35941c93f7781604d405a63752a928e0bb39c3ecad94d8d9e0df049b07a","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","CODEX_API_KEY","COPILOT_GITHUB_TOKEN","GEMINI_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"},{"repo":"docker/build-push-action","sha":"c3c9e263c25d99ce0380d002d59b67737d91b0dc","version":"v7.4.0"},{"repo":"docker/setup-buildx-action","sha":"f87e5991a6d7451dcb8d9637bfbc97413f497069","version":"v4.4.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70","digest":"sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70@sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70","digest":"sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70@sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.70","digest":"sha256:372ac51a5cccffdfe484f3f7be427d2d1c566e8ae550d55bd45d9fd166a154b4","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.70@sha256:372ac51a5cccffdfe484f3f7be427d2d1c566e8ae550d55bd45d9fd166a154b4"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70","digest":"sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70@sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.36","digest":"sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.36@sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382","pinned_image":"ghcr.io/github/gh-aw-node@sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"},{"image":"ghcr.io/oraios/serena:1.7.0","digest":"sha256:6c9459e4246a39c9deaa4f23fb05a526ac6e237b24c8e84a927a098fa1ab6730","pinned_image":"ghcr.io/oraios/serena:1.7.0@sha256:6c9459e4246a39c9deaa4f23fb05a526ac6e237b24c8e84a927a098fa1ab6730"}],"mcp_servers":[{"name":"agenticworkflows","tools":["*"]},{"name":"mcpscripts","tools":["github-discussion-query","github-issue-query","github-pr-query"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_check_run","create_discussion","create_issue","create_pull_request_review_comment","dispatch_workflow","haiku_printer","missing_data","missing_tool","noop","remove_labels","reply_to_pull_request_review_comment","report_incomplete","send_slack_message","set_issue_type","submit_pull_request_review","upload_artifact"]},{"name":"serena","tools":["*"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -209,7 +209,7 @@ jobs: GH_AW_EXPERIMENT_SPEC: '{"caveman":{"variants":["yes","no"]},"subagent_model":{"variants":["small","large"]}}' GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.jsonl GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments - GH_AW_HARNESS_VERSION: 87daa01645c01a6306e488cdefdde65a2a32cddd1b6dbb0306c6cb4d7975c193:cd71e47b9b57310c4a132adb44ac4d3e8fed2afba30f07b721f1ec349a1f2e11 + GH_AW_HARNESS_VERSION: b634f8a4621f1eda519893cb3bd1b95516d1dff9b4693f855ca7198c56f29f44:a8c8e35941c93f7781604d405a63752a928e0bb39c3ecad94d8d9e0df049b07a with: script: | const path = require('path'); @@ -646,7 +646,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "87daa01645c01a6306e488cdefdde65a2a32cddd1b6dbb0306c6cb4d7975c193:cd71e47b9b57310c4a132adb44ac4d3e8fed2afba30f07b721f1ec349a1f2e11" + GH_AW_ADMISSION_WORKFLOW_HASH: "b634f8a4621f1eda519893cb3bd1b95516d1dff9b4693f855ca7198c56f29f44:a8c8e35941c93f7781604d405a63752a928e0bb39c3ecad94d8d9e0df049b07a" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} @@ -2677,7 +2677,7 @@ jobs: name: Assert unified-session contract uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const path = require(\"node:path\");\nconst { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait sessionContractMain({\n core,\n engine: \"copilot\",\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [\"main\"],\n});" + script: "const path = require(\"node:path\");\nconst { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait sessionContractMain({\n core,\n engine: \"copilot\",\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [\"main\"],\n requiredSubAgents: [\"file-summarizer\"],\n});" - if: always() name: Stop Playwright title test server run: | diff --git a/.github/workflows/smoke-copilot.md b/.github/workflows/smoke-copilot.md index c963989ee9f..6060a105046 100644 --- a/.github/workflows/smoke-copilot.md +++ b/.github/workflows/smoke-copilot.md @@ -195,6 +195,7 @@ post-steps: expectModelInventory: true, copilotInventory: true, resolutionRoles: ["main"], + requiredSubAgents: ["file-summarizer"], }); --- @@ -213,6 +214,10 @@ Talk like a caveman in all your responses and outputs. Use short, broken sentenc - A `pull-request-number` is shown in ``: post the brief PR summary comment (pass `item_number` with that PR number explicitly), and **skip** the fun discussion follow-up comment. - No `pull-request-number` in ``: **skip** the PR summary comment and post the fun discussion follow-up comment. +## Required First Step: `file-summarizer` Sub-agent + +Your first action in this run must be a `task` tool call to the `file-summarizer` agent asking it to summarize `README.md`. Do this before any numbered test below. This step is mandatory and must not be skipped. Record the sub-agent's one-line summary of `README.md`; you will report it as test 14. + ## Test Requirements Run these checks and mark each as ✅/❌: @@ -230,7 +235,7 @@ Run these checks and mark each as ✅/❌: 11. Workflow dispatch: call `dispatch_workflow` for `haiku-printer`, set top-level `ref` to `${{ github.event.repository.default_branch }}`, and include `inputs.message` with an original testing/automation haiku (non-empty string). 12. PR review tools: add 1-2 inline `create_pull_request_review_comment` comments, submit review with event `COMMENT`, then reply to most recent existing review comment ID when available. 13. Comment memory: append an original 3-line haiku to `/tmp/gh-aw/comment-memory/*.md`. -14. Sub-agent: use `file-summarizer` on `README.md`. +14. Sub-agent: report the result of the required first step (the `task` call to `file-summarizer` on `README.md`), including its one-line summary. Do not call it again here. 15. Check run: call `create_check_run` with `conclusion=success`, title `Smoke Copilot - Run ${{ github.run_id }}`, summary `All smoke tests completed.`, text `Detailed results attached.` 16. **LSP TypeScript Testing**: Use the TypeScript language server (configured via `lsp.typescript` frontmatter) to count the number of functions in `${{ github.workspace }}/actions/setup/js/safe_output_helpers.cjs`: - Open the file `${{ github.workspace }}/actions/setup/js/safe_output_helpers.cjs` via LSP diff --git a/actions/setup/js/extract_inline_sub_agents.cjs b/actions/setup/js/extract_inline_sub_agents.cjs index 226269cecc8..3c6d3010a50 100644 --- a/actions/setup/js/extract_inline_sub_agents.cjs +++ b/actions/setup/js/extract_inline_sub_agents.cjs @@ -35,6 +35,7 @@ const fs = require("fs"); const path = require("path"); const { collectInlineEndMarkers, unknownInlineEndMarkerError } = require("./inline_marker_helpers.cjs"); const { inlineArtifactTarget } = require("./inline_artifact_target.cjs"); +const { ERR_CONFIG } = require("./error_codes.cjs"); // Regex for the start marker: ## agent: `name` (lowercase identifier) const START_MARKER_RE = /^##[ \t]+agent:[ \t]+`([a-z][a-z0-9_-]*)`[ \t]*$/gm; @@ -64,6 +65,72 @@ function preserveSubAgentFrontmatter(content) { return content; } +// Experiment references allowed in sub-agent frontmatter: the direct +// ${{ experiments. }} form (runtime-imported content keeps it verbatim) +// and the __GH_AW_EXPERIMENTS___ placeholder (compile-time inlined content). +// Compound expressions are rejected at compile time and are not evaluated here. +const EXPERIMENT_REFERENCE_RE = /\$\{\{[ \t]*experiments\.([A-Za-z_][A-Za-z0-9_-]*)[ \t]*\}\}|__(GH_AW_EXPERIMENTS_[A-Z0-9_]+)__/g; + +// Selected variants are written into YAML frontmatter, so only plain scalars +// such as model IDs and aliases (small, large, provider/model-1.2) are accepted. +const SAFE_EXPERIMENT_VALUE_RE = /^[A-Za-z0-9][A-Za-z0-9._/@+:?=&%-]*$/; + +/** + * Env var holding the selected variant for an experiment, matching + * ExperimentEnvVarName in pkg/workflow/expression_extraction.go. + * @param {string} experimentName + * @returns {string} + */ +function experimentEnvVarName(experimentName) { + return "GH_AW_EXPERIMENTS_" + experimentName.toUpperCase().replace(/-/g, "_"); +} + +/** + * Replaces experiment references in a sub-agent's frontmatter with the + * variant already selected by the pick-experiment step (exposed to the + * activation job as GH_AW_EXPERIMENTS_* env vars). No variant is selected and + * no other expression is evaluated here. + * + * Throws when a referenced experiment has no assignment or its value is not a + * plain scalar, so a sub-agent is never written with an unresolved expression. + * + * @param {string} agentName - Sub-agent name (for error messages). + * @param {string} content - Raw agent block content (frontmatter + prompt). + * @param {NodeJS.ProcessEnv} [env] - Environment with GH_AW_EXPERIMENTS_* values. + * @returns {string} Content with frontmatter experiment references resolved. + */ +function resolveSubAgentExperimentReferences(agentName, content, env = process.env) { + const match = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*\r?(?=\n|$)/.exec(content); + if (!match) return content; + const frontmatter = match[1]; + const resolved = frontmatter + .split("\n") + .map(line => + line.replace(EXPERIMENT_REFERENCE_RE, (reference, experimentName, placeholderEnvVar) => { + const envVar = experimentName ? experimentEnvVarName(experimentName) : placeholderEnvVar; + const field = /^[ \t]*([A-Za-z0-9_-]+)[ \t]*:/.exec(line)?.[1] ?? "frontmatter"; + const value = env[envVar]; + if (!value) { + throw new Error( + `${ERR_CONFIG}: inline sub-agent "${agentName}" field "${field}" references ${reference}, but no experiment assignment is available (${envVar} is not set). Declare the experiment under "experiments:" in the workflow frontmatter.` + ); + } + if (!SAFE_EXPERIMENT_VALUE_RE.test(value)) { + throw new Error(`${ERR_CONFIG}: inline sub-agent "${agentName}" field "${field}" references ${reference}, whose selected variant ${JSON.stringify(value)} is not a plain model or alias value.`); + } + core.info(`[extractInlineSubAgents] Sub-agent "${agentName}" ${field}: experiment expression ${reference} → selected variant "${value}"`); + return value; + }) + ) + .join("\n"); + if (/^[ \t]*model[ \t]*:[^\n]*(?:\$\{\{|__GH_AW_)/m.test(resolved)) { + throw new Error(`${ERR_CONFIG}: inline sub-agent "${agentName}" field "model" contains an unresolved expression or placeholder. Use a literal model selector or a declared experiment assignment.`); + } + if (resolved === frontmatter) return content; + const start = match.index + match[0].indexOf("\n") + 1; + return content.slice(0, start) + resolved + content.slice(start + frontmatter.length); +} + /** * Extracts inline sub-agents from markdown content. * @@ -316,7 +383,7 @@ function writeInlineSubAgents(content, workspaceDir, agentsBaseDir, engineId) { for (const agent of agents) { const agentPath = path.join(agentsDir, agent.name + ext); - const filteredContent = preserveSubAgentFrontmatter(agent.content); + const filteredContent = resolveSubAgentExperimentReferences(agent.name, preserveSubAgentFrontmatter(agent.content)); const agentContent = filteredContent.endsWith("\n") ? filteredContent : filteredContent + "\n"; try { fs.writeFileSync(agentPath, agentContent, "utf8"); @@ -388,4 +455,12 @@ function loadCopilotSDKCustomAgents(workspaceDir, agentsBaseDir, logger) { return [...agents.values()]; } -module.exports = { extractInlineSubAgents, writeInlineSubAgents, getEngineSubAgentTarget, preserveSubAgentFrontmatter, closeUnterminatedSubAgentMarkers, loadCopilotSDKCustomAgents }; +module.exports = { + extractInlineSubAgents, + writeInlineSubAgents, + getEngineSubAgentTarget, + preserveSubAgentFrontmatter, + resolveSubAgentExperimentReferences, + closeUnterminatedSubAgentMarkers, + loadCopilotSDKCustomAgents, +}; diff --git a/actions/setup/js/extract_inline_sub_agents.test.cjs b/actions/setup/js/extract_inline_sub_agents.test.cjs index da252ae39ee..edb5bd34969 100644 --- a/actions/setup/js/extract_inline_sub_agents.test.cjs +++ b/actions/setup/js/extract_inline_sub_agents.test.cjs @@ -13,7 +13,7 @@ global.core = { setFailed: () => {}, }; -const { extractInlineSubAgents, writeInlineSubAgents, preserveSubAgentFrontmatter, closeUnterminatedSubAgentMarkers, loadCopilotSDKCustomAgents } = require("./extract_inline_sub_agents.cjs"); +const { extractInlineSubAgents, writeInlineSubAgents, preserveSubAgentFrontmatter, resolveSubAgentExperimentReferences, closeUnterminatedSubAgentMarkers, loadCopilotSDKCustomAgents } = require("./extract_inline_sub_agents.cjs"); // Helper: returns a ## agent: `name` start marker line. const agentMarker = name => `## agent: \`${name}\``; @@ -395,3 +395,43 @@ describe("preserveSubAgentFrontmatter", () => { expect(result).toBe(content); }); }); + +describe("resolveSubAgentExperimentReferences", () => { + const env = { GH_AW_EXPERIMENTS_SUBAGENT_MODEL: "large", GH_AW_EXPERIMENTS_MY_MODEL: "gpt-5.4-mini" }; + + it("resolves direct and placeholder references in frontmatter only", () => { + const content = "---\nname: reader\nmodel: ${{experiments.subagent_model}}\ntools: [read]\n---\nKeep ${{ experiments.subagent_model }} and __GH_AW_EXPERIMENTS_SUBAGENT_MODEL__ in the body."; + expect(resolveSubAgentExperimentReferences("reader", content, env)).toBe("---\nname: reader\nmodel: large\ntools: [read]\n---\nKeep ${{ experiments.subagent_model }} and __GH_AW_EXPERIMENTS_SUBAGENT_MODEL__ in the body."); + expect(resolveSubAgentExperimentReferences("reader", "---\nmodel: __GH_AW_EXPERIMENTS_SUBAGENT_MODEL__\n---\nBody", env)).toBe("---\nmodel: large\n---\nBody"); + }); + + it("maps hyphenated experiment names to the normalized env var", () => { + expect(resolveSubAgentExperimentReferences("reader", "---\nmodel: ${{ experiments.my-model }}\n---\nBody", env)).toBe("---\nmodel: gpt-5.4-mini\n---\nBody"); + }); + + it.each(["gpt-5.4?effort=low", "gpt-5.4?effort=low&context=128000"])("preserves parameterized model selector %s", value => { + expect(resolveSubAgentExperimentReferences("reader", "---\nmodel: ${{ experiments.subagent_model }}\n---\nBody", { GH_AW_EXPERIMENTS_SUBAGENT_MODEL: value })).toBe(`---\nmodel: ${value}\n---\nBody`); + }); + + it.each(["__GH_AW_EXPR_0498E4C3__", "${{ secrets.MODEL }}", "__GH_AW_INPUTS_MODEL__"])("rejects unresolved model token %s without evaluating other expressions", model => { + expect(() => resolveSubAgentExperimentReferences("reader", `---\nmodel: ${model}\n---\nBody`, { ...env, GH_AW_EXPR_0498E4C3: "large", GH_AW_INPUTS_MODEL: "large" })).toThrow( + 'inline sub-agent "reader" field "model" contains an unresolved expression or placeholder' + ); + }); + + it("returns content without frontmatter or references unchanged", () => { + expect(resolveSubAgentExperimentReferences("reader", "No frontmatter ${{ experiments.subagent_model }}", env)).toBe("No frontmatter ${{ experiments.subagent_model }}"); + const plain = "---\nmodel: small\n---\nBody"; + expect(resolveSubAgentExperimentReferences("reader", plain, env)).toBe(plain); + }); + + it("fails with the sub-agent and field name when no assignment is available", () => { + expect(() => resolveSubAgentExperimentReferences("file-summarizer", "---\nmodel: ${{ experiments.subagent_model }}\n---\nBody", {})).toThrow( + 'inline sub-agent "file-summarizer" field "model" references ${{ experiments.subagent_model }}, but no experiment assignment is available (GH_AW_EXPERIMENTS_SUBAGENT_MODEL is not set)' + ); + }); + + it.each(["small\nmodel: other", "a b", "${{ secrets.TOKEN }}", "-dash"])("rejects selected variant %j that is not a plain scalar", value => { + expect(() => resolveSubAgentExperimentReferences("reader", "---\nmodel: ${{ experiments.subagent_model }}\n---\nBody", { GH_AW_EXPERIMENTS_SUBAGENT_MODEL: value })).toThrow("is not a plain model or alias value"); + }); +}); diff --git a/actions/setup/js/interpolate_prompt.test.cjs b/actions/setup/js/interpolate_prompt.test.cjs index 1092dabbbb8..5443d7497f7 100644 --- a/actions/setup/js/interpolate_prompt.test.cjs +++ b/actions/setup/js/interpolate_prompt.test.cjs @@ -488,3 +488,134 @@ describe("interpolate_prompt", () => { })); })); }); + +describe("inline sub-agent experiment model expansion", () => { + const { main } = require("./interpolate_prompt.cjs"); + const agentBlock = model => ["## agent: `file-summarizer`", "---", `model: ${model}`, "description: Summarizes the content of a file in a few concise sentences", "---", "You are a file summarization assistant."].join("\n"); + let tmpDir, promptPath, originalEnv; + + beforeEach(() => { + originalEnv = { ...process.env }; + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "interpolate-subagent-test-")); + promptPath = path.join(tmpDir, "prompt.txt"); + process.env.GH_AW_PROMPT = promptPath; + process.env.GITHUB_WORKSPACE = tmpDir; + process.env.GH_AW_ENGINE_ID = "copilot"; + core.debug ??= vi.fn(); + core.info.mockClear(); + core.setFailed.mockClear(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + Object.keys(process.env).forEach(k => { + if (!(k in originalEnv)) delete process.env[k]; + }); + Object.assign(process.env, originalEnv); + }); + + /** + * Runs the real interpolate_prompt main() and returns the sub-agent file it writes. + * @param {string} prompt + * @returns {Promise} + */ + async function runAndReadAgent(prompt) { + fs.writeFileSync(promptPath, prompt, "utf8"); + const realWriteFileSync = fs.writeFileSync; + /** @type {Map} */ + const agentWrites = new Map(); + vi.spyOn(fs, "writeFileSync").mockImplementation((file, data, ...args) => { + if (String(file).endsWith("file-summarizer.agent.md")) agentWrites.set(String(file), String(data)); + else realWriteFileSync(file, data, ...args); + }); + await main(); + vi.restoreAllMocks(); + expect(core.setFailed).not.toHaveBeenCalled(); + expect([...agentWrites.keys()]).toEqual([expect.stringMatching(/\.github\/agents\/file-summarizer\.agent\.md$/)]); + return [...agentWrites.values()][0]; + } + + const forms = { + "runtime-import": async model => { + fs.mkdirSync(path.join(tmpDir, ".github", "workflows"), { recursive: true }); + fs.writeFileSync(path.join(tmpDir, ".github", "workflows", "smoke.md"), `---\non: workflow_dispatch\nexperiments:\n subagent_model: [small, large]\n---\nUse file-summarizer on README.md.\n\n${agentBlock(model)}\n`); + return runAndReadAgent("{{#runtime-import .github/workflows/smoke.md}}\n"); + }, + inlined: async model => runAndReadAgent(`Use file-summarizer on README.md.\n\n${agentBlock(model)}\n`), + }; + + describe.each(Object.keys(forms))("%s prompt content", form => { + it.each([ + ["${{ experiments.subagent_model }}", "small"], + ["${{ experiments.subagent_model }}", "large"], + ["__GH_AW_EXPERIMENTS_SUBAGENT_MODEL__", "small"], + ["__GH_AW_EXPERIMENTS_SUBAGENT_MODEL__", "large"], + ["${{ experiments.subagent_model }}", "gpt-5.4?effort=low&context=128000"], + ])("writes_selected_experiment_model_before_agent_file_write: %s with assignment %s", async (reference, variant) => { + process.env.GH_AW_EXPERIMENTS_SUBAGENT_MODEL = variant; + const written = await forms[form](reference); + expect(written.startsWith(`---\nmodel: ${variant}\n`)).toBe(true); + expect(written).not.toContain("${{"); + expect(written).not.toContain("experiments."); + expect(written).not.toContain("__GH_AW_"); + expect(written).toContain("description: Summarizes the content of a file in a few concise sentences"); + expect(written).toContain("You are a file summarization assistant."); + expect(fs.readFileSync(promptPath, "utf8")).not.toContain("file summarization assistant"); + }); + + it("both_subagent_model_variants_are_reachable", async () => { + /** @type {string[]} */ + const models = []; + for (const variant of ["small", "large"]) { + process.env.GH_AW_EXPERIMENTS_SUBAGENT_MODEL = variant; + const written = await forms[form]("${{ experiments.subagent_model }}"); + models.push(/^model: (.*)$/m.exec(written)?.[1] ?? ""); + } + expect(models).toEqual(["small", "large"]); + }); + + it("preserves_plain_subagent_model_and_frontmatter", async () => { + process.env.GH_AW_EXPERIMENTS_SUBAGENT_MODEL = "large"; + const written = await forms[form]("claude-haiku-4.5"); + expect(written).toBe("---\nmodel: claude-haiku-4.5\ndescription: Summarizes the content of a file in a few concise sentences\n---\nYou are a file summarization assistant.\n"); + }); + + it("fails activation instead of writing an unresolved experiment model", async () => { + delete process.env.GH_AW_EXPERIMENTS_SUBAGENT_MODEL; + fs.writeFileSync(promptPath, "", "utf8"); + const writeSpy = vi.spyOn(fs, "writeFileSync"); + if (form === "runtime-import") { + fs.mkdirSync(path.join(tmpDir, ".github", "workflows"), { recursive: true }); + fs.writeFileSync(path.join(tmpDir, ".github", "workflows", "smoke.md"), `Task.\n\n${agentBlock("${{ experiments.subagent_model }}")}\n`); + fs.writeFileSync(promptPath, "{{#runtime-import .github/workflows/smoke.md}}\n", "utf8"); + } else { + fs.writeFileSync(promptPath, `Task.\n\n${agentBlock("${{ experiments.subagent_model }}")}\n`, "utf8"); + } + writeSpy.mockClear(); + await main(); + expect(core.setFailed).toHaveBeenCalledWith(expect.stringContaining('inline sub-agent "file-summarizer" field "model" references ${{ experiments.subagent_model }}')); + expect(writeSpy.mock.calls.some(call => String(call[0]).endsWith("file-summarizer.agent.md"))).toBe(false); + }); + + it.each(["small", undefined])("handles CRLF runtime imports with assignment %s", async variant => { + fs.mkdirSync(path.join(tmpDir, ".github", "workflows"), { recursive: true }); + const imported = `Task.\n\n${agentBlock("${{ experiments.subagent_model }}")}\n`.replace(/\n/g, "\r\n"); + fs.writeFileSync(path.join(tmpDir, ".github", "workflows", "smoke.md"), imported); + const prompt = "{{#runtime-import .github/workflows/smoke.md}}\n"; + if (variant) { + process.env.GH_AW_EXPERIMENTS_SUBAGENT_MODEL = variant; + const written = await runAndReadAgent(prompt); + expect(written).toContain(`---\r\nmodel: ${variant}\r\n`); + expect(written).not.toContain("${{"); + } else { + delete process.env.GH_AW_EXPERIMENTS_SUBAGENT_MODEL; + fs.writeFileSync(promptPath, prompt); + const writeSpy = vi.spyOn(fs, "writeFileSync"); + await main(); + expect(core.setFailed).toHaveBeenCalledWith(expect.stringContaining('inline sub-agent "file-summarizer" field "model" references ${{ experiments.subagent_model }}')); + expect(writeSpy.mock.calls.some(call => String(call[0]).endsWith("file-summarizer.agent.md"))).toBe(false); + } + }); + }); +}); diff --git a/actions/setup/js/smoke_model_routing_assertions.cjs b/actions/setup/js/smoke_model_routing_assertions.cjs index edeada8ab42..3e09162fa90 100644 --- a/actions/setup/js/smoke_model_routing_assertions.cjs +++ b/actions/setup/js/smoke_model_routing_assertions.cjs @@ -65,6 +65,12 @@ const { iterateJsonlEvents } = require("./jsonl_helpers.cjs"); * line reports that U2 was not evaluated); U3 every role in `resolutionRoles` has * exactly one `model.resolution` with status exact or resolved and a * resolved model. U1–U3 are not evaluated when the session file is missing. + * - `requiredSubAgents` (array of names) opts in and enables U4: each named + * sub-agent has at least one `subagent.started`, every started invocation + * has a `subagent.completed`, no `subagent.failed` is recorded for it, and + * its events record a model that is not an unresolved `${{ ... }}` + * expression or `__GH_AW_*__` placeholder. A sub-agent that never started + * fails U4. Lifecycle events are correlated as in S1/S2. * * Every check that iterates over requests fails when there are none to check. * @@ -105,6 +111,7 @@ const RESOLVED_STATUSES = ["exact", "resolved"]; * expectModelInventory?: boolean, * copilotInventory?: boolean, * resolutionRoles?: string[], + * requiredSubAgents?: string[], * }} SessionContractExpectations * @typedef {SessionContractExpectations & { * allowedModels: string[], @@ -249,6 +256,37 @@ function supportedEfforts(model, reflect) { return efforts; } +/** @param {any} event @returns {string} */ +function invocationKey(event) { + return idString(event?.data?.invocationId ?? event?.agentId ?? event?.data?.toolCallId); +} + +/** + * Lifecycle events for one declared sub-agent: its `subagent.started` events + * (by agent name) and every `subagent.*` event correlated to them by + * invocation or agent ID (or tool call ID when no agent ID is recorded). + * @param {any[]} subagentEvents - All `subagent.*` events in the session. + * @param {string} name - Declared sub-agent name. + * @returns {{ started: any[], ids: Set, activity: any[] }} + */ +function subAgentLifecycle(subagentEvents, name) { + const started = subagentEvents.filter(event => event.type === "subagent.started" && event.data?.agentName === name); + const ids = new Set(started.map(event => idString(event.data?.invocationId ?? event.agentId)).filter(Boolean)); + const toolCallIds = new Set( + started + .filter(event => !idString(event.data?.invocationId ?? event.agentId)) + .map(event => idString(event.data?.toolCallId)) + .filter(Boolean) + ); + const activity = subagentEvents.filter(event => ids.has(idString(event.data?.invocationId)) || ids.has(idString(event.agentId)) || toolCallIds.has(idString(event.data?.toolCallId))); + return { started, ids, activity }; +} + +/** @param {any[]} events @returns {any[]} */ +function subagentLifecycleEvents(events) { + return events.filter(event => typeof event?.type === "string" && event.type.startsWith("subagent.")); +} + /** * Evaluate model-routing evidence against the smoke expectations. * @param {{ events: any[], requests: any[], reflect?: any, expectations: RoutingExpectations }} input @@ -320,7 +358,7 @@ function evaluateModelRoutingEvidence({ events, requests, reflect = null, expect // agent made it. const agentTraffic = requests.filter(request => !isClassifierRequest(request)); - const subagentEvents = events.filter(event => typeof event?.type === "string" && event.type.startsWith("subagent.")); + const subagentEvents = subagentLifecycleEvents(events); const startedEvents = subagentEvents.filter(event => event.type === "subagent.started"); const observedNames = [...new Set(startedEvents.map(event => event.data?.agentName ?? ""))]; @@ -328,9 +366,7 @@ function evaluateModelRoutingEvidence({ events, requests, reflect = null, expect // (request IDs on sub-agent events, or sub-agent IDs on proxy records), and // fall back to the declared model. const subAgentEvidence = subAgents.map(agent => { - const started = startedEvents.filter(event => event.data?.agentName === agent.name); - const ids = new Set(started.map(event => idString(event.data?.invocationId ?? event.agentId)).filter(Boolean)); - const activity = subagentEvents.filter(event => ids.has(idString(event.data?.invocationId)) || ids.has(idString(event.agentId))); + const { started, ids, activity } = subAgentLifecycle(subagentEvents, agent.name); const eventRequestIds = new Set(activity.map(eventRequestId).filter(Boolean)); const correlated = agentTraffic.filter(request => eventRequestIds.has(requestId(request)) || ids.has(requestAgentId(request))); return { agent, model: servedModel(agent.model), started, activity, correlated }; @@ -586,7 +622,7 @@ function evaluateSteeringEvidence({ events, requests, routingRecords = [], expec * @returns {boolean} */ function hasSessionContract(expectations) { - return expectations.expectModelInventory === true || expectations.copilotInventory === true || Array.isArray(expectations.resolutionRoles); + return expectations.expectModelInventory === true || expectations.copilotInventory === true || Array.isArray(expectations.resolutionRoles) || Array.isArray(expectations.requiredSubAgents); } /** @param {any} inventory @returns {string} */ @@ -676,6 +712,29 @@ function evaluateSessionContract({ events, expectations }) { else if (roles.length) pass("U3", `model.resolution recorded for ${describeResolutions(resolutions.filter(event => roles.includes(event.data?.role)))}`); else pass("U3", `no model.resolution roles required; observed model.resolution: ${describeResolutions(resolutions)}`); + // U4: each required sub-agent was invoked, every invocation completed, none + // failed, and its recorded model is not an unresolved expression or placeholder. + const subagentEvents = subagentLifecycleEvents(events); + for (const name of expectations.requiredSubAgents ?? []) { + const label = `sub-agent ${name}`; + const { started, activity } = subAgentLifecycle(subagentEvents, name); + const failed = subagentEvents.filter(event => event.type === "subagent.failed" && (activity.includes(event) || event.data?.agentName === name)); + const completedKeys = new Set(activity.filter(event => event.type === "subagent.completed" && event.data?.cancelled !== true).map(invocationKey)); + const incomplete = started.filter(event => !completedKeys.has(invocationKey(event))); + const models = [ + ...new Set([...activity, ...failed].flatMap(event => [event.data?.model, event.data?.selectedModel, event.data?.resolvedModel, event.data?.firstDispatchedModel]).filter(value => typeof value === "string" && value !== "")), + ]; + const unresolved = models.filter(model => /\$\{\{|__GH_AW_/.test(model)); + const u4Problems = []; + if (!started.length) u4Problems.push("never started (no subagent.started)"); + if (failed.length) u4Problems.push(`${failed.length} subagent.failed event(s)${failed.map(event => (event.data?.error ? `: ${String(event.data.error).slice(0, 200)}` : "")).join("")}`); + if (started.length && incomplete.length) u4Problems.push(`${incomplete.length} of ${started.length} invocation(s) have no subagent.completed`); + if (started.length && !models.length) u4Problems.push("no model recorded on its lifecycle events"); + if (unresolved.length) u4Problems.push(`unresolved model expression(s) [${unresolved.join(", ")}]`); + if (u4Problems.length) fail("U4", `${label}: ${u4Problems.join("; ")}; observed events: ${describeEvents(started.length ? [...new Set([...activity, ...failed])] : subagentEvents)}`); + else pass("U4", `${label} started ${started.length} time(s) and completed on model(s) [${models.join(", ")}]`); + } + return results; } diff --git a/actions/setup/js/smoke_model_routing_assertions.test.cjs b/actions/setup/js/smoke_model_routing_assertions.test.cjs index 3fd5cd92fdf..10d152ac545 100644 --- a/actions/setup/js/smoke_model_routing_assertions.test.cjs +++ b/actions/setup/js/smoke_model_routing_assertions.test.cjs @@ -1147,6 +1147,91 @@ describe("smoke_model_routing_assertions", () => { expect(core.info).toHaveBeenLastCalledWith("All 4 unified-session smoke assertions passed"); }); + describe("required sub-agent lifecycle (U4)", () => { + // Copilot CLI lifecycle record shapes (copilot-cli-builtin-subagents golden + // and the failed start in run 39181772988). + const FS_ID = "6f1c9a52-3d0e-4c55-9a7e-0b2f6d1e8c41"; + const cliEvent = (type, data, agentId = FS_ID) => ({ + type, + agentId, + data: { toolCallId: "call_fs", ...data }, + provenance: { component: "agent", phase: "agent", path: "sandbox/agent/logs/copilot-session-state/s/events.jsonl", index: 0 }, + }); + const started = (model = "claude-haiku-4.5", agentId = FS_ID) => + cliEvent("subagent.started", { agentName: "file-summarizer", agentType: "file-summarizer", model, modelSelectionSource: "agent_definition_default", executionMode: "sync" }, agentId); + const completed = (model = "claude-haiku-4.5", agentId = FS_ID) => cliEvent("subagent.completed", { agentName: "file-summarizer", model, firstDispatchedModel: model, durationMs: 4100, totalToolCalls: 1 }, agentId); + const failed = (model, error) => cliEvent("subagent.failed", { agentName: "file-summarizer", durationMs: 280, totalToolCalls: 0, error: `400 Unsupported model selector "${model}" on copilot: no advertised display name matches.` }); + const smokeCopilot = { ...unroutedCopilot, requiredSubAgents: ["file-summarizer"] }; + const run = async session => { + write("usage/aw_session.jsonl", [...postFixCopilotSession(), ...session]); + const core = { info: vi.fn(), error: vi.fn(), setFailed: vi.fn() }; + const results = await sessionContractMain({ core, rootDir: root, buildSession: false, ...smokeCopilot }); + return { results, core }; + }; + + it("passes when file-summarizer starts and completes on a resolved model", async () => { + const { results, core } = await run([started(), cliEvent("subagent.configured", { model: "claude-haiku-4.5" }), completed()]); + expect(results.failures).toEqual([]); + expect(lineFor(results.passes, "U4")).toBe("PASS U4 sub-agent file-summarizer started 1 time(s) and completed on model(s) [claude-haiku-4.5]"); + expect(checkIds(results.passes)).toEqual(["E2", "U1", "U2", "U3", "U4"]); + expect(core.setFailed).not.toHaveBeenCalled(); + }); + + it("smoke_fails_when_file_summarizer_fails_to_start", async () => { + const expression = "${{ experiments.subagent_model }}"; + const { results, core } = await run([started(expression), failed(expression)]); + expect(results.failures).toEqual([ + `FAIL U4 sub-agent file-summarizer: 1 subagent.failed event(s): 400 Unsupported model selector "${expression}" on copilot: no advertised display name matches.; 1 of 1 invocation(s) have no subagent.completed; unresolved model expression(s) [${expression}]; observed events: subagent.started(agentName=file-summarizer model=${expression}), subagent.failed(agentName=file-summarizer)`, + ]); + expect(core.setFailed).toHaveBeenCalled(); + + // A failed start on an unresolved alias fails as well, without the expression problem. + const alias = await run([started("small"), failed("small")]); + expect(lineFor(alias.results.failures, "U4")).toMatch( + /^FAIL U4 sub-agent file-summarizer: 1 subagent\.failed event\(s\): 400 Unsupported model selector "small".*; 1 of 1 invocation\(s\) have no subagent\.completed; observed events: / + ); + }); + + it("fails when a later invocation fails or never completes", async () => { + const second = "9a0d7c11-5b2e-4f8a-8c3d-2e6f4b1a7d90"; + const { results } = await run([started(), completed(), started("claude-haiku-4.5", second)]); + expect(lineFor(results.failures, "U4")).toMatch(/^FAIL U4 sub-agent file-summarizer: 1 of 2 invocation\(s\) have no subagent\.completed; /); + }); + + it("fails when completion is cancellation only", async () => { + const cancelled = { ...completed(), data: { ...completed().data, cancelled: true } }; + const { results, core } = await run([started(), cancelled]); + expect(lineFor(results.failures, "U4")).toMatch(/1 of 1 invocation\(s\) have no subagent\.completed/); + expect(core.setFailed).toHaveBeenCalled(); + }); + + it("preserves a genuine completion followed by cleanup cancellation", async () => { + const cancelled = { ...completed(), data: { ...completed().data, cancelled: true } }; + const { results, core } = await run([started(), completed(), cancelled]); + expect(results.failures).toEqual([]); + expect(lineFor(results.passes, "U4")).toContain("started 1 time(s) and completed"); + expect(core.setFailed).not.toHaveBeenCalled(); + }); + + it("requires non-cancelled completion for each invocation", async () => { + const second = "second-invocation"; + const cancelled = { ...completed("claude-haiku-4.5", second), data: { ...completed().data, cancelled: true } }; + const { results } = await run([started(), completed(), started("claude-haiku-4.5", second), cancelled]); + expect(lineFor(results.failures, "U4")).toMatch(/1 of 2 invocation\(s\) have no subagent\.completed/); + }); + + it("smoke_fails_when_file_summarizer_is_not_invoked", async () => { + const { results, core } = await run([]); + expect(results.failures).toEqual(["FAIL U4 sub-agent file-summarizer: never started (no subagent.started); observed events: none"]); + expect(checkIds(results.passes)).toEqual(["E2", "U1", "U2", "U3"]); + expect(core.setFailed).toHaveBeenCalled(); + + // Another sub-agent's successful lifecycle does not satisfy the requirement. + const other = await run([{ ...started(), data: { ...started().data, agentName: "explore" } }, completed()]); + expect(lineFor(other.results.failures, "U4")).toMatch(/^FAIL U4 sub-agent file-summarizer: never started \(no subagent\.started\); observed events: subagent\.started\(agentName=explore /); + }); + }); + it("counts U failures in T4 when steering is configured", () => { write("usage/aw_session.jsonl", preFixCopilotSession()); write(TOKEN_USAGE, copilotRequests()); diff --git a/docs/src/content/docs/experimental/experiments.md b/docs/src/content/docs/experimental/experiments.md index d95d1d51c8f..1a3079172c4 100644 --- a/docs/src/content/docs/experimental/experiments.md +++ b/docs/src/content/docs/experimental/experiments.md @@ -187,8 +187,10 @@ syntax. With Copilot, you can also declare `subagent_model: [small, large]` in `experiments` and set the sub-agent's `model` to -`${{ experiments.subagent_model }}`. The selected variant is used in the -sub-agent definition and recorded in the run's declared-model metadata. +`${{ experiments.subagent_model }}`. The activation job replaces the reference +with the selected variant before it writes the sub-agent file, and records the +same variant in the run's declared-model metadata. If no assignment is available, +activation fails instead of writing the expression. Audit patterns and model-routing request policy expand every declared variant through the workflow's model aliases, while `models.allowed` and `models.blocked` still constrain admission. Router candidates remain unchanged. These patterns diff --git a/pkg/workflow/compiler_experiments_test.go b/pkg/workflow/compiler_experiments_test.go index ba24d151491..2952b77bc59 100644 --- a/pkg/workflow/compiler_experiments_test.go +++ b/pkg/workflow/compiler_experiments_test.go @@ -4,7 +4,9 @@ package workflow import ( "encoding/json" + "fmt" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -325,6 +327,133 @@ func TestCompileCopilotWithExperimentalSubagentModels(t *testing.T) { } } +// TestSubagentMetadataMatchesSelectedExperimentVariant verifies that the recorded sub-agent +// model metadata and the value interpolate_prompt.cjs writes into the inline sub-agent file +// both come from the same pick-experiment assignment, so after Actions evaluates the step +// output they agree with each other and with the selected variant. +func TestSubagentMetadataMatchesSelectedExperimentVariant(t *testing.T) { + source := filepath.Join(t.TempDir(), "workflow.md") + content := "---\non: workflow_dispatch\nstrict: false\npermissions:\n contents: read\nengine: copilot\nexperiments:\n subagent_model: [small, large]\n---\n# Work\nSummarize README.md using file-summarizer.\n\n" + + "## agent: `file-summarizer`\n---\nmodel: ${{ experiments.subagent_model }}\ndescription: Summarizes a file\n---\nSummarize the file.\n" + require.NoError(t, os.WriteFile(source, []byte(content), 0600)) + compiler := NewCompiler() + data, err := compiler.ParseWorkflowFile(source) + require.NoError(t, err) + generated, err := compiler.CompileToYAML(data, source) + require.NoError(t, err) + + var workflow struct { + Jobs map[string]struct { + Steps []struct { + ID string `yaml:"id"` + Name string `yaml:"name"` + Env map[string]string `yaml:"env"` + } `yaml:"steps"` + } `yaml:"jobs"` + } + require.NoError(t, yaml.Unmarshal([]byte(generated), &workflow)) + activation, ok := workflow.Jobs["activation"] + require.True(t, ok, "activation job must exist") + + var metadataEnv, agentFileEnv string + pickIndex, metadataIndex, agentFileIndex := -1, -1, -1 + for i, step := range activation.Steps { + switch { + case step.ID == "pick-experiment": + pickIndex = i + case step.ID == "generate_aw_info": + metadataIndex = i + metadataEnv = step.Env["GH_AW_INFO_SUB_AGENT_MODELS"] + case step.Name == "Interpolate variables and render templates": + agentFileIndex = i + agentFileEnv = step.Env["GH_AW_EXPERIMENTS_SUBAGENT_MODEL"] + } + } + require.NotEqual(t, -1, pickIndex, "pick-experiment step must exist") + require.NotEqual(t, -1, metadataIndex, "generate_aw_info step must exist") + require.NotEqual(t, -1, agentFileIndex, "interpolation step that writes inline sub-agents must exist") + assert.Less(t, pickIndex, metadataIndex, "experiment selection must precede sub-agent metadata") + assert.Less(t, pickIndex, agentFileIndex, "experiment selection must precede the sub-agent file write") + + const selectedVariant = "${{ steps.pick-experiment.outputs.subagent_model }}" + assert.Equal(t, selectedVariant, agentFileEnv, "the sub-agent file must be written from the selected assignment") + + for _, variant := range []string{"small", "large"} { + t.Run(variant, func(t *testing.T) { + // Simulate Actions evaluating the pick-experiment output for this assignment. + var declarations []struct { + Name string `json:"name"` + Model string `json:"model"` + } + require.NoError(t, json.Unmarshal([]byte(strings.ReplaceAll(metadataEnv, selectedVariant, variant)), &declarations)) + require.Len(t, declarations, 1) + assert.Equal(t, "file-summarizer", declarations[0].Name) + assert.Equal(t, variant, declarations[0].Model, "recorded sub-agent model must be the selected variant") + assert.Equal(t, strings.ReplaceAll(agentFileEnv, selectedVariant, variant), declarations[0].Model, "recorded model must match the model written to the sub-agent file") + assert.NotContains(t, declarations[0].Model, "experiments.", "a literal experiment expression must not be recorded as a model ID") + assert.NotContains(t, declarations[0].Model, "${{", "an unevaluated expression must not be recorded as a model ID") + }) + } +} + +func TestInlinedSubagentExperimentCompilerToRuntime(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node is required for compiler-to-runtime coverage") + } + runtime, err := filepath.Abs("../../actions/setup/js/interpolate_prompt.cjs") + require.NoError(t, err) + workspace := t.TempDir() + workflows := filepath.Join(workspace, ".github", "workflows") + require.NoError(t, os.MkdirAll(workflows, 0700)) + const importPath = ".github/workflows/reader.md" + require.NoError(t, os.WriteFile(filepath.Join(workspace, importPath), []byte( + "## agent: `reader`\n---\nmodel: ${{ experiments.subagent_model }}\n---\nSummarize the file.\n"), 0600)) + compiler := NewCompiler() + compiler.markdownPath = filepath.Join(workflows, "workflow.md") + + for _, ordered := range []bool{true, false} { + data := &WorkflowData{InlinedImports: true, ImportPaths: []string{importPath}} + if ordered { + data.PromptImports = []parser.PromptImportEntry{{ImportPath: importPath}} + } + chunks, mappings := compiler.processPromptImportEntries(data) + prompt := strings.Join(chunks, "\n") + require.Contains(t, prompt, "model: __GH_AW_EXPERIMENTS_SUBAGENT_MODEL__") + require.Len(t, mappings, 1) + assert.Equal(t, "GH_AW_EXPERIMENTS_SUBAGENT_MODEL", mappings[0].EnvVar) + assert.Equal(t, "steps.pick-experiment.outputs.subagent_model", mappings[0].Content) + + for _, variant := range []string{"small", "large", "gpt-5.4?effort=low"} { + t.Run(fmt.Sprintf("ordered=%t/%s", ordered, variant), func(t *testing.T) { + promptPath := filepath.Join(t.TempDir(), "prompt.txt") + require.NoError(t, os.WriteFile(promptPath, []byte(prompt), 0600)) + cmd := exec.Command(node, "-e", ` +const fs = require("fs"); +global.core = { info() {}, warning() {}, debug() {}, setFailed(message) { throw new Error(message); } }; +const write = fs.writeFileSync; +const agents = []; +fs.writeFileSync = (file, content, ...args) => { + if (String(file).endsWith(".agent.md")) agents.push(String(content)); + else write(file, content, ...args); +}; +require(process.argv[1]).main().then(() => { + if (agents.length !== 1) throw new Error("Expected one agent file write"); + process.stdout.write(agents[0]); +}).catch(error => { console.error(error); process.exitCode = 1; }); +`, runtime) + cmd.Dir = workspace + cmd.Env = append(os.Environ(), "GH_AW_PROMPT="+promptPath, "GITHUB_WORKSPACE="+workspace, "GH_AW_ENGINE_ID=copilot", "GH_AW_EXPERIMENTS_SUBAGENT_MODEL="+variant) + output, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", output) + assert.Contains(t, string(output), "model: "+variant+"\n") + assert.NotContains(t, string(output), "__GH_AW_") + assert.NotContains(t, string(output), "${{") + }) + } + } +} + // TestRewriteExperimentPrefixCollision guards against one declared experiment name being a // prefix of another (e.g. "model" and "model_variant"); the word-boundary-anchored regex // rewrite must not partially corrupt the longer name's reference while rewriting the shorter diff --git a/pkg/workflow/expression_extraction.go b/pkg/workflow/expression_extraction.go index 911bfa68525..b51751e7880 100644 --- a/pkg/workflow/expression_extraction.go +++ b/pkg/workflow/expression_extraction.go @@ -128,9 +128,13 @@ func (e *ExpressionExtractor) processMatch(originalExpr, rawContent string) { )) } + envVar := e.generateEnvVarName(content) + if name, ok := regexSubmatchAt(experimentNameRegex.FindStringSubmatch(originalContent), 1); ok { + envVar = ExperimentEnvVarName(name) + } e.mappings[originalExpr] = &ExpressionMapping{ Original: originalExpr, - EnvVar: e.generateEnvVarName(content), + EnvVar: envVar, Content: content, }