Resolve experiment references in inline sub-agent frontmatter before writing the agent file; fail smoke-copilot when file-summarizer doesn't run

.github/workflows/smoke-copilot.md

@@ -195,6 +195,7 @@ post-steps:
           expectModelInventory: true,
           copilotInventory: true,
           resolutionRoles: ["main"],
+          requiredSubAgents: ["file-summarizer"],
         });
 ---
 
@@ -213,6 +214,10 @@ Talk like a caveman in all your responses and outputs. Use short, broken sentenc
   - A `pull-request-number` is shown in `<github-context>`: post the brief PR summary comment (pass `item_number` with that PR number explicitly), and **skip** the fun discussion follow-up comment.
   - No `pull-request-number` in `<github-context>`: **skip** the PR summary comment and post the fun discussion follow-up comment.
 
+## Required First Step: `file-summarizer` Sub-agent
+
+Your first action in this run must be a `task` tool call to the `file-summarizer` agent asking it to summarize `README.md`. Do this before any numbered test below. This step is mandatory and must not be skipped. Record the sub-agent's one-line summary of `README.md`; you will report it as test 14.
+
 ## Test Requirements
 
 Run these checks and mark each as ✅/❌:
@@ -230,7 +235,7 @@ Run these checks and mark each as ✅/❌:
 11. Workflow dispatch: call `dispatch_workflow` for `haiku-printer`, set top-level `ref` to `${{ github.event.repository.default_branch }}`, and include `inputs.message` with an original testing/automation haiku (non-empty string).
 12. PR review tools: add 1-2 inline `create_pull_request_review_comment` comments, submit review with event `COMMENT`, then reply to most recent existing review comment ID when available.
 13. Comment memory: append an original 3-line haiku to `/tmp/gh-aw/comment-memory/*.md`.
-14. Sub-agent: use `file-summarizer` on `README.md`.
+14. Sub-agent: report the result of the required first step (the `task` call to `file-summarizer` on `README.md`), including its one-line summary. Do not call it again here.
 15. Check run: call `create_check_run` with `conclusion=success`, title `Smoke Copilot - Run ${{ github.run_id }}`, summary `All smoke tests completed.`, text `Detailed results attached.`
 16. **LSP TypeScript Testing**: Use the TypeScript language server (configured via `lsp.typescript` frontmatter) to count the number of functions in `${{ github.workspace }}/actions/setup/js/safe_output_helpers.cjs`:
     - Open the file `${{ github.workspace }}/actions/setup/js/safe_output_helpers.cjs` via LSP

actions/setup/js/extract_inline_sub_agents.cjs

@@ -35,6 +35,7 @@ const fs = require("fs");
 const path = require("path");
 const { collectInlineEndMarkers, unknownInlineEndMarkerError } = require("./inline_marker_helpers.cjs");
 const { inlineArtifactTarget } = require("./inline_artifact_target.cjs");
+const { ERR_CONFIG } = require("./error_codes.cjs");
 
 // Regex for the start marker: ## agent: `name` (lowercase identifier)
 const START_MARKER_RE = /^##[ \t]+agent:[ \t]+`([a-z][a-z0-9_-]*)`[ \t]*$/gm;
@@ -64,6 +65,72 @@ function preserveSubAgentFrontmatter(content) {
   return content;
 }
 
+// Experiment references allowed in sub-agent frontmatter: the direct
+// ${{ experiments.<name> }} form (runtime-imported content keeps it verbatim)
+// and the __GH_AW_EXPERIMENTS_<NAME>__ placeholder (compile-time inlined content).
+// Compound expressions are rejected at compile time and are not evaluated here.
+const EXPERIMENT_REFERENCE_RE = /\$\{\{[ \t]*experiments\.([A-Za-z_][A-Za-z0-9_-]*)[ \t]*\}\}|__(GH_AW_EXPERIMENTS_[A-Z0-9_]+)__/g;
+
+// Selected variants are written into YAML frontmatter, so only plain scalars
+// such as model IDs and aliases (small, large, provider/model-1.2) are accepted.
+const SAFE_EXPERIMENT_VALUE_RE = /^[A-Za-z0-9][A-Za-z0-9._/@+:?=&%-]*$/;
+
+/**
+ * Env var holding the selected variant for an experiment, matching
+ * ExperimentEnvVarName in pkg/workflow/expression_extraction.go.
+ * @param {string} experimentName
+ * @returns {string}
+ */
+function experimentEnvVarName(experimentName) {
+  return "GH_AW_EXPERIMENTS_" + experimentName.toUpperCase().replace(/-/g, "_");
+}
+
+/**
+ * Replaces experiment references in a sub-agent's frontmatter with the
+ * variant already selected by the pick-experiment step (exposed to the
+ * activation job as GH_AW_EXPERIMENTS_* env vars). No variant is selected and
+ * no other expression is evaluated here.
+ *
+ * Throws when a referenced experiment has no assignment or its value is not a
+ * plain scalar, so a sub-agent is never written with an unresolved expression.
+ *
+ * @param {string} agentName - Sub-agent name (for error messages).
+ * @param {string} content - Raw agent block content (frontmatter + prompt).
+ * @param {NodeJS.ProcessEnv} [env] - Environment with GH_AW_EXPERIMENTS_* values.
+ * @returns {string} Content with frontmatter experiment references resolved.
+ */
+function resolveSubAgentExperimentReferences(agentName, content, env = process.env) {
+  const match = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*\r?(?=\n|$)/.exec(content);
+  if (!match) return content;
+  const frontmatter = match[1];
+  const resolved = frontmatter
+    .split("\n")
+    .map(line =>
+      line.replace(EXPERIMENT_REFERENCE_RE, (reference, experimentName, placeholderEnvVar) => {
+        const envVar = experimentName ? experimentEnvVarName(experimentName) : placeholderEnvVar;
+        const field = /^[ \t]*([A-Za-z0-9_-]+)[ \t]*:/.exec(line)?.[1] ?? "frontmatter";
+        const value = env[envVar];
+        if (!value) {
+          throw new Error(
+            `${ERR_CONFIG}: inline sub-agent "${agentName}" field "${field}" references ${reference}, but no experiment assignment is available (${envVar} is not set). Declare the experiment under "experiments:" in the workflow frontmatter.`
+          );
+        }
+        if (!SAFE_EXPERIMENT_VALUE_RE.test(value)) {
+          throw new Error(`${ERR_CONFIG}: inline sub-agent "${agentName}" field "${field}" references ${reference}, whose selected variant ${JSON.stringify(value)} is not a plain model or alias value.`);
+        }
+        core.info(`[extractInlineSubAgents] Sub-agent "${agentName}" ${field}: experiment expression ${reference} → selected variant "${value}"`);
+        return value;
+      })
+    )
+    .join("\n");
+  if (/^[ \t]*model[ \t]*:[^\n]*(?:\$\{\{|__GH_AW_)/m.test(resolved)) {
+    throw new Error(`${ERR_CONFIG}: inline sub-agent "${agentName}" field "model" contains an unresolved expression or placeholder. Use a literal model selector or a declared experiment assignment.`);
+  }
+  if (resolved === frontmatter) return content;
+  const start = match.index + match[0].indexOf("\n") + 1;
+  return content.slice(0, start) + resolved + content.slice(start + frontmatter.length);
+}
+
 /**
  * Extracts inline sub-agents from markdown content.
  *
@@ -316,7 +383,7 @@ function writeInlineSubAgents(content, workspaceDir, agentsBaseDir, engineId) {
 
   for (const agent of agents) {
     const agentPath = path.join(agentsDir, agent.name + ext);
-    const filteredContent = preserveSubAgentFrontmatter(agent.content);
+    const filteredContent = resolveSubAgentExperimentReferences(agent.name, preserveSubAgentFrontmatter(agent.content));
     const agentContent = filteredContent.endsWith("\n") ? filteredContent : filteredContent + "\n";
     try {
       fs.writeFileSync(agentPath, agentContent, "utf8");
@@ -388,4 +455,12 @@ function loadCopilotSDKCustomAgents(workspaceDir, agentsBaseDir, logger) {
   return [...agents.values()];
 }
 
-module.exports = { extractInlineSubAgents, writeInlineSubAgents, getEngineSubAgentTarget, preserveSubAgentFrontmatter, closeUnterminatedSubAgentMarkers, loadCopilotSDKCustomAgents };
+module.exports = {
+  extractInlineSubAgents,
+  writeInlineSubAgents,
+  getEngineSubAgentTarget,
+  preserveSubAgentFrontmatter,
+  resolveSubAgentExperimentReferences,
+  closeUnterminatedSubAgentMarkers,
+  loadCopilotSDKCustomAgents,
+};

actions/setup/js/extract_inline_sub_agents.test.cjs

@@ -13,7 +13,7 @@ global.core = {
   setFailed: () => {},
 };
 
-const { extractInlineSubAgents, writeInlineSubAgents, preserveSubAgentFrontmatter, closeUnterminatedSubAgentMarkers, loadCopilotSDKCustomAgents } = require("./extract_inline_sub_agents.cjs");
+const { extractInlineSubAgents, writeInlineSubAgents, preserveSubAgentFrontmatter, resolveSubAgentExperimentReferences, closeUnterminatedSubAgentMarkers, loadCopilotSDKCustomAgents } = require("./extract_inline_sub_agents.cjs");
 
 // Helper: returns a ## agent: `name` start marker line.
 const agentMarker = name => `## agent: \`${name}\``;
@@ -395,3 +395,43 @@ describe("preserveSubAgentFrontmatter", () => {
     expect(result).toBe(content);
   });
 });
+
+describe("resolveSubAgentExperimentReferences", () => {
+  const env = { GH_AW_EXPERIMENTS_SUBAGENT_MODEL: "large", GH_AW_EXPERIMENTS_MY_MODEL: "gpt-5.4-mini" };
+
+  it("resolves direct and placeholder references in frontmatter only", () => {
+    const content = "---\nname: reader\nmodel: ${{experiments.subagent_model}}\ntools: [read]\n---\nKeep ${{ experiments.subagent_model }} and __GH_AW_EXPERIMENTS_SUBAGENT_MODEL__ in the body.";
+    expect(resolveSubAgentExperimentReferences("reader", content, env)).toBe("---\nname: reader\nmodel: large\ntools: [read]\n---\nKeep ${{ experiments.subagent_model }} and __GH_AW_EXPERIMENTS_SUBAGENT_MODEL__ in the body.");
+    expect(resolveSubAgentExperimentReferences("reader", "---\nmodel: __GH_AW_EXPERIMENTS_SUBAGENT_MODEL__\n---\nBody", env)).toBe("---\nmodel: large\n---\nBody");
+  });
+
+  it("maps hyphenated experiment names to the normalized env var", () => {
+    expect(resolveSubAgentExperimentReferences("reader", "---\nmodel: ${{ experiments.my-model }}\n---\nBody", env)).toBe("---\nmodel: gpt-5.4-mini\n---\nBody");
+  });
+
+  it.each(["gpt-5.4?effort=low", "gpt-5.4?effort=low&context=128000"])("preserves parameterized model selector %s", value => {
+    expect(resolveSubAgentExperimentReferences("reader", "---\nmodel: ${{ experiments.subagent_model }}\n---\nBody", { GH_AW_EXPERIMENTS_SUBAGENT_MODEL: value })).toBe(`---\nmodel: ${value}\n---\nBody`);
+  });
+
+  it.each(["__GH_AW_EXPR_0498E4C3__", "${{ secrets.MODEL }}", "__GH_AW_INPUTS_MODEL__"])("rejects unresolved model token %s without evaluating other expressions", model => {
+    expect(() => resolveSubAgentExperimentReferences("reader", `---\nmodel: ${model}\n---\nBody`, { ...env, GH_AW_EXPR_0498E4C3: "large", GH_AW_INPUTS_MODEL: "large" })).toThrow(
+      'inline sub-agent "reader" field "model" contains an unresolved expression or placeholder'
+    );
+  });
+
+  it("returns content without frontmatter or references unchanged", () => {
+    expect(resolveSubAgentExperimentReferences("reader", "No frontmatter ${{ experiments.subagent_model }}", env)).toBe("No frontmatter ${{ experiments.subagent_model }}");
+    const plain = "---\nmodel: small\n---\nBody";
+    expect(resolveSubAgentExperimentReferences("reader", plain, env)).toBe(plain);
+  });
+
+  it("fails with the sub-agent and field name when no assignment is available", () => {
+    expect(() => resolveSubAgentExperimentReferences("file-summarizer", "---\nmodel: ${{ experiments.subagent_model }}\n---\nBody", {})).toThrow(
+      'inline sub-agent "file-summarizer" field "model" references ${{ experiments.subagent_model }}, but no experiment assignment is available (GH_AW_EXPERIMENTS_SUBAGENT_MODEL is not set)'
+    );
+  });
+
+  it.each(["small\nmodel: other", "a b", "${{ secrets.TOKEN }}", "-dash"])("rejects selected variant %j that is not a plain scalar", value => {
+    expect(() => resolveSubAgentExperimentReferences("reader", "---\nmodel: ${{ experiments.subagent_model }}\n---\nBody", { GH_AW_EXPERIMENTS_SUBAGENT_MODEL: value })).toThrow("is not a plain model or alias value");
+  });
+});