From 12ff139b5679e59bdb3fb2a4546e873b9aa64992 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:25:14 +0000 Subject: [PATCH 1/5] Initial plan From 6110f97c1a197c877ef20a9e65722ef5ee533a80 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:34:58 +0000 Subject: [PATCH 2/5] Make smoke-copilot-sub-agents contract mandatory and independent of the experiment Co-authored-by: SivaKesava1 <11771739+SivaKesava1@users.noreply.github.com> --- .../smoke-copilot-sub-agents.lock.yml | 17 ++-- .github/workflows/smoke-copilot-sub-agents.md | 86 +++++++++---------- actions/setup/js/interpolate_prompt.test.cjs | 3 +- .../js/smoke_copilot_sub_agents.test.cjs | 12 +-- 4 files changed, 56 insertions(+), 62 deletions(-) diff --git a/.github/workflows/smoke-copilot-sub-agents.lock.yml b/.github/workflows/smoke-copilot-sub-agents.lock.yml index 8c081af3646..43c2509f128 100644 --- a/.github/workflows/smoke-copilot-sub-agents.lock.yml +++ b/.github/workflows/smoke-copilot-sub-agents.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"77090e7016ef53fcfb0f9afb99510eb5429319cf88ea6274982fdaf291d71e52","body_hash":"62af0943eec9eb3c97faee594017b8b312f93be82bf72d2e575b15d9239140c3","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"3958c29c5e2c8ed81e5255399d4f425ba9a5c2eccd5c8aeca3a7eac2aafc01e6","body_hash":"1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70","digest":"sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70@sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70","digest":"sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70@sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70","digest":"sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70@sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.36","digest":"sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.36@sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382","pinned_image":"ghcr.io/github/gh-aw-node@sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","noop","report_incomplete","report_missing_data","report_missing_tool"]}],"threat_detection":{"mode":"enabled"},"admission":true} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -164,10 +164,10 @@ jobs: id: pick-experiment uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_EXPERIMENT_SPEC: '{"sub_agent_strategy":{"variants":["inline_strict","delegated_sequential","single_agent_control"],"description":"Measure whether inline sub-agent orchestration is the best reliability/cost tradeoff for a model-identity smoke test.","hypothesis":"H0: no change in pass_rate between inline_strict and delegated_sequential. H1: delegated_sequential improves pass_rate by \u003e= 0.15 absolute versus inline_strict. Note: single_agent_control is a synthetic negative baseline (always FAIL by design) excluded from H1 comparisons.","metric":"pass_rate","secondary_metrics":["run_duration_seconds","output_validity_rate"],"guardrail_metrics":[{"name":"empty_output_rate","direction":"min","threshold":"0.01"},{"name":"false_pass_rate","direction":"min","threshold":"0.05"}],"min_samples":30,"weight":[34,33,33],"issue":47551,"start_date":"2026-07-23","end_date":"2026-10-23"}}' + GH_AW_EXPERIMENT_SPEC: '{"sub_agent_strategy":{"variants":["inline_strict","delegated_sequential"],"description":"Measure whether inline sub-agent orchestration is the best reliability/cost tradeoff for a model-identity smoke test. Both variants run the mandatory sub-agent contract; the experiment outcome is scored separately and never relaxes the smoke post-step gate.","hypothesis":"H0: no change in the sub_agent_strategy_goal_met YES rate between inline_strict and delegated_sequential. H1: delegated_sequential improves it by \u003e= 0.15 absolute versus inline_strict.","metric":"eval:sub_agent_strategy_goal_met","secondary_metrics":["run_duration_seconds","output_validity_rate"],"guardrail_metrics":[{"name":"empty_output_rate","direction":"min","threshold":"0.01"},{"name":"false_pass_rate","direction":"min","threshold":"0.05"}],"min_samples":30,"weight":[50,50],"issue":47551,"start_date":"2026-07-23","end_date":"2026-10-23"}}' GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.jsonl GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments - GH_AW_HARNESS_VERSION: 77090e7016ef53fcfb0f9afb99510eb5429319cf88ea6274982fdaf291d71e52:62af0943eec9eb3c97faee594017b8b312f93be82bf72d2e575b15d9239140c3 + GH_AW_HARNESS_VERSION: 3958c29c5e2c8ed81e5255399d4f425ba9a5c2eccd5c8aeca3a7eac2aafc01e6:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463 with: script: | const path = require('path'); @@ -502,7 +502,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "77090e7016ef53fcfb0f9afb99510eb5429319cf88ea6274982fdaf291d71e52:62af0943eec9eb3c97faee594017b8b312f93be82bf72d2e575b15d9239140c3" + GH_AW_ADMISSION_WORKFLOW_HASH: "3958c29c5e2c8ed81e5255399d4f425ba9a5c2eccd5c8aeca3a7eac2aafc01e6:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} @@ -1294,12 +1294,11 @@ jobs: fi - env: SMOKE_EXECUTION: ${{ steps.agentic_execution.outcome }} - SMOKE_VARIANT: ${{ needs.activation.outputs.sub_agent_strategy }} if: always() name: Assert sub-agent model and wire API evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"copilot\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\")\n .split(\"\\n\").filter(line => line.trim()).map(line => JSON.parse(line));\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst started = events.filter(event => event.type === \"subagent.started\");\nif (process.env.SMOKE_VARIANT === \"single_agent_control\") {\n assert.equal(started.length, 0, \"Control variant must not invoke sub-agents\");\n return;\n}\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Copilot SDK execution failed\");\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst requests = readJsonl(proxyFile);\n// [agent, configured model, expected model.resolution status, endpoint]\nconst expected = [\n [\"haiku-whoami\", \"claude-haiku-4.5\", \"exact\", \"/chat/completions\"],\n [\"gpt5-whoami\", \"gpt-5\", \"resolved\", \"/responses\"],\n [\"terra-whoami\", \"gpt-5.6-terra\", \"exact\", \"/responses\"],\n];\nconst servedModel = model => String(model || \"\")\n .replace(/^copilot(?:-(?:responses|completions))?(?:-\\d+)?\\//, \"\")\n .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, \"\")\n .replace(/^(claude-.+)-([0-9]+)-([0-9]+)$/, \"$1-$2.$3\");\nconst servedModels = new Set();\nfor (const [agentName, configuredModel, status, endpoint] of expected) {\n const resolutions = events.filter(event => event.type === \"model.resolution\" && event.data?.engine === \"copilot\" && event.data?.role === agentName);\n assert.equal(resolutions.length, 1, `${agentName} must have exactly one model.resolution record`);\n const resolution = resolutions[0].data;\n assert.equal(resolution.configured_model, configuredModel, `${agentName} model.resolution is for ${resolution.configured_model}, expected ${configuredModel}`);\n assert.equal(resolution.status, status, `${agentName} model.resolution status is ${resolution.status}, expected ${status}`);\n const model = servedModel(resolution.resolved_model);\n assert.ok(model, `${agentName} has no resolved model`);\n assert.ok(!servedModels.has(model), `${agentName} resolved to ${model}, which another sub-agent also uses, so its proxy requests are indistinguishable`);\n servedModels.add(model);\n if (status === \"exact\") assert.equal(model, configuredModel, `${agentName} resolved to ${model}, expected ${configuredModel}`);\n const invocations = started.filter(event => event.data.agentName === agentName);\n assert.equal(invocations.length, 1, `${agentName} must actually run exactly once`);\n const invocation = invocations[0];\n const invocationId = invocation.data.invocationId || invocation.agentId;\n assert.ok(invocationId, `${agentName} is missing an invocation ID`);\n const activity = events.filter(event => event.data?.invocationId === invocationId || event.agentId === invocationId);\n assert.ok(activity.some(event => event.type === \"subagent.completed\"), `${agentName} did not complete`);\n assert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${agentName} failed`);\n const models = activity.flatMap(event => [\n event.data.model, event.data.selectedModel, event.data.resolvedModel, event.data.firstDispatchedModel,\n ]).filter(Boolean);\n assert.ok(models.length > 0, `${agentName} has no recorded model`);\n assert.ok(models.every(value => servedModel(value) === model), `${agentName} changed or fell back from ${model}`);\n assert.ok(requests.some(request =>\n servedModel(request.model) === model &&\n (request.path || request.endpoint) === endpoint &&\n Number(request.status) === 200 &&\n request.purpose !== \"routing_classification\"),\n `${agentName} has no successful ${model} request on ${endpoint}`);\n}" + script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"copilot\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\")\n .split(\"\\n\").filter(line => line.trim()).map(line => JSON.parse(line));\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst started = events.filter(event => event.type === \"subagent.started\");\n// [agent, configured model, expected model.resolution status, endpoint]\nconst expected = [\n [\"haiku-whoami\", \"claude-haiku-4.5\", \"exact\", \"/chat/completions\"],\n [\"gpt5-whoami\", \"gpt-5\", \"resolved\", \"/responses\"],\n [\"terra-whoami\", \"gpt-5.6-terra\", \"exact\", \"/responses\"],\n];\nconst servedModel = model => String(model || \"\")\n .replace(/^copilot(?:-(?:responses|completions))?(?:-\\d+)?\\//, \"\")\n .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, \"\")\n .replace(/^(claude-.+)-([0-9]+)-([0-9]+)$/, \"$1-$2.$3\");\n// Check model resolution for every sub-agent first so a startup failure reports its cause.\nconst servedModels = new Map();\nfor (const [agentName, configuredModel, status] of expected) {\n const resolutions = events.filter(event => event.type === \"model.resolution\" && event.data?.engine === \"copilot\" && event.data?.role === agentName);\n assert.equal(resolutions.length, 1, `${agentName} must have exactly one model.resolution record`);\n const resolution = resolutions[0].data;\n assert.equal(resolution.configured_model, configuredModel, `${agentName} model.resolution is for ${resolution.configured_model}, expected ${configuredModel}`);\n assert.equal(resolution.status, status, `${agentName} model.resolution status is ${resolution.status}, expected ${status} (cause: ${resolution.cause || \"none\"})`);\n const model = servedModel(resolution.resolved_model);\n assert.ok(model, `${agentName} has no resolved model`);\n assert.ok(![...servedModels.values()].includes(model), `${agentName} resolved to ${model}, which another sub-agent also uses, so its proxy requests are indistinguishable`);\n servedModels.set(agentName, model);\n if (status === \"exact\") assert.equal(model, configuredModel, `${agentName} resolved to ${model}, expected ${configuredModel}`);\n}\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Copilot SDK execution failed\");\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst requests = readJsonl(proxyFile);\nfor (const [agentName, , , endpoint] of expected) {\n const model = servedModels.get(agentName);\n const invocations = started.filter(event => event.data.agentName === agentName);\n assert.equal(invocations.length, 1, `${agentName} must actually run exactly once`);\n const invocation = invocations[0];\n const invocationId = invocation.data.invocationId || invocation.agentId;\n assert.ok(invocationId, `${agentName} is missing an invocation ID`);\n const activity = events.filter(event => event.data?.invocationId === invocationId || event.agentId === invocationId);\n assert.ok(activity.some(event => event.type === \"subagent.completed\"), `${agentName} did not complete`);\n assert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${agentName} failed`);\n const models = activity.flatMap(event => [\n event.data.model, event.data.selectedModel, event.data.resolvedModel, event.data.firstDispatchedModel,\n ]).filter(Boolean);\n assert.ok(models.length > 0, `${agentName} has no recorded model`);\n assert.ok(models.every(value => servedModel(value) === model), `${agentName} changed or fell back from ${model}`);\n assert.ok(requests.some(request =>\n servedModel(request.model) === model &&\n (request.path || request.endpoint) === endpoint &&\n Number(request.status) === 200 &&\n request.purpose !== \"routing_classification\"),\n `${agentName} has no successful ${model} request on ${endpoint}`);\n}" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact @@ -1801,7 +1800,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "77090e7016ef53fcfb0f9afb99510eb5429319cf88ea6274982fdaf291d71e52:62af0943eec9eb3c97faee594017b8b312f93be82bf72d2e575b15d9239140c3" + GH_AW_ADMISSION_WORKFLOW_HASH: "3958c29c5e2c8ed81e5255399d4f425ba9a5c2eccd5c8aeca3a7eac2aafc01e6:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} @@ -2303,7 +2302,7 @@ jobs: if: always() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the orchestration objective for experiment sub_agent_strategy was met, with all three sub-agents invoked exactly once and their exact identities validated using the assigned orchestration style?"}]' GH_AW_EVALS_MODEL: "gpt-5.3-codex" GH_AW_EVALS_PHASE: setup with: @@ -2465,7 +2464,7 @@ jobs: continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the orchestration objective for experiment sub_agent_strategy was met, with all three sub-agents invoked exactly once and their exact identities validated using the assigned orchestration style?"}]' GH_AW_EVALS_MODEL: "gpt-5.3-codex" GH_AW_EVALS_PHASE: parse GITHUB_RUN_ID: ${{ github.run_id }} diff --git a/.github/workflows/smoke-copilot-sub-agents.md b/.github/workflows/smoke-copilot-sub-agents.md index 5258068a704..d64c28e26c5 100644 --- a/.github/workflows/smoke-copilot-sub-agents.md +++ b/.github/workflows/smoke-copilot-sub-agents.md @@ -18,10 +18,10 @@ engine: max-tool-denials: 3 experiments: sub_agent_strategy: - variants: [inline_strict, delegated_sequential, single_agent_control] - description: "Measure whether inline sub-agent orchestration is the best reliability/cost tradeoff for a model-identity smoke test." - hypothesis: "H0: no change in pass_rate between inline_strict and delegated_sequential. H1: delegated_sequential improves pass_rate by >= 0.15 absolute versus inline_strict. Note: single_agent_control is a synthetic negative baseline (always FAIL by design) excluded from H1 comparisons." - metric: pass_rate + variants: [inline_strict, delegated_sequential] + description: "Measure whether inline sub-agent orchestration is the best reliability/cost tradeoff for a model-identity smoke test. Both variants run the mandatory sub-agent contract; the experiment outcome is scored separately and never relaxes the smoke post-step gate." + hypothesis: "H0: no change in the sub_agent_strategy_goal_met YES rate between inline_strict and delegated_sequential. H1: delegated_sequential improves it by >= 0.15 absolute versus inline_strict." + metric: eval:sub_agent_strategy_goal_met secondary_metrics: [run_duration_seconds, output_validity_rate] guardrail_metrics: - name: empty_output_rate @@ -31,7 +31,7 @@ experiments: direction: min threshold: 0.05 min_samples: 30 - weight: [34, 33, 33] + weight: [50, 50] start_date: "2026-07-23" end_date: "2026-10-23" issue: 47551 @@ -49,7 +49,7 @@ imports: - shared/reporting.md evals: - id: sub_agent_strategy_goal_met - question: Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed? + question: Does the agent output show that the orchestration objective for experiment sub_agent_strategy was met, with all three sub-agents invoked exactly once and their exact identities validated using the assigned orchestration style? sandbox: agent: @@ -59,7 +59,6 @@ post-steps: if: always() uses: actions/github-script@v9.0.0 env: - SMOKE_VARIANT: ${{ needs.activation.outputs.sub_agent_strategy }} SMOKE_EXECUTION: ${{ steps.agentic_execution.outcome }} with: script: | @@ -73,19 +72,6 @@ post-steps: .split("\n").filter(line => line.trim()).map(line => JSON.parse(line)); const events = readJsonl(path.join(root, "usage/aw_session.jsonl")); const started = events.filter(event => event.type === "subagent.started"); - if (process.env.SMOKE_VARIANT === "single_agent_control") { - assert.equal(started.length, 0, "Control variant must not invoke sub-agents"); - return; - } - assert.equal(process.env.SMOKE_EXECUTION, "success", "Copilot SDK execution failed"); - const proxyPaths = [ - "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl", - "sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl", - "sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl", - ]; - const proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file)); - assert.ok(proxyFile, "Missing api-proxy token usage evidence"); - const requests = readJsonl(proxyFile); // [agent, configured model, expected model.resolution status, endpoint] const expected = [ ["haiku-whoami", "claude-haiku-4.5", "exact", "/chat/completions"], @@ -96,18 +82,31 @@ post-steps: .replace(/^copilot(?:-(?:responses|completions))?(?:-\d+)?\//, "") .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, "") .replace(/^(claude-.+)-([0-9]+)-([0-9]+)$/, "$1-$2.$3"); - const servedModels = new Set(); - for (const [agentName, configuredModel, status, endpoint] of expected) { + // Check model resolution for every sub-agent first so a startup failure reports its cause. + const servedModels = new Map(); + for (const [agentName, configuredModel, status] of expected) { const resolutions = events.filter(event => event.type === "model.resolution" && event.data?.engine === "copilot" && event.data?.role === agentName); assert.equal(resolutions.length, 1, `${agentName} must have exactly one model.resolution record`); const resolution = resolutions[0].data; assert.equal(resolution.configured_model, configuredModel, `${agentName} model.resolution is for ${resolution.configured_model}, expected ${configuredModel}`); - assert.equal(resolution.status, status, `${agentName} model.resolution status is ${resolution.status}, expected ${status}`); + assert.equal(resolution.status, status, `${agentName} model.resolution status is ${resolution.status}, expected ${status} (cause: ${resolution.cause || "none"})`); const model = servedModel(resolution.resolved_model); assert.ok(model, `${agentName} has no resolved model`); - assert.ok(!servedModels.has(model), `${agentName} resolved to ${model}, which another sub-agent also uses, so its proxy requests are indistinguishable`); - servedModels.add(model); + assert.ok(![...servedModels.values()].includes(model), `${agentName} resolved to ${model}, which another sub-agent also uses, so its proxy requests are indistinguishable`); + servedModels.set(agentName, model); if (status === "exact") assert.equal(model, configuredModel, `${agentName} resolved to ${model}, expected ${configuredModel}`); + } + assert.equal(process.env.SMOKE_EXECUTION, "success", "Copilot SDK execution failed"); + const proxyPaths = [ + "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl", + "sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl", + "sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl", + ]; + const proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file)); + assert.ok(proxyFile, "Missing api-proxy token usage evidence"); + const requests = readJsonl(proxyFile); + for (const [agentName, , , endpoint] of expected) { + const model = servedModels.get(agentName); const invocations = started.filter(event => event.data.agentName === agentName); assert.equal(invocations.length, 1, `${agentName} must actually run exactly once`); const invocation = invocations[0]; @@ -136,39 +135,34 @@ post-steps: ## Tasks -{{#if experiments.sub_agent_strategy == 'single_agent_control' }} -1. Do not call any sub-agent. -2. Produce the issue in the same format, but mark each agent as `not_invoked`. -3. Set overall status to FAIL. -4. Do not use unnecessary tool calls. -{{/if}} -{{#if experiments.sub_agent_strategy == 'delegated_sequential' }} -1. Call `haiku-whoami`, `gpt5-whoami`, and `terra-whoami` exactly once each. -2. Ask each sub-agent exactly this question: `who am i?` +These steps are mandatory in every run. Reporting an expected identity without actually executing the sub-agent through a `task` tool call does not satisfy this test. + +1. **Required first step:** your first action in this run must be a `task` tool call. Call `haiku-whoami` with exactly this prompt: `who am i?`. Do this before any other tool call, including skills and safe outputs. +2. Then make exactly one `task` tool call each to `gpt5-whoami` and `terra-whoami`, each with exactly this prompt: `who am i?`. Do not call `haiku-whoami` again. 3. Check the exact responses: - `haiku-whoami` → `claude-haiku-4.5` - `gpt5-whoami` → `gpt-5` - `terra-whoami` → `gpt-5.6-terra` -4. Execute and validate each sub-agent one at a time before moving to the next. -5. Do not use any other agent or any unnecessary tool calls. +4. Do not use any other agent or any unnecessary tool calls. + +{{#if experiments.sub_agent_strategy == 'delegated_sequential' }} +Orchestration: Execute and validate each sub-agent one at a time before moving to the next. {{/if}} {{#if experiments.sub_agent_strategy == 'inline_strict' }} -1. Call `haiku-whoami`, `gpt5-whoami`, and `terra-whoami` exactly once each. -2. Ask each sub-agent exactly this question: `who am i?` -3. Check the exact responses: - - `haiku-whoami` → `claude-haiku-4.5` - - `gpt5-whoami` → `gpt-5` - - `terra-whoami` → `gpt-5.6-terra` -4. Validate all three results and keep the current inline orchestration behavior. -5. Do not use any other agent or any unnecessary tool calls. +Orchestration: Validate all three results and keep the current inline orchestration behavior. {{/if}} ## Output Always create an issue titled **"Smoke Test: Copilot Sub Agents - ${{ github.run_id }}"** with: - One line per sub-agent showing expected value, actual value, and ✅/❌ -- Overall status: PASS only if all three exact matches succeed, otherwise FAIL -- Experiment variant: `{{ experiments.sub_agent_strategy }}` +- Overall status: PASS only if all three sub-agents actually ran exactly once and all three exact matches succeed, otherwise FAIL +{{#if experiments.sub_agent_strategy == 'delegated_sequential' }} +- Experiment variant: `delegated_sequential` (scored separately; it does not change the overall status) +{{/if}} +{{#if experiments.sub_agent_strategy == 'inline_strict' }} +- Experiment variant: `inline_strict` (scored separately; it does not change the overall status) +{{/if}} - Run URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ## agent: `haiku-whoami` diff --git a/actions/setup/js/interpolate_prompt.test.cjs b/actions/setup/js/interpolate_prompt.test.cjs index 0a8b25b91b4..69efcaa9035 100644 --- a/actions/setup/js/interpolate_prompt.test.cjs +++ b/actions/setup/js/interpolate_prompt.test.cjs @@ -255,7 +255,6 @@ describe("interpolate_prompt", () => { const { main } = require("./interpolate_prompt.cjs"); let tmpDir, promptPath, originalEnv; const smokeStrategies = [ - ["single_agent_control", "Do not call any sub-agent."], ["delegated_sequential", "Execute and validate each sub-agent one at a time before moving to the next."], ["inline_strict", "Validate all three results and keep the current inline orchestration behavior."], ]; @@ -311,7 +310,7 @@ describe("interpolate_prompt", () => { for (const [otherVariant, otherInstruction] of smokeStrategies) { if (otherVariant !== variant) expect(rendered).not.toContain(otherInstruction); } - expect(rendered.includes("Call `haiku-whoami`")).toBe(variant !== "single_agent_control"); + expect(rendered).toContain("Call `haiku-whoami`"); expect(rendered).not.toContain("{{"); }), it("should fail when GH_AW_PROMPT is not set", () => { diff --git a/actions/setup/js/smoke_copilot_sub_agents.test.cjs b/actions/setup/js/smoke_copilot_sub_agents.test.cjs index ed427ec3942..9a355d41e60 100644 --- a/actions/setup/js/smoke_copilot_sub_agents.test.cjs +++ b/actions/setup/js/smoke_copilot_sub_agents.test.cjs @@ -31,7 +31,7 @@ function evidence() { }; } -async function assertEvidence({ events, requests }, variant = "inline_strict") { +async function assertEvidence({ events, requests }, execution = "success") { const mockRequire = name => { if (name.endsWith("unified_session.cjs")) return { writeUnifiedSession() {} }; if (name === "node:fs") { @@ -43,7 +43,7 @@ async function assertEvidence({ events, requests }, variant = "inline_strict") { return require(name); }; await new AsyncFunction("require", "process", script)(mockRequire, { - env: { RUNNER_TEMP: "/tmp", SMOKE_VARIANT: variant, SMOKE_EXECUTION: "success" }, + env: { RUNNER_TEMP: "/tmp", SMOKE_EXECUTION: execution }, }); } @@ -121,8 +121,10 @@ describe("Copilot sub-agent smoke evidence", () => { await expect(assertEvidence(data)).rejects.toThrow("failed"); }); - test("requires no delegation in the negative control", async () => { - await expect(assertEvidence({ events: [], requests: [] }, "single_agent_control")).resolves.toBeUndefined(); - await expect(assertEvidence(evidence(), "single_agent_control")).rejects.toThrow("must not invoke"); + test("reports a dead child pin's model.resolution cause before the failed execution", async () => { + const data = evidence(); + data.events = data.events.filter(event => event.type === "model.resolution"); + Object.assign(data.events.find(event => event.data.role === "gpt5-whoami").data, { resolved_model: null, status: "failed", cause: "model_unavailable" }); + await expect(assertEvidence({ events: data.events, requests: [] }, "failure")).rejects.toThrow("gpt5-whoami model.resolution status is failed, expected resolved (cause: model_unavailable)"); }); }); From 2883a290580f9476892b3d2c216d953f40766555 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:36:55 +0000 Subject: [PATCH 3/5] Test per-variant Experiment variant lines in sub-agent smoke Output section Co-authored-by: SivaKesava1 <11771739+SivaKesava1@users.noreply.github.com> --- actions/setup/js/interpolate_prompt.test.cjs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/actions/setup/js/interpolate_prompt.test.cjs b/actions/setup/js/interpolate_prompt.test.cjs index 69efcaa9035..48e2f9d6f6e 100644 --- a/actions/setup/js/interpolate_prompt.test.cjs +++ b/actions/setup/js/interpolate_prompt.test.cjs @@ -313,6 +313,22 @@ describe("interpolate_prompt", () => { expect(rendered).toContain("Call `haiku-whoami`"); expect(rendered).not.toContain("{{"); }), + it.each(smokeStrategies.map(([variant]) => variant))("renders only the assigned %s experiment variant line in the sub-agent smoke output", async variant => { + const workflow = fs.readFileSync(path.join(__dirname, "../../../.github/workflows/smoke-copilot-sub-agents.md"), "utf8"); + const output = workflow.split("\n## Output\n")[1].split("\n## agent:")[0]; + // GitHub expressions are substituted before interpolate_prompt runs; replace them so only template blocks remain. + const outputWithoutExpressions = output.replace(/\$\{\{[^}]*\}\}/g, "EXPR"); + process.env.GH_AW_EXPERIMENTS_SUB_AGENT_STRATEGY = variant; + fs.writeFileSync(promptPath, outputWithoutExpressions); + + await main(); + + expect(core.setFailed).not.toHaveBeenCalled(); + const rendered = fs.readFileSync(promptPath, "utf8"); + const variantLines = rendered.split("\n").filter(line => line.includes("Experiment variant:")); + expect(variantLines).toEqual([`- Experiment variant: \`${variant}\` (scored separately; it does not change the overall status)`]); + expect(rendered).not.toContain("{{"); + }), it("should fail when GH_AW_PROMPT is not set", () => { delete process.env.GH_AW_PROMPT; const mainMatch = interpolatePromptScript.match(/async function main\(\)\s*{[\s\S]*?^}/m); From 7592c28d9abdd72b28f79dc60deba9e8b6748dd7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 22:47:29 +0000 Subject: [PATCH 4/5] Report execution failure before missing child resolutions, check dead pin cause first, and rename sub-agent eval ID Co-authored-by: SivaKesava1 <11771739+SivaKesava1@users.noreply.github.com> --- .../smoke-copilot-sub-agents.lock.yml | 16 ++++++------- .github/workflows/smoke-copilot-sub-agents.md | 16 +++++++++---- .../js/smoke_copilot_sub_agents.test.cjs | 23 +++++++++++++++++++ 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/.github/workflows/smoke-copilot-sub-agents.lock.yml b/.github/workflows/smoke-copilot-sub-agents.lock.yml index 43c2509f128..c94bf55cd15 100644 --- a/.github/workflows/smoke-copilot-sub-agents.lock.yml +++ b/.github/workflows/smoke-copilot-sub-agents.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"3958c29c5e2c8ed81e5255399d4f425ba9a5c2eccd5c8aeca3a7eac2aafc01e6","body_hash":"1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"510a49b80d0df8f8022d1eec687ddd378c4eef6119aea3f0378e9048e5dc1b24","body_hash":"1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70","digest":"sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70@sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70","digest":"sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70@sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70","digest":"sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70@sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.36","digest":"sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.36@sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382","pinned_image":"ghcr.io/github/gh-aw-node@sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","noop","report_incomplete","report_missing_data","report_missing_tool"]}],"threat_detection":{"mode":"enabled"},"admission":true} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -164,10 +164,10 @@ jobs: id: pick-experiment uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_EXPERIMENT_SPEC: '{"sub_agent_strategy":{"variants":["inline_strict","delegated_sequential"],"description":"Measure whether inline sub-agent orchestration is the best reliability/cost tradeoff for a model-identity smoke test. Both variants run the mandatory sub-agent contract; the experiment outcome is scored separately and never relaxes the smoke post-step gate.","hypothesis":"H0: no change in the sub_agent_strategy_goal_met YES rate between inline_strict and delegated_sequential. H1: delegated_sequential improves it by \u003e= 0.15 absolute versus inline_strict.","metric":"eval:sub_agent_strategy_goal_met","secondary_metrics":["run_duration_seconds","output_validity_rate"],"guardrail_metrics":[{"name":"empty_output_rate","direction":"min","threshold":"0.01"},{"name":"false_pass_rate","direction":"min","threshold":"0.05"}],"min_samples":30,"weight":[50,50],"issue":47551,"start_date":"2026-07-23","end_date":"2026-10-23"}}' + GH_AW_EXPERIMENT_SPEC: '{"sub_agent_strategy":{"variants":["inline_strict","delegated_sequential"],"description":"Measure whether inline sub-agent orchestration is the best reliability/cost tradeoff for a model-identity smoke test. Both variants run the mandatory sub-agent contract; the experiment outcome is scored separately and never relaxes the smoke post-step gate.","hypothesis":"H0: no change in the sub_agent_strategy_contract_met YES rate between inline_strict and delegated_sequential. H1: delegated_sequential improves it by \u003e= 0.15 absolute versus inline_strict.","metric":"eval:sub_agent_strategy_contract_met","secondary_metrics":["run_duration_seconds","output_validity_rate"],"guardrail_metrics":[{"name":"empty_output_rate","direction":"min","threshold":"0.01"},{"name":"false_pass_rate","direction":"min","threshold":"0.05"}],"min_samples":30,"weight":[50,50],"issue":47551,"start_date":"2026-07-23","end_date":"2026-10-23"}}' GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.jsonl GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments - GH_AW_HARNESS_VERSION: 3958c29c5e2c8ed81e5255399d4f425ba9a5c2eccd5c8aeca3a7eac2aafc01e6:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463 + GH_AW_HARNESS_VERSION: 510a49b80d0df8f8022d1eec687ddd378c4eef6119aea3f0378e9048e5dc1b24:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463 with: script: | const path = require('path'); @@ -502,7 +502,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "3958c29c5e2c8ed81e5255399d4f425ba9a5c2eccd5c8aeca3a7eac2aafc01e6:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463" + GH_AW_ADMISSION_WORKFLOW_HASH: "510a49b80d0df8f8022d1eec687ddd378c4eef6119aea3f0378e9048e5dc1b24:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} @@ -1298,7 +1298,7 @@ jobs: name: Assert sub-agent model and wire API evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"copilot\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\")\n .split(\"\\n\").filter(line => line.trim()).map(line => JSON.parse(line));\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst started = events.filter(event => event.type === \"subagent.started\");\n// [agent, configured model, expected model.resolution status, endpoint]\nconst expected = [\n [\"haiku-whoami\", \"claude-haiku-4.5\", \"exact\", \"/chat/completions\"],\n [\"gpt5-whoami\", \"gpt-5\", \"resolved\", \"/responses\"],\n [\"terra-whoami\", \"gpt-5.6-terra\", \"exact\", \"/responses\"],\n];\nconst servedModel = model => String(model || \"\")\n .replace(/^copilot(?:-(?:responses|completions))?(?:-\\d+)?\\//, \"\")\n .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, \"\")\n .replace(/^(claude-.+)-([0-9]+)-([0-9]+)$/, \"$1-$2.$3\");\n// Check model resolution for every sub-agent first so a startup failure reports its cause.\nconst servedModels = new Map();\nfor (const [agentName, configuredModel, status] of expected) {\n const resolutions = events.filter(event => event.type === \"model.resolution\" && event.data?.engine === \"copilot\" && event.data?.role === agentName);\n assert.equal(resolutions.length, 1, `${agentName} must have exactly one model.resolution record`);\n const resolution = resolutions[0].data;\n assert.equal(resolution.configured_model, configuredModel, `${agentName} model.resolution is for ${resolution.configured_model}, expected ${configuredModel}`);\n assert.equal(resolution.status, status, `${agentName} model.resolution status is ${resolution.status}, expected ${status} (cause: ${resolution.cause || \"none\"})`);\n const model = servedModel(resolution.resolved_model);\n assert.ok(model, `${agentName} has no resolved model`);\n assert.ok(![...servedModels.values()].includes(model), `${agentName} resolved to ${model}, which another sub-agent also uses, so its proxy requests are indistinguishable`);\n servedModels.set(agentName, model);\n if (status === \"exact\") assert.equal(model, configuredModel, `${agentName} resolved to ${model}, expected ${configuredModel}`);\n}\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Copilot SDK execution failed\");\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst requests = readJsonl(proxyFile);\nfor (const [agentName, , , endpoint] of expected) {\n const model = servedModels.get(agentName);\n const invocations = started.filter(event => event.data.agentName === agentName);\n assert.equal(invocations.length, 1, `${agentName} must actually run exactly once`);\n const invocation = invocations[0];\n const invocationId = invocation.data.invocationId || invocation.agentId;\n assert.ok(invocationId, `${agentName} is missing an invocation ID`);\n const activity = events.filter(event => event.data?.invocationId === invocationId || event.agentId === invocationId);\n assert.ok(activity.some(event => event.type === \"subagent.completed\"), `${agentName} did not complete`);\n assert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${agentName} failed`);\n const models = activity.flatMap(event => [\n event.data.model, event.data.selectedModel, event.data.resolvedModel, event.data.firstDispatchedModel,\n ]).filter(Boolean);\n assert.ok(models.length > 0, `${agentName} has no recorded model`);\n assert.ok(models.every(value => servedModel(value) === model), `${agentName} changed or fell back from ${model}`);\n assert.ok(requests.some(request =>\n servedModel(request.model) === model &&\n (request.path || request.endpoint) === endpoint &&\n Number(request.status) === 200 &&\n request.purpose !== \"routing_classification\"),\n `${agentName} has no successful ${model} request on ${endpoint}`);\n}" + script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"copilot\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\")\n .split(\"\\n\").filter(line => line.trim()).map(line => JSON.parse(line));\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst started = events.filter(event => event.type === \"subagent.started\");\n// [agent, configured model, expected model.resolution status, endpoint]\nconst expected = [\n [\"haiku-whoami\", \"claude-haiku-4.5\", \"exact\", \"/chat/completions\"],\n [\"gpt5-whoami\", \"gpt-5\", \"resolved\", \"/responses\"],\n [\"terra-whoami\", \"gpt-5.6-terra\", \"exact\", \"/responses\"],\n];\nconst servedModel = model => String(model || \"\")\n .replace(/^copilot(?:-(?:responses|completions))?(?:-\\d+)?\\//, \"\")\n .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, \"\")\n .replace(/^(claude-.+)-([0-9]+)-([0-9]+)$/, \"$1-$2.$3\");\nconst resolutionRecords = events.filter(event => event.type === \"model.resolution\" && event.data?.engine === \"copilot\");\nconst executionSucceeded = process.env.SMOKE_EXECUTION === \"success\";\n// Check model resolution for every sub-agent first so a startup failure reports its cause.\nconst servedModels = new Map();\nfor (const [agentName, configuredModel, status] of expected) {\n const resolutions = resolutionRecords.filter(event => event.data.role === agentName);\n if (resolutions.length === 0 && !executionSucceeded) {\n const causes = resolutionRecords.filter(event => event.data.cause).map(event => `${event.data.role}: ${event.data.cause}`);\n assert.fail(`Copilot SDK execution failed (outcome: ${process.env.SMOKE_EXECUTION || \"unknown\"}) before ${agentName} model.resolution was recorded; recorded resolution causes: ${causes.join(\", \") || \"none\"}`);\n }\n assert.equal(resolutions.length, 1, `${agentName} must have exactly one model.resolution record`);\n const resolution = resolutions[0].data;\n assert.equal(resolution.status, status, `${agentName} model.resolution status is ${resolution.status}, expected ${status} (cause: ${resolution.cause || \"none\"})`);\n assert.equal(resolution.configured_model, configuredModel, `${agentName} model.resolution is for ${resolution.configured_model}, expected ${configuredModel}`);\n const model = servedModel(resolution.resolved_model);\n assert.ok(model, `${agentName} has no resolved model`);\n assert.ok(![...servedModels.values()].includes(model), `${agentName} resolved to ${model}, which another sub-agent also uses, so its proxy requests are indistinguishable`);\n servedModels.set(agentName, model);\n if (status === \"exact\") assert.equal(model, configuredModel, `${agentName} resolved to ${model}, expected ${configuredModel}`);\n}\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Copilot SDK execution failed\");\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst requests = readJsonl(proxyFile);\nfor (const [agentName, , , endpoint] of expected) {\n const model = servedModels.get(agentName);\n const invocations = started.filter(event => event.data.agentName === agentName);\n assert.equal(invocations.length, 1, `${agentName} must actually run exactly once`);\n const invocation = invocations[0];\n const invocationId = invocation.data.invocationId || invocation.agentId;\n assert.ok(invocationId, `${agentName} is missing an invocation ID`);\n const activity = events.filter(event => event.data?.invocationId === invocationId || event.agentId === invocationId);\n assert.ok(activity.some(event => event.type === \"subagent.completed\"), `${agentName} did not complete`);\n assert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${agentName} failed`);\n const models = activity.flatMap(event => [\n event.data.model, event.data.selectedModel, event.data.resolvedModel, event.data.firstDispatchedModel,\n ]).filter(Boolean);\n assert.ok(models.length > 0, `${agentName} has no recorded model`);\n assert.ok(models.every(value => servedModel(value) === model), `${agentName} changed or fell back from ${model}`);\n assert.ok(requests.some(request =>\n servedModel(request.model) === model &&\n (request.path || request.endpoint) === endpoint &&\n Number(request.status) === 200 &&\n request.purpose !== \"routing_classification\"),\n `${agentName} has no successful ${model} request on ${endpoint}`);\n}" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact @@ -1800,7 +1800,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "3958c29c5e2c8ed81e5255399d4f425ba9a5c2eccd5c8aeca3a7eac2aafc01e6:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463" + GH_AW_ADMISSION_WORKFLOW_HASH: "510a49b80d0df8f8022d1eec687ddd378c4eef6119aea3f0378e9048e5dc1b24:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} @@ -2302,7 +2302,7 @@ jobs: if: always() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the orchestration objective for experiment sub_agent_strategy was met, with all three sub-agents invoked exactly once and their exact identities validated using the assigned orchestration style?"}]' + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_contract_met","question":"Does the agent output show that the orchestration objective for experiment sub_agent_strategy was met, with all three sub-agents invoked exactly once and their exact identities validated using the assigned orchestration style?"}]' GH_AW_EVALS_MODEL: "gpt-5.3-codex" GH_AW_EVALS_PHASE: setup with: @@ -2464,7 +2464,7 @@ jobs: continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the orchestration objective for experiment sub_agent_strategy was met, with all three sub-agents invoked exactly once and their exact identities validated using the assigned orchestration style?"}]' + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_contract_met","question":"Does the agent output show that the orchestration objective for experiment sub_agent_strategy was met, with all three sub-agents invoked exactly once and their exact identities validated using the assigned orchestration style?"}]' GH_AW_EVALS_MODEL: "gpt-5.3-codex" GH_AW_EVALS_PHASE: parse GITHUB_RUN_ID: ${{ github.run_id }} diff --git a/.github/workflows/smoke-copilot-sub-agents.md b/.github/workflows/smoke-copilot-sub-agents.md index d64c28e26c5..8c67ce353e7 100644 --- a/.github/workflows/smoke-copilot-sub-agents.md +++ b/.github/workflows/smoke-copilot-sub-agents.md @@ -20,8 +20,8 @@ experiments: sub_agent_strategy: variants: [inline_strict, delegated_sequential] description: "Measure whether inline sub-agent orchestration is the best reliability/cost tradeoff for a model-identity smoke test. Both variants run the mandatory sub-agent contract; the experiment outcome is scored separately and never relaxes the smoke post-step gate." - hypothesis: "H0: no change in the sub_agent_strategy_goal_met YES rate between inline_strict and delegated_sequential. H1: delegated_sequential improves it by >= 0.15 absolute versus inline_strict." - metric: eval:sub_agent_strategy_goal_met + hypothesis: "H0: no change in the sub_agent_strategy_contract_met YES rate between inline_strict and delegated_sequential. H1: delegated_sequential improves it by >= 0.15 absolute versus inline_strict." + metric: eval:sub_agent_strategy_contract_met secondary_metrics: [run_duration_seconds, output_validity_rate] guardrail_metrics: - name: empty_output_rate @@ -48,7 +48,7 @@ features: imports: - shared/reporting.md evals: - - id: sub_agent_strategy_goal_met + - id: sub_agent_strategy_contract_met question: Does the agent output show that the orchestration objective for experiment sub_agent_strategy was met, with all three sub-agents invoked exactly once and their exact identities validated using the assigned orchestration style? sandbox: @@ -82,14 +82,20 @@ post-steps: .replace(/^copilot(?:-(?:responses|completions))?(?:-\d+)?\//, "") .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, "") .replace(/^(claude-.+)-([0-9]+)-([0-9]+)$/, "$1-$2.$3"); + const resolutionRecords = events.filter(event => event.type === "model.resolution" && event.data?.engine === "copilot"); + const executionSucceeded = process.env.SMOKE_EXECUTION === "success"; // Check model resolution for every sub-agent first so a startup failure reports its cause. const servedModels = new Map(); for (const [agentName, configuredModel, status] of expected) { - const resolutions = events.filter(event => event.type === "model.resolution" && event.data?.engine === "copilot" && event.data?.role === agentName); + const resolutions = resolutionRecords.filter(event => event.data.role === agentName); + if (resolutions.length === 0 && !executionSucceeded) { + const causes = resolutionRecords.filter(event => event.data.cause).map(event => `${event.data.role}: ${event.data.cause}`); + assert.fail(`Copilot SDK execution failed (outcome: ${process.env.SMOKE_EXECUTION || "unknown"}) before ${agentName} model.resolution was recorded; recorded resolution causes: ${causes.join(", ") || "none"}`); + } assert.equal(resolutions.length, 1, `${agentName} must have exactly one model.resolution record`); const resolution = resolutions[0].data; - assert.equal(resolution.configured_model, configuredModel, `${agentName} model.resolution is for ${resolution.configured_model}, expected ${configuredModel}`); assert.equal(resolution.status, status, `${agentName} model.resolution status is ${resolution.status}, expected ${status} (cause: ${resolution.cause || "none"})`); + assert.equal(resolution.configured_model, configuredModel, `${agentName} model.resolution is for ${resolution.configured_model}, expected ${configuredModel}`); const model = servedModel(resolution.resolved_model); assert.ok(model, `${agentName} has no resolved model`); assert.ok(![...servedModels.values()].includes(model), `${agentName} resolved to ${model}, which another sub-agent also uses, so its proxy requests are indistinguishable`); diff --git a/actions/setup/js/smoke_copilot_sub_agents.test.cjs b/actions/setup/js/smoke_copilot_sub_agents.test.cjs index 9a355d41e60..bc243e51fb2 100644 --- a/actions/setup/js/smoke_copilot_sub_agents.test.cjs +++ b/actions/setup/js/smoke_copilot_sub_agents.test.cjs @@ -127,4 +127,27 @@ describe("Copilot sub-agent smoke evidence", () => { Object.assign(data.events.find(event => event.data.role === "gpt5-whoami").data, { resolved_model: null, status: "failed", cause: "model_unavailable" }); await expect(assertEvidence({ events: data.events, requests: [] }, "failure")).rejects.toThrow("gpt5-whoami model.resolution status is failed, expected resolved (cause: model_unavailable)"); }); + + test("reports a changed dead child pin's model.resolution cause before the pin mismatch", async () => { + const data = evidence(); + data.events = data.events.filter(event => event.type === "model.resolution"); + Object.assign(data.events.find(event => event.data.role === "terra-whoami").data, { configured_model: "dead-child-pin", resolved_model: null, status: "failed", cause: "model_unavailable" }); + await expect(assertEvidence({ events: data.events, requests: [] }, "failure")).rejects.toThrow("terra-whoami model.resolution status is failed, expected exact (cause: model_unavailable)"); + }); + + test("still rejects a changed child pin that resolves", async () => { + const data = evidence(); + data.events.find(event => event.type === "model.resolution" && event.data.role === "gpt5-whoami").data.configured_model = "gpt-5-mini"; + await expect(assertEvidence(data)).rejects.toThrow("gpt5-whoami model.resolution is for gpt-5-mini, expected gpt-5"); + }); + + test("reports a failed main-model resolution instead of missing child records", async () => { + const events = [{ type: "model.resolution", data: { engine: "copilot", role: "main", configured_model: "gpt-5.3-codex", resolved_model: null, provider: "copilot", status: "failed", cause: "model_unavailable" } }]; + const result = assertEvidence({ events, requests: [] }, "failure"); + await expect(result).rejects.toThrow("Copilot SDK execution failed (outcome: failure) before haiku-whoami model.resolution was recorded; recorded resolution causes: main: model_unavailable"); + }); + + test("reports skipped execution with an empty session instead of missing child records", async () => { + await expect(assertEvidence({ events: [], requests: [] }, "skipped")).rejects.toThrow("Copilot SDK execution failed (outcome: skipped) before haiku-whoami model.resolution was recorded; recorded resolution causes: none"); + }); }); From 93c96a420b35366dffd6a2e00dc6947c0b3438a1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:37:37 +0000 Subject: [PATCH 5/5] Enforce sub-agent invocation contract and schedule Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- .../smoke-copilot-sub-agents.lock.yml | 12 +++++----- .github/workflows/smoke-copilot-sub-agents.md | 7 +++++- .../js/smoke_copilot_sub_agents.test.cjs | 22 ++++++++++++++++++- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/smoke-copilot-sub-agents.lock.yml b/.github/workflows/smoke-copilot-sub-agents.lock.yml index c94bf55cd15..fed96cd7df5 100644 --- a/.github/workflows/smoke-copilot-sub-agents.lock.yml +++ b/.github/workflows/smoke-copilot-sub-agents.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"510a49b80d0df8f8022d1eec687ddd378c4eef6119aea3f0378e9048e5dc1b24","body_hash":"1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"21f93428e20d2b559ae76934cdc98de7b82d6a24feef6777bc67b9ca973d561b","body_hash":"1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70","digest":"sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70@sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70","digest":"sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70@sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70","digest":"sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70@sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.36","digest":"sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.36@sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382","pinned_image":"ghcr.io/github/gh-aw-node@sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","noop","report_incomplete","report_missing_data","report_missing_tool"]}],"threat_detection":{"mode":"enabled"},"admission":true} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -164,10 +164,10 @@ jobs: id: pick-experiment uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_EXPERIMENT_SPEC: '{"sub_agent_strategy":{"variants":["inline_strict","delegated_sequential"],"description":"Measure whether inline sub-agent orchestration is the best reliability/cost tradeoff for a model-identity smoke test. Both variants run the mandatory sub-agent contract; the experiment outcome is scored separately and never relaxes the smoke post-step gate.","hypothesis":"H0: no change in the sub_agent_strategy_contract_met YES rate between inline_strict and delegated_sequential. H1: delegated_sequential improves it by \u003e= 0.15 absolute versus inline_strict.","metric":"eval:sub_agent_strategy_contract_met","secondary_metrics":["run_duration_seconds","output_validity_rate"],"guardrail_metrics":[{"name":"empty_output_rate","direction":"min","threshold":"0.01"},{"name":"false_pass_rate","direction":"min","threshold":"0.05"}],"min_samples":30,"weight":[50,50],"issue":47551,"start_date":"2026-07-23","end_date":"2026-10-23"}}' + GH_AW_EXPERIMENT_SPEC: '{"sub_agent_strategy":{"variants":["inline_strict","delegated_sequential"],"description":"Measure whether inline sub-agent orchestration is the best reliability/cost tradeoff for a model-identity smoke test. Both variants run the mandatory sub-agent contract; the experiment outcome is scored separately and never relaxes the smoke post-step gate.","hypothesis":"H0: no change in the sub_agent_strategy_contract_met YES rate between inline_strict and delegated_sequential. H1: delegated_sequential improves it by \u003e= 0.15 absolute versus inline_strict.","metric":"eval:sub_agent_strategy_contract_met","secondary_metrics":["run_duration_seconds","output_validity_rate"],"guardrail_metrics":[{"name":"empty_output_rate","direction":"min","threshold":"0.01"},{"name":"false_pass_rate","direction":"min","threshold":"0.05"}],"min_samples":30,"weight":[50,50],"issue":47551,"start_date":"2026-07-23","end_date":"2027-03-01"}}' GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.jsonl GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments - GH_AW_HARNESS_VERSION: 510a49b80d0df8f8022d1eec687ddd378c4eef6119aea3f0378e9048e5dc1b24:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463 + GH_AW_HARNESS_VERSION: 21f93428e20d2b559ae76934cdc98de7b82d6a24feef6777bc67b9ca973d561b:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463 with: script: | const path = require('path'); @@ -502,7 +502,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "510a49b80d0df8f8022d1eec687ddd378c4eef6119aea3f0378e9048e5dc1b24:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463" + GH_AW_ADMISSION_WORKFLOW_HASH: "21f93428e20d2b559ae76934cdc98de7b82d6a24feef6777bc67b9ca973d561b:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} @@ -1298,7 +1298,7 @@ jobs: name: Assert sub-agent model and wire API evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"copilot\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\")\n .split(\"\\n\").filter(line => line.trim()).map(line => JSON.parse(line));\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst started = events.filter(event => event.type === \"subagent.started\");\n// [agent, configured model, expected model.resolution status, endpoint]\nconst expected = [\n [\"haiku-whoami\", \"claude-haiku-4.5\", \"exact\", \"/chat/completions\"],\n [\"gpt5-whoami\", \"gpt-5\", \"resolved\", \"/responses\"],\n [\"terra-whoami\", \"gpt-5.6-terra\", \"exact\", \"/responses\"],\n];\nconst servedModel = model => String(model || \"\")\n .replace(/^copilot(?:-(?:responses|completions))?(?:-\\d+)?\\//, \"\")\n .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, \"\")\n .replace(/^(claude-.+)-([0-9]+)-([0-9]+)$/, \"$1-$2.$3\");\nconst resolutionRecords = events.filter(event => event.type === \"model.resolution\" && event.data?.engine === \"copilot\");\nconst executionSucceeded = process.env.SMOKE_EXECUTION === \"success\";\n// Check model resolution for every sub-agent first so a startup failure reports its cause.\nconst servedModels = new Map();\nfor (const [agentName, configuredModel, status] of expected) {\n const resolutions = resolutionRecords.filter(event => event.data.role === agentName);\n if (resolutions.length === 0 && !executionSucceeded) {\n const causes = resolutionRecords.filter(event => event.data.cause).map(event => `${event.data.role}: ${event.data.cause}`);\n assert.fail(`Copilot SDK execution failed (outcome: ${process.env.SMOKE_EXECUTION || \"unknown\"}) before ${agentName} model.resolution was recorded; recorded resolution causes: ${causes.join(\", \") || \"none\"}`);\n }\n assert.equal(resolutions.length, 1, `${agentName} must have exactly one model.resolution record`);\n const resolution = resolutions[0].data;\n assert.equal(resolution.status, status, `${agentName} model.resolution status is ${resolution.status}, expected ${status} (cause: ${resolution.cause || \"none\"})`);\n assert.equal(resolution.configured_model, configuredModel, `${agentName} model.resolution is for ${resolution.configured_model}, expected ${configuredModel}`);\n const model = servedModel(resolution.resolved_model);\n assert.ok(model, `${agentName} has no resolved model`);\n assert.ok(![...servedModels.values()].includes(model), `${agentName} resolved to ${model}, which another sub-agent also uses, so its proxy requests are indistinguishable`);\n servedModels.set(agentName, model);\n if (status === \"exact\") assert.equal(model, configuredModel, `${agentName} resolved to ${model}, expected ${configuredModel}`);\n}\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Copilot SDK execution failed\");\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst requests = readJsonl(proxyFile);\nfor (const [agentName, , , endpoint] of expected) {\n const model = servedModels.get(agentName);\n const invocations = started.filter(event => event.data.agentName === agentName);\n assert.equal(invocations.length, 1, `${agentName} must actually run exactly once`);\n const invocation = invocations[0];\n const invocationId = invocation.data.invocationId || invocation.agentId;\n assert.ok(invocationId, `${agentName} is missing an invocation ID`);\n const activity = events.filter(event => event.data?.invocationId === invocationId || event.agentId === invocationId);\n assert.ok(activity.some(event => event.type === \"subagent.completed\"), `${agentName} did not complete`);\n assert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${agentName} failed`);\n const models = activity.flatMap(event => [\n event.data.model, event.data.selectedModel, event.data.resolvedModel, event.data.firstDispatchedModel,\n ]).filter(Boolean);\n assert.ok(models.length > 0, `${agentName} has no recorded model`);\n assert.ok(models.every(value => servedModel(value) === model), `${agentName} changed or fell back from ${model}`);\n assert.ok(requests.some(request =>\n servedModel(request.model) === model &&\n (request.path || request.endpoint) === endpoint &&\n Number(request.status) === 200 &&\n request.purpose !== \"routing_classification\"),\n `${agentName} has no successful ${model} request on ${endpoint}`);\n}" + script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"copilot\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\")\n .split(\"\\n\").filter(line => line.trim()).map(line => JSON.parse(line));\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst started = events.filter(event => event.type === \"subagent.started\");\n// [agent, configured model, expected model.resolution status, endpoint]\nconst expected = [\n [\"haiku-whoami\", \"claude-haiku-4.5\", \"exact\", \"/chat/completions\"],\n [\"gpt5-whoami\", \"gpt-5\", \"resolved\", \"/responses\"],\n [\"terra-whoami\", \"gpt-5.6-terra\", \"exact\", \"/responses\"],\n];\nconst expectedNames = new Set(expected.map(([agentName]) => agentName));\nconst servedModel = model => String(model || \"\")\n .replace(/^copilot(?:-(?:responses|completions))?(?:-\\d+)?\\//, \"\")\n .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, \"\")\n .replace(/^(claude-.+)-([0-9]+)-([0-9]+)$/, \"$1-$2.$3\");\nconst resolutionRecords = events.filter(event => event.type === \"model.resolution\" && event.data?.engine === \"copilot\");\nconst executionSucceeded = process.env.SMOKE_EXECUTION === \"success\";\n// Check model resolution for every sub-agent first so a startup failure reports its cause.\nconst servedModels = new Map();\nfor (const [agentName, configuredModel, status] of expected) {\n const resolutions = resolutionRecords.filter(event => event.data.role === agentName);\n if (resolutions.length === 0 && !executionSucceeded) {\n const causes = resolutionRecords.filter(event => event.data.cause).map(event => `${event.data.role}: ${event.data.cause}`);\n assert.fail(`Copilot SDK execution failed (outcome: ${process.env.SMOKE_EXECUTION || \"unknown\"}) before ${agentName} model.resolution was recorded; recorded resolution causes: ${causes.join(\", \") || \"none\"}`);\n }\n assert.equal(resolutions.length, 1, `${agentName} must have exactly one model.resolution record`);\n const resolution = resolutions[0].data;\n assert.equal(resolution.status, status, `${agentName} model.resolution status is ${resolution.status}, expected ${status} (cause: ${resolution.cause || \"none\"})`);\n assert.equal(resolution.configured_model, configuredModel, `${agentName} model.resolution is for ${resolution.configured_model}, expected ${configuredModel}`);\n const model = servedModel(resolution.resolved_model);\n assert.ok(model, `${agentName} has no resolved model`);\n assert.ok(![...servedModels.values()].includes(model), `${agentName} resolved to ${model}, which another sub-agent also uses, so its proxy requests are indistinguishable`);\n servedModels.set(agentName, model);\n if (status === \"exact\") assert.equal(model, configuredModel, `${agentName} resolved to ${model}, expected ${configuredModel}`);\n}\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Copilot SDK execution failed\");\nconst unexpected = [...new Set(started.map(event => event.data?.agentName).filter(agentName => !expectedNames.has(agentName)))];\nassert.equal(unexpected.length, 0, `Unexpected sub-agent invocation(s): ${unexpected.join(\", \")}`);\nassert.equal(started.length, expected.length, `Expected exactly ${expected.length} sub-agent invocations, found ${started.length}`);\nassert.equal(started[0]?.data.agentName, expected[0][0], `${expected[0][0]} must be the first sub-agent invoked`);\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst requests = readJsonl(proxyFile);\nfor (const [agentName, , , endpoint] of expected) {\n const model = servedModels.get(agentName);\n const invocations = started.filter(event => event.data.agentName === agentName);\n assert.equal(invocations.length, 1, `${agentName} must actually run exactly once`);\n const invocation = invocations[0];\n const invocationId = invocation.data.invocationId || invocation.agentId;\n assert.ok(invocationId, `${agentName} is missing an invocation ID`);\n const activity = events.filter(event => event.data?.invocationId === invocationId || event.agentId === invocationId);\n assert.ok(activity.some(event => event.type === \"subagent.completed\"), `${agentName} did not complete`);\n assert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${agentName} failed`);\n const models = activity.flatMap(event => [\n event.data.model, event.data.selectedModel, event.data.resolvedModel, event.data.firstDispatchedModel,\n ]).filter(Boolean);\n assert.ok(models.length > 0, `${agentName} has no recorded model`);\n assert.ok(models.every(value => servedModel(value) === model), `${agentName} changed or fell back from ${model}`);\n assert.ok(requests.some(request =>\n servedModel(request.model) === model &&\n (request.path || request.endpoint) === endpoint &&\n Number(request.status) === 200 &&\n request.purpose !== \"routing_classification\"),\n `${agentName} has no successful ${model} request on ${endpoint}`);\n}" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact @@ -1800,7 +1800,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "510a49b80d0df8f8022d1eec687ddd378c4eef6119aea3f0378e9048e5dc1b24:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463" + GH_AW_ADMISSION_WORKFLOW_HASH: "21f93428e20d2b559ae76934cdc98de7b82d6a24feef6777bc67b9ca973d561b:1f6e6ee71bb7af701b79d0f682a1c6288cef6a19ece7e4e4924e61cbe09ed463" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} diff --git a/.github/workflows/smoke-copilot-sub-agents.md b/.github/workflows/smoke-copilot-sub-agents.md index 8c67ce353e7..afc5bc7d9cc 100644 --- a/.github/workflows/smoke-copilot-sub-agents.md +++ b/.github/workflows/smoke-copilot-sub-agents.md @@ -33,7 +33,7 @@ experiments: min_samples: 30 weight: [50, 50] start_date: "2026-07-23" - end_date: "2026-10-23" + end_date: "2027-03-01" issue: 47551 safe-outputs: create-issue: @@ -78,6 +78,7 @@ post-steps: ["gpt5-whoami", "gpt-5", "resolved", "/responses"], ["terra-whoami", "gpt-5.6-terra", "exact", "/responses"], ]; + const expectedNames = new Set(expected.map(([agentName]) => agentName)); const servedModel = model => String(model || "") .replace(/^copilot(?:-(?:responses|completions))?(?:-\d+)?\//, "") .replace(/-[0-9]{4}(?:-[0-9]{2}-[0-9]{2}|[0-9]{4})$/, "") @@ -103,6 +104,10 @@ post-steps: if (status === "exact") assert.equal(model, configuredModel, `${agentName} resolved to ${model}, expected ${configuredModel}`); } assert.equal(process.env.SMOKE_EXECUTION, "success", "Copilot SDK execution failed"); + const unexpected = [...new Set(started.map(event => event.data?.agentName).filter(agentName => !expectedNames.has(agentName)))]; + assert.equal(unexpected.length, 0, `Unexpected sub-agent invocation(s): ${unexpected.join(", ")}`); + assert.equal(started.length, expected.length, `Expected exactly ${expected.length} sub-agent invocations, found ${started.length}`); + assert.equal(started[0]?.data.agentName, expected[0][0], `${expected[0][0]} must be the first sub-agent invoked`); const proxyPaths = [ "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl", "sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl", diff --git a/actions/setup/js/smoke_copilot_sub_agents.test.cjs b/actions/setup/js/smoke_copilot_sub_agents.test.cjs index bc243e51fb2..8aece52c849 100644 --- a/actions/setup/js/smoke_copilot_sub_agents.test.cjs +++ b/actions/setup/js/smoke_copilot_sub_agents.test.cjs @@ -52,6 +52,26 @@ describe("Copilot sub-agent smoke evidence", () => { await expect(assertEvidence(evidence())).resolves.toBeUndefined(); }); + test("rejects an unexpected sub-agent invocation", async () => { + const data = evidence(); + data.events.push({ type: "subagent.started", data: { agentName: "unexpected-agent", invocationId: "unexpected-agent" } }); + await expect(assertEvidence(data)).rejects.toThrow("Unexpected sub-agent invocation(s): unexpected-agent"); + }); + + test("rejects extra invocations of an expected sub-agent", async () => { + const data = evidence(); + data.events.push({ type: "subagent.started", data: { agentName: "haiku-whoami", invocationId: "haiku-again" } }); + await expect(assertEvidence(data)).rejects.toThrow("Expected exactly 3 sub-agent invocations, found 4"); + }); + + test("requires haiku-whoami to be the first sub-agent invoked", async () => { + const data = evidence(); + const first = data.events.findIndex(event => event.type === "subagent.started" && event.data.agentName === "haiku-whoami"); + const second = data.events.findIndex(event => event.type === "subagent.started" && event.data.agentName === "gpt5-whoami"); + [data.events[first], data.events[second]] = [data.events[second], data.events[first]]; + await expect(assertEvidence(data)).rejects.toThrow("haiku-whoami must be the first sub-agent invoked"); + }); + test("accepts the served dated Claude model identity", async () => { const data = evidence(); data.events[1].data.model = "copilot-completions/claude-haiku-4-5-20251001"; @@ -60,7 +80,7 @@ describe("Copilot sub-agent smoke evidence", () => { test("rejects success without delegation", async () => { const data = evidence(); - await expect(assertEvidence({ events: data.events.filter(event => event.type === "model.resolution"), requests: data.requests })).rejects.toThrow("must actually run"); + await expect(assertEvidence({ events: data.events.filter(event => event.type === "model.resolution"), requests: data.requests })).rejects.toThrow("Expected exactly 3 sub-agent invocations, found 0"); }); test("rejects a sub-agent without a model.resolution record", async () => {