diff --git a/.github/workflows/smoke-claude-on-copilot.lock.yml b/.github/workflows/smoke-claude-on-copilot.lock.yml index 754c2c07863..d7cb6c44904 100644 --- a/.github/workflows/smoke-claude-on-copilot.lock.yml +++ b/.github/workflows/smoke-claude-on-copilot.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"4d78e724c8f96c4829993e82bb574d77a1429dc9309959611d5a4f2913124baf","body_hash":"117d2cf544866a20b639076117d11283d9ae2877dbc52622cf47b4d587d5d93f","strict":true,"agent_id":"claude","agent_model":"copilot/haiku","engine_versions":{"claude":"2.1.296"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"6441523ce0759f76c1b84f101cb2546b42efbcd6a33d9eb24ed1af3017244fe8","body_hash":"117d2cf544866a20b639076117d11283d9ae2877dbc52622cf47b4d587d5d93f","strict":true,"agent_id":"claude","agent_model":"copilot/haiku","engine_versions":{"claude":"2.1.296"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62","digest":"sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62","digest":"sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.62","digest":"sha256:b23bbe2c025f60ea9aacde67c078ea937b54b4039552bca072ae9610d9e1c294","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.62@sha256:b23bbe2c025f60ea9aacde67c078ea937b54b4039552bca072ae9610d9e1c294"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62","digest":"sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"safeoutputs","tools":["add_comment","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1240,6 +1240,12 @@ jobs: if [ ! -f /tmp/gh-aw/agent_output.json ]; then echo '{"items":[]}' > /tmp/gh-aw/agent_output.json fi + - if: always() + name: Assert unified-session contract + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: "const path = require(\"node:path\");\nconst { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait sessionContractMain({\n core,\n engine: \"claude\",\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [\"main\"],\n});" + # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact - name: Upload agent output fallback artifact diff --git a/.github/workflows/smoke-claude-on-copilot.md b/.github/workflows/smoke-claude-on-copilot.md index 7ec01943f88..2845aa63ec2 100644 --- a/.github/workflows/smoke-claude-on-copilot.md +++ b/.github/workflows/smoke-claude-on-copilot.md @@ -33,6 +33,21 @@ timeout-minutes: 10 sandbox: agent: id: awf +post-steps: + - name: Assert unified-session contract + if: always() + uses: actions/github-script@v9.0.0 + with: + script: | + const path = require("node:path"); + const { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, "gh-aw/actions/smoke_model_routing_assertions.cjs")); + await sessionContractMain({ + core, + engine: "claude", + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: ["main"], + }); --- # Smoke Test: Claude on GitHub Inference PR Summary diff --git a/.github/workflows/smoke-claude-routed.lock.yml b/.github/workflows/smoke-claude-routed.lock.yml index 613dbc84fd0..e904b941632 100644 --- a/.github/workflows/smoke-claude-routed.lock.yml +++ b/.github/workflows/smoke-claude-routed.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"46c78ee160b4953c98324c6a5aa6ea59a496b772b5d75e6f6f508b006c207bac","body_hash":"c805ccd25917cdfd375847fea1df4becfcf14fad183c794f7c0894ad01954180","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.296"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"b5178a6d4503bdae443538914e302f93f62c38504f9d6f106ed87aae119ab8fa","body_hash":"c805ccd25917cdfd375847fea1df4becfcf14fad183c794f7c0894ad01954180","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.296"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/router:0.28.62@sha256:fb222d01480a2e8eda09f59e9bb8fdda04b0940e24fd6868d233ad24125cf85d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1279,7 +1279,7 @@ jobs: name: Assert model-routing evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"claude\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"claude-sonnet-5\"],\n mainEndpoint: \"/v1/messages\",\n});" + script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"claude\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"claude-sonnet-5\"],\n mainEndpoint: \"/v1/messages\",\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [],\n});" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact diff --git a/.github/workflows/smoke-claude-routed.md b/.github/workflows/smoke-claude-routed.md index 68030a4ac69..fd6129d4669 100644 --- a/.github/workflows/smoke-claude-routed.md +++ b/.github/workflows/smoke-claude-routed.md @@ -66,6 +66,9 @@ post-steps: executionOutcome: process.env.SMOKE_EXECUTION, allowedModels: ["claude-sonnet-5"], mainEndpoint: "/v1/messages", + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: [], }); --- diff --git a/.github/workflows/smoke-claude.lock.yml b/.github/workflows/smoke-claude.lock.yml index fee256acbe8..84625e63d84 100644 --- a/.github/workflows/smoke-claude.lock.yml +++ b/.github/workflows/smoke-claude.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"54bbec355922db85485ca464f003a18c4dd44e8685048d39f46389071948ecd8","body_hash":"5f6944eddd4844652e8741847bc7c01061d0cedf3193fb809e6114eed875cd33","strict":true,"agent_id":"claude","agent_model":"claude-sonnet-4-6","engine_versions":{"claude":"2.1.296"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"e5c061c7fa142fbb9ad06761177ea29aa3c45b3b53499030f9fab8a1f8264aef","body_hash":"5f6944eddd4844652e8741847bc7c01061d0cedf3193fb809e6114eed875cd33","strict":true,"agent_id":"claude","agent_model":"claude-sonnet-4-6","engine_versions":{"claude":"2.1.296"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN","TAVILY_API_KEY"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"},{"repo":"docker/build-push-action","sha":"c3c9e263c25d99ce0380d002d59b67737d91b0dc","version":"v7.4.0"},{"repo":"docker/setup-buildx-action","sha":"f87e5991a6d7451dcb8d9637bfbc97413f497069","version":"v4.4.1"},{"repo":"github/codeql-action/upload-sarif","sha":"24c54180a607b1449ed407dd24f251e4e9147c8d","version":"v4.38.3"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62","digest":"sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62","digest":"sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.62","digest":"sha256:b23bbe2c025f60ea9aacde67c078ea937b54b4039552bca072ae9610d9e1c294","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.62@sha256:b23bbe2c025f60ea9aacde67c078ea937b54b4039552bca072ae9610d9e1c294"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62","digest":"sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"has_pull_request":true,"mcp_servers":[{"name":"agenticworkflows","tools":["*"]},{"name":"mcpscripts","tools":["go","make"]},{"name":"safeoutputs","tools":["add_comment","add_labels","add_reviewer","close_pull_request","create_check_run","create_code_scanning_alert","create_issue","create_pull_request_review_comment","missing_data","missing_tool","noop","post_slack_message","push_to_pull_request_branch","report_incomplete","resolve_pull_request_review_thread","submit_pull_request_review","update_pull_request"]},{"name":"tavily","tools":["*"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1994,6 +1994,11 @@ jobs: include-hidden-files: true path: /tmp/gh-aw/cache-memory retention-days: ${{ vars.GH_AW_DEFAULT_ARTIFACT_RETENTION_DAYS || '0' }} + - if: always() + name: Assert unified-session contract + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: "const path = require(\"node:path\");\nconst { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait sessionContractMain({\n core,\n engine: \"claude\",\n expectModelInventory: true,\n copilotInventory: false,\n resolutionRoles: [\"main\"],\n});" - if: always() name: Stop Playwright title test server run: | diff --git a/.github/workflows/smoke-claude.md b/.github/workflows/smoke-claude.md index ad264cf0129..b6f07f10372 100644 --- a/.github/workflows/smoke-claude.md +++ b/.github/workflows/smoke-claude.md @@ -151,6 +151,21 @@ safe-outputs: timeout-minutes: 10 features: gh-aw-detection: false +post-steps: + - name: Assert unified-session contract + if: always() + uses: actions/github-script@v9.0.0 + with: + script: | + const path = require("node:path"); + const { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, "gh-aw/actions/smoke_model_routing_assertions.cjs")); + await sessionContractMain({ + core, + engine: "claude", + expectModelInventory: true, + copilotInventory: false, + resolutionRoles: ["main"], + }); --- # Smoke Test: Claude Engine Validation. diff --git a/.github/workflows/smoke-codex.lock.yml b/.github/workflows/smoke-codex.lock.yml index d7704adc02f..3a5c52ebfef 100644 --- a/.github/workflows/smoke-codex.lock.yml +++ b/.github/workflows/smoke-codex.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"4d942d9f12d726a153fb4f9f22bdbf62c271aba0d456b9a9390e5c35028c644b","body_hash":"971a1bd89429bf149cdc42a60f675515a3de5ce46b0d52ac7a76ea85530c1302","strict":true,"agent_id":"codex","engine_versions":{"codex":"0.163.0"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"4eade3d14f422842dfab467fbf3f0212b89307d5da64d112368ae0d4143b4c4a","body_hash":"971a1bd89429bf149cdc42a60f675515a3de5ce46b0d52ac7a76ea85530c1302","strict":true,"agent_id":"codex","engine_versions":{"codex":"0.163.0"}} # gh-aw-manifest: {"version":1,"secrets":["CODEX_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions-ecosystem/action-add-labels","sha":"18f1af5e3544586314bbe15c0273249c770b2daf","version":"v1.1.3"},{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62","digest":"sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62","digest":"sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62","digest":"sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"},{"image":"ghcr.io/oraios/serena:1.7.0","digest":"sha256:6c9459e4246a39c9deaa4f23fb05a526ac6e237b24c8e84a927a098fa1ab6730","pinned_image":"ghcr.io/oraios/serena:1.7.0@sha256:6c9459e4246a39c9deaa4f23fb05a526ac6e237b24c8e84a927a098fa1ab6730"}],"has_pull_request":true,"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","list_branches","list_commits","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","add_smoked_label","create_issue","hide_comment","missing_data","missing_tool","noop","remove_labels","report_incomplete","set_issue_field","unassign_from_user"]},{"name":"serena","tools":["*"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1605,6 +1605,11 @@ jobs: include-hidden-files: true path: /tmp/gh-aw/cache-memory retention-days: ${{ vars.GH_AW_DEFAULT_ARTIFACT_RETENTION_DAYS || '0' }} + - if: always() + name: Assert unified-session contract + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: "const path = require(\"node:path\");\nconst { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait sessionContractMain({\n core,\n engine: \"codex\",\n expectModelInventory: true,\n copilotInventory: false,\n resolutionRoles: [],\n});" - if: always() name: Stop Playwright title test server run: | diff --git a/.github/workflows/smoke-codex.md b/.github/workflows/smoke-codex.md index 69d799b1497..c5d0b3a58b8 100644 --- a/.github/workflows/smoke-codex.md +++ b/.github/workflows/smoke-codex.md @@ -102,6 +102,21 @@ features: sandbox: agent: id: awf +post-steps: + - name: Assert unified-session contract + if: always() + uses: actions/github-script@v9.0.0 + with: + script: | + const path = require("node:path"); + const { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, "gh-aw/actions/smoke_model_routing_assertions.cjs")); + await sessionContractMain({ + core, + engine: "codex", + expectModelInventory: true, + copilotInventory: false, + resolutionRoles: [], + }); --- # Smoke Test: Codex Engine Validation diff --git a/.github/workflows/smoke-copilot-routed-claude.lock.yml b/.github/workflows/smoke-copilot-routed-claude.lock.yml index 1e424596b41..cd67b9ca82f 100644 --- a/.github/workflows/smoke-copilot-routed-claude.lock.yml +++ b/.github/workflows/smoke-copilot-routed-claude.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"9ca2af3258308d16aaf8c52b242962e9379a422a9d7c58d7009e32262b94afd9","body_hash":"d00ffd48c6aa8bd9e98a8076b0216a30fbda73df9d54948304b0bcde9a5510e6","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"107982e9bdad08337744ab45cc20fc582a060f3af2a4914a407a921bff90e15e","body_hash":"d00ffd48c6aa8bd9e98a8076b0216a30fbda73df9d54948304b0bcde9a5510e6","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/router:0.28.62@sha256:fb222d01480a2e8eda09f59e9bb8fdda04b0940e24fd6868d233ad24125cf85d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1259,7 +1259,7 @@ jobs: name: Assert model-routing evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"copilot\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"claude-sonnet-5\"],\n mainEndpoint: \"/chat/completions\",\n routingMode: \"auto\",\n});" + script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"copilot\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"claude-sonnet-5\"],\n mainEndpoint: \"/chat/completions\",\n routingMode: \"auto\",\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [],\n});" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact diff --git a/.github/workflows/smoke-copilot-routed-claude.md b/.github/workflows/smoke-copilot-routed-claude.md index f9bb5fd250c..50f20f4cff1 100644 --- a/.github/workflows/smoke-copilot-routed-claude.md +++ b/.github/workflows/smoke-copilot-routed-claude.md @@ -61,6 +61,9 @@ post-steps: allowedModels: ["claude-sonnet-5"], mainEndpoint: "/chat/completions", routingMode: "auto", + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: [], }); --- diff --git a/.github/workflows/smoke-copilot-routed-steering.lock.yml b/.github/workflows/smoke-copilot-routed-steering.lock.yml index 5c1f65b729e..e9709c5c389 100644 --- a/.github/workflows/smoke-copilot-routed-steering.lock.yml +++ b/.github/workflows/smoke-copilot-routed-steering.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0f6ce8a3fefd1d6e8e074275629ac3b3c8e1deecae628c820eaf7d21b52c7de5","body_hash":"0b103fe1ea3f69f66614c7f840937012fc7ab87747299e0bb4bb9ea62454ab62","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ecd69152a1fdc0431a1f886936aa0ac772715e16ff87ae42ee5a7d9036520850","body_hash":"0b103fe1ea3f69f66614c7f840937012fc7ab87747299e0bb4bb9ea62454ab62","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/router:0.28.62@sha256:fb222d01480a2e8eda09f59e9bb8fdda04b0940e24fd6868d233ad24125cf85d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1255,7 +1255,7 @@ jobs: name: Assert model-routing and steering evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"copilot\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"gpt-5.6-luna\"],\n steering: { maxAiCredits: 7 },\n});" + script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"copilot\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"gpt-5.6-luna\"],\n steering: { maxAiCredits: 7 },\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [],\n});" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact diff --git a/.github/workflows/smoke-copilot-routed-steering.md b/.github/workflows/smoke-copilot-routed-steering.md index eb187697b61..3b95fc1b8e0 100644 --- a/.github/workflows/smoke-copilot-routed-steering.md +++ b/.github/workflows/smoke-copilot-routed-steering.md @@ -63,6 +63,9 @@ post-steps: executionOutcome: process.env.SMOKE_EXECUTION, allowedModels: ["gpt-5.6-luna"], steering: { maxAiCredits: 7 }, + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: [], }); --- diff --git a/.github/workflows/smoke-copilot-routed.lock.yml b/.github/workflows/smoke-copilot-routed.lock.yml index dd0f1a6a2fe..09c43d391d9 100644 --- a/.github/workflows/smoke-copilot-routed.lock.yml +++ b/.github/workflows/smoke-copilot-routed.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"cd6ef3e77213bad92265474ddd287131bd05a794ed4d37f2b18d0358f2a7f17f","body_hash":"ededd1cf82d2194c865500236c5f4d14b1233f5423bed6539ec6ecc8cb041b9d","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"9b95405aedba88acc11027e69f2cea70768139df2e50e6b6722d6bcc37a06947","body_hash":"ededd1cf82d2194c865500236c5f4d14b1233f5423bed6539ec6ecc8cb041b9d","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"3a2844b7e9c422d3c10d287c895573f7108da1b3"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/router:0.28.62@sha256:fb222d01480a2e8eda09f59e9bb8fdda04b0940e24fd6868d233ad24125cf85d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1241,7 +1241,7 @@ jobs: name: Assert model-routing evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"copilot\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"gpt-5.4-mini\", \"gpt-5.6-luna\", \"claude-haiku-4.5\"],\n});" + script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"copilot\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"gpt-5.4-mini\", \"gpt-5.6-luna\", \"claude-haiku-4.5\"],\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [],\n});" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact diff --git a/.github/workflows/smoke-copilot-routed.md b/.github/workflows/smoke-copilot-routed.md index e02f0fc92ee..634b19009a1 100644 --- a/.github/workflows/smoke-copilot-routed.md +++ b/.github/workflows/smoke-copilot-routed.md @@ -78,6 +78,9 @@ post-steps: engine: "copilot", executionOutcome: process.env.SMOKE_EXECUTION, allowedModels: ["gpt-5.4-mini", "gpt-5.6-luna", "claude-haiku-4.5"], + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: [], }); --- diff --git a/.github/workflows/smoke-copilot-sdk-routed-claude.lock.yml b/.github/workflows/smoke-copilot-sdk-routed-claude.lock.yml index c2ccee7167a..df2d5306f8b 100644 --- a/.github/workflows/smoke-copilot-sdk-routed-claude.lock.yml +++ b/.github/workflows/smoke-copilot-sdk-routed-claude.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"63192b4bac4378491a55949c6ada9e4f51fe6c4116b9b70f92fbeebc54dfb3b5","body_hash":"7914b56316f14e79b056f97cb5491665c9c5515745d37803de77b1c23143be82","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a570e8e01a5f7557ac9eb86a9cbc9993b7c5f20d275221a7be5bc3af6b5de40a","body_hash":"7914b56316f14e79b056f97cb5491665c9c5515745d37803de77b1c23143be82","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/router:0.28.62@sha256:fb222d01480a2e8eda09f59e9bb8fdda04b0940e24fd6868d233ad24125cf85d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1251,7 +1251,7 @@ jobs: name: Assert model-routing and sub-agent evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"copilot\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"claude-haiku-4.5\"],\n mainEndpoint: \"/chat/completions\",\n subAgents: [{ name: \"mini-whoami\", model: \"gpt-5.4-mini\", endpoint: \"/responses\" }],\n requireDeclaredAgentNames: true,\n});" + script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"copilot\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"claude-haiku-4.5\"],\n mainEndpoint: \"/chat/completions\",\n subAgents: [{ name: \"mini-whoami\", model: \"gpt-5.4-mini\", endpoint: \"/responses\" }],\n requireDeclaredAgentNames: true,\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [],\n});" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact diff --git a/.github/workflows/smoke-copilot-sdk-routed-claude.md b/.github/workflows/smoke-copilot-sdk-routed-claude.md index c0911a8f58d..5e58e489c66 100644 --- a/.github/workflows/smoke-copilot-sdk-routed-claude.md +++ b/.github/workflows/smoke-copilot-sdk-routed-claude.md @@ -60,6 +60,9 @@ post-steps: mainEndpoint: "/chat/completions", subAgents: [{ name: "mini-whoami", model: "gpt-5.4-mini", endpoint: "/responses" }], requireDeclaredAgentNames: true, + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: [], }); --- diff --git a/.github/workflows/smoke-copilot-sdk-routed.lock.yml b/.github/workflows/smoke-copilot-sdk-routed.lock.yml index bb5df57246f..718e7d15085 100644 --- a/.github/workflows/smoke-copilot-sdk-routed.lock.yml +++ b/.github/workflows/smoke-copilot-sdk-routed.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8f2e395dc5c2e9a0e2180d1e5a9978d64bb280fa517fe475c7d62a211ed9f9ae","body_hash":"8564571141aa0d50ad2609fd55e4d59578c99f5441fd2200546e11209344c79e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"98c791677f399f6817a12c767bea7af39f88ac097296b621abea3a303825e9be","body_hash":"8564571141aa0d50ad2609fd55e4d59578c99f5441fd2200546e11209344c79e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/router:0.28.62@sha256:fb222d01480a2e8eda09f59e9bb8fdda04b0940e24fd6868d233ad24125cf85d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1233,7 +1233,7 @@ jobs: name: Assert model-routing and sub-agent evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"copilot\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"gpt-5.6-luna\"],\n mainEndpoint: \"/responses\",\n subAgents: [{ name: \"haiku-whoami\", model: \"claude-haiku-4.5\", endpoint: \"/chat/completions\" }],\n requireDeclaredAgentNames: true,\n});" + script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"copilot\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"gpt-5.6-luna\"],\n mainEndpoint: \"/responses\",\n subAgents: [{ name: \"haiku-whoami\", model: \"claude-haiku-4.5\", endpoint: \"/chat/completions\" }],\n requireDeclaredAgentNames: true,\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [],\n});" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact diff --git a/.github/workflows/smoke-copilot-sdk-routed.md b/.github/workflows/smoke-copilot-sdk-routed.md index 656e302e945..1275b0995cb 100644 --- a/.github/workflows/smoke-copilot-sdk-routed.md +++ b/.github/workflows/smoke-copilot-sdk-routed.md @@ -63,6 +63,9 @@ post-steps: mainEndpoint: "/responses", subAgents: [{ name: "haiku-whoami", model: "claude-haiku-4.5", endpoint: "/chat/completions" }], requireDeclaredAgentNames: true, + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: [], }); --- diff --git a/.github/workflows/smoke-copilot-sdk.lock.yml b/.github/workflows/smoke-copilot-sdk.lock.yml index 3ab2b2246bd..32976697bf2 100644 --- a/.github/workflows/smoke-copilot-sdk.lock.yml +++ b/.github/workflows/smoke-copilot-sdk.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"e1000de2c4b8d0f6a5697d9980eb36dd1fe83fd8835b4a3251c4dbe994c3328b","body_hash":"82380b963821042b8628dd544778081dee749a55dbf32743ddeb6ff1c6e678ac","strict":true,"agent_id":"copilot","agent_model":"copilot/gpt-5","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"23f965f403ee993b81dd9eabd120486b57e717afc04dbbfd3e3da4724958e9ca","body_hash":"82380b963821042b8628dd544778081dee749a55dbf32743ddeb6ff1c6e678ac","strict":true,"agent_id":"copilot","agent_model":"copilot/gpt-5","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62","digest":"sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62","digest":"sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62","digest":"sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1260,6 +1260,12 @@ jobs: if [ ! -f /tmp/gh-aw/agent_output.json ]; then echo '{"items":[]}' > /tmp/gh-aw/agent_output.json fi + - if: always() + name: Assert unified-session contract + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: "const path = require(\"node:path\");\nconst { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait sessionContractMain({\n core,\n engine: \"copilot\",\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [\"main\"],\n});" + # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact - name: Upload agent output fallback artifact diff --git a/.github/workflows/smoke-copilot-sdk.md b/.github/workflows/smoke-copilot-sdk.md index e3e4f9fa90a..bdf11688b2f 100644 --- a/.github/workflows/smoke-copilot-sdk.md +++ b/.github/workflows/smoke-copilot-sdk.md @@ -45,6 +45,21 @@ features: sandbox: agent: id: awf +post-steps: + - name: Assert unified-session contract + if: always() + uses: actions/github-script@v9.0.0 + with: + script: | + const path = require("node:path"); + const { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, "gh-aw/actions/smoke_model_routing_assertions.cjs")); + await sessionContractMain({ + core, + engine: "copilot", + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: ["main"], + }); --- # Smoke Test: Copilot SDK Engine Validation diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index ed58f582633..244f9eb80c2 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"7494ba4dcc99ff8204f07cdb997d4a5663f0e891c3e968a529f6fe883df6220f","body_hash":"fc5aea3c76a9631fb266b3a70f2838dbdbfdc53300b0efaf0b03a593389258ea","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"9198495cab046a28283e739831528c072d4a6703d94e176262c1d34ee00fee64","body_hash":"fc5aea3c76a9631fb266b3a70f2838dbdbfdc53300b0efaf0b03a593389258ea","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","CODEX_API_KEY","COPILOT_GITHUB_TOKEN","GEMINI_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"},{"repo":"docker/build-push-action","sha":"c3c9e263c25d99ce0380d002d59b67737d91b0dc","version":"v7.4.0"},{"repo":"docker/setup-buildx-action","sha":"f87e5991a6d7451dcb8d9637bfbc97413f497069","version":"v4.4.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62","digest":"sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62","digest":"sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.62","digest":"sha256:b23bbe2c025f60ea9aacde67c078ea937b54b4039552bca072ae9610d9e1c294","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.62@sha256:b23bbe2c025f60ea9aacde67c078ea937b54b4039552bca072ae9610d9e1c294"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62","digest":"sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"},{"image":"ghcr.io/oraios/serena:1.7.0","digest":"sha256:6c9459e4246a39c9deaa4f23fb05a526ac6e237b24c8e84a927a098fa1ab6730","pinned_image":"ghcr.io/oraios/serena:1.7.0@sha256:6c9459e4246a39c9deaa4f23fb05a526ac6e237b24c8e84a927a098fa1ab6730"}],"mcp_servers":[{"name":"agenticworkflows","tools":["*"]},{"name":"mcpscripts","tools":["github-discussion-query","github-issue-query","github-pr-query"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_check_run","create_discussion","create_issue","create_pull_request_review_comment","dispatch_workflow","haiku_printer","missing_data","missing_tool","noop","remove_labels","reply_to_pull_request_review_comment","report_incomplete","send_slack_message","set_issue_type","submit_pull_request_review","upload_artifact"]},{"name":"serena","tools":["*"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -201,7 +201,7 @@ jobs: GH_AW_EXPERIMENT_SPEC: '{"caveman":{"variants":["yes","no"]},"subagent_model":{"variants":["small","large"]}}' GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.jsonl GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments - GH_AW_HARNESS_VERSION: 7494ba4dcc99ff8204f07cdb997d4a5663f0e891c3e968a529f6fe883df6220f:fc5aea3c76a9631fb266b3a70f2838dbdbfdc53300b0efaf0b03a593389258ea + GH_AW_HARNESS_VERSION: 9198495cab046a28283e739831528c072d4a6703d94e176262c1d34ee00fee64:fc5aea3c76a9631fb266b3a70f2838dbdbfdc53300b0efaf0b03a593389258ea with: script: | const path = require('path'); @@ -2579,6 +2579,11 @@ jobs: path: ${{ runner.temp }}/gh-aw/safeoutputs/upload-artifacts/ retention-days: ${{ vars.GH_AW_DEFAULT_ARTIFACT_RETENTION_DAYS || '1' }} if-no-files-found: ignore + - if: always() + name: Assert unified-session contract + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: "const path = require(\"node:path\");\nconst { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait sessionContractMain({\n core,\n engine: \"copilot\",\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [\"main\"],\n});" - if: always() name: Stop Playwright title test server run: | diff --git a/.github/workflows/smoke-copilot.md b/.github/workflows/smoke-copilot.md index ee023350676..67bf0f0c20a 100644 --- a/.github/workflows/smoke-copilot.md +++ b/.github/workflows/smoke-copilot.md @@ -181,6 +181,21 @@ evals: sandbox: agent: id: awf +post-steps: + - name: Assert unified-session contract + if: always() + uses: actions/github-script@v9.0.0 + with: + script: | + const path = require("node:path"); + const { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, "gh-aw/actions/smoke_model_routing_assertions.cjs")); + await sessionContractMain({ + core, + engine: "copilot", + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: ["main"], + }); --- # Smoke Test: Copilot Engine Validation diff --git a/.github/workflows/smoke-pi-routed-claude.lock.yml b/.github/workflows/smoke-pi-routed-claude.lock.yml index 37c348e8de5..f346c6ee68a 100644 --- a/.github/workflows/smoke-pi-routed-claude.lock.yml +++ b/.github/workflows/smoke-pi-routed-claude.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"61eee8bb61e4856ebc304c2f766adcaf582367f9859c67552757edf7d803df49","body_hash":"1f517db5c69952c5a3cf63f43c27f208e7c59ca51552b3c50ecc3d621a9573df","strict":true,"agent_id":"pi","engine_versions":{"pi":"1.3.0"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"237be814f0a8fabe4aefb0625da42de3ce094252a1d82f76f73870418a75606f","body_hash":"1f517db5c69952c5a3cf63f43c27f208e7c59ca51552b3c50ecc3d621a9573df","strict":true,"agent_id":"pi","engine_versions":{"pi":"1.3.0"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/router:0.28.62@sha256:fb222d01480a2e8eda09f59e9bb8fdda04b0940e24fd6868d233ad24125cf85d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1218,7 +1218,7 @@ jobs: name: Assert model-routing evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"pi\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"claude-sonnet-5\"],\n mainEndpoint: \"/v1/messages\",\n routingMode: \"robust\",\n});" + script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"pi\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"claude-sonnet-5\"],\n mainEndpoint: \"/v1/messages\",\n routingMode: \"robust\",\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [],\n});" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact diff --git a/.github/workflows/smoke-pi-routed-claude.md b/.github/workflows/smoke-pi-routed-claude.md index f00e05863a5..cb76bb102b3 100644 --- a/.github/workflows/smoke-pi-routed-claude.md +++ b/.github/workflows/smoke-pi-routed-claude.md @@ -63,6 +63,9 @@ post-steps: allowedModels: ["claude-sonnet-5"], mainEndpoint: "/v1/messages", routingMode: "robust", + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: [], }); --- diff --git a/.github/workflows/smoke-pi-routed.lock.yml b/.github/workflows/smoke-pi-routed.lock.yml index 779a3398613..b8ebaba35f9 100644 --- a/.github/workflows/smoke-pi-routed.lock.yml +++ b/.github/workflows/smoke-pi-routed.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"9b4105acb175035fecafb6df2d49d4f10c9723e1ae1a88142c8486d412f88525","body_hash":"96f2f06cedaa8ac0160459530432f9093efd2b3f90da1a51abe4f76377e8d3da","strict":true,"agent_id":"pi","engine_versions":{"pi":"1.3.0"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"cb847307274a5c5de82904b2dc78ebb7bc58435958b964ae58d860fb0ff00b67","body_hash":"96f2f06cedaa8ac0160459530432f9093efd2b3f90da1a51abe4f76377e8d3da","strict":true,"agent_id":"pi","engine_versions":{"pi":"1.3.0"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/router:0.28.62@sha256:fb222d01480a2e8eda09f59e9bb8fdda04b0940e24fd6868d233ad24125cf85d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1224,7 +1224,7 @@ jobs: name: Assert model-routing and sub-agent evidence uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"pi\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"gpt-5.6-luna\"],\n subAgents: [\n { name: \"haiku-whoami\", model: \"claude-haiku-4.5\", endpoint: \"/v1/messages\" },\n { name: \"mini-whoami\", model: \"gpt-5.4-mini\", endpoint: \"/responses\" },\n ],\n});" + script: "const path = require(\"node:path\");\nconst { main } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait main({\n core,\n engine: \"pi\",\n executionOutcome: process.env.SMOKE_EXECUTION,\n allowedModels: [\"gpt-5.6-luna\"],\n subAgents: [\n { name: \"haiku-whoami\", model: \"claude-haiku-4.5\", endpoint: \"/v1/messages\" },\n { name: \"mini-whoami\", model: \"gpt-5.4-mini\", endpoint: \"/responses\" },\n ],\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [\"haiku-whoami\", \"mini-whoami\"],\n});" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact diff --git a/.github/workflows/smoke-pi-routed.md b/.github/workflows/smoke-pi-routed.md index 2fb5f8ef769..7257f87e736 100644 --- a/.github/workflows/smoke-pi-routed.md +++ b/.github/workflows/smoke-pi-routed.md @@ -71,6 +71,9 @@ post-steps: { name: "haiku-whoami", model: "claude-haiku-4.5", endpoint: "/v1/messages" }, { name: "mini-whoami", model: "gpt-5.4-mini", endpoint: "/responses" }, ], + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: ["haiku-whoami", "mini-whoami"], }); --- diff --git a/.github/workflows/smoke-pi.lock.yml b/.github/workflows/smoke-pi.lock.yml index 543ae72ffe3..7201418a9af 100644 --- a/.github/workflows/smoke-pi.lock.yml +++ b/.github/workflows/smoke-pi.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ba51ed045d802290ef8e4ceae79bbb36a0f07bb64494e0fe22afd51a1c0ab731","body_hash":"a833fe6897d00dc3344cb9d4dc85c2fa37389b97ce03387ecd332d282d93a04a","strict":true,"agent_id":"pi","agent_model":"copilot/auto","engine_versions":{"pi":"1.3.0"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"67a1f666c73a4125d1edea9f10730bfc38f8d5ec397995d593cd4fd9055f6152","body_hash":"a833fe6897d00dc3344cb9d4dc85c2fa37389b97ce03387ecd332d282d93a04a","strict":true,"agent_id":"pi","agent_model":"copilot/auto","engine_versions":{"pi":"1.3.0"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62","digest":"sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62","digest":"sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.62","digest":"sha256:b23bbe2c025f60ea9aacde67c078ea937b54b4039552bca072ae9610d9e1c294","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.28.62@sha256:b23bbe2c025f60ea9aacde67c078ea937b54b4039552bca072ae9610d9e1c294"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62","digest":"sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"has_pull_request":true,"mcp_servers":[{"name":"safeoutputs","tools":["add_comment","add_labels","create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1304,6 +1304,12 @@ jobs: include-hidden-files: true path: /tmp/gh-aw/cache-memory retention-days: ${{ vars.GH_AW_DEFAULT_ARTIFACT_RETENTION_DAYS || '0' }} + - if: always() + name: Assert unified-session contract + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: "const path = require(\"node:path\");\nconst { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/smoke_model_routing_assertions.cjs\"));\nawait sessionContractMain({\n core,\n engine: \"pi\",\n expectModelInventory: true,\n copilotInventory: true,\n resolutionRoles: [\"main\"],\n});" + # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact - name: Upload agent output fallback artifact diff --git a/.github/workflows/smoke-pi.md b/.github/workflows/smoke-pi.md index 9fc530106d1..b124b27672e 100644 --- a/.github/workflows/smoke-pi.md +++ b/.github/workflows/smoke-pi.md @@ -77,6 +77,21 @@ safe-outputs: timeout-minutes: 10 features: gh-aw-detection: false +post-steps: + - name: Assert unified-session contract + if: always() + uses: actions/github-script@v9.0.0 + with: + script: | + const path = require("node:path"); + const { sessionContractMain } = require(path.join(process.env.RUNNER_TEMP, "gh-aw/actions/smoke_model_routing_assertions.cjs")); + await sessionContractMain({ + core, + engine: "pi", + expectModelInventory: true, + copilotInventory: true, + resolutionRoles: ["main"], + }); --- # Smoke Test: Pi Engine Validation diff --git a/DEVGUIDE.md b/DEVGUIDE.md index d863f7fa5c4..2ad1eaf821d 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -993,7 +993,9 @@ This documentation provides: - `smoke-copilot-routed-claude`: Copilot CLI with routing over a Claude-only pool in `mode: auto`; the main agent must use `/chat/completions` - `smoke-pi-routed-claude`: pi with routing over a Claude-only pool in `mode: robust`; the main agent must use `/v1/messages` - `smoke-copilot-sdk-routed-claude`: Copilot SDK with routing over `claude-haiku-4.5` (no reasoning effort) and a GPT sub-agent on `/responses` - - Each routed smoke's post-step (`actions/setup/js/smoke_model_routing_assertions.cjs`) checks runner- and proxy-written evidence, not the agent's reply. Every failed check logs its ID (`R1`–`R4`, `S1`–`S4`, `M1`, `P1`, `E1`, `E2`, and `T1`–`T4` for the steering smoke) with the observed model, endpoint, status, event or steering notice, so the step log is enough to diagnose a failure + - Each routed smoke's post-step (`actions/setup/js/smoke_model_routing_assertions.cjs`) checks runner- and proxy-written evidence, not the agent's reply. Every failed check logs its ID (`R1`–`R4`, `S1`–`S4`, `M1`, `P1`, `E1`, `E2`, `U1`–`U3`, and `T1`–`T4` for the steering smoke) with the observed model, endpoint, status, event or steering notice, so the step log is enough to diagnose a failure + - `U1`–`U3` check the unified session (`usage/aw_session.jsonl`) that audit, logs and footers read: `U1` no `session.collection_warning`; `U2` exactly one agent-phase `model.inventory` (with `copilotInventory`, a configured `copilot` endpoint with models); `U3` one successful `model.resolution` for each role in the smoke's explicit `resolutionRoles`. The unrouted engine smokes (`smoke-copilot`, `smoke-copilot-sdk`, `smoke-claude`, `smoke-claude-on-copilot`, `smoke-codex`, `smoke-pi`) run `E2` and `U1`–`U3` alone through `sessionContractMain` + - A routed smoke opts in to `U1`–`U3` by setting `expectModelInventory: true`, `copilotInventory: true` or a `resolutionRoles` array (`[]` requires no roles); `false` flags do not opt in. `U2` runs when `expectModelInventory` or `copilotInventory` is true. If neither is true, the log explicitly reports `U2 not evaluated`. With `expectModelInventory: true`, missing inventory evidence fails gracefully as `FAIL U2 ... observed 0`; it does not throw. A missing session instead fails `E2` and logs `U1–U3 not evaluated`. Unavailable evidence paths also fail `E2`. Repeated `U1` warnings are grouped by `(code, path)` with a count and an example line when available; every distinct pair remains visible - `label_command` removes the `smoke-routing` label when the runs start. To run them again, re-add the label, then approve workflows again ## Release Process diff --git a/actions/setup/js/fixtures/session-contract/smoke-copilot-routed-run-39176939097.jsonl b/actions/setup/js/fixtures/session-contract/smoke-copilot-routed-run-39176939097.jsonl new file mode 100644 index 00000000000..f796d02321c --- /dev/null +++ b/actions/setup/js/fixtures/session-contract/smoke-copilot-routed-run-39176939097.jsonl @@ -0,0 +1,2 @@ +{"type":"session.format","data":{"version":1},"provenance":{"component":"collector","phase":"conclusion","path":"usage/aw_session.jsonl","index":0}} +{"type":"model.inventory","data":{"endpoints":[{"provider":"openai","configured":false},{"provider":"anthropic","configured":false},{"provider":"copilot","configured":true,"models":["claude-haiku-4.5","claude-haiku-5.5","claude-opus-5.5","claude-sonnet-5","claude-sonnet-5.5","gpt-3.5-turbo","gpt-3.5-turbo-0613","gpt-4","gpt-4-0613","gpt-4-o-preview","gpt-4.1","gpt-4.1-2025-04-14","gpt-41-copilot","gpt-4o","gpt-4o-2024-05-13","gpt-4o-2024-08-06","gpt-4o-2024-11-20","gpt-4o-mini","gpt-4o-mini-2024-07-18","gpt-5-mini","gpt-5.3-codex","gpt-5.4","gpt-5.4-mini","gpt-5.5","gpt-5.6-luna","gpt-5.6-luna-utility","gpt-5.6-sol","gpt-5.6-terra","gpt-6-astra","gpt-6-luna","gpt-6-sol","gpt-6.1-sol","grok-4.5","grok-4.6","grok-4.7","mai-code-1.1-flash","text-embedding-3-small","text-embedding-3-small-inference","text-embedding-ada-002","trajectory-compaction"],"routingModels":[{"modelId":"claude-haiku-4.5","supportedEndpoints":["/chat/completions","/v1/messages"],"supportedReasoningEfforts":[],"contextWindowTokens":200000},{"modelId":"claude-haiku-5.5","supportedEndpoints":["/v1/messages","/chat/completions"],"supportedReasoningEfforts":["low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"claude-opus-5.5","supportedEndpoints":["/v1/messages","/chat/completions"],"supportedReasoningEfforts":["low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"claude-sonnet-5","supportedEndpoints":["/v1/messages","/chat/completions"],"supportedReasoningEfforts":["low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"claude-sonnet-5.5","supportedEndpoints":["/v1/messages","/chat/completions"],"supportedReasoningEfforts":["low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"gpt-3.5-turbo","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":16384},{"modelId":"gpt-3.5-turbo-0613","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":16384},{"modelId":"gpt-4","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":32768},{"modelId":"gpt-4-0613","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":32768},{"modelId":"gpt-4-o-preview","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4.1","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4.1-2025-04-14","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-41-copilot","supportedEndpoints":[],"supportedReasoningEfforts":[]},{"modelId":"gpt-4o","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4o-2024-05-13","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4o-2024-08-06","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4o-2024-11-20","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4o-mini","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4o-mini-2024-07-18","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-5-mini","supportedEndpoints":["/chat/completions","/responses","ws:/responses"],"supportedReasoningEfforts":["low","medium","high"],"contextWindowTokens":264000},{"modelId":"gpt-5.3-codex","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["low","medium","high","xhigh"],"contextWindowTokens":400000},{"modelId":"gpt-5.4","supportedEndpoints":["/responses","/chat/completions","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh"],"contextWindowTokens":1050000},{"modelId":"gpt-5.4-mini","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh"],"contextWindowTokens":400000},{"modelId":"gpt-5.5","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh"],"contextWindowTokens":1050000},{"modelId":"gpt-5.6-luna","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"gpt-5.6-luna-utility","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"gpt-5.6-sol","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"gpt-5.6-terra","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"gpt-6-astra","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"gpt-6-luna","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"gpt-6-sol","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"gpt-6.1-sol","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"grok-4.5","supportedEndpoints":["/responses"],"supportedReasoningEfforts":["low","medium","high"],"contextWindowTokens":500000},{"modelId":"grok-4.6","supportedEndpoints":["/responses"],"supportedReasoningEfforts":["low","medium","high","xhigh"],"contextWindowTokens":500000},{"modelId":"grok-4.7","supportedEndpoints":["/responses"],"supportedReasoningEfforts":["low","medium","high","xhigh"],"contextWindowTokens":500000},{"modelId":"mai-code-1.1-flash","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["low","medium","high"],"contextWindowTokens":256000},{"modelId":"text-embedding-3-small","supportedEndpoints":[],"supportedReasoningEfforts":[]},{"modelId":"text-embedding-3-small-inference","supportedEndpoints":[],"supportedReasoningEfforts":[]},{"modelId":"text-embedding-ada-002","supportedEndpoints":[],"supportedReasoningEfforts":[]},{"modelId":"trajectory-compaction","supportedEndpoints":["/chat/completions"],"supportedReasoningEfforts":[],"contextWindowTokens":262144}]},{"provider":"gemini","configured":false},{"provider":"vertex","configured":false}],"modelsFetchComplete":true},"provenance":{"component":"firewall","phase":"agent","path":"sandbox/firewall/awf-reflect.json","index":0}} diff --git a/actions/setup/js/fixtures/session-contract/smoke-copilot-run-39176941475.jsonl b/actions/setup/js/fixtures/session-contract/smoke-copilot-run-39176941475.jsonl new file mode 100644 index 00000000000..e6453562fb4 --- /dev/null +++ b/actions/setup/js/fixtures/session-contract/smoke-copilot-run-39176941475.jsonl @@ -0,0 +1,3 @@ +{"type":"session.format","data":{"version":1},"provenance":{"component":"collector","phase":"conclusion","path":"usage/aw_session.jsonl","index":0}} +{"type":"model.inventory","data":{"endpoints":[{"provider":"openai","configured":false},{"provider":"anthropic","configured":false},{"provider":"copilot","configured":true,"models":["claude-haiku-4.5","claude-haiku-5.5","claude-opus-5.5","claude-sonnet-5","claude-sonnet-5.5","gpt-3.5-turbo","gpt-3.5-turbo-0613","gpt-4","gpt-4-0613","gpt-4-o-preview","gpt-4.1","gpt-4.1-2025-04-14","gpt-41-copilot","gpt-4o","gpt-4o-2024-05-13","gpt-4o-2024-08-06","gpt-4o-2024-11-20","gpt-4o-mini","gpt-4o-mini-2024-07-18","gpt-5-mini","gpt-5.3-codex","gpt-5.4","gpt-5.4-mini","gpt-5.5","gpt-5.6-luna","gpt-5.6-luna-utility","gpt-5.6-sol","gpt-5.6-terra","gpt-6-astra","gpt-6-luna","gpt-6-sol","gpt-6.1-sol","grok-4.5","grok-4.6","grok-4.7","mai-code-1.1-flash","text-embedding-3-small","text-embedding-3-small-inference","text-embedding-ada-002","trajectory-compaction"],"routingModels":[{"modelId":"claude-haiku-4.5","supportedEndpoints":["/chat/completions","/v1/messages"],"supportedReasoningEfforts":[],"contextWindowTokens":200000},{"modelId":"claude-haiku-5.5","supportedEndpoints":["/v1/messages","/chat/completions"],"supportedReasoningEfforts":["low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"claude-opus-5.5","supportedEndpoints":["/v1/messages","/chat/completions"],"supportedReasoningEfforts":["low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"claude-sonnet-5","supportedEndpoints":["/v1/messages","/chat/completions"],"supportedReasoningEfforts":["low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"claude-sonnet-5.5","supportedEndpoints":["/v1/messages","/chat/completions"],"supportedReasoningEfforts":["low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"gpt-3.5-turbo","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":16384},{"modelId":"gpt-3.5-turbo-0613","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":16384},{"modelId":"gpt-4","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":32768},{"modelId":"gpt-4-0613","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":32768},{"modelId":"gpt-4-o-preview","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4.1","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4.1-2025-04-14","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-41-copilot","supportedEndpoints":[],"supportedReasoningEfforts":[]},{"modelId":"gpt-4o","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4o-2024-05-13","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4o-2024-08-06","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4o-2024-11-20","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4o-mini","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-4o-mini-2024-07-18","supportedEndpoints":[],"supportedReasoningEfforts":[],"contextWindowTokens":128000},{"modelId":"gpt-5-mini","supportedEndpoints":["/chat/completions","/responses","ws:/responses"],"supportedReasoningEfforts":["low","medium","high"],"contextWindowTokens":264000},{"modelId":"gpt-5.3-codex","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["low","medium","high","xhigh"],"contextWindowTokens":400000},{"modelId":"gpt-5.4","supportedEndpoints":["/responses","/chat/completions","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh"],"contextWindowTokens":1050000},{"modelId":"gpt-5.4-mini","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh"],"contextWindowTokens":400000},{"modelId":"gpt-5.5","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh"],"contextWindowTokens":1050000},{"modelId":"gpt-5.6-luna","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"gpt-5.6-luna-utility","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"gpt-5.6-sol","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"gpt-5.6-terra","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"gpt-6-astra","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"gpt-6-luna","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"gpt-6-sol","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1000000},{"modelId":"gpt-6.1-sol","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["none","low","medium","high","xhigh","max"],"contextWindowTokens":1050000},{"modelId":"grok-4.5","supportedEndpoints":["/responses"],"supportedReasoningEfforts":["low","medium","high"],"contextWindowTokens":500000},{"modelId":"grok-4.6","supportedEndpoints":["/responses"],"supportedReasoningEfforts":["low","medium","high","xhigh"],"contextWindowTokens":500000},{"modelId":"grok-4.7","supportedEndpoints":["/responses"],"supportedReasoningEfforts":["low","medium","high","xhigh"],"contextWindowTokens":500000},{"modelId":"mai-code-1.1-flash","supportedEndpoints":["/responses","ws:/responses"],"supportedReasoningEfforts":["low","medium","high"],"contextWindowTokens":256000},{"modelId":"text-embedding-3-small","supportedEndpoints":[],"supportedReasoningEfforts":[]},{"modelId":"text-embedding-3-small-inference","supportedEndpoints":[],"supportedReasoningEfforts":[]},{"modelId":"text-embedding-ada-002","supportedEndpoints":[],"supportedReasoningEfforts":[]},{"modelId":"trajectory-compaction","supportedEndpoints":["/chat/completions"],"supportedReasoningEfforts":[],"contextWindowTokens":262144}]},{"provider":"gemini","configured":false},{"provider":"vertex","configured":false}],"modelsFetchComplete":true},"provenance":{"component":"firewall","phase":"agent","path":"sandbox/firewall/awf-reflect.json","index":0}} +{"type":"model.resolution","data":{"engine":"copilot","role":"main","configured_model":"auto","resolved_model":"claude-sonnet-5.5","provider":"copilot","status":"resolved","cause":null},"provenance":{"component":"model_resolution","phase":"agent","path":"agent/model-resolution.json","index":0}} diff --git a/actions/setup/js/fixtures/session-contract/smoke-copilot-run-39181772988.jsonl b/actions/setup/js/fixtures/session-contract/smoke-copilot-run-39181772988.jsonl new file mode 100644 index 00000000000..24f555a9d7a --- /dev/null +++ b/actions/setup/js/fixtures/session-contract/smoke-copilot-run-39181772988.jsonl @@ -0,0 +1,3 @@ +{"type":"session.format","data":{"version":1},"provenance":{"component":"collector","phase":"conclusion","path":"usage/aw_session.jsonl","index":0}} +{"type":"model.resolution","data":{"engine":"copilot","role":"main","configured_model":"auto","resolved_model":"claude-sonnet-5.5","provider":"copilot","status":"resolved","cause":null},"provenance":{"component":"model_resolution","phase":"agent","path":"agent/model-resolution.json","index":0}} +{"type":"session.collection_warning","data":{"path":"sandbox/firewall/awf-reflect.json","code":"invalid_model_inventory"},"provenance":{"component":"collector","phase":"conclusion","path":"usage/aw_session.jsonl","index":1}} diff --git a/actions/setup/js/fixtures/session-contract/smoke-pi-routed-run-39178909557.jsonl b/actions/setup/js/fixtures/session-contract/smoke-pi-routed-run-39178909557.jsonl new file mode 100644 index 00000000000..a1c6588616a --- /dev/null +++ b/actions/setup/js/fixtures/session-contract/smoke-pi-routed-run-39178909557.jsonl @@ -0,0 +1,4 @@ +{"type":"session.format","data":{"version":1},"provenance":{"component":"collector","phase":"conclusion","path":"usage/aw_session.jsonl","index":0}} +{"type":"model.resolution","data":{"engine":"pi","role":"haiku-whoami","configured_model":"copilot/claude-haiku-4.5","resolved_model":"claude-haiku-4.5","provider":"copilot","status":"exact","cause":null},"provenance":{"component":"model_resolution","phase":"agent","path":"agent/model-resolution.json","index":0}} +{"type":"model.resolution","data":{"engine":"pi","role":"mini-whoami","configured_model":"copilot/gpt-5.4-mini","resolved_model":"gpt-5.4-mini","provider":"copilot","status":"exact","cause":null},"provenance":{"component":"model_resolution","phase":"agent","path":"agent/model-resolution.json","index":1}} +{"type":"session.collection_warning","data":{"path":"sandbox/firewall/awf-reflect.json","code":"invalid_model_inventory"},"provenance":{"component":"collector","phase":"conclusion","path":"usage/aw_session.jsonl","index":1}} diff --git a/actions/setup/js/fixtures/session_contract_sessions.cjs b/actions/setup/js/fixtures/session_contract_sessions.cjs new file mode 100644 index 00000000000..ac57d0bd6c8 --- /dev/null +++ b/actions/setup/js/fixtures/session_contract_sessions.cjs @@ -0,0 +1,50 @@ +// @ts-check +"use strict"; + +/** + * Unified-session fixtures for the U1–U3 session-contract smoke checks. + * + * Each file under `session-contract/` keeps, verbatim and in order, the + * `session.format`, `session.collection_warning`, `model.inventory` and + * `model.resolution` records of the published `usage/aw_session.jsonl` (usage + * artifact, conclusion job) of one run. All other records are dropped. + * + * - smoke-copilot-run-39181772988: Smoke Copilot on main before #70013, + * https://github.com/github/gh-aw/actions/runs/39181772988 — + * `invalid_model_inventory` for sandbox/firewall/awf-reflect.json, no + * `model.inventory`, main resolved `auto` -> `claude-sonnet-5.5`. + * - smoke-copilot-run-39176941475: Smoke Copilot after #70013, + * https://github.com/github/gh-aw/actions/runs/39176941475 — one agent-phase + * `model.inventory` (copilot configured, 40 models), main resolved. + * - smoke-copilot-routed-run-39176939097: Smoke Copilot Routed after #70013, + * https://github.com/github/gh-aw/actions/runs/39176939097 — one agent-phase + * `model.inventory`, no `model.resolution`. + * - smoke-pi-routed-run-39178909557: Smoke Pi Routed on main before #70013, + * https://github.com/github/gh-aw/actions/runs/39178909557 — exact + * `model.resolution` for haiku-whoami and mini-whoami, none for main, and + * `invalid_model_inventory` for sandbox/firewall/awf-reflect.json. + */ + +const path = require("path"); +const { readJsonlFile } = require("../jsonl_helpers.cjs"); + +const DIR = path.join(__dirname, "session-contract"); + +/** + * @param {string} name + * @returns {any[]} A fresh copy of the recorded session events. + */ +function load(name) { + return [...readJsonlFile(path.join(DIR, `${name}.jsonl`), { strict: true })]; +} + +module.exports = { + /** @returns {any[]} */ + preFixCopilotSession: () => load("smoke-copilot-run-39181772988"), + /** @returns {any[]} */ + postFixCopilotSession: () => load("smoke-copilot-run-39176941475"), + /** @returns {any[]} */ + postFixCopilotRoutedSession: () => load("smoke-copilot-routed-run-39176939097"), + /** @returns {any[]} */ + piRoutedSession: () => load("smoke-pi-routed-run-39178909557"), +}; diff --git a/actions/setup/js/smoke_model_routing_assertions.cjs b/actions/setup/js/smoke_model_routing_assertions.cjs index 5ad3b388c9d..edeada8ab42 100644 --- a/actions/setup/js/smoke_model_routing_assertions.cjs +++ b/actions/setup/js/smoke_model_routing_assertions.cjs @@ -48,6 +48,24 @@ const { iterateJsonlEvents } = require("./jsonl_helpers.cjs"); * `ai_credits_limit_exceeded` routing failure, and agent execution * succeeded; T4 every other check passed * + * - U1–U3 unified-session contract. `main` runs them only when the smoke opts + * in; `sessionContractMain` always runs E2 and U1–U3 alone for unrouted + * smokes. Session-contract fields: + * - `expectModelInventory: true` opts in and enables U2 + * - `copilotInventory: true` opts in, enables U2 and also requires a + * configured `copilot` endpoint with models + * - `resolutionRoles` (any array, including `[]`) opts in and lists the + * roles U3 requires; `[]` requires none + * `false` or omitted booleans and an omitted `resolutionRoles` do not opt + * in, so a smoke that sets only `false` values runs no U checks in `main`. + * U1 no `session.collection_warning` events; U2 exactly one + * agent-phase `model.inventory`, and with `copilotInventory` its `copilot` + * endpoint is configured with a non-empty model list (when + * `expectModelInventory` or `copilotInventory` is true; otherwise an info + * line reports that U2 was not evaluated); U3 every role in `resolutionRoles` has + * exactly one `model.resolution` with status exact or resolved and a + * resolved model. U1–U3 are not evaluated when the session file is missing. + * * Every check that iterates over requests fails when there are none to check. * * Threat-detection checks (`detectionMain`, run from @@ -78,11 +96,17 @@ const AI_CREDITS_HARD_CAP = 10000; const AI_CREDITS_LIMIT_CODE = "ai_credits_limit_exceeded"; const CREDIT_EPSILON = 1e-6; const ROUTING_PROFILES = ["economy", "balanced", "robust"]; +const RESOLVED_STATUSES = ["exact", "resolved"]; /** * @typedef {{ name: string, model: string, endpoint: string | string[] }} SubAgentExpectation * @typedef {{ maxAiCredits: number }} SteeringExpectation * @typedef {{ + * expectModelInventory?: boolean, + * copilotInventory?: boolean, + * resolutionRoles?: string[], + * }} SessionContractExpectations + * @typedef {SessionContractExpectations & { * allowedModels: string[], * mainEndpoint?: string, * routingMode?: string, @@ -555,6 +579,106 @@ function evaluateSteeringEvidence({ events, requests, routingRecords = [], expec return results; } +/** + * Whether the smoke opted in to the unified-session contract: a `true` + * inventory flag or an explicit `resolutionRoles` array. `false` does not opt in. + * @param {SessionContractExpectations} expectations + * @returns {boolean} + */ +function hasSessionContract(expectations) { + return expectations.expectModelInventory === true || expectations.copilotInventory === true || Array.isArray(expectations.resolutionRoles); +} + +/** @param {any} inventory @returns {string} */ +function describeInventoryEndpoints(inventory) { + const endpoints = Array.isArray(inventory?.data?.endpoints) ? inventory.data.endpoints : []; + if (!endpoints.length) return "none"; + return endpoints.map(endpoint => `${endpoint?.provider ?? ""} configured=${endpoint?.configured} models=${Array.isArray(endpoint?.models) ? endpoint.models.length : ""}`).join(", "); +} + +/** @param {any[]} resolutions @returns {string} */ +function describeResolutions(resolutions) { + if (!resolutions.length) return "none"; + return resolutions.map(event => `${event.data?.role ?? ""}: ${event.data?.configured_model ?? ""} -> ${event.data?.resolved_model || ""} (${event.data?.status ?? ""})`).join(", "); +} + +/** + * Evaluate the unified-session contract (U1–U3). Every expectation is explicit + * per smoke; nothing is inferred from sub-agent declarations. + * @param {{ events: any[], expectations: SessionContractExpectations }} input + * @returns {RoutingAssertionResults} + */ +function evaluateSessionContract({ events, expectations }) { + /** @type {RoutingAssertionResults} */ + const results = { passes: [], failures: [], info: [] }; + /** @param {string} id @param {string} message */ + const pass = (id, message) => results.passes.push(`PASS ${id} ${message}`); + /** @param {string} id @param {string} message */ + const fail = (id, message) => results.failures.push(`FAIL ${id} ${message}`); + + // U1: the collector reported no data it had to drop. + const warnings = events.filter(event => event?.type === "session.collection_warning"); + if (warnings.length) { + const groups = new Map(); + for (const event of warnings) { + const code = event.data?.code ?? ""; + const source = event.data?.path ?? ""; + const key = JSON.stringify([code, source]); + const group = groups.get(key) ?? { code, source, count: 0, line: undefined }; + group.count++; + group.line ??= event.data?.line; + groups.set(key, group); + } + const observed = [...groups.values()].map(group => `${group.code} in ${group.source} (${group.count} occurrence(s)${group.line === undefined ? "" : `, example line ${group.line}`})`); + fail("U1", `${warnings.length} session.collection_warning event(s): ${observed.join(", ")}`); + } else pass("U1", "no session.collection_warning events"); + + // U2: exactly one agent-phase model inventory (the collector can also add + // detection-phase inventories and one per candidate group). + if (expectations.expectModelInventory || expectations.copilotInventory) { + const inventories = events.filter(event => event?.type === "model.inventory"); + const agentInventories = inventories.filter(event => event.provenance?.phase === "agent"); + const observed = inventories.map(event => `${event.provenance?.phase ?? ""} ${event.provenance?.path ?? ""}`).join(", ") || "none"; + if (agentInventories.length !== 1) { + fail("U2", `expected exactly 1 agent-phase model.inventory, observed ${agentInventories.length}; model.inventory events: ${observed}`); + } else { + const inventory = agentInventories[0]; + const source = inventory.provenance?.path ?? ""; + const copilot = (Array.isArray(inventory.data?.endpoints) ? inventory.data.endpoints : []).find(endpoint => endpoint?.provider === "copilot"); + const copilotModels = Array.isArray(copilot?.models) ? copilot.models.length : 0; + if (expectations.copilotInventory && (copilot?.configured !== true || !copilotModels)) { + fail("U2", `agent-phase model.inventory from ${source} has no configured copilot endpoint with models; observed endpoints: ${describeInventoryEndpoints(inventory)}`); + } else { + pass("U2", `one agent-phase model.inventory from ${source}${expectations.copilotInventory ? `; copilot endpoint configured with ${copilotModels} model(s)` : ""}`); + } + } + } else (results.info ??= []).push("U2 not evaluated: neither expectModelInventory nor copilotInventory is true"); + + // U3: each required role recorded exactly one successful model resolution. + const roles = expectations.resolutionRoles ?? []; + const resolutions = events.filter(event => event?.type === "model.resolution"); + const u3Problems = []; + for (const role of roles) { + const forRole = resolutions.filter(event => event.data?.role === role); + if (!forRole.length) { + u3Problems.push(`no model.resolution for role ${role}`); + continue; + } + if (forRole.length > 1) { + u3Problems.push(`expected exactly 1 model.resolution for role ${role}, observed ${forRole.length}`); + continue; + } + const data = forRole[0].data ?? {}; + if (!RESOLVED_STATUSES.includes(data.status)) u3Problems.push(`role ${role} status is ${data.status ?? ""}${data.cause ? ` (cause ${data.cause})` : ""}, expected exact or resolved`); + else if (data.resolved_model === null || data.resolved_model === undefined || data.resolved_model === "") u3Problems.push(`role ${role} has no resolved_model`); + } + if (u3Problems.length) fail("U3", `${u3Problems.join("; ")}; observed model.resolution: ${describeResolutions(resolutions)}`); + else if (roles.length) pass("U3", `model.resolution recorded for ${describeResolutions(resolutions.filter(event => roles.includes(event.data?.role)))}`); + else pass("U3", `no model.resolution roles required; observed model.resolution: ${describeResolutions(resolutions)}`); + + return results; +} + /** * @param {string} file * @param {string[]} problems @@ -562,7 +686,14 @@ function evaluateSteeringEvidence({ events, requests, routingRecords = [], expec */ function readJsonl(file, problems) { const records = []; - for (const event of iterateJsonlEvents(fs.readFileSync(file, "utf8"))) { + let content; + try { + content = fs.readFileSync(file, "utf8"); + } catch (error) { + problems.push(`failed to read ${file}: ${getErrorMessage(error)}`); + return records; + } + for (const event of iterateJsonlEvents(content)) { if (event.type === "json") records.push(event.value); else if (event.reason !== "blank") problems.push(`${file}:${event.lineNumber} is not valid JSON: ${getErrorMessage(event.error)}`); } @@ -572,10 +703,11 @@ function readJsonl(file, problems) { /** * Collect evidence from the gh-aw temp directory. Post-steps run before the * usage-artifact step writes aw_session.jsonl, so the unified session is built first. - * @param {{ rootDir?: string, engine?: string, buildSession?: boolean, withRoutingRecords?: boolean }} [options] - * @returns {{ events: any[], requests: any[], routingRecords: any[], reflect: any, problems: string[] }} + * With `sessionOnly`, only the unified session is read. + * @param {{ rootDir?: string, engine?: string, buildSession?: boolean, withRoutingRecords?: boolean, sessionOnly?: boolean }} [options] + * @returns {{ events: any[], requests: any[], routingRecords: any[], reflect: any, problems: string[], sessionFound: boolean }} */ -function collectModelRoutingEvidence({ rootDir = DEFAULT_ROOT_DIR, engine, buildSession = true, withRoutingRecords = false } = {}) { +function collectModelRoutingEvidence({ rootDir = DEFAULT_ROOT_DIR, engine, buildSession = true, withRoutingRecords = false, sessionOnly = false } = {}) { /** @type {string[]} */ const problems = []; if (buildSession) { @@ -589,8 +721,10 @@ function collectModelRoutingEvidence({ rootDir = DEFAULT_ROOT_DIR, engine, build let events = []; const sessionFile = path.join(rootDir, SESSION_PATH); - if (fs.existsSync(sessionFile)) events = readJsonl(sessionFile, problems); + const sessionFound = fs.existsSync(sessionFile); + if (sessionFound) events = readJsonl(sessionFile, problems); else problems.push(`missing ${SESSION_PATH}`); + if (sessionOnly) return { events, requests: [], routingRecords: [], reflect: null, problems, sessionFound }; let requests = []; const tokenFile = TOKEN_USAGE_PATHS.map(file => path.join(rootDir, file)).find(file => fs.existsSync(file)); @@ -615,7 +749,25 @@ function collectModelRoutingEvidence({ rootDir = DEFAULT_ROOT_DIR, engine, build problems.push(`${reflectFile} is not valid JSON: ${getErrorMessage(error)}`); } } - return { events, requests, routingRecords, reflect, problems }; + return { events, requests, routingRecords, reflect, problems, sessionFound }; +} + +/** + * Append U1–U3 to `results`, or note that they were not evaluated when the + * session file is missing (E2 already reports that). + * @param {RoutingAssertionResults} results + * @param {{ events: any[], sessionFound: boolean }} evidence + * @param {SessionContractExpectations} expectations + */ +function appendSessionContract(results, evidence, expectations) { + if (!evidence.sessionFound) { + (results.info ??= []).push(`U1–U3 not evaluated: missing ${SESSION_PATH}`); + return; + } + const contract = evaluateSessionContract({ events: evidence.events, expectations }); + results.passes.push(...contract.passes); + results.failures.push(...contract.failures); + (results.info ??= []).push(...(contract.info ?? [])); } /** @@ -629,6 +781,7 @@ function checkModelRoutingEvidence(options) { const results = evaluateModelRoutingEvidence({ events: evidence.events, requests: evidence.requests, reflect: evidence.reflect, expectations }); if (evidence.problems.length) results.failures.unshift(...evidence.problems.map(problem => `FAIL E2 evidence: ${problem}`)); else results.passes.unshift("PASS E2 evidence: unified session and api-proxy token usage present"); + if (hasSessionContract(expectations)) appendSessionContract(results, evidence, expectations); if (expectations.steering) { // T4: the existing checks still pass alongside the steering checks. const failedIds = [...new Set(results.failures.map(line => line.split(" ")[1]))]; @@ -647,21 +800,60 @@ function checkModelRoutingEvidence(options) { } /** - * Post-step entry point: logs every check and fails the step on any failure. - * @param {RoutingExpectations & { rootDir?: string, engine?: string, buildSession?: boolean, core?: any }} options - * @returns {Promise} + * @param {any} actionsCore + * @param {RoutingAssertionResults} results + * @param {string} label */ -async function main(options) { - const { core: actionsCore = global.core, ...rest } = options; - const results = checkModelRoutingEvidence(rest); +function reportResults(actionsCore, results, label) { for (const line of results.info ?? []) actionsCore.info(line); for (const line of results.passes) actionsCore.info(line); for (const line of results.failures) actionsCore.error(line); if (results.failures.length) { - actionsCore.setFailed(`Model-routing smoke assertions failed (${results.failures.length}):\n${results.failures.join("\n")}`); + actionsCore.setFailed(`${label} smoke assertions failed (${results.failures.length}):\n${results.failures.join("\n")}`); } else { - actionsCore.info(`All ${results.passes.length} model-routing smoke assertions passed`); + actionsCore.info(`All ${results.passes.length} ${label.toLowerCase()} smoke assertions passed`); } +} + +/** + * Post-step entry point: logs every check and fails the step on any failure. + * @param {RoutingExpectations & { rootDir?: string, engine?: string, buildSession?: boolean, core?: any }} options + * @returns {Promise} + */ +async function main(options) { + const { core: actionsCore = global.core, ...rest } = options; + const results = checkModelRoutingEvidence(rest); + reportResults(actionsCore, results, "Model-routing"); + return results; +} + +/** + * Collect the unified session and evaluate E2 (session presence) and U1–U3, + * without the routing checks. + * @param {SessionContractExpectations & { rootDir?: string, engine?: string, buildSession?: boolean }} options + * @returns {RoutingAssertionResults} + */ +function checkSessionContract(options) { + const { rootDir, engine, buildSession, ...expectations } = options; + const evidence = collectModelRoutingEvidence({ rootDir, engine, buildSession, sessionOnly: true }); + /** @type {RoutingAssertionResults} */ + const results = { passes: [], failures: [], info: [] }; + if (evidence.problems.length) results.failures.push(...evidence.problems.map(problem => `FAIL E2 evidence: ${problem}`)); + else results.passes.push("PASS E2 evidence: unified session present"); + appendSessionContract(results, evidence, expectations); + return results; +} + +/** + * Post-step entry point for unrouted smokes: logs E2 and U1–U3 and fails the + * step on any failure. + * @param {SessionContractExpectations & { rootDir?: string, engine?: string, buildSession?: boolean, core?: any }} options + * @returns {Promise} + */ +async function sessionContractMain(options) { + const { core: actionsCore = global.core, ...rest } = options; + const results = checkSessionContract(rest); + reportResults(actionsCore, results, "Unified-session"); return results; } @@ -845,6 +1037,9 @@ module.exports = { collectModelRoutingEvidence, checkModelRoutingEvidence, main, + evaluateSessionContract, + checkSessionContract, + sessionContractMain, evaluateDetectionEvidence, collectDetectionEvidence, checkDetectionEvidence, diff --git a/actions/setup/js/smoke_model_routing_assertions.test.cjs b/actions/setup/js/smoke_model_routing_assertions.test.cjs index 0f5399e0208..3fd5cd92fdf 100644 --- a/actions/setup/js/smoke_model_routing_assertions.test.cjs +++ b/actions/setup/js/smoke_model_routing_assertions.test.cjs @@ -3,7 +3,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { checkDetectionEvidence, checkModelRoutingEvidence, detectionMain, evaluateDetectionEvidence, main, normalizeEndpoint, servedModel } from "./smoke_model_routing_assertions.cjs"; +import { checkDetectionEvidence, checkModelRoutingEvidence, checkSessionContract, detectionMain, evaluateDetectionEvidence, main, normalizeEndpoint, servedModel, sessionContractMain } from "./smoke_model_routing_assertions.cjs"; +import { piRoutedSession, postFixCopilotRoutedSession, postFixCopilotSession, preFixCopilotSession } from "./fixtures/session_contract_sessions.cjs"; const TOKEN_USAGE = "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl"; const ROUTED_ALLOWED = ["gpt-5.4-mini", "gpt-5.6-luna", "claude-haiku-4.5"]; @@ -955,6 +956,204 @@ describe("smoke_model_routing_assertions", () => { expect(core.info).toHaveBeenLastCalledWith("All 5 model-routing smoke assertions passed"); }); }); + + // Sessions are cut from published usage/aw_session.jsonl files; see + // fixtures/session_contract_sessions.cjs for the source run of each. + describe("unified-session contract (U1–U3)", () => { + const unroutedCopilot = { engine: "copilot", expectModelInventory: true, copilotInventory: true, resolutionRoles: ["main"] }; + const contract = (expectations = unroutedCopilot) => checkSessionContract({ rootDir: root, buildSession: false, ...expectations }); + const withoutWarnings = events => events.filter(event => event.type !== "session.collection_warning"); + const lineFor = (lines, id) => lines.find(line => line.split(" ")[1] === id); + + it("fails U1 and U2 for the pre-fix Smoke Copilot session (run 39181772988)", () => { + write("usage/aw_session.jsonl", preFixCopilotSession()); + const results = contract(); + expect(results.failures).toEqual([ + "FAIL U1 1 session.collection_warning event(s): invalid_model_inventory in sandbox/firewall/awf-reflect.json (1 occurrence(s))", + "FAIL U2 expected exactly 1 agent-phase model.inventory, observed 0; model.inventory events: none", + ]); + expect(checkIds(results.passes)).toEqual(["E2", "U3"]); + }); + + it("groups repeated U1 warnings by code and path with counts and an example line", () => { + const warning = (code, file, line) => ({ type: "session.collection_warning", data: { code, path: file, ...(line === undefined ? {} : { line }) } }); + write("usage/aw_session.jsonl", [ + ...postFixCopilotSession(), + warning("invalid_json", "agent/log.jsonl"), + ...Array.from({ length: 1000 }, (_, index) => warning("invalid_json", "agent/log.jsonl", index + 2)), + warning("invalid_record", "agent/log.jsonl", 0), + warning("invalid_json", "agent/other.jsonl", 9), + ]); + expect(lineFor(contract().failures, "U1")).toBe( + "FAIL U1 1003 session.collection_warning event(s): invalid_json in agent/log.jsonl (1001 occurrence(s), example line 2), invalid_record in agent/log.jsonl (1 occurrence(s), example line 0), invalid_json in agent/other.jsonl (1 occurrence(s), example line 9)" + ); + }); + + it("passes U1–U3 for the post-fix Smoke Copilot session (run 39176941475)", () => { + write("usage/aw_session.jsonl", postFixCopilotSession()); + const results = contract(); + expect(results.failures).toEqual([]); + expect(results.passes).toEqual([ + "PASS E2 evidence: unified session present", + "PASS U1 no session.collection_warning events", + "PASS U2 one agent-phase model.inventory from sandbox/firewall/awf-reflect.json; copilot endpoint configured with 40 model(s)", + "PASS U3 model.resolution recorded for main: auto -> claude-sonnet-5.5 (resolved)", + ]); + }); + + it("passes U1–U3 alongside R1–R4 for the post-fix routed session without a main model.resolution (run 39176939097)", () => { + fixture({ session: [...postFixCopilotRoutedSession(), workflowInfo(), selection(), harnessOutcome()], requests: copilotRequests() }); + const results = check({ ...copilotExpectations, expectModelInventory: true, copilotInventory: true, resolutionRoles: [] }); + expect(results.failures).toEqual([]); + expect(checkIds(results.passes)).toEqual(["E2", "R1", "R2", "R3", "R4", "U1", "U2", "U3"]); + expect(lineFor(results.passes, "U3")).toBe("PASS U3 no model.resolution roles required; observed model.resolution: none"); + }); + + it("passes U3 for both Pi sub-agent roles and names a missing role (run 39178909557)", () => { + const piRoles = { engine: "pi", resolutionRoles: ["haiku-whoami", "mini-whoami"] }; + write("usage/aw_session.jsonl", withoutWarnings(piRoutedSession())); + const results = contract(piRoles); + expect(results.failures).toEqual([]); + expect(lineFor(results.passes, "U3")).toBe("PASS U3 model.resolution recorded for haiku-whoami: copilot/claude-haiku-4.5 -> claude-haiku-4.5 (exact), mini-whoami: copilot/gpt-5.4-mini -> gpt-5.4-mini (exact)"); + + write( + "usage/aw_session.jsonl", + withoutWarnings(piRoutedSession()).filter(event => event.data?.role !== "mini-whoami") + ); + expect(contract(piRoles).failures).toEqual(["FAIL U3 no model.resolution for role mini-whoami; observed model.resolution: haiku-whoami: copilot/claude-haiku-4.5 -> claude-haiku-4.5 (exact)"]); + }); + + it("fails U2 for two agent-phase inventories and for an unconfigured copilot endpoint", () => { + const session = postFixCopilotSession(); + const inventory = session.find(event => event.type === "model.inventory"); + write("usage/aw_session.jsonl", [...session, { ...inventory, provenance: { ...inventory.provenance, path: "sandbox/firewall/models.json" } }]); + expect(lineFor(contract().failures, "U2")).toBe("FAIL U2 expected exactly 1 agent-phase model.inventory, observed 2; model.inventory events: agent sandbox/firewall/awf-reflect.json, agent sandbox/firewall/models.json"); + + // Detection-phase inventories are not counted. + write("usage/aw_session.jsonl", [...session, { ...inventory, provenance: { ...inventory.provenance, phase: "detection", path: "threat-detection/sandbox/firewall/awf-reflect.json" } }]); + expect(contract().failures).toEqual([]); + + const unconfigured = postFixCopilotSession().map(event => + event.type === "model.inventory" ? { ...event, data: { ...event.data, endpoints: event.data.endpoints.map(endpoint => (endpoint.provider === "copilot" ? { provider: "copilot", configured: false } : endpoint)) } } : event + ); + write("usage/aw_session.jsonl", unconfigured); + expect(contract().failures).toEqual([ + "FAIL U2 agent-phase model.inventory from sandbox/firewall/awf-reflect.json has no configured copilot endpoint with models; observed endpoints: openai configured=false models=, anthropic configured=false models=, copilot configured=false models=, gemini configured=false models=, vertex configured=false models=", + ]); + expect(contract({ ...unroutedCopilot, copilotInventory: false }).failures).toEqual([]); + }); + + it("fails U3 and prints the cause for a failed model.resolution", () => { + const failed = postFixCopilotSession().map(event => (event.type === "model.resolution" ? { ...event, data: { ...event.data, resolved_model: null, status: "failed", cause: "model_unavailable" } } : event)); + write("usage/aw_session.jsonl", failed); + expect(contract().failures).toEqual(["FAIL U3 role main status is failed (cause model_unavailable), expected exact or resolved; observed model.resolution: main: auto -> (failed)"]); + }); + + it("fails U3 for a duplicate role resolution", () => { + const session = postFixCopilotSession(); + const resolution = session.find(event => event.type === "model.resolution"); + write("usage/aw_session.jsonl", [...session, { ...resolution }]); + expect(lineFor(contract().failures, "U3")).toBe( + "FAIL U3 expected exactly 1 model.resolution for role main, observed 2; observed model.resolution: main: auto -> claude-sonnet-5.5 (resolved), main: auto -> claude-sonnet-5.5 (resolved)" + ); + }); + + it.each(["", null, undefined])("fails U3 for resolved_model %s despite a successful status", resolvedModel => { + const session = postFixCopilotSession().map(event => (event.type === "model.resolution" ? { ...event, data: { ...event.data, resolved_model: resolvedModel } } : event)); + write("usage/aw_session.jsonl", session); + expect(lineFor(contract().failures, "U3")).toBe("FAIL U3 role main has no resolved_model; observed model.resolution: main: auto -> (resolved)"); + }); + + it("fails U2 gracefully when the session has no inventory", async () => { + write( + "usage/aw_session.jsonl", + postFixCopilotSession().filter(event => event.type !== "model.inventory") + ); + const core = { info: vi.fn(), error: vi.fn(), setFailed: vi.fn() }; + const results = await sessionContractMain({ core, rootDir: root, buildSession: false, ...unroutedCopilot }); + expect(results.failures).toEqual(["FAIL U2 expected exactly 1 agent-phase model.inventory, observed 0; model.inventory events: none"]); + expect(core.setFailed).toHaveBeenCalledWith(`Unified-session smoke assertions failed (1):\n${results.failures[0]}`); + }); + + it.each([{ resolutionRoles: [] }, { copilotInventory: false, resolutionRoles: [] }, { expectModelInventory: false, copilotInventory: false, resolutionRoles: [] }])("logs when U2 is not requested: %j", async expectations => { + fixture({ session: [...postFixCopilotRoutedSession(), workflowInfo(), selection(), harnessOutcome()], requests: copilotRequests() }); + for (const [entryPoint, extra] of [ + [main, copilotExpectations], + [sessionContractMain, {}], + ]) { + const core = { info: vi.fn(), error: vi.fn(), setFailed: vi.fn() }; + const results = await entryPoint({ core, rootDir: root, buildSession: false, ...extra, ...expectations }); + expect(results.failures).toEqual([]); + expect(core.info).toHaveBeenCalledWith("U2 not evaluated: neither expectModelInventory nor copilotInventory is true"); + expect([...results.passes, ...results.failures].filter(line => /^(PASS|FAIL) U2 /.test(line))).toEqual([]); + } + }); + + it.each([ + [{}, []], + [{ copilotInventory: false }, []], + [{ expectModelInventory: false, copilotInventory: false }, []], + [{ expectModelInventory: true }, ["U1", "U2", "U3"]], + [{ copilotInventory: true }, ["U1", "U2", "U3"]], + [{ resolutionRoles: [] }, ["U1", "U3"]], + ])("runs U1–U3 from main only when the smoke opts in: %j", async (expectations, expectedIds) => { + fixture({ session: [...postFixCopilotRoutedSession(), workflowInfo(), selection(), harnessOutcome()], requests: copilotRequests() }); + const core = { info: vi.fn(), error: vi.fn(), setFailed: vi.fn() }; + const results = await main({ core, rootDir: root, buildSession: false, ...copilotExpectations, ...expectations }); + expect(results.failures).toEqual([]); + expect(checkIds(results.passes).filter(id => /^U\d$/.test(id))).toEqual(expectedIds); + }); + + it("reports E2 without throwing when the evidence root is unavailable", async () => { + const unavailableRoot = path.join(root, "not-a-directory"); + fs.writeFileSync(unavailableRoot, ""); + const core = { info: vi.fn(), error: vi.fn(), setFailed: vi.fn() }; + const results = await sessionContractMain({ core, rootDir: unavailableRoot, ...unroutedCopilot }); + expect(results.failures.some(line => line.startsWith("FAIL E2 evidence: failed to build the unified session:"))).toBe(true); + expect(results.failures).toContain("FAIL E2 evidence: missing usage/aw_session.jsonl"); + expect(core.setFailed).toHaveBeenCalled(); + }); + + it("reports E2 without throwing when the session path cannot be read", async () => { + fs.mkdirSync(path.join(root, "usage/aw_session.jsonl"), { recursive: true }); + const core = { info: vi.fn(), error: vi.fn(), setFailed: vi.fn() }; + const results = await sessionContractMain({ core, rootDir: root, buildSession: false, ...unroutedCopilot }); + expect(results.failures.some(line => line.startsWith("FAIL E2 evidence: failed to read ") && line.includes("aw_session.jsonl"))).toBe(true); + expect(core.setFailed).toHaveBeenCalled(); + }); + + it("reports FAIL E2 without U checks when the session file is missing, for both entry points", async () => { + write(TOKEN_USAGE, copilotRequests()); + const routedCore = { info: vi.fn(), error: vi.fn(), setFailed: vi.fn() }; + const routed = await main({ core: routedCore, rootDir: root, buildSession: false, ...copilotExpectations, expectModelInventory: true, copilotInventory: true, resolutionRoles: [] }); + expect(routed.failures[0]).toBe("FAIL E2 evidence: missing usage/aw_session.jsonl"); + expect([...routed.passes, ...routed.failures].filter(line => /^(PASS|FAIL) U/.test(line))).toEqual([]); + expect(routedCore.info).toHaveBeenCalledWith("U1–U3 not evaluated: missing usage/aw_session.jsonl"); + expect(routedCore.setFailed).toHaveBeenCalled(); + + const unroutedCore = { info: vi.fn(), error: vi.fn(), setFailed: vi.fn() }; + const unrouted = await sessionContractMain({ core: unroutedCore, rootDir: root, buildSession: false, ...unroutedCopilot }); + expect(unrouted.failures).toEqual(["FAIL E2 evidence: missing usage/aw_session.jsonl"]); + expect(unrouted.passes).toEqual([]); + expect(unroutedCore.info).toHaveBeenCalledWith("U1–U3 not evaluated: missing usage/aw_session.jsonl"); + expect(unroutedCore.setFailed).toHaveBeenCalledWith("Unified-session smoke assertions failed (1):\nFAIL E2 evidence: missing usage/aw_session.jsonl"); + }); + + it("logs every check and passes the step from sessionContractMain", async () => { + write("usage/aw_session.jsonl", postFixCopilotSession()); + const core = { info: vi.fn(), error: vi.fn(), setFailed: vi.fn() }; + await sessionContractMain({ core, rootDir: root, buildSession: false, ...unroutedCopilot }); + expect(core.setFailed).not.toHaveBeenCalled(); + expect(core.info).toHaveBeenLastCalledWith("All 4 unified-session smoke assertions passed"); + }); + + it("counts U failures in T4 when steering is configured", () => { + write("usage/aw_session.jsonl", preFixCopilotSession()); + write(TOKEN_USAGE, copilotRequests()); + const { failures } = check({ ...copilotExpectations, steering: { maxAiCredits: 7 }, expectModelInventory: true, copilotInventory: true, resolutionRoles: [] }); + expect(lineFor(failures, "T4")).toMatch(/^FAIL T4 existing check\(s\) failed: .*U1, U2$/); + }); + }); }); // Detection fixtures are copied from real records with request IDs and timestamps