From c28546c7b93e5cb7fc431db7e2aa422b5ca40de4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:42:04 +0000 Subject: [PATCH 1/6] Initial plan From 5e56aa44b9c3758e3b99dc87865134ceaacec51e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:05:51 +0000 Subject: [PATCH 2/6] Add deterministic workflow model pin check to Daily Model Inventory Checker Co-authored-by: SivaKesava1 <11771739+SivaKesava1@users.noreply.github.com> --- .../workflows/daily-model-inventory.lock.yml | 30 +- .github/workflows/daily-model-inventory.md | 63 +- actions/setup/js/model_pin_check.cjs | 765 ++++++++++++++++++ actions/setup/js/model_pin_check.test.cjs | 207 +++++ 4 files changed, 1057 insertions(+), 8 deletions(-) create mode 100644 actions/setup/js/model_pin_check.cjs create mode 100644 actions/setup/js/model_pin_check.test.cjs diff --git a/.github/workflows/daily-model-inventory.lock.yml b/.github/workflows/daily-model-inventory.lock.yml index 2c996960fff..7bae62c1f93 100644 --- a/.github/workflows/daily-model-inventory.lock.yml +++ b/.github/workflows/daily-model-inventory.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"bc0e2fcc9d46eac88f38c378e15bc4595fba9b1ce5c550886e859059856ac66f","body_hash":"1021d8e5f0f5f45cff34282cb573c3f40cf95f80095d978746133ca19d7fd36e","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"06eb59a5454e80efc72f7ca32a152ac8809728cd9e8e0d30a32f539b744fd7bd","body_hash":"93a1c9b06366c3b548bf480b4db8240ece21b171b6ebe98362fc9d384c3a10e2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GEMINI_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62","digest":"sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62","digest":"sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62","digest":"sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -23,7 +23,7 @@ # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# Queries model lists from OpenAI, Anthropic, and Google APIs daily, uses AWF /reflect for Copilot models, then analyzes the combined inventory to propose updates to the builtin model alias mapping +# Queries model lists from OpenAI, Anthropic, and Google APIs daily, uses AWF /reflect for Copilot models, checks every workflow model pin against the live Copilot catalogue, then analyzes the combined inventory to propose updates to the builtin model alias mapping # # Resolved workflow manifest: # Imports: @@ -1208,6 +1208,28 @@ jobs: if [ ! -f /tmp/gh-aw/agent_output.json ]; then echo '{"items":[]}' > /tmp/gh-aw/agent_output.json fi + - if: always() + name: Check workflow model pins + run: | + set -euo pipefail + OUT="/tmp/gh-aw/agent/model-inventory/model-pins" + REFLECT=/dev/null + for candidate in /tmp/gh-aw/agent/awf-reflect.json /tmp/gh-aw/sandbox/firewall/awf-reflect.json /tmp/gh-aw/agent/model-inventory/reflect.json; do + if [ -s "$candidate" ]; then REFLECT="$candidate"; break; fi + done + echo "Checking workflow model pins against $REFLECT" + node "$GITHUB_WORKSPACE/actions/setup/js/model_pin_check.cjs" --reflect "$REFLECT" --out-dir "$OUT" --repo "$GITHUB_WORKSPACE" > /dev/null + cat "$OUT/model-pins.md" >> "$GITHUB_STEP_SUMMARY" + shell: bash + - if: always() + name: Upload workflow model pin report + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + if-no-files-found: warn + name: model-pins + path: /tmp/gh-aw/agent/model-inventory/model-pins/ + retention-days: ${{ vars.GH_AW_DEFAULT_ARTIFACT_RETENTION_DAYS || '30' }} + # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact - name: Upload agent output fallback artifact @@ -2084,7 +2106,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Daily Model Inventory Checker" - WORKFLOW_DESCRIPTION: "Queries model lists from OpenAI, Anthropic, and Google APIs daily, uses AWF /reflect for Copilot models, then analyzes the combined inventory to propose updates to the builtin model alias mapping" + WORKFLOW_DESCRIPTION: "Queries model lists from OpenAI, Anthropic, and Google APIs daily, uses AWF /reflect for Copilot models, checks every workflow model pin against the live Copilot catalogue, then analyzes the combined inventory to propose updates to the builtin model alias mapping" HAS_PATCH: ${{ needs.agent.outputs.has_patch }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" GH_AW_DETECTION_SKIP_PROMPT_SUMMARY: "true" @@ -2157,7 +2179,7 @@ jobs: S2STOKENS: true TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} WORKFLOW_NAME: "Daily Model Inventory Checker" - WORKFLOW_DESCRIPTION: "Queries model lists from OpenAI, Anthropic, and Google APIs daily, uses AWF /reflect for Copilot models, then analyzes the combined inventory to propose updates to the builtin model alias mapping" + WORKFLOW_DESCRIPTION: "Queries model lists from OpenAI, Anthropic, and Google APIs daily, uses AWF /reflect for Copilot models, checks every workflow model pin against the live Copilot catalogue, then analyzes the combined inventory to propose updates to the builtin model alias mapping" HAS_PATCH: ${{ needs.agent.outputs.has_patch }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" run: | diff --git a/.github/workflows/daily-model-inventory.md b/.github/workflows/daily-model-inventory.md index 391358b8557..56a5854fb59 100644 --- a/.github/workflows/daily-model-inventory.md +++ b/.github/workflows/daily-model-inventory.md @@ -2,7 +2,7 @@ private: true emoji: "📦" name: Daily Model Inventory Checker -description: Queries model lists from OpenAI, Anthropic, and Google APIs daily, uses AWF /reflect for Copilot models, then analyzes the combined inventory to propose updates to the builtin model alias mapping +description: Queries model lists from OpenAI, Anthropic, and Google APIs daily, uses AWF /reflect for Copilot models, checks every workflow model pin against the live Copilot catalogue, then analyzes the combined inventory to propose updates to the builtin model alias mapping on: schedule: - cron: daily @@ -339,6 +339,32 @@ steps: echo "Combined inventory written to $INVENTORY" cat "$INVENTORY" +# The pin check runs again after the agent, preferring the harness's own /reflect capture, +# so the report artifact and step summary do not depend on the agent. +post-steps: + - name: Check workflow model pins + if: always() + shell: bash + run: | + set -euo pipefail + OUT="/tmp/gh-aw/agent/model-inventory/model-pins" + REFLECT=/dev/null + for candidate in /tmp/gh-aw/agent/awf-reflect.json /tmp/gh-aw/sandbox/firewall/awf-reflect.json /tmp/gh-aw/agent/model-inventory/reflect.json; do + if [ -s "$candidate" ]; then REFLECT="$candidate"; break; fi + done + echo "Checking workflow model pins against $REFLECT" + node "$GITHUB_WORKSPACE/actions/setup/js/model_pin_check.cjs" --reflect "$REFLECT" --out-dir "$OUT" --repo "$GITHUB_WORKSPACE" > /dev/null + cat "$OUT/model-pins.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload workflow model pin report + if: always() + uses: actions/upload-artifact@v7.0.2 + with: + name: model-pins + path: /tmp/gh-aw/agent/model-inventory/model-pins/ + if-no-files-found: warn + retention-days: 30 + sandbox: agent: id: awf @@ -484,6 +510,26 @@ done < <(jq -c '.endpoints[]? | select(.configured == true and .models == null a echo "Copilot reflect metadata written to $OUT" ``` +### Step 0.5: Check Workflow Model Pins + +Run the deterministic pin checker right after Step 0. It classifies every model pin configured +by `.github/workflows/*.md` (and their imports) against the Copilot catalogue in the reflect +file, using the same resolver as the engine harnesses. Do not classify pins yourself. + +```bash +set -euo pipefail +REPO="${GITHUB_WORKSPACE:-$PWD}" +node "$REPO/actions/setup/js/model_pin_check.cjs" \ + --reflect /tmp/gh-aw/agent/model-inventory/reflect.json \ + --out-dir /tmp/gh-aw/agent/model-inventory/model-pins \ + --repo "$REPO" +``` + +It writes `model-pins-summary.md` (the issue section), `model-pins.md` (every pin with its +classification) and `model-pins.json` to `/tmp/gh-aw/agent/model-inventory/model-pins/`. The +run uploads that directory as the `model-pins` artifact. Read `counts.unresolvable` from +`model-pins.json` for Step 6. + ### Step 1: Load and Validate the Inventory Read the combined inventory from `/tmp/gh-aw/agent/model-inventory/inventory.json`. Then read @@ -693,6 +739,13 @@ If you found any meaningful updates to propose, create a GitHub issue using `cre **Issue title format**: `Model alias inventory update - YYYY-MM-DD` +**Workflow model pins**: start the issue body with the contents of +`/tmp/gh-aw/agent/model-inventory/model-pins/model-pins-summary.md` from Step 0.5, copied +verbatim (it begins with `### Action required` when any pin is unresolvable). Do not edit, +reorder or summarize it. If `counts.unresolvable` in `model-pins.json` is greater than 0, +the title must be `Model alias inventory - action required - YYYY-MM-DD`, and the issue +must be created even if no alias or pricing updates are needed. + **Issue body structure** (use h3 `###` or lower — never h1/h2): ```markdown @@ -760,9 +813,11 @@ Any caveats, historical entries retained, or aliases that are already well-cover ``` If no updates are needed (all live models are already covered by existing aliases, all -`models.json` pricing entries are up to date, and no new task-oriented aliases are warranted), create an issue with -title `Model alias inventory - no changes needed - YYYY-MM-DD` and a brief summary confirming -coverage is up to date. +`models.json` pricing entries are up to date, no new task-oriented aliases are warranted, and +`counts.unresolvable` in `model-pins.json` is 0), create an issue with +title `Model alias inventory - no changes needed - YYYY-MM-DD`, starting with the verbatim +`model-pins-summary.md` contents and then a brief summary confirming coverage is up to date. +Never use the "no changes needed" title when any workflow model pin is unresolvable. ### Output Format diff --git a/actions/setup/js/model_pin_check.cjs b/actions/setup/js/model_pin_check.cjs new file mode 100644 index 00000000000..7f6fa9d16ec --- /dev/null +++ b/actions/setup/js/model_pin_check.cjs @@ -0,0 +1,765 @@ +// @ts-check +"use strict"; + +/** + * Deterministic workflow model pin check for the Daily Model Inventory Checker. + * + * Collects every model name configured by `.github/workflows/*.md` (and their local + * imports, imported agent files and inline `## agent:` blocks), then classifies each + * pin against the live Copilot catalogue from AWF `/reflect` with the same resolver the + * engine harnesses use (`resolveConfiguredModel`). The resolver returns `exact` for a + * non-alias name without checking liveness, so an exact name is also tested with + * `isLiveModelId` and a non-live exact name is reported as `model_unavailable`. + * + * Classification: + * - exact: a live Copilot model id. + * - alias: an alias that resolves; the resolved model is reported. + * - unresolvable: the resolver's cause (`model_unavailable` or `catalog_unavailable`). + * - not_checked: with a reason: + * - `expression`: the value contains a GitHub Actions expression (`${{ … }}`). + * - `provider not in the Copilot catalogue`: the pin targets another provider (an + * explicit non-Copilot prefix, the engine's own provider such as Anthropic or + * OpenAI, or Copilot BYOK via `engine.env.COPILOT_PROVIDER_BASE_URL`). + * - `sentinel`: `inherited` (a sub-agent that uses its parent's model; the parent's + * pin is checked in its own row) or `auto` on an engine or field where `auto` is + * the gateway's own model selection rather than a gh-aw alias (every engine except + * Copilot, and `safe-outputs.assign-to-agent.model`). + * + * Usage (from the repository root): + * node actions/setup/js/model_pin_check.cjs --reflect --out-dir [--repo ] + * Writes /model-pins.json, /model-pins.md (every pin) and /model-pins-summary.md + * (the issue section) and prints the summary. + */ + +const fs = require("fs"); +const path = require("path"); +const { parseDocument } = require("./work_queue_yaml.cjs"); +const { extractInlineSubAgents } = require("./extract_inline_sub_agents.cjs"); +const { buildProviderCatalog, canonicalModelProvider, isLiveModelId, loadBuiltinModelAliases, ModelAliasResolutionError, resolveConfiguredModel, splitModelIdentifier } = require("./resolve_model_alias.cjs"); + +/** Keys under `models:` that are not alias names (see reservedModelsKeys in pkg/workflow/model_aliases.go). */ +const RESERVED_MODELS_KEYS = new Set(["providers", "default-ai-credits-pricing", "allowed", "blocked"]); + +/** Native provider of each built-in engine when neither the model nor the engine names one. */ +const ENGINE_DEFAULT_PROVIDERS = Object.freeze({ copilot: "copilot", claude: "anthropic", codex: "openai", pi: "copilot", gemini: "gemini", agy: "gemini" }); + +/** Engines whose provider follows a `copilot/` main model when no provider is configured (resolveEngineLLMProviderFromModel). */ +const MODEL_PREFIX_PROVIDER_ENGINES = new Set(["claude", "codex", "pi"]); + +const REASON_EXPRESSION = "expression"; +const REASON_PROVIDER = "provider not in the Copilot catalogue"; +const REASON_BYOK = "provider not in the Copilot catalogue (BYOK via COPILOT_PROVIDER_BASE_URL)"; +const REASON_INHERITED = "sentinel `inherited`: the sub-agent uses its parent's model, which is checked in its own row"; +const REASON_AUTO = "sentinel `auto`: the gateway selects the model"; + +const STATUS_LABELS = Object.freeze({ unresolvable: "unresolvable", alias: "alias resolves", exact: "exact", not_checked: "not checked" }); +const STATUS_ORDER = ["unresolvable", "alias", "exact", "not_checked"]; + +/** + * @typedef {{ + * workflow: string, + * source: string, + * field: string, + * model: string, + * kind: "model" | "alias-definition", + * provider: string, + * engine: string, + * autoSentinel: boolean, + * aliasEntries?: string[], + * aliasMap: Record, + * }} ModelPin + */ + +/** + * @typedef {{ + * workflow: string, + * source: string, + * field: string, + * configured: string, + * engine: string, + * provider: string, + * status: "exact" | "alias" | "unresolvable" | "not_checked", + * resolved: string | null, + * cause: string | null, + * reason: string | null, + * }} PinResult + */ + +/** + * Split a markdown file into its YAML frontmatter object and body. + * @param {string} content + * @returns {{ frontmatter: Record | null, body: string, error: string | null }} + */ +function parseMarkdownFrontmatter(content) { + const text = String(content || "").replace(/^\uFEFF/, ""); + const lines = text.split(/\r?\n/); + if (lines.length === 0 || lines[0].trim() !== "---") return { frontmatter: null, body: text, error: null }; + const end = lines.findIndex((line, index) => index > 0 && line.trim() === "---"); + if (end < 0) return { frontmatter: null, body: text, error: "unterminated frontmatter" }; + const yaml = lines.slice(1, end).join("\n"); + const body = lines.slice(end + 1).join("\n"); + try { + /** @type {any} */ + const document = parseDocument(yaml, { version: "1.2", uniqueKeys: false }); + if (!document) return { frontmatter: {}, body, error: null }; + if (document.errors.length) return { frontmatter: null, body, error: String(document.errors[0]) }; + const value = document.toJS(); + return { frontmatter: value && typeof value === "object" && !Array.isArray(value) ? value : {}, body, error: null }; + } catch (error) { + return { frontmatter: null, body, error: error instanceof Error ? error.message : String(error) }; + } +} + +/** + * @param {unknown} value + * @returns {value is Record} + */ +function isObject(value) { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +/** + * @param {unknown} value + * @returns {string} + */ +function stringValue(value) { + return typeof value === "string" ? value.trim() : ""; +} + +/** + * Canonical provider named by a model's `provider/` prefix, or "" when unprefixed. + * @param {string} model + * @returns {string} + */ +function modelPrefixProvider(model) { + const { base } = splitModelIdentifier(model); + const slash = base.indexOf("/"); + return slash > 0 ? canonicalModelProvider(base.slice(0, slash)) : ""; +} + +/** + * The model id without provider prefix and parameters, lower-cased. + * @param {string} model + * @returns {string} + */ +function bareModelName(model) { + const { base } = splitModelIdentifier(model); + const slash = base.indexOf("/"); + return (slash >= 0 ? base.slice(slash + 1) : base).toLowerCase(); +} + +/** + * Import specs from an `imports:` value: strings, `{path|uses, inputs|with}` objects, + * or an object with an `aw:` list (parseImportSpecsFromField in pkg/parser/import_bfs.go). + * @param {unknown} imports + * @returns {string[]} + */ +function importPaths(imports) { + const list = Array.isArray(imports) ? imports : isObject(imports) && Array.isArray(imports.aw) ? imports.aw : []; + /** @type {string[]} */ + const paths = []; + for (const item of list) { + const value = typeof item === "string" ? item : isObject(item) ? stringValue(item.path) || stringValue(item.uses) : ""; + if (value) paths.push(value.trim()); + } + return paths; +} + +/** + * Whether an import path is a remote workflowspec (owner/repo/path[@ref]); mirrors + * IsWorkflowSpec in pkg/parser/remote_path_predicates.go. + * @param {string} importPath + * @returns {boolean} + */ +function isRemoteImport(importPath) { + const clean = importPath.split("#")[0].split("@")[0]; + const parts = clean.split("/"); + if (parts.length < 3) return false; + if (clean.includes("://")) return true; + if (clean.startsWith(".") || clean.startsWith("shared/") || clean.startsWith("/")) return false; + return !!parts[0] && !!parts[1]; +} + +/** + * Resolve a local import path the way the compiler does (determineNestedBaseDir and + * computeIncludeResolveAndSecurityBases in pkg/parser): `.github/…` and `/…` paths are + * repository-relative; a nested import without a directory (or starting with `./`) is + * relative to the importing file; anything else is relative to the workflow directory. + * @param {string} importPath + * @param {{ importerDir: string, workflowDir: string, repoRoot: string, nested: boolean }} bases absolute directories + * @returns {string} + */ +function resolveLocalImport(importPath, bases) { + const clean = importPath.split("#")[0]; + if (clean.startsWith(".github/")) return path.join(bases.repoRoot, clean); + if (clean.startsWith("/")) return path.join(bases.repoRoot, clean.slice(1)); + const localRelative = !clean.includes("/") || clean.startsWith("./"); + return path.join(bases.nested && localRelative ? bases.importerDir : bases.workflowDir, clean); +} + +/** + * @param {string} file absolute path + * @param {string} repoRoot + * @returns {string} + */ +function relativePath(file, repoRoot) { + return path.relative(repoRoot, file).split(path.sep).join("/"); +} + +/** + * @typedef {{ file: string, rel: string, frontmatter: Record, body: string, agentFile: boolean }} LoadedFile + */ + +/** + * Load a workflow and its local imports in breadth-first order (first import first), + * skipping remote and `@builtin:` imports. + * @param {string} workflowFile absolute path + * @param {string} repoRoot + * @returns {{ files: LoadedFile[], skipped: Array<{ workflow: string, import: string, reason: string }>, errors: Array<{ file: string, error: string }> }} + */ +function loadWorkflowFiles(workflowFile, repoRoot) { + /** @type {LoadedFile[]} */ + const files = []; + /** @type {Array<{ workflow: string, import: string, reason: string }>} */ + const skipped = []; + /** @type {Array<{ file: string, error: string }>} */ + const errors = []; + const workflowRel = relativePath(workflowFile, repoRoot); + const visited = new Set(); + /** @type {Array<{ file: string, importPath: string | null }>} */ + const queue = [{ file: workflowFile, importPath: null }]; + while (queue.length > 0) { + const item = queue.shift(); + if (!item) break; + const file = path.resolve(item.file); + if (visited.has(file)) continue; + visited.add(file); + const rel = relativePath(file, repoRoot); + let content; + try { + content = fs.readFileSync(file, "utf8"); + } catch (error) { + errors.push({ file: rel, error: `cannot read import${item.importPath ? ` '${item.importPath}'` : ""}: ${error instanceof Error ? error.message : String(error)}` }); + continue; + } + const parsed = parseMarkdownFrontmatter(content); + if (parsed.error) errors.push({ file: rel, error: `frontmatter: ${parsed.error}` }); + const frontmatter = parsed.frontmatter || {}; + const agentFile = item.importPath !== null && /(^|\/)\.github\/agents\/[^/]+\.md$/i.test(rel); + files.push({ file, rel, frontmatter, body: parsed.body, agentFile }); + // Imported agent files contribute only their model (handleAgentImportItem); they have no nested imports. + if (agentFile) continue; + for (const importPath of importPaths(frontmatter.imports)) { + if (importPath.startsWith("@builtin:")) continue; + if (isRemoteImport(importPath)) { + skipped.push({ workflow: workflowRel, import: importPath, reason: "remote import" }); + continue; + } + queue.push({ file: resolveLocalImport(importPath, { importerDir: path.dirname(file), workflowDir: path.dirname(workflowFile), repoRoot, nested: item.importPath !== null }), importPath }); + } + } + return { files, skipped, errors }; +} + +/** + * @typedef {{ id: string, explicitProvider: string, byok: boolean, definitionProvider: string }} EngineInfo + */ + +/** + * Describe an `engine:` value (string id, engine object, or inline definition). + * @param {unknown} engine + * @returns {EngineInfo | null} + */ +function describeEngine(engine) { + if (typeof engine === "string" && engine.trim()) return { id: engine.trim().toLowerCase(), explicitProvider: "", byok: false, definitionProvider: "" }; + if (!isObject(engine)) return null; + const runtime = engine.runtime; + const id = (stringValue(engine.id) || (typeof runtime === "string" ? runtime.trim() : isObject(runtime) ? stringValue(runtime.id) : "")).toLowerCase(); + const provider = engine.provider; + const explicitProvider = stringValue(engine["model-provider"]) || (typeof provider === "string" ? provider.trim() : isObject(provider) && !isObject(engine.behaviors) ? stringValue(provider.id) : ""); + const definitionProvider = isObject(provider) ? stringValue(provider.name) || stringValue(provider.id) : ""; + const byok = isObject(engine.env) && stringValue(engine.env.COPILOT_PROVIDER_BASE_URL) !== ""; + return { id, explicitProvider, byok, definitionProvider }; +} + +/** + * The provider a bare (unprefixed) model is sent to for an engine. + * @param {EngineInfo} engine + * @param {string} mainModel + * @param {Record} definitionProviders engine id → provider from imported engine definitions + * @returns {string} canonical provider, or "byok" + */ +function engineProvider(engine, mainModel, definitionProviders) { + if (engine.byok) return "byok"; + if (engine.explicitProvider) return canonicalModelProvider(engine.explicitProvider); + const id = engine.id || "copilot"; + if (id === "copilot") return "copilot"; + const mainPrefix = modelPrefixProvider(mainModel); + if (MODEL_PREFIX_PROVIDER_ENGINES.has(id)) return mainPrefix || ENGINE_DEFAULT_PROVIDERS[/** @type {keyof typeof ENGINE_DEFAULT_PROVIDERS} */ id]; + if (id === "gemini" || id === "agy") return "gemini"; + if (definitionProviders[id]) return canonicalModelProvider(definitionProviders[id]); + // Universal consumer engines default to the Copilot backend (resolveBackend). + return mainPrefix || "copilot"; +} + +/** + * Collect the model pins of one workflow. + * @param {string} workflowFile absolute path + * @param {{ repoRoot: string, builtinAliases: Record }} options + * @returns {{ pins: ModelPin[], skipped: Array<{ workflow: string, import: string, reason: string }>, errors: Array<{ file: string, error: string }> }} + */ +function collectWorkflowPins(workflowFile, options) { + const { repoRoot, builtinAliases } = options; + const { files, skipped, errors } = loadWorkflowFiles(workflowFile, repoRoot); + const workflow = relativePath(workflowFile, repoRoot); + if (files.length === 0) return { pins: [], skipped, errors }; + const main = files[0]; + const imports = files.slice(1).filter(file => !file.agentFile); + + // Merged alias map: built-ins, then imports (first import wins), then the main workflow. + /** @type {Record} */ + const aliasMap = { ...builtinAliases }; + /** @type {Set} */ + const importedAliasKeys = new Set(); + /** @type {Array<{ file: LoadedFile, name: string, entries: string[] }>} */ + const aliasDefinitions = []; + for (const file of [...imports, main]) { + const models = file.frontmatter.models; + if (!isObject(models)) continue; + for (const [name, value] of Object.entries(models)) { + if (RESERVED_MODELS_KEYS.has(name) || !Array.isArray(value)) continue; + const entries = value.filter(entry => typeof entry === "string" && entry.trim()).map(entry => entry.trim()); + if (entries.length === 0) continue; + aliasDefinitions.push({ file, name, entries }); + if (file === main) { + aliasMap[name] = entries; + } else if (!importedAliasKeys.has(name)) { + importedAliasKeys.add(name); + aliasMap[name] = entries; + } + } + } + + /** @type {Record} */ + const definitionProviders = {}; + /** @type {EngineInfo | null} */ + let mainEngine = describeEngine(main.frontmatter.engine); + for (const file of imports) { + const info = describeEngine(file.frontmatter.engine); + if (!info) continue; + if (info.id && info.definitionProvider && !definitionProviders[info.id]) definitionProviders[info.id] = info.definitionProvider; + if (!mainEngine && info.id && !isObject(file.frontmatter.engine?.behaviors)) mainEngine = info; + } + if (mainEngine && !mainEngine.id) mainEngine = { ...mainEngine, id: "copilot" }; + const engine = mainEngine || { id: "copilot", explicitProvider: "", byok: false, definitionProvider: "" }; + const mainModel = stringValue(main.frontmatter.model) || (isObject(main.frontmatter.engine) ? stringValue(main.frontmatter.engine.model) : ""); + const mainProvider = engineProvider(engine, mainModel, definitionProviders); + + const threatDetection = isObject(main.frontmatter["safe-outputs"]) ? main.frontmatter["safe-outputs"]["threat-detection"] : undefined; + const detectionEngineValue = isObject(threatDetection) ? (threatDetection.engine !== undefined && threatDetection.engine !== false ? threatDetection.engine : threatDetection["engine-config"]) : undefined; + const detectionInfo = describeEngine(detectionEngineValue); + const detectionEngine = detectionInfo ? { ...detectionInfo, id: detectionInfo.id || engine.id } : engine; + const detectionModel = (isObject(threatDetection) && stringValue(threatDetection.model)) || (isObject(detectionEngineValue) ? stringValue(detectionEngineValue.model) : "") || mainModel; + const detectionProvider = engineProvider(detectionEngine, detectionModel, definitionProviders); + + /** @type {ModelPin[]} */ + const pins = []; + /** + * @param {LoadedFile} file + * @param {string} field + * @param {unknown} value + * @param {{ engine: EngineInfo, provider: string, autoSentinel?: boolean }} scope + */ + const addPin = (file, field, value, scope) => { + const model = stringValue(value); + if (!model) return; + pins.push({ + workflow, + source: file.rel, + field, + model, + kind: "model", + provider: scope.provider, + engine: scope.engine.id || "copilot", + autoSentinel: scope.autoSentinel ?? scope.engine.id !== "copilot", + aliasMap, + }); + }; + const mainScope = { engine, provider: mainProvider }; + const detectionScope = { engine: detectionEngine, provider: detectionProvider }; + const copilotScope = { engine: { id: "copilot", explicitProvider: "", byok: false, definitionProvider: "" }, provider: "copilot" }; + + /** + * Pins of one engine object (engine:, threat-detection.engine:, threat-detection.engine-config:). + * @param {LoadedFile} file + * @param {string} prefix + * @param {unknown} engineValue + * @param {{ engine: EngineInfo, provider: string }} scope + */ + const addEnginePins = (file, prefix, engineValue, scope) => { + if (!isObject(engineValue)) return; + addPin(file, `${prefix}.model`, engineValue.model, scope); + if (Array.isArray(engineValue["fallback-models"])) engineValue["fallback-models"].forEach((model, index) => addPin(file, `${prefix}.fallback-models[${index}]`, model, scope)); + if (isObject(engineValue.provider)) addPin(file, `${prefix}.provider.model`, engineValue.provider.model, scope); + if (isObject(engineValue.models)) { + addPin(file, `${prefix}.models.default`, engineValue.models.default, scope); + if (Array.isArray(engineValue.models.supported)) engineValue.models.supported.forEach((model, index) => addPin(file, `${prefix}.models.supported[${index}]`, model, scope)); + } + const routing = engineValue["model-routing"]; + if (isObject(routing) && Array.isArray(routing["allowed-models"])) { + // Model routing always selects among Copilot models. + routing["allowed-models"].forEach((model, index) => addPin(file, `${prefix}.model-routing.allowed-models[${index}]`, model, { ...scope, provider: scope.provider === "byok" ? "byok" : "copilot" })); + } + }; + + for (const file of files) { + const fm = file.frontmatter; + if (file.agentFile) { + addPin(file, "model (imported agent file)", fm.model, mainScope); + continue; + } + addPin(file, "model", fm.model, mainScope); + addEnginePins(file, "engine", fm.engine, mainScope); + + const safeOutputs = isObject(fm["safe-outputs"]) ? fm["safe-outputs"] : {}; + const detection = safeOutputs["threat-detection"]; + if (isObject(detection)) { + addPin(file, "safe-outputs.threat-detection.model", detection.model, detectionScope); + addEnginePins(file, "safe-outputs.threat-detection.engine", detection.engine, detectionScope); + addEnginePins(file, "safe-outputs.threat-detection.engine-config", detection["engine-config"], detectionScope); + } + if (isObject(safeOutputs["assign-to-agent"])) addPin(file, "safe-outputs.assign-to-agent.model", safeOutputs["assign-to-agent"].model, { ...copilotScope, autoSentinel: true }); + + const evals = fm.evals; + if (Array.isArray(evals)) { + evals.forEach((question, index) => isObject(question) && addPin(file, `evals[${index}].model`, question.model, mainScope)); + } else if (isObject(evals)) { + addPin(file, "evals.model", evals.model, mainScope); + if (Array.isArray(evals.questions)) evals.questions.forEach((question, index) => isObject(question) && addPin(file, `evals.questions[${index}].model`, question.model, mainScope)); + } + + if (Array.isArray(fm.enclaves)) { + fm.enclaves.forEach((enclave, index) => { + if (!isObject(enclave) || !isObject(enclave.agent)) return; + const enclaveEngine = describeEngine(stringValue(enclave.agent.engine) || "copilot") || copilotScope.engine; + addPin(file, `enclaves[${index}].agent.model`, enclave.agent.model, { engine: enclaveEngine, provider: engineProvider(enclaveEngine, "", definitionProviders) }); + }); + } + + let agents = []; + try { + agents = extractInlineSubAgents(file.body).agents; + } catch (error) { + errors.push({ file: file.rel, error: `inline sub-agents: ${error instanceof Error ? error.message : String(error)}` }); + } + for (const agent of agents) { + const parsed = parseMarkdownFrontmatter(agent.content); + if (parsed.frontmatter) addPin(file, `## agent: '${agent.name}' model`, parsed.frontmatter.model, mainScope); + } + } + + for (const definition of aliasDefinitions) { + // An alias can reach the Copilot catalogue through a copilot/ entry, or through a bare + // entry when the workflow's engine sends bare models to Copilot. + const targetsCopilot = definition.entries.some(entry => { + const prefix = modelPrefixProvider(entry); + return prefix ? prefix === "copilot" : mainProvider === "copilot"; + }); + pins.push({ + workflow, + source: definition.file.rel, + field: `models.${definition.name}`, + model: definition.name, + kind: "alias-definition", + provider: targetsCopilot ? "copilot" : mainProvider === "byok" ? "byok" : modelPrefixProvider(definition.entries[0]) || mainProvider, + engine: engine.id || "copilot", + autoSentinel: false, + aliasEntries: definition.entries, + aliasMap, + }); + } + return { pins, skipped, errors }; +} + +/** + * Collect the model pins of every `*.md` workflow in a directory. + * @param {{ repoRoot: string, workflowsDir?: string, builtinAliases?: Record | null }} options + * @returns {{ pins: ModelPin[], workflows: string[], skipped: Array<{ workflow: string, import: string, reason: string }>, errors: Array<{ file: string, error: string }> }} + */ +function collectModelPins(options) { + const repoRoot = path.resolve(options.repoRoot); + const workflowsDir = path.resolve(repoRoot, options.workflowsDir || ".github/workflows"); + const builtinAliases = options.builtinAliases || loadBuiltinModelAliases() || {}; + const workflows = fs + .readdirSync(workflowsDir, { withFileTypes: true }) + .filter(entry => entry.isFile() && entry.name.endsWith(".md")) + .map(entry => entry.name) + .sort(); + /** @type {ModelPin[]} */ + const pins = []; + /** @type {Array<{ workflow: string, import: string, reason: string }>} */ + const skipped = []; + /** @type {Array<{ file: string, error: string }>} */ + const errors = []; + for (const name of workflows) { + const result = collectWorkflowPins(path.join(workflowsDir, name), { repoRoot, builtinAliases }); + pins.push(...result.pins); + skipped.push(...result.skipped); + errors.push(...result.errors); + } + return { pins, workflows: workflows.map(name => relativePath(path.join(workflowsDir, name), repoRoot)), skipped, errors }; +} + +/** + * Classify one pin against the live Copilot catalogue. + * @param {ModelPin} pin + * @param {any} reflectData + * @returns {PinResult} + */ +function classifyPin(pin, reflectData) { + /** @type {PinResult} */ + const result = { + workflow: pin.workflow, + source: pin.source, + field: pin.field, + configured: pin.kind === "alias-definition" ? `${pin.model}: [${(pin.aliasEntries || []).join(", ")}]` : pin.model, + engine: pin.engine, + provider: pin.provider, + status: "not_checked", + resolved: null, + cause: null, + reason: null, + }; + const values = pin.kind === "alias-definition" ? [pin.model, ...(pin.aliasEntries || [])] : [pin.model]; + if (values.some(value => value.includes("${{"))) return { ...result, reason: REASON_EXPRESSION }; + const name = pin.kind === "model" ? bareModelName(pin.model) : ""; + if (name === "inherited") return { ...result, reason: REASON_INHERITED }; + if (pin.provider === "byok") return { ...result, reason: REASON_BYOK }; + const prefix = pin.kind === "model" ? modelPrefixProvider(pin.model) : ""; + if ((prefix || pin.provider) !== "copilot") return { ...result, provider: prefix || pin.provider, reason: REASON_PROVIDER }; + if (name === "auto" && pin.autoSentinel) return { ...result, provider: "copilot", reason: REASON_AUTO }; + + const catalog = buildProviderCatalog({ reflectData, provider: "copilot" }); + try { + // An alias definition is checked with its own entries, even when another file overrides the name. + const aliasMap = pin.kind === "alias-definition" ? { ...pin.aliasMap, [pin.model]: pin.aliasEntries || [] } : pin.aliasMap; + const resolution = resolveConfiguredModel({ configuredModel: pin.model, aliasMap, reflectData, provider: "copilot" }); + if (resolution.status === "resolved") return { ...result, provider: "copilot", status: "alias", resolved: resolution.resolvedModel }; + if (catalog.length === 0) return { ...result, provider: "copilot", status: "unresolvable", cause: "catalog_unavailable" }; + if (isLiveModelId(splitModelIdentifier(resolution.modelId).base, catalog)) return { ...result, provider: "copilot", status: "exact", resolved: resolution.resolvedModel }; + return { ...result, provider: "copilot", status: "unresolvable", cause: "model_unavailable" }; + } catch (error) { + if (error instanceof ModelAliasResolutionError) return { ...result, provider: "copilot", status: "unresolvable", cause: error.reason }; + throw error; + } +} + +/** + * Summary of the Copilot endpoint in a /reflect payload. + * @param {any} reflectData + * @returns {{ available: boolean, model_count: number, error: string | null }} + */ +function describeCatalogue(reflectData) { + const catalog = buildProviderCatalog({ reflectData, provider: "copilot" }); + const endpoint = (Array.isArray(reflectData?.endpoints) ? reflectData.endpoints : []).find(entry => entry?.configured === true && canonicalModelProvider(entry.provider) === "copilot"); + const models = Array.isArray(endpoint?.models) ? endpoint.models : []; + const error = typeof reflectData?.error === "string" ? reflectData.error : !endpoint ? "no configured Copilot endpoint in /reflect" : !Array.isArray(endpoint.models) ? "Copilot endpoint returned no model list" : null; + return { available: catalog.length > 0, model_count: models.length, error }; +} + +/** + * Classify every pin and build the report data. + * @param {{ pins: ModelPin[], reflectData: any, workflows?: string[], skipped?: Array<{ workflow: string, import: string, reason: string }>, errors?: Array<{ file: string, error: string }> }} options + */ +function buildModelPinReport(options) { + const results = options.pins.map(pin => classifyPin(pin, options.reflectData)); + /** @type {Record} */ + const counts = { total: results.length, exact: 0, alias: 0, unresolvable: 0, not_checked: 0 }; + for (const result of results) counts[result.status] += 1; + return { + catalogue: describeCatalogue(options.reflectData), + counts, + action_required: results.filter(result => result.status === "unresolvable"), + pins: results, + workflows: options.workflows || [], + skipped_imports: options.skipped || [], + errors: options.errors || [], + }; +} + +/** + * @param {string | null} value + * @returns {string} + */ +function cell(value) { + return String(value ?? "") + .replace(/\|/g, "\\|") + .replace(/\r?\n/g, " "); +} + +/** + * @param {string | null} value + * @returns {string} + */ +function code(value) { + return `\`${cell(value).replace(/`/g, "'")}\``; +} + +/** + * @param {PinResult} result + * @returns {string} + */ +function workflowCell(result) { + return result.source === result.workflow ? code(result.workflow) : `${code(result.workflow)} (via ${code(result.source)})`; +} + +/** + * @param {PinResult} result + * @returns {string} + */ +function detailCell(result) { + if (result.status === "alias") return `→ ${code(result.resolved)}`; + if (result.status === "unresolvable") return code(result.cause); + if (result.status === "not_checked") return cell(result.reason); + return ""; +} + +/** + * @param {ReturnType} report + * @returns {string[]} + */ +function renderActionRequired(report) { + if (report.action_required.length === 0) return []; + const lines = ["### Action required", ""]; + // Without a catalogue every Copilot pin is catalog_unavailable; report that once instead of per pin. + const uncatalogued = report.action_required.filter(result => result.cause === "catalog_unavailable"); + const rows = report.action_required.filter(result => result.cause !== "catalog_unavailable"); + if (uncatalogued.length > 0) lines.push(`${uncatalogued.length} Copilot workflow model pin(s) could not be checked because the live Copilot catalogue is unavailable (\`catalog_unavailable\`); they are listed in \`model-pins.md\`.`, ""); + if (rows.length > 0) { + lines.push(`${rows.length} workflow model pin(s) do not resolve against the live Copilot catalogue.`, ""); + lines.push("| Workflow | Field | Configured | Cause |", "|---|---|---|---|"); + for (const result of rows) lines.push(`| ${workflowCell(result)} | ${code(result.field)} | ${code(result.configured)} | ${code(result.cause)} |`); + lines.push(""); + } + return lines; +} + +/** + * Render the issue section: unresolvable pins first under "Action required" (omitted + * when there are none), then the catalogue status and counts. Headers are h3 or lower. + * @param {ReturnType} report + * @returns {string} + */ +function renderModelPinSummary(report) { + const lines = renderActionRequired(report); + lines.push("### Workflow Model Pins", ""); + const catalogue = report.catalogue; + lines.push(catalogue.available ? `Live Copilot catalogue: ${catalogue.model_count} model(s).` : `Live Copilot catalogue: **unavailable** (${catalogue.error || "empty model list"}); Copilot pins could not be checked.`, ""); + lines.push("| Result | Pins |", "|---|---:|"); + for (const status of STATUS_ORDER) lines.push(`| ${STATUS_LABELS[status]} | ${report.counts[status]} |`); + lines.push(`| **total** | **${report.counts.total}** |`, ""); + /** @type {Map} */ + const reasons = new Map(); + for (const result of report.pins) if (result.status === "not_checked") reasons.set(result.reason || "", (reasons.get(result.reason || "") || 0) + 1); + if (reasons.size > 0) { + lines.push("Not checked:", ""); + for (const [reason, count] of [...reasons].sort((left, right) => right[1] - left[1] || (left[0] < right[0] ? -1 : 1))) lines.push(`- ${count} × ${cell(reason)}`); + lines.push(""); + } + if (report.skipped_imports.length > 0 || report.errors.length > 0) lines.push(`Not scanned: ${report.skipped_imports.length} remote import(s), ${report.errors.length} unreadable file(s) (listed in \`model-pins.md\`).`, ""); + lines.push(`Every pin of the ${report.workflows.length} workflow(s) with its classification is in \`model-pins.md\` and \`model-pins.json\` (run artifact \`model-pins\`).`); + return lines.join("\n") + "\n"; +} + +/** + * Render the full report: the summary followed by every pin with its classification. + * @param {ReturnType} report + * @returns {string} + */ +function renderModelPinReport(report) { + const lines = [renderModelPinSummary(report)]; + lines.push("### All Workflow Model Pins", ""); + lines.push("| Workflow | Field | Configured | Provider | Result | Detail |", "|---|---|---|---|---|---|"); + const ordered = [...report.pins].sort((left, right) => STATUS_ORDER.indexOf(left.status) - STATUS_ORDER.indexOf(right.status) || (left.workflow < right.workflow ? -1 : left.workflow > right.workflow ? 1 : 0)); + for (const result of ordered) { + lines.push(`| ${workflowCell(result)} | ${code(result.field)} | ${code(result.configured)} | ${cell(result.provider)} | ${STATUS_LABELS[result.status]} | ${detailCell(result)} |`); + } + if (report.skipped_imports.length > 0 || report.errors.length > 0) { + lines.push("", "### Not Scanned", ""); + for (const item of report.skipped_imports) lines.push(`- ${code(item.workflow)}: ${item.reason} ${code(item.import)}`); + for (const item of report.errors) lines.push(`- ${code(item.file)}: ${cell(item.error)}`); + } + return lines.join("\n") + "\n"; +} + +/** + * @param {string} file + * @returns {any} + */ +function readReflect(file) { + try { + return JSON.parse(fs.readFileSync(file, "utf8")); + } catch (error) { + return { endpoints: [], error: `cannot read ${file}: ${error instanceof Error ? error.message : String(error)}` }; + } +} + +/** + * @param {string[]} argv + * @returns {Record} + */ +function parseArgs(argv) { + /** @type {Record} */ + const args = {}; + for (let index = 0; index < argv.length; index += 1) { + const match = argv[index].match(/^--([a-z-]+)(?:=(.*))?$/); + if (!match) throw new Error(`unexpected argument: ${argv[index]}`); + args[match[1]] = match[2] !== undefined ? match[2] : argv[++index] || ""; + } + return args; +} + +/** + * CLI entry point. + * @param {string[]} argv + * @returns {ReturnType} + */ +function main(argv = process.argv.slice(2)) { + const args = parseArgs(argv); + if (!args.reflect) throw new Error("--reflect is required"); + const repoRoot = path.resolve(args.repo || process.cwd()); + const collected = collectModelPins({ repoRoot, workflowsDir: args["workflows-dir"] }); + const report = buildModelPinReport({ ...collected, reflectData: readReflect(args.reflect) }); + const summary = renderModelPinSummary(report); + if (args["out-dir"]) { + fs.mkdirSync(args["out-dir"], { recursive: true }); + fs.writeFileSync(path.join(args["out-dir"], "model-pins.json"), JSON.stringify(report, null, 2) + "\n"); + fs.writeFileSync(path.join(args["out-dir"], "model-pins.md"), renderModelPinReport(report)); + fs.writeFileSync(path.join(args["out-dir"], "model-pins-summary.md"), summary); + } + process.stdout.write(summary); + return report; +} + +if (require.main === module) { + main(); +} + +module.exports = { + buildModelPinReport, + classifyPin, + collectModelPins, + collectWorkflowPins, + describeEngine, + engineProvider, + isRemoteImport, + main, + parseMarkdownFrontmatter, + renderModelPinReport, + renderModelPinSummary, + REASON_AUTO, + REASON_BYOK, + REASON_EXPRESSION, + REASON_INHERITED, + REASON_PROVIDER, +}; diff --git a/actions/setup/js/model_pin_check.test.cjs b/actions/setup/js/model_pin_check.test.cjs new file mode 100644 index 00000000000..5f6f1b43798 --- /dev/null +++ b/actions/setup/js/model_pin_check.test.cjs @@ -0,0 +1,207 @@ +// @ts-check +import { afterEach, describe, expect, it } from "vitest"; +import fs from "fs"; +import os from "os"; +import path from "path"; +import { createRequire } from "module"; + +const require = createRequire(import.meta.url); +const { buildModelPinReport, collectModelPins, main, renderModelPinReport, renderModelPinSummary } = require("./model_pin_check.cjs"); +// Recorded /reflect from Design Decision Gate run 38945917145 (40 Copilot models; no claude-opus-5). +const { ddgReflect, incompleteDiscovery } = require("./fixtures/model_resolution_reflect.cjs"); + +/** @type {string[]} */ +const tempDirs = []; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +/** + * Write small fixture workflows into a temporary repository. + * @param {Record} files repository-relative path → content + * @returns {string} repository root + */ +function fixtureRepo(files) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "model-pin-check-")); + tempDirs.push(root); + fs.mkdirSync(path.join(root, ".github", "workflows"), { recursive: true }); + for (const [rel, content] of Object.entries(files)) { + const file = path.join(root, rel); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, content); + } + return root; +} + +/** + * @param {Record} files + * @param {any} [reflectData] + */ +function check(files, reflectData = ddgReflect()) { + const collected = collectModelPins({ repoRoot: fixtureRepo(files) }); + return buildModelPinReport({ ...collected, reflectData }); +} + +/** + * @param {ReturnType} report + * @param {string} field + */ +function pin(report, field) { + const found = report.pins.find(result => result.field === field); + expect(found, `pin ${field}`).toBeDefined(); + return /** @type {NonNullable} */ found; +} + +describe("model_pin_check acceptance cases", () => { + it("1. a retired copilot/claude-opus-5 pin is unresolvable and listed under Action required", () => { + const report = check({ ".github/workflows/gate.md": "---\nengine: pi\nmodel: copilot/claude-opus-5\n---\n# Gate\n" }); + const result = pin(report, "model"); + expect(result).toMatchObject({ workflow: ".github/workflows/gate.md", configured: "copilot/claude-opus-5", status: "unresolvable", cause: "model_unavailable" }); + expect(report.action_required).toEqual([result]); + const summary = renderModelPinSummary(report); + expect(summary.startsWith("### Action required")).toBe(true); + expect(summary).toContain("| `.github/workflows/gate.md` | `model` | `copilot/claude-opus-5` | `model_unavailable` |"); + }); + + it("2. copilot/opus resolves to the newest live Opus", () => { + const report = check({ ".github/workflows/opus.md": "---\nengine: copilot\nmodel: copilot/opus\n---\n" }); + expect(pin(report, "model")).toMatchObject({ status: "alias", resolved: "copilot/claude-opus-5.5" }); + expect(renderModelPinReport(report)).toContain("| alias resolves | → `copilot/claude-opus-5.5` |"); + }); + + it("3. gpt-5-mini (an alias name and a live id) is exact", () => { + const report = check({ ".github/workflows/mini.md": "---\nengine: copilot\nmodel: gpt-5-mini\n---\n" }); + expect(pin(report, "model")).toMatchObject({ status: "exact", resolved: "gpt-5-mini", cause: null }); + }); + + it("4. an expression pin is not checked with reason expression", () => { + const report = check({ ".github/workflows/expr.md": "---\nengine:\n id: copilot\n model: ${{ vars.GH_AW_MODEL_AGENT_COPILOT }}\n---\n" }); + expect(pin(report, "engine.model")).toMatchObject({ status: "not_checked", reason: "expression" }); + }); + + it("5. a Claude workflow on the Anthropic provider is not checked", () => { + const report = check({ ".github/workflows/claude.md": "---\nengine: claude\nmodel: claude-sonnet-4-6\n---\n" }); + expect(pin(report, "model")).toMatchObject({ provider: "anthropic", status: "not_checked", reason: "provider not in the Copilot catalogue" }); + }); + + it("6. a workflow models: alias whose every candidate is gone is unresolvable", () => { + const report = check({ ".github/workflows/alias.md": "---\nengine: copilot\nmodel: retired\nmodels:\n retired:\n - copilot/claude-opus-5\n - copilot/claude-opus-4.8\n---\n" }); + expect(pin(report, "models.retired")).toMatchObject({ status: "unresolvable", cause: "model_unavailable" }); + expect(pin(report, "model")).toMatchObject({ status: "unresolvable", cause: "model_unavailable" }); + expect(report.action_required).toHaveLength(2); + }); + + it("7. no pins gives an empty table and no Action required section", () => { + const report = check({ ".github/workflows/plain.md": "---\non: push\n---\n# Plain\n" }); + expect(report.pins).toEqual([]); + expect(report.counts).toEqual({ total: 0, exact: 0, alias: 0, unresolvable: 0, not_checked: 0 }); + const full = renderModelPinReport(report); + expect(full).not.toContain("Action required"); + expect(full).toMatch(/\| Workflow \| Field \| Configured \| Provider \| Result \| Detail \|\n\|---\|---\|---\|---\|---\|---\|\n$/); + }); +}); + +describe("model_pin_check collection and rules", () => { + it("collects engine, detection, evals, sub-agent and imported pins with file and field", () => { + const report = check({ + ".github/workflows/full.md": [ + "---", + "engine:", + " id: copilot", + " model: gpt-5-mini", + " fallback-models: [copilot/claude-opus-5]", + "imports:", + " - shared/models.md", + " - ../agents/helper.md", + "safe-outputs:", + " threat-detection:", + " model: sonnet", + "evals:", + " - question: ok?", + " model: mini", + "permissions:", + " models: read", + "---", + "# Full", + "", + "## agent: `nano`", + "---", + "model: inherited", + "---", + "Help.", + ].join("\n"), + ".github/workflows/shared/models.md": "---\nmodels:\n providers:\n copilot:\n models: {}\n team-fast:\n - copilot/claude-haiku-4.5\n---\n", + ".github/agents/helper.md": "---\nname: helper\nmodel: copilot/claude-opus-4.8\n---\nHelper.\n", + }); + const rows = report.pins.map(result => [result.source, result.field, result.status]); + expect(rows).toEqual( + expect.arrayContaining([ + [".github/workflows/full.md", "engine.model", "exact"], + [".github/workflows/full.md", "engine.fallback-models[0]", "unresolvable"], + [".github/workflows/full.md", "safe-outputs.threat-detection.model", "alias"], + [".github/workflows/full.md", "evals[0].model", "alias"], + [".github/workflows/full.md", "## agent: 'nano' model", "not_checked"], + [".github/agents/helper.md", "model (imported agent file)", "unresolvable"], + [".github/workflows/shared/models.md", "models.team-fast", "alias"], + ]) + ); + expect(report.pins.some(result => result.field.includes("providers"))).toBe(false); + expect(pin(report, "## agent: 'nano' model").reason).toMatch(/^sentinel `inherited`/); + }); + + it("resolves nested imports like the compiler (shared/ paths from the workflow directory)", () => { + const report = check({ + ".github/workflows/nested.md": "---\nengine: copilot\nimports:\n - shared/a.md\n---\n", + ".github/workflows/shared/a.md": "---\nimports:\n - shared/b.md\n - ./c.md\n---\n", + ".github/workflows/shared/b.md": "---\nmodels:\n b-alias: [copilot/gpt-5-mini]\n---\n", + ".github/workflows/shared/c.md": "---\nmodels:\n c-alias: [copilot/gpt-5-mini]\n---\n", + }); + expect(report.errors).toEqual([]); + expect(pin(report, "models.b-alias").source).toBe(".github/workflows/shared/b.md"); + expect(pin(report, "models.c-alias").source).toBe(".github/workflows/shared/c.md"); + }); + + it("does not check Copilot BYOK pins", () => { + const report = check({ ".github/workflows/byok.md": "---\nengine:\n id: copilot\n env:\n COPILOT_PROVIDER_BASE_URL: https://example.invalid/openai/v1\nmodel: o4-mini-aw\n---\n" }); + expect(pin(report, "model")).toMatchObject({ status: "not_checked", reason: expect.stringContaining("BYOK") }); + }); + + it("honours a copilot/ prefix over the engine provider and keeps Claude copilot/auto as a sentinel", () => { + const report = check({ ".github/workflows/claude.md": "---\nengine:\n id: claude\n model: copilot/auto\n fallback-models: [copilot/claude-sonnet-4.6, claude-sonnet-4-6]\n---\n" }); + expect(pin(report, "engine.fallback-models[0]")).toMatchObject({ provider: "copilot", status: "unresolvable", cause: "model_unavailable" }); + expect(pin(report, "engine.model")).toMatchObject({ status: "not_checked", reason: expect.stringMatching(/^sentinel `auto`/) }); + // A bare fallback follows the engine provider, which a copilot/ main model selects. + expect(pin(report, "engine.fallback-models[1]")).toMatchObject({ provider: "copilot", status: "unresolvable", cause: "model_unavailable" }); + }); + + it("reports catalog_unavailable when /reflect has no Copilot model list", () => { + const report = check({ ".github/workflows/mini.md": "---\nengine: copilot\nmodel: gpt-5-mini\n---\n" }, incompleteDiscovery()); + expect(pin(report, "model")).toMatchObject({ status: "unresolvable", cause: "catalog_unavailable" }); + const summary = renderModelPinSummary(report); + expect(summary.startsWith("### Action required")).toBe(true); + expect(summary).toContain("1 Copilot workflow model pin(s) could not be checked"); + expect(summary).not.toContain("| Workflow | Field | Configured | Cause |"); + expect(summary).toContain("Live Copilot catalogue: **unavailable**"); + }); + + it("main writes the JSON, full and summary reports", () => { + const root = fixtureRepo({ ".github/workflows/gate.md": "---\nengine: pi\nmodel: copilot/claude-opus-5\n---\n" }); + const reflectFile = path.join(root, "reflect.json"); + fs.writeFileSync(reflectFile, JSON.stringify(ddgReflect())); + const outDir = path.join(root, "out"); + const write = process.stdout.write; + /** @type {string[]} */ + const printed = []; + process.stdout.write = /** @type {any} */ chunk => printed.push(String(chunk)); + try { + main(["--reflect", reflectFile, "--out-dir", outDir, "--repo", root]); + } finally { + process.stdout.write = write; + } + const json = JSON.parse(fs.readFileSync(path.join(outDir, "model-pins.json"), "utf8")); + expect(json.counts.unresolvable).toBe(1); + expect(fs.readFileSync(path.join(outDir, "model-pins.md"), "utf8")).toContain("### All Workflow Model Pins"); + expect(fs.readFileSync(path.join(outDir, "model-pins-summary.md"), "utf8")).toBe(printed.join("")); + }); +}); From 64f48c4e8628deeb0e22f24af65039b8d9b9a0e7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:03:03 +0000 Subject: [PATCH 3/6] Make model pin check post-step non-fatal (continue-on-error) Co-authored-by: SivaKesava1 <11771739+SivaKesava1@users.noreply.github.com> --- .github/workflows/daily-model-inventory.lock.yml | 5 +++-- .github/workflows/daily-model-inventory.md | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-model-inventory.lock.yml b/.github/workflows/daily-model-inventory.lock.yml index 7bae62c1f93..3ba306e0903 100644 --- a/.github/workflows/daily-model-inventory.lock.yml +++ b/.github/workflows/daily-model-inventory.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"06eb59a5454e80efc72f7ca32a152ac8809728cd9e8e0d30a32f539b744fd7bd","body_hash":"93a1c9b06366c3b548bf480b4db8240ece21b171b6ebe98362fc9d384c3a10e2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8d793a2e0f8f3fd0d237a18b198ba3de8e62c31d63d71d708ed83a2f452c3be9","body_hash":"93a1c9b06366c3b548bf480b4db8240ece21b171b6ebe98362fc9d384c3a10e2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GEMINI_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62","digest":"sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62","digest":"sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62","digest":"sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1208,7 +1208,8 @@ jobs: if [ ! -f /tmp/gh-aw/agent_output.json ]; then echo '{"items":[]}' > /tmp/gh-aw/agent_output.json fi - - if: always() + - continue-on-error: true + if: always() name: Check workflow model pins run: | set -euo pipefail diff --git a/.github/workflows/daily-model-inventory.md b/.github/workflows/daily-model-inventory.md index 56a5854fb59..00140ca360b 100644 --- a/.github/workflows/daily-model-inventory.md +++ b/.github/workflows/daily-model-inventory.md @@ -344,6 +344,7 @@ steps: post-steps: - name: Check workflow model pins if: always() + continue-on-error: true shell: bash run: | set -euo pipefail From c4e23971561a5774379547e26ab63459fceff781 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:12:02 +0000 Subject: [PATCH 4/6] Fix model pin checker precedence and execution safety Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- .../workflows/daily-model-inventory.lock.yml | 4 +- .github/workflows/daily-model-inventory.md | 2 +- actions/setup/js/model_pin_check.cjs | 189 ++++++++++++++++-- actions/setup/js/model_pin_check.test.cjs | 179 +++++++++++++++++ 4 files changed, 351 insertions(+), 23 deletions(-) diff --git a/.github/workflows/daily-model-inventory.lock.yml b/.github/workflows/daily-model-inventory.lock.yml index 3ba306e0903..7aa8bbb1587 100644 --- a/.github/workflows/daily-model-inventory.lock.yml +++ b/.github/workflows/daily-model-inventory.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8d793a2e0f8f3fd0d237a18b198ba3de8e62c31d63d71d708ed83a2f452c3be9","body_hash":"93a1c9b06366c3b548bf480b4db8240ece21b171b6ebe98362fc9d384c3a10e2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"19697d0832e356947c1ef23e7b2faf659064387aa708c8a3c412e875a1c6e78a","body_hash":"93a1c9b06366c3b548bf480b4db8240ece21b171b6ebe98362fc9d384c3a10e2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GEMINI_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62","digest":"sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62","digest":"sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62","digest":"sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1219,7 +1219,7 @@ jobs: if [ -s "$candidate" ]; then REFLECT="$candidate"; break; fi done echo "Checking workflow model pins against $REFLECT" - node "$GITHUB_WORKSPACE/actions/setup/js/model_pin_check.cjs" --reflect "$REFLECT" --out-dir "$OUT" --repo "$GITHUB_WORKSPACE" > /dev/null + node "${RUNNER_TEMP}/gh-aw/actions/model_pin_check.cjs" --reflect "$REFLECT" --out-dir "$OUT" --repo "$GITHUB_WORKSPACE" > /dev/null cat "$OUT/model-pins.md" >> "$GITHUB_STEP_SUMMARY" shell: bash - if: always() diff --git a/.github/workflows/daily-model-inventory.md b/.github/workflows/daily-model-inventory.md index 00140ca360b..17dc8e5d27d 100644 --- a/.github/workflows/daily-model-inventory.md +++ b/.github/workflows/daily-model-inventory.md @@ -354,7 +354,7 @@ post-steps: if [ -s "$candidate" ]; then REFLECT="$candidate"; break; fi done echo "Checking workflow model pins against $REFLECT" - node "$GITHUB_WORKSPACE/actions/setup/js/model_pin_check.cjs" --reflect "$REFLECT" --out-dir "$OUT" --repo "$GITHUB_WORKSPACE" > /dev/null + node "${RUNNER_TEMP}/gh-aw/actions/model_pin_check.cjs" --reflect "$REFLECT" --out-dir "$OUT" --repo "$GITHUB_WORKSPACE" > /dev/null cat "$OUT/model-pins.md" >> "$GITHUB_STEP_SUMMARY" - name: Upload workflow model pin report diff --git a/actions/setup/js/model_pin_check.cjs b/actions/setup/js/model_pin_check.cjs index 7f6fa9d16ec..7869def35ec 100644 --- a/actions/setup/js/model_pin_check.cjs +++ b/actions/setup/js/model_pin_check.cjs @@ -35,7 +35,7 @@ const fs = require("fs"); const path = require("path"); const { parseDocument } = require("./work_queue_yaml.cjs"); const { extractInlineSubAgents } = require("./extract_inline_sub_agents.cjs"); -const { buildProviderCatalog, canonicalModelProvider, isLiveModelId, loadBuiltinModelAliases, ModelAliasResolutionError, resolveConfiguredModel, splitModelIdentifier } = require("./resolve_model_alias.cjs"); +const { buildProviderCatalog, canonicalModelProvider, isLiveModelId, loadBuiltinModelAliases, ModelAliasResolutionError, resolveConfiguredModel, resolveModelAlias, splitModelIdentifier } = require("./resolve_model_alias.cjs"); /** Keys under `models:` that are not alias names (see reservedModelsKeys in pkg/workflow/model_aliases.go). */ const RESERVED_MODELS_KEYS = new Set(["providers", "default-ai-credits-pricing", "allowed", "blocked"]); @@ -49,6 +49,7 @@ const MODEL_PREFIX_PROVIDER_ENGINES = new Set(["claude", "codex", "pi"]); const REASON_EXPRESSION = "expression"; const REASON_PROVIDER = "provider not in the Copilot catalogue"; const REASON_BYOK = "provider not in the Copilot catalogue (BYOK via COPILOT_PROVIDER_BASE_URL)"; +const REASON_DETECTION_DISABLED = "threat detection disabled by the main workflow"; const REASON_INHERITED = "sentinel `inherited`: the sub-agent uses its parent's model, which is checked in its own row"; const REASON_AUTO = "sentinel `auto`: the gateway selects the model"; @@ -65,6 +66,7 @@ const STATUS_ORDER = ["unresolvable", "alias", "exact", "not_checked"]; * provider: string, * engine: string, * autoSentinel: boolean, + * inactiveReason?: string, * aliasEntries?: string[], * aliasMap: Record, * }} ModelPin @@ -262,7 +264,7 @@ function loadWorkflowFiles(workflowFile, repoRoot) { } /** - * @typedef {{ id: string, explicitProvider: string, byok: boolean, definitionProvider: string }} EngineInfo + * @typedef {{ id: string, explicitProvider: string, byok: boolean, definitionProvider: string, env?: Record }} EngineInfo */ /** @@ -278,8 +280,82 @@ function describeEngine(engine) { const provider = engine.provider; const explicitProvider = stringValue(engine["model-provider"]) || (typeof provider === "string" ? provider.trim() : isObject(provider) && !isObject(engine.behaviors) ? stringValue(provider.id) : ""); const definitionProvider = isObject(provider) ? stringValue(provider.name) || stringValue(provider.id) : ""; - const byok = isObject(engine.env) && stringValue(engine.env.COPILOT_PROVIDER_BASE_URL) !== ""; - return { id, explicitProvider, byok, definitionProvider }; + const env = isObject(engine.env) ? engine.env : {}; + const byok = stringValue(env.COPILOT_PROVIDER_BASE_URL) !== ""; + return { id, explicitProvider, byok, definitionProvider, env }; +} + +/** + * @param {EngineInfo | null} defaults + * @param {EngineInfo | null} overrides + * @returns {EngineInfo | null} + */ +function mergeEngineInfo(defaults, overrides) { + if (!defaults) return overrides; + if (!overrides) return defaults; + const env = { ...(defaults.env || {}), ...(overrides.env || {}) }; + return { + id: overrides.id || defaults.id, + explicitProvider: overrides.explicitProvider || defaults.explicitProvider, + byok: stringValue(env.COPILOT_PROVIDER_BASE_URL) !== "", + definitionProvider: overrides.definitionProvider || defaults.definitionProvider, + env, + }; +} + +/** + * @param {unknown} engine + * @param {unknown} topLevelModel + * @param {unknown} fallbackModel + * @returns {string} + */ +function effectiveEngineModel(engine, topLevelModel, fallbackModel) { + return (isObject(engine) && stringValue(engine.model)) || stringValue(topLevelModel) || stringValue(fallbackModel); +} + +/** + * @param {LoadedFile} file + * @returns {{ present: boolean, value: any }} + */ +function threatDetectionConfig(file) { + const safeOutputs = file.frontmatter["safe-outputs"]; + if (!isObject(safeOutputs) || !Object.prototype.hasOwnProperty.call(safeOutputs, "threat-detection")) return { present: false, value: undefined }; + return { present: true, value: safeOutputs["threat-detection"] }; +} + +/** + * Imports provide defaults, while any main-workflow threat-detection value replaces or + * overrides them. In particular, an explicit false in the main workflow disables an + * imported detector. + * @param {LoadedFile[]} imports + * @param {LoadedFile} main + * @returns {any} + */ +function effectiveThreatDetectionConfig(imports, main) { + const imported = imports.map(threatDetectionConfig).find(config => config.present); + const mainConfig = threatDetectionConfig(main); + if (!mainConfig.present) return imported?.value; + if (!isObject(mainConfig.value) || !isObject(imported?.value)) return mainConfig.value; + return { ...imported.value, ...mainConfig.value }; +} + +/** + * @param {EngineInfo} engine + * @param {string} provider + * @param {Record} definitionDetectionEngines + * @returns {string} + */ +function threatDetectionEngineID(engine, provider, definitionDetectionEngines) { + const id = engine.id || "copilot"; + if (id === "pi") return "copilot"; + if (Object.prototype.hasOwnProperty.call(ENGINE_DEFAULT_PROVIDERS, id)) { + if (provider === "copilot") return "copilot"; + if (provider === "anthropic") return "claude"; + if (provider === "openai") return "codex"; + return id === "copilot" || id === "claude" || id === "codex" ? id : "copilot"; + } + const declared = definitionDetectionEngines[id]; + return declared === "copilot" || declared === "claude" || declared === "codex" ? declared : "copilot"; } /** @@ -302,6 +378,48 @@ function engineProvider(engine, mainModel, definitionProviders) { return mainPrefix || "copilot"; } +/** + * Whether resolving a model alias could depend on a GitHub Actions expression before + * reaching a statically resolvable candidate. A live exact model ID always takes + * precedence over an alias with the same name. + * @param {string} model + * @param {Record} aliasMap + * @param {string[]} catalog + * @returns {boolean} + */ +function aliasResolutionUsesExpression(model, aliasMap, catalog) { + const { base } = splitModelIdentifier(model); + const slash = base.indexOf("/"); + const aliasKey = slash > 0 ? base.slice(slash + 1) : base; + if (isLiveModelId(aliasKey, catalog)) return false; + + /** + * @param {string} key + * @param {Set} visited + * @returns {boolean} + */ + function hasExpressionBeforeResolution(key, visited) { + if (isLiveModelId(key, catalog) || visited.has(key) || !Array.isArray(aliasMap[key])) return false; + const nextVisited = new Set(visited); + nextVisited.add(key); + for (const entry of aliasMap[key]) { + if (entry.includes("${{")) return true; + const { base: entryBase } = splitModelIdentifier(entry); + const entrySlash = entryBase.indexOf("/"); + const entryProvider = entrySlash > 0 ? canonicalModelProvider(entryBase.slice(0, entrySlash)) : ""; + const entryAlias = entrySlash > 0 ? entryBase.slice(entrySlash + 1) : entryBase; + if (!entryProvider && Object.prototype.hasOwnProperty.call(aliasMap, entryAlias)) { + if (hasExpressionBeforeResolution(entryAlias, nextVisited)) return true; + } + if ((!entryProvider || entryProvider === "copilot") && isLiveModelId(entryAlias, catalog)) return false; + if (resolveModelAlias(entry, aliasMap, catalog)) return false; + } + return false; + } + + return hasExpressionBeforeResolution(aliasKey, new Set()); +} + /** * Collect the model pins of one workflow. * @param {string} workflowFile absolute path @@ -316,7 +434,7 @@ function collectWorkflowPins(workflowFile, options) { const main = files[0]; const imports = files.slice(1).filter(file => !file.agentFile); - // Merged alias map: built-ins, then imports (first import wins), then the main workflow. + // Merged alias map: built-ins, then imports (first definition wins), then the main workflow. /** @type {Record} */ const aliasMap = { ...builtinAliases }; /** @type {Set} */ @@ -330,36 +448,55 @@ function collectWorkflowPins(workflowFile, options) { if (RESERVED_MODELS_KEYS.has(name) || !Array.isArray(value)) continue; const entries = value.filter(entry => typeof entry === "string" && entry.trim()).map(entry => entry.trim()); if (entries.length === 0) continue; - aliasDefinitions.push({ file, name, entries }); if (file === main) { aliasMap[name] = entries; - } else if (!importedAliasKeys.has(name)) { + aliasDefinitions.push({ file, name, entries }); + } else if (!Object.prototype.hasOwnProperty.call(aliasMap, name) && !importedAliasKeys.has(name)) { importedAliasKeys.add(name); aliasMap[name] = entries; + aliasDefinitions.push({ file, name, entries }); } } } /** @type {Record} */ const definitionProviders = {}; + /** @type {Record} */ + const definitionDetectionEngines = {}; /** @type {EngineInfo | null} */ - let mainEngine = describeEngine(main.frontmatter.engine); + let importedEngine = null; + let importedModel = ""; for (const file of imports) { const info = describeEngine(file.frontmatter.engine); if (!info) continue; if (info.id && info.definitionProvider && !definitionProviders[info.id]) definitionProviders[info.id] = info.definitionProvider; - if (!mainEngine && info.id && !isObject(file.frontmatter.engine?.behaviors)) mainEngine = info; + const detectionEngine = stringValue(file.frontmatter.engine?.["detection-engine"]); + if (info.id && detectionEngine && !definitionDetectionEngines[info.id]) definitionDetectionEngines[info.id] = detectionEngine; + if (!importedEngine && info.id && !isObject(file.frontmatter.engine?.behaviors)) { + importedEngine = info; + importedModel = effectiveEngineModel(file.frontmatter.engine, file.frontmatter.model, ""); + } } - if (mainEngine && !mainEngine.id) mainEngine = { ...mainEngine, id: "copilot" }; - const engine = mainEngine || { id: "copilot", explicitProvider: "", byok: false, definitionProvider: "" }; - const mainModel = stringValue(main.frontmatter.model) || (isObject(main.frontmatter.engine) ? stringValue(main.frontmatter.engine.model) : ""); + const configuredMainEngine = describeEngine(main.frontmatter.engine); + const engine = configuredMainEngine?.id ? configuredMainEngine : mergeEngineInfo(importedEngine, configuredMainEngine) || { id: "copilot", explicitProvider: "", byok: false, definitionProvider: "" }; + const mainModel = effectiveEngineModel(main.frontmatter.engine, main.frontmatter.model, importedModel); const mainProvider = engineProvider(engine, mainModel, definitionProviders); - const threatDetection = isObject(main.frontmatter["safe-outputs"]) ? main.frontmatter["safe-outputs"]["threat-detection"] : undefined; + const threatDetection = effectiveThreatDetectionConfig(imports, main); + const detectionDisabled = threatDetection === false || (isObject(threatDetection) && (threatDetection.enabled === false || threatDetection.engine === false)); const detectionEngineValue = isObject(threatDetection) ? (threatDetection.engine !== undefined && threatDetection.engine !== false ? threatDetection.engine : threatDetection["engine-config"]) : undefined; const detectionInfo = describeEngine(detectionEngineValue); - const detectionEngine = detectionInfo ? { ...detectionInfo, id: detectionInfo.id || engine.id } : engine; - const detectionModel = (isObject(threatDetection) && stringValue(threatDetection.model)) || (isObject(detectionEngineValue) ? stringValue(detectionEngineValue.model) : "") || mainModel; + const detectionModel = (isObject(detectionEngineValue) && stringValue(detectionEngineValue.model)) || (isObject(threatDetection) && stringValue(threatDetection.model)) || mainModel; + const detectionEngineID = detectionInfo?.id + ? threatDetectionEngineID(detectionInfo, engineProvider(detectionInfo, detectionModel, definitionProviders), definitionDetectionEngines) + : threatDetectionEngineID(engine, mainProvider, definitionDetectionEngines); + const detectionEnv = detectionEngineID === engine.id ? { ...(engine.env || {}), ...(detectionInfo?.env || {}) } : detectionInfo?.env || {}; + const detectionEngine = { + ...(detectionInfo || { id: detectionEngineID, explicitProvider: "", definitionProvider: "" }), + id: detectionEngineID, + env: detectionEnv, + byok: stringValue(detectionEnv.COPILOT_PROVIDER_BASE_URL) !== "", + }; const detectionProvider = engineProvider(detectionEngine, detectionModel, definitionProviders); /** @type {ModelPin[]} */ @@ -368,7 +505,7 @@ function collectWorkflowPins(workflowFile, options) { * @param {LoadedFile} file * @param {string} field * @param {unknown} value - * @param {{ engine: EngineInfo, provider: string, autoSentinel?: boolean }} scope + * @param {{ engine: EngineInfo, provider: string, autoSentinel?: boolean, inactiveReason?: string }} scope */ const addPin = (file, field, value, scope) => { const model = stringValue(value); @@ -382,11 +519,12 @@ function collectWorkflowPins(workflowFile, options) { provider: scope.provider, engine: scope.engine.id || "copilot", autoSentinel: scope.autoSentinel ?? scope.engine.id !== "copilot", + ...(scope.inactiveReason ? { inactiveReason: scope.inactiveReason } : {}), aliasMap, }); }; const mainScope = { engine, provider: mainProvider }; - const detectionScope = { engine: detectionEngine, provider: detectionProvider }; + const detectionScope = { engine: detectionEngine, provider: detectionProvider, ...(detectionDisabled ? { inactiveReason: REASON_DETECTION_DISABLED } : {}) }; const copilotScope = { engine: { id: "copilot", explicitProvider: "", byok: false, definitionProvider: "" }, provider: "copilot" }; /** @@ -432,10 +570,17 @@ function collectWorkflowPins(workflowFile, options) { const evals = fm.evals; if (Array.isArray(evals)) { - evals.forEach((question, index) => isObject(question) && addPin(file, `evals[${index}].model`, question.model, mainScope)); + evals.forEach((question, index) => isObject(question) && addPin(file, `evals[${index}].model`, question.model, scopeForModel(question.model))); } else if (isObject(evals)) { - addPin(file, "evals.model", evals.model, mainScope); - if (Array.isArray(evals.questions)) evals.questions.forEach((question, index) => isObject(question) && addPin(file, `evals.questions[${index}].model`, question.model, mainScope)); + addPin(file, "evals.model", evals.model, scopeForModel(evals.model)); + if (Array.isArray(evals.questions)) evals.questions.forEach((question, index) => isObject(question) && addPin(file, `evals.questions[${index}].model`, question.model, scopeForModel(question.model || evals.model))); + } + + /** + * @param {unknown} model + */ + function scopeForModel(model) { + return { engine, provider: engineProvider(engine, stringValue(model) || mainModel, definitionProviders) }; } if (Array.isArray(fm.enclaves)) { @@ -534,12 +679,16 @@ function classifyPin(pin, reflectData) { if (values.some(value => value.includes("${{"))) return { ...result, reason: REASON_EXPRESSION }; const name = pin.kind === "model" ? bareModelName(pin.model) : ""; if (name === "inherited") return { ...result, reason: REASON_INHERITED }; + if (pin.inactiveReason) return { ...result, reason: pin.inactiveReason }; if (pin.provider === "byok") return { ...result, reason: REASON_BYOK }; const prefix = pin.kind === "model" ? modelPrefixProvider(pin.model) : ""; if ((prefix || pin.provider) !== "copilot") return { ...result, provider: prefix || pin.provider, reason: REASON_PROVIDER }; if (name === "auto" && pin.autoSentinel) return { ...result, provider: "copilot", reason: REASON_AUTO }; const catalog = buildProviderCatalog({ reflectData, provider: "copilot" }); + if (pin.kind === "model" && aliasResolutionUsesExpression(pin.model, pin.aliasMap, catalog)) { + return { ...result, provider: "copilot", reason: REASON_EXPRESSION }; + } try { // An alias definition is checked with its own entries, even when another file overrides the name. const aliasMap = pin.kind === "alias-definition" ? { ...pin.aliasMap, [pin.model]: pin.aliasEntries || [] } : pin.aliasMap; diff --git a/actions/setup/js/model_pin_check.test.cjs b/actions/setup/js/model_pin_check.test.cjs index 5f6f1b43798..75eb1ffdb1b 100644 --- a/actions/setup/js/model_pin_check.test.cjs +++ b/actions/setup/js/model_pin_check.test.cjs @@ -162,6 +162,185 @@ describe("model_pin_check collection and rules", () => { expect(pin(report, "models.c-alias").source).toBe(".github/workflows/shared/c.md"); }); + it("does not let imported aliases override built-ins, but keeps main-workflow overrides", () => { + const report = check({ + ".github/workflows/aliases.md": "---\nimports:\n - shared/aliases.md\nengine: copilot\nmodel: sonnet\n---\n", + ".github/workflows/shared/aliases.md": "---\nmodels:\n sonnet: [copilot/claude-opus-5]\n---\n", + }); + expect(pin(report, "model")).toMatchObject({ status: "alias", resolved: "claude-sonnet-5.5" }); + expect(report.action_required).toEqual([]); + expect(report.pins.some(result => result.field === "models.sonnet")).toBe(false); + + const mainOverride = check({ + ".github/workflows/aliases.md": "---\nimports:\n - shared/aliases.md\nengine: copilot\nmodel: sonnet\nmodels:\n sonnet: [copilot/claude-opus-5]\n---\n", + ".github/workflows/shared/aliases.md": "---\nmodels:\n sonnet: [copilot/claude-opus-5]\n---\n", + }); + expect(pin(mainOverride, "model")).toMatchObject({ status: "unresolvable", cause: "model_unavailable" }); + }); + + it("lets an imported engine supply a missing ID while retaining the main model preference", () => { + const report = check({ + ".github/workflows/preference.md": "---\nimports:\n - shared/engine.md\nengine:\n model: claude-sonnet-4-6\n---\n", + ".github/workflows/shared/engine.md": "---\nengine: claude\n---\n", + }); + expect(pin(report, "engine.model")).toMatchObject({ engine: "claude", provider: "anthropic", status: "not_checked", reason: "provider not in the Copilot catalogue" }); + }); + + it("uses engine.model before top-level model and an imported model as fallback", () => { + const explicit = check({ + ".github/workflows/precedence.md": ["---", "engine:", " id: claude", " model: copilot/gpt-5-mini", " fallback-models: [claude-opus-5]", "model: anthropic/claude-sonnet-4-6", "---"].join("\n"), + }); + expect(pin(explicit, "engine.fallback-models[0]")).toMatchObject({ provider: "copilot", status: "unresolvable", cause: "model_unavailable" }); + + const imported = check({ + ".github/workflows/default.md": "---\nimports:\n - shared/default.md\n---\n", + ".github/workflows/shared/default.md": "---\nengine: claude\nmodel: copilot/claude-opus-5\n---\n", + }); + expect(pin(imported, "model")).toMatchObject({ engine: "claude", provider: "copilot", status: "unresolvable", cause: "model_unavailable" }); + }); + + it("uses imported detection configuration unless the main workflow overrides or disables it", () => { + const imported = { + ".github/workflows/detection.md": "---\nengine: claude\nimports:\n - shared/detection.md\n---\n", + ".github/workflows/shared/detection.md": "---\nsafe-outputs:\n threat-detection:\n engine: copilot\n model: claude-opus-5\n---\n", + }; + expect(pin(check(imported), "safe-outputs.threat-detection.model")).toMatchObject({ + engine: "copilot", + provider: "copilot", + status: "unresolvable", + cause: "model_unavailable", + }); + + const overridden = check({ + ...imported, + ".github/workflows/detection.md": "---\nengine: claude\nimports:\n - shared/detection.md\nsafe-outputs:\n threat-detection:\n model: anthropic/claude-sonnet-4-6\n---\n", + }); + expect(pin(overridden, "safe-outputs.threat-detection.model")).toMatchObject({ + provider: "anthropic", + status: "not_checked", + reason: "provider not in the Copilot catalogue", + }); + + const disabled = check({ + ...imported, + ".github/workflows/detection.md": "---\nengine: claude\nimports:\n - shared/detection.md\nsafe-outputs:\n threat-detection: false\n---\n", + }); + expect(pin(disabled, "safe-outputs.threat-detection.model")).toMatchObject({ + status: "not_checked", + reason: "threat detection disabled by the main workflow", + }); + expect(disabled.action_required).toEqual([]); + }); + + it("inherits detector BYOK settings for matching engines and honors an explicit empty override", () => { + const inherited = check({ + ".github/workflows/byok-detection.md": [ + "---", + "engine:", + " id: copilot", + " env:", + " COPILOT_PROVIDER_BASE_URL: https://example.invalid/openai/v1", + "safe-outputs:", + " threat-detection:", + " engine: copilot", + " model: o4-mini-aw", + "---", + ].join("\n"), + }); + expect(pin(inherited, "safe-outputs.threat-detection.model")).toMatchObject({ provider: "byok", status: "not_checked", reason: expect.stringContaining("BYOK") }); + + const overridden = check({ + ".github/workflows/byok-detection.md": [ + "---", + "engine:", + " id: copilot", + " env:", + " COPILOT_PROVIDER_BASE_URL: https://example.invalid/openai/v1", + "safe-outputs:", + " threat-detection:", + " engine:", + " id: copilot", + " env:", + " COPILOT_PROVIDER_BASE_URL: ''", + " model: claude-opus-5", + "---", + ].join("\n"), + }); + expect(pin(overridden, "safe-outputs.threat-detection.model")).toMatchObject({ provider: "copilot", status: "unresolvable", cause: "model_unavailable" }); + }); + + it("uses detection engine.model and the effective detector selected from the main provider", () => { + const explicitModel = check({ + ".github/workflows/detection-model.md": [ + "---", + "engine: claude", + "safe-outputs:", + " threat-detection:", + " model: anthropic/claude-sonnet-4-6", + " engine:", + " id: claude", + " model: copilot/gpt-5-mini", + " fallback-models: [claude-opus-5]", + "---", + ].join("\n"), + }); + expect(pin(explicitModel, "safe-outputs.threat-detection.model")).toMatchObject({ provider: "anthropic", status: "not_checked" }); + expect(pin(explicitModel, "safe-outputs.threat-detection.engine.fallback-models[0]")).toMatchObject({ provider: "copilot", status: "unresolvable", cause: "model_unavailable" }); + + const providerSelected = check({ + ".github/workflows/provider-detection.md": "---\nengine: claude\nmodel: copilot/gpt-5-mini\nsafe-outputs:\n threat-detection:\n model: claude-opus-5\n---\n", + }); + expect(pin(providerSelected, "safe-outputs.threat-detection.model")).toMatchObject({ engine: "copilot", provider: "copilot", status: "unresolvable", cause: "model_unavailable" }); + }); + + it("uses a custom engine's declared detection engine", () => { + const report = check({ + ".github/workflows/custom-detection.md": "---\nimports:\n - shared/custom-engine.md\nengine: cursor\nsafe-outputs:\n threat-detection:\n model: claude-opus-5\n---\n", + ".github/workflows/shared/custom-engine.md": "---\nengine:\n id: cursor\n provider: anthropic\n detection-engine: copilot\n---\n", + }); + expect(pin(report, "safe-outputs.threat-detection.model")).toMatchObject({ engine: "copilot", provider: "copilot", status: "unresolvable", cause: "model_unavailable" }); + }); + + it("resolves eval model providers from the eval override", () => { + const report = check({ + ".github/workflows/eval.md": "---\nengine: claude\nmodel: copilot/gpt-5-mini\nevals:\n model: claude-sonnet-4-6\n---\n", + }); + expect(pin(report, "model")).toMatchObject({ provider: "copilot", status: "exact" }); + expect(pin(report, "evals.model")).toMatchObject({ engine: "claude", provider: "anthropic", status: "not_checked" }); + + const explicitProvider = check({ + ".github/workflows/eval-provider.md": "---\nengine:\n id: claude\n model-provider: openai\nmodel: copilot/gpt-5-mini\nevals:\n model: claude-sonnet-4-6\n---\n", + }); + expect(pin(explicitProvider, "evals.model")).toMatchObject({ provider: "openai", status: "not_checked" }); + + const byok = check({ + ".github/workflows/eval-byok.md": "---\nengine:\n id: copilot\n env:\n COPILOT_PROVIDER_BASE_URL: https://example.invalid/openai/v1\nevals:\n model: o4-mini-aw\n---\n", + }); + expect(pin(byok, "evals.model")).toMatchObject({ provider: "byok", status: "not_checked", reason: expect.stringContaining("BYOK") }); + }); + + it("does not check direct or nested expression aliases, but preserves ordered fallbacks and exact-ID precedence", () => { + const direct = check({ + ".github/workflows/direct.md": '---\nengine: copilot\nmodel: dynamic\nmodels:\n dynamic: ["copilot/${{ vars.MODEL }}"]\n---\n', + }); + expect(pin(direct, "model")).toMatchObject({ status: "not_checked", reason: "expression" }); + + const nested = check({ + ".github/workflows/nested.md": '---\nengine: copilot\nmodel: dynamic\nmodels:\n dynamic: [intermediate]\n intermediate: ["copilot/${{ vars.MODEL }}"]\n---\n', + }); + expect(pin(nested, "model")).toMatchObject({ status: "not_checked", reason: "expression" }); + + const ordered = check({ + ".github/workflows/ordered.md": '---\nengine: copilot\nmodel: dynamic\nmodels:\n dynamic: [copilot/gpt-5-mini, "copilot/${{ vars.MODEL }}"]\n---\n', + }); + expect(pin(ordered, "model")).toMatchObject({ status: "alias", resolved: "gpt-5-mini" }); + + const exact = check({ + ".github/workflows/exact.md": '---\nengine: copilot\nmodel: gpt-5-mini\nmodels:\n gpt-5-mini: ["copilot/${{ vars.MODEL }}"]\n---\n', + }); + expect(pin(exact, "model")).toMatchObject({ status: "exact", resolved: "gpt-5-mini" }); + }); + it("does not check Copilot BYOK pins", () => { const report = check({ ".github/workflows/byok.md": "---\nengine:\n id: copilot\n env:\n COPILOT_PROVIDER_BASE_URL: https://example.invalid/openai/v1\nmodel: o4-mini-aw\n---\n" }); expect(pin(report, "model")).toMatchObject({ status: "not_checked", reason: expect.stringContaining("BYOK") }); From 9f39165f583c4abaf701b1ac3b64a279baf92098 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:45:22 +0000 Subject: [PATCH 5/6] Fix outdated sub-agent smoke fixture Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/smoke_copilot_sub_agents.test.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/setup/js/smoke_copilot_sub_agents.test.cjs b/actions/setup/js/smoke_copilot_sub_agents.test.cjs index 18f9bd8e485..ab2e16967ae 100644 --- a/actions/setup/js/smoke_copilot_sub_agents.test.cjs +++ b/actions/setup/js/smoke_copilot_sub_agents.test.cjs @@ -14,7 +14,7 @@ function evidence() { const agents = [ ["haiku-whoami", "copilot-completions/claude-haiku-4.5", "/chat/completions"], ["mini-whoami", "copilot-responses/gpt-5-mini", "/responses"], - ["nano-whoami", "copilot-responses/gpt-5-nano", "/responses"], + ["mini54-whoami", "copilot-responses/gpt-5.4-mini", "/responses"], ]; return { events: agents.flatMap(([agentName, model]) => [ From 27be880983729c10ce1c5dbcd007375ee127e736 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:38:05 +0000 Subject: [PATCH 6/6] Test imported expression alias in model pin checker Co-authored-by: SivaKesava1 <11771739+SivaKesava1@users.noreply.github.com> --- actions/setup/js/model_pin_check.test.cjs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/actions/setup/js/model_pin_check.test.cjs b/actions/setup/js/model_pin_check.test.cjs index 75eb1ffdb1b..5086114cd24 100644 --- a/actions/setup/js/model_pin_check.test.cjs +++ b/actions/setup/js/model_pin_check.test.cjs @@ -330,6 +330,12 @@ describe("model_pin_check collection and rules", () => { }); expect(pin(nested, "model")).toMatchObject({ status: "not_checked", reason: "expression" }); + const imported = check({ + ".github/workflows/imported.md": "---\nengine: copilot\nmodel: dynamic\nimports:\n - shared/models.md\n---\n", + ".github/workflows/shared/models.md": '---\nmodels:\n dynamic: ["copilot/${{ vars.MODEL }}"]\n---\n', + }); + expect(pin(imported, "model")).toMatchObject({ status: "not_checked", reason: "expression" }); + const ordered = check({ ".github/workflows/ordered.md": '---\nengine: copilot\nmodel: dynamic\nmodels:\n dynamic: [copilot/gpt-5-mini, "copilot/${{ vars.MODEL }}"]\n---\n', });