diff --git a/actions/setup/js/types/unified_session.d.ts b/actions/setup/js/types/unified_session.d.ts index d3f46de7117..99c0e679019 100644 --- a/actions/setup/js/types/unified_session.d.ts +++ b/actions/setup/js/types/unified_session.d.ts @@ -153,6 +153,26 @@ export interface ModelRoutingData { deviations?: JsonValue; requestedModel?: JsonValue; requestedEffort?: JsonValue; + /** AWF route choices in rank order; ranks are one-based array positions. */ + rankedChoices?: ModelRoutingRankedChoice[]; + eligibleChoices?: JsonValue; + catalogueOverlap?: JsonValue; + selectedEndpoint?: JsonValue; + requestedEndpoint?: JsonValue; + upstreamEndpoint?: JsonValue; + /** Request-telemetry dimensions AWF did not observe, such as `model` and `effort`; not unavailable models. */ + unavailable?: string[]; + /** Observed HTTP status, independent of `outcome`. */ + status?: JsonValue; + /** AWF failure phase, such as `primary` or `bootstrap`; not the collection provenance phase. */ + phase?: JsonValue; +} + +export interface ModelRoutingRankedChoice { + rank: number; + id?: string; + model?: string; + effort?: string | null; } export interface ModelEndpointMismatchData { diff --git a/actions/setup/js/unified_session_payload.cjs b/actions/setup/js/unified_session_payload.cjs index bd297bbe254..ee206e6a0b1 100644 --- a/actions/setup/js/unified_session_payload.cjs +++ b/actions/setup/js/unified_session_payload.cjs @@ -349,6 +349,15 @@ const EVENT_FIELDS = { requestedModel: ["requested_model", "requestedModel"], requestedEffort: ["requested_effort", "requestedEffort"], outcome: ["outcome"], + rankedChoices: ["ranked_choices", "rankedChoices"], + eligibleChoices: ["eligible_choices", "eligibleChoices"], + catalogueOverlap: ["catalogue_overlap", "catalogueOverlap"], + selectedEndpoint: ["selected_endpoint", "selectedEndpoint"], + requestedEndpoint: ["requested_endpoint", "requestedEndpoint"], + upstreamEndpoint: ["upstream_endpoint", "upstreamEndpoint"], + unavailable: ["unavailable"], + status: ["status"], + phase: ["phase"], }, "safe_output.request": SAFE_OUTPUT_FIELDS, "safe_output.result": SAFE_OUTPUT_FIELDS, @@ -526,6 +535,24 @@ function normalizeModelInventoryData(value) { return { endpoints, ...(modelsFetchComplete !== undefined ? { modelsFetchComplete } : {}) }; } +/** + * AWF persists route choices in rank order; keep only the position, optional ID, model, and supplied effort. + * @param {any} choices + * @returns {import("./types/unified_session").ModelRoutingRankedChoice[] | undefined} + */ +function normalizeRankedChoices(choices) { + if (!Array.isArray(choices)) return undefined; + return choices.map((choice, index) => { + /** @type {import("./types/unified_session").ModelRoutingRankedChoice} */ + const entry = { rank: index + 1 }; + if (choice === null || typeof choice !== "object" || Array.isArray(choice)) return entry; + if (typeof choice.id === "string") entry.id = choice.id; + if (typeof choice.model === "string") entry.model = choice.model; + if (typeof choice.effort === "string" || choice.effort === null) entry.effort = choice.effort; + return entry; + }); +} + /** * Compact known payloads without truncating text or changing native parser traces. * Unknown extension data remains opaque for forward compatibility. @@ -543,6 +570,18 @@ function normalizeUnifiedSessionEvent(event, phase) { if (inventory === undefined) throw new Error(`${ERR_VALIDATION}: Model inventory snapshot must contain valid endpoint model collections`); Object.assign(data, inventory); } + if (event.type === "firewall.model_routing") { + if (Object.hasOwn(data, "rankedChoices")) { + const rankedChoices = normalizeRankedChoices(data.rankedChoices); + if (rankedChoices === undefined) delete data.rankedChoices; + else data.rankedChoices = rankedChoices; + } + // AWF lists unobserved request-telemetry dimensions here, not unavailable models. + if (Object.hasOwn(data, "unavailable") && !(Array.isArray(data.unavailable) && data.unavailable.every(value => typeof value === "string"))) delete data.unavailable; + for (const key of ["eligibleChoices", "catalogueOverlap", "selectedEndpoint", "requestedEndpoint", "upstreamEndpoint", "status", "phase"]) { + if (data[key] !== null && typeof data[key] === "object") delete data[key]; + } + } if (event.type === "workflow.info") { const awContext = Object.hasOwn(source, "awContext") ? source.awContext : source.context; if (awContext && typeof awContext === "object" && !Array.isArray(awContext)) { diff --git a/actions/setup/js/unified_session_payload.test.cjs b/actions/setup/js/unified_session_payload.test.cjs index b17ce4e129b..df14c8f961a 100644 --- a/actions/setup/js/unified_session_payload.test.cjs +++ b/actions/setup/js/unified_session_payload.test.cjs @@ -1,4 +1,6 @@ import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import { fileURLToPath } from "node:url"; import { normalizeUnifiedSessionEvent, normalizeModelInventoryData } from "./unified_session_payload.cjs"; import { mergeSessionSources } from "./unified_session.cjs"; import { generatePlainTextSummary, generateCopilotCliStyleSummary, convertCopilotEventsToLegacyLogEntries } from "./log_parser_shared.cjs"; @@ -915,3 +917,301 @@ describe("essential unified session payloads", () => { expect(output).toContain("type=add_comment"); }); }); + +describe("AWF model routing candidate and request metadata", () => { + const GOLDEN_DIR = fileURLToPath(new URL("../../../pkg/cli/testdata/model_routing_golden/", import.meta.url)); + const ROUTING_LOG = "sandbox/firewall/logs/api-proxy-logs/model-routing.jsonl"; + const NEW_FIELDS = ["rankedChoices", "eligibleChoices", "catalogueOverlap", "selectedEndpoint", "requestedEndpoint", "upstreamEndpoint", "unavailable", "status", "phase"]; + const SOURCE_FIELDS = { + rankedChoices: ["ranked_choices", "rankedChoices"], + eligibleChoices: ["eligible_choices", "eligibleChoices"], + catalogueOverlap: ["catalogue_overlap", "catalogueOverlap"], + selectedEndpoint: ["selected_endpoint", "selectedEndpoint"], + requestedEndpoint: ["requested_endpoint", "requestedEndpoint"], + upstreamEndpoint: ["upstream_endpoint", "upstreamEndpoint"], + unavailable: ["unavailable"], + status: ["status"], + phase: ["phase"], + }; + const readJsonl = file => + fs + .readFileSync(file, "utf8") + .split("\n") + .filter(line => line.trim()) + .map(line => JSON.parse(line)); + const readRoutingLog = name => readJsonl(`${GOLDEN_DIR}${name}/${ROUTING_LOG}`); + const camelize = record => Object.fromEntries(Object.entries(record).map(([key, value]) => [key === "_schema" ? "schema" : key.replace(/_([a-z0-9])/g, (_, char) => char.toUpperCase()), value])); + const SPELLINGS = [ + ["snake_case", record => structuredClone(record)], + ["camelCase", camelize], + ]; + const routingEvent = record => ({ type: "firewall.model_routing", ...(record.timestamp !== undefined ? { timestamp: record.timestamp } : {}), data: record }); + // Normalize once, checking input immutability, idempotence, and JSON round trips. + const project = event => { + const original = structuredClone(event); + const projected = normalizeUnifiedSessionEvent(event, "agent"); + expect(event).toEqual(original); + expect(normalizeUnifiedSessionEvent(structuredClone(projected), "agent")).toEqual(projected); + expect(JSON.parse(JSON.stringify(projected))).toEqual(projected); + return projected; + }; + const withoutNewFields = record => Object.fromEntries(Object.entries(record).filter(([key]) => !Object.values(SOURCE_FIELDS).flat().includes(key))); + const ranked = choices => choices.map((choice, index) => ({ rank: index + 1, ...choice })); + + // Minimized, sanitized copies from smoke-copilot-routed run 38984582061 (commit 354b5c3), + // artifact `usage`, path `agent/model-routing.jsonl`, schema model-routing/v0.28.59. + // conversation_sha256, interaction_id, github_repository, and github_workflow_ref were removed. + const SMOKE_SELECTION = { + _schema: "model-routing/v0.28.59", + timestamp: "2026-10-17T00:05:52.673Z", + event: "model_routing", + stage: "selection", + objective: { goal: "cost", mode: "economy" }, + provider: "copilot", + selected_provider: "copilot", + wire_model: "gpt-5.6-luna", + endpoint: "/responses", + labels: { task_type: "chore", scope: "local", task_complexity: "trivial" }, + mode: "economy", + classifier_model: "github-copilot/gpt-5.6-luna", + classifier_effort: "high", + router: { name: "gh-aw-router", version: "0.1.3" }, + ranked_choices: [ + { model: "github-copilot/gpt-5.6-luna", effort: "high" }, + { model: "github-copilot/gpt-5.6-luna", effort: "xhigh" }, + { model: "github-copilot/gpt-5.6-luna", effort: "max" }, + { model: "github-copilot/gpt-5.6-luna", effort: "medium" }, + { model: "github-copilot/gpt-5.4-mini", effort: "xhigh" }, + { model: "github-copilot/gpt-5.4-mini", effort: "high" }, + { model: "github-copilot/gpt-5.4-mini", effort: "medium" }, + { model: "github-copilot/gpt-5.6-luna", effort: "low" }, + { model: "github-copilot/gpt-5.4-mini", effort: "low" }, + { model: "github-copilot/claude-haiku-4.5", effort: null }, + { model: "github-copilot/gpt-5.6-luna", effort: "none" }, + { model: "github-copilot/gpt-5.4-mini", effort: "none" }, + ], + selected_id: "choice-0007", + selected_model: "github-copilot/gpt-5.6-luna", + selected_effort: "high", + degraded_classification: false, + degraded_reason: null, + classifier_attempts: 1, + eligible_choices: 12, + catalogue_overlap: 12, + latency_ms: 4212, + }; + const SMOKE_REQUEST = { + _schema: "model-routing/v0.28.59", + timestamp: "2026-10-17T00:06:03.778Z", + event: "model_routing", + stage: "request", + routed: "as_selected", + deviations: [], + unavailable: [], + method: "POST", + pathname: "/responses", + provider: "copilot", + requested_model: "gpt-5.6-luna", + requested_effort: "high", + selected_provider: "copilot", + selected_model: "gpt-5.6-luna", + selected_effort: "high", + selected_endpoint: "/responses", + requested_endpoint: "/responses", + upstream_endpoint: "/responses", + request_id: "3e8b8c51-1a07-46a5-a21f-c8ecea381bf4", + outcome: "completed", + status: 200, + }; + // Recorded raw AWF logs: run 37728959715 (model-routing/v0.28.44) and run 38174216426 (model-routing/v0.28.51). + const CODEX = readRoutingLog("codex-routed-provider-unavailable"); + const BUDGET = readRoutingLog("copilot-budget-exhausted-steering"); + const recorded = (records, predicate) => { + const record = records.find(predicate); + expect(record).toBeDefined(); + return record; + }; + const CODEX_SELECTION = recorded(CODEX, record => record.stage === "selection"); + const BUDGET_SELECTION = recorded(BUDGET, record => record.stage === "selection"); + const CODEX_REQUEST = recorded(CODEX, record => record.request_id === "ff510188-891a-4ad1-97eb-a12c47c2d7fc"); + const BUDGET_REQUEST = recorded(BUDGET, record => record.request_id === "2698b73b-55b7-4c97-8d6d-8d7c88b69461"); + const CODEX_FAILURE = recorded(CODEX, record => record.timestamp === "2026-10-08T04:47:41.353Z"); + const BUDGET_FAILURE = recorded(BUDGET, record => record.timestamp === "2026-10-11T20:59:12.712Z"); + + it.each(SPELLINGS)("preserves recorded AWF selection rankings and pool counts in source order (%s)", (_, spell) => { + const smoke = project(routingEvent(spell(SMOKE_SELECTION))); + expect(smoke.data.rankedChoices).toEqual([ + { rank: 1, model: "github-copilot/gpt-5.6-luna", effort: "high" }, + { rank: 2, model: "github-copilot/gpt-5.6-luna", effort: "xhigh" }, + { rank: 3, model: "github-copilot/gpt-5.6-luna", effort: "max" }, + { rank: 4, model: "github-copilot/gpt-5.6-luna", effort: "medium" }, + { rank: 5, model: "github-copilot/gpt-5.4-mini", effort: "xhigh" }, + { rank: 6, model: "github-copilot/gpt-5.4-mini", effort: "high" }, + { rank: 7, model: "github-copilot/gpt-5.4-mini", effort: "medium" }, + { rank: 8, model: "github-copilot/gpt-5.6-luna", effort: "low" }, + { rank: 9, model: "github-copilot/gpt-5.4-mini", effort: "low" }, + { rank: 10, model: "github-copilot/claude-haiku-4.5", effort: null }, + { rank: 11, model: "github-copilot/gpt-5.6-luna", effort: "none" }, + { rank: 12, model: "github-copilot/gpt-5.4-mini", effort: "none" }, + ]); + expect(smoke.data.rankedChoices.some(choice => Object.hasOwn(choice, "id"))).toBe(false); + expect(smoke.data).toMatchObject({ eligibleChoices: 12, catalogueOverlap: 12, endpoint: "/responses", selectedId: "choice-0007" }); + expect(smoke.data).not.toHaveProperty("selectedEndpoint"); + + for (const [record, count] of [ + [CODEX_SELECTION, 23], + [BUDGET_SELECTION, 6], + ]) { + const projected = project(routingEvent(spell(record))); + expect(projected.data.rankedChoices).toHaveLength(count); + expect(projected.data.rankedChoices.map(choice => choice.rank)).toEqual(Array.from({ length: count }, (_, index) => index + 1)); + expect(projected.data.rankedChoices.map(({ model, effort }) => ({ model, effort }))).toEqual(record.ranked_choices); + expect(projected.data).toMatchObject({ eligibleChoices: count, catalogueOverlap: count }); + } + + // Contract-compatibility variant: AWF route choices carry an `id` (routing-controller.js createSelection), + // but current persistence does not emit it. A supplied ID is kept; none is invented. + const withIds = structuredClone(SMOKE_SELECTION); + withIds.ranked_choices[0].id = "choice-0007"; + const projected = project(routingEvent(spell(withIds))); + expect(projected.data.rankedChoices[0]).toEqual({ rank: 1, id: "choice-0007", model: "github-copilot/gpt-5.6-luna", effort: "high" }); + expect(projected.data.rankedChoices.slice(1).some(choice => Object.hasOwn(choice, "id"))).toBe(false); + + // A supplied selected endpoint on a selection is retained exactly; an empty ranking and zero counts stay supplied values. + const supplied = project(routingEvent(spell({ ...SMOKE_SELECTION, selected_endpoint: "/chat/completions", ranked_choices: [], eligible_choices: 0, catalogue_overlap: 0 }))); + expect(supplied.data).toMatchObject({ endpoint: "/responses", selectedEndpoint: "/chat/completions", rankedChoices: [], eligibleChoices: 0, catalogueOverlap: 0 }); + }); + + it.each(SPELLINGS)("preserves recorded AWF request endpoints HTTP status and unavailable metadata (%s)", (_, spell) => { + for (const [record, status, outcome] of [ + [SMOKE_REQUEST, 200, "completed"], + [CODEX_REQUEST, 400, "failed"], + [BUDGET_REQUEST, 403, "rejected"], + ]) { + const projected = project(routingEvent(spell(record))); + expect(projected.data).toMatchObject({ + selectedEndpoint: "/responses", + requestedEndpoint: "/responses", + upstreamEndpoint: "/responses", + unavailable: [], + status, + outcome, + requestId: record.request_id, + routed: record.routed, + deviations: record.deviations, + requestedModel: record.requested_model, + requestedEffort: record.requested_effort, + selectedModel: record.selected_model, + selectedEffort: record.selected_effort, + }); + expect(projected.data).not.toHaveProperty("endpoint"); + const { unavailable: _unavailable, ...withoutUnavailable } = record; + expect(project(routingEvent(spell(withoutUnavailable))).data).not.toHaveProperty("unavailable"); + } + // AWF lists unobserved telemetry dimensions, not unavailable models; the array is kept as supplied. + const unobserved = project(routingEvent(spell({ ...SMOKE_REQUEST, unavailable: ["model", "effort"] }))); + expect(unobserved.data.unavailable).toEqual(["model", "effort"]); + }); + + it.each(SPELLINGS)("preserves recorded AWF failure phase independently of collection provenance (%s)", (_, spell) => { + for (const [record, code] of [ + [CODEX_FAILURE, "provider_unavailable"], + [BUDGET_FAILURE, "ai_credits_limit_exceeded"], + ]) { + const event = routingEvent(spell(record)); + const projected = project(event); + expect(projected.timestamp).toBe(record.timestamp); + expect(projected.data).toEqual({ schema: record._schema, stage: "failure", phase: "primary", code }); + const [merged] = mergeSessionSources([{ component: "firewall", phase: "agent", path: ROUTING_LOG, events: [structuredClone(event)] }]); + expect(merged.provenance).toMatchObject({ component: "firewall", phase: "agent", path: ROUTING_LOG }); + expect(merged.data.phase).toBe("primary"); + expect(merged.timestamp).toBe(record.timestamp); + const { phase: _phase, ...withoutPhase } = record; + expect(project(routingEvent(spell(withoutPhase))).data).not.toHaveProperty("phase"); + } + }); + + it.each(SPELLINGS)("leaves absent AWF routing metadata absent for older session records (%s)", (_, spell) => { + // Already-projected model-routing/v0.28.51 selection from copilot-budget-exhausted-steering/usage/aw_session.jsonl. + const session = readJsonl(`${GOLDEN_DIR}copilot-budget-exhausted-steering/usage/aw_session.jsonl`); + const older = session.find(event => event.type === "firewall.model_routing" && event.timestamp === "2026-10-11T20:58:04.898Z"); + expect(older.data.stage).toBe("selection"); + const { provenance: _provenance, ...olderEvent } = older; + const projectedOlder = project(olderEvent); + expect(projectedOlder.data).toEqual(older.data); + for (const field of NEW_FIELDS) expect(Object.hasOwn(projectedOlder.data, field)).toBe(false); + + // Compatibility variants: the real selection, request, and failure inputs with the new fields omitted. + for (const record of [SMOKE_SELECTION, CODEX_SELECTION, SMOKE_REQUEST, CODEX_REQUEST, BUDGET_REQUEST, CODEX_FAILURE, BUDGET_FAILURE]) { + const full = project(routingEvent(spell(record))); + const stripped = project(routingEvent(spell(withoutNewFields(record)))); + for (const field of NEW_FIELDS) expect(Object.hasOwn(stripped.data, field)).toBe(false); + expect(stripped.data).toEqual(Object.fromEntries(Object.entries(full.data).filter(([key]) => !NEW_FIELDS.includes(key)))); + } + }); + + it.each(SPELLINGS)("excludes private AWF routing fields while retaining only candidate metadata (%s)", (_, spell) => { + // Privacy-test sentinels in real AWF field names; these values are not recorded telemetry. + const sentinels = { conversation_sha256: "PRIVACY_TEST_CONVERSATION_SHA256", detail: "PRIVACY_TEST_DETAIL", method: "PRIVACY_TEST_METHOD", pathname: "PRIVACY_TEST_PATHNAME" }; + const selection = { ...SMOKE_SELECTION, ...sentinels }; + // Privacy variant: unexpected nested candidate properties must not be copied through. + selection.ranked_choices = selection.ranked_choices.map(choice => ({ ...choice, predicted_success: 0.9, explanation: "PRIVACY_TEST_EXPLANATION", payload: { prompt: "PRIVACY_TEST_PROMPT" } })); + // AWF controller failure shape (routing-controller.js, stage "failure" with `detail`), not a recorded run. + const controllerFailure = { + _schema: "model-routing/v0.28.59", + timestamp: "2026-10-17T00:05:52.700Z", + event: "model_routing", + stage: "failure", + objective: { goal: "cost", mode: "economy" }, + code: "routing_configuration_error", + catalogue_overlap: 12, + latency_ms: 10, + ...sentinels, + }; + for (const [record, expected] of [ + [selection, project(routingEvent(spell(SMOKE_SELECTION))).data], + [{ ...SMOKE_REQUEST, ...sentinels }, project(routingEvent(spell(SMOKE_REQUEST))).data], + [{ ...CODEX_FAILURE, ...sentinels }, project(routingEvent(spell(CODEX_FAILURE))).data], + [controllerFailure, { schema: "model-routing/v0.28.59", stage: "failure", objective: { goal: "cost", mode: "economy" }, code: "routing_configuration_error", catalogueOverlap: 12, latencyMs: 10 }], + ]) { + const projected = project(routingEvent(spell(record))); + expect(projected.data).toEqual(expected); + const serialized = JSON.stringify(projected); + expect(serialized).not.toContain("PRIVACY_TEST"); + for (const key of ["conversation_sha256", "conversationSha256", "detail", "method", "pathname", "predicted_success", "explanation", "payload"]) expect(serialized).not.toContain(`"${key}"`); + for (const choice of projected.data.rankedChoices ?? []) expect(Object.keys(choice).every(key => ["rank", "id", "model", "effort"].includes(key))).toBe(true); + } + }); + + it("projects recorded model-routing goldens with only the intended metadata additions", () => { + const validate = createSessionValidator("unified").event; + const cases = fs.readdirSync(GOLDEN_DIR).filter(name => fs.existsSync(`${GOLDEN_DIR}${name}/${ROUTING_LOG}`)); + expect(cases).toEqual(expect.arrayContaining(["codex-routed-provider-unavailable", "copilot-budget-exhausted-steering"])); + const sessionRouting = name => { + const file = `${GOLDEN_DIR}${name}/usage/aw_session.jsonl`; + return fs.existsSync(file) ? readJsonl(file).filter(event => event.type === "firewall.model_routing") : []; + }; + for (const name of cases) { + const raw = readRoutingLog(name).map(record => ({ record, event: routingEvent(record) })); + const sessions = sessionRouting(name).map(({ provenance: _provenance, ...event }) => ({ record: event.data, event })); + for (const { record, event } of [...raw, ...sessions]) { + const projected = project(event); + for (const field of NEW_FIELDS) { + const alias = SOURCE_FIELDS[field].find(key => Object.hasOwn(record, key)); + expect(Object.hasOwn(projected.data, field), `${name} ${field}`).toBe(alias !== undefined); + if (alias === undefined) continue; + if (field === "rankedChoices") expect(projected.data.rankedChoices).toEqual(ranked(record[alias].map(({ model, effort }) => ({ model, effort })))); + else expect(projected.data[field]).toEqual(record[alias]); + } + const [published] = serializeSessionArtifact(mergeSessionSources([{ component: "firewall", phase: "agent", path: ROUTING_LOG, events: [event] }])) + .trim() + .split("\n") + .map(line => JSON.parse(line)); + expect(validate(published), `${name}: ${JSON.stringify(validate.errors)}`).toBe(true); + expect(published.data).toEqual(projected.data); + } + // Already-projected sessions gain nothing they did not supply. + for (const { event } of sessions.filter(({ record }) => !Object.hasOwn(record, "_schema"))) expect(project(event).data).toEqual(event.data); + } + }); +}); diff --git a/docs/public/schemas/unified-session.schema.json b/docs/public/schemas/unified-session.schema.json index cedf0d0ce24..3e611aadf76 100644 --- a/docs/public/schemas/unified-session.schema.json +++ b/docs/public/schemas/unified-session.schema.json @@ -856,7 +856,16 @@ "classifierAttempts": { "$ref": "#/definitions/JsonValue" }, "deviations": { "$ref": "#/definitions/JsonValue" }, "requestedModel": { "$ref": "#/definitions/JsonValue" }, - "requestedEffort": { "$ref": "#/definitions/JsonValue" } + "requestedEffort": { "$ref": "#/definitions/JsonValue" }, + "rankedChoices": { "type": "array", "items": { "$ref": "#/definitions/ModelRoutingRankedChoice" }, "description": "AWF route choices in rank order; ranks are one-based array positions." }, + "eligibleChoices": { "$ref": "#/definitions/JsonValue" }, + "catalogueOverlap": { "$ref": "#/definitions/JsonValue" }, + "selectedEndpoint": { "$ref": "#/definitions/JsonValue" }, + "requestedEndpoint": { "$ref": "#/definitions/JsonValue" }, + "upstreamEndpoint": { "$ref": "#/definitions/JsonValue" }, + "unavailable": { "type": "array", "items": { "type": "string" }, "description": "Request-telemetry dimensions AWF did not observe, such as `model` and `effort`; not unavailable models." }, + "status": { "$ref": "#/definitions/JsonValue", "description": "Observed HTTP status, independent of `outcome`." }, + "phase": { "$ref": "#/definitions/JsonValue", "description": "AWF failure phase, such as `primary` or `bootstrap`; not the collection provenance phase." } }, "additionalProperties": false }, @@ -873,6 +882,12 @@ }, "additionalProperties": false }, + "ModelRoutingRankedChoice": { + "type": "object", + "properties": { "rank": { "type": "number" }, "id": { "type": "string" }, "model": { "type": "string" }, "effort": { "type": ["string", "null"] } }, + "required": ["rank"], + "additionalProperties": false + }, "ModelSelectionData": { "type": "object", "properties": { diff --git a/docs/src/content/docs/specs/unified-agent-session-specification.md b/docs/src/content/docs/specs/unified-agent-session-specification.md index 542c39ef101..a393a721aee 100644 --- a/docs/src/content/docs/specs/unified-agent-session-specification.md +++ b/docs/src/content/docs/specs/unified-agent-session-specification.md @@ -3,9 +3,9 @@ title: Unified Agent Session Specification description: Draft contract for canonical engine traces and essential unified session payloads across gh-aw runtime components. sidebar: order: 1365 -version: "1.11.0" +version: "1.12.0" status: Draft -publication_date: "2026-10-16" +publication_date: "2026-10-18" editors: - name: GitHub Agentic Workflows Team organization: GitHub @@ -13,9 +13,9 @@ editors: # Unified Agent Session Specification -**Version**: 1.11.0
+**Version**: 1.12.0
**Status**: Draft
-**Publication Date**: 2026-10-16
+**Publication Date**: 2026-10-18
**Editors**: GitHub Agentic Workflows Team (GitHub)
**This Version**: [unified-agent-session-specification](/gh-aw/specs/unified-agent-session-specification/)
**Latest Version**: This document @@ -30,7 +30,7 @@ This specification defines the session traces used by GitHub Agentic Workflows: This is a **GitHub Agentic Workflows project specification**, written using W3C-inspired document conventions. It is **not an official W3C standard**, W3C publication, or W3C-endorsed recommendation. -Version 1.11.0 is a draft governed by the project's normal review process. It may be updated, replaced, or superseded. The accompanying implementation and regression suites exercise this contract, including the sampled CI sessions identified in Section 9.4. This is not a blanket declaration of conformance for every engine version or source format. Section 10 records historical pre-implementation gaps, not current defects. Approval and ongoing compliance testing remain project responsibilities. +Version 1.12.0 is a draft governed by the project's normal review process. It may be updated, replaced, or superseded. The accompanying implementation and regression suites exercise this contract, including the sampled CI sessions identified in Section 9.4. This is not a blanket declaration of conformance for every engine version or source format. Section 10 records historical pre-implementation gaps, not current defects. Approval and ongoing compliance testing remain project responsibilities. The specification version belongs to this document. The unified file's leading `session.format` record carries an independent numeric serialization-format @@ -247,7 +247,7 @@ opaque because its essential fields are not defined by this specification. | Graders and evals | Grader IDs/names, sources, enabled flags, bounds, values, units, statuses, decisions, thresholds, errors, severity, and diagnostic messages. Manifest/result versions, run ID/attempt and counts, implementation ID/version/digest, and metric ID/value pairs are retained when supplied. Eval ID, answer, model, error, run ID, and experiment assignments are retained. No grader scripts, opaque diagnostics, or eval questions. | | Runtime accounting | Provider, model, request ID, status, AIC, cumulative/checkpoint AIC, premium requests, duration, and normalized `usage`; overlapping reports stay separate. Detection and eval usage reports resolve per-observation AIC from known model pricing only when explicit AIC is unavailable; unknown pricing leaves AIC absent, while explicit zero and checkpoints remain authoritative. | | Execution, detection, workflow | Observed outcomes, exit code, duration/start/end, detection job result/conclusion/categorical reason and verdict flags, engine ID, requested model, trigger type, workflow/repository/run ID, and available gh-aw, AWF, MCPG, and agent versions. No detector transcript or free-form reasons. | -| Model routing | `firewall.model_routing` retains its stage, purpose, provider, labels, selection, endpoint, router, request, outcome, and deviation fields. `workflow.info` includes the observed `model`, `requestedModel`, and a compact `modelRouting` object. `model_routing.outcome` records the harness's status, wire model, effective and selected endpoints, effort, applied effort, and failure code. | +| Model routing | `firewall.model_routing` retains its stage, purpose, provider, labels, selection, endpoint, router, request, outcome, and deviation fields, plus AWF's ranked choices, pool counts, request endpoints, telemetry availability, HTTP status, and failure phase when supplied. `workflow.info` includes the observed `model`, `requestedModel`, and a compact `modelRouting` object. `model_routing.outcome` records the harness's status, wire model, effective and selected endpoints, effort, applied effort, and failure code. | | Caller context and episode lineage | `workflow.info.awContext` retains the supplied `aw_context` object from runner-owned `aw_info.json.context`, including original keys and nested work-queue data. Text and Markdown summaries display it as compact JSON alongside the workflow metadata, subject to existing output-size limits. Registered secrets are redacted during artifact serialization and summary rendering. `workflow.info.episode` additionally projects supplied `episodeId`, `hopId`, `parentHopId`, `originEvent`, `rootRepo`, `rootWorkflowId`, and `rootRunId`. | | Skills | Activation inventory records installed skill name, relative source, scope, SHA-256 content digest, and declared purpose, without skill content. Exact-name and cross-scope collisions retain names, kinds, and source identifiers: current sources are workspace-relative paths, while overwritten frontmatter installations use ordinal identifiers such as `frontmatter-1`. Observed engine invocations retain the selected skill name, path, source, trigger, model, and turn when supplied; a declared or installed skill is not proof of invocation. | @@ -749,6 +749,29 @@ schema, stage, purpose, attempt, classifier model and effort, objective, provider, labels, mode, classifier attempts, degraded-classification state, selected ID/provider/model/effort, wire model, endpoint, router, latency, code, request ID, routed outcome, deviations, requested model, and requested effort. +It also retains the following AWF metadata when the source record supplies it, +accepting each snake_case source spelling and its camelCase equivalent: + +| Field | AWF source | Meaning | +| --- | --- | --- | +| `rankedChoices` | `ranked_choices` | Route choices in AWF order. Each entry has a one-based `rank` equal to its array position, `model`, and `effort` as supplied, including an explicit `null`. `id` appears only when supplied. | +| `eligibleChoices` | `eligible_choices` | Count of choices remaining after inventory and policy filtering. | +| `catalogueOverlap` | `catalogue_overlap` | Count of routing-table choices present in the live catalogue. | +| `selectedEndpoint` | `selected_endpoint` | Selected endpoint, including on request records. The existing `endpoint` field is unchanged. | +| `requestedEndpoint` | `requested_endpoint` | Endpoint requested by the harness. | +| `upstreamEndpoint` | `upstream_endpoint` | Endpoint sent upstream; never substituted from the requested endpoint. | +| `unavailable` | `unavailable` | Request-telemetry dimensions AWF did not observe, such as `model` and `effort`. It is not a list of unavailable models. | +| `status` | `status` | Observed HTTP status, independent of `outcome`. | +| `phase` | `phase` | AWF failure phase, such as `primary` or `bootstrap`; distinct from the provenance phase. | + +The collector MUST NOT sort, deduplicate, truncate, or otherwise reorder ranked +choices, and MUST NOT infer counts from their length. Absent fields MUST remain +absent: no inferred count, manufactured endpoint, zero, empty array, or `null` +placeholder is added. Supplied zeros, `null` efforts, and empty arrays are +retained. Candidate entries MUST contain only rank, optional ID, model, and +effort; per-candidate predictions or explanations MUST NOT be recorded. +`conversation_sha256`, `detail`, `method`, `pathname`, their camelCase aliases, +and other free-form or payload-bearing fields MUST be excluded. #### Model/endpoint mismatch @@ -1569,7 +1592,7 @@ Recommended execution is fixture parsing, canonical structural assertions, accou | T-UAS-065–T-UAS-066 | Unified file through both publication sinks and the conversation renderer; colliding source IDs; overlapping accounting; hostile/secret text; exhausted summary budget | Known runtime types remain visible, scopes and accounting remain independent, private prompts/payloads stay omitted, output is bounded and safely redacted, source artifact remains intact. | | T-UAS-067 | Repeated native/canonical/detector error observations, live timeout evidence, final-zero and missing exits, malformed or duplicate aggregate records, quoted errors and tool failures | One deterministic `agent.execution` record with distinct native codes/types, stable categories, observed exit precedence, unchanged error evidence, and matching JS/Go reader validation. | | T-UAS-068 | Goose stream messages/deltas, tool requests/results, cumulative completion usage, source-second timestamps, malformed records, error-only and partial sessions | Exact supported content and IDs, observed outcomes only, no duplicated snapshots or invented partial results, explicit diagnostics. | -| T-UAS-070 | Firewall token usage with supplied or absent purpose/path/initiator; `usage.report` alias | Supplied routing metadata survives projection; consumers treat absent purpose as unknown, not agent. | +| T-UAS-070 | Firewall token usage with supplied or absent purpose/path/initiator; `usage.report` alias; recorded AWF selection, request, and failure records in snake_case and camelCase, with and without ranked choices, pool counts, endpoints, `unavailable`, status, and phase, and with private fields | Supplied routing metadata survives projection; consumers treat absent purpose as unknown, not agent; ranked choices keep source order with one-based ranks; absent fields stay absent; private fields and nested candidate payloads are excluded; projection is idempotent. | | T-UAS-071 | Firewall token usage with supplied, absent, or malformed `steering`; AI-credit, token, and time steering event-log entries | Type and threshold survive projection, malformed notices are dropped, steering events keep threshold and request ID, and absent notices remain unrecorded rather than undelivered. | | T-UAS-073 | Copilot built-in and declared sub-agents with distinct display names, provider-qualified SDK model IDs, declared concrete and alias models, and sessions without `agentName` | Rows keyed by `agentName` with sorted `display_names`, one audit row per definition and one logs row per definition and effort, bare served model IDs without aliases, unchanged counts and credits, and display-name rows when `agentName` is absent. | | T-UAS-074 | `agent/model-resolution.json` with resolved, exact, and failed records for main and sub-agent roles; empty, non-array, malformed, and contract-violating records | One `model.resolution` event per record in array order with `model_resolution` provenance and only the contract fields; invalid sources emit collection warnings and no events. | @@ -2095,6 +2118,11 @@ Native IDs can collide, timestamps can be out of order, and a trace can contain ## 13. Change Log (Informative) +### Version 1.12.0 — Draft (2026-10-18) + +- Retained AWF ranked choices, eligible-choice and catalogue-overlap counts, selected, requested, and upstream endpoints, request-telemetry `unavailable` dimensions, HTTP status, and failure phase in `firewall.model_routing` when supplied (T-UAS-070). +- Retained serialization-format version 1. + ### Version 1.11.0 — Draft (2026-10-16) - Added the runner-written `model.resolution` observation from `agent/model-resolution.json`, recording each engine's configured and resolved model, provider, status, and failure cause (T-UAS-074).