diff --git a/.github/scripts/test_aw_issue_clustering_publish.cjs b/.github/scripts/test_aw_issue_clustering_publish.cjs index 2e4b4e979bc..7b6da4c5598 100644 --- a/.github/scripts/test_aw_issue_clustering_publish.cjs +++ b/.github/scripts/test_aw_issue_clustering_publish.cjs @@ -240,6 +240,7 @@ test("live queue refuses an eleventh issue", async () => { test("new summaries use attributed issue creation and preserve managed metadata", async () => { const { setupGlobals, createIssue } = require("../../actions/setup/js/index.cjs"); + const { renderIssueParameters } = require("../../actions/setup/js/issue_report.cjs"); const data = corpus(); const calls = []; const github = { @@ -261,6 +262,13 @@ test("new summaries use attributed issue creation and preserve managed metadata" GH_AW_CALLER_WORKFLOW_ID: "github/gh-aw/aw-issue-clustering", GH_AW_PROMPTS_DIR: path.resolve(__dirname, "../../actions/setup/md"), GH_AW_SAFE_OUTPUTS_STAGED: "false", + GITHUB_REPOSITORY: "github/gh-aw", + GITHUB_REPOSITORY_ID: "123", + GITHUB_SERVER_URL: "https://github.com", + GITHUB_RUN_ID: "123", + GITHUB_RUN_ATTEMPT: "1", + GITHUB_JOB: "safe_outputs", + GH_AW_JOB_CHECK_RUN_ID: "456", }; const previousEnv = Object.fromEntries(Object.keys(env).map(key => [key, process.env[key]])); const globalKeys = ["core", "github", "context", "exec", "io", "getOctokit", "runtimeFeatures", "hasRuntimeFeature", "getRuntimeFeatureValue"]; @@ -271,7 +279,7 @@ test("new summaries use attributed issue creation and preserve managed metadata" setupGlobals(core, github, { repo: { owner: "github", repo: "gh-aw" }, runId: 123, payload: {} }, {}, {}, () => github); const result = await reconcile(github, "github", "gh-aw", [cluster()], data, false, core, createIssue); assert.equal(calls.length, 1); - assert.ok(calls[0].body.includes(island(cluster(), 1, repo))); + assert.ok(calls[0].body.includes(renderIssueParameters({ title: "Summary", body: island(cluster(), 1, repo) }).body.trimEnd())); assert.match(calls[0].body, /> Generated by \[AW Essential Issue Clustering\]/); assert.match(calls[0].body, /gh-aw-agentic-workflow: AW Essential Issue Clustering/); assert.ok(calls[0].body.includes("https://github.com/github/gh-aw/actions/runs/123")); diff --git a/actions/setup/README.md b/actions/setup/README.md index 259d25f24cc..8a9b879087d 100644 --- a/actions/setup/README.md +++ b/actions/setup/README.md @@ -73,6 +73,65 @@ All files are copied from the committed `js/` and `sh/` directories which contai ## Development +### Structured issue reports + +All JavaScript GitHub issue creators publish through `js/issue_report.cjs`. +`createReportIssue(client, parameters, report?)` renders the report AST before +calling the supplied authenticated client's issue API. Existing issue Markdown +is imported when no document is supplied. GraphQL work-queue creation uses the +same `renderIssueParameters` boundary, as do staged safe-output previews. +Linear uses the Markdown renderer. Jira uses text-preserving normalization before +converting its literal description to ADF, without replacing ordinary language +characters with visually similar Latin letters. + +`js/report_ast.cjs` defines the typed inline and block nodes, normalization, and +GitHub Markdown renderer. The renderer owns escaping, HTTPS link validation, +heading levels, balanced details, code fences, and whitespace. It rejects raw +HTML/Markdown nodes and redacts diagnostic secrets before choosing delimiters. + +`js/failure_report.cjs` builds reports from `md/agent_failure_issue.md`. +`js/report_markdown.cjs` imports the runtime template dialect: headings, +paragraphs, emphasis, links, images, bare HTTPS URLs, code, lists, task lists, +tables, quotes, alerts, separators, and details. Link destinations retain balanced +parentheses, and image sources use the same HTTPS validation as links. +Source paragraph newlines and explicit Markdown hard breaks use typed break nodes; +newlines in scalar metadata still normalize to spaces. +Templates are parsed **before** scalar metadata is substituted. Diagnostic +fragments enter through explicit block fields, with headings below the report +sections; unsupported HTML remains literal text. New programmatic report content +should use AST nodes rather than concatenated Markdown. Empty optional blocks +are omitted and rendering is deterministic. Complete fragments are redacted before +tokenization, and URL-domain policies apply to fully normalized content. + +`js/report_footer.cjs` builds the footer from the shared attribution template and +model/cost metadata. Attribution, expiration, provenance, failure-match markers, +and the detection preamble are rendered with the document, not appended afterward. +Marker names and fields are validated; marker values cannot close an HTML comment +or inject another metadata field. Ordinary double hyphens in identifiers remain +unchanged. Unknown placeholders in user-configured footer templates remain literal; +internal templates retain strict missing-field validation. Expiration remains a +checked task-list item so existing cleanup and deduplication readers continue to work. + +The issue publication importer explicitly enables typed metadata comments to +preserve grouping, execution provenance, managed islands, and work-queue IDs. +Diagnostic imports do not enable them. JSON metadata is serialized without HTML +delimiters while preserving decoded values. Configured mentions use explicit +mention nodes after the existing safe-output allowlist and budget checks; other +text cannot create notifications. Body limits apply after rendering, including +escaping and attribution. A repository-wide regression guards against direct +REST issue-creation calls outside the shared publisher. + +Special syntax has semantic nodes rather than interpolated strings: +`closing_reference` records the closing action, issue number, optional +`owner/repo`, and an explicit `enabled` flag. Disabled references render an +escaped `#`, including in PR fallback issues; scalar text never becomes an active +closing directive. `structured_data` accepts JSON values and renders a safely +fenced JSON block. `xml_marker` holds a validated name, scalar value, and optional +key/value fields; `json_marker` holds an actual data object, not serialized JSON. +Footer provenance uses the same XML-marker node and renderer. Structured data +preserves entity-looking strings, redacts secrets, and rejects cycles, getters, +non-finite numbers, and other values JSON would silently drop or coerce. + ### JSONL runtime helpers Use `js/jsonl_helpers.cjs` instead of splitting files or maintaining chunk buffers: diff --git a/actions/setup/js/check_version_updates.cjs b/actions/setup/js/check_version_updates.cjs index 2d3b9856e0d..0dbd86e857d 100644 --- a/actions/setup/js/check_version_updates.cjs +++ b/actions/setup/js/check_version_updates.cjs @@ -16,6 +16,8 @@ * Fails the activation job when validation fails. */ +const { createReportIssue } = require("./issue_report.cjs"); + const { withRetry, isTransientError } = require("./error_recovery.cjs"); const { getErrorMessage } = require("./error_helpers.cjs"); const { renderTemplateFromFile, getPromptPath } = require("./messages_core.cjs"); @@ -188,7 +190,7 @@ async function reportBlockedVersionIssue(compiledVersion) { const newIssue = await withRetry( () => - github.rest.issues.create({ + createReportIssue(github, { owner, repo, title, diff --git a/actions/setup/js/check_workflow_recompile_needed.cjs b/actions/setup/js/check_workflow_recompile_needed.cjs index 19cb159e96c..39ca3300e3a 100644 --- a/actions/setup/js/check_workflow_recompile_needed.cjs +++ b/actions/setup/js/check_workflow_recompile_needed.cjs @@ -1,6 +1,8 @@ // @ts-check /// +const { createReportIssue } = require("./issue_report.cjs"); + const { getErrorMessage } = require("./error_helpers.cjs"); const { getFooterWorkflowRecompileMessage, getFooterWorkflowRecompileCommentMessage, generateXMLMarker, getDetectionCautionAlert } = require("./messages_footer.cjs"); const fs = require("fs"); @@ -420,7 +422,7 @@ async function main() { issueBody += "\n\n---\n" + footer + "\n\n" + xmlMarker + "\n"; try { - const newIssue = await github.rest.issues.create({ + const newIssue = await createReportIssue(github, { owner, repo, title: RECOMPILE_ISSUE_TITLE, diff --git a/actions/setup/js/create_forecast_issue.cjs b/actions/setup/js/create_forecast_issue.cjs index 753f99ea93d..275360d1162 100644 --- a/actions/setup/js/create_forecast_issue.cjs +++ b/actions/setup/js/create_forecast_issue.cjs @@ -2,6 +2,8 @@ // @safe-outputs-exempt SEC-004 /// +const { createReportIssue } = require("./issue_report.cjs"); + const fs = require("node:fs"); const { getPromptPath, renderTemplateFromFile } = require("./messages_core.cjs"); const { getErrorMessage } = require("./error_helpers.cjs"); @@ -348,7 +350,7 @@ async function main() { await core.summary.addRaw(summary).write(); } - const createdIssue = await github.rest.issues.create({ + const createdIssue = await createReportIssue(github, { owner: context.repo.owner, repo: context.repo.repo, title: isErrorOutcome ? FORECAST_ERROR_ISSUE_TITLE : FORECAST_ISSUE_TITLE, diff --git a/actions/setup/js/create_issue.cjs b/actions/setup/js/create_issue.cjs index 1f7bfe00ef3..0f3d5a1a042 100644 --- a/actions/setup/js/create_issue.cjs +++ b/actions/setup/js/create_issue.cjs @@ -1,9 +1,12 @@ // @ts-check /// +const { createReportIssue, renderIssueParameters, buildIssueReport } = require("./issue_report.cjs"); +const { normalizeReportText, normalizeReportTitle } = require("./report_ast.cjs"); + const { sanitizeLabelContent } = require("./sanitize_label_content.cjs"); const { sanitizeTitle, applyTitlePrefix } = require("./sanitize_title.cjs"); -const { sanitizeContent } = require("./sanitize_content.cjs"); +const { sanitizeContent, expandAllowedAliases } = require("./sanitize_content.cjs"); const { normalizeCloseOlderKey } = require("./generate_footer.cjs"); const { appendExecutionProvenance } = require("./execution_provenance.cjs"); const { formatIssueBody } = require("./issue_body.cjs"); @@ -87,19 +90,17 @@ async function createIssue(parameters) { if (issueArrayLimitError) { throw new Error(`${ERR_VALIDATION}: ${issueArrayLimitError}`); } - const body = formatIssueBody(parameters.body, { repo: { owner, repo } }); - if (body.length > MAX_GITHUB_BODY_LENGTH) { - throw new Error(`${ERR_VALIDATION}: Issue body exceeds GitHub's maximum length of ${MAX_GITHUB_BODY_LENGTH} characters including attribution`); - } - /** @type {IssueParameters} */ - const request = { ...parameters, owner, repo, body }; + const report = buildIssueReport(parameters.title, formatIssueBody(parameters.body, { repo: { owner, repo } })); + const rendered = renderIssueParameters({ title: parameters.title }, report); + /** @type {IssueParameters & {title: string, body: string}} */ + const request = { ...parameters, owner, repo, ...rendered }; if (labels) request.labels = labels; if (assignees) request.assignees = assignees; if (isStagedMode()) { - core.info(`Staged: would create issue in ${owner}/${repo} with title: ${parameters.title}`); + core.info(`Staged: would create issue in ${owner}/${repo} with title: ${request.title}`); return { staged: true, preview: request }; } - const { data: issue } = await withRetry(() => github.rest.issues.create(request), RATE_LIMIT_RETRY_CONFIG, `create_issue in ${owner}/${repo}`); + const { data: issue } = await withRetry(() => createReportIssue(github, request, report), RATE_LIMIT_RETRY_CONFIG, `create_issue in ${owner}/${repo}`); core.info(`Created issue ${owner}/${repo}#${issue.number}: ${issue.html_url}`); return { staged: false, issue }; } @@ -237,7 +238,7 @@ async function findOrCreateParentIssue({ githubClient, groupId, owner, repo, tit const template = createParentIssueTemplate(groupId, titlePrefix, workflowName, workflowSourceURL, expiresHours); const { data: parentIssue } = await withRetry( () => - githubClient.rest.issues.create({ + createReportIssue(githubClient, { owner, repo, title: template.title, @@ -994,7 +995,7 @@ async function main(config = {}) { processedBody = removeDuplicateTitleFromDescription(title, processedBody); // Sanitize body content to neutralize @mentions, URLs, and other security risks - processedBody = sanitizeContent(processedBody, { allowedAliases: allowedMentionAliases, maxMentions }); + processedBody = sanitizeContent(normalizeReportText(processedBody), { allowedAliases: allowedMentionAliases, maxMentions }); const bodyLines = processedBody.split("\n"); @@ -1021,7 +1022,7 @@ async function main(config = {}) { } // Apply title prefix (only if it doesn't already exist) - title = applyTitlePrefix(title, titlePrefix); + title = normalizeReportTitle(applyTitlePrefix(title, titlePrefix)); const normalizedTitle = normalizeTitleForDedup(title); @@ -1107,13 +1108,15 @@ async function main(config = {}) { const workflowId = process.env.GH_AW_WORKFLOW_ID ?? ""; const callerWorkflowId = process.env.GH_AW_CALLER_WORKFLOW_ID ?? ""; const runUrl = buildWorkflowRunUrl(context, context.repo); - const body = formatIssueBody(bodyLines.join("\n"), { + const formattedBody = formatIssueBody(bodyLines.join("\n"), { repo: repoParts, footer: includeFooter, bodyFooter: config.body_footer, expiresHours, closeOlderKey, }); + const report = buildIssueReport(title, formattedBody, { allowedMentions: expandAllowedAliases(allowedMentionAliases) }); + const { body } = renderIssueParameters({ title }, report); // Reserve a max-count slot synchronously before any async pre-creation work. // There is no await between check and increment, so concurrent invocations @@ -1216,14 +1219,17 @@ async function main(config = {}) { } const { data: issue } = await withRetry( () => - githubClient.rest.issues.create({ - owner: repoParts.owner, - repo: repoParts.repo, - title, - body, - labels, - assignees, - }), + createReportIssue( + githubClient, + { + owner: repoParts.owner, + repo: repoParts.repo, + title, + labels, + assignees, + }, + report + ), RATE_LIMIT_RETRY_CONFIG, `create_issue in ${qualifiedItemRepo}` ); diff --git a/actions/setup/js/create_issue.test.cjs b/actions/setup/js/create_issue.test.cjs index 22c1e681d95..c533cad15cd 100644 --- a/actions/setup/js/create_issue.test.cjs +++ b/actions/setup/js/create_issue.test.cjs @@ -113,6 +113,26 @@ describe("create_issue", () => { }); describe("basic issue creation", () => { + it.each([{ allowedBotMentions: [] }, { allowedBotMentions: ["copilot"] }])("normalizes complete source fragments before URL policy with allowed mentions $allowedBotMentions", async ({ allowedBotMentions }) => { + vi.stubEnv("GH_AW_ALLOWED_DOMAINS", "github.com"); + vi.stubEnv("GH_AW_SECRET_NAMES", "REPORT_REFERENCE"); + vi.stubEnv("SECRET_REPORT_REFERENCE", "report-secret#123"); + const handler = await main({ allowedMentionAliases: allowedBotMentions, footer: false }); + const result = await handler({ + title: "Normalized report", + body: "Agent job: build \nConclusion: failure\nreport-secret#123 https:/&amp;amp;amp;amp;ZeroWidthSpace;/outside.invalid\n\n![Chart](https://github.com/o/r/chart(a).png)\n\nSee https://github.com/o/r/create_issue.cjs#L1118.\n\n@copilot @other", + }); + expect(result.success).toBe(true); + const body = mockGithub.rest.issues.create.mock.calls[0][0].body; + expect(body).not.toContain("report-secret"); + expect(body).not.toContain("https://outside.invalid"); + expect(body).not.toContain("ZeroWidthSpace"); + expect(body).toContain("Agent job: build \nConclusion: failure"); + expect(body).toContain("![Chart](https://github.com/o/r/chart%28a%29.png)"); + expect(body).toContain("](https://github.com/o/r/create_issue.cjs#L1118)."); + expect(body).toContain(allowedBotMentions.length ? "@copilot `@other`" : "`@copilot` `@other`"); + }); + it.each(requiredExecutionEnvKeys)("should not publish an issue when %s is missing", async key => { await expectMissingRuntimeFailure(async () => { const handler = await main({}); @@ -163,7 +183,7 @@ describe("create_issue", () => { title: "Tracker regression", }) ); - expect(request.body).toContain("Issue content with safe HTML"); + expect(request.body).toContain("Issue content with <b>safe HTML</b>"); expect(request.body).not.toContain("