Add weekly AAT and VRT flaky test reporting

script/find-flaky-tests.mts

@@ -0,0 +1,249 @@
+#!/usr/bin/env node
+
+import {execFile as execFileCallback} from 'node:child_process'
+import {promisify} from 'node:util'
+
+const execFile = promisify(execFileCallback)
+const defaultWorkflows = ['reports.yml', 'deploy_preview.yml', 'deploy_preview_forks.yml']
+const workflowPattern = /^(aat|vrt)-reports \/ (?:aat|vrt)-runner \(\d+\)$/
+const testPattern = /(?:\[[^\]]+\]\s+›\s+)?(\S+\.test\.[cm]?[jt]sx?:\d+:\d+\s+›\s+.+)$/
+const escapeCharacter = String.fromCharCode(27)
+
+interface Run {
+  readonly databaseId: number
+  readonly createdAt: string
+  readonly url: string
+}
+
+interface FlakyOccurrence {
+  readonly suite: 'aat' | 'vrt'
+  readonly test: string
+  readonly run: Run
+}
+
+interface Options {
+  readonly days: number
+  readonly repo: string
+  readonly workflows: Array<string>
+}
+
+export function parseFlakyTests(log: string, run: Run): Array<FlakyOccurrence> {
+  const jobs = new Map<string, Array<string>>()
+
+  for (const line of log.split('\n')) {
+    const fields = line.split('\t')
+    const jobName = fields[0]
+
+    if (!workflowPattern.test(jobName)) {
+      continue
+    }
+
+    const content = cleanLogLine(fields.slice(2).join('\t'))
+    const lines = jobs.get(jobName) ?? []
+    lines.push(content)
+    jobs.set(jobName, lines)
+  }
+
+  const occurrences: Array<FlakyOccurrence> = []
+
+  for (const [jobName, lines] of jobs) {
+    const suite = jobName.startsWith('aat-') ? 'aat' : 'vrt'
+    let remaining = 0
+
+    for (const line of lines) {
+      const summary = line.match(/^\s*(\d+)\s+flaky\s*$/)
+
+      if (summary) {
+        remaining = Number(summary[1])
+        continue
+      }
+
+      if (remaining === 0) {
+        continue
+      }
+
+      const test = line.match(testPattern)?.[1]
+
+      if (test) {
+        occurrences.push({suite, test: test.trim(), run})
+        remaining -= 1
+      }
+    }
+  }
+
+  return occurrences
+}
+
+export function formatReport(occurrences: Array<FlakyOccurrence>, days: number): string {
+  if (occurrences.length === 0) {
+    return `No flaky AAT or VRT tests found in the past ${days} days.`
+  }
+
+  const flakyTests = new Map<string, {count: number; latestRun: Run; suite: 'aat' | 'vrt'; test: string}>()
+
+  for (const occurrence of occurrences) {
+    const key = `${occurrence.suite}\0${occurrence.test}`
+    const current = flakyTests.get(key)
+
+    if (current) {
+      current.count += 1
+      if (occurrence.run.createdAt > current.latestRun.createdAt) {
+        current.latestRun = occurrence.run
+      }
+    } else {
+      flakyTests.set(key, {
+        count: 1,
+        latestRun: occurrence.run,
+        suite: occurrence.suite,
+        test: occurrence.test,
+      })
+    }
+  }
+
+  const rows = [...flakyTests.values()]
+    .sort((a, b) => b.count - a.count || a.test.localeCompare(b.test))
+    .map(
+      ({count, latestRun, suite, test}) =>
+        `| ${suite.toUpperCase()} | ${escapeMarkdown(test)} | ${count} | [run](${latestRun.url}) |`,
+    )
+
+  return [
+    `# Flaky AAT and VRT tests (past ${days} days)`,
+    '',
+    '| Workflow | Test | Flaky runs | Latest occurrence |',
+    '| :------- | :--- | ---------: | :---------------- |',
+    ...rows,
+  ].join('\n')
+}
+
+async function main() {
+  const options = await parseOptions(process.argv.slice(2))
+  const since = new Date(Date.now() - options.days * 24 * 60 * 60 * 1000).toISOString()
+  const runs = (await Promise.all(options.workflows.map(workflow => listRuns(options.repo, workflow, since)))).flat()
+  const uniqueRuns = [...new Map(runs.map(run => [run.databaseId, run])).values()]
+  const occurrences = (
+    await mapLimit(uniqueRuns, 5, async run => {
+      try {
+        const {stdout} = await execFile(
+          'gh',
+          ['run', 'view', String(run.databaseId), '--repo', options.repo, '--log'],
+          {maxBuffer: 100 * 1024 * 1024},
+        )
+        return parseFlakyTests(stdout, run)
+      } catch (error) {
+        process.stderr.write(`Could not read logs for ${run.url}: ${getErrorMessage(error)}\n`)
+        return []
+      }
+    })
+  ).flat()
+
+  process.stdout.write(`${formatReport(occurrences, options.days)}\n`)
+}
+
+async function listRuns(repo: string, workflow: string, since: string): Promise<Array<Run>> {
+  const {stdout} = await execFile(
+    'gh',
+    [
+      'run',
+      'list',
+      '--repo',
+      repo,
+      '--workflow',
+      workflow,
+      '--status',
+      'completed',
+      '--created',
+      `>=${since}`,
+      '--limit',
+      '1000',
+      '--json',
+      'databaseId,createdAt,url',
+    ],
+    {maxBuffer: 10 * 1024 * 1024},
+  )
+  return JSON.parse(stdout) as Array<Run>
+}
+
+async function parseOptions(args: Array<string>): Promise<Options> {
+  let days = 7
+  let repo = process.env.GITHUB_REPOSITORY
+  const workflows: Array<string> = []
+
+  for (let index = 0; index < args.length; index += 1) {
+    const argument = args[index]
+    const value = args[index + 1]
+
+    if (argument === '--days' && value) {
+      days = Number(value)
+      index += 1
+    } else if (argument === '--repo' && value) {
+      repo = value
+      index += 1
+    } else if (argument === '--workflow' && value) {
+      workflows.push(value)
+      index += 1
+    } else {
+      throw new Error(`Unknown or incomplete argument: ${argument}`)
+    }
+  }
+
+  if (!repo) {
+    const {stdout} = await execFile('gh', ['repo', 'view', '--json', 'nameWithOwner', '--jq', '.nameWithOwner'])
+    repo = stdout.trim()
+  }
+
+  if (!/^[\w.-]+\/[\w.-]+$/.test(repo)) {
+    throw new Error(`Invalid repository: ${repo}`)
+  }
+
+  if (!Number.isInteger(days) || days < 1 || days > 365) {
+    throw new Error('--days must be an integer between 1 and 365')
+  }
+
+  return {days, repo, workflows: workflows.length > 0 ? workflows : defaultWorkflows}
+}
+
+function cleanLogLine(line: string): string {
+  return line
+    .split(escapeCharacter)
+    .map((part, index) => (index === 0 ? part : part.replace(/^\[[0-?]*[ -/]*[@-~]/, '')))
+    .join('')
+    .replace(/^\d{4}-\d{2}-\d{2}T\S+\s/, '')
+}
+
+function escapeMarkdown(value: string): string {
+  return value.replaceAll('|', '\\|')
+}
+
+function getErrorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error)
+}
+
+async function mapLimit<T, U>(values: Array<T>, limit: number, callback: (value: T) => Promise<U>): Promise<Array<U>> {
+  const results = new Array<U>(values.length)
+  let nextIndex = 0
+
+  async function worker() {
+    while (nextIndex < values.length) {
+      const index = nextIndex
+      nextIndex += 1
+      results[index] = await callback(values[index])
+    }
+  }
+
+  await Promise.all(Array.from({length: Math.min(limit, values.length)}, () => worker()))
+  return results
+}
+
+async function run() {
+  try {
+    await main()
+  } catch (error) {
+    process.stderr.write(`${getErrorMessage(error)}\n`)
+    process.exitCode = 1
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  await run()
+}

script/find-flaky-tests.tests.mts

@@ -0,0 +1,49 @@
+import assert from 'node:assert/strict'
+import {describe, it} from 'node:test'
+import {formatReport, parseFlakyTests} from './find-flaky-tests.mts'
+
+const run = {
+  databaseId: 123,
+  createdAt: '2026-09-01T12:00:00Z',
+  url: 'https://github.com/primer/react/actions/runs/123',
+}
+
+describe('parseFlakyTests', () => {
+  it('extracts flaky test summaries from AAT and VRT runner logs', () => {
+    const log = [
+      'aat-reports / aat-runner (1)\tRun AAT\t2026-09-01T12:00:00Z \u001B[31m  1 flaky\u001B[39m',
+      'aat-reports / aat-runner (1)\tRun AAT\t2026-09-01T12:00:00Z     e2e/Axe.test.ts:10:2 › Axe › @aat test',
+      'vrt-reports / vrt-runner (8)\tRun VRT\t2026-09-01T12:00:00Z   1 flaky',
+      'vrt-reports / vrt-runner (8)\tRun VRT\t2026-09-01T12:00:00Z     [chromium] › e2e/Visual.test.ts:20:4 › @vrt test',
+      'aat-reports / aat\tMerge\t2026-09-01T12:00:00Z   1 flaky',
+    ].join('\n')
+
+    assert.deepEqual(parseFlakyTests(log, run), [
+      {suite: 'aat', test: 'e2e/Axe.test.ts:10:2 › Axe › @aat test', run},
+      {suite: 'vrt', test: 'e2e/Visual.test.ts:20:4 › @vrt test', run},
+    ])
+  })
+})
+
+describe('formatReport', () => {
+  it('aggregates repeated occurrences and links the latest run', () => {
+    const report = formatReport(
+      [
+        {suite: 'aat', test: 'e2e/Axe.test.ts:10:2 › test', run},
+        {
+          suite: 'aat',
+          test: 'e2e/Axe.test.ts:10:2 › test',
+          run: {
+            databaseId: 456,
+            createdAt: '2026-09-02T12:00:00Z',
+            url: 'https://github.com/primer/react/actions/runs/456',
+          },
+        },
+      ],
+      7,
+    )
+
+    assert.match(report, /\| AAT \| e2e\/Axe\.test\.ts:10:2 › test \| 2 \|/)
+    assert.match(report, /\[run\]\(https:\/\/github\.com\/primer\/react\/actions\/runs\/456\)/)
+  })
+})