diff --git a/.github/workflows/e2e-failure-report.yml b/.github/workflows/e2e-failure-report.yml new file mode 100644 index 00000000000..64f7fbad548 --- /dev/null +++ b/.github/workflows/e2e-failure-report.yml @@ -0,0 +1,57 @@ +name: E2E failure report + +# Nightly classification of failed E2E shard jobs into named failure modes. +# Job logs expire after 90 days; the classified dataset on the +# e2e-failure-data branch is what survives. See bin/e2e-failure-report.js. + +on: + schedule: + - cron: '23 5 * * *' + workflow_dispatch: + inputs: + days_back: + description: 'How many days of runs to scan (backfills are idempotent)' + default: '2' + +permissions: + contents: write + actions: read + +jobs: + report: + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@v6 + - uses: actions/checkout@v6 + # The data branch does not exist until the first successful run. + continue-on-error: true + with: + ref: e2e-failure-data + path: .e2e-data + - name: Classify failed E2E shards + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: node bin/e2e-failure-report.js --days ${{ inputs.days_back || 2 }} --data-dir .e2e-data + - name: Publish job summary + run: cat .e2e-data/summary.md >> "$GITHUB_STEP_SUMMARY" + - name: Push data branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cd .e2e-data + if [ ! -d .git ]; then + git init -b e2e-failure-data + git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + fi + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "Update E2E failure data" || exit 0 + git push origin e2e-failure-data + - name: Post summary to Slack + env: + SLACK_WEBHOOK: ${{ secrets.E2E_REPORT_SLACK_WEBHOOK }} + if: env.SLACK_WEBHOOK != '' + run: | + jq -Rs '{text: .}' < .e2e-data/summary.md | curl -s -X POST -H 'Content-Type: application/json' -d @- "$SLACK_WEBHOOK" diff --git a/bin/e2e-failure-modes.json b/bin/e2e-failure-modes.json new file mode 100644 index 00000000000..f1c3332be5a --- /dev/null +++ b/bin/e2e-failure-modes.json @@ -0,0 +1,23 @@ +{ + "comment": "Ordered failure-mode patterns for bin/e2e-failure-report.js. First match wins as the job's primary mode, so job-fatal signatures come first and cleanup noise (which co-occurs with everything) comes last. Add new patterns when the report shows unclassified failures.", + "modes": [ + {"mode": "build-typescript-error", "category": "not-e2e", "pattern": "error TS\\d+:"}, + {"mode": "commands-snapshot-drift", "category": "not-e2e", "pattern": "SNAPSHOT TEST FAILED"}, + {"mode": "pnpm-install-failed", "category": "ci-infra", "pattern": "ERR_PNPM"}, + {"mode": "auth-setup-failed", "category": "auth-browser", "pattern": "globalSetup|global-auth\\.ts"}, + {"mode": "org-app-cap", "category": "capacity", "pattern": "maximum number of apps allowed"}, + {"mode": "create-app-no-org", "category": "cli-command", "pattern": "No Organization found"}, + {"mode": "partners-throttled", "category": "backend", "pattern": "Throttled"}, + {"mode": "unprocessable-entity", "category": "backend", "pattern": "Unprocessable entity"}, + {"mode": "deploy-version-error", "category": "cli-command", "pattern": "Version couldn't be created"}, + {"mode": "uninstall-api-failed", "category": "cleanup", "pattern": "admin API uninstall failed|access token request failed"}, + {"mode": "store-provision-timeout", "category": "store", "pattern": "page\\.waitForURL"}, + {"mode": "cli-died-waiting-ready", "category": "cli-process", "pattern": "Process exited \\(code \\d+\\) while waiting for output: \"Ready"}, + {"mode": "cli-died-waiting-other", "category": "cli-process", "pattern": "Process exited \\(code \\d+\\) while waiting for output"}, + {"mode": "dev-exit-hang", "category": "cli-hang", "pattern": "waiting for process exit"}, + {"mode": "cli-output-timeout", "category": "cli-hang", "pattern": "Timed out after \\d+ms waiting for output"}, + {"mode": "playwright-test-timeout", "category": "cli-hang", "pattern": "Test timeout of \\d+ms exceeded"}, + {"mode": "dashboard-delete-timeout", "category": "cleanup", "pattern": "scrollIntoViewIfNeeded: Timeout"}, + {"mode": "browser-context-closed", "category": "cleanup", "pattern": "Target page, context or browser has been closed"} + ] +} diff --git a/bin/e2e-failure-report.js b/bin/e2e-failure-report.js new file mode 100644 index 00000000000..8fbbfd6904b --- /dev/null +++ b/bin/e2e-failure-report.js @@ -0,0 +1,240 @@ +import {appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs' +import * as path from 'node:path' +import {fileURLToPath} from 'node:url' + +// Classifies failed E2E shard jobs from the PR workflow into named failure +// modes and appends them to a durable dataset. Job logs expire after 90 days +// and run history after ~8 months; this dataset is what survives. +// +// Because the E2E job is continue-on-error, runs with failed shards still +// conclude as success, so we cannot filter for failed runs — every run's job +// list is scanned. +// +// Usage: +// GITHUB_TOKEN=... node bin/e2e-failure-report.js --days 2 --data-dir .e2e-data +// GITHUB_TOKEN=... node bin/e2e-failure-report.js --since 2026-05-15 --until 2026-05-22 --data-dir .e2e-data +// +// Data files (JSONL, idempotent — re-scans skip already-recorded IDs): +// runs.jsonl — one line per completed PR-workflow run (the denominator) +// failures.jsonl — one line per failed E2E shard job, with matched modes +// summary.md — last-7-days mode table vs the prior 7 days + +const repoRoot = path.dirname(path.dirname(fileURLToPath(import.meta.url))) +const token = process.env.GITHUB_TOKEN +if (!token) { + console.error('GITHUB_TOKEN is required') + process.exit(1) +} +const repo = process.env.GITHUB_REPOSITORY ?? 'Shopify/cli' + +const args = process.argv.slice(2) +const argValue = (name) => { + const index = args.indexOf(name) + return index === -1 ? undefined : args[index + 1] +} +const dataDir = argValue('--data-dir') ?? '.e2e-data' +const days = Number(argValue('--days') ?? 2) +const until = argValue('--until') ?? new Date().toISOString().slice(0, 10) +const since = + argValue('--since') ?? new Date(new Date(until).getTime() - days * 24 * 60 * 60 * 1000).toISOString().slice(0, 10) + +const {modes: MODES} = JSON.parse(readFileSync(path.join(repoRoot, 'bin/e2e-failure-modes.json'), 'utf8')) +for (const mode of MODES) mode.regex = new RegExp(mode.pattern) + +// --------------------------------------------------------------------------- +// GitHub API helpers +// --------------------------------------------------------------------------- + +async function apiRequest(url, {raw = false} = {}) { + for (let attempt = 1; attempt <= 5; attempt++) { + const response = await fetch(url, { + headers: {Authorization: `Bearer ${token}`, Accept: 'application/vnd.github+json'}, + redirect: 'manual', + }) + // Log downloads redirect to blob storage; refetch without the auth header. + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get('location') + const blob = await fetch(location) + return raw ? blob.text() : blob.json() + } + if (response.status === 403 || response.status === 429) { + const reset = Number(response.headers.get('x-ratelimit-reset') ?? 0) * 1000 + const waitMs = Math.min(Math.max(reset - Date.now(), 30_000), 15 * 60_000) + console.error(`rate limited, waiting ${Math.round(waitMs / 1000)}s`) + await new Promise((resolve) => setTimeout(resolve, waitMs)) + continue + } + if (response.status === 404 || response.status === 410) return undefined + if (!response.ok) throw new Error(`${url} failed: ${response.status}`) + return raw ? response.text() : response.json() + } + throw new Error(`${url} still rate limited after retries`) +} + +async function listRunsForDay(day) { + const runs = [] + for (let page = 1; page <= 10; page++) { + const result = await apiRequest( + `https://api.github.com/repos/${repo}/actions/workflows/tests-pr.yml/runs?created=${day}..${day}&per_page=100&page=${page}`, + ) + runs.push(...(result?.workflow_runs ?? [])) + if (!result || result.workflow_runs.length < 100) break + } + return runs +} + +async function listJobs(runId) { + const jobs = [] + for (let page = 1; page <= 5; page++) { + const result = await apiRequest( + `https://api.github.com/repos/${repo}/actions/runs/${runId}/jobs?per_page=100&page=${page}`, + ) + jobs.push(...(result?.jobs ?? [])) + if (!result || result.jobs.length < 100) break + } + return jobs +} + +// --------------------------------------------------------------------------- +// Classification and dataset +// --------------------------------------------------------------------------- + +function classify(logText) { + const matched = MODES.filter((mode) => mode.regex.test(logText)).map((mode) => mode.mode) + if (matched.length === 0) { + const firstError = logText.match(/^.*(Error:|error).*$/m)?.[0]?.slice(0, 200) ?? '' + return {primary: 'unclassified', modes: [], firstError} + } + return {primary: matched[0], modes: matched} +} + +function readJsonl(filePath) { + if (!existsSync(filePath)) return [] + return readFileSync(filePath, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)) +} + +mkdirSync(dataDir, {recursive: true}) +const runsPath = path.join(dataDir, 'runs.jsonl') +const failuresPath = path.join(dataDir, 'failures.jsonl') +const seenRuns = new Set(readJsonl(runsPath).map((run) => run.runId)) +const seenJobs = new Set(readJsonl(failuresPath).map((failure) => failure.jobId)) + +// --------------------------------------------------------------------------- +// Scan +// --------------------------------------------------------------------------- + +const dayCursor = new Date(since) +const untilDate = new Date(until) +let scanned = 0 +let newFailures = 0 + +while (dayCursor <= untilDate) { + const day = dayCursor.toISOString().slice(0, 10) + dayCursor.setDate(dayCursor.getDate() + 1) + + const runs = await listRunsForDay(day) + console.error(`${day}: ${runs.length} runs`) + + for (const run of runs) { + if (run.status !== 'completed' || seenRuns.has(run.id)) continue + scanned++ + + const jobs = await listJobs(run.id) + const shardJobs = jobs.filter((job) => job.name.startsWith('E2E tests (shard')) + const ranJobs = shardJobs.filter((job) => job.conclusion && job.conclusion !== 'skipped') + const failedJobs = ranJobs.filter((job) => job.conclusion === 'failure') + + appendFileSync( + runsPath, + `${JSON.stringify({ + runId: run.id, + createdAt: run.created_at, + e2eRan: ranJobs.length > 0, + shards: ranJobs.length, + failedShards: failedJobs.length, + })}\n`, + ) + seenRuns.add(run.id) + + for (const job of failedJobs) { + if (seenJobs.has(job.id)) continue + const logText = await apiRequest(`https://api.github.com/repos/${repo}/actions/jobs/${job.id}/logs`, {raw: true}) + const result = logText ? classify(logText) : {primary: 'log-unavailable', modes: []} + appendFileSync( + failuresPath, + `${JSON.stringify({ + date: run.created_at.slice(0, 10), + runId: run.id, + jobId: job.id, + shard: job.name, + pr: run.pull_requests?.[0]?.number ?? null, + primary: result.primary, + modes: result.modes, + firstError: result.firstError, + url: job.html_url, + })}\n`, + ) + seenJobs.add(job.id) + newFailures++ + } + } +} + +console.error(`scanned ${scanned} new runs, recorded ${newFailures} new failed shards`) + +// --------------------------------------------------------------------------- +// Summary — last 7 days vs the 7 before, by primary mode +// --------------------------------------------------------------------------- + +const now = Date.now() +const weekMs = 7 * 24 * 60 * 60 * 1000 +const inWindow = (dateString, start, end) => { + const time = new Date(dateString).getTime() + return time >= start && time < end +} + +const allRuns = readJsonl(runsPath) +const allFailures = readJsonl(failuresPath) + +function windowStats(start, end) { + const runs = allRuns.filter((run) => inWindow(run.createdAt, start, end) && run.e2eRan) + const failures = allFailures.filter((failure) => inWindow(failure.date, start, end)) + const runsWithFailure = runs.filter((run) => run.failedShards > 0).length + const modeCounts = {} + for (const failure of failures) modeCounts[failure.primary] = (modeCounts[failure.primary] ?? 0) + 1 + return {runs: runs.length, runsWithFailure, failures: failures.length, modeCounts} +} + +const thisWeek = windowStats(now - weekMs, now) +const lastWeek = windowStats(now - 2 * weekMs, now - weekMs) +const rate = (stats) => (stats.runs === 0 ? '—' : `${((stats.runsWithFailure / stats.runs) * 100).toFixed(1)}%`) + +const allModeNames = [...new Set([...Object.keys(thisWeek.modeCounts), ...Object.keys(lastWeek.modeCounts)])].sort( + (a, b) => (thisWeek.modeCounts[b] ?? 0) - (thisWeek.modeCounts[a] ?? 0), +) + +const lines = [ + '# E2E failure report', + '', + `| | last 7 days | previous 7 days |`, + `|---|---|---|`, + `| Runs where E2E ran | ${thisWeek.runs} | ${lastWeek.runs} |`, + `| Runs with a failed shard | ${thisWeek.runsWithFailure} (${rate(thisWeek)}) | ${lastWeek.runsWithFailure} (${rate(lastWeek)}) |`, + `| Failed shard jobs | ${thisWeek.failures} | ${lastWeek.failures} |`, + '', + '## Failure modes (primary, per failed shard)', + '', + '| Mode | last 7 days | previous 7 days |', + '|---|---|---|', + ...allModeNames.map( + (mode) => `| ${mode} | ${thisWeek.modeCounts[mode] ?? 0} | ${lastWeek.modeCounts[mode] ?? 0} |`, + ), + '', + `_Window scanned this execution: ${since}..${until}. Unclassified failures carry a \`firstError\` snippet in failures.jsonl — name them in bin/e2e-failure-modes.json._`, +] + +writeFileSync(path.join(dataDir, 'summary.md'), `${lines.join('\n')}\n`) +console.log(lines.join('\n'))