diff --git a/.github/workflows/tests-pr.yml b/.github/workflows/tests-pr.yml index 5e35457511..0a805cf8a7 100644 --- a/.github/workflows/tests-pr.yml +++ b/.github/workflows/tests-pr.yml @@ -316,3 +316,17 @@ jobs: run: | echo '::error::Breaking changes detected. See the sticky comment on the PR for details.' exit 1 + + ci-gate-sync: + name: 'CI gate manifest' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.DEFAULT_NODE_VERSION }} + - name: Test the gate checker + run: node --test bin/check-ci-gates.test.js + - name: Check the CI gate manifest matches this workflow + run: node bin/check-ci-gates.js diff --git a/bin/check-ci-gates.js b/bin/check-ci-gates.js new file mode 100644 index 0000000000..fd90781721 --- /dev/null +++ b/bin/check-ci-gates.js @@ -0,0 +1,91 @@ +// Drift guard: fails if the local CI-gate manifest (bin/ci-gates.js) falls out of +// sync with .github/workflows/tests-pr.yml, or if the pinned tool versions in +// dev.yml and tests-pr.yml disagree. Kept dependency-free (no YAML library) so the +// CI job can run on bare Node; the parsing below is hardened for the formats these +// two files actually use. +import {readFileSync} from 'node:fs' +import {fileURLToPath, pathToFileURL} from 'node:url' +import {dirname, join} from 'node:path' + +import {MANIFEST_JOB_IDS} from './ci-gates.js' + +// Job ids are the keys directly under `jobs:`. Bound the search to the jobs block +// (up to the next top-level key) and allow a trailing comment after the id. Job +// keys are always bare (their mapping is on following lines), so nested keys — +// indented deeper than 2 spaces — and `key: value` anchors are naturally excluded. +export function parseJobIds(workflowText) { + const workflow = workflowText.replace(/\r\n/g, '\n') + const jobsAt = workflow.search(/^jobs:/m) + if (jobsAt === -1) return [] + const afterHeader = workflow.slice(jobsAt).replace(/^jobs:.*\n/, '') + const nextTopLevel = afterHeader.search(/^\S/m) + const block = nextTopLevel === -1 ? afterHeader : afterHeader.slice(0, nextTopLevel) + return [...block.matchAll(/^ {2}([A-Za-z0-9_-]+):[ \t]*(?:#.*)?$/gm)].map((match) => match[1]) +} + +// Pure and testable: given the two YAML texts and the manifest job ids, return the +// list of human-readable problems (empty when everything is in sync). +export function findProblems({workflow: workflowText, devYml: devYmlText, manifestJobIds}) { + const problems = [] + + // Normalize line endings so the parsing below is robust to CRLF working trees. + const workflow = workflowText.replace(/\r\n/g, '\n') + const devYml = devYmlText.replace(/\r\n/g, '\n') + + const workflowJobIds = parseJobIds(workflow) + const manifestSet = new Set(manifestJobIds) + const workflowSet = new Set(workflowJobIds) + const missingFromManifest = workflowJobIds.filter((id) => !manifestSet.has(id)) + const staleInManifest = manifestJobIds.filter((id) => !workflowSet.has(id)) + + if (missingFromManifest.length > 0) { + problems.push( + `Workflow jobs not classified in bin/ci-gates.js: ${missingFromManifest.join(', ')}.\n` + + ` Add each to CI_GATES as a 'pre-ci' gate (with a local command) or 'ci-only' (with a reason).`, + ) + } + if (staleInManifest.length > 0) { + problems.push(`bin/ci-gates.js lists jobs absent from tests-pr.yml: ${staleInManifest.join(', ')}.`) + } + + const pick = (source, regex, label) => { + const match = source.match(regex) + if (!match) problems.push(`Could not read ${label}.`) + return match ? match[1] : undefined + } + const ciNode = pick(workflow, /DEFAULT_NODE_VERSION:\s*['"]?([0-9][\w.-]*)/, 'DEFAULT_NODE_VERSION from tests-pr.yml') + const ciPnpm = pick(workflow, /PNPM_VERSION:\s*['"]?([0-9][\w.-]*)/, 'PNPM_VERSION from tests-pr.yml') + // dev.yml pins these on the `version:`/`package_manager:` lines under the `node:` step. + const devNode = pick(devYml, /node:\s*\n\s+version:\s*['"]?([0-9][\w.-]*)/, 'the node version from dev.yml') + const devPnpm = pick(devYml, /package_manager:\s*['"]?pnpm@([0-9][\w.-]*)/, 'the pnpm version from dev.yml') + + if (ciNode && devNode && ciNode !== devNode) { + problems.push(`Node version mismatch: dev.yml ${devNode} vs tests-pr.yml DEFAULT_NODE_VERSION ${ciNode}.`) + } + if (ciPnpm && devPnpm && ciPnpm !== devPnpm) { + problems.push(`pnpm version mismatch: dev.yml ${devPnpm} vs tests-pr.yml PNPM_VERSION ${ciPnpm}.`) + } + + return {problems, workflowJobIds, ciNode, ciPnpm} +} + +// Run as a CLI when invoked directly (not when imported by a test). +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const root = join(dirname(fileURLToPath(import.meta.url)), '..') + const read = (rel) => readFileSync(join(root, rel), 'utf8') + + const {problems, workflowJobIds, ciNode, ciPnpm} = findProblems({ + workflow: read('.github/workflows/tests-pr.yml'), + devYml: read('dev.yml'), + manifestJobIds: MANIFEST_JOB_IDS, + }) + + if (problems.length > 0) { + console.error('CI gate manifest is out of sync with the workflow:\n') + for (const problem of problems) console.error(`- ${problem}`) + process.exit(1) + } + console.log( + `CI gate manifest in sync: ${workflowJobIds.length} workflow jobs classified; tool versions match (node ${ciNode}, pnpm ${ciPnpm}).`, + ) +} diff --git a/bin/check-ci-gates.test.js b/bin/check-ci-gates.test.js new file mode 100644 index 0000000000..21df4c48c6 --- /dev/null +++ b/bin/check-ci-gates.test.js @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import {findProblems, parseJobIds} from './check-ci-gates.js' + +// Versions below are arbitrary fixtures, not the repo's real pins — the guard +// reads those from dev.yml / tests-pr.yml at runtime, never hard-coded. +const workflow = (jobIds, {node = '1.2.3', pnpm = '4.5.6'} = {}) => + `name: tests\non: pull_request\nenv:\n DEFAULT_NODE_VERSION: '${node}'\n PNPM_VERSION: '${pnpm}'\njobs:\n` + + jobIds.map((id) => ` ${id}:\n runs-on: ubuntu-latest\n steps: []`).join('\n') + + '\n' + +const devYml = ({node = '1.2.3', pnpm = '4.5.6'} = {}) => + `name: cli\nup:\n - node:\n version: ${node}\n package_manager: pnpm@${pnpm}\n - packages:\n - jq\n` + +test('in sync: no problems', () => { + const {problems} = findProblems({workflow: workflow(['a', 'b']), devYml: devYml(), manifestJobIds: ['a', 'b']}) + assert.deepEqual(problems, []) +}) + +test('workflow job missing from the manifest', () => { + const {problems} = findProblems({workflow: workflow(['a', 'b', 'c']), devYml: devYml(), manifestJobIds: ['a', 'b']}) + assert.match(problems.join('\n'), /not classified.*\bc\b/) +}) + +test('manifest lists a job absent from the workflow', () => { + const {problems} = findProblems({workflow: workflow(['a']), devYml: devYml(), manifestJobIds: ['a', 'b']}) + assert.match(problems.join('\n'), /absent from tests-pr\.yml.*\bb\b/) +}) + +test('node version mismatch is detected', () => { + const {problems} = findProblems({workflow: workflow(['a'], {node: '9.9.9'}), devYml: devYml({node: '1.2.3'}), manifestJobIds: ['a']}) + assert.match(problems.join('\n'), /Node version mismatch/) +}) + +test('pnpm version mismatch is detected', () => { + const {problems} = findProblems({workflow: workflow(['a']), devYml: devYml({pnpm: '9.0.0'}), manifestJobIds: ['a']}) + assert.match(problems.join('\n'), /pnpm version mismatch/) +}) + +test('a missing version pin is reported, not silently passed', () => { + const noEnv = `name: t\non: pull_request\njobs:\n a:\n runs-on: ubuntu-latest\n steps: []\n` + const {problems} = findProblems({workflow: noEnv, devYml: devYml(), manifestJobIds: ['a']}) + assert.match(problems.join('\n'), /Could not read DEFAULT_NODE_VERSION/) +}) + +// Hardening: tolerate a trailing comment after the job id, and ignore deeper-indented +// keys, blank lines, comment lines, and any top-level section after `jobs:`. +test('parseJobIds tolerates comments and ignores non-job lines', () => { + const wf = `env:\n DEFAULT_NODE_VERSION: '1.2.3'\njobs:\n build: # freshness gate\n runs-on: ubuntu-latest\n env:\n NESTED: 1\n\n # a comment line\n test-job:\n steps: []\nconcurrency:\n group: x\n` + assert.deepEqual(parseJobIds(wf), ['build', 'test-job']) +}) + +test('parseJobIds returns empty when there is no jobs block', () => { + assert.deepEqual(parseJobIds('name: t\non: push\n'), []) +}) + +test('CRLF line endings are handled', () => { + const wf = workflow(['a', 'b']).replace(/\n/g, '\r\n') + assert.deepEqual(parseJobIds(wf), ['a', 'b']) + const {problems} = findProblems({workflow: wf, devYml: devYml().replace(/\n/g, '\r\n'), manifestJobIds: ['a', 'b']}) + assert.deepEqual(problems, []) +}) diff --git a/bin/ci-gates.js b/bin/ci-gates.js new file mode 100644 index 0000000000..715f50f29a --- /dev/null +++ b/bin/ci-gates.js @@ -0,0 +1,52 @@ +// Single source of truth mapping every job in .github/workflows/tests-pr.yml to +// either a local `pre-ci` command (run-what-CI-runs) or an explicit reason it is +// CI-only. `bin/pre-ci.js` runs the pre-ci gates; `bin/check-ci-gates.js` asserts +// this list stays in sync with the workflow so the two cannot silently drift. +// +// `job` is the workflow job id (the key under `jobs:`), which is stable, unlike +// the rendered display name (matrix jobs interpolate `${{ ... }}`). + +export const CI_GATES = [ + // --- gates a contributor can reproduce locally before pushing --- + // Ordered as pre-ci should run them: build precedes the oclif codegen check, + // and the graphql check precedes the oclif check (their whole-repo `git status` + // asserts otherwise cross-contaminate in a single working tree). + {job: 'type-check', kind: 'pre-ci', command: 'pnpm type-check'}, + {job: 'lint', kind: 'pre-ci', command: 'pnpm lint'}, + {job: 'bundle', kind: 'pre-ci', command: 'pnpm build'}, + {job: 'knip', kind: 'pre-ci', command: 'pnpm knip'}, + {job: 'graphql-schema', kind: 'pre-ci', command: 'pnpm codegen:check:graphql'}, + {job: 'oclif-checks', kind: 'pre-ci', command: 'pnpm codegen:check:oclif'}, + {job: 'unit-tests', kind: 'pre-ci', command: 'pnpm test'}, + + // --- CI-only jobs, with the reason they are not part of pre-ci --- + { + job: 'unit-tests-gate', + kind: 'ci-only', + reason: 'Aggregation gate that only collects the unit-tests matrix results; nothing to run locally.', + }, + { + job: 'e2e-tests', + kind: 'ci-only', + reason: 'Needs Playwright browsers and real test stores/credentials; too slow and credentialed for a pre-push check.', + }, + { + job: 'type-diff', + kind: 'ci-only', + reason: 'Diffs the public type surface against the main branch; needs a base checkout, not a single local working tree.', + }, + { + job: 'major-change-check', + kind: 'ci-only', + reason: 'Breaking-change detection against the PR base; advisory and diff-based, not reproducible from one local tree.', + }, + { + job: 'ci-gate-sync', + kind: 'ci-only', + reason: 'Meta gate: runs `pnpm check-ci-gates` to keep this manifest in sync with tests-pr.yml. pre-ci runs the same check locally.', + }, +] + +export const PRE_CI_GATES = CI_GATES.filter((gate) => gate.kind === 'pre-ci') +export const CI_ONLY_GATES = CI_GATES.filter((gate) => gate.kind === 'ci-only') +export const MANIFEST_JOB_IDS = CI_GATES.map((gate) => gate.job) diff --git a/bin/pre-ci.js b/bin/pre-ci.js new file mode 100644 index 0000000000..e148a35d30 --- /dev/null +++ b/bin/pre-ci.js @@ -0,0 +1,42 @@ +// Runs the local subset of PR CI gates ("run what CI runs") so contributors can +// catch failures before pushing. The gate list and its parity with the workflow +// are defined in bin/ci-gates.js and enforced by bin/check-ci-gates.js. +// +// pre-ci mirrors CI's full (`--all`) targets so that green locally implies green +// in CI. It is intentionally slower than the affected-only `dev check`. +import {execSync} from 'node:child_process' + +import {PRE_CI_GATES, CI_ONLY_GATES} from './ci-gates.js' + +const steps = [ + {label: 'CI gate manifest in sync', command: 'pnpm check-ci-gates'}, + ...PRE_CI_GATES.map((gate) => ({label: gate.job, command: gate.command})), +] + +const results = [] +for (const step of steps) { + process.stdout.write(`\n▶ ${step.label}: ${step.command}\n`) + try { + execSync(step.command, {stdio: 'inherit'}) + results.push({...step, ok: true}) + } catch { + results.push({...step, ok: false}) + } +} + +console.log('\n──────── pre-ci summary ────────') +for (const result of results) { + console.log(`${result.ok ? '✓' : '✗'} ${result.label}`) +} + +console.log('\nNot run locally (CI-only):') +for (const gate of CI_ONLY_GATES) { + console.log(`· ${gate.job} — ${gate.reason}`) +} + +const failed = results.filter((result) => !result.ok) +if (failed.length > 0) { + console.error(`\npre-ci failed: ${failed.map((result) => result.label).join(', ')}`) + process.exit(1) +} +console.log('\npre-ci passed. Note: codegen checks regenerate files — review `git status` for any uncommitted generated changes.') diff --git a/dev.yml b/dev.yml index 6c6bffb6bf..8f64660a55 100644 --- a/dev.yml +++ b/dev.yml @@ -66,6 +66,9 @@ commands: type-check: desc: 'Type-check the project' run: pnpm run type-check:affected + pre-ci: + desc: 'Run the local subset of PR CI gates (run what CI runs) before pushing' + run: pnpm run pre-ci check: type-check: pnpm nx affected --target=type-check diff --git a/package.json b/package.json index ab8b564e89..b53670b7e4 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "clean": "nx run-many --target=clean --all --skip-nx-cache && nx reset", "codegen": "pnpm graphql-codegen:get-graphql-schemas && pnpm graphql-codegen && pnpm refresh-manifests && pnpm refresh-code-documentation && pnpm build-dev-docs", "codegen:check:graphql": "pnpm graphql-codegen:get-graphql-schemas && pnpm graphql-codegen && node ./bin/check-codegen-clean.js graphql", + "check-ci-gates": "node bin/check-ci-gates.js", "codegen:check:oclif": "pnpm refresh-manifests && node ./bin/check-codegen-clean.js oclif:manifests && pnpm refresh-readme && node ./bin/check-codegen-clean.js oclif:readme && pnpm refresh-code-documentation && node ./bin/check-codegen-clean.js oclif:code-docs && pnpm build-dev-docs && node ./bin/check-codegen-clean.js oclif:dev-docs", "create-app": "nx build create-app && node packages/create-app/bin/dev.js --package-manager pnpm", "deploy-experimental": "node bin/deploy-experimental.js", @@ -29,6 +30,7 @@ "refresh-readme": "nx run-many --target=refresh-readme --all --skip-nx-cache", "release": "./bin/release", "post-release": "./bin/post-release", + "pre-ci": "node bin/pre-ci.js", "update-observe": "node bin/update-observe.js", "shopify:run": "node packages/cli/bin/dev.js", "shopify": "nx build cli && node packages/cli/bin/dev.js",