From da09cd3a03f5aeb40c769043be5aeee9df7db2d1 Mon Sep 17 00:00:00 2001 From: Charles Lowell Date: Thu, 12 Nov 2026 09:50:29 -0600 Subject: [PATCH] Make focused WPT runs register only matching cases --- .changeset/wpt-focused-runs.md | 5 + .github/workflows/benchmark.yml | 1 + .github/workflows/check.yml | 1 + CONTRIBUTING.md | 10 +- package.json | 4 +- packages/ui/package.json | 2 +- packages/ui/src/dom.compatibility.test.ts | 4 +- packages/ui/test/README.md | 13 +- packages/web-platform-tests/README.md | 38 +++- packages/web-platform-tests/bin/wpt.mjs | 2 + packages/web-platform-tests/knip.json | 5 + packages/web-platform-tests/package.json | 30 ++- packages/web-platform-tests/src/cli.ts | 68 +++++++ .../web-platform-tests/src/shared/context.ts | 2 + .../src/shared/test-dom-compatibility.ts | 12 +- packages/web-platform-tests/src/vitest.ts | 29 +++ .../web-platform-tests/src/vitest/runner.ts | 37 ++++ packages/web-platform-tests/test/cli.test.ts | 178 ++++++++++++++++++ .../test/compatibility-harness.test.ts | 67 ++++++- pnpm-lock.yaml | 21 +++ vitest.config.mts | 27 +-- 21 files changed, 517 insertions(+), 39 deletions(-) create mode 100644 .changeset/wpt-focused-runs.md create mode 100755 packages/web-platform-tests/bin/wpt.mjs create mode 100644 packages/web-platform-tests/src/cli.ts create mode 100644 packages/web-platform-tests/src/vitest.ts create mode 100644 packages/web-platform-tests/src/vitest/runner.ts create mode 100644 packages/web-platform-tests/test/cli.test.ts diff --git a/.changeset/wpt-focused-runs.md b/.changeset/wpt-focused-runs.md new file mode 100644 index 0000000..c164499 --- /dev/null +++ b/.changeset/wpt-focused-runs.md @@ -0,0 +1,5 @@ +--- +'@bomb.sh/web-platform-tests': patch +--- + +Allow runners to filter configured WPT cases before test registration. Provide a Vitest integration and a Router-based `wpt` command that accepts literal search text, so focused runs omit unrelated cases instead of reporting them as skipped. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4058f46..49ed6be 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,6 +22,7 @@ jobs: persist-credentials: false - uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 - run: pnpm install --frozen-lockfile + - run: pnpm --filter @bomb.sh/web-platform-tests build - name: Run benchmarks with CodSpeed uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5 with: diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 04f3145..5d39482 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -40,6 +40,7 @@ jobs: persist-credentials: false - uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 - run: pnpm install --frozen-lockfile + - run: pnpm --filter @bomb.sh/web-platform-tests build - run: pnpm test dom-wpt: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b3f57c1..ff1ac08 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,15 +15,18 @@ You need the exact Node and pnpm versions listed under `devEngines` in the root Any version manager works. If you use [mise](https://mise.jdx.dev), which is what our lint, format, and test checks use, export `MISE_IDIOMATIC_VERSION_FILE_ENABLE_TOOLS=node,pnpm` and run `mise install` so it reads those versions from `package.json`. -Then install the dependencies: +Then install the dependencies and build the packages: ```sh pnpm install +pnpm build ``` ## Commands you'll use -Run these from the repo root. You don't need to build before you run the tests. +Run these from the repo root. Tests and benchmarks reuse the built WPT package. +Rebuild it after changing its sources with `pnpm --filter @bomb.sh/web-platform-tests build`; +you can run tests repeatedly without rebuilding otherwise. | Command | What it does | | ------------------------------------------ | ----------------------------------------- | @@ -63,12 +66,13 @@ Use `expect.poll` here, because the screen doesn't update the moment you change Before you push, make sure these pass: ```sh +pnpm build pnpm lint pnpm format:check pnpm test ``` -CI runs the same three on every pull request, along with the benchmarks and a workflow that publishes a preview build. In your PR, tell us why the change is needed and link the issue if there is one. +CI runs these checks on every pull request, building the WPT package once before tests or benchmarks. A separate workflow publishes a preview build. In your PR, tell us why the change is needed and link the issue if there is one. ## Be kind diff --git a/package.json b/package.json index cf51ceb..32201f4 100644 --- a/package.json +++ b/package.json @@ -33,10 +33,12 @@ "format": "bsh format", "format:check": "bsh format --check", "lint": "bsh lint .", - "test": "pnpm --filter @bomb.sh/web-platform-tests build && bsh test" + "test": "vitest run --config vitest.config.mts --experimental.viteModuleRunner", + "wpt": "wpt --file packages/ui/src/dom.compatibility.test.ts" }, "devDependencies": { "@bomb.sh/tools": "^0.8.0", + "@bomb.sh/web-platform-tests": "workspace:*", "@codspeed/vitest-plugin": "^6.1.0", "@termless/core": "^0.10.2", "@termless/test": "^0.10.2", diff --git a/packages/ui/package.json b/packages/ui/package.json index 5e696f1..17593c2 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -33,7 +33,7 @@ "build": "bsh build --dts", "format": "bsh format", "lint": "bsh lint", - "test": "pnpm --filter @bomb.sh/web-platform-tests build && bsh test" + "test": "vitest run --config ../../vitest.config.mts --experimental.viteModuleRunner" }, "dependencies": { "@bomb.sh/tty": "catalog:" diff --git a/packages/ui/src/dom.compatibility.test.ts b/packages/ui/src/dom.compatibility.test.ts index 01055f5..2258433 100644 --- a/packages/ui/src/dom.compatibility.test.ts +++ b/packages/ui/src/dom.compatibility.test.ts @@ -1,10 +1,8 @@ -import { describe, it, expect } from 'vitest'; -import { testDOMCompatibility } from '@bomb.sh/web-platform-tests'; +import { testDOMCompatibility } from '@bomb.sh/web-platform-tests/vitest/runner'; import { Document } from './dom.ts'; import config from '../test/wpt.config.ts'; testDOMCompatibility({ - runner: { describe, it, expect }, config, create: () => new Document(() => {}), }); diff --git a/packages/ui/test/README.md b/packages/ui/test/README.md index 2e0750d..5da7e56 100644 --- a/packages/ui/test/README.md +++ b/packages/ui/test/README.md @@ -1,12 +1,12 @@ # UI tests -Tests live alongside the implementation in `src/`; this directory contains their shared fixtures and WPT configuration. Run the UI tests from the workspace root: +Tests live in `src/` and this directory, alongside shared fixtures and WPT configuration. Run the UI tests from the workspace root: ```sh pnpm --filter @clack/ui test ``` -The test script builds the private `@bomb.sh/web-platform-tests` package before running Vitest. Use `pnpm test` from the workspace root to run the workspace tests. +Run `pnpm build` once before testing, and rebuild the private `@bomb.sh/web-platform-tests` package after changing its sources. Test commands reuse that build. Use `pnpm test` from the workspace root to run the workspace tests. ## Terminal fixtures @@ -26,13 +26,16 @@ Rendering, scheduling, notifications, and lifecycle behavior belong in the UI te The configuration explicitly excludes unsupported directories. Add includes or narrow exclusions as cases pass. Selecting a source runs all of our adapted cases from that file; it does not imply complete coverage of the upstream file or interface. -Include the source or case in the configuration while implementing it. Vitest's `-t` can narrow the configured selection by source path and case name: +Include the source or case in the configuration while implementing it. Run matching cases by supplying plain text from a source path or case name: ```sh -pnpm --filter @clack/ui test src/dom.compatibility.test.ts -t 'Document-createElement.html.*createElement\(foo\)' +pnpm wpt createElement +pnpm wpt 'createElement(foo) in HTML document' ``` -If the shared package is already built, avoid rebuilding by using `pnpm --filter @clack/ui exec bsh test src/dom.compatibility.test.ts` with the same name filter. +The script invokes the package-owned `wpt` CLI, which uses `@bomb.sh/router` to parse arguments and launches Vitest with the UI test entrypoint. It reuses the existing WPT build. The search is a literal substring, so punctuation needs no regex escaping. Omit the search text (`pnpm wpt`) to run all configured cases. Matching happens before Vitest registers tests, so unrelated cases do not appear as skipped. + +The root Vitest configuration imports `wptNameFilter()` from `@bomb.sh/web-platform-tests/vitest`; the UI entrypoint uses the worker adapter from `@bomb.sh/web-platform-tests/vitest/runner`. Direct Vitest runs still support `-t` regular expressions. Watch mode retains the full configured catalogue so changing its interactive name filter can discover other cases. When implementing behavior, read the source-named file under [`src/wpt`](../../web-platform-tests/src/wpt), implement its operations, and update the configuration once its cases pass. Extend the fixture when cases need document parsing, host input, or cleanup. Implementation-specific rendering, notification, and lifecycle checks remain in the UI tests. diff --git a/packages/web-platform-tests/README.md b/packages/web-platform-tests/README.md index 68f1f8e..7ba9f01 100644 --- a/packages/web-platform-tests/README.md +++ b/packages/web-platform-tests/README.md @@ -16,14 +16,25 @@ The WPT suite must contain only faithful adaptations of existing upstream cases. The shared suite is the long-term home of upstream WPT coverage for the UI. Implementation-specific rendering, notification, and lifecycle checks belong in the corresponding UI integration tests. +### Vitest integration + +Register `wptNameFilter()` in the consumer's Vitest configuration: + +```ts +import { defineConfig } from 'vitest/config'; +import { wptNameFilter } from '@bomb.sh/web-platform-tests/vitest'; + +export default defineConfig({ plugins: [wptNameFilter()] }); +``` + +The test entrypoint supplies the DOM implementation and case selection: + ```ts -import { testDOMCompatibility } from '@bomb.sh/web-platform-tests'; -import { describe, it, expect } from 'vitest'; +import { testDOMCompatibility } from '@bomb.sh/web-platform-tests/vitest/runner'; import { createDocument, dispose } from './implementation-fixture.ts'; import config from './wpt.config.ts'; testDOMCompatibility({ - runner: { describe, it, expect }, config, create: createDocument, dispose, @@ -31,6 +42,25 @@ testDOMCompatibility({ }); ``` +The configuration plugin is safe to import outside a test worker. The separate worker adapter supplies Vitest's assertions and filters cases before registration. Direct Vitest `-t` filters work too; watch mode retains the full configured catalogue so interactive filter changes can discover other cases. + +The main `@bomb.sh/web-platform-tests` entrypoint remains independent of Vitest. Other runners can use its `testDOMCompatibility()` with their own `runner: { describe, it, expect, filter? }`. + +### CLI + +The package provides a `wpt` executable using `@bomb.sh/router`. Install Vitest in the consuming workspace and build the WPT package before running it: + +```sh +pnpm --filter @bomb.sh/web-platform-tests build +pnpm exec wpt --file path/to/dom.compatibility.test.ts createElement +pnpm exec wpt --file path/to/dom.compatibility.test.ts 'createElement(foo) in HTML document' +pnpm exec wpt --help +``` + +`--file` selects the consumer's test entrypoint. `--config` optionally selects its Vitest configuration; otherwise Vitest discovers the configuration from the current working directory. Paths are relative to the caller, so the CLI does not depend on this repository's layout or package manager. The configuration must include `wptNameFilter()` and the entrypoint must use the worker adapter shown above. + +Search text is an optional literal substring of a source path or case name. Multiple words are joined with spaces; use `--` before search text beginning with a dash. Omit the search to run all configured cases. The CLI forwards Vitest's exit status and never runs a build. Rebuild after changing WPT package sources, then reuse the output across focused runs. + ### Selecting tests The `config` option is optional; omitting it runs all adapted cases. A `WPTConfig` supplies source-path selectors in `include` and `exclude`; an explicit include always wins. Each value is `true` for the entire source file or an array of exact case names and regular expressions matching case names. @@ -50,7 +80,7 @@ export default { Source paths support `*` within a path segment and `**` across directories; `**/` also matches zero directories. Other characters are literal. Case-name strings match exactly. Empty, invalid, or unmatched selectors fail configuration before any cases register, so stale exclusions remain visible. -Cases have no test-to-test prerequisite gates. A selected case exercises its supporting DOM operations and reports failures normally. Globals and host input are resolved lazily when the case uses them. The runner can narrow selected cases further with its usual name filter. +Cases have no test-to-test prerequisite gates. A selected case exercises its supporting DOM operations and reports failures normally. Globals and host input are resolved lazily when the case uses them. An optional `runner.filter(source, name)` narrows configured cases before registering suites or tests with the runner. Configuration is still validated against the full catalogue. If this filter matches nothing, the harness registers nothing. Run `pnpm generate` after adding, moving, or removing source files. The build also regenerates the registry from filenames. Only files exporting `register` are test entrypoints; upstream support files are imported by those tests. `node scripts/generate-registry.mjs --check` verifies that the checked-in registry is current. diff --git a/packages/web-platform-tests/bin/wpt.mjs b/packages/web-platform-tests/bin/wpt.mjs new file mode 100755 index 0000000..fadb052 --- /dev/null +++ b/packages/web-platform-tests/bin/wpt.mjs @@ -0,0 +1,2 @@ +#!/usr/bin/env node +await import('../dist/cli.mjs'); diff --git a/packages/web-platform-tests/knip.json b/packages/web-platform-tests/knip.json index ab35cd0..57f4133 100644 --- a/packages/web-platform-tests/knip.json +++ b/packages/web-platform-tests/knip.json @@ -2,6 +2,10 @@ "$schema": "https://unpkg.com/knip@6/schema.json", "entry": [ "src/index.ts", + "src/cli.ts", + "src/vitest.ts", + "src/vitest/runner.ts", + "bin/*.mjs", "test/*.test.ts", "test/browser/compatibility.mjs", "test/browser/run-compatibility.mjs", @@ -10,6 +14,7 @@ ], "project": [ "src/**/*.ts", + "bin/*.mjs", "test/**/*.{ts,mts,mjs}", "scripts/*.mjs" ] diff --git a/packages/web-platform-tests/package.json b/packages/web-platform-tests/package.json index c3aec0c..c681fb4 100644 --- a/packages/web-platform-tests/package.json +++ b/packages/web-platform-tests/package.json @@ -10,11 +10,19 @@ "types": "./dist/index.d.mts", "import": "./dist/index.mjs" }, - "./package.json": "./package.json" + "./package.json": "./package.json", + "./vitest": { + "types": "./dist/vitest.d.mts", + "import": "./dist/vitest.mjs" + }, + "./vitest/runner": { + "types": "./dist/vitest/runner.d.mts", + "import": "./dist/vitest/runner.mjs" + } }, "scripts": { "generate": "node scripts/generate-registry.mjs", - "build": "pnpm generate && bsh build src/index.ts --dts", + "build": "pnpm generate && bsh build src/index.ts src/vitest.ts src/vitest/runner.ts src/cli.ts --dts", "test": "bsh test", "typecheck": "tsc -p tsconfig.json && tsc -p test/browser/tsconfig.json", "test:browser:build": "pnpm generate && vite build --config test/browser/vite.config.mjs", @@ -30,6 +38,22 @@ "chai": "^6.2.2", "playwright": "^1.64.1", "typescript": "^6.0.2", - "vite": "^8.1.0" + "vite": "^8.1.0", + "vitest": "5.0.5" + }, + "bin": { + "wpt": "./bin/wpt.mjs" + }, + "dependencies": { + "@bomb.sh/router": "^0.8.0", + "zod": "^4.4.3" + }, + "peerDependencies": { + "vitest": "^5.0.5" + }, + "peerDependenciesMeta": { + "vitest": { + "optional": true + } } } diff --git a/packages/web-platform-tests/src/cli.ts b/packages/web-platform-tests/src/cli.ts new file mode 100644 index 0000000..4ee20a9 --- /dev/null +++ b/packages/web-platform-tests/src/cli.ts @@ -0,0 +1,68 @@ +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { + argument, + command, + description, + multiple, + name, + option, + parse, + printErrors, + printHelp, + schema, +} from '@bomb.sh/router'; +import * as z from 'zod'; + +const app = command( + name('wpt'), + description('Run configured Web Platform Tests with literal-text filtering.'), + option( + name('file'), + description('Consumer test entrypoint, relative to the current directory.'), + schema(z.string().min(1)), + ), + option( + name('config'), + description('Vitest configuration path; otherwise use normal Vitest discovery.'), + schema(z.string().min(1).optional()), + ), + argument( + name('search'), + description('Optional literal search text. Multiple words are joined with spaces.'), + multiple(), + schema(z.array(z.string()).default([])), + ), +); + +const argv = process.argv.slice(2); +const result = parse(app, { argv: argv.length === 0 ? ['--help'] : argv }); +if (!result.ok) { + console.error(printErrors(result)); + process.exitCode = 1; +} else if (result.method === 'help') { + process.stdout.write(`${printHelp(result)}\n`); +} else { + const text = [ + ...result.model.search, + ...Array.from(result.literals, (literal) => literal.text), + ].join(' '); + const args = ['run', result.model.file]; + if (result.model.config) { + args.push('--config', result.model.config); + } + if (text) { + // Keep numeric-looking searches as strings through Vitest's argument parser. + args.push('-t', `(?:${text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`); + } + const vitest = fileURLToPath(new URL('./vitest.mjs', import.meta.resolve('vitest/package.json'))); + const child = spawnSync(process.execPath, [vitest, ...args], { stdio: 'inherit' }); + if (child.error) { + console.error(child.error.message); + } + if (child.signal) { + process.kill(process.pid, child.signal); + } else { + process.exitCode = child.status ?? 1; + } +} diff --git a/packages/web-platform-tests/src/shared/context.ts b/packages/web-platform-tests/src/shared/context.ts index f05714f..3d49625 100644 --- a/packages/web-platform-tests/src/shared/context.ts +++ b/packages/web-platform-tests/src/shared/context.ts @@ -26,6 +26,8 @@ export interface CompatibilityExpect { } export interface CompatibilityRunner { + /** Narrow configured cases before registering suites or tests with the runner. */ + filter?(source: string, name: string): boolean; describe(name: string, register: () => void): unknown; it: { (name: string, run: () => void | Promise): unknown; diff --git a/packages/web-platform-tests/src/shared/test-dom-compatibility.ts b/packages/web-platform-tests/src/shared/test-dom-compatibility.ts index 59fd7e1..2893e45 100644 --- a/packages/web-platform-tests/src/shared/test-dom-compatibility.ts +++ b/packages/web-platform-tests/src/shared/test-dom-compatibility.ts @@ -30,16 +30,18 @@ export function testDOMCompatibility(options: DOMCompatibilityOptions): vo }, }); } + const selected = selectTests(cases, options.config); + if (selected.length === 0) { + options.runner.it.todo('implement DOM features and enable their WPT cases'); + return; + } const groups = new Map(); - for (const test of selectTests(cases, options.config)) { + for (const test of selected) { + if (options.runner.filter && !options.runner.filter(test.source, test.name)) continue; const group = groups.get(test.source) ?? []; group.push(test); groups.set(test.source, group); } - if (groups.size === 0) { - options.runner.it.todo('implement DOM features and enable their WPT cases'); - return; - } for (const [source, tests] of groups) { options.runner.describe(source, () => { for (const test of tests) diff --git a/packages/web-platform-tests/src/vitest.ts b/packages/web-platform-tests/src/vitest.ts new file mode 100644 index 0000000..453068e --- /dev/null +++ b/packages/web-platform-tests/src/vitest.ts @@ -0,0 +1,29 @@ +/* oxlint-disable bombshell-dev/exported-function-async -- Vitest plugin creation is synchronous. */ +import type { Plugin } from 'vitest/config'; + +/** Serializable name filter passed from Vitest configuration to test workers. */ +export interface WPTNamePattern { + source: string; + flags: string; +} + +declare module 'vitest' { + interface ProvidedContext { + wptTestNamePattern: WPTNamePattern | null; + } +} + +/** Pass Vitest's resolved name filter to WPT before test registration. */ +export function wptNameFilter(): Plugin { + return { + name: 'wpt-name-filter', + configureVitest({ vitest, project }) { + // Watch mode needs the full catalogue to discover cases when its filter changes. + const pattern = vitest.config.watch ? undefined : vitest.getGlobalTestNamePattern(); + project.provide( + 'wptTestNamePattern', + pattern ? { source: pattern.source, flags: pattern.flags } : null, + ); + }, + }; +} diff --git a/packages/web-platform-tests/src/vitest/runner.ts b/packages/web-platform-tests/src/vitest/runner.ts new file mode 100644 index 0000000..39b37dd --- /dev/null +++ b/packages/web-platform-tests/src/vitest/runner.ts @@ -0,0 +1,37 @@ +/* oxlint-disable bombshell-dev/exported-function-async -- Vitest test registration is synchronous. */ +import { describe, it, expect, inject } from 'vitest'; +import type { WPTNamePattern } from '../vitest.ts'; +import { + testDOMCompatibility as registerDOMCompatibility, + type DOMCompatibilityOptions, +} from '../shared/test-dom-compatibility.ts'; + +/** Register configured WPT cases with Vitest's assertions and name filter. */ +export function testDOMCompatibility(options: Omit, 'runner'>): void { + const selection: WPTNamePattern | null = inject('wptTestNamePattern'); + const pattern = selection ? new RegExp(selection.source, selection.flags) : undefined; + let matched = false; + + registerDOMCompatibility({ + ...options, + runner: { + describe, + it, + expect, + filter(source, name) { + if (pattern) { + pattern.lastIndex = 0; + } + const matches = !pattern || pattern.test(`${source} > ${name}`); + matched ||= matches; + return matches; + }, + }, + }); + + // An empty skipped suite lets Vitest report no matches without inventing a test + // or treating a filter for another test file as an empty-file failure. + if (pattern && !matched) { + describe.skip('No WPT cases match the test name filter', () => {}); + } +} diff --git a/packages/web-platform-tests/test/cli.test.ts b/packages/web-platform-tests/test/cli.test.ts new file mode 100644 index 0000000..7068d2d --- /dev/null +++ b/packages/web-platform-tests/test/cli.test.ts @@ -0,0 +1,178 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { afterEach, describe, expect, it } from 'vitest'; + +const executable = fileURLToPath(new URL('../bin/wpt.mjs', import.meta.url)); +const dependencies = fileURLToPath(new URL('../../../node_modules', import.meta.url)); +const implementation = fileURLToPath(new URL('../../ui/src/dom.ts', import.meta.url)); +const consumers: URL[] = []; + +interface CLIResult { + status: number | null; + output: string; +} + +interface Report { + numTotalTests: number; + numPassedTests: number; + numPendingTests: number; + numFailedTests: number; + testResults: { assertionResults: { fullName: string }[] }[]; +} + +afterEach(() => { + for (const directory of consumers.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function consumer(): URL { + const directory = pathToFileURL(`${mkdtempSync(`${tmpdir()}/wpt-cli-consumer-`)}/`); + consumers.push(directory); + symlinkSync(dependencies, new URL('node_modules', directory), 'dir'); + writeFileSync(new URL('package.json', directory), JSON.stringify({ type: 'module' })); + writeFileSync(new URL('consumer.txt', directory), 'consumer fixture'); + writeFileSync( + new URL('consumer.config.mjs', directory), + `import { defineConfig } from 'vitest/config'; +import { wptNameFilter } from '@bomb.sh/web-platform-tests/vitest'; +export default defineConfig({ + plugins: [wptNameFilter()], + test: { + setupFiles: ['./setup.mjs'], + reporters: ['json'], + outputFile: './results.json', + maxWorkers: 1, + experimental: { viteModuleRunner: false }, + }, +});`, + ); + writeFileSync( + new URL('setup.mjs', directory), + `import { readFileSync } from 'node:fs'; +if (readFileSync('./consumer.txt', 'utf8') !== 'consumer fixture') { + throw new Error('The consumer working directory was not preserved'); +}`, + ); + writeFileSync( + new URL('wpt.test.ts', directory), + `import { testDOMCompatibility } from '@bomb.sh/web-platform-tests/vitest/runner'; +import { Document, Text, Comment, CharacterData, Node } from ${JSON.stringify(implementation)}; +testDOMCompatibility({ + create: () => new Document(() => {}), + config: { + globals: { Text, Comment, CharacterData, Node }, + exclude: { '**': true }, + include: { + 'dom/nodes/Document-createTextNode.html': true, + 'dom/nodes/Document-createComment.html': true, + }, + }, +});`, + ); + // These are ordinary harness tests, not shared WPT conformance cases. + writeFileSync( + new URL('literals.test.ts', directory), + `import { it, expect } from 'vitest'; +it('007', () => expect(true).toBe(true)); +it('7', () => expect.fail('The leading zeroes were lost')); +it('--switch [value]+ (literal)', () => expect(true).toBe(true)); +it('intentional failure', () => expect('actual').toBe('expected'));`, + ); + return directory; +} + +function invoke(directory: URL, args: string[]): CLIResult { + rmSync(new URL('results.json', directory), { force: true }); + const result = spawnSync(process.execPath, [executable, ...args], { + cwd: directory, + encoding: 'utf8', + timeout: 20_000, + }); + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + return { status: result.status, output: result.stdout + result.stderr }; +} + +function run( + directory: URL, + { file, search = [] }: { file: string; search?: string[] }, +): CLIResult { + return invoke(directory, ['--file', file, '--config', 'consumer.config.mjs', ...search]); +} + +function report(directory: URL): Report { + return JSON.parse(readFileSync(new URL('results.json', directory), 'utf8')) as Report; +} + +describe('wpt CLI in a separate consumer project', () => { + it('uses consumer configuration and filters literal WPT names before registration', () => { + const directory = consumer(); + const result = run(directory, { + file: 'wpt.test.ts', + search: ['createTextNode(a', '--', '--', 'b)'], + }); + expect(result.status, result.output).toBe(0); + const results = report(directory); + expect(results.numTotalTests).toBe(1); + expect(results.numPassedTests).toBe(1); + expect(results.numPendingTests).toBe(0); + expect(results.testResults[0]?.assertionResults[0]?.fullName).toContain( + 'createTextNode(a -- b)', + ); + }, 30_000); + + it('succeeds when the literal filter matches no WPT cases', () => { + const directory = consumer(); + const result = run(directory, { + file: 'wpt.test.ts', + search: ['no such WPT case [missing]'], + }); + expect(result.status, result.output).toBe(0); + expect(report(directory).numTotalTests).toBe(0); + expect(report(directory).numFailedTests).toBe(0); + expect(result.output).not.toContain('No test suite found'); + }, 30_000); + + it('preserves leading zeroes and literal arguments after --', () => { + const directory = consumer(); + const numeric = run(directory, { file: 'literals.test.ts', search: ['007'] }); + expect(numeric.status, numeric.output).toBe(0); + expect(report(directory).numPassedTests).toBe(1); + const literal = run(directory, { + file: 'literals.test.ts', + search: ['--', '--switch', '[value]+', '(literal)'], + }); + expect(literal.status, literal.output).toBe(0); + expect(report(directory).numPassedTests).toBe(1); + }, 30_000); + + it('forwards failing test runs as a nonzero exit status', () => { + const directory = consumer(); + const result = run(directory, { file: 'literals.test.ts', search: ['intentional failure'] }); + expect(result.status, result.output).toBe(1); + expect(report(directory).numFailedTests).toBe(1); + }, 30_000); + + it('shows help and rejects invalid arguments or missing entrypoints', () => { + const directory = consumer(); + for (const args of [[], ['--help']]) { + const result = invoke(directory, args); + expect(result.status, result.output).toBe(0); + expect(result.output).toContain('wpt [OPTIONS]'); + } + for (const args of [ + ['createTextNode'], + ['--config', 'consumer.config.mjs'], + ['--file'], + ['--file', 'wpt.test.ts', '--unknown-option'], + ]) { + const result = invoke(directory, args); + expect(result.status, result.output).not.toBe(0); + } + const missing = run(directory, { file: 'missing.test.ts' }); + expect(missing.status, missing.output).not.toBe(0); + }, 30_000); +}); diff --git a/packages/web-platform-tests/test/compatibility-harness.test.ts b/packages/web-platform-tests/test/compatibility-harness.test.ts index aad9a04..ee36566 100644 --- a/packages/web-platform-tests/test/compatibility-harness.test.ts +++ b/packages/web-platform-tests/test/compatibility-harness.test.ts @@ -76,11 +76,14 @@ function recordingRunner(): { }; } -function register(config?: WPTConfig): ReturnType { +function register( + config?: WPTConfig, + filter?: CompatibilityRunner['filter'], +): ReturnType { const recording = recordingRunner(); testDOMCompatibility({ config, - runner: recording.runner, + runner: { ...recording.runner, filter }, create: () => expect.fail('Registration must not create a document'), }); @@ -129,7 +132,7 @@ describe('compatibility catalogue', () => { } const knownFiles = new Set(allFiles); const reachable = new Set(); - const pending = ['index.ts', ...entrypoints]; + const pending = ['index.ts', 'vitest.ts', 'vitest/runner.ts', 'cli.ts', ...entrypoints]; while (pending.length) { const file = pending.pop()!; if (reachable.has(file)) continue; @@ -329,6 +332,64 @@ describe('WPT include and exclude selection', () => { }); }); +describe('runner filtering', () => { + it('registers only matching cases and their suites without creating fixtures', () => { + const recording = register( + undefined, + (source, name) => source === onceSource && name === onceName, + ); + expect(recording.suites).toEqual([onceSource]); + expect(recording.tests.map(({ source, name }) => ({ source, name }))).toEqual([ + { source: onceSource, name: onceName }, + ]); + expect(recording.todos).toEqual([]); + }); + it('cannot enable configured exclusions and receives only configured cases', () => { + const filtered: { source: string; name: string }[] = []; + const recording = register({ exclude: { [onceSource]: [onceName] } }, (source, name) => { + filtered.push({ source, name }); + return source === onceSource; + }); + const expected = register(only(onceSource)) + .tests.filter((test) => test.name !== onceName) + .map(({ source, name }) => ({ source, name })); + expect(filtered).not.toContainEqual({ source: onceSource, name: onceName }); + expect(recording.tests.map(({ source, name }) => ({ source, name }))).toEqual(expected); + expect(recording.suites).toEqual([onceSource]); + expect(recording.todos).toEqual([]); + }); + it('registers no suites, tests, or todos when the runner filter matches nothing', () => { + const recording = register(only(onceSource), () => false); + expect(recording.suites).toEqual([]); + expect(recording.tests).toEqual([]); + expect(recording.todos).toEqual([]); + }); + it.each([ + { include: { 'missing/upstream.html': true as const } }, + { exclude: { [onceSource]: ['no such upstream case'] } }, + ])('validates configuration before applying a filter that rejects all cases: %j', (config) => { + const recording = recordingRunner(); + let filterCalls = 0; + expect(() => + testDOMCompatibility({ + config, + runner: { + ...recording.runner, + filter() { + filterCalls++; + return false; + }, + }, + create: () => expect.fail('Invalid selection must not create a document'), + }), + ).toThrow(CompatibilityConfigurationError); + expect(filterCalls).toBe(0); + expect(recording.suites).toEqual([]); + expect(recording.tests).toEqual([]); + expect(recording.todos).toEqual([]); + }); +}); + describe('registration and fixture lifetime', () => { it('uses global bindings supplied in the configuration', async () => { const recording = recordingRunner(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48ad6a8..9c44bd8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: '@bomb.sh/tools': specifier: ^0.8.0 version: 0.8.0(@types/node@22.20.0)(oxc-resolver@11.20.0)(unrun@0.2.39)(vite@8.3.5(@types/node@22.20.0)(jiti@2.7.0)(yaml@2.9.1)) + '@bomb.sh/web-platform-tests': + specifier: workspace:* + version: link:packages/web-platform-tests '@codspeed/vitest-plugin': specifier: ^6.1.0 version: 6.1.0(tinybench@6.2.2)(vite@8.3.5(@types/node@22.20.0)(jiti@2.7.0)(yaml@2.9.1))(vitest@5.0.5(@types/node@22.20.0)(vite@8.3.5(@types/node@22.20.0)(jiti@2.7.0)(yaml@2.9.1))) @@ -47,6 +50,13 @@ importers: version: link:../web-platform-tests packages/web-platform-tests: + dependencies: + '@bomb.sh/router': + specifier: ^0.8.0 + version: 0.8.0 + zod: + specifier: ^4.4.3 + version: 4.4.3 devDependencies: '@vitest/expect': specifier: 5.0.5 @@ -63,12 +73,19 @@ importers: vite: specifier: ^8.1.0 version: 8.3.5(@types/node@22.20.0)(jiti@2.7.0)(yaml@2.9.1) + vitest: + specifier: 5.0.5 + version: 5.0.5(@types/node@22.20.0)(vite@8.3.5(@types/node@22.20.0)(jiti@2.7.0)(yaml@2.9.1)) packages: '@bomb.sh/args@0.3.1': resolution: {integrity: sha512-CwxKrfgcorUPP6KfYD59aRdBYWBTsfsxT+GmoLVnKo5Tmyoqbpo0UNcjngRMyU+6tiPbd18RuIYxhgAn44wU/Q==} + '@bomb.sh/router@0.8.0': + resolution: {integrity: sha512-9Y0fA76+dlFL8aRf8PVfgXrI0qHr4GQrLLv09E75tGRqCtCI5+E/ZzLnyTGJMo/j4CzYt44c4jdXDLv+m9/g9g==} + engines: {node: '>= 20'} + '@bomb.sh/tools@0.8.0': resolution: {integrity: sha512-iaWws8S5kMuS6zImKXPSguUoQYA5M8XlK4ObVJI5jvubNeSlCI6feGPwex9GHyO+eIsi6izv/swO1ceO2MWEvw==} hasBin: true @@ -1826,6 +1843,10 @@ snapshots: '@bomb.sh/args@0.3.1': {} + '@bomb.sh/router@0.8.0': + dependencies: + '@standard-schema/spec': 1.1.0 + '@bomb.sh/tools@0.8.0(@types/node@22.20.0)(oxc-resolver@11.20.0)(unrun@0.2.39)(vite@8.3.5(@types/node@22.20.0)(jiti@2.7.0)(yaml@2.9.1))': dependencies: '@bomb.sh/args': 0.3.1 diff --git a/vitest.config.mts b/vitest.config.mts index f1b09f9..ad4059c 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -1,13 +1,18 @@ import codspeedPlugin from '@codspeed/vitest-plugin'; -import { defineConfig } from 'vitest/config'; +import base from '@bomb.sh/tools/test-utils/vitest.config.mjs'; +import { defineConfig, mergeConfig } from 'vitest/config'; +import { wptNameFilter } from '@bomb.sh/web-platform-tests/vitest'; -export default defineConfig({ - plugins: [codspeedPlugin()], - test: { - benchmark: { include: ['**/*.bench.ts'] }, - experimental: { viteModuleRunner: false }, - // ensure we running only one test at a time - maxWorkers: 1, - isolate: false, - }, -}); +export default mergeConfig( + base, + defineConfig({ + plugins: [codspeedPlugin(), wptNameFilter()], + test: { + benchmark: { include: ['**/*.bench.ts'] }, + experimental: { viteModuleRunner: false }, + // ensure we running only one test at a time + maxWorkers: 1, + isolate: false, + }, + }), +);