diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b4c35a21..4f051d47b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,7 +186,7 @@ jobs: - name: Check REPL bench tools working-directory: repl bench checkout run: | - deno test --allow-all --cached-only --frozen --node-modules-dir=none \ + deno test --reporter=dot --allow-all --cached-only --frozen --node-modules-dir=none \ --ignore=examples/repl-bench/tests/driver-exit.test.ts,examples/repl-bench/tests/teardown.test.ts \ examples/repl-bench/tests @@ -194,11 +194,12 @@ jobs: if: github.event_name == 'push' working-directory: repl bench checkout run: | - deno test --allow-all --cached-only --frozen --node-modules-dir=none \ + deno test --reporter=dot --allow-all --cached-only --frozen --node-modules-dir=none \ examples/repl-bench/tests/driver-exit.test.ts \ examples/repl-bench/tests/teardown.test.ts + echo "Running Markdown REPL Bench journeys" deno run --allow-all --cached-only --frozen --node-modules-dir=none \ - packages/cli/src/deno.ts test examples/repl-bench/journeys + packages/cli/src/deno.ts test examples/repl-bench/journeys --reporter=failures - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: github.event_name == 'push' && (failure() || cancelled()) @@ -284,19 +285,19 @@ jobs: ./dist/xmd test smoke-test/README.md \ --include smoke-test \ --include packages/core/components \ - --raw + --reporter=failures # Progress on stderr and a failed test's journal, from the binary rather # than the source entrypoint the runtime shards launch. - name: Test diagnostics from the compiled binary - run: deno test --allow-all --frozen scripts/tests/test-diagnostics-compiled.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/test-diagnostics-compiled.test.ts - name: Smoke test attached-service ping-pong with the compiled binary run: | ./dist/xmd test smoke-test/attached-service-ping-pong.test.md \ --include smoke-test \ --include packages/core/components \ - --raw + --reporter=failures # Foreground execution is about what a caller sees and what status they # get, and only the binary shows both surviving `deno compile`. @@ -314,14 +315,14 @@ jobs: # search path supplied the implementation. It lives here because its # subject is `dist/xmd`, and this is the job that builds one. - name: Form dispatch across the compiled binary's loaded copies - run: deno test --allow-all --frozen scripts/tests/component-form-dispatch.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/component-form-dispatch.test.ts # `` is packaged Markdown rather than a module, so it is exactly the # kind of thing `deno compile` ships only because an `--include` names it. # Asking the binary which Component source it carries is what catches a forgotten # include here rather than at a person's first `xmd plan`. - name: The Component the compiled binary carries - run: deno test --allow-all --frozen scripts/tests/plan-component-compiled.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/plan-component-compiled.test.ts # The compiled entrypoint states its own Prompt usage host facts, a # different assembly from the source CLI's, so its carriage is asserted @@ -336,13 +337,13 @@ jobs: # reopened location. This drives the same journey the source row drives, # on a real terminal, against the binary this job just built. - name: A REPL entry's files across the compiled binary - run: deno test --allow-all --frozen scripts/tests/repl-repositories-compiled.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/repl-repositories-compiled.test.ts - name: A REPL entry's issues across the compiled binary - run: deno test --allow-all --frozen scripts/tests/repl-issues-compiled.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/repl-issues-compiled.test.ts - name: A REPL entry's publications across the compiled binary - run: deno test --allow-all --frozen scripts/tests/repl-push-compiled.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/repl-push-compiled.test.ts # Each package's `components.md` is a packaged asset on the same footing, # and the long-form documentation is the only thing that reports whether @@ -374,7 +375,7 @@ jobs: # source at once — the built-in components under `components/`, and the # structural directives, which resolve no file at all. - name: Smoke test the built-ins with no search path - run: ./dist/xmd test packages/core/src --raw + run: ./dist/xmd test packages/core/src --reporter=failures # An inline root document exercises the compiled module graph the same way # a file does, and only the binary proves the graph survived `deno compile`. @@ -520,13 +521,13 @@ jobs: # Covers the compiled binary relaunching itself as `xmd test-agent`, # which the source-mode worker command never exercises. - name: Smoke test the compiled binary as a test agent - run: ./dist/xmd test smoke-test/test-agent/README.md --raw + run: ./dist/xmd test smoke-test/test-agent/README.md --reporter=failures - name: Smoke test xmd run through ACPX run: | ./dist/xmd test smoke-test/agent/README.md \ --include smoke-test/agent/components \ - --raw + --reporter=failures # A failed or cancelled run leaves the journals of the executions that # failed; a passing one uploads nothing. @@ -611,7 +612,7 @@ jobs: # each rewrite `node_modules` into their own layout, so a Deno step after # one of them resolves through links the other pruned (#279). - name: Host contract under Deno - run: deno test --allow-all --frozen packages/runtime/tests/host-files.test.ts + run: deno test --reporter=dot --allow-all --frozen packages/runtime/tests/host-files.test.ts - name: Host contract as a compiled binary run: | diff --git a/README.md b/README.md index d6d8f1ec3..de6a8e121 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,14 @@ documents. Name an affected suite's path instead, beside any TypeScript paths: deno task test examples/adversarial-implementation/tests/Implementation.test.md ``` +Call `xmd test` directly to choose the reporter for a Markdown test. + +```bash +deno task xmd test examples/adversarial-implementation/tests/Implementation.test.md --reporter=failures +``` + +[Choose a test reporter](specs/testing-spec.md#choose-a-reporter). + ### Complete Run the complete applicable battery concurrently, yielding delivery diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 37aae43ee..db69cbce5 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -430,6 +430,9 @@ const planConfig = object({ }, }); +const TEST_REPORTER_OPTION = "--reporter"; +const testReporter = z.enum(["pretty", "failures"]); + const testConfig = object({ path: { description: "markdown document or directory to test (defaults to the current directory)", @@ -439,6 +442,10 @@ const testConfig = object({ description: "glob for test documents, relative to a directory target (repeatable)", ...field(z.array(z.string()), field.default(["**/*.test.md"]), field.array()), }, + reporter: { + description: "test output: pretty (default) or failures", + ...field(testReporter, field.default("pretty")), + }, include: { description: "component search directory", ...field(z.array(z.string()), field.default(["components", "."]), field.array()), @@ -1776,9 +1783,7 @@ type TestPlugins = (document: string) => Operation; /** * `xmd test` — one document, or every document a directory holds. * - * A directory keeps going after a failure and decides the status once at the - * end. A single document behaves exactly as it always has: one reported - * failure, no heading, no summary. + * A directory runs every matching document and settles the exit status once. */ function* test( config: TestConfig, @@ -1786,7 +1791,15 @@ function* test( installService: HostServiceInstaller, testPlugins: TestPlugins, ): Operation { - const patterns = readPatternFlags(args); + const reporterFlags = readTestOption(args, TEST_REPORTER_OPTION); + const reporter = testReporter.safeParse(reporterFlags.values[0] ?? "pretty"); + if (reporterFlags.missingValue || reporterFlags.values.length > 1 || !reporter.success) { + console.error("--reporter accepts one value: pretty or failures"); + yield* exit(1); + return; + } + const failuresOnly = reporter.data === "failures"; + const patterns = readTestOption(args, PATTERN_OPTION); if (patterns.missingValue) { console.error( `${PATTERN_OPTION} requires a value — write \`${PATTERN_OPTION} \`, or ` + @@ -1814,14 +1827,27 @@ function* test( return; } announceSecretDetection(config.secretDetection); - const { value: result, delivery } = yield* withTestProgress((diagnostics) => - runTestDocument({ ...config, root: { path } }, testPlugins, installService, diagnostics), + const { + value: result, + delivery, + summary, + } = yield* withTestProgress(failuresOnly, (diagnostics) => + runTestDocument( + { ...config, root: { path }, discardOutput: failuresOnly }, + testPlugins, + installService, + diagnostics, + ), ); if (!result.ok) { reportFailure(result.error); - yield* exit(1); } else if (!delivery.ok) { reportFailure(delivery.error); + } + if (failuresOnly) { + console.error(summary); + } + if (!result.ok || !delivery.ok) { yield* exit(1); } return; @@ -1847,15 +1873,22 @@ function* test( // and a directory of fifty documents would otherwise say so fifty times. announceSecretDetection(config.secretDetection); - const { value: failures, delivery } = yield* withTestProgress(function* (diagnostics) { + const { + value: failures, + delivery, + summary, + } = yield* withTestProgress(failuresOnly, function* (diagnostics) { const failed: string[] = []; for (const document of target.documents) { - process.stdout.write(`\n# ${document.relativePath}\n\n`); + if (!failuresOnly) { + process.stdout.write(`\n# ${document.relativePath}\n\n`); + } const result = yield* runTestDocument( { ...config, root: { path: document.path }, include: componentSearchPath(document, target.root, config.include), + discardOutput: failuresOnly, }, testPlugins, installService, @@ -1869,11 +1902,19 @@ function* test( return failed; }); - if (failures.length > 0) { - console.error(`\n${failures.length} of ${target.documents.length} documents failed`); - yield* exit(1); - } else if (!delivery.ok) { + const failedDocuments = + failures.length > 0 + ? `${failures.length} of ${target.documents.length} documents failed` + : undefined; + if (failedDocuments === undefined && !delivery.ok) { reportFailure(delivery.error); + } + if (failuresOnly) { + console.error(failedDocuments === undefined ? summary : `${summary} · ${failedDocuments}`); + } else if (failedDocuments !== undefined) { + console.error(`\n${failedDocuments}`); + } + if (failures.length > 0 || !delivery.ok) { yield* exit(1); } } @@ -1886,15 +1927,22 @@ function* test( * returns, so whatever the command reports next follows them. */ function* withTestProgress( + failuresOnly: boolean, body: (diagnostics: TestDiagnostics) => Operation, -): Operation<{ value: T; delivery: Result }> { - let delivery: () => Result = () => Ok(undefined); - const value = yield* scoped(function* () { - const diagnostics = yield* useTestDiagnostics(process.stderr); - delivery = diagnostics.delivery; - return yield* body(diagnostics); +): Operation<{ value: T; delivery: Result; summary: string }> { + const started = performance.now(); + const run = yield* scoped(function* () { + const diagnostics = yield* useTestDiagnostics(process.stderr, failuresOnly); + const value = yield* body(diagnostics); + return { value, diagnostics }; }); - return { value, delivery: delivery() }; + const { pass, fail } = run.diagnostics.results(); + const elapsed = Math.round(performance.now() - started); + return { + value: run.value, + delivery: run.diagnostics.delivery(), + summary: `[xmd test] ${pass} passed | ${fail} failed (${elapsed}ms)`, + }; } /** @@ -2002,15 +2050,13 @@ function findPropsFlag(args: string[]): string | undefined { const PATTERN_OPTION = "--pattern"; -interface PatternFlags { - /** Values the caller wrote, in the order they wrote them. */ +interface TestOption { values: string[]; - /** A `--pattern` that ran out of argv or was followed by another option. */ missingValue: boolean; } /** - * Read `--pattern` from argv. + * Read an explicitly supplied test option from argv. * * The resolved configuration answers none of the questions this serves. It * cannot say whether the option was given at all — the default is a real @@ -2021,7 +2067,7 @@ interface PatternFlags { * A separated value that begins with `-` is another option, not a glob; * `--pattern=` expresses a glob that really does begin with one. */ -function readPatternFlags(args: string[]): PatternFlags { +function readTestOption(args: string[], option: string): TestOption { const values: string[] = []; let missingValue = false; @@ -2029,7 +2075,7 @@ function readPatternFlags(args: string[]): PatternFlags { if (arg === "--") { break; } - if (arg === PATTERN_OPTION) { + if (arg === option) { const value = args[index + 1]; if (value === undefined || value.startsWith("-")) { missingValue = true; @@ -2038,8 +2084,8 @@ function readPatternFlags(args: string[]): PatternFlags { values.push(value); continue; } - if (arg.startsWith(`${PATTERN_OPTION}=`)) { - values.push(arg.slice(PATTERN_OPTION.length + 1)); + if (arg.startsWith(`${option}=`)) { + values.push(arg.slice(option.length + 1)); } } diff --git a/packages/cli/src/test-diagnostics.ts b/packages/cli/src/test-diagnostics.ts index d02b71af3..fdc371422 100644 --- a/packages/cli/src/test-diagnostics.ts +++ b/packages/cli/src/test-diagnostics.ts @@ -56,6 +56,8 @@ export interface TestDiagnostics { execution(kind: "root" | "child", source: string): ExecutionDiagnostics; /** The first failure to deliver a progress line, or `Ok` when none failed. */ delivery(): Result; + /** Counts recorded by root documents. Nested execution results are evidence. */ + results(): Readonly<{ pass: number; fail: number }>; } /** What one execution is observed and preserved through. */ @@ -95,7 +97,10 @@ function* artifactDirectory(working: string): Operation { * diagnostics drains what was accepted before it returns — on success, failure * and cancellation alike — and nothing is written after that. */ -export function useTestDiagnostics(sink: DeliverySink): Operation { +export function useTestDiagnostics( + sink: DeliverySink, + failuresOnly = false, +): Operation { return resource(function* (provide) { const working = yield* cwd(); const directory = yield* artifactDirectory(working); @@ -125,6 +130,7 @@ export function useTestDiagnostics(sink: DeliverySink): Operation observe(owner, shown, (line) => emit(`${context}${line}`)), + observe: () => + observe( + owner, + shown, + (line, failure = false) => { + if (!failuresOnly || failure) { + emit(`${context}${line}`); + } + }, + (status) => { + if (kind === "root") { + results[status] += 1; + } + }, + ), journal: (failed: (value: T) => boolean): FailureJournalHost => ({ failed, *save(events: readonly DurableEvent[]) { @@ -149,6 +169,7 @@ export function useTestDiagnostics(sink: DeliverySink): Operation delivery, + results: () => results, }); }); } @@ -206,7 +227,8 @@ interface ObservedTest { function* observe( owner: Scope, shown: (path: string) => string, - report: (line: string) => void, + report: (line: string, failure?: boolean) => void, + record: (status: TestResult["status"]) => void, ): Operation { const tests = new Map(); let current: ObservedTest | undefined; @@ -228,12 +250,12 @@ function* observe( } if (phase.phase === "complete") { if (active) { - report(`${phase.result.ok ? "done" : "failed"} ${what}`); + report(`${phase.result.ok ? "done" : "failed"} ${what}`, !phase.result.ok); } } else if (phase.phase === "cancelled") { interrupted(test, what); if (active) { - report(`cancelled ${what}`); + report(`cancelled ${what}`, true); } } yield* each.next(); @@ -280,7 +302,7 @@ function* observe( result.error?.kind ?? "error", ...(test?.interrupted === undefined ? [] : [`interrupted ${test.interrupted}`]), ]; - report(`fail ${name} at ${where} (${why.join("; ")})`); + report(`fail ${name} at ${where} (${why.join("; ")})`, true); }; yield* Component.around({ @@ -334,7 +356,7 @@ function* observe( if (!settled && !test.recorded) { const detail = test.interrupted === undefined ? "" : ` (interrupted ${test.interrupted})`; - report(`cancelled ${named} at ${test.shown}${detail}`); + report(`cancelled ${named} at ${test.shown}${detail}`, true); } } } @@ -344,6 +366,7 @@ function* observe( yield* Test.around({ *record([result], next) { finished(result); + record(result.status); yield* next(result); }, }); diff --git a/packages/cli/tests/test-diagnostics.test.ts b/packages/cli/tests/test-diagnostics.test.ts index e8675715a..ad5bb151a 100644 --- a/packages/cli/tests/test-diagnostics.test.ts +++ b/packages/cli/tests/test-diagnostics.test.ts @@ -550,3 +550,66 @@ describe("xmd test live progress", () => { expect(run.stderr()).toBe(delivered); }); }); + +describe("xmd test reporters", { sanitizeOps: false, sanitizeResources: false }, () => { + it("reports failures and totals across documents without printing passing tests", function* () { + const directory = yield* useProject({ + "a.test.md": doc( + "Routine suite output", + '', + " ", + "", + '', + " ", + "", + ), + "b.test.md": doc( + '', + " ", + "", + ), + }); + const artifacts = join(directory, "artifacts"); + + const result = yield* runTest(directory, [".", "--reporter=failures"], artifacts); + expect(result.code).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain('fail test "failing witness" at a.test.md:5:1 (assertion)'); + expect(result.stderr).toContain("a.test.md: tests failed:"); + expect(result.stderr).not.toContain("passing witness"); + expect(result.stderr).not.toContain("later witness"); + expect(progress(result.stderr).at(-1)).toMatch( + /^2 passed \| 1 failed \(\d+ms\) · 1 of 2 documents failed$/, + ); + expect(fileNames(yield* journals(artifacts))).toEqual(["root-1.jsonl"]); + }); + + it("reports a passing single document and rejects unsupported reporters", function* () { + const directory = yield* useProject({ + "suite.test.md": doc( + '', + " ", + "", + ), + }); + const artifacts = join(directory, "artifacts"); + + const result = yield* runTest( + directory, + ["--reporter", "failures", "suite.test.md"], + artifacts, + ); + expect(result.code).toBe(0); + expect(result.stdout).toBe(""); + expect(progress(result.stderr)).toEqual([ + expect.stringMatching(/^1 passed \| 0 failed \(\d+ms\)$/), + ]); + expect(yield* exists(artifacts)).toBe(false); + + const invalid = yield* runTest(directory, ["suite.test.md", "--reporter=unknown"], artifacts); + expect(invalid.code).toBe(1); + expect(invalid.stdout).toBe(""); + expect(invalid.stderr).toContain("--reporter"); + expect(progress(invalid.stderr)).toEqual([]); + }); +}); diff --git a/scripts/lib/measure-weights.ts b/scripts/lib/measure-weights.ts index 1bcff6257..4f1da416f 100644 --- a/scripts/lib/measure-weights.ts +++ b/scripts/lib/measure-weights.ts @@ -72,7 +72,7 @@ export function* measureRuntime( for (const file of applicable) { const started = performance.now(); - const code = yield* runOneFile(runtime, file, root); + const code = yield* runOneFile(runtime, file, root, { compactReport: true }); // The schema admits positive integers only, and a file that finishes in // under half a millisecond would otherwise round to a weight of zero. const ms = Math.max(1, Math.round(performance.now() - started)); diff --git a/scripts/lib/runtime-tests.ts b/scripts/lib/runtime-tests.ts index 0ba358bca..0c5fe50d3 100644 --- a/scripts/lib/runtime-tests.ts +++ b/scripts/lib/runtime-tests.ts @@ -18,7 +18,7 @@ import { exit, scoped, until } from "effection"; import type { Context, Operation } from "effection"; import { createContext } from "effection"; -import { exec } from "@effectionx/process"; +import { exec, Stdio } from "@effectionx/process"; import { exists, readTextFile, writeTextFile } from "@effectionx/fs"; import { timebox } from "@effectionx/timebox"; import { launchEnvironment, shellQuote } from "@executablemd/test-support/launch"; @@ -131,16 +131,23 @@ function retainedEvidence(suite: string): Record { * `xmd test ` under `runtime`. The adversarial suites also receive * their fixture plugin and absolute component directories. */ -function markdownCommand(runtime: Runtime, file: string, root: URL): Launch { +function markdownCommand( + runtime: Runtime, + file: string, + root: URL, + options: TestLaunchOptions, +): Launch { const [command, ...entry] = cliEntry(runtime); + const args = [...entry, "test", file]; + if (options.compactReport) { + args.push("--reporter=failures"); + } const document = DOCUMENT_SUITES.get(file); if (document !== undefined) { return { command, arguments: [ - ...entry, - "test", - file, + ...args, ...(runtime === "deno" && document.worker ? ["--plugin", checkoutPath("packages/cli/tests/fixtures/prepared-test-worker.mjs")] : []), @@ -155,9 +162,7 @@ function markdownCommand(runtime: Runtime, file: string, root: URL): Launch { return { command, arguments: [ - ...entry, - "test", - file, + ...args, "--plugin", checkoutPath("packages/cli/tests/fixtures/workflow-test-permissions.mjs"), "--include", @@ -192,21 +197,35 @@ export const RUNNERS: Record = { bun: { command: "bun", prefix: ["test", `--timeout=${BUN_TEST_TIMEOUT_MS}`], serial: [] }, }; +interface TestLaunchOptions { + compactReport?: boolean; +} + /** - * The exact command that runs one test file alone under `runtime`. + * Build the command that runs one test file alone under `runtime`. * * Weight measurement times this command, and a shard invokes it once per * assigned file — so the millisecond a weight records is the millisecond that * shard will spend, rather than a fraction of some larger invocation. */ -export function oneFileCommand(runtime: Runtime, file: string, root: URL = CHECKOUT): Launch { +export function oneFileCommand( + runtime: Runtime, + file: string, + root: URL = CHECKOUT, + options: TestLaunchOptions = {}, +): Launch { if (isMarkdownSuite(file)) { - return markdownCommand(runtime, file, root); + return markdownCommand(runtime, file, root, options); } const runner = RUNNERS[runtime]; return { command: runner.command, - arguments: [...runner.prefix, ...runner.serial, file], + arguments: [ + ...runner.prefix, + ...runner.serial, + ...(runtime === "deno" && options.compactReport ? ["--reporter=dot"] : []), + file, + ], }; } @@ -268,9 +287,14 @@ function* preparedLink(root: URL): Operation { * the prepared dependency link as it found it. A changed or missing link * fails the file. An earlier failure retains its own status. */ -export function* runOneFile(runtime: Runtime, file: string, root: URL): Operation { +export function* runOneFile( + runtime: Runtime, + file: string, + root: URL, + options: TestLaunchOptions = {}, +): Operation { const run = yield* TestRuns.expect(); - const launch = { ...oneFileCommand(runtime, file, root), cwd: fileURLToPath(root) }; + const launch = { ...oneFileCommand(runtime, file, root, options), cwd: fileURLToPath(root) }; if (!isMarkdownSuite(file)) { return yield* run(launch); } @@ -513,10 +537,14 @@ export function* runShardFiles( ): Operation { const ran: string[] = []; let failure: { file: string; status: number } | undefined; + const encoder = new TextEncoder(); for (const file of files) { ran.push(file); - const status = yield* runOneFile(runtime, file, root); + if (isMarkdownSuite(file)) { + yield* Stdio.operations.stderr(encoder.encode(`Running ${file}\n`)); + } + const status = yield* runOneFile(runtime, file, root, { compactReport: true }); if (status !== 0 && failure === undefined) { failure = { file, status }; } diff --git a/scripts/tests/filesystem-contract-workflow.test.ts b/scripts/tests/filesystem-contract-workflow.test.ts index 6a87affd9..7d17d06a7 100644 --- a/scripts/tests/filesystem-contract-workflow.test.ts +++ b/scripts/tests/filesystem-contract-workflow.test.ts @@ -121,7 +121,7 @@ describe("the filesystem contract matrix", () => { it("runs the contract under Deno, Node, Bun, and a compiled binary", function* () { const commands = (yield* job()).commands.join("\n"); - expect(commands).toContain(`deno test --allow-all --frozen ${SUITE}`); + expect(commands).toContain(`deno test --reporter=dot --allow-all --frozen ${SUITE}`); expect(commands).toContain(`tsx --tsconfig tsconfig.node.json --test ${SUITE}`); expect(commands).toContain(`bun test --timeout=300000 ${SUITE}`); expect(commands).toContain(`deno compile`); diff --git a/scripts/tests/runtime-tests.test.ts b/scripts/tests/runtime-tests.test.ts index 423445b3d..665b40c4f 100644 --- a/scripts/tests/runtime-tests.test.ts +++ b/scripts/tests/runtime-tests.test.ts @@ -125,7 +125,7 @@ describe("running a shard", () => { expect(runs.peak).toEqual(1); expect(runs.launches).toEqual( [ONE, TWO, THREE].map((file) => { - const launch = oneFileCommand("deno", file); + const launch = oneFileCommand("deno", file, ROOT, { compactReport: true }); return `${launch.command} ${launch.arguments.join(" ")}`; }), ); @@ -136,12 +136,13 @@ describe("running a shard", () => { const runs = recorder(); yield* TestRuns.with(runs.run, () => runShardFiles(runtime, [ONE], ROOT)); - const launch = oneFileCommand(runtime, ONE); + const launch = oneFileCommand(runtime, ONE, ROOT, { compactReport: true }); + expect(launch.arguments.includes("--reporter=dot")).toBe(runtime === "deno"); expect(runs.launches).toEqual([`${launch.command} ${launch.arguments.join(" ")}`]); } }); - it("routes a Markdown suite through its one-file command, from the repository", function* () { + it("uses the failures reporter for Markdown in each runtime shard", function* () { for (const runtime of RUNTIMES) { const seen: Array<{ line: string; cwd: string; env?: Record }> = []; yield* TestRuns.with( @@ -153,9 +154,10 @@ describe("running a shard", () => { ); expect(seen.map((entry) => entry.line)).toEqual([ - line(oneFileCommand(runtime, ONE)), - line(oneFileCommand(runtime, SUITE, ROOT)), + line(oneFileCommand(runtime, ONE, ROOT, { compactReport: true })), + line(oneFileCommand(runtime, SUITE, ROOT, { compactReport: true })), ]); + expect(seen[1].line).toContain(`test ${SUITE} --reporter=failures`); expect(seen.map((entry) => entry.cwd)).toEqual([fileURLToPath(ROOT), fileURLToPath(ROOT)]); expect(seen[1].env).toEqual(oneFileCommand(runtime, SUITE, ROOT).env); } diff --git a/scripts/tests/shard-execution.test.ts b/scripts/tests/shard-execution.test.ts index de2cdc715..6475886b1 100644 --- a/scripts/tests/shard-execution.test.ts +++ b/scripts/tests/shard-execution.test.ts @@ -69,7 +69,7 @@ function* workspace(files: Record): Operation { } function passing(marker: string): string { - return `Deno.test("${marker}", () => {\n console.log("${marker}_RAN");\n});\n`; + return `Deno.test("${marker}", () => {\n Deno.writeTextFileSync("${marker}.marker", "");\n console.log("${marker}_RAN");\n});\n`; } /** Multiline on both channels, so truncation or a summary would be visible. */ @@ -111,7 +111,7 @@ function* waitFor(file: URL): Operation { } describe("a shard running real children", () => { - it("runs the files after a failing one and keeps its complete output", function* () { + it("runs later files and keeps the failure diagnostic with Deno's dot reporter", function* () { const root = yield* workspace({ "a.test.ts": passing("A"), "b.test.ts": FAILING, @@ -127,19 +127,13 @@ describe("a shard running real children", () => { expect(outcome.failure?.status).toBeGreaterThan(0); const displayed = `${seen.stdout}\n${seen.stderr}`; - // The file after the failure ran, which is the whole point of continuing. - expect(displayed).toContain("A_RAN"); - expect(displayed).toContain("C_RAN"); - // And the failure's own text survived, every line of it, on both channels. - for (const line of [ - "B_STDOUT_ONE", - "B_STDOUT_TWO", - "B_STDERR_ONE", - "B_STDERR_TWO", - "B_FAILED_DELIBERATELY", - ]) { - expect(displayed).toContain(line); - } + expect(yield* exists(new URL("A.marker", root))).toBe(true); + expect(yield* exists(new URL("C.marker", root))).toBe(true); + expect(displayed).toContain("B_FAILED_DELIBERATELY"); + expect(displayed).not.toContain("A_RAN"); + expect(displayed).not.toContain("C_RAN"); + expect(displayed).not.toContain("B_STDOUT_ONE"); + expect(displayed).not.toContain("B_STDERR_ONE"); }); it("succeeds and stays quiet about failures when every file passes", function* () { @@ -167,7 +161,7 @@ describe("a shard running real children", () => { const marker = new URL("slow.marker", root); if (!(yield* waitFor(marker))) { - const launch = oneFileCommand("deno", "tests/a-slow.test.ts"); + const launch = oneFileCommand("deno", "tests/a-slow.test.ts", root, { compactReport: true }); throw new Error( `child did not write expected startup marker ${fileURLToPath(marker)}\ncommand: ${launch.command} ${launch.arguments.join(" ")}\noutcome: ${JSON.stringify(outcome)}\nstdout:\n${seen.stdout}\nstderr:\n${seen.stderr}`, ); @@ -257,7 +251,7 @@ function* collected(directory: URL): Operation { } describe("a shard running real Markdown suites", () => { - it("shows a suite's named start live, saves its journal, runs the files after it and keeps its failure first", function* () { + it("shows the active suite live, saves its journal and keeps its failure first", function* () { const { root, journals } = yield* suites({ "Held.test.md": HELD_FAILURE, "Later.test.md": MD_LATER, @@ -272,7 +266,7 @@ describe("a shard running real Markdown suites", () => { outcome = yield* runShardFiles("deno", assigned, root); }); - const start = `[xmd test] start test "fails deliberately" at ${held}`; + const start = `Running ${held}\n`; let visible = false; for (let attempt = 0; attempt < 300 && !visible; attempt += 1) { visible = seen.stderr.includes(start); @@ -289,6 +283,8 @@ describe("a shard running real Markdown suites", () => { expect(outcome?.failure).toEqual({ file: held, status: 1 }); expect(yield* exists(new URL("later.marker", root))).toBe(true); expect(seen.stderr).toContain("MD_FAILED_DELIBERATELY"); + expect(seen.stderr).toContain("0 passed | 1 failed"); + expect(seen.stderr).not.toContain('[xmd test] start test "fails deliberately"'); expect(`${seen.stdout}\n${seen.stderr}`).toContain("B_FAILED_DELIBERATELY"); const saved = yield* collected(journals); diff --git a/scripts/tests/test-weights.test.ts b/scripts/tests/test-weights.test.ts index d9d6b3e90..825a78b9f 100644 --- a/scripts/tests/test-weights.test.ts +++ b/scripts/tests/test-weights.test.ts @@ -361,7 +361,7 @@ describe("measuring one runtime", () => { expect(runs.launches).toEqual( [ALPHA, BETA].map((file) => { - const launch = oneFileCommand(runtime, file); + const launch = oneFileCommand(runtime, file, root, { compactReport: true }); return `${launch.command} ${launch.arguments.join(" ")}`; }), ); @@ -392,7 +392,9 @@ describe("measuring one runtime", () => { () => measureRuntime(runtime, root, SOURCE, measurementFile(root, runtime)), ); - const [alpha, suite] = [ALPHA, SUITE].map((file) => oneFileCommand(runtime, file, root)); + const alpha = oneFileCommand(runtime, ALPHA, root, { compactReport: true }); + const suite = oneFileCommand(runtime, SUITE, root, { compactReport: true }); + expect(suite.arguments).toContain("--reporter=failures"); expect(seen).toEqual([ { line: `${suite.command} ${suite.arguments.join(" ")}`, diff --git a/specs/testing-spec.md b/specs/testing-spec.md index bffc6eb6b..e6ab68cc7 100644 --- a/specs/testing-spec.md +++ b/specs/testing-spec.md @@ -81,10 +81,34 @@ recorded results are preserved. On live and partial runs nothing is restored: re-expansion records each result exactly once, in discovery order, with completed records replaying in place. +### Choose a reporter + +The default `--reporter=pretty` prints the rendered report on stdout and +progress on stderr. Use `--reporter=failures` for a compact report in CI or a +Sidekick Round. + +```sh +xmd test ./tests --reporter=failures +``` + +The failures reporter omits the rendered report, directory headings, test +starts, and successful outcomes. It writes failure and cancellation +diagnostics and journal paths to stderr. A completed invocation ends with one +summary on stderr. + +```text +[xmd test] 11 passed | 1 failed (324ms) · 1 of 3 documents failed +``` + +The counts cover tests recorded by the root documents. Nested executions can +add diagnostics and journals. Exit status and journal retention follow the same +rules for either reporter. Output that commands write directly to the terminal +keeps its normal routing. + ### Progress and failure journals -While `xmd test` runs, it writes a short progress report to stderr, so a pipe or -CI log shows which test is running and which operation a timeout interrupted: +The pretty reporter writes live progress to stderr. A pipe or CI log shows +which test is running and which operation a timeout interrupted. ```text [xmd test] start test "held" at suite.test.md:1:1 @@ -240,8 +264,8 @@ registrations, and testing sessions belong to one document and are torn down before the next one starts. A document that declares no tests fails on its own rather than inheriting an earlier document's results. -Each document's report is preceded by a heading naming its relative path, and a -failure is reported against that path. A failing document does not stop the +The pretty reporter precedes each document's report with a heading naming its +relative path. Both reporters identify a failed document by that path. A failing document does not stop the run; the remaining documents still execute, and a closing line names how many of them failed. The run exits `0` only when every discovered document passes, and `1` when any document fails or when no document matches. @@ -930,7 +954,7 @@ the first failure of either portion. With a selection it resolves that shard and invokes its files serially, in sorted order, each through that runtime's one-file command: -- Deno — `deno test --allow-all --frozen ` +- Deno — `deno test --allow-all --frozen --reporter=dot ` - Node — `tsx --tsconfig tsconfig.node.json --test --test-concurrency=1 ` - Bun — `bun test --timeout=300000 ` @@ -941,8 +965,11 @@ source entrypoint, resolved from the runner's checkout: - Node — `tsx --tsconfig tsconfig.node.json packages/cli/src/node.ts` - Bun — `bun packages/cli/src/bun.ts` -The CLI receives `test ` and runs from the repository root with its inherited -environment (`PATH`, the Deno cache and install variables, `XDG_CACHE_HOME`, +The CLI runs from the repository root. A shard names each Markdown suite when +it starts and passes `test --reporter=failures`. Weight measurement +passes the same arguments. Local unsharded runs use the default reporter. +Deno weight measurement also uses the same dot reporter as Deno shards. +The CLI inherits the environment (`PATH`, the Deno cache and install variables, `XDG_CACHE_HOME`, `TMPDIR` and `XMD_TEST_ARTIFACTS`). Individually registered suites use the CLI's default includes and file permissions. Deno launches of `Plan`, `NativeSessionLaunch` and `SessionConfiguration` select the test-only worker @@ -988,8 +1015,9 @@ A malformed runtime, index, or count is refused before any test process starts. A numeric failure does not prevent the files assigned after it: the shard continues, retains the first failed file and its status, and exits with that status once every assigned file has had its chance. A status without a numeric -code — a signalled child — is failure. Native output is neither captured, -truncated, nor replaced by a summary. +code — a signalled child — is failure. The shard streams the output its +selected reporter emits. Deno's dot reporter keeps failure details and omits +test console logs. Cancellation is distinct from failure. Halting a shard halts the process group of the file it is running, releases that file's scratch state and starts no