From 14225acc3495dd4aaca86669a26e38b4ba3a4057 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Mon, 16 Nov 2026 18:16:35 -0500 Subject: [PATCH 1/4] Add failures reporter to xmd test --- packages/cli/src/cli.ts | 102 ++++++++++++++------ packages/cli/src/test-diagnostics.ts | 37 +++++-- packages/cli/tests/test-diagnostics.test.ts | 63 ++++++++++++ specs/testing-spec.md | 14 ++- 4 files changed, 179 insertions(+), 37 deletions(-) diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 37aae43ee..db69cbce5 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -430,6 +430,9 @@ const planConfig = object({ }, }); +const TEST_REPORTER_OPTION = "--reporter"; +const testReporter = z.enum(["pretty", "failures"]); + const testConfig = object({ path: { description: "markdown document or directory to test (defaults to the current directory)", @@ -439,6 +442,10 @@ const testConfig = object({ description: "glob for test documents, relative to a directory target (repeatable)", ...field(z.array(z.string()), field.default(["**/*.test.md"]), field.array()), }, + reporter: { + description: "test output: pretty (default) or failures", + ...field(testReporter, field.default("pretty")), + }, include: { description: "component search directory", ...field(z.array(z.string()), field.default(["components", "."]), field.array()), @@ -1776,9 +1783,7 @@ type TestPlugins = (document: string) => Operation; /** * `xmd test` — one document, or every document a directory holds. * - * A directory keeps going after a failure and decides the status once at the - * end. A single document behaves exactly as it always has: one reported - * failure, no heading, no summary. + * A directory runs every matching document and settles the exit status once. */ function* test( config: TestConfig, @@ -1786,7 +1791,15 @@ function* test( installService: HostServiceInstaller, testPlugins: TestPlugins, ): Operation { - const patterns = readPatternFlags(args); + const reporterFlags = readTestOption(args, TEST_REPORTER_OPTION); + const reporter = testReporter.safeParse(reporterFlags.values[0] ?? "pretty"); + if (reporterFlags.missingValue || reporterFlags.values.length > 1 || !reporter.success) { + console.error("--reporter accepts one value: pretty or failures"); + yield* exit(1); + return; + } + const failuresOnly = reporter.data === "failures"; + const patterns = readTestOption(args, PATTERN_OPTION); if (patterns.missingValue) { console.error( `${PATTERN_OPTION} requires a value — write \`${PATTERN_OPTION} \`, or ` + @@ -1814,14 +1827,27 @@ function* test( return; } announceSecretDetection(config.secretDetection); - const { value: result, delivery } = yield* withTestProgress((diagnostics) => - runTestDocument({ ...config, root: { path } }, testPlugins, installService, diagnostics), + const { + value: result, + delivery, + summary, + } = yield* withTestProgress(failuresOnly, (diagnostics) => + runTestDocument( + { ...config, root: { path }, discardOutput: failuresOnly }, + testPlugins, + installService, + diagnostics, + ), ); if (!result.ok) { reportFailure(result.error); - yield* exit(1); } else if (!delivery.ok) { reportFailure(delivery.error); + } + if (failuresOnly) { + console.error(summary); + } + if (!result.ok || !delivery.ok) { yield* exit(1); } return; @@ -1847,15 +1873,22 @@ function* test( // and a directory of fifty documents would otherwise say so fifty times. announceSecretDetection(config.secretDetection); - const { value: failures, delivery } = yield* withTestProgress(function* (diagnostics) { + const { + value: failures, + delivery, + summary, + } = yield* withTestProgress(failuresOnly, function* (diagnostics) { const failed: string[] = []; for (const document of target.documents) { - process.stdout.write(`\n# ${document.relativePath}\n\n`); + if (!failuresOnly) { + process.stdout.write(`\n# ${document.relativePath}\n\n`); + } const result = yield* runTestDocument( { ...config, root: { path: document.path }, include: componentSearchPath(document, target.root, config.include), + discardOutput: failuresOnly, }, testPlugins, installService, @@ -1869,11 +1902,19 @@ function* test( return failed; }); - if (failures.length > 0) { - console.error(`\n${failures.length} of ${target.documents.length} documents failed`); - yield* exit(1); - } else if (!delivery.ok) { + const failedDocuments = + failures.length > 0 + ? `${failures.length} of ${target.documents.length} documents failed` + : undefined; + if (failedDocuments === undefined && !delivery.ok) { reportFailure(delivery.error); + } + if (failuresOnly) { + console.error(failedDocuments === undefined ? summary : `${summary} · ${failedDocuments}`); + } else if (failedDocuments !== undefined) { + console.error(`\n${failedDocuments}`); + } + if (failures.length > 0 || !delivery.ok) { yield* exit(1); } } @@ -1886,15 +1927,22 @@ function* test( * returns, so whatever the command reports next follows them. */ function* withTestProgress( + failuresOnly: boolean, body: (diagnostics: TestDiagnostics) => Operation, -): Operation<{ value: T; delivery: Result }> { - let delivery: () => Result = () => Ok(undefined); - const value = yield* scoped(function* () { - const diagnostics = yield* useTestDiagnostics(process.stderr); - delivery = diagnostics.delivery; - return yield* body(diagnostics); +): Operation<{ value: T; delivery: Result; summary: string }> { + const started = performance.now(); + const run = yield* scoped(function* () { + const diagnostics = yield* useTestDiagnostics(process.stderr, failuresOnly); + const value = yield* body(diagnostics); + return { value, diagnostics }; }); - return { value, delivery: delivery() }; + const { pass, fail } = run.diagnostics.results(); + const elapsed = Math.round(performance.now() - started); + return { + value: run.value, + delivery: run.diagnostics.delivery(), + summary: `[xmd test] ${pass} passed | ${fail} failed (${elapsed}ms)`, + }; } /** @@ -2002,15 +2050,13 @@ function findPropsFlag(args: string[]): string | undefined { const PATTERN_OPTION = "--pattern"; -interface PatternFlags { - /** Values the caller wrote, in the order they wrote them. */ +interface TestOption { values: string[]; - /** A `--pattern` that ran out of argv or was followed by another option. */ missingValue: boolean; } /** - * Read `--pattern` from argv. + * Read an explicitly supplied test option from argv. * * The resolved configuration answers none of the questions this serves. It * cannot say whether the option was given at all — the default is a real @@ -2021,7 +2067,7 @@ interface PatternFlags { * A separated value that begins with `-` is another option, not a glob; * `--pattern=` expresses a glob that really does begin with one. */ -function readPatternFlags(args: string[]): PatternFlags { +function readTestOption(args: string[], option: string): TestOption { const values: string[] = []; let missingValue = false; @@ -2029,7 +2075,7 @@ function readPatternFlags(args: string[]): PatternFlags { if (arg === "--") { break; } - if (arg === PATTERN_OPTION) { + if (arg === option) { const value = args[index + 1]; if (value === undefined || value.startsWith("-")) { missingValue = true; @@ -2038,8 +2084,8 @@ function readPatternFlags(args: string[]): PatternFlags { values.push(value); continue; } - if (arg.startsWith(`${PATTERN_OPTION}=`)) { - values.push(arg.slice(PATTERN_OPTION.length + 1)); + if (arg.startsWith(`${option}=`)) { + values.push(arg.slice(option.length + 1)); } } diff --git a/packages/cli/src/test-diagnostics.ts b/packages/cli/src/test-diagnostics.ts index d02b71af3..fdc371422 100644 --- a/packages/cli/src/test-diagnostics.ts +++ b/packages/cli/src/test-diagnostics.ts @@ -56,6 +56,8 @@ export interface TestDiagnostics { execution(kind: "root" | "child", source: string): ExecutionDiagnostics; /** The first failure to deliver a progress line, or `Ok` when none failed. */ delivery(): Result; + /** Counts recorded by root documents. Nested execution results are evidence. */ + results(): Readonly<{ pass: number; fail: number }>; } /** What one execution is observed and preserved through. */ @@ -95,7 +97,10 @@ function* artifactDirectory(working: string): Operation { * diagnostics drains what was accepted before it returns — on success, failure * and cancellation alike — and nothing is written after that. */ -export function useTestDiagnostics(sink: DeliverySink): Operation { +export function useTestDiagnostics( + sink: DeliverySink, + failuresOnly = false, +): Operation { return resource(function* (provide) { const working = yield* cwd(); const directory = yield* artifactDirectory(working); @@ -125,6 +130,7 @@ export function useTestDiagnostics(sink: DeliverySink): Operation observe(owner, shown, (line) => emit(`${context}${line}`)), + observe: () => + observe( + owner, + shown, + (line, failure = false) => { + if (!failuresOnly || failure) { + emit(`${context}${line}`); + } + }, + (status) => { + if (kind === "root") { + results[status] += 1; + } + }, + ), journal: (failed: (value: T) => boolean): FailureJournalHost => ({ failed, *save(events: readonly DurableEvent[]) { @@ -149,6 +169,7 @@ export function useTestDiagnostics(sink: DeliverySink): Operation delivery, + results: () => results, }); }); } @@ -206,7 +227,8 @@ interface ObservedTest { function* observe( owner: Scope, shown: (path: string) => string, - report: (line: string) => void, + report: (line: string, failure?: boolean) => void, + record: (status: TestResult["status"]) => void, ): Operation { const tests = new Map(); let current: ObservedTest | undefined; @@ -228,12 +250,12 @@ function* observe( } if (phase.phase === "complete") { if (active) { - report(`${phase.result.ok ? "done" : "failed"} ${what}`); + report(`${phase.result.ok ? "done" : "failed"} ${what}`, !phase.result.ok); } } else if (phase.phase === "cancelled") { interrupted(test, what); if (active) { - report(`cancelled ${what}`); + report(`cancelled ${what}`, true); } } yield* each.next(); @@ -280,7 +302,7 @@ function* observe( result.error?.kind ?? "error", ...(test?.interrupted === undefined ? [] : [`interrupted ${test.interrupted}`]), ]; - report(`fail ${name} at ${where} (${why.join("; ")})`); + report(`fail ${name} at ${where} (${why.join("; ")})`, true); }; yield* Component.around({ @@ -334,7 +356,7 @@ function* observe( if (!settled && !test.recorded) { const detail = test.interrupted === undefined ? "" : ` (interrupted ${test.interrupted})`; - report(`cancelled ${named} at ${test.shown}${detail}`); + report(`cancelled ${named} at ${test.shown}${detail}`, true); } } } @@ -344,6 +366,7 @@ function* observe( yield* Test.around({ *record([result], next) { finished(result); + record(result.status); yield* next(result); }, }); diff --git a/packages/cli/tests/test-diagnostics.test.ts b/packages/cli/tests/test-diagnostics.test.ts index e8675715a..ad5bb151a 100644 --- a/packages/cli/tests/test-diagnostics.test.ts +++ b/packages/cli/tests/test-diagnostics.test.ts @@ -550,3 +550,66 @@ describe("xmd test live progress", () => { expect(run.stderr()).toBe(delivered); }); }); + +describe("xmd test reporters", { sanitizeOps: false, sanitizeResources: false }, () => { + it("reports failures and totals across documents without printing passing tests", function* () { + const directory = yield* useProject({ + "a.test.md": doc( + "Routine suite output", + '', + " ", + "", + '', + " ", + "", + ), + "b.test.md": doc( + '', + " ", + "", + ), + }); + const artifacts = join(directory, "artifacts"); + + const result = yield* runTest(directory, [".", "--reporter=failures"], artifacts); + expect(result.code).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain('fail test "failing witness" at a.test.md:5:1 (assertion)'); + expect(result.stderr).toContain("a.test.md: tests failed:"); + expect(result.stderr).not.toContain("passing witness"); + expect(result.stderr).not.toContain("later witness"); + expect(progress(result.stderr).at(-1)).toMatch( + /^2 passed \| 1 failed \(\d+ms\) · 1 of 2 documents failed$/, + ); + expect(fileNames(yield* journals(artifacts))).toEqual(["root-1.jsonl"]); + }); + + it("reports a passing single document and rejects unsupported reporters", function* () { + const directory = yield* useProject({ + "suite.test.md": doc( + '', + " ", + "", + ), + }); + const artifacts = join(directory, "artifacts"); + + const result = yield* runTest( + directory, + ["--reporter", "failures", "suite.test.md"], + artifacts, + ); + expect(result.code).toBe(0); + expect(result.stdout).toBe(""); + expect(progress(result.stderr)).toEqual([ + expect.stringMatching(/^1 passed \| 0 failed \(\d+ms\)$/), + ]); + expect(yield* exists(artifacts)).toBe(false); + + const invalid = yield* runTest(directory, ["suite.test.md", "--reporter=unknown"], artifacts); + expect(invalid.code).toBe(1); + expect(invalid.stdout).toBe(""); + expect(invalid.stderr).toContain("--reporter"); + expect(progress(invalid.stderr)).toEqual([]); + }); +}); diff --git a/specs/testing-spec.md b/specs/testing-spec.md index bffc6eb6b..5e02f0ba3 100644 --- a/specs/testing-spec.md +++ b/specs/testing-spec.md @@ -117,6 +117,16 @@ never carries a captured value, a prop, command or prompt text, an assertion's operands or anything a document rendered, and the rendered report on stdout is unchanged by it. +The default `--reporter=pretty` prints the rendered report on stdout and +progress on stderr. `--reporter=failures` suppresses the rendered report, +directory headings, test starts, and successful outcomes. It keeps failure +and cancellation diagnostics and journal notices on stderr. After the run, +it prints one line on stderr with the number of passing and failing tests +recorded by the root documents and elapsed time. Nested execution results +provide diagnostics without adding to those counts. Both reporters use the +same exit statuses and retain the same journals. Output that commands write +directly to the terminal keeps its normal routing. + Each root document and each nested `run` child keeps its journal in memory. When that execution fails, times out or is cancelled, the command saves the recorded events as JSONL after the execution's assertions and teardown have @@ -240,8 +250,8 @@ registrations, and testing sessions belong to one document and are torn down before the next one starts. A document that declares no tests fails on its own rather than inheriting an earlier document's results. -Each document's report is preceded by a heading naming its relative path, and a -failure is reported against that path. A failing document does not stop the +The pretty reporter precedes each document's report with a heading naming its +relative path. Both reporters identify a failed document by that path. A failing document does not stop the run; the remaining documents still execute, and a closing line names how many of them failed. The run exits `0` only when every discovered document passes, and `1` when any document fails or when no document matches. From 73642e8a98f0ecb9be1544d3a73a111ce3337fb1 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Mon, 16 Nov 2026 18:20:57 -0500 Subject: [PATCH 2/4] Document choosing an xmd test reporter --- README.md | 8 ++++++++ specs/testing-spec.md | 38 ++++++++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d6d8f1ec3..de6a8e121 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,14 @@ documents. Name an affected suite's path instead, beside any TypeScript paths: deno task test examples/adversarial-implementation/tests/Implementation.test.md ``` +Call `xmd test` directly to choose the reporter for a Markdown test. + +```bash +deno task xmd test examples/adversarial-implementation/tests/Implementation.test.md --reporter=failures +``` + +[Choose a test reporter](specs/testing-spec.md#choose-a-reporter). + ### Complete Run the complete applicable battery concurrently, yielding delivery diff --git a/specs/testing-spec.md b/specs/testing-spec.md index 5e02f0ba3..5bbc3112b 100644 --- a/specs/testing-spec.md +++ b/specs/testing-spec.md @@ -81,10 +81,34 @@ recorded results are preserved. On live and partial runs nothing is restored: re-expansion records each result exactly once, in discovery order, with completed records replaying in place. +### Choose a reporter + +The default `--reporter=pretty` prints the rendered report on stdout and +progress on stderr. Use `--reporter=failures` for a compact report in CI or a +Sidekick Round. + +```sh +xmd test ./tests --reporter=failures +``` + +The failures reporter omits the rendered report, directory headings, test +starts, and successful outcomes. It writes failure and cancellation +diagnostics and journal paths to stderr. A completed invocation ends with one +summary on stderr. + +```text +[xmd test] 11 passed | 1 failed (324ms) · 1 of 3 documents failed +``` + +The counts cover tests recorded by the root documents. Nested executions can +add diagnostics and journals. Exit status and journal retention follow the same +rules for either reporter. Output that commands write directly to the terminal +keeps its normal routing. + ### Progress and failure journals -While `xmd test` runs, it writes a short progress report to stderr, so a pipe or -CI log shows which test is running and which operation a timeout interrupted: +The pretty reporter writes live progress to stderr. A pipe or CI log shows +which test is running and which operation a timeout interrupted. ```text [xmd test] start test "held" at suite.test.md:1:1 @@ -117,16 +141,6 @@ never carries a captured value, a prop, command or prompt text, an assertion's operands or anything a document rendered, and the rendered report on stdout is unchanged by it. -The default `--reporter=pretty` prints the rendered report on stdout and -progress on stderr. `--reporter=failures` suppresses the rendered report, -directory headings, test starts, and successful outcomes. It keeps failure -and cancellation diagnostics and journal notices on stderr. After the run, -it prints one line on stderr with the number of passing and failing tests -recorded by the root documents and elapsed time. Nested execution results -provide diagnostics without adding to those counts. Both reporters use the -same exit statuses and retain the same journals. Output that commands write -directly to the terminal keeps its normal routing. - Each root document and each nested `run` child keeps its journal in memory. When that execution fails, times out or is cancelled, the command saves the recorded events as JSONL after the execution's assertions and teardown have From 582122f5aa83457ec1524b6884ff73d74270023a Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Mon, 16 Nov 2026 18:32:15 -0500 Subject: [PATCH 3/4] Use compact Markdown and Deno reporters in CI --- .github/workflows/ci.yml | 31 +++++++------- scripts/lib/measure-weights.ts | 2 +- scripts/lib/runtime-tests.ts | 58 ++++++++++++++++++++------- scripts/tests/runtime-tests.test.ts | 12 +++--- scripts/tests/shard-execution.test.ts | 32 +++++++-------- scripts/tests/test-weights.test.ts | 6 ++- specs/testing-spec.md | 14 ++++--- 7 files changed, 94 insertions(+), 61 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b4c35a21..4f051d47b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,7 +186,7 @@ jobs: - name: Check REPL bench tools working-directory: repl bench checkout run: | - deno test --allow-all --cached-only --frozen --node-modules-dir=none \ + deno test --reporter=dot --allow-all --cached-only --frozen --node-modules-dir=none \ --ignore=examples/repl-bench/tests/driver-exit.test.ts,examples/repl-bench/tests/teardown.test.ts \ examples/repl-bench/tests @@ -194,11 +194,12 @@ jobs: if: github.event_name == 'push' working-directory: repl bench checkout run: | - deno test --allow-all --cached-only --frozen --node-modules-dir=none \ + deno test --reporter=dot --allow-all --cached-only --frozen --node-modules-dir=none \ examples/repl-bench/tests/driver-exit.test.ts \ examples/repl-bench/tests/teardown.test.ts + echo "Running Markdown REPL Bench journeys" deno run --allow-all --cached-only --frozen --node-modules-dir=none \ - packages/cli/src/deno.ts test examples/repl-bench/journeys + packages/cli/src/deno.ts test examples/repl-bench/journeys --reporter=failures - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: github.event_name == 'push' && (failure() || cancelled()) @@ -284,19 +285,19 @@ jobs: ./dist/xmd test smoke-test/README.md \ --include smoke-test \ --include packages/core/components \ - --raw + --reporter=failures # Progress on stderr and a failed test's journal, from the binary rather # than the source entrypoint the runtime shards launch. - name: Test diagnostics from the compiled binary - run: deno test --allow-all --frozen scripts/tests/test-diagnostics-compiled.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/test-diagnostics-compiled.test.ts - name: Smoke test attached-service ping-pong with the compiled binary run: | ./dist/xmd test smoke-test/attached-service-ping-pong.test.md \ --include smoke-test \ --include packages/core/components \ - --raw + --reporter=failures # Foreground execution is about what a caller sees and what status they # get, and only the binary shows both surviving `deno compile`. @@ -314,14 +315,14 @@ jobs: # search path supplied the implementation. It lives here because its # subject is `dist/xmd`, and this is the job that builds one. - name: Form dispatch across the compiled binary's loaded copies - run: deno test --allow-all --frozen scripts/tests/component-form-dispatch.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/component-form-dispatch.test.ts # `` is packaged Markdown rather than a module, so it is exactly the # kind of thing `deno compile` ships only because an `--include` names it. # Asking the binary which Component source it carries is what catches a forgotten # include here rather than at a person's first `xmd plan`. - name: The Component the compiled binary carries - run: deno test --allow-all --frozen scripts/tests/plan-component-compiled.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/plan-component-compiled.test.ts # The compiled entrypoint states its own Prompt usage host facts, a # different assembly from the source CLI's, so its carriage is asserted @@ -336,13 +337,13 @@ jobs: # reopened location. This drives the same journey the source row drives, # on a real terminal, against the binary this job just built. - name: A REPL entry's files across the compiled binary - run: deno test --allow-all --frozen scripts/tests/repl-repositories-compiled.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/repl-repositories-compiled.test.ts - name: A REPL entry's issues across the compiled binary - run: deno test --allow-all --frozen scripts/tests/repl-issues-compiled.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/repl-issues-compiled.test.ts - name: A REPL entry's publications across the compiled binary - run: deno test --allow-all --frozen scripts/tests/repl-push-compiled.test.ts + run: deno test --reporter=dot --allow-all --frozen scripts/tests/repl-push-compiled.test.ts # Each package's `components.md` is a packaged asset on the same footing, # and the long-form documentation is the only thing that reports whether @@ -374,7 +375,7 @@ jobs: # source at once — the built-in components under `components/`, and the # structural directives, which resolve no file at all. - name: Smoke test the built-ins with no search path - run: ./dist/xmd test packages/core/src --raw + run: ./dist/xmd test packages/core/src --reporter=failures # An inline root document exercises the compiled module graph the same way # a file does, and only the binary proves the graph survived `deno compile`. @@ -520,13 +521,13 @@ jobs: # Covers the compiled binary relaunching itself as `xmd test-agent`, # which the source-mode worker command never exercises. - name: Smoke test the compiled binary as a test agent - run: ./dist/xmd test smoke-test/test-agent/README.md --raw + run: ./dist/xmd test smoke-test/test-agent/README.md --reporter=failures - name: Smoke test xmd run through ACPX run: | ./dist/xmd test smoke-test/agent/README.md \ --include smoke-test/agent/components \ - --raw + --reporter=failures # A failed or cancelled run leaves the journals of the executions that # failed; a passing one uploads nothing. @@ -611,7 +612,7 @@ jobs: # each rewrite `node_modules` into their own layout, so a Deno step after # one of them resolves through links the other pruned (#279). - name: Host contract under Deno - run: deno test --allow-all --frozen packages/runtime/tests/host-files.test.ts + run: deno test --reporter=dot --allow-all --frozen packages/runtime/tests/host-files.test.ts - name: Host contract as a compiled binary run: | diff --git a/scripts/lib/measure-weights.ts b/scripts/lib/measure-weights.ts index 1bcff6257..4f1da416f 100644 --- a/scripts/lib/measure-weights.ts +++ b/scripts/lib/measure-weights.ts @@ -72,7 +72,7 @@ export function* measureRuntime( for (const file of applicable) { const started = performance.now(); - const code = yield* runOneFile(runtime, file, root); + const code = yield* runOneFile(runtime, file, root, { compactReport: true }); // The schema admits positive integers only, and a file that finishes in // under half a millisecond would otherwise round to a weight of zero. const ms = Math.max(1, Math.round(performance.now() - started)); diff --git a/scripts/lib/runtime-tests.ts b/scripts/lib/runtime-tests.ts index 0ba358bca..0c5fe50d3 100644 --- a/scripts/lib/runtime-tests.ts +++ b/scripts/lib/runtime-tests.ts @@ -18,7 +18,7 @@ import { exit, scoped, until } from "effection"; import type { Context, Operation } from "effection"; import { createContext } from "effection"; -import { exec } from "@effectionx/process"; +import { exec, Stdio } from "@effectionx/process"; import { exists, readTextFile, writeTextFile } from "@effectionx/fs"; import { timebox } from "@effectionx/timebox"; import { launchEnvironment, shellQuote } from "@executablemd/test-support/launch"; @@ -131,16 +131,23 @@ function retainedEvidence(suite: string): Record { * `xmd test ` under `runtime`. The adversarial suites also receive * their fixture plugin and absolute component directories. */ -function markdownCommand(runtime: Runtime, file: string, root: URL): Launch { +function markdownCommand( + runtime: Runtime, + file: string, + root: URL, + options: TestLaunchOptions, +): Launch { const [command, ...entry] = cliEntry(runtime); + const args = [...entry, "test", file]; + if (options.compactReport) { + args.push("--reporter=failures"); + } const document = DOCUMENT_SUITES.get(file); if (document !== undefined) { return { command, arguments: [ - ...entry, - "test", - file, + ...args, ...(runtime === "deno" && document.worker ? ["--plugin", checkoutPath("packages/cli/tests/fixtures/prepared-test-worker.mjs")] : []), @@ -155,9 +162,7 @@ function markdownCommand(runtime: Runtime, file: string, root: URL): Launch { return { command, arguments: [ - ...entry, - "test", - file, + ...args, "--plugin", checkoutPath("packages/cli/tests/fixtures/workflow-test-permissions.mjs"), "--include", @@ -192,21 +197,35 @@ export const RUNNERS: Record = { bun: { command: "bun", prefix: ["test", `--timeout=${BUN_TEST_TIMEOUT_MS}`], serial: [] }, }; +interface TestLaunchOptions { + compactReport?: boolean; +} + /** - * The exact command that runs one test file alone under `runtime`. + * Build the command that runs one test file alone under `runtime`. * * Weight measurement times this command, and a shard invokes it once per * assigned file — so the millisecond a weight records is the millisecond that * shard will spend, rather than a fraction of some larger invocation. */ -export function oneFileCommand(runtime: Runtime, file: string, root: URL = CHECKOUT): Launch { +export function oneFileCommand( + runtime: Runtime, + file: string, + root: URL = CHECKOUT, + options: TestLaunchOptions = {}, +): Launch { if (isMarkdownSuite(file)) { - return markdownCommand(runtime, file, root); + return markdownCommand(runtime, file, root, options); } const runner = RUNNERS[runtime]; return { command: runner.command, - arguments: [...runner.prefix, ...runner.serial, file], + arguments: [ + ...runner.prefix, + ...runner.serial, + ...(runtime === "deno" && options.compactReport ? ["--reporter=dot"] : []), + file, + ], }; } @@ -268,9 +287,14 @@ function* preparedLink(root: URL): Operation { * the prepared dependency link as it found it. A changed or missing link * fails the file. An earlier failure retains its own status. */ -export function* runOneFile(runtime: Runtime, file: string, root: URL): Operation { +export function* runOneFile( + runtime: Runtime, + file: string, + root: URL, + options: TestLaunchOptions = {}, +): Operation { const run = yield* TestRuns.expect(); - const launch = { ...oneFileCommand(runtime, file, root), cwd: fileURLToPath(root) }; + const launch = { ...oneFileCommand(runtime, file, root, options), cwd: fileURLToPath(root) }; if (!isMarkdownSuite(file)) { return yield* run(launch); } @@ -513,10 +537,14 @@ export function* runShardFiles( ): Operation { const ran: string[] = []; let failure: { file: string; status: number } | undefined; + const encoder = new TextEncoder(); for (const file of files) { ran.push(file); - const status = yield* runOneFile(runtime, file, root); + if (isMarkdownSuite(file)) { + yield* Stdio.operations.stderr(encoder.encode(`Running ${file}\n`)); + } + const status = yield* runOneFile(runtime, file, root, { compactReport: true }); if (status !== 0 && failure === undefined) { failure = { file, status }; } diff --git a/scripts/tests/runtime-tests.test.ts b/scripts/tests/runtime-tests.test.ts index 423445b3d..665b40c4f 100644 --- a/scripts/tests/runtime-tests.test.ts +++ b/scripts/tests/runtime-tests.test.ts @@ -125,7 +125,7 @@ describe("running a shard", () => { expect(runs.peak).toEqual(1); expect(runs.launches).toEqual( [ONE, TWO, THREE].map((file) => { - const launch = oneFileCommand("deno", file); + const launch = oneFileCommand("deno", file, ROOT, { compactReport: true }); return `${launch.command} ${launch.arguments.join(" ")}`; }), ); @@ -136,12 +136,13 @@ describe("running a shard", () => { const runs = recorder(); yield* TestRuns.with(runs.run, () => runShardFiles(runtime, [ONE], ROOT)); - const launch = oneFileCommand(runtime, ONE); + const launch = oneFileCommand(runtime, ONE, ROOT, { compactReport: true }); + expect(launch.arguments.includes("--reporter=dot")).toBe(runtime === "deno"); expect(runs.launches).toEqual([`${launch.command} ${launch.arguments.join(" ")}`]); } }); - it("routes a Markdown suite through its one-file command, from the repository", function* () { + it("uses the failures reporter for Markdown in each runtime shard", function* () { for (const runtime of RUNTIMES) { const seen: Array<{ line: string; cwd: string; env?: Record }> = []; yield* TestRuns.with( @@ -153,9 +154,10 @@ describe("running a shard", () => { ); expect(seen.map((entry) => entry.line)).toEqual([ - line(oneFileCommand(runtime, ONE)), - line(oneFileCommand(runtime, SUITE, ROOT)), + line(oneFileCommand(runtime, ONE, ROOT, { compactReport: true })), + line(oneFileCommand(runtime, SUITE, ROOT, { compactReport: true })), ]); + expect(seen[1].line).toContain(`test ${SUITE} --reporter=failures`); expect(seen.map((entry) => entry.cwd)).toEqual([fileURLToPath(ROOT), fileURLToPath(ROOT)]); expect(seen[1].env).toEqual(oneFileCommand(runtime, SUITE, ROOT).env); } diff --git a/scripts/tests/shard-execution.test.ts b/scripts/tests/shard-execution.test.ts index de2cdc715..6475886b1 100644 --- a/scripts/tests/shard-execution.test.ts +++ b/scripts/tests/shard-execution.test.ts @@ -69,7 +69,7 @@ function* workspace(files: Record): Operation { } function passing(marker: string): string { - return `Deno.test("${marker}", () => {\n console.log("${marker}_RAN");\n});\n`; + return `Deno.test("${marker}", () => {\n Deno.writeTextFileSync("${marker}.marker", "");\n console.log("${marker}_RAN");\n});\n`; } /** Multiline on both channels, so truncation or a summary would be visible. */ @@ -111,7 +111,7 @@ function* waitFor(file: URL): Operation { } describe("a shard running real children", () => { - it("runs the files after a failing one and keeps its complete output", function* () { + it("runs later files and keeps the failure diagnostic with Deno's dot reporter", function* () { const root = yield* workspace({ "a.test.ts": passing("A"), "b.test.ts": FAILING, @@ -127,19 +127,13 @@ describe("a shard running real children", () => { expect(outcome.failure?.status).toBeGreaterThan(0); const displayed = `${seen.stdout}\n${seen.stderr}`; - // The file after the failure ran, which is the whole point of continuing. - expect(displayed).toContain("A_RAN"); - expect(displayed).toContain("C_RAN"); - // And the failure's own text survived, every line of it, on both channels. - for (const line of [ - "B_STDOUT_ONE", - "B_STDOUT_TWO", - "B_STDERR_ONE", - "B_STDERR_TWO", - "B_FAILED_DELIBERATELY", - ]) { - expect(displayed).toContain(line); - } + expect(yield* exists(new URL("A.marker", root))).toBe(true); + expect(yield* exists(new URL("C.marker", root))).toBe(true); + expect(displayed).toContain("B_FAILED_DELIBERATELY"); + expect(displayed).not.toContain("A_RAN"); + expect(displayed).not.toContain("C_RAN"); + expect(displayed).not.toContain("B_STDOUT_ONE"); + expect(displayed).not.toContain("B_STDERR_ONE"); }); it("succeeds and stays quiet about failures when every file passes", function* () { @@ -167,7 +161,7 @@ describe("a shard running real children", () => { const marker = new URL("slow.marker", root); if (!(yield* waitFor(marker))) { - const launch = oneFileCommand("deno", "tests/a-slow.test.ts"); + const launch = oneFileCommand("deno", "tests/a-slow.test.ts", root, { compactReport: true }); throw new Error( `child did not write expected startup marker ${fileURLToPath(marker)}\ncommand: ${launch.command} ${launch.arguments.join(" ")}\noutcome: ${JSON.stringify(outcome)}\nstdout:\n${seen.stdout}\nstderr:\n${seen.stderr}`, ); @@ -257,7 +251,7 @@ function* collected(directory: URL): Operation { } describe("a shard running real Markdown suites", () => { - it("shows a suite's named start live, saves its journal, runs the files after it and keeps its failure first", function* () { + it("shows the active suite live, saves its journal and keeps its failure first", function* () { const { root, journals } = yield* suites({ "Held.test.md": HELD_FAILURE, "Later.test.md": MD_LATER, @@ -272,7 +266,7 @@ describe("a shard running real Markdown suites", () => { outcome = yield* runShardFiles("deno", assigned, root); }); - const start = `[xmd test] start test "fails deliberately" at ${held}`; + const start = `Running ${held}\n`; let visible = false; for (let attempt = 0; attempt < 300 && !visible; attempt += 1) { visible = seen.stderr.includes(start); @@ -289,6 +283,8 @@ describe("a shard running real Markdown suites", () => { expect(outcome?.failure).toEqual({ file: held, status: 1 }); expect(yield* exists(new URL("later.marker", root))).toBe(true); expect(seen.stderr).toContain("MD_FAILED_DELIBERATELY"); + expect(seen.stderr).toContain("0 passed | 1 failed"); + expect(seen.stderr).not.toContain('[xmd test] start test "fails deliberately"'); expect(`${seen.stdout}\n${seen.stderr}`).toContain("B_FAILED_DELIBERATELY"); const saved = yield* collected(journals); diff --git a/scripts/tests/test-weights.test.ts b/scripts/tests/test-weights.test.ts index d9d6b3e90..825a78b9f 100644 --- a/scripts/tests/test-weights.test.ts +++ b/scripts/tests/test-weights.test.ts @@ -361,7 +361,7 @@ describe("measuring one runtime", () => { expect(runs.launches).toEqual( [ALPHA, BETA].map((file) => { - const launch = oneFileCommand(runtime, file); + const launch = oneFileCommand(runtime, file, root, { compactReport: true }); return `${launch.command} ${launch.arguments.join(" ")}`; }), ); @@ -392,7 +392,9 @@ describe("measuring one runtime", () => { () => measureRuntime(runtime, root, SOURCE, measurementFile(root, runtime)), ); - const [alpha, suite] = [ALPHA, SUITE].map((file) => oneFileCommand(runtime, file, root)); + const alpha = oneFileCommand(runtime, ALPHA, root, { compactReport: true }); + const suite = oneFileCommand(runtime, SUITE, root, { compactReport: true }); + expect(suite.arguments).toContain("--reporter=failures"); expect(seen).toEqual([ { line: `${suite.command} ${suite.arguments.join(" ")}`, diff --git a/specs/testing-spec.md b/specs/testing-spec.md index 5bbc3112b..e6ab68cc7 100644 --- a/specs/testing-spec.md +++ b/specs/testing-spec.md @@ -954,7 +954,7 @@ the first failure of either portion. With a selection it resolves that shard and invokes its files serially, in sorted order, each through that runtime's one-file command: -- Deno — `deno test --allow-all --frozen ` +- Deno — `deno test --allow-all --frozen --reporter=dot ` - Node — `tsx --tsconfig tsconfig.node.json --test --test-concurrency=1 ` - Bun — `bun test --timeout=300000 ` @@ -965,8 +965,11 @@ source entrypoint, resolved from the runner's checkout: - Node — `tsx --tsconfig tsconfig.node.json packages/cli/src/node.ts` - Bun — `bun packages/cli/src/bun.ts` -The CLI receives `test ` and runs from the repository root with its inherited -environment (`PATH`, the Deno cache and install variables, `XDG_CACHE_HOME`, +The CLI runs from the repository root. A shard names each Markdown suite when +it starts and passes `test --reporter=failures`. Weight measurement +passes the same arguments. Local unsharded runs use the default reporter. +Deno weight measurement also uses the same dot reporter as Deno shards. +The CLI inherits the environment (`PATH`, the Deno cache and install variables, `XDG_CACHE_HOME`, `TMPDIR` and `XMD_TEST_ARTIFACTS`). Individually registered suites use the CLI's default includes and file permissions. Deno launches of `Plan`, `NativeSessionLaunch` and `SessionConfiguration` select the test-only worker @@ -1012,8 +1015,9 @@ A malformed runtime, index, or count is refused before any test process starts. A numeric failure does not prevent the files assigned after it: the shard continues, retains the first failed file and its status, and exits with that status once every assigned file has had its chance. A status without a numeric -code — a signalled child — is failure. Native output is neither captured, -truncated, nor replaced by a summary. +code — a signalled child — is failure. The shard streams the output its +selected reporter emits. Deno's dot reporter keeps failure details and omits +test console logs. Cancellation is distinct from failure. Halting a shard halts the process group of the file it is running, releases that file's scratch state and starts no From 723cbcb002953df69c449b9de513cfe794d8bfee Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Mon, 16 Nov 2026 19:14:59 -0500 Subject: [PATCH 4/4] Check the compact Deno filesystem contract command --- scripts/tests/filesystem-contract-workflow.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/filesystem-contract-workflow.test.ts b/scripts/tests/filesystem-contract-workflow.test.ts index 6a87affd9..7d17d06a7 100644 --- a/scripts/tests/filesystem-contract-workflow.test.ts +++ b/scripts/tests/filesystem-contract-workflow.test.ts @@ -121,7 +121,7 @@ describe("the filesystem contract matrix", () => { it("runs the contract under Deno, Node, Bun, and a compiled binary", function* () { const commands = (yield* job()).commands.join("\n"); - expect(commands).toContain(`deno test --allow-all --frozen ${SUITE}`); + expect(commands).toContain(`deno test --reporter=dot --allow-all --frozen ${SUITE}`); expect(commands).toContain(`tsx --tsconfig tsconfig.node.json --test ${SUITE}`); expect(commands).toContain(`bun test --timeout=300000 ${SUITE}`); expect(commands).toContain(`deno compile`);