Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 16 additions & 15 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -186,19 +186,20 @@ jobs:
- name: Check REPL bench tools
working-directory: repl bench checkout
run: |
deno test --allow-all --cached-only --frozen --node-modules-dir=none \
deno test --reporter=dot --allow-all --cached-only --frozen --node-modules-dir=none \
--ignore=examples/repl-bench/tests/driver-exit.test.ts,examples/repl-bench/tests/teardown.test.ts \
examples/repl-bench/tests

- name: Exercise the REPL through a host terminal
if: github.event_name == 'push'
working-directory: repl bench checkout
run: |
deno test --allow-all --cached-only --frozen --node-modules-dir=none \
deno test --reporter=dot --allow-all --cached-only --frozen --node-modules-dir=none \
examples/repl-bench/tests/driver-exit.test.ts \
examples/repl-bench/tests/teardown.test.ts
echo "Running Markdown REPL Bench journeys"
deno run --allow-all --cached-only --frozen --node-modules-dir=none \
packages/cli/src/deno.ts test examples/repl-bench/journeys
packages/cli/src/deno.ts test examples/repl-bench/journeys --reporter=failures

- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: github.event_name == 'push' && (failure() || cancelled())
Expand Down Expand Up @@ -284,19 +285,19 @@ jobs:
./dist/xmd test smoke-test/README.md \
--include smoke-test \
--include packages/core/components \
--raw
--reporter=failures

# Progress on stderr and a failed test's journal, from the binary rather
# than the source entrypoint the runtime shards launch.
- name: Test diagnostics from the compiled binary
run: deno test --allow-all --frozen scripts/tests/test-diagnostics-compiled.test.ts
run: deno test --reporter=dot --allow-all --frozen scripts/tests/test-diagnostics-compiled.test.ts

- name: Smoke test attached-service ping-pong with the compiled binary
run: |
./dist/xmd test smoke-test/attached-service-ping-pong.test.md \
--include smoke-test \
--include packages/core/components \
--raw
--reporter=failures

# Foreground execution is about what a caller sees and what status they
# get, and only the binary shows both surviving `deno compile`.
Expand All @@ -314,14 +315,14 @@ jobs:
# search path supplied the implementation. It lives here because its
# subject is `dist/xmd`, and this is the job that builds one.
- name: Form dispatch across the compiled binary's loaded copies
run: deno test --allow-all --frozen scripts/tests/component-form-dispatch.test.ts
run: deno test --reporter=dot --allow-all --frozen scripts/tests/component-form-dispatch.test.ts

# `<Plan>` is packaged Markdown rather than a module, so it is exactly the
# kind of thing `deno compile` ships only because an `--include` names it.
# Asking the binary which Component source it carries is what catches a forgotten
# include here rather than at a person's first `xmd plan`.
- name: The <Plan> Component the compiled binary carries
run: deno test --allow-all --frozen scripts/tests/plan-component-compiled.test.ts
run: deno test --reporter=dot --allow-all --frozen scripts/tests/plan-component-compiled.test.ts

# The compiled entrypoint states its own Prompt usage host facts, a
# different assembly from the source CLI's, so its carriage is asserted
Expand All @@ -336,13 +337,13 @@ jobs:
# reopened location. This drives the same journey the source row drives,
# on a real terminal, against the binary this job just built.
- name: A REPL entry's files across the compiled binary
run: deno test --allow-all --frozen scripts/tests/repl-repositories-compiled.test.ts
run: deno test --reporter=dot --allow-all --frozen scripts/tests/repl-repositories-compiled.test.ts

- name: A REPL entry's issues across the compiled binary
run: deno test --allow-all --frozen scripts/tests/repl-issues-compiled.test.ts
run: deno test --reporter=dot --allow-all --frozen scripts/tests/repl-issues-compiled.test.ts

- name: A REPL entry's publications across the compiled binary
run: deno test --allow-all --frozen scripts/tests/repl-push-compiled.test.ts
run: deno test --reporter=dot --allow-all --frozen scripts/tests/repl-push-compiled.test.ts

# Each package's `components.md` is a packaged asset on the same footing,
# and the long-form documentation is the only thing that reports whether
Expand Down Expand Up @@ -374,7 +375,7 @@ jobs:
# source at once — the built-in components under `components/`, and the
# structural directives, which resolve no file at all.
- name: Smoke test the built-ins with no search path
run: ./dist/xmd test packages/core/src --raw
run: ./dist/xmd test packages/core/src --reporter=failures

# An inline root document exercises the compiled module graph the same way
# a file does, and only the binary proves the graph survived `deno compile`.
Expand Down Expand Up @@ -520,13 +521,13 @@ jobs:
# Covers the compiled binary relaunching itself as `xmd test-agent`,
# which the source-mode worker command never exercises.
- name: Smoke test the compiled binary as a test agent
run: ./dist/xmd test smoke-test/test-agent/README.md --raw
run: ./dist/xmd test smoke-test/test-agent/README.md --reporter=failures

- name: Smoke test xmd run through ACPX
run: |
./dist/xmd test smoke-test/agent/README.md \
--include smoke-test/agent/components \
--raw
--reporter=failures

# A failed or cancelled run leaves the journals of the executions that
# failed; a passing one uploads nothing.
Expand Down Expand Up @@ -611,7 +612,7 @@ jobs:
# each rewrite `node_modules` into their own layout, so a Deno step after
# one of them resolves through links the other pruned (#279).
- name: Host contract under Deno
run: deno test --allow-all --frozen packages/runtime/tests/host-files.test.ts
run: deno test --reporter=dot --allow-all --frozen packages/runtime/tests/host-files.test.ts

- name: Host contract as a compiled binary
run: |
Expand Down
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,14 @@ documents. Name an affected suite's path instead, beside any TypeScript paths:
deno task test examples/adversarial-implementation/tests/Implementation.test.md
```

Call `xmd test` directly to choose the reporter for a Markdown test.

```bash
deno task xmd test examples/adversarial-implementation/tests/Implementation.test.md --reporter=failures
```

[Choose a test reporter](specs/testing-spec.md#choose-a-reporter).

### Complete

Run the complete applicable battery concurrently, yielding delivery
Expand Down
102 changes: 74 additions & 28 deletions packages/cli/src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,9 @@ const planConfig = object({
},
});

const TEST_REPORTER_OPTION = "--reporter";
const testReporter = z.enum(["pretty", "failures"]);

const testConfig = object({
path: {
description: "markdown document or directory to test (defaults to the current directory)",
Expand All @@ -439,6 +442,10 @@ const testConfig = object({
description: "glob for test documents, relative to a directory target (repeatable)",
...field(z.array(z.string()), field.default(["**/*.test.md"]), field.array()),
},
reporter: {
description: "test output: pretty (default) or failures",
...field(testReporter, field.default("pretty")),
},
include: {
description: "component search directory",
...field(z.array(z.string()), field.default(["components", "."]), field.array()),
Expand Down Expand Up @@ -1776,17 +1783,23 @@ type TestPlugins = (document: string) => Operation<CommandPlugins>;
/**
* `xmd test` — one document, or every document a directory holds.
*
* A directory keeps going after a failure and decides the status once at the
* end. A single document behaves exactly as it always has: one reported
* failure, no heading, no summary.
* A directory runs every matching document and settles the exit status once.
*/
function* test(
config: TestConfig,
args: string[],
installService: HostServiceInstaller,
testPlugins: TestPlugins,
): Operation<void> {
const patterns = readPatternFlags(args);
const reporterFlags = readTestOption(args, TEST_REPORTER_OPTION);
const reporter = testReporter.safeParse(reporterFlags.values[0] ?? "pretty");
if (reporterFlags.missingValue || reporterFlags.values.length > 1 || !reporter.success) {
console.error("--reporter accepts one value: pretty or failures");
yield* exit(1);
return;
}
const failuresOnly = reporter.data === "failures";
const patterns = readTestOption(args, PATTERN_OPTION);
if (patterns.missingValue) {
console.error(
`${PATTERN_OPTION} requires a value — write \`${PATTERN_OPTION} <glob>\`, or ` +
Expand Down Expand Up @@ -1814,14 +1827,27 @@ function* test(
return;
}
announceSecretDetection(config.secretDetection);
const { value: result, delivery } = yield* withTestProgress((diagnostics) =>
runTestDocument({ ...config, root: { path } }, testPlugins, installService, diagnostics),
const {
value: result,
delivery,
summary,
} = yield* withTestProgress(failuresOnly, (diagnostics) =>
runTestDocument(
{ ...config, root: { path }, discardOutput: failuresOnly },
testPlugins,
installService,
diagnostics,
),
);
if (!result.ok) {
reportFailure(result.error);
yield* exit(1);
} else if (!delivery.ok) {
reportFailure(delivery.error);
}
if (failuresOnly) {
console.error(summary);
}
if (!result.ok || !delivery.ok) {
yield* exit(1);
}
return;
Expand All @@ -1847,15 +1873,22 @@ function* test(
// and a directory of fifty documents would otherwise say so fifty times.
announceSecretDetection(config.secretDetection);

const { value: failures, delivery } = yield* withTestProgress(function* (diagnostics) {
const {
value: failures,
delivery,
summary,
} = yield* withTestProgress(failuresOnly, function* (diagnostics) {
const failed: string[] = [];
for (const document of target.documents) {
process.stdout.write(`\n# ${document.relativePath}\n\n`);
if (!failuresOnly) {
process.stdout.write(`\n# ${document.relativePath}\n\n`);
}
const result = yield* runTestDocument(
{
...config,
root: { path: document.path },
include: componentSearchPath(document, target.root, config.include),
discardOutput: failuresOnly,
},
testPlugins,
installService,
Expand All @@ -1869,11 +1902,19 @@ function* test(
return failed;
});

if (failures.length > 0) {
console.error(`\n${failures.length} of ${target.documents.length} documents failed`);
yield* exit(1);
} else if (!delivery.ok) {
const failedDocuments =
failures.length > 0
? `${failures.length} of ${target.documents.length} documents failed`
: undefined;
if (failedDocuments === undefined && !delivery.ok) {
reportFailure(delivery.error);
}
if (failuresOnly) {
console.error(failedDocuments === undefined ? summary : `${summary} · ${failedDocuments}`);
} else if (failedDocuments !== undefined) {
console.error(`\n${failedDocuments}`);
}
if (failures.length > 0 || !delivery.ok) {
yield* exit(1);
}
}
Expand All @@ -1886,15 +1927,22 @@ function* test(
* returns, so whatever the command reports next follows them.
*/
function* withTestProgress<T>(
failuresOnly: boolean,
body: (diagnostics: TestDiagnostics) => Operation<T>,
): Operation<{ value: T; delivery: Result<void> }> {
let delivery: () => Result<void> = () => Ok(undefined);
const value = yield* scoped(function* () {
const diagnostics = yield* useTestDiagnostics(process.stderr);
delivery = diagnostics.delivery;
return yield* body(diagnostics);
): Operation<{ value: T; delivery: Result<void>; summary: string }> {
const started = performance.now();
const run = yield* scoped(function* () {
const diagnostics = yield* useTestDiagnostics(process.stderr, failuresOnly);
const value = yield* body(diagnostics);
return { value, diagnostics };
});
return { value, delivery: delivery() };
const { pass, fail } = run.diagnostics.results();
const elapsed = Math.round(performance.now() - started);
return {
value: run.value,
delivery: run.diagnostics.delivery(),
summary: `[xmd test] ${pass} passed | ${fail} failed (${elapsed}ms)`,
};
}

/**
Expand Down Expand Up @@ -2002,15 +2050,13 @@ function findPropsFlag(args: string[]): string | undefined {

const PATTERN_OPTION = "--pattern";

interface PatternFlags {
/** Values the caller wrote, in the order they wrote them. */
interface TestOption {
values: string[];
/** A `--pattern` that ran out of argv or was followed by another option. */
missingValue: boolean;
}

/**
* Read `--pattern` from argv.
* Read an explicitly supplied test option from argv.
*
* The resolved configuration answers none of the questions this serves. It
* cannot say whether the option was given at all — the default is a real
Expand All @@ -2021,15 +2067,15 @@ interface PatternFlags {
* A separated value that begins with `-` is another option, not a glob;
* `--pattern=<glob>` expresses a glob that really does begin with one.
*/
function readPatternFlags(args: string[]): PatternFlags {
function readTestOption(args: string[], option: string): TestOption {
const values: string[] = [];
let missingValue = false;

for (const [index, arg] of args.entries()) {
if (arg === "--") {
break;
}
if (arg === PATTERN_OPTION) {
if (arg === option) {
const value = args[index + 1];
if (value === undefined || value.startsWith("-")) {
missingValue = true;
Expand All @@ -2038,8 +2084,8 @@ function readPatternFlags(args: string[]): PatternFlags {
values.push(value);
continue;
}
if (arg.startsWith(`${PATTERN_OPTION}=`)) {
values.push(arg.slice(PATTERN_OPTION.length + 1));
if (arg.startsWith(`${option}=`)) {
values.push(arg.slice(option.length + 1));
}
}

Expand Down
Loading
Loading