diff --git a/packages/core/src/testing-run.ts b/packages/core/src/testing-run.ts index 85735c38..0f24be67 100644 --- a/packages/core/src/testing-run.ts +++ b/packages/core/src/testing-run.ts @@ -29,7 +29,9 @@ import { ExpectedOperationError } from "@mobile-dev/shared/error-reporting.ts"; /** One step of the run, as the panel shows it against a line of the preview. */ export interface TestRunStep { /** The 1-based line in the test file the step came from. */ - line: number; + line?: number; + /** Position among this test's awaited calls when its runner couldn't locate the source file. */ + stepIndex?: number; /** The runner's own name for the call, e.g. `locator.tap`. */ api: string; label: string; @@ -88,7 +90,9 @@ function errorMessageOf(value: unknown): string | undefined { export function reportSteps(report: unknown, file: string): TestRunStep[] { const run = record(record(report)?.run); const steps: TestRunStep[] = []; - for (const entry of list(run?.results)) { + const results = list(run?.results); + const requestedFile = file.replaceAll("\\", "/"); + for (const entry of results) { const result = record(entry); if (!result) continue; const attempts = list(result.attempts); @@ -97,11 +101,18 @@ export function reportSteps(report: unknown, file: string): TestRunStep[] { const step = record(item); const source = record(step?.source); const line = typeof source?.line === "number" ? source.line : 0; - // A step the runner attributes to another file is another test's; `app.open` has no line. - if (!step || line <= 0 || (str(source?.file) && str(source?.file) !== file)) continue; + const sourceFile = str(source?.file).replaceAll("\\", "/"); + if (!step) continue; + // On Windows e2e can attribute every call to "unknown:1". Its index still identifies + // the call in a single test, just as it does for the panel's live progress. + const location = sourceFile === "unknown" && results.length === 1 + && str(result.file).replaceAll("\\", "/") === requestedFile && typeof step.index === "number" + ? { stepIndex: step.index } + : line > 0 && (!sourceFile || sourceFile === requestedFile) ? { line } : undefined; + if (!location) continue; const error = errorMessageOf(step.error); steps.push({ - line, api: str(step.api), label: str(step.label), status: str(step.status) || "unknown", + ...location, api: str(step.api), label: str(step.label), status: str(step.status) || "unknown", ...(typeof step.durationMs === "number" ? { durationMs: step.durationMs } : {}), ...(error ? { error } : {}), }); @@ -335,7 +346,7 @@ export function runnerEnv(environment: NodeJS.ProcessEnv = process.env): NodeJS. * progress spinner must never be the reason Run stopped working. */ function refusedTheConfig(result: TestRunResult): boolean { - return result.steps.length === 0 && /config/i.test(result.output ?? ""); + return result.status !== "passed" && result.steps.length === 0 && /config/i.test(result.output ?? ""); } export async function runRecordedTest(request: TestRunRequest, deps: TestRunDeps = {}): Promise { diff --git a/packages/core/test/testing-run.test.ts b/packages/core/test/testing-run.test.ts index 699d6bbe..33ba60a2 100644 --- a/packages/core/test/testing-run.test.ts +++ b/packages/core/test/testing-run.test.ts @@ -60,6 +60,20 @@ test("a retried test is read from the attempt that decided it", () => { assert.deepEqual(reportSteps(raw, FILE).map(step => step.status), ["passed"]); }); +test("the final runner report retains step indexes for one selected test without source locations", () => { + const raw = report([{ line: 1, status: "passed" }, { line: 1, status: "failed" }]); + const result = raw.run.results[0]!; + Object.assign(result, { file: FILE.replaceAll("/", "\\") }); + for (const step of result.attempts[0]!.steps) step.source.file = "unknown"; + assert.deepEqual(reportSteps(raw, FILE).map(step => [step.stepIndex, step.line, step.status]), + [[0, undefined, "passed"], [1, undefined, "failed"]]); + assert.deepEqual(reportSteps(raw, "tests/another.e2e.ts"), []); + Object.assign(result, { file: FILE }); + assert.deepEqual(reportSteps(raw, FILE).map(step => step.stepIndex), [0, 1]); + raw.run.results.push(result); + assert.deepEqual(reportSteps(raw, FILE), [], "step indexes restart for each test"); +}); + test("the report is found even when something else wrote to the same stream", () => { const document = JSON.stringify(report([{ line: 8, status: "passed" }])); assert.notEqual(parseReport(`npm warn exec something\n${document}\n`), undefined); @@ -177,7 +191,7 @@ test("the runner is not handed the app's pinned agent-device state directory", ( test("the generated config adds a reporter and changes nothing else", () => { const source = progressConfigSource("/app/e2e.config.ts"); // The project's own config is spread, so its targets, agents and workers all stand. - assert.match(source, /import base from "file:\/\/\/app\/e2e\.config\.ts";/); + assert.match(source, /import base from "file:\/\/\/(?:[A-Za-z]:\/)?app\/e2e\.config\.ts";/); assert.match(source, /\.\.\.resolved,/); // A config can export a function; the runner resolves one, so the generated config does too. assert.match(source, /typeof base === "function" \? await base\(\) : base/); @@ -263,6 +277,20 @@ test("a test that really failed is not retried into a different answer", async ( assert.equal(result.status, "failed"); }); +test("a passing test with no source locations runs only once when its output mentions config", async () => { + const root = await project(); + const output = JSON.stringify({ ...report([]), config: "e2e.config.ts" }); + let runs = 0; + const spawn = ((command: string, args: readonly string[], options: { cwd?: string }) => { + runs++; + return spawner(output)(command, args, options as never); + }) as never; + const result = await runRecordedTest({ projectPath: root, relativePath: FILE, configPath: join(root, "e2e.config.ts") }, + { spawner: spawn, onProgress: () => undefined }); + assert.equal(result.status, "passed"); + assert.equal(runs, 1); +}); + test("the generated config is syntactically whole, line by line", () => { // It is source written by source, so an escape that survives one level and not the other produces // a file the runner refuses with `CONFIG_LOAD_FAILED: Unterminated string literal` — and a run diff --git a/packages/ui/src/testing-panel.ts b/packages/ui/src/testing-panel.ts index a7a52b23..c6eeafc9 100644 --- a/packages/ui/src/testing-panel.ts +++ b/packages/ui/src/testing-panel.ts @@ -995,6 +995,21 @@ export class TestingPanel { && !this.snapshot.loadingTest && !this.snapshot.agent?.active && !this.snapshot.running && !this.snapshot.saving); } + private finishRun(data: Record, calls: readonly number[]) { + const steps = (Array.isArray(data.steps) ? data.steps : []).flatMap(item => { + const step = object(item); + if (!step) return []; + const line = typeof step.line === "number" ? step.line : typeof step.stepIndex === "number" ? calls[step.stepIndex] : undefined; + if (line === undefined || line <= 0) return []; + return [{ line, status: text(step.status), ...(text(step.error) ? { error: text(step.error) } : {}) }]; + }); + const status = text(data.status) || "failed"; + testRecorder.finishRun({ status, steps, + ...(text(data.output) ? { output: text(data.output) } : {}), + ...(typeof data.durationMs === "number" && Number.isFinite(data.durationMs) ? { durationMs: data.durationMs } : {}) }); + countUiEvent(status === "passed" ? "ui.testing.run_passed" : "ui.testing.run_failed"); + } + /** Runs a temporary sibling copy so Play and Run never save the working copy implicitly. */ async runStoredTest(untilLine?: number) { if (!this.canRunStored) return; @@ -1018,15 +1033,7 @@ export class TestingPanel { ...(this.snapshot.configPath ? { configPath: this.snapshot.configPath } : {}), ...(this.snapshot.targetName ? { target: this.snapshot.targetName } : {}), }, 15 * 60_000); - const data = object(response.structuredContent) ?? {}; - const steps = (Array.isArray(data.steps) ? data.steps : []).map(entry => object(entry)) - .filter((entry): entry is Record => entry !== null) - .map(entry => ({ line: typeof entry.line === "number" ? entry.line : 0, status: text(entry.status), ...(text(entry.error) ? { error: text(entry.error) } : {}) })) - .filter(step => step.line > 0); - testRecorder.finishRun({ status: text(data.status) || "failed", steps, - ...(text(data.output) ? { output: text(data.output) } : {}), - ...(typeof data.durationMs === "number" ? { durationMs: data.durationMs } : {}) }); - countUiEvent(text(data.status) === "passed" ? "ui.testing.run_passed" : "ui.testing.run_failed"); + this.finishRun(object(response.structuredContent) ?? {}, callLines); } catch (error) { const message = error instanceof Error ? error.message : String(error); testRecorder.failRun(message); @@ -1090,7 +1097,9 @@ export class TestingPanel { const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes(); // The run's own tool call is one request and one response, so progress comes from polling a // second tool while it is in flight. See `mobile_testing_run_progress`. - const poll = setInterval(() => { void this.pollRunProgress(); }, 400); + const callLines = testRecorder.getSnapshot().sourceOverride !== undefined + ? storedTestCallLines(testRecorder.source) : testRecorder.emitted.calls; + const poll = setInterval(() => { void this.pollRunProgress(callLines); }, 400); try { const response = await this.call("mobile_testing_run_test", { projectPath: this.snapshot.projectPath, relativePath, @@ -1098,24 +1107,7 @@ export class TestingPanel { ...(this.snapshot.targetName ? { target: this.snapshot.targetName } : {}), ...(until ? { untilSource: until.source } : {}), }, 15 * 60_000); - const data = object(response.structuredContent) ?? {}; - const steps = (Array.isArray(data.steps) ? data.steps : []).map(entry => object(entry)) - .filter((entry): entry is Record => entry !== null) - .map(entry => ({ - line: typeof entry.line === "number" ? entry.line : 0, - status: text(entry.status), - ...(text(entry.error) ? { error: text(entry.error) } : {}), - })) - .filter(step => step.line > 0); - // The runner's own number for the run, which is not the timing below: that one brackets the - // whole call, spawning the CLI and resolving the project included. - const durationMs = typeof data.durationMs === "number" && Number.isFinite(data.durationMs) ? data.durationMs : undefined; - testRecorder.finishRun({ - status: text(data.status) || "failed", steps, - ...(text(data.output) ? { output: text(data.output) } : {}), - ...(durationMs === undefined ? {} : { durationMs }), - }); - countUiEvent(text(data.status) === "passed" ? "ui.testing.run_passed" : "ui.testing.run_failed"); + this.finishRun(object(response.structuredContent) ?? {}, callLines); } catch (error) { const message = error instanceof Error ? error.message : String(error); testRecorder.failRun(message); diff --git a/packages/ui/test/testing-panel.test.ts b/packages/ui/test/testing-panel.test.ts index 14c80545..981d025c 100644 --- a/packages/ui/test/testing-panel.test.ts +++ b/packages/ui/test/testing-panel.test.ts @@ -150,13 +150,36 @@ test("a project test polls live progress against its own source lines", async () assert.ok(calls.some(call => call.name === "mobile_testing_run_progress"), "project runs poll while the runner is in flight"); assert.deepEqual(testRecorder.getSnapshot().run?.lines, { 2: "passed", 3: "running" }, "runner indexes map to the awaited calls in the saved project's source"); - finishRun({ status: "passed", exitCode: 0, steps: [{ line: 2, status: "passed" }, { line: 3, status: "passed" }] }); + finishRun({ status: "passed", exitCode: 0, steps: [{ stepIndex: 0, status: "passed" }, { stepIndex: 1, status: "passed" }] }); await running; + assert.deepEqual(testRecorder.getSnapshot().run?.lines, { 2: "passed", 3: "passed" }, + "a runner that omitted source locations still marks the final result against its awaited calls"); testRecorder.clearRun(); panel.dispose(); }); +test("step indexes in an edited recording use the saved source's awaited calls", async () => { + const panel = new TestingPanel(host({ + mobile_testing_write_file: { written: true, existed: false, path: `${PROJECT}/tests/flow.e2e.ts` }, + mobile_testing_run_test: { status: "failed", steps: [ + { stepIndex: 1, status: "failed", error: "Button missing" }, { stepIndex: 0, status: "passed" }, + ] }, + }, []), deviceApps); + panel.setAvailable(true); + (panel as unknown as { update(fields: Record): void }).update({ projectPath: PROJECT, fileName: "flow" }); + testRecorder.editSource('test("flow", async ({ app, screen }) => {\n await app.open();\n\n await screen.getByText("Done").tap();\n});'); + try { + await panel.runTest(); + assert.deepEqual(testRecorder.getSnapshot().run?.lines, { 2: "passed", 4: "failed" }); + assert.equal(testRecorder.getSnapshot().run?.error, "Button missing"); + } finally { + testRecorder.clearRun(); + testRecorder.clearSteps(); + panel.dispose(); + } +}); + test("Play until here runs the flow cut short, and never writes the test", async () => { const calls: { name: string; arguments: Record }[] = []; const panel = new TestingPanel(host({