From a2c173e71375422ddcf46c44c4b0d7e841b7fcbd Mon Sep 17 00:00:00 2001 From: omermorad Date: Tue, 29 Sep 2026 08:46:41 +0300 Subject: [PATCH 1/4] feat: support external SUT URLs without Docker Compose --- .github/actions/plan-e2e/action.yml | 2 +- .github/actions/plan-e2e/plan.rb | 15 +- .github/actions/run-e2e/action.yml | 14 +- .github/actions/run-e2e/complete-shard.mjs | 31 +- .github/actions/run-e2e/prepare-shard.mjs | 58 ++-- .github/tests/sut-lifecycle.test.mjs | 345 +++++++++++++++++++++ .github/workflows/action-tests.yml | 21 ++ README.md | 43 ++- action.yml | 2 +- 9 files changed, 475 insertions(+), 56 deletions(-) create mode 100644 .github/tests/sut-lifecycle.test.mjs create mode 100644 .github/workflows/action-tests.yml diff --git a/.github/actions/plan-e2e/action.yml b/.github/actions/plan-e2e/action.yml index e8d3e97..0fc0738 100644 --- a/.github/actions/plan-e2e/action.yml +++ b/.github/actions/plan-e2e/action.yml @@ -29,7 +29,7 @@ outputs: description: Playwright config path relative to working-directory. value: ${{ steps.plan.outputs.playwright-config }} compose-file: - description: Docker Compose file relative to the repository root. + description: Docker Compose file relative to the repository root; empty for an externally managed SUT. value: ${{ steps.plan.outputs.compose-file }} base-url: description: Ready SUT URL used by Playwright. diff --git a/.github/actions/plan-e2e/plan.rb b/.github/actions/plan-e2e/plan.rb index d10b4e3..c6911cb 100644 --- a/.github/actions/plan-e2e/plan.rb +++ b/.github/actions/plan-e2e/plan.rb @@ -114,7 +114,7 @@ def append_output(file, name, value) sut = require_mapping(manifest["sut"], "sut") reject_unknown_keys(sut, %w[composeFile baseUrl], "sut") -compose_file = require_relative_path(sut["composeFile"], "sut.composeFile") +compose_file = sut.key?("composeFile") ? require_relative_path(sut["composeFile"], "sut.composeFile") : "" begin base_url = URI(sut["baseUrl"].to_s) rescue URI::InvalidURIError @@ -163,17 +163,20 @@ def append_output(file, name, value) dockerfile_path = working_directory_path.join(dockerfile).cleanpath playwright_config_path = working_directory_path.join(playwright_config).cleanpath -compose_path = workspace.join(compose_file).cleanpath -[ +required_paths = [ [dockerfile_path, "runner.dockerfile"], [playwright_config_path, "playwright.config"], - [compose_path, "sut.composeFile"], -].each do |path, label| +] +compose_path = workspace.join(compose_file).cleanpath unless compose_file.empty? +required_paths << [compose_path, "sut.composeFile"] if compose_path +required_paths.each do |path, label| fail_plan("#{label} does not exist: #{path}") unless path.file? end fail_plan("runner.dockerfile must remain inside the E2E directory") unless dockerfile_path.realpath.to_s.start_with?("#{working_directory_path}/") fail_plan("playwright.config must remain inside the E2E directory") unless playwright_config_path.realpath.to_s.start_with?("#{working_directory_path}/") -fail_plan("sut.composeFile must remain inside the repository") unless compose_path.realpath.to_s.start_with?("#{workspace}/") +if compose_path && !compose_path.realpath.to_s.start_with?("#{workspace}/") + fail_plan("sut.composeFile must remain inside the repository") +end files_output, files_error, files_status = Open3.capture3( "git", diff --git a/.github/actions/run-e2e/action.yml b/.github/actions/run-e2e/action.yml index c83ecbe..f170c07 100644 --- a/.github/actions/run-e2e/action.yml +++ b/.github/actions/run-e2e/action.yml @@ -11,11 +11,11 @@ inputs: required: false default: playwright.config.ts compose-file: - description: Docker Compose file for the SUT, relative to the repository root. + description: Optional Docker Compose file for the SUT; empty targets the base URL without managing services. required: false - default: compose.e2e.yml + default: "" base-url: - description: Base URL exposed by the Dockerized SUT. + description: Base URL of the Compose-managed or externally managed SUT. required: false default: http://127.0.0.1:4173 runner-image-name: @@ -157,7 +157,7 @@ runs: id: start if: >- always() && steps.initialize.outcome == 'success' && - steps.runner.outputs.exit-code == '0' + steps.runner.outputs.exit-code == '0' && inputs.compose-file != '' continue-on-error: true shell: bash env: @@ -182,7 +182,7 @@ runs: if: >- steps.initialize.outcome == 'success' && steps.runner.outputs.exit-code == '0' && - steps.start.outputs.exit-code == '0' + (inputs.compose-file == '' || steps.start.outputs.exit-code == '0') continue-on-error: true shell: bash env: @@ -245,7 +245,7 @@ runs: - name: Capture Dockerized SUT logs id: logs - if: always() && steps.initialize.outcome == 'success' + if: always() && steps.initialize.outcome == 'success' && inputs.compose-file != '' continue-on-error: true shell: bash env: @@ -266,7 +266,7 @@ runs: - name: Tear down Dockerized SUT id: teardown - if: always() && steps.initialize.outcome == 'success' + if: always() && steps.initialize.outcome == 'success' && inputs.compose-file != '' continue-on-error: true shell: bash env: diff --git a/.github/actions/run-e2e/complete-shard.mjs b/.github/actions/run-e2e/complete-shard.mjs index 77534b8..fa4f892 100644 --- a/.github/actions/run-e2e/complete-shard.mjs +++ b/.github/actions/run-e2e/complete-shard.mjs @@ -23,10 +23,7 @@ const parseExitCode = (requestedValue, label) => { return value; }; -const startupExitCode = parseExitCode( - requestedStartupExitCode, - "startup", -); +const startupExitCode = parseExitCode(requestedStartupExitCode, "startup"); const runnerExitCode = parseExitCode( requestedRunnerExitCode, "runner image verification", @@ -36,12 +33,10 @@ const playwrightExitCode = parseExitCode( "Playwright", ); const logExitCode = parseExitCode(requestedLogExitCode, "log capture"); -const teardownExitCode = parseExitCode( - requestedTeardownExitCode, - "teardown", -); +const teardownExitCode = parseExitCode(requestedTeardownExitCode, "teardown"); const status = JSON.parse(readFileSync(statusFile, "utf8")); +const managesSut = status.lifecycle.sut.mode !== "external"; const finishedAt = new Date(); const startedAt = new Date(status.startedAt); @@ -76,6 +71,7 @@ if (runnerTransportSucceeded && runnerExitCode === null) { }); } if ( + managesSut && runnerTransportSucceeded && runnerExitCode === 0 && startupExitCode === null @@ -86,6 +82,7 @@ if ( message: "Dockerized SUT startup did not report an exit code", }); } else if ( + managesSut && runnerTransportSucceeded && runnerExitCode === 0 && startupExitCode !== 0 @@ -96,7 +93,11 @@ if ( message: `Dockerized SUT startup exited with ${startupExitCode}`, }); } -if (runnerExitCode === 0 && startupExitCode === 0 && playwrightExitCode === null) { +if ( + runnerExitCode === 0 && + (!managesSut || startupExitCode === 0) && + playwrightExitCode === null +) { failures.push({ kind: "infrastructure", phase: "playwright-execution", @@ -125,26 +126,26 @@ if (runnerExitCode === 0 && startupExitCode === 0 && playwrightExitCode === null message: `Playwright exited with ${playwrightExitCode}`, }); } -if (logExitCode === null) { +if (managesSut && logExitCode === null) { failures.push({ kind: "infrastructure", phase: "results", message: "Docker Compose log capture did not report an exit code", }); -} else if (logExitCode !== 0) { +} else if (managesSut && logExitCode !== 0) { failures.push({ kind: "infrastructure", phase: "results", message: `Docker Compose log capture exited with ${logExitCode}`, }); } -if (teardownExitCode === null) { +if (managesSut && teardownExitCode === null) { failures.push({ kind: "infrastructure", phase: "cleanup", message: "Docker Compose teardown did not report an exit code", }); -} else if (teardownExitCode !== 0) { +} else if (managesSut && teardownExitCode !== 0) { failures.push({ kind: "infrastructure", phase: "cleanup", @@ -210,7 +211,9 @@ for (const [name, value] of [ } if (status.primaryFailure) { - console.error(`${status.primaryFailure.kind}: ${status.primaryFailure.message}`); + console.error( + `${status.primaryFailure.kind}: ${status.primaryFailure.message}`, + ); } for (const failure of status.secondaryFailures) { console.error(`additional ${failure.kind}: ${failure.message}`); diff --git a/.github/actions/run-e2e/prepare-shard.mjs b/.github/actions/run-e2e/prepare-shard.mjs index 2813ccc..ef897e0 100644 --- a/.github/actions/run-e2e/prepare-shard.mjs +++ b/.github/actions/run-e2e/prepare-shard.mjs @@ -48,7 +48,8 @@ const assertRelative = (value, label) => { assertRelative(requestedWorkingDirectory, "working-directory"); assertRelative(requestedConfig, "playwright-config"); -assertRelative(requestedComposeFile, "compose-file"); +const managesSut = requestedComposeFile !== ""; +if (managesSut) assertRelative(requestedComposeFile, "compose-file"); assertRelative(requestedResultsDirectory, "results-directory"); let baseUrl; @@ -60,11 +61,18 @@ try { if (!["http:", "https:"].includes(baseUrl.protocol)) { fail("base-url must use HTTP or HTTPS"); } +if (baseUrl.username || baseUrl.password || baseUrl.hash) { + fail("base-url must not contain credentials or a fragment"); +} if (!requestedRunnerImageName || /[\s\0]/.test(requestedRunnerImageName)) { - fail("runner-image-name must be a non-empty Docker reference without whitespace"); + fail( + "runner-image-name must be a non-empty Docker reference without whitespace", + ); } if (!/^sha256:[a-f0-9]{64}$/.test(requestedRunnerImageId)) { - fail(`runner-image-id must be a sha256 image ID; received ${requestedRunnerImageId}`); + fail( + `runner-image-id must be a sha256 image ID; received ${requestedRunnerImageId}`, + ); } const allowedStepOutcomes = new Set([ "success", @@ -120,14 +128,17 @@ if (!existsSync(configPath)) { fail(`Playwright configuration does not exist: ${requestedConfig}`); } -const composeFile = resolve(workspacePath, requestedComposeFile); -if (!existsSync(composeFile)) { - fail(`Docker Compose file does not exist: ${requestedComposeFile}`); -} -const resolvedComposeFile = realpathSync(composeFile); -const relativeComposeFile = relative(workspacePath, resolvedComposeFile); -if (relativeComposeFile.startsWith("..") || isAbsolute(relativeComposeFile)) { - fail("compose-file must not escape GITHUB_WORKSPACE"); +let resolvedComposeFile = ""; +if (managesSut) { + const composeFile = resolve(workspacePath, requestedComposeFile); + if (!existsSync(composeFile)) { + fail(`Docker Compose file does not exist: ${requestedComposeFile}`); + } + resolvedComposeFile = realpathSync(composeFile); + const relativeComposeFile = relative(workspacePath, resolvedComposeFile); + if (relativeComposeFile.startsWith("..") || isAbsolute(relativeComposeFile)) { + fail("compose-file must not escape GITHUB_WORKSPACE"); + } } const resultsRoot = resolve( @@ -174,7 +185,8 @@ const status = { verificationExitCode: null, }, sut: { - composeFile: requestedComposeFile, + mode: managesSut ? "compose" : "external", + composeFile: managesSut ? requestedComposeFile : null, baseUrl: baseUrl.toString(), projectName: null, startupExitCode: null, @@ -193,16 +205,18 @@ const status = { }; writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}\n`); -const composeProject = [ - "e2e", - process.env.GITHUB_RUN_ID ?? "local", - process.env.GITHUB_RUN_ATTEMPT ?? "1", - shardIndex, -] - .join("-") - .toLowerCase() - .replaceAll(/[^a-z0-9_-]/g, "-"); -status.lifecycle.sut.projectName = composeProject; +const composeProject = managesSut + ? [ + "e2e", + process.env.GITHUB_RUN_ID ?? "local", + process.env.GITHUB_RUN_ATTEMPT ?? "1", + shardIndex, + ] + .join("-") + .toLowerCase() + .replaceAll(/[^a-z0-9_-]/g, "-") + : ""; +status.lifecycle.sut.projectName = composeProject || null; writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}\n`); const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; diff --git a/.github/tests/sut-lifecycle.test.mjs b/.github/tests/sut-lifecycle.test.mjs new file mode 100644 index 0000000..1ae0e69 --- /dev/null +++ b/.github/tests/sut-lifecycle.test.mjs @@ -0,0 +1,345 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const root = fileURLToPath(new URL("../../", import.meta.url)); +const imageId = `sha256:${"a".repeat(64)}`; + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { encoding: "utf8", ...options }); + assert.ifError(result.error); + return result; +} + +function workspace(t, compose = false) { + const directory = mkdtempSync(join(tmpdir(), "playwright-sut-test-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + mkdirSync(join(directory, "e2e")); + writeFileSync(join(directory, "e2e/Dockerfile"), "FROM scratch\n"); + writeFileSync( + join(directory, "e2e/playwright.config.ts"), + "export default {};\n", + ); + writeFileSync( + join(directory, "e2e/ci.yml"), + `version: 1 +runner: + dockerfile: Dockerfile +playwright: + config: playwright.config.ts +sut: + baseUrl: https://staging.example.com/api +${compose ? " composeFile: compose.e2e.yml\n" : ""}execution: + shards: 1 + artifactRetentionDays: 10 +profiles: + pull-request: + projects: [api] + labels: [] + labelMatch: all +`, + ); + if (compose) + writeFileSync(join(directory, "compose.e2e.yml"), "services: {}\n"); + assert.equal(run("git", ["init", "--quiet", directory]).status, 0); + return directory; +} + +function plan(directory) { + return run("ruby", [ + join(root, ".github/actions/plan-e2e/plan.rb"), + directory, + "pull-request", + "", + "", + "", + join(directory, "plan-output"), + ]); +} + +function prepare( + directory, + compose = "", + baseUrl = "https://staging.example.com/api", +) { + return run(process.execPath, [ + join(root, ".github/actions/run-e2e/prepare-shard.mjs"), + directory, + "e2e", + "playwright.config.ts", + compose, + baseUrl, + "playwright-e2e-runner:test", + imageId, + "success", + "success", + "api", + "", + "all", + "1", + "1", + "test-results", + ]); +} + +function finish(directory, overrides = {}) { + const codes = { + download: "success", + load: "success", + runner: "0", + startup: "", + playwright: "0", + kind: "test", + logs: "", + teardown: "", + ...overrides, + }; + const statusFile = join( + directory, + "e2e/test-results/shard-1/shard-status.json", + ); + const result = run(process.execPath, [ + join(root, ".github/actions/run-e2e/complete-shard.mjs"), + statusFile, + codes.download, + codes.load, + codes.runner, + imageId, + codes.startup, + codes.playwright, + codes.kind, + codes.logs, + codes.teardown, + join(directory, "output"), + ]); + return { ...result, report: JSON.parse(readFileSync(statusFile, "utf8")) }; +} + +test("URL-only planning succeeds without a Compose file and preserves the runner identity", (t) => { + const directory = workspace(t); + const remote = plan(directory); + assert.equal(remote.status, 0, remote.stderr); + const remoteOutput = readFileSync(join(directory, "plan-output"), "utf8"); + assert.match(remoteOutput, /^compose-file=$/m); + assert.match(remoteOutput, /^base-url=https:\/\/staging.example.com\/api$/m); + + const config = join(directory, "e2e/ci.yml"); + writeFileSync( + config, + readFileSync(config, "utf8").replace( + "sut:\n", + "sut:\n composeFile: compose.e2e.yml\n", + ), + ); + writeFileSync(join(directory, "compose.e2e.yml"), "services: {}\n"); + const managed = plan(directory); + assert.equal(managed.status, 0, managed.stderr); + const hashes = readFileSync(join(directory, "plan-output"), "utf8").match( + /^runner-content-hash=.+$/gm, + ); + assert.equal( + hashes[0], + hashes[1], + "Changing SUT ownership must not rebuild the test runner", + ); +}); + +for (const value of [ + "missing.yml", + "../outside.yml", + "/tmp/outside.yml", + "''", + "null", +]) { + test(`Reject an explicitly invalid Compose configuration: ${value}`, (t) => { + const directory = workspace(t); + const config = join(directory, "e2e/ci.yml"); + writeFileSync( + config, + readFileSync(config, "utf8").replace( + "sut:\n", + `sut:\n composeFile: ${value}\n`, + ), + ); + assert.notEqual(plan(directory).status, 0); + }); +} + +test("External shard metadata records no Compose ownership", (t) => { + const directory = workspace(t); + const prepared = prepare(directory); + assert.equal(prepared.status, 0, prepared.stderr); + assert.match(prepared.stdout, /E2E_ABSOLUTE_COMPOSE_FILE=''/); + assert.match(prepared.stdout, /E2E_COMPOSE_PROJECT=''/); + const completed = finish(directory); + assert.equal(completed.status, 0, completed.stderr); + assert.equal(completed.report.result, "passed"); + assert.deepEqual(completed.report.lifecycle.sut, { + mode: "external", + composeFile: null, + baseUrl: "https://staging.example.com/api", + projectName: null, + startupExitCode: null, + }); + assert.equal(completed.report.lifecycle.results.logCaptureExitCode, null); + assert.equal(completed.report.lifecycle.cleanup.teardownExitCode, null); +}); + +for (const [name, overrides, result, phase] of [ + ["test failure", { playwright: "1" }, "failed", "playwright"], + [ + "missing test execution", + { playwright: "" }, + "infrastructure-error", + "playwright-execution", + ], + [ + "image failure", + { runner: "42", playwright: "" }, + "infrastructure-error", + "runner-image", + ], + [ + "download failure", + { download: "failure", runner: "", playwright: "" }, + "infrastructure-error", + "runner-image-download", + ], + [ + "container failure", + { playwright: "125", kind: "runner-container" }, + "infrastructure-error", + "runner-container", + ], +]) { + test(`External mode preserves ${name}`, (t) => { + const directory = workspace(t); + assert.equal(prepare(directory).status, 0); + const completed = finish(directory, overrides); + assert.notEqual(completed.status, 0); + assert.equal(completed.report.result, result); + assert.equal(completed.report.primaryFailure.phase, phase); + }); +} + +for (const [name, overrides, result, phase] of [ + ["success", {}, "passed", undefined], + [ + "missing startup", + { startup: "", playwright: "" }, + "infrastructure-error", + "sut-startup", + ], + [ + "failed startup", + { startup: "1", playwright: "" }, + "infrastructure-error", + "sut-startup", + ], + ["missing logs", { logs: "" }, "infrastructure-error", "results"], + ["failed teardown", { teardown: "1" }, "infrastructure-error", "cleanup"], + [ + "test failure before cleanup failure", + { playwright: "1", teardown: "1" }, + "failed", + "playwright", + ], +]) { + test(`Compose mode preserves ${name}`, (t) => { + const directory = workspace(t, true); + assert.equal(prepare(directory, "compose.e2e.yml").status, 0); + const completed = finish(directory, { + startup: "0", + logs: "0", + teardown: "0", + ...overrides, + }); + assert.equal(completed.report.result, result); + assert.equal(completed.report.primaryFailure?.phase, phase); + assert.equal(completed.status === 0, result === "passed"); + assert.equal(completed.report.lifecycle.sut.mode, "compose"); + }); +} + +for (const baseUrl of [ + "ftp://example.com", + "https://user:secret@example.com", + "https://example.com/#fragment", +]) { + test(`Reject an unsafe target URL: ${new URL(baseUrl).protocol} ${new URL(baseUrl).hostname}`, (t) => { + const directory = workspace(t); + assert.notEqual(prepare(directory, "", baseUrl).status, 0); + const config = join(directory, "e2e/ci.yml"); + writeFileSync( + config, + readFileSync(config, "utf8").replace( + "https://staging.example.com/api", + baseUrl, + ), + ); + assert.notEqual(plan(directory).status, 0); + }); +} + +test("Compose paths cannot escape through symlinks", (t) => { + const directory = workspace(t); + const outside = mkdtempSync(join(tmpdir(), "playwright-outside-")); + t.after(() => rmSync(outside, { recursive: true, force: true })); + writeFileSync(join(outside, "compose.yml"), "services: {}\n"); + symlinkSync(join(outside, "compose.yml"), join(directory, "compose.e2e.yml")); + assert.notEqual(prepare(directory, "compose.e2e.yml").status, 0); + const config = join(directory, "e2e/ci.yml"); + writeFileSync( + config, + readFileSync(config, "utf8").replace( + "sut:\n", + "sut:\n composeFile: compose.e2e.yml\n", + ), + ); + assert.notEqual(plan(directory).status, 0); +}); + +test("The action skips every Compose step for a URL-only SUT but still executes Playwright", () => { + const parsed = run("ruby", [ + "-ryaml", + "-rjson", + "-e", + "puts JSON.generate(YAML.load_file(ARGV[0]))", + join(root, ".github/actions/run-e2e/action.yml"), + ]); + assert.equal(parsed.status, 0, parsed.stderr); + const action = JSON.parse(parsed.stdout); + assert.equal(action.inputs["compose-file"].default, ""); + const enabled = (id, compose, startup = "") => { + const condition = action.runs.steps + .find((step) => step.id === id) + .if.replaceAll("always()", "true") + .replaceAll("steps.initialize.outcome", '"success"') + .replaceAll("steps.runner.outputs.exit-code", '"0"') + .replaceAll("steps.start.outputs.exit-code", JSON.stringify(startup)) + .replaceAll("inputs.compose-file", JSON.stringify(compose)); + return Function(`return (${condition});`)(); + }; + for (const id of ["start", "logs", "teardown"]) { + assert.equal( + enabled(id, ""), + false, + `${id} must never touch an external SUT`, + ); + assert.equal(enabled(id, "compose.e2e.yml"), true); + } + assert.equal(enabled("playwright", ""), true); + assert.equal(enabled("playwright", "compose.e2e.yml", "0"), true); + assert.equal(enabled("playwright", "compose.e2e.yml", "1"), false); +}); diff --git a/.github/workflows/action-tests.yml b/.github/workflows/action-tests.yml new file mode 100644 index 0000000..d6bb849 --- /dev/null +++ b/.github/workflows/action-tests.yml @@ -0,0 +1,21 @@ +name: Action tests + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + sut-lifecycle: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Verify external and Compose SUT lifecycles + run: node --test .github/tests/*.test.mjs diff --git a/README.md b/README.md index 257e039..22a96b9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Playwright E2E -A portable GitHub Action and lightweight framework for running Playwright against a Dockerized system under test (SUT). +A portable GitHub Action and lightweight framework for running Playwright against a Docker Compose or externally managed system under test (SUT). It keeps the application repository in control of its tests and services while providing a consistent CI engine: deterministic runner images, smart reuse, SUT lifecycle management, test filtering, Playwright reports, traces, portable artifacts, and a fail-closed result. @@ -101,6 +101,29 @@ The root action is deliberately a single GitHub job. Playwright can still use mu `playwright.config.ts` defines projects, browser/device settings, matching, dependencies, reporters, and runtime behavior. `e2e/ci.yml` selects what CI runs. Unknown keys, invalid paths, unsafe values, nonexistent projects, or malformed tags fail before the SUT starts. +### Target an existing environment + +Omit `sut.composeFile` to test an already running local service or remote environment: + +```yaml +sut: + baseUrl: https://staging.example.com +``` + +Keep the other sections of `e2e/ci.yml` unchanged. The action passes this URL as `BASE_URL` to the runner and skips all Compose startup, service-log collection, and teardown. It does not deploy, reset, or stop the target; the tests themselves may still create or change resources. + +The environment must already be ready and reachable from the runner. Connection failures remain test failures. Test reports, runner images, and result enforcement work in both modes. Docker is still required for the test runner and report image; Docker Compose is only needed when `composeFile` is present. + +To run the same suites locally after installing their dependencies: + +```bash +BASE_URL=https://staging.example.com pnpm --dir e2e test +``` + +No Compose commands are needed. Use only environments you are authorized to test. Do not embed credentials in the URL. Arbitrary workflow environment variables, including API tokens, are not currently forwarded into the runner container. + +This support is unreleased; `v0.2.0` requires Compose. Use a release or commit containing this change before selecting URL-only mode. + ## What rebuilds the runner The runner identity covers every Git-tracked or non-ignored untracked entry below `e2e/`, except `e2e/ci.yml`. Paths, file types, Unix modes, file bytes, and symlink targets all contribute. @@ -115,10 +138,10 @@ The action performs one fail-closed sequence: 1. Validate configuration and resolve the requested profile. 2. Reuse or build the content-addressed Playwright runner. -3. Build and start the SUT with Docker Compose and wait for health checks. +3. If `sut.composeFile` is present, build and start the SUT and wait for Compose health checks; otherwise use the existing URL. 4. Run the selected Playwright projects and tags. -5. Capture results, traces, screenshots, SUT logs, and teardown logs. -6. Tear down containers, volumes, and networks even after failure. +5. Capture results, traces, and screenshots; collect SUT logs for Compose-managed runs. +6. Tear down Compose-managed containers, volumes, and networks even after failure. Never tear down an external SUT. 7. Build the HTML report and portable report image. 8. Upload artifacts and enforce the final result. @@ -161,12 +184,22 @@ Always run the final cleanup command, including after a failed test. Run `pnpm - ## Platform support -- GitHub.com hosted or self-hosted Linux runners with Docker and Docker Compose +- GitHub.com hosted or self-hosted Linux runners with Docker; Docker Compose is required only for managed SUTs - GitHub Actions runner 2.336.0 or newer, required for repository-relative `$/` action references - One checked-out application repository per job No inherited secrets are required by the action. The SUT may use repository or environment secrets supplied by its own workflow. +## Test action lifecycle changes + +With Node.js 22 and Ruby installed, run the offline regression checks: + +```bash +node --test .github/tests/*.test.mjs +``` + +These check both SUT modes, failure propagation, URL and path validation, and the action's lifecycle conditions without contacting a remote environment. + ## License [MIT](LICENSE) diff --git a/action.yml b/action.yml index aa3f592..3bb0fc9 100644 --- a/action.yml +++ b/action.yml @@ -1,6 +1,6 @@ name: Playwright E2E author: Codotech -description: Run a Dockerized system under test with a portable Playwright runner and publish the test results. +description: Test a Docker Compose or externally managed system with a portable Playwright runner and publish the results. branding: icon: check-circle From 2c01fc684d6153783c3df98b4a48502521fdb32b Mon Sep 17 00:00:00 2001 From: omermorad Date: Tue, 29 Sep 2026 08:53:59 +0300 Subject: [PATCH 2/4] refactor!: leave all SUT lifecycle management to callers --- .github/actions/plan-e2e/action.yml | 3 - .github/actions/plan-e2e/plan.rb | 9 +- .github/actions/run-e2e/action.yml | 90 +--------- .github/actions/run-e2e/complete-shard.mjs | 66 ------- .github/actions/run-e2e/prepare-shard.mjs | 38 ---- .github/tests/caller-workflow.test.mjs | 124 +++++++++++++ .github/tests/sut-lifecycle.test.mjs | 200 ++++++++++----------- .github/workflows/action-tests.yml | 4 +- .github/workflows/e2e.yml | 70 +++++++- README.md | 53 +++--- action.yml | 3 +- docs/diagrams/e2e-flow.svg | 10 +- docs/diagrams/repository-contract.svg | 14 +- e2e/ci.yml | 1 - 14 files changed, 327 insertions(+), 358 deletions(-) create mode 100644 .github/tests/caller-workflow.test.mjs diff --git a/.github/actions/plan-e2e/action.yml b/.github/actions/plan-e2e/action.yml index 0fc0738..66f8e11 100644 --- a/.github/actions/plan-e2e/action.yml +++ b/.github/actions/plan-e2e/action.yml @@ -28,9 +28,6 @@ outputs: playwright-config: description: Playwright config path relative to working-directory. value: ${{ steps.plan.outputs.playwright-config }} - compose-file: - description: Docker Compose file relative to the repository root; empty for an externally managed SUT. - value: ${{ steps.plan.outputs.compose-file }} base-url: description: Ready SUT URL used by Playwright. value: ${{ steps.plan.outputs.base-url }} diff --git a/.github/actions/plan-e2e/plan.rb b/.github/actions/plan-e2e/plan.rb index c6911cb..8afd9dd 100644 --- a/.github/actions/plan-e2e/plan.rb +++ b/.github/actions/plan-e2e/plan.rb @@ -113,8 +113,7 @@ def append_output(file, name, value) playwright_config = require_relative_path(playwright["config"], "playwright.config") sut = require_mapping(manifest["sut"], "sut") -reject_unknown_keys(sut, %w[composeFile baseUrl], "sut") -compose_file = sut.key?("composeFile") ? require_relative_path(sut["composeFile"], "sut.composeFile") : "" +reject_unknown_keys(sut, %w[baseUrl], "sut") begin base_url = URI(sut["baseUrl"].to_s) rescue URI::InvalidURIError @@ -167,16 +166,11 @@ def append_output(file, name, value) [dockerfile_path, "runner.dockerfile"], [playwright_config_path, "playwright.config"], ] -compose_path = workspace.join(compose_file).cleanpath unless compose_file.empty? -required_paths << [compose_path, "sut.composeFile"] if compose_path required_paths.each do |path, label| fail_plan("#{label} does not exist: #{path}") unless path.file? end fail_plan("runner.dockerfile must remain inside the E2E directory") unless dockerfile_path.realpath.to_s.start_with?("#{working_directory_path}/") fail_plan("playwright.config must remain inside the E2E directory") unless playwright_config_path.realpath.to_s.start_with?("#{working_directory_path}/") -if compose_path && !compose_path.realpath.to_s.start_with?("#{workspace}/") - fail_plan("sut.composeFile must remain inside the repository") -end files_output, files_error, files_status = Open3.capture3( "git", @@ -221,7 +215,6 @@ def append_output(file, name, value) append_output(output_file, "working-directory", working_directory) append_output(output_file, "dockerfile", dockerfile) append_output(output_file, "playwright-config", playwright_config) -append_output(output_file, "compose-file", compose_file) append_output(output_file, "base-url", base_url.to_s) append_output(output_file, "projects", projects.join("\n")) append_output(output_file, "labels", labels.join("\n")) diff --git a/.github/actions/run-e2e/action.yml b/.github/actions/run-e2e/action.yml index f170c07..971eddd 100644 --- a/.github/actions/run-e2e/action.yml +++ b/.github/actions/run-e2e/action.yml @@ -10,14 +10,9 @@ inputs: description: Path to the Playwright configuration, relative to working-directory. required: false default: playwright.config.ts - compose-file: - description: Optional Docker Compose file for the SUT; empty targets the base URL without managing services. - required: false - default: "" base-url: - description: Base URL of the Compose-managed or externally managed SUT. - required: false - default: http://127.0.0.1:4173 + description: HTTP or HTTPS base URL of the ready, caller-managed SUT. + required: true runner-image-name: description: Exact local Docker image reference loaded from the runner artifact. required: true @@ -58,7 +53,7 @@ outputs: description: "Shard result: passed, failed, or infrastructure-error." value: ${{ steps.finalize.outputs.result }} exit-code: - description: Effective shard exit code after SUT, Playwright, results, and cleanup checks. + description: Effective shard exit code after runner verification and Playwright execution. value: ${{ steps.finalize.outputs.exit-code }} shard-directory: description: Absolute path containing this shard's results. @@ -76,7 +71,6 @@ runs: env: E2E_WORKING_DIRECTORY: ${{ inputs.working-directory }} E2E_PLAYWRIGHT_CONFIG: ${{ inputs.playwright-config }} - E2E_COMPOSE_FILE: ${{ inputs.compose-file }} E2E_BASE_URL: ${{ inputs.base-url }} E2E_RUNNER_IMAGE_NAME: ${{ inputs.runner-image-name }} E2E_RUNNER_IMAGE_ID: ${{ inputs.runner-image-id }} @@ -95,7 +89,6 @@ runs: "$GITHUB_WORKSPACE" \ "$E2E_WORKING_DIRECTORY" \ "$E2E_PLAYWRIGHT_CONFIG" \ - "$E2E_COMPOSE_FILE" \ "$E2E_BASE_URL" \ "$E2E_RUNNER_IMAGE_NAME" \ "$E2E_RUNNER_IMAGE_ID" \ @@ -111,9 +104,7 @@ runs: { echo "working-directory=$E2E_ABSOLUTE_WORKING_DIRECTORY" echo "config=$E2E_CONTAINER_CONFIG" - echo "compose-file=$E2E_ABSOLUTE_COMPOSE_FILE" echo "base-url=$E2E_BASE_URL" - echo "compose-project=$E2E_COMPOSE_PROJECT" echo "runner-image-name=$E2E_RUNNER_IMAGE_NAME" echo "runner-image-id=$E2E_RUNNER_IMAGE_ID" echo "selection-file=$E2E_SELECTION_FILE" @@ -153,36 +144,11 @@ runs: } >> "$GITHUB_OUTPUT" exit "$runner_exit_code" - - name: Start Dockerized SUT - id: start - if: >- - always() && steps.initialize.outcome == 'success' && - steps.runner.outputs.exit-code == '0' && inputs.compose-file != '' - continue-on-error: true - shell: bash - env: - E2E_COMPOSE_FILE: ${{ steps.initialize.outputs.compose-file }} - E2E_COMPOSE_PROJECT: ${{ steps.initialize.outputs.compose-project }} - E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} - run: | - set +e - set -uo pipefail - startup_log="$E2E_SHARD_DIRECTORY/docker-compose-startup.log" - docker compose \ - --file "$E2E_COMPOSE_FILE" \ - --project-name "$E2E_COMPOSE_PROJECT" \ - up --detach --build --wait > "$startup_log" 2>&1 - startup_exit_code=$? - cat "$startup_log" - echo "exit-code=$startup_exit_code" >> "$GITHUB_OUTPUT" - exit "$startup_exit_code" - - name: Run Playwright shard id: playwright if: >- steps.initialize.outcome == 'success' && - steps.runner.outputs.exit-code == '0' && - (inputs.compose-file == '' || steps.start.outputs.exit-code == '0') + steps.runner.outputs.exit-code == '0' continue-on-error: true shell: bash env: @@ -243,48 +209,6 @@ runs: fi exit "$playwright_exit_code" - - name: Capture Dockerized SUT logs - id: logs - if: always() && steps.initialize.outcome == 'success' && inputs.compose-file != '' - continue-on-error: true - shell: bash - env: - E2E_COMPOSE_FILE: ${{ steps.initialize.outputs.compose-file }} - E2E_COMPOSE_PROJECT: ${{ steps.initialize.outputs.compose-project }} - E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} - run: | - set +e - service_log="$E2E_SHARD_DIRECTORY/docker-compose.log" - docker compose \ - --file "$E2E_COMPOSE_FILE" \ - --project-name "$E2E_COMPOSE_PROJECT" \ - logs --no-color --timestamps > "$service_log" 2>&1 - log_exit_code=$? - cat "$service_log" - echo "exit-code=$log_exit_code" >> "$GITHUB_OUTPUT" - exit "$log_exit_code" - - - name: Tear down Dockerized SUT - id: teardown - if: always() && steps.initialize.outcome == 'success' && inputs.compose-file != '' - continue-on-error: true - shell: bash - env: - E2E_COMPOSE_FILE: ${{ steps.initialize.outputs.compose-file }} - E2E_COMPOSE_PROJECT: ${{ steps.initialize.outputs.compose-project }} - E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} - run: | - set +e - teardown_log="$E2E_SHARD_DIRECTORY/docker-compose-teardown.log" - docker compose \ - --file "$E2E_COMPOSE_FILE" \ - --project-name "$E2E_COMPOSE_PROJECT" \ - down --volumes --remove-orphans > "$teardown_log" 2>&1 - teardown_exit_code=$? - cat "$teardown_log" - echo "exit-code=$teardown_exit_code" >> "$GITHUB_OUTPUT" - exit "$teardown_exit_code" - - name: Finalize shard results id: finalize if: always() && steps.initialize.outcome == 'success' @@ -295,11 +219,8 @@ runs: E2E_RUNNER_LOAD_OUTCOME: ${{ inputs.runner-load-outcome }} E2E_RUNNER_EXIT_CODE: ${{ steps.runner.outputs.exit-code }} E2E_ACTUAL_RUNNER_IMAGE_ID: ${{ steps.runner.outputs.actual-image-id }} - E2E_STARTUP_EXIT_CODE: ${{ steps.start.outputs.exit-code }} E2E_PLAYWRIGHT_EXIT_CODE: ${{ steps.playwright.outputs.exit-code }} E2E_PLAYWRIGHT_FAILURE_KIND: ${{ steps.playwright.outputs.failure-kind }} - E2E_LOG_EXIT_CODE: ${{ steps.logs.outputs.exit-code }} - E2E_TEARDOWN_EXIT_CODE: ${{ steps.teardown.outputs.exit-code }} run: | set +e node "$GITHUB_ACTION_PATH/complete-shard.mjs" \ @@ -308,11 +229,8 @@ runs: "$E2E_RUNNER_LOAD_OUTCOME" \ "$E2E_RUNNER_EXIT_CODE" \ "$E2E_ACTUAL_RUNNER_IMAGE_ID" \ - "$E2E_STARTUP_EXIT_CODE" \ "$E2E_PLAYWRIGHT_EXIT_CODE" \ "$E2E_PLAYWRIGHT_FAILURE_KIND" \ - "$E2E_LOG_EXIT_CODE" \ - "$E2E_TEARDOWN_EXIT_CODE" \ "$GITHUB_OUTPUT" completion_exit_code=$? exit "$completion_exit_code" diff --git a/.github/actions/run-e2e/complete-shard.mjs b/.github/actions/run-e2e/complete-shard.mjs index fa4f892..9aa2b69 100644 --- a/.github/actions/run-e2e/complete-shard.mjs +++ b/.github/actions/run-e2e/complete-shard.mjs @@ -7,11 +7,8 @@ const [ runnerLoadOutcome, requestedRunnerExitCode, actualRunnerImageId, - requestedStartupExitCode, requestedPlaywrightExitCode, requestedPlaywrightFailureKind, - requestedLogExitCode, - requestedTeardownExitCode, outputFile, ] = process.argv.slice(2); const parseExitCode = (requestedValue, label) => { @@ -23,7 +20,6 @@ const parseExitCode = (requestedValue, label) => { return value; }; -const startupExitCode = parseExitCode(requestedStartupExitCode, "startup"); const runnerExitCode = parseExitCode( requestedRunnerExitCode, "runner image verification", @@ -32,11 +28,8 @@ const playwrightExitCode = parseExitCode( requestedPlaywrightExitCode, "Playwright", ); -const logExitCode = parseExitCode(requestedLogExitCode, "log capture"); -const teardownExitCode = parseExitCode(requestedTeardownExitCode, "teardown"); const status = JSON.parse(readFileSync(statusFile, "utf8")); -const managesSut = status.lifecycle.sut.mode !== "external"; const finishedAt = new Date(); const startedAt = new Date(status.startedAt); @@ -71,31 +64,8 @@ if (runnerTransportSucceeded && runnerExitCode === null) { }); } if ( - managesSut && runnerTransportSucceeded && runnerExitCode === 0 && - startupExitCode === null -) { - failures.push({ - kind: "infrastructure", - phase: "sut-startup", - message: "Dockerized SUT startup did not report an exit code", - }); -} else if ( - managesSut && - runnerTransportSucceeded && - runnerExitCode === 0 && - startupExitCode !== 0 -) { - failures.push({ - kind: "infrastructure", - phase: "sut-startup", - message: `Dockerized SUT startup exited with ${startupExitCode}`, - }); -} -if ( - runnerExitCode === 0 && - (!managesSut || startupExitCode === 0) && playwrightExitCode === null ) { failures.push({ @@ -126,33 +96,6 @@ if ( message: `Playwright exited with ${playwrightExitCode}`, }); } -if (managesSut && logExitCode === null) { - failures.push({ - kind: "infrastructure", - phase: "results", - message: "Docker Compose log capture did not report an exit code", - }); -} else if (managesSut && logExitCode !== 0) { - failures.push({ - kind: "infrastructure", - phase: "results", - message: `Docker Compose log capture exited with ${logExitCode}`, - }); -} -if (managesSut && teardownExitCode === null) { - failures.push({ - kind: "infrastructure", - phase: "cleanup", - message: "Docker Compose teardown did not report an exit code", - }); -} else if (managesSut && teardownExitCode !== 0) { - failures.push({ - kind: "infrastructure", - phase: "cleanup", - message: `Docker Compose teardown exited with ${teardownExitCode}`, - }); -} - const testFailure = failures.find((failure) => failure.kind === "test"); const infrastructureFailure = failures.find( (failure) => failure.kind === "infrastructure", @@ -175,12 +118,6 @@ const exitCodeForFailure = (failure) => { return playwrightExitCode ?? 1; case "runner-image": return runnerExitCode ?? 1; - case "sut-startup": - return startupExitCode ?? 1; - case "results": - return logExitCode ?? 1; - case "cleanup": - return teardownExitCode ?? 1; default: return failure ? 1 : 0; } @@ -189,13 +126,10 @@ const effectiveExitCode = exitCodeForFailure(status.primaryFailure); status.exitCode = effectiveExitCode; status.lifecycle.runner.verificationExitCode = runnerExitCode; status.lifecycle.runner.actualImageId = actualRunnerImageId || null; -status.lifecycle.sut.startupExitCode = startupExitCode; status.lifecycle.playwright = { exitCode: playwrightExitCode, started: playwrightExitCode !== null, }; -status.lifecycle.results.logCaptureExitCode = logExitCode; -status.lifecycle.cleanup.teardownExitCode = teardownExitCode; status.finishedAt = finishedAt.toISOString(); status.durationMs = Math.max(0, finishedAt.getTime() - startedAt.getTime()); diff --git a/.github/actions/run-e2e/prepare-shard.mjs b/.github/actions/run-e2e/prepare-shard.mjs index ef897e0..399180f 100644 --- a/.github/actions/run-e2e/prepare-shard.mjs +++ b/.github/actions/run-e2e/prepare-shard.mjs @@ -12,7 +12,6 @@ const [ workspace, requestedWorkingDirectory, requestedConfig, - requestedComposeFile, requestedBaseUrl, requestedRunnerImageName, requestedRunnerImageId, @@ -48,8 +47,6 @@ const assertRelative = (value, label) => { assertRelative(requestedWorkingDirectory, "working-directory"); assertRelative(requestedConfig, "playwright-config"); -const managesSut = requestedComposeFile !== ""; -if (managesSut) assertRelative(requestedComposeFile, "compose-file"); assertRelative(requestedResultsDirectory, "results-directory"); let baseUrl; @@ -128,19 +125,6 @@ if (!existsSync(configPath)) { fail(`Playwright configuration does not exist: ${requestedConfig}`); } -let resolvedComposeFile = ""; -if (managesSut) { - const composeFile = resolve(workspacePath, requestedComposeFile); - if (!existsSync(composeFile)) { - fail(`Docker Compose file does not exist: ${requestedComposeFile}`); - } - resolvedComposeFile = realpathSync(composeFile); - const relativeComposeFile = relative(workspacePath, resolvedComposeFile); - if (relativeComposeFile.startsWith("..") || isAbsolute(relativeComposeFile)) { - fail("compose-file must not escape GITHUB_WORKSPACE"); - } -} - const resultsRoot = resolve( resolvedWorkingDirectory, requestedResultsDirectory, @@ -185,15 +169,9 @@ const status = { verificationExitCode: null, }, sut: { - mode: managesSut ? "compose" : "external", - composeFile: managesSut ? requestedComposeFile : null, baseUrl: baseUrl.toString(), - projectName: null, - startupExitCode: null, }, playwright: { exitCode: null, started: false }, - results: { logCaptureExitCode: null }, - cleanup: { teardownExitCode: null }, }, github: { repository: process.env.GITHUB_REPOSITORY ?? null, @@ -205,28 +183,12 @@ const status = { }; writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}\n`); -const composeProject = managesSut - ? [ - "e2e", - process.env.GITHUB_RUN_ID ?? "local", - process.env.GITHUB_RUN_ATTEMPT ?? "1", - shardIndex, - ] - .join("-") - .toLowerCase() - .replaceAll(/[^a-z0-9_-]/g, "-") - : ""; -status.lifecycle.sut.projectName = composeProject || null; -writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}\n`); - const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; const variables = { E2E_ABSOLUTE_WORKING_DIRECTORY: resolvedWorkingDirectory, E2E_ABSOLUTE_CONFIG: configPath, E2E_CONTAINER_CONFIG: requestedConfig, - E2E_ABSOLUTE_COMPOSE_FILE: resolvedComposeFile, E2E_BASE_URL: baseUrl.toString(), - E2E_COMPOSE_PROJECT: composeProject, E2E_RUNNER_IMAGE_NAME: requestedRunnerImageName, E2E_RUNNER_IMAGE_ID: requestedRunnerImageId, E2E_SELECTION_FILE: selectionFile, diff --git a/.github/tests/caller-workflow.test.mjs b/.github/tests/caller-workflow.test.mjs new file mode 100644 index 0000000..6fd78ab --- /dev/null +++ b/.github/tests/caller-workflow.test.mjs @@ -0,0 +1,124 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const workflowPath = fileURLToPath( + new URL("../workflows/e2e.yml", import.meta.url), +); +const parsed = spawnSync( + "ruby", + [ + "-ryaml", + "-rjson", + "-e", + "puts JSON.generate(YAML.load_file(ARGV[0]))", + workflowPath, + ], + { encoding: "utf8" }, +); +assert.equal(parsed.status, 0, parsed.stderr); +const job = JSON.parse(parsed.stdout).jobs.e2e; +const step = (id) => job.steps.find((entry) => entry.id === id); + +test("The caller starts and waits for its SUT before invoking the URL-only action", () => { + assert.match(step("sut").run, /docker compose .*up --detach --build --wait/); + assert.ok(job.steps.indexOf(step("sut")) < job.steps.indexOf(step("e2e"))); + assert.equal(step("e2e").if, "steps.sut.outcome == 'success'"); + assert.match( + job.env.COMPOSE_PROJECT_NAME, + /github.run_id.*github.run_attempt/, + ); +}); + +test("The caller collects and publishes its own service logs and always attempts cleanup", () => { + for (const id of ["sut-logs", "sut-cleanup", "sut-artifact"]) { + assert.match(step(id).if, /^always\(\)/); + assert.equal(step(id)["continue-on-error"], true); + } + assert.match(step("sut-cleanup").run, /down --volumes --remove-orphans/); + assert.equal(step("sut-artifact").with.name, "sut-logs"); + assert.ok( + job.steps.indexOf(step("sut-cleanup")) < + job.steps.indexOf(step("sut-artifact")), + ); +}); + +const gate = job.steps.find((entry) => entry.name === "Preserve E2E result"); +const successfulOutcomes = Object.fromEntries( + Object.keys(gate.env).map((name) => [name, "success"]), +); + +test("The caller gate passes only when tests and all caller-owned lifecycle steps succeed", () => { + const result = spawnSync("bash", ["-c", gate.run], { + env: { ...process.env, ...successfulOutcomes }, + }); + assert.equal(result.status, 0); +}); + +for (const name of Object.keys(successfulOutcomes)) { + test(`The caller gate fails when ${name} fails`, () => { + const result = spawnSync("bash", ["-c", gate.run], { + env: { ...process.env, ...successfulOutcomes, [name]: "failure" }, + }); + assert.equal(result.status, 1); + }); +} + +test("A skipped action after SUT startup failure cannot turn the caller gate green", () => { + const result = spawnSync("bash", ["-c", gate.run], { + env: { + ...process.env, + ...successfulOutcomes, + SUT_SETUP_OUTCOME: "failure", + E2E_STEP_OUTCOME: "skipped", + }, + }); + assert.equal(result.status, 1); +}); + +for (const [actionResult, cleanup, expected] of [ + ["passed", "success", "✅ passed"], + ["passed", "failure", "⚠️ infrastructure-error"], + ["failed", "failure", "❌ failed"], +]) { + test(`The PR comment reports ${expected} when tests are ${actionResult} and cleanup is ${cleanup}`, async () => { + const script = job.steps.find( + (entry) => entry.name === "Update pull request", + ).with.script; + let comment; + const github = { + paginate: async () => [], + rest: { + issues: { + listComments() {}, + async createComment(value) { + comment = value.body; + }, + }, + }, + }; + const AsyncFunction = Object.getPrototypeOf( + async function () {}, + ).constructor; + await new AsyncFunction("github", "context", "process", script)( + github, + { + serverUrl: "https://github.com", + repo: { owner: "example", repo: "test" }, + runId: 1, + issue: { number: 1 }, + }, + { + env: { + E2E_RESULT: actionResult, + E2E_SUT_SETUP: "success", + E2E_SUT_CLEANUP: cleanup, + E2E_SUT_LOGS: "success", + E2E_SUT_ARTIFACT: "success", + }, + }, + ); + assert.ok(comment.includes(expected), comment); + }); +} diff --git a/.github/tests/sut-lifecycle.test.mjs b/.github/tests/sut-lifecycle.test.mjs index 1ae0e69..fde1d18 100644 --- a/.github/tests/sut-lifecycle.test.mjs +++ b/.github/tests/sut-lifecycle.test.mjs @@ -5,7 +5,7 @@ import { mkdirSync, readFileSync, rmSync, - symlinkSync, + readdirSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -22,7 +22,7 @@ function run(command, args, options = {}) { return result; } -function workspace(t, compose = false) { +function workspace(t) { const directory = mkdtempSync(join(tmpdir(), "playwright-sut-test-")); t.after(() => rmSync(directory, { recursive: true, force: true })); mkdirSync(join(directory, "e2e")); @@ -40,7 +40,7 @@ playwright: config: playwright.config.ts sut: baseUrl: https://staging.example.com/api -${compose ? " composeFile: compose.e2e.yml\n" : ""}execution: +execution: shards: 1 artifactRetentionDays: 10 profiles: @@ -50,8 +50,6 @@ profiles: labelMatch: all `, ); - if (compose) - writeFileSync(join(directory, "compose.e2e.yml"), "services: {}\n"); assert.equal(run("git", ["init", "--quiet", directory]).status, 0); return directory; } @@ -68,17 +66,12 @@ function plan(directory) { ]); } -function prepare( - directory, - compose = "", - baseUrl = "https://staging.example.com/api", -) { +function prepare(directory, baseUrl = "https://staging.example.com/api") { return run(process.execPath, [ join(root, ".github/actions/run-e2e/prepare-shard.mjs"), directory, "e2e", "playwright.config.ts", - compose, baseUrl, "playwright-e2e-runner:test", imageId, @@ -98,11 +91,8 @@ function finish(directory, overrides = {}) { download: "success", load: "success", runner: "0", - startup: "", playwright: "0", kind: "test", - logs: "", - teardown: "", ...overrides, }; const statusFile = join( @@ -116,33 +106,29 @@ function finish(directory, overrides = {}) { codes.load, codes.runner, imageId, - codes.startup, codes.playwright, codes.kind, - codes.logs, - codes.teardown, join(directory, "output"), ]); return { ...result, report: JSON.parse(readFileSync(statusFile, "utf8")) }; } -test("URL-only planning succeeds without a Compose file and preserves the runner identity", (t) => { +test("Planning needs only a base URL, and changing it preserves the runner identity", (t) => { const directory = workspace(t); const remote = plan(directory); assert.equal(remote.status, 0, remote.stderr); const remoteOutput = readFileSync(join(directory, "plan-output"), "utf8"); - assert.match(remoteOutput, /^compose-file=$/m); + assert.doesNotMatch(remoteOutput, /compose/i); assert.match(remoteOutput, /^base-url=https:\/\/staging.example.com\/api$/m); const config = join(directory, "e2e/ci.yml"); writeFileSync( config, readFileSync(config, "utf8").replace( - "sut:\n", - "sut:\n composeFile: compose.e2e.yml\n", + "https://staging.example.com/api", + "http://127.0.0.1:4173", ), ); - writeFileSync(join(directory, "compose.e2e.yml"), "services: {}\n"); const managed = plan(directory); assert.equal(managed.status, 0, managed.stderr); const hashes = readFileSync(join(directory, "plan-output"), "utf8").match( @@ -151,19 +137,21 @@ test("URL-only planning succeeds without a Compose file and preserves the runner assert.equal( hashes[0], hashes[1], - "Changing SUT ownership must not rebuild the test runner", + "Changing the target must not rebuild the test runner", ); }); for (const value of [ + "compose.e2e.yml", "missing.yml", "../outside.yml", "/tmp/outside.yml", "''", "null", ]) { - test(`Reject an explicitly invalid Compose configuration: ${value}`, (t) => { + test(`Reject the removed composeFile option even when set to ${value}`, (t) => { const directory = workspace(t); + writeFileSync(join(directory, "compose.e2e.yml"), "services: {}\n"); const config = join(directory, "e2e/ci.yml"); writeFileSync( config, @@ -172,28 +160,32 @@ for (const value of [ `sut:\n composeFile: ${value}\n`, ), ); - assert.notEqual(plan(directory).status, 0); + const result = plan(directory); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /sut contains unknown keys: composeFile/); }); } -test("External shard metadata records no Compose ownership", (t) => { +test("Shard metadata records only the target URL and runner-owned lifecycle", (t) => { const directory = workspace(t); const prepared = prepare(directory); assert.equal(prepared.status, 0, prepared.stderr); - assert.match(prepared.stdout, /E2E_ABSOLUTE_COMPOSE_FILE=''/); - assert.match(prepared.stdout, /E2E_COMPOSE_PROJECT=''/); + assert.doesNotMatch(prepared.stdout, /COMPOSE|STARTUP|TEARDOWN/); const completed = finish(directory); assert.equal(completed.status, 0, completed.stderr); assert.equal(completed.report.result, "passed"); assert.deepEqual(completed.report.lifecycle.sut, { - mode: "external", - composeFile: null, baseUrl: "https://staging.example.com/api", - projectName: null, - startupExitCode: null, }); - assert.equal(completed.report.lifecycle.results.logCaptureExitCode, null); - assert.equal(completed.report.lifecycle.cleanup.teardownExitCode, null); + assert.deepEqual(Object.keys(completed.report.lifecycle), [ + "runner", + "sut", + "playwright", + ]); + assert.deepEqual(completed.report.lifecycle.playwright, { + exitCode: 0, + started: true, + }); }); for (const [name, overrides, result, phase] of [ @@ -222,64 +214,46 @@ for (const [name, overrides, result, phase] of [ "infrastructure-error", "runner-container", ], -]) { - test(`External mode preserves ${name}`, (t) => { - const directory = workspace(t); - assert.equal(prepare(directory).status, 0); - const completed = finish(directory, overrides); - assert.notEqual(completed.status, 0); - assert.equal(completed.report.result, result); - assert.equal(completed.report.primaryFailure.phase, phase); - }); -} - -for (const [name, overrides, result, phase] of [ - ["success", {}, "passed", undefined], [ - "missing startup", - { startup: "", playwright: "" }, + "missing image verification", + { runner: "", playwright: "" }, "infrastructure-error", - "sut-startup", + "runner-image", ], [ - "failed startup", - { startup: "1", playwright: "" }, + "load failure", + { load: "failure", runner: "", playwright: "" }, "infrastructure-error", - "sut-startup", + "runner-image-load", ], - ["missing logs", { logs: "" }, "infrastructure-error", "results"], - ["failed teardown", { teardown: "1" }, "infrastructure-error", "cleanup"], [ - "test failure before cleanup failure", - { playwright: "1", teardown: "1" }, - "failed", - "playwright", + "selection failure", + { playwright: "2", kind: "selection" }, + "infrastructure-error", + "playwright-selection", ], ]) { - test(`Compose mode preserves ${name}`, (t) => { - const directory = workspace(t, true); - assert.equal(prepare(directory, "compose.e2e.yml").status, 0); - const completed = finish(directory, { - startup: "0", - logs: "0", - teardown: "0", - ...overrides, - }); + test(`Preserve ${name} without managing the target`, (t) => { + const directory = workspace(t); + assert.equal(prepare(directory).status, 0); + const completed = finish(directory, overrides); + assert.notEqual(completed.status, 0); assert.equal(completed.report.result, result); - assert.equal(completed.report.primaryFailure?.phase, phase); - assert.equal(completed.status === 0, result === "passed"); - assert.equal(completed.report.lifecycle.sut.mode, "compose"); + assert.equal(completed.report.primaryFailure.phase, phase); + assert.equal(completed.report.secondaryFailures.length, 0); }); } for (const baseUrl of [ + "", + "not-a-url", "ftp://example.com", "https://user:secret@example.com", "https://example.com/#fragment", ]) { - test(`Reject an unsafe target URL: ${new URL(baseUrl).protocol} ${new URL(baseUrl).hostname}`, (t) => { + test(`Reject a missing, invalid, or unsafe target URL: ${baseUrl}`, (t) => { const directory = workspace(t); - assert.notEqual(prepare(directory, "", baseUrl).status, 0); + assert.notEqual(prepare(directory, baseUrl).status, 0); const config = join(directory, "e2e/ci.yml"); writeFileSync( config, @@ -292,25 +266,27 @@ for (const baseUrl of [ }); } -test("Compose paths cannot escape through symlinks", (t) => { - const directory = workspace(t); - const outside = mkdtempSync(join(tmpdir(), "playwright-outside-")); - t.after(() => rmSync(outside, { recursive: true, force: true })); - writeFileSync(join(outside, "compose.yml"), "services: {}\n"); - symlinkSync(join(outside, "compose.yml"), join(directory, "compose.e2e.yml")); - assert.notEqual(prepare(directory, "compose.e2e.yml").status, 0); - const config = join(directory, "e2e/ci.yml"); - writeFileSync( - config, - readFileSync(config, "utf8").replace( - "sut:\n", - "sut:\n composeFile: compose.e2e.yml\n", - ), - ); - assert.notEqual(plan(directory).status, 0); -}); +for (const baseUrl of [ + "http://127.0.0.1:4173", + "https://staging.example.com/api", +]) { + test(`Accept a caller-managed target at ${baseUrl}`, (t) => { + const directory = workspace(t); + assert.equal(prepare(directory, baseUrl).status, 0); + const config = join(directory, "e2e/ci.yml"); + writeFileSync( + config, + readFileSync(config, "utf8").replace( + "https://staging.example.com/api", + baseUrl, + ), + ); + const planned = plan(directory); + assert.equal(planned.status, 0, planned.stderr); + }); +} -test("The action skips every Compose step for a URL-only SUT but still executes Playwright", () => { +test("The action has no SUT lifecycle options or steps and still runs Playwright", () => { const parsed = run("ruby", [ "-ryaml", "-rjson", @@ -320,26 +296,34 @@ test("The action skips every Compose step for a URL-only SUT but still executes ]); assert.equal(parsed.status, 0, parsed.stderr); const action = JSON.parse(parsed.stdout); - assert.equal(action.inputs["compose-file"].default, ""); - const enabled = (id, compose, startup = "") => { + assert.equal(action.inputs["compose-file"], undefined); + assert.equal(action.inputs["base-url"].required, true); + assert.equal(action.inputs["base-url"].default, undefined); + assert.deepEqual( + action.runs.steps.map((step) => step.id), + ["initialize", "runner", "playwright", "finalize"], + ); + const enabled = (runnerCode) => { const condition = action.runs.steps - .find((step) => step.id === id) - .if.replaceAll("always()", "true") - .replaceAll("steps.initialize.outcome", '"success"') - .replaceAll("steps.runner.outputs.exit-code", '"0"') - .replaceAll("steps.start.outputs.exit-code", JSON.stringify(startup)) - .replaceAll("inputs.compose-file", JSON.stringify(compose)); + .find((step) => step.id === "playwright") + .if.replaceAll("steps.initialize.outcome", '"success"') + .replaceAll("steps.runner.outputs.exit-code", JSON.stringify(runnerCode)); return Function(`return (${condition});`)(); }; - for (const id of ["start", "logs", "teardown"]) { - assert.equal( - enabled(id, ""), - false, - `${id} must never touch an external SUT`, + assert.equal(enabled("0"), true); + assert.equal(enabled("42"), false); + const runtimeFiles = readdirSync(join(root, ".github/actions"), { + recursive: true, + withFileTypes: true, + }) + .filter((entry) => entry.isFile()) + .map((entry) => join(entry.parentPath, entry.name)); + runtimeFiles.push(join(root, "action.yml")); + for (const path of runtimeFiles) { + assert.doesNotMatch( + readFileSync(path, "utf8"), + /compose|sut-startup|startupExitCode|teardownExitCode/i, + path, ); - assert.equal(enabled(id, "compose.e2e.yml"), true); } - assert.equal(enabled("playwright", ""), true); - assert.equal(enabled("playwright", "compose.e2e.yml", "0"), true); - assert.equal(enabled("playwright", "compose.e2e.yml", "1"), false); }); diff --git a/.github/workflows/action-tests.yml b/.github/workflows/action-tests.yml index d6bb849..c07e570 100644 --- a/.github/workflows/action-tests.yml +++ b/.github/workflows/action-tests.yml @@ -9,7 +9,7 @@ permissions: contents: read jobs: - sut-lifecycle: + url-only-runner: runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -17,5 +17,5 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "22" - - name: Verify external and Compose SUT lifecycles + - name: Verify URL-only planning and runner results run: node --test .github/tests/*.test.mjs diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 68c02d2..ff1a7a8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -45,12 +45,23 @@ jobs: name: E2E Gate runs-on: ubuntu-latest timeout-minutes: 45 + env: + COMPOSE_PROJECT_NAME: e2e-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: Check out repository uses: actions/checkout@v7 + - id: sut + name: Start example SUT and wait for readiness + continue-on-error: true + shell: bash + run: | + mkdir -p sut-logs + docker compose -f compose.e2e.yml up --detach --build --wait 2>&1 | tee sut-logs/startup.log + - id: e2e name: Run E2E + if: steps.sut.outcome == 'success' continue-on-error: true uses: $/ with: @@ -63,11 +74,44 @@ jobs: ${{ github.event_name == 'workflow_dispatch' && inputs.label-match != 'profile' && inputs.label-match || '' }} + - id: sut-logs + name: Capture example SUT logs + if: always() && steps.sut.outcome != 'skipped' + continue-on-error: true + shell: bash + run: | + mkdir -p sut-logs + docker compose -f compose.e2e.yml logs --no-color --timestamps > sut-logs/services.log 2>&1 + + - id: sut-cleanup + name: Stop example SUT + if: always() && steps.sut.outcome != 'skipped' + continue-on-error: true + shell: bash + run: | + mkdir -p sut-logs + docker compose -f compose.e2e.yml down --volumes --remove-orphans > sut-logs/teardown.log 2>&1 + + - id: sut-artifact + name: Upload example SUT logs + if: always() && steps.sut.outcome != 'skipped' + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: sut-logs + path: sut-logs/ + if-no-files-found: error + retention-days: 10 + - name: Update pull request if: always() && github.event_name == 'pull_request' uses: actions/github-script@v8 env: E2E_RESULT: ${{ steps.e2e.outputs.result }} + E2E_SUT_SETUP: ${{ steps.sut.outcome }} + E2E_SUT_CLEANUP: ${{ steps.sut-cleanup.outcome }} + E2E_SUT_LOGS: ${{ steps.sut-logs.outcome }} + E2E_SUT_ARTIFACT: ${{ steps.sut-artifact.outcome }} E2E_PRIMARY_FAILURE: ${{ steps.e2e.outputs.primary-failure }} E2E_TOTAL: ${{ steps.e2e.outputs.total }} E2E_PASSED: ${{ steps.e2e.outputs.passed }} @@ -87,12 +131,18 @@ jobs: const marker = ''; const value = (name, fallback = 'n/a') => process.env[name] || fallback; const list = (name) => value(name).split('\n').filter(Boolean).join(', '); - const result = value('E2E_RESULT', 'infrastructure-error'); + const actionResult = value('E2E_RESULT', 'infrastructure-error'); + const lifecycleFailures = ['E2E_SUT_SETUP', 'E2E_SUT_CLEANUP', 'E2E_SUT_LOGS', 'E2E_SUT_ARTIFACT'] + .filter((name) => value(name) !== 'success'); + const result = actionResult === 'failed' + ? 'failed' + : lifecycleFailures.length ? 'infrastructure-error' : actionResult; const icon = result === 'passed' ? '✅' : result === 'failed' ? '❌' : '⚠️'; const cache = value('E2E_RUNNER_CACHE_HIT', 'false') === 'true' ? 'reused' : 'built'; const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const rows = [ ['Gate', `${icon} ${result}`], + ['Caller SUT', `setup: ${value('E2E_SUT_SETUP')} · cleanup: ${value('E2E_SUT_CLEANUP')} · artifact: sut-logs`], ['Tests', `${value('E2E_PASSED', '0')} passed · ${value('E2E_FAILED', '0')} failed · ${value('E2E_SKIPPED', '0')} skipped · ${value('E2E_TOTAL', '0')} total`], ['Profile', value('E2E_PROFILE')], ['Projects', list('E2E_PROJECTS')], @@ -102,8 +152,10 @@ jobs: ['Report image', `${value('E2E_REPORT_IMAGE')} · ${value('E2E_REPORT_ARTIFACT')}`], ['Run', `[Open workflow run](${runUrl})`], ]; - const failure = process.env.E2E_PRIMARY_FAILURE - ? `\n> ${process.env.E2E_PRIMARY_FAILURE}\n` + const failureMessage = process.env.E2E_PRIMARY_FAILURE || + (lifecycleFailures.length ? `Caller lifecycle did not succeed: ${lifecycleFailures.join(', ')}` : ''); + const failure = failureMessage + ? `\n> ${failureMessage}\n` : ''; const body = [ marker, @@ -140,7 +192,13 @@ jobs: shell: bash env: E2E_STEP_OUTCOME: ${{ steps.e2e.outcome }} + SUT_SETUP_OUTCOME: ${{ steps.sut.outcome }} + SUT_LOGS_OUTCOME: ${{ steps.sut-logs.outcome }} + SUT_CLEANUP_OUTCOME: ${{ steps.sut-cleanup.outcome }} + SUT_ARTIFACT_OUTCOME: ${{ steps.sut-artifact.outcome }} run: | - if [[ "$E2E_STEP_OUTCOME" != "success" ]]; then - exit 1 - fi + for outcome in "$E2E_STEP_OUTCOME" "$SUT_SETUP_OUTCOME" "$SUT_LOGS_OUTCOME" "$SUT_CLEANUP_OUTCOME" "$SUT_ARTIFACT_OUTCOME"; do + if [[ "$outcome" != "success" ]]; then + exit 1 + fi + done diff --git a/README.md b/README.md index 22a96b9..bbf710d 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,20 @@ # Playwright E2E -A portable GitHub Action and lightweight framework for running Playwright against a Docker Compose or externally managed system under test (SUT). +A portable GitHub Action for running Playwright against an already running system under test (SUT). -It keeps the application repository in control of its tests and services while providing a consistent CI engine: deterministic runner images, smart reuse, SUT lifecycle management, test filtering, Playwright reports, traces, portable artifacts, and a fail-closed result. +Your workflow starts the application and passes its base URL. The action owns the test runner, filtering, reports, traces, portable artifacts, and a fail-closed result. It never starts, collects logs from, or stops application services. -![E2E delivery flow from pull request, main, or manual trigger through runner reuse, Dockerized SUT testing, reports, and the required gate](docs/diagrams/e2e-flow.svg) +![E2E flow through runner reuse, testing a caller-provided URL, reports, and the required gate](docs/diagrams/e2e-flow.svg) ## Start from the template The quickest path is [codotech/playwright-e2e-starter](https://github.com/codotech/playwright-e2e-starter). Create a repository from that template, replace its example SUT and tests, then adapt the profiles in `e2e/ci.yml`. -The template owns repository policy: triggers, permissions, concurrency, checkout, the sticky pull-request comment, and preservation of the action result. This repository owns the portable execution engine. +The template owns repository policy and application lifecycle: triggers, permissions, concurrency, checkout, Compose startup, service logs, cleanup, the sticky pull-request comment, and preservation of the result. This repository owns the portable test execution engine. ## Use the action -Check out the caller repository first, then invoke the root action. Use major-version references for released actions: +Check out the caller repository and make the target ready before invoking the root action. The URL-only contract is unreleased: `v0.2.0` still requires Compose. Until a compatible release exists, use the implementation commit pinned by the [starter recipe PR](https://github.com/codotech/playwright-e2e-starter/pull/1), or the implementation branch while evaluating this change: ```yaml permissions: @@ -25,12 +25,12 @@ steps: - uses: actions/checkout@v7 - id: e2e - uses: codotech/playwright-e2e@v0 + uses: codotech/playwright-e2e@fix/optional-compose-remote-sut with: profile: pull-request ``` -`@v0` follows the latest compatible 0.x release. Pin `@v0.2.0` instead when an exact release is required. +Pin a reviewed full commit SHA for reproducible adoption. Do not use `@v0.2.0` with this recipe. ### Inputs @@ -64,7 +64,7 @@ The action fails at the end when tests fail or infrastructure is incomplete. A c The caller keeps these files: -![Repository contract showing the root Compose file, E2E configuration and tests, and replaceable system services](docs/diagrams/repository-contract.svg) +![Repository contract separating caller-owned application lifecycle from E2E configuration and tests](docs/diagrams/repository-contract.svg) `e2e/ci.yml` is the CI contract: @@ -78,7 +78,6 @@ playwright: config: playwright.config.ts sut: - composeFile: compose.e2e.yml baseUrl: http://127.0.0.1:4173 execution: @@ -99,20 +98,20 @@ profiles: The root action is deliberately a single GitHub job. Playwright can still use multiple workers inside the runner container. Keep `execution.shards: 1`; repository-level job matrices can be added later by the caller without changing the portable engine. -`playwright.config.ts` defines projects, browser/device settings, matching, dependencies, reporters, and runtime behavior. `e2e/ci.yml` selects what CI runs. Unknown keys, invalid paths, unsafe values, nonexistent projects, or malformed tags fail before the SUT starts. +`playwright.config.ts` defines projects, browser/device settings, matching, dependencies, reporters, and runtime behavior. `e2e/ci.yml` selects what CI runs. Unknown keys, invalid paths, unsafe values, nonexistent projects, or malformed tags fail before tests run. ### Target an existing environment -Omit `sut.composeFile` to test an already running local service or remote environment: +Set `sut.baseUrl` to an already running local service or remote environment: ```yaml sut: baseUrl: https://staging.example.com ``` -Keep the other sections of `e2e/ci.yml` unchanged. The action passes this URL as `BASE_URL` to the runner and skips all Compose startup, service-log collection, and teardown. It does not deploy, reset, or stop the target; the tests themselves may still create or change resources. +Keep the other sections of `e2e/ci.yml` unchanged. The action passes this URL as `BASE_URL` to the runner. It does not deploy, reset, or stop the target; the tests themselves may still create or change resources. -The environment must already be ready and reachable from the runner. Connection failures remain test failures. Test reports, runner images, and result enforcement work in both modes. Docker is still required for the test runner and report image; Docker Compose is only needed when `composeFile` is present. +The environment must already be ready and reachable from the runner. Connection failures remain test failures. Docker is required for the test runner and report image, not for managing your application. The action does not require Docker Compose. To run the same suites locally after installing their dependencies: @@ -122,13 +121,17 @@ BASE_URL=https://staging.example.com pnpm --dir e2e test No Compose commands are needed. Use only environments you are authorized to test. Do not embed credentials in the URL. Arbitrary workflow environment variables, including API tokens, are not currently forwarded into the runner container. -This support is unreleased; `v0.2.0` requires Compose. Use a release or commit containing this change before selecting URL-only mode. +### Migrate from action-managed Compose + +This is a breaking configuration change. Remove `sut.composeFile`; it is no longer accepted. Move startup and readiness checks before the action in your workflow. Collect service logs, upload them separately, and tear down your application in caller-owned steps with `if: always()`. The [starter](https://github.com/codotech/playwright-e2e-starter) demonstrates the complete Compose recipe. + +An existing staging environment needs none of those lifecycle steps. Only its base URL belongs in the action's contract. ## What rebuilds the runner The runner identity covers every Git-tracked or non-ignored untracked entry below `e2e/`, except `e2e/ci.yml`. Paths, file types, Unix modes, file bytes, and symlink targets all contribute. -A new runner is built when tests, fixtures, dependencies, the lockfile, Playwright configuration, runner entrypoint, Dockerfile, or Docker-ignore rules change. Profiles, filters, retention, and service-only changes reuse the same runner bytes while still running the suite against the newly built SUT. +A new runner is built when tests, fixtures, dependencies, the lockfile, Playwright configuration, runner entrypoint, Dockerfile, or Docker-ignore rules change. Profiles, filters, retention, base URL, and service-only changes reuse the same runner bytes while still running the suite against the supplied target. On an exact cache hit, the action validates the archive metadata, image reference, content hash, byte size, SHA-256, and image ID before using it. A missing, evicted, or invalid cache causes a clean rebuild. @@ -138,20 +141,18 @@ The action performs one fail-closed sequence: 1. Validate configuration and resolve the requested profile. 2. Reuse or build the content-addressed Playwright runner. -3. If `sut.composeFile` is present, build and start the SUT and wait for Compose health checks; otherwise use the existing URL. -4. Run the selected Playwright projects and tags. -5. Capture results, traces, and screenshots; collect SUT logs for Compose-managed runs. -6. Tear down Compose-managed containers, volumes, and networks even after failure. Never tear down an external SUT. -7. Build the HTML report and portable report image. -8. Upload artifacts and enforce the final result. +3. Run the selected Playwright projects and tags against `sut.baseUrl`. +4. Capture test results, traces, screenshots, and attachments. +5. Build the HTML report and portable report image. +6. Upload artifacts and enforce the final result. -A Playwright failure remains the primary failure even if later log collection or cleanup also fails. Missing results, runner identity problems, unhealthy services, or publication failures produce `infrastructure-error`; they never turn a run green. +A Playwright failure remains the primary failure if later report handling also fails. Missing results, runner identity problems, or publication failures produce `infrastructure-error`; they never turn a run green. Your workflow must separately enforce application startup and cleanup failures. ## Artifacts | Artifact | Contents | | --- | --- | -| `e2e-results` | Final JSON result, CTRF report, Playwright HTML report, test attachments, SUT logs, and teardown logs | +| `e2e-results` | Final JSON result, CTRF report, Playwright HTML report, and test attachments | | `e2e-runner-image` | Compressed Docker archive for the exact runner used by the run | | `e2e-report-image` | Compressed Docker archive serving the HTML report on port 8080 | @@ -168,7 +169,7 @@ The images are workflow artifacts, not registry publications. Artifact retention ## Run this example locally -Prerequisites: Docker, Node.js 22, Corepack, and pnpm 10.26.2. +This repository's sample uses caller-managed Compose. Prerequisites: Docker with Compose, Node.js 22, Corepack, and pnpm 10.26.2. ```bash corepack enable @@ -184,7 +185,7 @@ Always run the final cleanup command, including after a failed test. Run `pnpm - ## Platform support -- GitHub.com hosted or self-hosted Linux runners with Docker; Docker Compose is required only for managed SUTs +- GitHub.com hosted or self-hosted Linux runners with Docker - GitHub Actions runner 2.336.0 or newer, required for repository-relative `$/` action references - One checked-out application repository per job @@ -198,7 +199,7 @@ With Node.js 22 and Ruby installed, run the offline regression checks: node --test .github/tests/*.test.mjs ``` -These check both SUT modes, failure propagation, URL and path validation, and the action's lifecycle conditions without contacting a remote environment. +These check the URL-only contract, rejection of removed Compose configuration, failure propagation, and URL and path validation without contacting a remote environment. ## License diff --git a/action.yml b/action.yml index 3bb0fc9..bbf81f1 100644 --- a/action.yml +++ b/action.yml @@ -1,6 +1,6 @@ name: Playwright E2E author: Codotech -description: Test a Docker Compose or externally managed system with a portable Playwright runner and publish the results. +description: Test a caller-managed system at its base URL with a portable Playwright runner and publish the results. branding: icon: check-circle @@ -182,7 +182,6 @@ runs: with: working-directory: ${{ steps.plan.outputs.working-directory }} playwright-config: ${{ steps.plan.outputs.playwright-config }} - compose-file: ${{ steps.plan.outputs.compose-file }} base-url: ${{ steps.plan.outputs.base-url }} runner-image-name: ${{ steps.reuse.outputs.image-name || steps.build.outputs.image-name }} runner-image-id: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} diff --git a/docs/diagrams/e2e-flow.svg b/docs/diagrams/e2e-flow.svg index 6426346..558b33c 100644 --- a/docs/diagrams/e2e-flow.svg +++ b/docs/diagrams/e2e-flow.svg @@ -1,6 +1,6 @@ Playwright E2E delivery flow - A change enters CI, configuration and runner inputs are validated, a cached runner is reused or rebuilt, the Dockerized system is tested, and results and reports feed the E2E gate. + The caller makes its application ready. The action validates configuration, reuses or builds a runner, tests the supplied base URL, and publishes reports for the E2E gate. Application lifecycle remains outside the action. @@ -28,7 +28,7 @@ E2E DELIVERY FLOW - One portable action from change detection to a required test result + Caller makes the application ready; the action owns runner execution and reports @@ -92,8 +92,8 @@ - Start the SUT - Compose build · health wait + Use target URL + already ready · caller-owned @@ -107,7 +107,7 @@ Collect results - traces · logs · attachments + traces · test attachments diff --git a/docs/diagrams/repository-contract.svg b/docs/diagrams/repository-contract.svg index bb4829f..bc3a20d 100644 --- a/docs/diagrams/repository-contract.svg +++ b/docs/diagrams/repository-contract.svg @@ -1,6 +1,6 @@ Repository contract for the Playwright E2E action - The consumer repository contains a root Compose file, an e2e directory with the execution contract and tests, and the Dockerized services under test. + The consumer owns application lifecycle and supplies a ready base URL. Only the e2e directory containing runner configuration and tests is the action's repository contract. Compose is not required or accepted by the action. @@ -42,10 +42,10 @@ ROOT - Compose topology + Caller lifecycle - compose.e2e.yml - Build, health, and service graph + your workflow + Start · readiness · logs · cleanup @@ -77,10 +77,10 @@ SUT - System services + Ready target URL - sut/ - Example only; use real services + sut.baseUrl + Local or remote; caller-managed The directory name and manifest path stay stable; implementation details remain application-owned. diff --git a/e2e/ci.yml b/e2e/ci.yml index b9ade55..4ae98f4 100644 --- a/e2e/ci.yml +++ b/e2e/ci.yml @@ -7,7 +7,6 @@ playwright: config: playwright.config.ts sut: - composeFile: compose.e2e.yml baseUrl: http://127.0.0.1:4173 execution: From 94c9b827e861b71a4d62d0610cf92f87423d73cc Mon Sep 17 00:00:00 2001 From: omermorad Date: Tue, 29 Sep 2026 08:56:32 +0300 Subject: [PATCH 3/4] docs: use major-version action references --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bbf710d..866d591 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The template owns repository policy and application lifecycle: triggers, permiss ## Use the action -Check out the caller repository and make the target ready before invoking the root action. The URL-only contract is unreleased: `v0.2.0` still requires Compose. Until a compatible release exists, use the implementation commit pinned by the [starter recipe PR](https://github.com/codotech/playwright-e2e-starter/pull/1), or the implementation branch while evaluating this change: +Check out the caller repository and make the target ready before invoking the root action. Use a major-version tag. This URL-only recipe requires a compatible release: currently `v0` points to `v0.2.0`, which still requires Compose. The [starter recipe PR](https://github.com/codotech/playwright-e2e-starter/pull/1) depends on releasing this change first: ```yaml permissions: @@ -25,12 +25,12 @@ steps: - uses: actions/checkout@v7 - id: e2e - uses: codotech/playwright-e2e@fix/optional-compose-remote-sut + uses: codotech/playwright-e2e@v0 with: profile: pull-request ``` -Pin a reviewed full commit SHA for reproducible adoption. Do not use `@v0.2.0` with this recipe. +Use version tags such as `@v0` or `@v1`, not commit SHAs or feature branches. Verify that the selected release line includes the URL-only contract before adopting this recipe. ### Inputs From 6092d3012eead5b843076d57f02cbdf2814db6f1 Mon Sep 17 00:00:00 2001 From: omermorad Date: Tue, 29 Sep 2026 09:10:47 +0300 Subject: [PATCH 4/4] docs: show caller-owned local and remote targets in E2E flow --- README.md | 2 +- docs/diagrams/e2e-flow.svg | 44 +++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 866d591..a739433 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A portable GitHub Action for running Playwright against an already running syste Your workflow starts the application and passes its base URL. The action owns the test runner, filtering, reports, traces, portable artifacts, and a fail-closed result. It never starts, collects logs from, or stops application services. -![E2E flow through runner reuse, testing a caller-provided URL, reports, and the required gate](docs/diagrams/e2e-flow.svg) +![E2E flow: a caller-started local system or existing remote target supplies a base URL; codotech/playwright-e2e@v0 resolves the runner, runs tests, and publishes reports for the caller's gate](docs/diagrams/e2e-flow.svg) ## Start from the template diff --git a/docs/diagrams/e2e-flow.svg b/docs/diagrams/e2e-flow.svg index 558b33c..86eddf6 100644 --- a/docs/diagrams/e2e-flow.svg +++ b/docs/diagrams/e2e-flow.svg @@ -1,6 +1,6 @@ Playwright E2E delivery flow - The caller makes its application ready. The action validates configuration, reuses or builds a runner, tests the supplied base URL, and publishes reports for the E2E gate. Application lifecycle remains outside the action. + The caller supplies a ready base URL from either a caller-started local system or an existing remote environment. codotech/playwright-e2e@v0 validates configuration, reuses or builds a runner, tests the supplied URL, and publishes reports. The caller owns application lifecycle and enforces the E2E gate. @@ -28,7 +28,7 @@ E2E DELIVERY FLOW - Caller makes the application ready; the action owns runner execution and reports + codotech/playwright-e2e@v0 owns the runner and reports; the caller supplies a ready local or remote URL @@ -79,8 +79,8 @@ Build image - - + + @@ -88,12 +88,40 @@ exact image for this run - + + + CALLER CHOOSES ONE TARGET + + + + Caller-started system + Compose or other startup + readiness + + + OR + + + + Existing remote target + No service startup in this workflow + + + + + + + + Ready base URL + sut.baseUrl in e2e/ci.yml + Reachable by the runner + + + Use target URL - already ready · caller-owned + no SUT lifecycle in action @@ -121,8 +149,8 @@ E2E Gate - pass or fail closed + caller enforces the result - Every path produces the same result contract and retained debugging artifacts. + The caller owns any service startup, readiness, service logs, and cleanup. The action tests the supplied URL.