diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 06f0958..883e155 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -45,13 +45,25 @@ jobs: name: E2E Gate runs-on: ubuntu-latest timeout-minutes: 45 + env: + COMPOSE_PROJECT_NAME: e2e-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: Check out repository uses: actions/checkout@v7 + - id: sut + name: Start example SUT and wait for readiness + continue-on-error: true + shell: bash + run: | + mkdir -p sut-logs + docker compose -f compose.e2e.yml up --detach --build --wait 2>&1 | tee sut-logs/startup.log + - id: e2e name: Run E2E + if: steps.sut.outcome == 'success' continue-on-error: true + # Requires a v0 release containing the URL-only contract. uses: codotech/playwright-e2e@v0 with: profile: >- @@ -63,11 +75,44 @@ jobs: ${{ github.event_name == 'workflow_dispatch' && inputs.label-match != 'profile' && inputs.label-match || '' }} + - id: sut-logs + name: Capture example SUT logs + if: always() && steps.sut.outcome != 'skipped' + continue-on-error: true + shell: bash + run: | + mkdir -p sut-logs + docker compose -f compose.e2e.yml logs --no-color --timestamps > sut-logs/services.log 2>&1 + + - id: sut-cleanup + name: Stop example SUT + if: always() && steps.sut.outcome != 'skipped' + continue-on-error: true + shell: bash + run: | + mkdir -p sut-logs + docker compose -f compose.e2e.yml down --volumes --remove-orphans > sut-logs/teardown.log 2>&1 + + - id: sut-artifact + name: Upload example SUT logs + if: always() && steps.sut.outcome != 'skipped' + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: sut-logs + path: sut-logs/ + if-no-files-found: error + retention-days: 10 + - name: Update pull request if: always() && github.event_name == 'pull_request' uses: actions/github-script@v8 env: E2E_RESULT: ${{ steps.e2e.outputs.result }} + E2E_SUT_SETUP: ${{ steps.sut.outcome }} + E2E_SUT_CLEANUP: ${{ steps.sut-cleanup.outcome }} + E2E_SUT_LOGS: ${{ steps.sut-logs.outcome }} + E2E_SUT_ARTIFACT: ${{ steps.sut-artifact.outcome }} E2E_PRIMARY_FAILURE: ${{ steps.e2e.outputs.primary-failure }} E2E_TOTAL: ${{ steps.e2e.outputs.total }} E2E_PASSED: ${{ steps.e2e.outputs.passed }} @@ -87,12 +132,18 @@ jobs: const marker = ''; const value = (name, fallback = 'n/a') => process.env[name] || fallback; const list = (name) => value(name).split('\n').filter(Boolean).join(', '); - const result = value('E2E_RESULT', 'infrastructure-error'); + const actionResult = value('E2E_RESULT', 'infrastructure-error'); + const lifecycleFailures = ['E2E_SUT_SETUP', 'E2E_SUT_CLEANUP', 'E2E_SUT_LOGS', 'E2E_SUT_ARTIFACT'] + .filter((name) => value(name) !== 'success'); + const result = actionResult === 'failed' + ? 'failed' + : lifecycleFailures.length ? 'infrastructure-error' : actionResult; const icon = result === 'passed' ? '✅' : result === 'failed' ? '❌' : '⚠️'; const cache = value('E2E_RUNNER_CACHE_HIT', 'false') === 'true' ? 'reused' : 'built'; const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const rows = [ ['Gate', `${icon} ${result}`], + ['Caller SUT', `setup: ${value('E2E_SUT_SETUP')} · cleanup: ${value('E2E_SUT_CLEANUP')} · artifact: sut-logs`], ['Tests', `${value('E2E_PASSED', '0')} passed · ${value('E2E_FAILED', '0')} failed · ${value('E2E_SKIPPED', '0')} skipped · ${value('E2E_TOTAL', '0')} total`], ['Profile', value('E2E_PROFILE')], ['Projects', list('E2E_PROJECTS')], @@ -102,8 +153,10 @@ jobs: ['Report image', `${value('E2E_REPORT_IMAGE')} · ${value('E2E_REPORT_ARTIFACT')}`], ['Run', `[Open workflow run](${runUrl})`], ]; - const failure = process.env.E2E_PRIMARY_FAILURE - ? `\n> ${process.env.E2E_PRIMARY_FAILURE}\n` + const failureMessage = process.env.E2E_PRIMARY_FAILURE || + (lifecycleFailures.length ? `Caller lifecycle did not succeed: ${lifecycleFailures.join(', ')}` : ''); + const failure = failureMessage + ? `\n> ${failureMessage}\n` : ''; const body = [ marker, @@ -140,7 +193,13 @@ jobs: shell: bash env: E2E_STEP_OUTCOME: ${{ steps.e2e.outcome }} + SUT_SETUP_OUTCOME: ${{ steps.sut.outcome }} + SUT_LOGS_OUTCOME: ${{ steps.sut-logs.outcome }} + SUT_CLEANUP_OUTCOME: ${{ steps.sut-cleanup.outcome }} + SUT_ARTIFACT_OUTCOME: ${{ steps.sut-artifact.outcome }} run: | - if [[ "$E2E_STEP_OUTCOME" != "success" ]]; then - exit 1 - fi + for outcome in "$E2E_STEP_OUTCOME" "$SUT_SETUP_OUTCOME" "$SUT_LOGS_OUTCOME" "$SUT_CLEANUP_OUTCOME" "$SUT_ARTIFACT_OUTCOME"; do + if [[ "$outcome" != "success" ]]; then + exit 1 + fi + done diff --git a/README.md b/README.md index 8262614..86eede6 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,47 @@ # Playwright E2E Starter -A ready-to-run repository template for testing a Dockerized system with [codotech/playwright-e2e](https://github.com/codotech/playwright-e2e). +A repository template that starts its own application and tests its base URL with [codotech/playwright-e2e](https://github.com/codotech/playwright-e2e). -The included echo service is intentionally small. It proves the complete path through Docker Compose, API and browser projects, Playwright tags, runner reuse, traces, reports, portable images, pull-request comments, and a required E2E gate. +The included echo service is intentionally small. The caller workflow manages Docker Compose startup, readiness, service logs, and cleanup. The action only owns Playwright execution, reports, and portable images. You can also target an existing environment without starting any services. -![Starter flow from a change through repository workflow policy, the portable action, Dockerized SUT and Playwright runner, reports, and the required gate](docs/diagrams/starter-flow.svg) +![Caller-started system or existing remote environment provides a ready base URL to codotech/playwright-e2e@v0, which runs tests and produces reports for the caller's E2E gate](docs/diagrams/starter-flow.svg) ## Create your repository 1. Select **Use this template** on GitHub. -2. Keep the `e2e/` directory name and the root `compose.e2e.yml` contract. -3. Replace `sut/` and `compose.e2e.yml` with the services your tests need. +2. Keep the `e2e/` directory name. +3. Replace `sut/` and `compose.e2e.yml` with the services your tests need and adapt the workflow's lifecycle steps, or follow the existing-environment recipe below. 4. Set the SUT URL and execution profiles in `e2e/ci.yml`. 5. Replace the example tests under `e2e/tests/`. 6. Define projects, reporters, workers, retries, and trace policy in `e2e/playwright.config.ts`. 7. Require the **E2E Gate** check before merging. -The workflow follows the latest compatible 0.x action through `@v0`. Use `@v0.2.0` when an exact release is required. +The workflow uses the major-version tag `codotech/playwright-e2e@v0`, not a commit SHA. This recipe requires a release containing the URL-only contract. Currently `v0` points to `v0.2.0`, which still requires action-managed Compose, so this starter change must wait for the compatible release. See [.github/workflows/e2e.yml](.github/workflows/e2e.yml). ## What belongs where -![Starter repository layout showing workflow policy, Compose topology, replaceable services, and the stable E2E directory contract](docs/diagrams/repository-layout.svg) +![Starter repository layout showing workflow policy and the E2E directory, with Compose and service files marked as optional examples that remote targets do not need](docs/diagrams/repository-layout.svg) -The application repository owns the workflow policy and SUT topology. The external action owns the execution engine. Tests and their dependencies stay in `e2e/`, so any material runner change produces a new content hash and runner image. Service-only or profile-only changes reuse the validated runner while still rebuilding and testing the SUT. +The application repository owns workflow policy, SUT topology, readiness, and cleanup. The external action owns the test execution engine and accepts only the application's URL. Tests and their dependencies stay in `e2e/`, so any material runner change produces a new content hash and runner image. Service-only or profile-only changes reuse the validated runner while the caller independently manages the application. + +## Start the application in your workflow + +The supplied [.github/workflows/e2e.yml](.github/workflows/e2e.yml) runs this lifecycle: + +1. Check out the repository. +2. Start Compose with `docker compose -f compose.e2e.yml up --detach --build --wait`. +3. Invoke the action against the ready application. +4. Collect service logs and run `docker compose -f compose.e2e.yml down --volumes --remove-orphans` in caller-owned steps with `if: always()`. +5. Upload service logs separately, update the PR comment, and enforce the final gate. + +The action never reads a Compose file or manages these services. Its application configuration is only: + +```yaml +sut: + baseUrl: http://127.0.0.1:4173 +``` + +When migrating from an earlier recipe, remove `sut.composeFile` from `e2e/ci.yml`. That option is no longer supported; move its lifecycle responsibilities into your workflow. ## Configure CI selection @@ -67,16 +86,45 @@ Always run the final cleanup command, including after a failed test. Run the sta pnpm --dir e2e test:static ``` +### Target an existing environment + +Replace the `sut` section in `e2e/ci.yml` with: + +```yaml +sut: + baseUrl: https://staging.example.com +``` + +Adapt [.github/workflows/e2e.yml](.github/workflows/e2e.yml): + +1. Remove the Compose startup, service-log collection, service-log upload, and cleanup steps. +2. Remove `if: steps.sut.outcome == 'success'` from **Run E2E**; otherwise deleting startup also skips the action. +3. Remove lifecycle outcome references from the final gate and PR comment, retaining the action-result check. +4. Remove the unused `COMPOSE_PROJECT_NAME` environment setting. + +The example `sut/` directory and root Compose file are no longer needed. Keep the runner Dockerfile, entrypoint, Playwright configuration, reporters, and workflow result handling. + +After installing the E2E dependencies, run locally with: + +```bash +BASE_URL=https://staging.example.com pnpm --dir e2e test +``` + +CI passes the configured URL into the test container. The action does not start, collect service logs from, or tear down the target. Docker is still used for the runner and report images. The target must already be reachable and ready. Tests can modify data, so select an authorized test environment. + +Do not put credentials in the URL. Arbitrary workflow environment variables, including API tokens, are not currently forwarded into the test container. + ## CI behavior The supplied workflow runs for pull requests, pushes to `main`, and manual dispatches. It: - cancels stale runs for the same pull request or ref; -- checks out the application before invoking the action; +- checks out the application and starts Compose with readiness checks before invoking the action; - uses `pull-request`, `main`, or the manually selected profile; - creates or updates one E2E report comment on pull requests; - publishes the test results, Playwright HTML report, traces, runner image, and report image; -- preserves the failing result after the report comment is written. +- collects and uploads service logs separately and attempts Compose cleanup even after failure; +- preserves application lifecycle failures and the failing action result after the report comment is written. The action needs no inherited secrets. Add repository or environment secrets to the workflow only when your SUT requires them. @@ -84,11 +132,12 @@ The action needs no inherited secrets. Add repository or environment secrets to | Artifact | Purpose | | --- | --- | -| `e2e-results` | Final result, CTRF data, HTML report, traces, screenshots, video, and service logs | +| `e2e-results` | Final result, CTRF data, HTML report, traces, screenshots, and video | +| `sut-logs` | Service logs and teardown output collected by the caller workflow, not the action | | `e2e-runner-image` | Portable archive of the exact Playwright runner used | | `e2e-report-image` | Portable image serving the HTML report on port 8080 | -Artifacts are retained for the number of days configured in `e2e/ci.yml`. The image archives are not published to a registry. +Action artifacts use the retention configured in `e2e/ci.yml`; the workflow configures service-log retention separately. The image archives are not published to a registry. ## Replace the example SUT @@ -100,11 +149,11 @@ The example service exposes `/health` and `/echo` on port 4173. When adapting th - keep teardown safe for repeated and cancelled runs; - avoid installing application dependencies directly on the GitHub host. -The action runs Compose with build and wait enabled, always captures logs, and removes containers, networks, and volumes after the suite. +The supplied workflow runs Compose with build and wait enabled, collects logs, and attempts to remove containers, networks, and volumes after the suite. These are application-owned steps; the action has no lifecycle hooks or Compose configuration. ## Upgrade the action -Review changes in the core repository, then update the major version in `.github/workflows/e2e.yml`. The current line uses `@v0`; move it to `@v1` only when adopting a future 1.x release. +Use major-version tags in `.github/workflows/e2e.yml`, such as `@v0` or `@v1`, rather than commit SHAs or feature branches. Review breaking changes before changing the major version. Ensure the selected release line includes the URL-only contract before merging this recipe. ## License diff --git a/docs/diagrams/repository-layout.svg b/docs/diagrams/repository-layout.svg index c76317f..af24f74 100644 --- a/docs/diagrams/repository-layout.svg +++ b/docs/diagrams/repository-layout.svg @@ -1,6 +1,6 @@ Starter repository layout - The starter contains repository workflow policy, a root Compose topology, replaceable system services, and an e2e directory containing configuration, runner files, and tests. + The starter contains repository workflow policy and an e2e directory with configuration, runner files, and tests. The compose.e2e.yml file and sut directory are optional local-system examples, not action requirements. An existing remote target needs neither. @@ -26,7 +26,7 @@ STARTER REPOSITORY LAYOUT - Four replaceable areas, one stable E2E contract + Workflow policy and E2E files, plus optional local-system examples @@ -35,9 +35,9 @@ - - - + + + @@ -47,27 +47,27 @@ .github/workflows/ e2e.yml - triggers · comment · gate + lifecycle · comment · gate - - TOPOLOGY + + OPTIONAL EXAMPLE Compose compose.e2e.yml - build · health · network + omit for a remote target - - SUT + + OPTIONAL EXAMPLE Services sut/ - replace with your system + omit for a remote target @@ -91,5 +91,5 @@ replace examples with your suites - Keep the paths stable; replace the example implementation behind them. + The action consumes e2e/; Compose and sut/ are optional starter examples, not action requirements. diff --git a/docs/diagrams/starter-flow.svg b/docs/diagrams/starter-flow.svg index e1b6b79..86df06d 100644 --- a/docs/diagrams/starter-flow.svg +++ b/docs/diagrams/starter-flow.svg @@ -1,6 +1,6 @@ Starter repository E2E flow - A change activates repository workflow policy, invokes the portable E2E action, runs a Dockerized system against the Playwright runner, publishes results, and enforces the E2E gate. + The repository workflow chooses either a caller-started system or an existing remote environment. Both provide a ready base URL to codotech/playwright-e2e@v0, which runs Playwright and publishes reports. The caller enforces the E2E gate and owns any service startup, logs, and cleanup. @@ -26,67 +26,65 @@ - FROM CHANGE TO E2E GATE - The repository owns policy and topology; the action supplies the portable execution engine + FROM READY TARGET TO E2E GATE + On a change, the repository workflow chooses the target; the action owns tests and reports - - - INPUT - Change opened + + + CALLER-STARTED + Local system + Compose or your own startup steps + Caller waits until ready - - - - - REPO POLICY - Repository workflow - trigger · permission · concurrency - - + OR - - - E2E ENGINE - Portable action - codotech/playwright-e2e@v0 + + + ALREADY RUNNING + Remote environment + Staging or another reachable target + No service startup in this workflow - - + + - - Dockerized SUT - build · health · service graph - owned by the application + + Ready base URL + sut.baseUrl in e2e/ci.yml + Reachable by the runner + + - - Playwright runner - projects · tags · workers - reused by content hash + + + E2E ENGINE + codotech/playwright-e2e@v0 + Playwright runner · projects · tags · workers + No application startup or teardown - - + - - Results + report - traces · logs · portable images + + Results + report + traces · portable images - + - - E2E Gate - required before merge + + E2E Gate + Caller enforces result - Replace the example SUT and tests; keep the same action contract. + For caller-started services, the caller also collects logs and cleans up. The action only receives their URL. diff --git a/e2e/ci.yml b/e2e/ci.yml index b9ade55..4ae98f4 100644 --- a/e2e/ci.yml +++ b/e2e/ci.yml @@ -7,7 +7,6 @@ playwright: config: playwright.config.ts sut: - composeFile: compose.e2e.yml baseUrl: http://127.0.0.1:4173 execution: