From 12d3f5a771dd83e48e4b5739316b94e5764ef2a4 Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Fri, 23 Oct 2026 20:42:33 +0200 Subject: [PATCH 1/2] fix(ci): each uv job keeps its own cache, Playwright skips its apt step, and grype's database is cached All backend jobs shared one setup-uv key, so the first job to finish (bandit only) saved a 3 MB cache and the others downloaded CPython and every wheel on each run; each job now saves its own cache with the managed Python. The Playwright browser cache follows the browser builds in playwright-core's browsers.json instead of the whole package-lock.json, and the install-deps step goes: on the runner image it only adds fonts no test uses, and its apt download stalled two runs for more than 6 minutes. The SBOM scan restores grype's vulnerability database from a cache keyed on the checksum of the newest database, instead of spending about 80 s downloading and unpacking it in each job. --- .github/workflows/ci.yml | 23 ++++++++++++++++++----- .github/workflows/sbom-compliance.yml | 16 ++++++++++++++++ docs/operations/cicd.md | 6 +++--- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1aeb83b..6868c7bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,10 +89,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Each job keeps its own uv cache: they install different groups, and a shared key keeps whatever the first job to finish saved - name: Set up uv uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true + cache-suffix: ${{ github.job }} + cache-python: true cache-dependency-glob: "backend/uv.lock" - name: Install Python and dependencies @@ -118,6 +121,8 @@ jobs: uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true + cache-suffix: ${{ github.job }} + cache-python: true cache-dependency-glob: "backend/uv.lock" - name: Install Python and dependencies @@ -145,6 +150,8 @@ jobs: uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true + cache-suffix: ${{ github.job }} + cache-python: true cache-dependency-glob: "backend/uv.lock" - name: Install Python and dependencies @@ -170,6 +177,7 @@ jobs: uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true + cache-suffix: ${{ github.job }} cache-dependency-glob: "backend/uv.lock" - name: Run bandit @@ -226,6 +234,8 @@ jobs: uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true + cache-suffix: ${{ github.job }} + cache-python: true cache-dependency-glob: "backend/uv.lock" - name: Install backend dependencies @@ -273,6 +283,7 @@ jobs: uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true + cache-suffix: ${{ github.job }} - name: Install backend dependencies run: cd backend && uv sync --frozen --no-dev --no-install-project --no-build @@ -337,6 +348,8 @@ jobs: uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true + cache-suffix: ${{ github.job }} + cache-python: true cache-dependency-glob: "backend/uv.lock" - name: Install Python dependencies @@ -378,6 +391,8 @@ jobs: uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: enable-cache: true + cache-suffix: ${{ github.job }} + cache-python: true cache-dependency-glob: "backend/uv.lock" - name: Install Python dependencies @@ -652,12 +667,10 @@ jobs: id: playwright-cache with: path: ~/.cache/ms-playwright - key: playwright-${{ runner.os }}-${{ hashFiles('frontend/package-lock.json') }} - - - name: Install Playwright system dependencies - working-directory: frontend - run: npm run test:e2e:deps + # The browser builds this Playwright downloads, so only a new build invalidates the cache + key: playwright-${{ runner.os }}-${{ hashFiles('frontend/node_modules/playwright-core/browsers.json') }} + # No install-deps: the runner image has Chromium's libraries, and it only added fonts no test uses - name: Install Playwright browsers if: steps.playwright-cache.outputs.cache-hit != 'true' working-directory: frontend diff --git a/.github/workflows/sbom-compliance.yml b/.github/workflows/sbom-compliance.yml index 943d0a59..76efba9b 100644 --- a/.github/workflows/sbom-compliance.yml +++ b/.github/workflows/sbom-compliance.yml @@ -47,9 +47,25 @@ jobs: output-file: ${{ matrix.component }}-sbom.spdx.json upload-artifact: false + # The newest database's checksum keys the cache, so grype downloads only when it publishes a new one + - name: Look up grype's newest vulnerability database + id: grype-db + run: | + checksum=$(curl -sf https://grype.anchore.io/databases/v6/latest.json | jq -er .checksum) + echo "checksum=$checksum" >> "$GITHUB_OUTPUT" + + - name: Cache grype's vulnerability database + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/grype-db + key: grype-db-${{ steps.grype-db.outputs.checksum }} + restore-keys: grype-db- + - name: Scan SBOM for Vulnerabilities uses: anchore/scan-action@27805bf3b4e84b4a5c980df22ed233c00390a439 # v7.4.2 id: scan + env: + GRYPE_DB_CACHE_DIR: ${{ runner.temp }}/grype-db with: sbom: ${{ matrix.component }}-sbom.spdx.json fail-build: false # Don't block builds yet, just report diff --git a/docs/operations/cicd.md b/docs/operations/cicd.md index c4272ac8..7b9b9d8c 100644 --- a/docs/operations/cicd.md +++ b/docs/operations/cicd.md @@ -34,7 +34,7 @@ graph LR | Label Sync | label-sync.yml | push and PRs to `main` touching the label files or issue forms; Mondays at 03:00; manual | | Dependabot Auto-merge | dependabot-auto-merge.yml | Dependabot's PRs to `main` | -Besides the tests, CI runs one job per check. Ruff runs `ruff check`. MyPy runs `mypy --strict` with `SECRET_KEY` from the `TEST_SECRET_KEY` secret. Grimp runs `python -m scripts.check_orphan_modules`, which fails on any module nothing imports except `app.main` and `__init__.py` files that hold only a docstring. Security Scanning runs `uvx --no-build --python 3.14 bandit==1.9.4 -r backend/ -x backend/tests/,backend/.venv/ -ll`. Bandit parses with the Python it runs on, so on anything older it skips the files that use 3.14 syntax such as `except A, B:` without failing. The frontend job runs ESLint, Stylelint, the Prettier check, `svelte-check --fail-on-warnings` and a production build. The SBOM workflow generates SPDX SBOMs for backend and frontend with `anchore/sbom-action`, scans them with Grype through `anchore/scan-action` (cutoff `high`, `fail-build: false`) and uploads SARIF to the Security tab. +Besides the tests, CI runs one job per check. Ruff runs `ruff check`. MyPy runs `mypy --strict` with `SECRET_KEY` from the `TEST_SECRET_KEY` secret. Grimp runs `python -m scripts.check_orphan_modules`, which fails on any module nothing imports except `app.main` and `__init__.py` files that hold only a docstring. Security Scanning runs `uvx --no-build --python 3.14 bandit==1.9.4 -r backend/ -x backend/tests/,backend/.venv/ -ll`. Bandit parses with the Python it runs on, so on anything older it skips the files that use 3.14 syntax such as `except A, B:` without failing. The frontend job runs ESLint, Stylelint, the Prettier check, `svelte-check --fail-on-warnings` and a production build. The SBOM workflow generates SPDX SBOMs for backend and frontend with `anchore/sbom-action`, scans them with Grype through `anchore/scan-action` (cutoff `high`, `fail-build: false`), with Grype's vulnerability database cached under the checksum of the newest one Grype has published and uploads SARIF to the Security tab. The docs job installs the backend dependencies, runs `python -m scripts.export_openapi` and builds with `mkdocs build --strict`. On pushes to `main`, Deploy Docs publishes the result to GitHub Pages and then deletes the `github-pages` deployments before it, so the repository lists only the live one. @@ -84,7 +84,7 @@ Each E2E job gets its own runner with k3s and the full compose stack, pulling th `e2e-ready` copies k3s's kubeconfig for the runner's kubectl, waits for the node and then for the image pre-pull. The worker creates the `integr8scode` namespace, its NetworkPolicy and its ResourceQuota itself when it starts. It copies config.test.toml over config.toml, waits for the background infra start, runs `docker compose up -d --no-build` and polls `/api/v1/health/live` (and the frontend, when asked). It then waits until the five Kafka consumer groups report `Stable` in `kafka-consumer-groups --describe --state`. On a fresh Kafka most of them start at the newest message, so a request sent while they were still joining would never be read, and an execution test would time out. -The backend E2E job runs `pytest tests/e2e` inside the backend container after `uv sync --group test --no-dev --frozen --no-install-project --no-build`. The frontend E2E job runs as two shards. It caches Playwright browsers by package-lock.json, always runs `npm run test:e2e:deps`, runs `npm run test:e2e:browsers` on a cache miss, and runs `npm run test:e2e -- --shard=N/2`. On failure both jobs upload compose logs, the k3s container's log and `kubectl describe nodes`; the backend one adds `kubectl get events`. +The backend E2E job runs `pytest tests/e2e` inside the backend container after `uv sync --group test --no-dev --frozen --no-install-project --no-build`. The frontend E2E job runs as two shards. It caches Playwright browsers by the browser builds in playwright-core's `browsers.json`, runs `npm run test:e2e:browsers` on a cache miss, and skips `npm run test:e2e:deps`, since the runner image already has Chromium's libraries and that step only added fonts. It then runs `npm run test:e2e -- --shard=N/2`. On failure both jobs upload compose logs, the k3s container's log and `kubectl describe nodes`; the backend one adds `kubectl get events`. ## Docker Scan & Promote @@ -121,7 +121,7 @@ Compose falls back to public defaults for `SECRET_KEY`, the MongoDB and Redis cr Every action, the composite ones included, is pinned to a full commit SHA with the version in a comment, for example `actions/checkout@3d3c42e5... # v7.0.1`. The Trivy action is pinned to the commit of v0.36.0 because the older tags were deleted upstream. -Tools are pinned exactly: mkdocs 1.6.1 with material 9.7.7, mermaid2 1.2.3 and swagger-ui-tag 0.8.1; bandit 1.9.4; Trivy v0.75.0. Python installs use `uv sync --frozen` with `--no-build`, so only wheels from the lockfile get installed, and `--no-install-project`. `uvx` calls also pass `--no-build`. npm installs use `npm ci --ignore-scripts`, in CI and in the frontend image. +Tools are pinned exactly: mkdocs 1.6.1 with material 9.7.7, mermaid2 1.2.3 and swagger-ui-tag 0.8.1; bandit 1.9.4; Trivy v0.75.0. Python installs use `uv sync --frozen` with `--no-build`, so only wheels from the lockfile get installed, and `--no-install-project`. Each uv job keeps its own setup-uv cache, keyed on `backend/uv.lock` with the job's name as suffix and the managed Python included, since the jobs install different groups. `uvx` calls also pass `--no-build`. npm installs use `npm ci --ignore-scripts`, in CI and in the frontend image. k3s is the `rancher/k3s:v1.37.1-k3s1` image, pinned by digest in `K3S_IMAGE` in ci.yml. From ea87d800d242f765c9cc4b5a1713dea3ebae3f6c Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Fri, 23 Oct 2026 20:51:51 +0200 Subject: [PATCH 2/2] fix(ci): E2E pulls and starts only the app and its dependencies, not the observability services No E2E test uses Grafana, Loki, OpenSearch, VictoriaMetrics, Jaeger or the collectors. Since #456 the E2E jobs pulled their 4 GB of images too: the pull grew from 22 s to 49 s and the stack waited about 25 s on it, plus 13 s more startup. --- .github/actions/e2e-boot/action.yml | 2 +- .github/actions/e2e-ready/action.yml | 3 ++- docs/operations/cicd.md | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/actions/e2e-boot/action.yml b/.github/actions/e2e-boot/action.yml index 1a84cfb8..f53e050a 100644 --- a/.github/actions/e2e-boot/action.yml +++ b/.github/actions/e2e-boot/action.yml @@ -26,7 +26,7 @@ runs: run: | cp .env.example .env nohup bash -c ' - IMAGE_TAG='"$IMAGE_TAG"' docker compose pull --quiet 2>&1 + IMAGE_TAG='"$IMAGE_TAG"' docker compose pull --quiet --include-deps frontend worker mongo-backup 2>&1 echo "--- pull done, starting infra ---" docker compose up -d --no-build \ mongo redis kafka 2>&1 diff --git a/.github/actions/e2e-ready/action.yml b/.github/actions/e2e-ready/action.yml index 2d7c1ee3..d0b641d3 100644 --- a/.github/actions/e2e-ready/action.yml +++ b/.github/actions/e2e-ready/action.yml @@ -47,7 +47,8 @@ runs: shell: bash env: IMAGE_TAG: ${{ inputs.image-tag }} - run: docker compose up -d --no-build + # The app and what it depends on; no test uses the observability services, and their images are about 4 GB + run: docker compose up -d --no-build frontend worker mongo-backup - name: Wait for services shell: bash diff --git a/docs/operations/cicd.md b/docs/operations/cicd.md index 7b9b9d8c..2a1ed711 100644 --- a/docs/operations/cicd.md +++ b/docs/operations/cicd.md @@ -80,9 +80,9 @@ The pushes run in parallel and each failure is reported on its own. Fork PRs can Each E2E job gets its own runner with k3s and the full compose stack, pulling the images it just built. Two composite actions set that up, and the frontend job installs Node and Playwright between them so that work overlaps with k3s booting. -`e2e-boot` logs in to GHCR, copies `.env.example` to `.env`, starts `docker compose pull` and then `mongo redis kafka` in the background, and starts k3s in a privileged container named `k3s`, from the `rancher/k3s` image, with its network policy controller off. No E2E test checks policy enforcement, and the worker still creates the NetworkPolicy object. The container publishes 6443 and mounts the runner's `/etc/rancher/k3s`, so the kubeconfig lands where the worker reads it, the same as with a host install. k3s runs in a container because its pods then get their network interfaces inside it. With k3s on the runner itself, every executor pod changed the runner's network, and Chromium aborted Playwright navigations in flight with `net::ERR_NETWORK_CHANGED`. Also in the background, once k3s has written its kubeconfig, it pre-pulls `python:3.14-slim`, the default runtime and the only one the tests execute, into k3s with `docker exec k3s crictl`, so the first executor pods don't wait on an image pull. Nothing execs into the container before that kubeconfig exists: at startup k3s moves every process out of the container's root cgroup and then enables the cgroup controllers there, and a `docker exec` landing in between made that fail with "device or resource busy", so the kubelet exited and the node never came up. +`e2e-boot` logs in to GHCR, copies `.env.example` to `.env`, pulls the images of the frontend, the worker, the backup and their dependencies (no test uses the observability services), then starts `mongo redis kafka`, all in the background, and starts k3s in a privileged container named `k3s`, from the `rancher/k3s` image, with its network policy controller off. No E2E test checks policy enforcement, and the worker still creates the NetworkPolicy object. The container publishes 6443 and mounts the runner's `/etc/rancher/k3s`, so the kubeconfig lands where the worker reads it, the same as with a host install. k3s runs in a container because its pods then get their network interfaces inside it. With k3s on the runner itself, every executor pod changed the runner's network, and Chromium aborted Playwright navigations in flight with `net::ERR_NETWORK_CHANGED`. Also in the background, once k3s has written its kubeconfig, it pre-pulls `python:3.14-slim`, the default runtime and the only one the tests execute, into k3s with `docker exec k3s crictl`, so the first executor pods don't wait on an image pull. Nothing execs into the container before that kubeconfig exists: at startup k3s moves every process out of the container's root cgroup and then enables the cgroup controllers there, and a `docker exec` landing in between made that fail with "device or resource busy", so the kubelet exited and the node never came up. -`e2e-ready` copies k3s's kubeconfig for the runner's kubectl, waits for the node and then for the image pre-pull. The worker creates the `integr8scode` namespace, its NetworkPolicy and its ResourceQuota itself when it starts. It copies config.test.toml over config.toml, waits for the background infra start, runs `docker compose up -d --no-build` and polls `/api/v1/health/live` (and the frontend, when asked). It then waits until the five Kafka consumer groups report `Stable` in `kafka-consumer-groups --describe --state`. On a fresh Kafka most of them start at the newest message, so a request sent while they were still joining would never be read, and an execution test would time out. +`e2e-ready` copies k3s's kubeconfig for the runner's kubectl, waits for the node and then for the image pre-pull. The worker creates the `integr8scode` namespace, its NetworkPolicy and its ResourceQuota itself when it starts. It adds `BACKEND_CONFIG=config.test.toml` to `.env`, waits for the background infra start, runs `docker compose up -d --no-build frontend worker mongo-backup` and polls `/api/v1/health/live` (and the frontend, when asked). It then waits until the five Kafka consumer groups report `Stable` in `kafka-consumer-groups --describe --state`. On a fresh Kafka most of them start at the newest message, so a request sent while they were still joining would never be read, and an execution test would time out. The backend E2E job runs `pytest tests/e2e` inside the backend container after `uv sync --group test --no-dev --frozen --no-install-project --no-build`. The frontend E2E job runs as two shards. It caches Playwright browsers by the browser builds in playwright-core's `browsers.json`, runs `npm run test:e2e:browsers` on a cache miss, and skips `npm run test:e2e:deps`, since the runner image already has Chromium's libraries and that step only added fonts. It then runs `npm run test:e2e -- --shard=N/2`. On failure both jobs upload compose logs, the k3s container's log and `kubectl describe nodes`; the backend one adds `kubectl get events`.