test(cli-e2e): add live e2e suite covering the CLI command matrix

.github/scripts/sweep-live-projects.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Delete every staging project whose name starts with the given prefix (the live
+# e2e job's per-run prefix). Shared by the in-run retry sweep (called best-effort
+# with `|| true`) and the always() cleanup step (which propagates the exit code).
+#
+# Reads SUPABASE_ACCESS_TOKEN + CLI_E2E_API_URL from the environment. Exits
+# non-zero if any DELETE failed; a failed *listing* also exits non-zero (pipefail).
+set -o pipefail
+
+PREFIX="${1:?usage: sweep-live-projects.sh PREFIX}"
+: "${SUPABASE_ACCESS_TOKEN:?SUPABASE_ACCESS_TOKEN required}"
+: "${CLI_E2E_API_URL:?CLI_E2E_API_URL required}"
+
+# Capture the list in a var (not a pipe-to-while subshell) so a failed delete is
+# recorded in $failed; a failed listing aborts here via pipefail.
+refs=$(curl -fsS -H "Authorization: Bearer ${SUPABASE_ACCESS_TOKEN}" \
+  "${CLI_E2E_API_URL}/v1/projects" \
+  | jq -r --arg p "$PREFIX" '.[] | select(.name|startswith($p)) | .ref // .id')
+
+failed=0
+for ref in $refs; do
+  [ -n "$ref" ] || continue
+  echo "deleting leftover project $ref"
+  if ! curl -fsS -X DELETE -H "Authorization: Bearer ${SUPABASE_ACCESS_TOKEN}" \
+    "${CLI_E2E_API_URL}/v1/projects/${ref}" >/dev/null; then
+    echo "::error::failed to delete leftover project $ref"
+    failed=1
+  fi
+done
+exit "$failed"

.github/workflows/live-e2e.yml

@@ -0,0 +1,187 @@
+name: Live E2E
+
+# Live e2e suite (ADR-0013). Runs the real CLI against the real staging
+# Management API + Docker bundler, then invokes the deployed functions over HTTP.
+#
+# Non-blocking by construction: this is a standalone workflow, NOT part of the
+# required-checks set, and it never runs on the default PR path of test.yml.
+#
+# Triggers:
+#   - workflow_dispatch — manual run. The Actions UI branch picker selects the
+#     ref (github.ref), always a same-repo branch. We deliberately take NO
+#     free-form `ref` input: that would let a manual run check out arbitrary
+#     (e.g. external PR) code while the staging token is in the job env.
+#   - schedule (hourly) — exercises the `@beta` channel. `develop` is the default
+#     branch AND the beta release source, so a scheduled run checks it out and
+#     builds from source. The `gate` job skips the run unless the published
+#     `supabase@beta` version changed since the last green run (an actions/cache
+#     marker keyed on the version), so we only spend a staging project when there
+#     is actually a new beta to test.
+#
+# Secrets: workflow_dispatch and schedule both run on trusted same-repo refs, so
+# the staging token is never exposed to fork code.
+on:
+  workflow_dispatch:
+  schedule:
+    # Hourly, offset from other scheduled workflows. Cron timing is best-effort.
+    - cron: "23 * * * *"
+
+permissions:
+  contents: read
+
+jobs:
+  # Decide whether to run. Manual dispatch always runs. Scheduled runs only fire
+  # when the latest published `@beta` is newer than the last one we tested green.
+  gate:
+    name: Gate (newer @beta?)
+    runs-on: ubuntu-latest
+    outputs:
+      should_run: ${{ steps.decide.outputs.should_run }}
+      version: ${{ steps.ver.outputs.version }}
+    steps:
+      - name: Resolve latest @beta version
+        id: ver
+        run: |
+          set -euo pipefail
+          version="$(npm view supabase@beta version)"
+          # Validate the shape before it becomes a cache key (defense-in-depth
+          # against a garbage/poisoned registry value).
+          if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$ ]]; then
+            echo "::error::unexpected supabase@beta version '$version'"; exit 1
+          fi
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+
+      # Marker presence == "this beta already tested green". lookup-only so we
+      # download nothing; the marker is written by `finalize` after a green run.
+      - name: Check tested marker
+        id: cache
+        if: github.event_name == 'schedule'
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: .beta-marker
+          key: live-e2e-beta-${{ steps.ver.outputs.version }}
+          lookup-only: true
+
+      - name: Decide
+        id: decide
+        run: |
+          if [ "${{ github.event_name }}" != "schedule" ]; then
+            echo "manual dispatch -> run"
+            echo "should_run=true" >> "$GITHUB_OUTPUT"
+          elif [ "${{ steps.cache.outputs.cache-hit }}" = "true" ]; then
+            echo "beta ${{ steps.ver.outputs.version }} already tested green -> skip"
+            echo "should_run=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "new beta ${{ steps.ver.outputs.version }} -> run"
+            echo "should_run=true" >> "$GITHUB_OUTPUT"
+          fi
+
+  live-e2e:
+    needs: gate
+    if: needs.gate.outputs.should_run == 'true'
+    name: Live e2e (${{ matrix.target }})
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    # Serialize a target against itself across runs; go and ts-legacy run in
+    # parallel. Per-job-scoped project names mean this is mostly belt-and-braces.
+    concurrency:
+      group: live-e2e-${{ matrix.target }}-${{ github.ref }}
+      cancel-in-progress: true
+    strategy:
+      # Each target is an independent green/red signal.
+      fail-fast: false
+      matrix:
+        # go = source-of-truth Go binary; ts-legacy = the TS rewrite (shells out
+        # to Go for most commands). Authoring target is go; ts-legacy proves the
+        # shim matches. ts-next is a later axis.
+        target:
+          - go
+          - ts-legacy
+    # Non-secret config is job-level; the staging token is scoped to only the two
+    # steps that need it (run + cleanup) so build/checkout/docker never see it.
+    env:
+      CLI_E2E_MODE: live
+      CLI_E2E_TARGET_ENV: staging
+      CLI_E2E_API_URL: https://api.supabase.green
+      CLI_E2E_PROJECT_HOST: supabase.red
+      CLI_HARNESS_TARGET: ${{ matrix.target }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Setup
+        uses: ./.github/actions/setup
+
+      - name: Setup Go
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        with:
+          go-version-file: apps/cli-go/go.mod
+          cache-dependency-path: apps/cli-go/go.sum
+
+      # Build the Go binary for every target: `go` runs it directly and
+      # `ts-legacy` shells out to it for most commands.
+      - name: Build Go CLI
+        working-directory: apps/cli-go
+        run: go build -o supabase-go .
+
+      - name: Export Go binary path
+        run: echo "SUPABASE_GO_BINARY=${{ github.workspace }}/apps/cli-go/supabase-go" >> "$GITHUB_ENV"
+
+      # The ts-legacy harness runs the compiled supabase binary from apps/cli/dist.
+      - name: Build CLI
+        if: matrix.target == 'ts-legacy'
+        run: pnpm exec nx run supabase:build
+
+      # Docker is a hard requirement for the --use-docker bundler cell.
+      - name: Docker preflight
+        run: docker info
+
+      - name: Run live e2e (retry up to 3x)
+        env:
+          SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN }}
+        run: |
+          PREFIX="cli-e2e-live-${CLI_HARNESS_TARGET}-${GITHUB_RUN_ID}-"
+          # GitHub runs this step as `bash -e`; use `if cmd; then` (errexit-exempt)
+          # so a failing attempt does not abort the step before the retry.
+          for attempt in 1 2 3; do
+            echo "::group::live e2e attempt ${attempt}"
+            if [ "$attempt" -gt 1 ]; then
+              bash .github/scripts/sweep-live-projects.sh "$PREFIX" || true
+            fi
+            if pnpm --filter @supabase/cli-e2e test:e2e:live; then
+              echo "::endgroup::"
+              exit 0
+            fi
+            echo "::endgroup::"
+            echo "attempt ${attempt} failed"
+          done
+          exit 1
+
+      # Backstop: delete any project this job created that survived a crash.
+      # The script exits non-zero (failing this step) if any delete failed.
+      - name: Cleanup leftover projects
+        if: always()
+        env:
+          SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN }}
+        run: bash .github/scripts/sweep-live-projects.sh "cli-e2e-live-${CLI_HARNESS_TARGET}-${GITHUB_RUN_ID}-"
+
+  # Record that this beta tested green so the next scheduled run skips it. Needs
+  # the whole matrix: the marker is saved only if BOTH go and ts-legacy passed
+  # (a red leg leaves no marker, so the next hour re-runs the same beta).
+  finalize:
+    needs: [gate, live-e2e]
+    if: github.event_name == 'schedule' && needs.live-e2e.result == 'success'
+    name: Mark @beta tested
+    runs-on: ubuntu-latest
+    steps:
+      - name: Write marker
+        run: |
+          mkdir -p .beta-marker
+          echo "${{ needs.gate.outputs.version }}" > .beta-marker/version
+
+      - name: Save tested marker
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: .beta-marker
+          key: live-e2e-beta-${{ needs.gate.outputs.version }}

.gitignore

@@ -2,10 +2,16 @@ node_modules
 dist
 coverage/
 .env
+.env.*
+!.env.example
 .claude/
 .agents/.repos/effect-v3
 .worktrees/
 .supabase/
+# Stray `supabase` project dir created by running the CLI at the repo root
+# (e.g. supabase/.temp/linked-project.json). This monorepo has no top-level
+# Supabase project — real fixtures live under apps/cli-e2e/fixtures/.
+/supabase/
 .idea/
 
 # Local dev registry (verdaccio storage, generated config, auth tokens)

apps/cli-e2e/.env.example

@@ -0,0 +1,32 @@
+# cli-e2e environment — copy to `.env.local` (gitignored) and fill in.
+# Only the live/record modes need real values; replay mode (the default) needs none.
+
+# Mode: replay (default, no creds) | record (capture fixtures) | live (ADR-0013).
+CLI_E2E_MODE=live
+
+# Backend the live/record suite targets. Only `staging` is wired today.
+CLI_E2E_TARGET_ENV=staging
+
+# CLI target under test: go (source-of-truth binary) | ts-legacy (the rewrite) | ts-next.
+CLI_HARNESS_TARGET=go
+
+# Staging Management API token. Either name works (the suite also reads
+# SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN). Required in record/live mode.
+SUPABASE_ACCESS_TOKEN=sbp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+# For the `go` target, point at a freshly built binary so newly-added commands
+# resolve (the system `supabase` may be stale):
+#   cd apps/cli-go && go build -o /tmp/supabase-test-binary .
+SUPABASE_GO_BINARY=/tmp/supabase-test-binary
+
+# --- Optional overrides (sensible defaults in src/tests/env.ts) ---
+# Management API base + per-project host (default to staging: api.supabase.green / supabase.red).
+# CLI_E2E_API_URL=https://api.supabase.green
+# CLI_E2E_PROJECT_HOST=supabase.red
+# DB password for the ephemeral project (default: random per run).
+# CLI_E2E_DB_PASSWORD=
+# Skip org resolution / region / pick a specific org.
+# CLI_E2E_ORG_ID=
+# CLI_E2E_REGION=us-east-1
+# Leave the ephemeral live project alive after the run (debugging).
+# CLI_E2E_KEEP_PROJECT=1

apps/cli-e2e/.prettierignore

@@ -0,0 +1,4 @@
+# Live e2e fixtures are real Deno Edge Function projects (deno.json, jsr:/npm:
+# imports, Deno globals) — test data the CLI deploys, not workspace source.
+# oxfmt reads this file by default; keep it from formatting the fixtures.
+fixtures/

apps/cli-e2e/AGENTS.md

@@ -152,6 +152,18 @@ In **record mode**: global setup resolves the org, deletes any orphaned test pro
 
 The pre-recording cleanup deletes projects named `cli-e2e-test`, `my-project`, and `to-delete` so re-recording never hits a 409 name-conflict. Do not add tests that rely on pre-existing named projects existing on staging.
 
+## Live mode (ADR-0013)
+
+`live` is a third mode (`CLI_E2E_MODE=live`) that, unlike replay/record, **does not use the replay server**. The harness is wired straight at the real Management API (`CLI_E2E_API_URL`) and the real Docker socket; tests assert on **real outcomes**.
+
+- Live tests are `src/tests/live/**/*.live.e2e.test.ts`, run only via `vitest.live.config.ts` (the default config excludes them). They `skipIf(!isLive)`, so they are inert on the replay suite.
+- Global setup (`tests/live-setup.ts`) provisions **one ephemeral project per run** (`cli-e2e-live-{target}-{runId}-{short}`), waits for `ACTIVE_HEALTHY`, resolves the anon JWT, the IPv4 **session-pooler `dbUrl`** (for `--db-url` DB commands), the functions URL, and a seeded storage bucket, exposing them via `inject()`. It deletes the project on teardown (even on failure). Setup is intentionally **dumb** — no provisioning retry; the CI job re-runs the step on flake.
+- Use `testLive` from `src/tests/live/live-context.ts`: `run(cmd)` (direct-wired CLI), `invoke(slug)` (direct HTTP call sending the **anon JWT** in both `Authorization: Bearer` and `apikey`), plus `workspace` (a fresh `supabase init` config so golden paths exercise a generated config), `projectRef`, `anonKey`, `functionsUrl`, `dbUrl`, `storageBucket`. The functions deploy tests call `seedFunctions(workspace.path)` to layer the `deploy-e2e-*` fixtures + their `[functions.*]` config onto the init'd config.
+- **Assertion style:** outcome-based — assert `exitCode`/`stdout` substrings and the function's HTTP status + JSON body. This is ID-agnostic, so **no normalization/snapshots by default**. If the CLI's own diagnostic output is ever the assertion target, add a scoped normalizer for that one test — do not make normalization the default.
+- **Authoring target is `go`** (source of truth for the port); `ts-legacy` runs the same tests to prove the shim matches. Both run as separate CI jobs.
+- Retargeting to another env (e.g. `supabox`) is an env swap only: `CLI_E2E_TARGET_ENV` + `CLI_E2E_API_URL` + `CLI_E2E_PROJECT_HOST` + token. Tests assert on function output, not hostnames.
+- **CI triggers** (`.github/workflows/live-e2e.yml`): `workflow_dispatch` (manual; the Actions branch picker selects the ref — no free-form `ref` input, so the staging token never reaches arbitrary code) and an hourly `schedule`. There is **no `pull_request` trigger** — run it manually on a PR branch for pre-merge coverage. The scheduled run exercises the `@beta` channel: `develop` is the default branch and the beta release source, so it builds from `develop` source and runs the same `[go, ts-legacy]` matrix. A `gate` job skips the run unless the published `supabase@beta` version changed since the last green run (an `actions/cache` marker keyed on the version, written by `finalize` only after **both** legs pass), so a staging project is spent only when there is a new beta to test. Because the marker is written only on a fully-green matrix, a chronically-failing `@beta` keeps re-running every hour until it goes green or a newer beta supersedes it (intended — the failure stays visible).
+
 ## Running the suite
 
 ```sh
@@ -162,8 +174,19 @@ pnpm nx run @supabase/cli-e2e:test:go       # go binary target
 # Record (requires staging access)
 SUPABASE_ACCESS_TOKEN=sbp_... SUPABASE_STAGING_URL=https://api.supabase.green \
   pnpm nx run @supabase/cli-e2e:record
+
+# Live (requires staging access; creates + deletes a real project; needs Docker).
+# For the `go` target, build the binary first so newly-added commands resolve
+# (the system `supabase` may be stale) — mirrors what CI does.
+cd apps/cli-go && go build -o /tmp/supabase-test-binary . && cd -
+SUPABASE_GO_BINARY=/tmp/supabase-test-binary CLI_HARNESS_TARGET=go \
+  SUPABASE_ACCESS_TOKEN=sbp_... \
+  pnpm --filter @supabase/cli-e2e test:e2e:live
 ```
 
+See `apps/cli-e2e/.env.example` for the full set of live/record env vars (copy to
+a gitignored `.env.local`).
+
 After recording, replay must pass with no changes between the two commands.
 
 ### Sharding (replay only)

apps/cli-e2e/fixtures/live/functions-config.toml

@@ -0,0 +1,19 @@
+# Per-function config appended onto the `supabase init`-generated config.toml by
+# seedFunctions() for the functions deploy tests (the import-map, custom
+# entrypoint, static-file, and no-jwt fixtures need these). Everything else runs
+# against the bare generated config.
+
+[functions."deploy-e2e-root-map"]
+import_map = "./import_map.json"
+
+[functions."deploy-e2e-custom-entry"]
+entrypoint = "./functions/deploy-e2e-custom-entry/handler.ts"
+
+[functions."deploy-e2e-static-in-fn"]
+static_files = ["./functions/deploy-e2e-static-in-fn/static/*.txt"]
+
+[functions."deploy-e2e-static-asset"]
+static_files = ["./assets/*.svg", "./functions/deploy-e2e-static-asset/assets/*.svg"]
+
+[functions."deploy-e2e-no-jwt"]
+verify_jwt = false

apps/cli-e2e/fixtures/live/functions-project/functions/_shared/greet.ts

@@ -0,0 +1 @@
+export const greet = () => "hello";

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-basic/deno.json

@@ -0,0 +1,3 @@
+{
+  "imports": {}
+}

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-basic/index.ts

@@ -0,0 +1 @@
+Deno.serve(() => Response.json({ case: "deploy-e2e-basic", ok: true }));

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-custom-entry/deno.json

@@ -0,0 +1,3 @@
+{
+  "imports": {}
+}

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-custom-entry/handler.ts

@@ -0,0 +1,3 @@
+Deno.serve(() =>
+  Response.json({ case: "deploy-e2e-custom-entry", ok: true, entry: "handler.ts" })
+);

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deno-jsonc/deno.jsonc

@@ -0,0 +1,6 @@
+{
+  // scoped alias with comments
+  "imports": {
+    "@shared/": "../_shared/"
+  }
+}

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deno-jsonc/index.ts

@@ -0,0 +1,5 @@
+import { greet } from "@shared/greet.ts";
+
+Deno.serve(() =>
+  Response.json({ case: "deploy-e2e-deno-jsonc", ok: true, message: greet() })
+);

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deprecated-map/import_map.json

@@ -0,0 +1,5 @@
+{
+  "imports": {
+    "@shared/": "../_shared/"
+  }
+}

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deprecated-map/index.ts

@@ -0,0 +1,5 @@
+import { greet } from "@shared/greet.ts";
+
+Deno.serve(() =>
+  Response.json({ case: "deploy-e2e-deprecated-map", ok: true, message: greet() })
+);

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/deno.json

@@ -0,0 +1,3 @@
+{
+  "imports": {}
+}

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/index.ts

@@ -0,0 +1,4 @@
+Deno.serve(async () => {
+  const { value } = await import("./lazy.ts");
+  return Response.json({ case: "deploy-e2e-dynamic-import", ok: true, value });
+});

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/lazy.ts

@@ -0,0 +1 @@
+export const value = "lazy-ok";

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jsr/deno.json

@@ -0,0 +1,3 @@
+{
+  "imports": {}
+}

apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jsr/index.ts

@@ -0,0 +1,5 @@
+import "jsr:@supabase/functions-js/edge-runtime.d.ts";
+
+Deno.serve((req) =>
+  Response.json({ case: "deploy-e2e-jsr", ok: true, method: req.method })
+);