diff --git a/.github/scripts/sweep-live-projects.sh b/.github/scripts/sweep-live-projects.sh new file mode 100755 index 0000000000..19456281ed --- /dev/null +++ b/.github/scripts/sweep-live-projects.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Delete every staging project whose name starts with the given prefix (the live +# e2e job's per-run prefix). Shared by the in-run retry sweep (called best-effort +# with `|| true`) and the always() cleanup step (which propagates the exit code). +# +# Reads SUPABASE_ACCESS_TOKEN + CLI_E2E_API_URL from the environment. Exits +# non-zero if any DELETE failed; a failed *listing* also exits non-zero (pipefail). +set -o pipefail + +PREFIX="${1:?usage: sweep-live-projects.sh PREFIX}" +: "${SUPABASE_ACCESS_TOKEN:?SUPABASE_ACCESS_TOKEN required}" +: "${CLI_E2E_API_URL:?CLI_E2E_API_URL required}" + +# Capture the list in a var (not a pipe-to-while subshell) so a failed delete is +# recorded in $failed; a failed listing aborts here via pipefail. +refs=$(curl -fsS -H "Authorization: Bearer ${SUPABASE_ACCESS_TOKEN}" \ + "${CLI_E2E_API_URL}/v1/projects" \ + | jq -r --arg p "$PREFIX" '.[] | select(.name|startswith($p)) | .ref // .id') + +failed=0 +for ref in $refs; do + [ -n "$ref" ] || continue + echo "deleting leftover project $ref" + if ! curl -fsS -X DELETE -H "Authorization: Bearer ${SUPABASE_ACCESS_TOKEN}" \ + "${CLI_E2E_API_URL}/v1/projects/${ref}" >/dev/null; then + echo "::error::failed to delete leftover project $ref" + failed=1 + fi +done +exit "$failed" diff --git a/.github/workflows/live-e2e.yml b/.github/workflows/live-e2e.yml new file mode 100644 index 0000000000..fa4fea3fc1 --- /dev/null +++ b/.github/workflows/live-e2e.yml @@ -0,0 +1,187 @@ +name: Live E2E + +# Live e2e suite (ADR-0013). Runs the real CLI against the real staging +# Management API + Docker bundler, then invokes the deployed functions over HTTP. +# +# Non-blocking by construction: this is a standalone workflow, NOT part of the +# required-checks set, and it never runs on the default PR path of test.yml. +# +# Triggers: +# - workflow_dispatch — manual run. The Actions UI branch picker selects the +# ref (github.ref), always a same-repo branch. We deliberately take NO +# free-form `ref` input: that would let a manual run check out arbitrary +# (e.g. external PR) code while the staging token is in the job env. +# - schedule (hourly) — exercises the `@beta` channel. `develop` is the default +# branch AND the beta release source, so a scheduled run checks it out and +# builds from source. The `gate` job skips the run unless the published +# `supabase@beta` version changed since the last green run (an actions/cache +# marker keyed on the version), so we only spend a staging project when there +# is actually a new beta to test. +# +# Secrets: workflow_dispatch and schedule both run on trusted same-repo refs, so +# the staging token is never exposed to fork code. +on: + workflow_dispatch: + schedule: + # Hourly, offset from other scheduled workflows. Cron timing is best-effort. + - cron: "23 * * * *" + +permissions: + contents: read + +jobs: + # Decide whether to run. Manual dispatch always runs. Scheduled runs only fire + # when the latest published `@beta` is newer than the last one we tested green. + gate: + name: Gate (newer @beta?) + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.decide.outputs.should_run }} + version: ${{ steps.ver.outputs.version }} + steps: + - name: Resolve latest @beta version + id: ver + run: | + set -euo pipefail + version="$(npm view supabase@beta version)" + # Validate the shape before it becomes a cache key (defense-in-depth + # against a garbage/poisoned registry value). + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$ ]]; then + echo "::error::unexpected supabase@beta version '$version'"; exit 1 + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + + # Marker presence == "this beta already tested green". lookup-only so we + # download nothing; the marker is written by `finalize` after a green run. + - name: Check tested marker + id: cache + if: github.event_name == 'schedule' + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: .beta-marker + key: live-e2e-beta-${{ steps.ver.outputs.version }} + lookup-only: true + + - name: Decide + id: decide + run: | + if [ "${{ github.event_name }}" != "schedule" ]; then + echo "manual dispatch -> run" + echo "should_run=true" >> "$GITHUB_OUTPUT" + elif [ "${{ steps.cache.outputs.cache-hit }}" = "true" ]; then + echo "beta ${{ steps.ver.outputs.version }} already tested green -> skip" + echo "should_run=false" >> "$GITHUB_OUTPUT" + else + echo "new beta ${{ steps.ver.outputs.version }} -> run" + echo "should_run=true" >> "$GITHUB_OUTPUT" + fi + + live-e2e: + needs: gate + if: needs.gate.outputs.should_run == 'true' + name: Live e2e (${{ matrix.target }}) + runs-on: blacksmith-8vcpu-ubuntu-2404 + # Serialize a target against itself across runs; go and ts-legacy run in + # parallel. Per-job-scoped project names mean this is mostly belt-and-braces. + concurrency: + group: live-e2e-${{ matrix.target }}-${{ github.ref }} + cancel-in-progress: true + strategy: + # Each target is an independent green/red signal. + fail-fast: false + matrix: + # go = source-of-truth Go binary; ts-legacy = the TS rewrite (shells out + # to Go for most commands). Authoring target is go; ts-legacy proves the + # shim matches. ts-next is a later axis. + target: + - go + - ts-legacy + # Non-secret config is job-level; the staging token is scoped to only the two + # steps that need it (run + cleanup) so build/checkout/docker never see it. + env: + CLI_E2E_MODE: live + CLI_E2E_TARGET_ENV: staging + CLI_E2E_API_URL: https://api.supabase.green + CLI_E2E_PROJECT_HOST: supabase.red + CLI_HARNESS_TARGET: ${{ matrix.target }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + + - name: Setup Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version-file: apps/cli-go/go.mod + cache-dependency-path: apps/cli-go/go.sum + + # Build the Go binary for every target: `go` runs it directly and + # `ts-legacy` shells out to it for most commands. + - name: Build Go CLI + working-directory: apps/cli-go + run: go build -o supabase-go . + + - name: Export Go binary path + run: echo "SUPABASE_GO_BINARY=${{ github.workspace }}/apps/cli-go/supabase-go" >> "$GITHUB_ENV" + + # The ts-legacy harness runs the compiled supabase binary from apps/cli/dist. + - name: Build CLI + if: matrix.target == 'ts-legacy' + run: pnpm exec nx run supabase:build + + # Docker is a hard requirement for the --use-docker bundler cell. + - name: Docker preflight + run: docker info + + - name: Run live e2e (retry up to 3x) + env: + SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN }} + run: | + PREFIX="cli-e2e-live-${CLI_HARNESS_TARGET}-${GITHUB_RUN_ID}-" + # GitHub runs this step as `bash -e`; use `if cmd; then` (errexit-exempt) + # so a failing attempt does not abort the step before the retry. + for attempt in 1 2 3; do + echo "::group::live e2e attempt ${attempt}" + if [ "$attempt" -gt 1 ]; then + bash .github/scripts/sweep-live-projects.sh "$PREFIX" || true + fi + if pnpm --filter @supabase/cli-e2e test:e2e:live; then + echo "::endgroup::" + exit 0 + fi + echo "::endgroup::" + echo "attempt ${attempt} failed" + done + exit 1 + + # Backstop: delete any project this job created that survived a crash. + # The script exits non-zero (failing this step) if any delete failed. + - name: Cleanup leftover projects + if: always() + env: + SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN }} + run: bash .github/scripts/sweep-live-projects.sh "cli-e2e-live-${CLI_HARNESS_TARGET}-${GITHUB_RUN_ID}-" + + # Record that this beta tested green so the next scheduled run skips it. Needs + # the whole matrix: the marker is saved only if BOTH go and ts-legacy passed + # (a red leg leaves no marker, so the next hour re-runs the same beta). + finalize: + needs: [gate, live-e2e] + if: github.event_name == 'schedule' && needs.live-e2e.result == 'success' + name: Mark @beta tested + runs-on: ubuntu-latest + steps: + - name: Write marker + run: | + mkdir -p .beta-marker + echo "${{ needs.gate.outputs.version }}" > .beta-marker/version + + - name: Save tested marker + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: .beta-marker + key: live-e2e-beta-${{ needs.gate.outputs.version }} diff --git a/.gitignore b/.gitignore index 789bb71712..f743e832df 100644 --- a/.gitignore +++ b/.gitignore @@ -2,10 +2,16 @@ node_modules dist coverage/ .env +.env.* +!.env.example .claude/ .agents/.repos/effect-v3 .worktrees/ .supabase/ +# Stray `supabase` project dir created by running the CLI at the repo root +# (e.g. supabase/.temp/linked-project.json). This monorepo has no top-level +# Supabase project — real fixtures live under apps/cli-e2e/fixtures/. +/supabase/ .idea/ # Local dev registry (verdaccio storage, generated config, auth tokens) diff --git a/apps/cli-e2e/.env.example b/apps/cli-e2e/.env.example new file mode 100644 index 0000000000..b165e0b8d4 --- /dev/null +++ b/apps/cli-e2e/.env.example @@ -0,0 +1,32 @@ +# cli-e2e environment — copy to `.env.local` (gitignored) and fill in. +# Only the live/record modes need real values; replay mode (the default) needs none. + +# Mode: replay (default, no creds) | record (capture fixtures) | live (ADR-0013). +CLI_E2E_MODE=live + +# Backend the live/record suite targets. Only `staging` is wired today. +CLI_E2E_TARGET_ENV=staging + +# CLI target under test: go (source-of-truth binary) | ts-legacy (the rewrite) | ts-next. +CLI_HARNESS_TARGET=go + +# Staging Management API token. Either name works (the suite also reads +# SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN). Required in record/live mode. +SUPABASE_ACCESS_TOKEN=sbp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +# For the `go` target, point at a freshly built binary so newly-added commands +# resolve (the system `supabase` may be stale): +# cd apps/cli-go && go build -o /tmp/supabase-test-binary . +SUPABASE_GO_BINARY=/tmp/supabase-test-binary + +# --- Optional overrides (sensible defaults in src/tests/env.ts) --- +# Management API base + per-project host (default to staging: api.supabase.green / supabase.red). +# CLI_E2E_API_URL=https://api.supabase.green +# CLI_E2E_PROJECT_HOST=supabase.red +# DB password for the ephemeral project (default: random per run). +# CLI_E2E_DB_PASSWORD= +# Skip org resolution / region / pick a specific org. +# CLI_E2E_ORG_ID= +# CLI_E2E_REGION=us-east-1 +# Leave the ephemeral live project alive after the run (debugging). +# CLI_E2E_KEEP_PROJECT=1 diff --git a/apps/cli-e2e/.prettierignore b/apps/cli-e2e/.prettierignore new file mode 100644 index 0000000000..e00cd0d65c --- /dev/null +++ b/apps/cli-e2e/.prettierignore @@ -0,0 +1,4 @@ +# Live e2e fixtures are real Deno Edge Function projects (deno.json, jsr:/npm: +# imports, Deno globals) — test data the CLI deploys, not workspace source. +# oxfmt reads this file by default; keep it from formatting the fixtures. +fixtures/ diff --git a/apps/cli-e2e/AGENTS.md b/apps/cli-e2e/AGENTS.md index 41bbd51aba..804ebe26ba 100644 --- a/apps/cli-e2e/AGENTS.md +++ b/apps/cli-e2e/AGENTS.md @@ -152,6 +152,18 @@ In **record mode**: global setup resolves the org, deletes any orphaned test pro The pre-recording cleanup deletes projects named `cli-e2e-test`, `my-project`, and `to-delete` so re-recording never hits a 409 name-conflict. Do not add tests that rely on pre-existing named projects existing on staging. +## Live mode (ADR-0013) + +`live` is a third mode (`CLI_E2E_MODE=live`) that, unlike replay/record, **does not use the replay server**. The harness is wired straight at the real Management API (`CLI_E2E_API_URL`) and the real Docker socket; tests assert on **real outcomes**. + +- Live tests are `src/tests/live/**/*.live.e2e.test.ts`, run only via `vitest.live.config.ts` (the default config excludes them). They `skipIf(!isLive)`, so they are inert on the replay suite. +- Global setup (`tests/live-setup.ts`) provisions **one ephemeral project per run** (`cli-e2e-live-{target}-{runId}-{short}`), waits for `ACTIVE_HEALTHY`, resolves the anon JWT, the IPv4 **session-pooler `dbUrl`** (for `--db-url` DB commands), the functions URL, and a seeded storage bucket, exposing them via `inject()`. It deletes the project on teardown (even on failure). Setup is intentionally **dumb** — no provisioning retry; the CI job re-runs the step on flake. +- Use `testLive` from `src/tests/live/live-context.ts`: `run(cmd)` (direct-wired CLI), `invoke(slug)` (direct HTTP call sending the **anon JWT** in both `Authorization: Bearer` and `apikey`), plus `workspace` (a fresh `supabase init` config so golden paths exercise a generated config), `projectRef`, `anonKey`, `functionsUrl`, `dbUrl`, `storageBucket`. The functions deploy tests call `seedFunctions(workspace.path)` to layer the `deploy-e2e-*` fixtures + their `[functions.*]` config onto the init'd config. +- **Assertion style:** outcome-based — assert `exitCode`/`stdout` substrings and the function's HTTP status + JSON body. This is ID-agnostic, so **no normalization/snapshots by default**. If the CLI's own diagnostic output is ever the assertion target, add a scoped normalizer for that one test — do not make normalization the default. +- **Authoring target is `go`** (source of truth for the port); `ts-legacy` runs the same tests to prove the shim matches. Both run as separate CI jobs. +- Retargeting to another env (e.g. `supabox`) is an env swap only: `CLI_E2E_TARGET_ENV` + `CLI_E2E_API_URL` + `CLI_E2E_PROJECT_HOST` + token. Tests assert on function output, not hostnames. +- **CI triggers** (`.github/workflows/live-e2e.yml`): `workflow_dispatch` (manual; the Actions branch picker selects the ref — no free-form `ref` input, so the staging token never reaches arbitrary code) and an hourly `schedule`. There is **no `pull_request` trigger** — run it manually on a PR branch for pre-merge coverage. The scheduled run exercises the `@beta` channel: `develop` is the default branch and the beta release source, so it builds from `develop` source and runs the same `[go, ts-legacy]` matrix. A `gate` job skips the run unless the published `supabase@beta` version changed since the last green run (an `actions/cache` marker keyed on the version, written by `finalize` only after **both** legs pass), so a staging project is spent only when there is a new beta to test. Because the marker is written only on a fully-green matrix, a chronically-failing `@beta` keeps re-running every hour until it goes green or a newer beta supersedes it (intended — the failure stays visible). + ## Running the suite ```sh @@ -162,8 +174,19 @@ pnpm nx run @supabase/cli-e2e:test:go # go binary target # Record (requires staging access) SUPABASE_ACCESS_TOKEN=sbp_... SUPABASE_STAGING_URL=https://api.supabase.green \ pnpm nx run @supabase/cli-e2e:record + +# Live (requires staging access; creates + deletes a real project; needs Docker). +# For the `go` target, build the binary first so newly-added commands resolve +# (the system `supabase` may be stale) — mirrors what CI does. +cd apps/cli-go && go build -o /tmp/supabase-test-binary . && cd - +SUPABASE_GO_BINARY=/tmp/supabase-test-binary CLI_HARNESS_TARGET=go \ + SUPABASE_ACCESS_TOKEN=sbp_... \ + pnpm --filter @supabase/cli-e2e test:e2e:live ``` +See `apps/cli-e2e/.env.example` for the full set of live/record env vars (copy to +a gitignored `.env.local`). + After recording, replay must pass with no changes between the two commands. ### Sharding (replay only) diff --git a/apps/cli-e2e/fixtures/live/functions-config.toml b/apps/cli-e2e/fixtures/live/functions-config.toml new file mode 100644 index 0000000000..d210d7800e --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-config.toml @@ -0,0 +1,19 @@ +# Per-function config appended onto the `supabase init`-generated config.toml by +# seedFunctions() for the functions deploy tests (the import-map, custom +# entrypoint, static-file, and no-jwt fixtures need these). Everything else runs +# against the bare generated config. + +[functions."deploy-e2e-root-map"] +import_map = "./import_map.json" + +[functions."deploy-e2e-custom-entry"] +entrypoint = "./functions/deploy-e2e-custom-entry/handler.ts" + +[functions."deploy-e2e-static-in-fn"] +static_files = ["./functions/deploy-e2e-static-in-fn/static/*.txt"] + +[functions."deploy-e2e-static-asset"] +static_files = ["./assets/*.svg", "./functions/deploy-e2e-static-asset/assets/*.svg"] + +[functions."deploy-e2e-no-jwt"] +verify_jwt = false diff --git a/apps/cli-e2e/fixtures/live/functions-project/assets/badge.svg b/apps/cli-e2e/fixtures/live/functions-project/assets/badge.svg new file mode 100644 index 0000000000..914f94e2e0 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/assets/badge.svg @@ -0,0 +1,3 @@ + + outside-static + diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/_shared/greet.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/_shared/greet.ts new file mode 100644 index 0000000000..d901eb79d4 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/_shared/greet.ts @@ -0,0 +1 @@ +export const greet = () => "hello"; diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-basic/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-basic/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-basic/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-basic/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-basic/index.ts new file mode 100644 index 0000000000..cc000c3fc7 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-basic/index.ts @@ -0,0 +1 @@ +Deno.serve(() => Response.json({ case: "deploy-e2e-basic", ok: true })); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-custom-entry/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-custom-entry/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-custom-entry/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-custom-entry/handler.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-custom-entry/handler.ts new file mode 100644 index 0000000000..ff43ad2065 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-custom-entry/handler.ts @@ -0,0 +1,3 @@ +Deno.serve(() => + Response.json({ case: "deploy-e2e-custom-entry", ok: true, entry: "handler.ts" }) +); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deno-jsonc/deno.jsonc b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deno-jsonc/deno.jsonc new file mode 100644 index 0000000000..6f14fbcc6e --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deno-jsonc/deno.jsonc @@ -0,0 +1,6 @@ +{ + // scoped alias with comments + "imports": { + "@shared/": "../_shared/" + } +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deno-jsonc/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deno-jsonc/index.ts new file mode 100644 index 0000000000..8b1ba2da96 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deno-jsonc/index.ts @@ -0,0 +1,5 @@ +import { greet } from "@shared/greet.ts"; + +Deno.serve(() => + Response.json({ case: "deploy-e2e-deno-jsonc", ok: true, message: greet() }) +); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deprecated-map/import_map.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deprecated-map/import_map.json new file mode 100644 index 0000000000..4e99a415b5 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deprecated-map/import_map.json @@ -0,0 +1,5 @@ +{ + "imports": { + "@shared/": "../_shared/" + } +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deprecated-map/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deprecated-map/index.ts new file mode 100644 index 0000000000..21231dc870 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-deprecated-map/index.ts @@ -0,0 +1,5 @@ +import { greet } from "@shared/greet.ts"; + +Deno.serve(() => + Response.json({ case: "deploy-e2e-deprecated-map", ok: true, message: greet() }) +); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/index.ts new file mode 100644 index 0000000000..41a2055f44 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/index.ts @@ -0,0 +1,4 @@ +Deno.serve(async () => { + const { value } = await import("./lazy.ts"); + return Response.json({ case: "deploy-e2e-dynamic-import", ok: true, value }); +}); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/lazy.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/lazy.ts new file mode 100644 index 0000000000..636afa7830 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-dynamic-import/lazy.ts @@ -0,0 +1 @@ +export const value = "lazy-ok"; diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jsr/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jsr/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jsr/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jsr/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jsr/index.ts new file mode 100644 index 0000000000..b136d09c48 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jsr/index.ts @@ -0,0 +1,5 @@ +import "jsr:@supabase/functions-js/edge-runtime.d.ts"; + +Deno.serve((req) => + Response.json({ case: "deploy-e2e-jsr", ok: true, method: req.method }) +); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jwt-required/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jwt-required/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jwt-required/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jwt-required/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jwt-required/index.ts new file mode 100644 index 0000000000..81648d03de --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-jwt-required/index.ts @@ -0,0 +1 @@ +Deno.serve(() => Response.json({ case: "deploy-e2e-jwt-required", ok: true })); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-local-imports/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-local-imports/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-local-imports/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-local-imports/helpers.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-local-imports/helpers.ts new file mode 100644 index 0000000000..16e3e308e4 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-local-imports/helpers.ts @@ -0,0 +1 @@ +export const suffix = "-imports"; diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-local-imports/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-local-imports/index.ts new file mode 100644 index 0000000000..fb7ea13f9d --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-local-imports/index.ts @@ -0,0 +1,6 @@ +import { greet } from "../_shared/greet.ts"; +import { suffix } from "./helpers.ts"; + +Deno.serve(() => + Response.json({ case: "deploy-e2e-local-imports", ok: true, message: greet() + suffix }) +); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-api/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-api/deno.json new file mode 100644 index 0000000000..f6ca8454c5 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-api/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-api/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-api/index.ts new file mode 100644 index 0000000000..e344e16514 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-api/index.ts @@ -0,0 +1 @@ +Deno.serve(() => Response.json({ case: "deploy-e2e-mode-api", ok: true })); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-default/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-default/deno.json new file mode 100644 index 0000000000..f6ca8454c5 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-default/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-default/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-default/index.ts new file mode 100644 index 0000000000..dbdfe144ff --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-default/index.ts @@ -0,0 +1 @@ +Deno.serve(() => Response.json({ case: "deploy-e2e-mode-default", ok: true })); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-docker/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-docker/deno.json new file mode 100644 index 0000000000..f6ca8454c5 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-docker/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-docker/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-docker/index.ts new file mode 100644 index 0000000000..fcd8ea060a --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-mode-docker/index.ts @@ -0,0 +1 @@ +Deno.serve(() => Response.json({ case: "deploy-e2e-mode-docker", ok: true })); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-no-jwt/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-no-jwt/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-no-jwt/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-no-jwt/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-no-jwt/index.ts new file mode 100644 index 0000000000..1697305182 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-no-jwt/index.ts @@ -0,0 +1 @@ +Deno.serve(() => Response.json({ case: "deploy-e2e-no-jwt", ok: true })); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-npm/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-npm/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-npm/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-npm/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-npm/index.ts new file mode 100644 index 0000000000..76b0dbb54a --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-npm/index.ts @@ -0,0 +1,10 @@ +import { createClient } from "npm:@supabase/supabase-js@2"; + +Deno.serve(() => { + const client = createClient("https://example.supabase.co", "anon-key"); + return Response.json({ + case: "deploy-e2e-npm", + ok: true, + hasClient: typeof client.from === "function", + }); +}); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-package-json/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-package-json/index.ts new file mode 100644 index 0000000000..c2671f20ac --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-package-json/index.ts @@ -0,0 +1 @@ +Deno.serve(() => Response.json({ case: "deploy-e2e-package-json", ok: true })); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-package-json/package.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-package-json/package.json new file mode 100644 index 0000000000..b667d153ab --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-package-json/package.json @@ -0,0 +1,4 @@ +{ + "type": "module", + "dependencies": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-remote-only/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-remote-only/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-remote-only/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-remote-only/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-remote-only/index.ts new file mode 100644 index 0000000000..b911f4475e --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-remote-only/index.ts @@ -0,0 +1 @@ +Deno.serve(() => Response.json({ case: "deploy-e2e-remote-only", ok: true })); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-root-map/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-root-map/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-root-map/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-root-map/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-root-map/index.ts new file mode 100644 index 0000000000..fd1cd5a53f --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-root-map/index.ts @@ -0,0 +1,5 @@ +import { greet } from "@root/greet.ts"; + +Deno.serve(() => + Response.json({ case: "deploy-e2e-root-map", ok: true, message: greet() }) +); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-scoped-map/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-scoped-map/deno.json new file mode 100644 index 0000000000..4e99a415b5 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-scoped-map/deno.json @@ -0,0 +1,5 @@ +{ + "imports": { + "@shared/": "../_shared/" + } +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-scoped-map/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-scoped-map/index.ts new file mode 100644 index 0000000000..783b8506d6 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-scoped-map/index.ts @@ -0,0 +1,5 @@ +import { greet } from "@shared/greet.ts"; + +Deno.serve(() => + Response.json({ case: "deploy-e2e-scoped-map", ok: true, message: greet() }) +); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-asset/assets/badge.svg b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-asset/assets/badge.svg new file mode 100644 index 0000000000..914f94e2e0 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-asset/assets/badge.svg @@ -0,0 +1,3 @@ + + outside-static + diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-asset/deno.json b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-asset/deno.json new file mode 100644 index 0000000000..80c4e4a920 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-asset/deno.json @@ -0,0 +1,3 @@ +{ + "imports": {} +} diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-asset/index.ts b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-asset/index.ts new file mode 100644 index 0000000000..9a598ec2c9 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-asset/index.ts @@ -0,0 +1,10 @@ +// static_files bundles supabase/assets/*.svg (outside functions/) plus the function-local +// assets/ copy used at runtime (same pattern as deploy-e2e-static-in-fn). +Deno.serve(async () => { + const svg = await Deno.readTextFile(new URL("./assets/badge.svg", import.meta.url)); + return Response.json({ + case: "deploy-e2e-static-asset", + ok: true, + static: svg.includes("outside-static") || svg.includes(" { + const text = await Deno.readTextFile(new URL("./static/note.txt", import.meta.url)); + return Response.json({ + case: "deploy-e2e-static-in-fn", + ok: true, + static: text.trim(), + }); +}); diff --git a/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-in-fn/static/note.txt b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-in-fn/static/note.txt new file mode 100644 index 0000000000..99337dc661 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/functions/deploy-e2e-static-in-fn/static/note.txt @@ -0,0 +1 @@ +in-fn-static diff --git a/apps/cli-e2e/fixtures/live/functions-project/import_map.json b/apps/cli-e2e/fixtures/live/functions-project/import_map.json new file mode 100644 index 0000000000..c84d752202 --- /dev/null +++ b/apps/cli-e2e/fixtures/live/functions-project/import_map.json @@ -0,0 +1,5 @@ +{ + "imports": { + "@root/": "./functions/_shared/" + } +} diff --git a/apps/cli-e2e/package.json b/apps/cli-e2e/package.json index 6a5908fab7..3f4541fe05 100644 --- a/apps/cli-e2e/package.json +++ b/apps/cli-e2e/package.json @@ -9,6 +9,7 @@ "test:go": "CLI_HARNESS_TARGET=go bun --bun vitest run", "test:legacy": "CLI_HARNESS_TARGET=ts-legacy bun --bun vitest run", "test:next": "CLI_HARNESS_TARGET=ts-next bun --bun vitest run", + "test:e2e:live": "CLI_E2E_MODE=live CLI_E2E_TARGET_ENV=staging bun --bun vitest run --config vitest.live.config.ts", "record": "RECORD=true CLI_HARNESS_TARGET=go bun --bun vitest run", "check:all": "nx run-many -t types:check lint:check fmt:check knip:check --projects=$npm_package_name", "fix:all": "nx run-many -t lint:fix fmt:fix knip:fix --projects=$npm_package_name" @@ -30,7 +31,12 @@ "knip": { "entry": [ "src/**/*.e2e.test.ts", - "tests/**/*.ts" + "src/**/*.live.e2e.test.ts", + "tests/**/*.ts", + "vitest.live.config.ts" + ], + "ignore": [ + "fixtures/**" ], "ignoreDependencies": [ "@typescript/native-preview", diff --git a/apps/cli-e2e/src/tests/env.ts b/apps/cli-e2e/src/tests/env.ts index ae30c3a04a..e8803530cc 100644 --- a/apps/cli-e2e/src/tests/env.ts +++ b/apps/cli-e2e/src/tests/env.ts @@ -1,16 +1,84 @@ import type { CLITarget } from "@supabase/cli-test-helpers"; -export const isRecording = process.env["RECORD"] === "true"; +type CliE2eMode = "replay" | "record" | "live"; +type CliE2eTargetEnv = "staging" | "supabox"; + +// Runtime mode. `replay` (default) serves recorded fixtures; `record` proxies to +// staging and captures fixtures; `live` (ADR-0013) bypasses the replay server and +// wires the CLI straight at the real Management API + Docker socket. +// Back-compat: RECORD=true still maps to `record`. +const MODE: CliE2eMode = + (process.env["CLI_E2E_MODE"] as CliE2eMode | undefined) ?? + (process.env["RECORD"] === "true" ? "record" : "replay"); + +export const isRecording = MODE === "record"; +export const isLive = MODE === "live"; + +// The replay server + tests/setup.ts key recording off the RECORD env var +// directly. Keep RECORD in sync with MODE in BOTH directions so an explicit +// CLI_E2E_MODE wins over a stale RECORD env — e.g. CLI_E2E_MODE=replay must NOT +// record and wipe fixtures just because RECORD=true lingers in the shell. +if (isRecording) { + process.env["RECORD"] = "true"; +} else { + delete process.env["RECORD"]; +} + +// startReplayServer + tests/setup.ts read SUPABASE_STAGING_URL directly as the +// record proxy target. Normalise it from CLI_E2E_API_URL so +// `CLI_E2E_MODE=record CLI_E2E_API_URL=…` works without also setting the legacy var. +if (isRecording && !process.env["SUPABASE_STAGING_URL"] && process.env["CLI_E2E_API_URL"]) { + process.env["SUPABASE_STAGING_URL"] = process.env["CLI_E2E_API_URL"]; +} + +// Which backend the live/record suite targets. Only `staging` is wired today; +// `supabox` is a later env swap (CLI_E2E_API_URL + CLI_E2E_PROJECT_HOST + token). +const TARGET_ENV: CliE2eTargetEnv = + (process.env["CLI_E2E_TARGET_ENV"] as CliE2eTargetEnv | undefined) ?? "staging"; + +// Base Management API URL for record/live modes (the real API). In live mode the +// harness apiUrl is wired here directly — there is no replay server in front. +// Replay mode never reads this. +export const TARGET_API_URL = + process.env["CLI_E2E_API_URL"] ?? + process.env["SUPABASE_STAGING_URL"] ?? + "https://api.supabase.green"; + +// Host used to build the deployed-function invoke URL: +// https://{ref}.{PROJECT_HOST}/functions/v1 +// Environment-specific (staging is not supabase.co), so it is configurable. +export const PROJECT_HOST = + process.env["CLI_E2E_PROJECT_HOST"] ?? (TARGET_ENV === "staging" ? "supabase.red" : ""); // In replay mode the token never reaches a real API, but the Go CLI validates // the format before making any request (must match sbp_[a-f0-9]{40}). -// In record mode (RECORD=true) it must be a valid staging token. +// In record/live mode it must be a valid token for the target env. Falls back to +// the live staging secret name so a local `.env.local` works without remapping. export const ACCESS_TOKEN = - process.env["SUPABASE_ACCESS_TOKEN"] ?? "sbp_0000000000000000000000000000000000000000"; + process.env["SUPABASE_ACCESS_TOKEN"] ?? + process.env["SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN"] ?? + "sbp_0000000000000000000000000000000000000000"; + +// Whether a real token was supplied (vs the replay placeholder above). Live mode +// must fail fast on a missing token instead of letting every API call 401. +export const isAccessTokenProvided = Boolean( + process.env["SUPABASE_ACCESS_TOKEN"] ?? process.env["SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN"], +); -// Which target to run. Defaults to "ts-legacy"; set to "go" for recording. +// Which target to run. Defaults to "ts-legacy"; set to "go" for recording and as +// the source-of-truth target when authoring live tests. export const TARGET = (process.env["CLI_HARNESS_TARGET"] ?? "ts-legacy") as CLITarget; +// Optional org for the fresh live project. When unset, live-setup resolves it via +// `orgs list` (which also exercises that command against the real API). +export const ORG_ID_OVERRIDE = process.env["CLI_E2E_ORG_ID"]; + +// Region for the fresh live project. +export const REGION = process.env["CLI_E2E_REGION"] ?? "us-east-1"; + +// Skip live-project teardown for debugging. +export const KEEP_PROJECT = process.env["CLI_E2E_KEEP_PROJECT"] === "1"; + // In replay mode any 20-char lowercase alpha string normalises to __PROJECT_REF__ // in the fixture key. In record mode supply a real project ref via env. export const PROJECT_REF = process.env["SUPABASE_TEST_PROJECT_REF"] ?? "aaaaaaaaaaaaaaaaaaaa"; diff --git a/apps/cli-e2e/src/tests/live/branches.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/branches.live.e2e.test.ts new file mode 100644 index 0000000000..5c88ce20ce --- /dev/null +++ b/apps/cli-e2e/src/tests/live/branches.live.e2e.test.ts @@ -0,0 +1,49 @@ +import { describe, expect } from "vitest"; +import { testLive } from "./live-context.ts"; + +// Preview branches (workflow 3). `branches create` provisions a real branch and +// requires a paid plan; the cli-e2e test org may be on the free plan, in which +// case the CLI must surface the plan requirement rather than crash. Handle both: +// on a paid org, create → list → delete; on a free org, assert the plan-gate. +describe("branches (live)", () => { + testLive("create + list + delete (or surface the plan gate)", async ({ run, projectRef }) => { + // Unique per attempt so a retry (vitest retry:2) after a post-create flake + // can't collide on the name; a finally guarantees cleanup either way. + const name = `e2e-branch-${Date.now()}`; + const created = await run(["branches", "create", name, "--project-ref", projectRef]); + + if (created.exitCode !== 0) { + // Free-plan org: the command must clearly report that branching needs a + // paid plan (not fail opaquely). + expect(created.stderr, created.stderr).toMatch(/paid plan|upgrade|not.*support/i); + return; + } + + let branchDeleted = false; + try { + expect(created.stdout).toContain("Created preview branch"); + + const listed = await run([ + "branches", + "list", + "--output", + "json", + "--project-ref", + projectRef, + ]); + expect(listed.exitCode, listed.stderr).toBe(0); + const names = (JSON.parse(listed.stdout) as Array<{ name?: string }>).map((b) => b.name); + expect(names).toContain(name); + + const deleted = await run(["branches", "delete", name, "--project-ref", projectRef, "--yes"]); + expect(deleted.exitCode, deleted.stderr).toBe(0); + branchDeleted = true; + } finally { + // Retry/leak safety: clean up only if the in-try delete didn't already + // succeed (e.g. an earlier assertion threw). Tolerates a not-found branch. + if (!branchDeleted) { + await run(["branches", "delete", name, "--project-ref", projectRef, "--yes"]); + } + } + }); +}); diff --git a/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts new file mode 100644 index 0000000000..6a99f7131f --- /dev/null +++ b/apps/cli-e2e/src/tests/live/database.live.e2e.test.ts @@ -0,0 +1,32 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect } from "vitest"; +import { testLive } from "./live-context.ts"; + +// DB-connectivity commands against the fresh project's Postgres via the IPv4 +// session-mode Supavisor pooler (`dbUrl` from live-setup). The direct host +// (db..supabase.red) is IPv6-only and unreachable from IPv4-only CI +// runners; the pooler is IPv4, and session mode is required for pg_dump. +// A non-zero exit here means the connection itself failed. +describe("database (live, session pooler --db-url)", () => { + testLive("inspect db db-stats connects and reports stats", async ({ run, dbUrl }) => { + const res = await run(["inspect", "db", "db-stats", "--db-url", dbUrl]); + expect(res.exitCode, res.stderr).toBe(0); + expect(res.stdout).toContain("Database Size"); + }); + + testLive("migration list connects to the remote migration history", async ({ run, dbUrl }) => { + const res = await run(["migration", "list", "--db-url", dbUrl]); + // Fresh project has no migrations, but exit 0 proves it connected and + // queried the remote history table. + expect(res.exitCode, res.stderr).toBe(0); + }); + + testLive("db dump exports the remote schema", async ({ run, dbUrl, workspace }) => { + const file = join(workspace.path, "dump.sql"); + const res = await run(["db", "dump", "--db-url", dbUrl, "-f", file]); + expect(res.exitCode, res.stderr).toBe(0); + expect(existsSync(file)).toBe(true); + expect(readFileSync(file, "utf8")).toMatch(/CREATE|PostgreSQL database dump|SCHEMA/i); + }); +}); diff --git a/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts new file mode 100644 index 0000000000..4780b56537 --- /dev/null +++ b/apps/cli-e2e/src/tests/live/db-sync.live.e2e.test.ts @@ -0,0 +1,47 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect } from "vitest"; +import { testLive } from "./live-context.ts"; + +// Local↔remote schema sync (workflows 1-2) over the IPv4 session pooler. Done as +// one round-trip in a single workspace: pushing first makes the local migration +// history match the remote, so the subsequent pull's consistency check passes +// (a separate fresh-workspace pull would see a history mismatch on the shared +// per-run project). db push/pull confirm via a prompt that only auto-accepts +// with --yes. Mutates the throwaway project's schema — deleted on teardown. +describe("db push + pull (live, session pooler)", () => { + testLive( + "pushes a local migration and pulls the remote schema back", + async ({ run, dbUrl, workspace }) => { + const migrations = join(workspace.path, "supabase", "migrations"); + mkdirSync(migrations, { recursive: true }); + writeFileSync( + join(migrations, "20240101000000_e2e_push.sql"), + "create table if not exists e2e_push (id int);\n", + ); + + const pushed = await run(["db", "push", "--db-url", dbUrl, "--yes"]); + expect(pushed.exitCode, pushed.stderr).toBe(0); + + const listed = await run(["migration", "list", "--db-url", dbUrl]); + expect(listed.exitCode, listed.stderr).toBe(0); + expect(listed.stdout).toContain("20240101000000"); + + // Local history now matches remote, so pull connects and runs the diff. + // It either finds a remote-only change (exit 0, writes a migration) or + // reports no changes — both prove connectivity; only a real connection + // failure would surface a different error. + const pulled = await run(["db", "pull", "--db-url", dbUrl, "--yes"]); + const pullOutput = `${pulled.stdout}${pulled.stderr}`; + // The point of this test is connectivity over the pooler: a real connection + // failure must never be mistaken for a benign "no changes" outcome. + expect(pullOutput, "db pull hit a connection error").not.toMatch( + /dial|no route|connection refused|could not connect|server closed the connection|i\/o timeout/i, + ); + expect( + pulled.exitCode === 0 || /No schema changes found/i.test(pullOutput), + pulled.stderr, + ).toBe(true); + }, + ); +}); diff --git a/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts new file mode 100644 index 0000000000..e9101cc1be --- /dev/null +++ b/apps/cli-e2e/src/tests/live/functions-deploy.live.e2e.test.ts @@ -0,0 +1,66 @@ +import { readdirSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect } from "vitest"; +import { expectFunctionOk } from "./invoke.ts"; +import { seedFunctions, testLive } from "./live-context.ts"; + +// Pilot (ADR-0013): deploy with the real CLI across the three bundler paths, +// then invoke the deployed function over HTTP and assert the body it returns. +// Each mode deploys a DISTINCT slug so the invoke proves THAT mode's deploy +// produced a running function — the shared project means a single slug could +// otherwise be served by an earlier mode's deploy. Negative/arg-validation +// cases live in apps/cli integration tests. +const MODES = [ + { name: "default", slug: "deploy-e2e-mode-default", flags: [] as string[] }, + { name: "use-api", slug: "deploy-e2e-mode-api", flags: ["--use-api"] }, + { name: "use-docker", slug: "deploy-e2e-mode-docker", flags: ["--use-docker"] }, +] as const; + +describe.each(MODES)("functions deploy ($name)", ({ slug, flags }) => { + testLive("deploys and the function responds", async ({ run, invoke, workspace, projectRef }) => { + seedFunctions(workspace.path); + const deployed = await run([ + "functions", + "deploy", + slug, + "--project-ref", + projectRef, + ...flags, + ]); + expect(deployed.exitCode, deployed.stderr).toBe(0); + expect(deployed.stdout).toContain("Deployed Functions"); + + const res = await invoke(slug); + expectFunctionOk(res, slug); + }); +}); + +// No slug → the CLI walks every function declared under supabase/functions and +// deploys them all. Assert each declared function appears in the deploy output, +// then smoke-invoke a representative one. +testLive( + "deploys every declared function when no slug is given", + async ({ run, invoke, workspace, projectRef }) => { + seedFunctions(workspace.path); + const declared = readdirSync(join(workspace.path, "supabase", "functions"), { + withFileTypes: true, + }) + .filter((e) => e.isDirectory() && !e.name.startsWith("_")) + .map((e) => e.name); + expect(declared.length).toBeGreaterThan(1); + + const deployed = await run(["functions", "deploy", "--project-ref", projectRef]); + expect(deployed.exitCode, deployed.stderr).toBe(0); + expect(deployed.stdout).toContain("Deployed Functions"); + + // Each declared function must be listed in the deploy output AND respond + // with its own {case: slug, ok: true}. A handler returns that marker only if + // it actually executed — and for the npm/jsr/local-imports/scoped-map + // fixtures only if their imports resolved at runtime — so this proves the + // feature ran end-to-end, not merely that the function deployed and booted. + for (const slug of declared) { + expect(deployed.stdout, `expected "${slug}" in deploy output`).toContain(slug); + expectFunctionOk(await invoke(slug), slug); + } + }, +); diff --git a/apps/cli-e2e/src/tests/live/functions-lifecycle.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/functions-lifecycle.live.e2e.test.ts new file mode 100644 index 0000000000..b800b8bda2 --- /dev/null +++ b/apps/cli-e2e/src/tests/live/functions-lifecycle.live.e2e.test.ts @@ -0,0 +1,73 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect } from "vitest"; +import { testLive } from "./live-context.ts"; + +// Write a throwaway Edge Function into the test workspace so the lifecycle tests +// own a dedicated slug (the shared per-run project is cleaned up on teardown). +function writeFunction(workspacePath: string, slug: string, jsonBody: string): void { + const dir = join(workspacePath, "supabase", "functions", slug); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "index.ts"), `Deno.serve(() => Response.json(${jsonBody}));\n`); + writeFileSync(join(dir, "deno.json"), `{\n "imports": {}\n}\n`); +} + +// Active (non-REMOVED) function slugs. The Management API can keep deleted +// functions in the list with status REMOVED (the Go prune path skips them), so a +// successful delete may leave a REMOVED row — filter those out. +function activeSlugs(stdout: string): string[] { + return (JSON.parse(stdout) as Array<{ slug?: string; name?: string; status?: string }>) + .filter((f) => (f.status ?? "").toUpperCase() !== "REMOVED") + .map((f) => f.slug ?? f.name ?? ""); +} + +describe("functions update + delete (live)", () => { + // There is no dedicated `functions update` command — re-deploying a slug + // upserts it. Verify the second deploy replaces the running code. + testLive( + "re-deploying a function updates the running code", + async ({ run, invoke, workspace, projectRef }) => { + const slug = "deploy-e2e-update"; + + writeFunction(workspace.path, slug, `{ case: "${slug}", version: 1 }`); + expect((await run(["functions", "deploy", slug, "--project-ref", projectRef])).exitCode).toBe( + 0, + ); + expect((await invoke(slug)).body).toMatchObject({ case: slug, version: 1 }); + + writeFunction(workspace.path, slug, `{ case: "${slug}", version: 2 }`); + expect((await run(["functions", "deploy", slug, "--project-ref", projectRef])).exitCode).toBe( + 0, + ); + expect((await invoke(slug)).body).toMatchObject({ case: slug, version: 2 }); + }, + ); + + testLive("delete removes a deployed function", async ({ run, workspace, projectRef }) => { + const slug = "deploy-e2e-delete"; + + writeFunction(workspace.path, slug, `{ case: "${slug}", ok: true }`); + expect((await run(["functions", "deploy", slug, "--project-ref", projectRef])).exitCode).toBe( + 0, + ); + + const before = await run([ + "functions", + "list", + "--output", + "json", + "--project-ref", + projectRef, + ]); + expect(before.exitCode, before.stderr).toBe(0); + expect(activeSlugs(before.stdout)).toContain(slug); + + const del = await run(["functions", "delete", slug, "--project-ref", projectRef]); + expect(del.exitCode, del.stderr).toBe(0); + expect(del.stdout).toContain("Deleted Function"); + + const after = await run(["functions", "list", "--output", "json", "--project-ref", projectRef]); + expect(after.exitCode, after.stderr).toBe(0); + expect(activeSlugs(after.stdout)).not.toContain(slug); + }); +}); diff --git a/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts new file mode 100644 index 0000000000..e75455989b --- /dev/null +++ b/apps/cli-e2e/src/tests/live/gen-types.live.e2e.test.ts @@ -0,0 +1,13 @@ +import { describe, expect } from "vitest"; +import { testLive } from "./live-context.ts"; + +// gen types introspects the remote schema over the IPv4 session pooler and emits +// TypeScript types. It pulls the postgres-meta Docker image, so it needs Docker +// (present in the CI live job alongside the --use-docker bundler cell). +describe("gen types (live, session pooler)", () => { + testLive("generates TypeScript types from the remote schema", async ({ run, dbUrl }) => { + const res = await run(["gen", "types", "--db-url", dbUrl, "--lang", "typescript"]); + expect(res.exitCode, res.stderr).toBe(0); + expect(res.stdout).toMatch(/export type (Database|Json)/); + }); +}); diff --git a/apps/cli-e2e/src/tests/live/invoke.ts b/apps/cli-e2e/src/tests/live/invoke.ts new file mode 100644 index 0000000000..a5efcb3e58 --- /dev/null +++ b/apps/cli-e2e/src/tests/live/invoke.ts @@ -0,0 +1,47 @@ +import { expect } from "vitest"; + +export interface InvokeResult { + status: number; + body: unknown; + text: string; +} + +/** Direct HTTP-invoke a deployed Edge Function and return status + parsed body. + * The replay server is not involved (ADR-0013) — this is a real call to the + * deployed function. Staging expects the publishable/anon key in BOTH the + * Authorization Bearer header and the apikey header. */ +export async function invokeFunction(opts: { + functionsUrl: string; + slug: string; + anonKey?: string; + payload?: unknown; +}): Promise { + const headers: Record = { "Content-Type": "application/json" }; + if (opts.anonKey) { + headers["Authorization"] = `Bearer ${opts.anonKey}`; + headers["apikey"] = opts.anonKey; + } + const res = await fetch(`${opts.functionsUrl}/${opts.slug}`, { + method: "POST", + headers, + body: JSON.stringify(opts.payload ?? {}), + }); + const text = await res.text(); + let body: unknown; + try { + body = JSON.parse(text); + } catch { + body = text; + } + return { status: res.status, body, text }; +} + +/** Assert the playbook's default per-slug expectation: 200 + `{case: slug, ok: true}`. */ +export function expectFunctionOk( + result: InvokeResult, + slug: string, + extra?: Record, +): void { + expect(result.status, result.text).toBe(200); + expect(result.body).toMatchObject({ case: slug, ok: true, ...extra }); +} diff --git a/apps/cli-e2e/src/tests/live/link.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/link.live.e2e.test.ts new file mode 100644 index 0000000000..f0c75b5e7b --- /dev/null +++ b/apps/cli-e2e/src/tests/live/link.live.e2e.test.ts @@ -0,0 +1,21 @@ +import { describe, expect } from "vitest"; +import { testLive } from "./live-context.ts"; + +// `link` is the backbone of workflows 1-3. --skip-pooler keeps it +// Management-API-only (no IPv6-only DB connection): it validates the ref and +// writes the linked-project cache into the workspace's supabase/.temp. +describe("link (live)", () => { + testLive("links the project so ref-less commands resolve it", async ({ run, projectRef }) => { + const linked = await run(["link", "--project-ref", projectRef, "--skip-pooler"]); + expect(linked.exitCode, linked.stderr).toBe(0); + expect(linked.stdout).toContain("Finished supabase link"); + + // No --project-ref and no SUPABASE_PROJECT_ID env: a remote command must now + // resolve the ref from the link written above. + const listed = await run(["secrets", "list", "--output", "json"], { + env: { SUPABASE_PROJECT_ID: "" }, + }); + expect(listed.exitCode, listed.stderr).toBe(0); + expect(Array.isArray(JSON.parse(listed.stdout))).toBe(true); + }); +}); diff --git a/apps/cli-e2e/src/tests/live/live-context.ts b/apps/cli-e2e/src/tests/live/live-context.ts new file mode 100644 index 0000000000..2cabe016e3 --- /dev/null +++ b/apps/cli-e2e/src/tests/live/live-context.ts @@ -0,0 +1,122 @@ +import { appendFileSync, cpSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { inject, test } from "vitest"; +import { + createHarness, + exec, + makeTempDir, + type CLIResult, + type TempDir, +} from "@supabase/cli-test-helpers"; +import { ACCESS_TOKEN, isLive, PROJECT_HOST, TARGET, TARGET_API_URL } from "../env.ts"; +import { invokeFunction, type InvokeResult } from "./invoke.ts"; + +type ExecOptions = NonNullable[2]>; + +// deploy-e2e-* function files (functions/, import_map.json, assets/) + the +// [functions.*] config snippet, layered onto an init-generated config by +// seedFunctions() for the functions deploy tests. +const FUNCTIONS_PROJECT_DIR = new URL("../../../fixtures/live/functions-project", import.meta.url) + .pathname; +const FUNCTIONS_CONFIG_SNIPPET = new URL( + "../../../fixtures/live/functions-config.toml", + import.meta.url, +).pathname; + +function liveHarness(cwd: string) { + return createHarness(TARGET, { + apiUrl: TARGET_API_URL, + accessToken: ACCESS_TOKEN, + cwd, + projectId: inject("projectRef"), + // Real host so host-derived commands (storage --linked → .) reach + // the live endpoint instead of localhost. + projectHost: PROJECT_HOST, + }); +} + +/** Layer the deploy-e2e-* function files + their [functions.*] config onto an + * init-generated workspace. Used by the functions deploy tests; every other + * test runs against the bare `supabase init` config. */ +export function seedFunctions(workspacePath: string): void { + const supabaseDir = join(workspacePath, "supabase"); + cpSync(FUNCTIONS_PROJECT_DIR, supabaseDir, { recursive: true }); + appendFileSync( + join(supabaseDir, "config.toml"), + `\n${readFileSync(FUNCTIONS_CONFIG_SNIPPET, "utf8")}`, + ); +} + +interface LiveFixtures { + projectRef: string; + anonKey: string; + functionsUrl: string; + dbUrl: string; + dbPassword: string; + storageBucket: string; + workspace: TempDir; + run: (cmd: string[], execOpts?: ExecOptions) => Promise; + invoke: (slug: string, opts?: { anonKey?: string; payload?: unknown }) => Promise; +} + +const base = test.extend({ + // eslint-disable-next-line no-empty-pattern + projectRef: async ({}, use) => { + await use(inject("projectRef")); + }, + + // eslint-disable-next-line no-empty-pattern + anonKey: async ({}, use) => { + await use(inject("anonKey")); + }, + + // eslint-disable-next-line no-empty-pattern + functionsUrl: async ({}, use) => { + await use(inject("functionsUrl")); + }, + + // eslint-disable-next-line no-empty-pattern + dbUrl: async ({}, use) => { + await use(inject("dbUrl")); + }, + + // eslint-disable-next-line no-empty-pattern + dbPassword: async ({}, use) => { + await use(inject("dbPassword")); + }, + + // eslint-disable-next-line no-empty-pattern + storageBucket: async ({}, use) => { + await use(inject("storageBucket")); + }, + + workspace: async ({ task }, use) => { + const dir = makeTempDir(`cli-e2e-live-${task.name.slice(0, 30)}-`); + // Generate config.toml via `supabase init` so the golden paths run against a + // freshly-generated config (functions tests add functions via seedFunctions). + const init = await exec(liveHarness(dir.path), ["init"]); + if (init.exitCode !== 0) throw new Error(`supabase init failed: ${init.stderr}`); + await use(dir); + dir[Symbol.dispose](); + }, + + run: async ({ workspace }, use) => { + const harness = liveHarness(workspace.path); + await use((cmd, execOpts) => exec(harness, cmd, execOpts)); + }, + + invoke: async ({ functionsUrl, anonKey }, use) => { + await use((slug, opts) => + invokeFunction({ + functionsUrl, + slug, + anonKey: opts && "anonKey" in opts ? opts.anonKey : anonKey, + payload: opts?.payload, + }), + ); + }, +}); + +/** Live test API — skipped unless CLI_E2E_MODE=live, so files are inert on + * replay/PR runs (and globalSetup provisions nothing). */ +export const testLive = base.skipIf(!isLive); diff --git a/apps/cli-e2e/src/tests/live/projects.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/projects.live.e2e.test.ts new file mode 100644 index 0000000000..1b17aad672 --- /dev/null +++ b/apps/cli-e2e/src/tests/live/projects.live.e2e.test.ts @@ -0,0 +1,37 @@ +import { describe, expect } from "vitest"; +import { testLive } from "./live-context.ts"; + +// projects create/delete are exercised implicitly by live-setup (it provisions +// and tears down the per-run project). Here we cover the read paths against the +// real Management API: the fresh project shows up in `projects list`, and +// `projects api-keys` returns its keys. +describe("projects (live)", () => { + testLive( + "list includes the project and api-keys returns the anon key", + async ({ run, projectRef }) => { + const listed = await run(["projects", "list", "--output", "json"]); + expect(listed.exitCode, listed.stderr).toBe(0); + const refs = (JSON.parse(listed.stdout) as Array<{ id?: string; ref?: string }>).map( + (p) => p.ref ?? p.id, + ); + expect(refs).toContain(projectRef); + + const keys = await run([ + "projects", + "api-keys", + "--project-ref", + projectRef, + "--output", + "json", + ]); + expect(keys.exitCode, keys.stderr).toBe(0); + // Accept either a legacy anon JWT or a new-style publishable key — projects + // that only issue new keys still return a usable key. + const rows = JSON.parse(keys.stdout) as Array<{ name?: string; api_key?: string }>; + const hasUsableKey = rows.some( + (k) => k.name === "anon" || k.api_key?.startsWith("sb_publishable_"), + ); + expect(hasUsableKey, "expected an anon or publishable key").toBe(true); + }, + ); +}); diff --git a/apps/cli-e2e/src/tests/live/secrets.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/secrets.live.e2e.test.ts new file mode 100644 index 0000000000..b5c2170b68 --- /dev/null +++ b/apps/cli-e2e/src/tests/live/secrets.live.e2e.test.ts @@ -0,0 +1,48 @@ +import { describe, expect } from "vitest"; +import { testLive } from "./live-context.ts"; + +interface SecretRow { + name: string; +} + +// Live secrets flow (Management API only — no Docker, no DB). The fresh per-run +// project isolates the secret; the unset at the end cleans it up. Asserts on the +// real remote outcome: the key appears in `secrets list` after set and is gone +// after unset. +describe("secrets", () => { + testLive("set surfaces the key in list, unset removes it", async ({ run, projectRef }) => { + const key = "LIVE_E2E_SECRET"; + + const set = await run(["secrets", "set", `${key}=live-value`, "--project-ref", projectRef]); + expect(set.exitCode, set.stderr).toBe(0); + expect(set.stdout).toContain("Finished"); + + const afterSet = await run([ + "secrets", + "list", + "--output", + "json", + "--project-ref", + projectRef, + ]); + expect(afterSet.exitCode, afterSet.stderr).toBe(0); + const setNames = (JSON.parse(afterSet.stdout) as SecretRow[]).map((s) => s.name); + expect(setNames).toContain(key); + + const unset = await run(["secrets", "unset", key, "--project-ref", projectRef, "--yes"]); + expect(unset.exitCode, unset.stderr).toBe(0); + expect(unset.stdout).toContain("Finished"); + + const afterUnset = await run([ + "secrets", + "list", + "--output", + "json", + "--project-ref", + projectRef, + ]); + expect(afterUnset.exitCode, afterUnset.stderr).toBe(0); + const unsetNames = (JSON.parse(afterUnset.stdout) as SecretRow[]).map((s) => s.name); + expect(unsetNames).not.toContain(key); + }); +}); diff --git a/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts b/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts new file mode 100644 index 0000000000..2d705f690d --- /dev/null +++ b/apps/cli-e2e/src/tests/live/storage.live.e2e.test.ts @@ -0,0 +1,45 @@ +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect } from "vitest"; +import { testLive } from "./live-context.ts"; + +// Storage object round-trip against the project's real Storage API. `storage +// --linked` opens a DB connection to resolve storage config; the direct host is +// IPv6-only (unreachable from IPv4-only CI), so we `link` first (with the db +// password) to persist the IPv4 pooler connection that storage then reuses. +// The bucket is pre-seeded by live-setup; storage is gated behind --experimental. +const STORAGE_FLAGS = ["--linked", "--experimental"]; +describe("storage (live --linked)", () => { + testLive( + "uploads, lists, and removes an object", + async ({ run, workspace, projectRef, storageBucket, dbPassword }) => { + const linked = await run(["link", "--project-ref", projectRef], { + env: { SUPABASE_DB_PASSWORD: dbPassword }, + }); + expect(linked.exitCode, linked.stderr).toBe(0); + + const local = join(workspace.path, "upload.txt"); + writeFileSync(local, "live-e2e storage payload\n"); + const remote = `ss:///${storageBucket}/upload.txt`; + + const cp = await run(["storage", "cp", local, remote, ...STORAGE_FLAGS]); + expect(cp.exitCode, cp.stderr).toBe(0); + + // Trailing slash lists the bucket's contents (without it, ls returns the + // bucket entry itself). + const ls = await run(["storage", "ls", `ss:///${storageBucket}/`, ...STORAGE_FLAGS]); + expect(ls.exitCode, ls.stderr).toBe(0); + expect(ls.stdout).toContain("upload.txt"); + + // --yes: rm prompts (default No) and would otherwise skip deletion in the + // non-TTY harness yet still exit 0. + const rm = await run(["storage", "rm", remote, "--yes", ...STORAGE_FLAGS]); + expect(rm.exitCode, rm.stderr).toBe(0); + + // Confirm the object is actually gone (guards against a no-op delete). + const after = await run(["storage", "ls", `ss:///${storageBucket}/`, ...STORAGE_FLAGS]); + expect(after.exitCode, after.stderr).toBe(0); + expect(after.stdout).not.toContain("upload.txt"); + }, + ); +}); diff --git a/apps/cli-e2e/tests/live-setup.ts b/apps/cli-e2e/tests/live-setup.ts new file mode 100644 index 0000000000..4672cf8a2e --- /dev/null +++ b/apps/cli-e2e/tests/live-setup.ts @@ -0,0 +1,106 @@ +import { randomUUID } from "node:crypto"; +import type { ProvidedContext } from "vitest"; +import { + isAccessTokenProvided, + isLive, + KEEP_PROJECT, + ORG_ID_OVERRIDE, + PROJECT_HOST, + TARGET, + TARGET_API_URL, +} from "../src/tests/env.ts"; +import { + createStorageBucket, + createTestProject, + deleteTestProject, + generateDbPassword, + getAnonKey, + getPoolerSessionUrl, + getServiceRoleKey, + resolveOrgId, + waitForProjectReady, +} from "./staging-project.ts"; +import "./provided-context.ts"; // centralized `inject()` key augmentation + +const STORAGE_BUCKET = "cli-e2e-live-bucket"; + +// Live e2e global setup (ADR-0013). Provisions ONE ephemeral project per run, +// wired straight at the real Management API — no replay server. Intentionally +// dumb: no provisioning retry (the CI job re-runs the whole step on flake). +export async function setup({ + provide, +}: { + provide: (key: K, value: ProvidedContext[K]) => void; +}) { + if (!isLive) { + // The live config was invoked without CLI_E2E_MODE=live. Every test is + // skipIf(!isLive), so provision nothing. + return () => {}; + } + if (!isAccessTokenProvided) { + throw new Error( + "Live mode requires a staging access token: set SUPABASE_ACCESS_TOKEN " + + "(or SUPABASE_E2E_CLI_LIVE_STAGING_ACCESS_TOKEN). Refusing to provision against an empty token.", + ); + } + if (!PROJECT_HOST) { + throw new Error("CLI_E2E_PROJECT_HOST is required in live mode (function invoke host)"); + } + + // Resolving the org via `orgs list` also exercises that command against the + // real API; CLI_E2E_ORG_ID short-circuits it when set. + const orgId = ORG_ID_OVERRIDE ?? (await resolveOrgId(TARGET_API_URL)); + + // Per-job, per-run unique name so the CI cleanup can target only this job's + // project (never a sibling matrix job's). + const runId = process.env["GITHUB_RUN_ID"] ?? String(Date.now()); + const name = `cli-e2e-live-${TARGET}-${runId}-${randomUUID().slice(0, 8)}`; + + // Generated here (not a shared export) and routed through provide() so the + // password reaches tests only via inject(), never an importable module const. + const dbPassword = generateDbPassword(); + const projectRef = await createTestProject(TARGET_API_URL, orgId, name, dbPassword); + + // Once the project exists, any later setup failure must still delete it — + // setup returns before the teardown closure, so Vitest cannot clean up. + let anonKey: string; + let functionsUrl: string; + let dbUrl: string; + try { + await waitForProjectReady(TARGET_API_URL, projectRef); + anonKey = await getAnonKey(TARGET_API_URL, projectRef); + functionsUrl = `https://${projectRef}.${PROJECT_HOST}/functions/v1`; + // IPv4 session-mode pooler — the direct host is IPv6-only (unreachable from + // IPv4-only CI runners); the pooler is IPv4 and session mode supports pg_dump. + dbUrl = await getPoolerSessionUrl(TARGET_API_URL, projectRef, dbPassword); + // Seed a private bucket via the Storage API so the storage live tests have + // something to cp/ls/rm against (cleaned up with the project on teardown). + const serviceRoleKey = await getServiceRoleKey(TARGET_API_URL, projectRef); + await createStorageBucket(PROJECT_HOST, projectRef, serviceRoleKey, STORAGE_BUCKET); + } catch (err) { + // Delete the half-provisioned project, but never mask the original failure. + if (!KEEP_PROJECT) { + await deleteTestProject(TARGET_API_URL, projectRef, { throwOnError: true }).catch( + (cleanupErr) => console.error("Failed to delete project after setup failure:", cleanupErr), + ); + } + throw err; + } + + provide("projectRef", projectRef); + provide("anonKey", anonKey); + provide("functionsUrl", functionsUrl); + provide("dbUrl", dbUrl); + provide("dbPassword", dbPassword); + provide("storageBucket", STORAGE_BUCKET); + + return async () => { + if (KEEP_PROJECT) { + console.log(`CLI_E2E_KEEP_PROJECT set — leaving project ${projectRef} (${name}) alive`); + return; + } + // Surface a failed teardown so a leaked staging project is visible locally + // (CI also has the always() sweep as a backstop). + await deleteTestProject(TARGET_API_URL, projectRef, { throwOnError: true }); + }; +} diff --git a/apps/cli-e2e/tests/provided-context.ts b/apps/cli-e2e/tests/provided-context.ts new file mode 100644 index 0000000000..a02d045184 --- /dev/null +++ b/apps/cli-e2e/tests/provided-context.ts @@ -0,0 +1,30 @@ +// Single source of truth for Vitest's `inject()` keys across all three modes +// (replay/record use the replay-server keys; live uses the staging-project keys). +// Both global setups import this module so the augmentation is always in the +// build and `inject("…")` is typed without `as` casts. +export {}; + +declare module "vitest" { + export interface ProvidedContext { + // Shared by every mode. + projectRef: string; + storageBucket: string; + // Replay/record only (replay server + pg/docker mocks). + replayServerUrl: string; + orgId: string; + pgMockPort: number; + /** DOCKER_HOST value (tcp://host:port) pointing at the relay server. + * In record mode the relay forwards to the real Docker socket; in replay + * mode it serves recorded Docker API fixtures. */ + dockerHostUrl: string; + // Live only (ADR-0013): real ephemeral project wiring. + /** Legacy anon JWT for invoking deployed functions over HTTP. */ + anonKey: string; + /** https://{ref}.{CLI_E2E_PROJECT_HOST}/functions/v1 */ + functionsUrl: string; + /** IPv4 session-pooler Postgres URL for --db-url DB commands. */ + dbUrl: string; + /** DB password of the ephemeral project (for `link` → persisted pooler config). */ + dbPassword: string; + } +} diff --git a/apps/cli-e2e/tests/setup.ts b/apps/cli-e2e/tests/setup.ts index 60711237e7..76395763ac 100644 --- a/apps/cli-e2e/tests/setup.ts +++ b/apps/cli-e2e/tests/setup.ts @@ -1,114 +1,25 @@ import type { ProvidedContext } from "vitest"; -import { createHarness, exec } from "@supabase/cli-test-helpers"; import { startPgMock } from "../src/server/pg-mock.ts"; import { startReplayServer } from "../src/server/replay-server.ts"; -import { ACCESS_TOKEN, isRecording, ORG_ID, PROJECT_REF, TARGET } from "../src/tests/env.ts"; +import { ACCESS_TOKEN, isRecording, ORG_ID, PROJECT_REF } from "../src/tests/env.ts"; +import { + cleanupProjectsByName, + createTestProject, + deleteTestProject, + generateDbPassword, + resolveOrgId, + waitForProjectReady, +} from "./staging-project.ts"; +import "./provided-context.ts"; // centralized `inject()` key augmentation const FIXTURES_DIR = new URL("../fixtures", import.meta.url).pathname; -declare module "vitest" { - export interface ProvidedContext { - replayServerUrl: string; - projectRef: string; - orgId: string; - storageBucket: string; - pgMockPort: number; - /** DOCKER_HOST value (tcp://host:port) pointing at the relay server. - * In record mode the relay forwards to the real Docker socket; in replay - * mode it serves recorded Docker API fixtures. */ - dockerHostUrl: string; - } -} - function resolveDockerSocket(): string { const dockerHost = process.env["DOCKER_HOST"]; if (dockerHost?.startsWith("unix://")) return dockerHost.slice("unix://".length); return "/var/run/docker.sock"; } -function harness(serverUrl: string) { - return createHarness(TARGET, { apiUrl: serverUrl, accessToken: ACCESS_TOKEN }); -} - -async function resolveOrgId(serverUrl: string): Promise { - const result = await exec(harness(serverUrl), ["orgs", "list", "--output", "json"]); - if (result.exitCode !== 0) throw new Error(`orgs list failed: ${result.stderr}`); - const first = (JSON.parse(result.stdout) as Array<{ id: string }>)[0]?.id; - if (!first) throw new Error("No orgs found — cannot create test project"); - return first; -} - -async function cleanupProjectsByName(serverUrl: string, names: string[]): Promise { - const listResult = await exec(harness(serverUrl), ["projects", "list", "--output", "json"]); - if (listResult.exitCode !== 0) return; - - const projects = JSON.parse(listResult.stdout) as Array<{ - id: string; - ref?: string; - name: string; - }>; - - for (const project of projects.filter((p) => names.includes(p.name))) { - const ref = project.ref ?? project.id; - if (ref && /^[a-z]{20}$/.test(ref)) { - await exec(harness(serverUrl), ["projects", "delete", ref, "--yes"]); - } - } -} - -async function waitForProjectReady( - stagingApiUrl: string, - projectRef: string, - timeoutMs = 300_000, -): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const res = await fetch(`${stagingApiUrl}/v1/projects/${projectRef}`, { - headers: { Authorization: `Bearer ${ACCESS_TOKEN}` }, - }); - if (res.ok) { - const project = (await res.json()) as { status?: string }; - if (project.status === "ACTIVE_HEALTHY") return; - } - await new Promise((r) => setTimeout(r, 5_000)); - } - throw new Error(`Project ${projectRef} did not become ACTIVE_HEALTHY within ${timeoutMs}ms`); -} - -async function createTestProject(serverUrl: string, orgId: string): Promise { - const result = await exec(harness(serverUrl), [ - "projects", - "create", - "cli-e2e-test", - "--org-id", - orgId, - "--db-password", - "cli-e2e-password-123", - "--region", - "us-east-1", - "--output", - "json", - ]); - if (result.exitCode !== 0) throw new Error(`projects create failed: ${result.stderr}`); - const project = JSON.parse(result.stdout) as { id?: string; ref?: string }; - const ref = project.ref ?? project.id; - if (!ref || !/^[a-z]{20}$/.test(ref)) { - throw new Error(`Unexpected project ref from create: ${result.stdout}`); - } - return ref; -} - -async function deleteTestProject(serverUrl: string, projectRef: string): Promise { - try { - const result = await exec(harness(serverUrl), ["projects", "delete", projectRef, "--yes"]); - if (result.exitCode !== 0) { - console.error(`Warning: failed to delete test project ${projectRef}: ${result.stderr}`); - } - } catch (err) { - console.error(`Warning: exception deleting test project ${projectRef}:`, err); - } -} - export async function setup({ provide, }: { @@ -153,7 +64,12 @@ export async function setup({ // Create a fresh project for this recording run. Its ref is used by branches, // functions, secrets, and api-keys tests. - const projectRef = await createTestProject(server.url, orgId); + const projectRef = await createTestProject( + server.url, + orgId, + "cli-e2e-test", + generateDbPassword(), + ); provide("projectRef", projectRef); provide("orgId", orgId); diff --git a/apps/cli-e2e/tests/staging-project.ts b/apps/cli-e2e/tests/staging-project.ts new file mode 100644 index 0000000000..e0d54017fb --- /dev/null +++ b/apps/cli-e2e/tests/staging-project.ts @@ -0,0 +1,275 @@ +import { randomBytes } from "node:crypto"; +import { createHarness, exec } from "@supabase/cli-test-helpers"; +import { ACCESS_TOKEN, REGION, TARGET } from "../src/tests/env.ts"; + +// Shared staging-project helpers used by both record setup (tests/setup.ts) and +// live setup (tests/live-setup.ts). +// +// `apiUrl` is whatever the CLI talks to: in record mode that is the replay +// server (so calls are captured); in live mode it is the real Management API +// (CLI_E2E_API_URL). The harness target + token come from env. + +function harness(apiUrl: string) { + return createHarness(TARGET, { apiUrl, accessToken: ACCESS_TOKEN }); +} + +const PROJECT_REF_RE = /^[a-z]{20}$/; + +// Project statuses from which provisioning never recovers — fast-fail instead of +// polling to the timeout. +const TERMINAL_BAD_STATUSES = new Set(["INIT_FAILED", "RESTORE_FAILED", "REMOVED"]); + +/** A DB password for a throwaway project, used at creation and to build the live + * --db-url. Randomised per call (overridable via CLI_E2E_DB_PASSWORD) so no + * static credential is committed — the project is deleted on teardown anyway. + * Each setup generates its own and routes it through provide()/inject() rather + * than sharing a module-level export. */ +export function generateDbPassword(): string { + return process.env["CLI_E2E_DB_PASSWORD"] ?? `cli-e2e-${randomBytes(12).toString("hex")}`; +} + +export async function resolveOrgId(apiUrl: string): Promise { + const result = await exec(harness(apiUrl), ["orgs", "list", "--output", "json"]); + if (result.exitCode !== 0) throw new Error(`orgs list failed: ${result.stderr}`); + const first = (JSON.parse(result.stdout) as Array<{ id: string }>)[0]?.id; + if (!first) throw new Error("No orgs found — cannot create test project"); + return first; +} + +export async function createTestProject( + apiUrl: string, + orgId: string, + name: string, + password: string, +): Promise { + const result = await exec(harness(apiUrl), [ + "projects", + "create", + name, + "--org-id", + orgId, + "--db-password", + password, + "--region", + REGION, + "--output", + "json", + ]); + if (result.exitCode !== 0) throw new Error(`projects create failed: ${result.stderr}`); + const project = JSON.parse(result.stdout) as { id?: string; ref?: string }; + const ref = project.ref ?? project.id; + if (!ref || !PROJECT_REF_RE.test(ref)) { + throw new Error(`Unexpected project ref from create: ${result.stdout}`); + } + return ref; +} + +// `throwOnError` surfaces a failed deletion (live teardown uses it so a leaked +// staging project fails the run loudly; record setup keeps the lenient default). +export async function deleteTestProject( + apiUrl: string, + projectRef: string, + opts: { throwOnError?: boolean } = {}, +): Promise { + try { + const result = await exec(harness(apiUrl), ["projects", "delete", projectRef, "--yes"]); + if (result.exitCode !== 0) { + throw new Error(`projects delete exited ${result.exitCode}: ${result.stderr}`); + } + } catch (err) { + if (opts.throwOnError) throw err; + console.error(`Warning: failed to delete test project ${projectRef}:`, err); + } +} + +export async function cleanupProjectsByName(apiUrl: string, names: string[]): Promise { + const listResult = await exec(harness(apiUrl), ["projects", "list", "--output", "json"]); + if (listResult.exitCode !== 0) return; + + const projects = JSON.parse(listResult.stdout) as Array<{ + id: string; + ref?: string; + name: string; + }>; + + for (const project of projects.filter((p) => names.includes(p.name))) { + const ref = project.ref ?? project.id; + if (ref && PROJECT_REF_RE.test(ref)) { + await exec(harness(apiUrl), ["projects", "delete", ref, "--yes"]); + } + } +} + +/** Poll the real Management API until the project is ACTIVE_HEALTHY. Hits the API + * directly (not via any proxy) — this is setup-only and must not be recorded. */ +export async function waitForProjectReady( + apiBaseUrl: string, + projectRef: string, + timeoutMs = 300_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const res = await fetch(`${apiBaseUrl}/v1/projects/${projectRef}`, { + headers: { Authorization: `Bearer ${ACCESS_TOKEN}` }, + }); + if (res.ok) { + const project = (await res.json()) as { status?: string }; + if (project.status === "ACTIVE_HEALTHY") return; + if (project.status && TERMINAL_BAD_STATUSES.has(project.status)) { + throw new Error( + `Project ${projectRef} entered terminal status ${project.status} during provisioning`, + ); + } + } else { + await res.body?.cancel(); // free the socket before sleeping + } + await new Promise((r) => setTimeout(r, 5_000)); + } + throw new Error(`Project ${projectRef} did not become ACTIVE_HEALTHY within ${timeoutMs}ms`); +} + +interface ApiKey { + name?: string; + api_key?: string; +} + +/** Resolve a key for invoking the project's deployed functions over HTTP. + * Prefers the legacy `anon` JWT: Edge Functions default to verify_jwt=true and + * a publishable (sb_publishable_) key is NOT a JWT, so it fails the platform + * JWT check on a verified function. Falls back to the publishable key for + * projects that only issue new-style keys. Even after ACTIVE_HEALTHY the + * api-keys endpoint can briefly 4xx, so retry. */ +export async function getAnonKey( + apiBaseUrl: string, + projectRef: string, + attempts = 12, +): Promise { + for (let attempt = 1; attempt <= attempts; attempt++) { + const res = await fetch(`${apiBaseUrl}/v1/projects/${projectRef}/api-keys`, { + headers: { Authorization: `Bearer ${ACCESS_TOKEN}` }, + }); + if (res.ok) { + const keys = (await res.json()) as ApiKey[]; + const anonJwt = keys.find((k) => k.name === "anon" && k.api_key)?.api_key; + if (anonJwt) return anonJwt; + // Keys present but no legacy anon JWT. A publishable (sb_publishable_) key + // is NOT a JWT and 401s on the default verify_jwt=true functions, so fail + // loudly rather than proceed with a key that can't authenticate verified + // invokes (the suite would need to deploy with --no-verify-jwt instead). + if (keys.length > 0) { + throw new Error( + `Project ${projectRef} returned no anon JWT (only new-style keys); verified-function invokes require a JWT`, + ); + } + } else if (attempt < attempts) { + await res.body?.cancel(); // free the socket before sleeping + } + if (attempt === attempts) { + const detail = res.bodyUsed ? res.status : await res.text().catch(() => res.status); + throw new Error( + `Failed to resolve anon key for ${projectRef} after ${attempts} attempts: ${detail}`, + ); + } + await new Promise((r) => setTimeout(r, 10_000)); + } + // Unreachable — the loop either returns a key or throws on the last attempt. + throw new Error(`Failed to resolve anon key for ${projectRef}`); +} + +/** Service-role / secret key, used to seed a storage bucket for the live storage + * tests (the same way record setup does). Retries like getAnonKey. */ +export async function getServiceRoleKey( + apiBaseUrl: string, + projectRef: string, + attempts = 12, +): Promise { + for (let attempt = 1; attempt <= attempts; attempt++) { + const res = await fetch(`${apiBaseUrl}/v1/projects/${projectRef}/api-keys`, { + headers: { Authorization: `Bearer ${ACCESS_TOKEN}` }, + }); + if (res.ok) { + const keys = (await res.json()) as ApiKey[]; + const secret = + keys.find((k) => k.name === "service_role" && k.api_key)?.api_key ?? + keys.find((k) => k.api_key?.startsWith("sb_secret_"))?.api_key; + if (secret) return secret; + } else { + await res.body?.cancel(); // free the socket before sleeping + } + if (attempt === attempts) { + throw new Error(`Failed to resolve service-role key for ${projectRef}`); + } + await new Promise((r) => setTimeout(r, 10_000)); + } + throw new Error(`Failed to resolve service-role key for ${projectRef}`); +} + +/** Create a private storage bucket via the project's Storage API (host derived + * from projectHost, IPv4-reachable). Idempotent — treats an existing bucket as + * success. */ +export async function createStorageBucket( + projectHost: string, + projectRef: string, + serviceRoleKey: string, + bucket: string, +): Promise { + const res = await fetch(`https://${projectRef}.${projectHost}/storage/v1/bucket`, { + method: "POST", + headers: { Authorization: `Bearer ${serviceRoleKey}`, "Content-Type": "application/json" }, + body: JSON.stringify({ id: bucket, name: bucket, public: false }), + }); + if (!res.ok && res.status !== 409) { + throw new Error(`Failed to create bucket ${bucket}: ${res.status} ${await res.text()}`); + } +} + +interface PoolerConfig { + database_type?: string; + connection_string?: string; +} + +/** Build a SESSION-mode (port 5432) Supavisor pooler connection string for the + * project's Postgres. The direct host (db....) is IPv6-only and unreachable + * from IPv4-only CI runners, so DB commands go through the pooler, which is IPv4. + * Session mode (not the API's default transaction 6543) is required for pg_dump + * (`db dump`). + * + * Reuses the Management API's `connection_string` verbatim — it carries tenant + * routing (e.g. options=reference=... query params) that a field-reconstructed + * URL would drop — and only swaps in our password and the session port. Mirrors + * the Go connector by selecting the PRIMARY pooler config. */ +export async function getPoolerSessionUrl( + apiBaseUrl: string, + projectRef: string, + password: string, + attempts = 12, +): Promise { + for (let attempt = 1; attempt <= attempts; attempt++) { + const res = await fetch(`${apiBaseUrl}/v1/projects/${projectRef}/config/database/pooler`, { + headers: { Authorization: `Bearer ${ACCESS_TOKEN}` }, + }); + if (res.ok) { + const raw = (await res.json()) as PoolerConfig | PoolerConfig[]; + const configs = Array.isArray(raw) ? raw : [raw]; + const primary = configs.find((c) => c.database_type === "PRIMARY") ?? configs[0]; + if (primary?.connection_string) { + const url = new URL(primary.connection_string); + url.password = password; // overwrites the [YOUR-PASSWORD] placeholder (URL-encoded) + url.port = "5432"; // session mode (API returns the 6543 transaction port) + if (!url.searchParams.has("connect_timeout")) url.searchParams.set("connect_timeout", "30"); + return url.toString(); + } + } else if (attempt < attempts) { + await res.body?.cancel(); // free the socket before sleeping + } + if (attempt === attempts) { + const detail = res.bodyUsed ? res.status : await res.text().catch(() => res.status); + throw new Error( + `Failed to resolve pooler config for ${projectRef} after ${attempts} attempts: ${detail}`, + ); + } + await new Promise((r) => setTimeout(r, 10_000)); + } + // Unreachable — the loop either returns a URL or throws on the last attempt. + throw new Error(`Failed to resolve pooler config for ${projectRef}`); +} diff --git a/apps/cli-e2e/tsconfig.json b/apps/cli-e2e/tsconfig.json index eef2f2a863..fe8e1e0b3e 100644 --- a/apps/cli-e2e/tsconfig.json +++ b/apps/cli-e2e/tsconfig.json @@ -3,5 +3,6 @@ "compilerOptions": { "lib": ["ESNext", "DOM"], "types": ["bun"] - } + }, + "exclude": ["node_modules", "fixtures"] } diff --git a/apps/cli-e2e/vitest.config.ts b/apps/cli-e2e/vitest.config.ts index 157645d066..74c9117ecc 100644 --- a/apps/cli-e2e/vitest.config.ts +++ b/apps/cli-e2e/vitest.config.ts @@ -5,6 +5,10 @@ export default defineConfig({ test: { passWithNoTests: true, include: ["**/*.e2e.test.ts"], + // Live tests are *.live.e2e.test.ts and run only via vitest.live.config.ts. + // They also match the include glob, so exclude them here to keep the + // PR-blocking replay suite from globbing them. + exclude: ["**/node_modules/**", "**/*.live.e2e.test.ts"], fileParallelism: false, maxWorkers: 1, globalSetup: ["tests/setup.ts"], diff --git a/apps/cli-e2e/vitest.live.config.ts b/apps/cli-e2e/vitest.live.config.ts new file mode 100644 index 0000000000..5a6ea689ff --- /dev/null +++ b/apps/cli-e2e/vitest.live.config.ts @@ -0,0 +1,21 @@ +import { defineConfig } from "vitest/config"; + +// Live e2e project (ADR-0013): runs *.live.e2e.test.ts against a real backend. +// Separate from vitest.config.ts so the PR-blocking replay suite never globs +// live tests. The replay server is NOT started here — live-setup wires the CLI +// straight at the real Management API + Docker socket. +export default defineConfig({ + test: { + passWithNoTests: true, + include: ["**/*.live.e2e.test.ts"], + fileParallelism: false, + maxWorkers: 1, + globalSetup: ["tests/live-setup.ts"], + // Real provisioning + Docker bundling are slow; give each test plenty of room. + testTimeout: 600_000, + hookTimeout: 600_000, + // Per-test flake (a single invoke/deploy blip) retries here; provisioning / + // setup flake is handled by the CI job re-running the whole step. + retry: 2, + }, +}); diff --git a/apps/cli/src/legacy/commands/functions/deploy/deploy.e2e.test.ts b/apps/cli/src/legacy/commands/functions/deploy/deploy.e2e.test.ts new file mode 100644 index 0000000000..5f8df92de9 --- /dev/null +++ b/apps/cli/src/legacy/commands/functions/deploy/deploy.e2e.test.ts @@ -0,0 +1,81 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { makeTempHome, runSupabase } from "../../../../../tests/helpers/cli.ts"; + +// Argument-validation negatives for `functions deploy`. This validation lives in +// the Go CLI today (the legacy TS command proxies to it); a black-box subprocess +// test keeps these assertions valid through the eventual native TS port — it +// guards behavior, not implementation. Asserting the SPECIFIC error text also +// avoids a false pass from an unrelated non-zero exit (e.g. a missing Go binary). +// +// All cases fail before any network call (cobra flag parsing / pre-resolution), +// so no auth or linked project is required. + +const E2E_TIMEOUT_MS = 30_000; +const SLUG = "deploy-e2e-basic"; +// Valid-format token + ref to clear the auth and project-ref gates (both checked +// before the Go bundler-flag validation under test). These cases all fail before +// any network call (cobra flag-group validation / the jobs check at the top of +// RunE), so neither value is ever used against a real API. +const FAKE_TOKEN = `sbp_${"0".repeat(40)}`; +const FAKE_REF = "a".repeat(20); + +describe("supabase functions deploy (legacy) — argument validation", () => { + const conflicts = [ + { name: "--use-api + --use-docker", flags: ["--use-api", "--use-docker"] }, + { name: "--use-api + --legacy-bundle", flags: ["--use-api", "--legacy-bundle"] }, + { name: "--use-docker + --legacy-bundle", flags: ["--use-docker", "--legacy-bundle"] }, + ] as const; + + for (const { name, flags } of conflicts) { + test(`rejects ${name} as mutually exclusive`, { timeout: E2E_TIMEOUT_MS }, async () => { + using home = makeTempHome(); + const { exitCode, stderr } = await runSupabase( + ["functions", "deploy", SLUG, "--project-ref", FAKE_REF, ...flags], + { + entrypoint: "legacy", + home: home.dir, + env: { HOME: home.dir, SUPABASE_ACCESS_TOKEN: FAKE_TOKEN }, + }, + ); + expect(exitCode).not.toBe(0); + expect(stderr).toMatch(/none of the others can be|mutually exclusive/i); + }); + } + + test("rejects --jobs without --use-api", { timeout: E2E_TIMEOUT_MS }, async () => { + using home = makeTempHome(); + const { exitCode, stderr } = await runSupabase( + ["functions", "deploy", SLUG, "--project-ref", FAKE_REF, "--use-docker", "--jobs", "2"], + { + entrypoint: "legacy", + home: home.dir, + env: { HOME: home.dir, SUPABASE_ACCESS_TOKEN: FAKE_TOKEN }, + }, + ); + expect(exitCode).not.toBe(0); + // The Go CLI phrases this as either "must be used together with --use-api" + // or "cannot be used with local bundling" depending on version — both mean + // --jobs is rejected without server-side (--use-api) bundling. + expect(stderr).toMatch(/--jobs\b.*(--use-api|local bundling)/i); + }); + + test("fails without a linked project or --project-ref", { timeout: E2E_TIMEOUT_MS }, async () => { + using home = makeTempHome(); + const workdir = mkdtempSync(join(tmpdir(), "fn-deploy-nolink-")); + try { + const { exitCode, stderr } = await runSupabase(["functions", "deploy", SLUG], { + entrypoint: "legacy", + home: home.dir, + cwd: workdir, + env: { HOME: home.dir, SUPABASE_ACCESS_TOKEN: FAKE_TOKEN }, + }); + expect(exitCode).not.toBe(0); + expect(stderr).toMatch(/Cannot find project ref|Have you run|supabase link/i); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } + }); +}); diff --git a/docs/adr/0013-live-e2e-bypasses-replay-server.md b/docs/adr/0013-live-e2e-bypasses-replay-server.md new file mode 100644 index 0000000000..097bb467df --- /dev/null +++ b/docs/adr/0013-live-e2e-bypasses-replay-server.md @@ -0,0 +1,132 @@ +# 0013. Live E2E Tests Bypass the Replay Server + +**Status**: accepted +**Date**: 2026-06-16 + +## Problem Statement + +The CLI has no true end-to-end tests. `apps/cli-e2e` is a replay/record harness: +in **replay** mode it serves recorded HTTP fixtures (fast, deterministic, no +network); in **record** mode it proxies the CLI's Management API and Docker +traffic to staging only to *capture* those fixtures. Tests always assert against +replayed fixtures, never live responses. Behaviour that cannot be mocked — real +Management API calls and the real Docker bundler (e.g. `functions deploy`) — is +therefore untested. + +[CLI-1630](https://linear.app/supabase/issue/CLI-1630/set-up-proper-live-e2e-tests-for-the-cli) +adds a structured Vitest **live** suite that runs the real CLI against a real +backend (staging today, the dockerized `supabox` stack later) as a non-blocking +smoke test before a stable deploy. + +The open architectural question was *how* live mode should reach the backend. +The first instinct was to add a third runtime mode inside `replay-server.ts` +alongside `replay` and `record` — taking record mode's passthrough path +(CLI → replay server → real API) but skipping fixture I/O. That keeps the +existing Docker and storage proxies "for free." + +## Decision + +Live mode **does not route through the replay server**. It is a harness-wiring +mode, not a `replay-server.ts` branch. + +- Live tests reuse `createHarness`/`exec` from `@supabase/cli-test-helpers`, but + the harness is wired **directly**: `apiUrl = CLI_E2E_API_URL` (the real + Management API) and `DOCKER_HOST` points at the **real Docker socket**. +- `replay-server.ts` is untouched — no `live` branch, no live Docker or storage + proxy. +- Assertions are **outcome-based**, modeled on the manual deploy playbook: + 1. run the real CLI (`run([...])`) and assert `exitCode` / `stdout`; + 2. **invoke the deployed function over HTTP directly** and assert HTTP status + + the JSON body the function itself returns (e.g. `{case, ok:true}`). + The invoke is a direct HTTP call to `https://{ref}.{CLI_E2E_PROJECT_HOST}/functions/v1`, + not a proxied call — the replay server is nowhere in the assertion path. +- Because the assertion target is the function's own deterministic response (plus + exit codes / stdout substrings), the suite is **ID-agnostic** — no response + normalization or snapshot machinery by default. The function invoke URL and + anon key are resolved at setup from the freshly created project (anon key via + `GET /v1/projects/{ref}/api-keys`). + +The CLI target is a CI **matrix axis** (`CLI_HARNESS_TARGET`): each target runs +as its own job with `fail-fast: false`, so each implementation is independently +green/red. The pilot covers `go` (raw Go binary) and `ts-legacy` (the TS rewrite +that shells out to Go for most commands and runs native TS logic for ported +ones); `ts-next` is a later axis. + +## Rationale + +For the assertions live mode actually makes, intercepting the Management API buys +nothing — nothing inspects a proxied API body. The only thing the replay server +would do in live mode for `functions deploy` is relay Docker traffic +(CLI → relay → real socket) through its streaming/idle-timeout proxy. That +streaming relay is the most complex, most failure-prone code path in the harness, +and it would sit in front of the slowest, flakiest real operation (image pull + +bundle) for zero assertion benefit. Pointing `DOCKER_HOST` at the real socket +removes that failure surface entirely. + +Keeping `replay-server.ts` out of the live path also means live and record modes +stay decoupled: record mode's destructive fixture-tree rewrite, scenario logging, +and placeholder normalization never have to grow `isLive` guards, and a future +reader is not left wondering why a "transparent proxy" mode exists that records +nothing. + +The storage proxy (the other "free" proxy) is not exercised by the +`functions deploy` pilot, so it is not a reason to keep the server in front. If a +later live command genuinely needs host rewriting (e.g. storage on a different +host than the Management API), a scoped passthrough can be introduced *then* for +that command — YAGNI until a concrete need exists. + +The per-target matrix exists because `go` and `ts-legacy` are different code +paths reaching the same backend; running them as separate jobs gives two +independent green signals instead of one averaged result. + +## Consequences + +### Positive + +- The live path has fewer moving parts: no proxy, no streaming relay, no fixture + guards. The Docker bundler talks to the real daemon as users' machines do. +- `replay-server.ts` and the replay/record contract are unchanged, so the + PR-blocking `e2e` suite is unaffected. +- Tests are trivial to add: drop a `deploy-e2e-foo` fixture function returning a + known body, add one `testLive` that runs deploy → invoke → asserts body. +- Retargeting from staging to `supabox` is genuinely an env swap + (`CLI_E2E_TARGET_ENV` + `CLI_E2E_API_URL` + `CLI_E2E_PROJECT_HOST` + token), + because assertions key off function output, not hostnames. + +### Negative + +- Live mode requires a working Docker daemon on the runner (enforced by a + `docker info` preflight) — unlike the replay suite, which served Docker + fixtures and needed no daemon. +- Each live run provisions and tears down a real staging project, so the suite is + inherently slower and subject to provisioning flake. Mitigated by a CI-level + re-run (up to 3×) rather than in-setup retry. +- A second wiring path now exists for the same harness (replay-via-server vs + live-direct); contributors must know which mode wires the CLI how. + +## Alternatives Considered + +1. **Third `live` branch inside `replay-server.ts`** (the initial plan): rejected. + It adds `isLive` guards throughout record-mode code, keeps the fragile Docker + stream relay in the hot path for no assertion benefit, and couples live mode to + machinery it does not use. +2. **Snapshot/normalization-first assertions**: rejected as the default. Outcome + assertions on function bodies are naturally ID-agnostic; a scoped normalizer is + added only if a future case makes CLI diagnostic output itself the assertion + target. +3. **Single CLI target**: rejected. `go` and `ts-legacy` are distinct + implementations of the same commands; one job would hide a regression in + whichever target was not chosen. +4. **One shared long-lived staging project**: rejected. State would leak between + runs and overlapping runs would collide; ephemeral per-job projects with + scoped teardown keep runs isolated. + +## Related Decisions + +- [ADR 0012](0012-compiled-bun-runtime-dispatch.md): Compiled Bun Runtime Dispatch + (the next CLI e2e harness runs against the compiled binary) +- [ADR 0011](0011-cli-release-and-distribution-strategy.md): CLI Release & Distribution Strategy + +## See Also + +- [cli-e2e harness](../../apps/cli-e2e/AGENTS.md) diff --git a/docs/adr/README.md b/docs/adr/README.md index abffe26a78..91deb69cea 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -56,6 +56,7 @@ When an ADR becomes outdated, mark it as `deprecated` or reference the supersedi | 0010 | [Process Manager Architecture](0010-process-manager-architecture.md) | proposed | | 0011 | [CLI Release & Distribution Strategy](0011-cli-release-and-distribution-strategy.md) | proposed | | 0012 | [Compiled Bun Runtime Dispatch](0012-compiled-bun-runtime-dispatch.md) | proposed | +| 0013 | [Live E2E Tests Bypass the Replay Server](0013-live-e2e-bypasses-replay-server.md) | proposed | ## Template diff --git a/packages/cli-test-helpers/src/harness.ts b/packages/cli-test-helpers/src/harness.ts index bbe93e18e8..5aa0028c4f 100644 --- a/packages/cli-test-helpers/src/harness.ts +++ b/packages/cli-test-helpers/src/harness.ts @@ -22,6 +22,11 @@ export interface HarnessOptions { /** Set as SUPABASE_PROJECT_ID in the subprocess env. Storage commands read * this via viper (no --project-ref flag) for config validation in --local mode. */ projectId?: string; + /** Profile `project_host` — the domain the CLI derives per-project hosts from + * (storage `.`, db `db..`, etc.). Defaults to "localhost" + * for replay/mock runs; live mode sets the real target host (e.g. supabase.red) + * so host-derived commands like `storage --linked` reach the real endpoint. */ + projectHost?: string; } export interface CLIHarness { @@ -175,7 +180,7 @@ export async function exec( `name: test`, `api_url: "${url}"`, `dashboard_url: "${url}"`, - `project_host: localhost`, + `project_host: ${harness.options.projectHost ?? "localhost"}`, ].join("\n"), ); env["SUPABASE_PROFILE"] = profilePath;