ci: post-deploy smoke, nightly e2e, deploy paths-ignore and manual deploy

.github/workflows/deploy.yml

@@ -1,8 +1,60 @@
 name: Deploy
 
+run-name: Deploy ${{ inputs.environment || github.ref_name }} (${{ github.event_name }})
+
 on:
   push:
     branches: [main, staging, production]
+    # A push is skipped only when EVERY changed file matches one of these. Anything not
+    # listed (new files included) still deploys. Keep the installer and its inputs out of this list.
+    paths-ignore:
+      - "docs/**"
+      - "packages/**"
+      - ".agents/**"
+      - ".changeset/**"
+      - "**/*.md"
+      - "LICENSE-APACHE"
+      - "LICENSE-MIT"
+      - ".gitignore"
+      - ".git-blame-ignore-revs"
+      - "biome.json"
+      - "lefthook.yml"
+      - "tsconfig.base.json"
+      - "tooling/**"
+      - ".github/workflows/hypercerts-api.yml"
+      - ".github/workflows/release.yml"
+      - "happyview-modules/tests/**"
+      - "happyview-modules/.luacheckrc"
+      - "happyview-modules/tsconfig.json"
+      - "happyview-modules/docker-compose.contracts.yml"
+      # Installer (installer.js, lexicon-source.js) is deliberately NOT listed. Other scripts are
+      # listed by name so a new file the installer starts importing fails safe (deploys).
+      - "happyview-modules/tooling/build-lua.js"
+      - "happyview-modules/tooling/lint-lua.js"
+      - "happyview-modules/tooling/lua-bundles.js"
+      - "happyview-modules/tooling/lua-maintainability.js"
+      - "happyview-modules/tooling/run-http-tests.js"
+      - "happyview-modules/tooling/run-unit-tests.js"
+      - "happyview-modules/tooling/seed.js"
+      - "happyview-modules/tooling/validate-lexicons.js"
+      - "happyview-modules/tooling/smoke.js"
+      - "happyview-modules/tooling/smoke-core.js"
+      - "happyview-modules/tooling/smoke-cases.js"
+      - "happyview-modules/tooling/e2e.js"
+      - ".github/workflows/smoke.yml"
+      - ".github/workflows/e2e-nightly.yml"
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to deploy. Run the workflow from that environment's branch (Test=main, Staging=staging, Production=production).
+        type: choice
+        required: true
+        options: [Test, Staging, Production]
+      commit:
+        description: Optional full commit SHA to redeploy (rollback). Must already be on the environment's branch. Leave empty for the branch head.
+        type: string
+        required: false
+        default: ""
 
 permissions:
   contents: read
@@ -15,23 +67,42 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       environment: ${{ steps.map.outputs.environment }}
+      branch: ${{ steps.map.outputs.branch }}
     steps:
       - id: map
         env:
-          BRANCH: ${{ github.ref_name }}
+          EVENT: ${{ github.event_name }}
+          REF_NAME: ${{ github.ref_name }}
+          REF_TYPE: ${{ github.ref_type }}
+          INPUT_ENVIRONMENT: ${{ inputs.environment }}
+          INPUT_COMMIT: ${{ inputs.commit }}
         run: |
-          case "$BRANCH" in
+          set -euo pipefail
+          if [ "$REF_TYPE" != branch ]; then
+            echo "::error::Deploys run from a branch, not a $REF_TYPE ('$REF_NAME'). Pick main, staging or production under 'Use workflow from'." >&2; exit 1
+          fi
+          case "$REF_NAME" in
             main) environment=Test ;;
             staging) environment=Staging ;;
             production) environment=Production ;;
-            *) echo "::error::No deploy environment is mapped to branch '$BRANCH'; add it to deploy.yml." >&2; exit 1 ;;
+            *) echo "::error::No deploy environment is mapped to branch '$REF_NAME'. Run from main, staging or production, or add the branch to deploy.yml." >&2; exit 1 ;;
           esac
+          if [ "$EVENT" = workflow_dispatch ] && [ "$INPUT_ENVIRONMENT" != "$environment" ]; then
+            echo "::error::Deploying $INPUT_ENVIRONMENT must run from its own branch, but this run is from '$REF_NAME' (which deploys $environment). Re-run with 'Use workflow from' set to the matching branch." >&2; exit 1
+          fi
+          if [ -n "$INPUT_COMMIT" ] && ! printf '%s' "$INPUT_COMMIT" | grep -Eq '^[0-9a-f]{40}$'; then
+            echo "::error::'commit' must be a full 40-character lowercase SHA, got '$INPUT_COMMIT'." >&2; exit 1
+          fi
           echo "environment=$environment" >> "$GITHUB_OUTPUT"
+          echo "branch=$REF_NAME" >> "$GITHUB_OUTPUT"
 
   deploy:
     name: Install HappyView modules (${{ needs.resolve.outputs.environment }})
     needs: resolve
     runs-on: ubuntu-latest
+    outputs:
+      api_url: ${{ vars.HAPPYVIEW_BASE_URL }}
+      environment: ${{ needs.resolve.outputs.environment }}
     environment: ${{ needs.resolve.outputs.environment }}
     # Deploys to one environment run one at a time and are never cancelled midway.
     concurrency:
@@ -41,8 +112,20 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
+          ref: ${{ inputs.commit || github.sha }}
+          # Full history: the rollback ancestry check needs origin/<branch>.
+          fetch-depth: 0
           persist-credentials: false
 
+      - name: Check the commit is on the deploy branch
+        if: inputs.commit != ''
+        env:
+          COMMIT: ${{ inputs.commit }}
+          BRANCH: ${{ needs.resolve.outputs.branch }}
+        run: |
+          git merge-base --is-ancestor "$COMMIT" "origin/$BRANCH" || {
+            echo "::error::$COMMIT is not on $BRANCH. Only commits that were merged to the environment's branch can be redeployed to it." >&2; exit 1; }
+
       - name: Set up pnpm
         uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
 
@@ -62,3 +145,52 @@ jobs:
           HYPERCERTS_HANDLE_RESOLVER_URL: ${{ vars.HYPERCERTS_HANDLE_RESOLVER_URL }}
         # The repo is the source of truth: replace any differing installed assets.
         run: pnpm --filter @hypercerts-org/happyview-modules install:api --override
+
+  # Read-only check that the live API answers every installed query after the install.
+  # Anonymous GETs only, so it needs no secrets. It has no `environment:` on purpose: that would
+  # ask required reviewers to approve a second time. The API URL comes from the deploy job's
+  # `api_url` output instead. HAPPYVIEW_BASE_URL is the HappyView origin, which serves both
+  # /admin (used above) and the public /xrpc API.
+  smoke:
+    name: Smoke test (${{ needs.deploy.outputs.environment }})
+    needs: [deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          # Same commit that was deployed, so a rollback is checked by its own smoke cases.
+          ref: ${{ inputs.commit || github.sha }}
+          persist-credentials: false
+
+      - name: Check that this commit has smoke tests
+        id: smoke_check
+        run: |
+          if [ -f happyview-modules/tooling/smoke.js ]; then
+            echo "has_smoke=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_smoke=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::Skipping smoke test: the deployed commit predates it."
+          fi
+
+      - name: Set up pnpm
+        if: steps.smoke_check.outputs.has_smoke == 'true'
+        uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+
+      - name: Set up Node.js
+        if: steps.smoke_check.outputs.has_smoke == 'true'
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: .nvmrc
+          cache: pnpm
+
+      - name: Install dependencies
+        if: steps.smoke_check.outputs.has_smoke == 'true'
+        run: pnpm install --frozen-lockfile
+
+      - name: Smoke test the deployed API
+        if: steps.smoke_check.outputs.has_smoke == 'true'
+        env:
+          API_BASE_URL: ${{ needs.deploy.outputs.api_url }}
+        run: pnpm smoke

.github/workflows/e2e-nightly.yml

@@ -0,0 +1,56 @@
+name: Nightly e2e
+
+# Deep read-only checks (pagination walks, 100-subject batches, cross-endpoint count invariants).
+# Too slow for the deploy path, so they run off-peak, one environment at a time, and every
+# environment reports even when an earlier one fails.
+on:
+  schedule:
+    - cron: "17 3 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  e2e:
+    name: E2E (${{ matrix.environment }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    concurrency:
+      group: e2e-${{ matrix.environment }}
+      cancel-in-progress: false
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      # URLs are mapped here because these environments only deploy from their own branches,
+      # so a job running from main cannot use `environment:` to read their variables.
+      matrix:
+        include:
+          - environment: Test
+            url: https://test.api.hypercerts.dev
+          - environment: Staging
+            url: https://api.staging.hypercerts.dev
+          - environment: Production
+            url: https://api.hypercerts.dev
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up pnpm
+        uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: .nvmrc
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run e2e checks
+        env:
+          API_BASE_URL: ${{ matrix.url }}
+        run: pnpm e2e

.github/workflows/smoke.yml

@@ -0,0 +1,66 @@
+name: Smoke
+
+# Catches drift between deploys: runs the same read-only smoke test as deploy.yml against the
+# live API. Scheduled runs target Production; use "Run workflow" for Test or Staging.
+on:
+  schedule:
+    - cron: "*/30 * * * *"
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to check
+        type: choice
+        options: [Test, Staging, Production]
+        default: Production
+
+permissions:
+  contents: read
+
+jobs:
+  smoke:
+    name: Smoke test (${{ inputs.environment || 'Production' }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    # Own group per environment, deliberately not the deploy group: a smoke run never queues
+    # behind or cancels a deploy. A run that is still going when the next one starts is left alone.
+    concurrency:
+      group: smoke-${{ inputs.environment || 'Production' }}
+      cancel-in-progress: false
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up pnpm
+        uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: .nvmrc
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      # No `environment:` here: Test/Staging/Production deployment branch policies only allow
+      # main/staging/production, and this workflow runs from main. So the URLs are mapped here
+      # instead of read from the environment's HAPPYVIEW_BASE_URL variable.
+      - name: Resolve API URL
+        id: target
+        env:
+          ENVIRONMENT: ${{ inputs.environment || 'Production' }}
+        run: |
+          case "$ENVIRONMENT" in
+            Test) url=https://test.api.hypercerts.dev ;;
+            Staging) url=https://api.staging.hypercerts.dev ;;
+            Production) url=https://api.hypercerts.dev ;;
+            *) echo "::error::Unknown environment '$ENVIRONMENT'; add its URL to smoke.yml." >&2; exit 1 ;;
+          esac
+          echo "url=$url" >> "$GITHUB_OUTPUT"
+
+      - name: Smoke test the live API
+        env:
+          API_BASE_URL: ${{ steps.target.outputs.url }}
+        run: pnpm smoke

CONTRIBUTING.md

@@ -73,6 +73,22 @@ The HTTP gate fails if it discovers no suites, executes no `node:test` cases, or
 
 For the pinned HappyView release, ordinary Lua `error()` exceptions return HTTP 500 JSON with `error: "script_error"` and `errorType: "runtime"`; the error name appears in `message`. Negative HTTP tests assert this observed behavior. It is not a statement of the ideal public HTTP status contract, and does not guarantee 4xx mapping for `RecordNotFound` or `InvalidRequest`.
 
+## Smoke and e2e checks against a live API
+
+Two read-only runners check a deployed API. They send anonymous `GET` requests only and never write.
+
+```sh
+API_BASE_URL=https://test.api.hypercerts.dev pnpm smoke   # one call per installed query, about 10 seconds
+API_BASE_URL=https://test.api.hypercerts.dev pnpm e2e     # pagination walks, batches, count invariants, about 1-2 minutes
+```
+
+`--base-url <url>` works instead of `API_BASE_URL`. Run them against Test; Staging and Production are checked by CI.
+
+- `pnpm smoke` takes its query list from `happyview-modules/manifest.json`, finds real records through the list endpoints, and checks each response against its Lexicon output schema plus basic invariants (page size, `totalCount`, count order, `uri` echo). A case is skipped when an environment has no record of that kind. It retries once on a timeout or 5xx. Adding a query without a case in `happyview-modules/tooling/smoke-cases.js` fails `pnpm test:unit`.
+- `pnpm e2e` compares live counts only loosely (small drift allowed, never fixed numbers) and paces requests 300 ms apart. It does not test invalid input.
+- CI: `deploy.yml` runs the smoke test after each install (a push that only touches docs, tests or tooling skips the deploy and smoke test; run the Deploy workflow by hand, from the environment's branch, to force one); `smoke.yml` runs it on Production every 30 minutes (or on demand for any environment); `e2e-nightly.yml` runs the e2e checks on Test, Staging and Production in turn at 03:17 UTC. `smoke.yml` and `e2e-nightly.yml` run from `main` without a GitHub environment (the environments only accept their own deploy branch), so their API URLs are written in the workflow files; update them there if a public URL changes. Failures show up as a failed run (default GitHub notifications); there is no other alerting.
+- Both runners write a results table to the GitHub step summary.
+
 ## Operations with external effects
 
 For pull requests that change the installable API bundle or operator-visible behavior, add a release note following the [Changesets guidance](.changeset/README.md). Documentation-only or endpoint-explorer-only changes do not need a Changeset.

biome.json

@@ -102,6 +102,8 @@
       "includes": [
         "happyview-modules/tooling/installer.js",
         "happyview-modules/tooling/seed.js",
+        "happyview-modules/tooling/smoke.js",
+        "happyview-modules/tooling/e2e.js",
         "happyview-modules/tooling/lint-lua.js"
       ],
       "linter": {

happyview-modules/README.md

@@ -70,4 +70,6 @@ Each environment must allow deployments from its branch and define its own value
 - secret `HAPPYVIEW_ADMIN_TOKEN`: a scoped admin token for that HappyView, including `script-variables:read`
 - variable `HYPERCERTS_HANDLE_RESOLVER_URL`: needed only if that HappyView does not already have this setting; the token then also needs `script-variables:create`
 
+Pushes that only change docs, tests or tooling skip the deploy (`paths-ignore` in `deploy.yml`). To deploy by hand, run the workflow with `workflow_dispatch`: choose the environment, run it from that environment's branch, and optionally give a rollback commit SHA, which must be on that branch. A smoke job runs after every deploy and checks the live API.
+
 Deploys to each environment run one at a time and are never cancelled midway; deploys to different environments do not block each other.

happyview-modules/package.json

@@ -21,6 +21,8 @@
     "test:unit": "node tooling/run-unit-tests.js",
     "test:http": "node tooling/run-http-tests.js",
     "install:api": "node tooling/installer.js",
+    "smoke": "node tooling/smoke.js",
+    "e2e": "node tooling/e2e.js",
     "seed:test": "node tooling/seed.js",
     "seed:bad-dates": "node tooling/seed.js --bad-dates",
     "test:contracts": "pnpm run test:http"