diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4bdfe82..fde1f38 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,8 +1,60 @@ name: Deploy +run-name: Deploy ${{ inputs.environment || github.ref_name }} (${{ github.event_name }}) + on: push: branches: [main, staging, production] + # A push is skipped only when EVERY changed file matches one of these. Anything not + # listed (new files included) still deploys. Keep the installer and its inputs out of this list. + paths-ignore: + - "docs/**" + - "packages/**" + - ".agents/**" + - ".changeset/**" + - "**/*.md" + - "LICENSE-APACHE" + - "LICENSE-MIT" + - ".gitignore" + - ".git-blame-ignore-revs" + - "biome.json" + - "lefthook.yml" + - "tsconfig.base.json" + - "tooling/**" + - ".github/workflows/hypercerts-api.yml" + - ".github/workflows/release.yml" + - "happyview-modules/tests/**" + - "happyview-modules/.luacheckrc" + - "happyview-modules/tsconfig.json" + - "happyview-modules/docker-compose.contracts.yml" + # Installer (installer.js, lexicon-source.js) is deliberately NOT listed. Other scripts are + # listed by name so a new file the installer starts importing fails safe (deploys). + - "happyview-modules/tooling/build-lua.js" + - "happyview-modules/tooling/lint-lua.js" + - "happyview-modules/tooling/lua-bundles.js" + - "happyview-modules/tooling/lua-maintainability.js" + - "happyview-modules/tooling/run-http-tests.js" + - "happyview-modules/tooling/run-unit-tests.js" + - "happyview-modules/tooling/seed.js" + - "happyview-modules/tooling/validate-lexicons.js" + - "happyview-modules/tooling/smoke.js" + - "happyview-modules/tooling/smoke-core.js" + - "happyview-modules/tooling/smoke-cases.js" + - "happyview-modules/tooling/e2e.js" + - ".github/workflows/smoke.yml" + - ".github/workflows/e2e-nightly.yml" + workflow_dispatch: + inputs: + environment: + description: Environment to deploy. Run the workflow from that environment's branch (Test=main, Staging=staging, Production=production). + type: choice + required: true + options: [Test, Staging, Production] + commit: + description: Optional full commit SHA to redeploy (rollback). Must already be on the environment's branch. Leave empty for the branch head. + type: string + required: false + default: "" permissions: contents: read @@ -15,23 +67,42 @@ jobs: runs-on: ubuntu-latest outputs: environment: ${{ steps.map.outputs.environment }} + branch: ${{ steps.map.outputs.branch }} steps: - id: map env: - BRANCH: ${{ github.ref_name }} + EVENT: ${{ github.event_name }} + REF_NAME: ${{ github.ref_name }} + REF_TYPE: ${{ github.ref_type }} + INPUT_ENVIRONMENT: ${{ inputs.environment }} + INPUT_COMMIT: ${{ inputs.commit }} run: | - case "$BRANCH" in + set -euo pipefail + if [ "$REF_TYPE" != branch ]; then + echo "::error::Deploys run from a branch, not a $REF_TYPE ('$REF_NAME'). Pick main, staging or production under 'Use workflow from'." >&2; exit 1 + fi + case "$REF_NAME" in main) environment=Test ;; staging) environment=Staging ;; production) environment=Production ;; - *) echo "::error::No deploy environment is mapped to branch '$BRANCH'; add it to deploy.yml." >&2; exit 1 ;; + *) echo "::error::No deploy environment is mapped to branch '$REF_NAME'. Run from main, staging or production, or add the branch to deploy.yml." >&2; exit 1 ;; esac + if [ "$EVENT" = workflow_dispatch ] && [ "$INPUT_ENVIRONMENT" != "$environment" ]; then + echo "::error::Deploying $INPUT_ENVIRONMENT must run from its own branch, but this run is from '$REF_NAME' (which deploys $environment). Re-run with 'Use workflow from' set to the matching branch." >&2; exit 1 + fi + if [ -n "$INPUT_COMMIT" ] && ! printf '%s' "$INPUT_COMMIT" | grep -Eq '^[0-9a-f]{40}$'; then + echo "::error::'commit' must be a full 40-character lowercase SHA, got '$INPUT_COMMIT'." >&2; exit 1 + fi echo "environment=$environment" >> "$GITHUB_OUTPUT" + echo "branch=$REF_NAME" >> "$GITHUB_OUTPUT" deploy: name: Install HappyView modules (${{ needs.resolve.outputs.environment }}) needs: resolve runs-on: ubuntu-latest + outputs: + api_url: ${{ vars.HAPPYVIEW_BASE_URL }} + environment: ${{ needs.resolve.outputs.environment }} environment: ${{ needs.resolve.outputs.environment }} # Deploys to one environment run one at a time and are never cancelled midway. concurrency: @@ -41,8 +112,20 @@ jobs: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ inputs.commit || github.sha }} + # Full history: the rollback ancestry check needs origin/. + fetch-depth: 0 persist-credentials: false + - name: Check the commit is on the deploy branch + if: inputs.commit != '' + env: + COMMIT: ${{ inputs.commit }} + BRANCH: ${{ needs.resolve.outputs.branch }} + run: | + git merge-base --is-ancestor "$COMMIT" "origin/$BRANCH" || { + echo "::error::$COMMIT is not on $BRANCH. Only commits that were merged to the environment's branch can be redeployed to it." >&2; exit 1; } + - name: Set up pnpm uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 @@ -62,3 +145,52 @@ jobs: HYPERCERTS_HANDLE_RESOLVER_URL: ${{ vars.HYPERCERTS_HANDLE_RESOLVER_URL }} # The repo is the source of truth: replace any differing installed assets. run: pnpm --filter @hypercerts-org/happyview-modules install:api --override + + # Read-only check that the live API answers every installed query after the install. + # Anonymous GETs only, so it needs no secrets. It has no `environment:` on purpose: that would + # ask required reviewers to approve a second time. The API URL comes from the deploy job's + # `api_url` output instead. HAPPYVIEW_BASE_URL is the HappyView origin, which serves both + # /admin (used above) and the public /xrpc API. + smoke: + name: Smoke test (${{ needs.deploy.outputs.environment }}) + needs: [deploy] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Same commit that was deployed, so a rollback is checked by its own smoke cases. + ref: ${{ inputs.commit || github.sha }} + persist-credentials: false + + - name: Check that this commit has smoke tests + id: smoke_check + run: | + if [ -f happyview-modules/tooling/smoke.js ]; then + echo "has_smoke=true" >> "$GITHUB_OUTPUT" + else + echo "has_smoke=false" >> "$GITHUB_OUTPUT" + echo "::notice::Skipping smoke test: the deployed commit predates it." + fi + + - name: Set up pnpm + if: steps.smoke_check.outputs.has_smoke == 'true' + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - name: Set up Node.js + if: steps.smoke_check.outputs.has_smoke == 'true' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .nvmrc + cache: pnpm + + - name: Install dependencies + if: steps.smoke_check.outputs.has_smoke == 'true' + run: pnpm install --frozen-lockfile + + - name: Smoke test the deployed API + if: steps.smoke_check.outputs.has_smoke == 'true' + env: + API_BASE_URL: ${{ needs.deploy.outputs.api_url }} + run: pnpm smoke diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml new file mode 100644 index 0000000..14375e5 --- /dev/null +++ b/.github/workflows/e2e-nightly.yml @@ -0,0 +1,56 @@ +name: Nightly e2e + +# Deep read-only checks (pagination walks, 100-subject batches, cross-endpoint count invariants). +# Too slow for the deploy path, so they run off-peak, one environment at a time, and every +# environment reports even when an earlier one fails. +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + e2e: + name: E2E (${{ matrix.environment }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + concurrency: + group: e2e-${{ matrix.environment }} + cancel-in-progress: false + strategy: + fail-fast: false + max-parallel: 1 + # URLs are mapped here because these environments only deploy from their own branches, + # so a job running from main cannot use `environment:` to read their variables. + matrix: + include: + - environment: Test + url: https://test.api.hypercerts.dev + - environment: Staging + url: https://api.staging.hypercerts.dev + - environment: Production + url: https://api.hypercerts.dev + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .nvmrc + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run e2e checks + env: + API_BASE_URL: ${{ matrix.url }} + run: pnpm e2e diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 0000000..2c68aaa --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,66 @@ +name: Smoke + +# Catches drift between deploys: runs the same read-only smoke test as deploy.yml against the +# live API. Scheduled runs target Production; use "Run workflow" for Test or Staging. +on: + schedule: + - cron: "*/30 * * * *" + workflow_dispatch: + inputs: + environment: + description: Environment to check + type: choice + options: [Test, Staging, Production] + default: Production + +permissions: + contents: read + +jobs: + smoke: + name: Smoke test (${{ inputs.environment || 'Production' }}) + runs-on: ubuntu-latest + timeout-minutes: 5 + # Own group per environment, deliberately not the deploy group: a smoke run never queues + # behind or cancels a deploy. A run that is still going when the next one starts is left alone. + concurrency: + group: smoke-${{ inputs.environment || 'Production' }} + cancel-in-progress: false + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: .nvmrc + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # No `environment:` here: Test/Staging/Production deployment branch policies only allow + # main/staging/production, and this workflow runs from main. So the URLs are mapped here + # instead of read from the environment's HAPPYVIEW_BASE_URL variable. + - name: Resolve API URL + id: target + env: + ENVIRONMENT: ${{ inputs.environment || 'Production' }} + run: | + case "$ENVIRONMENT" in + Test) url=https://test.api.hypercerts.dev ;; + Staging) url=https://api.staging.hypercerts.dev ;; + Production) url=https://api.hypercerts.dev ;; + *) echo "::error::Unknown environment '$ENVIRONMENT'; add its URL to smoke.yml." >&2; exit 1 ;; + esac + echo "url=$url" >> "$GITHUB_OUTPUT" + + - name: Smoke test the live API + env: + API_BASE_URL: ${{ steps.target.outputs.url }} + run: pnpm smoke diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9e86d96..d742f43 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,6 +73,22 @@ The HTTP gate fails if it discovers no suites, executes no `node:test` cases, or For the pinned HappyView release, ordinary Lua `error()` exceptions return HTTP 500 JSON with `error: "script_error"` and `errorType: "runtime"`; the error name appears in `message`. Negative HTTP tests assert this observed behavior. It is not a statement of the ideal public HTTP status contract, and does not guarantee 4xx mapping for `RecordNotFound` or `InvalidRequest`. +## Smoke and e2e checks against a live API + +Two read-only runners check a deployed API. They send anonymous `GET` requests only and never write. + +```sh +API_BASE_URL=https://test.api.hypercerts.dev pnpm smoke # one call per installed query, about 10 seconds +API_BASE_URL=https://test.api.hypercerts.dev pnpm e2e # pagination walks, batches, count invariants, about 1-2 minutes +``` + +`--base-url ` works instead of `API_BASE_URL`. Run them against Test; Staging and Production are checked by CI. + +- `pnpm smoke` takes its query list from `happyview-modules/manifest.json`, finds real records through the list endpoints, and checks each response against its Lexicon output schema plus basic invariants (page size, `totalCount`, count order, `uri` echo). A case is skipped when an environment has no record of that kind. It retries once on a timeout or 5xx. Adding a query without a case in `happyview-modules/tooling/smoke-cases.js` fails `pnpm test:unit`. +- `pnpm e2e` compares live counts only loosely (small drift allowed, never fixed numbers) and paces requests 300 ms apart. It does not test invalid input. +- CI: `deploy.yml` runs the smoke test after each install (a push that only touches docs, tests or tooling skips the deploy and smoke test; run the Deploy workflow by hand, from the environment's branch, to force one); `smoke.yml` runs it on Production every 30 minutes (or on demand for any environment); `e2e-nightly.yml` runs the e2e checks on Test, Staging and Production in turn at 03:17 UTC. `smoke.yml` and `e2e-nightly.yml` run from `main` without a GitHub environment (the environments only accept their own deploy branch), so their API URLs are written in the workflow files; update them there if a public URL changes. Failures show up as a failed run (default GitHub notifications); there is no other alerting. +- Both runners write a results table to the GitHub step summary. + ## Operations with external effects For pull requests that change the installable API bundle or operator-visible behavior, add a release note following the [Changesets guidance](.changeset/README.md). Documentation-only or endpoint-explorer-only changes do not need a Changeset. diff --git a/biome.json b/biome.json index 1472562..697d718 100644 --- a/biome.json +++ b/biome.json @@ -102,6 +102,8 @@ "includes": [ "happyview-modules/tooling/installer.js", "happyview-modules/tooling/seed.js", + "happyview-modules/tooling/smoke.js", + "happyview-modules/tooling/e2e.js", "happyview-modules/tooling/lint-lua.js" ], "linter": { diff --git a/happyview-modules/README.md b/happyview-modules/README.md index a6f1a81..93fd06a 100644 --- a/happyview-modules/README.md +++ b/happyview-modules/README.md @@ -70,4 +70,6 @@ Each environment must allow deployments from its branch and define its own value - secret `HAPPYVIEW_ADMIN_TOKEN`: a scoped admin token for that HappyView, including `script-variables:read` - variable `HYPERCERTS_HANDLE_RESOLVER_URL`: needed only if that HappyView does not already have this setting; the token then also needs `script-variables:create` +Pushes that only change docs, tests or tooling skip the deploy (`paths-ignore` in `deploy.yml`). To deploy by hand, run the workflow with `workflow_dispatch`: choose the environment, run it from that environment's branch, and optionally give a rollback commit SHA, which must be on that branch. A smoke job runs after every deploy and checks the live API. + Deploys to each environment run one at a time and are never cancelled midway; deploys to different environments do not block each other. diff --git a/happyview-modules/package.json b/happyview-modules/package.json index 6edcd1a..ca65ed7 100644 --- a/happyview-modules/package.json +++ b/happyview-modules/package.json @@ -21,6 +21,8 @@ "test:unit": "node tooling/run-unit-tests.js", "test:http": "node tooling/run-http-tests.js", "install:api": "node tooling/installer.js", + "smoke": "node tooling/smoke.js", + "e2e": "node tooling/e2e.js", "seed:test": "node tooling/seed.js", "seed:bad-dates": "node tooling/seed.js --bad-dates", "test:contracts": "pnpm run test:http" diff --git a/happyview-modules/tests/unit/tooling/smoke-core.test.js b/happyview-modules/tests/unit/tooling/smoke-core.test.js new file mode 100644 index 0000000..cc73918 --- /dev/null +++ b/happyview-modules/tests/unit/tooling/smoke-core.test.js @@ -0,0 +1,103 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { genericInvariants, installedQueryIds } from "../../../tooling/smoke.js"; +import { SMOKE_CASES } from "../../../tooling/smoke-cases.js"; +import { + assertReadOnlyRequest, + createHttpClient, + findCoverageGaps, + normalizeBaseUrl, + runPool, +} from "../../../tooling/smoke-core.js"; + +const response = (status, body = {}) => new Response(JSON.stringify(body), { status }); + +/** Client over a scripted fetch; returns the calls it saw. */ +function scripted(...replies) { + const calls = []; + const fetchImpl = async (url, init) => { + calls.push({ url, method: init.method }); + const reply = replies.shift(); + if (reply instanceof Error) throw reply; + return reply; + }; + return { calls, client: createHttpClient({ baseUrl: "https://api.example/", fetchImpl }) }; +} + +test("only GET requests to /xrpc/ are allowed", () => { + assert.doesNotThrow(() => assertReadOnlyRequest("GET", "https://x.test/xrpc/a.b.c")); + assert.throws(() => assertReadOnlyRequest("POST", "https://x.test/xrpc/a.b.c"), /only send GET/); + assert.throws(() => assertReadOnlyRequest("GET", "https://x.test/admin/lexicons"), /\/xrpc\//); +}); + +test("base URL rejects credentials and non-http schemes", () => { + assert.equal(normalizeBaseUrl("https://api.example/"), "https://api.example"); + assert.throws(() => normalizeBaseUrl("https://u:p@api.example"), /credentials/); + assert.throws(() => normalizeBaseUrl("ftp://api.example"), /http/); + assert.throws(() => normalizeBaseUrl("not a url"), /Invalid API base URL/); +}); + +test("retries once on 5xx and returns the second result", async () => { + const { client, calls } = scripted(response(503), response(200, { ok: 1 })); + const res = await client.get("a.b.c", { x: ["1", "2"] }); + assert.equal(res.status, 200); + assert.equal(res.attempts, 2); + assert.deepEqual( + calls.map((c) => c.method), + ["GET", "GET"], + ); + assert.equal(calls[0]?.url, "https://api.example/xrpc/a.b.c?x=1&x=2"); +}); + +test("retries once on a network failure, then reports status 0", async () => { + const { client, calls } = scripted(new Error("reset"), new Error("reset")); + const res = await client.get("a.b.c"); + assert.equal(res.status, 0); + assert.equal(res.attempts, 2); + assert.equal(calls.length, 2); +}); + +test("never retries a 4xx", async () => { + const { client, calls } = scripted(response(400, { error: "InvalidRequest" })); + const res = await client.get("a.b.c"); + assert.equal(res.attempts, 1); + assert.equal(calls.length, 1); +}); + +test("coverage guard reports missing and stale cases", () => { + assert.deepEqual(findCoverageGaps(["a", "b"], ["b", "c"]), { missing: ["a"], stale: ["c"] }); + assert.deepEqual(findCoverageGaps(["a"], ["a"]), { missing: [], stale: [] }); +}); + +test("runPool never exceeds the concurrency limit and keeps order", async () => { + let active = 0; + let peak = 0; + const tasks = Array.from({ length: 10 }, (_, i) => async () => { + active += 1; + peak = Math.max(peak, active); + await new Promise((resolve) => setTimeout(resolve, 2)); + active -= 1; + return i; + }); + assert.deepEqual(await runPool(tasks, 3), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + assert.ok(peak <= 3); +}); + +test("generic invariants flag oversized pages, small totals, reordered counts and uri mismatch", () => { + assert.deepEqual(genericInvariants({ items: [1, 2], totalCount: 2 }, { limit: 5 }), []); + assert.match(genericInvariants({ items: [1, 2, 3] }, { limit: 2 })[0] ?? "", /more than limit/); + assert.match(genericInvariants({ items: [1, 2], totalCount: 1 }, {})[0] ?? "", /totalCount/); + assert.match( + genericInvariants( + { counts: [{ subject: "b" }, { subject: "a" }] }, + { subjects: ["a", "b"] }, + )[0] ?? "", + /request order/, + ); + assert.match(genericInvariants({ record: { uri: "x" } }, { uri: "y" })[0] ?? "", /echo/); +}); + +test("every installed query has a smoke case (adding a query lexicon requires one)", async () => { + const gaps = findCoverageGaps(await installedQueryIds(), Object.keys(SMOKE_CASES)); + assert.deepEqual(gaps, { missing: [], stale: [] }); +}); diff --git a/happyview-modules/tooling/e2e.js b/happyview-modules/tooling/e2e.js new file mode 100644 index 0000000..1639500 --- /dev/null +++ b/happyview-modules/tooling/e2e.js @@ -0,0 +1,436 @@ +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + createHttpClient, + parseFlags, + renderSummary, + resolveBaseUrl, + writeStepSummary, +} from "./smoke-core.js"; + +/** + * Deep, read-only checks on live data: pagination walks, 100-subject batches and loose + * cross-endpoint count invariants. Ported from the one-off Python e2e run. + * + * Live data keeps changing, so no check pins a number. Counts are compared with `near` + * (a small absolute/relative drift allowance) and everything else is >=, <= or non-empty. + * Requests are sequential and paced; invalid-input behaviour is deliberately not tested here. + */ + +const PACE_MS = 300; +const TIMEOUT_MS = 30_000; +const MAX_PAGES = 60; +const FAKE_DID = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"; +const FAKE_URI = `at://${FAKE_DID}/org.hypercerts.collection/3zzzzzzzzzzzz`; +const GRAPH = "app.certified.graph"; +const FC = `${GRAPH}.getFollowerCounts`; +const CC = "org.hypercerts.context.getCommentCounts"; +const LB = "app.certified.badge.listBadgeAwards"; + +/** Counts from two live reads may differ by a little between requests. */ +export function near(a, b) { + return Math.abs(a - b) <= Math.max(3, 0.02 * Math.max(a, b)); +} + +/** Thrown to mark a check as not applicable in this environment. */ +class Skip extends Error {} +const need = (value, what) => { + if (value === undefined || value === null || (Array.isArray(value) && !value.length)) + throw new Skip(`no ${what} in this environment`); + return value; +}; +const expect = (condition, message) => { + if (!condition) throw new Error(message); +}; + +/** + * @param {ReturnType["get"]} get + * @param {string} nsid @param {Record} params @param {string} key @param {number} limit + */ +export async function walk(get, nsid, params, key, limit = 100, maxPages = MAX_PAGES) { + const items = []; + const totals = new Set(); + let cursor; + let pages = 0; + do { + const res = await get(nsid, { ...params, limit, ...(cursor ? { cursor } : {}) }); + expect(res.status === 200, `${nsid} page ${pages + 1} returned HTTP ${res.status}`); + pages += 1; + if (typeof res.json.totalCount === "number") totals.add(res.json.totalCount); + items.push(...res.json[key]); + cursor = res.json.cursor; + } while (cursor && pages < maxPages); + return { items, pages, totals: [...totals], complete: !cursor }; +} + +const unique = (xs) => new Set(xs).size === xs.length; + +/** Core of every walk check: no duplicates, and (when complete) totalCount agrees with the walk. */ +function expectConsistentWalk(w, idOf, label) { + const ids = w.items.map(idOf); + expect(unique(ids), `${label}: duplicate items across pages`); + if (w.complete && w.totals.length) { + const last = w.totals[w.totals.length - 1]; + expect(near(last, ids.length), `${label}: totalCount ${last} vs ${ids.length} items walked`); + } + return `walked ${ids.length} in ${w.pages} page(s)${w.complete ? "" : " (capped)"}`; +} + +export function buildChecks(get) { + /** @type {Record} */ + const s = {}; + /** @type {Array<[string, () => Promise]>} */ + const checks = []; + const check = (name, fn) => checks.push([name, fn]); + const getOk = async (nsid, params) => { + const res = await get(nsid, params); + expect(res.status === 200, `${nsid} returned HTTP ${res.status}`); + return res.json; + }; + + check("discover: recent follows", async () => { + const w = await walk(get, `${GRAPH}.listRecentFollows`, {}, "follows", 100, 3); + const dids = w.items + .map((f) => f.record?.subject) + .filter((x) => typeof x === "string" && x.startsWith("did:")); + const tally = new Map(); + for (const d of dids) tally.set(d, (tally.get(d) ?? 0) + 1); + s.actors = [...tally] + .sort((a, b) => b[1] - a[1]) + .slice(0, 8) + .map(([d]) => d); + s.entities = [...new Set(w.items.map((f) => f.record?.subject?.uri).filter(Boolean))].slice( + 0, + 4, + ); + need(s.actors, "followed accounts"); + return `${s.actors.length} accounts, ${s.entities.length} entities`; + }); + + check( + "getFollowerCounts: mixed batch keeps order, unknowns are 0, duplicates repeated", + async () => { + const subjects = [ + ...s.entities.slice(0, 2), + ...need(s.actors, "followed accounts"), + FAKE_DID, + FAKE_URI, + s.actors[0], + ]; + const { counts } = await getOk(FC, { subjects }); + expect( + JSON.stringify(counts.map((c) => c.subject)) === JSON.stringify(subjects), + "counts not in request order", + ); + expect( + counts.every((c) => Number.isInteger(c.followerCount) && c.followerCount >= 0), + "followerCount not a non-negative integer", + ); + const byKey = new Map(counts.map((c) => [c.subject, c.followerCount])); + expect( + byKey.get(FAKE_DID) === 0 && byKey.get(FAKE_URI) === 0, + "unknown subjects should count 0", + ); + expect( + counts.at(-1).followerCount === counts[s.entities.slice(0, 2).length].followerCount, + "duplicate subject got a different count", + ); + s.followerCounts = byKey; + return `${counts.length} counts`; + }, + ); + + check("getFollowerCounts: every recently followed account has >= 1 follower", async () => { + const { counts } = await getOk(FC, { subjects: need(s.actors, "followed accounts") }); + const zero = counts.filter((c) => c.followerCount < 1).map((c) => c.subject); + expect( + !zero.length, + `accounts from the recent-follows feed with 0 followers: ${zero.join(", ")}`, + ); + return `${counts.length} accounts`; + }); + + check("getFollowerCounts: 100 DIDs in one request", async () => { + const subjects = Array.from( + { length: 100 }, + (_, i) => `did:plc:${String(i).padStart(24, "0")}`, + ); + const { counts } = await getOk(FC, { subjects }); + expect(counts.length === 100, `expected 100 counts, got ${counts.length}`); + return "100 counts"; + }); + + check("getFollowerCounts: 50 realistic entity URIs in one request", async () => { + const subjects = Array.from( + { length: 50 }, + (_, i) => + `at://did:plc:${"a".repeat(24)}/org.hypercerts.collection/3mvnvx${String(i).padStart(7, "0")}`, + ); + const { counts } = await getOk(FC, { subjects }); + expect( + counts.length === 50 && counts.every((c) => c.followerCount === 0), + "unknown entities should all count 0", + ); + return "50 counts"; + }); + + check("listEntityFollowers walk vs getFollowerCounts", async () => { + const entity = need(s.entities, "followed entity")[0]; + const w = await walk(get, `${GRAPH}.listEntityFollowers`, { entity }, "followers"); + const note = expectConsistentWalk(w, (f) => f.did, "listEntityFollowers"); + const count = s.followerCounts?.get(entity); + if (w.complete && count !== undefined) + expect(near(count, w.items.length), `getFollowerCounts ${count} vs ${w.items.length} walked`); + return note; + }); + + check("listEntityFollowers small pages (limit=7) match the large-page walk", async () => { + const entity = need(s.entities, "followed entity")[0]; + const small = await walk(get, `${GRAPH}.listEntityFollowers`, { entity }, "followers", 7); + const note = expectConsistentWalk(small, (f) => f.did, "limit=7"); + expect( + small.totals.length <= 2, + `totalCount changed ${small.totals.length} times during the walk`, + ); + return note; + }); + + check("listActorFollowers walks vs getFollowerCounts", async () => { + const actors = need(s.actors, "followed accounts").slice(0, 3); + const notes = []; + for (const actor of actors) { + const w = await walk(get, `${GRAPH}.listActorFollowers`, { actor }, "followers"); + notes.push( + expectConsistentWalk(w, (f) => f.did ?? f.uri, `listActorFollowers ${actor.slice(-6)}`), + ); + const count = s.followerCounts?.get(actor); + if (w.complete && count !== undefined) + expect( + near(count, w.items.length), + `${actor.slice(-6)}: count ${count} vs ${w.items.length} walked`, + ); + } + return notes.join("; "); + }); + + check("discover: commented subjects", async () => { + const w = await walk( + get, + "org.hypercerts.context.listAttachments", + { contentTypes: "comment" }, + "attachments", + 100, + 5, + ); + const tally = new Map(); + for (const a of w.items) + for (const sub of a.record?.subjects ?? []) + if (sub.uri) tally.set(sub.uri, (tally.get(sub.uri) ?? 0) + 1); + s.commented = [...tally] + .sort((a, b) => b[1] - a[1]) + .slice(0, 6) + .map(([u]) => u); + need(s.commented, "comment attachments"); + return `${s.commented.length} subjects`; + }); + + check("getCommentCounts: batch order, unknown = 0, >= 1 for commented subjects", async () => { + const subjects = [...need(s.commented, "commented subjects"), FAKE_URI, s.commented[0]]; + const { counts } = await getOk(CC, { subjects }); + expect( + JSON.stringify(counts.map((c) => c.subject)) === JSON.stringify(subjects), + "counts not in request order", + ); + const bySubject = new Map(counts.map((c) => [c.subject, c.commentCount])); + expect(bySubject.get(FAKE_URI) === 0, "unknown subject should count 0"); + const low = s.commented.filter((u) => !(bySubject.get(u) >= 1)); + expect( + !low.length, + `subjects with comment attachments but commentCount < 1: ${low.join(", ")}`, + ); + s.commentCounts = bySubject; + return `${counts.length} counts`; + }); + + check("getCommentCounts vs listAttachments walk (limit=1)", async () => { + const notes = []; + for (const subject of need(s.commented, "commented subjects").slice(0, 3)) { + const w = await walk( + get, + "org.hypercerts.context.listAttachments", + { subjects: subject, contentTypes: "comment" }, + "attachments", + 1, + 30, + ); + expect(unique(w.items.map((a) => a.uri)), "duplicate attachments across pages"); + const count = s.commentCounts?.get(subject); + if (w.complete && count !== undefined) + expect(near(count, w.items.length), `count ${count} vs ${w.items.length} walked`); + notes.push(`${w.items.length}`); + } + return `walked ${notes.join("/")}`; + }); + + // 100 realistic ~80-character activity URIs hit a server-side limit (HTTP 500 on Test), so + // this check stays at 50; do not raise it without checking the current limit. + check("getCommentCounts: 50 subjects in one request", async () => { + const subjects = Array.from( + { length: 50 }, + (_, i) => + `at://did:plc:${"a".repeat(24)}/org.hypercerts.claim.activity/3mzfw${String(i).padStart(8, "0")}`, + ); + const { counts } = await getOk(CC, { subjects }); + expect(counts.length === 50, `expected 50 counts, got ${counts.length}`); + return "50 counts"; + }); + + check("listBadgeAwards: totalCount only when includeTotal=true", async () => { + const plain = await getOk(LB, { limit: 5 }); + expect(!("totalCount" in plain), "totalCount present without includeTotal"); + const off = await getOk(LB, { limit: 5, includeTotal: "false" }); + expect(!("totalCount" in off), "totalCount present with includeTotal=false"); + const on = await getOk(LB, { limit: 1, includeTotal: "true" }); + expect( + Number.isInteger(on.totalCount) && on.totalCount >= on.badgeAwards.length, + "includeTotal=true gave no usable totalCount", + ); + s.awardTotal = on.totalCount; + return `total ${on.totalCount}`; + }); + + check("listBadgeAwards: unknown author → empty page with totalCount 0", async () => { + const j = await getOk(LB, { authors: FAKE_DID, includeTotal: "true" }); + expect( + j.totalCount === 0 && j.badgeAwards.length === 0 && !j.cursor, + `expected an empty result, got ${JSON.stringify(j).slice(0, 120)}`, + ); + return "empty"; + }); + + check("listBadgeAwards: per-author walks at limit 7 and 100 agree with totalCount", async () => { + const page = await getOk(LB, { limit: 100 }); + const authors = [...new Set(page.badgeAwards.map((a) => a.did))].slice(0, 3); + need(authors, "badge award authors"); + const notes = []; + for (const author of authors) { + const walks = []; + for (const [limit, maxPages] of [ + [7, 25], + [100, MAX_PAGES], + ]) { + const w = await walk( + get, + LB, + { authors: author, includeTotal: "true" }, + "badgeAwards", + limit, + maxPages, + ); + notes.push( + expectConsistentWalk(w, (a) => a.uri, `author ${author.slice(-6)} limit=${limit}`), + ); + walks.push(w); + } + const [small, large] = walks; + if (small?.complete && large?.complete) + expect( + near(small.items.length, large.items.length), + `author ${author.slice(-6)}: limit=7 walked ${small.items.length}, limit=100 walked ${large.items.length}`, + ); + } + return notes.length ? `${authors.length} authors` : ""; + }); + + check("listBadgeAwards: sortDirection=asc walk reverses desc walk", async () => { + const page = await getOk(LB, { limit: 100 }); + const author = need(page.badgeAwards, "badge awards")[0].did; + const desc = await walk(get, LB, { authors: author, sortDirection: "desc" }, "badgeAwards"); + const asc = await walk(get, LB, { authors: author, sortDirection: "asc" }, "badgeAwards"); + if (!desc.complete || !asc.complete) throw new Skip("walk was capped"); + expect( + near(desc.items.length, asc.items.length), + `desc ${desc.items.length} vs asc ${asc.items.length} items`, + ); + if (desc.items.length === asc.items.length && desc.items.length < 100) + expect( + JSON.stringify(desc.items.map((a) => a.uri)) === + JSON.stringify(asc.items.map((a) => a.uri).reverse()), + "asc is not the reverse of desc", + ); + return `${desc.items.length} items`; + }); + + check("listBadgeAwards: responses=rejected walk agrees with totalCount", async () => { + const w = await walk(get, LB, { responses: "rejected", includeTotal: "true" }, "badgeAwards"); + return expectConsistentWalk(w, (a) => a.uri, "responses=rejected"); + }); + + for (const [nsid, key] of [ + ["org.hypercerts.claim.listActivities", "activities"], + ["org.hypercerts.collection.listCollections", "collections"], + ["org.hypercerts.context.listAttachments", "attachments"], + ]) { + check( + `${nsid}: small-page walk has no duplicates; asc and desc differ at the ends`, + async () => { + const w = await walk(get, nsid, {}, key, 7, 12); + const note = expectConsistentWalk(w, (i) => i.uri, nsid); + const desc = await getOk(nsid, { limit: 1, sortDirection: "desc" }); + const asc = await getOk(nsid, { limit: 1, sortDirection: "asc" }); + if (desc[key][0] && asc[key][0] && w.complete && w.items.length > 1) + expect(desc[key][0].uri !== asc[key][0].uri, "asc and desc start with the same record"); + return note; + }, + ); + } + + return checks; +} + +/** @param {string[]} argv */ +export async function main(argv) { + if (argv.includes("--help")) { + console.log( + "Usage: pnpm e2e [--base-url ]\nRead-only deep checks; API_BASE_URL is used when --base-url is not given.", + ); + return 0; + } + const baseUrl = resolveBaseUrl(parseFlags(argv, ["base-url"])); + const client = createHttpClient({ baseUrl, timeoutMs: TIMEOUT_MS, retries: 1, paceMs: PACE_MS }); + const started = Date.now(); + /** @type {import("./smoke-core.js").CheckResult[]} */ + const results = []; + for (const [name, fn] of buildChecks(client.get)) { + const t = Date.now(); + try { + results.push({ name, status: "pass", ms: Date.now() - t, detail: await fn() }); + } catch (error) { + const skipped = error instanceof Skip; + results.push({ + name, + status: skipped ? "skip" : "fail", + ms: Date.now() - t, + detail: /** @type {Error} */ (error).message, + }); + } + const last = results.at(-1); + console.log(`${last?.status.toUpperCase().padEnd(4)} ${name}: ${last?.detail}`); + } + const markdown = renderSummary("Nightly e2e", results, { + baseUrl, + requests: client.requestCount, + seconds: (Date.now() - started) / 1000, + }); + await writeStepSummary(markdown); + return results.some((r) => r.status === "fail") ? 1 : 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + try { + process.exitCode = await main(process.argv.slice(2)); + } catch (error) { + console.error(/** @type {Error} */ (error).message); + process.exitCode = 1; + } +} diff --git a/happyview-modules/tooling/smoke-cases.js b/happyview-modules/tooling/smoke-cases.js new file mode 100644 index 0000000..f94cfab --- /dev/null +++ b/happyview-modules/tooling/smoke-cases.js @@ -0,0 +1,252 @@ +/** + * One smoke case per installed query. Seeds (real records) are discovered at runtime from + * list endpoints, so nothing here pins data that could drift between environments. + * + * - phase 1: list endpoints that need no input; their first items become seeds. + * - phase 2: everything that needs a seed. A case whose seed is missing is skipped, not failed, + * because an environment may legitimately have no data of that kind. + * + * Output shape is validated generically against the lexicon (see smoke.js); `check` adds + * endpoint-specific invariants. + */ + +/** @typedef {Record} Seeds */ +/** @typedef {import("./smoke-core.js").QueryParams} QueryParams */ +/** + * @typedef {object} SmokeCase + * @property {1 | 2} phase + * @property {(seeds: Seeds) => QueryParams | undefined} params undefined = no seed, skip. + * @property {(json: any) => Seeds} [collect] Seeds to record from a phase 1 response. + * @property {(json: any, params: QueryParams) => string[]} [check] Extra invariants; [] = ok. + */ + +const NS = { + actor: "app.certified.actor", + badge: "app.certified.badge", + feed: "app.certified.feed", + graph: "app.certified.graph", + link: "app.certified.link", + location: "app.certified.location", + claim: "org.hypercerts.claim", + collection: "org.hypercerts.collection", + context: "org.hypercerts.context", + entity: "org.hypercerts.entity", + funding: "org.hypercerts.funding", + vocab: "org.hypercerts.vocab", + workscope: "org.hypercerts.workscope", +}; + +const LIST_PARAMS = () => ({ limit: 5 }); + +/** First word of at least 4 letters from the first string field found; a stable search term. */ +export function searchTermFrom(record, fields = ["title", "name", "displayName"]) { + for (const field of fields) { + const value = record?.[field]; + const word = typeof value === "string" ? /\p{L}{4,}/u.exec(value)?.[0] : undefined; + if (word) return word; + } + return undefined; +} + +/** Phase 1 list case that records the first item's URI under `seedKey` plus any `extra` seeds. */ +function listCase(arrayKey, seedKey, extra) { + return { + phase: 1, + params: LIST_PARAMS, + collect: (json) => ({ + [seedKey]: json[arrayKey]?.[0]?.uri, + ...extra?.(json[arrayKey] ?? []), + }), + }; +} + +/** Phase 2 `get*` case taking one record URI. */ +const getByUri = (seedKey) => ({ + phase: 2, + params: (s) => (s[seedKey] ? { uri: s[seedKey] } : undefined), +}); + +/** Phase 2 list/search case with optional seed-derived params. */ +const withSeed = (build, check) => ({ + phase: 2, + params: (s) => { + try { + return build(s); + } catch { + return undefined; + } + }, + check, +}); + +const nonEmpty = (key) => (json) => + (json[key]?.length ?? 0) > 0 + ? [] + : [`search for a term taken from an existing record returned no ${key}`]; + +/** @type {Record} */ +export const SMOKE_CASES = { + // ---- phase 1: lists that need no input ---- + [`${NS.actor}.listProfiles`]: listCase("profiles", "profileUri", (items) => ({ + actor: items[0]?.did, + profileTerm: searchTermFrom(items[0]?.record), + })), + [`${NS.actor}.listOrganizations`]: listCase("actors", "orgUri", (items) => ({ + org: items[0]?.did, + orgTerm: searchTermFrom(items[0]?.organization?.record, ["legalName"]), + })), + [`${NS.badge}.listBadgeAwards`]: listCase("badgeAwards", "badgeAward"), + [`${NS.badge}.listBadgeDefinitions`]: listCase( + "badgeDefinitions", + "badgeDefinition", + (items) => ({ + badgeTerm: searchTermFrom(items[0]?.record), + }), + ), + [`${NS.badge}.listBadgeResponses`]: listCase("badgeResponses", "badgeResponse"), + [`${NS.graph}.listRecentFollows`]: { + phase: 1, + params: LIST_PARAMS, + collect: (json) => { + const follows = json.follows ?? []; + return { + follower: follows[0]?.did, + followed: follows + .map((f) => f.record?.subject) + .find((subject) => typeof subject === "string" && subject.startsWith("did:")), + }; + }, + }, + [`${NS.link}.listEvmLinks`]: listCase("evmLinks", "evmLink"), + [`${NS.location}.listLocations`]: listCase("locations", "location"), + [`${NS.claim}.listActivities`]: listCase("activities", "activity", (items) => ({ + activityTerm: searchTermFrom(items[0]?.record), + })), + [`${NS.claim}.listContributions`]: listCase("contributions", "contribution"), + [`${NS.claim}.listContributorInformation`]: listCase( + "contributorInformation", + "contributorInformation", + ), + [`${NS.claim}.listRights`]: listCase("rights", "rights"), + [`${NS.collection}.listCollections`]: listCase("collections", "collection", (items) => ({ + collectionTerm: searchTermFrom(items[0]?.record), + })), + [`${NS.context}.listAcknowledgements`]: listCase("acknowledgements", "acknowledgement"), + [`${NS.context}.listAttachments`]: listCase("attachments", "attachment"), + [`${NS.context}.listEvaluations`]: listCase("evaluations", "evaluation"), + [`${NS.context}.listMeasurements`]: listCase("measurements", "measurement"), + [`${NS.entity}.listFeatures`]: listCase("features", "feature"), + [`${NS.funding}.listReceipts`]: listCase("receipts", "receipt"), + [`${NS.vocab}.listVocabTags`]: listCase("vocabTags", "vocabTag"), + [`${NS.workscope}.listWorkscopeTags`]: listCase("workscopeTags", "workscopeTag"), + + // ---- phase 2: get by URI ---- + [`${NS.badge}.getBadgeAward`]: getByUri("badgeAward"), + [`${NS.badge}.getBadgeDefinition`]: getByUri("badgeDefinition"), + [`${NS.badge}.getBadgeResponse`]: getByUri("badgeResponse"), + [`${NS.link}.getEvmLink`]: getByUri("evmLink"), + [`${NS.location}.getLocation`]: getByUri("location"), + [`${NS.claim}.getActivity`]: getByUri("activity"), + [`${NS.claim}.getContribution`]: getByUri("contribution"), + [`${NS.claim}.getContributorInformation`]: getByUri("contributorInformation"), + [`${NS.claim}.getRights`]: getByUri("rights"), + [`${NS.collection}.getCollection`]: getByUri("collection"), + [`${NS.context}.getAcknowledgement`]: getByUri("acknowledgement"), + [`${NS.context}.getAttachment`]: getByUri("attachment"), + [`${NS.context}.getEvaluation`]: getByUri("evaluation"), + [`${NS.context}.getMeasurement`]: getByUri("measurement"), + [`${NS.entity}.getFeature`]: getByUri("feature"), + [`${NS.funding}.getReceipt`]: getByUri("receipt"), + [`${NS.vocab}.getVocabTag`]: getByUri("vocabTag"), + [`${NS.workscope}.getWorkscopeTag`]: getByUri("workscopeTag"), + + // ---- phase 2: actors, graph, feed ---- + [`${NS.actor}.getProfile`]: withSeed((s) => ({ actor: need(s.actor) })), + [`${NS.actor}.getProfiles`]: withSeed((s) => ({ actors: [need(s.actor)] })), + [`${NS.actor}.getOrganization`]: withSeed((s) => ({ actor: need(s.org) })), + [`${NS.actor}.getOrganizations`]: withSeed((s) => ({ actors: [need(s.org)] })), + [`${NS.actor}.searchProfiles`]: withSeed( + (s) => ({ search: need(s.profileTerm) }), + nonEmpty("profiles"), + ), + [`${NS.actor}.searchOrganizations`]: withSeed( + (s) => ({ search: need(s.orgTerm) }), + nonEmpty("actors"), + ), + [`${NS.badge}.searchBadgeDefinitions`]: withSeed( + (s) => ({ search: need(s.badgeTerm) }), + nonEmpty("badgeDefinitions"), + ), + [`${NS.claim}.searchActivities`]: withSeed( + (s) => ({ search: need(s.activityTerm) }), + nonEmpty("activities"), + ), + [`${NS.collection}.searchCollections`]: withSeed( + (s) => ({ search: need(s.collectionTerm) }), + nonEmpty("collections"), + ), + [`${NS.collection}.listCollectionItems`]: withSeed((s) => ({ + collection: need(s.collection), + limit: 5, + })), + [`${NS.graph}.getFollow`]: withSeed((s) => ({ + actor: need(s.follower), + subject: need(s.followed), + })), + [`${NS.graph}.getFollows`]: withSeed((s) => ({ + actor: need(s.follower), + subjects: [need(s.followed)], + })), + [`${NS.graph}.getFollowerCounts`]: withSeed((s) => ({ + subjects: [need(s.followed), need(s.follower)], + })), + [`${NS.graph}.listActorFollowers`]: withSeed((s) => ({ actor: need(s.followed), limit: 5 })), + [`${NS.graph}.listActorFollowing`]: withSeed((s) => ({ actor: need(s.follower), limit: 5 })), + [`${NS.graph}.listEntityFollowers`]: withSeed((s) => ({ + entity: need(s.entity ?? s.collection), + limit: 5, + })), + [`${NS.graph}.listEntityFollowing`]: withSeed((s) => ({ + actor: need(s.entityFollower ?? s.follower), + limit: 5, + })), + [`${NS.graph}.getEntityFollow`]: withSeed((s) => ({ + actor: need(s.entityFollower), + entity: need(s.entity), + })), + [`${NS.graph}.getEntityFollows`]: withSeed((s) => ({ + actor: need(s.entityFollower), + entities: [need(s.entity)], + })), + [`${NS.feed}.getLikes`]: withSeed((s) => ({ subject: need(s.activity), limit: 5 })), + [`${NS.feed}.getReposts`]: withSeed((s) => ({ subject: need(s.activity), limit: 5 })), + [`${NS.feed}.getReactionCounts`]: withSeed((s) => ({ subjects: [need(s.activity)] })), + [`${NS.feed}.getActorLikes`]: withSeed((s) => ({ actor: need(s.follower), limit: 5 })), + [`${NS.feed}.getActorReposts`]: withSeed((s) => ({ actor: need(s.follower), limit: 5 })), + [`${NS.feed}.getActorReactions`]: withSeed((s) => ({ + actor: need(s.follower), + subjects: [need(s.activity)], + })), + [`${NS.context}.getCommentCounts`]: withSeed((s) => ({ subjects: [need(s.activity)] })), +}; + +/** Throws (caught by withSeed → skip) when a seed was not discovered. */ +function need(value) { + if (value === undefined || value === null || value === "") throw new Error("missing seed"); + return value; +} + +/** + * Entity follows have no list endpoint of their own, so take one from the recent-follows feed + * (it mixes account and entity follows). Returns no seeds when the feed has none; dependent + * cases then skip. + * @param {(nsid: string, params: QueryParams) => Promise<{ status: number; json: any }>} get + * @returns {Promise} + */ +export async function discoverEntityFollow(get) { + const res = await get(`${NS.graph}.listRecentFollows`, { limit: 100 }); + const follow = (res.status === 200 ? (res.json?.follows ?? []) : []).find( + (f) => f.record?.$type === "app.certified.graph.entityFollow" && f.record?.subject?.uri, + ); + return follow ? { entity: follow.record.subject.uri, entityFollower: follow.did } : {}; +} diff --git a/happyview-modules/tooling/smoke-core.js b/happyview-modules/tooling/smoke-core.js new file mode 100644 index 0000000..116b4ef --- /dev/null +++ b/happyview-modules/tooling/smoke-core.js @@ -0,0 +1,232 @@ +/** + * Shared building blocks for the read-only post-deploy checks (`smoke.js`, `e2e.js`). + * Everything here is anonymous and GET-only by construction: the HTTP client has no way to + * send another method, a body, or credentials. + */ + +/** @typedef {Record>} QueryParams */ +/** + * @typedef {object} HttpResult + * @property {string} url + * @property {number} status 0 when no HTTP response arrived (timeout or network failure). + * @property {number} ms Wall time of the final attempt. + * @property {number} attempts + * @property {any} json Parsed JSON body, or undefined when the body was not JSON. + * @property {string} [error] Transport failure description when status is 0. + */ + +export const DEFAULT_TIMEOUT_MS = 15_000; + +/** @param {string} raw @returns {string} origin-and-path base without a trailing slash */ +export function normalizeBaseUrl(raw) { + let url; + try { + url = new URL(raw); + } catch { + throw new Error( + `Invalid API base URL "${raw}"; pass --base-url or set API_BASE_URL to an https:// origin such as https://test.api.hypercerts.dev`, + ); + } + if (url.protocol !== "https:" && url.protocol !== "http:") + throw new Error(`API base URL must be http(s), got "${raw}"`); + if (url.username || url.password) + throw new Error("API base URL must not contain credentials; these checks are anonymous"); + let end = url.pathname.length; + while (end > 0 && url.pathname[end - 1] === "/") end -= 1; + return `${url.origin}${url.pathname.slice(0, end)}`; +} + +/** + * Hard guard: only anonymous GET requests to /xrpc/ query endpoints may leave this process. + * @param {string} method @param {string} url + */ +export function assertReadOnlyRequest(method, url) { + if (method !== "GET") + throw new Error(`Refusing ${method} ${url}: post-deploy checks only send GET requests`); + if (!new URL(url).pathname.includes("/xrpc/")) + throw new Error(`Refusing ${url}: post-deploy checks only call /xrpc/ query endpoints`); +} + +/** @param {QueryParams} params */ +function toSearch(params) { + const search = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + for (const item of Array.isArray(value) ? value : [value]) search.append(key, String(item)); + } + return search.toString(); +} + +/** @param {number} status */ +const isRetryableStatus = (status) => status === 0 || status >= 500; + +/** + * @param {object} options + * @param {string} options.baseUrl + * @param {number} [options.timeoutMs] + * @param {number} [options.retries] Extra attempts after a timeout, network error or 5xx. + * @param {number} [options.paceMs] Minimum gap between the start of consecutive requests. + * @param {typeof fetch} [options.fetchImpl] + * @param {(ms: number) => Promise} [options.sleep] + */ +export function createHttpClient({ + baseUrl, + timeoutMs = DEFAULT_TIMEOUT_MS, + retries = 1, + paceMs = 0, + fetchImpl = globalThis.fetch, + sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), +}) { + const base = normalizeBaseUrl(baseUrl); + let requests = 0; + let lastStart = 0; + + /** @param {string} url @returns {Promise>} */ + async function attempt(url) { + if (paceMs > 0) { + const wait = lastStart + paceMs - Date.now(); + lastStart = Date.now() + Math.max(wait, 0); + if (wait > 0) await sleep(wait); + } + const started = Date.now(); + requests += 1; + try { + const response = await fetchImpl(url, { + method: "GET", + headers: { accept: "application/json" }, + redirect: "error", + signal: AbortSignal.timeout(timeoutMs), + }); + const text = await response.text(); + /** @type {any} */ + let json; + try { + json = JSON.parse(text); + } catch { + json = undefined; + } + return { status: response.status, ms: Date.now() - started, json }; + } catch (cause) { + const name = cause instanceof Error ? cause.name : ""; + const reason = name === "TimeoutError" ? `timed out after ${timeoutMs}ms` : String(cause); + return { status: 0, ms: Date.now() - started, json: undefined, error: reason }; + } + } + + return { + get requestCount() { + return requests; + }, + /** + * One GET with a single retry on timeout/network error/5xx. 4xx is never retried. + * @param {string} nsid @param {QueryParams} [params] @returns {Promise} + */ + async get(nsid, params = {}) { + const search = toSearch(params); + const query = search ? `?${search}` : ""; + const url = `${base}/xrpc/${nsid}${query}`; + assertReadOnlyRequest("GET", url); + let result = await attempt(url); + let attempts = 1; + while (isRetryableStatus(result.status) && attempts <= retries) { + result = await attempt(url); + attempts += 1; + } + return { url, attempts, ...result }; + }, + }; +} + +/** + * Coverage guard. Installed queries without a case would ship unchecked; cases for queries + * that are not installed are stale. + * @param {string[]} installed @param {string[]} cased + */ +export function findCoverageGaps(installed, cased) { + const have = new Set(cased); + const want = new Set(installed); + return { + missing: installed.filter((nsid) => !have.has(nsid)).sort((a, b) => a.localeCompare(b)), + stale: cased.filter((nsid) => !want.has(nsid)).sort((a, b) => a.localeCompare(b)), + }; +} + +/** + * Runs async tasks with bounded concurrency; results keep input order. + * @template T @param {Array<() => Promise>} tasks @param {number} limit @returns {Promise} + */ +export async function runPool(tasks, limit) { + /** @type {T[]} */ + const results = new Array(tasks.length); + let next = 0; + async function worker() { + while (next < tasks.length) { + const index = next++; + const task = tasks[index]; + if (task) results[index] = await task(); + } + } + await Promise.all(Array.from({ length: Math.min(limit, tasks.length) }, worker)); + return results; +} + +/** @typedef {{ name: string; status: "pass" | "fail" | "skip" | "info"; ms: number; detail: string }} CheckResult */ + +/** @param {string} text */ +const cell = (text) => text.replaceAll("|", String.raw`\|`).replace(/\s+/g, " ").slice(0, 300); + +/** @param {string} title @param {CheckResult[]} results @param {{ baseUrl: string; requests: number; seconds: number }} meta */ +export function renderSummary(title, results, meta) { + const count = (/** @type {string} */ status) => results.filter((r) => r.status === status).length; + const lines = [ + `## ${title}`, + "", + `Target: \`${meta.baseUrl}\` · ${meta.requests} requests · ${meta.seconds.toFixed(1)}s · ` + + `${count("pass")} passed, ${count("fail")} failed, ${count("skip")} skipped`, + "", + "| Check | Result | ms | Detail |", + "| --- | --- | ---: | --- |", + ]; + const icon = { pass: "pass", fail: "**FAIL**", skip: "skip", info: "info" }; + // Failures first so they are visible without scrolling. + const ordered = [...results].sort( + (a, b) => Number(b.status === "fail") - Number(a.status === "fail"), + ); + for (const r of ordered) + lines.push(`| ${cell(r.name)} | ${icon[r.status]} | ${r.ms} | ${cell(r.detail)} |`); + return `${lines.join("\n")}\n`; +} + +/** @param {string} markdown */ +export async function writeStepSummary(markdown) { + const file = process.env.GITHUB_STEP_SUMMARY; + if (!file) return; + const { appendFile } = await import("node:fs/promises"); + await appendFile(file, `${markdown}\n`); +} + +/** @param {string[]} argv @param {string[]} flagsWithValue @returns {Record} */ +export function parseFlags(argv, flagsWithValue) { + /** @type {Record} */ + const flags = {}; + const supported = flagsWithValue.map((f) => `--${f} `).join(", "); + for (let i = 0; i < argv.length; i++) { + const arg = argv[i] ?? ""; + const name = arg.replace(/^--/, ""); + if (!arg.startsWith("--") || !flagsWithValue.includes(name)) + throw new Error(`Unknown option ${arg}; supported: ${supported}`); + const value = argv[++i]; + if (value === undefined) throw new Error(`Option ${arg} needs a value`); + flags[name] = value; + } + return flags; +} + +/** @param {Record} flags */ +export function resolveBaseUrl(flags) { + const raw = flags["base-url"] ?? process.env.API_BASE_URL; + if (!raw) + throw new Error( + "No API base URL: set API_BASE_URL or pass --base-url (for example https://test.api.hypercerts.dev)", + ); + return normalizeBaseUrl(raw); +} diff --git a/happyview-modules/tooling/smoke.js b/happyview-modules/tooling/smoke.js new file mode 100644 index 0000000..4b2be4a --- /dev/null +++ b/happyview-modules/tooling/smoke.js @@ -0,0 +1,230 @@ +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { Lexicons } from "@atproto/lexicon"; +import { loadAssets } from "./installer.js"; +import { discoverEntityFollow, SMOKE_CASES } from "./smoke-cases.js"; +import { + createHttpClient, + findCoverageGaps, + parseFlags, + renderSummary, + resolveBaseUrl, + runPool, + writeStepSummary, +} from "./smoke-core.js"; +import { validatePackageLexicons } from "./validate-lexicons.js"; + +const CONCURRENCY = 4; +const SLOW_MS = 5_000; +const USAGE = `Usage: pnpm smoke [--base-url ] [--only ] +Calls every installed query once (anonymous GET) and checks status, lexicon output schema +and basic invariants. API_BASE_URL is used when --base-url is not given.`; + +const packageRoot = fileURLToPath(new URL("../", import.meta.url)); + +/** Installed query NSIDs, from the same bundle manifest the installer uploads. */ +export async function installedQueryIds() { + const { assets } = await loadAssets(path.join(packageRoot, "manifest.json")); + return assets + .filter((asset) => asset.kind === "script" && asset.id.startsWith("xrpc.query:")) + .map((asset) => asset.id.slice("xrpc.query:".length)) + .sort((a, b) => a.localeCompare(b)); +} + +/** + * Output views embed whole records by ref, which Lexicons.validate rejects ("Unexpected lexicon + * type: record"). Validate the view shape and treat embedded records as opaque objects; the + * records themselves are the indexer's responsibility, not this deploy's. + * @param {any[]} documents @param {any} lexicons @returns {Lexicons} + */ +export function createViewValidator(documents, lexicons) { + const isRecordRef = (/** @type {string} */ ref, /** @type {string} */ docId) => + lexicons.getDef(ref.startsWith("#") ? `${docId}${ref}` : ref)?.type === "record"; + const rewrite = (/** @type {any} */ node, /** @type {string} */ docId) => { + if (Array.isArray(node)) return node.map((n) => rewrite(n, docId)); + if (!node || typeof node !== "object") return node; + if (node.type === "ref" && isRecordRef(node.ref, docId)) return { type: "unknown" }; + if (node.type === "union" && node.refs.some((/** @type {string} */ r) => isRecordRef(r, docId))) + return { type: "unknown" }; + return Object.fromEntries(Object.entries(node).map(([k, v]) => [k, rewrite(v, docId)])); + }; + const copy = documents.map((doc) => rewrite(doc, doc.id)); + return new Lexicons(copy); +} + +/** + * Cheap, data-independent invariants derived from the request and response. + * @param {any} json @param {Record} params @returns {string[]} + */ +export function genericInvariants(json, params) { + return [ + ...limitErrors(json, params.limit), + ...countErrors(json), + ...subjectOrderErrors(json, params.subjects), + ...uriEchoErrors(json, params.uri), + ]; +} + +/** @param {any} json @param {unknown} limit @returns {string[]} */ +function limitErrors(json, limit) { + if (typeof limit !== "number") return []; + return Object.entries(json ?? {}) + .filter(([, value]) => Array.isArray(value) && value.length > limit) + .map( + ([key, value]) => + `${key} has ${/** @type {any[]} */ (value).length} items, more than limit=${limit}`, + ); +} + +/** @param {any} json @returns {string[]} */ +function countErrors(json) { + const longest = Math.max( + 0, + ...Object.values(json ?? {}) + .filter(Array.isArray) + .map((a) => a.length), + ); + return ["totalCount", "likeCount", "repostCount"] + .filter((key) => typeof json?.[key] === "number" && json[key] < longest) + .map((key) => `${key} ${json[key]} is smaller than the ${longest} items returned`); +} + +/** @param {any} json @param {unknown} subjects @returns {string[]} */ +function subjectOrderErrors(json, subjects) { + if (!Array.isArray(json?.counts) || !Array.isArray(subjects)) return []; + const got = json.counts.map((/** @type {any} */ c) => c.subject); + return JSON.stringify(got) === JSON.stringify(subjects) + ? [] + : ["counts are not returned in request order (subject list differs)"]; +} + +/** @param {any} json @param {unknown} uri @returns {string[]} */ +function uriEchoErrors(json, uri) { + if (typeof uri !== "string") return []; + const record = /** @type {any} */ ( + Object.values(json ?? {}).find((v) => v && typeof v === "object" && "uri" in v) + ); + return record && record.uri !== uri + ? [`returned uri ${record.uri} does not echo requested ${uri}`] + : []; +} + +/** @param {any} json @returns {string[]} URIs of the first array of records, for overlap checks */ +function pageUris(json) { + const items = Object.values(json ?? {}).find(Array.isArray) ?? []; + return items.map((/** @type {any} */ i) => i?.uri).filter(Boolean); +} + +/** + * @param {string} nsid + * @param {import("./smoke-cases.js").SmokeCase} smokeCase + * @param {Record} seeds + * @param {{ get: ReturnType["get"]; lexicons: any }} deps + * @returns {Promise} + */ +async function runCase(nsid, smokeCase, seeds, { get, lexicons }) { + const params = smokeCase.params(seeds); + if (!params) + return { name: nsid, status: "skip", ms: 0, detail: "no seed record in this environment" }; + + const res = await get(nsid, params); + /** @type {string[]} */ + const errors = []; + if (res.status !== 200) { + const why = + res.status === 0 ? res.error : `${res.json?.error ?? ""} ${res.json?.message ?? ""}`.trim(); + errors.push(`HTTP ${res.status} after ${res.attempts} attempt(s): ${why}`); + } else if (res.json === undefined) { + errors.push("response body is not JSON"); + } else { + try { + lexicons.assertValidXrpcOutput(nsid, res.json); + } catch (error) { + errors.push(`output does not match lexicon: ${/** @type {Error} */ (error).message}`); + } + errors.push( + ...genericInvariants(res.json, params), + ...(smokeCase.check?.(res.json, params) ?? []), + ); + if (smokeCase.phase === 1 && !errors.length) + errors.push(...(await checkSecondPage(nsid, params, res.json, get))); + Object.assign(seeds, smokeCase.collect?.(res.json)); + } + const slow = res.ms > SLOW_MS ? ` (slow: ${res.ms}ms)` : ""; + return errors.length + ? { name: nsid, status: "fail", ms: res.ms, detail: errors.join("; ") } + : { name: nsid, status: "pass", ms: res.ms, detail: `ok${slow}` }; +} + +/** Page 2 must not repeat page 1. @returns {Promise} */ +async function checkSecondPage(nsid, params, first, get) { + if (typeof first.cursor !== "string") return []; + const second = await get(nsid, { ...params, cursor: first.cursor }); + if (second.status !== 200) return [`page 2 returned HTTP ${second.status}`]; + const seen = new Set(pageUris(first)); + const repeated = pageUris(second.json).filter((uri) => seen.has(uri)); + return repeated.length ? [`page 2 repeats ${repeated.length} item(s) from page 1`] : []; +} + +/** @param {string[]} argv */ +export async function main(argv) { + if (argv.includes("--help")) { + console.log(USAGE); + return 0; + } + const flags = parseFlags(argv, ["base-url", "only"]); + const baseUrl = resolveBaseUrl(flags); + const started = Date.now(); + + const installed = await installedQueryIds(); + const gaps = findCoverageGaps(installed, Object.keys(SMOKE_CASES)); + if (gaps.missing.length || gaps.stale.length) { + const lines = [ + ...gaps.missing.map((n) => ` installed query has no smoke case: ${n}`), + ...gaps.stale.map((n) => ` smoke case for a query that is not installed: ${n}`), + ]; + throw new Error( + `Smoke coverage is out of sync with happyview-modules/manifest.json:\n${lines.join("\n")}\nAdd or remove the case in tooling/smoke-cases.js.`, + ); + } + + const { lexicons: full, documents } = await validatePackageLexicons(packageRoot); + const lexicons = createViewValidator(documents, full); + const client = createHttpClient({ baseUrl }); + const deps = { get: client.get, lexicons }; + const only = flags.only; + /** @type {Record} */ + const seeds = {}; + const select = (/** @type {number} */ phase) => + installed.filter( + (nsid) => SMOKE_CASES[nsid]?.phase === phase && (!only || nsid.includes(only)), + ); + const tasks = (/** @type {string[]} */ ids) => + ids.map((nsid) => () => runCase(nsid, /** @type {any} */ (SMOKE_CASES[nsid]), seeds, deps)); + + const results = await runPool(tasks(select(1)), CONCURRENCY); + Object.assign(seeds, await discoverEntityFollow(client.get)); + results.push(...(await runPool(tasks(select(2)), CONCURRENCY))); + + const markdown = renderSummary("Post-deploy smoke", results, { + baseUrl, + requests: client.requestCount, + seconds: (Date.now() - started) / 1000, + }); + for (const r of results) { + if (r.status !== "pass") + console.log(`${r.status.toUpperCase().padEnd(4)} ${r.name}: ${r.detail}`); + } + console.log(markdown.split("\n")[2]); + await writeStepSummary(markdown); + return results.some((r) => r.status === "fail") ? 1 : 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + try { + process.exitCode = await main(process.argv.slice(2)); + } catch (error) { + console.error(/** @type {Error} */ (error).message); + process.exitCode = 1; + } +} diff --git a/package.json b/package.json index 635e30f..2897a66 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "format:check": "biome format .", "check": "pnpm lint && pnpm format:check && pnpm --filter @hypercerts-org/happyview-modules check && pnpm --filter hypercerts-api-endpoints test && pnpm --filter @hypercerts-org/api check", "test:unit": "pnpm --filter @hypercerts-org/happyview-modules run test:unit", + "smoke": "pnpm --filter @hypercerts-org/happyview-modules run smoke", + "e2e": "pnpm --filter @hypercerts-org/happyview-modules run e2e", "test:http": "pnpm --filter @hypercerts-org/happyview-modules run test:http", "build": "pnpm --filter @hypercerts-org/happyview-modules run build:lua && pnpm --filter hypercerts-api-endpoints build && pnpm --filter @hypercerts-org/api build", "docs:dev": "pnpm --filter hypercerts-api-endpoints dev",