From 4ed0c4a83ef86235309717c64eff1d74b42b06bf Mon Sep 17 00:00:00 2001 From: MK Date: Sat, 12 Sep 2026 09:49:42 +0800 Subject: [PATCH 01/17] feat(remote-cache): add self-hosted public cache server Co-authored-by: GPT-6 Codex --- .github/workflows/ci.yml | 18 + CHANGELOG.md | 1 + justfile | 3 + package.json | 3 +- packages/remote-cache/.gitignore | 5 + packages/remote-cache/README.md | 151 +++ packages/remote-cache/measurements/local.json | 130 ++ .../remote-cache/migrations/0001_cache.sql | 159 +++ packages/remote-cache/package.json | 33 + packages/remote-cache/scripts/benchmark.ts | 174 +++ packages/remote-cache/scripts/operator.ts | 579 +++++++++ packages/remote-cache/src/admission.ts | 17 + packages/remote-cache/src/auth.ts | 142 +++ packages/remote-cache/src/cbor.ts | 146 +++ packages/remote-cache/src/database.ts | 296 +++++ packages/remote-cache/src/errors.ts | 42 + packages/remote-cache/src/index.ts | 155 +++ packages/remote-cache/src/limits.ts | 42 + packages/remote-cache/src/multipart.ts | 116 ++ packages/remote-cache/src/observations.ts | 41 + packages/remote-cache/src/store.ts | 130 ++ packages/remote-cache/src/streams.ts | 135 +++ packages/remote-cache/test/failures.test.ts | 306 +++++ .../remote-cache/test/fixtures/protocol.json | 18 + packages/remote-cache/test/helpers.ts | 164 +++ packages/remote-cache/test/index.test.ts | 4 + packages/remote-cache/test/operator.test.ts | 184 +++ packages/remote-cache/test/protocol.test.ts | 102 ++ packages/remote-cache/test/service.test.ts | 410 +++++++ packages/remote-cache/tsconfig.json | 23 + packages/remote-cache/wrangler.jsonc | 30 + pnpm-lock.yaml | 1079 +++++++++++++++-- pnpm-workspace.yaml | 10 + 33 files changed, 4760 insertions(+), 88 deletions(-) create mode 100644 packages/remote-cache/.gitignore create mode 100644 packages/remote-cache/README.md create mode 100644 packages/remote-cache/measurements/local.json create mode 100644 packages/remote-cache/migrations/0001_cache.sql create mode 100644 packages/remote-cache/package.json create mode 100644 packages/remote-cache/scripts/benchmark.ts create mode 100644 packages/remote-cache/scripts/operator.ts create mode 100644 packages/remote-cache/src/admission.ts create mode 100644 packages/remote-cache/src/auth.ts create mode 100644 packages/remote-cache/src/cbor.ts create mode 100644 packages/remote-cache/src/database.ts create mode 100644 packages/remote-cache/src/errors.ts create mode 100644 packages/remote-cache/src/index.ts create mode 100644 packages/remote-cache/src/limits.ts create mode 100644 packages/remote-cache/src/multipart.ts create mode 100644 packages/remote-cache/src/observations.ts create mode 100644 packages/remote-cache/src/store.ts create mode 100644 packages/remote-cache/src/streams.ts create mode 100644 packages/remote-cache/test/failures.test.ts create mode 100644 packages/remote-cache/test/fixtures/protocol.json create mode 100644 packages/remote-cache/test/helpers.ts create mode 100644 packages/remote-cache/test/index.test.ts create mode 100644 packages/remote-cache/test/operator.test.ts create mode 100644 packages/remote-cache/test/protocol.test.ts create mode 100644 packages/remote-cache/test/service.test.ts create mode 100644 packages/remote-cache/tsconfig.json create mode 100644 packages/remote-cache/wrangler.jsonc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d8c3355b..240cd8f3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -383,6 +383,23 @@ jobs: - name: Check test utility types run: pnpm --filter vite-task-tools check + remote-cache: + needs: detect-changes + if: needs.detect-changes.outputs.code-changed == 'true' + name: Remote cache (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: + - namespace-profile-linux-x64-default + - namespace-profile-mac-default + - namespace-profile-windows-4c-8g + runs-on: ${{ matrix.os }} + steps: + - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 + - run: pnpm check-remote-cache + done: runs-on: namespace-profile-linux-x64-default if: always() @@ -393,6 +410,7 @@ jobs: - build-windows-tests - test-windows - fmt + - remote-cache steps: - run: exit 1 # Thank you, next https://github.com/vercel/next.js/blob/canary/.github/workflows/build_and_test.yml#L379 diff --git a/CHANGELOG.md b/CHANGELOG.md index 17a3d0c85..ccda6386e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Changelog +- **Added** A self-hosted public remote cache service with GitHub Actions write authorization, storage limits, and automatic cleanup ([#???](https://github.com/voidzero-dev/vite-task/pull/???)). - **Fixed** An invalid glob in `--filter` no longer shows its error message twice ([#763](https://github.com/voidzero-dev/vite-task/pull/763)). - **Changed** The detailed summary from `vp run --verbose` and `vp run --last-details` now shows each underlying cause of an error on its own line ([#761](https://github.com/voidzero-dev/vite-task/pull/761)). - **Added** Remote caching. Configure an endpoint with the workspace's `cache: { remote: { url } }` or `VP_REMOTE_CACHE_URL`, and choose access with `--remote-cache=off|read|read-write` or `VP_REMOTE_CACHE`. The default is `read` with an endpoint and `off` without one. After a local cache miss, `vp run` looks the task up in the remote cache and, on a hit, restores its outputs and caches it locally. The task output and the run summary show which hits came from the remote cache. A failed read is just a cache miss, with the failure as its reason. In `read-write` mode, `vp run` also uploads the results of successful, cacheable tasks after caching them locally. A failed upload doesn't fail the task; the run summary shows a warning instead. Ctrl-C, or a failing task, stops remote cache requests right away, and a task still being looked up doesn't start. Tasks can opt out with `cache: { remote: false }`. Requests use the proxy environment variables or, on macOS and Windows, the system proxy settings ([#727](https://github.com/voidzero-dev/vite-task/pull/727), [#755](https://github.com/voidzero-dev/vite-task/pull/755), [#756](https://github.com/voidzero-dev/vite-task/pull/756), [#757](https://github.com/voidzero-dev/vite-task/pull/757), [#764](https://github.com/voidzero-dev/vite-task/pull/764), [#771](https://github.com/voidzero-dev/vite-task/pull/771), [#772](https://github.com/voidzero-dev/vite-task/pull/772)). diff --git a/justfile b/justfile index 628d8123d..46fa43c95 100644 --- a/justfile +++ b/justfile @@ -38,6 +38,9 @@ watch-check: test: cargo test +remote-cache: + pnpm check-remote-cache + lint: cargo clippy --workspace --all-targets --all-features -- --deny warnings diff --git a/package.json b/package.json index f76d82e9e..7bc6d429a 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "prepare": "vp config", - "build-vite-task-client-types": "tsc -p packages/vite-task-client/tsconfig.json" + "build-vite-task-client-types": "tsc -p packages/vite-task-client/tsconfig.json", + "check-remote-cache": "pnpm --filter @voidzero-dev/remote-cache check && pnpm --filter @voidzero-dev/remote-cache smoke" }, "devDependencies": { "@tsconfig/strictest": "catalog:", diff --git a/packages/remote-cache/.gitignore b/packages/remote-cache/.gitignore new file mode 100644 index 000000000..0959f873a --- /dev/null +++ b/packages/remote-cache/.gitignore @@ -0,0 +1,5 @@ +.wrangler/ +.dev.vars* +operator-state.json +wrangler.operator.json +benchmark-results.json diff --git a/packages/remote-cache/README.md b/packages/remote-cache/README.md new file mode 100644 index 000000000..8717dd454 --- /dev/null +++ b/packages/remote-cache/README.md @@ -0,0 +1,151 @@ +# Public remote cache service + +This package implements the server in [RFC #716](https://github.com/voidzero-dev/vite-task/pull/716): a TypeScript Worker, primary D1 metadata, and a private R2 Standard bucket. Anyone can read an enabled namespace. Only a signed GitHub Actions token for the registered public repository's main-branch `push` job can publish. + +The package contains no `vp run` client adapter. Cache keys, values, and blobs remain opaque. A successful lookup does not prove that a result is reusable; a client must validate its inputs and output archive. + +## Protocol + +The endpoint is `https:///projects/`. Namespaces use 1–63 lowercase letters, digits, or hyphens, starting with a letter or digit. An operator can register up to 100 namespaces per deployment. + +| Request | Success | +| ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| `POST /fetch`, `application/cbor`, `{ key: bytes, secondary_key: bytes }` | CBOR `{ kind: "exact", value: bytes, blob_id: string \| null }`, or `{ kind: "fallback", key: bytes, value: bytes, blob_id: string \| null }` | +| `GET /blob/{blob_id}` | Raw bytes with `application/octet-stream` | +| `POST /store`, `multipart/form-data` | CBOR `{ blob_id: string \| null }` | + +`/store` requires one `metadata` part with content type `application/cbor` and fields `{ key: bytes, secondary_key: bytes, value: bytes }`. An optional `blob` part has content type `application/octet-stream`. Either part order works. An omitted blob returns `null`; an empty blob gets an ID. Duplicate parts, unknown parts, duplicate envelope fields, extra fields, invalid types, and truncated bodies are rejected. + +Empty and non-UTF-8 keys work. CBOR definite and indefinite maps and byte strings work, including noncanonical lengths. The bounded envelope decoder supports only the protocol's map and string types. It never decodes `value` contents. Chunked strings have a 4,096-chunk bookkeeping limit. + +Fetch gives exact matches priority. A fallback follows the latest secondary-key association. A store replaces both mappings atomically. Reassigning a secondary key does not delete its former target. Replacing an entry changes the value seen through every association to that entry. + +**Draft contract difference:** this implementation follows RFC #716 and returns `404` for a fetch miss. The still-open API draft [#713 at `362f5bd9`](https://github.com/voidzero-dev/vite-task/blob/362f5bd91bb32806b512d3d9a5339ff435bf5f0a/docs/remote-cache-server-api.md) instead specifies HTTP `200` with `kind: "not_found"`. Reconcile the drafts before releasing a client against this service. + +Errors have `Content-Type: text/plain; charset=utf-8`. Codes are `400` for invalid input, `401` for invalid/missing/expired tokens, `403` for a signed token that fails write policy, `404` for absent data or unavailable namespaces/routes, `413` for size limits, `429` for admission limits, `500` for an incomplete operation, and `503` for unavailable authorization/storage, failed publication guards, quotas, or concurrency admission. `429` and `503` include `Retry-After: 60`. + +All responses disable caching. No CDN cache, R2 public URL, presigned URL, administrative HTTP route, or authentication redirect is exposed. The request host is never used as the JWT audience. + +## Local checks + +Use Node.js 22.12 or newer and the repository's pinned pnpm version. From the repository root: + +```sh +pnpm install --frozen-lockfile +pnpm check-remote-cache +``` + +`just remote-cache` runs the same checks. The command generates binding/runtime types, checks all source, operator, benchmark, and test files, runs isolated workerd/D1/R2 tests, and bundles a deployment dry run. Tests generate a temporary RSA key and intercept only GitHub's fixed JWKS URL. They need no Cloudflare account, GitHub credentials, or client adapter. CI runs them on Linux, macOS, and Windows. + +From this package directory, `pnpm dev` starts Wrangler with local bindings. Apply the local schema with `pnpm exec wrangler d1 migrations apply INDEX --local`. No namespace is enabled in the template. Use `pnpm test` for a fully initialized, isolated smoke test; it exercises authorized writes without creating a development signing-key bypass in production. + +## Setup + +Use a dedicated Worker, D1 database, and bucket. The operator requires `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` in its environment. The token needs account permissions for Workers Scripts, D1, and Workers R2 Storage, plus route/zone permissions if using a custom domain. Enable R2 in the account first. Credentials stay in the operator process and Wrangler; they are never stored in namespace policy or passed as command arguments. + +Run from this package directory: + +```sh +pnpm operator setup --name my-public-cache --namespace docs --repo owner/repository --origin https://my-public-cache.account-subdomain.workers.dev +``` + +Use the account's actual Workers subdomain. For a custom domain, set `--origin https://cache.example.com`. Setup disables the unused `workers.dev` alias for a custom domain and disables preview URLs. It rejects an R2 bucket with public custom domains and disables its `r2.dev` access. + +Setup discovers resources by name, creates missing resources, applies migrations, resolves public repository and owner IDs through GitHub, records the default branch and exact audience, installs lifecycle/Cron settings, deploys, and prints the endpoint. Repeated setup does not duplicate resources or re-enable a withdrawn namespace. Reusing an existing namespace for a different repository is rejected. Configuration is written to the ignored `wrangler.operator.json`; keep a secure backup of this non-secret file. + +The default `free` **resource profile** retains the RFC's 8 GB budget, 20,000 entries, 20,000 associations, and 16 generations per Cron run. It is not proof of compatibility with the Workers Free CPU limit. **Use Workers Paid for the full payload limits until production CPU measurements establish a supported Free profile.** Local measurements already exceed 10 ms in several cases. Setup does not purchase or change a Workers subscription. + +`--profile paid` selects 256 generations per Cron run. It does not increase storage or retention. Select those independently with `--byte-limit` and `--retention-days`. Retention can be 1–365 days. For example: + +```sh +pnpm operator setup --name my-public-cache --namespace docs --repo owner/repository --origin https://cache.example.com --profile paid --retention-days 30 --byte-limit 30000000000 +``` + +Set GitHub Actions `id-token: write` only on the trusted publishing job. The token audience must equal the printed endpoint. No cache secret is needed in GitHub. A token with a customized `sub` can work because policy uses signed IDs, branch, visibility, and event claims. Tokens for fork repositories, tags, PRs, `pull_request_target`, or `workflow_run` are denied. + +## Limits and authorization + +| Resource | Default maximum | +| -------------------------------------------- | ----------------- | +| Each key | 16 KiB | +| Opaque value | 4 MiB | +| Store metadata | 5 MiB | +| Fetch request | 40 KiB | +| Blob | 64 MiB | +| Entire store request | 72 MiB | +| Multipart headers per part | 8 KiB | +| Multipart parts | 2 | +| R2 upload part | 5 MiB, sequential | +| Store request deadline | 2 minutes | +| Metadata / blob lookup deadline | 15 seconds | +| Upload lease | 15 minutes | +| Replacement/late-upload grace | 10 minutes | +| Concurrent buffered stores/reads per isolate | 2 / 4 | + +The `LIMITS` Wrangler variable is a JSON object with optional keys `key`, `value`, `metadata`, `fetch`, `blob`, `store`, `headers`, and `deadlineMs`. It can lower the tested ceilings. Envelope limits must accommodate their field limits and framing. Raising ceilings requires code changes and new memory, CPU, and account-limit measurements. An HTTP request without `Content-Length` reserves the full store limit. Actual bytes replace that reservation only at publication; pending, retired, and deleting generations remain charged. + +GitHub authorization uses `jose`, `RS256`, the fixed issuer `https://token.actions.githubusercontent.com`, and its fixed HTTPS JWKS endpoint. The server checks exact string types for audience, repository/owner IDs, visibility, branch, ref type, and event. It requires integer `exp`, `nbf`, and `iat`, permits 30 seconds of skew for not-before/issued-at checks, accepts at most a 15-minute token lifetime, and never publishes after expiry. Token headers cannot supply signing-key URLs or embedded keys. + +Tokens are limited to 16 KiB. JWKS bodies are limited to 64 KiB, requests to five seconds, cached keys to ten minutes, and refresh attempts to one per 30 seconds per isolate, including failures. Unknown key IDs cannot cause unlimited refreshes. Missing usable keys fail closed. A maintainer's policy change increments its version; publication checks that version, current scope/deployment state, token expiry, lease, bytes, and identity quotas inside the D1 transaction. + +The rate bindings apply before database or object access. Defaults are 600 requests/minute for each namespace's fetch/blob operation and 30 store attempts/minute. Invalid credentials consume store admission too. Unknown routes share catch-all identities. Limits are approximate per Cloudflare location; they are not a global billing cap. Reads do not write D1 counters or refresh retention. + +## Operations + +```sh +pnpm operator status +pnpm operator bind --namespace another-project --repo owner/another-repository +pnpm operator bind --namespace docs --repo owner/repository +pnpm operator policy --namespace docs --writes off +pnpm operator policy --namespace docs --enabled off +pnpm operator policy --namespace docs --enabled on --writes on +pnpm operator policy --namespace docs --retention-days 30 --byte-limit 30000000000 +pnpm operator deployment --byte-limit 30000000000 +pnpm operator deployment --writes off +pnpm operator deployment --enabled off +pnpm operator upgrade +``` + +`bind` refreshes repository display data, owner ID, and default branch for an existing repository, or registers a new namespace. It never silently moves existing public data to a different repository. Per-scope and deployment byte/entry/association limits all apply; use `--entry-limit` and `--association-limit` to change them explicitly. Lowering a limit below current use blocks further publication until cleanup or an operator change restores capacity. + +`status` reports policy, charged bytes, entry/association counts, generation states, cleanup eligibility, and actual D1 storage. It warns at 400 MB. Set provider alerts at 80% of request, CPU, R2 operation/storage, and D1 read/write/storage allowances. Monitor cleanup delay and pending bytes, and pause writes before a backlog reaches the storage ceiling. Public requests and other services in the same account can exhaust allowances even when this cache's byte budget is respected. + +Sampled structured logs include request ID, namespace, operation, exact/fallback/miss/error outcome, HTTP status, request/response sizes, duration, D1 rows/query latency/storage, and R2 operation attempts. Verified writes add signed repository ID, workflow ref, run ID/attempt, and commit SHA. They exclude tokens, keys, values, blobs, and database errors. `LOG_SAMPLE_RATE` defaults to `0.1` and applies to failures too. Provider transfer metrics account for disconnected downloads; logged response size describes the selected response. A server exact hit is distinct from a client-validated cache hit. + +### Cleanup and withdrawal + +Every five minutes Cron claims a bounded indexed set of eligible generations, aborts known multipart uploads, deletes immutable object names, then releases charges. Failed deletion is retried after ten minutes. Conditional claims and generation-specific foreign keys preserve concurrent replacement entries. An indexed cursor scans and deletes orphan associations in bounded batches. + +Current entries expire seven days after commit by default. Replaced blob IDs retain their original bytes for ten minutes. Abandoned uploads get a late-operation grace period after their lease ends. R2 lifecycle rules abort unfinished multipart uploads after one day and expire objects after retention plus two days (9/32 days for 7/30-day retention). A retention high-water mark prevents later policy reductions from deleting older generations early. Lifecycle covers the crash window between multipart creation and recording its ID; it supplements D1 accounting. + +To withdraw and delete one namespace: + +```sh +pnpm operator purge --namespace docs --confirm docs +``` + +To delete all deployment data and resources: + +```sh +pnpm operator teardown --confirm my-public-cache +``` + +Teardown first disables access and writes and schedules deletion. If generations remain, it exits with a message to check `status` and repeat after Cron drains them. It keeps Cron and accounting available until R2 accepts deletion of the empty bucket, then deletes D1 and the Worker. Lifecycle may need to finish orphan cleanup before the bucket is empty. These commands require the operator's Cloudflare credentials; no HTTP caller can administer the service. + +Public data can include logs, source maps, and input metadata. Only publish results intended for public distribution. Making a GitHub repository private does not withdraw existing cache data; disable/purge its namespace. Disabling writes or changing policy stops pending publication at the D1 guard. Job cancellation alone does not revoke an already issued bearer token, and downloaded public copies cannot be recalled. + +### Upgrade and recovery + +Keep the lockfile and compatibility date pinned. `upgrade` applies additive migrations before deploying the Worker. Back up `wrangler.operator.json` and the `scopes`/`deployment` policy rows separately from disposable cache data. `status` exports readable policy data. Before a migration that changes the schema contract, retain a compatible Worker bundle; do not roll back to code that predates a required schema change. + +A D1 restore cannot restore deleted R2 objects. Missing live values return `503`; missing blobs return `404`. After partial recovery, create a new namespace and retire the old one, or reconcile the generation records and objects before enabling it. Do not restore stale policy over an intentional withdrawal. + +## Measurements and release checks + +Run `pnpm benchmark` for isolated workerd profiling of cold/warm JWKS stores, 250 KB and 4 MiB values, and 5/20/50 MB plus 64 MiB blobs at concurrency one and two. It writes ignored `benchmark-results.json`. The initial run is in [measurements/local.json](measurements/local.json). + +The benchmark records wall time, V8 CPU samples, and heap/backing-store observations. Codec encode/decode CPU is measured separately in Node. Sampling is diagnostic, excludes some native runtime work, and is not Cloudflare's CPU billing metric. Post-operation heap readings are not peak isolate memory. Maximum-sized concurrent requests also pass the local runtime's memory checks, but production CPU, memory, R2/D1 latency, and usage still require a limited trial before a Free-plan release claim. Use [Cloudflare's CPU profiling tools](https://developers.cloudflare.com/workers/observability/dev-tools/cpu-usage/) and deployed request metrics for that gate. + +The tests cover protocol fixtures, every multipart split position, binary identity semantics, all write-policy claim classes, JWKS outages/cooldowns, no-blob/empty-blob behavior, atomic concurrent stores, failed publication guards, quota accounting, R2 failures, unknown-length cancellation, namespace withdrawal, and deletion retries. No platform is skipped. The Worker deployment template and operator commands are checked locally; actual resource provisioning requires an operator account. + +Implementation references: [D1 transactions](https://developers.cloudflare.com/d1/worker-api/d1-database/), [R2 Worker API](https://developers.cloudflare.com/r2/api/workers/workers-api-reference/), [lifecycle rules](https://developers.cloudflare.com/r2/buckets/object-lifecycles/), and [rate-limit bindings](https://developers.cloudflare.com/workers/runtime-apis/bindings/rate-limit/). diff --git a/packages/remote-cache/measurements/local.json b/packages/remote-cache/measurements/local.json new file mode 100644 index 000000000..ef3a0459a --- /dev/null +++ b/packages/remote-cache/measurements/local.json @@ -0,0 +1,130 @@ +{ + "measured_at": "2026-09-11T16:35:37.230Z", + "platform": "darwin", + "arch": "arm64", + "cpu": "Apple M4 Max", + "node": "v22.21.0", + "runtime": "workerd 1.20260911.1", + "concurrency": { + "stores": 2, + "metadata_reads": 4 + }, + "free_cpu_verified": false, + "caveat": "Local V8 sampling is diagnostic. It excludes some native work and is not provider CPU billing. Validate production CPU before enabling a Free release profile.", + "measurements": [ + { + "name": "cold-jwks", + "wall_ms": 40.12, + "sampled_cpu_ms": 28.731, + "heap_bytes": 1552516, + "backing_storage_bytes": 6097748 + }, + { + "name": "warm-jwks", + "wall_ms": 9.59, + "sampled_cpu_ms": 8.312, + "heap_bytes": 1343620, + "backing_storage_bytes": 5529717 + }, + { + "name": "cbor-decode-250000", + "node_cpu_ms_per_operation": 0.00419 + }, + { + "name": "cbor-encode-250000", + "node_cpu_ms_per_operation": 0.06711 + }, + { + "name": "fetch-value-250000", + "wall_ms": 8.48, + "sampled_cpu_ms": 7.78, + "heap_bytes": 1608140, + "backing_storage_bytes": 11293169 + }, + { + "name": "fetch-response-250000", + "wall_ms": 7.23, + "sampled_cpu_ms": 5.831, + "heap_bytes": 1852152, + "backing_storage_bytes": 16294767 + }, + { + "name": "cbor-decode-4194304", + "node_cpu_ms_per_operation": 0.00368 + }, + { + "name": "cbor-encode-4194304", + "node_cpu_ms_per_operation": 0.27475 + }, + { + "name": "fetch-value-4194304", + "wall_ms": 50.1, + "sampled_cpu_ms": 45.651, + "heap_bytes": 3161464, + "backing_storage_bytes": 7968897 + }, + { + "name": "fetch-response-4194304", + "wall_ms": 17.15, + "sampled_cpu_ms": 13.373, + "heap_bytes": 4710864, + "backing_storage_bytes": 18648273 + }, + { + "name": "store-5000000-concurrency-1", + "wall_ms": 99.68, + "sampled_cpu_ms": 34.529, + "heap_bytes": 6111500, + "backing_storage_bytes": 20504341 + }, + { + "name": "store-5000000-concurrency-2", + "wall_ms": 197.69, + "sampled_cpu_ms": 193.841, + "heap_bytes": 12435204, + "backing_storage_bytes": 38597083 + }, + { + "name": "store-20000000-concurrency-1", + "wall_ms": 283.25, + "sampled_cpu_ms": 280.233, + "heap_bytes": 19072604, + "backing_storage_bytes": 41544287 + }, + { + "name": "store-20000000-concurrency-2", + "wall_ms": 504.93, + "sampled_cpu_ms": 458.933, + "heap_bytes": 19583376, + "backing_storage_bytes": 48028243 + }, + { + "name": "store-50000000-concurrency-1", + "wall_ms": 560.81, + "sampled_cpu_ms": 472.501, + "heap_bytes": 18188432, + "backing_storage_bytes": 28738528 + }, + { + "name": "store-50000000-concurrency-2", + "wall_ms": 1103.99, + "sampled_cpu_ms": 896.194, + "heap_bytes": 19111428, + "backing_storage_bytes": 34509917 + }, + { + "name": "store-67108864-concurrency-1", + "wall_ms": 773.37, + "sampled_cpu_ms": 729.105, + "heap_bytes": 17239424, + "backing_storage_bytes": 55903659 + }, + { + "name": "store-67108864-concurrency-2", + "wall_ms": 1443.97, + "sampled_cpu_ms": 1207.561, + "heap_bytes": 21261328, + "backing_storage_bytes": 40246099 + } + ] +} diff --git a/packages/remote-cache/migrations/0001_cache.sql b/packages/remote-cache/migrations/0001_cache.sql new file mode 100644 index 000000000..2d9d8f145 --- /dev/null +++ b/packages/remote-cache/migrations/0001_cache.sql @@ -0,0 +1,159 @@ +-- Primary D1 owns policy and accounting. Back up policies separately from cache data. +CREATE TABLE deployment ( + id INTEGER PRIMARY KEY CHECK (id = 1), + enabled INTEGER NOT NULL DEFAULT 1 CHECK (enabled IN (0, 1)), + writes_enabled INTEGER NOT NULL DEFAULT 1 CHECK (writes_enabled IN (0, 1)), + byte_limit INTEGER NOT NULL DEFAULT 8000000000 CHECK (byte_limit > 0), + entry_limit INTEGER NOT NULL DEFAULT 20000 CHECK (entry_limit > 0), + association_limit INTEGER NOT NULL DEFAULT 20000 CHECK (association_limit > 0), + retention_high_water_seconds INTEGER NOT NULL DEFAULT 604800, + charged_bytes INTEGER NOT NULL DEFAULT 0 CHECK (charged_bytes >= 0), + entry_count INTEGER NOT NULL DEFAULT 0 CHECK (entry_count >= 0), + association_count INTEGER NOT NULL DEFAULT 0 CHECK (association_count >= 0) +); +INSERT INTO deployment (id) VALUES (1); +CREATE TABLE scopes ( + scope_id TEXT PRIMARY KEY, + endpoint TEXT NOT NULL UNIQUE, + repository TEXT NOT NULL, + repository_id TEXT NOT NULL, + repository_owner_id TEXT NOT NULL, + branch TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1 CHECK (enabled IN (0, 1)), + writes_enabled INTEGER NOT NULL DEFAULT 1 CHECK (writes_enabled IN (0, 1)), + policy_version INTEGER NOT NULL DEFAULT 1, + retention_seconds INTEGER NOT NULL DEFAULT 604800 CHECK (retention_seconds > 0), + byte_limit INTEGER NOT NULL DEFAULT 8000000000 CHECK (byte_limit > 0), + entry_limit INTEGER NOT NULL DEFAULT 20000 CHECK (entry_limit > 0), + association_limit INTEGER NOT NULL DEFAULT 20000 CHECK (association_limit > 0), + charged_bytes INTEGER NOT NULL DEFAULT 0 CHECK (charged_bytes >= 0), + entry_count INTEGER NOT NULL DEFAULT 0 CHECK (entry_count >= 0), + association_count INTEGER NOT NULL DEFAULT 0 CHECK (association_count >= 0) +); +CREATE TABLE generations ( + generation_id TEXT PRIMARY KEY, + scope_id TEXT NOT NULL REFERENCES scopes(scope_id), + state TEXT NOT NULL CHECK (state IN ('uploading', 'ready', 'retired', 'deleting')), + policy_version INTEGER NOT NULL, + token_exp INTEGER NOT NULL, + lease_until INTEGER NOT NULL, + gc_after INTEGER NOT NULL, + gc_claim TEXT, + key BLOB, + secondary_key BLOB, + value_object TEXT NOT NULL UNIQUE, + blob_object TEXT NOT NULL UNIQUE, + blob_id TEXT, + upload_id TEXT, + value_size INTEGER NOT NULL DEFAULT 0 CHECK (value_size >= 0), + blob_size INTEGER NOT NULL DEFAULT 0 CHECK (blob_size >= 0), + charged_bytes INTEGER NOT NULL CHECK (charged_bytes >= 0), + expires_at INTEGER, + retired_at INTEGER, + UNIQUE (scope_id, generation_id) +); +CREATE UNIQUE INDEX generations_blob ON generations(scope_id, blob_id) WHERE blob_id IS NOT NULL; +CREATE INDEX generations_gc ON generations(gc_after); +CREATE TABLE entries ( + scope_id TEXT NOT NULL REFERENCES scopes(scope_id), + key BLOB NOT NULL CHECK (typeof(key) = 'blob'), + generation_id TEXT NOT NULL, + PRIMARY KEY (scope_id, key), + FOREIGN KEY (scope_id, generation_id) REFERENCES generations(scope_id, generation_id) ON DELETE CASCADE +) WITHOUT ROWID; +CREATE INDEX entries_generation ON entries(generation_id); +CREATE TABLE associations ( + scope_id TEXT NOT NULL REFERENCES scopes(scope_id), + secondary_key BLOB NOT NULL CHECK (typeof(secondary_key) = 'blob'), + target_key BLOB NOT NULL CHECK (typeof(target_key) = 'blob'), + PRIMARY KEY (scope_id, secondary_key) +) WITHOUT ROWID; +CREATE INDEX associations_target ON associations(scope_id, target_key); +CREATE TABLE maintenance (id INTEGER PRIMARY KEY CHECK (id = 1), scope_id TEXT NOT NULL, secondary_key BLOB NOT NULL); +INSERT INTO maintenance VALUES (1, '', X''); + +-- Lifecycle rules must never shorten the life of an already published generation. +CREATE TRIGGER initial_retention AFTER INSERT ON scopes BEGIN + UPDATE deployment SET retention_high_water_seconds = max(retention_high_water_seconds, NEW.retention_seconds); +END; +CREATE TRIGGER increased_retention AFTER UPDATE OF retention_seconds ON scopes BEGIN + UPDATE deployment SET retention_high_water_seconds = max(retention_high_water_seconds, NEW.retention_seconds); +END; +CREATE TRIGGER scope_limit BEFORE INSERT ON scopes +WHEN NOT EXISTS (SELECT 1 FROM scopes WHERE scope_id = NEW.scope_id) +BEGIN + SELECT CASE WHEN (SELECT count(*) FROM scopes) >= 100 THEN RAISE(ABORT, 'cache_scope_limit') END; +END; + +-- Direct policy edits must invalidate uploads that captured the previous policy. +CREATE TRIGGER changed_policy AFTER UPDATE OF endpoint, repository_id, repository_owner_id, branch, + enabled, writes_enabled, retention_seconds, byte_limit, entry_limit, association_limit ON scopes +WHEN NEW.policy_version = OLD.policy_version +BEGIN + UPDATE scopes SET policy_version = OLD.policy_version + 1 WHERE scope_id = NEW.scope_id; +END; + +CREATE TRIGGER reserve_capacity BEFORE INSERT ON generations BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM scopes s, deployment d WHERE s.scope_id = NEW.scope_id + AND s.enabled = 1 AND s.writes_enabled = 1 AND d.enabled = 1 AND d.writes_enabled = 1 + AND s.policy_version = NEW.policy_version AND NEW.token_exp > unixepoch() + AND s.charged_bytes + NEW.charged_bytes <= s.byte_limit + AND d.charged_bytes + NEW.charged_bytes <= d.byte_limit + ) THEN RAISE(ABORT, 'cache_admission_denied') END; +END; +CREATE TRIGGER charge_generation AFTER INSERT ON generations BEGIN + UPDATE scopes SET charged_bytes = charged_bytes + NEW.charged_bytes WHERE scope_id = NEW.scope_id; + UPDATE deployment SET charged_bytes = charged_bytes + NEW.charged_bytes; +END; +CREATE TRIGGER adjust_generation AFTER UPDATE OF charged_bytes ON generations BEGIN + UPDATE scopes SET charged_bytes = charged_bytes + NEW.charged_bytes - OLD.charged_bytes WHERE scope_id = NEW.scope_id; + UPDATE deployment SET charged_bytes = charged_bytes + NEW.charged_bytes - OLD.charged_bytes; +END; +CREATE TRIGGER release_generation AFTER DELETE ON generations BEGIN + UPDATE scopes SET charged_bytes = charged_bytes - OLD.charged_bytes WHERE scope_id = OLD.scope_id; + UPDATE deployment SET charged_bytes = charged_bytes - OLD.charged_bytes; +END; + +CREATE TRIGGER count_entry BEFORE INSERT ON entries +WHEN NOT EXISTS (SELECT 1 FROM entries WHERE scope_id = NEW.scope_id AND key = NEW.key) +BEGIN + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM scopes s, deployment d WHERE s.scope_id = NEW.scope_id + AND (s.entry_count >= s.entry_limit OR d.entry_count >= d.entry_limit) + ) THEN RAISE(ABORT, 'cache_entry_limit') END; + UPDATE scopes SET entry_count = entry_count + 1 WHERE scope_id = NEW.scope_id; + UPDATE deployment SET entry_count = entry_count + 1; +END; +CREATE TRIGGER uncount_entry AFTER DELETE ON entries BEGIN + UPDATE scopes SET entry_count = entry_count - 1 WHERE scope_id = OLD.scope_id; + UPDATE deployment SET entry_count = entry_count - 1; +END; +CREATE TRIGGER count_association BEFORE INSERT ON associations +WHEN NOT EXISTS (SELECT 1 FROM associations WHERE scope_id = NEW.scope_id AND secondary_key = NEW.secondary_key) +BEGIN + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM scopes s, deployment d WHERE s.scope_id = NEW.scope_id + AND (s.association_count >= s.association_limit OR d.association_count >= d.association_limit) + ) THEN RAISE(ABORT, 'cache_association_limit') END; + UPDATE scopes SET association_count = association_count + 1 WHERE scope_id = NEW.scope_id; + UPDATE deployment SET association_count = association_count + 1; +END; +CREATE TRIGGER uncount_association AFTER DELETE ON associations BEGIN + UPDATE scopes SET association_count = association_count - 1 WHERE scope_id = OLD.scope_id; + UPDATE deployment SET association_count = association_count - 1; +END; + +-- All publication mutations inherit the guarded state change's transaction. +-- A zero-row guard fires no trigger. A quota failure rolls every mutation back. +CREATE TRIGGER publish_generation AFTER UPDATE OF state ON generations +WHEN OLD.state = 'uploading' AND NEW.state = 'ready' +BEGIN + UPDATE generations SET state = 'retired', retired_at = unixepoch(), + gc_after = CASE WHEN expires_at > unixepoch() THEN unixepoch() + 600 ELSE unixepoch() END + WHERE generation_id = (SELECT generation_id FROM entries WHERE scope_id = NEW.scope_id AND key = NEW.key) AND state = 'ready'; + INSERT INTO entries (scope_id, key, generation_id) VALUES (NEW.scope_id, NEW.key, NEW.generation_id) + ON CONFLICT (scope_id, key) DO UPDATE SET generation_id = excluded.generation_id; + INSERT INTO associations (scope_id, secondary_key, target_key) VALUES (NEW.scope_id, NEW.secondary_key, NEW.key) + ON CONFLICT (scope_id, secondary_key) DO UPDATE SET target_key = excluded.target_key; +END; diff --git a/packages/remote-cache/package.json b/packages/remote-cache/package.json new file mode 100644 index 000000000..e22f1eab0 --- /dev/null +++ b/packages/remote-cache/package.json @@ -0,0 +1,33 @@ +{ + "name": "@voidzero-dev/remote-cache", + "version": "0.0.0", + "private": true, + "license": "MIT", + "type": "module", + "scripts": { + "dev": "wrangler dev --test-scheduled", + "build": "wrangler deploy --dry-run --outdir dist", + "types": "wrangler types .wrangler/worker-configuration.d.ts --env-interface Env --strict-vars false", + "check": "pnpm types && tsc --noEmit", + "test": "node --import tsx --test test/index.test.ts", + "smoke": "pnpm test && pnpm build", + "operator": "node --import tsx scripts/operator.ts", + "benchmark": "node --import tsx scripts/benchmark.ts" + }, + "dependencies": { + "jose": "6.2.12" + }, + "devDependencies": { + "@types/node": "catalog:", + "cborg": "6.1.2", + "esbuild": "0.28.1", + "jsonc-parser": "3.3.1", + "miniflare": "5.20260911.0-alpha", + "tsx": "4.23.13", + "typescript": "catalog:", + "wrangler": "4.131.1" + }, + "engines": { + "node": ">=22.12.0" + } +} diff --git a/packages/remote-cache/scripts/benchmark.ts b/packages/remote-cache/scripts/benchmark.ts new file mode 100644 index 000000000..6f0f446ee --- /dev/null +++ b/packages/remote-cache/scripts/benchmark.ts @@ -0,0 +1,174 @@ +import assert from 'node:assert/strict'; +import { writeFile } from 'node:fs/promises'; +import { URL } from 'node:url'; +import { arch, platform, cpus } from 'node:os'; +import { harness, bytes } from '../test/helpers.ts'; +import { decodeEnvelope, encodeEnvelope } from '../src/cbor.ts'; +import { defaults, MiB } from '../src/limits.ts'; + +// Profile only the application isolate, not the Node client or D1/R2 emulators. +class Inspector { + private id = 0; + private pending = new Map< + number, + { resolve: (value: Record) => void; reject: (error: Error) => void } + >(); + constructor(private socket: WebSocket) { + socket.addEventListener('message', (event) => { + const response = JSON.parse(String(event.data)); + const request = this.pending.get(response.id); + if (!request) return; + this.pending.delete(response.id); + if (response.error) request.reject(new Error(response.error.message)); + else request.resolve(response.result); + }); + } + async call(method: string, params = {}): Promise> { + const id = ++this.id; + const response = new Promise>((resolve, reject) => + this.pending.set(id, { resolve, reject }), + ); + this.socket.send(JSON.stringify({ id, method, params })); + return response; + } + close() { + this.socket.close(); + } +} + +function sampledCpu(profile: Record): number { + const nodes = profile['nodes'] as { id: number; callFrame: { functionName: string } }[]; + const samples = profile['samples'] as number[]; + const deltas = profile['timeDeltas'] as number[]; + const excluded = new Set( + nodes + .filter((node) => ['(idle)', '(root)'].includes(node.callFrame.functionName)) + .map((node) => node.id), + ); + return ( + samples.reduce((sum, node, i) => sum + (excluded.has(node) ? 0 : (deltas[i] ?? 0)), 0) / 1000 + ); +} + +const h = await harness({ inspector: true }); +let inspector: Inspector | undefined; +const measurements: Record[] = []; +try { + const address = await h.mf.getInspectorURL(); + const listing = new URL('/json', address.href); + listing.protocol = 'http:'; + const targets = (await (await fetch(listing)).json()) as { + id: string; + webSocketDebuggerUrl: string; + title: string; + }[]; + const target = + targets.find((target) => target.id.includes('core:user:')) ?? + targets.find((target) => !target.id.includes('core:')); + if (!target) throw new Error('Cannot find application isolate in inspector targets'); + const socket = new WebSocket(target.webSocketDebuggerUrl); + await new Promise((resolve, reject) => { + socket.addEventListener('open', () => resolve(), { once: true }); + socket.addEventListener('error', () => reject(new Error('Inspector connection failed')), { + once: true, + }); + }); + inspector = new Inspector(socket); + await inspector.call('Profiler.enable'); + await inspector.call('Profiler.setSamplingInterval', { interval: 100 }); + const token = await h.token(); + async function measure(name: string, action: () => Promise) { + await inspector!.call('Profiler.start'); + const start = performance.now(); + await action(); + const wall = performance.now() - start; + const profile = (await inspector!.call('Profiler.stop'))['profile'] as Record; + const heap = await inspector!.call('Runtime.getHeapUsage'); + const row = { + name, + wall_ms: Math.round(wall * 100) / 100, + sampled_cpu_ms: sampledCpu(profile), + heap_bytes: heap['usedSize'], + backing_storage_bytes: heap['backingStorageSize'], + }; + measurements.push(row); + console.log(JSON.stringify(row)); + } + for (const mode of ['cold-jwks', 'warm-jwks']) + await measure(mode, async () => { + const response = await h.store(bytes(mode), bytes(mode), new Uint8Array(250000), undefined, { + token, + }); + assert.equal(response.status, 200, await response.clone().text()); + await response.arrayBuffer(); + }); + for (const valueSize of [250000, defaults.value]) { + // Codec timings exclude JWT, blob size, and transport. Node CPU is labelled + // separately; it must not be interpreted as Cloudflare's billed Worker CPU. + const value = new Uint8Array(valueSize); + const envelope = encodeEnvelope({ key: bytes('codec'), secondary_key: bytes('codec'), value }); + for (const operation of ['decode', 'encode']) { + const start = process.cpuUsage(); + for (let i = 0; i < 100; i++) { + if (operation === 'decode') decodeEnvelope(envelope, true, defaults); + else encodeEnvelope({ kind: 'exact', value, blob_id: null }); + } + const cpu = process.cpuUsage(start); + measurements.push({ + name: `cbor-${operation}-${valueSize}`, + node_cpu_ms_per_operation: (cpu.user + cpu.system) / 100000, + }); + } + await measure(`fetch-value-${valueSize}`, async () => { + assert.equal( + (await h.store(bytes('fetch'), bytes('fetch'), value, undefined, { token })).status, + 200, + ); + }); + await measure(`fetch-response-${valueSize}`, async () => { + const response = await h.fetch(bytes('fetch'), bytes('fetch')); + assert.equal(response.status, 200); + await response.arrayBuffer(); + }); + } + for (const blobSize of [5_000_000, 20_000_000, 50_000_000, 64 * MiB]) { + for (const concurrency of [1, 2]) + await measure(`store-${blobSize}-concurrency-${concurrency}`, async () => { + const result = await Promise.all( + Array.from({ length: concurrency }, (_, i) => + h.store( + bytes(`size-${blobSize}-${i}`), + bytes(`size-${blobSize}-${i}`), + new Uint8Array(defaults.value), + new Uint8Array(blobSize), + { token, blobFirst: i % 2 === 0 }, + ), + ), + ); + for (const response of result) { + assert.equal(response.status, 200, await response.clone().text()); + await response.arrayBuffer(); + } + }); + } + const result = { + measured_at: new Date().toISOString(), + platform: platform(), + arch: arch(), + cpu: cpus()[0]?.model, + node: process.version, + runtime: 'workerd 1.20260911.1', + concurrency: { stores: 2, metadata_reads: 4 }, + free_cpu_verified: false, + caveat: + 'Local V8 sampling is diagnostic. It excludes some native work and is not provider CPU billing. Validate production CPU before enabling a Free release profile.', + measurements, + }; + await writeFile( + new URL('../benchmark-results.json', import.meta.url), + JSON.stringify(result, null, 2) + '\n', + ); +} finally { + inspector?.close(); + await h.close(); +} diff --git a/packages/remote-cache/scripts/operator.ts b/packages/remote-cache/scripts/operator.ts new file mode 100644 index 000000000..536f020b3 --- /dev/null +++ b/packages/remote-cache/scripts/operator.ts @@ -0,0 +1,579 @@ +import { spawn } from 'node:child_process'; +import { readFile, writeFile, mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath, pathToFileURL, URL } from 'node:url'; +import { createRequire } from 'node:module'; +import { parseArgs } from 'node:util'; +import { parse, type ParseError } from 'jsonc-parser'; + +const root = fileURLToPath(new URL('../', import.meta.url)); +const configPath = join(root, 'wrangler.operator.json'); +const require = createRequire(import.meta.url); + +export async function readTemplate(): Promise { + const errors: ParseError[] = []; + const config = parse(await readFile(join(root, 'wrangler.jsonc'), 'utf8'), errors, { + allowTrailingComma: true, + }); + if (errors.length) throw new Error('Invalid Wrangler JSONC template'); + return config; +} + +export interface OperatorIO { + api(path: string, method?: string, body?: unknown): Promise; + github(repository: string): Promise; + wrangler(args: string[]): Promise; + readConfig(): Promise; + writeConfig(config: Config): Promise; + lifecycle(bucket: string, rules: unknown): Promise; + print(message: string): void; +} +type Config = { + name: string; + main: string; + compatibility_date: string; + compatibility_flags: string[]; + workers_dev: boolean; + preview_urls: boolean; + routes?: { pattern: string; custom_domain: boolean }[]; + observability: { enabled: boolean; head_sampling_rate: number }; + triggers: { crons: string[] }; + d1_databases: { + binding: string; + database_name: string; + database_id: string; + migrations_dir: string; + }[]; + r2_buckets: { binding: string; bucket_name: string }[]; + ratelimits: { name: string; namespace_id: string; simple: { limit: number; period: number } }[]; + vars: Record; +}; + +function object(value: unknown): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid API response'); + return value as Record; +} +function text(value: unknown): string { + if (typeof value !== 'string' || !value) throw new Error('Expected a nonempty string'); + return value; +} +function positive(value: string | undefined, fallback: number): number { + const number = value === undefined ? fallback : Number(value); + if (!Number.isSafeInteger(number) || number <= 0) throw new Error('Expected a positive integer'); + return number; +} +function toggle(value: string | undefined): number | null { + if (value === undefined) return null; + if (value !== 'on' && value !== 'off') throw new Error('Use on or off'); + return Number(value === 'on'); +} +function identifier(value: string | undefined): string { + if (!value || !/^[a-z0-9][a-z0-9-]{0,62}$/.test(value)) + throw new Error('Use a lowercase name with letters, digits, and hyphens (1–63 characters)'); + return value; +} + +export async function resolveRepository(io: Pick, name: string) { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(name)) throw new Error('Use owner/repository'); + const repo = object(await io.github(name)); + if (repo['private'] !== false || repo['visibility'] !== 'public') + throw new Error('Only public repositories can publish'); + const owner = object(repo['owner']); + if (!Number.isSafeInteger(repo['id']) || !Number.isSafeInteger(owner['id'])) + throw new Error('Invalid GitHub IDs'); + return { + name: text(repo['full_name']), + id: String(repo['id']), + owner: String(owner['id']), + branch: `refs/heads/${text(repo['default_branch'])}`, + }; +} + +async function query( + io: OperatorIO, + config: Config, + sql: string, + params: (string | number | null)[] = [], +) { + const result = await io.api(`/d1/database/${config.d1_databases[0]!.database_id}/query`, 'POST', { + sql, + params, + }); + if (!Array.isArray(result) || result.some((row) => object(row)['success'] !== true)) + throw new Error('D1 query failed'); + return result.flatMap((row) => { + const rows = object(row)['results']; + if (!Array.isArray(rows)) throw new Error('Invalid D1 results'); + return rows.map(object); + }); +} + +async function exists(io: OperatorIO, path: string): Promise { + try { + await io.api(path); + return true; + } catch (error) { + if (error instanceof ApiError && error.status === 404) return false; + throw error; + } +} + +export function lifecycleRules(retentionDays: number) { + return { + rules: [ + { + id: 'remote-cache-generations', + enabled: true, + conditions: { prefix: '' }, + deleteObjectsTransition: { + condition: { type: 'Age', maxAge: (retentionDays + 2) * 86400 }, + }, + abortMultipartUploadsTransition: { condition: { type: 'Age', maxAge: 86400 } }, + }, + ], + }; +} + +async function updateLifecycle(io: OperatorIO, config: Config) { + const rows = await query( + io, + config, + 'SELECT retention_high_water_seconds AS retention FROM deployment', + ); + const days = Math.ceil(Number(rows[0]?.['retention'] ?? 604800) / 86400); + await io.lifecycle(config.r2_buckets[0]!.bucket_name, lifecycleRules(days)); +} + +export async function runOperator(argv: string[], io: OperatorIO): Promise { + const { values, positionals } = parseArgs({ + args: argv, + allowPositionals: true, + options: { + name: { type: 'string' }, + namespace: { type: 'string' }, + repo: { type: 'string' }, + origin: { type: 'string' }, + profile: { type: 'string' }, + 'retention-days': { type: 'string' }, + 'byte-limit': { type: 'string' }, + 'entry-limit': { type: 'string' }, + 'association-limit': { type: 'string' }, + enabled: { type: 'string' }, + writes: { type: 'string' }, + confirm: { type: 'string' }, + help: { type: 'boolean' }, + }, + }); + const command = positionals[0]; + if (values.help || !command) { + io.print( + 'Commands: setup, bind, policy, deployment, status, upgrade, purge, teardown. See README.md for options.', + ); + return; + } + if ( + !['setup', 'bind', 'policy', 'deployment', 'status', 'upgrade', 'purge', 'teardown'].includes( + command, + ) + ) + throw new Error('Unknown command'); + for (const name of [ + 'retention-days', + 'byte-limit', + 'entry-limit', + 'association-limit', + ] as const) { + if (values[name] !== undefined) positive(values[name], 1); + } + if (values['retention-days'] && positive(values['retention-days'], 7) > 365) + throw new Error('Retention cannot exceed 365 days'); + toggle(values.enabled); + toggle(values.writes); + let config: Config; + if (command === 'setup') { + const name = identifier(values.name); + if (name.length < 3 || name.endsWith('-')) + throw new Error('Use a deployment name of 3–63 characters that ends with a letter or digit'); + const namespace = identifier(values.namespace); + const repo = await resolveRepository(io, text(values.repo)); + const origin = new URL(text(values.origin)); + if ( + origin.protocol !== 'https:' || + origin.username || + origin.password || + origin.pathname !== '/' || + origin.search || + origin.hash || + origin.port + ) + throw new Error('Use an HTTPS origin without a path'); + if ( + origin.hostname.endsWith('.workers.dev') && + (!origin.hostname.startsWith(`${name}.`) || origin.hostname.split('.').length !== 4) + ) + throw new Error('Use the deployment name and account subdomain in the workers.dev origin'); + const profile = values.profile ?? 'free'; + if (profile !== 'free' && profile !== 'paid') throw new Error('Use free or paid'); + const databases = await io.api(`/d1/database?name=${encodeURIComponent(name)}&per_page=100`); + if (!Array.isArray(databases)) throw new Error('Invalid D1 list'); + let db = databases.map(object).find((item) => item['name'] === name); + if (!db) { + await io.wrangler(['d1', 'create', name, '--no-update-config']); + const created = await io.api(`/d1/database?name=${encodeURIComponent(name)}&per_page=100`); + if (!Array.isArray(created)) throw new Error('Invalid D1 list'); + db = created.map(object).find((item) => item['name'] === name); + } + if (!db) throw new Error('D1 creation did not return the database'); + try { + const bucket = object(await io.api(`/r2/buckets/${name}`)); + if (bucket['storage_class'] && bucket['storage_class'] !== 'Standard') + throw new Error('Use an R2 Standard bucket'); + } catch (error) { + if (!(error instanceof ApiError) || error.status !== 404) throw error; + await io.wrangler([ + 'r2', + 'bucket', + 'create', + name, + '--storage-class', + 'Standard', + '--no-update-config', + ]); + } + // Refuse to adopt a bucket with public custom domains, then disable r2.dev. + const domains = object(await io.api(`/r2/buckets/${name}/domains/custom`)); + if (!Array.isArray(domains['domains']) || domains['domains'].length) + throw new Error('Remove R2 public custom domains before setup'); + await io.api(`/r2/buckets/${name}/domains/managed`, 'PUT', { enabled: false }); + config = await readTemplate(); + config.name = name; + config.d1_databases[0] = { + binding: 'INDEX', + database_name: name, + database_id: text(db['uuid']), + migrations_dir: 'migrations', + }; + config.r2_buckets[0] = { binding: 'ARTIFACTS', bucket_name: name }; + config.workers_dev = origin.hostname.endsWith('.workers.dev'); + if (!config.workers_dev) config.routes = [{ pattern: origin.hostname, custom_domain: true }]; + config.vars['GC_BATCH_SIZE'] = profile === 'free' ? '16' : '256'; + await io.writeConfig(config); + await io.wrangler(['d1', 'migrations', 'apply', 'INDEX', '--remote', '--config', configPath]); + const prior = await query(io, config, 'SELECT repository_id FROM scopes WHERE scope_id = ?', [ + namespace, + ]); + if (prior.length && prior[0]!['repository_id'] !== repo.id) + throw new Error('Use a new namespace for a different repository'); + // Repeated setup only creates a missing policy; it never re-enables a withdrawn scope. + await query( + io, + config, + `INSERT INTO scopes (scope_id, endpoint, repository, repository_id, repository_owner_id, branch, retention_seconds, byte_limit, entry_limit, association_limit) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(scope_id) DO NOTHING`, + [ + namespace, + `${origin.origin}/projects/${namespace}`, + repo.name, + repo.id, + repo.owner, + repo.branch, + positive(values['retention-days'], 7) * 86400, + positive(values['byte-limit'], 8000000000), + positive(values['entry-limit'], 20000), + positive(values['association-limit'], 20000), + ], + ); + await query( + io, + config, + `UPDATE deployment SET byte_limit = coalesce(?, byte_limit), + entry_limit = coalesce(?, entry_limit), association_limit = coalesce(?, association_limit)`, + [ + values['byte-limit'] ? positive(values['byte-limit'], 1) : null, + values['entry-limit'] ? positive(values['entry-limit'], 1) : null, + values['association-limit'] ? positive(values['association-limit'], 1) : null, + ], + ); + const scopes = await query(io, config, 'SELECT scope_id, endpoint FROM scopes'); + if (scopes.some((row) => new URL(text(row['endpoint'])).origin !== origin.origin)) + throw new Error('Use a separate deployment for a different public origin'); + config.vars['NAMESPACES'] = JSON.stringify(scopes.map((row) => text(row['scope_id']))); + await io.writeConfig(config); + await updateLifecycle(io, config); + await io.wrangler(['deploy', '--config', configPath]); + io.print(text(scopes.find((row) => row['scope_id'] === namespace)?.['endpoint'])); + return; + } + config = await io.readConfig(); + if (command === 'upgrade') { + await io.wrangler(['d1', 'migrations', 'apply', 'INDEX', '--remote', '--config', configPath]); + await updateLifecycle(io, config); + await io.wrangler(['deploy', '--config', configPath]); + return; + } + if (command === 'status') { + const deployment = await query(io, config, 'SELECT * FROM deployment'); + const scopes = await query(io, config, 'SELECT * FROM scopes'); + const generations = await query( + io, + config, + `SELECT state, count(*) AS count, sum(charged_bytes) AS bytes, + min(gc_after) AS earliest_cleanup FROM generations GROUP BY state`, + ); + const db = object(await io.api(`/d1/database/${config.d1_databases[0]!.database_id}`)); + io.print( + JSON.stringify( + { + deployment, + scopes, + generations, + database_bytes: db['file_size'], + warning: + Number(db['file_size']) >= 400000000 ? 'D1 storage is at or above 400 MB' : undefined, + }, + null, + 2, + ), + ); + return; + } + if (command === 'deployment') { + await query( + io, + config, + `UPDATE deployment SET enabled = coalesce(?, enabled), writes_enabled = coalesce(?, writes_enabled), + byte_limit = coalesce(?, byte_limit), entry_limit = coalesce(?, entry_limit), association_limit = coalesce(?, association_limit)`, + [ + toggle(values.enabled), + toggle(values.writes), + values['byte-limit'] ? positive(values['byte-limit'], 1) : null, + values['entry-limit'] ? positive(values['entry-limit'], 1) : null, + values['association-limit'] ? positive(values['association-limit'], 1) : null, + ], + ); + return; + } + if (command === 'teardown') { + if (values.confirm !== config.name) + throw new Error( + 'Pass --confirm with the exact deployment name to delete its data and resources', + ); + const database = config.d1_databases[0]!.database_id; + const bucket = config.r2_buckets[0]!.bucket_name; + const databaseExists = await exists(io, `/d1/database/${database}`); + if (databaseExists) { + await query(io, config, 'UPDATE deployment SET enabled = 0, writes_enabled = 0'); + await query( + io, + config, + `UPDATE generations SET lease_until = min(lease_until, unixepoch()), + gc_after = min(gc_after, unixepoch() + 600), expires_at = 0`, + ); + const remaining = await query(io, config, 'SELECT count(*) AS count FROM generations'); + if (Number(remaining[0]?.['count']) > 0) + throw new Error( + 'Data is withdrawn. Cron will delete it after the upload grace period. Run status, then repeat teardown when generations reach zero.', + ); + } + // R2 rejects deletion of a nonempty bucket. Keep D1 and Cron until this succeeds. + if (await exists(io, `/r2/buckets/${bucket}`)) + await io.wrangler(['r2', 'bucket', 'delete', bucket]); + if (databaseExists) await io.wrangler(['d1', 'delete', database, '--skip-confirmation']); + await io.wrangler(['delete', '--config', configPath, '--force']); + io.print('Removed cache storage and Worker.'); + return; + } + const namespace = identifier(values.namespace); + const existing = await query(io, config, 'SELECT * FROM scopes WHERE scope_id = ?', [namespace]); + if (command === 'bind') { + const repo = await resolveRepository(io, text(values.repo)); + if (existing.length) { + if (existing[0]!['repository_id'] !== repo.id) + throw new Error('Use a new namespace for a different repository'); + await query( + io, + config, + `UPDATE scopes SET repository = ?, repository_owner_id = ?, branch = ?, policy_version = policy_version + 1 WHERE scope_id = ?`, + [repo.name, repo.owner, repo.branch, namespace], + ); + } else { + const scopes = await query(io, config, 'SELECT endpoint FROM scopes LIMIT 1'); + const origin = new URL(text(scopes[0]?.['endpoint'])).origin; + await query( + io, + config, + `INSERT INTO scopes (scope_id, endpoint, repository, repository_id, repository_owner_id, branch) VALUES (?, ?, ?, ?, ?, ?)`, + [namespace, `${origin}/projects/${namespace}`, repo.name, repo.id, repo.owner, repo.branch], + ); + } + // Retry deployment even if an earlier attempt committed the binding first. + const all = await query(io, config, 'SELECT scope_id FROM scopes'); + config.vars['NAMESPACES'] = JSON.stringify(all.map((row) => text(row['scope_id']))); + await io.writeConfig(config); + await io.wrangler(['deploy', '--config', configPath]); + io.print( + text( + ( + await query(io, config, 'SELECT endpoint FROM scopes WHERE scope_id = ?', [namespace]) + )[0]?.['endpoint'], + ), + ); + return; + } + if (!existing.length) throw new Error('Namespace does not exist'); + if (command === 'purge') { + if (values.confirm !== namespace) + throw new Error( + 'Pass --confirm with the exact namespace to withdraw and delete its public data', + ); + await query( + io, + config, + 'UPDATE scopes SET enabled = 0, writes_enabled = 0, policy_version = policy_version + 1 WHERE scope_id = ?', + [namespace], + ); + await query( + io, + config, + `UPDATE generations SET expires_at = 0, lease_until = min(lease_until, unixepoch()), + gc_after = min(gc_after, unixepoch() + 600) WHERE scope_id = ?`, + [namespace], + ); + io.print('Namespace withdrawn. Cron will remove its data after the upload grace period.'); + return; + } + // Install a longer lifecycle before publishing a longer retention policy. + if (values['retention-days']) { + const days = positive(values['retention-days'], 7); + const max = await query( + io, + config, + 'SELECT retention_high_water_seconds AS retention FROM deployment', + ); + await io.lifecycle( + config.r2_buckets[0]!.bucket_name, + lifecycleRules(Math.max(days, Math.ceil(Number(max[0]?.['retention']) / 86400))), + ); + } + await query( + io, + config, + `UPDATE scopes SET enabled = coalesce(?, enabled), writes_enabled = coalesce(?, writes_enabled), + retention_seconds = coalesce(?, retention_seconds), byte_limit = coalesce(?, byte_limit), entry_limit = coalesce(?, entry_limit), + association_limit = coalesce(?, association_limit), policy_version = policy_version + 1 WHERE scope_id = ?`, + [ + toggle(values.enabled), + toggle(values.writes), + values['retention-days'] ? positive(values['retention-days'], 7) * 86400 : null, + values['byte-limit'] ? positive(values['byte-limit'], 1) : null, + values['entry-limit'] ? positive(values['entry-limit'], 1) : null, + values['association-limit'] ? positive(values['association-limit'], 1) : null, + namespace, + ], + ); +} + +export class ApiError extends Error { + constructor(public status: number) { + super(`Cloudflare API failed (HTTP ${status})`); + } +} + +async function jsonResponse(response: Response): Promise { + if (!response.ok) { + void response.body?.cancel(); + throw new ApiError(response.status); + } + const reader = response.body!.getReader(); + let size = 0; + const chunks: Uint8Array[] = []; + try { + while (true) { + const part = await reader.read(); + if (part.done) break; + size += part.value.length; + if (size > 4 * 1024 * 1024) throw new Error('API response too large'); + chunks.push(part.value); + } + } finally { + await reader.cancel(); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8')); +} + +const io: OperatorIO = { + async api(path, method = 'GET', body) { + const account = process.env['CLOUDFLARE_ACCOUNT_ID']; + const token = process.env['CLOUDFLARE_API_TOKEN']; + if (!account || !/^[a-f0-9]{32}$/.test(account) || !token) + throw new Error( + 'Set CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN in the operator environment', + ); + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${account}${path}`, + { + method, + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + signal: AbortSignal.timeout(30_000), + redirect: 'error', + }, + ); + const result = object(await jsonResponse(response)); + if (result['success'] !== true) throw new Error('Cloudflare API operation failed'); + return result['result']; + }, + async github(repository) { + return jsonResponse( + await fetch(`https://api.github.com/repos/${repository}`, { + headers: { + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'vp-remote-cache-operator', + }, + redirect: 'error', + signal: AbortSignal.timeout(15_000), + }), + ); + }, + async wrangler(args) { + const wrangler = join(dirname(require.resolve('wrangler/package.json')), 'bin/wrangler.js'); + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [wrangler, ...args], { + cwd: root, + stdio: 'inherit', + shell: false, + }); + child.once('error', reject); + child.once('exit', (code) => + code === 0 ? resolve() : reject(new Error(`Wrangler failed (exit ${code})`)), + ); + }); + }, + async readConfig() { + return JSON.parse(await readFile(configPath, 'utf8')); + }, + async writeConfig(config) { + await writeFile(configPath, JSON.stringify(config, null, 2) + '\n'); + }, + async lifecycle(bucket, rules) { + const dir = await mkdtemp(join(tmpdir(), 'vp-cache-lifecycle-')); + try { + const path = join(dir, 'lifecycle.json'); + await writeFile(path, JSON.stringify(rules)); + await io.wrangler(['r2', 'bucket', 'lifecycle', 'set', bucket, '--file', path, '--force']); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + print: (message) => console.log(message), +}; + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runOperator(process.argv.slice(2), io).catch((error) => { + console.error(error instanceof Error ? error.message : 'Operator command failed'); + process.exitCode = 1; + }); +} diff --git a/packages/remote-cache/src/admission.ts b/packages/remote-cache/src/admission.ts new file mode 100644 index 000000000..0de93ba47 --- /dev/null +++ b/packages/remote-cache/src/admission.ts @@ -0,0 +1,17 @@ +import { HttpError } from './errors.ts'; + +// Isolate-wide resource counters contain no request data or I/O promises. Rate +// limiting controls traffic; these counters additionally bound simultaneous buffers. +export class Admission { + private stores = 0; + private reads = 0; + acquire(store: boolean): () => void { + if (store ? this.stores >= 2 : this.reads >= 4) throw new HttpError(503, 'concurrency_limit'); + if (store) this.stores++; + else this.reads++; + return () => { + if (store) this.stores--; + else this.reads--; + }; + } +} diff --git a/packages/remote-cache/src/auth.ts b/packages/remote-cache/src/auth.ts new file mode 100644 index 000000000..b4dd1fbed --- /dev/null +++ b/packages/remote-cache/src/auth.ts @@ -0,0 +1,142 @@ +import { + createRemoteJWKSet, + customFetch, + decodeProtectedHeader, + errors, + jwtVerify, + type JWTVerifyGetKey, + type JWTPayload, +} from 'jose'; +import { HttpError, unavailable } from './errors.ts'; +import { Deadline, readBody } from './streams.ts'; +import type { Scope } from './database.ts'; + +export const ISSUER = 'https://token.actions.githubusercontent.com'; +export const JWKS_URL = `${ISSUER}/.well-known/jwks`; + +export function githubKeys(): JWTVerifyGetKey { + // Only reusable public-key cache state lives across requests. Failed refreshes + // also have a cooldown; jose's own cooldown starts only after a successful fetch. + let lastAttempt = -Infinity; + return createRemoteJWKSet(new URL(JWKS_URL), { + timeoutDuration: 5000, + cooldownDuration: 30_000, + cacheMaxAge: 10 * 60_000, + [customFetch]: async (url, options) => { + if (Date.now() - lastAttempt < 30_000) unavailable(); + lastAttempt = Date.now(); + const deadline = new Deadline(5000, options.signal); + try { + const response = await deadline.run(fetch(url, options)); + if (response.status !== 200) { + void response.body?.cancel(); + unavailable(); + } + const bytes = await readBody(response.body, 64 * 1024, deadline); + return new Response(bytes, { headers: { 'Content-Type': 'application/json' } }); + } catch { + unavailable(); + } finally { + deadline.dispose(); + } + }, + }); +} + +export interface WriteIdentity { + exp: number; + repository_id: string; + workflow_ref?: string; + run_id?: string; + run_attempt?: string; + sha?: string; +} + +export async function authorize( + request: Request, + scope: Scope, + keys: JWTVerifyGetKey, +): Promise { + const authorization = request.headers.get('Authorization'); + if ( + !authorization || + authorization.length > 16 * 1024 || + !/^Bearer [A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/i.test(authorization) + ) { + throw new HttpError(401, 'invalid_token'); + } + const token = authorization.slice(7); + let payload: JWTPayload; + try { + const header = decodeProtectedHeader(token); + if ( + header.alg !== 'RS256' || + typeof header.kid !== 'string' || + header.kid.length > 256 || + header.jku || + header.x5u || + header.jwk + ) { + throw new HttpError(401, 'invalid_token'); + } + ({ payload } = await jwtVerify(token, keys, { + algorithms: ['RS256'], + issuer: ISSUER, + requiredClaims: ['exp', 'nbf', 'iat'], + clockTolerance: 30, + maxTokenAge: '15 minutes', + })); + } catch (error) { + if (error instanceof HttpError) throw error; + if ( + error instanceof errors.JWTExpired || + error instanceof errors.JWTClaimValidationFailed || + error instanceof errors.JWSSignatureVerificationFailed || + error instanceof errors.JWSInvalid || + error instanceof errors.JWTInvalid || + error instanceof errors.JOSENotSupported || + error instanceof errors.JWKSNoMatchingKey || + error instanceof errors.JOSEAlgNotAllowed + ) { + throw new HttpError(401, 'invalid_token'); + } + unavailable(); + } + const now = Date.now() / 1000; + const { exp, nbf, iat } = payload; + if ( + typeof exp !== 'number' || + typeof nbf !== 'number' || + typeof iat !== 'number' || + !Number.isSafeInteger(exp) || + !Number.isSafeInteger(nbf) || + !Number.isSafeInteger(iat) || + exp <= now || + nbf > now + 30 || + iat > now + 30 || + iat < now - 930 || + nbf > exp || + iat >= exp || + exp - iat > 900 + ) { + throw new HttpError(401, 'invalid_token'); + } + if ( + !scope.writes_enabled || + payload.aud !== scope.endpoint || + payload.repository_id !== scope.repository_id || + payload.repository_owner_id !== scope.repository_owner_id || + payload.repository_visibility !== 'public' || + payload.ref !== scope.branch || + payload.ref_type !== 'branch' || + payload.event_name !== 'push' + ) { + throw new HttpError(403, 'write_policy'); + } + const identity: WriteIdentity = { exp, repository_id: scope.repository_id }; + for (const name of ['workflow_ref', 'run_id', 'run_attempt', 'sha'] as const) { + const value = payload[name]; + if (typeof value === 'string' && value.length <= 512) identity[name] = value; + } + return identity; +} diff --git a/packages/remote-cache/src/cbor.ts b/packages/remote-cache/src/cbor.ts new file mode 100644 index 000000000..3d3e3976e --- /dev/null +++ b/packages/remote-cache/src/cbor.ts @@ -0,0 +1,146 @@ +import { badRequest, tooLarge } from './errors.ts'; +import type { Limits } from './limits.ts'; + +// This codec only decodes the protocol envelope. Opaque bytes are never decoded. +// The accepted containers are one map and (possibly chunked) strings: depth <= 2. +class Decoder { + offset = 0; + constructor(private data: Uint8Array) {} + byte(): number { + return this.data[this.offset++] ?? badRequest(); + } + head(major: number): number | null { + const first = this.byte(); + if (first >> 5 !== major) badRequest(); + const info = first & 31; + if (info < 24) return info; + if (info === 31) return null; + if (info > 27) badRequest(); + let length = 0; + for (let i = 0; i < 2 ** (info - 24); i++) length = length * 256 + this.byte(); + if (!Number.isSafeInteger(length)) tooLarge(); + return length; + } + string(major: number, limit: number): Uint8Array { + const length = this.head(major); + if (length !== null) return this.take(length, limit); + const chunks: Uint8Array[] = []; + let size = 0; + while (this.data[this.offset] !== 255) { + const chunkLength = this.head(major); + if (chunkLength === null) badRequest(); + size += chunkLength; + if (size > limit) tooLarge(); + chunks.push(this.take(chunkLength, limit)); + // Bound bookkeeping even for a malicious sequence of empty chunks. + if (chunks.length > 4096) tooLarge(); + } + this.offset++; + return join(chunks, size); + } + take(size: number, limit: number): Uint8Array { + if (size > limit) tooLarge(); + if (size > this.data.length - this.offset) badRequest(); + const value = this.data.subarray(this.offset, this.offset + size); + this.offset += size; + return value; + } + finished(): boolean { + return this.offset === this.data.length; + } + break(): boolean { + if (this.data[this.offset] !== 255) return false; + this.offset++; + return true; + } +} + +export function join(chunks: Uint8Array[], size: number): Uint8Array { + const data = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + data.set(chunk, offset); + offset += chunk.length; + } + return data; +} + +export function decodeEnvelope( + data: Uint8Array, + store: false, + limits: Limits, +): { key: Uint8Array; secondary_key: Uint8Array }; +export function decodeEnvelope( + data: Uint8Array, + store: true, + limits: Limits, +): { key: Uint8Array; secondary_key: Uint8Array; value: Uint8Array }; +export function decodeEnvelope(data: Uint8Array, store: boolean, limits: Limits) { + const decoder = new Decoder(data); + const count = decoder.head(5); + const expected = store ? 3 : 2; + if (count !== null && count !== expected) badRequest(); + const fields = new Map(); + for (let i = 0; i < expected; i++) { + let name: string; + try { + name = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode( + decoder.string(3, 32), + ); + } catch { + badRequest(); + } + if (fields.has(name) || !['key', 'secondary_key', ...(store ? ['value'] : [])].includes(name)) + badRequest(); + fields.set(name, decoder.string(2, name === 'value' ? limits.value : limits.key)); + } + if ((count === null && !decoder.break()) || !decoder.finished()) badRequest(); + return { + key: fields.get('key')!, + secondary_key: fields.get('secondary_key')!, + ...(store ? { value: fields.get('value')! } : {}), + }; +} + +function header(major: number, length: number): Uint8Array { + if (length < 24) return Uint8Array.of((major << 5) | length); + if (length <= 255) return Uint8Array.of((major << 5) | 24, length); + if (length <= 65535) return Uint8Array.of((major << 5) | 25, length >> 8, length & 255); + return Uint8Array.of( + (major << 5) | 26, + length >>> 24, + (length >>> 16) & 255, + (length >>> 8) & 255, + length & 255, + ); +} + +export function encodeEnvelope( + fields: Record, +): Uint8Array { + const chunks = [header(5, Object.keys(fields).length)]; + const string = (value: string | Uint8Array) => { + const bytes = typeof value === 'string' ? new TextEncoder().encode(value) : value; + chunks.push(header(typeof value === 'string' ? 3 : 2, bytes.length), bytes); + }; + for (const [key, value] of Object.entries(fields)) { + string(key); + if (value === null) chunks.push(Uint8Array.of(246)); + else string(value); + } + return join( + chunks, + chunks.reduce((sum, chunk) => sum + chunk.length, 0), + ); +} + +export function cborResponse(fields: Record): Response { + const body = encodeEnvelope(fields); + return new Response(body, { + headers: { + 'Content-Type': 'application/cbor', + 'Content-Length': String(body.length), + 'Cache-Control': 'no-store', + }, + }); +} diff --git a/packages/remote-cache/src/database.ts b/packages/remote-cache/src/database.ts new file mode 100644 index 000000000..d1ec57f5c --- /dev/null +++ b/packages/remote-cache/src/database.ts @@ -0,0 +1,296 @@ +import { HttpError, unavailable } from './errors.ts'; +import { GRACE_SECONDS, LEASE_SECONDS } from './limits.ts'; +import { measured, type Observations } from './observations.ts'; + +export interface Scope { + scope_id: string; + endpoint: string; + repository_id: string; + repository_owner_id: string; + branch: string; + policy_version: number; + writes_enabled: number; +} +export interface Generation { + generation_id: string; + scope_id: string; + value_object: string; + blob_object: string; + blob_id: string | null; + value_size: number; + blob_size: number; +} +export interface Selection extends Generation { + kind: 'exact' | 'fallback'; + key: number[]; +} + +// Preserve empty and non-UTF-8 keys as BLOBs, without text or hash conversions. +export function binary(bytes: Uint8Array): ArrayBuffer { + return bytes.slice().buffer; +} + +export async function getScope(db: D1Database, id: string, stats?: Observations): Promise { + let scope: Scope | null; + try { + scope = + ( + await measured( + db + .prepare(`SELECT s.*, (s.writes_enabled AND d.writes_enabled) AS writes_enabled + FROM scopes s, deployment d WHERE s.scope_id = ? AND s.enabled = 1 AND d.enabled = 1`) + .bind(id) + .all(), + stats, + ) + ).results[0] ?? null; + } catch { + unavailable(); + } + if (!scope) throw new HttpError(404, 'unknown_scope'); + return scope; +} + +export async function selectEntry( + db: D1Database, + scope: string, + key: Uint8Array, + secondary: Uint8Array, + stats?: Observations, +): Promise { + // Both branches use indexed identities in one SQLite snapshot. + try { + return ( + ( + await measured( + db + .prepare(` + SELECT g.*, e.key, 'exact' AS kind, 0 AS priority + FROM entries e JOIN generations g ON g.generation_id = e.generation_id + JOIN scopes s ON s.scope_id = e.scope_id CROSS JOIN deployment d + WHERE e.scope_id = ?1 AND e.key = ?2 AND g.state = 'ready' AND g.expires_at > unixepoch() + AND s.enabled = 1 AND d.enabled = 1 + UNION ALL + SELECT g.*, e.key, 'fallback' AS kind, 1 AS priority + FROM associations a JOIN entries e ON e.scope_id = a.scope_id AND e.key = a.target_key + JOIN generations g ON g.generation_id = e.generation_id + JOIN scopes s ON s.scope_id = e.scope_id CROSS JOIN deployment d + WHERE a.scope_id = ?1 AND a.secondary_key = ?3 AND g.state = 'ready' AND g.expires_at > unixepoch() + AND s.enabled = 1 AND d.enabled = 1 + ORDER BY priority LIMIT 1`) + .bind(scope, binary(key), binary(secondary)) + .all(), + stats, + ) + ).results[0] ?? null + ); + } catch { + unavailable(); + } +} + +export async function selectBlob( + db: D1Database, + scope: string, + blob: string, + stats?: Observations, +): Promise { + try { + return ( + ( + await measured( + db + .prepare(`SELECT g.* FROM generations g JOIN scopes s ON s.scope_id = g.scope_id CROSS JOIN deployment d + WHERE g.scope_id = ? AND g.blob_id = ? AND s.enabled = 1 AND d.enabled = 1 + AND ((g.state = 'ready' AND g.expires_at > unixepoch()) OR (g.state = 'retired' AND g.gc_after > unixepoch()))`) + .bind(scope, blob) + .all(), + stats, + ) + ).results[0] ?? null + ); + } catch { + unavailable(); + } +} + +export async function reserve( + db: D1Database, + scope: Scope, + tokenExp: number, + bytes: number, + stats?: Observations, +): Promise { + const id = crypto.randomUUID(); + const prefix = `${scope.scope_id}/${id}`; + try { + await measured( + db + .prepare(`INSERT INTO generations + (generation_id, scope_id, state, policy_version, token_exp, lease_until, gc_after, value_object, blob_object, charged_bytes) + VALUES (?, ?, 'uploading', ?, ?, unixepoch() + ?, unixepoch() + ?, ?, ?, ?)`) + .bind( + id, + scope.scope_id, + scope.policy_version, + tokenExp, + LEASE_SECONDS, + LEASE_SECONDS + GRACE_SECONDS, + `${prefix}/value`, + `${prefix}/blob`, + bytes, + ) + .run(), + stats, + ); + } catch { + unavailable(); + } + return { + generation_id: id, + scope_id: scope.scope_id, + value_object: `${prefix}/value`, + blob_object: `${prefix}/blob`, + blob_id: null, + value_size: 0, + blob_size: 0, + }; +} + +export async function recordMultipart( + db: D1Database, + generation: Generation, + uploadId: string, + stats?: Observations, +) { + const result = await measured( + db + .prepare( + `UPDATE generations SET upload_id = ? WHERE generation_id = ? AND state = 'uploading' AND lease_until > unixepoch()`, + ) + .bind(uploadId, generation.generation_id) + .run(), + stats, + ); + if (result.meta.changes !== 1) unavailable(); +} + +export async function publish( + db: D1Database, + generation: Generation, + key: Uint8Array, + secondary: Uint8Array, + stats?: Observations, +): Promise { + const actual = generation.value_size + generation.blob_size; + let results: D1Result[]; + try { + results = await measured( + db.batch([ + db + .prepare(`UPDATE generations SET state = 'ready', key = ?, secondary_key = ?, blob_id = ?, + value_size = ?, blob_size = ?, charged_bytes = ?, + expires_at = unixepoch() + (SELECT retention_seconds FROM scopes WHERE scope_id = generations.scope_id), + gc_after = unixepoch() + (SELECT retention_seconds FROM scopes WHERE scope_id = generations.scope_id) + WHERE generation_id = ? AND state = 'uploading' AND lease_until > unixepoch() AND token_exp > unixepoch() + AND charged_bytes >= ? AND EXISTS ( + SELECT 1 FROM scopes s, deployment d WHERE s.scope_id = generations.scope_id + AND s.policy_version = generations.policy_version AND s.enabled = 1 AND s.writes_enabled = 1 + AND d.enabled = 1 AND d.writes_enabled = 1 AND s.charged_bytes <= s.byte_limit AND d.charged_bytes <= d.byte_limit + ) RETURNING generation_id`) + .bind( + binary(key), + binary(secondary), + generation.blob_id, + generation.value_size, + generation.blob_size, + actual, + generation.generation_id, + actual, + ), + ]), + stats, + ); + } catch { + unavailable(); + } + if (results[0]?.results.length !== 1) throw new HttpError(503, 'publication_guard'); +} + +export async function abandon(db: D1Database, id: string): Promise { + // Keep charges during the grace period for R2 operations that finish late. + await db + .prepare(`UPDATE generations SET lease_until = min(lease_until, unixepoch()), gc_after = min(gc_after, unixepoch() + ?) + WHERE generation_id = ? AND state = 'uploading'`) + .bind(GRACE_SECONDS, id) + .run(); +} + +export async function cleanup(env: Env): Promise { + const limit = Number(env.GC_BATCH_SIZE); + if (!Number.isInteger(limit) || limit < 1 || limit > 256) + throw new Error('Invalid GC batch size'); + const claim = crypto.randomUUID(); + const claimed = + await env.INDEX.prepare(`UPDATE generations SET state = 'deleting', gc_claim = ?, gc_after = unixepoch() + 600 + WHERE generation_id IN (SELECT generation_id FROM generations WHERE gc_after <= unixepoch() ORDER BY gc_after LIMIT ?) + RETURNING generation_id, value_object, blob_object, upload_id`) + .bind(claim, limit) + .all<{ + generation_id: string; + value_object: string; + blob_object: string; + upload_id: string | null; + }>(); + let deleted = 0; + for (const generation of claimed.results) { + try { + if (generation.upload_id) + await env.ARTIFACTS.resumeMultipartUpload( + generation.blob_object, + generation.upload_id, + ).abort(); + await env.ARTIFACTS.delete([generation.value_object, generation.blob_object]); + // FK deletion only removes entries still selecting this generation. + await env.INDEX.prepare( + `DELETE FROM generations WHERE generation_id = ? AND state = 'deleting' AND gc_claim = ?`, + ) + .bind(generation.generation_id, claim) + .run(); + deleted++; + } catch { + console.warn( + JSON.stringify({ + operation: 'cleanup', + error: 'deletion_failed', + generation: generation.generation_id, + }), + ); + } + } + await cleanAssociations(env.INDEX, limit); + console.log(JSON.stringify({ operation: 'cleanup', claimed: claimed.results.length, deleted })); +} + +async function cleanAssociations(db: D1Database, limit: number) { + // Cursor bounds scanning as well as deletion, even when all associations are live. + const batch = await db + .prepare(`SELECT scope_id, secondary_key FROM associations + WHERE (scope_id, secondary_key) > (SELECT scope_id, secondary_key FROM maintenance WHERE id = 1) + ORDER BY scope_id, secondary_key LIMIT ?`) + .bind(limit) + .all<{ scope_id: string; secondary_key: number[] }>(); + const last = batch.results.at(-1); + await db.batch([ + db + .prepare(`DELETE FROM associations WHERE (scope_id, secondary_key) IN ( + SELECT scope_id, secondary_key FROM associations + WHERE (scope_id, secondary_key) > (SELECT scope_id, secondary_key FROM maintenance WHERE id = 1) + ORDER BY scope_id, secondary_key LIMIT ?) + AND NOT EXISTS (SELECT 1 FROM entries e WHERE e.scope_id = associations.scope_id AND e.key = associations.target_key)`) + .bind(limit), + db + .prepare('UPDATE maintenance SET scope_id = ?, secondary_key = ? WHERE id = 1') + .bind(last?.scope_id ?? '', binary(new Uint8Array(last?.secondary_key ?? []))), + ]); +} diff --git a/packages/remote-cache/src/errors.ts b/packages/remote-cache/src/errors.ts new file mode 100644 index 000000000..9b54ffce5 --- /dev/null +++ b/packages/remote-cache/src/errors.ts @@ -0,0 +1,42 @@ +export class HttpError extends Error { + constructor( + public status: number, + public code: string, + ) { + super(code); + } +} + +export function badRequest(): never { + throw new HttpError(400, 'invalid_request'); +} + +export function tooLarge(): never { + throw new HttpError(413, 'size_limit'); +} + +export function unavailable(): never { + throw new HttpError(503, 'unavailable'); +} + +export function errorResponse(error: unknown): Response { + const status = error instanceof HttpError ? error.status : 500; + const messages: Record = { + 400: 'Invalid request', + 401: 'Invalid credentials', + 403: 'Write not permitted', + 404: 'Not found', + 413: 'Request too large', + 429: 'Rate limit exceeded', + 500: 'Operation failed', + 503: 'Service unavailable', + }; + return new Response(messages[status] ?? messages[500], { + status, + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'no-store', + ...(status === 429 || status === 503 ? { 'Retry-After': '60' } : {}), + }, + }); +} diff --git a/packages/remote-cache/src/index.ts b/packages/remote-cache/src/index.ts new file mode 100644 index 000000000..c6609ba69 --- /dev/null +++ b/packages/remote-cache/src/index.ts @@ -0,0 +1,155 @@ +import { authorize, githubKeys, type WriteIdentity } from './auth.ts'; +import { cborResponse, decodeEnvelope } from './cbor.ts'; +import { cleanup, getScope, selectBlob, selectEntry } from './database.ts'; +import { badRequest, errorResponse, HttpError, unavailable } from './errors.ts'; +import { limitsFrom } from './limits.ts'; +import { parameters } from './multipart.ts'; +import { store } from './store.ts'; +import { contentLength, Deadline, readBody } from './streams.ts'; +import { Admission } from './admission.ts'; +import { Observations } from './observations.ts'; + +const keys = githubKeys(); +const admission = new Admission(); + +export default { + async fetch(request, env, ctx) { + const started = Date.now(); + const id = crypto.randomUUID(); + const stats = new Observations(); + let scopeId = 'unknown'; + let operation = 'unknown'; + let outcome = 'error'; + let identity: WriteIdentity | undefined; + let deadline: Deadline | undefined; + let release: (() => void) | undefined; + let response: Response; + try { + const limits = limitsFrom(env.LIMITS); + const url = new URL(request.url); + deadline = new Deadline( + url.pathname.endsWith('/store') ? limits.deadlineMs : Math.min(limits.deadlineMs, 15000), + request.signal, + ); + const route = + /^\/projects\/([a-z0-9][a-z0-9-]{0,62})\/(fetch|store|blob\/([0-9a-f-]{36}))$/.exec( + url.pathname, + ); + const namespaces: unknown = JSON.parse(env.NAMESPACES); + if ( + !Array.isArray(namespaces) || + namespaces.length > 100 || + namespaces.some((value) => typeof value !== 'string') + ) + throw new Error('Invalid namespaces'); + const known = route && namespaces.includes(route[1]); + if (known) { + scopeId = route[1]!; + operation = route[2]!.startsWith('blob/') ? 'blob' : route[2]!; + } + const limiter = + operation === 'store' || url.pathname.endsWith('/store') + ? env.STORE_LIMITER + : env.READ_LIMITER; + if ( + !(await deadline.run(limiter.limit({ key: known ? `${scopeId}:${operation}` : 'unknown' }))) + .success + ) + throw new HttpError(429, 'rate_limit'); + if (!known || url.search || request.method !== (operation === 'blob' ? 'GET' : 'POST')) + throw new HttpError(404, 'unknown_route'); + release = admission.acquire(operation === 'store'); + const scope = await deadline.run(getScope(env.INDEX, scopeId, stats)); + if (operation === 'store') { + identity = await deadline.run(authorize(request, scope, keys)); + response = await store(request, env, ctx, scope, identity, limits, deadline, stats); + outcome = 'stored'; + } else if (operation === 'fetch') { + if (parameters(request.headers.get('Content-Type') ?? '').type !== 'application/cbor') + badRequest(); + contentLength(request, limits.fetch); + const body = await readBody(request.body, limits.fetch, deadline); + stats.request_bytes = body.length; + const data = decodeEnvelope(body, false, limits); + const selected = await deadline.run( + selectEntry(env.INDEX, scopeId, data.key, data.secondary_key, stats), + ); + if (!selected) { + outcome = 'miss'; + throw new HttpError(404, 'miss'); + } + stats.r2_operations++; + const object = await deadline + .run(env.ARTIFACTS.get(selected.value_object)) + .catch(() => unavailable()); + if (!object || object.size !== selected.value_size || object.size > limits.value) + unavailable(); + const value = await readBody(object.body, limits.value, deadline).catch(() => + unavailable(), + ); + if (value.length !== selected.value_size) unavailable(); + response = cborResponse({ + kind: selected.kind, + ...(selected.kind === 'fallback' ? { key: new Uint8Array(selected.key) } : {}), + value, + blob_id: selected.blob_id, + }); + outcome = selected.kind; + } else { + const selected = await deadline.run(selectBlob(env.INDEX, scopeId, route[3]!, stats)); + if (!selected) throw new HttpError(404, 'blob_missing'); + stats.r2_operations++; + const object = await deadline + .run(env.ARTIFACTS.get(selected.blob_object)) + .catch(() => unavailable()); + if (!object) throw new HttpError(404, 'blob_missing'); + if (object.size !== selected.blob_size) unavailable(); + response = new Response(object.body, { + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Length': String(object.size), + 'Cache-Control': 'no-store', + }, + }); + outcome = 'blob'; + } + } catch (error) { + response = errorResponse(error); + if (error instanceof HttpError) outcome = error.code; + } finally { + deadline?.dispose(); + release?.(); + } + response.headers.set('X-Request-Id', id); + const sample = Math.min(1, Math.max(0, Number(env.LOG_SAMPLE_RATE) || 0)); + // Fixed sampling bounds error volume as well as successful request volume. + if (Math.random() < sample) + console.log( + JSON.stringify({ + request_id: id, + scope: scopeId, + operation, + status: response.status, + outcome, + ...(response.status >= 400 ? { error_class: outcome } : {}), + response_bytes: Number(response.headers.get('Content-Length') ?? 0), + duration_ms: Date.now() - started, + ...stats.toJSON(), + ...(stats.d1_storage_bytes >= 400000000 ? { warning: 'd1_storage_400mb' } : {}), + ...(identity + ? { + repository_id: identity.repository_id, + workflow_ref: identity.workflow_ref, + run_id: identity.run_id, + run_attempt: identity.run_attempt, + sha: identity.sha, + } + : {}), + }), + ); + return response; + }, + async scheduled(_event, env) { + await cleanup(env); + }, +} satisfies ExportedHandler; diff --git a/packages/remote-cache/src/limits.ts b/packages/remote-cache/src/limits.ts new file mode 100644 index 000000000..b3c1f2650 --- /dev/null +++ b/packages/remote-cache/src/limits.ts @@ -0,0 +1,42 @@ +export const MiB = 1024 * 1024; +export const PART_SIZE = 5 * MiB; +export const LEASE_SECONDS = 15 * 60; +export const GRACE_SECONDS = 10 * 60; + +export const defaults = { + key: 16 * 1024, + value: 4 * MiB, + metadata: 5 * MiB, + fetch: 40 * 1024, + blob: 64 * MiB, + store: 72 * MiB, + headers: 8 * 1024, + deadlineMs: 120_000, +}; +export type Limits = typeof defaults; + +export function limitsFrom(raw: string): Limits { + const input: unknown = JSON.parse(raw); + if (!input || typeof input !== 'object' || Array.isArray(input)) + throw new Error('Invalid limits'); + const result = { ...defaults }; + for (const [name, value] of Object.entries(input)) { + if ( + !(name in defaults) || + !Number.isSafeInteger(value) || + value <= 0 || + value > defaults[name as keyof Limits] + ) { + throw new Error('Limits must be positive integers no larger than the tested defaults'); + } + result[name as keyof Limits] = value; + } + if ( + result.metadata < 2 * result.key + result.value + 128 || + result.fetch < 2 * result.key + 128 || + result.store < result.metadata + result.blob + 1024 + ) { + throw new Error('Envelope limits must accommodate field limits and framing'); + } + return result; +} diff --git a/packages/remote-cache/src/multipart.ts b/packages/remote-cache/src/multipart.ts new file mode 100644 index 000000000..b68ee0e4b --- /dev/null +++ b/packages/remote-cache/src/multipart.ts @@ -0,0 +1,116 @@ +import { Buffer } from 'node:buffer'; +import { badRequest, tooLarge } from './errors.ts'; +import { collect, Input } from './streams.ts'; + +export function parameters(raw: string): { type: string; params: Map } { + const separator = raw.indexOf(';'); + const type = (separator === -1 ? raw : raw.slice(0, separator)).trim().toLowerCase(); + let rest = separator === -1 ? '' : raw.slice(separator); + const params = new Map(); + while (rest.length) { + const match = /^;\s*([\w-]+)\s*=\s*(?:"((?:[^"\\\r\n]|\\[^\r\n])*)"|([^\s;]+))\s*/.exec(rest); + if (!match) badRequest(); + const name = match[1]!.toLowerCase(); + if (params.has(name)) badRequest(); + params.set(name, match[2] !== undefined ? match[2].replace(/\\(.)/g, '$1') : match[3]!); + rest = rest.slice(match[0].length); + } + return { type, params }; +} + +export function boundaryFrom(contentType: string | null): string { + const { type, params } = parameters(contentType ?? ''); + const boundary = params.get('boundary'); + if ( + type !== 'multipart/form-data' || + !boundary || + boundary.length > 70 || + !/^[0-9A-Za-z'()+_,./:=? -]+$/.test(boundary) || + boundary.endsWith(' ') + ) + badRequest(); + return boundary; +} + +export async function* multipart( + input: Input, + boundary: string, + headerLimit: number, +): AsyncGenerator<{ + name: 'metadata' | 'blob'; + body: AsyncGenerator; +}> { + let preamble = 0; + while (true) { + const line = await input.until(Buffer.from('\r\n'), headerLimit); + preamble += line.length + 2; + if (preamble > headerLimit) tooLarge(); + if (line.toString('utf8').replace(/[ \t]+$/, '') === `--${boundary}`) break; + } + const delimiter = Buffer.from(`\r\n--${boundary}`); + const seen = new Set(); + let closed = false; + while (!closed) { + if (seen.size >= 2) badRequest(); + const raw = await input.until(Buffer.from('\r\n\r\n'), headerLimit); + const headers = new Map(); + for (const line of raw.toString('utf8').split('\r\n')) { + const match = /^([!#$%&'*+.^_`|~\w-]+):[ \t]*([^\r\n]*)$/.exec(line); + if (!match) badRequest(); + const name = match[1]!.toLowerCase(); + if (headers.has(name)) badRequest(); + headers.set(name, match[2]!.trim()); + } + const disposition = parameters(headers.get('content-disposition') ?? ''); + const name = disposition.params.get('name'); + if ( + disposition.type !== 'form-data' || + (name !== 'metadata' && name !== 'blob') || + seen.has(name) + ) + badRequest(); + if (headers.has('content-transfer-encoding')) badRequest(); + const expected = name === 'metadata' ? 'application/cbor' : 'application/octet-stream'; + if (parameters(headers.get('content-type') ?? '').type !== expected) badRequest(); + seen.add(name); + let consumed = false; + async function* body(): AsyncGenerator { + while (true) { + let index = input.buffer.indexOf(delimiter); + while (index >= 0) { + const match = await delimiterEnd(input, index + delimiter.length, headerLimit); + if (match) { + if (index) yield input.take(index); + input.take(match.end - index); + closed = match.closed; + consumed = true; + return; + } + index = input.buffer.indexOf(delimiter, index + 1); + } + const safe = input.buffer.length - delimiter.length - 2; + if (safe > 0) yield input.take(safe); + if (!(await input.fill())) badRequest(); + } + } + yield { name, body: body() }; + if (!consumed) throw new Error('Multipart consumer must drain each part'); + } + if (!seen.has('metadata')) badRequest(); + await collect(input.rest(), headerLimit); +} + +async function delimiterEnd(input: Input, start: number, limit: number) { + while (input.buffer.length < start + 2 && (await input.fill())) {} + const closed = input.buffer[start] === 45 && input.buffer[start + 1] === 45; + let end = start + (closed ? 2 : 0); + while (true) { + while (input.buffer.length < end + 2 && (await input.fill())) {} + if (input.buffer[end] !== 32 && input.buffer[end] !== 9) break; + if (++end - start > limit) tooLarge(); + } + if (input.buffer[end] === 13 && input.buffer[end + 1] === 10) return { end: end + 2, closed }; + if (closed && input.ended && end === input.buffer.length) return { end, closed }; + // A boundary prefix followed by arbitrary bytes is still part of the opaque body. + return null; +} diff --git a/packages/remote-cache/src/observations.ts b/packages/remote-cache/src/observations.ts new file mode 100644 index 000000000..78bdf7548 --- /dev/null +++ b/packages/remote-cache/src/observations.ts @@ -0,0 +1,41 @@ +export class Observations { + request_bytes = 0; + d1_rows_read = 0; + d1_rows_written = 0; + d1_ms = 0; + d1_queries = 0; + d1_storage_bytes = 0; + r2_operations = 0; + + toJSON() { + return { + request_bytes: this.request_bytes, + d1_rows_read: this.d1_rows_read, + d1_rows_written: this.d1_rows_written, + d1_ms: this.d1_ms, + d1_queries: this.d1_queries, + d1_storage_bytes: this.d1_storage_bytes, + r2_operations: this.r2_operations, + }; + } +} + +export async function measured( + operation: Promise, + stats?: Observations, +): Promise { + const start = Date.now(); + try { + const result = await operation; + if (stats) + for (const item of Array.isArray(result) ? result : [result]) { + stats.d1_rows_read += item.meta.rows_read; + stats.d1_rows_written += item.meta.rows_written; + stats.d1_storage_bytes = item.meta.size_after; + stats.d1_queries++; + } + return result; + } finally { + if (stats) stats.d1_ms += Date.now() - start; + } +} diff --git a/packages/remote-cache/src/store.ts b/packages/remote-cache/src/store.ts new file mode 100644 index 000000000..4482e74d7 --- /dev/null +++ b/packages/remote-cache/src/store.ts @@ -0,0 +1,130 @@ +import { decodeEnvelope, cborResponse } from './cbor.ts'; +import { + abandon, + publish, + recordMultipart, + reserve, + type Generation, + type Scope, +} from './database.ts'; +import { badRequest, tooLarge, unavailable } from './errors.ts'; +import { PART_SIZE, type Limits } from './limits.ts'; +import { boundaryFrom, multipart } from './multipart.ts'; +import { collect, contentLength, Deadline, Input } from './streams.ts'; +import type { WriteIdentity } from './auth.ts'; +import type { Observations } from './observations.ts'; + +async function uploadBlob( + env: Env, + generation: Generation, + source: AsyncIterable, + limit: number, + deadline: Deadline, + stats: Observations, +) { + let buffer = new Uint8Array(PART_SIZE); + let used = 0; + let upload: R2MultipartUpload | undefined; + const parts: R2UploadedPart[] = []; + for await (const chunk of source) { + generation.blob_size += chunk.length; + if (generation.blob_size > limit) tooLarge(); + let offset = 0; + while (offset < chunk.length) { + if (used === PART_SIZE) { + if (!upload) { + stats.r2_operations++; + upload = await deadline.run(env.ARTIFACTS.createMultipartUpload(generation.blob_object)); + try { + await deadline.run(recordMultipart(env.INDEX, generation, upload.uploadId, stats)); + } catch (error) { + // Also cover creation succeeding but D1 failing to record its ID. + try { + await deadline.run(upload.abort()); + } catch { + /* Lifecycle is the final backstop. */ + } + throw error; + } + } + stats.r2_operations++; + parts.push(await deadline.run(upload.uploadPart(parts.length + 1, buffer))); + buffer = new Uint8Array(PART_SIZE); + used = 0; + } + const size = Math.min(PART_SIZE - used, chunk.length - offset); + buffer.set(chunk.subarray(offset, offset + size), used); + used += size; + offset += size; + } + } + if (upload) { + stats.r2_operations += Number(used > 0) + 1; + if (used) + parts.push(await deadline.run(upload.uploadPart(parts.length + 1, buffer.subarray(0, used)))); + await deadline.run(upload.complete(parts)); + } else { + stats.r2_operations++; + const result = await deadline.run( + env.ARTIFACTS.put(generation.blob_object, buffer.subarray(0, used)), + ); + if (!result) unavailable(); + } + generation.blob_id = crypto.randomUUID(); +} + +export async function store( + request: Request, + env: Env, + ctx: Pick, + scope: Scope, + identity: WriteIdentity, + limits: Limits, + deadline: Deadline, + stats: Observations, +): Promise { + const boundary = boundaryFrom(request.headers.get('Content-Type')); + const length = contentLength(request, limits.store); + const generation = await deadline.run( + reserve(env.INDEX, scope, identity.exp, length ?? limits.store, stats), + ); + let input: Input | undefined; + try { + input = new Input(request.body, Math.min(length ?? limits.store, limits.store), deadline); + let metadata: ReturnType | undefined; + for await (const part of multipart(input, boundary, limits.headers)) { + if (part.name === 'metadata') { + metadata = decodeEnvelope(await collect(part.body, limits.metadata), true, limits); + generation.value_size = metadata.value.length; + stats.r2_operations++; + const result = await deadline.run( + env.ARTIFACTS.put(generation.value_object, metadata.value), + ); + if (!result) unavailable(); + } else { + await uploadBlob(env, generation, part.body, limits.blob, deadline, stats); + } + } + if (!metadata || (length !== null && input.bytes !== length)) badRequest(); + deadline.check(); + await deadline.run(publish(env.INDEX, generation, metadata.key, metadata.secondary_key, stats)); + return cborResponse({ blob_id: generation.blob_id }); + } catch (error) { + // Publication is never deferred. Only cleanup can continue after the response. + ctx.waitUntil( + abandon(env.INDEX, generation.generation_id).catch(() => { + console.warn( + JSON.stringify({ + operation: 'cleanup', + error: 'abandon_failed', + generation: generation.generation_id, + }), + ); + }), + ); + throw error; + } finally { + stats.request_bytes = input?.bytes ?? 0; + input?.close(); + } +} diff --git a/packages/remote-cache/src/streams.ts b/packages/remote-cache/src/streams.ts new file mode 100644 index 000000000..0682b60eb --- /dev/null +++ b/packages/remote-cache/src/streams.ts @@ -0,0 +1,135 @@ +import { Buffer } from 'node:buffer'; +import { badRequest, HttpError, tooLarge } from './errors.ts'; + +export class Deadline { + private controller = new AbortController(); + private timer: ReturnType; + private cancelled: Promise; + private onAbort = () => this.controller.abort(); + constructor( + ms: number, + private signal?: AbortSignal, + ) { + this.timer = setTimeout(this.onAbort, ms); + signal?.addEventListener('abort', this.onAbort, { once: true }); + this.cancelled = new Promise((_, reject) => + this.controller.signal.addEventListener( + 'abort', + () => reject(new HttpError(503, 'deadline')), + { once: true }, + ), + ); + // Cancellation can precede the first awaited operation. + void this.cancelled.catch(() => {}); + if (signal?.aborted) this.onAbort(); + } + check() { + if (this.controller.signal.aborted) throw new HttpError(503, 'deadline'); + } + async run(operation: Promise): Promise { + this.check(); + return Promise.race([operation, this.cancelled]); + } + dispose() { + clearTimeout(this.timer); + this.signal?.removeEventListener('abort', this.onAbort); + } +} + +export async function collect( + source: AsyncIterable, + limit: number, +): Promise> { + let size = 0; + // Fixed-capacity accumulation also bounds bookkeeping for one-byte chunks. + const buffer = new Uint8Array(limit); + for await (const chunk of source) { + if (chunk.length > limit - size) tooLarge(); + buffer.set(chunk, size); + size += chunk.length; + } + return buffer.subarray(0, size); +} + +export class Input { + private reader: ReadableStreamDefaultReader; + private pending: Uint8Array = new Uint8Array(0); + private offset = 0; + bytes = 0; + ended = false; + buffer: Buffer = Buffer.alloc(0); + constructor( + body: ReadableStream | null, + private max: number, + private deadline: Deadline, + ) { + if (!body) badRequest(); + this.reader = body.getReader(); + } + async fill(): Promise { + if (this.ended) return false; + if (this.offset === this.pending.length) { + const result = await this.deadline.run(this.reader.read()); + if (result.done) { + this.ended = true; + return false; + } + this.bytes += result.value.length; + if (this.bytes > this.max) tooLarge(); + this.pending = result.value; + this.offset = 0; + } + const next = this.pending.subarray(this.offset, this.offset + 64 * 1024); + this.offset += next.length; + this.buffer = Buffer.concat([this.buffer, next]); + return true; + } + take(size: number): Buffer { + const result = this.buffer.subarray(0, size); + this.buffer = this.buffer.subarray(size); + return result; + } + async until(delimiter: Buffer, limit: number): Promise { + while (true) { + const index = this.buffer.indexOf(delimiter); + if (index >= 0) { + if (index > limit) tooLarge(); + const value = this.take(index); + this.take(delimiter.length); + return value; + } + if (this.buffer.length > limit + delimiter.length) tooLarge(); + if (!(await this.fill())) badRequest(); + } + } + async *rest(): AsyncGenerator { + do { + if (this.buffer.length) yield this.take(this.buffer.length); + } while (await this.fill()); + } + close() { + void this.reader.cancel().catch(() => {}); + } +} + +export async function readBody( + body: ReadableStream | null, + limit: number, + deadline: Deadline, +) { + const input = new Input(body, limit, deadline); + try { + return await collect(input.rest(), limit); + } finally { + input.close(); + } +} + +export function contentLength(request: Request, maximum: number): number | null { + const raw = request.headers.get('Content-Length'); + if (raw === null) return null; + if (!/^\d+$/.test(raw)) badRequest(); + const length = Number(raw); + if (!Number.isSafeInteger(length) || length > maximum) tooLarge(); + return length; +} diff --git a/packages/remote-cache/test/failures.test.ts b/packages/remote-cache/test/failures.test.ts new file mode 100644 index 000000000..636d68109 --- /dev/null +++ b/packages/remote-cache/test/failures.test.ts @@ -0,0 +1,306 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { encode } from 'cborg'; +import { harness, bytes, decodeResponse } from './helpers.ts'; +import { cleanup, getScope, reserve, publish } from '../src/database.ts'; +import { store } from '../src/store.ts'; +import { defaults } from '../src/limits.ts'; +import { Deadline } from '../src/streams.ts'; +import { Observations } from '../src/observations.ts'; +import { Admission } from '../src/admission.ts'; + +function multipartRequest(blob?: Uint8Array): Request { + const form = new FormData(); + form.append( + 'metadata', + new Blob([encode({ key: bytes('A'), secondary_key: bytes('T'), value: bytes('new') })], { + type: 'application/cbor', + }), + ); + if (blob) form.append('blob', new Blob([blob], { type: 'application/octet-stream' })); + return new Request('https://cache.example.com/projects/test/store', { + method: 'POST', + body: form, + }); +} + +void test('R2 PUT, multipart creation, part, completion, and upload-ID recording failures leave mappings unchanged', async () => { + const h = await harness(); + try { + await h.store(bytes('A'), bytes('S'), bytes('old')); + const original = await h.mf.getBindings(); + for (const failure of ['value', 'blob', 'create', 'part', 'complete', 'record']) { + const pending: Promise[] = []; + const deadline = new Deadline(5000); + const bucket = new Proxy(original.ARTIFACTS, { + get(target, prop) { + if (prop === 'put') + return async (key: string, value: ArrayBuffer | ArrayBufferView) => { + if (key.endsWith(`/${failure}`)) throw new Error('Injected PUT failure'); + return target.put(key, value); + }; + if (prop === 'createMultipartUpload') + return async (key: string) => { + if (failure === 'create') throw new Error('Injected multipart creation failure'); + const upload = await target.createMultipartUpload(key); + if (failure === 'record') + await h.db + .prepare("UPDATE generations SET lease_until = 0 WHERE state = 'uploading'") + .run(); + return { + key: upload.key, + uploadId: upload.uploadId, + abort: () => upload.abort(), + uploadPart: async (number: number, value: ArrayBufferView) => { + if (failure === 'part') throw new Error('Injected part failure'); + return upload.uploadPart(number, value); + }, + complete: async (parts: R2UploadedPart[]) => { + if (failure === 'complete') throw new Error('Injected completion failure'); + return upload.complete(parts); + }, + }; + }; + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + try { + const blob = new Uint8Array(['value', 'blob'].includes(failure) ? 4 : 5 * 1024 * 1024 + 1); + await assert.rejects( + store( + multipartRequest(blob), + { ...original, ARTIFACTS: bucket }, + { waitUntil: (promise) => pending.push(promise) }, + await getScope(h.db, 'test'), + { exp: Math.floor(Date.now() / 1000) + 300, repository_id: '123' }, + defaults, + deadline, + new Observations(), + ), + ); + await Promise.all(pending); + assert.deepEqual( + (await decodeResponse(await h.fetch(bytes('A'), bytes('S')))).value, + bytes('old'), + ); + assert.equal((await h.fetch(bytes('missing'), bytes('T'))).status, 404); + } finally { + deadline.dispose(); + } + } + await h.db.prepare("UPDATE generations SET gc_after = 0 WHERE state = 'uploading'").run(); + await cleanup(original); + assert.equal((await h.db.prepare('SELECT count(*) AS n FROM generations').first())!.n, 1); + assert.equal((await h.bucket.list()).objects.length, 1); + } finally { + await h.close(); + } +}); + +void test('failed deletion remains charged and a retry removes only the claimed generation', async () => { + const h = await harness(); + try { + await h.store(bytes('A'), bytes('S'), bytes('old'), bytes('blob')); + await h.store(bytes('A'), bytes('T'), bytes('new')); + await h.db.prepare("UPDATE generations SET gc_after = 0 WHERE state = 'retired'").run(); + const original = await h.mf.getBindings(); + const bucket = new Proxy(original.ARTIFACTS, { + get(target, prop) { + if (prop === 'delete') + return async () => { + throw new Error('Injected deletion failure'); + }; + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + await cleanup({ ...original, ARTIFACTS: bucket }); + assert.equal( + (await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes, + 10, + ); + await h.db.prepare("UPDATE generations SET gc_after = 0 WHERE state = 'deleting'").run(); + await cleanup(original); + assert.equal( + (await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes, + 3, + ); + assert.deepEqual( + (await decodeResponse(await h.fetch(bytes('missing'), bytes('S')))).value, + bytes('new'), + ); + } finally { + await h.close(); + } +}); + +void test('unknown-length requests reserve the full limit and cancellation prevents publication', async () => { + const h = await harness(); + try { + const env = await h.mf.getBindings(); + const pending: Promise[] = []; + const deadline = new Deadline(100); + let cancelled = false; + const requestInit = { + method: 'POST', + headers: { 'Content-Type': 'multipart/form-data; boundary=x' }, + body: new ReadableStream({ + cancel() { + cancelled = true; + }, + }), + duplex: 'half', + }; + const request = new Request('https://cache.example.com/projects/test/store', requestInit); + try { + await assert.rejects( + store( + request, + env, + { waitUntil: (promise) => pending.push(promise) }, + await getScope(h.db, 'test'), + { exp: Math.floor(Date.now() / 1000) + 300, repository_id: '123' }, + defaults, + deadline, + new Observations(), + ), + { status: 503 }, + ); + await Promise.all(pending); + assert.equal(cancelled, true); + const row = await h.db.prepare('SELECT charged_bytes, state FROM generations').first(); + assert.equal(row!.charged_bytes, defaults.store); + assert.equal(row!.state, 'uploading'); + assert.equal((await h.fetch(bytes('A'), bytes('T'))).status, 404); + } finally { + deadline.dispose(); + } + } finally { + await h.close(); + } +}); + +void test('scope/deployment disable and capacity changes revoke pending publications', async () => { + const h = await harness(); + try { + for (const sql of [ + 'UPDATE scopes SET enabled = 0', + 'UPDATE scopes SET writes_enabled = 0', + 'UPDATE scopes SET retention_seconds = retention_seconds + 1', + 'UPDATE deployment SET enabled = 0', + 'UPDATE deployment SET writes_enabled = 0', + 'UPDATE deployment SET byte_limit = 1', + ]) { + const g = await reserve( + h.db, + await getScope(h.db, 'test'), + Math.floor(Date.now() / 1000) + 300, + 100, + ); + await h.db.prepare(sql).run(); + await assert.rejects(publish(h.db, g, bytes('A'), bytes('S')), { status: 503 }); + await h.db.prepare('UPDATE scopes SET enabled = 1, writes_enabled = 1').run(); + await h.db + .prepare('UPDATE deployment SET enabled = 1, writes_enabled = 1, byte_limit = 8000000000') + .run(); + } + assert.equal((await h.db.prepare('SELECT count(*) AS n FROM entries').first())!.n, 0); + assert.equal((await h.db.prepare('SELECT count(*) AS n FROM associations').first())!.n, 0); + } finally { + await h.close(); + } +}); + +void test('scope and deployment entry limits roll back association changes, while replacements use no new slot', async () => { + const h = await harness(); + try { + assert.equal((await h.store(bytes('A'), bytes('S'), bytes('old'))).status, 200); + for (const table of ['scopes', 'deployment']) { + await h.db.prepare(`UPDATE ${table} SET entry_limit = 1`).run(); + assert.equal((await h.store(bytes('B'), bytes('S'), bytes('rejected'))).status, 503); + assert.deepEqual( + (await decodeResponse(await h.fetch(bytes('missing'), bytes('S')))).key, + bytes('A'), + ); + assert.equal((await h.fetch(bytes('B'), bytes('missing'))).status, 404); + assert.equal((await h.store(bytes('A'), bytes('S'), bytes('replacement'))).status, 200); + await h.db.prepare(`UPDATE ${table} SET association_limit = 1`).run(); + assert.equal((await h.store(bytes('A'), bytes('T'), bytes('rejected'))).status, 503); + assert.deepEqual( + (await decodeResponse(await h.fetch(bytes('A'), bytes('S')))).value, + bytes('replacement'), + ); + await h.db + .prepare(`UPDATE ${table} SET entry_limit = 20000, association_limit = 20000`) + .run(); + } + const counters = await h.db + .prepare('SELECT entry_count, association_count FROM deployment') + .first(); + assert.deepEqual(counters, { entry_count: 1, association_count: 1 }); + } finally { + await h.close(); + } +}); + +void test('cleanup preserves a recreated target and claims at most 16 generations', async () => { + const h = await harness(); + try { + await h.store(bytes('A'), bytes('S'), bytes('old'), bytes('old')); + await h.db.prepare('UPDATE generations SET expires_at = 0, gc_after = 0').run(); + const original = await h.mf.getBindings(); + const bucket = new Proxy(original.ARTIFACTS, { + get(target, prop) { + if (prop === 'delete') + return async (keys: string[]) => { + assert.equal((await h.store(bytes('A'), bytes('T'), bytes('new'))).status, 200); + const claimed = await h.db + .prepare("SELECT count(*) AS n FROM generations WHERE state = 'deleting'") + .first(); + assert.equal(claimed!.n, 1); + await target.delete(keys); + }; + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + await cleanup({ ...original, ARTIFACTS: bucket }); + assert.deepEqual( + (await decodeResponse(await h.fetch(bytes('missing'), bytes('S')))).value, + bytes('new'), + ); + const scope = await getScope(h.db, 'test'); + for (let i = 0; i < 18; i++) + await reserve(h.db, scope, Math.floor(Date.now() / 1000) + 300, 100); + await h.db.prepare("UPDATE generations SET gc_after = 0 WHERE state = 'uploading'").run(); + await cleanup(original); + assert.equal((await h.db.prepare('SELECT count(*) AS n FROM generations').first())!.n, 3); + assert.equal( + (await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes, + 203, + ); + await cleanup(original); + assert.equal((await h.db.prepare('SELECT count(*) AS n FROM generations').first())!.n, 1); + assert.equal( + (await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes, + 3, + ); + } finally { + await h.close(); + } +}); + +void test('isolate admission bounds memory and releases capacity', () => { + const admission = new Admission(); + const first = admission.acquire(true), + second = admission.acquire(true); + assert.throws(() => admission.acquire(true), { status: 503 }); + first(); + const third = admission.acquire(true); + second(); + third(); + const readers = Array.from({ length: 4 }, () => admission.acquire(false)); + assert.throws(() => admission.acquire(false), { status: 503 }); + for (const release of readers) release(); +}); diff --git a/packages/remote-cache/test/fixtures/protocol.json b/packages/remote-cache/test/fixtures/protocol.json new file mode 100644 index 000000000..aa455f36a --- /dev/null +++ b/packages/remote-cache/test/fixtures/protocol.json @@ -0,0 +1,18 @@ +{ + "fetch": [ + { + "name": "empty key and binary secondary key", + "hex": "a2636b6579406d7365636f6e646172795f6b65794200ff" + }, + { + "name": "indefinite map and chunked byte strings", + "hex": "bf636b65795f40ff6d7365636f6e646172795f6b65795f410041ffffff" + }, + { + "name": "noncanonical lengths", + "hex": "b80278036b657958006d7365636f6e646172795f6b65795a0000000200ff" + } + ], + "expectedKey": [], + "expectedSecondaryKey": [0, 255] +} diff --git a/packages/remote-cache/test/helpers.ts b/packages/remote-cache/test/helpers.ts new file mode 100644 index 000000000..2e3cbd9d8 --- /dev/null +++ b/packages/remote-cache/test/helpers.ts @@ -0,0 +1,164 @@ +import { readFile } from 'node:fs/promises'; +import { fileURLToPath, URL } from 'node:url'; +import { build } from 'esbuild'; +import { Miniflare, convertV4MiniflareOptions } from 'miniflare'; +import { exportJWK, generateKeyPair, SignJWT } from 'jose'; +import { encode, decode } from 'cborg'; +import { ISSUER, JWKS_URL } from '../src/auth.ts'; + +export const endpoint = 'https://cache.example.com/projects/test'; +export const bytes = (value: string) => new TextEncoder().encode(value); +export const decodeResponse = async (response: Pick) => + decode(new Uint8Array(await response.arrayBuffer())); + +export async function harness( + options: { + readLimit?: number; + storeLimit?: number; + jwksStatus?: number; + limits?: Record; + inspector?: boolean; + } = {}, +) { + const pair = await generateKeyPair('RS256', { extractable: true }); + const jwk = { ...(await exportJWK(pair.publicKey)), kid: 'test-key', alg: 'RS256', use: 'sig' }; + const bundle = await build({ + entryPoints: [fileURLToPath(new URL('../src/index.ts', import.meta.url))], + bundle: true, + write: false, + format: 'esm', + platform: 'neutral', + target: 'es2022', + external: ['node:*', 'cloudflare:*'], + }); + let jwksRequests = 0; + const mf = new Miniflare( + convertV4MiniflareOptions({ + // Tests must not discover or restart other local Wrangler/Miniflare sessions. + unsafeDevRegistryPath: '', + unsafeRegisterWorker: false, + ...(options.inspector ? { inspectorPort: 0 } : {}), + modules: true, + script: bundle.outputFiles[0]!.text, + compatibilityDate: '2026-09-11', + compatibilityFlags: ['nodejs_compat'], + d1Databases: ['INDEX'], + r2Buckets: ['ARTIFACTS'], + bindings: { + NAMESPACES: '["test","other"]', + LIMITS: JSON.stringify(options.limits ?? {}), + GC_BATCH_SIZE: '16', + LOG_SAMPLE_RATE: '0', + }, + ratelimits: { + READ_LIMITER: { + namespace_id: '1001', + simple: { limit: options.readLimit ?? 10000, period: 60 }, + }, + STORE_LIMITER: { + namespace_id: '1002', + simple: { limit: options.storeLimit ?? 10000, period: 60 }, + }, + }, + outboundService: async (request) => { + if (request.url !== JWKS_URL) throw new Error('Unexpected outbound request'); + jwksRequests++; + return new Response(JSON.stringify({ keys: [jwk] }), { + status: options.jwksStatus ?? 200, + headers: { 'Content-Type': 'application/json' }, + }); + }, + }), + ); + try { + const db = await mf.getD1Database('INDEX'); + const bucket = await mf.getR2Bucket('ARTIFACTS'); + const migration = await readFile( + new URL('../migrations/0001_cache.sql', import.meta.url), + 'utf8', + ); + // Preserve complete trigger bodies. Each prepared statement is a real D1 migration statement. + const statements = migration.match( + /CREATE TRIGGER[\s\S]*?\nEND;|(?:CREATE TABLE|CREATE (?:UNIQUE )?INDEX|INSERT INTO)[\s\S]*?;/g, + )!; + await db.batch(statements.map((sql) => db.prepare(sql))); + for (const name of ['test', 'other']) + await db + .prepare(`INSERT INTO scopes (scope_id, endpoint, repository, repository_id, repository_owner_id, branch) + VALUES (?, ?, 'owner/repo', '123', '456', 'refs/heads/main')`) + .bind(name, `https://cache.example.com/projects/${name}`) + .run(); + async function token(claims: Record = {}, kid = 'test-key') { + const now = Math.floor(Date.now() / 1000); + return new SignJWT({ + iss: ISSUER, + aud: endpoint, + repository_id: '123', + repository_owner_id: '456', + repository_visibility: 'public', + ref: 'refs/heads/main', + ref_type: 'branch', + event_name: 'push', + exp: now + 300, + iat: now, + nbf: now - 1, + ...claims, + }) + .setProtectedHeader({ alg: 'RS256', kid }) + .sign(pair.privateKey); + } + async function store( + key: Uint8Array, + secondary: Uint8Array, + value: Uint8Array, + blob?: Uint8Array, + options: { token?: string; scope?: string; blobFirst?: boolean } = {}, + ) { + const form = new FormData(); + const addBlob = () => { + if (blob !== undefined) + form.append('blob', new Blob([blob], { type: 'application/octet-stream' }), 'blob'); + }; + if (options.blobFirst) addBlob(); + form.append( + 'metadata', + new Blob([encode({ key, secondary_key: secondary, value })], { type: 'application/cbor' }), + 'metadata', + ); + if (!options.blobFirst) addBlob(); + const request = new Request( + `https://cache.example.com/projects/${options.scope ?? 'test'}/store`, + { + method: 'POST', + headers: { Authorization: `Bearer ${options.token ?? (await token())}` }, + body: form, + }, + ); + return mf.dispatchFetch(request.url, { + method: 'POST', + headers: Object.fromEntries(request.headers), + body: await request.arrayBuffer(), + }); + } + function fetch(key: Uint8Array, secondary: Uint8Array, scope = 'test') { + return mf.dispatchFetch(`https://cache.example.com/projects/${scope}/fetch`, { + method: 'POST', + headers: { 'Content-Type': 'application/cbor' }, + body: encode({ key, secondary_key: secondary }), + }); + } + return { + mf, + db, + bucket, + token, + store, + fetch, + jwksRequests: () => jwksRequests, + close: () => mf.dispose(), + }; + } catch (error) { + await mf.dispose(); + throw error; + } +} diff --git a/packages/remote-cache/test/index.test.ts b/packages/remote-cache/test/index.test.ts new file mode 100644 index 000000000..fd2243249 --- /dev/null +++ b/packages/remote-cache/test/index.test.ts @@ -0,0 +1,4 @@ +import './protocol.test.ts'; +import './service.test.ts'; +import './operator.test.ts'; +import './failures.test.ts'; diff --git a/packages/remote-cache/test/operator.test.ts b/packages/remote-cache/test/operator.test.ts new file mode 100644 index 000000000..2032602ad --- /dev/null +++ b/packages/remote-cache/test/operator.test.ts @@ -0,0 +1,184 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + runOperator, + resolveRepository, + lifecycleRules, + readTemplate, + ApiError, + type OperatorIO, +} from '../scripts/operator.ts'; +import { harness } from './helpers.ts'; + +void test('operator setup is repeatable and policy changes preserve withdrawal', async () => { + let config = await readTemplate(); + const h = await harness(); + const commands: string[][] = []; + const output: string[] = []; + const lifecycle: unknown[] = []; + let failDeploy = false; + const io: OperatorIO = { + async api(path, _method, body) { + if (path.startsWith('/d1/database?')) return [{ name: 'cache', uuid: 'test-database' }]; + if (path.endsWith('/query')) { + const { sql, params } = body as { sql: string; params: unknown[] }; + return [ + await h.db + .prepare(sql) + .bind(...params) + .all(), + ]; + } + if (path.endsWith('/domains/custom')) return { domains: [] }; + return {}; + }, + async github() { + return { + id: 123, + full_name: 'owner/repo', + owner: { id: 456 }, + private: false, + visibility: 'public', + default_branch: 'main', + }; + }, + async wrangler(args) { + commands.push(args); + if (failDeploy && args[0] === 'deploy') throw new Error('Deployment failed'); + }, + async readConfig() { + return config; + }, + async writeConfig(value) { + config = value; + }, + async lifecycle(_bucket, rules) { + lifecycle.push(rules); + }, + print(message) { + output.push(message); + }, + }; + try { + const args = [ + 'setup', + '--name', + 'cache', + '--namespace', + 'new', + '--repo', + 'owner/repo', + '--origin', + 'https://cache.example.com', + ]; + await runOperator(args, io); + await runOperator(['policy', '--namespace', 'new', '--enabled', 'off', '--writes', 'off'], io); + await runOperator(args, io); + const row = await h.db.prepare("SELECT * FROM scopes WHERE scope_id = 'new'").first(); + assert.equal(row!.enabled, 0); + assert.equal(row!.writes_enabled, 0); + assert.equal(row!.policy_version, 2); + assert.equal(config.workers_dev, false); + assert.equal(config.preview_urls, false); + assert.deepEqual(config.routes, [{ pattern: 'cache.example.com', custom_domain: true }]); + assert.equal( + JSON.parse(config.vars.NAMESPACES!).filter((name: string) => name === 'new').length, + 1, + ); + assert.equal( + commands.some((args) => args.includes('create')), + false, + ); + assert.equal(lifecycle.length, 2); + assert.equal(output.at(-1), 'https://cache.example.com/projects/new'); + assert.deepEqual(lifecycleRules(30), { + rules: [ + { + id: 'remote-cache-generations', + enabled: true, + conditions: { prefix: '' }, + deleteObjectsTransition: { condition: { type: 'Age', maxAge: 32 * 86400 } }, + abortMultipartUploadsTransition: { condition: { type: 'Age', maxAge: 86400 } }, + }, + ], + }); + await assert.rejects(runOperator(['teardown'], io), /confirm/); + await assert.rejects(runOperator(['purge', '--namespace', 'test'], io), /confirm/); + failDeploy = true; + const bind = ['bind', '--namespace', 'retry', '--repo', 'owner/repo']; + await assert.rejects(runOperator(bind, io), /Deployment failed/); + const attempts = commands.length; + failDeploy = false; + await runOperator(bind, io); + assert.equal(commands.length, attempts + 1); + assert.equal(commands.at(-1)![0], 'deploy'); + assert.ok(JSON.parse(config.vars.NAMESPACES!).includes('retry')); + } finally { + await h.close(); + } +}); + +void test('teardown resumes after partial resource deletion', async () => { + const config = await readTemplate(); + let database = true, + bucket = true, + failDatabase = true, + failWorker = true; + const deleted: string[] = []; + const io: OperatorIO = { + async api(path) { + if (path.endsWith('/query')) return [{ success: true, results: [{ count: 0 }] }]; + if ((path.startsWith('/d1/') && !database) || (path.startsWith('/r2/') && !bucket)) + throw new ApiError(404); + return {}; + }, + async github() { + throw new Error('Unexpected GitHub request'); + }, + async readConfig() { + return config; + }, + async writeConfig() { + throw new Error('Unexpected config write'); + }, + async lifecycle() { + throw new Error('Unexpected lifecycle write'); + }, + print() {}, + async wrangler(args) { + if (args[0] === 'r2') { + bucket = false; + deleted.push('bucket'); + } else if (args[0] === 'd1') { + if (failDatabase) { + failDatabase = false; + throw new Error('D1 deletion failed'); + } + database = false; + deleted.push('database'); + } else { + if (failWorker) { + failWorker = false; + throw new Error('Worker deletion failed'); + } + deleted.push('worker'); + } + }, + }; + const args = ['teardown', '--confirm', config.name]; + await assert.rejects(runOperator(args, io), /D1 deletion failed/); + await assert.rejects(runOperator(args, io), /Worker deletion failed/); + await runOperator(args, io); + assert.deepEqual(deleted, ['bucket', 'database', 'worker']); +}); + +void test('operator rejects private repositories and injection-shaped names', async () => { + const io = { + github: async () => ({ id: 123, owner: { id: 456 }, private: true, visibility: 'private' }), + }; + await assert.rejects(resolveRepository(io, 'owner/repo'), /public/); + await assert.rejects( + resolveRepository(io, "owner/repo'; DROP TABLE scopes;"), + /owner\/repository/, + ); +}); diff --git a/packages/remote-cache/test/protocol.test.ts b/packages/remote-cache/test/protocol.test.ts new file mode 100644 index 000000000..a0acf3438 --- /dev/null +++ b/packages/remote-cache/test/protocol.test.ts @@ -0,0 +1,102 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { decode, encode } from 'cborg'; +import { decodeEnvelope, encodeEnvelope } from '../src/cbor.ts'; +import { defaults } from '../src/limits.ts'; +import { multipart } from '../src/multipart.ts'; +import { collect, Deadline, Input } from '../src/streams.ts'; +import fixtures from './fixtures/protocol.json'; + +void test('CBOR fixtures preserve opaque, empty, indefinite, and noncanonical bytes', () => { + for (const fixture of fixtures.fetch) { + const actual = decodeEnvelope(Buffer.from(fixture.hex, 'hex'), false, defaults); + assert.deepEqual(Array.from(actual.key), fixtures.expectedKey, fixture.name); + assert.deepEqual(Array.from(actual.secondary_key), fixtures.expectedSecondaryKey, fixture.name); + } + const value = Uint8Array.of(0, 255, 159, 255); + assert.deepEqual(decode(encodeEnvelope({ kind: 'exact', value, blob_id: null })), { + kind: 'exact', + value, + blob_id: null, + }); +}); + +void test('multipart accepts bounded preamble, padding, and epilogue and rejects invalid parts', async () => { + const headers = + 'Content-Disposition: form-data; name="metadata"\r\nContent-Type: application/cbor\r\n\r\n'; + const valid = `preamble\r\n--x \t\r\n${headers}abc\r\n--x-- \t\r\nepilogue`; + async function parse(data: string, limit = 1024) { + const deadline = new Deadline(5000); + const input = new Input(new Blob([data]).stream(), 4096, deadline); + try { + const parts = []; + for await (const part of multipart(input, 'x', limit)) + parts.push(Buffer.from(await collect(part.body, 1024)).toString()); + return parts; + } finally { + input.close(); + deadline.dispose(); + } + } + assert.deepEqual(await parse(valid), ['abc']); + for (const data of [ + `--x\r\n${headers}abc\r\n--x\r\n${headers}abc\r\n--x--`, + `--x\r\n${headers.replace('metadata', 'unknown')}abc\r\n--x--`, + `--x\r\n${headers.replace('application/cbor', 'text/plain')}abc\r\n--x--`, + `--x\r\n${headers}abc\r\n--x--invalid`, + ]) + await assert.rejects(parse(data), { status: 400 }); + await assert.rejects(parse('a'.repeat(1025) + '\r\n' + valid), { status: 413 }); +}); + +void test('CBOR rejects duplicate, missing, wrong-type, nested, oversized, and trailing fields', () => { + for (const hex of [ + 'a2636b657940636b657940', + 'a1636b657940', + 'a2636b6579606d7365636f6e646172795f6b657940', + 'a2636b657981406d7365636f6e646172795f6b657940', + 'a2636b65795bffffffffffffffff', + 'a2636b6579406d7365636f6e646172795f6b65794000', + ]) { + assert.throws(() => decodeEnvelope(Buffer.from(hex, 'hex'), false, defaults)); + } + assert.throws( + () => + decodeEnvelope( + encode({ key: new Uint8Array(defaults.key + 1), secondary_key: new Uint8Array() }), + false, + defaults, + ), + { status: 413 }, + ); +}); + +void test('multipart accepts every split point, either part order, and boundary-like blob bytes', async () => { + for (const blobFirst of [false, true]) { + const metadata = + '--test\r\nContent-Disposition: form-data; name="metadata"\r\nContent-Type: application/cbor\r\n\r\nabc\r\n'; + const blob = + '--test\r\nContent-Disposition: form-data; name="blob"; filename="out"\r\nContent-Type: application/octet-stream\r\n\r\nx\r\n--testXYz\r\n--test--XYz\r\n'; + const data = Buffer.from((blobFirst ? blob + metadata : metadata + blob) + '--test--\r\n'); + for (let split = 1; split < data.length; split++) { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(data.subarray(0, split)); + controller.enqueue(data.subarray(split)); + controller.close(); + }, + }); + const deadline = new Deadline(5000); + const input = new Input(stream, 4096, deadline); + try { + const result: Record = {}; + for await (const part of multipart(input, 'test', 1024)) + result[part.name] = Buffer.from(await collect(part.body, 1024)).toString(); + assert.deepEqual(result, { metadata: 'abc', blob: 'x\r\n--testXYz\r\n--test--XYz' }); + } finally { + deadline.dispose(); + input.close(); + } + } + } +}); diff --git a/packages/remote-cache/test/service.test.ts b/packages/remote-cache/test/service.test.ts new file mode 100644 index 000000000..a27bb4049 --- /dev/null +++ b/packages/remote-cache/test/service.test.ts @@ -0,0 +1,410 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { encode } from 'cborg'; +import { bytes, decodeResponse, endpoint, harness } from './helpers.ts'; +import { getScope, publish, reserve } from '../src/database.ts'; + +void test('Workers D1/R2: exact/fallback, replacement, empty blobs, namespace isolation, and read-only access', async () => { + const h = await harness(); + try { + const a = Uint8Array.of(0, 255), + b = bytes('B'), + c = bytes('C'), + s = bytes('S'), + t = new Uint8Array(); + assert.equal((await h.fetch(a, s)).status, 404); + const first = await h.store(a, s, bytes('VA'), bytes('old')); + assert.equal(first.status, 200, await first.clone().text()); + const firstBlob = (await decodeResponse(first)).blob_id; + assert.equal((await h.store(b, s, bytes('VB'))).status, 200); + assert.deepEqual(await decodeResponse(await h.fetch(a, s)), { + kind: 'exact', + value: bytes('VA'), + blob_id: firstBlob, + }); + assert.deepEqual(await decodeResponse(await h.fetch(c, s)), { + kind: 'fallback', + key: b, + value: bytes('VB'), + blob_id: null, + }); + const replacement = await h.store(a, t, bytes('VA2'), new Uint8Array(), { blobFirst: true }); + assert.equal(replacement.status, 200); + const emptyBlob = (await decodeResponse(replacement)).blob_id; + assert.equal(typeof emptyBlob, 'string'); + assert.equal(await (await h.mf.dispatchFetch(`${endpoint}/blob/${firstBlob}`)).text(), 'old'); + assert.equal((await h.mf.dispatchFetch(`${endpoint}/blob/${emptyBlob}`)).status, 200); + assert.equal( + (await h.mf.dispatchFetch(`${endpoint}/blob/${emptyBlob}`)).headers.get('Content-Length'), + '0', + ); + assert.equal((await h.fetch(a, s, 'other')).status, 404); + assert.equal( + (await h.mf.dispatchFetch(`https://cache.example.com/projects/other/blob/${firstBlob}`)) + .status, + 404, + ); + assert.equal((await h.store(a, s, bytes('other'), undefined, { scope: 'other' })).status, 403); + assert.equal( + ( + await h.store(a, s, bytes('other'), undefined, { + scope: 'other', + token: await h.token({ aud: endpoint.replace('/test', '/other') }), + }) + ).status, + 200, + ); + assert.deepEqual((await decodeResponse(await h.fetch(a, s, 'other'))).value, bytes('other')); + assert.deepEqual((await decodeResponse(await h.fetch(a, s))).value, bytes('VA2')); + const before = await h.db.prepare('SELECT * FROM deployment').first(); + await h.fetch(a, t); + await h.fetch(c, t); + assert.deepEqual(await h.db.prepare('SELECT * FROM deployment').first(), before); + await h.db.prepare("UPDATE scopes SET enabled = 0 WHERE scope_id = 'test'").run(); + assert.equal((await h.fetch(a, t)).status, 404); + assert.equal((await h.mf.dispatchFetch(`${endpoint}/blob/${firstBlob}`)).status, 404); + } finally { + await h.close(); + } +}); + +void test('signed GitHub policy denies forks, wrong owners, private repos, non-push events, branches, audiences, and invalid times', async () => { + const h = await harness(); + try { + for (const claims of [ + { repository_id: '999' }, + { repository_id: 123 }, + { repository_owner_id: '999' }, + { repository_visibility: 'private' }, + { event_name: 'pull_request' }, + { event_name: 'pull_request_target' }, + { event_name: 'workflow_run' }, + { ref: 'refs/heads/feature' }, + { ref_type: 'tag' }, + { aud: 'https://attacker.example' }, + { aud: [endpoint] }, + ]) + assert.equal( + ( + await h.store(bytes('A'), bytes('S'), bytes('V'), undefined, { + token: await h.token(claims), + }) + ).status, + 403, + ); + const now = Math.floor(Date.now() / 1000); + for (const claims of [ + { exp: now - 1 }, + { nbf: now + 100 }, + { iat: now + 100 }, + { iat: 'bad' }, + { exp: null }, + { iss: 'https://attacker.example' }, + ]) { + assert.equal( + ( + await h.store(bytes('A'), bytes('S'), bytes('V'), undefined, { + token: await h.token(claims), + }) + ).status, + 401, + ); + } + assert.equal( + (await h.mf.dispatchFetch(`${endpoint}/store`, { method: 'POST', body: 'not read' })).status, + 401, + ); + assert.equal( + (await h.store(bytes('A'), bytes('S'), bytes('V'), undefined, { token: 'forged' })).status, + 401, + ); + const valid = await h.token(); + const segments = valid.split('.'); + const signature = Buffer.from(segments[2]!, 'base64url'); + signature[0] = signature[0]! ^ 1; + segments[2] = signature.toString('base64url'); + assert.equal( + (await h.store(bytes('A'), bytes('S'), bytes('V'), undefined, { token: segments.join('.') })) + .status, + 401, + ); + for (let i = 0; i < 5; i++) + assert.equal( + ( + await h.store(bytes('A'), bytes('S'), bytes('V'), undefined, { + token: await h.token({}, `unknown-${i}`), + }) + ).status, + 401, + ); + assert.equal(h.jwksRequests(), 1); + assert.equal((await h.db.prepare('SELECT count(*) AS n FROM generations').first())!.n, 0); + assert.equal((await h.store(bytes('A'), bytes('S'), bytes('V'))).status, 200); + } finally { + await h.close(); + } +}); + +void test('JWKS outages fail closed and have a refresh cooldown', async () => { + const h = await harness({ jwksStatus: 503 }); + try { + for (let i = 0; i < 3; i++) + assert.equal((await h.store(bytes('A'), bytes('S'), bytes('V'))).status, 503); + assert.equal(h.jwksRequests(), 1); + } finally { + await h.close(); + } +}); + +void test('publication guard and quotas preserve BOTH mappings and charged bytes', async () => { + const h = await harness(); + try { + assert.equal((await h.store(bytes('A'), bytes('S'), bytes('old'))).status, 200); + const scope = await getScope(h.db, 'test'); + for (const mutate of [ + (id: string) => + h.db + .prepare('UPDATE generations SET lease_until = 0 WHERE generation_id = ?') + .bind(id) + .run(), + (id: string) => + h.db.prepare('UPDATE generations SET token_exp = 0 WHERE generation_id = ?').bind(id).run(), + () => + h.db + .prepare("UPDATE scopes SET policy_version = policy_version + 1 WHERE scope_id = 'test'") + .run(), + ]) { + const current = await getScope(h.db, 'test'); + const g = await reserve(h.db, current, Math.floor(Date.now() / 1000) + 300, 100); + g.value_size = 3; + await h.bucket.put(g.value_object, 'new'); + await mutate(g.generation_id); + await assert.rejects(publish(h.db, g, bytes('A'), bytes('T')), { status: 503 }); + assert.equal( + (await h.db + .prepare('SELECT state FROM generations WHERE generation_id = ?') + .bind(g.generation_id) + .first())!.state, + 'uploading', + ); + assert.deepEqual( + (await decodeResponse(await h.fetch(bytes('A'), bytes('S')))).value, + bytes('old'), + ); + assert.equal((await h.fetch(bytes('missing'), bytes('T'))).status, 404); + } + assert.equal(scope.repository_id, '123'); + await h.db.prepare('UPDATE deployment SET association_limit = 1').run(); + assert.equal((await h.store(bytes('A'), bytes('new-S'), bytes('new'))).status, 503); + assert.deepEqual( + (await decodeResponse(await h.fetch(bytes('A'), bytes('S')))).value, + bytes('old'), + ); + assert.equal((await h.store(bytes('A'), bytes('S'), bytes('update'))).status, 200); + await h.db.prepare('UPDATE deployment SET byte_limit = charged_bytes').run(); + assert.equal((await h.store(bytes('A'), bytes('S'), bytes('new'))).status, 503); + } finally { + await h.close(); + } +}); + +void test('large sequential R2 multipart stores and concurrent stores return coherent generations', async () => { + const h = await harness(); + try { + const blob = new Uint8Array(5 * 1024 * 1024 + 123).fill(37); + const response = await h.store(bytes('large'), bytes('S'), bytes('large-value'), blob, { + blobFirst: true, + }); + assert.equal(response.status, 200, await response.clone().text()); + const blobId = (await decodeResponse(response)).blob_id; + const download = await h.mf.dispatchFetch(`${endpoint}/blob/${blobId}`); + assert.deepEqual(new Uint8Array(await download.arrayBuffer()), blob); + const stored = await Promise.all( + Array.from({ length: 2 }, (_, i) => + h.store(bytes('race'), bytes('race'), bytes(String(i)), bytes(String(i))), + ), + ); + for (const result of stored) assert.equal(result.status, 200); + const final = await decodeResponse(await h.fetch(bytes('race'), bytes('race'))); + assert.deepEqual( + new Uint8Array( + await (await h.mf.dispatchFetch(`${endpoint}/blob/${final.blob_id}`)).arrayBuffer(), + ), + final.value, + ); + const accounting = await h.db + .prepare( + 'SELECT charged_bytes, (SELECT sum(charged_bytes) FROM generations) AS expected FROM deployment', + ) + .first(); + assert.ok(accounting); + assert.equal(accounting.charged_bytes, accounting.expected); + // Cleanup also accepts a recorded upload ID whose multipart upload completed. + await h.db + .prepare('UPDATE generations SET expires_at = 0, gc_after = 0 WHERE blob_id = ?') + .bind(blobId) + .run(); + await (await h.mf.getWorker()).scheduled(); + assert.equal((await h.mf.dispatchFetch(`${endpoint}/blob/${blobId}`)).status, 404); + assert.equal( + (await h.db + .prepare('SELECT count(*) AS n FROM generations WHERE blob_id = ?') + .bind(blobId) + .first())!.n, + 0, + ); + } finally { + await h.close(); + } +}); + +void test('retention, retirement, abandoned uploads, and bounded cleanup protect replacements', async () => { + const h = await harness(); + try { + const oldBlob = ( + await decodeResponse(await h.store(bytes('A'), bytes('S'), bytes('old'), bytes('old'))) + ).blob_id; + await h.store(bytes('A'), bytes('S'), bytes('new'), bytes('new')); + await h.db.prepare("UPDATE generations SET gc_after = 0 WHERE state = 'retired'").run(); + assert.equal((await h.mf.dispatchFetch(`${endpoint}/blob/${oldBlob}`)).status, 404); + await (await h.mf.getWorker()).scheduled(); + assert.deepEqual( + (await decodeResponse(await h.fetch(bytes('A'), bytes('S')))).value, + bytes('new'), + ); + const expiredBlob = (await decodeResponse(await h.fetch(bytes('A'), bytes('S')))).blob_id; + await h.db + .prepare("UPDATE generations SET expires_at = 0, gc_after = 0 WHERE state = 'ready'") + .run(); + assert.equal((await h.fetch(bytes('A'), bytes('S'))).status, 404); + assert.equal((await h.store(bytes('A'), bytes('S'), bytes('newest'))).status, 200); + assert.equal((await h.mf.dispatchFetch(`${endpoint}/blob/${expiredBlob}`)).status, 404); + await h.db + .prepare("UPDATE generations SET expires_at = 0, gc_after = 0 WHERE state = 'ready'") + .run(); + await (await h.mf.getWorker()).scheduled(); + await (await h.mf.getWorker()).scheduled(); + const accounting = await h.db.prepare('SELECT * FROM deployment').first(); + assert.ok(accounting); + assert.equal(accounting.charged_bytes, 0); + assert.equal(accounting.entry_count, 0); + assert.equal(accounting.association_count, 0); + assert.equal((await h.bucket.list()).objects.length, 0); + } finally { + await h.close(); + } +}); + +void test('rate admission uses bounded keys and precedes storage', async () => { + const h = await harness({ readLimit: 1, storeLimit: 1 }); + try { + assert.equal((await h.mf.dispatchFetch('https://cache.example.com/unknown-one')).status, 404); + const limited = await h.mf.dispatchFetch('https://cache.example.com/unknown-two'); + assert.equal(limited.status, 429); + assert.equal(limited.headers.get('Retry-After'), '60'); + assert.equal((await h.mf.dispatchFetch(`${endpoint}/store`, { method: 'POST' })).status, 401); + assert.equal((await h.mf.dispatchFetch(`${endpoint}/store`, { method: 'POST' })).status, 429); + assert.equal((await h.db.prepare('SELECT count(*) AS n FROM generations').first())!.n, 0); + } finally { + await h.close(); + } +}); + +void test('malformed stores cannot publish and missing live value is a storage failure', async () => { + const h = await harness(); + try { + const metadata = encode({ key: bytes('A'), secondary_key: bytes('S'), value: bytes('V') }); + for (const ending of ['', '\r\n--bad--', '\r\n--x\r\n']) { + const body = Buffer.concat([ + Buffer.from( + '--x\r\nContent-Disposition: form-data; name="metadata"\r\nContent-Type: application/cbor\r\n\r\n', + ), + metadata, + Buffer.from(ending), + ]); + const response = await h.mf.dispatchFetch(`${endpoint}/store`, { + method: 'POST', + headers: { + 'Content-Type': 'multipart/form-data; boundary=x', + Authorization: `Bearer ${await h.token()}`, + }, + body, + }); + assert.equal(response.status, 400); + } + assert.equal((await h.fetch(bytes('A'), bytes('S'))).status, 404); + await h.store(bytes('A'), bytes('S'), bytes('V')); + const g = await h.db + .prepare("SELECT value_object FROM generations WHERE state = 'ready'") + .first(); + assert.ok(g); + assert.equal(typeof g.value_object, 'string'); + await h.bucket.delete(String(g.value_object)); + assert.equal((await h.fetch(bytes('A'), bytes('S'))).status, 503); + } finally { + await h.close(); + } +}); + +void test('configured field and streamed request limits return 413 without publication', async () => { + const h = await harness({ + limits: { + key: 8, + value: 8, + metadata: 256, + fetch: 256, + blob: 16, + store: 2048, + headers: 512, + }, + }); + try { + for (const [key, secondary, value, blob] of [ + [new Uint8Array(9), bytes('S'), bytes('V')], + [bytes('A'), new Uint8Array(9), bytes('V')], + [bytes('A'), bytes('S'), new Uint8Array(9)], + [bytes('A'), bytes('S'), bytes('V'), new Uint8Array(17)], + ]) + assert.equal((await h.store(key!, secondary!, value!, blob)).status, 413); + assert.equal((await h.fetch(new Uint8Array(9), bytes('S'))).status, 413); + const token = await h.token(); + for (const body of [ + `--x\r\nContent-Disposition: form-data; name="metadata"\r\nContent-Type: application/cbor\r\n\r\n${'a'.repeat(257)}\r\n--x--`, + 'a'.repeat(2049), + ]) { + const response = await h.mf.dispatchFetch(`${endpoint}/store`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'multipart/form-data; boundary=x', + }, + body: new Blob([body]).stream(), + duplex: 'half', + }); + assert.equal(response.status, 413); + } + assert.equal((await h.db.prepare('SELECT count(*) AS n FROM entries').first())!.n, 0); + const form = new FormData(); + form.append( + 'metadata', + new Blob([encode({ key: bytes('A'), secondary_key: bytes('S'), value: bytes('V') })], { + type: 'application/cbor', + }), + ); + const request = new Request(`${endpoint}/store`, { method: 'POST', body: form }); + assert.equal(request.headers.has('Content-Length'), false); + const response = await h.mf.dispatchFetch(request.url, { + method: 'POST', + headers: { ...Object.fromEntries(request.headers), Authorization: `Bearer ${token}` }, + body: request.body, + duplex: 'half', + }); + assert.equal(response.status, 200, await response.clone().text()); + assert.deepEqual( + (await decodeResponse(await h.fetch(bytes('A'), bytes('S')))).value, + bytes('V'), + ); + } finally { + await h.close(); + } +}); diff --git a/packages/remote-cache/tsconfig.json b/packages/remote-cache/tsconfig.json new file mode 100644 index 000000000..219deca62 --- /dev/null +++ b/packages/remote-cache/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "lib": ["ES2022"], + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "skipLibCheck": true, + "allowImportingTsExtensions": true, + "noEmit": true + }, + "include": [ + "src/**/*.ts", + "scripts/**/*.ts", + "test/**/*.ts", + ".wrangler/worker-configuration.d.ts" + ] +} diff --git a/packages/remote-cache/wrangler.jsonc b/packages/remote-cache/wrangler.jsonc new file mode 100644 index 000000000..08cfa7a29 --- /dev/null +++ b/packages/remote-cache/wrangler.jsonc @@ -0,0 +1,30 @@ +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "vp-remote-cache", + "main": "src/index.ts", + "compatibility_date": "2026-09-11", + "compatibility_flags": ["nodejs_compat"], + "workers_dev": true, + "preview_urls": false, + "observability": { "enabled": true, "head_sampling_rate": 0.1 }, + "triggers": { "crons": ["*/5 * * * *"] }, + "d1_databases": [ + { + "binding": "INDEX", + "database_name": "vp-remote-cache", + "database_id": "00000000-0000-0000-0000-000000000000", + "migrations_dir": "migrations", + }, + ], + "r2_buckets": [{ "binding": "ARTIFACTS", "bucket_name": "vp-remote-cache" }], + "ratelimits": [ + { "name": "READ_LIMITER", "namespace_id": "1001", "simple": { "limit": 600, "period": 60 } }, + { "name": "STORE_LIMITER", "namespace_id": "1002", "simple": { "limit": 30, "period": 60 } }, + ], + "vars": { + "NAMESPACES": "[]", + "LIMITS": "{}", + "GC_BATCH_SIZE": "16", + "LOG_SAMPLE_RATE": "0.1", + }, +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e4680cd9c..a3c960387 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -81,10 +81,41 @@ importers: version: 6.0.3 vite: specifier: npm:@voidzero-dev/vite-plus-core@0.3.3 - version: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)' + version: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)' vite-plus: specifier: 'catalog:' - version: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(typescript@6.0.3) + version: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3) + + packages/remote-cache: + dependencies: + jose: + specifier: 6.2.12 + version: 6.2.12 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 25.0.3 + cborg: + specifier: 6.1.2 + version: 6.1.2 + esbuild: + specifier: 0.28.1 + version: 0.28.1 + jsonc-parser: + specifier: 3.3.1 + version: 3.3.1 + miniflare: + specifier: 5.20260911.0-alpha + version: 5.20260911.0-alpha(@types/node@25.0.3) + tsx: + specifier: 4.23.13 + version: 4.23.13 + typescript: + specifier: 'catalog:' + version: 6.0.3 + wrangler: + specifier: 4.131.1 + version: 4.131.1(@types/node@25.0.3) packages/tools: dependencies: @@ -102,7 +133,7 @@ importers: version: 1.63.0 '@vitest/browser-playwright': specifier: 'catalog:' - version: 4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11) + version: 4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11) '@voidzero-dev/vite-task-client': specifier: workspace:* version: link:../vite-task-client @@ -123,10 +154,10 @@ importers: version: 2.9.6 oxfmt: specifier: 'catalog:' - version: 0.70.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1)) + version: 0.70.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)) oxlint: specifier: 'catalog:' - version: 1.85.0(oxlint-tsgolint@7.0.2003)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1)) + version: 1.85.0(oxlint-tsgolint@7.0.2003)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)) oxlint-tsgolint: specifier: 'catalog:' version: 7.0.2003 @@ -138,10 +169,10 @@ importers: version: 7.1.0-dev.20260924.1 vite: specifier: 8.3.1 - version: 8.3.1(@types/node@25.0.3) + version: 8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13) vitest: specifier: 'catalog:' - version: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)) + version: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)) devDependencies: vite-task-tools: specifier: workspace:* @@ -171,6 +202,53 @@ packages: '@blazediff/core@1.9.1': resolution: {integrity: sha512-ehg3jIkYKulZh+8om/O25vkvSsXXwC+skXmyA87FFx6A/45eqOkZsBltMw/TVteb0mloiGT8oGRTcjRAz66zaA==} + '@cloudflare/kv-asset-handler@0.5.0': + resolution: {integrity: sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==} + engines: {node: '>=22.0.0'} + + '@cloudflare/unenv-preset@2.16.1': + resolution: {integrity: sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==} + peerDependencies: + unenv: 2.0.0-rc.24 + workerd: '>1.20260305.0 <2.0.0-0' + peerDependenciesMeta: + workerd: + optional: true + + '@cloudflare/workerd-darwin-64@1.20260911.1': + resolution: {integrity: sha512-785eaY1bkR1cm4Z/PCUeteZYmTMe6lre2zz63/GdGGimsoMsKxgl4brFPRukim8iv28EyD1XoCB/VPYF20BERA==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@cloudflare/workerd-darwin-arm64@1.20260911.1': + resolution: {integrity: sha512-WU4bFqEN0H7ndGWxoedegv95DmNVBtv0ncXcHG9nYFTUI78sxEb0qoT3U6Ga4hyBkzsJFBX/zvVBIGX3qKldGA==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@cloudflare/workerd-linux-64@1.20260911.1': + resolution: {integrity: sha512-0Y2gy62oxQxWa38qinSPE6zNL5+JmumJtDY9AWW1HB8KHuATxN71o5MGzmVFfB8PwZsiHfUd2Sv7O22krCOrhw==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@cloudflare/workerd-linux-arm64@1.20260911.1': + resolution: {integrity: sha512-kttNPnx1r2lCqFUoMH62z7CqGV+j4QBbw5fdtaz4pzOrzBv0AWkNATt7onFUe+SwP8zhcepMtbm2F4kKzTf6VA==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@cloudflare/workerd-windows-64@1.20260911.1': + resolution: {integrity: sha512-5iO/YfoBDOgO3CrHdkiiVP8SL3O2jC+c6Ux3d378TSPKLhU5+CgHjtE/ZSodWQrzr4FzFRqdW8S7n5nbyD1MHQ==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + + '@cspotcode/source-map-support@0.8.1': + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + '@deno/darwin-arm64@2.9.6': resolution: {integrity: sha512-V8uO1Aolrl/yvdMb5lOtgtigs8YuZBjrOrgviVMiYkQ0swymFfzkae1wZak4Xi24t7vf7d/eiMb/7ryjjVm+tg==} cpu: [arm64] @@ -203,15 +281,343 @@ packages: cpu: [x64] os: [win32] + '@emnapi/runtime@1.11.3': + resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} + '@epic-web/invariant@1.0.0': resolution: {integrity: sha512-lrTPqgvfFQtR/eY/qkIzp98OGdNJu0m5ji3q/nJI8v3SXkRKEnWiOxMmbvcSoAIzv/cGiuvRy57k4suKQSAdwA==} + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@fastify/busboy@3.2.2': resolution: {integrity: sha512-yXSS27qPExaXeuLvMRMXOLtpipzfQYNjG3FkunDWKGfMYjKuhFXko9CVzqxm8jcF+lmtS9Fd89QNdh9XDjnbNg==} + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.35.4': + resolution: {integrity: sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.35.4': + resolution: {integrity: sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [darwin] + + '@img/sharp-freebsd-wasm32@0.35.4': + resolution: {integrity: sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==} + engines: {node: '>=20.9.0'} + os: [freebsd] + + '@img/sharp-libvips-darwin-arm64@1.3.3': + resolution: {integrity: sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.3.3': + resolution: {integrity: sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.3.3': + resolution: {integrity: sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.3.3': + resolution: {integrity: sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-ppc64@1.3.3': + resolution: {integrity: sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-riscv64@1.3.3': + resolution: {integrity: sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.3.3': + resolution: {integrity: sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.3.3': + resolution: {integrity: sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.3.3': + resolution: {integrity: sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.3.3': + resolution: {integrity: sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.35.4': + resolution: {integrity: sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.35.4': + resolution: {integrity: sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==} + engines: {node: '>=20.9.0'} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-ppc64@0.35.4': + resolution: {integrity: sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==} + engines: {node: '>=20.9.0'} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-riscv64@0.35.4': + resolution: {integrity: sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==} + engines: {node: '>=20.9.0'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.35.4': + resolution: {integrity: sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==} + engines: {node: '>=20.9.0'} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.35.4': + resolution: {integrity: sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.35.4': + resolution: {integrity: sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.35.4': + resolution: {integrity: sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.35.4': + resolution: {integrity: sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==} + engines: {node: '>=20.9.0'} + + '@img/sharp-webcontainers-wasm32@0.35.4': + resolution: {integrity: sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==} + engines: {node: '>=20.9.0'} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.35.4': + resolution: {integrity: sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.35.4': + resolution: {integrity: sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==} + engines: {node: ^20.9.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.35.4': + resolution: {integrity: sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [win32] + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + '@jridgewell/trace-mapping@0.3.9': + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + '@openai/codex@0.156.1': resolution: {integrity: sha512-nI1iVl/n2SO2lSvlwEsJx63zdSI4C4Me2gR7AG0OWMJiGSakz2tY2hx43E39Zq5aEoeB5bZjJXzp5Sqhog6vyA==} engines: {node: '>=16'} @@ -889,6 +1295,15 @@ packages: '@pondwader/socks5-server@1.0.10': resolution: {integrity: sha512-bQY06wzzR8D2+vVCUoBsr5QS2U6UgPUQRmErNwtsuI6vLcyRKkafjkr3KxbtGFf9aBBIV2mcvlsKD1UYaIV+sg==} + '@poppinss/colors@4.1.6': + resolution: {integrity: sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==} + + '@poppinss/dumper@0.6.5': + resolution: {integrity: sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==} + + '@poppinss/exception@1.2.3': + resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} + '@rolldown/binding-android-arm-eabi@1.2.11': resolution: {integrity: sha512-A5kXfGKvKWWZE0TtPrfsvT+q4Y5d1QG8gGUzpYjGydM+fARM9MuX90PrXYXe0XbsDVgyxxNzHo6giCj90bsFNw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -988,6 +1403,13 @@ packages: '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + '@sindresorhus/is@7.2.0': + resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} + engines: {node: '>=18'} + + '@speed-highlight/core@1.2.24': + resolution: {integrity: sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -1365,6 +1787,9 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + blake3-wasm@2.1.5: + resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} + bun@1.4.2: resolution: {integrity: sha512-TrSXo6HJfIEaczpb3kjX82I2pL47vK1QUNmHRCUdz9IzaOwa9lzOXSWwu2l18YHE3sNfGRapVLd4nNm+22vVVA==} cpu: [arm64, x64] @@ -1384,6 +1809,10 @@ packages: resolution: {integrity: sha512-3Cco8XQhi27DogSp9Ri6LYNZLi/TBY/JVnDe+mj06NkBjW/ZYOtekaEU4wZ4xcRMNrFkDv8KNtOAqHyDfz3lYg==} engines: {node: '>=18.7'} + cborg@6.1.2: + resolution: {integrity: sha512-hQfFh6FuuCDoycN68FtUpjtQx5kCtxOLA8msbpJZxjtxaMqxgHl+4GNwO+0YexH/lHmVzhhAKa0ua+vtmTRoVw==} + hasBin: true + chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -1395,6 +1824,10 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + cross-env@10.1.0: resolution: {integrity: sha512-GsYosgnACZTADcmEyJctkJIoqAhHjttw7RsFrVoJNXbsWWqaq6Ym+7kZjq6mS45O0jij6vtiReppKQEtqWy6Dw==} engines: {node: '>=20'} @@ -1419,9 +1852,17 @@ packages: dom-accessibility-api@0.5.16: resolution: {integrity: sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==} + error-stack-parser-es@1.0.5: + resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} + es-module-lexer@2.1.0: resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + estree-walker@3.0.3: resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} @@ -1446,9 +1887,19 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + jose@6.2.12: + resolution: {integrity: sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + jsonc-parser@3.3.1: + resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} + + kleur@4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + lightningcss-android-arm64@1.33.0: resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} engines: {node: '>= 12.0.0'} @@ -1530,6 +1981,10 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + miniflare@5.20260911.0-alpha: + resolution: {integrity: sha512-CRieJmvHx+7rNqnA5SKdsYsER6rfkUIE/jruIUw+fLhsQ4sORfuMtr3+FQzsQ9/y8lhk061V4Fl1DFdHiyBB6g==} + engines: {node: '>=22.0.0'} + mrmime@2.0.1: resolution: {integrity: sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==} engines: {node: '>=10'} @@ -1610,6 +2065,9 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + path-to-regexp@6.3.0: + resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} + pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} @@ -1650,6 +2108,20 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + sharp@0.35.4: + resolution: {integrity: sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==} + engines: {node: '>=20.9.0'} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -1675,6 +2147,10 @@ packages: std-env@4.1.0: resolution: {integrity: sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==} + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -1698,6 +2174,14 @@ packages: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tsx@4.23.13: + resolution: {integrity: sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==} + engines: {node: '>=18.0.0'} + hasBin: true + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -1711,6 +2195,13 @@ packages: undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + + unenv@2.0.0-rc.24: + resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} + vite-plus@0.3.3: resolution: {integrity: sha512-gr3t8ZIsytJnpPE0YPxtkouf+asMsc+TWlKXYzEdY6rRXUdCGYxgaPmzRePx1Gyip2gmARRLogAwoMMW9BvSUA==} engines: {node: ^20.19.0 || ^22.18.0 || >=24.11.0} @@ -1818,6 +2309,21 @@ packages: engines: {node: '>=8'} hasBin: true + workerd@1.20260911.1: + resolution: {integrity: sha512-vRr8QdBxueQOZJO1hRCI73EZlix87IAyBAcSyI3rA1VB+6oxjw3oaqzYnIV8C4IOPtUgihbdMAgzkb5GM4V7DQ==} + engines: {node: '>=16'} + hasBin: true + + wrangler@4.131.1: + resolution: {integrity: sha512-1u5FMdJAn6UOcL02cVsIITcnHrk6mC7N+RF10EkVhPL18R/o9g5BZb4PCjByL+3AsRP5wQpppCIPHhYPRmIJwg==} + engines: {node: '>=22.0.0'} + hasBin: true + peerDependencies: + '@cloudflare/workers-types': ^5.20260911.1 + peerDependenciesMeta: + '@cloudflare/workers-types': + optional: true + ws@8.21.0: resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} engines: {node: '>=10.0.0'} @@ -1830,6 +2336,12 @@ packages: utf-8-validate: optional: true + youch-core@0.3.3: + resolution: {integrity: sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==} + + youch@4.1.0-beta.10: + resolution: {integrity: sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==} + yuku-ast@0.9.5: resolution: {integrity: sha512-Q8qW8WwQnN5Cm0ZZivdRIfv0sRLTjUq0YumXJkw8CYN1aCdICH9rk4C47/4n6kA5EqDtlj+F608twoTSV2MwdQ==} @@ -1863,6 +2375,33 @@ snapshots: '@blazediff/core@1.9.1': {} + '@cloudflare/kv-asset-handler@0.5.0': {} + + '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260911.1)': + dependencies: + unenv: 2.0.0-rc.24 + optionalDependencies: + workerd: 1.20260911.1 + + '@cloudflare/workerd-darwin-64@1.20260911.1': + optional: true + + '@cloudflare/workerd-darwin-arm64@1.20260911.1': + optional: true + + '@cloudflare/workerd-linux-64@1.20260911.1': + optional: true + + '@cloudflare/workerd-linux-arm64@1.20260911.1': + optional: true + + '@cloudflare/workerd-windows-64@1.20260911.1': + optional: true + + '@cspotcode/source-map-support@0.8.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + '@deno/darwin-arm64@2.9.6': optional: true @@ -1881,12 +2420,208 @@ snapshots: '@deno/win32-x64@2.9.6': optional: true + '@emnapi/runtime@1.11.3': + dependencies: + tslib: 2.8.1 + optional: true + '@epic-web/invariant@1.0.0': {} + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.28.1': + optional: true + '@fastify/busboy@3.2.2': {} + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.3.3 + optional: true + + '@img/sharp-darwin-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.3.3 + optional: true + + '@img/sharp-freebsd-wasm32@0.35.4': + dependencies: + '@img/sharp-wasm32': 0.35.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-darwin-x64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-arm@1.3.3': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-s390x@1.3.3': + optional: true + + '@img/sharp-libvips-linux-x64@1.3.3': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.3.3': + optional: true + + '@img/sharp-linux-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.3.3 + optional: true + + '@img/sharp-linux-arm@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.3.3 + optional: true + + '@img/sharp-linux-ppc64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.3.3 + optional: true + + '@img/sharp-linux-riscv64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.3.3 + optional: true + + '@img/sharp-linux-s390x@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.3.3 + optional: true + + '@img/sharp-linux-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.3.3 + optional: true + + '@img/sharp-linuxmusl-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 + optional: true + + '@img/sharp-linuxmusl-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.3.3 + optional: true + + '@img/sharp-wasm32@0.35.4': + dependencies: + '@emnapi/runtime': 1.11.3 + optional: true + + '@img/sharp-webcontainers-wasm32@0.35.4': + dependencies: + '@img/sharp-wasm32': 0.35.4 + optional: true + + '@img/sharp-win32-arm64@0.35.4': + optional: true + + '@img/sharp-win32-ia32@0.35.4': + optional: true + + '@img/sharp-win32-x64@0.35.4': + optional: true + + '@jridgewell/resolve-uri@3.1.2': {} + '@jridgewell/sourcemap-codec@1.5.5': {} + '@jridgewell/trace-mapping@0.3.9': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + '@openai/codex@0.156.1': optionalDependencies: '@openai/codex-darwin-arm64': '@openai/codex@0.156.1-darwin-arm64' @@ -2230,6 +2965,18 @@ snapshots: '@pondwader/socks5-server@1.0.10': {} + '@poppinss/colors@4.1.6': + dependencies: + kleur: 4.1.5 + + '@poppinss/dumper@0.6.5': + dependencies: + '@poppinss/colors': 4.1.6 + '@sindresorhus/is': 7.2.0 + supports-color: 10.2.2 + + '@poppinss/exception@1.2.3': {} + '@rolldown/binding-android-arm-eabi@1.2.11': optional: true @@ -2277,6 +3024,10 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} + '@sindresorhus/is@7.2.0': {} + + '@speed-highlight/core@1.2.24': {} + '@standard-schema/spec@1.1.0': {} '@testing-library/dom@10.4.1': @@ -2332,13 +3083,13 @@ snapshots: '@typescript/typescript-win32-x64@7.1.0-dev.20260924.1': optional: true - '@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11)': + '@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11)': dependencies: - '@vitest/browser': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(vitest@4.1.11) - '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)) + '@vitest/browser': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(vitest@4.1.11) + '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)) playwright: 1.63.0 tinyrainbow: 3.1.0 - vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)) + vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)) transitivePeerDependencies: - bufferutil - msw @@ -2346,37 +3097,37 @@ snapshots: - vite optional: true - '@vitest/browser-playwright@4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11)': + '@vitest/browser-playwright@4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11)': dependencies: - '@vitest/browser': 4.1.11(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11) - '@vitest/mocker': 4.1.11(vite@8.3.1(@types/node@25.0.3)) + '@vitest/browser': 4.1.11(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11) + '@vitest/mocker': 4.1.11(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)) playwright: 1.63.0 tinyrainbow: 3.1.0 - vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)) + vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)) transitivePeerDependencies: - bufferutil - msw - utf-8-validate - vite - '@vitest/browser-preview@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(vitest@4.1.11)': + '@vitest/browser-preview@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(vitest@4.1.11)': dependencies: '@testing-library/dom': 10.4.1 '@testing-library/user-event': 14.6.1(@testing-library/dom@10.4.1) - '@vitest/browser': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(vitest@4.1.11) - vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)) + '@vitest/browser': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(vitest@4.1.11) + vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)) transitivePeerDependencies: - bufferutil - msw - utf-8-validate - vite - '@vitest/browser-preview@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1))(vitest@4.1.11)': + '@vitest/browser-preview@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1))(vitest@4.1.11)': dependencies: '@testing-library/dom': 10.4.1 '@testing-library/user-event': 14.6.1(@testing-library/dom@10.4.1) - '@vitest/browser': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1))(vitest@4.1.11) - vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)) + '@vitest/browser': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1))(vitest@4.1.11) + vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)) transitivePeerDependencies: - bufferutil - msw @@ -2384,12 +3135,12 @@ snapshots: - vite optional: true - '@vitest/browser-preview@4.1.11(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11)': + '@vitest/browser-preview@4.1.11(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11)': dependencies: '@testing-library/dom': 10.4.1 '@testing-library/user-event': 14.6.1(@testing-library/dom@10.4.1) - '@vitest/browser': 4.1.11(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11) - vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)) + '@vitest/browser': 4.1.11(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11) + vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)) transitivePeerDependencies: - bufferutil - msw @@ -2397,16 +3148,16 @@ snapshots: - vite optional: true - '@vitest/browser@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(vitest@4.1.11)': + '@vitest/browser@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(vitest@4.1.11)': dependencies: '@blazediff/core': 1.9.1 - '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)) + '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)) '@vitest/utils': 4.1.11 magic-string: 0.30.21 pngjs: 7.0.0 sirv: 3.0.2 tinyrainbow: 3.1.0 - vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)) + vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)) ws: 8.21.0 transitivePeerDependencies: - bufferutil @@ -2414,16 +3165,16 @@ snapshots: - utf-8-validate - vite - '@vitest/browser@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1))(vitest@4.1.11)': + '@vitest/browser@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1))(vitest@4.1.11)': dependencies: '@blazediff/core': 1.9.1 - '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1)) + '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)) '@vitest/utils': 4.1.11 magic-string: 0.30.21 pngjs: 7.0.0 sirv: 3.0.2 tinyrainbow: 3.1.0 - vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)) + vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)) ws: 8.21.0 transitivePeerDependencies: - bufferutil @@ -2432,16 +3183,16 @@ snapshots: - vite optional: true - '@vitest/browser@4.1.11(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11)': + '@vitest/browser@4.1.11(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11)': dependencies: '@blazediff/core': 1.9.1 - '@vitest/mocker': 4.1.11(vite@8.3.1(@types/node@25.0.3)) + '@vitest/mocker': 4.1.11(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)) '@vitest/utils': 4.1.11 magic-string: 0.30.21 pngjs: 7.0.0 sirv: 3.0.2 tinyrainbow: 3.1.0 - vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)) + vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)) ws: 8.21.0 transitivePeerDependencies: - bufferutil @@ -2458,30 +3209,30 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))': + '@vitest/mocker@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))': dependencies: '@vitest/spy': 4.1.11 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)' + vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)' - '@vitest/mocker@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1))': + '@vitest/mocker@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1))': dependencies: '@vitest/spy': 4.1.11 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1)' + vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)' optional: true - '@vitest/mocker@4.1.11(vite@8.3.1(@types/node@25.0.3))': + '@vitest/mocker@4.1.11(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))': dependencies: '@vitest/spy': 4.1.11 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.3.1(@types/node@25.0.3) + vite: 8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13) '@vitest/pretty-format@4.1.11': dependencies: @@ -2507,7 +3258,7 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.1.0 - '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)': + '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)': dependencies: '@oxc-project/runtime': 0.150.0 '@oxc-project/types': 0.150.0 @@ -2525,10 +3276,12 @@ snapshots: '@voidzero-dev/vite-plus-linux-x64-musl': 0.3.3 '@voidzero-dev/vite-plus-win32-arm64-msvc': 0.3.3 '@voidzero-dev/vite-plus-win32-x64-msvc': 0.3.3 + esbuild: 0.28.1 fsevents: 2.3.3 + tsx: 4.23.13 typescript: 6.0.3 - '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1)': + '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)': dependencies: '@oxc-project/runtime': 0.150.0 '@oxc-project/types': 0.150.0 @@ -2546,7 +3299,9 @@ snapshots: '@voidzero-dev/vite-plus-linux-x64-musl': 0.3.3 '@voidzero-dev/vite-plus-win32-arm64-msvc': 0.3.3 '@voidzero-dev/vite-plus-win32-x64-msvc': 0.3.3 + esbuild: 0.28.1 fsevents: 2.3.3 + tsx: 4.23.13 typescript: 7.1.0-dev.20260924.1 optional: true @@ -2658,6 +3413,8 @@ snapshots: assertion-error@2.0.1: {} + blake3-wasm@2.1.5: {} + bun@1.4.2: optionalDependencies: '@oven/bun-darwin-aarch64': 1.4.2 @@ -2681,12 +3438,16 @@ snapshots: cbor2@1.12.0: {} + cborg@6.1.2: {} + chai@6.2.2: {} commander@12.1.0: {} convert-source-map@2.0.0: {} + cookie@1.1.1: {} + cross-env@10.1.0: dependencies: '@epic-web/invariant': 1.0.0 @@ -2713,8 +3474,39 @@ snapshots: dom-accessibility-api@0.5.16: {} + error-stack-parser-es@1.0.5: {} + es-module-lexer@2.1.0: {} + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + estree-walker@3.0.3: dependencies: '@types/estree': 1.0.9 @@ -2730,8 +3522,14 @@ snapshots: isexe@2.0.0: {} + jose@6.2.12: {} + js-tokens@4.0.0: {} + jsonc-parser@3.3.1: {} + + kleur@4.1.5: {} + lightningcss-android-arm64@1.33.0: optional: true @@ -2787,6 +3585,19 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + miniflare@5.20260911.0-alpha(@types/node@25.0.3): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + sharp: 0.35.4(@types/node@25.0.3) + undici: 7.29.0 + workerd: 1.20260911.1 + ws: 8.21.0 + youch: 4.1.0-beta.10 + transitivePeerDependencies: + - '@types/node' + - bufferutil + - utf-8-validate + mrmime@2.0.1: {} nanoid@3.3.18: {} @@ -2795,7 +3606,7 @@ snapshots: obug@2.1.1: {} - oxfmt@0.68.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(typescript@6.0.3)): + oxfmt@0.68.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)): dependencies: tinypool: 2.1.2 optionalDependencies: @@ -2818,9 +3629,9 @@ snapshots: '@oxfmt/binding-win32-arm64-msvc': 0.68.0 '@oxfmt/binding-win32-ia32-msvc': 0.68.0 '@oxfmt/binding-win32-x64-msvc': 0.68.0 - vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(typescript@6.0.3) + vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3) - oxfmt@0.68.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1)): + oxfmt@0.68.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)): dependencies: tinypool: 2.1.2 optionalDependencies: @@ -2843,10 +3654,10 @@ snapshots: '@oxfmt/binding-win32-arm64-msvc': 0.68.0 '@oxfmt/binding-win32-ia32-msvc': 0.68.0 '@oxfmt/binding-win32-x64-msvc': 0.68.0 - vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1) + vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1) optional: true - oxfmt@0.70.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1)): + oxfmt@0.70.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)): dependencies: tinypool: 2.1.2 optionalDependencies: @@ -2869,7 +3680,7 @@ snapshots: '@oxfmt/binding-win32-arm64-msvc': 0.70.0 '@oxfmt/binding-win32-ia32-msvc': 0.70.0 '@oxfmt/binding-win32-x64-msvc': 0.70.0 - vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1) + vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1) oxlint-tsgolint@7.0.2001: optionalDependencies: @@ -2889,7 +3700,7 @@ snapshots: '@oxlint-tsgolint/win32-arm64': 7.0.2003 '@oxlint-tsgolint/win32-x64': 7.0.2003 - oxlint@1.83.0(oxlint-tsgolint@7.0.2001)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(typescript@6.0.3)): + oxlint@1.83.0(oxlint-tsgolint@7.0.2001)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)): optionalDependencies: '@oxlint/binding-android-arm-eabi': 1.83.0 '@oxlint/binding-android-arm64': 1.83.0 @@ -2911,9 +3722,9 @@ snapshots: '@oxlint/binding-win32-ia32-msvc': 1.83.0 '@oxlint/binding-win32-x64-msvc': 1.83.0 oxlint-tsgolint: 7.0.2001 - vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(typescript@6.0.3) + vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3) - oxlint@1.83.0(oxlint-tsgolint@7.0.2001)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1)): + oxlint@1.83.0(oxlint-tsgolint@7.0.2001)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)): optionalDependencies: '@oxlint/binding-android-arm-eabi': 1.83.0 '@oxlint/binding-android-arm64': 1.83.0 @@ -2935,10 +3746,10 @@ snapshots: '@oxlint/binding-win32-ia32-msvc': 1.83.0 '@oxlint/binding-win32-x64-msvc': 1.83.0 oxlint-tsgolint: 7.0.2001 - vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1) + vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1) optional: true - oxlint@1.85.0(oxlint-tsgolint@7.0.2003)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1)): + oxlint@1.85.0(oxlint-tsgolint@7.0.2003)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)): optionalDependencies: '@oxlint/binding-android-arm-eabi': 1.85.0 '@oxlint/binding-android-arm64': 1.85.0 @@ -2960,10 +3771,12 @@ snapshots: '@oxlint/binding-win32-ia32-msvc': 1.85.0 '@oxlint/binding-win32-x64-msvc': 1.85.0 oxlint-tsgolint: 7.0.2003 - vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1) + vite-plus: 0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1) path-key@3.1.1: {} + path-to-regexp@6.3.0: {} + pathe@2.0.3: {} picocolors@1.1.1: {} @@ -3013,6 +3826,41 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.2.11 '@rolldown/binding-win32-x64-msvc': 1.2.11 + semver@7.8.5: {} + + sharp@0.35.4(@types/node@25.0.3): + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.8.5 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.35.4 + '@img/sharp-darwin-x64': 0.35.4 + '@img/sharp-freebsd-wasm32': 0.35.4 + '@img/sharp-libvips-darwin-arm64': 1.3.3 + '@img/sharp-libvips-darwin-x64': 1.3.3 + '@img/sharp-libvips-linux-arm': 1.3.3 + '@img/sharp-libvips-linux-arm64': 1.3.3 + '@img/sharp-libvips-linux-ppc64': 1.3.3 + '@img/sharp-libvips-linux-riscv64': 1.3.3 + '@img/sharp-libvips-linux-s390x': 1.3.3 + '@img/sharp-libvips-linux-x64': 1.3.3 + '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 + '@img/sharp-libvips-linuxmusl-x64': 1.3.3 + '@img/sharp-linux-arm': 0.35.4 + '@img/sharp-linux-arm64': 0.35.4 + '@img/sharp-linux-ppc64': 0.35.4 + '@img/sharp-linux-riscv64': 0.35.4 + '@img/sharp-linux-s390x': 0.35.4 + '@img/sharp-linux-x64': 0.35.4 + '@img/sharp-linuxmusl-arm64': 0.35.4 + '@img/sharp-linuxmusl-x64': 0.35.4 + '@img/sharp-webcontainers-wasm32': 0.35.4 + '@img/sharp-win32-arm64': 0.35.4 + '@img/sharp-win32-ia32': 0.35.4 + '@img/sharp-win32-x64': 0.35.4 + '@types/node': 25.0.3 + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -3033,6 +3881,8 @@ snapshots: std-env@4.1.0: {} + supports-color@10.2.2: {} + tinybench@2.9.0: {} tinyexec@1.1.2: {} @@ -3048,6 +3898,15 @@ snapshots: totalist@3.0.1: {} + tslib@2.8.1: + optional: true + + tsx@4.23.13: + dependencies: + esbuild: 0.28.1 + optionalDependencies: + fsevents: 2.3.3 + typescript@6.0.3: {} typescript@7.1.0-dev.20260924.1: @@ -3062,26 +3921,32 @@ snapshots: undici-types@7.16.0: {} - vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(typescript@6.0.3): + undici@7.29.0: {} + + unenv@2.0.0-rc.24: + dependencies: + pathe: 2.0.3 + + vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3): dependencies: '@oxc-project/types': 0.150.0 '@oxlint/plugins': 1.79.0 - '@vitest/browser': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(vitest@4.1.11) - '@vitest/browser-preview': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(vitest@4.1.11) + '@vitest/browser': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(vitest@4.1.11) + '@vitest/browser-preview': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(vitest@4.1.11) '@vitest/expect': 4.1.11 - '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)) + '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)) '@vitest/pretty-format': 4.1.11 '@vitest/runner': 4.1.11 '@vitest/snapshot': 4.1.11 '@vitest/spy': 4.1.11 '@vitest/utils': 4.1.11 - oxfmt: 0.68.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(typescript@6.0.3)) - oxlint: 1.83.0(oxlint-tsgolint@7.0.2001)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(typescript@6.0.3)) + oxfmt: 0.68.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)) + oxlint: 1.83.0(oxlint-tsgolint@7.0.2001)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)) oxlint-tsgolint: 7.0.2001 - vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)' - vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)) + vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)' + vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)) optionalDependencies: - '@vitest/browser-playwright': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11) + '@vitest/browser-playwright': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11) '@voidzero-dev/vite-plus-darwin-arm64': 0.3.3 '@voidzero-dev/vite-plus-darwin-x64': 0.3.3 '@voidzero-dev/vite-plus-linux-arm64-gnu': 0.3.3 @@ -3120,26 +3985,26 @@ snapshots: - utf-8-validate - yaml - vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1): + vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1): dependencies: '@oxc-project/types': 0.150.0 '@oxlint/plugins': 1.79.0 - '@vitest/browser': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1))(vitest@4.1.11) - '@vitest/browser-preview': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1))(vitest@4.1.11) + '@vitest/browser': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1))(vitest@4.1.11) + '@vitest/browser-preview': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1))(vitest@4.1.11) '@vitest/expect': 4.1.11 - '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1)) + '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)) '@vitest/pretty-format': 4.1.11 '@vitest/runner': 4.1.11 '@vitest/snapshot': 4.1.11 '@vitest/spy': 4.1.11 '@vitest/utils': 4.1.11 - oxfmt: 0.68.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1)) - oxlint: 1.83.0(oxlint-tsgolint@7.0.2001)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(typescript@7.1.0-dev.20260924.1)) + oxfmt: 0.68.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)) + oxlint: 1.83.0(oxlint-tsgolint@7.0.2001)(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)) oxlint-tsgolint: 7.0.2001 - vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1)' - vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1)) + vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)' + vitest: 4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)) optionalDependencies: - '@vitest/browser-playwright': 4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11) + '@vitest/browser-playwright': 4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11) '@voidzero-dev/vite-plus-darwin-arm64': 0.3.3 '@voidzero-dev/vite-plus-darwin-x64': 0.3.3 '@voidzero-dev/vite-plus-linux-arm64-gnu': 0.3.3 @@ -3179,7 +4044,7 @@ snapshots: - yaml optional: true - vite@8.3.1(@types/node@25.0.3): + vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13): dependencies: lightningcss: 1.33.0 picomatch: 4.0.7 @@ -3188,12 +4053,14 @@ snapshots: tinyglobby: 0.2.17 optionalDependencies: '@types/node': 25.0.3 + esbuild: 0.28.1 fsevents: 2.3.3 + tsx: 4.23.13 - vitest@4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)): + vitest@4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11))(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)): dependencies: '@vitest/expect': 4.1.11 - '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)) + '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)) '@vitest/pretty-format': 4.1.11 '@vitest/runner': 4.1.11 '@vitest/snapshot': 4.1.11 @@ -3210,19 +4077,19 @@ snapshots: tinyexec: 1.1.2 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3)' + vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3)' why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 25.0.3 - '@vitest/browser-playwright': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11) - '@vitest/browser-preview': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@6.0.3))(vitest@4.1.11) + '@vitest/browser-playwright': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(playwright@1.63.0)(vitest@4.1.11) + '@vitest/browser-preview': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@6.0.3))(vitest@4.1.11) transitivePeerDependencies: - msw - vitest@4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1)): + vitest@4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)): dependencies: '@vitest/expect': 4.1.11 - '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1)) + '@vitest/mocker': 4.1.11(@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)) '@vitest/pretty-format': 4.1.11 '@vitest/runner': 4.1.11 '@vitest/snapshot': 4.1.11 @@ -3239,20 +4106,20 @@ snapshots: tinyexec: 1.1.2 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(typescript@7.1.0-dev.20260924.1)' + vite: '@voidzero-dev/vite-plus-core@0.3.3(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)' why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 25.0.3 - '@vitest/browser-playwright': 4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11) - '@vitest/browser-preview': 4.1.11(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11) + '@vitest/browser-playwright': 4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11) + '@vitest/browser-preview': 4.1.11(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11) transitivePeerDependencies: - msw optional: true - vitest@4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)): + vitest@4.1.11(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(@vitest/browser-preview@4.1.11)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)): dependencies: '@vitest/expect': 4.1.11 - '@vitest/mocker': 4.1.11(vite@8.3.1(@types/node@25.0.3)) + '@vitest/mocker': 4.1.11(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13)) '@vitest/pretty-format': 4.1.11 '@vitest/runner': 4.1.11 '@vitest/snapshot': 4.1.11 @@ -3269,12 +4136,12 @@ snapshots: tinyexec: 1.1.2 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vite: 8.3.1(@types/node@25.0.3) + vite: 8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 25.0.3 - '@vitest/browser-playwright': 4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11) - '@vitest/browser-preview': 4.1.11(vite@8.3.1(@types/node@25.0.3))(vitest@4.1.11) + '@vitest/browser-playwright': 4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11) + '@vitest/browser-preview': 4.1.11(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11) transitivePeerDependencies: - msw @@ -3287,8 +4154,46 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + workerd@1.20260911.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260911.1 + '@cloudflare/workerd-darwin-arm64': 1.20260911.1 + '@cloudflare/workerd-linux-64': 1.20260911.1 + '@cloudflare/workerd-linux-arm64': 1.20260911.1 + '@cloudflare/workerd-windows-64': 1.20260911.1 + + wrangler@4.131.1(@types/node@25.0.3): + dependencies: + '@cloudflare/kv-asset-handler': 0.5.0 + '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260911.1) + blake3-wasm: 2.1.5 + esbuild: 0.28.1 + miniflare: 5.20260911.0-alpha(@types/node@25.0.3) + path-to-regexp: 6.3.0 + unenv: 2.0.0-rc.24 + workerd: 1.20260911.1 + optionalDependencies: + fsevents: 2.3.3 + transitivePeerDependencies: + - '@types/node' + - bufferutil + - utf-8-validate + ws@8.21.0: {} + youch-core@0.3.3: + dependencies: + '@poppinss/exception': 1.2.3 + error-stack-parser-es: 1.0.5 + + youch@4.1.0-beta.10: + dependencies: + '@poppinss/colors': 4.1.6 + '@poppinss/dumper': 0.6.5 + '@speed-highlight/core': 1.2.24 + cookie: 1.1.1 + youch-core: 0.3.3 + yuku-ast@0.9.5: dependencies: '@yuku-toolchain/types': 0.9.5 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 6b0aa8e56..cb08fb5f9 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -2,6 +2,7 @@ packages: - . - packages/tools - packages/vite-task-client + - packages/remote-cache allowBuilds: '@playwright/browser-chromium': true @@ -11,6 +12,15 @@ allowBuilds: minimumReleaseAgeExclude: - vite-plus - '@voidzero-dev/*' + - '@cloudflare/workerd-darwin-64@1.20260911.1' + - '@cloudflare/workerd-darwin-arm64@1.20260911.1' + - '@cloudflare/workerd-linux-64@1.20260911.1' + - '@cloudflare/workerd-linux-arm64@1.20260911.1' + - '@cloudflare/workerd-windows-64@1.20260911.1' + - '@cloudflare/workers-types@5.20260911.1' + - miniflare@5.20260911.0-alpha + - workerd@1.20260911.1 + - wrangler@4.131.1 catalog: '@anthropic-ai/sandbox-runtime': 0.0.77 From 62498088963dbb53182e9f9d4b95a6e122b93ccd Mon Sep 17 00:00:00 2001 From: MK Date: Sat, 12 Sep 2026 09:50:41 +0800 Subject: [PATCH 02/17] docs: link remote cache changelog to pull request Co-authored-by: GPT-6 Codex --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccda6386e..e5b0dfd3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -- **Added** A self-hosted public remote cache service with GitHub Actions write authorization, storage limits, and automatic cleanup ([#???](https://github.com/voidzero-dev/vite-task/pull/???)). +- **Added** A self-hosted public remote cache service with GitHub Actions write authorization, storage limits, and automatic cleanup ([#718](https://github.com/voidzero-dev/vite-task/pull/718)). - **Fixed** An invalid glob in `--filter` no longer shows its error message twice ([#763](https://github.com/voidzero-dev/vite-task/pull/763)). - **Changed** The detailed summary from `vp run --verbose` and `vp run --last-details` now shows each underlying cause of an error on its own line ([#761](https://github.com/voidzero-dev/vite-task/pull/761)). - **Added** Remote caching. Configure an endpoint with the workspace's `cache: { remote: { url } }` or `VP_REMOTE_CACHE_URL`, and choose access with `--remote-cache=off|read|read-write` or `VP_REMOTE_CACHE`. The default is `read` with an endpoint and `off` without one. After a local cache miss, `vp run` looks the task up in the remote cache and, on a hit, restores its outputs and caches it locally. The task output and the run summary show which hits came from the remote cache. A failed read is just a cache miss, with the failure as its reason. In `read-write` mode, `vp run` also uploads the results of successful, cacheable tasks after caching them locally. A failed upload doesn't fail the task; the run summary shows a warning instead. Ctrl-C, or a failing task, stops remote cache requests right away, and a task still being looked up doesn't start. Tasks can opt out with `cache: { remote: false }`. Requests use the proxy environment variables or, on macOS and Windows, the system proxy settings ([#727](https://github.com/voidzero-dev/vite-task/pull/727), [#755](https://github.com/voidzero-dev/vite-task/pull/755), [#756](https://github.com/voidzero-dev/vite-task/pull/756), [#757](https://github.com/voidzero-dev/vite-task/pull/757), [#764](https://github.com/voidzero-dev/vite-task/pull/764), [#771](https://github.com/voidzero-dev/vite-task/pull/771), [#772](https://github.com/voidzero-dev/vite-task/pull/772)). From 8ae2f705c6d375058be6bee25966c3d20dab0307 Mon Sep 17 00:00:00 2001 From: MK Date: Sat, 12 Sep 2026 10:47:15 +0800 Subject: [PATCH 03/17] ci(remote-cache): deploy previews and verify Cloudflare Co-authored-by: GPT-6 Codex --- .github/workflows/remote-cache-deploy.yml | 159 ++++++ .../workflows/remote-cache-preview-events.yml | 112 +++++ packages/remote-cache/.gitignore | 1 + packages/remote-cache/README.md | 2 + packages/remote-cache/docs/e2e-plan.md | 162 +++++++ packages/remote-cache/package.json | 5 +- packages/remote-cache/scripts/ci.ts | 398 +++++++++++++++ packages/remote-cache/scripts/e2e/fixtures.ts | 76 +++ packages/remote-cache/scripts/e2e/suite.ts | 459 ++++++++++++++++++ packages/remote-cache/scripts/operator.ts | 19 +- packages/remote-cache/src/index.ts | 1 + packages/remote-cache/test/deployed.test.ts | 257 ++++++++++ packages/remote-cache/test/helpers.ts | 5 +- packages/remote-cache/test/index.test.ts | 1 + packages/remote-cache/wrangler.jsonc | 1 + 15 files changed, 1651 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/remote-cache-deploy.yml create mode 100644 .github/workflows/remote-cache-preview-events.yml create mode 100644 packages/remote-cache/docs/e2e-plan.md create mode 100644 packages/remote-cache/scripts/ci.ts create mode 100644 packages/remote-cache/scripts/e2e/fixtures.ts create mode 100644 packages/remote-cache/scripts/e2e/suite.ts create mode 100644 packages/remote-cache/test/deployed.test.ts diff --git a/.github/workflows/remote-cache-deploy.yml b/.github/workflows/remote-cache-deploy.yml new file mode 100644 index 000000000..ef49b1c3d --- /dev/null +++ b/.github/workflows/remote-cache-deploy.yml @@ -0,0 +1,159 @@ +name: Remote cache deployment + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - packages/remote-cache/** + - .github/workflows/remote-cache-*.yml + - package.json + - pnpm-lock.yaml + - pnpm-workspace.yaml + - vite.config.ts + push: + branches: [main] + paths: + - packages/remote-cache/** + - .github/workflows/remote-cache-*.yml + - package.json + - pnpm-lock.yaml + - pnpm-workspace.yaml + - vite.config.ts + workflow_dispatch: + inputs: + full: + description: Check real Cron cleanup (manual runs cannot authorize HTTP stores) + type: boolean + default: true + +permissions: + contents: read + +# Share the group with PR teardown. Do not cancel a partially completed deployment. +concurrency: + group: remote-cache-cloudflare-${{ github.event.pull_request.number || 'main' }} + cancel-in-progress: false + +env: + REMOTE_CACHE_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + REMOTE_CACHE_PR_NUMBER: ${{ github.event.pull_request.number }} + REMOTE_CACHE_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + REMOTE_CACHE_RESOURCE_PREFIX: ${{ vars.REMOTE_CACHE_RESOURCE_PREFIX || 'vp-cache-ci' }} + REMOTE_CACHE_WORKERS_SUBDOMAIN: ${{ vars.REMOTE_CACHE_WORKERS_SUBDOMAIN }} + +jobs: + check: + name: Check deployment source + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ env.REMOTE_CACHE_SOURCE_SHA }} + persist-credentials: false + - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 + - run: pnpm check-remote-cache + + deploy: + name: Deploy and verify + needs: check + if: >- + vars.REMOTE_CACHE_DEPLOY_ENABLED == 'true' && + github.event.pull_request.user.login != 'dependabot[bot]' && + (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && + (github.event_name != 'workflow_dispatch' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) + runs-on: ubuntu-latest + timeout-minutes: 40 + environment: + name: remote-cache-staging + url: ${{ steps.deploy.outputs.endpoint }} + permissions: + contents: read + id-token: write + pull-requests: read + outputs: + endpoint: ${{ steps.deploy.outputs.endpoint }} + steps: + - name: Check that the PR still selects this commit + if: github.event_name == 'pull_request' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: context.issue.number }); + if (pr.state !== 'open' || pr.head.sha !== process.env.REMOTE_CACHE_SOURCE_SHA) { + core.setFailed('This PR deployment is obsolete.'); + } + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ env.REMOTE_CACHE_SOURCE_SHA }} + persist-credentials: false + - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 + - name: Deploy isolated Cloudflare resources + id: deploy + run: pnpm --filter @voidzero-dev/remote-cache ci:deploy + env: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + - name: Test the deployed Worker, D1, and R2 + run: pnpm --filter @voidzero-dev/remote-cache e2e + env: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + REMOTE_CACHE_E2E_FULL: ${{ github.event_name == 'push' || inputs.full == true }} + - name: Save verification results and manual fixtures + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: remote-cache-e2e-${{ github.run_id }}-${{ github.run_attempt }} + path: packages/remote-cache/e2e-results/ + retention-days: 7 + if-no-files-found: warn + + notify: + name: Update PR verification instructions + if: >- + always() && github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login != 'dependabot[bot]' + needs: [check, deploy] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: write + steps: + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + DEPLOY_RESULT: ${{ needs.deploy.result }} + CHECK_RESULT: ${{ needs.check.result }} + DEPLOY_ENDPOINT: ${{ needs.deploy.outputs.endpoint }} + with: + script: | + const marker = ''; + const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: context.issue.number }); + if (pr.state !== 'open' || pr.head.sha !== process.env.REMOTE_CACHE_SOURCE_SHA) return; + const run = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const docs = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/blob/${pr.head.sha}/packages/remote-cache/docs/e2e-plan.md`; + const passed = process.env.DEPLOY_RESULT === 'success'; + let status; + if (passed) { + const endpoint = process.env.DEPLOY_ENDPOINT; + const expected = `https://${process.env.REMOTE_CACHE_RESOURCE_PREFIX}-pr-${pr.number}.${process.env.REMOTE_CACHE_WORKERS_SUBDOMAIN}.workers.dev/projects/manual`; + if (endpoint !== expected) throw new Error('Unexpected deployment endpoint'); + status = `Cloudflare deployment and PR e2e checks passed. You can now perform manual verification.\n\n` + + `Endpoint: ${endpoint}\n\n` + + `Download the **remote-cache-e2e-${context.runId}-${process.env.GITHUB_RUN_ATTEMPT}** artifact from the [workflow run](${run}). ` + + `It contains \`manual-fetch.cbor\`, \`manual-manifest.json\`, and \`report.json\`.\n\n` + + `PR checks cover public reads and rejected writes. Main-branch push checks also cover authorized HTTP stores and real Cron cleanup. ` + + `The endpoint remains available until this PR closes or a later commit replaces it.`; + } else if (process.env.CHECK_RESULT !== 'success') { + status = `The source checks did not pass. No verified Cloudflare preview is ready. See the [workflow run](${run}).`; + } else if (process.env.DEPLOY_RESULT === 'skipped') { + status = `Cloudflare deployment is disabled. Configure the staging environment described in the [e2e plan](${docs}), ` + + `then set \`REMOTE_CACHE_DEPLOY_ENABLED=true\` and rerun this workflow. No live deployment passed verification.`; + } else { + status = `Cloudflare deployment or e2e verification failed. The preview is not marked ready. See the [workflow run](${run}).`; + } + const body = `${marker}\n### Remote cache preview\n\nCommit: \`${pr.head.sha}\`\n\n${status}\n\n[Manual checks and complete e2e plan](${docs}).`; + const comments = await github.paginate(github.rest.issues.listComments, { ...context.repo, issue_number: pr.number, per_page: 100 }); + const previous = comments.find(c => c.user?.login === 'github-actions[bot]' && c.body?.includes(marker)); + if (previous) await github.rest.issues.updateComment({ ...context.repo, comment_id: previous.id, body }); + else await github.rest.issues.createComment({ ...context.repo, issue_number: pr.number, body }); diff --git a/.github/workflows/remote-cache-preview-events.yml b/.github/workflows/remote-cache-preview-events.yml new file mode 100644 index 000000000..ff1924f2e --- /dev/null +++ b/.github/workflows/remote-cache-preview-events.yml @@ -0,0 +1,112 @@ +name: Remote cache preview events + +on: + pull_request_target: + types: [opened, synchronize, reopened, closed] + paths: + - packages/remote-cache/** + - .github/workflows/remote-cache-*.yml + - package.json + - pnpm-lock.yaml + - pnpm-workspace.yaml + - vite.config.ts + workflow_dispatch: + inputs: + pr: + description: Closed PR number whose preview resources should be removed + type: string + required: true + +permissions: {} + +jobs: + fork-notice: + if: >- + github.event_name == 'pull_request_target' && github.event.action != 'closed' && + (github.event.pull_request.head.repo.full_name != github.repository || github.event.pull_request.user.login == 'dependabot[bot]') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: write + steps: + # This privileged workflow never checks out or executes PR code. + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const marker = ''; + const body = `${marker}\n### Remote cache preview\n\nCloudflare credentials are not available to fork or Dependabot workflows. ` + + `Local protocol and integration checks still run. A maintainer can copy the reviewed commit to a branch in this repository ` + + `and open a PR to deploy an isolated preview. No live deployment passed verification for this PR.`; + const comments = await github.paginate(github.rest.issues.listComments, { ...context.repo, issue_number: context.issue.number, per_page: 100 }); + const previous = comments.find(c => c.user?.login === 'github-actions[bot]' && c.body?.includes(marker)); + if (previous) await github.rest.issues.updateComment({ ...context.repo, comment_id: previous.id, body }); + else await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body }); + + cleanup: + name: Remove closed PR resources + if: >- + vars.REMOTE_CACHE_DEPLOY_ENABLED == 'true' && + ((github.event.action == 'closed' && github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'workflow_dispatch' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))) + runs-on: ubuntu-latest + timeout-minutes: 35 + concurrency: + group: remote-cache-cloudflare-${{ github.event.pull_request.number || inputs.pr }} + cancel-in-progress: false + environment: remote-cache-staging + permissions: + contents: read + pull-requests: read + env: + REMOTE_CACHE_PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr }} + REMOTE_CACHE_SOURCE_SHA: ${{ github.sha }} + REMOTE_CACHE_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + REMOTE_CACHE_RESOURCE_PREFIX: ${{ vars.REMOTE_CACHE_RESOURCE_PREFIX || 'vp-cache-ci' }} + REMOTE_CACHE_WORKERS_SUBDOMAIN: ${{ vars.REMOTE_CACHE_WORKERS_SUBDOMAIN }} + steps: + - name: Verify that the PR is closed + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const number = process.env.REMOTE_CACHE_PR_NUMBER; + if (!/^[1-9][0-9]{0,9}$/.test(number)) throw new Error('Invalid PR number'); + const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: Number(number) }); + if (pr.state !== 'closed') throw new Error('Only closed PR previews can be removed'); + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 + - run: pnpm --filter @voidzero-dev/remote-cache ci:cleanup + env: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + + cleanup-notice: + if: always() && needs.cleanup.result != 'skipped' + needs: cleanup + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: write + steps: + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr }} + CLEANUP_RESULT: ${{ needs.cleanup.result }} + with: + script: | + if (!/^[1-9][0-9]{0,9}$/.test(process.env.PR_NUMBER)) return; + const number = Number(process.env.PR_NUMBER); + const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: number }); + if (pr.state !== 'closed') return; + const marker = ''; + const comments = await github.paginate(github.rest.issues.listComments, { ...context.repo, issue_number: number, per_page: 100 }); + const previous = comments.find(c => c.user?.login === 'github-actions[bot]' && c.body?.includes(marker)); + if (!previous) return; + const run = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const status = process.env.CLEANUP_RESULT === 'success' + ? 'The PR is closed. Its Cloudflare preview resources were removed.' + : 'The PR is closed, but preview cleanup did not finish. Check the workflow and rerun cleanup after Cron or lifecycle completes.'; + await github.rest.issues.updateComment({ ...context.repo, comment_id: previous.id, + body: `${marker}\n### Remote cache preview\n\n${status}\n\n[Cleanup run](${run}).` }); diff --git a/packages/remote-cache/.gitignore b/packages/remote-cache/.gitignore index 0959f873a..36aba5f3b 100644 --- a/packages/remote-cache/.gitignore +++ b/packages/remote-cache/.gitignore @@ -3,3 +3,4 @@ operator-state.json wrangler.operator.json benchmark-results.json +e2e-results/ diff --git a/packages/remote-cache/README.md b/packages/remote-cache/README.md index 8717dd454..436d34de9 100644 --- a/packages/remote-cache/README.md +++ b/packages/remote-cache/README.md @@ -41,6 +41,8 @@ From this package directory, `pnpm dev` starts Wrangler with local bindings. App ## Setup +For automatic PR previews, main-branch staging deployments, and tests against real Cloudflare resources, follow the [deployment and e2e plan](docs/e2e-plan.md). It includes GitHub environment configuration, the complete test matrix, manual verification, and preview teardown. + Use a dedicated Worker, D1 database, and bucket. The operator requires `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` in its environment. The token needs account permissions for Workers Scripts, D1, and Workers R2 Storage, plus route/zone permissions if using a custom domain. Enable R2 in the account first. Credentials stay in the operator process and Wrangler; they are never stored in namespace policy or passed as command arguments. Run from this package directory: diff --git a/packages/remote-cache/docs/e2e-plan.md b/packages/remote-cache/docs/e2e-plan.md new file mode 100644 index 000000000..ecf3539be --- /dev/null +++ b/packages/remote-cache/docs/e2e-plan.md @@ -0,0 +1,162 @@ +# Cloudflare deployment and e2e verification + +This plan checks the service through its public HTTP endpoints after deployment to Cloudflare. A deployment dry run or a passing Miniflare test is not evidence of a successful Cloudflare deployment. + +## Deployment flow + +`.github/workflows/remote-cache-deploy.yml` runs when a PR or a push to `main` changes this package, its deployment workflows, or the root dependency/build configuration. It also supports manual runs from the default branch. + +1. Check the exact source commit with `pnpm check-remote-cache`. +2. Create or update dedicated staging resources and apply D1 migrations. +3. Check that the R2 bucket has no public domain. Seed public test fixtures through the authenticated operator API. +4. Test the deployed HTTP endpoints. Every response must identify the expected commit and workflow attempt through `X-Remote-Cache-Deployment`. +5. Save a JSON report and manual fixtures as a seven-day workflow artifact. +6. Update one PR comment with the commit, result, endpoint, artifact link, and manual instructions. A failed or skipped deployment never gets a ready message. + +Each PR uses `-pr-` for its Worker, database, and bucket. Main uses `-main`. The default prefix is `vp-cache-ci`. Each deployment has three public namespaces: `e2e`, `other`, and `manual`. Resources contain synthetic data only. They are separate from production and from other PRs. + +Deployment and teardown use the same concurrency group. They do not cancel a resource operation in progress. An obsolete PR commit cannot start deployment or replace the current PR comment. A later commit can replace a preview at the same URL; manual checks must compare the deployment ID from the saved manifest. + +## GitHub and Cloudflare setup + +Use a dedicated Cloudflare staging account or dedicated resources in an account that permits this CI workload. Select Workers Paid for the configured payload sizes. The `paid` operator profile controls cleanup batch size; it does not purchase a subscription. + +Create the GitHub environment `remote-cache-staging` and add these environment secrets: + +| Secret | Purpose | +| ----------------------- | ---------------------------------------------------------------------------- | +| `CLOUDFLARE_ACCOUNT_ID` | Account that owns the staging resources | +| `CLOUDFLARE_API_TOKEN` | Account-scoped Workers Scripts, D1, and Workers R2 Storage write permissions | + +Set these **repository variables**, which the notification job also needs: + +| Variable | Value | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `REMOTE_CACHE_WORKERS_SUBDOMAIN` | Account subdomain only, for example `example` for `example.workers.dev` | +| `REMOTE_CACHE_RESOURCE_PREFIX` | Optional; defaults to `vp-cache-ci`. A custom prefix must end in `-ci` and use at most 34 lowercase letters, digits, or hyphens | +| `REMOTE_CACHE_DEPLOY_ENABLED` | Set to `true` after the environment and account are ready | + +Enable R2 in the account. The token must allow resource creation and deletion as well as data access. No custom domain or DNS permission is needed. Do not attach production bindings or secrets to these Workers. The GitHub environment can require a maintainer review if the repository needs one before a deployment. + +The workflow exposes Cloudflare credentials only to deployment, verification, and cleanup commands. Dependency installation runs before those credentials enter the step environment. Internal PR contributors must be trusted to change deployment code. Fork and Dependabot PRs run local checks without Cloudflare credentials. A separate `pull_request_target` workflow only posts their notice or executes cleanup code from the default branch; it never checks out PR code. To preview a fork change, a maintainer must copy the reviewed commit to a branch in this repository and open a PR. + +The preview-events workflow must exist on the default branch before fork notices and automatic closed-PR cleanup can run. Merge the workflow setup before relying on those events. Manual deployment dispatches use the default branch and cannot select an arbitrary PR revision. + +## Authorization and test levels + +The production write policy requires a real GitHub token with `event_name=push`, the registered repository and owner IDs, its configured main branch, and the exact namespace audience. A PR token or a `workflow_dispatch` token cannot pass this policy. GitHub signs the event claim; the test runner cannot change it. + +| Level | Trigger | What it proves | +| ------------------------------ | ------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Local regression | Every normal CI run, on Linux/macOS/Windows | Protocol, storage state transitions, failures, and the e2e driver against workerd/D1/R2 emulators | +| PR Cloudflare verification | Related changes in an internal PR | Deployment revision, migrations, private R2, real public HTTP reads, real rejected OIDC writes, policy changes, and expiry | +| Main Cloudflare verification | Related push to `main` | All PR cases plus real authorized HTTP writes, multipart uploads, concurrent maximum payloads, and the real Cron schedule | +| Manual Cloudflare verification | Default-branch workflow dispatch | Public reads and rejected writes; `full=true` also waits for real Cron cleanup | +| Release / incident exercises | Maintainer-controlled staging session | Long-running limits, provider outages, lifecycle delays, restore, rollback, and real hostile workflow identities | + +The runner obtains tokens directly from GitHub with `id-token: write`. It checks the token request host, rejects redirects, bounds responses, and caches each audience separately for three minutes. Tokens remain in memory. Reports and artifacts contain no GitHub or Cloudflare credentials. The Worker has no alternate issuer, test signing key, administrative HTTP endpoint, or weakened write policy. + +The operator seeds immutable objects and generation rows for read tests. This validates reads independently of upload authorization. Seeded fixtures do **not** count as successful `/store` coverage. Only main-branch push runs report authorized HTTP write coverage. + +## Automated case matrix + +“Local” refers to the existing regression suite and the new shared e2e-driver tests. “PR” and “Main” refer to real Cloudflare runs. + +| Case | Local | PR | Main | Required result | +| ------------------------------------------------ | ----------------------------- | ----------------------------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| Exact source revision | Yes | Yes | Yes | Expected deployment ID on responses; fail if the URL serves another revision | +| Migrations and private R2 | Yes | Yes | Yes | Setup succeeds, seeded data works, `r2.dev` is disabled, and custom R2 domains are absent | +| Anonymous exact lookup | Yes | Yes | Yes | `200` CBOR with exact opaque value and matching blob ID | +| Anonymous fallback | Yes | Yes | Yes | `200` CBOR with the associated stored key and matching value | +| Read-only accounting | Yes | Yes | Yes | Reads do not change charged bytes or entry/association counts | +| Missing entry/blob | Yes | Yes | Yes | Plain-text `404`, without redirects | +| Namespace isolation | Yes | Yes | Yes | Another namespace cannot resolve the key, association, or blob ID | +| Malformed / oversized fetch | Yes | Yes | Yes | `400` / `413`; no truncation | +| Missing or forged token | Yes | Yes | Yes | `401` and no new generation | +| Wrong audience | Yes | Yes | Yes | A real signed token for another namespace gets `403` | +| PR / dispatch token | Yes | Yes | Applicable on manual runs | `403` and no new generation | +| Immediate scope withdrawal | Yes | Yes | Yes | Existing metadata/blob reads get `404` without redeployment; restore re-enables reads | +| Missing live value / blob object | Yes | Yes | Yes | Value fetch gets `503`; blob download gets `404`; restore works | +| Real authorized store | Emulated GitHub keys | No | Yes | GitHub-signed main-branch push token permits `/store` | +| Opaque / empty fields | Yes | Read fixtures | Yes | Preserve binary values; an omitted blob gets `null`, an empty blob gets an ID | +| Unknown request length | Yes | No | Yes | A streamed body without `Content-Length` succeeds within limits | +| Association reassignment | Yes | No | Yes | Reassign the secondary key without deleting its previous target | +| Entry replacement / grace | Yes | No | Yes | New mappings select the new generation; the old blob still has its original bytes during grace | +| R2 multipart upload | Yes | No | Yes | Upload more than 5 MiB; download and compare the SHA-256 digest | +| Concurrent same-key stores | Yes | No | Yes | The selected value and blob belong to one complete generation | +| Malformed store / quota | Yes | No | Yes | `400` / `503`; preserve the previous value and mappings | +| Maximum payload concurrency | Yes | No | Yes | Two simultaneous 64 MiB blobs with 4 MiB values succeed and preserve digests | +| Expired generation | Yes | Yes | Yes | Metadata and blob become unavailable immediately | +| Real Cron deletion | Direct scheduled-handler call | No | Yes | Scheduled cleanup removes the D1 generation and both R2 objects, and preserves accounting | +| JWT claim classes / clock bounds / JWKS failures | Yes | Real wrong audience/event | Real valid identity | Full forged/fork/owner/visibility/ref/event/time matrix remains in local tests; controlled live identity exercises supplement it | +| R2 failure injection / lease and policy races | Yes | Selected missing-object cases | Selected quota cases | Local deterministic failures never publish partial state or release charges before deletion | +| Operator retries and teardown guards | Yes | Provisioning path | Provisioning path | Retry setup without duplicate resources; recover interrupted policy checks; refuse unrelated resources | + +The main run has 14 grouped e2e checks; the PR run has eight. A group can contain several requests and assertions. The local driver test runs the same main suite, including maximum payloads, with a directly invoked scheduled handler. That does not replace the Cloudflare Cron check. Both modes also check that the advertised manual endpoint serves its expected fixture. + +## Pass criteria and failure evidence + +Every executed assertion must pass. The e2e command exits nonzero on failure. `report.json` records each completed group, its status, duration, deployment ID, endpoint, and authorization mode. The workflow uploads partial reports after failures too. Reports use generic error classes; credentials and opaque request bodies are excluded. + +Deployment readiness permits two minutes for the expected revision and fixture to appear. Normal HTTP requests have a 30-second deadline; stores have 130 seconds. Behavior tests do not retry failed writes or turn errors into passes. Only readiness and asynchronous cleanup use bounded polling. + +The main run waits at most 25 minutes for its expired fixture to disappear through the real five-minute Cron schedule. Cloudflare notes that Cron changes can take up to 15 minutes to propagate. The test also checks that both R2 objects disappear, the selected entry is gone, and charged bytes match generation totals. It never calls a fake public scheduled endpoint or performs the sentinel deletion itself. + +Use the artifact and workflow logs to identify the first failed group. For server diagnosis, correlate `X-Request-Id` with sampled Worker logs. Do not retry a failed assertion solely to obtain a green result. A new run should follow a fix or an identified transient deployment issue. + +## Manual verification from a PR + +1. Wait for the PR comment to say that Cloudflare deployment and PR e2e checks passed. Confirm that its commit is the current PR head. +2. Open the linked workflow run. Download its `remote-cache-e2e--` artifact and extract the files. +3. Read `manual-manifest.json`. It contains the expected deployment ID, endpoint, blob URL, and expected public fixture contents. +4. From the extracted directory, set `CACHE_ENDPOINT` to the manifest's endpoint and run: + +```sh +curl --fail-with-body --silent --show-error \ + --dump-header fetch-headers.txt \ + --header 'Content-Type: application/cbor' \ + --data-binary @manual-fetch.cbor \ + "$CACHE_ENDPOINT/fetch" --output fetch-response.cbor +``` + +5. Check `X-Remote-Cache-Deployment` against the manifest. Check `Content-Type: application/cbor` and `Cache-Control: no-store`. Decode the response with a CBOR tool; expect `kind: "exact"` and the manifest's value. +6. Use `curl --fail-with-body --dump-header blob-headers.txt --output blob.txt` with the manifest's `blob_url`. Compare the downloaded text with `blob_utf8`. +7. Run `curl --silent --show-error --include --request POST "$CACHE_ENDPOINT/store"` without credentials. Expect plain-text `401`, not a login page or redirect. + +For a CBOR decoder already available in this repository, run this from `packages/remote-cache` after dependency installation, replacing the file path: + +```sh +node --input-type=module -e 'import { readFileSync } from "node:fs"; import { decode } from "cborg"; console.log(decode(readFileSync(process.argv[1])))' /path/to/fetch-response.cbor +``` + +PR previews intentionally reject publication tokens from PR workflows. Use the main staging workflow to validate successful publication. The manual namespace contains synthetic public data and expires after one day unless another run refreshes it. + +## Cleanup, retries, and cost limits + +Each CI deployment has a 2 GB byte budget, 1,000 entries, and 2,000 associations. Pending and retired objects remain charged. After verification, the runner expires `e2e` and `other` data and gives in-flight operations the normal ten-minute cleanup grace. It retains the `manual` fixture for developer checks. Subsequent runs explicitly restore CI-owned policy switches in case a prior process stopped during a withdrawal test. + +Closing a PR triggers `.github/workflows/remote-cache-preview-events.yml`. The cleanup job executes code from the default branch, confirms that the PR is closed, and checks resource ownership. It disables access, waits for Cron to drain generations, then deletes the empty bucket, D1 database, and Worker. It does not delete main staging. The PR comment reports cleanup success or failure. + +If Cron or orphan multipart uploads delay cleanup, the job fails instead of claiming that resources are gone. R2 lifecycle can require a day to abort an unrecorded upload. Rerun the cleanup workflow with the closed PR number after the backlog drains. Failed resource creation and partial teardown can also be retried. A nonempty bucket always blocks final removal. + +Staging policies belong to CI. Do not use these names for an operator-managed production cache. Monitor resource counts and Cloudflare allowances. Resource budgets do not guarantee zero cost. Disable `REMOTE_CACHE_DEPLOY_ENABLED` to stop automatic deployments; remove existing PR resources explicitly after pending runs finish. + +## Release and incident exercises + +Before a production release, record the commit, workflow URL, Cloudflare plan, region, and outcome for these controlled staging exercises: + +| Exercise | Procedure and acceptance criterion | +| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Provider failure and recovery | Restrict D1/R2 access in staging, exercise reads and stores, then restore access. Existing mappings survive; failures remain explicit; retries recover | +| Upload interruption | Disconnect a real authorized multipart request after its first R2 part. Verify unchanged mappings, retained reservation, upload abortion, and eventual byte release | +| Lease/token expiry and policy race | Delay a staging upload across expiry or change policy before publication. Neither mapping changes; old data remains readable | +| Real hostile workflow identities | Use controlled fork, tag, PR, `pull_request_target`, and `workflow_run` jobs. All writes fail; public reads still work | +| Rate limits | Send a bounded burst to the staging namespace. Check `429`, `Retry-After`, recovery, and a finite identity set for unknown routes | +| Retention and lifecycle | Observe expiry, replacement grace, unfinished multipart abortion, and lifecycle deletion over actual retention windows | +| Migration and rollback | Upgrade a populated staging database, verify old entries, and restore a compatible previous Worker version. Never roll code back across an incompatible schema change | +| Partial restore | Restore D1 without deleted R2 data. Expect `503` for live missing values and `404` for missing blobs; withdraw or replace the namespace before reuse | +| Load and CPU | Measure deployed CPU, memory, D1 rows/latency, R2 operations, and cleanup lag under sustained concurrency. Compare observed costs with configured budgets | + +These are explicit release exercises, not claims that fault injection or multi-day lifecycle behavior already ran in the PR workflow. Workers Free support still needs provider CPU measurements within its limits. + +References: [Cloudflare GitHub Actions deployment](https://developers.cloudflare.com/workers/ci-cd/external-cicd/github-actions/), [Cron propagation](https://developers.cloudflare.com/workers/configuration/cron-triggers/), [GitHub OIDC claims](https://docs.github.com/en/actions/reference/security/oidc), and [secure workflow use](https://docs.github.com/en/actions/reference/security/secure-use). diff --git a/packages/remote-cache/package.json b/packages/remote-cache/package.json index e22f1eab0..ee1461991 100644 --- a/packages/remote-cache/package.json +++ b/packages/remote-cache/package.json @@ -12,7 +12,10 @@ "test": "node --import tsx --test test/index.test.ts", "smoke": "pnpm test && pnpm build", "operator": "node --import tsx scripts/operator.ts", - "benchmark": "node --import tsx scripts/benchmark.ts" + "benchmark": "node --import tsx scripts/benchmark.ts", + "ci:deploy": "node --import tsx scripts/ci.ts deploy", + "ci:cleanup": "node --import tsx scripts/ci.ts cleanup", + "e2e": "node --import tsx scripts/ci.ts test" }, "dependencies": { "jose": "6.2.12" diff --git a/packages/remote-cache/scripts/ci.ts b/packages/remote-cache/scripts/ci.ts new file mode 100644 index 000000000..f840ea59f --- /dev/null +++ b/packages/remote-cache/scripts/ci.ts @@ -0,0 +1,398 @@ +import assert from 'node:assert/strict'; +import { appendFile, mkdir, writeFile } from 'node:fs/promises'; +import { URL, pathToFileURL } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; +import { encode } from 'cborg'; +import { + ApiError, + operatorIO, + query, + readTemplate, + runOperator, + type Config, + type OperatorIO, +} from './operator.ts'; +import { retireTestData, seedManual, type Admin } from './e2e/fixtures.ts'; +import { runSuite, type Report } from './e2e/suite.ts'; + +const resultsDir = new URL('../e2e-results/', import.meta.url); + +export function settingsFrom(env: Record) { + const prefix = env['REMOTE_CACHE_RESOURCE_PREFIX'] || 'vp-cache-ci'; + if (!/^[a-z0-9][a-z0-9-]{0,30}-ci$/.test(prefix)) + throw new Error( + 'The resource prefix must end in -ci and contain at most 34 lowercase letters, digits, or hyphens', + ); + const subdomain = env['REMOTE_CACHE_WORKERS_SUBDOMAIN']; + if (!subdomain || !/^[a-z0-9][a-z0-9-]{0,62}$/.test(subdomain)) + throw new Error( + 'Set REMOTE_CACHE_WORKERS_SUBDOMAIN to the account subdomain, without workers.dev', + ); + const pr = env['REMOTE_CACHE_PR_NUMBER'] || undefined; + if (pr && !/^[1-9][0-9]{0,9}$/.test(pr)) throw new Error('Invalid PR number'); + const name = `${prefix}-${pr ? `pr-${pr}` : 'main'}`; + const repository = env['GITHUB_REPOSITORY']; + if (!repository || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) + throw new Error('Invalid repository'); + const repositoryId = env['GITHUB_REPOSITORY_ID']; + if (!repositoryId || !/^[1-9][0-9]*$/.test(repositoryId)) + throw new Error('Invalid repository ID'); + const revision = env['REMOTE_CACHE_SOURCE_SHA']; + if (!revision || !/^[a-f0-9]{40}$/.test(revision)) + throw new Error('Use the full source commit SHA'); + const run = env['GITHUB_RUN_ID'], + attempt = env['GITHUB_RUN_ATTEMPT']; + if (!run || !attempt || !/^\d+$/.test(run) || !/^\d+$/.test(attempt)) + throw new Error('Invalid workflow run identity'); + const defaultBranch = env['REMOTE_CACHE_DEFAULT_BRANCH']; + const writes = + !pr && + env['GITHUB_EVENT_NAME'] === 'push' && + env['GITHUB_REF'] === `refs/heads/${defaultBranch}`; + if (!pr && env['GITHUB_REF'] !== `refs/heads/${defaultBranch}`) + throw new Error('The main staging deployment requires the default branch'); + return { + name, + pr, + repository, + repositoryId, + revision, + deployment: `${revision}-${run}-${attempt}`, + origin: `https://${name}.${subdomain}.workers.dev`, + writes, + full: writes || env['REMOTE_CACHE_E2E_FULL'] === 'true', + }; +} +type Settings = ReturnType; + +function checkConfig(config: Config, settings: Settings) { + if ( + config.name !== settings.name || + config.r2_buckets[0]?.bucket_name !== settings.name || + config.d1_databases[0]?.database_name !== settings.name + ) + throw new Error('CI can only operate on its dedicated staging resources'); +} + +async function checkAccountOrigin(settings: Settings) { + const account = (await operatorIO.api('/workers/subdomain')) as { subdomain: string }; + assert.equal( + new URL(settings.origin).hostname, + `${settings.name}.${account.subdomain}.workers.dev`, + 'The configured Workers subdomain must belong to the authenticated Cloudflare account', + ); +} + +export function cloudflareAdmin(config: Config): Admin { + const account = process.env['CLOUDFLARE_ACCOUNT_ID'], + token = process.env['CLOUDFLARE_API_TOKEN']; + if (!account || !/^[a-f0-9]{32}$/.test(account) || !token) + throw new Error('Cloudflare staging credentials are required'); + const bucket = config.r2_buckets[0]!.bucket_name; + // This is the authenticated object API used by the pinned Wrangler CLI. + // Binary responses cannot pass through the operator's JSON API adapter. + async function object(key: string, method: string, value?: Uint8Array) { + if (!/^(e2e|other|manual)\/[a-f0-9-]{36}\/(value|blob)$/.test(key)) + throw new Error('Invalid fixture object key'); + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${account}/r2/buckets/${bucket}/objects/${key}`, + { + method, + headers: { Authorization: `Bearer ${token}` }, + ...(value === undefined ? {} : { body: value }), + redirect: 'error', + signal: AbortSignal.timeout(60000), + }, + ); + await response.body?.cancel(); + if (!response.ok && response.status !== 404) throw new ApiError(response.status); + return response.status !== 404; + } + return { + sql: (sql, params = []) => query(operatorIO, config, sql, params), + async put(key, value) { + if (!(await object(key, 'PUT', value))) throw new Error('Fixture bucket is missing'); + }, + async delete(key) { + await object(key, 'DELETE'); + }, + exists: (key) => object(key, 'GET'), + }; +} + +export function githubTokens( + env: Record, + request: typeof fetch = fetch, +) { + const cached = new Map(); + return async (audience: string) => { + const previous = cached.get(audience); + if (previous && previous.until > Date.now()) return previous.token; + const raw = env['ACTIONS_ID_TOKEN_REQUEST_URL'], + credential = env['ACTIONS_ID_TOKEN_REQUEST_TOKEN']; + if (!raw || !credential) throw new Error('The e2e job requires id-token: write'); + const url = new URL(raw); + if ( + url.protocol !== 'https:' || + !url.hostname.endsWith('.actions.githubusercontent.com') || + url.username || + url.password || + url.port + ) + throw new Error('Unexpected GitHub OIDC request URL'); + url.searchParams.set('audience', audience); + const response = await request(url, { + headers: { Authorization: `Bearer ${credential}` }, + redirect: 'error', + signal: AbortSignal.timeout(15000), + }); + if (!response.ok) { + await response.body?.cancel(); + throw new Error('GitHub OIDC request failed'); + } + const reader = response.body!.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const part = await reader.read(); + if (part.done) break; + size += part.value.length; + if (size > 32768) throw new Error('OIDC response exceeds its limit'); + chunks.push(part.value); + } + } finally { + await reader.cancel(); + } + const data = JSON.parse(Buffer.concat(chunks).toString('utf8')) as { value?: unknown }; + if (typeof data.value !== 'string' || data.value.length > 16384) + throw new Error('Invalid GitHub OIDC response'); + cached.set(audience, { token: data.value, until: Date.now() + 180000 }); + return data.value; + }; +} + +async function deploy(settings: Settings) { + await checkAccountOrigin(settings); + await runOperator( + [ + 'setup', + '--name', + settings.name, + '--namespace', + 'e2e', + '--repo', + settings.repository, + '--origin', + settings.origin, + '--profile', + 'paid', + '--retention-days', + '1', + '--byte-limit', + '2000000000', + '--entry-limit', + '1000', + '--association-limit', + '2000', + ], + { + ...operatorIO, + async wrangler(args) { + // Install all CI namespace policies before the single final deployment. + if (args[0] !== 'deploy') await operatorIO.wrangler(args); + }, + async writeConfig(config) { + config.vars['DEPLOYMENT_ID'] = settings.deployment; + await operatorIO.writeConfig(config); + }, + }, + ); + const config = await operatorIO.readConfig(); + checkConfig(config, settings); + const existing = await query( + operatorIO, + config, + 'SELECT scope_id, repository_id, endpoint FROM scopes', + ); + if ( + existing.some( + (scope) => + !['e2e', 'other', 'manual'].includes(String(scope['scope_id'])) || + scope['repository_id'] !== settings.repositoryId || + scope['endpoint'] !== `${settings.origin}/projects/${String(scope['scope_id'])}`, + ) + ) + throw new Error('CI resources must contain only this repository’s verification namespaces'); + for (const scope of ['other', 'manual']) + await query( + operatorIO, + config, + `INSERT INTO scopes + (scope_id, endpoint, repository, repository_id, repository_owner_id, branch, retention_seconds) + SELECT ?, ?, repository, repository_id, repository_owner_id, branch, 86400 FROM scopes WHERE scope_id = 'e2e' + ON CONFLICT(scope_id) DO NOTHING`, + [scope, `${settings.origin}/projects/${scope}`], + ); + // Recover policy changes left by an interrupted e2e run. These resources belong to CI only. + await query(operatorIO, config, 'UPDATE deployment SET enabled = 1, writes_enabled = 1'); + await query( + operatorIO, + config, + `UPDATE scopes SET enabled = 1, writes_enabled = 1, + byte_limit = 2000000000, entry_limit = 1000, association_limit = 2000`, + ); + config.vars['NAMESPACES'] = '["e2e","other","manual"]'; + await operatorIO.writeConfig(config); + await operatorIO.wrangler(['deploy', '--config', 'wrangler.operator.json']); + const managed = (await operatorIO.api(`/r2/buckets/${settings.name}/domains/managed`)) as { + enabled: boolean; + }; + const custom = (await operatorIO.api(`/r2/buckets/${settings.name}/domains/custom`)) as { + domains: unknown[]; + }; + assert.equal(managed.enabled, false, 'R2 must remain private'); + assert.deepEqual(custom.domains, [], 'R2 must have no public custom domain'); + const fixture = await seedManual(cloudflareAdmin(config), settings.deployment); + await mkdir(resultsDir, { recursive: true }); + await writeFile( + new URL('manual-fetch.cbor', resultsDir), + encode({ key: fixture.key, secondary_key: fixture.secondary }), + ); + await writeFile( + new URL('manual-manifest.json', resultsDir), + JSON.stringify( + { + deployment: settings.deployment, + source_sha: settings.revision, + endpoint: `${settings.origin}/projects/manual`, + blob_url: `${settings.origin}/projects/manual/blob/${fixture.blobId}`, + value_utf8: Buffer.from(fixture.value).toString(), + blob_utf8: Buffer.from(fixture.blob!).toString(), + }, + null, + 2, + ) + '\n', + ); + if (process.env['GITHUB_OUTPUT']) + await appendFile( + process.env['GITHUB_OUTPUT'], + `endpoint=${settings.origin}/projects/manual\ndeployment=${settings.deployment}\n`, + ); +} + +async function verify(settings: Settings) { + await checkAccountOrigin(settings); + const config = await operatorIO.readConfig(); + checkConfig(config, settings); + const admin = cloudflareAdmin(config); + let report: Report | undefined; + await mkdir(resultsDir, { recursive: true }); + try { + await runSuite({ + origin: settings.origin, + deployment: settings.deployment, + admin, + request: fetch, + token: githubTokens(process.env), + writes: settings.writes, + full: settings.full, + cron: settings.full, + async record(value) { + report = value; + await writeFile(new URL('report.json', resultsDir), JSON.stringify(report, null, 2) + '\n'); + }, + }); + } finally { + await retireTestData(admin); + if (process.env['GITHUB_STEP_SUMMARY'] && report) + await appendFile( + process.env['GITHUB_STEP_SUMMARY'], + `### Remote cache Cloudflare verification\n\nDeployment: \`${settings.deployment}\`\n\n` + + `Mode: ${report.mode}. Manual endpoint: ${settings.origin}/projects/manual\n\n` + + report.results.map((result) => `- ${result.status}: ${result.name}`).join('\n') + + '\n', + ); + } +} + +export async function cleanup( + settings: Settings, + io: OperatorIO = operatorIO, + sleep: (ms: number) => Promise = delay, +) { + if (!settings.pr) throw new Error('Automatic teardown is restricted to PR resources'); + const config = await readTemplate(); + const databases = (await io.api(`/d1/database?name=${settings.name}&per_page=100`)) as { + name: string; + uuid: string; + }[]; + const db = databases.find((db) => db.name === settings.name); + config.name = settings.name; + config.r2_buckets[0]!.bucket_name = settings.name; + config.d1_databases[0]!.database_name = settings.name; + if (db) config.d1_databases[0]!.database_id = db.uuid; + await io.writeConfig(config); + if (db) { + const tables = await query( + io, + config, + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'generations'", + ); + if (tables.length) { + const scopes = await query(io, config, 'SELECT repository_id, endpoint FROM scopes'); + if ( + scopes.some( + (scope) => + scope['repository_id'] !== settings.repositoryId || + !String(scope['endpoint']).startsWith(`${settings.origin}/projects/`), + ) + ) + throw new Error('Refusing to remove resources owned by another deployment'); + await query(io, config, 'UPDATE deployment SET enabled = 0, writes_enabled = 0'); + await query( + io, + config, + `UPDATE generations SET expires_at = 0, + lease_until = min(lease_until, unixepoch()), gc_after = min(gc_after, unixepoch() + 600)`, + ); + const until = Date.now() + 25 * 60000; + while ( + Number((await query(io, config, 'SELECT count(*) AS count FROM generations'))[0]!['count']) + ) { + if (Date.now() >= until) + throw new Error( + 'Cleanup is still pending; rerun the cleanup workflow after Cron or lifecycle finishes', + ); + await sleep(30000); + } + } + } + // Empty-bucket deletion also protects multipart uploads without a recorded ID. + try { + await io.api(`/r2/buckets/${settings.name}`); + await io.wrangler(['r2', 'bucket', 'delete', settings.name]); + } catch (error) { + if (!(error instanceof ApiError) || error.status !== 404) throw error; + } + if (db) await io.wrangler(['d1', 'delete', db.uuid, '--skip-confirmation']); + try { + await io.api(`/workers/scripts/${settings.name}`, 'DELETE'); + } catch (error) { + if (!(error instanceof ApiError) || error.status !== 404) throw error; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const settings = settingsFrom(process.env); + const command = process.argv[2]; + if (command === 'deploy') await deploy(settings); + else if (command === 'test') await verify(settings); + else if (command === 'cleanup') await cleanup(settings); + else throw new Error('Use deploy, test, or cleanup'); + } catch (error) { + console.error(error instanceof Error ? error.message : 'Cloudflare CI failed'); + process.exitCode = 1; + } +} diff --git a/packages/remote-cache/scripts/e2e/fixtures.ts b/packages/remote-cache/scripts/e2e/fixtures.ts new file mode 100644 index 000000000..404fb89ca --- /dev/null +++ b/packages/remote-cache/scripts/e2e/fixtures.ts @@ -0,0 +1,76 @@ +import { randomUUID } from 'node:crypto'; + +export type Row = Record; +export interface Admin { + sql(sql: string, params?: (string | number | null)[]): Promise; + put(key: string, value: Uint8Array): Promise; + delete(key: string): Promise; + exists(key: string): Promise; +} + +export const bytes = (value: string) => new TextEncoder().encode(value); +// The REST fixture loader uses generated hexadecimal SQL literals for binary fields. +// This conversion accepts bytes only, never SQL or HTTP input. +export const sqlBytes = (value: Uint8Array) => `X'${Buffer.from(value).toString('hex')}'`; + +export interface Fixture { + scope: string; + generation: string; + key: Uint8Array; + secondary: Uint8Array; + value: Uint8Array; + blob: Uint8Array | undefined; + blobId: string | null; + valueObject: string; + blobObject: string; +} + +export async function seed( + admin: Admin, + scope: string, + label: string, + value: Uint8Array, + blob?: Uint8Array, +): Promise { + const generation = randomUUID(); + const key = bytes(label), + secondary = bytes(`${label}-secondary`); + const valueObject = `${scope}/${generation}/value`, + blobObject = `${scope}/${generation}/blob`; + const blobId = blob === undefined ? null : randomUUID(); + await admin.sql( + `INSERT INTO generations + (generation_id, scope_id, state, policy_version, token_exp, lease_until, gc_after, + value_object, blob_object, charged_bytes) + SELECT ?, scope_id, 'uploading', policy_version, unixepoch() + 900, unixepoch() + 900, + unixepoch() + 1500, ?, ?, ? FROM scopes WHERE scope_id = ?`, + [generation, valueObject, blobObject, value.length + (blob?.length ?? 0), scope], + ); + await admin.put(valueObject, value); + if (blob !== undefined) await admin.put(blobObject, blob); + // Use the production publication trigger; no test endpoint or signing-key bypass exists. + await admin.sql( + `UPDATE generations SET state = 'ready', key = ${sqlBytes(key)}, + secondary_key = ${sqlBytes(secondary)}, blob_id = ?, value_size = ?, blob_size = ?, + expires_at = unixepoch() + 86400, gc_after = unixepoch() + 86400 + WHERE generation_id = ?`, + [blobId, value.length, blob?.length ?? 0, generation], + ); + return { scope, generation, key, secondary, value, blob, blobId, valueObject, blobObject }; +} + +export async function retireTestData(admin: Admin): Promise { + await admin.sql(`UPDATE generations SET expires_at = 0, + lease_until = min(lease_until, unixepoch()), gc_after = min(gc_after, unixepoch() + 600) + WHERE scope_id IN ('e2e', 'other')`); +} + +export function seedManual(admin: Admin, deployment: string): Promise { + return seed( + admin, + 'manual', + 'manual-verification', + bytes(`deployment:${deployment}`), + bytes('Public remote cache verification blob\n'), + ); +} diff --git a/packages/remote-cache/scripts/e2e/suite.ts b/packages/remote-cache/scripts/e2e/suite.ts new file mode 100644 index 000000000..9a68ae6f7 --- /dev/null +++ b/packages/remote-cache/scripts/e2e/suite.ts @@ -0,0 +1,459 @@ +import assert from 'node:assert/strict'; +import { createHash, randomUUID } from 'node:crypto'; +import { setTimeout as delay } from 'node:timers/promises'; +import { encode, decode } from 'cborg'; +import { defaults, MiB } from '../../src/limits.ts'; +import { bytes, seed, sqlBytes, type Admin, type Fixture } from './fixtures.ts'; + +export interface Result { + name: string; + status: 'passed' | 'failed'; + duration_ms: number; + error?: string; +} +export interface Report { + deployment: string; + endpoint: string; + mode: 'push' | 'read-only'; + started_at: string; + results: Result[]; +} +export interface Options { + origin: string; + deployment: string; + admin: Admin; + request: (url: string, init?: RequestInit) => Promise; + token: (audience: string) => Promise; + writes: boolean; + full: boolean; + cron: boolean; + cronTimeoutMs?: number; + pollMs?: number; + record?: (report: Report) => Promise; +} + +const digest = (value: Uint8Array) => createHash('sha256').update(value).digest('hex'); +function blobId(result: Record): string { + assert.equal(typeof result['blob_id'], 'string'); + return result['blob_id'] as string; +} + +export async function runSuite(options: Options): Promise { + const { origin, admin, deployment } = options; + const report: Report = { + deployment, + endpoint: `${origin}/projects/e2e`, + mode: options.writes ? 'push' : 'read-only', + started_at: new Date().toISOString(), + results: [], + }; + const prefix = randomUUID(); + async function check(name: string, action: () => Promise) { + const start = Date.now(); + try { + await action(); + report.results.push({ name, status: 'passed', duration_ms: Date.now() - start }); + } catch (error) { + // Reports contain test names, never request headers, tokens, or server bodies. + report.results.push({ + name, + status: 'failed', + duration_ms: Date.now() - start, + error: + error instanceof assert.AssertionError + ? 'Assertion failed' + : 'Request or administration failed', + }); + await options.record?.(report); + throw new Error(`Cloudflare e2e failed: ${name}`, { cause: error }); + } + await options.record?.(report); + } + async function call(scope: string, path: string, status: number, init: RequestInit = {}) { + const response = await options.request(`${origin}/projects/${scope}/${path}`, { + ...init, + redirect: 'error', + signal: AbortSignal.timeout(path === 'store' ? 130000 : 30000), + }); + assert.equal(response.status, status, `${scope}/${path}: unexpected HTTP status`); + assert.equal( + response.headers.get('X-Remote-Cache-Deployment'), + deployment, + 'Deployment changed during verification', + ); + assert.equal(response.headers.get('Cache-Control'), 'no-store'); + assert.ok(response.headers.get('X-Request-Id')); + if (status === 200 && path.startsWith('blob/')) + assert.equal(response.headers.get('Content-Type'), 'application/octet-stream'); + if (status >= 400) + assert.equal(response.headers.get('Content-Type'), 'text/plain; charset=utf-8'); + return response; + } + async function body(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) return new Uint8Array(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const part = await reader.read(); + if (part.done) break; + size += part.value.length; + assert.ok(size <= defaults.blob + MiB, 'Response exceeds the test limit'); + chunks.push(part.value); + } + const combined = Buffer.concat(chunks); + return new Uint8Array(combined.buffer, combined.byteOffset, combined.byteLength); + } finally { + await reader.cancel(); + } + } + async function envelope(response: Response): Promise> { + assert.equal(response.headers.get('Content-Type'), 'application/cbor'); + const value: unknown = decode(await body(response)); + assert.ok(value && typeof value === 'object' && !Array.isArray(value)); + return value as Record; + } + async function lookup(key: Uint8Array, secondary: Uint8Array, status = 200, scope = 'e2e') { + return call(scope, 'fetch', status, { + method: 'POST', + headers: { 'Content-Type': 'application/cbor' }, + body: encode({ key, secondary_key: secondary }), + }); + } + async function store( + key: Uint8Array, + secondary: Uint8Array, + value: Uint8Array, + blob?: Uint8Array, + options: { blobFirst?: boolean; status?: number; token?: string; stream?: boolean } = {}, + ) { + const form = new FormData(); + const addBlob = () => { + if (blob !== undefined) + form.append('blob', new Blob([blob], { type: 'application/octet-stream' })); + }; + if (options.blobFirst) addBlob(); + form.append( + 'metadata', + new Blob([encode({ key, secondary_key: secondary, value })], { type: 'application/cbor' }), + ); + if (!options.blobFirst) addBlob(); + const authorization = `Bearer ${options.token ?? (await optionsToken())}`; + if (!options.stream) + return call('e2e', 'store', options.status ?? 200, { + method: 'POST', + headers: { Authorization: authorization }, + body: form, + }); + const request = new Request(`${origin}/projects/e2e/store`, { method: 'POST', body: form }); + return call('e2e', 'store', options.status ?? 200, { + method: 'POST', + headers: { ...Object.fromEntries(request.headers), Authorization: authorization }, + body: request.body, + duplex: 'half', + } as RequestInit); + } + const optionsToken = () => options.token(`${origin}/projects/e2e`); + let fixture: Fixture; + await check('deployed revision and D1/R2 readiness', async () => { + fixture = await seed( + admin, + 'e2e', + `${prefix}-fixture`, + bytes(`value:${deployment}`), + bytes(`blob:${deployment}`), + ); + const until = Date.now() + 120000; + while (true) { + const response = await options.request(`${origin}/projects/e2e/fetch`, { + method: 'POST', + headers: { 'Content-Type': 'application/cbor' }, + body: encode({ key: fixture.key, secondary_key: fixture.secondary }), + redirect: 'error', + signal: AbortSignal.timeout(15000), + }); + if ( + response.status === 200 && + response.headers.get('X-Remote-Cache-Deployment') === deployment + ) { + assert.deepEqual((await envelope(response)).value, fixture.value); + break; + } + await response.body?.cancel(); + assert.ok(Date.now() < until, 'Deployment did not become ready'); + await delay(options.pollMs ?? 3000); + } + }); + const f = fixture!; + await check('manual verification fixture is available at the advertised endpoint', async () => { + const manual = await envelope( + await lookup( + bytes('manual-verification'), + bytes('manual-verification-secondary'), + 200, + 'manual', + ), + ); + assert.equal(manual['kind'], 'exact'); + assert.deepEqual(manual['value'], bytes(`deployment:${deployment}`)); + assert.deepEqual( + await body(await call('manual', `blob/${blobId(manual)}`, 200)), + bytes('Public remote cache verification blob\n'), + ); + }); + await check('anonymous exact/fallback reads preserve data and counters', async () => { + // Real Cron can delete older runs during these reads. Compare invariants and + // this live fixture, rather than assuming global totals cannot decrease. + const accounting = `SELECT + charged_bytes - (SELECT coalesce(sum(charged_bytes), 0) FROM generations) AS bytes_delta, + entry_count - (SELECT count(*) FROM entries) AS entries_delta, + association_count - (SELECT count(*) FROM associations) AS associations_delta FROM deployment`; + const before = await admin.sql( + 'SELECT charged_bytes, expires_at, state FROM generations WHERE generation_id = ?', + [f.generation], + ); + assert.deepEqual(await admin.sql(accounting), [ + { bytes_delta: 0, entries_delta: 0, associations_delta: 0 }, + ]); + assert.deepEqual(await envelope(await lookup(f.key, f.secondary)), { + kind: 'exact', + value: f.value, + blob_id: f.blobId, + }); + assert.deepEqual(await envelope(await lookup(bytes('missing'), f.secondary)), { + kind: 'fallback', + key: f.key, + value: f.value, + blob_id: f.blobId, + }); + assert.deepEqual(await body(await call('e2e', `blob/${f.blobId}`, 200)), f.blob); + assert.deepEqual( + await admin.sql( + 'SELECT charged_bytes, expires_at, state FROM generations WHERE generation_id = ?', + [f.generation], + ), + before, + ); + assert.deepEqual(await admin.sql(accounting), [ + { bytes_delta: 0, entries_delta: 0, associations_delta: 0 }, + ]); + }); + await check('missing data, namespace isolation, and malformed reads', async () => { + await body(await lookup(bytes(`${prefix}-missing`), bytes('missing'), 404)); + await body(await lookup(f.key, f.secondary, 404, 'other')); + await body(await call('other', `blob/${f.blobId}`, 404)); + await body(await call('unknown', 'fetch', 404, { method: 'POST' })); + await body( + await call('e2e', 'fetch', 400, { + method: 'POST', + headers: { 'Content-Type': 'application/cbor' }, + body: Uint8Array.of(255), + }), + ); + await body(await lookup(new Uint8Array(defaults.key + 1), bytes('S'), 413)); + }); + await check('missing, forged, and wrong-audience credentials cannot write', async () => { + const before = new Set( + (await admin.sql('SELECT generation_id FROM generations')).map((row) => row['generation_id']), + ); + await body(await call('e2e', 'store', 401, { method: 'POST' })); + await body( + await call('e2e', 'store', 401, { + method: 'POST', + headers: { Authorization: 'Bearer forged' }, + }), + ); + await body( + await store(f.key, f.secondary, bytes('rejected'), undefined, { + token: await options.token(`${origin}/projects/other`), + status: 403, + }), + ); + if (!options.writes) + await body(await store(f.key, f.secondary, bytes('rejected'), undefined, { status: 403 })); + assert.ok( + (await admin.sql('SELECT generation_id FROM generations')).every((row) => + before.has(row['generation_id']), + ), + ); + }); + await check('policy withdrawal takes effect without redeployment', async () => { + try { + await admin.sql("UPDATE scopes SET enabled = 0 WHERE scope_id = 'e2e'"); + await body(await lookup(f.key, f.secondary, 404)); + await body(await call('e2e', `blob/${f.blobId}`, 404)); + } finally { + await admin.sql("UPDATE scopes SET enabled = 1 WHERE scope_id = 'e2e'"); + } + assert.deepEqual((await envelope(await lookup(f.key, f.secondary))).value, f.value); + }); + await check('missing live R2 objects have the correct errors', async () => { + try { + await admin.delete(f.valueObject); + await body(await lookup(f.key, f.secondary, 503)); + } finally { + await admin.put(f.valueObject, f.value); + } + try { + await admin.delete(f.blobObject); + await body(await call('e2e', `blob/${f.blobId}`, 404)); + } finally { + await admin.put(f.blobObject, f.blob!); + } + }); + + if (options.writes) { + await check('real GitHub OIDC permits opaque and empty HTTP stores', async () => { + const key = new Uint8Array(), + secondary = Uint8Array.of(0, 255); + const value = Uint8Array.of(0, 255, 159, 255); + assert.deepEqual(await envelope(await store(key, secondary, value)), { blob_id: null }); + const empty = await envelope( + await store(key, secondary, value, new Uint8Array(), { blobFirst: true, stream: true }), + ); + assert.equal(typeof empty.blob_id, 'string'); + assert.equal((await body(await call('e2e', `blob/${blobId(empty)}`, 200))).length, 0); + assert.deepEqual((await envelope(await lookup(key, secondary))).value, value); + }); + await check('secondary reassignment and replacement keep both mappings coherent', async () => { + const a = bytes(`${prefix}-A`), + b = bytes(`${prefix}-B`), + s = bytes(`${prefix}-S`), + t = bytes(`${prefix}-T`); + const old = await envelope(await store(a, s, bytes('old'), bytes('old-blob'))); + await body(await store(b, s, bytes('B'))); + assert.deepEqual((await envelope(await lookup(a, s))).value, bytes('old')); + assert.deepEqual((await envelope(await lookup(bytes('missing'), s))).key, b); + await body(await store(a, t, bytes('replacement'))); + assert.deepEqual( + (await envelope(await lookup(bytes('missing'), t))).value, + bytes('replacement'), + ); + assert.deepEqual( + await body(await call('e2e', `blob/${blobId(old)}`, 200)), + bytes('old-blob'), + ); + }); + await check('R2 multipart upload and download preserve bytes', async () => { + const value = new Uint8Array(250000).fill(149), + blob = new Uint8Array(5 * MiB + 17).fill(61); + const key = bytes(`${prefix}-large`); + const stored = await envelope(await store(key, key, value, blob, { blobFirst: true })); + assert.deepEqual((await envelope(await lookup(key, key))).value, value); + assert.equal( + digest(await body(await call('e2e', `blob/${blobId(stored)}`, 200))), + digest(blob), + ); + }); + await check('concurrent HTTP stores select one complete generation', async () => { + const key = bytes(`${prefix}-concurrent`); + await Promise.all( + [0, 1].map(async (i) => body(await store(key, key, bytes(String(i)), bytes(String(i))))), + ); + const selected = await envelope(await lookup(key, key)); + assert.deepEqual( + await body(await call('e2e', `blob/${blobId(selected)}`, 200)), + selected.value, + ); + }); + await check('malformed uploads and exhausted quotas preserve published data', async () => { + const token = await optionsToken(); + await body( + await call('e2e', 'store', 400, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'multipart/form-data; boundary=x', + }, + body: '--x\r\nContent-Disposition: form-data; name="metadata"\r\nContent-Type: application/cbor\r\n\r\ntruncated', + }), + ); + const limit = (await admin.sql("SELECT byte_limit FROM scopes WHERE scope_id = 'e2e'"))[0]![ + 'byte_limit' + ]; + assert.equal(typeof limit, 'number'); + try { + await admin.sql( + "UPDATE scopes SET byte_limit = max(1, charged_bytes) WHERE scope_id = 'e2e'", + ); + await body(await store(f.key, f.secondary, bytes('rejected'), undefined, { status: 503 })); + } finally { + await admin.sql("UPDATE scopes SET byte_limit = ? WHERE scope_id = 'e2e'", [Number(limit)]); + } + assert.deepEqual((await envelope(await lookup(f.key, f.secondary))).value, f.value); + }); + if (options.full) + await check('maximum values and concurrent 64 MiB HTTP uploads', async () => { + await Promise.all( + [0, 1].map(async (i) => { + const key = bytes(`${prefix}-maximum-${i}`), + value = new Uint8Array(defaults.value).fill(17 + i); + const blob = new Uint8Array(defaults.blob).fill(29 + i); + const stored = await envelope( + await store(key, key, value, blob, { blobFirst: i === 0 }), + ); + assert.equal( + digest((await envelope(await lookup(key, key))).value as Uint8Array), + digest(value), + ); + assert.equal( + digest(await body(await call('e2e', `blob/${blobId(stored)}`, 200))), + digest(blob), + ); + }), + ); + }); + } + + await check( + options.cron + ? 'real Cron deletes expired objects and releases accounting' + : 'expired generations are immediately unavailable', + async () => { + const expired = await seed( + admin, + 'e2e', + `${prefix}-expired`, + bytes('expired'), + bytes('expired-blob'), + ); + await admin.sql( + 'UPDATE generations SET expires_at = 0, gc_after = 0 WHERE generation_id = ?', + [expired.generation], + ); + await body(await lookup(expired.key, expired.secondary, 404)); + await body(await call('e2e', `blob/${expired.blobId}`, 404)); + if (options.cron) { + const until = Date.now() + (options.cronTimeoutMs ?? 25 * 60000); + while ( + ( + await admin.sql('SELECT generation_id FROM generations WHERE generation_id = ?', [ + expired.generation, + ]) + ).length + ) { + assert.ok(Date.now() < until, 'Cron did not remove the generation'); + await delay(options.pollMs ?? 30000); + } + assert.equal(await admin.exists(expired.valueObject), false); + assert.equal(await admin.exists(expired.blobObject), false); + assert.equal( + ( + await admin.sql( + `SELECT count(*) AS count FROM entries WHERE scope_id = 'e2e' AND key = ${sqlBytes(expired.key)}`, + ) + )[0]!['count'], + 0, + ); + const accounting = ( + await admin.sql( + 'SELECT charged_bytes, (SELECT coalesce(sum(charged_bytes), 0) FROM generations) AS expected FROM deployment', + ) + )[0]!; + assert.equal(accounting['charged_bytes'], accounting['expected']); + } + }, + ); + return report; +} diff --git a/packages/remote-cache/scripts/operator.ts b/packages/remote-cache/scripts/operator.ts index 536f020b3..a19304fde 100644 --- a/packages/remote-cache/scripts/operator.ts +++ b/packages/remote-cache/scripts/operator.ts @@ -29,7 +29,7 @@ export interface OperatorIO { lifecycle(bucket: string, rules: unknown): Promise; print(message: string): void; } -type Config = { +export type Config = { name: string; main: string; compatibility_date: string; @@ -91,7 +91,7 @@ export async function resolveRepository(io: Pick, name: st }; } -async function query( +export async function query( io: OperatorIO, config: Config, sql: string, @@ -503,7 +503,7 @@ async function jsonResponse(response: Response): Promise { return JSON.parse(Buffer.concat(chunks).toString('utf8')); } -const io: OperatorIO = { +export const operatorIO: OperatorIO = { async api(path, method = 'GET', body) { const account = process.env['CLOUDFLARE_ACCOUNT_ID']; const token = process.env['CLOUDFLARE_API_TOKEN']; @@ -563,7 +563,16 @@ const io: OperatorIO = { try { const path = join(dir, 'lifecycle.json'); await writeFile(path, JSON.stringify(rules)); - await io.wrangler(['r2', 'bucket', 'lifecycle', 'set', bucket, '--file', path, '--force']); + await operatorIO.wrangler([ + 'r2', + 'bucket', + 'lifecycle', + 'set', + bucket, + '--file', + path, + '--force', + ]); } finally { await rm(dir, { recursive: true, force: true }); } @@ -572,7 +581,7 @@ const io: OperatorIO = { }; if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - runOperator(process.argv.slice(2), io).catch((error) => { + runOperator(process.argv.slice(2), operatorIO).catch((error) => { console.error(error instanceof Error ? error.message : 'Operator command failed'); process.exitCode = 1; }); diff --git a/packages/remote-cache/src/index.ts b/packages/remote-cache/src/index.ts index c6609ba69..b1e13cc68 100644 --- a/packages/remote-cache/src/index.ts +++ b/packages/remote-cache/src/index.ts @@ -121,6 +121,7 @@ export default { release?.(); } response.headers.set('X-Request-Id', id); + response.headers.set('X-Remote-Cache-Deployment', env.DEPLOYMENT_ID ?? 'local'); const sample = Math.min(1, Math.max(0, Number(env.LOG_SAMPLE_RATE) || 0)); // Fixed sampling bounds error volume as well as successful request volume. if (Math.random() < sample) diff --git a/packages/remote-cache/test/deployed.test.ts b/packages/remote-cache/test/deployed.test.ts new file mode 100644 index 000000000..d1ddd768c --- /dev/null +++ b/packages/remote-cache/test/deployed.test.ts @@ -0,0 +1,257 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { harness } from './helpers.ts'; +import { runSuite } from '../scripts/e2e/suite.ts'; +import { seed, seedManual, bytes, type Admin } from '../scripts/e2e/fixtures.ts'; +import { cleanup, githubTokens, settingsFrom } from '../scripts/ci.ts'; +import { ApiError, readTemplate, type OperatorIO } from '../scripts/operator.ts'; +import { randomUUID } from 'node:crypto'; + +for (const writes of [false, true]) + void test(`deployed HTTP suite runs against workerd with ${writes ? 'push' : 'PR'} claims`, async () => { + const h = await harness({ + deploymentId: 'suite-local', + namespaces: ['test', 'other', 'e2e', 'manual'], + }); + try { + for (const scope of ['e2e', 'manual']) + await h.db + .prepare(`INSERT INTO scopes + (scope_id, endpoint, repository, repository_id, repository_owner_id, branch) + VALUES (?, ?, 'owner/repo', '123', '456', 'refs/heads/main')`) + .bind(scope, `https://cache.example.com/projects/${scope}`) + .run(); + const admin: Admin = { + async sql(sql, params = []) { + if (sql.startsWith('SELECT generation_id FROM generations WHERE generation_id = ?')) + await (await h.mf.getWorker()).scheduled(); + return ( + await h.db + .prepare(sql) + .bind(...params) + .all() + ).results; + }, + async put(key, value) { + await h.bucket.put(key, value); + }, + async delete(key) { + await h.bucket.delete(key); + }, + async exists(key) { + return (await h.bucket.head(key)) !== null; + }, + }; + await seedManual(admin, 'suite-local'); + const report = await runSuite({ + origin: 'https://cache.example.com', + deployment: 'suite-local', + admin, + writes, + full: writes, + cron: true, + cronTimeoutMs: 5000, + pollMs: 1, + token: (aud) => + h.token({ + aud, + ...(writes ? {} : { event_name: 'pull_request', ref: 'refs/pull/718/merge' }), + }), + async request(url, init) { + // Serialize Node FormData before crossing Miniflare's fetch implementation. + const request = new Request(url, init); + const response = await h.mf.dispatchFetch(url, { + method: request.method, + headers: Object.fromEntries(request.headers), + ...(request.method === 'GET' ? {} : { body: await request.arrayBuffer() }), + }); + return new Response(await response.arrayBuffer(), { + status: response.status, + headers: response.headers, + }); + }, + }); + assert.equal(report.results.length, writes ? 14 : 8); + assert.ok(report.results.every((result) => result.status === 'passed')); + } finally { + await h.close(); + } + }); + +void test('CI settings isolate PRs and only enable writes for default-branch pushes', () => { + const env = { + REMOTE_CACHE_WORKERS_SUBDOMAIN: 'example', + GITHUB_REPOSITORY: 'owner/repo', + GITHUB_REPOSITORY_ID: '123', + REMOTE_CACHE_SOURCE_SHA: 'a'.repeat(40), + GITHUB_RUN_ID: '42', + GITHUB_RUN_ATTEMPT: '2', + REMOTE_CACHE_DEFAULT_BRANCH: 'main', + GITHUB_REF: 'refs/heads/main', + GITHUB_EVENT_NAME: 'push', + }; + assert.equal(settingsFrom(env).writes, true); + assert.equal(settingsFrom({ ...env, GITHUB_EVENT_NAME: 'workflow_dispatch' }).writes, false); + const pr = settingsFrom({ + ...env, + REMOTE_CACHE_PR_NUMBER: '718', + GITHUB_EVENT_NAME: 'pull_request', + }); + assert.equal(pr.name, 'vp-cache-ci-pr-718'); + assert.equal(pr.writes, false); + assert.throws(() => settingsFrom({ ...env, REMOTE_CACHE_RESOURCE_PREFIX: 'production' })); + assert.throws(() => settingsFrom({ ...env, REMOTE_CACHE_PR_NUMBER: '718;rm -rf /' })); + assert.throws(() => settingsFrom({ ...env, GITHUB_REF: 'refs/heads/feature' })); +}); + +void test('OIDC requests use only GitHub, reject redirects, and reuse tokens only for the same audience', async () => { + const calls: { url: string; init?: RequestInit }[] = []; + const request: typeof fetch = async (url, init) => { + calls.push({ url: typeof url === 'string' ? url : 'href' in url ? url.href : url.url, init }); + return new Response(JSON.stringify({ value: 'signed-token' })); + }; + const env = { + ACTIONS_ID_TOKEN_REQUEST_URL: 'https://run-actions.example.actions.githubusercontent.com/token', + ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-credential', + }; + const token = githubTokens(env, request); + await token('https://cache.example/projects/e2e'); + await token('https://cache.example/projects/e2e'); + await token('https://cache.example/projects/other'); + assert.equal(calls.length, 2); + assert.equal( + new URL(calls[0]!.url).searchParams.get('audience'), + 'https://cache.example/projects/e2e', + ); + assert.equal(calls[0]!.init?.redirect, 'error'); + await assert.rejects( + githubTokens( + { ...env, ACTIONS_ID_TOKEN_REQUEST_URL: 'https://attacker.example/token' }, + request, + )('audience'), + ); + assert.equal(calls.length, 2); +}); + +void test('CI cleanup checks ownership, drains through Cron, preserves nonempty resources, and retries', async () => { + const settings = settingsFrom({ + REMOTE_CACHE_WORKERS_SUBDOMAIN: 'example', + REMOTE_CACHE_PR_NUMBER: '718', + GITHUB_REPOSITORY: 'owner/repo', + GITHUB_REPOSITORY_ID: '123', + REMOTE_CACHE_SOURCE_SHA: 'a'.repeat(40), + GITHUB_RUN_ID: '42', + GITHUB_RUN_ATTEMPT: '1', + REMOTE_CACHE_DEFAULT_BRANCH: 'main', + GITHUB_REF: 'refs/heads/main', + GITHUB_EVENT_NAME: 'workflow_dispatch', + }); + const h = await harness(); + let config = await readTemplate(); + let database = true, + bucket = true, + worker = true; + const deleted: string[] = []; + const io: OperatorIO = { + async api(path, method, body) { + if (path.startsWith('/d1/database?')) + return database ? [{ name: settings.name, uuid: 'test-db' }] : []; + if (path.endsWith('/query')) { + const { sql, params } = body as { sql: string; params: (string | number | null)[] }; + return [ + await h.db + .prepare(sql) + .bind(...params) + .all(), + ]; + } + if (path.startsWith('/r2/')) { + if (!bucket) throw new ApiError(404); + return {}; + } + if (path.startsWith('/workers/') && method === 'DELETE') { + if (!worker) throw new ApiError(404); + assert.equal(database, false); + worker = false; + deleted.push('worker'); + return null; + } + throw new Error('Unexpected Cloudflare API call'); + }, + async github() { + throw new Error('Unexpected GitHub API call'); + }, + async lifecycle() { + throw new Error('Unexpected lifecycle change'); + }, + async readConfig() { + return config; + }, + async writeConfig(value) { + config = value; + }, + print() {}, + async wrangler(args) { + if (args[0] === 'r2') { + if ((await h.bucket.list()).objects.length) throw new Error('Bucket is not empty'); + bucket = false; + deleted.push('bucket'); + } else if (args[0] === 'd1') { + assert.equal(bucket, false); + database = false; + deleted.push('database'); + } else throw new Error('Unexpected Wrangler command'); + }, + }; + try { + await h.db + .prepare("UPDATE scopes SET endpoint = ? || '/projects/' || scope_id") + .bind(settings.origin) + .run(); + const admin: Admin = { + async sql(sql, params = []) { + return ( + await h.db + .prepare(sql) + .bind(...params) + .all() + ).results; + }, + async put(key, value) { + await h.bucket.put(key, value); + }, + async delete(key) { + await h.bucket.delete(key); + }, + async exists(key) { + return (await h.bucket.head(key)) !== null; + }, + }; + await seed(admin, 'other', 'cleanup', bytes('value'), bytes('blob')); + const orphan = `other/${randomUUID()}/value`; + await h.bucket.put(orphan, 'orphan'); + await assert.rejects(cleanup({ ...settings, repositoryId: '999' }, io), /another deployment/); + assert.deepEqual(deleted, []); + await assert.rejects( + cleanup(settings, io, async () => { + await h.db.prepare('UPDATE generations SET gc_after = 0').run(); + await (await h.mf.getWorker()).scheduled(); + }), + /not empty/, + ); + assert.equal(database, true); + assert.equal(worker, true); + assert.deepEqual(deleted, []); + assert.equal( + (await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes, + 0, + ); + await h.bucket.delete(orphan); + await cleanup(settings, io); + await cleanup(settings, io); + assert.deepEqual(deleted, ['bucket', 'database', 'worker']); + await assert.rejects(cleanup({ ...settings, pr: undefined }, io), /restricted to PR/); + } finally { + await h.close(); + } +}); diff --git a/packages/remote-cache/test/helpers.ts b/packages/remote-cache/test/helpers.ts index 2e3cbd9d8..bdd13dec6 100644 --- a/packages/remote-cache/test/helpers.ts +++ b/packages/remote-cache/test/helpers.ts @@ -18,6 +18,8 @@ export async function harness( jwksStatus?: number; limits?: Record; inspector?: boolean; + deploymentId?: string; + namespaces?: string[]; } = {}, ) { const pair = await generateKeyPair('RS256', { extractable: true }); @@ -45,7 +47,8 @@ export async function harness( d1Databases: ['INDEX'], r2Buckets: ['ARTIFACTS'], bindings: { - NAMESPACES: '["test","other"]', + DEPLOYMENT_ID: options.deploymentId ?? 'local', + NAMESPACES: JSON.stringify(options.namespaces ?? ['test', 'other']), LIMITS: JSON.stringify(options.limits ?? {}), GC_BATCH_SIZE: '16', LOG_SAMPLE_RATE: '0', diff --git a/packages/remote-cache/test/index.test.ts b/packages/remote-cache/test/index.test.ts index fd2243249..1c37c3520 100644 --- a/packages/remote-cache/test/index.test.ts +++ b/packages/remote-cache/test/index.test.ts @@ -2,3 +2,4 @@ import './protocol.test.ts'; import './service.test.ts'; import './operator.test.ts'; import './failures.test.ts'; +import './deployed.test.ts'; diff --git a/packages/remote-cache/wrangler.jsonc b/packages/remote-cache/wrangler.jsonc index 08cfa7a29..e0bed4736 100644 --- a/packages/remote-cache/wrangler.jsonc +++ b/packages/remote-cache/wrangler.jsonc @@ -22,6 +22,7 @@ { "name": "STORE_LIMITER", "namespace_id": "1002", "simple": { "limit": 30, "period": 60 } }, ], "vars": { + "DEPLOYMENT_ID": "local", "NAMESPACES": "[]", "LIMITS": "{}", "GC_BATCH_SIZE": "16", From 099a946c444e08ef6686b1fe9b3bd47936cc7131 Mon Sep 17 00:00:00 2001 From: MK Date: Sat, 12 Sep 2026 10:54:30 +0800 Subject: [PATCH 04/17] fix(remote-cache): initialize CI types and revise cleanup trigger Co-authored-by: GPT-6 Codex --- .github/workflows/remote-cache-deploy.yml | 12 +++++++ .../workflows/remote-cache-preview-events.yml | 31 +++---------------- packages/remote-cache/docs/e2e-plan.md | 6 ++-- packages/remote-cache/package.json | 2 +- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/.github/workflows/remote-cache-deploy.yml b/.github/workflows/remote-cache-deploy.yml index ef49b1c3d..5cee1fc8d 100644 --- a/.github/workflows/remote-cache-deploy.yml +++ b/.github/workflows/remote-cache-deploy.yml @@ -53,6 +53,18 @@ jobs: persist-credentials: false - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 - run: pnpm check-remote-cache + - name: Explain fork preview availability + if: >- + always() && github.event_name == 'pull_request' && + (github.event.pull_request.head.repo.full_name != github.repository || github.event.pull_request.user.login == 'dependabot[bot]') + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + await core.summary.addHeading('Remote cache preview').addRaw( + 'Fork and Dependabot PRs run local checks without Cloudflare credentials. ' + + 'A maintainer can copy the reviewed commit to an internal branch and open a PR for a preview. ' + + 'No live deployment passed verification for this PR.' + ).write(); deploy: name: Deploy and verify diff --git a/.github/workflows/remote-cache-preview-events.yml b/.github/workflows/remote-cache-preview-events.yml index ff1924f2e..21e9ce074 100644 --- a/.github/workflows/remote-cache-preview-events.yml +++ b/.github/workflows/remote-cache-preview-events.yml @@ -1,8 +1,8 @@ -name: Remote cache preview events +name: Remote cache preview cleanup on: - pull_request_target: - types: [opened, synchronize, reopened, closed] + pull_request: + types: [closed] paths: - packages/remote-cache/** - .github/workflows/remote-cache-*.yml @@ -20,33 +20,12 @@ on: permissions: {} jobs: - fork-notice: - if: >- - github.event_name == 'pull_request_target' && github.event.action != 'closed' && - (github.event.pull_request.head.repo.full_name != github.repository || github.event.pull_request.user.login == 'dependabot[bot]') - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - pull-requests: write - steps: - # This privileged workflow never checks out or executes PR code. - - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - script: | - const marker = ''; - const body = `${marker}\n### Remote cache preview\n\nCloudflare credentials are not available to fork or Dependabot workflows. ` + - `Local protocol and integration checks still run. A maintainer can copy the reviewed commit to a branch in this repository ` + - `and open a PR to deploy an isolated preview. No live deployment passed verification for this PR.`; - const comments = await github.paginate(github.rest.issues.listComments, { ...context.repo, issue_number: context.issue.number, per_page: 100 }); - const previous = comments.find(c => c.user?.login === 'github-actions[bot]' && c.body?.includes(marker)); - if (previous) await github.rest.issues.updateComment({ ...context.repo, comment_id: previous.id, body }); - else await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body }); - cleanup: name: Remove closed PR resources if: >- vars.REMOTE_CACHE_DEPLOY_ENABLED == 'true' && - ((github.event.action == 'closed' && github.event.pull_request.head.repo.full_name == github.repository) || + ((github.event.action == 'closed' && github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login != 'dependabot[bot]') || (github.event_name == 'workflow_dispatch' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))) runs-on: ubuntu-latest timeout-minutes: 35 diff --git a/packages/remote-cache/docs/e2e-plan.md b/packages/remote-cache/docs/e2e-plan.md index ecf3539be..3792c08c0 100644 --- a/packages/remote-cache/docs/e2e-plan.md +++ b/packages/remote-cache/docs/e2e-plan.md @@ -38,9 +38,9 @@ Set these **repository variables**, which the notification job also needs: Enable R2 in the account. The token must allow resource creation and deletion as well as data access. No custom domain or DNS permission is needed. Do not attach production bindings or secrets to these Workers. The GitHub environment can require a maintainer review if the repository needs one before a deployment. -The workflow exposes Cloudflare credentials only to deployment, verification, and cleanup commands. Dependency installation runs before those credentials enter the step environment. Internal PR contributors must be trusted to change deployment code. Fork and Dependabot PRs run local checks without Cloudflare credentials. A separate `pull_request_target` workflow only posts their notice or executes cleanup code from the default branch; it never checks out PR code. To preview a fork change, a maintainer must copy the reviewed commit to a branch in this repository and open a PR. +The workflow exposes Cloudflare credentials only to deployment, verification, and cleanup commands. Dependency installation runs before those credentials enter the step environment. Internal PR contributors must be trusted to change deployment code. Fork and Dependabot PRs run local checks without Cloudflare credentials. Their check summary explains why no preview is available; they receive no deployment comment. Cleanup uses ordinary closed-PR events for internal branches and executes code from the default branch. To preview a fork change, a maintainer must copy the reviewed commit to a branch in this repository and open a PR. -The preview-events workflow must exist on the default branch before fork notices and automatic closed-PR cleanup can run. Merge the workflow setup before relying on those events. Manual deployment dispatches use the default branch and cannot select an arbitrary PR revision. +Merge the workflow and cleanup script to the default branch before relying on automatic cleanup. GitHub can suppress `pull_request` workflows when a PR has merge conflicts. Use the manual cleanup workflow with the closed PR number if the close event does not run. Manual deployment dispatches use the default branch and cannot select an arbitrary PR revision. ## Authorization and test levels @@ -135,7 +135,7 @@ PR previews intentionally reject publication tokens from PR workflows. Use the m Each CI deployment has a 2 GB byte budget, 1,000 entries, and 2,000 associations. Pending and retired objects remain charged. After verification, the runner expires `e2e` and `other` data and gives in-flight operations the normal ten-minute cleanup grace. It retains the `manual` fixture for developer checks. Subsequent runs explicitly restore CI-owned policy switches in case a prior process stopped during a withdrawal test. -Closing a PR triggers `.github/workflows/remote-cache-preview-events.yml`. The cleanup job executes code from the default branch, confirms that the PR is closed, and checks resource ownership. It disables access, waits for Cron to drain generations, then deletes the empty bucket, D1 database, and Worker. It does not delete main staging. The PR comment reports cleanup success or failure. +Closing an internal PR triggers `.github/workflows/remote-cache-preview-events.yml`. The cleanup job executes code from the default branch, confirms that the PR is closed, and checks resource ownership. It disables access, waits for Cron to drain generations, then deletes the empty bucket, D1 database, and Worker. It does not delete main staging. The PR comment reports cleanup success or failure. If Cron or orphan multipart uploads delay cleanup, the job fails instead of claiming that resources are gone. R2 lifecycle can require a day to abort an unrecorded upload. Rerun the cleanup workflow with the closed PR number after the backlog drains. Failed resource creation and partial teardown can also be retried. A nonempty bucket always blocks final removal. diff --git a/packages/remote-cache/package.json b/packages/remote-cache/package.json index ee1461991..897e81954 100644 --- a/packages/remote-cache/package.json +++ b/packages/remote-cache/package.json @@ -7,7 +7,7 @@ "scripts": { "dev": "wrangler dev --test-scheduled", "build": "wrangler deploy --dry-run --outdir dist", - "types": "wrangler types .wrangler/worker-configuration.d.ts --env-interface Env --strict-vars false", + "types": "node -e \"require('node:fs').mkdirSync('.wrangler', { recursive: true })\" && wrangler types .wrangler/worker-configuration.d.ts --env-interface Env --strict-vars false", "check": "pnpm types && tsc --noEmit", "test": "node --import tsx --test test/index.test.ts", "smoke": "pnpm test && pnpm build", From 443fe7f7ea458a8298446daec207393007cddb11 Mon Sep 17 00:00:00 2001 From: MK Date: Sat, 12 Sep 2026 10:58:49 +0800 Subject: [PATCH 05/17] test(remote-cache): synchronize upload cancellation with stream reads Co-authored-by: GPT-6 Codex --- packages/remote-cache/test/failures.test.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/remote-cache/test/failures.test.ts b/packages/remote-cache/test/failures.test.ts index 636d68109..cad574066 100644 --- a/packages/remote-cache/test/failures.test.ts +++ b/packages/remote-cache/test/failures.test.ts @@ -140,16 +140,24 @@ void test('unknown-length requests reserve the full limit and cancellation preve try { const env = await h.mf.getBindings(); const pending: Promise[] = []; - const deadline = new Deadline(100); + const controller = new AbortController(); + const deadline = new Deadline(10000, controller.signal); let cancelled = false; const requestInit = { method: 'POST', headers: { 'Content-Type': 'multipart/form-data; boundary=x' }, - body: new ReadableStream({ - cancel() { - cancelled = true; + body: new ReadableStream( + { + pull() { + controller.abort(); + }, + cancel() { + cancelled = true; + }, }, - }), + // Pull only after the reservation completes and the parser starts reading. + { highWaterMark: 0 }, + ), duplex: 'half', }; const request = new Request('https://cache.example.com/projects/test/store', requestInit); @@ -168,6 +176,7 @@ void test('unknown-length requests reserve the full limit and cancellation preve { status: 503 }, ); await Promise.all(pending); + assert.equal(controller.signal.aborted, true); assert.equal(cancelled, true); const row = await h.db.prepare('SELECT charged_bytes, state FROM generations').first(); assert.equal(row!.charged_bytes, defaults.store); From 5d22b21c193841a0ec1cbbf85691c034d89e3d52 Mon Sep 17 00:00:00 2001 From: MK Date: Sat, 12 Sep 2026 11:36:27 +0800 Subject: [PATCH 06/17] docs(remote-cache): include RFC and supporting material Copy RFC #716 from c201f8eff9d9d2827426c88667be39e8b1d21999 and adjust relative paths. Co-authored-by: GPT-6 Codex --- packages/remote-cache/README.md | 2 +- .../remote-cache/docs/0001-remote-cache.md | 813 ++++++++++++++++++ .../docs/images/repository-binding-form.svg | 47 + .../docs/remote-cache-size-study/README.md | 107 +++ .../docs/remote-cache-size-study/measure.py | 175 ++++ .../docs/remote-cache-size-study/results.json | 355 ++++++++ .../docs/remote-cache-size-study/sources.json | 71 ++ 7 files changed, 1569 insertions(+), 1 deletion(-) create mode 100644 packages/remote-cache/docs/0001-remote-cache.md create mode 100644 packages/remote-cache/docs/images/repository-binding-form.svg create mode 100644 packages/remote-cache/docs/remote-cache-size-study/README.md create mode 100644 packages/remote-cache/docs/remote-cache-size-study/measure.py create mode 100644 packages/remote-cache/docs/remote-cache-size-study/results.json create mode 100644 packages/remote-cache/docs/remote-cache-size-study/sources.json diff --git a/packages/remote-cache/README.md b/packages/remote-cache/README.md index 436d34de9..03425c96d 100644 --- a/packages/remote-cache/README.md +++ b/packages/remote-cache/README.md @@ -1,6 +1,6 @@ # Public remote cache service -This package implements the server in [RFC #716](https://github.com/voidzero-dev/vite-task/pull/716): a TypeScript Worker, primary D1 metadata, and a private R2 Standard bucket. Anyone can read an enabled namespace. Only a signed GitHub Actions token for the registered public repository's main-branch `push` job can publish. +This package implements the server in the [remote cache RFC](docs/0001-remote-cache.md) ([PR #716](https://github.com/voidzero-dev/vite-task/pull/716)): a TypeScript Worker, primary D1 metadata, and a private R2 Standard bucket. Anyone can read an enabled namespace. Only a signed GitHub Actions token for the registered public repository's main-branch `push` job can publish. The package contains no `vp run` client adapter. Cache keys, values, and blobs remain opaque. A successful lookup does not prove that a result is reusable; a client must validate its inputs and output archive. diff --git a/packages/remote-cache/docs/0001-remote-cache.md b/packages/remote-cache/docs/0001-remote-cache.md new file mode 100644 index 000000000..a424d280a --- /dev/null +++ b/packages/remote-cache/docs/0001-remote-cache.md @@ -0,0 +1,813 @@ +# RFC: Public remote cache for GitHub projects with `vp run` + +Status: Draft design. + +Updated: 2026-09-10. Repository baseline: `9a1d32cf`. API baseline: [PR #713](https://github.com/voidzero-dev/vite-task/pull/713), commit [`362f5bd9`](https://github.com/voidzero-dev/vite-task/blob/362f5bd91bb32806b512d3d9a5339ff435bf5f0a/docs/remote-cache-server-api.md). The API proposal remains a draft; check its final contract before implementation. + +## 1. Motivation + +Open-source maintainers should publish successful task results from the main branch through GitHub Actions. They should use a service in their own Cloudflare account. Developers and fork contributors should reuse these public results without login. The service needs no Vite+ hosted account or license service. + +The client checks the local cache first. If a remote read fails, the client executes the task. + +The current [docs deployment action](https://github.com/voidzero-dev/vite-plus/blob/ed1710f7aff8941436907a4d755d6b38c798ed41/.github/actions/deploy-docs/action.yml) transfers a whole task-cache directory through GitHub Actions Cache. A native remote cache can transfer one task's metadata and output blob. Compatible developer machines can reuse these results. + +This RFC describes an implementation of PR #713 on Workers, D1, and R2. Reads need no authentication. Writes use GitHub Actions OpenID Connect (OIDC), which supplies signed tokens that identify jobs. The RFC sets operational defaults and estimates when an individual or small team can stay within Cloudflare's free allowances. + +## 2. Contract and scope + +PR #713 defines the HTTP contract. This RFC defines the Cloudflare storage, authorization, limits, deployment, and cleanup. A namespace is a project's cache scope at a configured endpoint. Version 1 requires public reads and write authorization for each registered repository. All three endpoints must keep namespace data separate. The client defines its fingerprint format and cache-validation policy. + +| Area | Decision | +| -------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| Hosting | Open-source TypeScript Worker, private R2 Standard bucket, D1 database, five-minute Cron Trigger | +| Endpoints | `POST /fetch`, `GET /blob/{blob_id}`, `POST /store`, relative to a configured namespace endpoint | +| Data | CBOR envelope; opaque binary keys and values; optional opaque blob | +| Lookup | Exact key first, then one secondary-key association | +| Store | Replace the entry and secondary association together after object storage succeeds | +| Access control | Anonymous reads; GitHub OIDC writes restricted to the registered repository and main-branch push events | +| Transfer | One multipart HTTP store request; internal R2 multipart upload for larger blobs | +| Client mode | `--remote-cache` or `VP_REMOTE_CACHE` selects `off`, `read`, or `read-write`; defaults to `read` with an endpoint and `off` without one | +| Defaults | Seven-day retention, 8 GB total R2 budget, 64 MiB maximum blob | +| Failure | Bounded waits; read failures become misses; upload failures warn without changing the task exit status | + +The first delivery includes a template for self-deployment and a native Rust client adapter. The client must work on macOS, Linux, and Windows. Reuse across different operating systems or architectures requires a separate agreement about client compatibility. + +This plan excludes remote execution, a hosted SaaS, anonymous writes, a web dashboard, and deduplication across projects. Version 2 covers private caches and Cloudflare One authorization. + +## 3. Relationship to the current local cache + +The current engine separates exact lookup from a task association that explains misses: + +| Source evidence | Client integration consequence | +| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| [`ExecutionCache::try_hit` and `CacheEntryKey`](../../../crates/vt/src/session/cache/mod.rs) use the spawn fingerprint and resolved input/output configuration for exact lookup. | Construct `key` before execution. Input changes can replace the value at the same key. | +| [`ExecutionCacheKey`](../../../crates/vt_plan/src/cache_metadata.rs) identifies the task for the diagnostic association. | Supply a corresponding `secondary_key`; a fallback result can explain what changed. | +| [`PostRunFingerprint`](../../../crates/vt/src/session/execute/fingerprint.rs) and explicit glob checks validate observed inputs. | An exact server response still needs local validation before reuse. | +| [`update_cache`](../../../crates/vt/src/session/execute/cache_update.rs) rejects failed, cancelled, incompletely traced, or otherwise ineligible executions. | Apply the same eligibility checks before remote storage. | +| [`archive`](../../../crates/vt/src/session/cache/archive.rs) and [`replay_cache_hit`](../../../crates/vt/src/session/execute/mod.rs) handle output files and terminal replay. | Import verified data through a bounded staging path before reporting a hit. | + +An exact response means that the server found identical key bytes. It does not prove that current input files, inferred dependencies, or tracked environment values match. In the current engine, fallback data explains a miss. The client does not reuse fallback outputs. Keep this distinction in the remote adapter. An exact entry that fails validation is a cache miss. + +The client must define a portable, versioned encoding before it supports reuse across machines. An opaque field contains bytes that the Worker does not interpret. The encoding must preserve these observations: + +- Negative file dependencies, which record that a file does not exist. +- Directory observations. +- Tracked environment queries. +- Explicit glob membership, which records the files that match each pattern. + +The client must include schema, toolchain, and platform compatibility in its identity or validation data. PR #713 does not define this encoding. The Worker must not decode it. Local schema `v18` and a serialized SQLite directory do not define a portable wire format. + +The client follows this sequence: + +1. Validate the local cache. A local hit makes no remote request during `vp run`. +2. On a local miss, call `/fetch`. +3. Validate an exact response. Download its blob only if the result passes validation. +4. Restore the outputs. Save the result in local storage. + +A fallback response or HTTP `404` from `/fetch` leads to task execution. A successful eligible execution updates local storage, then queues its result for `/store` if uploads are enabled for that run. Cache hits do not trigger uploads. + +## 4. HTTP API mapping + +All paths are relative to an endpoint such as `https://cache.example.com/projects/docs-trusted-v1`. The endpoint includes the namespace. The examples describe fields in Concise Binary Object Representation (CBOR), a binary data format. `bytes` means a binary byte string. `string` means text. Nullable fields must remain present with CBOR `null` when they have no value. + +### Fetch metadata + +```text +POST {endpoint}/fetch +Content-Type: application/cbor + +{ key: bytes, secondary_key: bytes } +``` + +For a match, return HTTP `200`, `Content-Type: application/cbor`, with one of: + +```text +{ kind: "exact", value: bytes, blob_id: string | null } +{ kind: "fallback", key: bytes, value: bytes, blob_id: string | null } +``` + +Check `key` first. If no live entry exists, resolve `secondary_key` to a stored key. Check that entry. Include the stored key only in the fallback variant. If neither resolves, return HTTP `404`. Fetch does not change entries or associations. + +### Download a blob + +```text +GET {endpoint}/blob/{blob_id} +``` + +Return HTTP `200`, `Content-Type: application/octet-stream`, with the raw blob. Return `404` if the blob is unavailable. The server generates an opaque blob ID within the endpoint's namespace. The ID is neither an R2 URL nor an authorization credential. + +### Store an entry + +```text +POST {endpoint}/store +Content-Type: multipart/form-data; boundary=... + +metadata part (required), Content-Type: application/cbor: + { key: bytes, secondary_key: bytes, value: bytes } + +blob part (optional), Content-Type: application/octet-stream: + raw blob bytes +``` + +Accept either part order. Return HTTP `200`, `Content-Type: application/cbor`: + +```text +{ blob_id: string | null } +``` + +If the request omits the blob, return `null`. A present zero-byte blob receives a non-null ID. Its download has an empty body. + +Each successful store replaces `entries[key]` and sets `associations[secondary_key] = key`. For example: + +| Operation | Entries afterward | Association afterward | +| ------------------- | ----------------------------------------------- | --------------------- | +| Store `(A, S, VA)` | `A → VA` | `S → A` | +| Store `(B, S, VB)` | `A → VA`, `B → VB` | `S → B` | +| Fetch `(A, S)` | Unchanged; returns exact `VA` | Still `S → B` | +| Fetch `(C, S)` | Unchanged; returns fallback key `B`, value `VB` | Still `S → B` | +| Store `(A, T, VA2)` | `A → VA2`, `B → VB` | `S → B`, `T → A` | + +Other secondary keys that already point to `A` also resolve to `VA2`. A change to `S` does not evict entry `A`. + +### Errors + +API errors use `Content-Type: text/plain; charset=utf-8`. Clients use the status code, not the human-readable message, to classify them. + +| Status | Meaning | +| ------ | ------------------------------------------------------------------------ | +| `400` | Malformed request or invalid field types | +| `404` | No matching entry on fetch, or blob unavailable | +| `413` | Request exceeds configured size limits | +| `500` | Operation could not complete | +| `503` | Service temporarily unavailable, including exhausted application budgets | + +If metadata is absent, return `404`. Version 1 reads require no credentials. For `/store`, this deployment adds these errors: + +- `401`: The JSON Web Token (JWT) is missing, invalid, or expired. +- `403`: The verified JWT fails the namespace's write policy. +- `503`: The Worker cannot establish authorization because D1 or required signing keys are unavailable. + +Keep these errors generic. Use plain text. PR #713 does not define authentication. Rate limiting can return `429` with `Retry-After`. Cloudflare can reject requests before the Worker runs. Clients must handle error bodies outside the protocol without authentication redirects. + +## 5. Public reads, GitHub OIDC writes, and cache uploads + +Version 1 serves public cache data for open-source repositories on GitHub.com. Anyone can call `/fetch` and `/blob/{blob_id}` without credentials. Only an authorized GitHub Actions job can call `/store`. Developers use the checked-in endpoint without login, secrets, or individual permission setup. Version 2 covers private projects with Cloudflare One authorization. + +### Client configuration and remote cache modes + +```ts +export default { + run: { + remoteCache: { + url: 'https://cache.example.com/projects/docs-trusted-v1', + }, + }, +}; +``` + +Use `--remote-cache` to select a mode for one invocation, or set `VP_REMOTE_CACHE` to configure all `vp run` commands in an environment. Both accept the same values: + +| Command | Environment variable | Remote reads | Uploads | +| ---------------------------------------- | ---------------------------- | ------------ | -------- | +| `vp run build --remote-cache=off` | `VP_REMOTE_CACHE=off` | Disabled | Disabled | +| `vp run build --remote-cache=read` | `VP_REMOTE_CACHE=read` | Enabled | Disabled | +| `vp run build --remote-cache=read-write` | `VP_REMOTE_CACHE=read-write` | Enabled | Enabled | + +The command-line option takes precedence over `VP_REMOTE_CACHE`. If neither is set, use `read` when an endpoint is configured; otherwise, use `off`. The mode leaves local caching unchanged. + +Use `VP_REMOTE_CACHE_URL` to override the endpoint on a host. Without an endpoint, the client makes no remote requests. Selecting `read` or `read-write` through the command-line option or `VP_REMOTE_CACHE` without an endpoint reports a configuration error before execution. + +Task-level `remoteCache: false` excludes remote reads and uploads but retains local caching. `cache: false`, `--no-cache`, and tool-requested cache disabling also prevent uploads. The remote cache mode does not override these exclusions. + +Version 1 uploads require GitHub Actions OIDC authorization for a main-branch push job. Enable uploads in that job with `VP_REMOTE_CACHE=read-write` or `--remote-cache=read-write`. + +### Upload lifecycle + +In `read-write` mode, queue one `/store` request after each successful eligible task saves its result locally. Upload only results generated by the current invocation. Cache hits do not trigger uploads. + +Upload in the background with bounded concurrency so dependent tasks can start without waiting for network transfers. A successful task's result remains eligible even if another task fails. Before exiting after task execution, wait for pending uploads within a bounded deadline. + +### One-time repository binding + +The operator binds a public repository during deployment. This form illustrates the setup inputs and automatic values: + +![Configuration form: repository, namespace, and main branch inputs; automatic IDs, access policy, and public endpoint.](images/repository-binding-form.svg) + +Setup saves both immutable IDs, the branch ref, and the audience in the server policy. The audience identifies the token's intended recipient. The client derives the same audience from its configured endpoint, without a trailing slash. + +After a repository transfer, review the binding. Update the owner ID. After an endpoint alias or namespace change, update the policy. + +### GitHub Actions token acquisition + +The publishing job grants `permissions: id-token: write`. This permission lets the job request an OIDC token. The Worker decides whether the token grants write access. The native client uses GitHub's `ACTIONS_ID_TOKEN_REQUEST_URL` and `ACTIONS_ID_TOKEN_REQUEST_TOKEN` to request a token with the namespace audience. See GitHub's [OIDC workflow configuration](https://docs.github.com/en/actions/how-tos/secure-your-work/security-harden-deployments/oidc-in-cloud-providers). + +`vp run` requests the token only when uploads are enabled and a newly generated result is ready to upload. Send the returned JWT to `/store` as `Authorization: Bearer `. The Worker verifies it directly, without a custom endpoint for token exchange. + +Keep the JWT in process memory. Reuse it only while it remains valid for the same audience. Obtain a fresh token before expiry. Send the runner's request token only to GitHub's token endpoint. Never send that request token to the cache Worker. If GitHub OIDC is unavailable, report one warning and stop upload attempts for the invocation without changing the task exit status. + +Do not forward either token across redirects. Do not write either token to config, command arguments, output, or the cache. Remove the OIDC request variables and remote-cache controls from these locations, including through wildcard environment selection: + +- Environments of task child processes. +- Fingerprints. +- Runner-aware environment APIs. +- Serialized plans. +- Debug output. + +These controls do not isolate a privileged job from other code that runs as the same OS user. Publication jobs execute trusted code from the main branch. + +### Worker write policy + +Authorize only registered namespaces. Use the stored repository and owner IDs; names serve as display data. Do not let a token claim a namespace. Use the stored audience, never the request's `Host` header. + +Verify the JWT signature before the Worker reads a store body or reserves quota. Use a maintained library such as [`jose`](https://github.com/panva/jose), which supports Workers and remote JSON Web Key Sets (JWKS). A JWKS supplies public keys for signature verification. + +Use GitHub's fixed [OIDC issuer metadata](https://token.actions.githubusercontent.com/.well-known/openid-configuration) and HTTPS JWKS endpoint. Allow only its supported signing algorithm (`RS256` initially). Reject `none`, symmetric algorithms, key URLs from tokens, and claims without signature verification. + +Set limits for token size, JWKS response size, fetch time, cache lifetime, and refresh frequency. Unknown key IDs must not trigger unlimited outbound requests. Return `503` if key retrieval fails and the cache has no usable key. Do not skip signature verification. + +After signature verification, check these signed claims against the enabled namespace policy: + +| Claim | Required value | +| ----------------------- | --------------------------------------------------- | +| `iss` | `https://token.actions.githubusercontent.com` | +| `aud` | Exact configured namespace endpoint | +| `repository_id` | Namespace's registered repository ID | +| `repository_owner_id` | Namespace's registered owner ID | +| `repository_visibility` | `public` | +| `ref` / `ref_type` | Configured main-branch ref / `branch` | +| `event_name` | `push` | +| `exp`, `nbf`, `iat` | Present and valid under a bounded clock-skew policy | + +These conditions use [GitHub's documented claims](https://docs.github.com/en/actions/reference/security/oidc). Require exact types and values. Do not authorize writes from any of these inputs: + +- `actor`. +- A repository URL from the client. +- A branch environment variable. +- A substring match in `sub`. + +GitHub supports different subject formats. Repository IDs and branch/event claims avoid dependence on one text format for `sub`. A write grant for an organization or repository does not replace the complete set of checks. + +A `pull_request_target` job can run in the base repository's default-branch context. Thus, `ref=refs/heads/main` alone cannot authorize writes. Version 1 denies `pull_request`, `pull_request_target`, `workflow_run`, tags, branches other than main, and all other event types. + +Fork repositories have different IDs. They cannot write to the upstream namespace, even from their own `main` branch. Reusable workflows must have matching repository, branch, and event claims for the caller. The called workflow's identity alone grants no permission. See GitHub's [workflow event behavior](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#pull_request_target). + +| Caller | `POST /fetch` | `GET /blob/{blob_id}` | `POST /store` | +| --------------------------------------------------------------------------------- | ----------------------------- | ----------------------------- | ------------- | +| Anonymous developer or fork contributor | Allow | Allow | `401` | +| Registered repository, public, main-branch push, valid audience/token | Allow | Allow | Allow | +| Valid GitHub token with wrong repo, owner, visibility, branch, event, or audience | Allow | Allow | `403` | +| Invalid, expired, or forged token | Allow without using the token | Allow without using the token | `401` | + +A local command cannot obtain write permission through environment variables alone. Unknown or disabled namespaces expose no cache data. Apply the namespace restriction to every exact, fallback, and blob lookup and every mutation. Return `404` for a blob ID from another namespace. + +This separation prevents results from different projects from mixing. All enabled version 1 namespaces remain public. Keep the R2 bucket private so reads pass through Worker routing, retention checks, and budgets. Apply the same store verifier to every exposed route and alias. + +### Publication trust and revocation + +A GitHub token proves the job's identity. It does not prove that uploaded bytes match the commit or contain no secrets. The trusted publishing workflow builds the commit that triggered the main-branch job. It selects only results intended for public release. + +Values, input metadata, terminal logs, source maps, and blobs are public. Tasks that use private inputs or produce sensitive output must opt out. The Worker treats these fields as opaque and cannot redact them. The client still validates a public result before reuse. + +At publication, the guarded D1 transaction checks these conditions again: + +- The token has not expired according to server time. +- The scope remains enabled for access and writes. +- The policy version has not changed. +- The lease and quotas remain valid. + +If the token expires or the policy changes, keep existing mappings unchanged. Mark the staged objects for cleanup. Do not call the GitHub API inside that transaction. + +Tokens are short-lived bearer credentials. A caller can reuse a token within its validity period. Version 1 keeps no ledger for token revocation or single-use enforcement. Job cancellation does not immediately revoke a token. The maintainer can disable writes or change the namespace policy in primary D1. If the maintainer disables the whole scope, subsequent reads also stop. + +A repository visibility change to private does not remove previously published cache data. To withdraw publication, maintainers must disable the public namespace. They must also remove its objects. They cannot recall downloaded copies. Version 2 covers access control for private caches. + +## 6. Cloudflare storage model + +D1 commits key mappings atomically. R2 holds opaque values and optional blobs. All values use R2 because they can exceed D1's [2 MB row limit](https://developers.cloudflare.com/d1/platform/limits/). + +### D1 relationships + +The diagram shows the logical relationships and key fields. `PK` marks primary-key columns; two marked columns form one composite key. Implementation defines the final schema and foreign-key constraints. + +```mermaid +erDiagram + direction LR + scopes ||--o{ entries : contains + scopes ||--o{ associations : contains + scopes ||..o{ generations : contains + entries |o..o{ associations : "is the target of" + entries o|..|| generations : "selects current" + + scopes { + ID scope_id PK + } + entries { + ID scope_id PK + BLOB key PK + ID generation_id + } + associations { + ID scope_id PK + BLOB secondary_key PK + BLOB target_key + } + generations { + ID generation_id PK + ID scope_id + TEXT state + } +``` + +All records and references stay within their namespace. An entry selects one current generation and can have many secondary-key associations. Unreferenced generations await publication or cleanup. Associations can remain after their targets disappear, until cleanup. + +Other metadata: + +- `scopes`: Public endpoint, enabled/write-enabled state, GitHub repository and owner IDs, branch, audience, and policy version. It also stores retention, budgets, and counters. +- `generations`: R2 object keys, optional blob ID, actual sizes, and lease/expiry/retirement times. Its random ID identifies one store. Multipart uploads also record an R2 upload ID. + +### R2 objects + +Each store creates a generation with unique, immutable R2 object names. The paths below illustrate the scope/generation prefix: + +```mermaid +flowchart LR + subgraph D1["D1 · metadata"] + E["Entry
(scope_id, key)"] -->|current generation| G["Generation
random generation_id"] + end + subgraph R2["Private R2 · immutable objects"] + V["Value object
scope/generation/value"] + B["Blob object · optional
scope/generation/blob"] + end + G -->|value object key| V + G -.->|blob object key| B +``` + +Publish only after all required objects are complete. Switch the entry pointer atomically. Retire its previous generation for cleanup. This keeps the value and blob from the same execution together during concurrent stores. + +### Storage rules + +- Use bound binary parameters and byte equality. Accept empty and non-UTF-8 keys within the size limits. Do not convert keys to strings, normalize them, or interpret them as hex hashes. +- Index exact lookups, secondary lookups, blob IDs, and cleanup eligibility. Limit associations separately because many can target one entry. +- Use generation states `uploading`, `ready`, `retired`, and `deleting`. Record object names and a bounded upload lease before R2 writes. Record multipart upload IDs for abort or recovery. These records remain internal; clients receive no upload-session API. +- Use primary D1 in version 1. Read replicas need a consistency agreement to prevent old mappings after store commits or scope disable. +- Serve public data through the Worker without an additional CDN cache. Review edge caching separately, including namespace withdrawal and expiry behavior. + +## 7. Fetch and download implementation + +First, apply rate limits. Check that the public scope is enabled. Decode CBOR within the configured limits. + +Use one indexed D1 query to select the exact live entry, or the secondary fallback if no exact entry exists. Select the response kind, stored key, and generation references from one database snapshot. Separate exact and fallback reads could observe different commits. + +Read the selected generation's value from R2. Return the corresponding CBOR variant. Return `503` if D1 identifies a live generation but its value object is missing or unreadable. This condition is a storage failure. Expired or deleted entries count as absent. The client still validates the exact value before it downloads outputs. + +For `/blob/{blob_id}`, check that the public scope is enabled, without authentication. Resolve the blob ID in D1. Stream the R2 object to the response. A blob ID from another scope must not expose data. + +Ready blobs remain available until expiry. Replaced blobs remain available during the retirement grace period described below. Return `404` for unavailable IDs, including IDs whose R2 object is gone. + +Reads do not update last-access timestamps, extend retention, change associations, or create analytics rows for each request. Fixed retention and a short replacement grace period keep fetch and download read-only. A blob can expire between fetch and download. The client treats the resulting `404` as a miss and executes the task. + +## 8. Store implementation and concurrency + +1. Verify the GitHub OIDC JWT. Check the namespace's write policy from section 5. Record the policy version and token expiry. + + Reserve storage capacity in D1. If the request supplies `Content-Length`, use it within the request limit. Otherwise, reserve the total request limit. Do not require that header. Create the generation and a 15-minute internal lease. Count concurrent reservations against scope and deployment budgets. + +2. Parse multipart input incrementally with backpressure, so reads wait when the upload cannot accept more data. Limit headers, part count, metadata bytes, blob bytes, and total bytes. Reject duplicate metadata or blob parts, invalid types, missing metadata, and truncated bodies. Accept either part order. Do not buffer the whole request with `formData()` or `arrayBuffer()`. + +3. Buffer the metadata part within its limit. Decode its outer CBOR map. Preserve its byte-string fields. Write `value` to the generation's R2 object. Do not inspect nested client data. + +4. For a blob up to 5 MiB, buffer the blob. Upload it with one R2 PUT. + + For a larger blob, use internal R2 multipart upload with 5 MiB parts and a smaller final part. Upload one part at a time. Release buffers when the upload no longer needs them. A present empty blob still requires an R2 object. Cloudflare documents the [multipart minimum and API](https://developers.cloudflare.com/r2/objects/multipart-objects/). + +5. Wait for both R2 objects and the full multipart request to complete, including the closing boundary. Record actual sizes. Publish through one guarded D1 batch, as described below. + +6. Return the new blob ID, or `null`. Complete publication before the response. Use `waitUntil()` only for cleanup or observations that do not require guaranteed completion. + +The publication batch checks token expiry against server time. It also checks the lease, captured scope, unchanged policy version, enabled/write-enabled state, and budgets. Under the same guard, the batch performs these changes atomically: + +- Mark the generation ready. +- Replace `entries[key]`. +- Set `associations[secondary_key] = key`. +- Retire the old generation of the same key. +- Adjust reserved bytes to match actual bytes. + +D1 [batches roll back the transaction if a statement fails](https://developers.cloudflare.com/d1/worker-api/d1-database/#batch). A conditional update that affects zero rows is not a SQL failure. Apply the same valid-generation guard to all publication mutations. Check their results. A failed guard must not leave either mapping changed. Test this condition with concurrent stores and lease expiry. + +Readers see either the previous complete entry or the new complete entry. Concurrent stores follow the order of successful D1 commits. The last commit determines each affected mapping. A secondary-key reassignment does not retire the different entry that it previously referenced. An entry replacement changes the result for all associations to that key. + +If R2 or parsing fails before publication, keep existing mappings unchanged. Clean up the staged generation. If the response is lost after commit, the client cannot know whether storage succeeded. A retry creates another store. It can return a different blob ID or overwrite a newer concurrent store. PR #713 supplies no idempotency key or exactly-once guarantee. + +R2 multipart uploads run inside one incoming HTTP request. A client cannot resume an upload after disconnection. Cancellation and Worker termination require cleanup. Cleanup must also cover failures between R2 multipart creation and upload-ID recording. Bucket lifecycle rules provide additional cleanup protection. + +## 9. Size limits and runtime budgets + +PR #713 defines no fixed maximum length for keys, values, or blobs from the client. Its [size guidance](https://github.com/voidzero-dev/vite-task/blob/362f5bd91bb32806b512d3d9a5339ff435bf5f0a/docs/remote-cache-server-api.md#sizes) permits servers to impose resource limits and return `413`. The following configurable defaults apply to this deployment. They do not limit the protocol as a whole: + +| Resource | Initial limit | +| ------------------------------------------------------ | --------------------- | +| Each `key` or `secondary_key` | 16 KiB | +| Opaque `value` | 4 MiB | +| Store metadata part | 5 MiB | +| Fetch request body | 40 KiB | +| Blob | 64 MiB | +| Entire store request, including multipart overhead | 72 MiB | +| Internal R2 part buffer | 5 MiB | +| Upload lease | 15 minutes | +| Store request deadline / client blob-download deadline | 2 minutes / 5 minutes | + +Return `413` when a field or request exceeds its byte limit, including during streaming. Use `400` for malformed input. Do not truncate or transform opaque fields to make them fit. + +Publish the configured limits in the deployment guide. Keep field limits, envelope sizes, total request size, and measured runtime budgets consistent after changes. The client reports rejected stores as skipped remote publication and retains local results. + +Limit multipart headers and CBOR container depth before memory allocation. Reject ambiguous duplicate envelope fields. Support valid CBOR byte strings without a requirement for canonical encoding. Test streaming boundaries and request bodies of unknown length. Envelope limits do not authorize the Worker to decode the opaque `value`. + +Cloudflare's [request-body limits](https://developers.cloudflare.com/workers/platform/limits/#request-limits) depend on the Cloudflare account plan. Free and Pro allow 100 MB. Business allows 200 MB. Workers Paid alone does not raise a Free account's 100 MB body limit. The 72 MiB cap leaves space below that limit. Larger transfers require compatible field limits, request limits, account allowances, and measured Worker settings. + +Workers provides 128 MB per isolate, which concurrent requests share. Budget metadata, CBOR copies, stream buffers, and concurrency together. Streaming reduces memory use, but parsing costs still depend on the multipart input. Workers Free allows 10 ms CPU per HTTP or Cron invocation. + +Measure these operations before release: + +- Measure store costs with JWT signature verification and key loading. Test both cold and warm JWKS caches. +- Measure CBOR decoding and fetch encoding independently of blob size. Use 250 KB values and values up to the configured 4 MiB maximum. +- Measure stores at 5, 20, and 50 MB and at the configured maximum. Include concurrent uploads. + +The number of output files does not limit input metadata size. Release on Free only for sizes that pass these measurements. Lower the limits or select Workers Paid if the implementation cannot meet them. + +Keep no more than six external connections open. Upload R2 parts sequentially. Keep SQL batches and cleanup work within the Free plan's limits for each invocation. Average CPU estimates in the cost table do not prove that large stores fit Free. + +## 10. Retention, quotas, and cleanup + +By default, retain current entries for seven days after a successful store commit. A key replacement starts a new retention interval for the new generation. Fetch does not refresh retention. An association follows its target entry's lifetime. An association change does not shorten the old target's retention. + +Retain a replaced generation's value and blob for ten minutes after replacement. This grace period covers fetch and download sequences already in progress. During this period, the old blob ID continues to identify the old bytes. It must never return the replacement blob. + +The Free profile reserves 8 GB across live, pending, retired, and deleting objects. It also limits live entries and associations to 20,000 each. Reserve space for new associations and entries during publication. Updates to existing identities do not consume new slots. + +Warn at 400 MB of actual D1 storage. Maximum-sized keys and many associations can fill the database before it reaches the entry count limit. + +A Cron invocation runs every five minutes. It performs cleanup in this order: + +1. Claim a bounded batch of expired, retired, or abandoned generations in D1. +2. Delete their known R2 objects or abort their uploads. +3. Release the charged bytes after successful deletion. + +Use generation IDs and conditional state transitions for garbage collection (GC). GC must not remove a replacement entry or an association whose target was recreated. Remove associations with no target in bounded, indexed batches. Keep objects charged while deletion remains incomplete. + +An expired upload lease prevents publication. Allow an additional cleanup grace period for late R2 operations. Retry deletion until the generation is gone. + +Configure [R2 lifecycle rules](https://developers.cloudflare.com/r2/buckets/object-lifecycles/) as additional cleanup protection: + +- Abort unfinished multipart uploads after one day. +- For seven-day retention, expire generation objects after nine days. +- For 30-day retention, expire generation objects after 32 days. + +Lifecycle age starts at object creation. Its margin must cover upload leases and retirement grace. Lifecycle deletion is asynchronous. It does not replace D1 accounting or prompt cleanup. + +Start with at most 16 generations per Free Cron run and 256 per Paid run. These limits depend on measured CPU, query, and subrequest costs. The theoretical Free ceiling is 4,608 generations/day. Actual cleanup can be lower. Both overwritten and expired generations add to the backlog. + +Pause stores with `503` before cleanup delays threaten the byte budget. Normal cleanup does not need an R2 LIST for each entry. + +Apply resource limits to public reads before D1 or R2 work. Use a [Workers rate-limiting binding](https://developers.cloudflare.com/workers/runtime-apis/bindings/rate-limit/). Limit the set of keys for each namespace and operation. Apply stricter admission limits to unauthenticated stores. Return `429` with `Retry-After` when a request exceeds its rate limit. + +Use one catch-all key for unknown paths. This prevents callers from creating unlimited limiter identities. The binding provides approximate limits at each Cloudflare location. It does not provide a global billing cap. Rejected requests still invoke the Worker. + +Track anonymous misses and denied writes separately. Public traffic can exhaust the Free daily allowance even when storage fits. Keep controls to disable a deployment or namespace. Do not write D1 counters for each read. + +Application budgets keep ordinary storage growth within the configured allowance. They cannot guarantee a zero bill for arbitrary traffic, shared-account usage, failed uploads, or delayed lifecycle cleanup. Alert at 80% of provider allowances. Keep hard admission limits and spare capacity for cleanup. + +## 11. Failure handling and operations + +The client preserves local results if remote reads, validation, downloads, or uploads fail. `vp run` executes the task after a failed read. Upload failures produce warnings and a run-summary count without changing the task exit status. Reject invalid mode values or a missing required endpoint before execution. + +Use short metadata deadlines and bounded transfer deadlines. Support cancellation. Limit concurrency. After repeated failures, use a circuit breaker to stop remote attempts for the rest of the invocation. + +For authentication or size failures, report one diagnostic that explains the required action. Do not repeat the same failed attempt for every task. The client can retry transient read failures within its time budget. A retry after an uncertain store outcome follows the same replacement rules. + +Before archive extraction, the client must validate the remote blob's format, compatibility, integrity information, output paths, and decompression/file-count limits. Reject path traversal, absolute paths, unsafe links, and malformed archives. Restore into a staging area before terminal replay or promotion to local storage. The Worker stores opaque bytes and cannot perform these task-specific checks. + +Log the request ID, scope, operation, status, bytes, duration, and error class. For verified writes, also log repository ID, workflow ref, run ID/attempt, and commit SHA from signed claims. Reads have no authenticated identity. Do not add an identity API call. Do not log credentials or opaque request contents. + +Measure these operational values: + +- Exact, fallback, and not-found rates. +- Hits that pass client validation. +- Transferred bytes. +- D1 rows and latency. +- R2 operations. +- Pending bytes and cleanup delays. + +Sample successful Worker logs. Limit error logging. The service needs no central telemetry service or paid analytics. Client hit metrics must distinguish exact lookup from successful reuse. + +The Worker verifies GitHub tokens on writes and checks scope state in primary D1. Follow section 5's rules for policy changes and token expiry. Back up repository bindings and namespace policy separately from disposable cache data. + +D1 restoration does not restore deleted R2 objects. After partial recovery, reconcile references or create a new namespace. Use additive migrations. Document rollback compatibility. + +## 12. Self-deployment and GitHub Actions migration + +Deliver `packages/remote-cache` with these files and tools: + +- TypeScript sources. +- Pinned dependencies and a lockfile. +- `wrangler.jsonc`. +- D1 migrations. +- Protocol fixtures. +- An operator CLI and guide. + +Setup creates a private R2 Standard bucket and D1 database. It binds them as `ARTIFACTS` and `INDEX` and installs lifecycle and Cron settings. The maintainer supplies the public GitHub repository. Setup resolves the repository's IDs through the [GitHub repository API](https://docs.github.com/en/rest/repos/repos#get-a-repository). It stores the namespace's write policy and prints the public endpoint. Only the operator can create namespaces. + +Deploy to `workers.dev` or an optional custom domain. Every exposed route must permit public reads and enforce the same GitHub JWT policy for stores. Disable unused aliases. Require the operator's Cloudflare credentials for administration. + +Provide setup that can run repeatedly without duplicate resources. Support policy changes, write/scope disable, upgrades, and isolated smoke tests. Provide explicit teardown of stored data and Worker resources. Pin tested versions of the JWT library, tools, and runtime compatibility settings. Setup needs no cache secret in GitHub. + +Use the Free profile in section 10 by default. Change operational budgets for Paid only after the operator selects them. The operator must explicitly select more retention or R2 storage, independently of the Workers subscription. + +The following workflow excerpt shows the client flow. Keep the existing checkout, Vite+ setup, and dependency-installation steps. The repository can store the endpoint in `vite.config.*`. This example uses a non-secret repository variable as a protected CI override. The build saves local results and uploads newly generated eligible entries: + +```yaml +on: + push: + branches: [main] + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + env: + VP_REMOTE_CACHE_URL: ${{ vars.VP_REMOTE_CACHE_URL }} + VP_REMOTE_CACHE: read-write + steps: + # Existing checkout, Vite+ setup, and dependency installation steps. + - run: vp run build + working-directory: docs + env: + DOCS_SITE_ORIGIN: ${{ vars.DOCS_SITE_ORIGIN }} +``` + +The Worker checks signed repository, branch, and event claims. PR workflows use the default `read` mode with the same command and endpoint, and omit `id-token: write`. + +A workflow that handles both main-branch pushes and PRs can select the mode once for all run commands: + +```yaml +env: + VP_REMOTE_CACHE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'read-write' || 'read' }} +``` + +Preserve the existing [`DOCS_SITE_ORIGIN` input tracking](https://github.com/voidzero-dev/vite-plus/blob/ed1710f7aff8941436907a4d755d6b38c798ed41/docs/vite.config.ts). Keep the workflow's configured site-origin value. Keep dependency installation and package-manager caching. + +During a limited production trial, or canary, keep the existing task-directory restore/save steps within their trust boundary. Do not republish restored entries by default. Remove those steps after compatible clients pass anonymous reuse tests from fresh checkouts and explicit publication tests. + +To stop uploads while retaining remote reads, select `read` through `--remote-cache` or `VP_REMOTE_CACHE`, or disable server writes. To disable remote reads and uploads, select `off`. + +## 13. Free and Paid capacity comparison + +Prices below use USD before tax. We checked them on 2026-09-09. Estimates use a 30-day month and decimal MB/GB. Allowances assume that no other service uses the account. The workload examples illustrate costs. The frontend samples measure sizes and do not establish typical user traffic. + +### Provider allowances + +A Cloudflare account plan, Workers Free/Paid, and R2 billing are separate choices. The service can use `workers.dev` without a Pro website plan. Users must [enable R2](https://developers.cloudflare.com/r2/get-started/). R2 usage above its free allowance can incur charges while Workers remains Free. A Workers upgrade does not increase R2's free allowance. See Cloudflare's [billing model](https://developers.cloudflare.com/billing/understand/how-billing-works/). + +| Workers resource | Free | Paid Standard | +| -------------------- | ---------------- | --------------------------------------------------------------------------------------------------------------- | +| Subscription | $0 | $5/month minimum | +| Dynamic requests | 100,000/day | 10 million/month included; then $0.30/million | +| HTTP CPU | 10 ms/invocation | 30 million CPU ms/month included; then $0.02/million CPU ms; 30 s default per invocation, configurable to 5 min | +| Five-minute Cron CPU | 10 ms/invocation | 30 s/invocation | +| Memory | 128 MB/isolate | 128 MB/isolate | + +Sources: [Workers pricing](https://developers.cloudflare.com/workers/platform/pricing/) and [limits](https://developers.cloudflare.com/workers/platform/limits/). Storage and network wait time do not consume Worker CPU. The Free request allowance applies each day. CPU limits apply to each operation. + +| D1 resource | Free | Paid Standard | +| ----------------------------- | ----------------------------- | -------------------------------------------------- | +| Rows read | 5 million/day | 25 billion/month; then $0.001/million | +| Rows written | 100,000/day | 50 million/month; then $1/million | +| Storage | 5 GB/account; 500 MB/database | 5 GB included, then $0.75/GB-month; 10 GB/database | +| Queries per Worker invocation | 50 | 1,000 | + +Sources: [D1 pricing](https://developers.cloudflare.com/d1/platform/pricing/) and [limits](https://developers.cloudflare.com/d1/platform/limits/). Count index maintenance and deletion as writes. If the account exhausts a Free daily allowance, database work stops until that allowance resets. + +| R2 Standard resource | Included with either Workers plan | Overage | +| ------------------------------- | --------------------------------- | --------------- | +| Storage | 10 GB-month/month | $0.015/GB-month | +| Class A | 1 million/month | $4.50/million | +| Class B | 10 million/month | $0.36/million | +| Egress, DELETE, multipart abort | Free | Free | + +[R2 pricing](https://developers.cloudflare.com/r2/pricing/) counts multipart creation, each part, completion, and PUT as Class A operations. It counts GET and HEAD as Class B operations. Storage billing averages daily peaks. Billable units round up to whole GB-months and million-operation units. Include unfinished and retired objects in observed peaks. + +### Version 1 authorization cost + +Public reads require no identity subscription. GitHub issues tokens for publishing jobs. GitHub Actions compute billing is separate from this cache estimate. JWT checks run in the Worker. Bounded JWKS refreshes add subrequests and latency. Benchmark cold and warm signature verification with store parsing before a claim of Workers Free support. + +### Usage model + +Use these variables for daily traffic: + +- `L`: Public fetches after local misses. +- `F`: Fetches that return a value. +- `H`: Blob downloads after successful client validation. +- `P`: Entries successfully published through explicit CI pushes from the main branch, including overwrites. + +`P` counts entries, not command invocations. It is independent of developer misses. Exclude GitHub token requests from cache-Worker request counts. Include JWT verification and JWKS retrieval in measured Worker CPU and subrequest budgets. + +Use these variables for stored data: + +- `B`: Mean blob bytes. +- `V`: Mean value bytes. +- `S = (B + V) / 1,000,000`: Mean stored size in MB. +- `E`: Live exact keys. +- `R`: Retention days. + +Each store takes one Worker request. Each metadata fetch takes one request. The client makes a blob request only when it needs the blob. The Worker also reads the opaque value from R2. Internal R2 multipart upload changes storage operation counts but adds no client requests: + +```text +A per store = 1 # no blob: value PUT + = 2 # blob <= 5 MiB: value PUT + blob PUT + = 3 + ceil(B / 5 MiB) # larger: value PUT + create + parts + complete + +Worker invocations/day = ceil(1.10 * (L + H + P)) + 300 +R2 Class A/month = ceil(30 * 1.10 * P * A) +R2 Class B/month = ceil(30 * 1.10 * (F + H)) +Current R2 GB = E * S / 1000 +Total R2 GB = current + pending + retired + awaiting deletion +``` + +For mixed workloads, group stores by size or sum the operations for each store. `ceil(mean size)` can undercount multipart operations. The 10% operation reserve covers ordinary retries and maintenance. The 300 daily invocations cover 288 Cron runs and routine management. This reserve does not limit costs during outages or arbitrary traffic. + +R2 completion and PUT must succeed before publication. The plan adds no HEAD request for each object. + +If stores arrive steadily and each creates a different retained exact key, `E = P * R`. Current storage then equals `P * S * R / 1000` GB. Repeated stores to one key retain its current generation and a short retirement backlog. Input changes do not necessarily create a different exact key. + +Do not multiply all stores by seven days and report the result as actual storage usage. Overwrites still consume Worker, R2, D1, and cleanup operations. + +Use these D1 planning budgets: + +- 64 rows read per fetch/download cycle. +- 32 rows read per store. +- 40 rows written per store over its full lifecycle. + +Include scope lookups, indexes, accounting, association replacement, and cleanup. GitHub token verification needs no D1 credential table. Keep conservative row budgets for repository-policy checks until measurements support a reduction. For daily maintenance, add 10% plus 5,000 reads and 1,000 writes: + +```text +D1 reads/day = ceil(1.10 * (64 * L + 32 * P)) + 5000 +D1 writes/day = ceil(1.10 * 40 * P) + 1000 +D1 storage MB = E * 4096 / 1000000 # provisional, ordinary small keys +``` + +These budgets are estimates. They are not measured costs. The storage estimate assumes roughly one association and one current generation per key. Account separately for extra associations, pending and retired generations, and large keys. R2 part receipts do not require one D1 row per part. + +Check `rows_read`, `rows_written`, index plans, actual database bytes, and cleanup costs before a claim of these capacities. + +### Key and value size evidence + +The [PR #713 size example](https://github.com/voidzero-dev/vite-task/blob/362f5bd91bb32806b512d3d9a5339ff435bf5f0a/docs/remote-cache-server-api.md#sizes) reports a build tracking about 4,200 input paths and producing eight output files: + +| Field | Reported size | +| --------------- | -------------: | +| `key` | About 1 KB | +| `secondary_key` | About 50 bytes | +| `value` | About 250 KB | +| `blob` | About 250 KB | + +This example comes from the upstream API proposal. It is neither a measurement from our frontend study nor a population average. The client can record many input paths and hashes even when a task produces few output files. Here the value is as large as the blob. Budget and measure each separately. The Worker treats both as opaque bytes. + +For planning, interpret the approximate KB values as decimal. `V = 250,000` bytes and `B = 250,000` bytes give `S = 0.5 MB`. At 200 new distinct keys/day with seven-day retention, value and blob storage totals about 0.7 GB. This excludes staging and spare capacity for cleanup. Both objects remain in R2. + +D1's provisional 4 KiB estimate for each key covers index and generation records only. Validate it with the reported key sizes. Include repeated key bytes in indexes and associations. + +Every exact or fallback response transfers the value, even if client validation prevents a blob download. Before protocol overhead and retries, daily response payload equals `F * V + H * B` bytes. At the example's sizes, 1,000 fetches with values and 800 blob downloads transfer about 450 MB/day. This includes 250 MB of values. These transfers affect time, CBOR work, and memory. Request and R2 operation counts follow the same equations. + +### Artifact-size evidence + +Use 5 MB for a scenario with small outputs. On 2026-09-07, we measured published Vite frontend outputs from four established products. The [artifact study](remote-cache-size-study/README.md) records pinned sources, exact bytes, and a reproduction script. + +| Product/release | Output MB | `tar.zst` MB | `tar.zst` MB without source maps | +| ------------------------------- | --------: | -----------: | -------------------------------: | +| Directus `@directus/app@17.1.1` | 20.81 | 6.97 | 6.97 | +| Docmost `v0.95.0` | 14.83 | 4.77 | 4.77 | +| Hoppscotch `2026.8.0` | 127.53 | 32.18 | 12.18 | +| n8n `n8n-editor-ui@2.16.2` | 162.76 | 34.71 | 13.15 | + +We recompressed official npm/Docker frontend outputs with zstd level 3. We did not rebuild them locally or measure private cloud deployments. The measurements exclude backend tasks, dependencies, terminal events, and client validation metadata. Keep source maps when the task requires them. All four sampled archives fit the 64 MiB blob limit. + +Three samples exceed 5 MB. Use 50 MB as an additional planning case for complete results from mature frontends. This leaves room above the sampled archive sizes. It is neither a measured population average nor an upper bound. + +Measure one task at a time. A whole local-cache directory can contain several tasks and keys. During the canary, record these values: + +- Compressed blob bytes and value bytes. +- Store counts and overwrite counts. +- Live distinct keys and retention. +- Peak pending bytes. +- Task types. +- Lookup rates and hit rates after validation. + +Report mean, median, p95, maximum, and sample count by task type over at least two retention windows. Here, p95 means the 95th percentile. Measure the fraction of sampled deployments that stay free before a claim of support for most average users. + +### Public-read workloads with explicit CI publication + +In version 1, many developers can read a small set of results published from the main branch. The examples below use `H = 0.8 * L` and `F = L`. Each published entry contains 5 MB. Retention is seven days, and every store creates a distinct key. Publication counts are independent inputs: + +| Public-cache workload | Fetches/day | Published entries/day | Current R2 | Worker/day | D1 reads/day | D1 writes/day | Free monthly estimate | Paid monthly estimate | +| --------------------- | ----------: | --------------------: | ---------: | ---------: | -----------: | ------------: | --------------------: | --------------------: | +| Public project | 1,000 | 20 | 0.7 GB | 2,302 | 76,104 | 1,880 | $0 | $5.00 | +| High read traffic | 40,000 | 100 | 3.5 GB | 79,610 | 2,824,520 | 5,400 | $0 | $5.00 | + +Both examples fit the estimated request, row, storage, and R2-operation allowances. Each request must also meet the CPU limit, with spare capacity for operation. Over a 30-day month, the second case uses: + +- 2.3883 million Worker requests. +- 6,600 R2 Class A operations. +- 2.376 million R2 Class B operations. + +With the provisional average of 5 ms CPU per invocation, Workers Paid remains at its $5 base charge. The number of readers adds no identity subscription fees. These examples do not measure typical projects or guarantee costs under arbitrary public traffic. + +### Illustrative workloads and prices + +These scenarios compare storage and operation costs with `H = 0.8 * L`, `P = 0.2 * L`, and `F = L`. Only CI publications from the main branch count toward `P`. The ratio is an assumption for this model. Assume each store creates a distinct key. + +Set `V = 250 KB` (250,000 bytes), based on the upstream example. Include this value in `S`. Calculate multipart counts from the remaining blob bytes. An average of 5 ms CPU per invocation is an assumption for Paid costs. Free support requires measurements for each request. + +| Workload | Fetches/day | Stores/day | Mean size | Retention | Current R2 | Worker/day | D1 writes/day | +| -------------------- | ----------: | ---------: | --------: | --------: | ---------: | ---------: | ------------: | +| Individual | 100 | 20 | 1 MB | 7 days | 0.14 GB | 520 | 1,880 | +| Small team | 500 | 100 | 5 MB | 7 days | 3.5 GB | 1,400 | 5,400 | +| Active small team | 1,000 | 200 | 5 MB | 7 days | 7 GB | 2,500 | 9,800 | +| Longer retention | 1,000 | 200 | 5 MB | 30 days | 30 GB | 2,500 | 9,800 | +| Larger outputs | 1,000 | 200 | 20 MB | 7 days | 28 GB | 2,500 | 9,800 | +| Mature frontend case | 1,000 | 200 | 50 MB | 7 days | 70 GB | 2,500 | 9,800 | +| Busy team | 20,000 | 4,000 | 5 MB | 7 days | 140 GB | 44,300 | 177,000 | +| Large organization | 100,000 | 20,000 | 5 MB | 7 days | 700 GB | 220,300 | 881,000 | + +| Workload | Workers Free + R2 monthly estimate | Workers Paid + R2/D1 monthly estimate | +| ------------------------------------------- | --------------------------------------------------------------- | ------------------------------------: | +| Individual / small team / active small team | $0 within modeled allowances | $5.00 | +| Longer retention | $0.30; requires higher storage budget | $5.30 | +| Larger outputs | $0.27; requires higher storage budget | $5.27 | +| Mature frontend case | $0.90; requires higher storage budget and Free CPU verification | $5.90 | +| Busy team | Does not fit D1 Free writes | $6.95 | +| Large organization | Exceeds Free requests, writes, and single-database storage | $19.91 | + +The public cache has no subscription charge for each reader. These prices assume steady usage. They exclude pending/retired storage, other account usage, domain costs, GitHub Actions compute, and unusual or abusive traffic. Free support still requires CPU measurements, including JWT verification on stores. + +The default 8 GB profile rejects excess stores. It does not automatically increase the storage budget. Under this multipart model, the 20 MB case uses 46,200 R2 Class A operations each month. The 50 MB case uses 85,800. Both remain within the one-million monthly allowance. + +The large-organization example has these monthly costs: + +- 6.609 million Worker requests and 33.045 million CPU ms cost about $5.06. +- 700 GB of R2 storage costs $10.35. +- 1.32 million Class A operations cost $4.50 after the free allowance and unit rounding. +- 5.94 million Class B operations remain within the included allowance. + +The D1 estimate includes 232.47 million reads, 26.43 million writes, and about 573 MB of metadata for current keys. These values fit Paid allowances. The cache infrastructure subtotal is about $19.91/month. Bursts, unusually high CPU use, and D1 throughput still require load tests. Monthly allowances do not guarantee a request rate. + +### How much can remain free? + +For distinct keys, the 8 GB application budget gives these storage ceilings. They exclude pending bytes and spare capacity for cleanup: + +| Mean stored size | New distinct keys/day, 7 days | New distinct keys/day, 30 days | +| ---------------- | ----------------------------: | -----------------------------: | +| 1 MB | 1,142 | 266 | +| 5 MB | 228 | 53 | +| 20 MB | 57 | 13 | +| 50 MB | 22 | 5 | + +Calculate the ceiling with `floor(8000 / (S * R))`. Other limits can reduce it. If 200 daily stores repeatedly replace the same ten 50 MB keys, current storage totals about 0.5 GB. Add temporary old generations and staging to that total. If each store creates a different key, seven-day current storage reaches 70 GB. The client's actual key reuse matters as much as archive size. + +The stress comparison uses `P = 0.2 * L`, 80% blob downloads, and no storage constraint. Under these assumptions, Free Worker requests allow about 45,318 fetches/day. The provisional D1 write budget reduces this to 11,250 fetches/day, or about 8,977 at the 80% write alert threshold. Cleanup and CPU can reduce these numbers further. + +For read-only workloads, D1 reads and Worker requests determine the limits. Apply the equations with `P = 0`. + +The plan targets free operation with these choices: + +- Check the local cache before public remote reads. +- Use one metadata fetch. Download blobs only after validation. +- Publish explicitly from the main branch. +- Replace existing entries for repeated keys. +- Make no D1 writes during reads. +- Limit retention. Use private Standard R2. + +Read traffic can grow without more writers or user subscription fees. The examples fit free allowances only while traffic, storage, and CPU for each request remain within budget. Production measurements must support any claim about typical users. + +## 14. Version 2 and implementation questions + +Version 2 can add private projects through Cloudflare One: + +- Access policies protect reads and writes. +- Service tokens support automation that cannot use GitHub OIDC. +- Managed OAuth provides developer login. + +Keep namespace isolation and explicit publication. Separate private namespaces or deployments from public version 1 endpoints. A private authorization failure must never permit public access. + +For version 2, review [Workers Access integration](https://developers.cloudflare.com/workers/configuration/cloudflare-access/), [Managed OAuth](https://developers.cloudflare.com/cloudflare-one/access-controls/applications/http-apps/managed-oauth/), revocation behavior, and [Access pricing](https://www.cloudflare.com/sase/products/access/). None adds a dependency or cost to version 1. + +Resolve these questions during implementation: + +- Upload concurrency, shutdown deadlines, and protection of archives while uploads are pending. +- Portable client encoding and compatibility. +- JWT time tolerances and limits for cached keys. +- Measured Free CPU and D1 costs. +- Representative rates for public traffic and publication. + +Check compatibility with the merged version of PR #713 before release. diff --git a/packages/remote-cache/docs/images/repository-binding-form.svg b/packages/remote-cache/docs/images/repository-binding-form.svg new file mode 100644 index 000000000..7b21a7fd2 --- /dev/null +++ b/packages/remote-cache/docs/images/repository-binding-form.svg @@ -0,0 +1,47 @@ + + Bind a public GitHub repository + The maintainer enters a GitHub repository, namespace, and main branch. Setup resolves immutable repository and owner IDs, checks public visibility, and produces the public endpoint and OIDC audience. Reads are public. Writes require an authorized main-branch push job from this repository. + + + + Bind a public GitHub repository + Setup form schematic · example values + + MAINTAINER INPUTS + AUTOMATIC VALUES · READ-ONLY + GitHub repository + + your-org/your-project + Public repositories on GitHub.com + Cache namespace + + docs-trusted-v1 + Main branch + + main + Default: main · saved as refs/heads/main + Repository and owner IDs + + Resolved from GitHub + repository_id + repository_owner_id + Required visibility + + Public + Read access + Anyone · no credentials + Write access + This repository · push jobs on main + + Public cache endpoint + https://cache.example.com/projects/docs-trusted-v1 + OIDC audience: this exact URL, without a trailing slash + + Save binding + No developer registration or cache secret + diff --git a/packages/remote-cache/docs/remote-cache-size-study/README.md b/packages/remote-cache/docs/remote-cache-size-study/README.md new file mode 100644 index 000000000..44afada6a --- /dev/null +++ b/packages/remote-cache/docs/remote-cache-size-study/README.md @@ -0,0 +1,107 @@ +# Vite SaaS frontend artifact measurements + +Measured on 2026-09-07 for [the remote-cache RFC](../0001-remote-cache.md#13-free-and-paid-capacity-comparison). + +The four sampled frontend outputs compress to **4.77–34.71 MB** with zstd level 3. A 5 MB result is useful for a scenario with small outputs. Three of these four releases exceed it. Use an additional 50 MB planning case for mature SaaS frontends. This sample does not establish the average cache size across users or tasks. + +## Projects and scope + +We selected established products with public source and downloadable release artifacts. Each repository had more than 20,000 GitHub stars when we checked it. This method selects recognizable projects. It does not produce a random sample of Vite users. The sample includes commercial products with different source licenses. + +| Project | Product | GitHub stars at observation | Measured release | Published frontend output | +| ------------------------------------------------------ | ---------------------------- | --------------------------: | -------------------------------------- | -------------------------------------------------- | +| [Directus](https://github.com/directus/directus) | Data platform / headless CMS | 37,786 | `@directus/app@17.1.1`, from `v12.3.1` | npm package `dist/` | +| [Docmost](https://github.com/docmost/docmost) | Collaborative wiki | 21,602 | `v0.95.0` | Docker build's `/app/apps/client/dist/` COPY layer | +| [Hoppscotch](https://github.com/hoppscotch/hoppscotch) | API development platform | 80,228 | `2026.8.0` | Docker build's `/site/selfhost-web/` COPY layer | +| [n8n](https://github.com/n8n-io/n8n) | Workflow automation | 203,576 | `n8n-editor-ui@2.16.2` | npm package `dist/` | + +These measurements cover **published build products**. We did not run local builds or measure the vendors' private cloud deployments. Docker samples use `linux/amd64` manifests. We selected the layer that copies the frontend from the build stage. This layer precedes runtime dependency installation and startup transformations. + +We exclude the Docker base image, server code, dependencies, and unrelated npm package files. + +The pinned source confirms Vite usage: + +| Project | Build evidence | Output/configuration evidence | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Directus | [`build: vite build`](https://github.com/directus/directus/blob/973be10df8b0305569dc0dc53e187c648133c8d6/app/package.json) | [Vite configuration](https://github.com/directus/directus/blob/973be10df8b0305569dc0dc53e187c648133c8d6/app/vite.config.js) | +| Docmost | [`build: tsc && vite build`](https://github.com/docmost/docmost/blob/4132dd597c956a27423607d008708c0e214690da/apps/client/package.json) | [Vite configuration](https://github.com/docmost/docmost/blob/4132dd597c956a27423607d008708c0e214690da/apps/client/vite.config.ts), [Dockerfile COPY](https://github.com/docmost/docmost/blob/4132dd597c956a27423607d008708c0e214690da/Dockerfile) | +| Hoppscotch | [`generate` calls `build`, which invokes Vite](https://github.com/hoppscotch/hoppscotch/blob/ac145e7f758151b41fd46d3e5f513886ce9068ba/packages/hoppscotch-selfhost-web/package.json) | [Vite configuration](https://github.com/hoppscotch/hoppscotch/blob/ac145e7f758151b41fd46d3e5f513886ce9068ba/packages/hoppscotch-selfhost-web/vite.config.ts), [production Dockerfile](https://github.com/hoppscotch/hoppscotch/blob/ac145e7f758151b41fd46d3e5f513886ce9068ba/prod.Dockerfile) | +| n8n | [`build` invokes `vite build`](https://github.com/n8n-io/n8n/blob/9bdde69954a4d7d1569d37b6fd3a3f55f55b297a/packages/frontend/editor-ui/package.json) | [Vite configuration](https://github.com/n8n-io/n8n/blob/9bdde69954a4d7d1569d37b6fd3a3f55f55b297a/packages/frontend/editor-ui/vite.config.mts) | + +[sources.json](sources.json) records artifact URLs, SHA-256 digests, source commits, npm provenance URLs, and Docker manifest/layer digests. We checked npm downloads against the registry's SHA-512 integrity values. We checked Docker layers against their SHA-256 digests. + +We obtained npm source commits from the registry's provenance records. The Docker source references identify the corresponding release tags. We did not verify attestation signatures or independently rebuild the releases. + +## Measured sizes + +All sizes use decimal MB (`1 MB = 1,000,000 bytes`). “Output” sums regular-file bytes, including static assets and source maps in the selected directory. “Archive” measures one sorted GNU tar stream compressed with `zstd -3 -T1`, using zstd `1.5.7`. We normalize tar paths and metadata. Archive sizes estimate the compressed output payload. A complete cache entry also includes task metadata. + +| Project | Files | Output MB | Archive MB | Source-map MB before compression | Archive MB with `.map` files omitted | +| ---------- | ----: | --------: | ---------: | -------------------------------: | -----------------------------------: | +| Directus | 401 | 20.81 | **6.97** | 0 | 6.97 | +| Docmost | 230 | 14.83 | **4.77** | 0 | 4.77 | +| Hoppscotch | 471 | 127.53 | **32.18** | 89.82 | 12.18 | +| n8n | 1,477 | 162.76 | **34.71** | 118.66 | 13.15 | + +Source maps account for about 70% of Hoppscotch's uncompressed output and 73% of n8n's. Their configurations enable source maps for these builds. n8n also enables its legacy-browser plugin for releases. Hoppscotch includes a TypeScript worker. n8n includes worker and WebAssembly assets. + +A page's initial JavaScript download omits much of this build output. Its size cannot substitute for the cache size. + +The last column measures the same files without source maps. It does not represent another build. Without maps, Hoppscotch's compressed archive measures 12.18 MB and n8n's measures 13.15 MB. Both remain above 5 MB. + +A cache must preserve the outputs that its task requires. These measurements do not justify silent removal of source maps. The published frontend directories for Directus and Docmost contain no `.map` files. + +These archives contain frontend output files only. A complete cached task also needs terminal events and client validation metadata. Under [PR #713](https://github.com/voidzero-dev/vite-task/pull/713), the client encodes these in an opaque value and optional blob. The Worker does not define or interpret their internal format. + +A cached task can produce additional files outside `dist/`. Release packaging can omit these files. Measure their bytes during implementation before you assign a complete size to each result. Backend and shared-package builds are separate task results unless the operator caches them as one task. + +## Effect on the free storage budget + +The table uses the RFC's 8 GB application budget and seven-day retention. Each store creates a different exact key at a steady rate. The ceilings include only the measured output archives: + +| Project-sized result | New results/day within 8 GB | Storage at 200 new results/day, seven days | +| -------------------- | --------------------------: | -----------------------------------------: | +| Directus | 164 | 9.75 GB | +| Docmost | 239 | 6.68 GB | +| Hoppscotch | 35 | 45.06 GB | +| n8n | 32 | 48.59 GB | + +Calculate the ceiling as `floor(8,000,000,000 / (archive_bytes * 7))`. These upper bounds include only artifacts. Client metadata, logs, pending uploads, retired generations, and delayed deletion reduce them. + +A new exact key stores another complete archive, even when many assets match the previous build. Repeated stores to the same key replace its value and blob. The old generation remains only for a short download grace period. Input changes do not necessarily create a different exact key. Read hits do not create another copy. Count separate namespaces and client compatibility identities when the service stores separate results. + +For a **50 MB planning case for complete results**, the same budget supports at most **22 new distinct keys/day** with seven-day retention. This excludes spare capacity for operation. At 200 new distinct keys/day, storage reaches 70 GB. + +Under the RFC's operation assumptions, this higher budget costs about **$0.90/month in R2 storage** beyond the included 10 GB. Workers can remain Free if the streaming implementation meets its CPU limit. The estimate uses steady usage, a 30-day month, and [R2 Standard pricing](https://developers.cloudflare.com/r2/pricing/), which we rechecked on 2026-09-09. The default profile rejects additional stores instead of increasing its budget. + +Use 5 MB for a scenario with small outputs and 50 MB for mature frontend builds. All four measured archives fit the RFC's 64 MiB blob limit. These values are planning inputs. They do not estimate population averages. + +A few new keys for complete frontend outputs each day can fit the free budget. Hundreds of different keys each day require smaller results, shorter retention, or more storage. Frequent overwrites can retain much less data but still consume operations. + +The number of readers does not determine this storage case. In version 1, only explicit CI publication from the main branch adds remote results. Public readers need no credentials or subscription for each user. + +This sample covers one release per product and favors mature applications. It does not measure these values: + +- Average size weighted by store counts. +- Daily change rate. +- Exact-key replacement rate. +- Cache hit rate. +- The fraction of ordinary users that stay free. + +The RFC's limited production trial still needs these measurements across task types and successive changes. + +## Reproduce + +Run from the repository root with Python 3 and the `zstd` CLI installed: + +```sh +python3 packages/remote-cache/docs/remote-cache-size-study/measure.py \ + --cache-dir /tmp/vite-cache-size-study \ + --output /tmp/vite-cache-size-study-results.json +``` + +The script downloads about 84 MB of pinned npm archives and Docker layers. It checks SHA-256 digests and decompresses the downloads into temporary tar files for inspection. It does not install packages, execute project code, or start containers. It writes compressed comparison archives in the cache directory. + +Allow about 600 MB of disk space. Registry availability and anonymous Docker pull limits can affect repeated runs. + +[results.json](results.json) contains exact byte counts, compressed archive digests, file-type totals, and the five largest files for each sample. Use those counts for calculations. The tables round values for readability. diff --git a/packages/remote-cache/docs/remote-cache-size-study/measure.py b/packages/remote-cache/docs/remote-cache-size-study/measure.py new file mode 100644 index 000000000..795e7da43 --- /dev/null +++ b/packages/remote-cache/docs/remote-cache-size-study/measure.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Measure pinned, published Vite outputs without installing or running projects. + +Requires Python 3 and the zstd CLI. Usage: + python3 measure.py --cache-dir /tmp/vite-cache-size-study --output results.json +""" + +import argparse +import collections +import gzip +import hashlib +import json +from pathlib import Path, PurePosixPath +import re +import shutil +import subprocess +import tarfile +import urllib.parse +import urllib.request + + +def read_url(url, headers=None): + return urllib.request.urlopen( + urllib.request.Request( + url, headers={"User-Agent": "vite-task-cache-size-study", **(headers or {})} + ), + timeout=60, + ) + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def download(sample, cache): + target = cache / (sample["id"] + ".tgz") + if not target.exists() or sha256(target) != sample["sha256"]: + headers = {} + if sample.get("docker_repository"): + query = urllib.parse.urlencode( + { + "service": "registry.docker.io", + "scope": "repository:" + sample["docker_repository"] + ":pull", + } + ) + with read_url("https://auth.docker.io/token?" + query) as response: + token = json.load(response)["token"] + headers["Authorization"] = "Bearer " + token + partial = target.with_suffix(".partial") + with read_url(sample["url"], headers) as response, partial.open("wb") as output: + shutil.copyfileobj(response, output) + if sha256(partial) != sample["sha256"]: + raise ValueError("Source digest mismatch: " + sample["id"]) + partial.replace(target) + return target + + +def compress(source, members, destination): + # Match the engine's ordinary zstd level (3). Normalize GNU tar headers, + # sort paths, and include regular files only. No filesystem extraction. + with destination.open("wb") as output: + process = subprocess.Popen( + ["zstd", "-3", "-T1", "-q", "-c"], stdin=subprocess.PIPE, stdout=output + ) + try: + with tarfile.open(fileobj=process.stdin, mode="w|", format=tarfile.GNU_FORMAT) as archive: + for name, member in members: + header = tarfile.TarInfo("outputs/" + name) + header.size = member.size + header.mode = 0o644 + with source.extractfile(member) as data: + archive.addfile(header, data) + finally: + process.stdin.close() + status = process.wait() + if status: + raise RuntimeError("zstd failed: " + str(status)) + return {"bytes": destination.stat().st_size, "sha256": sha256(destination)} + + +def measure(sample, cache): + packed = download(sample, cache) + unpacked = cache / (sample["id"] + ".tar") + # Decompress once so sorted reads do not repeatedly rewind a gzip stream. + with gzip.open(packed, "rb") as source, unpacked.open("wb") as output: + shutil.copyfileobj(source, output) + + with tarfile.open(unpacked) as source: + prefix = sample["prefix"] + members = [] + for member in source: + if not member.name.startswith(prefix): + continue + name = member.name[len(prefix):] + if member.isdir(): + continue + if not member.isfile() or ".." in PurePosixPath(name).parts or name.startswith("/"): + raise ValueError("Unsupported output member: " + member.name) + members.append((name, member)) + members.sort(key=lambda item: item[0]) + if not members or len({name for name, _ in members}) != len(members): + raise ValueError("Empty or duplicate output paths: " + sample["id"]) + + by_extension = collections.defaultdict(lambda: {"files": 0, "bytes": 0}) + for name, member in members: + group = by_extension[PurePosixPath(name).suffix or "(none)"] + group["files"] += 1 + group["bytes"] += member.size + without_maps = [(name, member) for name, member in members if not name.endswith(".map")] + complete = compress(source, members, cache / (sample["id"] + ".tar.zst")) + no_maps = ( + compress(source, without_maps, cache / (sample["id"] + "-no-maps.tar.zst")) + if len(without_maps) != len(members) + else complete.copy() + ) + return { + "id": sample["id"], + "source_sha256": sample["sha256"], + "source_download_bytes": packed.stat().st_size, + "prefix": prefix, + "files": len(members), + "output_bytes": sum(member.size for _, member in members), + "tar_zstd": complete, + "without_maps": { + "files": len(without_maps), + "output_bytes": sum(member.size for _, member in without_maps), + "tar_zstd": no_maps, + }, + "by_extension": dict(sorted(by_extension.items())), + "largest_files": [ + {"path": name, "bytes": member.size} + for name, member in sorted(members, key=lambda item: item[1].size, reverse=True)[:5] + ], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cache-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + samples = json.loads(Path(__file__).with_name("sources.json").read_text()) + args.cache_dir.mkdir(parents=True, exist_ok=True) + results = [] + for sample in samples["samples"]: + result = measure(sample, args.cache_dir) + results.append(result) + print( + sample["id"], + "files=" + str(result["files"]), + "output_bytes=" + str(result["output_bytes"]), + "zstd_bytes=" + str(result["tar_zstd"]["bytes"]), + "without_maps_zstd_bytes=" + str(result["without_maps"]["tar_zstd"]["bytes"]), + flush=True, + ) + version = subprocess.check_output(["zstd", "--version"], text=True) + args.output.write_text( + json.dumps( + { + "measurement_date": samples["measurement_date"], + "zstd_version": re.search(r"\bv(\d+\.\d+\.\d+)\b", version).group(1), + "method": "Sorted GNU tar regular files; outputs/ prefix; mode 0644; uid/gid/mtime 0; zstd -3 -T1 stream", + "samples": results, + }, + indent=2, + ) + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/packages/remote-cache/docs/remote-cache-size-study/results.json b/packages/remote-cache/docs/remote-cache-size-study/results.json new file mode 100644 index 000000000..93fa5f257 --- /dev/null +++ b/packages/remote-cache/docs/remote-cache-size-study/results.json @@ -0,0 +1,355 @@ +{ + "measurement_date": "2026-09-07", + "zstd_version": "1.5.7", + "method": "Sorted GNU tar regular files; outputs/ prefix; mode 0644; uid/gid/mtime 0; zstd -3 -T1 stream", + "samples": [ + { + "id": "directus", + "source_sha256": "aeb56c7f70c6ba09782a5b649d0ab2cdf9cd746ab6dcaaa80bb3db9490f64f94", + "source_download_bytes": 7521159, + "prefix": "package/dist/", + "files": 401, + "output_bytes": 20806766, + "tar_zstd": { + "bytes": 6967198, + "sha256": "bf7b16909bfb430f6c31605e2026b5ae0b6f458040943fcdf9b3a60178cd51ae" + }, + "without_maps": { + "files": 401, + "output_bytes": 20806766, + "tar_zstd": { + "bytes": 6967198, + "sha256": "bf7b16909bfb430f6c31605e2026b5ae0b6f458040943fcdf9b3a60178cd51ae" + } + }, + "by_extension": { + ".css": { + "files": 5, + "bytes": 719721 + }, + ".html": { + "files": 1, + "bytes": 1694 + }, + ".ico": { + "files": 1, + "bytes": 1150 + }, + ".js": { + "files": 369, + "bytes": 17819785 + }, + ".png": { + "files": 1, + "bytes": 14915 + }, + ".svg": { + "files": 1, + "bytes": 155481 + }, + ".woff": { + "files": 11, + "bytes": 972248 + }, + ".woff2": { + "files": 12, + "bytes": 1121772 + } + }, + "largest_files": [ + { + "path": "assets/index.C9zFwJTK.entry.js", + "bytes": 6724943 + }, + { + "path": "assets/shader-background-B-taT_fk.js", + "bytes": 1449199 + }, + { + "path": "assets/index-BMHQmbsG.css", + "bytes": 699610 + }, + { + "path": "assets/dist-chJWdeb_.js", + "bytes": 657240 + }, + { + "path": "assets/social-icon-Dxox5INO.js", + "bytes": 570996 + } + ] + }, + { + "id": "docmost", + "source_sha256": "4e7fffb4a1e8feb4e7213d2e8701c09df286de364177860e1c230e2374a25190", + "source_download_bytes": 5015721, + "prefix": "app/apps/client/dist/", + "files": 230, + "output_bytes": 14828770, + "tar_zstd": { + "bytes": 4769432, + "sha256": "cad27b93f705537b0abf7cfe7cc47dbb416c6e2120bd91dfce7316cc54a00157" + }, + "without_maps": { + "files": 230, + "output_bytes": 14828770, + "tar_zstd": { + "bytes": 4769432, + "sha256": "cad27b93f705537b0abf7cfe7cc47dbb416c6e2120bd91dfce7316cc54a00157" + } + }, + "by_extension": { + ".css": { + "files": 3, + "bytes": 660300 + }, + ".html": { + "files": 1, + "bytes": 1504 + }, + ".js": { + "files": 145, + "bytes": 11997158 + }, + ".json": { + "files": 13, + "bytes": 994411 + }, + ".png": { + "files": 4, + "bytes": 19808 + }, + ".svg": { + "files": 1, + "bytes": 1497 + }, + ".ttf": { + "files": 20, + "bytes": 513664 + }, + ".woff": { + "files": 20, + "bytes": 303116 + }, + ".woff2": { + "files": 23, + "bytes": 337312 + } + }, + "largest_files": [ + { + "path": "assets/index-BgC9hh6t.js", + "bytes": 3469981 + }, + { + "path": "assets/chunk-Z5NKEFVG-B3tsbN9K.js", + "bytes": 1821041 + }, + { + "path": "assets/excalidraw-utils-DacaKd1Y.js", + "bytes": 1350729 + }, + { + "path": "assets/chunk-NNHCCRGN-DlpIbxXb.js", + "bytes": 593667 + }, + { + "path": "assets/chunk-LZXEDZCA-4GiMzlKw.js", + "bytes": 536756 + } + ] + }, + { + "id": "hoppscotch", + "source_sha256": "9823634d487dac47140c008685e31f548e33644ba890c8bba66ba8b776aea98e", + "source_download_bytes": 33455805, + "prefix": "site/selfhost-web/", + "files": 471, + "output_bytes": 127526484, + "tar_zstd": { + "bytes": 32182975, + "sha256": "958a98d1aba6d70f315d593c6549898c803670d995513fb242d3958405bad2e0" + }, + "without_maps": { + "files": 279, + "output_bytes": 37709018, + "tar_zstd": { + "bytes": 12175754, + "sha256": "ed31d530071727007670fbd42e1009091cea063f0df6dc44ad0b82e7a8740c0c" + } + }, + "by_extension": { + "(none)": { + "files": 1, + "bytes": 13 + }, + ".css": { + "files": 13, + "bytes": 305947 + }, + ".html": { + "files": 1, + "bytes": 5219 + }, + ".ico": { + "files": 1, + "bytes": 15086 + }, + ".js": { + "files": 192, + "bytes": 33464992 + }, + ".map": { + "files": 192, + "bytes": 89817466 + }, + ".png": { + "files": 12, + "bytes": 2105842 + }, + ".svg": { + "files": 41, + "bytes": 372789 + }, + ".ttf": { + "files": 1, + "bytes": 121972 + }, + ".txt": { + "files": 1, + "bytes": 66 + }, + ".webmanifest": { + "files": 1, + "bytes": 845 + }, + ".woff2": { + "files": 14, + "bytes": 1313348 + }, + ".xml": { + "files": 1, + "bytes": 2899 + } + }, + "largest_files": [ + { + "path": "assets/index-DSckl1TP.js.map", + "bytes": 37094987 + }, + { + "path": "assets/ts.worker-BAgXcSxE.js.map", + "bytes": 18859801 + }, + { + "path": "assets/index-DSckl1TP.js", + "bytes": 12071781 + }, + { + "path": "assets/har-DRQ_daXH.js.map", + "bytes": 8016993 + }, + { + "path": "assets/ts.worker-BAgXcSxE.js", + "bytes": 7091224 + } + ] + }, + { + "id": "n8n", + "source_sha256": "14636be99659fca25eaa19ed20ec52c5286745eb3e1cef80b699adb23144eb1f", + "source_download_bytes": 37936612, + "prefix": "package/dist/", + "files": 1477, + "output_bytes": 162756849, + "tar_zstd": { + "bytes": 34705298, + "sha256": "7c916ddcdbb090123824840e55669de6500e294d9d419a6cb91cba5c7dc77cac" + }, + "without_maps": { + "files": 875, + "output_bytes": 44092939, + "tar_zstd": { + "bytes": 13146351, + "sha256": "4c1473f0d932be64dd7ee412b7ef1aa9ec0921a02010dc82a61bda7e9ebef21f" + } + }, + "by_extension": { + ".css": { + "files": 149, + "bytes": 1098799 + }, + ".gif": { + "files": 1, + "bytes": 158971 + }, + ".html": { + "files": 1, + "bytes": 18822 + }, + ".ico": { + "files": 1, + "bytes": 15086 + }, + ".js": { + "files": 626, + "bytes": 35334179 + }, + ".map": { + "files": 602, + "bytes": 118663910 + }, + ".png": { + "files": 24, + "bytes": 820355 + }, + ".svg": { + "files": 3, + "bytes": 367325 + }, + ".ttf": { + "files": 21, + "bytes": 569620 + }, + ".wasm": { + "files": 5, + "bytes": 3980250 + }, + ".webp": { + "files": 1, + "bytes": 315064 + }, + ".woff": { + "files": 21, + "bytes": 331316 + }, + ".woff2": { + "files": 22, + "bytes": 1083152 + } + }, + "largest_files": [ + { + "path": "assets/typescript.worker-C3GNzKj8.js.map", + "bytes": 18592487 + }, + { + "path": "assets/constants-j6z_fOkc.js.map", + "bytes": 5567385 + }, + { + "path": "assets/worker-FLvH_Wit.js.map", + "bytes": 5457892 + }, + { + "path": "assets/constants-legacy-i87_3HzL.js.map", + "bytes": 5432841 + }, + { + "path": "assets/typescript.worker-C3GNzKj8.js", + "bytes": 5049634 + } + ] + } + ] +} diff --git a/packages/remote-cache/docs/remote-cache-size-study/sources.json b/packages/remote-cache/docs/remote-cache-size-study/sources.json new file mode 100644 index 000000000..02cc4c162 --- /dev/null +++ b/packages/remote-cache/docs/remote-cache-size-study/sources.json @@ -0,0 +1,71 @@ +{ + "measurement_date": "2026-09-07", + "samples": [ + { + "id": "directus", + "project": "Directus", + "version": "@directus/app 17.1.1 (Directus v12.3.1)", + "url": "https://registry.npmjs.org/@directus/app/-/app-17.1.1.tgz", + "sha256": "aeb56c7f70c6ba09782a5b649d0ab2cdf9cd746ab6dcaaa80bb3db9490f64f94", + "prefix": "package/dist/", + "repository": "https://github.com/directus/directus", + "source_commit": "973be10df8b0305569dc0dc53e187c648133c8d6", + "vite_config": "app/vite.config.js", + "build_script": "app/package.json", + "provenance": "https://registry.npmjs.org/-/npm/v1/attestations/@directus%2fapp@17.1.1", + "stars": 37786 + }, + { + "id": "docmost", + "project": "Docmost", + "version": "v0.95.0", + "url": "https://registry-1.docker.io/v2/docmost/docmost/blobs/sha256:4e7fffb4a1e8feb4e7213d2e8701c09df286de364177860e1c230e2374a25190", + "sha256": "4e7fffb4a1e8feb4e7213d2e8701c09df286de364177860e1c230e2374a25190", + "prefix": "app/apps/client/dist/", + "repository": "https://github.com/docmost/docmost", + "source_commit": "4132dd597c956a27423607d008708c0e214690da", + "vite_config": "apps/client/vite.config.ts", + "build_script": "apps/client/package.json", + "docker_repository": "docmost/docmost", + "image": "docmost/docmost:0.95.0", + "image_manifest": "sha256:2b0a3f73e57951b726bf67259c4e8cb5269eb24530cde521003ff382d7ad8ab6", + "image_platform": "linux/amd64", + "layer_index": 9, + "dockerfile": "Dockerfile", + "stars": 21602 + }, + { + "id": "hoppscotch", + "project": "Hoppscotch", + "version": "2026.8.0", + "url": "https://registry-1.docker.io/v2/hoppscotch/hoppscotch-frontend/blobs/sha256:9823634d487dac47140c008685e31f548e33644ba890c8bba66ba8b776aea98e", + "sha256": "9823634d487dac47140c008685e31f548e33644ba890c8bba66ba8b776aea98e", + "prefix": "site/selfhost-web/", + "repository": "https://github.com/hoppscotch/hoppscotch", + "source_commit": "ac145e7f758151b41fd46d3e5f513886ce9068ba", + "vite_config": "packages/hoppscotch-selfhost-web/vite.config.ts", + "build_script": "packages/hoppscotch-selfhost-web/package.json", + "docker_repository": "hoppscotch/hoppscotch-frontend", + "image": "hoppscotch/hoppscotch-frontend:2026.8.0", + "image_manifest": "sha256:c5acdcfa5e00d3500ff809a33d25b10d36eb2fe2b8ef23e0fd74a157c6abc362", + "image_platform": "linux/amd64", + "layer_index": 18, + "dockerfile": "prod.Dockerfile", + "stars": 80228 + }, + { + "id": "n8n", + "project": "n8n", + "version": "n8n-editor-ui 2.16.2", + "url": "https://registry.npmjs.org/n8n-editor-ui/-/n8n-editor-ui-2.16.2.tgz", + "sha256": "14636be99659fca25eaa19ed20ec52c5286745eb3e1cef80b699adb23144eb1f", + "prefix": "package/dist/", + "repository": "https://github.com/n8n-io/n8n", + "source_commit": "9bdde69954a4d7d1569d37b6fd3a3f55f55b297a", + "vite_config": "packages/frontend/editor-ui/vite.config.mts", + "build_script": "packages/frontend/editor-ui/package.json", + "provenance": "https://registry.npmjs.org/-/npm/v1/attestations/n8n-editor-ui@2.16.2", + "stars": 203576 + } + ] +} From d21e04617e40d6f1ec5ca3e9338c91f9e4553b6e Mon Sep 17 00:00:00 2001 From: MK Date: Sat, 12 Sep 2026 11:40:43 +0800 Subject: [PATCH 07/17] docs(remote-cache): use the local RFC status contract Co-authored-by: GPT-6 Codex --- packages/remote-cache/README.md | 2 +- packages/remote-cache/docs/e2e-plan.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/remote-cache/README.md b/packages/remote-cache/README.md index 03425c96d..264788032 100644 --- a/packages/remote-cache/README.md +++ b/packages/remote-cache/README.md @@ -20,7 +20,7 @@ Empty and non-UTF-8 keys work. CBOR definite and indefinite maps and byte string Fetch gives exact matches priority. A fallback follows the latest secondary-key association. A store replaces both mappings atomically. Reassigning a secondary key does not delete its former target. Replacing an entry changes the value seen through every association to that entry. -**Draft contract difference:** this implementation follows RFC #716 and returns `404` for a fetch miss. The still-open API draft [#713 at `362f5bd9`](https://github.com/voidzero-dev/vite-task/blob/362f5bd91bb32806b512d3d9a5339ff435bf5f0a/docs/remote-cache-server-api.md) instead specifies HTTP `200` with `kind: "not_found"`. Reconcile the drafts before releasing a client against this service. +HTTP status codes follow the [local RFC](docs/0001-remote-cache.md#4-http-api-mapping). A fetch returns `404` when neither key resolves to a live entry. An unavailable blob also returns `404`, including when its R2 object is missing. These responses use plain text. A missing or unreadable value for a live entry is a storage failure and returns `503`, as specified in [the read design](docs/0001-remote-cache.md#7-fetch-and-download-implementation). Errors have `Content-Type: text/plain; charset=utf-8`. Codes are `400` for invalid input, `401` for invalid/missing/expired tokens, `403` for a signed token that fails write policy, `404` for absent data or unavailable namespaces/routes, `413` for size limits, `429` for admission limits, `500` for an incomplete operation, and `503` for unavailable authorization/storage, failed publication guards, quotas, or concurrency admission. `429` and `503` include `Retry-After: 60`. diff --git a/packages/remote-cache/docs/e2e-plan.md b/packages/remote-cache/docs/e2e-plan.md index 3792c08c0..d67763815 100644 --- a/packages/remote-cache/docs/e2e-plan.md +++ b/packages/remote-cache/docs/e2e-plan.md @@ -62,6 +62,8 @@ The operator seeds immutable objects and generation rows for read tests. This va “Local” refers to the existing regression suite and the new shared e2e-driver tests. “PR” and “Main” refer to real Cloudflare runs. +Status assertions follow the [local RFC](0001-remote-cache.md#4-http-api-mapping). Fetch misses and unavailable blobs return plain-text `404`. A missing or unreadable value for a live entry returns `503`. + | Case | Local | PR | Main | Required result | | ------------------------------------------------ | ----------------------------- | ----------------------------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | | Exact source revision | Yes | Yes | Yes | Expected deployment ID on responses; fail if the URL serves another revision | From 54c4870b9df3736741b04a5473253585d5be4517 Mon Sep 17 00:00:00 2001 From: MK Date: Mon, 14 Sep 2026 23:01:09 +0800 Subject: [PATCH 08/17] fix(remote-cache): align RFC responses and isolate deployments Co-authored-by: GPT-6 Codex --- packages/remote-cache/README.md | 18 +- .../remote-cache/docs/0001-remote-cache.md | 26 +-- packages/remote-cache/docs/e2e-plan.md | 26 +-- packages/remote-cache/scripts/e2e/suite.ts | 21 ++- packages/remote-cache/scripts/operator.ts | 55 ++++-- packages/remote-cache/src/fetch.ts | 42 +++++ packages/remote-cache/src/index.ts | 43 +---- packages/remote-cache/test/failures.test.ts | 75 ++++++++- packages/remote-cache/test/helpers.ts | 17 +- packages/remote-cache/test/operator.test.ts | 158 +++++++++++++++++- packages/remote-cache/test/service.test.ts | 9 +- packages/remote-cache/wrangler.jsonc | 1 + 12 files changed, 386 insertions(+), 105 deletions(-) create mode 100644 packages/remote-cache/src/fetch.ts diff --git a/packages/remote-cache/README.md b/packages/remote-cache/README.md index 264788032..6ca11564e 100644 --- a/packages/remote-cache/README.md +++ b/packages/remote-cache/README.md @@ -8,19 +8,19 @@ The package contains no `vp run` client adapter. Cache keys, values, and blobs r The endpoint is `https:///projects/`. Namespaces use 1–63 lowercase letters, digits, or hyphens, starting with a letter or digit. An operator can register up to 100 namespaces per deployment. -| Request | Success | -| ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -| `POST /fetch`, `application/cbor`, `{ key: bytes, secondary_key: bytes }` | CBOR `{ kind: "exact", value: bytes, blob_id: string \| null }`, or `{ kind: "fallback", key: bytes, value: bytes, blob_id: string \| null }` | -| `GET /blob/{blob_id}` | Raw bytes with `application/octet-stream` | -| `POST /store`, `multipart/form-data` | CBOR `{ blob_id: string \| null }` | +| Request | Success | +| ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `POST /fetch`, `application/cbor`, `{ key: bytes, secondary_key: bytes }` | CBOR `{ kind: "exact", value: bytes, blob_id: string \| null }`, or `{ kind: "fallback", key: bytes }` | +| `GET /blob/{blob_id}` | Raw bytes with `application/octet-stream` | +| `POST /store`, `multipart/form-data` | CBOR `{ blob_id: string \| null }` | `/store` requires one `metadata` part with content type `application/cbor` and fields `{ key: bytes, secondary_key: bytes, value: bytes }`. An optional `blob` part has content type `application/octet-stream`. Either part order works. An omitted blob returns `null`; an empty blob gets an ID. Duplicate parts, unknown parts, duplicate envelope fields, extra fields, invalid types, and truncated bodies are rejected. Empty and non-UTF-8 keys work. CBOR definite and indefinite maps and byte strings work, including noncanonical lengths. The bounded envelope decoder supports only the protocol's map and string types. It never decodes `value` contents. Chunked strings have a 4,096-chunk bookkeeping limit. -Fetch gives exact matches priority. A fallback follows the latest secondary-key association. A store replaces both mappings atomically. Reassigning a secondary key does not delete its former target. Replacing an entry changes the value seen through every association to that entry. +Fetch gives exact matches priority. A fallback follows the latest secondary-key association and returns only the stored key, without reading R2. A store replaces both mappings atomically. Reassigning a secondary key does not delete its former target. Replacing an entry preserves the other associations to its key. -HTTP status codes follow the [local RFC](docs/0001-remote-cache.md#4-http-api-mapping). A fetch returns `404` when neither key resolves to a live entry. An unavailable blob also returns `404`, including when its R2 object is missing. These responses use plain text. A missing or unreadable value for a live entry is a storage failure and returns `503`, as specified in [the read design](docs/0001-remote-cache.md#7-fetch-and-download-implementation). +HTTP status codes follow the [local RFC](docs/0001-remote-cache.md#4-http-api-mapping). A fetch returns `404` when neither key resolves to a live entry. An unavailable blob also returns `404`, including when its R2 object is missing. These responses use plain text. A missing or unreadable value for a live exact match is a storage failure and returns `503`, as specified in [the read design](docs/0001-remote-cache.md#7-fetch-and-download-implementation). Errors have `Content-Type: text/plain; charset=utf-8`. Codes are `400` for invalid input, `401` for invalid/missing/expired tokens, `403` for a signed token that fails write policy, `404` for absent data or unavailable namespaces/routes, `413` for size limits, `429` for admission limits, `500` for an incomplete operation, and `503` for unavailable authorization/storage, failed publication guards, quotas, or concurrency admission. `429` and `503` include `Retry-After: 60`. @@ -53,7 +53,7 @@ pnpm operator setup --name my-public-cache --namespace docs --repo owner/reposit Use the account's actual Workers subdomain. For a custom domain, set `--origin https://cache.example.com`. Setup disables the unused `workers.dev` alias for a custom domain and disables preview URLs. It rejects an R2 bucket with public custom domains and disables its `r2.dev` access. -Setup discovers resources by name, creates missing resources, applies migrations, resolves public repository and owner IDs through GitHub, records the default branch and exact audience, installs lifecycle/Cron settings, deploys, and prints the endpoint. Repeated setup does not duplicate resources or re-enable a withdrawn namespace. Reusing an existing namespace for a different repository is rejected. Configuration is written to the ignored `wrangler.operator.json`; keep a secure backup of this non-secret file. +Setup discovers resources by name, creates missing resources, applies migrations, resolves public repository and owner IDs through GitHub, records the default branch and exact audience, installs lifecycle/Cron settings, deploys, and prints the endpoint. Repeated setup does not duplicate resources or re-enable a withdrawn namespace. An incompatible origin or repository is rejected before setup changes existing policies, bucket settings, or saved configuration. Configuration is written to the ignored `wrangler.operator.json`; keep a secure backup of this non-secret file. The default `free` **resource profile** retains the RFC's 8 GB budget, 20,000 entries, 20,000 associations, and 16 generations per Cron run. It is not proof of compatibility with the Workers Free CPU limit. **Use Workers Paid for the full payload limits until production CPU measurements establish a supported Free profile.** Local measurements already exceed 10 ms in several cases. Setup does not purchase or change a Workers subscription. @@ -90,7 +90,7 @@ GitHub authorization uses `jose`, `RS256`, the fixed issuer `https://token.actio Tokens are limited to 16 KiB. JWKS bodies are limited to 64 KiB, requests to five seconds, cached keys to ten minutes, and refresh attempts to one per 30 seconds per isolate, including failures. Unknown key IDs cannot cause unlimited refreshes. Missing usable keys fail closed. A maintainer's policy change increments its version; publication checks that version, current scope/deployment state, token expiry, lease, bytes, and identity quotas inside the D1 transaction. -The rate bindings apply before database or object access. Defaults are 600 requests/minute for each namespace's fetch/blob operation and 30 store attempts/minute. Invalid credentials consume store admission too. Unknown routes share catch-all identities. Limits are approximate per Cloudflare location; they are not a global billing cap. Reads do not write D1 counters or refresh retention. +The rate bindings apply before database or object access. Defaults are 600 requests/minute for each namespace's fetch/blob operation and 30 store attempts/minute. Invalid credentials consume store admission too. Unknown routes share catch-all identities. Setup and upgrade derive stable limiter IDs from the Worker and binding names, so separate deployments have separate counters. Run `pnpm operator upgrade` for an existing deployment to replace the old shared IDs. Limits are approximate per Cloudflare location; they are not a global billing cap. Reads do not write D1 counters or refresh retention. ## Operations diff --git a/packages/remote-cache/docs/0001-remote-cache.md b/packages/remote-cache/docs/0001-remote-cache.md index a424d280a..14b61019e 100644 --- a/packages/remote-cache/docs/0001-remote-cache.md +++ b/packages/remote-cache/docs/0001-remote-cache.md @@ -84,10 +84,10 @@ For a match, return HTTP `200`, `Content-Type: application/cbor`, with one of: ```text { kind: "exact", value: bytes, blob_id: string | null } -{ kind: "fallback", key: bytes, value: bytes, blob_id: string | null } +{ kind: "fallback", key: bytes } ``` -Check `key` first. If no live entry exists, resolve `secondary_key` to a stored key. Check that entry. Include the stored key only in the fallback variant. If neither resolves, return HTTP `404`. Fetch does not change entries or associations. +Check `key` first. If no live entry exists, resolve `secondary_key` to a stored key. Check that entry. For a fallback match, return only the stored key so the client can explain what changed. If neither resolves, return HTTP `404`. Fetch does not change entries or associations. ### Download a blob @@ -120,13 +120,13 @@ If the request omits the blob, return `null`. A present zero-byte blob receives Each successful store replaces `entries[key]` and sets `associations[secondary_key] = key`. For example: -| Operation | Entries afterward | Association afterward | -| ------------------- | ----------------------------------------------- | --------------------- | -| Store `(A, S, VA)` | `A → VA` | `S → A` | -| Store `(B, S, VB)` | `A → VA`, `B → VB` | `S → B` | -| Fetch `(A, S)` | Unchanged; returns exact `VA` | Still `S → B` | -| Fetch `(C, S)` | Unchanged; returns fallback key `B`, value `VB` | Still `S → B` | -| Store `(A, T, VA2)` | `A → VA2`, `B → VB` | `S → B`, `T → A` | +| Operation | Entries afterward | Association afterward | +| ------------------- | ----------------------------------- | --------------------- | +| Store `(A, S, VA)` | `A → VA` | `S → A` | +| Store `(B, S, VB)` | `A → VA`, `B → VB` | `S → B` | +| Fetch `(A, S)` | Unchanged; returns exact `VA` | Still `S → B` | +| Fetch `(C, S)` | Unchanged; returns fallback key `B` | Still `S → B` | +| Store `(A, T, VA2)` | `A → VA2`, `B → VB` | `S → B`, `T → A` | Other secondary keys that already point to `A` also resolve to `VA2`. A change to `S` does not evict entry `A`. @@ -359,7 +359,7 @@ First, apply rate limits. Check that the public scope is enabled. Decode CBOR wi Use one indexed D1 query to select the exact live entry, or the secondary fallback if no exact entry exists. Select the response kind, stored key, and generation references from one database snapshot. Separate exact and fallback reads could observe different commits. -Read the selected generation's value from R2. Return the corresponding CBOR variant. Return `503` if D1 identifies a live generation but its value object is missing or unreadable. This condition is a storage failure. Expired or deleted entries count as absent. The client still validates the exact value before it downloads outputs. +For an exact match, read the selected generation's value from R2 and return the exact CBOR variant. Return `503` if its value object is missing or unreadable. This condition is a storage failure. For a fallback match, return the stored key without reading R2. Expired or deleted entries count as absent. The client still validates the exact value before it downloads outputs. For `/blob/{blob_id}`, check that the public scope is enabled, without authentication. Resolve the blob ID in D1. Stream the R2 object to the response. A blob ID from another scope must not expose data. @@ -607,7 +607,7 @@ Public reads require no identity subscription. GitHub issues tokens for publishi Use these variables for daily traffic: - `L`: Public fetches after local misses. -- `F`: Fetches that return a value. +- `F`: Exact fetches that return a value. - `H`: Blob downloads after successful client validation. - `P`: Entries successfully published through explicit CI pushes from the main branch, including overwrites. @@ -621,7 +621,7 @@ Use these variables for stored data: - `E`: Live exact keys. - `R`: Retention days. -Each store takes one Worker request. Each metadata fetch takes one request. The client makes a blob request only when it needs the blob. The Worker also reads the opaque value from R2. Internal R2 multipart upload changes storage operation counts but adds no client requests: +Each store takes one Worker request. Each metadata fetch takes one request. The client makes a blob request only when it needs the blob. For an exact match, the Worker also reads the opaque value from R2. Internal R2 multipart upload changes storage operation counts but adds no client requests: ```text A per store = 1 # no blob: value PUT @@ -678,7 +678,7 @@ For planning, interpret the approximate KB values as decimal. `V = 250,000` byte D1's provisional 4 KiB estimate for each key covers index and generation records only. Validate it with the reported key sizes. Include repeated key bytes in indexes and associations. -Every exact or fallback response transfers the value, even if client validation prevents a blob download. Before protocol overhead and retries, daily response payload equals `F * V + H * B` bytes. At the example's sizes, 1,000 fetches with values and 800 blob downloads transfer about 450 MB/day. This includes 250 MB of values. These transfers affect time, CBOR work, and memory. Request and R2 operation counts follow the same equations. +Every exact response transfers the value, even if client validation prevents a blob download. Before protocol overhead and retries, daily value and blob payload equals `F * V + H * B` bytes. At the example's sizes, 1,000 fetches with values and 800 blob downloads transfer about 450 MB/day. This includes 250 MB of values. These transfers affect time, CBOR work, and memory. Request and R2 operation counts follow the same equations. ### Artifact-size evidence diff --git a/packages/remote-cache/docs/e2e-plan.md b/packages/remote-cache/docs/e2e-plan.md index d67763815..52c761205 100644 --- a/packages/remote-cache/docs/e2e-plan.md +++ b/packages/remote-cache/docs/e2e-plan.md @@ -62,14 +62,14 @@ The operator seeds immutable objects and generation rows for read tests. This va “Local” refers to the existing regression suite and the new shared e2e-driver tests. “PR” and “Main” refer to real Cloudflare runs. -Status assertions follow the [local RFC](0001-remote-cache.md#4-http-api-mapping). Fetch misses and unavailable blobs return plain-text `404`. A missing or unreadable value for a live entry returns `503`. +Status assertions follow the [local RFC](0001-remote-cache.md#4-http-api-mapping). Fetch misses and unavailable blobs return plain-text `404`. An exact match with a missing or unreadable value returns `503`. A fallback returns only the stored key and makes no R2 read. | Case | Local | PR | Main | Required result | | ------------------------------------------------ | ----------------------------- | ----------------------------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | | Exact source revision | Yes | Yes | Yes | Expected deployment ID on responses; fail if the URL serves another revision | | Migrations and private R2 | Yes | Yes | Yes | Setup succeeds, seeded data works, `r2.dev` is disabled, and custom R2 domains are absent | | Anonymous exact lookup | Yes | Yes | Yes | `200` CBOR with exact opaque value and matching blob ID | -| Anonymous fallback | Yes | Yes | Yes | `200` CBOR with the associated stored key and matching value | +| Anonymous fallback | Yes | Yes | Yes | `200` CBOR with only `kind: "fallback"` and the associated stored key; no R2 read | | Read-only accounting | Yes | Yes | Yes | Reads do not change charged bytes or entry/association counts | | Missing entry/blob | Yes | Yes | Yes | Plain-text `404`, without redirects | | Namespace isolation | Yes | Yes | Yes | Another namespace cannot resolve the key, association, or blob ID | @@ -147,17 +147,17 @@ Staging policies belong to CI. Do not use these names for an operator-managed pr Before a production release, record the commit, workflow URL, Cloudflare plan, region, and outcome for these controlled staging exercises: -| Exercise | Procedure and acceptance criterion | -| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Provider failure and recovery | Restrict D1/R2 access in staging, exercise reads and stores, then restore access. Existing mappings survive; failures remain explicit; retries recover | -| Upload interruption | Disconnect a real authorized multipart request after its first R2 part. Verify unchanged mappings, retained reservation, upload abortion, and eventual byte release | -| Lease/token expiry and policy race | Delay a staging upload across expiry or change policy before publication. Neither mapping changes; old data remains readable | -| Real hostile workflow identities | Use controlled fork, tag, PR, `pull_request_target`, and `workflow_run` jobs. All writes fail; public reads still work | -| Rate limits | Send a bounded burst to the staging namespace. Check `429`, `Retry-After`, recovery, and a finite identity set for unknown routes | -| Retention and lifecycle | Observe expiry, replacement grace, unfinished multipart abortion, and lifecycle deletion over actual retention windows | -| Migration and rollback | Upgrade a populated staging database, verify old entries, and restore a compatible previous Worker version. Never roll code back across an incompatible schema change | -| Partial restore | Restore D1 without deleted R2 data. Expect `503` for live missing values and `404` for missing blobs; withdraw or replace the namespace before reuse | -| Load and CPU | Measure deployed CPU, memory, D1 rows/latency, R2 operations, and cleanup lag under sustained concurrency. Compare observed costs with configured budgets | +| Exercise | Procedure and acceptance criterion | +| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Provider failure and recovery | Restrict D1/R2 access in staging, exercise reads and stores, then restore access. Existing mappings survive; failures remain explicit; retries recover | +| Upload interruption | Disconnect a real authorized multipart request after its first R2 part. Verify unchanged mappings, retained reservation, upload abortion, and eventual byte release | +| Lease/token expiry and policy race | Delay a staging upload across expiry or change policy before publication. Neither mapping changes; old data remains readable | +| Real hostile workflow identities | Use controlled fork, tag, PR, `pull_request_target`, and `workflow_run` jobs. All writes fail; public reads still work | +| Rate limits | Send a bounded burst to the staging namespace. Check `429`, `Retry-After`, recovery, and a finite identity set for unknown routes | +| Retention and lifecycle | Observe expiry, replacement grace, unfinished multipart abortion, and lifecycle deletion over actual retention windows | +| Migration and rollback | Upgrade a populated staging database, verify old entries, and restore a compatible previous Worker version. Never roll code back across an incompatible schema change | +| Partial restore | Restore D1 without deleted R2 data. Expect `503` for exact matches with missing values, key-only fallbacks, and `404` for missing blobs; withdraw or replace the namespace before reuse | +| Load and CPU | Measure deployed CPU, memory, D1 rows/latency, R2 operations, and cleanup lag under sustained concurrency. Compare observed costs with configured budgets | These are explicit release exercises, not claims that fault injection or multi-day lifecycle behavior already ran in the PR workflow. Workers Free support still needs provider CPU measurements within its limits. diff --git a/packages/remote-cache/scripts/e2e/suite.ts b/packages/remote-cache/scripts/e2e/suite.ts index 9a68ae6f7..1d2a5bb3b 100644 --- a/packages/remote-cache/scripts/e2e/suite.ts +++ b/packages/remote-cache/scripts/e2e/suite.ts @@ -224,8 +224,6 @@ export async function runSuite(options: Options): Promise { assert.deepEqual(await envelope(await lookup(bytes('missing'), f.secondary)), { kind: 'fallback', key: f.key, - value: f.value, - blob_id: f.blobId, }); assert.deepEqual(await body(await call('e2e', `blob/${f.blobId}`, 200)), f.blob); assert.deepEqual( @@ -292,6 +290,10 @@ export async function runSuite(options: Options): Promise { try { await admin.delete(f.valueObject); await body(await lookup(f.key, f.secondary, 503)); + assert.deepEqual(await envelope(await lookup(bytes('missing'), f.secondary)), { + kind: 'fallback', + key: f.key, + }); } finally { await admin.put(f.valueObject, f.value); } @@ -324,12 +326,16 @@ export async function runSuite(options: Options): Promise { const old = await envelope(await store(a, s, bytes('old'), bytes('old-blob'))); await body(await store(b, s, bytes('B'))); assert.deepEqual((await envelope(await lookup(a, s))).value, bytes('old')); - assert.deepEqual((await envelope(await lookup(bytes('missing'), s))).key, b); + assert.deepEqual(await envelope(await lookup(bytes('missing'), s)), { + kind: 'fallback', + key: b, + }); await body(await store(a, t, bytes('replacement'))); - assert.deepEqual( - (await envelope(await lookup(bytes('missing'), t))).value, - bytes('replacement'), - ); + assert.deepEqual(await envelope(await lookup(bytes('missing'), t)), { + kind: 'fallback', + key: a, + }); + assert.deepEqual((await envelope(await lookup(a, t))).value, bytes('replacement')); assert.deepEqual( await body(await call('e2e', `blob/${blobId(old)}`, 200)), bytes('old-blob'), @@ -423,6 +429,7 @@ export async function runSuite(options: Options): Promise { [expired.generation], ); await body(await lookup(expired.key, expired.secondary, 404)); + await body(await lookup(bytes('missing'), expired.secondary, 404)); await body(await call('e2e', `blob/${expired.blobId}`, 404)); if (options.cron) { const until = Date.now() + (options.cronTimeoutMs ?? 25 * 60000); diff --git a/packages/remote-cache/scripts/operator.ts b/packages/remote-cache/scripts/operator.ts index a19304fde..a1daf29ac 100644 --- a/packages/remote-cache/scripts/operator.ts +++ b/packages/remote-cache/scripts/operator.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'; import { join, dirname } from 'node:path'; import { fileURLToPath, pathToFileURL, URL } from 'node:url'; import { createRequire } from 'node:module'; +import { createHash } from 'node:crypto'; import { parseArgs } from 'node:util'; import { parse, type ParseError } from 'jsonc-parser'; @@ -50,6 +51,16 @@ export type Config = { vars: Record; }; +function isolateRateLimits(config: Config) { + for (const binding of config.ratelimits) { + // Account-wide IDs must stay stable across revisions and differ between Workers/bindings. + const hash = createHash('sha256') + .update(`remote-cache:${config.name}:${binding.name}`) + .digest(); + binding.namespace_id = String(hash.readUIntBE(0, 6) + 1); + } +} + function object(value: unknown): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Invalid API response'); @@ -226,6 +237,32 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise db = created.map(object).find((item) => item['name'] === name); } if (!db) throw new Error('D1 creation did not return the database'); + config = await readTemplate(); + config.name = name; + config.d1_databases[0] = { + binding: 'INDEX', + database_name: name, + database_id: text(db['uuid']), + migrations_dir: 'migrations', + }; + // New databases have no schema. Existing policies must pass validation before any writes. + const tables = await query( + io, + config, + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'scopes'", + ); + if (tables.length) { + const scopes = await query( + io, + config, + 'SELECT scope_id, endpoint, repository_id FROM scopes', + ); + if (scopes.some((row) => new URL(text(row['endpoint'])).origin !== origin.origin)) + throw new Error('Use a separate deployment for a different public origin'); + const prior = scopes.find((row) => row['scope_id'] === namespace); + if (prior && prior['repository_id'] !== repo.id) + throw new Error('Use a new namespace for a different repository'); + } try { const bucket = object(await io.api(`/r2/buckets/${name}`)); if (bucket['storage_class'] && bucket['storage_class'] !== 'Standard') @@ -247,25 +284,13 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise if (!Array.isArray(domains['domains']) || domains['domains'].length) throw new Error('Remove R2 public custom domains before setup'); await io.api(`/r2/buckets/${name}/domains/managed`, 'PUT', { enabled: false }); - config = await readTemplate(); - config.name = name; - config.d1_databases[0] = { - binding: 'INDEX', - database_name: name, - database_id: text(db['uuid']), - migrations_dir: 'migrations', - }; config.r2_buckets[0] = { binding: 'ARTIFACTS', bucket_name: name }; config.workers_dev = origin.hostname.endsWith('.workers.dev'); if (!config.workers_dev) config.routes = [{ pattern: origin.hostname, custom_domain: true }]; config.vars['GC_BATCH_SIZE'] = profile === 'free' ? '16' : '256'; + isolateRateLimits(config); await io.writeConfig(config); await io.wrangler(['d1', 'migrations', 'apply', 'INDEX', '--remote', '--config', configPath]); - const prior = await query(io, config, 'SELECT repository_id FROM scopes WHERE scope_id = ?', [ - namespace, - ]); - if (prior.length && prior[0]!['repository_id'] !== repo.id) - throw new Error('Use a new namespace for a different repository'); // Repeated setup only creates a missing policy; it never re-enables a withdrawn scope. await query( io, @@ -297,8 +322,6 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise ], ); const scopes = await query(io, config, 'SELECT scope_id, endpoint FROM scopes'); - if (scopes.some((row) => new URL(text(row['endpoint'])).origin !== origin.origin)) - throw new Error('Use a separate deployment for a different public origin'); config.vars['NAMESPACES'] = JSON.stringify(scopes.map((row) => text(row['scope_id']))); await io.writeConfig(config); await updateLifecycle(io, config); @@ -308,6 +331,8 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise } config = await io.readConfig(); if (command === 'upgrade') { + isolateRateLimits(config); + await io.writeConfig(config); await io.wrangler(['d1', 'migrations', 'apply', 'INDEX', '--remote', '--config', configPath]); await updateLifecycle(io, config); await io.wrangler(['deploy', '--config', configPath]); diff --git a/packages/remote-cache/src/fetch.ts b/packages/remote-cache/src/fetch.ts new file mode 100644 index 000000000..73b8283e2 --- /dev/null +++ b/packages/remote-cache/src/fetch.ts @@ -0,0 +1,42 @@ +import { decodeEnvelope } from './cbor.ts'; +import { selectEntry } from './database.ts'; +import { badRequest, HttpError, unavailable } from './errors.ts'; +import type { Limits } from './limits.ts'; +import { parameters } from './multipart.ts'; +import type { Observations } from './observations.ts'; +import { contentLength, Deadline, readBody } from './streams.ts'; + +type FetchResult = + | { kind: 'exact'; value: Uint8Array; blob_id: string | null } + | { kind: 'fallback'; key: Uint8Array }; + +export async function fetchMetadata( + request: Request, + env: Pick, + scope: string, + limits: Limits, + deadline: Deadline, + stats: Observations, +): Promise { + if (parameters(request.headers.get('Content-Type') ?? '').type !== 'application/cbor') + badRequest(); + contentLength(request, limits.fetch); + const body = await readBody(request.body, limits.fetch, deadline); + stats.request_bytes = body.length; + const data = decodeEnvelope(body, false, limits); + const selected = await deadline.run( + selectEntry(env.INDEX, scope, data.key, data.secondary_key, stats), + ); + if (!selected) throw new HttpError(404, 'miss'); + // Fallbacks only identify the associated key. They must not depend on R2 availability. + if (selected.kind === 'fallback') return { kind: 'fallback', key: new Uint8Array(selected.key) }; + + stats.r2_operations++; + const object = await deadline + .run(env.ARTIFACTS.get(selected.value_object)) + .catch(() => unavailable()); + if (!object || object.size !== selected.value_size || object.size > limits.value) unavailable(); + const value = await readBody(object.body, limits.value, deadline).catch(() => unavailable()); + if (value.length !== selected.value_size) unavailable(); + return { kind: 'exact', value, blob_id: selected.blob_id }; +} diff --git a/packages/remote-cache/src/index.ts b/packages/remote-cache/src/index.ts index b1e13cc68..f05c06f5b 100644 --- a/packages/remote-cache/src/index.ts +++ b/packages/remote-cache/src/index.ts @@ -1,11 +1,11 @@ import { authorize, githubKeys, type WriteIdentity } from './auth.ts'; -import { cborResponse, decodeEnvelope } from './cbor.ts'; -import { cleanup, getScope, selectBlob, selectEntry } from './database.ts'; -import { badRequest, errorResponse, HttpError, unavailable } from './errors.ts'; +import { cborResponse } from './cbor.ts'; +import { cleanup, getScope, selectBlob } from './database.ts'; +import { errorResponse, HttpError, unavailable } from './errors.ts'; +import { fetchMetadata } from './fetch.ts'; import { limitsFrom } from './limits.ts'; -import { parameters } from './multipart.ts'; import { store } from './store.ts'; -import { contentLength, Deadline, readBody } from './streams.ts'; +import { Deadline } from './streams.ts'; import { Admission } from './admission.ts'; import { Observations } from './observations.ts'; @@ -65,36 +65,9 @@ export default { response = await store(request, env, ctx, scope, identity, limits, deadline, stats); outcome = 'stored'; } else if (operation === 'fetch') { - if (parameters(request.headers.get('Content-Type') ?? '').type !== 'application/cbor') - badRequest(); - contentLength(request, limits.fetch); - const body = await readBody(request.body, limits.fetch, deadline); - stats.request_bytes = body.length; - const data = decodeEnvelope(body, false, limits); - const selected = await deadline.run( - selectEntry(env.INDEX, scopeId, data.key, data.secondary_key, stats), - ); - if (!selected) { - outcome = 'miss'; - throw new HttpError(404, 'miss'); - } - stats.r2_operations++; - const object = await deadline - .run(env.ARTIFACTS.get(selected.value_object)) - .catch(() => unavailable()); - if (!object || object.size !== selected.value_size || object.size > limits.value) - unavailable(); - const value = await readBody(object.body, limits.value, deadline).catch(() => - unavailable(), - ); - if (value.length !== selected.value_size) unavailable(); - response = cborResponse({ - kind: selected.kind, - ...(selected.kind === 'fallback' ? { key: new Uint8Array(selected.key) } : {}), - value, - blob_id: selected.blob_id, - }); - outcome = selected.kind; + const result = await fetchMetadata(request, env, scopeId, limits, deadline, stats); + response = cborResponse(result); + outcome = result.kind; } else { const selected = await deadline.run(selectBlob(env.INDEX, scopeId, route[3]!, stats)); if (!selected) throw new HttpError(404, 'blob_missing'); diff --git a/packages/remote-cache/test/failures.test.ts b/packages/remote-cache/test/failures.test.ts index cad574066..8e2597845 100644 --- a/packages/remote-cache/test/failures.test.ts +++ b/packages/remote-cache/test/failures.test.ts @@ -4,11 +4,74 @@ import { encode } from 'cborg'; import { harness, bytes, decodeResponse } from './helpers.ts'; import { cleanup, getScope, reserve, publish } from '../src/database.ts'; import { store } from '../src/store.ts'; +import { fetchMetadata } from '../src/fetch.ts'; import { defaults } from '../src/limits.ts'; import { Deadline } from '../src/streams.ts'; import { Observations } from '../src/observations.ts'; import { Admission } from '../src/admission.ts'; +void test('fallbacks return only opaque keys without R2 reads, while exact failures remain 503', async () => { + const h = await harness(); + try { + const original = await h.mf.getBindings(); + let reads = 0; + const bucket = new Proxy(original.ARTIFACTS, { + get(target, prop) { + if (prop === 'get') + return async () => { + reads++; + throw new Error('Injected R2 read failure'); + }; + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const request = (key: Uint8Array) => + new Request('https://cache.example.com/projects/test/fetch', { + method: 'POST', + headers: { 'Content-Type': 'application/cbor' }, + body: encode({ key, secondary_key: bytes('S') }), + }); + const deadline = new Deadline(10000); + const stats = new Observations(); + const env = { ...original, ARTIFACTS: bucket }; + try { + for (const key of [ + Uint8Array.of(0, 255), + new Uint8Array(), + new Uint8Array(defaults.key).fill(165), + ]) { + const stored = await h.store(key, bytes('S'), bytes('value'), bytes('blob')); + assert.equal(stored.status, 200); + await stored.arrayBuffer(); + assert.deepEqual( + await fetchMetadata(request(bytes('missing')), env, 'test', defaults, deadline, stats), + { kind: 'fallback', key }, + ); + assert.equal(reads, 0); + assert.equal(stats.r2_operations, 0); + } + // An exact match still wins when the secondary key points at a different live entry. + await assert.rejects( + fetchMetadata(request(Uint8Array.of(0, 255)), env, 'test', defaults, deadline, stats), + { status: 503 }, + ); + assert.equal(reads, 1); + assert.equal(stats.r2_operations, 1); + await h.db.prepare("UPDATE generations SET expires_at = 0 WHERE state = 'ready'").run(); + await assert.rejects( + fetchMetadata(request(bytes('missing')), env, 'test', defaults, deadline, stats), + { status: 404 }, + ); + assert.equal(reads, 1); + } finally { + deadline.dispose(); + } + } finally { + await h.close(); + } +}); + function multipartRequest(blob?: Uint8Array): Request { const form = new FormData(); form.append( @@ -126,8 +189,12 @@ void test('failed deletion remains charged and a retry removes only the claimed (await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes, 3, ); + assert.deepEqual(await decodeResponse(await h.fetch(bytes('missing'), bytes('S'))), { + kind: 'fallback', + key: bytes('A'), + }); assert.deepEqual( - (await decodeResponse(await h.fetch(bytes('missing'), bytes('S')))).value, + (await decodeResponse(await h.fetch(bytes('A'), bytes('S')))).value, bytes('new'), ); } finally { @@ -275,8 +342,12 @@ void test('cleanup preserves a recreated target and claims at most 16 generation }, }); await cleanup({ ...original, ARTIFACTS: bucket }); + assert.deepEqual(await decodeResponse(await h.fetch(bytes('missing'), bytes('S'))), { + kind: 'fallback', + key: bytes('A'), + }); assert.deepEqual( - (await decodeResponse(await h.fetch(bytes('missing'), bytes('S')))).value, + (await decodeResponse(await h.fetch(bytes('A'), bytes('S')))).value, bytes('new'), ); const scope = await getScope(h.db, 'test'); diff --git a/packages/remote-cache/test/helpers.ts b/packages/remote-cache/test/helpers.ts index bdd13dec6..537c7cdf1 100644 --- a/packages/remote-cache/test/helpers.ts +++ b/packages/remote-cache/test/helpers.ts @@ -20,6 +20,7 @@ export async function harness( inspector?: boolean; deploymentId?: string; namespaces?: string[]; + initializeDatabase?: boolean; } = {}, ) { const pair = await generateKeyPair('RS256', { extractable: true }); @@ -84,13 +85,16 @@ export async function harness( const statements = migration.match( /CREATE TRIGGER[\s\S]*?\nEND;|(?:CREATE TABLE|CREATE (?:UNIQUE )?INDEX|INSERT INTO)[\s\S]*?;/g, )!; - await db.batch(statements.map((sql) => db.prepare(sql))); - for (const name of ['test', 'other']) - await db - .prepare(`INSERT INTO scopes (scope_id, endpoint, repository, repository_id, repository_owner_id, branch) + const migrate = () => db.batch(statements.map((sql) => db.prepare(sql))); + if (options.initializeDatabase !== false) { + await migrate(); + for (const name of ['test', 'other']) + await db + .prepare(`INSERT INTO scopes (scope_id, endpoint, repository, repository_id, repository_owner_id, branch) VALUES (?, ?, 'owner/repo', '123', '456', 'refs/heads/main')`) - .bind(name, `https://cache.example.com/projects/${name}`) - .run(); + .bind(name, `https://cache.example.com/projects/${name}`) + .run(); + } async function token(claims: Record = {}, kid = 'test-key') { const now = Math.floor(Date.now() / 1000); return new SignJWT({ @@ -154,6 +158,7 @@ export async function harness( mf, db, bucket, + migrate, token, store, fetch, diff --git a/packages/remote-cache/test/operator.test.ts b/packages/remote-cache/test/operator.test.ts index 2032602ad..4fc7ab13e 100644 --- a/packages/remote-cache/test/operator.test.ts +++ b/packages/remote-cache/test/operator.test.ts @@ -10,18 +10,21 @@ import { } from '../scripts/operator.ts'; import { harness } from './helpers.ts'; -void test('operator setup is repeatable and policy changes preserve withdrawal', async () => { +void test('operator setup validates before writes and preserves withdrawal on retries', async () => { let config = await readTemplate(); const h = await harness(); const commands: string[][] = []; const output: string[] = []; const lifecycle: unknown[] = []; + const writes: string[] = []; + let repositoryId = 123; let failDeploy = false; const io: OperatorIO = { async api(path, _method, body) { if (path.startsWith('/d1/database?')) return [{ name: 'cache', uuid: 'test-database' }]; if (path.endsWith('/query')) { const { sql, params } = body as { sql: string; params: unknown[] }; + if (!sql.startsWith('SELECT')) writes.push(sql); return [ await h.db .prepare(sql) @@ -30,11 +33,12 @@ void test('operator setup is repeatable and policy changes preserve withdrawal', ]; } if (path.endsWith('/domains/custom')) return { domains: [] }; + if (_method && _method !== 'GET') writes.push(path); return {}; }, async github() { return { - id: 123, + id: repositoryId, full_name: 'owner/repo', owner: { id: 456 }, private: false, @@ -43,6 +47,7 @@ void test('operator setup is repeatable and policy changes preserve withdrawal', }; }, async wrangler(args) { + writes.push(args.join(' ')); commands.push(args); if (failDeploy && args[0] === 'deploy') throw new Error('Deployment failed'); }, @@ -50,9 +55,11 @@ void test('operator setup is repeatable and policy changes preserve withdrawal', return config; }, async writeConfig(value) { - config = value; + writes.push('config'); + config = structuredClone(value); }, async lifecycle(_bucket, rules) { + writes.push('lifecycle'); lifecycle.push(rules); }, print(message) { @@ -71,9 +78,35 @@ void test('operator setup is repeatable and policy changes preserve withdrawal', '--origin', 'https://cache.example.com', ]; + const originalConfig = structuredClone(config); + const originalScopes = await h.db.prepare('SELECT * FROM scopes ORDER BY scope_id').all(); + const originalDeployment = await h.db.prepare('SELECT * FROM deployment').first(); + await assert.rejects( + runOperator([...args.slice(0, -1), 'https://wrong.example.com', '--byte-limit', '1'], io), + /different public origin/, + ); + repositoryId = 999; + await assert.rejects( + runOperator( + args.map((arg) => (arg === 'new' ? 'test' : arg)), + io, + ), + /different repository/, + ); + repositoryId = 123; + assert.deepEqual(writes, []); + assert.deepEqual(config, originalConfig); + assert.deepEqual( + (await h.db.prepare('SELECT * FROM scopes ORDER BY scope_id').all()).results, + originalScopes.results, + ); + assert.deepEqual(await h.db.prepare('SELECT * FROM deployment').first(), originalDeployment); await runOperator(args, io); + const rateLimits = structuredClone(config.ratelimits); + assert.ok(rateLimits.every((binding) => !['1001', '1002'].includes(binding.namespace_id))); await runOperator(['policy', '--namespace', 'new', '--enabled', 'off', '--writes', 'off'], io); await runOperator(args, io); + assert.deepEqual(config.ratelimits, rateLimits); const row = await h.db.prepare("SELECT * FROM scopes WHERE scope_id = 'new'").first(); assert.equal(row!.enabled, 0); assert.equal(row!.writes_enabled, 0); @@ -118,6 +151,125 @@ void test('operator setup is repeatable and policy changes preserve withdrawal', } }); +void test('operator upgrades isolate rate counters by Worker and retain them across revisions', async () => { + const namespaces = new Set(); + for (const name of ['vp-cache-ci-pr-1', 'vp-cache-ci-pr-2', 'vp-cache-ci-main', 'production']) { + let config = await readTemplate(); + config.name = name; + const limits = config.ratelimits.map((binding) => binding.simple); + const deployed: (typeof config)[] = []; + const io: OperatorIO = { + async api() { + return [{ success: true, results: [{ retention: 604800 }] }]; + }, + async github() { + throw new Error('Unexpected GitHub request'); + }, + async readConfig() { + return structuredClone(config); + }, + async writeConfig(value) { + config = structuredClone(value); + }, + async wrangler(args) { + if (args[0] === 'deploy') deployed.push(structuredClone(config)); + }, + async lifecycle() {}, + print() {}, + }; + await runOperator(['upgrade'], io); + for (const binding of config.ratelimits) { + assert.match(binding.namespace_id, /^[1-9][0-9]*$/); + assert.ok(Number.isSafeInteger(Number(binding.namespace_id))); + assert.ok( + !namespaces.has(binding.namespace_id), + 'deployments and bindings must not share counters', + ); + namespaces.add(binding.namespace_id); + } + assert.deepEqual( + config.ratelimits.map((binding) => binding.simple), + limits, + ); + config.vars['DEPLOYMENT_ID'] = 'next-revision'; + await runOperator(['upgrade'], io); + assert.equal(deployed.length, 2); + assert.deepEqual(deployed[0]!.ratelimits, deployed[1]!.ratelimits); + } +}); + +void test('operator setup creates resources and initializes an empty database', async () => { + const h = await harness({ initializeDatabase: false }); + let config = await readTemplate(); + let database = false; + let bucket = false; + let deployed = false; + const io: OperatorIO = { + async api(path, _method, body) { + if (path.startsWith('/d1/database?')) + return database ? [{ name: 'cache', uuid: 'test-database' }] : []; + if (path.endsWith('/query')) { + const { sql, params } = body as { sql: string; params: unknown[] }; + return [ + await h.db + .prepare(sql) + .bind(...params) + .all(), + ]; + } + if (path === '/r2/buckets/cache' && !bucket) throw new ApiError(404); + if (path.endsWith('/domains/custom')) return { domains: [] }; + return {}; + }, + async github() { + return { + id: 123, + full_name: 'owner/repo', + owner: { id: 456 }, + private: false, + visibility: 'public', + default_branch: 'main', + }; + }, + async readConfig() { + return structuredClone(config); + }, + async writeConfig(value) { + config = structuredClone(value); + }, + async wrangler(args) { + if (args[0] === 'd1' && args[1] === 'create') database = true; + if (args[0] === 'r2' && args[2] === 'create') bucket = true; + if (args[1] === 'migrations') await h.migrate(); + if (args[0] === 'deploy') deployed = true; + }, + async lifecycle() {}, + print() {}, + }; + try { + await runOperator( + [ + 'setup', + '--name', + 'cache', + '--namespace', + 'test', + '--repo', + 'owner/repo', + '--origin', + 'https://cache.example.com', + ], + io, + ); + assert.ok(database && bucket && deployed); + assert.deepEqual(JSON.parse(config.vars['NAMESPACES']!), ['test']); + const scope = await h.db.prepare("SELECT endpoint FROM scopes WHERE scope_id = 'test'").first(); + assert.equal(scope!.endpoint, 'https://cache.example.com/projects/test'); + } finally { + await h.close(); + } +}); + void test('teardown resumes after partial resource deletion', async () => { const config = await readTemplate(); let database = true, diff --git a/packages/remote-cache/test/service.test.ts b/packages/remote-cache/test/service.test.ts index a27bb4049..26c6239dc 100644 --- a/packages/remote-cache/test/service.test.ts +++ b/packages/remote-cache/test/service.test.ts @@ -25,8 +25,6 @@ void test('Workers D1/R2: exact/fallback, replacement, empty blobs, namespace is assert.deepEqual(await decodeResponse(await h.fetch(c, s)), { kind: 'fallback', key: b, - value: bytes('VB'), - blob_id: null, }); const replacement = await h.store(a, t, bytes('VA2'), new Uint8Array(), { blobFirst: true }); assert.equal(replacement.status, 200); @@ -56,12 +54,15 @@ void test('Workers D1/R2: exact/fallback, replacement, empty blobs, namespace is ); assert.deepEqual((await decodeResponse(await h.fetch(a, s, 'other'))).value, bytes('other')); assert.deepEqual((await decodeResponse(await h.fetch(a, s))).value, bytes('VA2')); + assert.deepEqual(await decodeResponse(await h.fetch(c, t)), { kind: 'fallback', key: a }); + assert.equal((await h.fetch(c, t, 'other')).status, 404); const before = await h.db.prepare('SELECT * FROM deployment').first(); await h.fetch(a, t); await h.fetch(c, t); assert.deepEqual(await h.db.prepare('SELECT * FROM deployment').first(), before); await h.db.prepare("UPDATE scopes SET enabled = 0 WHERE scope_id = 'test'").run(); assert.equal((await h.fetch(a, t)).status, 404); + assert.equal((await h.fetch(c, t)).status, 404); assert.equal((await h.mf.dispatchFetch(`${endpoint}/blob/${firstBlob}`)).status, 404); } finally { await h.close(); @@ -341,6 +342,10 @@ void test('malformed stores cannot publish and missing live value is a storage f assert.equal(typeof g.value_object, 'string'); await h.bucket.delete(String(g.value_object)); assert.equal((await h.fetch(bytes('A'), bytes('S'))).status, 503); + assert.deepEqual(await decodeResponse(await h.fetch(bytes('missing'), bytes('S'))), { + kind: 'fallback', + key: bytes('A'), + }); } finally { await h.close(); } diff --git a/packages/remote-cache/wrangler.jsonc b/packages/remote-cache/wrangler.jsonc index e0bed4736..33f239c85 100644 --- a/packages/remote-cache/wrangler.jsonc +++ b/packages/remote-cache/wrangler.jsonc @@ -17,6 +17,7 @@ }, ], "r2_buckets": [{ "binding": "ARTIFACTS", "bucket_name": "vp-remote-cache" }], + // Local defaults. Operator setup/upgrade derives stable IDs for each Worker and binding. "ratelimits": [ { "name": "READ_LIMITER", "namespace_id": "1001", "simple": { "limit": 600, "period": 60 } }, { "name": "STORE_LIMITER", "namespace_id": "1002", "simple": { "limit": 30, "period": 60 } }, From ab6892815475b4d2248104c757afbe77c672e15f Mon Sep 17 00:00:00 2001 From: MK Date: Wed, 16 Sep 2026 00:04:58 +0800 Subject: [PATCH 09/17] ci(remote-cache): use one persistent staging deployment Co-authored-by: GPT-6 Codex --- .github/workflows/remote-cache-deploy.yml | 45 ++-- .../workflows/remote-cache-preview-events.yml | 91 ------- packages/remote-cache/README.md | 2 +- packages/remote-cache/docs/e2e-plan.md | 61 +++-- packages/remote-cache/package.json | 1 - packages/remote-cache/scripts/ci.ts | 104 +------ packages/remote-cache/test/deployed.test.ts | 254 ++++++------------ packages/remote-cache/test/operator.test.ts | 2 +- 8 files changed, 146 insertions(+), 414 deletions(-) delete mode 100644 .github/workflows/remote-cache-preview-events.yml diff --git a/.github/workflows/remote-cache-deploy.yml b/.github/workflows/remote-cache-deploy.yml index 5cee1fc8d..d130df94c 100644 --- a/.github/workflows/remote-cache-deploy.yml +++ b/.github/workflows/remote-cache-deploy.yml @@ -1,4 +1,4 @@ -name: Remote cache deployment +name: Remote cache staging on: pull_request: @@ -20,23 +20,17 @@ on: - pnpm-workspace.yaml - vite.config.ts workflow_dispatch: - inputs: - full: - description: Check real Cron cleanup (manual runs cannot authorize HTTP stores) - type: boolean - default: true permissions: contents: read -# Share the group with PR teardown. Do not cancel a partially completed deployment. +# All PRs and main share staging. Finish deployment, smoke tests, and notification before replacing it. concurrency: - group: remote-cache-cloudflare-${{ github.event.pull_request.number || 'main' }} + group: remote-cache-staging cancel-in-progress: false env: REMOTE_CACHE_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - REMOTE_CACHE_PR_NUMBER: ${{ github.event.pull_request.number }} REMOTE_CACHE_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} REMOTE_CACHE_RESOURCE_PREFIX: ${{ vars.REMOTE_CACHE_RESOURCE_PREFIX || 'vp-cache-ci' }} REMOTE_CACHE_WORKERS_SUBDOMAIN: ${{ vars.REMOTE_CACHE_WORKERS_SUBDOMAIN }} @@ -53,21 +47,21 @@ jobs: persist-credentials: false - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 - run: pnpm check-remote-cache - - name: Explain fork preview availability + - name: Explain staging deployment availability if: >- always() && github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name != github.repository || github.event.pull_request.user.login == 'dependabot[bot]') uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 with: script: | - await core.summary.addHeading('Remote cache preview').addRaw( + await core.summary.addHeading('Remote cache staging').addRaw( 'Fork and Dependabot PRs run local checks without Cloudflare credentials. ' + - 'A maintainer can copy the reviewed commit to an internal branch and open a PR for a preview. ' + + 'A maintainer can copy the reviewed commit to an internal branch and open a PR to deploy to staging. ' + 'No live deployment passed verification for this PR.' ).write(); deploy: - name: Deploy and verify + name: Deploy staging and run smoke tests needs: check if: >- vars.REMOTE_CACHE_DEPLOY_ENABLED == 'true' && @@ -75,7 +69,7 @@ jobs: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name != 'workflow_dispatch' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 15 environment: name: remote-cache-staging url: ${{ steps.deploy.outputs.endpoint }} @@ -100,18 +94,17 @@ jobs: ref: ${{ env.REMOTE_CACHE_SOURCE_SHA }} persist-credentials: false - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 - - name: Deploy isolated Cloudflare resources + - name: Create or update persistent staging resources id: deploy run: pnpm --filter @voidzero-dev/remote-cache ci:deploy env: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - - name: Test the deployed Worker, D1, and R2 + - name: Run smoke tests against the staging Worker, D1, and R2 run: pnpm --filter @voidzero-dev/remote-cache e2e env: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - REMOTE_CACHE_E2E_FULL: ${{ github.event_name == 'push' || inputs.full == true }} - name: Save verification results and manual fixtures if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -122,7 +115,7 @@ jobs: if-no-files-found: warn notify: - name: Update PR verification instructions + name: Update PR staging verification instructions if: >- always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login != 'dependabot[bot]' @@ -139,6 +132,7 @@ jobs: DEPLOY_ENDPOINT: ${{ needs.deploy.outputs.endpoint }} with: script: | + // Reuse the marker so existing PR comments are updated. const marker = ''; const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: context.issue.number }); if (pr.state !== 'open' || pr.head.sha !== process.env.REMOTE_CACHE_SOURCE_SHA) return; @@ -148,23 +142,24 @@ jobs: let status; if (passed) { const endpoint = process.env.DEPLOY_ENDPOINT; - const expected = `https://${process.env.REMOTE_CACHE_RESOURCE_PREFIX}-pr-${pr.number}.${process.env.REMOTE_CACHE_WORKERS_SUBDOMAIN}.workers.dev/projects/manual`; + const expected = `https://${process.env.REMOTE_CACHE_RESOURCE_PREFIX}-staging.${process.env.REMOTE_CACHE_WORKERS_SUBDOMAIN}.workers.dev/projects/manual`; if (endpoint !== expected) throw new Error('Unexpected deployment endpoint'); - status = `Cloudflare deployment and PR e2e checks passed. You can now perform manual verification.\n\n` + + status = `Cloudflare staging deployment and smoke tests passed. You can now perform manual verification.\n\n` + `Endpoint: ${endpoint}\n\n` + `Download the **remote-cache-e2e-${context.runId}-${process.env.GITHUB_RUN_ATTEMPT}** artifact from the [workflow run](${run}). ` + `It contains \`manual-fetch.cbor\`, \`manual-manifest.json\`, and \`report.json\`.\n\n` + - `PR checks cover public reads and rejected writes. Main-branch push checks also cover authorized HTTP stores and real Cron cleanup. ` + - `The endpoint remains available until this PR closes or a later commit replaces it.`; + `PR checks cover public reads and rejected writes. Main-branch push checks also cover authorized HTTP stores. ` + + `This staging endpoint is shared by all PRs and main. A later deployment replaces its code and manual fixture. ` + + `Compare the deployment ID in the response with the artifact before manual verification. Closing this PR does not remove staging.`; } else if (process.env.CHECK_RESULT !== 'success') { - status = `The source checks did not pass. No verified Cloudflare preview is ready. See the [workflow run](${run}).`; + status = `The source checks did not pass. No verified Cloudflare staging deployment is ready. See the [workflow run](${run}).`; } else if (process.env.DEPLOY_RESULT === 'skipped') { status = `Cloudflare deployment is disabled. Configure the staging environment described in the [e2e plan](${docs}), ` + `then set \`REMOTE_CACHE_DEPLOY_ENABLED=true\` and rerun this workflow. No live deployment passed verification.`; } else { - status = `Cloudflare deployment or e2e verification failed. The preview is not marked ready. See the [workflow run](${run}).`; + status = `Cloudflare deployment or e2e verification failed. The staging deployment is not marked ready. See the [workflow run](${run}).`; } - const body = `${marker}\n### Remote cache preview\n\nCommit: \`${pr.head.sha}\`\n\n${status}\n\n[Manual checks and complete e2e plan](${docs}).`; + const body = `${marker}\n### Remote cache staging\n\nCommit: \`${pr.head.sha}\`\n\n${status}\n\n[Manual checks and complete e2e plan](${docs}).`; const comments = await github.paginate(github.rest.issues.listComments, { ...context.repo, issue_number: pr.number, per_page: 100 }); const previous = comments.find(c => c.user?.login === 'github-actions[bot]' && c.body?.includes(marker)); if (previous) await github.rest.issues.updateComment({ ...context.repo, comment_id: previous.id, body }); diff --git a/.github/workflows/remote-cache-preview-events.yml b/.github/workflows/remote-cache-preview-events.yml deleted file mode 100644 index 21e9ce074..000000000 --- a/.github/workflows/remote-cache-preview-events.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Remote cache preview cleanup - -on: - pull_request: - types: [closed] - paths: - - packages/remote-cache/** - - .github/workflows/remote-cache-*.yml - - package.json - - pnpm-lock.yaml - - pnpm-workspace.yaml - - vite.config.ts - workflow_dispatch: - inputs: - pr: - description: Closed PR number whose preview resources should be removed - type: string - required: true - -permissions: {} - -jobs: - cleanup: - name: Remove closed PR resources - if: >- - vars.REMOTE_CACHE_DEPLOY_ENABLED == 'true' && - ((github.event.action == 'closed' && github.event.pull_request.head.repo.full_name == github.repository && - github.event.pull_request.user.login != 'dependabot[bot]') || - (github.event_name == 'workflow_dispatch' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))) - runs-on: ubuntu-latest - timeout-minutes: 35 - concurrency: - group: remote-cache-cloudflare-${{ github.event.pull_request.number || inputs.pr }} - cancel-in-progress: false - environment: remote-cache-staging - permissions: - contents: read - pull-requests: read - env: - REMOTE_CACHE_PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr }} - REMOTE_CACHE_SOURCE_SHA: ${{ github.sha }} - REMOTE_CACHE_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - REMOTE_CACHE_RESOURCE_PREFIX: ${{ vars.REMOTE_CACHE_RESOURCE_PREFIX || 'vp-cache-ci' }} - REMOTE_CACHE_WORKERS_SUBDOMAIN: ${{ vars.REMOTE_CACHE_WORKERS_SUBDOMAIN }} - steps: - - name: Verify that the PR is closed - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - with: - script: | - const number = process.env.REMOTE_CACHE_PR_NUMBER; - if (!/^[1-9][0-9]{0,9}$/.test(number)) throw new Error('Invalid PR number'); - const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: Number(number) }); - if (pr.state !== 'closed') throw new Error('Only closed PR previews can be removed'); - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - with: - ref: ${{ github.event.repository.default_branch }} - persist-credentials: false - - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 - - run: pnpm --filter @voidzero-dev/remote-cache ci:cleanup - env: - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - - cleanup-notice: - if: always() && needs.cleanup.result != 'skipped' - needs: cleanup - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - pull-requests: write - steps: - - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - env: - PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr }} - CLEANUP_RESULT: ${{ needs.cleanup.result }} - with: - script: | - if (!/^[1-9][0-9]{0,9}$/.test(process.env.PR_NUMBER)) return; - const number = Number(process.env.PR_NUMBER); - const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: number }); - if (pr.state !== 'closed') return; - const marker = ''; - const comments = await github.paginate(github.rest.issues.listComments, { ...context.repo, issue_number: number, per_page: 100 }); - const previous = comments.find(c => c.user?.login === 'github-actions[bot]' && c.body?.includes(marker)); - if (!previous) return; - const run = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const status = process.env.CLEANUP_RESULT === 'success' - ? 'The PR is closed. Its Cloudflare preview resources were removed.' - : 'The PR is closed, but preview cleanup did not finish. Check the workflow and rerun cleanup after Cron or lifecycle completes.'; - await github.rest.issues.updateComment({ ...context.repo, comment_id: previous.id, - body: `${marker}\n### Remote cache preview\n\n${status}\n\n[Cleanup run](${run}).` }); diff --git a/packages/remote-cache/README.md b/packages/remote-cache/README.md index 6ca11564e..c13aedf14 100644 --- a/packages/remote-cache/README.md +++ b/packages/remote-cache/README.md @@ -41,7 +41,7 @@ From this package directory, `pnpm dev` starts Wrangler with local bindings. App ## Setup -For automatic PR previews, main-branch staging deployments, and tests against real Cloudflare resources, follow the [deployment and e2e plan](docs/e2e-plan.md). It includes GitHub environment configuration, the complete test matrix, manual verification, and preview teardown. +For continuous deployment and smoke tests against one persistent Cloudflare staging environment, follow the [deployment and e2e plan](docs/e2e-plan.md). Internal PRs and main-branch pushes share the same Worker, D1 database, and R2 bucket. The plan includes GitHub configuration, the complete test matrix, and manual verification. Closing a PR leaves staging available. Use a dedicated Worker, D1 database, and bucket. The operator requires `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` in its environment. The token needs account permissions for Workers Scripts, D1, and Workers R2 Storage, plus route/zone permissions if using a custom domain. Enable R2 in the account first. Credentials stay in the operator process and Wrangler; they are never stored in namespace policy or passed as command arguments. diff --git a/packages/remote-cache/docs/e2e-plan.md b/packages/remote-cache/docs/e2e-plan.md index 52c761205..ac8c73f6e 100644 --- a/packages/remote-cache/docs/e2e-plan.md +++ b/packages/remote-cache/docs/e2e-plan.md @@ -1,4 +1,4 @@ -# Cloudflare deployment and e2e verification +# Cloudflare staging deployment and e2e verification This plan checks the service through its public HTTP endpoints after deployment to Cloudflare. A deployment dry run or a passing Miniflare test is not evidence of a successful Cloudflare deployment. @@ -7,15 +7,15 @@ This plan checks the service through its public HTTP endpoints after deployment `.github/workflows/remote-cache-deploy.yml` runs when a PR or a push to `main` changes this package, its deployment workflows, or the root dependency/build configuration. It also supports manual runs from the default branch. 1. Check the exact source commit with `pnpm check-remote-cache`. -2. Create or update dedicated staging resources and apply D1 migrations. +2. Create or update the shared staging Worker, D1 database, and R2 bucket. Apply D1 migrations. 3. Check that the R2 bucket has no public domain. Seed public test fixtures through the authenticated operator API. -4. Test the deployed HTTP endpoints. Every response must identify the expected commit and workflow attempt through `X-Remote-Cache-Deployment`. +4. Run smoke tests against the deployed HTTP endpoints. Every response must identify the expected commit and workflow attempt through `X-Remote-Cache-Deployment`. 5. Save a JSON report and manual fixtures as a seven-day workflow artifact. 6. Update one PR comment with the commit, result, endpoint, artifact link, and manual instructions. A failed or skipped deployment never gets a ready message. -Each PR uses `-pr-` for its Worker, database, and bucket. Main uses `-main`. The default prefix is `vp-cache-ci`. Each deployment has three public namespaces: `e2e`, `other`, and `manual`. Resources contain synthetic data only. They are separate from production and from other PRs. +All internal PRs, main-branch pushes, and manual runs use `-staging` for the Worker, database, and bucket. The default name is `vp-cache-ci-staging`. Setup reuses existing resources. This single environment has three public namespaces: `e2e`, `other`, and `manual`. Resources contain synthetic data only and are separate from production. -Deployment and teardown use the same concurrency group. They do not cancel a resource operation in progress. An obsolete PR commit cannot start deployment or replace the current PR comment. A later commit can replace a preview at the same URL; manual checks must compare the deployment ID from the saved manifest. +The workflow uses one `remote-cache-staging` concurrency group for all branches. It finishes deployment, smoke tests, and PR notification before another run can replace staging. It does not cancel a run in progress. An obsolete PR commit cannot start deployment or replace its PR comment. A later run from any PR or `main` replaces the code and manual fixture at the same URL. Manual checks must compare the response deployment ID with the saved manifest. Closing a PR leaves the staging resources in place. ## GitHub and Cloudflare setup @@ -36,23 +36,23 @@ Set these **repository variables**, which the notification job also needs: | `REMOTE_CACHE_RESOURCE_PREFIX` | Optional; defaults to `vp-cache-ci`. A custom prefix must end in `-ci` and use at most 34 lowercase letters, digits, or hyphens | | `REMOTE_CACHE_DEPLOY_ENABLED` | Set to `true` after the environment and account are ready | -Enable R2 in the account. The token must allow resource creation and deletion as well as data access. No custom domain or DNS permission is needed. Do not attach production bindings or secrets to these Workers. The GitHub environment can require a maintainer review if the repository needs one before a deployment. +Enable R2 in the account. The token must allow resource provisioning, updates, and fixture data access. No custom domain or DNS permission is needed. Do not attach production bindings or secrets to the staging Worker. The GitHub environment can require a maintainer review if the repository needs one before a deployment. -The workflow exposes Cloudflare credentials only to deployment, verification, and cleanup commands. Dependency installation runs before those credentials enter the step environment. Internal PR contributors must be trusted to change deployment code. Fork and Dependabot PRs run local checks without Cloudflare credentials. Their check summary explains why no preview is available; they receive no deployment comment. Cleanup uses ordinary closed-PR events for internal branches and executes code from the default branch. To preview a fork change, a maintainer must copy the reviewed commit to a branch in this repository and open a PR. +The workflow exposes Cloudflare credentials only to deployment and smoke-test commands. Dependency installation runs before those credentials enter the step environment. Internal PR contributors must be trusted to change deployment code. Fork and Dependabot PRs run local checks without Cloudflare credentials. Their check summary explains why staging deployment is unavailable; they receive no deployment comment. To deploy a fork change to staging, a maintainer must copy the reviewed commit to a branch in this repository and open a PR. -Merge the workflow and cleanup script to the default branch before relying on automatic cleanup. GitHub can suppress `pull_request` workflows when a PR has merge conflicts. Use the manual cleanup workflow with the closed PR number if the close event does not run. Manual deployment dispatches use the default branch and cannot select an arbitrary PR revision. +GitHub can suppress `pull_request` workflows when a PR has merge conflicts. Resolve the conflicts to run its checks and deployment. Manual deployment dispatches use the default branch and cannot select an arbitrary PR revision. There is no closed-PR trigger, resource teardown job, or cleanup workflow. ## Authorization and test levels The production write policy requires a real GitHub token with `event_name=push`, the registered repository and owner IDs, its configured main branch, and the exact namespace audience. A PR token or a `workflow_dispatch` token cannot pass this policy. GitHub signs the event claim; the test runner cannot change it. -| Level | Trigger | What it proves | -| ------------------------------ | ------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | -| Local regression | Every normal CI run, on Linux/macOS/Windows | Protocol, storage state transitions, failures, and the e2e driver against workerd/D1/R2 emulators | -| PR Cloudflare verification | Related changes in an internal PR | Deployment revision, migrations, private R2, real public HTTP reads, real rejected OIDC writes, policy changes, and expiry | -| Main Cloudflare verification | Related push to `main` | All PR cases plus real authorized HTTP writes, multipart uploads, concurrent maximum payloads, and the real Cron schedule | -| Manual Cloudflare verification | Default-branch workflow dispatch | Public reads and rejected writes; `full=true` also waits for real Cron cleanup | -| Release / incident exercises | Maintainer-controlled staging session | Long-running limits, provider outages, lifecycle delays, restore, rollback, and real hostile workflow identities | +| Level | Trigger | What it proves | +| ---------------------------- | ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| Local regression | Every normal CI run, on Linux/macOS/Windows | Protocol, storage state transitions, failures, and the e2e driver against workerd/D1/R2 emulators | +| PR staging smoke tests | Related changes in an internal PR | Deployment revision, migrations, private R2, real public HTTP reads, real rejected OIDC writes, policy changes, and expiry | +| Main staging smoke tests | Related push to `main` | All PR cases plus real authorized HTTP writes, multipart uploads, replacement, concurrency, and quota checks | +| Manual staging smoke tests | Default-branch workflow dispatch | The same public-read and rejected-write checks as PR runs | +| Release / incident exercises | Maintainer-controlled staging session | Maximum payloads, real Cron, long-running limits, provider outages, lifecycle delays, restore, rollback, and hostile workflow identities | The runner obtains tokens directly from GitHub with `id-token: write`. It checks the token request host, rejects redirects, bounds responses, and caches each audience separately for three minutes. Tokens remain in memory. Reports and artifacts contain no GitHub or Cloudflare credentials. The Worker has no alternate issuer, test signing key, administrative HTTP endpoint, or weakened write policy. @@ -60,7 +60,7 @@ The operator seeds immutable objects and generation rows for read tests. This va ## Automated case matrix -“Local” refers to the existing regression suite and the new shared e2e-driver tests. “PR” and “Main” refer to real Cloudflare runs. +“Local” refers to the existing regression suite and the new shared e2e-driver tests. “PR” and “Main” refer to smoke tests against the shared Cloudflare staging environment. Status assertions follow the [local RFC](0001-remote-cache.md#4-http-api-mapping). Fetch misses and unavailable blobs return plain-text `404`. An exact match with a missing or unreadable value returns `503`. A fallback returns only the stored key and makes no R2 read. @@ -87,28 +87,28 @@ Status assertions follow the [local RFC](0001-remote-cache.md#4-http-api-mapping | R2 multipart upload | Yes | No | Yes | Upload more than 5 MiB; download and compare the SHA-256 digest | | Concurrent same-key stores | Yes | No | Yes | The selected value and blob belong to one complete generation | | Malformed store / quota | Yes | No | Yes | `400` / `503`; preserve the previous value and mappings | -| Maximum payload concurrency | Yes | No | Yes | Two simultaneous 64 MiB blobs with 4 MiB values succeed and preserve digests | +| Maximum payload concurrency | Yes | No | Release exercise | Two simultaneous 64 MiB blobs with 4 MiB values succeed and preserve digests | | Expired generation | Yes | Yes | Yes | Metadata and blob become unavailable immediately | -| Real Cron deletion | Direct scheduled-handler call | No | Yes | Scheduled cleanup removes the D1 generation and both R2 objects, and preserves accounting | +| Real Cron deletion | Direct scheduled-handler call | No | Release exercise | Scheduled cleanup removes the D1 generation and both R2 objects, and preserves accounting | | JWT claim classes / clock bounds / JWKS failures | Yes | Real wrong audience/event | Real valid identity | Full forged/fork/owner/visibility/ref/event/time matrix remains in local tests; controlled live identity exercises supplement it | | R2 failure injection / lease and policy races | Yes | Selected missing-object cases | Selected quota cases | Local deterministic failures never publish partial state or release charges before deletion | | Operator retries and teardown guards | Yes | Provisioning path | Provisioning path | Retry setup without duplicate resources; recover interrupted policy checks; refuse unrelated resources | -The main run has 14 grouped e2e checks; the PR run has eight. A group can contain several requests and assertions. The local driver test runs the same main suite, including maximum payloads, with a directly invoked scheduled handler. That does not replace the Cloudflare Cron check. Both modes also check that the advertised manual endpoint serves its expected fixture. +Main smoke runs have 13 grouped checks; PR and manual smoke runs have eight. A group can contain several requests and assertions. All modes check that the advertised manual endpoint serves its expected fixture. Local driver tests repeat both smoke modes against reused storage and retain the 14-group full suite, including maximum payloads and a directly invoked scheduled handler. Routine deployments do not wait for real Cron or run the maximum-payload case. Those require a controlled release exercise on Cloudflare. ## Pass criteria and failure evidence Every executed assertion must pass. The e2e command exits nonzero on failure. `report.json` records each completed group, its status, duration, deployment ID, endpoint, and authorization mode. The workflow uploads partial reports after failures too. Reports use generic error classes; credentials and opaque request bodies are excluded. -Deployment readiness permits two minutes for the expected revision and fixture to appear. Normal HTTP requests have a 30-second deadline; stores have 130 seconds. Behavior tests do not retry failed writes or turn errors into passes. Only readiness and asynchronous cleanup use bounded polling. +Deployment readiness permits two minutes for the expected revision and fixture to appear. Normal HTTP requests have a 30-second deadline; stores have 130 seconds. Behavior tests do not retry failed writes or turn errors into passes. Only deployment readiness uses polling during smoke tests. -The main run waits at most 25 minutes for its expired fixture to disappear through the real five-minute Cron schedule. Cloudflare notes that Cron changes can take up to 15 minutes to propagate. The test also checks that both R2 objects disappear, the selected entry is gone, and charged bytes match generation totals. It never calls a fake public scheduled endpoint or performs the sentinel deletion itself. +Smoke tests check that expired fixtures immediately become unavailable. The service’s normal Cron schedule later removes expired objects and releases accounting. A separate release exercise must verify real Cron execution: both R2 objects and the selected entry disappear, and charged bytes match generation totals. A directly invoked local scheduled handler does not prove the deployed schedule works. Use the artifact and workflow logs to identify the first failed group. For server diagnosis, correlate `X-Request-Id` with sampled Worker logs. Do not retry a failed assertion solely to obtain a green result. A new run should follow a fix or an identified transient deployment issue. ## Manual verification from a PR -1. Wait for the PR comment to say that Cloudflare deployment and PR e2e checks passed. Confirm that its commit is the current PR head. +1. Wait for the PR comment to say that Cloudflare staging deployment and smoke tests passed. Confirm that its commit is the current PR head. 2. Open the linked workflow run. Download its `remote-cache-e2e--` artifact and extract the files. 3. Read `manual-manifest.json`. It contains the expected deployment ID, endpoint, blob URL, and expected public fixture contents. 4. From the extracted directory, set `CACHE_ENDPOINT` to the manifest's endpoint and run: @@ -131,17 +131,15 @@ For a CBOR decoder already available in this repository, run this from `packages node --input-type=module -e 'import { readFileSync } from "node:fs"; import { decode } from "cborg"; console.log(decode(readFileSync(process.argv[1])))' /path/to/fetch-response.cbor ``` -PR previews intentionally reject publication tokens from PR workflows. Use the main staging workflow to validate successful publication. The manual namespace contains synthetic public data and expires after one day unless another run refreshes it. +Staging rejects publication tokens from PR and manual workflows. Use a main-branch push to validate successful publication. The manual namespace contains synthetic public data and expires after one day unless another run refreshes it. If the deployment ID differs from the manifest, another run has replaced staging; use the artifact for the currently deployed revision. -## Cleanup, retries, and cost limits +## Persistent resources, retries, and test data -Each CI deployment has a 2 GB byte budget, 1,000 entries, and 2,000 associations. Pending and retired objects remain charged. After verification, the runner expires `e2e` and `other` data and gives in-flight operations the normal ten-minute cleanup grace. It retains the `manual` fixture for developer checks. Subsequent runs explicitly restore CI-owned policy switches in case a prior process stopped during a withdrawal test. +Staging has a 2 GB byte budget, 1,000 entries, and 2,000 associations. Pending and retired objects remain charged. After verification, the runner expires `e2e` and `other` test data and gives in-flight operations the normal ten-minute deletion grace. It retains the `manual` fixture for developer checks. Normal cache retention, Cron, and R2 lifecycle rules bound test data growth; the workflow never deletes the Worker, database, or bucket. -Closing an internal PR triggers `.github/workflows/remote-cache-preview-events.yml`. The cleanup job executes code from the default branch, confirms that the PR is closed, and checks resource ownership. It disables access, waits for Cron to drain generations, then deletes the empty bucket, D1 database, and Worker. It does not delete main staging. The PR comment reports cleanup success or failure. +Subsequent runs restore CI-owned policy switches in case a prior process stopped during a withdrawal or quota test. Setup retries discover resources by name and apply outstanding migrations. Closing a PR has no effect on staging. No environment teardown or cleanup command is required. -If Cron or orphan multipart uploads delay cleanup, the job fails instead of claiming that resources are gone. R2 lifecycle can require a day to abort an unrecorded upload. Rerun the cleanup workflow with the closed PR number after the backlog drains. Failed resource creation and partial teardown can also be retried. A nonempty bucket always blocks final removal. - -Staging policies belong to CI. Do not use these names for an operator-managed production cache. Monitor resource counts and Cloudflare allowances. Resource budgets do not guarantee zero cost. Disable `REMOTE_CACHE_DEPLOY_ENABLED` to stop automatic deployments; remove existing PR resources explicitly after pending runs finish. +Staging policies belong to CI. Do not use these names for an operator-managed production cache. Monitor storage, deletion backlogs, and Cloudflare allowances. Resource budgets do not guarantee zero cost. Set `REMOTE_CACHE_DEPLOY_ENABLED=false` to stop automatic deployments; the existing staging environment remains available. ## Release and incident exercises @@ -150,6 +148,7 @@ Before a production release, record the commit, workflow URL, Cloudflare plan, r | Exercise | Procedure and acceptance criterion | | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Provider failure and recovery | Restrict D1/R2 access in staging, exercise reads and stores, then restore access. Existing mappings survive; failures remain explicit; retries recover | +| Maximum payloads and real Cron | Run two concurrent authorized stores with 4 MiB values and 64 MiB blobs; verify their digests. Expire a fixture and observe scheduled D1/R2 deletion and accounting release | | Upload interruption | Disconnect a real authorized multipart request after its first R2 part. Verify unchanged mappings, retained reservation, upload abortion, and eventual byte release | | Lease/token expiry and policy race | Delay a staging upload across expiry or change policy before publication. Neither mapping changes; old data remains readable | | Real hostile workflow identities | Use controlled fork, tag, PR, `pull_request_target`, and `workflow_run` jobs. All writes fail; public reads still work | @@ -159,6 +158,6 @@ Before a production release, record the commit, workflow URL, Cloudflare plan, r | Partial restore | Restore D1 without deleted R2 data. Expect `503` for exact matches with missing values, key-only fallbacks, and `404` for missing blobs; withdraw or replace the namespace before reuse | | Load and CPU | Measure deployed CPU, memory, D1 rows/latency, R2 operations, and cleanup lag under sustained concurrency. Compare observed costs with configured budgets | -These are explicit release exercises, not claims that fault injection or multi-day lifecycle behavior already ran in the PR workflow. Workers Free support still needs provider CPU measurements within its limits. +These are explicit release exercises, not claims that maximum payloads, real Cron, fault injection, or multi-day lifecycle behavior ran in the staging smoke workflow. Workers Free support still needs provider CPU measurements within its limits. -References: [Cloudflare GitHub Actions deployment](https://developers.cloudflare.com/workers/ci-cd/external-cicd/github-actions/), [Cron propagation](https://developers.cloudflare.com/workers/configuration/cron-triggers/), [GitHub OIDC claims](https://docs.github.com/en/actions/reference/security/oidc), and [secure workflow use](https://docs.github.com/en/actions/reference/security/secure-use). +References: [Cloudflare GitHub Actions deployment](https://developers.cloudflare.com/workers/ci-cd/external-cicd/github-actions/), [workflow concurrency](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency), [GitHub OIDC claims](https://docs.github.com/en/actions/reference/security/oidc), and [secure workflow use](https://docs.github.com/en/actions/reference/security/secure-use). diff --git a/packages/remote-cache/package.json b/packages/remote-cache/package.json index 897e81954..3641cc450 100644 --- a/packages/remote-cache/package.json +++ b/packages/remote-cache/package.json @@ -14,7 +14,6 @@ "operator": "node --import tsx scripts/operator.ts", "benchmark": "node --import tsx scripts/benchmark.ts", "ci:deploy": "node --import tsx scripts/ci.ts deploy", - "ci:cleanup": "node --import tsx scripts/ci.ts cleanup", "e2e": "node --import tsx scripts/ci.ts test" }, "dependencies": { diff --git a/packages/remote-cache/scripts/ci.ts b/packages/remote-cache/scripts/ci.ts index f840ea59f..e553b2a43 100644 --- a/packages/remote-cache/scripts/ci.ts +++ b/packages/remote-cache/scripts/ci.ts @@ -1,17 +1,8 @@ import assert from 'node:assert/strict'; import { appendFile, mkdir, writeFile } from 'node:fs/promises'; import { URL, pathToFileURL } from 'node:url'; -import { setTimeout as delay } from 'node:timers/promises'; import { encode } from 'cborg'; -import { - ApiError, - operatorIO, - query, - readTemplate, - runOperator, - type Config, - type OperatorIO, -} from './operator.ts'; +import { ApiError, operatorIO, query, runOperator, type Config } from './operator.ts'; import { retireTestData, seedManual, type Admin } from './e2e/fixtures.ts'; import { runSuite, type Report } from './e2e/suite.ts'; @@ -28,9 +19,7 @@ export function settingsFrom(env: Record) { throw new Error( 'Set REMOTE_CACHE_WORKERS_SUBDOMAIN to the account subdomain, without workers.dev', ); - const pr = env['REMOTE_CACHE_PR_NUMBER'] || undefined; - if (pr && !/^[1-9][0-9]{0,9}$/.test(pr)) throw new Error('Invalid PR number'); - const name = `${prefix}-${pr ? `pr-${pr}` : 'main'}`; + const name = `${prefix}-staging`; const repository = env['GITHUB_REPOSITORY']; if (!repository || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) throw new Error('Invalid repository'); @@ -45,22 +34,21 @@ export function settingsFrom(env: Record) { if (!run || !attempt || !/^\d+$/.test(run) || !/^\d+$/.test(attempt)) throw new Error('Invalid workflow run identity'); const defaultBranch = env['REMOTE_CACHE_DEFAULT_BRANCH']; - const writes = - !pr && - env['GITHUB_EVENT_NAME'] === 'push' && - env['GITHUB_REF'] === `refs/heads/${defaultBranch}`; - if (!pr && env['GITHUB_REF'] !== `refs/heads/${defaultBranch}`) - throw new Error('The main staging deployment requires the default branch'); + const event = env['GITHUB_EVENT_NAME']; + if (!['pull_request', 'push', 'workflow_dispatch'].includes(event ?? '')) + throw new Error('Unsupported staging deployment event'); + const onDefaultBranch = env['GITHUB_REF'] === `refs/heads/${defaultBranch}`; + if (event !== 'pull_request' && !onDefaultBranch) + throw new Error('Push and manual staging deployments require the default branch'); + const writes = event === 'push' && onDefaultBranch; return { name, - pr, repository, repositoryId, revision, deployment: `${revision}-${run}-${attempt}`, origin: `https://${name}.${subdomain}.workers.dev`, writes, - full: writes || env['REMOTE_CACHE_E2E_FULL'] === 'true', }; } type Settings = ReturnType; @@ -296,8 +284,8 @@ async function verify(settings: Settings) { request: fetch, token: githubTokens(process.env), writes: settings.writes, - full: settings.full, - cron: settings.full, + full: false, + cron: false, async record(value) { report = value; await writeFile(new URL('report.json', resultsDir), JSON.stringify(report, null, 2) + '\n'); @@ -316,81 +304,13 @@ async function verify(settings: Settings) { } } -export async function cleanup( - settings: Settings, - io: OperatorIO = operatorIO, - sleep: (ms: number) => Promise = delay, -) { - if (!settings.pr) throw new Error('Automatic teardown is restricted to PR resources'); - const config = await readTemplate(); - const databases = (await io.api(`/d1/database?name=${settings.name}&per_page=100`)) as { - name: string; - uuid: string; - }[]; - const db = databases.find((db) => db.name === settings.name); - config.name = settings.name; - config.r2_buckets[0]!.bucket_name = settings.name; - config.d1_databases[0]!.database_name = settings.name; - if (db) config.d1_databases[0]!.database_id = db.uuid; - await io.writeConfig(config); - if (db) { - const tables = await query( - io, - config, - "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'generations'", - ); - if (tables.length) { - const scopes = await query(io, config, 'SELECT repository_id, endpoint FROM scopes'); - if ( - scopes.some( - (scope) => - scope['repository_id'] !== settings.repositoryId || - !String(scope['endpoint']).startsWith(`${settings.origin}/projects/`), - ) - ) - throw new Error('Refusing to remove resources owned by another deployment'); - await query(io, config, 'UPDATE deployment SET enabled = 0, writes_enabled = 0'); - await query( - io, - config, - `UPDATE generations SET expires_at = 0, - lease_until = min(lease_until, unixepoch()), gc_after = min(gc_after, unixepoch() + 600)`, - ); - const until = Date.now() + 25 * 60000; - while ( - Number((await query(io, config, 'SELECT count(*) AS count FROM generations'))[0]!['count']) - ) { - if (Date.now() >= until) - throw new Error( - 'Cleanup is still pending; rerun the cleanup workflow after Cron or lifecycle finishes', - ); - await sleep(30000); - } - } - } - // Empty-bucket deletion also protects multipart uploads without a recorded ID. - try { - await io.api(`/r2/buckets/${settings.name}`); - await io.wrangler(['r2', 'bucket', 'delete', settings.name]); - } catch (error) { - if (!(error instanceof ApiError) || error.status !== 404) throw error; - } - if (db) await io.wrangler(['d1', 'delete', db.uuid, '--skip-confirmation']); - try { - await io.api(`/workers/scripts/${settings.name}`, 'DELETE'); - } catch (error) { - if (!(error instanceof ApiError) || error.status !== 404) throw error; - } -} - if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { try { const settings = settingsFrom(process.env); const command = process.argv[2]; if (command === 'deploy') await deploy(settings); else if (command === 'test') await verify(settings); - else if (command === 'cleanup') await cleanup(settings); - else throw new Error('Use deploy, test, or cleanup'); + else throw new Error('Use deploy or test'); } catch (error) { console.error(error instanceof Error ? error.message : 'Cloudflare CI failed'); process.exitCode = 1; diff --git a/packages/remote-cache/test/deployed.test.ts b/packages/remote-cache/test/deployed.test.ts index d1ddd768c..5976f8fe1 100644 --- a/packages/remote-cache/test/deployed.test.ts +++ b/packages/remote-cache/test/deployed.test.ts @@ -2,13 +2,13 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { harness } from './helpers.ts'; import { runSuite } from '../scripts/e2e/suite.ts'; -import { seed, seedManual, bytes, type Admin } from '../scripts/e2e/fixtures.ts'; -import { cleanup, githubTokens, settingsFrom } from '../scripts/ci.ts'; -import { ApiError, readTemplate, type OperatorIO } from '../scripts/operator.ts'; -import { randomUUID } from 'node:crypto'; +import { retireTestData, seedManual, type Admin } from '../scripts/e2e/fixtures.ts'; +import { githubTokens, settingsFrom } from '../scripts/ci.ts'; -for (const writes of [false, true]) - void test(`deployed HTTP suite runs against workerd with ${writes ? 'push' : 'PR'} claims`, async () => { +for (const mode of ['read-only smoke', 'push smoke', 'push full'] as const) + void test(`deployed HTTP suite runs against workerd: ${mode}`, async () => { + const writes = mode !== 'read-only smoke'; + const full = mode === 'push full'; const h = await harness({ deploymentId: 'suite-local', namespaces: ['test', 'other', 'e2e', 'manual'], @@ -42,43 +42,61 @@ for (const writes of [false, true]) return (await h.bucket.head(key)) !== null; }, }; - await seedManual(admin, 'suite-local'); - const report = await runSuite({ - origin: 'https://cache.example.com', - deployment: 'suite-local', - admin, - writes, - full: writes, - cron: true, - cronTimeoutMs: 5000, - pollMs: 1, - token: (aud) => - h.token({ - aud, - ...(writes ? {} : { event_name: 'pull_request', ref: 'refs/pull/718/merge' }), - }), - async request(url, init) { - // Serialize Node FormData before crossing Miniflare's fetch implementation. - const request = new Request(url, init); - const response = await h.mf.dispatchFetch(url, { - method: request.method, - headers: Object.fromEntries(request.headers), - ...(request.method === 'GET' ? {} : { body: await request.arrayBuffer() }), - }); - return new Response(await response.arrayBuffer(), { - status: response.status, - headers: response.headers, - }); - }, - }); - assert.equal(report.results.length, writes ? 14 : 8); - assert.ok(report.results.every((result) => result.status === 'passed')); + // Reuse storage for consecutive smoke runs, including previous test generations. + for (let attempt = 0; attempt < (full ? 1 : 2); attempt++) { + const manual = await seedManual(admin, 'suite-local'); + const report = await runSuite({ + origin: 'https://cache.example.com', + deployment: 'suite-local', + admin, + writes, + full, + cron: full, + cronTimeoutMs: 5000, + pollMs: 1, + token: (aud) => + h.token({ + aud, + ...(writes ? {} : { event_name: 'pull_request', ref: 'refs/pull/718/merge' }), + }), + async request(url, init) { + // Serialize Node FormData before crossing Miniflare's fetch implementation. + const request = new Request(url, init); + const response = await h.mf.dispatchFetch(url, { + method: request.method, + headers: Object.fromEntries(request.headers), + ...(request.method === 'GET' ? {} : { body: await request.arrayBuffer() }), + }); + return new Response(await response.arrayBuffer(), { + status: response.status, + headers: response.headers, + }); + }, + }); + assert.equal(report.results.length, writes ? (full ? 14 : 13) : 8); + assert.ok(report.results.every((result) => result.status === 'passed')); + assert.equal( + report.results.some((result) => result.name.startsWith('real Cron')), + full, + ); + await retireTestData(admin); + assert.ok(await admin.exists(manual.valueObject)); + assert.ok(await admin.exists(manual.blobObject)); + assert.equal( + ( + await admin.sql('SELECT state FROM generations WHERE generation_id = ?', [ + manual.generation, + ]) + )[0]?.['state'], + 'ready', + ); + } } finally { await h.close(); } }); -void test('CI settings isolate PRs and only enable writes for default-branch pushes', () => { +void test('CI shares persistent staging across events and only permits writes on default-branch pushes', () => { const env = { REMOTE_CACHE_WORKERS_SUBDOMAIN: 'example', GITHUB_REPOSITORY: 'owner/repo', @@ -90,18 +108,33 @@ void test('CI settings isolate PRs and only enable writes for default-branch pus GITHUB_REF: 'refs/heads/main', GITHUB_EVENT_NAME: 'push', }; - assert.equal(settingsFrom(env).writes, true); - assert.equal(settingsFrom({ ...env, GITHUB_EVENT_NAME: 'workflow_dispatch' }).writes, false); - const pr = settingsFrom({ - ...env, - REMOTE_CACHE_PR_NUMBER: '718', - GITHUB_EVENT_NAME: 'pull_request', - }); - assert.equal(pr.name, 'vp-cache-ci-pr-718'); - assert.equal(pr.writes, false); + const main = settingsFrom(env); + assert.equal(main.name, 'vp-cache-ci-staging'); + assert.equal(main.origin, 'https://vp-cache-ci-staging.example.workers.dev'); + assert.equal(main.writes, true); + for (const override of [ + { GITHUB_EVENT_NAME: 'workflow_dispatch' }, + { GITHUB_EVENT_NAME: 'pull_request', GITHUB_REF: 'refs/pull/718/merge' }, + { GITHUB_EVENT_NAME: 'pull_request', GITHUB_REF: 'refs/pull/719/merge' }, + ]) { + const settings = settingsFrom({ ...env, ...override }); + assert.equal(settings.name, main.name); + assert.equal(settings.origin, main.origin); + assert.equal(settings.writes, false); + } + const retry = settingsFrom({ ...env, GITHUB_RUN_ATTEMPT: '3' }); + assert.equal(retry.origin, main.origin); + assert.notEqual(retry.deployment, main.deployment); + assert.equal( + settingsFrom({ ...env, REMOTE_CACHE_RESOURCE_PREFIX: 'custom-ci' }).name, + 'custom-ci-staging', + ); assert.throws(() => settingsFrom({ ...env, REMOTE_CACHE_RESOURCE_PREFIX: 'production' })); - assert.throws(() => settingsFrom({ ...env, REMOTE_CACHE_PR_NUMBER: '718;rm -rf /' })); - assert.throws(() => settingsFrom({ ...env, GITHUB_REF: 'refs/heads/feature' })); + for (const GITHUB_EVENT_NAME of ['push', 'workflow_dispatch']) + assert.throws(() => + settingsFrom({ ...env, GITHUB_EVENT_NAME, GITHUB_REF: 'refs/heads/feature' }), + ); + assert.throws(() => settingsFrom({ ...env, GITHUB_EVENT_NAME: 'pull_request_target' })); }); void test('OIDC requests use only GitHub, reject redirects, and reuse tokens only for the same audience', async () => { @@ -132,126 +165,3 @@ void test('OIDC requests use only GitHub, reject redirects, and reuse tokens onl ); assert.equal(calls.length, 2); }); - -void test('CI cleanup checks ownership, drains through Cron, preserves nonempty resources, and retries', async () => { - const settings = settingsFrom({ - REMOTE_CACHE_WORKERS_SUBDOMAIN: 'example', - REMOTE_CACHE_PR_NUMBER: '718', - GITHUB_REPOSITORY: 'owner/repo', - GITHUB_REPOSITORY_ID: '123', - REMOTE_CACHE_SOURCE_SHA: 'a'.repeat(40), - GITHUB_RUN_ID: '42', - GITHUB_RUN_ATTEMPT: '1', - REMOTE_CACHE_DEFAULT_BRANCH: 'main', - GITHUB_REF: 'refs/heads/main', - GITHUB_EVENT_NAME: 'workflow_dispatch', - }); - const h = await harness(); - let config = await readTemplate(); - let database = true, - bucket = true, - worker = true; - const deleted: string[] = []; - const io: OperatorIO = { - async api(path, method, body) { - if (path.startsWith('/d1/database?')) - return database ? [{ name: settings.name, uuid: 'test-db' }] : []; - if (path.endsWith('/query')) { - const { sql, params } = body as { sql: string; params: (string | number | null)[] }; - return [ - await h.db - .prepare(sql) - .bind(...params) - .all(), - ]; - } - if (path.startsWith('/r2/')) { - if (!bucket) throw new ApiError(404); - return {}; - } - if (path.startsWith('/workers/') && method === 'DELETE') { - if (!worker) throw new ApiError(404); - assert.equal(database, false); - worker = false; - deleted.push('worker'); - return null; - } - throw new Error('Unexpected Cloudflare API call'); - }, - async github() { - throw new Error('Unexpected GitHub API call'); - }, - async lifecycle() { - throw new Error('Unexpected lifecycle change'); - }, - async readConfig() { - return config; - }, - async writeConfig(value) { - config = value; - }, - print() {}, - async wrangler(args) { - if (args[0] === 'r2') { - if ((await h.bucket.list()).objects.length) throw new Error('Bucket is not empty'); - bucket = false; - deleted.push('bucket'); - } else if (args[0] === 'd1') { - assert.equal(bucket, false); - database = false; - deleted.push('database'); - } else throw new Error('Unexpected Wrangler command'); - }, - }; - try { - await h.db - .prepare("UPDATE scopes SET endpoint = ? || '/projects/' || scope_id") - .bind(settings.origin) - .run(); - const admin: Admin = { - async sql(sql, params = []) { - return ( - await h.db - .prepare(sql) - .bind(...params) - .all() - ).results; - }, - async put(key, value) { - await h.bucket.put(key, value); - }, - async delete(key) { - await h.bucket.delete(key); - }, - async exists(key) { - return (await h.bucket.head(key)) !== null; - }, - }; - await seed(admin, 'other', 'cleanup', bytes('value'), bytes('blob')); - const orphan = `other/${randomUUID()}/value`; - await h.bucket.put(orphan, 'orphan'); - await assert.rejects(cleanup({ ...settings, repositoryId: '999' }, io), /another deployment/); - assert.deepEqual(deleted, []); - await assert.rejects( - cleanup(settings, io, async () => { - await h.db.prepare('UPDATE generations SET gc_after = 0').run(); - await (await h.mf.getWorker()).scheduled(); - }), - /not empty/, - ); - assert.equal(database, true); - assert.equal(worker, true); - assert.deepEqual(deleted, []); - assert.equal( - (await h.db.prepare('SELECT charged_bytes FROM deployment').first())!.charged_bytes, - 0, - ); - await h.bucket.delete(orphan); - await cleanup(settings, io); - await cleanup(settings, io); - assert.deepEqual(deleted, ['bucket', 'database', 'worker']); - await assert.rejects(cleanup({ ...settings, pr: undefined }, io), /restricted to PR/); - } finally { - await h.close(); - } -}); diff --git a/packages/remote-cache/test/operator.test.ts b/packages/remote-cache/test/operator.test.ts index 4fc7ab13e..ae2af1027 100644 --- a/packages/remote-cache/test/operator.test.ts +++ b/packages/remote-cache/test/operator.test.ts @@ -153,7 +153,7 @@ void test('operator setup validates before writes and preserves withdrawal on re void test('operator upgrades isolate rate counters by Worker and retain them across revisions', async () => { const namespaces = new Set(); - for (const name of ['vp-cache-ci-pr-1', 'vp-cache-ci-pr-2', 'vp-cache-ci-main', 'production']) { + for (const name of ['vp-cache-ci-staging', 'custom-ci-staging', 'production']) { let config = await readTemplate(); config.name = name; const limits = config.ratelimits.map((binding) => binding.simple); From 117a7ff1c319df1cc5d93ef017299c9dd05389bf Mon Sep 17 00:00:00 2001 From: MK Date: Wed, 16 Sep 2026 00:25:14 +0800 Subject: [PATCH 10/17] ci(remote-cache): use repository secrets for staging Co-authored-by: GPT-6 Codex --- .github/workflows/remote-cache-deploy.yml | 5 +---- packages/remote-cache/docs/e2e-plan.md | 15 +++++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/remote-cache-deploy.yml b/.github/workflows/remote-cache-deploy.yml index d130df94c..fa3a25b92 100644 --- a/.github/workflows/remote-cache-deploy.yml +++ b/.github/workflows/remote-cache-deploy.yml @@ -70,9 +70,6 @@ jobs: (github.event_name != 'workflow_dispatch' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) runs-on: ubuntu-latest timeout-minutes: 15 - environment: - name: remote-cache-staging - url: ${{ steps.deploy.outputs.endpoint }} permissions: contents: read id-token: write @@ -154,7 +151,7 @@ jobs: } else if (process.env.CHECK_RESULT !== 'success') { status = `The source checks did not pass. No verified Cloudflare staging deployment is ready. See the [workflow run](${run}).`; } else if (process.env.DEPLOY_RESULT === 'skipped') { - status = `Cloudflare deployment is disabled. Configure the staging environment described in the [e2e plan](${docs}), ` + + status = `Cloudflare deployment is disabled. Configure the repository secrets and variables described in the [e2e plan](${docs}), ` + `then set \`REMOTE_CACHE_DEPLOY_ENABLED=true\` and rerun this workflow. No live deployment passed verification.`; } else { status = `Cloudflare deployment or e2e verification failed. The staging deployment is not marked ready. See the [workflow run](${run}).`; diff --git a/packages/remote-cache/docs/e2e-plan.md b/packages/remote-cache/docs/e2e-plan.md index ac8c73f6e..f9521b06f 100644 --- a/packages/remote-cache/docs/e2e-plan.md +++ b/packages/remote-cache/docs/e2e-plan.md @@ -21,22 +21,29 @@ The workflow uses one `remote-cache-staging` concurrency group for all branches. Use a dedicated Cloudflare staging account or dedicated resources in an account that permits this CI workload. Select Workers Paid for the configured payload sizes. The `paid` operator profile controls cleanup batch size; it does not purchase a subscription. -Create the GitHub environment `remote-cache-staging` and add these environment secrets: +Add these repository secrets under **Settings → Secrets and variables → Actions**. Repository write access is sufficient. The workflow does not require a GitHub environment: | Secret | Purpose | | ----------------------- | ---------------------------------------------------------------------------- | | `CLOUDFLARE_ACCOUNT_ID` | Account that owns the staging resources | | `CLOUDFLARE_API_TOKEN` | Account-scoped Workers Scripts, D1, and Workers R2 Storage write permissions | +Use the GitHub CLI from an interactive terminal to enter each secret at its prompt: + +```sh +gh secret set CLOUDFLARE_ACCOUNT_ID --repo voidzero-dev/vite-task +gh secret set CLOUDFLARE_API_TOKEN --repo voidzero-dev/vite-task +``` + Set these **repository variables**, which the notification job also needs: | Variable | Value | | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | | `REMOTE_CACHE_WORKERS_SUBDOMAIN` | Account subdomain only, for example `example` for `example.workers.dev` | | `REMOTE_CACHE_RESOURCE_PREFIX` | Optional; defaults to `vp-cache-ci`. A custom prefix must end in `-ci` and use at most 34 lowercase letters, digits, or hyphens | -| `REMOTE_CACHE_DEPLOY_ENABLED` | Set to `true` after the environment and account are ready | +| `REMOTE_CACHE_DEPLOY_ENABLED` | Set to `true` after the repository secrets, variables, and Cloudflare account are ready | -Enable R2 in the account. The token must allow resource provisioning, updates, and fixture data access. No custom domain or DNS permission is needed. Do not attach production bindings or secrets to the staging Worker. The GitHub environment can require a maintainer review if the repository needs one before a deployment. +Enable R2 in the account. The token must allow resource provisioning, updates, and fixture data access. No custom domain or DNS permission is needed. Do not attach production bindings or secrets to the staging Worker. The workflow exposes Cloudflare credentials only to deployment and smoke-test commands. Dependency installation runs before those credentials enter the step environment. Internal PR contributors must be trusted to change deployment code. Fork and Dependabot PRs run local checks without Cloudflare credentials. Their check summary explains why staging deployment is unavailable; they receive no deployment comment. To deploy a fork change to staging, a maintainer must copy the reviewed commit to a branch in this repository and open a PR. @@ -160,4 +167,4 @@ Before a production release, record the commit, workflow URL, Cloudflare plan, r These are explicit release exercises, not claims that maximum payloads, real Cron, fault injection, or multi-day lifecycle behavior ran in the staging smoke workflow. Workers Free support still needs provider CPU measurements within its limits. -References: [Cloudflare GitHub Actions deployment](https://developers.cloudflare.com/workers/ci-cd/external-cicd/github-actions/), [workflow concurrency](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency), [GitHub OIDC claims](https://docs.github.com/en/actions/reference/security/oidc), and [secure workflow use](https://docs.github.com/en/actions/reference/security/secure-use). +References: [GitHub repository secrets](https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/use-secrets#creating-secrets-for-a-repository), [Cloudflare GitHub Actions deployment](https://developers.cloudflare.com/workers/ci-cd/external-cicd/github-actions/), [workflow concurrency](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency), [GitHub OIDC claims](https://docs.github.com/en/actions/reference/security/oidc), and [secure workflow use](https://docs.github.com/en/actions/reference/security/secure-use). From d7534cdd241c63556580f5f531c9f6792b8fe4d8 Mon Sep 17 00:00:00 2001 From: MK Date: Wed, 16 Sep 2026 00:45:53 +0800 Subject: [PATCH 11/17] fix(remote-cache): preserve trigger bodies in D1 migrations Co-authored-by: GPT-6 Codex --- packages/remote-cache/migrations/0001_cache.sql | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/remote-cache/migrations/0001_cache.sql b/packages/remote-cache/migrations/0001_cache.sql index 2d9d8f145..31f73fa03 100644 --- a/packages/remote-cache/migrations/0001_cache.sql +++ b/packages/remote-cache/migrations/0001_cache.sql @@ -72,6 +72,7 @@ CREATE INDEX associations_target ON associations(scope_id, target_key); CREATE TABLE maintenance (id INTEGER PRIMARY KEY CHECK (id = 1), scope_id TEXT NOT NULL, secondary_key BLOB NOT NULL); INSERT INTO maintenance VALUES (1, '', X''); +-- Parenthesize CASE expressions so the D1 REST parser preserves each trigger body. -- Lifecycle rules must never shorten the life of an already published generation. CREATE TRIGGER initial_retention AFTER INSERT ON scopes BEGIN UPDATE deployment SET retention_high_water_seconds = max(retention_high_water_seconds, NEW.retention_seconds); @@ -82,7 +83,7 @@ END; CREATE TRIGGER scope_limit BEFORE INSERT ON scopes WHEN NOT EXISTS (SELECT 1 FROM scopes WHERE scope_id = NEW.scope_id) BEGIN - SELECT CASE WHEN (SELECT count(*) FROM scopes) >= 100 THEN RAISE(ABORT, 'cache_scope_limit') END; + SELECT (CASE WHEN (SELECT count(*) FROM scopes) >= 100 THEN RAISE(ABORT, 'cache_scope_limit') END); END; -- Direct policy edits must invalidate uploads that captured the previous policy. @@ -94,13 +95,13 @@ BEGIN END; CREATE TRIGGER reserve_capacity BEFORE INSERT ON generations BEGIN - SELECT CASE WHEN NOT EXISTS ( + SELECT (CASE WHEN NOT EXISTS ( SELECT 1 FROM scopes s, deployment d WHERE s.scope_id = NEW.scope_id AND s.enabled = 1 AND s.writes_enabled = 1 AND d.enabled = 1 AND d.writes_enabled = 1 AND s.policy_version = NEW.policy_version AND NEW.token_exp > unixepoch() AND s.charged_bytes + NEW.charged_bytes <= s.byte_limit AND d.charged_bytes + NEW.charged_bytes <= d.byte_limit - ) THEN RAISE(ABORT, 'cache_admission_denied') END; + ) THEN RAISE(ABORT, 'cache_admission_denied') END); END; CREATE TRIGGER charge_generation AFTER INSERT ON generations BEGIN UPDATE scopes SET charged_bytes = charged_bytes + NEW.charged_bytes WHERE scope_id = NEW.scope_id; @@ -118,10 +119,10 @@ END; CREATE TRIGGER count_entry BEFORE INSERT ON entries WHEN NOT EXISTS (SELECT 1 FROM entries WHERE scope_id = NEW.scope_id AND key = NEW.key) BEGIN - SELECT CASE WHEN EXISTS ( + SELECT (CASE WHEN EXISTS ( SELECT 1 FROM scopes s, deployment d WHERE s.scope_id = NEW.scope_id AND (s.entry_count >= s.entry_limit OR d.entry_count >= d.entry_limit) - ) THEN RAISE(ABORT, 'cache_entry_limit') END; + ) THEN RAISE(ABORT, 'cache_entry_limit') END); UPDATE scopes SET entry_count = entry_count + 1 WHERE scope_id = NEW.scope_id; UPDATE deployment SET entry_count = entry_count + 1; END; @@ -132,10 +133,10 @@ END; CREATE TRIGGER count_association BEFORE INSERT ON associations WHEN NOT EXISTS (SELECT 1 FROM associations WHERE scope_id = NEW.scope_id AND secondary_key = NEW.secondary_key) BEGIN - SELECT CASE WHEN EXISTS ( + SELECT (CASE WHEN EXISTS ( SELECT 1 FROM scopes s, deployment d WHERE s.scope_id = NEW.scope_id AND (s.association_count >= s.association_limit OR d.association_count >= d.association_limit) - ) THEN RAISE(ABORT, 'cache_association_limit') END; + ) THEN RAISE(ABORT, 'cache_association_limit') END); UPDATE scopes SET association_count = association_count + 1 WHERE scope_id = NEW.scope_id; UPDATE deployment SET association_count = association_count + 1; END; @@ -150,7 +151,7 @@ CREATE TRIGGER publish_generation AFTER UPDATE OF state ON generations WHEN OLD.state = 'uploading' AND NEW.state = 'ready' BEGIN UPDATE generations SET state = 'retired', retired_at = unixepoch(), - gc_after = CASE WHEN expires_at > unixepoch() THEN unixepoch() + 600 ELSE unixepoch() END + gc_after = (CASE WHEN expires_at > unixepoch() THEN unixepoch() + 600 ELSE unixepoch() END) WHERE generation_id = (SELECT generation_id FROM entries WHERE scope_id = NEW.scope_id AND key = NEW.key) AND state = 'ready'; INSERT INTO entries (scope_id, key, generation_id) VALUES (NEW.scope_id, NEW.key, NEW.generation_id) ON CONFLICT (scope_id, key) DO UPDATE SET generation_id = excluded.generation_id; From 311ba633097af2f49e62cd1b05972ffc9d3e0808 Mon Sep 17 00:00:00 2001 From: MK Date: Wed, 16 Sep 2026 12:24:15 +0800 Subject: [PATCH 12/17] refactor(remote-cache): simplify service and deployment code Remove the obsolete cache size study and its RFC link. Co-authored-by: GPT-6 Codex --- .../remote-cache/docs/0001-remote-cache.md | 2 +- .../docs/remote-cache-size-study/README.md | 107 ------ .../docs/remote-cache-size-study/measure.py | 175 --------- .../docs/remote-cache-size-study/results.json | 355 ------------------ .../docs/remote-cache-size-study/sources.json | 71 ---- packages/remote-cache/scripts/ci.ts | 56 ++- packages/remote-cache/scripts/e2e/fixtures.ts | 16 +- packages/remote-cache/scripts/e2e/suite.ts | 100 ++--- packages/remote-cache/scripts/http.ts | 21 ++ packages/remote-cache/scripts/operator.ts | 143 +++---- packages/remote-cache/src/cbor.ts | 25 +- packages/remote-cache/src/database.ts | 56 ++- packages/remote-cache/src/index.ts | 9 +- packages/remote-cache/src/multipart.ts | 6 +- packages/remote-cache/src/store.ts | 6 +- packages/remote-cache/src/streams.ts | 8 +- packages/remote-cache/test/deployed.test.ts | 17 +- 17 files changed, 257 insertions(+), 916 deletions(-) delete mode 100644 packages/remote-cache/docs/remote-cache-size-study/README.md delete mode 100644 packages/remote-cache/docs/remote-cache-size-study/measure.py delete mode 100644 packages/remote-cache/docs/remote-cache-size-study/results.json delete mode 100644 packages/remote-cache/docs/remote-cache-size-study/sources.json create mode 100644 packages/remote-cache/scripts/http.ts diff --git a/packages/remote-cache/docs/0001-remote-cache.md b/packages/remote-cache/docs/0001-remote-cache.md index 14b61019e..78d338d3e 100644 --- a/packages/remote-cache/docs/0001-remote-cache.md +++ b/packages/remote-cache/docs/0001-remote-cache.md @@ -682,7 +682,7 @@ Every exact response transfers the value, even if client validation prevents a b ### Artifact-size evidence -Use 5 MB for a scenario with small outputs. On 2026-09-07, we measured published Vite frontend outputs from four established products. The [artifact study](remote-cache-size-study/README.md) records pinned sources, exact bytes, and a reproduction script. +Use 5 MB for a scenario with small outputs. On 2026-09-07, we measured published Vite frontend outputs from four established products. | Product/release | Output MB | `tar.zst` MB | `tar.zst` MB without source maps | | ------------------------------- | --------: | -----------: | -------------------------------: | diff --git a/packages/remote-cache/docs/remote-cache-size-study/README.md b/packages/remote-cache/docs/remote-cache-size-study/README.md deleted file mode 100644 index 44afada6a..000000000 --- a/packages/remote-cache/docs/remote-cache-size-study/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# Vite SaaS frontend artifact measurements - -Measured on 2026-09-07 for [the remote-cache RFC](../0001-remote-cache.md#13-free-and-paid-capacity-comparison). - -The four sampled frontend outputs compress to **4.77–34.71 MB** with zstd level 3. A 5 MB result is useful for a scenario with small outputs. Three of these four releases exceed it. Use an additional 50 MB planning case for mature SaaS frontends. This sample does not establish the average cache size across users or tasks. - -## Projects and scope - -We selected established products with public source and downloadable release artifacts. Each repository had more than 20,000 GitHub stars when we checked it. This method selects recognizable projects. It does not produce a random sample of Vite users. The sample includes commercial products with different source licenses. - -| Project | Product | GitHub stars at observation | Measured release | Published frontend output | -| ------------------------------------------------------ | ---------------------------- | --------------------------: | -------------------------------------- | -------------------------------------------------- | -| [Directus](https://github.com/directus/directus) | Data platform / headless CMS | 37,786 | `@directus/app@17.1.1`, from `v12.3.1` | npm package `dist/` | -| [Docmost](https://github.com/docmost/docmost) | Collaborative wiki | 21,602 | `v0.95.0` | Docker build's `/app/apps/client/dist/` COPY layer | -| [Hoppscotch](https://github.com/hoppscotch/hoppscotch) | API development platform | 80,228 | `2026.8.0` | Docker build's `/site/selfhost-web/` COPY layer | -| [n8n](https://github.com/n8n-io/n8n) | Workflow automation | 203,576 | `n8n-editor-ui@2.16.2` | npm package `dist/` | - -These measurements cover **published build products**. We did not run local builds or measure the vendors' private cloud deployments. Docker samples use `linux/amd64` manifests. We selected the layer that copies the frontend from the build stage. This layer precedes runtime dependency installation and startup transformations. - -We exclude the Docker base image, server code, dependencies, and unrelated npm package files. - -The pinned source confirms Vite usage: - -| Project | Build evidence | Output/configuration evidence | -| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Directus | [`build: vite build`](https://github.com/directus/directus/blob/973be10df8b0305569dc0dc53e187c648133c8d6/app/package.json) | [Vite configuration](https://github.com/directus/directus/blob/973be10df8b0305569dc0dc53e187c648133c8d6/app/vite.config.js) | -| Docmost | [`build: tsc && vite build`](https://github.com/docmost/docmost/blob/4132dd597c956a27423607d008708c0e214690da/apps/client/package.json) | [Vite configuration](https://github.com/docmost/docmost/blob/4132dd597c956a27423607d008708c0e214690da/apps/client/vite.config.ts), [Dockerfile COPY](https://github.com/docmost/docmost/blob/4132dd597c956a27423607d008708c0e214690da/Dockerfile) | -| Hoppscotch | [`generate` calls `build`, which invokes Vite](https://github.com/hoppscotch/hoppscotch/blob/ac145e7f758151b41fd46d3e5f513886ce9068ba/packages/hoppscotch-selfhost-web/package.json) | [Vite configuration](https://github.com/hoppscotch/hoppscotch/blob/ac145e7f758151b41fd46d3e5f513886ce9068ba/packages/hoppscotch-selfhost-web/vite.config.ts), [production Dockerfile](https://github.com/hoppscotch/hoppscotch/blob/ac145e7f758151b41fd46d3e5f513886ce9068ba/prod.Dockerfile) | -| n8n | [`build` invokes `vite build`](https://github.com/n8n-io/n8n/blob/9bdde69954a4d7d1569d37b6fd3a3f55f55b297a/packages/frontend/editor-ui/package.json) | [Vite configuration](https://github.com/n8n-io/n8n/blob/9bdde69954a4d7d1569d37b6fd3a3f55f55b297a/packages/frontend/editor-ui/vite.config.mts) | - -[sources.json](sources.json) records artifact URLs, SHA-256 digests, source commits, npm provenance URLs, and Docker manifest/layer digests. We checked npm downloads against the registry's SHA-512 integrity values. We checked Docker layers against their SHA-256 digests. - -We obtained npm source commits from the registry's provenance records. The Docker source references identify the corresponding release tags. We did not verify attestation signatures or independently rebuild the releases. - -## Measured sizes - -All sizes use decimal MB (`1 MB = 1,000,000 bytes`). “Output” sums regular-file bytes, including static assets and source maps in the selected directory. “Archive” measures one sorted GNU tar stream compressed with `zstd -3 -T1`, using zstd `1.5.7`. We normalize tar paths and metadata. Archive sizes estimate the compressed output payload. A complete cache entry also includes task metadata. - -| Project | Files | Output MB | Archive MB | Source-map MB before compression | Archive MB with `.map` files omitted | -| ---------- | ----: | --------: | ---------: | -------------------------------: | -----------------------------------: | -| Directus | 401 | 20.81 | **6.97** | 0 | 6.97 | -| Docmost | 230 | 14.83 | **4.77** | 0 | 4.77 | -| Hoppscotch | 471 | 127.53 | **32.18** | 89.82 | 12.18 | -| n8n | 1,477 | 162.76 | **34.71** | 118.66 | 13.15 | - -Source maps account for about 70% of Hoppscotch's uncompressed output and 73% of n8n's. Their configurations enable source maps for these builds. n8n also enables its legacy-browser plugin for releases. Hoppscotch includes a TypeScript worker. n8n includes worker and WebAssembly assets. - -A page's initial JavaScript download omits much of this build output. Its size cannot substitute for the cache size. - -The last column measures the same files without source maps. It does not represent another build. Without maps, Hoppscotch's compressed archive measures 12.18 MB and n8n's measures 13.15 MB. Both remain above 5 MB. - -A cache must preserve the outputs that its task requires. These measurements do not justify silent removal of source maps. The published frontend directories for Directus and Docmost contain no `.map` files. - -These archives contain frontend output files only. A complete cached task also needs terminal events and client validation metadata. Under [PR #713](https://github.com/voidzero-dev/vite-task/pull/713), the client encodes these in an opaque value and optional blob. The Worker does not define or interpret their internal format. - -A cached task can produce additional files outside `dist/`. Release packaging can omit these files. Measure their bytes during implementation before you assign a complete size to each result. Backend and shared-package builds are separate task results unless the operator caches them as one task. - -## Effect on the free storage budget - -The table uses the RFC's 8 GB application budget and seven-day retention. Each store creates a different exact key at a steady rate. The ceilings include only the measured output archives: - -| Project-sized result | New results/day within 8 GB | Storage at 200 new results/day, seven days | -| -------------------- | --------------------------: | -----------------------------------------: | -| Directus | 164 | 9.75 GB | -| Docmost | 239 | 6.68 GB | -| Hoppscotch | 35 | 45.06 GB | -| n8n | 32 | 48.59 GB | - -Calculate the ceiling as `floor(8,000,000,000 / (archive_bytes * 7))`. These upper bounds include only artifacts. Client metadata, logs, pending uploads, retired generations, and delayed deletion reduce them. - -A new exact key stores another complete archive, even when many assets match the previous build. Repeated stores to the same key replace its value and blob. The old generation remains only for a short download grace period. Input changes do not necessarily create a different exact key. Read hits do not create another copy. Count separate namespaces and client compatibility identities when the service stores separate results. - -For a **50 MB planning case for complete results**, the same budget supports at most **22 new distinct keys/day** with seven-day retention. This excludes spare capacity for operation. At 200 new distinct keys/day, storage reaches 70 GB. - -Under the RFC's operation assumptions, this higher budget costs about **$0.90/month in R2 storage** beyond the included 10 GB. Workers can remain Free if the streaming implementation meets its CPU limit. The estimate uses steady usage, a 30-day month, and [R2 Standard pricing](https://developers.cloudflare.com/r2/pricing/), which we rechecked on 2026-09-09. The default profile rejects additional stores instead of increasing its budget. - -Use 5 MB for a scenario with small outputs and 50 MB for mature frontend builds. All four measured archives fit the RFC's 64 MiB blob limit. These values are planning inputs. They do not estimate population averages. - -A few new keys for complete frontend outputs each day can fit the free budget. Hundreds of different keys each day require smaller results, shorter retention, or more storage. Frequent overwrites can retain much less data but still consume operations. - -The number of readers does not determine this storage case. In version 1, only explicit CI publication from the main branch adds remote results. Public readers need no credentials or subscription for each user. - -This sample covers one release per product and favors mature applications. It does not measure these values: - -- Average size weighted by store counts. -- Daily change rate. -- Exact-key replacement rate. -- Cache hit rate. -- The fraction of ordinary users that stay free. - -The RFC's limited production trial still needs these measurements across task types and successive changes. - -## Reproduce - -Run from the repository root with Python 3 and the `zstd` CLI installed: - -```sh -python3 packages/remote-cache/docs/remote-cache-size-study/measure.py \ - --cache-dir /tmp/vite-cache-size-study \ - --output /tmp/vite-cache-size-study-results.json -``` - -The script downloads about 84 MB of pinned npm archives and Docker layers. It checks SHA-256 digests and decompresses the downloads into temporary tar files for inspection. It does not install packages, execute project code, or start containers. It writes compressed comparison archives in the cache directory. - -Allow about 600 MB of disk space. Registry availability and anonymous Docker pull limits can affect repeated runs. - -[results.json](results.json) contains exact byte counts, compressed archive digests, file-type totals, and the five largest files for each sample. Use those counts for calculations. The tables round values for readability. diff --git a/packages/remote-cache/docs/remote-cache-size-study/measure.py b/packages/remote-cache/docs/remote-cache-size-study/measure.py deleted file mode 100644 index 795e7da43..000000000 --- a/packages/remote-cache/docs/remote-cache-size-study/measure.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 -"""Measure pinned, published Vite outputs without installing or running projects. - -Requires Python 3 and the zstd CLI. Usage: - python3 measure.py --cache-dir /tmp/vite-cache-size-study --output results.json -""" - -import argparse -import collections -import gzip -import hashlib -import json -from pathlib import Path, PurePosixPath -import re -import shutil -import subprocess -import tarfile -import urllib.parse -import urllib.request - - -def read_url(url, headers=None): - return urllib.request.urlopen( - urllib.request.Request( - url, headers={"User-Agent": "vite-task-cache-size-study", **(headers or {})} - ), - timeout=60, - ) - - -def sha256(path): - digest = hashlib.sha256() - with path.open("rb") as source: - for block in iter(lambda: source.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def download(sample, cache): - target = cache / (sample["id"] + ".tgz") - if not target.exists() or sha256(target) != sample["sha256"]: - headers = {} - if sample.get("docker_repository"): - query = urllib.parse.urlencode( - { - "service": "registry.docker.io", - "scope": "repository:" + sample["docker_repository"] + ":pull", - } - ) - with read_url("https://auth.docker.io/token?" + query) as response: - token = json.load(response)["token"] - headers["Authorization"] = "Bearer " + token - partial = target.with_suffix(".partial") - with read_url(sample["url"], headers) as response, partial.open("wb") as output: - shutil.copyfileobj(response, output) - if sha256(partial) != sample["sha256"]: - raise ValueError("Source digest mismatch: " + sample["id"]) - partial.replace(target) - return target - - -def compress(source, members, destination): - # Match the engine's ordinary zstd level (3). Normalize GNU tar headers, - # sort paths, and include regular files only. No filesystem extraction. - with destination.open("wb") as output: - process = subprocess.Popen( - ["zstd", "-3", "-T1", "-q", "-c"], stdin=subprocess.PIPE, stdout=output - ) - try: - with tarfile.open(fileobj=process.stdin, mode="w|", format=tarfile.GNU_FORMAT) as archive: - for name, member in members: - header = tarfile.TarInfo("outputs/" + name) - header.size = member.size - header.mode = 0o644 - with source.extractfile(member) as data: - archive.addfile(header, data) - finally: - process.stdin.close() - status = process.wait() - if status: - raise RuntimeError("zstd failed: " + str(status)) - return {"bytes": destination.stat().st_size, "sha256": sha256(destination)} - - -def measure(sample, cache): - packed = download(sample, cache) - unpacked = cache / (sample["id"] + ".tar") - # Decompress once so sorted reads do not repeatedly rewind a gzip stream. - with gzip.open(packed, "rb") as source, unpacked.open("wb") as output: - shutil.copyfileobj(source, output) - - with tarfile.open(unpacked) as source: - prefix = sample["prefix"] - members = [] - for member in source: - if not member.name.startswith(prefix): - continue - name = member.name[len(prefix):] - if member.isdir(): - continue - if not member.isfile() or ".." in PurePosixPath(name).parts or name.startswith("/"): - raise ValueError("Unsupported output member: " + member.name) - members.append((name, member)) - members.sort(key=lambda item: item[0]) - if not members or len({name for name, _ in members}) != len(members): - raise ValueError("Empty or duplicate output paths: " + sample["id"]) - - by_extension = collections.defaultdict(lambda: {"files": 0, "bytes": 0}) - for name, member in members: - group = by_extension[PurePosixPath(name).suffix or "(none)"] - group["files"] += 1 - group["bytes"] += member.size - without_maps = [(name, member) for name, member in members if not name.endswith(".map")] - complete = compress(source, members, cache / (sample["id"] + ".tar.zst")) - no_maps = ( - compress(source, without_maps, cache / (sample["id"] + "-no-maps.tar.zst")) - if len(without_maps) != len(members) - else complete.copy() - ) - return { - "id": sample["id"], - "source_sha256": sample["sha256"], - "source_download_bytes": packed.stat().st_size, - "prefix": prefix, - "files": len(members), - "output_bytes": sum(member.size for _, member in members), - "tar_zstd": complete, - "without_maps": { - "files": len(without_maps), - "output_bytes": sum(member.size for _, member in without_maps), - "tar_zstd": no_maps, - }, - "by_extension": dict(sorted(by_extension.items())), - "largest_files": [ - {"path": name, "bytes": member.size} - for name, member in sorted(members, key=lambda item: item[1].size, reverse=True)[:5] - ], - } - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--cache-dir", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - samples = json.loads(Path(__file__).with_name("sources.json").read_text()) - args.cache_dir.mkdir(parents=True, exist_ok=True) - results = [] - for sample in samples["samples"]: - result = measure(sample, args.cache_dir) - results.append(result) - print( - sample["id"], - "files=" + str(result["files"]), - "output_bytes=" + str(result["output_bytes"]), - "zstd_bytes=" + str(result["tar_zstd"]["bytes"]), - "without_maps_zstd_bytes=" + str(result["without_maps"]["tar_zstd"]["bytes"]), - flush=True, - ) - version = subprocess.check_output(["zstd", "--version"], text=True) - args.output.write_text( - json.dumps( - { - "measurement_date": samples["measurement_date"], - "zstd_version": re.search(r"\bv(\d+\.\d+\.\d+)\b", version).group(1), - "method": "Sorted GNU tar regular files; outputs/ prefix; mode 0644; uid/gid/mtime 0; zstd -3 -T1 stream", - "samples": results, - }, - indent=2, - ) + "\n" - ) - - -if __name__ == "__main__": - main() diff --git a/packages/remote-cache/docs/remote-cache-size-study/results.json b/packages/remote-cache/docs/remote-cache-size-study/results.json deleted file mode 100644 index 93fa5f257..000000000 --- a/packages/remote-cache/docs/remote-cache-size-study/results.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "measurement_date": "2026-09-07", - "zstd_version": "1.5.7", - "method": "Sorted GNU tar regular files; outputs/ prefix; mode 0644; uid/gid/mtime 0; zstd -3 -T1 stream", - "samples": [ - { - "id": "directus", - "source_sha256": "aeb56c7f70c6ba09782a5b649d0ab2cdf9cd746ab6dcaaa80bb3db9490f64f94", - "source_download_bytes": 7521159, - "prefix": "package/dist/", - "files": 401, - "output_bytes": 20806766, - "tar_zstd": { - "bytes": 6967198, - "sha256": "bf7b16909bfb430f6c31605e2026b5ae0b6f458040943fcdf9b3a60178cd51ae" - }, - "without_maps": { - "files": 401, - "output_bytes": 20806766, - "tar_zstd": { - "bytes": 6967198, - "sha256": "bf7b16909bfb430f6c31605e2026b5ae0b6f458040943fcdf9b3a60178cd51ae" - } - }, - "by_extension": { - ".css": { - "files": 5, - "bytes": 719721 - }, - ".html": { - "files": 1, - "bytes": 1694 - }, - ".ico": { - "files": 1, - "bytes": 1150 - }, - ".js": { - "files": 369, - "bytes": 17819785 - }, - ".png": { - "files": 1, - "bytes": 14915 - }, - ".svg": { - "files": 1, - "bytes": 155481 - }, - ".woff": { - "files": 11, - "bytes": 972248 - }, - ".woff2": { - "files": 12, - "bytes": 1121772 - } - }, - "largest_files": [ - { - "path": "assets/index.C9zFwJTK.entry.js", - "bytes": 6724943 - }, - { - "path": "assets/shader-background-B-taT_fk.js", - "bytes": 1449199 - }, - { - "path": "assets/index-BMHQmbsG.css", - "bytes": 699610 - }, - { - "path": "assets/dist-chJWdeb_.js", - "bytes": 657240 - }, - { - "path": "assets/social-icon-Dxox5INO.js", - "bytes": 570996 - } - ] - }, - { - "id": "docmost", - "source_sha256": "4e7fffb4a1e8feb4e7213d2e8701c09df286de364177860e1c230e2374a25190", - "source_download_bytes": 5015721, - "prefix": "app/apps/client/dist/", - "files": 230, - "output_bytes": 14828770, - "tar_zstd": { - "bytes": 4769432, - "sha256": "cad27b93f705537b0abf7cfe7cc47dbb416c6e2120bd91dfce7316cc54a00157" - }, - "without_maps": { - "files": 230, - "output_bytes": 14828770, - "tar_zstd": { - "bytes": 4769432, - "sha256": "cad27b93f705537b0abf7cfe7cc47dbb416c6e2120bd91dfce7316cc54a00157" - } - }, - "by_extension": { - ".css": { - "files": 3, - "bytes": 660300 - }, - ".html": { - "files": 1, - "bytes": 1504 - }, - ".js": { - "files": 145, - "bytes": 11997158 - }, - ".json": { - "files": 13, - "bytes": 994411 - }, - ".png": { - "files": 4, - "bytes": 19808 - }, - ".svg": { - "files": 1, - "bytes": 1497 - }, - ".ttf": { - "files": 20, - "bytes": 513664 - }, - ".woff": { - "files": 20, - "bytes": 303116 - }, - ".woff2": { - "files": 23, - "bytes": 337312 - } - }, - "largest_files": [ - { - "path": "assets/index-BgC9hh6t.js", - "bytes": 3469981 - }, - { - "path": "assets/chunk-Z5NKEFVG-B3tsbN9K.js", - "bytes": 1821041 - }, - { - "path": "assets/excalidraw-utils-DacaKd1Y.js", - "bytes": 1350729 - }, - { - "path": "assets/chunk-NNHCCRGN-DlpIbxXb.js", - "bytes": 593667 - }, - { - "path": "assets/chunk-LZXEDZCA-4GiMzlKw.js", - "bytes": 536756 - } - ] - }, - { - "id": "hoppscotch", - "source_sha256": "9823634d487dac47140c008685e31f548e33644ba890c8bba66ba8b776aea98e", - "source_download_bytes": 33455805, - "prefix": "site/selfhost-web/", - "files": 471, - "output_bytes": 127526484, - "tar_zstd": { - "bytes": 32182975, - "sha256": "958a98d1aba6d70f315d593c6549898c803670d995513fb242d3958405bad2e0" - }, - "without_maps": { - "files": 279, - "output_bytes": 37709018, - "tar_zstd": { - "bytes": 12175754, - "sha256": "ed31d530071727007670fbd42e1009091cea063f0df6dc44ad0b82e7a8740c0c" - } - }, - "by_extension": { - "(none)": { - "files": 1, - "bytes": 13 - }, - ".css": { - "files": 13, - "bytes": 305947 - }, - ".html": { - "files": 1, - "bytes": 5219 - }, - ".ico": { - "files": 1, - "bytes": 15086 - }, - ".js": { - "files": 192, - "bytes": 33464992 - }, - ".map": { - "files": 192, - "bytes": 89817466 - }, - ".png": { - "files": 12, - "bytes": 2105842 - }, - ".svg": { - "files": 41, - "bytes": 372789 - }, - ".ttf": { - "files": 1, - "bytes": 121972 - }, - ".txt": { - "files": 1, - "bytes": 66 - }, - ".webmanifest": { - "files": 1, - "bytes": 845 - }, - ".woff2": { - "files": 14, - "bytes": 1313348 - }, - ".xml": { - "files": 1, - "bytes": 2899 - } - }, - "largest_files": [ - { - "path": "assets/index-DSckl1TP.js.map", - "bytes": 37094987 - }, - { - "path": "assets/ts.worker-BAgXcSxE.js.map", - "bytes": 18859801 - }, - { - "path": "assets/index-DSckl1TP.js", - "bytes": 12071781 - }, - { - "path": "assets/har-DRQ_daXH.js.map", - "bytes": 8016993 - }, - { - "path": "assets/ts.worker-BAgXcSxE.js", - "bytes": 7091224 - } - ] - }, - { - "id": "n8n", - "source_sha256": "14636be99659fca25eaa19ed20ec52c5286745eb3e1cef80b699adb23144eb1f", - "source_download_bytes": 37936612, - "prefix": "package/dist/", - "files": 1477, - "output_bytes": 162756849, - "tar_zstd": { - "bytes": 34705298, - "sha256": "7c916ddcdbb090123824840e55669de6500e294d9d419a6cb91cba5c7dc77cac" - }, - "without_maps": { - "files": 875, - "output_bytes": 44092939, - "tar_zstd": { - "bytes": 13146351, - "sha256": "4c1473f0d932be64dd7ee412b7ef1aa9ec0921a02010dc82a61bda7e9ebef21f" - } - }, - "by_extension": { - ".css": { - "files": 149, - "bytes": 1098799 - }, - ".gif": { - "files": 1, - "bytes": 158971 - }, - ".html": { - "files": 1, - "bytes": 18822 - }, - ".ico": { - "files": 1, - "bytes": 15086 - }, - ".js": { - "files": 626, - "bytes": 35334179 - }, - ".map": { - "files": 602, - "bytes": 118663910 - }, - ".png": { - "files": 24, - "bytes": 820355 - }, - ".svg": { - "files": 3, - "bytes": 367325 - }, - ".ttf": { - "files": 21, - "bytes": 569620 - }, - ".wasm": { - "files": 5, - "bytes": 3980250 - }, - ".webp": { - "files": 1, - "bytes": 315064 - }, - ".woff": { - "files": 21, - "bytes": 331316 - }, - ".woff2": { - "files": 22, - "bytes": 1083152 - } - }, - "largest_files": [ - { - "path": "assets/typescript.worker-C3GNzKj8.js.map", - "bytes": 18592487 - }, - { - "path": "assets/constants-j6z_fOkc.js.map", - "bytes": 5567385 - }, - { - "path": "assets/worker-FLvH_Wit.js.map", - "bytes": 5457892 - }, - { - "path": "assets/constants-legacy-i87_3HzL.js.map", - "bytes": 5432841 - }, - { - "path": "assets/typescript.worker-C3GNzKj8.js", - "bytes": 5049634 - } - ] - } - ] -} diff --git a/packages/remote-cache/docs/remote-cache-size-study/sources.json b/packages/remote-cache/docs/remote-cache-size-study/sources.json deleted file mode 100644 index 02cc4c162..000000000 --- a/packages/remote-cache/docs/remote-cache-size-study/sources.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "measurement_date": "2026-09-07", - "samples": [ - { - "id": "directus", - "project": "Directus", - "version": "@directus/app 17.1.1 (Directus v12.3.1)", - "url": "https://registry.npmjs.org/@directus/app/-/app-17.1.1.tgz", - "sha256": "aeb56c7f70c6ba09782a5b649d0ab2cdf9cd746ab6dcaaa80bb3db9490f64f94", - "prefix": "package/dist/", - "repository": "https://github.com/directus/directus", - "source_commit": "973be10df8b0305569dc0dc53e187c648133c8d6", - "vite_config": "app/vite.config.js", - "build_script": "app/package.json", - "provenance": "https://registry.npmjs.org/-/npm/v1/attestations/@directus%2fapp@17.1.1", - "stars": 37786 - }, - { - "id": "docmost", - "project": "Docmost", - "version": "v0.95.0", - "url": "https://registry-1.docker.io/v2/docmost/docmost/blobs/sha256:4e7fffb4a1e8feb4e7213d2e8701c09df286de364177860e1c230e2374a25190", - "sha256": "4e7fffb4a1e8feb4e7213d2e8701c09df286de364177860e1c230e2374a25190", - "prefix": "app/apps/client/dist/", - "repository": "https://github.com/docmost/docmost", - "source_commit": "4132dd597c956a27423607d008708c0e214690da", - "vite_config": "apps/client/vite.config.ts", - "build_script": "apps/client/package.json", - "docker_repository": "docmost/docmost", - "image": "docmost/docmost:0.95.0", - "image_manifest": "sha256:2b0a3f73e57951b726bf67259c4e8cb5269eb24530cde521003ff382d7ad8ab6", - "image_platform": "linux/amd64", - "layer_index": 9, - "dockerfile": "Dockerfile", - "stars": 21602 - }, - { - "id": "hoppscotch", - "project": "Hoppscotch", - "version": "2026.8.0", - "url": "https://registry-1.docker.io/v2/hoppscotch/hoppscotch-frontend/blobs/sha256:9823634d487dac47140c008685e31f548e33644ba890c8bba66ba8b776aea98e", - "sha256": "9823634d487dac47140c008685e31f548e33644ba890c8bba66ba8b776aea98e", - "prefix": "site/selfhost-web/", - "repository": "https://github.com/hoppscotch/hoppscotch", - "source_commit": "ac145e7f758151b41fd46d3e5f513886ce9068ba", - "vite_config": "packages/hoppscotch-selfhost-web/vite.config.ts", - "build_script": "packages/hoppscotch-selfhost-web/package.json", - "docker_repository": "hoppscotch/hoppscotch-frontend", - "image": "hoppscotch/hoppscotch-frontend:2026.8.0", - "image_manifest": "sha256:c5acdcfa5e00d3500ff809a33d25b10d36eb2fe2b8ef23e0fd74a157c6abc362", - "image_platform": "linux/amd64", - "layer_index": 18, - "dockerfile": "prod.Dockerfile", - "stars": 80228 - }, - { - "id": "n8n", - "project": "n8n", - "version": "n8n-editor-ui 2.16.2", - "url": "https://registry.npmjs.org/n8n-editor-ui/-/n8n-editor-ui-2.16.2.tgz", - "sha256": "14636be99659fca25eaa19ed20ec52c5286745eb3e1cef80b699adb23144eb1f", - "prefix": "package/dist/", - "repository": "https://github.com/n8n-io/n8n", - "source_commit": "9bdde69954a4d7d1569d37b6fd3a3f55f55b297a", - "vite_config": "packages/frontend/editor-ui/vite.config.mts", - "build_script": "packages/frontend/editor-ui/package.json", - "provenance": "https://registry.npmjs.org/-/npm/v1/attestations/n8n-editor-ui@2.16.2", - "stars": 203576 - } - ] -} diff --git a/packages/remote-cache/scripts/ci.ts b/packages/remote-cache/scripts/ci.ts index e553b2a43..2ab56bdf7 100644 --- a/packages/remote-cache/scripts/ci.ts +++ b/packages/remote-cache/scripts/ci.ts @@ -5,10 +5,21 @@ import { encode } from 'cborg'; import { ApiError, operatorIO, query, runOperator, type Config } from './operator.ts'; import { retireTestData, seedManual, type Admin } from './e2e/fixtures.ts'; import { runSuite, type Report } from './e2e/suite.ts'; +import { readJson } from './http.ts'; const resultsDir = new URL('../e2e-results/', import.meta.url); -export function settingsFrom(env: Record) { +interface Settings { + name: string; + repository: string; + repositoryId: string; + revision: string; + deployment: string; + origin: string; + writes: boolean; +} + +export function settingsFrom(env: Record): Settings { const prefix = env['REMOTE_CACHE_RESOURCE_PREFIX'] || 'vp-cache-ci'; if (!/^[a-z0-9][a-z0-9-]{0,30}-ci$/.test(prefix)) throw new Error( @@ -29,8 +40,8 @@ export function settingsFrom(env: Record) { const revision = env['REMOTE_CACHE_SOURCE_SHA']; if (!revision || !/^[a-f0-9]{40}$/.test(revision)) throw new Error('Use the full source commit SHA'); - const run = env['GITHUB_RUN_ID'], - attempt = env['GITHUB_RUN_ATTEMPT']; + const run = env['GITHUB_RUN_ID']; + const attempt = env['GITHUB_RUN_ATTEMPT']; if (!run || !attempt || !/^\d+$/.test(run) || !/^\d+$/.test(attempt)) throw new Error('Invalid workflow run identity'); const defaultBranch = env['REMOTE_CACHE_DEFAULT_BRANCH']; @@ -51,9 +62,8 @@ export function settingsFrom(env: Record) { writes, }; } -type Settings = ReturnType; -function checkConfig(config: Config, settings: Settings) { +function checkConfig(config: Config, settings: Settings): void { if ( config.name !== settings.name || config.r2_buckets[0]?.bucket_name !== settings.name || @@ -62,7 +72,7 @@ function checkConfig(config: Config, settings: Settings) { throw new Error('CI can only operate on its dedicated staging resources'); } -async function checkAccountOrigin(settings: Settings) { +async function checkAccountOrigin(settings: Settings): Promise { const account = (await operatorIO.api('/workers/subdomain')) as { subdomain: string }; assert.equal( new URL(settings.origin).hostname, @@ -72,8 +82,8 @@ async function checkAccountOrigin(settings: Settings) { } export function cloudflareAdmin(config: Config): Admin { - const account = process.env['CLOUDFLARE_ACCOUNT_ID'], - token = process.env['CLOUDFLARE_API_TOKEN']; + const account = process.env['CLOUDFLARE_ACCOUNT_ID']; + const token = process.env['CLOUDFLARE_API_TOKEN']; if (!account || !/^[a-f0-9]{32}$/.test(account) || !token) throw new Error('Cloudflare staging credentials are required'); const bucket = config.r2_buckets[0]!.bucket_name; @@ -111,13 +121,13 @@ export function cloudflareAdmin(config: Config): Admin { export function githubTokens( env: Record, request: typeof fetch = fetch, -) { +): (audience: string) => Promise { const cached = new Map(); - return async (audience: string) => { + return async function token(audience: string): Promise { const previous = cached.get(audience); if (previous && previous.until > Date.now()) return previous.token; - const raw = env['ACTIONS_ID_TOKEN_REQUEST_URL'], - credential = env['ACTIONS_ID_TOKEN_REQUEST_TOKEN']; + const raw = env['ACTIONS_ID_TOKEN_REQUEST_URL']; + const credential = env['ACTIONS_ID_TOKEN_REQUEST_TOKEN']; if (!raw || !credential) throw new Error('The e2e job requires id-token: write'); const url = new URL(raw); if ( @@ -138,21 +148,9 @@ export function githubTokens( await response.body?.cancel(); throw new Error('GitHub OIDC request failed'); } - const reader = response.body!.getReader(); - const chunks: Uint8Array[] = []; - let size = 0; - try { - while (true) { - const part = await reader.read(); - if (part.done) break; - size += part.value.length; - if (size > 32768) throw new Error('OIDC response exceeds its limit'); - chunks.push(part.value); - } - } finally { - await reader.cancel(); - } - const data = JSON.parse(Buffer.concat(chunks).toString('utf8')) as { value?: unknown }; + const data = (await readJson(response, 32768, 'OIDC response exceeds its limit')) as { + value?: unknown; + }; if (typeof data.value !== 'string' || data.value.length > 16384) throw new Error('Invalid GitHub OIDC response'); cached.set(audience, { token: data.value, until: Date.now() + 180000 }); @@ -160,7 +158,7 @@ export function githubTokens( }; } -async function deploy(settings: Settings) { +async function deploy(settings: Settings): Promise { await checkAccountOrigin(settings); await runOperator( [ @@ -269,7 +267,7 @@ async function deploy(settings: Settings) { ); } -async function verify(settings: Settings) { +async function verify(settings: Settings): Promise { await checkAccountOrigin(settings); const config = await operatorIO.readConfig(); checkConfig(config, settings); diff --git a/packages/remote-cache/scripts/e2e/fixtures.ts b/packages/remote-cache/scripts/e2e/fixtures.ts index 404fb89ca..a5f826996 100644 --- a/packages/remote-cache/scripts/e2e/fixtures.ts +++ b/packages/remote-cache/scripts/e2e/fixtures.ts @@ -8,10 +8,14 @@ export interface Admin { exists(key: string): Promise; } -export const bytes = (value: string) => new TextEncoder().encode(value); +export function bytes(value: string): Uint8Array { + return new TextEncoder().encode(value); +} // The REST fixture loader uses generated hexadecimal SQL literals for binary fields. // This conversion accepts bytes only, never SQL or HTTP input. -export const sqlBytes = (value: Uint8Array) => `X'${Buffer.from(value).toString('hex')}'`; +export function sqlBytes(value: Uint8Array): string { + return `X'${Buffer.from(value).toString('hex')}'`; +} export interface Fixture { scope: string; @@ -33,10 +37,10 @@ export async function seed( blob?: Uint8Array, ): Promise { const generation = randomUUID(); - const key = bytes(label), - secondary = bytes(`${label}-secondary`); - const valueObject = `${scope}/${generation}/value`, - blobObject = `${scope}/${generation}/blob`; + const key = bytes(label); + const secondary = bytes(`${label}-secondary`); + const valueObject = `${scope}/${generation}/value`; + const blobObject = `${scope}/${generation}/blob`; const blobId = blob === undefined ? null : randomUUID(); await admin.sql( `INSERT INTO generations diff --git a/packages/remote-cache/scripts/e2e/suite.ts b/packages/remote-cache/scripts/e2e/suite.ts index 1d2a5bb3b..974edc320 100644 --- a/packages/remote-cache/scripts/e2e/suite.ts +++ b/packages/remote-cache/scripts/e2e/suite.ts @@ -1,9 +1,9 @@ import assert from 'node:assert/strict'; import { createHash, randomUUID } from 'node:crypto'; import { setTimeout as delay } from 'node:timers/promises'; -import { encode, decode } from 'cborg'; +import { decode, encode } from 'cborg'; import { defaults, MiB } from '../../src/limits.ts'; -import { bytes, seed, sqlBytes, type Admin, type Fixture } from './fixtures.ts'; +import { bytes, seed, sqlBytes, type Admin } from './fixtures.ts'; export interface Result { name: string; @@ -32,7 +32,9 @@ export interface Options { record?: (report: Report) => Promise; } -const digest = (value: Uint8Array) => createHash('sha256').update(value).digest('hex'); +function digest(value: Uint8Array): string { + return createHash('sha256').update(value).digest('hex'); +} function blobId(result: Record): string { assert.equal(typeof result['blob_id'], 'string'); return result['blob_id'] as string; @@ -48,10 +50,11 @@ export async function runSuite(options: Options): Promise { results: [], }; const prefix = randomUUID(); - async function check(name: string, action: () => Promise) { + async function check(name: string, action: () => Promise): Promise { const start = Date.now(); + let result: T; try { - await action(); + result = await action(); report.results.push({ name, status: 'passed', duration_ms: Date.now() - start }); } catch (error) { // Reports contain test names, never request headers, tokens, or server bodies. @@ -68,6 +71,7 @@ export async function runSuite(options: Options): Promise { throw new Error(`Cloudflare e2e failed: ${name}`, { cause: error }); } await options.record?.(report); + return result; } async function call(scope: string, path: string, status: number, init: RequestInit = {}) { const response = await options.request(`${origin}/projects/${scope}/${path}`, { @@ -129,17 +133,17 @@ export async function runSuite(options: Options): Promise { options: { blobFirst?: boolean; status?: number; token?: string; stream?: boolean } = {}, ) { const form = new FormData(); - const addBlob = () => { + function addBlob(): void { if (blob !== undefined) form.append('blob', new Blob([blob], { type: 'application/octet-stream' })); - }; + } if (options.blobFirst) addBlob(); form.append( 'metadata', new Blob([encode({ key, secondary_key: secondary, value })], { type: 'application/cbor' }), ); if (!options.blobFirst) addBlob(); - const authorization = `Bearer ${options.token ?? (await optionsToken())}`; + const authorization = `Bearer ${options.token ?? (await storeToken())}`; if (!options.stream) return call('e2e', 'store', options.status ?? 200, { method: 'POST', @@ -154,10 +158,11 @@ export async function runSuite(options: Options): Promise { duplex: 'half', } as RequestInit); } - const optionsToken = () => options.token(`${origin}/projects/e2e`); - let fixture: Fixture; - await check('deployed revision and D1/R2 readiness', async () => { - fixture = await seed( + function storeToken(): Promise { + return options.token(`${origin}/projects/e2e`); + } + const fixture = await check('deployed revision and D1/R2 readiness', async () => { + const seeded = await seed( admin, 'e2e', `${prefix}-fixture`, @@ -169,7 +174,7 @@ export async function runSuite(options: Options): Promise { const response = await options.request(`${origin}/projects/e2e/fetch`, { method: 'POST', headers: { 'Content-Type': 'application/cbor' }, - body: encode({ key: fixture.key, secondary_key: fixture.secondary }), + body: encode({ key: seeded.key, secondary_key: seeded.secondary }), redirect: 'error', signal: AbortSignal.timeout(15000), }); @@ -177,15 +182,14 @@ export async function runSuite(options: Options): Promise { response.status === 200 && response.headers.get('X-Remote-Cache-Deployment') === deployment ) { - assert.deepEqual((await envelope(response)).value, fixture.value); - break; + assert.deepEqual((await envelope(response)).value, seeded.value); + return seeded; } await response.body?.cancel(); assert.ok(Date.now() < until, 'Deployment did not become ready'); await delay(options.pollMs ?? 3000); } }); - const f = fixture!; await check('manual verification fixture is available at the advertised endpoint', async () => { const manual = await envelope( await lookup( @@ -211,25 +215,25 @@ export async function runSuite(options: Options): Promise { association_count - (SELECT count(*) FROM associations) AS associations_delta FROM deployment`; const before = await admin.sql( 'SELECT charged_bytes, expires_at, state FROM generations WHERE generation_id = ?', - [f.generation], + [fixture.generation], ); assert.deepEqual(await admin.sql(accounting), [ { bytes_delta: 0, entries_delta: 0, associations_delta: 0 }, ]); - assert.deepEqual(await envelope(await lookup(f.key, f.secondary)), { + assert.deepEqual(await envelope(await lookup(fixture.key, fixture.secondary)), { kind: 'exact', - value: f.value, - blob_id: f.blobId, + value: fixture.value, + blob_id: fixture.blobId, }); - assert.deepEqual(await envelope(await lookup(bytes('missing'), f.secondary)), { + assert.deepEqual(await envelope(await lookup(bytes('missing'), fixture.secondary)), { kind: 'fallback', - key: f.key, + key: fixture.key, }); - assert.deepEqual(await body(await call('e2e', `blob/${f.blobId}`, 200)), f.blob); + assert.deepEqual(await body(await call('e2e', `blob/${fixture.blobId}`, 200)), fixture.blob); assert.deepEqual( await admin.sql( 'SELECT charged_bytes, expires_at, state FROM generations WHERE generation_id = ?', - [f.generation], + [fixture.generation], ), before, ); @@ -239,8 +243,8 @@ export async function runSuite(options: Options): Promise { }); await check('missing data, namespace isolation, and malformed reads', async () => { await body(await lookup(bytes(`${prefix}-missing`), bytes('missing'), 404)); - await body(await lookup(f.key, f.secondary, 404, 'other')); - await body(await call('other', `blob/${f.blobId}`, 404)); + await body(await lookup(fixture.key, fixture.secondary, 404, 'other')); + await body(await call('other', `blob/${fixture.blobId}`, 404)); await body(await call('unknown', 'fetch', 404, { method: 'POST' })); await body( await call('e2e', 'fetch', 400, { @@ -263,13 +267,15 @@ export async function runSuite(options: Options): Promise { }), ); await body( - await store(f.key, f.secondary, bytes('rejected'), undefined, { + await store(fixture.key, fixture.secondary, bytes('rejected'), undefined, { token: await options.token(`${origin}/projects/other`), status: 403, }), ); if (!options.writes) - await body(await store(f.key, f.secondary, bytes('rejected'), undefined, { status: 403 })); + await body( + await store(fixture.key, fixture.secondary, bytes('rejected'), undefined, { status: 403 }), + ); assert.ok( (await admin.sql('SELECT generation_id FROM generations')).every((row) => before.has(row['generation_id']), @@ -279,29 +285,32 @@ export async function runSuite(options: Options): Promise { await check('policy withdrawal takes effect without redeployment', async () => { try { await admin.sql("UPDATE scopes SET enabled = 0 WHERE scope_id = 'e2e'"); - await body(await lookup(f.key, f.secondary, 404)); - await body(await call('e2e', `blob/${f.blobId}`, 404)); + await body(await lookup(fixture.key, fixture.secondary, 404)); + await body(await call('e2e', `blob/${fixture.blobId}`, 404)); } finally { await admin.sql("UPDATE scopes SET enabled = 1 WHERE scope_id = 'e2e'"); } - assert.deepEqual((await envelope(await lookup(f.key, f.secondary))).value, f.value); + assert.deepEqual( + (await envelope(await lookup(fixture.key, fixture.secondary))).value, + fixture.value, + ); }); await check('missing live R2 objects have the correct errors', async () => { try { - await admin.delete(f.valueObject); - await body(await lookup(f.key, f.secondary, 503)); - assert.deepEqual(await envelope(await lookup(bytes('missing'), f.secondary)), { + await admin.delete(fixture.valueObject); + await body(await lookup(fixture.key, fixture.secondary, 503)); + assert.deepEqual(await envelope(await lookup(bytes('missing'), fixture.secondary)), { kind: 'fallback', - key: f.key, + key: fixture.key, }); } finally { - await admin.put(f.valueObject, f.value); + await admin.put(fixture.valueObject, fixture.value); } try { - await admin.delete(f.blobObject); - await body(await call('e2e', `blob/${f.blobId}`, 404)); + await admin.delete(fixture.blobObject); + await body(await call('e2e', `blob/${fixture.blobId}`, 404)); } finally { - await admin.put(f.blobObject, f.blob!); + await admin.put(fixture.blobObject, fixture.blob!); } }); @@ -364,7 +373,7 @@ export async function runSuite(options: Options): Promise { ); }); await check('malformed uploads and exhausted quotas preserve published data', async () => { - const token = await optionsToken(); + const token = await storeToken(); await body( await call('e2e', 'store', 400, { method: 'POST', @@ -383,11 +392,18 @@ export async function runSuite(options: Options): Promise { await admin.sql( "UPDATE scopes SET byte_limit = max(1, charged_bytes) WHERE scope_id = 'e2e'", ); - await body(await store(f.key, f.secondary, bytes('rejected'), undefined, { status: 503 })); + await body( + await store(fixture.key, fixture.secondary, bytes('rejected'), undefined, { + status: 503, + }), + ); } finally { await admin.sql("UPDATE scopes SET byte_limit = ? WHERE scope_id = 'e2e'", [Number(limit)]); } - assert.deepEqual((await envelope(await lookup(f.key, f.secondary))).value, f.value); + assert.deepEqual( + (await envelope(await lookup(fixture.key, fixture.secondary))).value, + fixture.value, + ); }); if (options.full) await check('maximum values and concurrent 64 MiB HTTP uploads', async () => { diff --git a/packages/remote-cache/scripts/http.ts b/packages/remote-cache/scripts/http.ts new file mode 100644 index 000000000..d8996ebac --- /dev/null +++ b/packages/remote-cache/scripts/http.ts @@ -0,0 +1,21 @@ +export async function readJson( + response: Response, + limit: number, + sizeError: string, +): Promise { + const reader = response.body!.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + size += value.length; + if (size > limit) throw new Error(sizeError); + chunks.push(value); + } + } finally { + await reader.cancel(); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8')); +} diff --git a/packages/remote-cache/scripts/operator.ts b/packages/remote-cache/scripts/operator.ts index a1daf29ac..d8ece6456 100644 --- a/packages/remote-cache/scripts/operator.ts +++ b/packages/remote-cache/scripts/operator.ts @@ -1,12 +1,13 @@ import { spawn } from 'node:child_process'; -import { readFile, writeFile, mkdtemp, rm } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; -import { join, dirname } from 'node:path'; +import { dirname, join } from 'node:path'; import { fileURLToPath, pathToFileURL, URL } from 'node:url'; -import { createRequire } from 'node:module'; -import { createHash } from 'node:crypto'; import { parseArgs } from 'node:util'; import { parse, type ParseError } from 'jsonc-parser'; +import { readJson } from './http.ts'; const root = fileURLToPath(new URL('../', import.meta.url)); const configPath = join(root, 'wrangler.operator.json'); @@ -51,7 +52,7 @@ export type Config = { vars: Record; }; -function isolateRateLimits(config: Config) { +function isolateRateLimits(config: Config): void { for (const binding of config.ratelimits) { // Account-wide IDs must stay stable across revisions and differ between Workers/bindings. const hash = createHash('sha256') @@ -75,6 +76,9 @@ function positive(value: string | undefined, fallback: number): number { if (!Number.isSafeInteger(number) || number <= 0) throw new Error('Expected a positive integer'); return number; } +function optionalPositive(value: string | undefined): number | null { + return value ? positive(value, 1) : null; +} function toggle(value: string | undefined): number | null { if (value === undefined) return null; if (value !== 'on' && value !== 'off') throw new Error('Use on or off'); @@ -86,7 +90,17 @@ function identifier(value: string | undefined): string { return value; } -export async function resolveRepository(io: Pick, name: string) { +interface Repository { + name: string; + id: string; + owner: string; + branch: string; +} + +export async function resolveRepository( + io: Pick, + name: string, +): Promise { if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(name)) throw new Error('Use owner/repository'); const repo = object(await io.github(name)); if (repo['private'] !== false || repo['visibility'] !== 'public') @@ -107,7 +121,7 @@ export async function query( config: Config, sql: string, params: (string | number | null)[] = [], -) { +): Promise[]> { const result = await io.api(`/d1/database/${config.d1_databases[0]!.database_id}/query`, 'POST', { sql, params, @@ -147,7 +161,7 @@ export function lifecycleRules(retentionDays: number) { }; } -async function updateLifecycle(io: OperatorIO, config: Config) { +async function updateLifecycle(io: OperatorIO, config: Config): Promise { const rows = await query( io, config, @@ -157,6 +171,39 @@ async function updateLifecycle(io: OperatorIO, config: Config) { await io.lifecycle(config.r2_buckets[0]!.bucket_name, lifecycleRules(days)); } +async function findDatabase( + io: OperatorIO, + name: string, +): Promise | undefined> { + const databases = await io.api(`/d1/database?name=${encodeURIComponent(name)}&per_page=100`); + if (!Array.isArray(databases)) throw new Error('Invalid D1 list'); + return databases.map(object).find((database) => database['name'] === name); +} + +async function ensurePrivateBucket(io: OperatorIO, name: string): Promise { + try { + const bucket = object(await io.api(`/r2/buckets/${name}`)); + if (bucket['storage_class'] && bucket['storage_class'] !== 'Standard') + throw new Error('Use an R2 Standard bucket'); + } catch (error) { + if (!(error instanceof ApiError) || error.status !== 404) throw error; + await io.wrangler([ + 'r2', + 'bucket', + 'create', + name, + '--storage-class', + 'Standard', + '--no-update-config', + ]); + } + // Refuse to adopt a bucket with public custom domains, then disable r2.dev. + const domains = object(await io.api(`/r2/buckets/${name}/domains/custom`)); + if (!Array.isArray(domains['domains']) || domains['domains'].length) + throw new Error('Remove R2 public custom domains before setup'); + await io.api(`/r2/buckets/${name}/domains/managed`, 'PUT', { enabled: false }); +} + export async function runOperator(argv: string[], io: OperatorIO): Promise { const { values, positionals } = parseArgs({ args: argv, @@ -202,7 +249,6 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise throw new Error('Retention cannot exceed 365 days'); toggle(values.enabled); toggle(values.writes); - let config: Config; if (command === 'setup') { const name = identifier(values.name); if (name.length < 3 || name.endsWith('-')) @@ -227,17 +273,13 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise throw new Error('Use the deployment name and account subdomain in the workers.dev origin'); const profile = values.profile ?? 'free'; if (profile !== 'free' && profile !== 'paid') throw new Error('Use free or paid'); - const databases = await io.api(`/d1/database?name=${encodeURIComponent(name)}&per_page=100`); - if (!Array.isArray(databases)) throw new Error('Invalid D1 list'); - let db = databases.map(object).find((item) => item['name'] === name); + let db = await findDatabase(io, name); if (!db) { await io.wrangler(['d1', 'create', name, '--no-update-config']); - const created = await io.api(`/d1/database?name=${encodeURIComponent(name)}&per_page=100`); - if (!Array.isArray(created)) throw new Error('Invalid D1 list'); - db = created.map(object).find((item) => item['name'] === name); + db = await findDatabase(io, name); } if (!db) throw new Error('D1 creation did not return the database'); - config = await readTemplate(); + const config = await readTemplate(); config.name = name; config.d1_databases[0] = { binding: 'INDEX', @@ -263,27 +305,7 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise if (prior && prior['repository_id'] !== repo.id) throw new Error('Use a new namespace for a different repository'); } - try { - const bucket = object(await io.api(`/r2/buckets/${name}`)); - if (bucket['storage_class'] && bucket['storage_class'] !== 'Standard') - throw new Error('Use an R2 Standard bucket'); - } catch (error) { - if (!(error instanceof ApiError) || error.status !== 404) throw error; - await io.wrangler([ - 'r2', - 'bucket', - 'create', - name, - '--storage-class', - 'Standard', - '--no-update-config', - ]); - } - // Refuse to adopt a bucket with public custom domains, then disable r2.dev. - const domains = object(await io.api(`/r2/buckets/${name}/domains/custom`)); - if (!Array.isArray(domains['domains']) || domains['domains'].length) - throw new Error('Remove R2 public custom domains before setup'); - await io.api(`/r2/buckets/${name}/domains/managed`, 'PUT', { enabled: false }); + await ensurePrivateBucket(io, name); config.r2_buckets[0] = { binding: 'ARTIFACTS', bucket_name: name }; config.workers_dev = origin.hostname.endsWith('.workers.dev'); if (!config.workers_dev) config.routes = [{ pattern: origin.hostname, custom_domain: true }]; @@ -316,9 +338,9 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise `UPDATE deployment SET byte_limit = coalesce(?, byte_limit), entry_limit = coalesce(?, entry_limit), association_limit = coalesce(?, association_limit)`, [ - values['byte-limit'] ? positive(values['byte-limit'], 1) : null, - values['entry-limit'] ? positive(values['entry-limit'], 1) : null, - values['association-limit'] ? positive(values['association-limit'], 1) : null, + optionalPositive(values['byte-limit']), + optionalPositive(values['entry-limit']), + optionalPositive(values['association-limit']), ], ); const scopes = await query(io, config, 'SELECT scope_id, endpoint FROM scopes'); @@ -329,7 +351,7 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise io.print(text(scopes.find((row) => row['scope_id'] === namespace)?.['endpoint'])); return; } - config = await io.readConfig(); + const config = await io.readConfig(); if (command === 'upgrade') { isolateRateLimits(config); await io.writeConfig(config); @@ -373,9 +395,9 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise [ toggle(values.enabled), toggle(values.writes), - values['byte-limit'] ? positive(values['byte-limit'], 1) : null, - values['entry-limit'] ? positive(values['entry-limit'], 1) : null, - values['association-limit'] ? positive(values['association-limit'], 1) : null, + optionalPositive(values['byte-limit']), + optionalPositive(values['entry-limit']), + optionalPositive(values['association-limit']), ], ); return; @@ -438,13 +460,10 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise config.vars['NAMESPACES'] = JSON.stringify(all.map((row) => text(row['scope_id']))); await io.writeConfig(config); await io.wrangler(['deploy', '--config', configPath]); - io.print( - text( - ( - await query(io, config, 'SELECT endpoint FROM scopes WHERE scope_id = ?', [namespace]) - )[0]?.['endpoint'], - ), - ); + const scopes = await query(io, config, 'SELECT endpoint FROM scopes WHERE scope_id = ?', [ + namespace, + ]); + io.print(text(scopes[0]?.['endpoint'])); return; } if (!existing.length) throw new Error('Namespace does not exist'); @@ -492,9 +511,9 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise toggle(values.enabled), toggle(values.writes), values['retention-days'] ? positive(values['retention-days'], 7) * 86400 : null, - values['byte-limit'] ? positive(values['byte-limit'], 1) : null, - values['entry-limit'] ? positive(values['entry-limit'], 1) : null, - values['association-limit'] ? positive(values['association-limit'], 1) : null, + optionalPositive(values['byte-limit']), + optionalPositive(values['entry-limit']), + optionalPositive(values['association-limit']), namespace, ], ); @@ -511,21 +530,7 @@ async function jsonResponse(response: Response): Promise { void response.body?.cancel(); throw new ApiError(response.status); } - const reader = response.body!.getReader(); - let size = 0; - const chunks: Uint8Array[] = []; - try { - while (true) { - const part = await reader.read(); - if (part.done) break; - size += part.value.length; - if (size > 4 * 1024 * 1024) throw new Error('API response too large'); - chunks.push(part.value); - } - } finally { - await reader.cancel(); - } - return JSON.parse(Buffer.concat(chunks).toString('utf8')); + return readJson(response, 4 * 1024 * 1024, 'API response too large'); } export const operatorIO: OperatorIO = { diff --git a/packages/remote-cache/src/cbor.ts b/packages/remote-cache/src/cbor.ts index 3d3e3976e..08510e394 100644 --- a/packages/remote-cache/src/cbor.ts +++ b/packages/remote-cache/src/cbor.ts @@ -1,6 +1,15 @@ import { badRequest, tooLarge } from './errors.ts'; import type { Limits } from './limits.ts'; +export interface FetchRequest { + key: Uint8Array; + secondary_key: Uint8Array; +} + +export interface StoreMetadata extends FetchRequest { + value: Uint8Array; +} + // This codec only decodes the protocol envelope. Opaque bytes are never decoded. // The accepted containers are one map and (possibly chunked) strings: depth <= 2. class Decoder { @@ -65,17 +74,13 @@ export function join(chunks: Uint8Array[], size: number): Uint8Array { let scope: Scope | null; try { - scope = - ( - await measured( - db - .prepare(`SELECT s.*, (s.writes_enabled AND d.writes_enabled) AS writes_enabled + const { results } = await measured( + db + .prepare(`SELECT s.*, (s.writes_enabled AND d.writes_enabled) AS writes_enabled FROM scopes s, deployment d WHERE s.scope_id = ? AND s.enabled = 1 AND d.enabled = 1`) - .bind(id) - .all(), - stats, - ) - ).results[0] ?? null; + .bind(id) + .all(), + stats, + ); + scope = results[0] ?? null; } catch { unavailable(); } @@ -60,11 +58,9 @@ export async function selectEntry( ): Promise { // Both branches use indexed identities in one SQLite snapshot. try { - return ( - ( - await measured( - db - .prepare(` + const { results } = await measured( + db + .prepare(` SELECT g.*, e.key, 'exact' AS kind, 0 AS priority FROM entries e JOIN generations g ON g.generation_id = e.generation_id JOIN scopes s ON s.scope_id = e.scope_id CROSS JOIN deployment d @@ -78,12 +74,11 @@ export async function selectEntry( WHERE a.scope_id = ?1 AND a.secondary_key = ?3 AND g.state = 'ready' AND g.expires_at > unixepoch() AND s.enabled = 1 AND d.enabled = 1 ORDER BY priority LIMIT 1`) - .bind(scope, binary(key), binary(secondary)) - .all(), - stats, - ) - ).results[0] ?? null + .bind(scope, binary(key), binary(secondary)) + .all(), + stats, ); + return results[0] ?? null; } catch { unavailable(); } @@ -96,19 +91,16 @@ export async function selectBlob( stats?: Observations, ): Promise { try { - return ( - ( - await measured( - db - .prepare(`SELECT g.* FROM generations g JOIN scopes s ON s.scope_id = g.scope_id CROSS JOIN deployment d + const { results } = await measured( + db + .prepare(`SELECT g.* FROM generations g JOIN scopes s ON s.scope_id = g.scope_id CROSS JOIN deployment d WHERE g.scope_id = ? AND g.blob_id = ? AND s.enabled = 1 AND d.enabled = 1 AND ((g.state = 'ready' AND g.expires_at > unixepoch()) OR (g.state = 'retired' AND g.gc_after > unixepoch()))`) - .bind(scope, blob) - .all(), - stats, - ) - ).results[0] ?? null + .bind(scope, blob) + .all(), + stats, ); + return results[0] ?? null; } catch { unavailable(); } @@ -162,7 +154,7 @@ export async function recordMultipart( generation: Generation, uploadId: string, stats?: Observations, -) { +): Promise { const result = await measured( db .prepare( @@ -272,7 +264,7 @@ export async function cleanup(env: Env): Promise { console.log(JSON.stringify({ operation: 'cleanup', claimed: claimed.results.length, deleted })); } -async function cleanAssociations(db: D1Database, limit: number) { +async function cleanAssociations(db: D1Database, limit: number): Promise { // Cursor bounds scanning as well as deletion, even when all associations are live. const batch = await db .prepare(`SELECT scope_id, secondary_key FROM associations diff --git a/packages/remote-cache/src/index.ts b/packages/remote-cache/src/index.ts index f05c06f5b..ffcd1c4b6 100644 --- a/packages/remote-cache/src/index.ts +++ b/packages/remote-cache/src/index.ts @@ -51,11 +51,10 @@ export default { operation === 'store' || url.pathname.endsWith('/store') ? env.STORE_LIMITER : env.READ_LIMITER; - if ( - !(await deadline.run(limiter.limit({ key: known ? `${scopeId}:${operation}` : 'unknown' }))) - .success - ) - throw new HttpError(429, 'rate_limit'); + const rateLimit = await deadline.run( + limiter.limit({ key: known ? `${scopeId}:${operation}` : 'unknown' }), + ); + if (!rateLimit.success) throw new HttpError(429, 'rate_limit'); if (!known || url.search || request.method !== (operation === 'blob' ? 'GET' : 'POST')) throw new HttpError(404, 'unknown_route'); release = admission.acquire(operation === 'store'); diff --git a/packages/remote-cache/src/multipart.ts b/packages/remote-cache/src/multipart.ts index b68ee0e4b..cacbdd672 100644 --- a/packages/remote-cache/src/multipart.ts +++ b/packages/remote-cache/src/multipart.ts @@ -100,7 +100,11 @@ export async function* multipart( await collect(input.rest(), headerLimit); } -async function delimiterEnd(input: Input, start: number, limit: number) { +async function delimiterEnd( + input: Input, + start: number, + limit: number, +): Promise<{ end: number; closed: boolean } | null> { while (input.buffer.length < start + 2 && (await input.fill())) {} const closed = input.buffer[start] === 45 && input.buffer[start + 1] === 45; let end = start + (closed ? 2 : 0); diff --git a/packages/remote-cache/src/store.ts b/packages/remote-cache/src/store.ts index 4482e74d7..d6ecc29f3 100644 --- a/packages/remote-cache/src/store.ts +++ b/packages/remote-cache/src/store.ts @@ -1,4 +1,4 @@ -import { decodeEnvelope, cborResponse } from './cbor.ts'; +import { cborResponse, decodeEnvelope, type StoreMetadata } from './cbor.ts'; import { abandon, publish, @@ -21,7 +21,7 @@ async function uploadBlob( limit: number, deadline: Deadline, stats: Observations, -) { +): Promise { let buffer = new Uint8Array(PART_SIZE); let used = 0; let upload: R2MultipartUpload | undefined; @@ -91,7 +91,7 @@ export async function store( let input: Input | undefined; try { input = new Input(request.body, Math.min(length ?? limits.store, limits.store), deadline); - let metadata: ReturnType | undefined; + let metadata: StoreMetadata | undefined; for await (const part of multipart(input, boundary, limits.headers)) { if (part.name === 'metadata') { metadata = decodeEnvelope(await collect(part.body, limits.metadata), true, limits); diff --git a/packages/remote-cache/src/streams.ts b/packages/remote-cache/src/streams.ts index 0682b60eb..5a47b0c1f 100644 --- a/packages/remote-cache/src/streams.ts +++ b/packages/remote-cache/src/streams.ts @@ -23,14 +23,14 @@ export class Deadline { void this.cancelled.catch(() => {}); if (signal?.aborted) this.onAbort(); } - check() { + check(): void { if (this.controller.signal.aborted) throw new HttpError(503, 'deadline'); } async run(operation: Promise): Promise { this.check(); return Promise.race([operation, this.cancelled]); } - dispose() { + dispose(): void { clearTimeout(this.timer); this.signal?.removeEventListener('abort', this.onAbort); } @@ -107,7 +107,7 @@ export class Input { if (this.buffer.length) yield this.take(this.buffer.length); } while (await this.fill()); } - close() { + close(): void { void this.reader.cancel().catch(() => {}); } } @@ -116,7 +116,7 @@ export async function readBody( body: ReadableStream | null, limit: number, deadline: Deadline, -) { +): Promise> { const input = new Input(body, limit, deadline); try { return await collect(input.rest(), limit); diff --git a/packages/remote-cache/test/deployed.test.ts b/packages/remote-cache/test/deployed.test.ts index 5976f8fe1..65ca16d58 100644 --- a/packages/remote-cache/test/deployed.test.ts +++ b/packages/remote-cache/test/deployed.test.ts @@ -5,10 +5,14 @@ import { runSuite } from '../scripts/e2e/suite.ts'; import { retireTestData, seedManual, type Admin } from '../scripts/e2e/fixtures.ts'; import { githubTokens, settingsFrom } from '../scripts/ci.ts'; -for (const mode of ['read-only smoke', 'push smoke', 'push full'] as const) +const modes = [ + { mode: 'read-only smoke', writes: false, full: false, checks: 8, runs: 2 }, + { mode: 'push smoke', writes: true, full: false, checks: 13, runs: 2 }, + { mode: 'push full', writes: true, full: true, checks: 14, runs: 1 }, +]; + +for (const { mode, writes, full, checks, runs } of modes) void test(`deployed HTTP suite runs against workerd: ${mode}`, async () => { - const writes = mode !== 'read-only smoke'; - const full = mode === 'push full'; const h = await harness({ deploymentId: 'suite-local', namespaces: ['test', 'other', 'e2e', 'manual'], @@ -43,7 +47,7 @@ for (const mode of ['read-only smoke', 'push smoke', 'push full'] as const) }, }; // Reuse storage for consecutive smoke runs, including previous test generations. - for (let attempt = 0; attempt < (full ? 1 : 2); attempt++) { + for (let attempt = 0; attempt < runs; attempt++) { const manual = await seedManual(admin, 'suite-local'); const report = await runSuite({ origin: 'https://cache.example.com', @@ -73,7 +77,7 @@ for (const mode of ['read-only smoke', 'push smoke', 'push full'] as const) }); }, }); - assert.equal(report.results.length, writes ? (full ? 14 : 13) : 8); + assert.equal(report.results.length, checks); assert.ok(report.results.every((result) => result.status === 'passed')); assert.equal( report.results.some((result) => result.name.startsWith('real Cron')), @@ -140,7 +144,8 @@ void test('CI shares persistent staging across events and only permits writes on void test('OIDC requests use only GitHub, reject redirects, and reuse tokens only for the same audience', async () => { const calls: { url: string; init?: RequestInit }[] = []; const request: typeof fetch = async (url, init) => { - calls.push({ url: typeof url === 'string' ? url : 'href' in url ? url.href : url.url, init }); + const requestUrl = url instanceof Request ? url.url : String(url); + calls.push({ url: requestUrl, init }); return new Response(JSON.stringify({ value: 'signed-token' })); }; const env = { From d34945f1d38a05fed39dd6046886d68f1464e946 Mon Sep 17 00:00:00 2001 From: MK Date: Wed, 16 Sep 2026 13:51:51 +0800 Subject: [PATCH 13/17] chore(remote-cache): document self-hosting and default staging to free Co-authored-by: GPT-6 Codex --- .github/workflows/remote-cache-deploy.yml | 1 + README.md | 4 + packages/remote-cache/README.md | 14 +- packages/remote-cache/docs/e2e-plan.md | 5 +- packages/remote-cache/docs/self-hosting.md | 204 ++++++++++++++++++ .../{docs => rfcs}/0001-remote-cache.md | 2 +- packages/remote-cache/scripts/ci.ts | 7 +- packages/remote-cache/test/deployed.test.ts | 5 + packages/remote-cache/test/operator.test.ts | 1 + 9 files changed, 235 insertions(+), 8 deletions(-) create mode 100644 packages/remote-cache/docs/self-hosting.md rename packages/remote-cache/{docs => rfcs}/0001-remote-cache.md (99%) diff --git a/.github/workflows/remote-cache-deploy.yml b/.github/workflows/remote-cache-deploy.yml index fa3a25b92..7fb4ea310 100644 --- a/.github/workflows/remote-cache-deploy.yml +++ b/.github/workflows/remote-cache-deploy.yml @@ -34,6 +34,7 @@ env: REMOTE_CACHE_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} REMOTE_CACHE_RESOURCE_PREFIX: ${{ vars.REMOTE_CACHE_RESOURCE_PREFIX || 'vp-cache-ci' }} REMOTE_CACHE_WORKERS_SUBDOMAIN: ${{ vars.REMOTE_CACHE_WORKERS_SUBDOMAIN }} + REMOTE_CACHE_PROFILE: ${{ vars.REMOTE_CACHE_PROFILE || 'free' }} jobs: check: diff --git a/README.md b/README.md index 26206b796..17cbf559e 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ vp run -t @my/app#build # run in a package and its transitive dependencies vp run --cache build # run with caching enabled ``` +## Remote cache service + +To deploy the public remote cache service in your own Cloudflare account and bind it to a GitHub repository, follow the [self-hosting guide](packages/remote-cache/docs/self-hosting.md). The service is available in this source tree; the `vp run` remote-cache client adapter is not yet implemented here. + ## Sponsors Thanks to [namespace.so](https://namespace.so) for powering our CI/CD pipelines with fast, free macOS and Linux runners. diff --git a/packages/remote-cache/README.md b/packages/remote-cache/README.md index c13aedf14..46e5f8608 100644 --- a/packages/remote-cache/README.md +++ b/packages/remote-cache/README.md @@ -1,9 +1,11 @@ # Public remote cache service -This package implements the server in the [remote cache RFC](docs/0001-remote-cache.md) ([PR #716](https://github.com/voidzero-dev/vite-task/pull/716)): a TypeScript Worker, primary D1 metadata, and a private R2 Standard bucket. Anyone can read an enabled namespace. Only a signed GitHub Actions token for the registered public repository's main-branch `push` job can publish. +This package implements the server in the [remote cache RFC](rfcs/0001-remote-cache.md) ([PR #716](https://github.com/voidzero-dev/vite-task/pull/716)): a TypeScript Worker, primary D1 metadata, and a private R2 Standard bucket. Anyone can read an enabled namespace. Only a signed GitHub Actions token for the registered public repository's main-branch `push` job can publish. The package contains no `vp run` client adapter. Cache keys, values, and blobs remain opaque. A successful lookup does not prove that a result is reusable; a client must validate its inputs and output archive. +**Start here:** [Deploy the service and bind your repository](docs/self-hosting.md). The guide covers Cloudflare credentials, deployment, custom domains, repository binding, and the current client-integration limitation. + ## Protocol The endpoint is `https:///projects/`. Namespaces use 1–63 lowercase letters, digits, or hyphens, starting with a letter or digit. An operator can register up to 100 namespaces per deployment. @@ -20,7 +22,7 @@ Empty and non-UTF-8 keys work. CBOR definite and indefinite maps and byte string Fetch gives exact matches priority. A fallback follows the latest secondary-key association and returns only the stored key, without reading R2. A store replaces both mappings atomically. Reassigning a secondary key does not delete its former target. Replacing an entry preserves the other associations to its key. -HTTP status codes follow the [local RFC](docs/0001-remote-cache.md#4-http-api-mapping). A fetch returns `404` when neither key resolves to a live entry. An unavailable blob also returns `404`, including when its R2 object is missing. These responses use plain text. A missing or unreadable value for a live exact match is a storage failure and returns `503`, as specified in [the read design](docs/0001-remote-cache.md#7-fetch-and-download-implementation). +HTTP status codes follow the [local RFC](rfcs/0001-remote-cache.md#4-http-api-mapping). A fetch returns `404` when neither key resolves to a live entry. An unavailable blob also returns `404`, including when its R2 object is missing. These responses use plain text. A missing or unreadable value for a live exact match is a storage failure and returns `503`, as specified in [the read design](rfcs/0001-remote-cache.md#7-fetch-and-download-implementation). Errors have `Content-Type: text/plain; charset=utf-8`. Codes are `400` for invalid input, `401` for invalid/missing/expired tokens, `403` for a signed token that fails write policy, `404` for absent data or unavailable namespaces/routes, `413` for size limits, `429` for admission limits, `500` for an incomplete operation, and `503` for unavailable authorization/storage, failed publication guards, quotas, or concurrency admission. `429` and `503` include `Retry-After: 60`. @@ -41,6 +43,8 @@ From this package directory, `pnpm dev` starts Wrangler with local bindings. App ## Setup +For your own service and application repository, use the [self-hosting guide](docs/self-hosting.md). The commands below are the operator reference. + For continuous deployment and smoke tests against one persistent Cloudflare staging environment, follow the [deployment and e2e plan](docs/e2e-plan.md). Internal PRs and main-branch pushes share the same Worker, D1 database, and R2 bucket. The plan includes GitHub configuration, the complete test matrix, and manual verification. Closing a PR leaves staging available. Use a dedicated Worker, D1 database, and bucket. The operator requires `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` in its environment. The token needs account permissions for Workers Scripts, D1, and Workers R2 Storage, plus route/zone permissions if using a custom domain. Enable R2 in the account first. Credentials stay in the operator process and Wrangler; they are never stored in namespace policy or passed as command arguments. @@ -55,9 +59,11 @@ Use the account's actual Workers subdomain. For a custom domain, set `--origin h Setup discovers resources by name, creates missing resources, applies migrations, resolves public repository and owner IDs through GitHub, records the default branch and exact audience, installs lifecycle/Cron settings, deploys, and prints the endpoint. Repeated setup does not duplicate resources or re-enable a withdrawn namespace. An incompatible origin or repository is rejected before setup changes existing policies, bucket settings, or saved configuration. Configuration is written to the ignored `wrangler.operator.json`; keep a secure backup of this non-secret file. -The default `free` **resource profile** retains the RFC's 8 GB budget, 20,000 entries, 20,000 associations, and 16 generations per Cron run. It is not proof of compatibility with the Workers Free CPU limit. **Use Workers Paid for the full payload limits until production CPU measurements establish a supported Free profile.** Local measurements already exceed 10 ms in several cases. Setup does not purchase or change a Workers subscription. +Start with Workers Free. The operator defaults to the `free` resource profile: an 8 GB budget, 20,000 entries, 20,000 associations, and 16 generations per Cron run. A paid subscription is optional. Setup does not purchase or change a Workers subscription, and R2 must be enabled separately. + +Check deployed CPU and usage for your workload. Workers Free currently allows [10 ms of CPU per invocation](https://developers.cloudflare.com/workers/platform/limits/); maximum payloads are not yet validated against that allowance. Local CPU measurements are diagnostic, not Cloudflare billing measurements. Lower payload limits or select Workers Paid if your workload exceeds Free limits. -`--profile paid` selects 256 generations per Cron run. It does not increase storage or retention. Select those independently with `--byte-limit` and `--retention-days`. Retention can be 1–365 days. For example: +For a deployment on Workers Paid, explicitly select `--profile paid` when you need the larger cleanup batch of 256 generations per Cron run. This option does not upgrade the account or increase storage or retention. Select those independently with `--byte-limit` and `--retention-days`. Retention can be 1–365 days. For example: ```sh pnpm operator setup --name my-public-cache --namespace docs --repo owner/repository --origin https://cache.example.com --profile paid --retention-days 30 --byte-limit 30000000000 diff --git a/packages/remote-cache/docs/e2e-plan.md b/packages/remote-cache/docs/e2e-plan.md index f9521b06f..7aa53fa36 100644 --- a/packages/remote-cache/docs/e2e-plan.md +++ b/packages/remote-cache/docs/e2e-plan.md @@ -19,7 +19,7 @@ The workflow uses one `remote-cache-staging` concurrency group for all branches. ## GitHub and Cloudflare setup -Use a dedicated Cloudflare staging account or dedicated resources in an account that permits this CI workload. Select Workers Paid for the configured payload sizes. The `paid` operator profile controls cleanup batch size; it does not purchase a subscription. +Use a dedicated Cloudflare staging account or dedicated resources in an account that permits this CI workload. Workers Free is the default starting point. Staging uses the `free` operator profile unless `REMOTE_CACHE_PROFILE=paid` is explicitly configured. The profile controls cleanup batch size; it does not purchase or change a subscription. Validate deployed CPU and usage against the account's limits, and select Workers Paid if the workload needs it. Passing local tests does not establish that maximum payloads fit Free. Add these repository secrets under **Settings → Secrets and variables → Actions**. Repository write access is sufficient. The workflow does not require a GitHub environment: @@ -41,6 +41,7 @@ Set these **repository variables**, which the notification job also needs: | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | | `REMOTE_CACHE_WORKERS_SUBDOMAIN` | Account subdomain only, for example `example` for `example.workers.dev` | | `REMOTE_CACHE_RESOURCE_PREFIX` | Optional; defaults to `vp-cache-ci`. A custom prefix must end in `-ci` and use at most 34 lowercase letters, digits, or hyphens | +| `REMOTE_CACHE_PROFILE` | Optional; defaults to `free`. Set to `paid` to opt into larger cleanup batches on a Workers Paid account | | `REMOTE_CACHE_DEPLOY_ENABLED` | Set to `true` after the repository secrets, variables, and Cloudflare account are ready | Enable R2 in the account. The token must allow resource provisioning, updates, and fixture data access. No custom domain or DNS permission is needed. Do not attach production bindings or secrets to the staging Worker. @@ -69,7 +70,7 @@ The operator seeds immutable objects and generation rows for read tests. This va “Local” refers to the existing regression suite and the new shared e2e-driver tests. “PR” and “Main” refer to smoke tests against the shared Cloudflare staging environment. -Status assertions follow the [local RFC](0001-remote-cache.md#4-http-api-mapping). Fetch misses and unavailable blobs return plain-text `404`. An exact match with a missing or unreadable value returns `503`. A fallback returns only the stored key and makes no R2 read. +Status assertions follow the [local RFC](../rfcs/0001-remote-cache.md#4-http-api-mapping). Fetch misses and unavailable blobs return plain-text `404`. An exact match with a missing or unreadable value returns `503`. A fallback returns only the stored key and makes no R2 read. | Case | Local | PR | Main | Required result | | ------------------------------------------------ | ----------------------------- | ----------------------------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | diff --git a/packages/remote-cache/docs/self-hosting.md b/packages/remote-cache/docs/self-hosting.md new file mode 100644 index 000000000..38949b36c --- /dev/null +++ b/packages/remote-cache/docs/self-hosting.md @@ -0,0 +1,204 @@ +# Deploy a remote cache for your GitHub repository + +This guide deploys the service in your Cloudflare account and gives your repository its own cache endpoint. You can bind any **public repository on GitHub.com**. Setup reads its default branch and permits uploads from GitHub Actions `push` jobs on that branch. All cache reads are public. Private repositories and other Git providers are not supported. + +**Current status:** the server and operator CLI work in this source tree. The `vp run` remote-cache client adapter is not implemented here. You can deploy and verify the HTTP service now. Automatic task uploads and reuse require a compatible client; the proposed Vite+ settings are described separately below. + +## 1. Prepare the deployment checkout + +Use a checkout of this repository that contains `packages/remote-cache`. Install Node.js 22.12 or newer and the pnpm version in the root `package.json`. From the repository root, run: + +```sh +pnpm install --frozen-lockfile +pnpm check-remote-cache +cd packages/remote-cache +``` + +Run all subsequent `pnpm operator` commands from `packages/remote-cache`. This is the service checkout. Your application repository can be a separate checkout; it does not need the Worker source or Cloudflare credentials. + +## 2. Prepare Cloudflare credentials + +Start with a Cloudflare account on Workers Free and enable R2. A Workers Paid subscription is optional. [R2 has its own free usage allowance and billing](https://developers.cloudflare.com/r2/pricing/), separate from the Workers plan. + +Find your account ID using [Cloudflare's account-ID instructions](https://developers.cloudflare.com/fundamentals/account/find-account-and-zone-ids/). This is the account ID, not a zone ID. Also find or create the account's [Workers subdomain](https://developers.cloudflare.com/workers/configuration/routing/workers-dev/). For a URL ending in `my-team.workers.dev`, the subdomain is `my-team`. + +[Create an API token](https://developers.cloudflare.com/fundamentals/api/get-started/create-token/) scoped to that account with these permissions: + +| Account permission | Access | +| ------------------ | --------------------------------------------------------------- | +| Workers | Admin at Workers product scope, to create and manage the Worker | +| D1 | Edit / Write | +| Workers R2 Storage | Edit / Write | + +These permissions cover deployment, database migrations, private bucket setup, and lifecycle rules. Accounts using the legacy token permissions can use Workers Scripts Edit / Write for Workers access. With the newer Workers roles, Editor only permits deployment to an existing Worker; initial setup requires product-level Admin. See [Workers roles](https://developers.cloudflare.com/workers/authorization/workers/) and the [API permission reference](https://developers.cloudflare.com/fundamentals/api/reference/permissions/) for the current labels. + +Set `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` in the terminal that will run the operator. Enter the token at a hidden prompt so it does not enter shell history. + +On macOS or Linux, in Bash or Zsh: + +```sh +printf 'Cloudflare account ID: ' +read -r CLOUDFLARE_ACCOUNT_ID +printf 'Cloudflare API token: ' +read -r -s CLOUDFLARE_API_TOKEN +printf '\n' +export CLOUDFLARE_ACCOUNT_ID CLOUDFLARE_API_TOKEN +``` + +On Windows, in PowerShell: + +```powershell +$env:CLOUDFLARE_ACCOUNT_ID = Read-Host 'Cloudflare account ID' +$token = Read-Host 'Cloudflare API token' -AsSecureString +$env:CLOUDFLARE_API_TOKEN = [System.Net.NetworkCredential]::new('', $token).Password +Remove-Variable token +``` + +The operator reads these environment variables directly. Putting them in `.dev.vars` or signing in with `wrangler login` alone does not configure the operator. Keep these credentials in the deployment terminal or deployment automation. Application build jobs use GitHub OIDC instead. + +## 3. Deploy and bind your repository + +Choose the following values. This example uses the public repository `acme/web-app` and a Workers subdomain of `my-team`: + +| Option | Example | Meaning | +| ------------- | ---------------------------------------------- | --------------------------------------------------------------- | +| `--name` | `acme-build-cache` | Name for the Worker, D1 database, and R2 bucket | +| `--namespace` | `web-app` | Repository's cache namespace | +| `--repo` | `acme/web-app` | GitHub repository allowed to upload; omit `https://github.com/` | +| `--origin` | `https://acme-build-cache.my-team.workers.dev` | Public service origin, without `/projects/...` | + +Use dedicated resource names. Replace the example repository and subdomain with your own values, then run: + +```sh +pnpm operator setup --name acme-build-cache --namespace web-app --repo acme/web-app --origin https://acme-build-cache.my-team.workers.dev +``` + +Setup creates or reuses the named resources, applies the schema, keeps R2 private, registers the repository's IDs and default branch, and deploys the Worker. The endpoint it prints is: + +```text +https://acme-build-cache.my-team.workers.dev/projects/web-app +``` + +Use that complete endpoint, without a trailing slash, in your client. It is also the exact audience required for upload tokens. The namespace does not have to match the repository name. + +Setup writes `wrangler.operator.json`, an ignored file containing resource IDs and deployment settings. Keep it for later operator commands and back it up. It contains no API token. Repeating setup preserves an existing repository binding and does not re-enable a disabled namespace. Use a separate service checkout and configuration file for each deployment. + +Omitting `--profile` selects `free`, with 16 generations per cleanup run. Defaults are seven days of retention, an 8 GB byte budget, 20,000 entries, and 20,000 secondary-key associations. Pending and retired uploads also consume the byte budget. To choose different limits during setup, add options such as: + +```text +--retention-days 30 --byte-limit 30000000000 --entry-limit 50000 --association-limit 50000 +``` + +### Select Paid when needed + +Measure your deployed workload before choosing higher allowances. Workers Free currently allows [10 ms of CPU per invocation](https://developers.cloudflare.com/workers/platform/limits/). The full 4 MiB value and 64 MiB blob limits have not been validated against that CPU allowance. Local benchmark results cannot establish Cloudflare's billed CPU usage. If your workload exceeds Free limits, lower payload limits or choose Workers Paid in Cloudflare. + +After choosing Workers Paid, you can rerun your setup command with `--profile paid` to process up to 256 generations per cleanup run. Preserve your deployment, namespace, repository, and origin arguments. The flag changes cleanup batch size only; it does not purchase a subscription or raise storage and retention budgets. You can keep the smaller `free` cleanup profile on a Paid account. + +### Use a custom domain + +Choose the custom origin during initial setup. For example: + +```sh +pnpm operator setup --name acme-build-cache --namespace web-app --repo acme/web-app --origin https://cache.example.com +``` + +The domain must be in an active Cloudflare zone in the same account. Add Workers Routes write access and Zone read access for that zone to the deployment token. The operator configures a Worker Custom Domain and disables the unused `workers.dev` alias. The resulting endpoint is `https://cache.example.com/projects/web-app`. See [Custom Domains](https://developers.cloudflare.com/workers/configuration/routing/custom-domains/) and [route permissions](https://developers.cloudflare.com/workers/authorization/workers/). + +Existing namespaces retain their original origin. Repeating setup with a different origin is rejected. Plan an origin change as a separate deployment and update clients to its new endpoint. + +## 4. Check the service + +Inspect the deployed repository binding and storage state: + +```sh +pnpm operator status +``` + +In `scopes`, confirm that `scope_id` is `web-app`, `repository` is your repository, `branch` matches its default branch, and `endpoint` matches the printed URL. `enabled` and `writes_enabled` should both be `1` for a new namespace. The deployment-level switches must also be enabled. + +Check that an anonymous upload is rejected: + +```sh +curl --include --request POST https://acme-build-cache.my-team.workers.dev/projects/web-app/store +``` + +Expect HTTP `401` with the plain-text body `Invalid credentials`. Use `curl.exe` in Windows PowerShell. The service has no landing page at `/`; a browser visit there returns `404`. + +An empty cache also returns `404` for a valid lookup until an authorized client publishes an entry. A complete integration check must upload from a permitted GitHub Actions job, fetch the same entry without credentials, and compare the returned value and blob. The [HTTP protocol reference](../README.md#protocol) defines those requests. The [e2e plan](e2e-plan.md) describes the project's separate shared staging checks. + +## 5. Connect your application build + +A compatible client needs two settings: the complete namespace endpoint and whether it can upload. Developer machines and PR jobs can read without credentials. For uploads, the job must run in the bound public repository on a `push` to its registered default branch. Grant that job `id-token: write`; GitHub documents this [OIDC permission and custom audiences](https://docs.github.com/en/actions/reference/security/oidc). + +The client requests a GitHub OIDC token whose audience is the endpoint, then sends the token as `Authorization: Bearer ` to `/store`. `GITHUB_TOKEN`, personal access tokens, and Cloudflare API tokens are not accepted as cache upload credentials. PR, tag, and `workflow_dispatch` tokens cannot upload. A default branch named `master` works when that is the branch saved by setup. + +### Proposed Vite+ configuration — client implementation required + +The following examples describe the [RFC's client configuration](../rfcs/0001-remote-cache.md#client-configuration-and-remote-cache-modes). They do **not** enable remote caching in the client code currently in this repository. + +Once a compatible `vp run` client implements that contract, merge this setting into your application's `vite.config.ts`: + +```ts +export default { + run: { + remoteCache: { + url: 'https://acme-build-cache.my-team.workers.dev/projects/web-app', + }, + }, +}; +``` + +The proposed default is public reads when an endpoint is configured. `VP_REMOTE_CACHE_URL` overrides the endpoint. To enable uploads, add these settings to the existing build job that runs only on default-branch pushes: + +```yaml +permissions: + contents: read + id-token: write +env: + VP_REMOTE_CACHE_URL: https://acme-build-cache.my-team.workers.dev/projects/web-app + VP_REMOTE_CACHE: read-write +``` + +Keep the job's checkout, dependency installation, and `vp run` build steps. For PR builds, use `VP_REMOTE_CACHE: read` and omit `id-token: write`. The RFC also proposes `VP_REMOTE_CACHE: off` to disable remote caching. The URL is public and can be stored as a GitHub repository variable. No Cloudflare secrets or GitHub environment are required for application cache access. + +## Add or update repository bindings + +One deployment can serve multiple repositories. Add another public repository with a separate namespace: + +```sh +pnpm operator bind --namespace shared-ui --repo acme/shared-ui +``` + +This redeploys the namespace configuration and prints `https://acme-build-cache.my-team.workers.dev/projects/shared-ui`. Configure that repository's client with its own endpoint. + +After renaming or transferring a repository, or changing its default branch, review and refresh its existing binding: + +```sh +pnpm operator bind --namespace web-app --repo acme/web-app +``` + +Use the new repository path after a rename or transfer. `bind` verifies the immutable repository ID and refreshes the owner and branch. To bind an unrelated repository, choose a new namespace. + +To pause uploads or withdraw public access: + +```sh +pnpm operator policy --namespace web-app --writes off +pnpm operator policy --namespace web-app --enabled off +``` + +Making the GitHub repository private does not withdraw previously published cache data. Use the operator to disable or purge the namespace. The [operations reference](../README.md#operations) covers retention changes, upgrades, backups, purge, and teardown. + +## Troubleshooting + +| Symptom | Check | +| ------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| Setup cannot find the repository | `--repo` must be an existing public GitHub.com `owner/repository` | +| Cloudflare API returns `403` | Token permissions, account scope, and R2 activation | +| Namespace requests return `404` | Exact endpoint, namespace, and scope/deployment switches; fetch misses are also `404` | +| `/store` returns `401` | Missing, invalid, or expired GitHub OIDC token | +| `/store` returns `403` | Token audience, repository/owner IDs, registered branch, `push` event, and write switch | +| `/store` returns `503` | Quotas, concurrent uploads, storage availability, and policy changes; inspect `pnpm operator status` | +| `vp run` does not contact the service | This source tree has no remote-cache client adapter; the RFC settings alone cannot add it | + +The `REMOTE_CACHE_*` variables in this project's staging workflow configure its own test deployment. They are not application-client settings. Use the dedicated resources and namespace you created above for your repository. diff --git a/packages/remote-cache/docs/0001-remote-cache.md b/packages/remote-cache/rfcs/0001-remote-cache.md similarity index 99% rename from packages/remote-cache/docs/0001-remote-cache.md rename to packages/remote-cache/rfcs/0001-remote-cache.md index 78d338d3e..540c4bb1a 100644 --- a/packages/remote-cache/docs/0001-remote-cache.md +++ b/packages/remote-cache/rfcs/0001-remote-cache.md @@ -192,7 +192,7 @@ Upload in the background with bounded concurrency so dependent tasks can start w The operator binds a public repository during deployment. This form illustrates the setup inputs and automatic values: -![Configuration form: repository, namespace, and main branch inputs; automatic IDs, access policy, and public endpoint.](images/repository-binding-form.svg) +![Configuration form: repository, namespace, and main branch inputs; automatic IDs, access policy, and public endpoint.](../docs/images/repository-binding-form.svg) Setup saves both immutable IDs, the branch ref, and the audience in the server policy. The audience identifies the token's intended recipient. The client derives the same audience from its configured endpoint, without a trailing slash. diff --git a/packages/remote-cache/scripts/ci.ts b/packages/remote-cache/scripts/ci.ts index 2ab56bdf7..f659b2854 100644 --- a/packages/remote-cache/scripts/ci.ts +++ b/packages/remote-cache/scripts/ci.ts @@ -17,9 +17,13 @@ interface Settings { deployment: string; origin: string; writes: boolean; + profile: 'free' | 'paid'; } export function settingsFrom(env: Record): Settings { + const profile = env['REMOTE_CACHE_PROFILE'] || 'free'; + if (profile !== 'free' && profile !== 'paid') + throw new Error('Set REMOTE_CACHE_PROFILE to free or paid'); const prefix = env['REMOTE_CACHE_RESOURCE_PREFIX'] || 'vp-cache-ci'; if (!/^[a-z0-9][a-z0-9-]{0,30}-ci$/.test(prefix)) throw new Error( @@ -60,6 +64,7 @@ export function settingsFrom(env: Record): Settings deployment: `${revision}-${run}-${attempt}`, origin: `https://${name}.${subdomain}.workers.dev`, writes, + profile, }; } @@ -172,7 +177,7 @@ async function deploy(settings: Settings): Promise { '--origin', settings.origin, '--profile', - 'paid', + settings.profile, '--retention-days', '1', '--byte-limit', diff --git a/packages/remote-cache/test/deployed.test.ts b/packages/remote-cache/test/deployed.test.ts index 65ca16d58..2e49db92b 100644 --- a/packages/remote-cache/test/deployed.test.ts +++ b/packages/remote-cache/test/deployed.test.ts @@ -116,6 +116,11 @@ void test('CI shares persistent staging across events and only permits writes on assert.equal(main.name, 'vp-cache-ci-staging'); assert.equal(main.origin, 'https://vp-cache-ci-staging.example.workers.dev'); assert.equal(main.writes, true); + assert.equal(main.profile, 'free'); + assert.equal(settingsFrom({ ...env, REMOTE_CACHE_PROFILE: '' }).profile, 'free'); + for (const profile of ['free', 'paid']) + assert.equal(settingsFrom({ ...env, REMOTE_CACHE_PROFILE: profile }).profile, profile); + assert.throws(() => settingsFrom({ ...env, REMOTE_CACHE_PROFILE: 'invalid' }), /free or paid/); for (const override of [ { GITHUB_EVENT_NAME: 'workflow_dispatch' }, { GITHUB_EVENT_NAME: 'pull_request', GITHUB_REF: 'refs/pull/718/merge' }, diff --git a/packages/remote-cache/test/operator.test.ts b/packages/remote-cache/test/operator.test.ts index ae2af1027..9bf6795ff 100644 --- a/packages/remote-cache/test/operator.test.ts +++ b/packages/remote-cache/test/operator.test.ts @@ -262,6 +262,7 @@ void test('operator setup creates resources and initializes an empty database', io, ); assert.ok(database && bucket && deployed); + assert.equal(config.vars['GC_BATCH_SIZE'], '16'); assert.deepEqual(JSON.parse(config.vars['NAMESPACES']!), ['test']); const scope = await h.db.prepare("SELECT endpoint FROM scopes WHERE scope_id = 'test'").first(); assert.equal(scope!.endpoint, 'https://cache.example.com/projects/test'); From 598003cb041ff6aaf0e67dff4e0050a79a7bbb79 Mon Sep 17 00:00:00 2001 From: MK Date: Wed, 16 Sep 2026 16:33:08 +0800 Subject: [PATCH 14/17] feat(remote-cache): add Deploy to Cloudflare button Co-authored-by: GPT-6 Codex --- .../scripts/check-remote-cache-standalone.mjs | 41 + .github/workflows/ci.yml | 1 + CHANGELOG.md | 2 +- README.md | 2 + package.json | 3 +- packages/remote-cache/.gitignore | 3 + packages/remote-cache/LICENSE | 21 + packages/remote-cache/README.md | 6 +- packages/remote-cache/docs/e2e-plan.md | 20 + packages/remote-cache/docs/self-hosting.md | 40 + packages/remote-cache/package.json | 26 +- packages/remote-cache/pnpm-lock.yaml | 976 ++++++++++++++++++ packages/remote-cache/pnpm-workspace.yaml | 7 + packages/remote-cache/scripts/deploy.ts | 137 +++ packages/remote-cache/scripts/operator.ts | 25 +- packages/remote-cache/test/deploy.test.ts | 198 ++++ packages/remote-cache/test/index.test.ts | 1 + packages/remote-cache/wrangler.jsonc | 4 + pnpm-lock.yaml | 17 +- 19 files changed, 1514 insertions(+), 16 deletions(-) create mode 100644 .github/scripts/check-remote-cache-standalone.mjs create mode 100644 packages/remote-cache/LICENSE create mode 100644 packages/remote-cache/pnpm-lock.yaml create mode 100644 packages/remote-cache/pnpm-workspace.yaml create mode 100644 packages/remote-cache/scripts/deploy.ts create mode 100644 packages/remote-cache/test/deploy.test.ts diff --git a/.github/scripts/check-remote-cache-standalone.mjs b/.github/scripts/check-remote-cache-standalone.mjs new file mode 100644 index 000000000..8e272f438 --- /dev/null +++ b/.github/scripts/check-remote-cache-standalone.mjs @@ -0,0 +1,41 @@ +import { spawn } from 'node:child_process'; +import { cp, mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; + +const pnpm = process.env.npm_execpath; +if (!pnpm) throw new Error('Run this check through pnpm check-remote-cache-standalone'); +const directory = await mkdtemp(join(tmpdir(), 'remote-cache-standalone-')); +const excluded = new Set([ + 'node_modules', + '.wrangler', + 'dist', + 'wrangler.operator.json', + 'operator-state.json', + 'benchmark-results.json', + 'e2e-results', +]); +try { + await cp(new URL('../../packages/remote-cache/', import.meta.url), directory, { + recursive: true, + filter(source) { + const name = basename(source); + return !excluded.has(name) && !name.startsWith('.dev.vars') && !name.startsWith('.env'); + }, + }); + for (const args of [['install', '--frozen-lockfile'], ['check'], ['build']]) { + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [pnpm, ...args], { + cwd: directory, + stdio: 'inherit', + shell: false, + }); + child.once('error', reject); + child.once('exit', (code) => + code === 0 ? resolve() : reject(new Error(`Standalone ${args[0]} failed (${code})`)), + ); + }); + } +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 240cd8f3d..74859233a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -399,6 +399,7 @@ jobs: - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 - run: pnpm check-remote-cache + - run: pnpm check-remote-cache-standalone done: runs-on: namespace-profile-linux-x64-default diff --git a/CHANGELOG.md b/CHANGELOG.md index e5b0dfd3c..18618d7d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -- **Added** A self-hosted public remote cache service with GitHub Actions write authorization, storage limits, and automatic cleanup ([#718](https://github.com/voidzero-dev/vite-task/pull/718)). +- **Added** A self-hosted public remote cache service with a Deploy to Cloudflare button, GitHub Actions write authorization, storage limits, and automatic cleanup ([#718](https://github.com/voidzero-dev/vite-task/pull/718)). - **Fixed** An invalid glob in `--filter` no longer shows its error message twice ([#763](https://github.com/voidzero-dev/vite-task/pull/763)). - **Changed** The detailed summary from `vp run --verbose` and `vp run --last-details` now shows each underlying cause of an error on its own line ([#761](https://github.com/voidzero-dev/vite-task/pull/761)). - **Added** Remote caching. Configure an endpoint with the workspace's `cache: { remote: { url } }` or `VP_REMOTE_CACHE_URL`, and choose access with `--remote-cache=off|read|read-write` or `VP_REMOTE_CACHE`. The default is `read` with an endpoint and `off` without one. After a local cache miss, `vp run` looks the task up in the remote cache and, on a hit, restores its outputs and caches it locally. The task output and the run summary show which hits came from the remote cache. A failed read is just a cache miss, with the failure as its reason. In `read-write` mode, `vp run` also uploads the results of successful, cacheable tasks after caching them locally. A failed upload doesn't fail the task; the run summary shows a warning instead. Ctrl-C, or a failing task, stops remote cache requests right away, and a task still being looked up doesn't start. Tasks can opt out with `cache: { remote: false }`. Requests use the proxy environment variables or, on macOS and Windows, the system proxy settings ([#727](https://github.com/voidzero-dev/vite-task/pull/727), [#755](https://github.com/voidzero-dev/vite-task/pull/755), [#756](https://github.com/voidzero-dev/vite-task/pull/756), [#757](https://github.com/voidzero-dev/vite-task/pull/757), [#764](https://github.com/voidzero-dev/vite-task/pull/764), [#771](https://github.com/voidzero-dev/vite-task/pull/771), [#772](https://github.com/voidzero-dev/vite-task/pull/772)). diff --git a/README.md b/README.md index 17cbf559e..ed373a24e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ vp run --cache build # run with caching enabled ## Remote cache service +[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https%3A%2F%2Fgithub.com%2Fvoidzero-dev%2Fvite-task%2Ftree%2Fmain%2Fpackages%2Fremote-cache) + To deploy the public remote cache service in your own Cloudflare account and bind it to a GitHub repository, follow the [self-hosting guide](packages/remote-cache/docs/self-hosting.md). The service is available in this source tree; the `vp run` remote-cache client adapter is not yet implemented here. ## Sponsors diff --git a/package.json b/package.json index 7bc6d429a..9af60d308 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "scripts": { "prepare": "vp config", "build-vite-task-client-types": "tsc -p packages/vite-task-client/tsconfig.json", - "check-remote-cache": "pnpm --filter @voidzero-dev/remote-cache check && pnpm --filter @voidzero-dev/remote-cache smoke" + "check-remote-cache": "pnpm --filter @voidzero-dev/remote-cache check && pnpm --filter @voidzero-dev/remote-cache smoke", + "check-remote-cache-standalone": "node .github/scripts/check-remote-cache-standalone.mjs" }, "devDependencies": { "@tsconfig/strictest": "catalog:", diff --git a/packages/remote-cache/.gitignore b/packages/remote-cache/.gitignore index 36aba5f3b..555b28d8a 100644 --- a/packages/remote-cache/.gitignore +++ b/packages/remote-cache/.gitignore @@ -1,4 +1,7 @@ .wrangler/ +node_modules/ +dist/ +.env* .dev.vars* operator-state.json wrangler.operator.json diff --git a/packages/remote-cache/LICENSE b/packages/remote-cache/LICENSE new file mode 100644 index 000000000..cf067b10e --- /dev/null +++ b/packages/remote-cache/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026-present, VoidZero Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/remote-cache/README.md b/packages/remote-cache/README.md index 46e5f8608..a256cf3d8 100644 --- a/packages/remote-cache/README.md +++ b/packages/remote-cache/README.md @@ -4,7 +4,9 @@ This package implements the server in the [remote cache RFC](rfcs/0001-remote-ca The package contains no `vp run` client adapter. Cache keys, values, and blobs remain opaque. A successful lookup does not prove that a result is reusable; a client must validate its inputs and output archive. -**Start here:** [Deploy the service and bind your repository](docs/self-hosting.md). The guide covers Cloudflare credentials, deployment, custom domains, repository binding, and the current client-integration limitation. +[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https%3A%2F%2Fgithub.com%2Fvoidzero-dev%2Fvite-task%2Ftree%2Fmain%2Fpackages%2Fremote-cache) + +**Start here:** [Deploy the service and bind your repository](docs/self-hosting.md#quick-start-deploy-to-cloudflare). The button creates a standalone repository and provisions storage. Set `CACHE_REPOSITORY` to your public GitHub repository and keep the default `free` profile. The guide covers build-token permissions, deployment checks, custom domains, and the current client-integration limitation. ## Protocol @@ -37,6 +39,8 @@ pnpm install --frozen-lockfile pnpm check-remote-cache ``` +In the standalone repository created by the deployment button, run `pnpm install --frozen-lockfile`, `pnpm check`, and `pnpm smoke` instead. In the monorepo, `pnpm check-remote-cache-standalone` checks that an isolated copy installs and builds without workspace dependencies. Update both lockfiles when this package's dependencies change. + `just remote-cache` runs the same checks. The command generates binding/runtime types, checks all source, operator, benchmark, and test files, runs isolated workerd/D1/R2 tests, and bundles a deployment dry run. Tests generate a temporary RSA key and intercept only GitHub's fixed JWKS URL. They need no Cloudflare account, GitHub credentials, or client adapter. CI runs them on Linux, macOS, and Windows. From this package directory, `pnpm dev` starts Wrangler with local bindings. Apply the local schema with `pnpm exec wrangler d1 migrations apply INDEX --local`. No namespace is enabled in the template. Use `pnpm test` for a fully initialized, isolated smoke test; it exercises authorized writes without creating a development signing-key bypass in production. diff --git a/packages/remote-cache/docs/e2e-plan.md b/packages/remote-cache/docs/e2e-plan.md index 7aa53fa36..04af4108c 100644 --- a/packages/remote-cache/docs/e2e-plan.md +++ b/packages/remote-cache/docs/e2e-plan.md @@ -2,6 +2,26 @@ This plan checks the service through its public HTTP endpoints after deployment to Cloudflare. A deployment dry run or a passing Miniflare test is not evidence of a successful Cloudflare deployment. +## Deploy to Cloudflare button + +The [self-hosting quick start](self-hosting.md#quick-start-deploy-to-cloudflare) uses Cloudflare's deployment button. Its `pnpm deploy` command reuses the provisioned `INDEX` and `ARTIFACTS` bindings, applies migrations, registers the selected public repository, disables public R2 access, installs lifecycle rules, and deploys the Worker. Free is the default. + +Automated regression checks cover independently named resources, an empty D1 database, repository binding, optional Paid cleanup batches, retries from a clean checkout, preserved withdrawal and quotas, rejected repository reassignment, missing storage, and placeholder IDs. Post-deployment HTTP checks require the current deployment ID and `Cache-Control: no-store`. They check an anonymous upload (`401`), malformed fetch (`400`), and a valid cache miss (`404`). A disabled namespace must continue to return `404`. Polling allows for route/revision propagation and fails the build if checks never pass. These checks do not publish data or validate authorized uploads. + +Linux, macOS, and Windows CI also copy the package outside the monorepo, install with its standalone frozen lockfile, check types, and bundle the Worker. This catches accidental workspace dependencies that would break the button's subdirectory copy. + +Before releasing the button, complete this live acceptance exercise in a dedicated account or with dedicated test resources: + +1. Use the package URL at the candidate commit in Deploy to Cloudflare. Start from an account with R2 enabled and a Workers subdomain. Use Workers Free first. +2. Choose distinct Worker, D1, and R2 names, and bind a public application repository different from the Worker source repository. Confirm build-token permissions, including D1, and the default build/deploy commands. +3. Confirm migrations, private bucket settings, cleanup rules, correct repository IDs/default branch/audience, and successful HTTP checks. Inspect build logs and the copied repository for credential leaks. +4. Push an update to the new source repository. Confirm the same resources and data remain, and the checks observe the new deployment ID. Retry after a failed deployment without creating duplicate resources. +5. Disable the namespace, redeploy, and confirm it remains unavailable. Restore it explicitly before testing uploads. Confirm an invalid repository and insufficient D1 permissions fail the build without a success message. +6. From a default-branch `push` job in the bound application repository, upload a small value and blob with a real GitHub OIDC token. Read both anonymously and compare bytes. Confirm a PR token cannot upload. This is separate from the unauthenticated deployment checks. +7. Keep non-production builds disabled, then check deployed CPU and account usage. Exercise maximum payloads and Paid separately using the release matrix below. + +Local tests cannot prove the Cloudflare setup form, resource provisioning, build-token injection, or Free-plan CPU behavior. Record the live build URL and results when this exercise is complete. The shared staging workflow below continues to validate the service independently of this user deployment path. + ## Deployment flow `.github/workflows/remote-cache-deploy.yml` runs when a PR or a push to `main` changes this package, its deployment workflows, or the root dependency/build configuration. It also supports manual runs from the default branch. diff --git a/packages/remote-cache/docs/self-hosting.md b/packages/remote-cache/docs/self-hosting.md index 38949b36c..49c2a82ca 100644 --- a/packages/remote-cache/docs/self-hosting.md +++ b/packages/remote-cache/docs/self-hosting.md @@ -4,6 +4,46 @@ This guide deploys the service in your Cloudflare account and gives your reposit **Current status:** the server and operator CLI work in this source tree. The `vp run` remote-cache client adapter is not implemented here. You can deploy and verify the HTTP service now. Automatic task uploads and reuse require a compatible client; the proposed Vite+ settings are described separately below. +## Quick start: Deploy to Cloudflare + +[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https%3A%2F%2Fgithub.com%2Fvoidzero-dev%2Fvite-task%2Ftree%2Fmain%2Fpackages%2Fremote-cache) + +The button copies this package into a new GitHub repository in your account and connects it to Workers Builds. It provisions a Worker, a D1 database, and an R2 bucket. You do not need a local checkout or GitHub Actions secrets. The repository that hosts the Worker can differ from the application repository that uses its cache. See [Cloudflare's deployment-button guide](https://developers.cloudflare.com/workers/platform/deploy-buttons/). + +1. Enable R2 in your Cloudflare account and create a [Workers subdomain](https://developers.cloudflare.com/workers/configuration/routing/workers-dev/). Workers Free is the default; R2 has separate usage allowances and billing. +2. Click the button and connect your GitHub and Cloudflare accounts. Choose a name for the new source repository, Worker, D1 database, and R2 bucket. Use dedicated storage. The storage names can differ from the Worker name. +3. Set the following variables in the setup form. They are public deployment settings, not secrets. + + | Variable | Value | + | ------------------ | ---------------------------------------------------------------------- | + | `CACHE_REPOSITORY` | Your public GitHub repository, for example `acme/web-app` | + | `CACHE_NAMESPACE` | Endpoint namespace; defaults to `cache` | + | `CACHE_PROFILE` | Keep `free`; choose `paid` only when you want the larger cleanup batch | + +4. Use `pnpm build` as the build command and `pnpm deploy` as the deploy command. The copied package includes its own dependency versions and lockfile. Use Node.js 22.12 or newer and the pinned pnpm version. The root directory in this new repository is `/`. +5. Check the selected build API token's permissions. Deployment needs Workers, D1, and Workers R2 Storage write access. Cloudflare manages the build token, but its [documented default permissions](https://developers.cloudflare.com/workers/ci-cd/builds/configuration/#api-token) do not include D1. Add D1 Edit / Write to that token under **My Profile → API Tokens**, or select a token with these permissions under **Worker → Settings → Build**. Keep the token in Cloudflare; do not add it as a Worker variable or commit it. +6. Deploy. If you changed the token after the first build started, retry the build. The deploy command applies migrations, configures private storage and cleanup, binds your application repository, and checks the public endpoint. A successful log ends with `Deployment checks passed. Cache endpoint: ...`. + +For a Worker named `acme-build-cache` on `my-team.workers.dev`, the default endpoint is: + +```text +https://acme-build-cache.my-team.workers.dev/projects/cache +``` + +Use that complete URL as the client endpoint and upload-token audience. Continue with [Connect your application build](#5-connect-your-application-build) below. Opening `/` in a browser returns `404`; the service has no dashboard. + +Workers Builds deploys subsequent pushes to the selected production branch. Disable non-production branch builds for this service so preview code does not share production storage. To change the deployment inputs, edit `vars.CACHE_REPOSITORY`, `vars.CACHE_NAMESPACE`, or `vars.CACHE_PROFILE` in the copied `wrangler.jsonc`, or set the same names as **build variables**. Build variables override the file. Runtime dashboard variable edits alone do not configure the deployment script. + +Keep the generated D1 ID and R2 name in `wrangler.jsonc`. Each build reconstructs `wrangler.operator.json` from those bindings and the stored policies. Repeat deployments preserve disabled namespaces and storage budgets. A namespace cannot be reassigned to a different repository. Repository transfers or default-branch changes require an explicit `operator bind`, as described below. Use the CLI path for custom domains. + +The automatic checks verify the new deployment ID, disabled HTTP caching, rejected anonymous uploads, invalid requests, and cache misses. They do not upload data or prove authorized GitHub OIDC writes. The [e2e plan](e2e-plan.md#deploy-to-cloudflare-button) covers the full acceptance checks. + +The button targets the package on `main`. Before this change is merged, test the flow by supplying the package URL at the reviewed PR commit to Cloudflare's deployment page. + +## CLI deployment + +Use the following steps for direct deployment or custom domains. If you already deployed with the button, skip to [Connect your application build](#5-connect-your-application-build). To run later operator commands, clone the repository created by the button, install dependencies at its root, set the Cloudflare credentials below, and run `pnpm deploy` once to reconstruct its operator configuration. Run `pnpm operator` commands from that root. + ## 1. Prepare the deployment checkout Use a checkout of this repository that contains `packages/remote-cache`. Install Node.js 22.12 or newer and the pnpm version in the root `package.json`. From the repository root, run: diff --git a/packages/remote-cache/package.json b/packages/remote-cache/package.json index 3641cc450..be26f1d88 100644 --- a/packages/remote-cache/package.json +++ b/packages/remote-cache/package.json @@ -7,6 +7,7 @@ "scripts": { "dev": "wrangler dev --test-scheduled", "build": "wrangler deploy --dry-run --outdir dist", + "deploy": "node --import tsx scripts/deploy.ts", "types": "node -e \"require('node:fs').mkdirSync('.wrangler', { recursive: true })\" && wrangler types .wrangler/worker-configuration.d.ts --env-interface Env --strict-vars false", "check": "pnpm types && tsc --noEmit", "test": "node --import tsx --test test/index.test.ts", @@ -20,16 +21,37 @@ "jose": "6.2.12" }, "devDependencies": { - "@types/node": "catalog:", + "@cloudflare/workers-types": "5.20260911.1", + "@types/node": "25.0.3", "cborg": "6.1.2", "esbuild": "0.28.1", "jsonc-parser": "3.3.1", "miniflare": "5.20260911.0-alpha", "tsx": "4.23.13", - "typescript": "catalog:", + "typescript": "6.0.3", "wrangler": "4.131.1" }, "engines": { "node": ">=22.12.0" + }, + "packageManager": "pnpm@11.24.0", + "cloudflare": { + "bindings": { + "CACHE_REPOSITORY": { + "description": "Public GitHub repository allowed to upload, in `owner/repository` form. This can differ from the repository that hosts this Worker." + }, + "CACHE_NAMESPACE": { + "description": "Cache namespace in the endpoint. Use 1–63 lowercase letters, digits, or hyphens." + }, + "CACHE_PROFILE": { + "description": "Use `free` by default. Choose `paid` for a larger cleanup batch on Workers Paid. This does not change your subscription." + }, + "INDEX": { + "description": "Dedicated D1 database for cache metadata and repository authorization." + }, + "ARTIFACTS": { + "description": "Dedicated private R2 Standard bucket for cache values and build artifacts. Enable R2 in your account first." + } + } } } diff --git a/packages/remote-cache/pnpm-lock.yaml b/packages/remote-cache/pnpm-lock.yaml new file mode 100644 index 000000000..608e47ab9 --- /dev/null +++ b/packages/remote-cache/pnpm-lock.yaml @@ -0,0 +1,976 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + jose: + specifier: 6.2.12 + version: 6.2.12 + devDependencies: + '@cloudflare/workers-types': + specifier: 5.20260911.1 + version: 5.20260911.1 + '@types/node': + specifier: 25.0.3 + version: 25.0.3 + cborg: + specifier: 6.1.2 + version: 6.1.2 + esbuild: + specifier: 0.28.1 + version: 0.28.1 + jsonc-parser: + specifier: 3.3.1 + version: 3.3.1 + miniflare: + specifier: 5.20260911.0-alpha + version: 5.20260911.0-alpha(@types/node@25.0.3) + tsx: + specifier: 4.23.13 + version: 4.23.13 + typescript: + specifier: 6.0.3 + version: 6.0.3 + wrangler: + specifier: 4.131.1 + version: 4.131.1(@cloudflare/workers-types@5.20260911.1)(@types/node@25.0.3) + +packages: + + '@cloudflare/kv-asset-handler@0.5.0': + resolution: {integrity: sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==} + engines: {node: '>=22.0.0'} + + '@cloudflare/unenv-preset@2.16.1': + resolution: {integrity: sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==} + peerDependencies: + unenv: 2.0.0-rc.24 + workerd: '>1.20260305.0 <2.0.0-0' + peerDependenciesMeta: + workerd: + optional: true + + '@cloudflare/workerd-darwin-64@1.20260911.1': + resolution: {integrity: sha512-785eaY1bkR1cm4Z/PCUeteZYmTMe6lre2zz63/GdGGimsoMsKxgl4brFPRukim8iv28EyD1XoCB/VPYF20BERA==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@cloudflare/workerd-darwin-arm64@1.20260911.1': + resolution: {integrity: sha512-WU4bFqEN0H7ndGWxoedegv95DmNVBtv0ncXcHG9nYFTUI78sxEb0qoT3U6Ga4hyBkzsJFBX/zvVBIGX3qKldGA==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@cloudflare/workerd-linux-64@1.20260911.1': + resolution: {integrity: sha512-0Y2gy62oxQxWa38qinSPE6zNL5+JmumJtDY9AWW1HB8KHuATxN71o5MGzmVFfB8PwZsiHfUd2Sv7O22krCOrhw==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@cloudflare/workerd-linux-arm64@1.20260911.1': + resolution: {integrity: sha512-kttNPnx1r2lCqFUoMH62z7CqGV+j4QBbw5fdtaz4pzOrzBv0AWkNATt7onFUe+SwP8zhcepMtbm2F4kKzTf6VA==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@cloudflare/workerd-windows-64@1.20260911.1': + resolution: {integrity: sha512-5iO/YfoBDOgO3CrHdkiiVP8SL3O2jC+c6Ux3d378TSPKLhU5+CgHjtE/ZSodWQrzr4FzFRqdW8S7n5nbyD1MHQ==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + + '@cloudflare/workers-types@5.20260911.1': + resolution: {integrity: sha512-yiAvknjulcU85B3yB4aKOn9+l+garWP+AbHgsdCFckeRYDdhZ1rPULi64BDf52R3VTaKqxi47kF+o9ZbjsCt5g==} + + '@cspotcode/source-map-support@0.8.1': + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + + '@emnapi/runtime@1.11.3': + resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} + + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.35.4': + resolution: {integrity: sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.35.4': + resolution: {integrity: sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [darwin] + + '@img/sharp-freebsd-wasm32@0.35.4': + resolution: {integrity: sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==} + engines: {node: '>=20.9.0'} + os: [freebsd] + + '@img/sharp-libvips-darwin-arm64@1.3.3': + resolution: {integrity: sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.3.3': + resolution: {integrity: sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.3.3': + resolution: {integrity: sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.3.3': + resolution: {integrity: sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-ppc64@1.3.3': + resolution: {integrity: sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-riscv64@1.3.3': + resolution: {integrity: sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.3.3': + resolution: {integrity: sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.3.3': + resolution: {integrity: sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.3.3': + resolution: {integrity: sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.3.3': + resolution: {integrity: sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.35.4': + resolution: {integrity: sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.35.4': + resolution: {integrity: sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==} + engines: {node: '>=20.9.0'} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-ppc64@0.35.4': + resolution: {integrity: sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==} + engines: {node: '>=20.9.0'} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-riscv64@0.35.4': + resolution: {integrity: sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==} + engines: {node: '>=20.9.0'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.35.4': + resolution: {integrity: sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==} + engines: {node: '>=20.9.0'} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.35.4': + resolution: {integrity: sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.35.4': + resolution: {integrity: sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.35.4': + resolution: {integrity: sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.35.4': + resolution: {integrity: sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==} + engines: {node: '>=20.9.0'} + + '@img/sharp-webcontainers-wasm32@0.35.4': + resolution: {integrity: sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==} + engines: {node: '>=20.9.0'} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.35.4': + resolution: {integrity: sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.35.4': + resolution: {integrity: sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==} + engines: {node: ^20.9.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.35.4': + resolution: {integrity: sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [win32] + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + + '@jridgewell/trace-mapping@0.3.9': + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + + '@poppinss/colors@4.1.6': + resolution: {integrity: sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==} + + '@poppinss/dumper@0.6.5': + resolution: {integrity: sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==} + + '@poppinss/exception@1.2.3': + resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} + + '@sindresorhus/is@7.2.0': + resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} + engines: {node: '>=18'} + + '@speed-highlight/core@1.2.24': + resolution: {integrity: sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==} + + '@types/node@25.0.3': + resolution: {integrity: sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==} + + blake3-wasm@2.1.5: + resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} + + cborg@6.1.2: + resolution: {integrity: sha512-hQfFh6FuuCDoycN68FtUpjtQx5kCtxOLA8msbpJZxjtxaMqxgHl+4GNwO+0YexH/lHmVzhhAKa0ua+vtmTRoVw==} + hasBin: true + + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + error-stack-parser-es@1.0.5: + resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} + + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + jose@6.2.12: + resolution: {integrity: sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==} + + jsonc-parser@3.3.1: + resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} + + kleur@4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + + miniflare@5.20260911.0-alpha: + resolution: {integrity: sha512-CRieJmvHx+7rNqnA5SKdsYsER6rfkUIE/jruIUw+fLhsQ4sORfuMtr3+FQzsQ9/y8lhk061V4Fl1DFdHiyBB6g==} + engines: {node: '>=22.0.0'} + + path-to-regexp@6.3.0: + resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + sharp@0.35.4: + resolution: {integrity: sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==} + engines: {node: '>=20.9.0'} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tsx@4.23.13: + resolution: {integrity: sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==} + engines: {node: '>=18.0.0'} + hasBin: true + + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@7.16.0: + resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + + unenv@2.0.0-rc.24: + resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} + + workerd@1.20260911.1: + resolution: {integrity: sha512-vRr8QdBxueQOZJO1hRCI73EZlix87IAyBAcSyI3rA1VB+6oxjw3oaqzYnIV8C4IOPtUgihbdMAgzkb5GM4V7DQ==} + engines: {node: '>=16'} + hasBin: true + + wrangler@4.131.1: + resolution: {integrity: sha512-1u5FMdJAn6UOcL02cVsIITcnHrk6mC7N+RF10EkVhPL18R/o9g5BZb4PCjByL+3AsRP5wQpppCIPHhYPRmIJwg==} + engines: {node: '>=22.0.0'} + hasBin: true + peerDependencies: + '@cloudflare/workers-types': ^5.20260911.1 + peerDependenciesMeta: + '@cloudflare/workers-types': + optional: true + + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + youch-core@0.3.3: + resolution: {integrity: sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==} + + youch@4.1.0-beta.10: + resolution: {integrity: sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==} + +snapshots: + + '@cloudflare/kv-asset-handler@0.5.0': {} + + '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260911.1)': + dependencies: + unenv: 2.0.0-rc.24 + optionalDependencies: + workerd: 1.20260911.1 + + '@cloudflare/workerd-darwin-64@1.20260911.1': + optional: true + + '@cloudflare/workerd-darwin-arm64@1.20260911.1': + optional: true + + '@cloudflare/workerd-linux-64@1.20260911.1': + optional: true + + '@cloudflare/workerd-linux-arm64@1.20260911.1': + optional: true + + '@cloudflare/workerd-windows-64@1.20260911.1': + optional: true + + '@cloudflare/workers-types@5.20260911.1': {} + + '@cspotcode/source-map-support@0.8.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + + '@emnapi/runtime@1.11.3': + dependencies: + tslib: 2.8.1 + optional: true + + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.28.1': + optional: true + + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.3.3 + optional: true + + '@img/sharp-darwin-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.3.3 + optional: true + + '@img/sharp-freebsd-wasm32@0.35.4': + dependencies: + '@img/sharp-wasm32': 0.35.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-darwin-x64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-arm@1.3.3': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.3.3': + optional: true + + '@img/sharp-libvips-linux-s390x@1.3.3': + optional: true + + '@img/sharp-libvips-linux-x64@1.3.3': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.3.3': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.3.3': + optional: true + + '@img/sharp-linux-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.3.3 + optional: true + + '@img/sharp-linux-arm@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.3.3 + optional: true + + '@img/sharp-linux-ppc64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.3.3 + optional: true + + '@img/sharp-linux-riscv64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.3.3 + optional: true + + '@img/sharp-linux-s390x@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.3.3 + optional: true + + '@img/sharp-linux-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.3.3 + optional: true + + '@img/sharp-linuxmusl-arm64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 + optional: true + + '@img/sharp-linuxmusl-x64@0.35.4': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.3.3 + optional: true + + '@img/sharp-wasm32@0.35.4': + dependencies: + '@emnapi/runtime': 1.11.3 + optional: true + + '@img/sharp-webcontainers-wasm32@0.35.4': + dependencies: + '@img/sharp-wasm32': 0.35.4 + optional: true + + '@img/sharp-win32-arm64@0.35.4': + optional: true + + '@img/sharp-win32-ia32@0.35.4': + optional: true + + '@img/sharp-win32-x64@0.35.4': + optional: true + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.6.0': {} + + '@jridgewell/trace-mapping@0.3.9': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.6.0 + + '@poppinss/colors@4.1.6': + dependencies: + kleur: 4.1.5 + + '@poppinss/dumper@0.6.5': + dependencies: + '@poppinss/colors': 4.1.6 + '@sindresorhus/is': 7.2.0 + supports-color: 10.2.2 + + '@poppinss/exception@1.2.3': {} + + '@sindresorhus/is@7.2.0': {} + + '@speed-highlight/core@1.2.24': {} + + '@types/node@25.0.3': + dependencies: + undici-types: 7.16.0 + + blake3-wasm@2.1.5: {} + + cborg@6.1.2: {} + + cookie@1.1.1: {} + + detect-libc@2.1.2: {} + + error-stack-parser-es@1.0.5: {} + + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + + fsevents@2.3.3: + optional: true + + jose@6.2.12: {} + + jsonc-parser@3.3.1: {} + + kleur@4.1.5: {} + + miniflare@5.20260911.0-alpha(@types/node@25.0.3): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + sharp: 0.35.4(@types/node@25.0.3) + undici: 7.29.0 + workerd: 1.20260911.1 + ws: 8.21.0 + youch: 4.1.0-beta.10 + transitivePeerDependencies: + - '@types/node' + - bufferutil + - utf-8-validate + + path-to-regexp@6.3.0: {} + + pathe@2.0.3: {} + + semver@7.8.5: {} + + sharp@0.35.4(@types/node@25.0.3): + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.8.5 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.35.4 + '@img/sharp-darwin-x64': 0.35.4 + '@img/sharp-freebsd-wasm32': 0.35.4 + '@img/sharp-libvips-darwin-arm64': 1.3.3 + '@img/sharp-libvips-darwin-x64': 1.3.3 + '@img/sharp-libvips-linux-arm': 1.3.3 + '@img/sharp-libvips-linux-arm64': 1.3.3 + '@img/sharp-libvips-linux-ppc64': 1.3.3 + '@img/sharp-libvips-linux-riscv64': 1.3.3 + '@img/sharp-libvips-linux-s390x': 1.3.3 + '@img/sharp-libvips-linux-x64': 1.3.3 + '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 + '@img/sharp-libvips-linuxmusl-x64': 1.3.3 + '@img/sharp-linux-arm': 0.35.4 + '@img/sharp-linux-arm64': 0.35.4 + '@img/sharp-linux-ppc64': 0.35.4 + '@img/sharp-linux-riscv64': 0.35.4 + '@img/sharp-linux-s390x': 0.35.4 + '@img/sharp-linux-x64': 0.35.4 + '@img/sharp-linuxmusl-arm64': 0.35.4 + '@img/sharp-linuxmusl-x64': 0.35.4 + '@img/sharp-webcontainers-wasm32': 0.35.4 + '@img/sharp-win32-arm64': 0.35.4 + '@img/sharp-win32-ia32': 0.35.4 + '@img/sharp-win32-x64': 0.35.4 + '@types/node': 25.0.3 + + supports-color@10.2.2: {} + + tslib@2.8.1: + optional: true + + tsx@4.23.13: + dependencies: + esbuild: 0.28.1 + optionalDependencies: + fsevents: 2.3.3 + + typescript@6.0.3: {} + + undici-types@7.16.0: {} + + undici@7.29.0: {} + + unenv@2.0.0-rc.24: + dependencies: + pathe: 2.0.3 + + workerd@1.20260911.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260911.1 + '@cloudflare/workerd-darwin-arm64': 1.20260911.1 + '@cloudflare/workerd-linux-64': 1.20260911.1 + '@cloudflare/workerd-linux-arm64': 1.20260911.1 + '@cloudflare/workerd-windows-64': 1.20260911.1 + + wrangler@4.131.1(@cloudflare/workers-types@5.20260911.1)(@types/node@25.0.3): + dependencies: + '@cloudflare/kv-asset-handler': 0.5.0 + '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260911.1) + blake3-wasm: 2.1.5 + esbuild: 0.28.1 + miniflare: 5.20260911.0-alpha(@types/node@25.0.3) + path-to-regexp: 6.3.0 + unenv: 2.0.0-rc.24 + workerd: 1.20260911.1 + optionalDependencies: + '@cloudflare/workers-types': 5.20260911.1 + fsevents: 2.3.3 + transitivePeerDependencies: + - '@types/node' + - bufferutil + - utf-8-validate + + ws@8.21.0: {} + + youch-core@0.3.3: + dependencies: + '@poppinss/exception': 1.2.3 + error-stack-parser-es: 1.0.5 + + youch@4.1.0-beta.10: + dependencies: + '@poppinss/colors': 4.1.6 + '@poppinss/dumper': 0.6.5 + '@speed-highlight/core': 1.2.24 + cookie: 1.1.1 + youch-core: 0.3.3 diff --git a/packages/remote-cache/pnpm-workspace.yaml b/packages/remote-cache/pnpm-workspace.yaml new file mode 100644 index 000000000..f14ed3aee --- /dev/null +++ b/packages/remote-cache/pnpm-workspace.yaml @@ -0,0 +1,7 @@ +# Used when Deploy to Cloudflare copies this package into its own repository. +packages: + - . + +allowBuilds: + esbuild: true + workerd: true diff --git a/packages/remote-cache/scripts/deploy.ts b/packages/remote-cache/scripts/deploy.ts new file mode 100644 index 000000000..e8a4f56e7 --- /dev/null +++ b/packages/remote-cache/scripts/deploy.ts @@ -0,0 +1,137 @@ +import { randomUUID } from 'node:crypto'; +import { setTimeout } from 'node:timers/promises'; +import { pathToFileURL } from 'node:url'; +import { encode } from 'cborg'; +import { + operatorIO, + query, + readTemplate, + runOperator, + type Config, + type OperatorIO, +} from './operator.ts'; + +// These checks use no upload credentials and do not publish cache data. +export async function checkDeployment( + endpoint: string, + deployment: string, + enabled: boolean, + request: typeof fetch = fetch, +): Promise { + const key = new TextEncoder().encode(`deployment-check:${randomUUID()}`); + const checks = [ + { path: 'store', status: enabled ? 401 : 404, body: undefined }, + { path: 'fetch', status: enabled ? 400 : 404, body: new Uint8Array() }, + { path: 'fetch', status: 404, body: encode({ key, secondary_key: key }) }, + ]; + for (const check of checks) { + const response = await request(`${endpoint}/${check.path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/cbor' }, + body: check.body, + redirect: 'error', + signal: AbortSignal.timeout(10_000), + }); + await response.body?.cancel(); + if ( + response.status !== check.status || + response.headers.get('X-Remote-Cache-Deployment') !== deployment || + response.headers.get('Cache-Control') !== 'no-store' + ) + throw new Error( + `Deployment check failed for /${check.path}: expected ${check.status}, received ${response.status}; check the deployed revision and cache headers`, + ); + } +} + +export async function deploy( + config: Config, + io: OperatorIO = operatorIO, + env: Record = process.env, + request: typeof fetch = fetch, + wait: (ms: number) => Promise = setTimeout, +): Promise { + const repository = env['CACHE_REPOSITORY'] ?? config.vars['CACHE_REPOSITORY']; + if (!repository || repository === 'owner/repository') + throw new Error( + 'Set CACHE_REPOSITORY to your public GitHub owner/repository in the deployment settings', + ); + const namespace = env['CACHE_NAMESPACE'] ?? config.vars['CACHE_NAMESPACE'] ?? 'cache'; + const profile = env['CACHE_PROFILE'] ?? config.vars['CACHE_PROFILE'] ?? 'free'; + const database = config.d1_databases.find((binding) => binding.binding === 'INDEX'); + const bucket = config.r2_buckets.find((binding) => binding.binding === 'ARTIFACTS'); + if ( + !database || + !bucket || + !/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(database.database_id) || + database.database_id === '00000000-0000-0000-0000-000000000000' || + !/^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/.test(bucket.bucket_name) + ) + throw new Error( + 'Use the D1 INDEX and R2 ARTIFACTS bindings provisioned by Deploy to Cloudflare', + ); + if (env['WRANGLER_CI_OVERRIDE_NAME'] && env['WRANGLER_CI_OVERRIDE_NAME'] !== config.name) + throw new Error( + 'The Worker name in wrangler.jsonc must match the connected Workers Builds project', + ); + const account = (await io.api('/workers/subdomain')) as { subdomain?: unknown }; + if (typeof account.subdomain !== 'string' || !/^[a-z0-9][a-z0-9-]{0,62}$/.test(account.subdomain)) + throw new Error('Create a workers.dev subdomain in your Cloudflare account before deploying'); + const origin = `https://${config.name}.${account.subdomain}.workers.dev`; + const deployment = env['WORKERS_CI_BUILD_UUID'] || randomUUID(); + await runOperator( + [ + 'setup', + '--name', + config.name, + '--namespace', + namespace, + '--repo', + repository, + '--origin', + origin, + '--profile', + profile, + ], + { + ...io, + async writeConfig(value) { + value.vars['DEPLOYMENT_ID'] = deployment; + await io.writeConfig(value); + }, + }, + { database, bucket }, + ); + const deployed = await io.readConfig(); + const policies = await query( + io, + deployed, + 'SELECT scopes.enabled AS scope_enabled, deployment.enabled AS deployment_enabled FROM scopes CROSS JOIN deployment WHERE scope_id = ?', + [namespace], + ); + if (policies.length !== 1) throw new Error('The deployed namespace has no policy'); + const enabled = policies[0]!['scope_enabled'] === 1 && policies[0]!['deployment_enabled'] === 1; + const endpoint = `${origin}/projects/${namespace}`; + // New workers.dev routes and revisions can take a short time to reach the edge. + for (let attempt = 0; ; attempt++) { + try { + await checkDeployment(endpoint, deployment, enabled, request); + break; + } catch (error) { + if (attempt === 9) throw error; + await wait(3000); + } + } + io.print(`Deployment checks passed. Cache endpoint: ${endpoint}`); + if (!enabled) + io.print('This namespace remains disabled. Deployment did not change its access policy.'); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + readTemplate() + .then((config) => deploy(config)) + .catch((error) => { + console.error(error instanceof Error ? error.message : 'Deployment failed'); + process.exitCode = 1; + }); +} diff --git a/packages/remote-cache/scripts/operator.ts b/packages/remote-cache/scripts/operator.ts index d8ece6456..b14aec255 100644 --- a/packages/remote-cache/scripts/operator.ts +++ b/packages/remote-cache/scripts/operator.ts @@ -180,13 +180,13 @@ async function findDatabase( return databases.map(object).find((database) => database['name'] === name); } -async function ensurePrivateBucket(io: OperatorIO, name: string): Promise { +async function ensurePrivateBucket(io: OperatorIO, name: string, create = true): Promise { try { const bucket = object(await io.api(`/r2/buckets/${name}`)); if (bucket['storage_class'] && bucket['storage_class'] !== 'Standard') throw new Error('Use an R2 Standard bucket'); } catch (error) { - if (!(error instanceof ApiError) || error.status !== 404) throw error; + if (!create || !(error instanceof ApiError) || error.status !== 404) throw error; await io.wrangler([ 'r2', 'bucket', @@ -204,7 +204,15 @@ async function ensurePrivateBucket(io: OperatorIO, name: string): Promise await io.api(`/r2/buckets/${name}/domains/managed`, 'PUT', { enabled: false }); } -export async function runOperator(argv: string[], io: OperatorIO): Promise { +export async function runOperator( + argv: string[], + io: OperatorIO, + // Deploy to Cloudflare supplies these bindings; never rediscover them by Worker name. + resources?: { + database: Config['d1_databases'][number]; + bucket: Config['r2_buckets'][number]; + }, +): Promise { const { values, positionals } = parseArgs({ args: argv, allowPositionals: true, @@ -273,7 +281,9 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise throw new Error('Use the deployment name and account subdomain in the workers.dev origin'); const profile = values.profile ?? 'free'; if (profile !== 'free' && profile !== 'paid') throw new Error('Use free or paid'); - let db = await findDatabase(io, name); + let db = resources + ? object(await io.api(`/d1/database/${resources.database.database_id}`)) + : await findDatabase(io, name); if (!db) { await io.wrangler(['d1', 'create', name, '--no-update-config']); db = await findDatabase(io, name); @@ -283,7 +293,7 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise config.name = name; config.d1_databases[0] = { binding: 'INDEX', - database_name: name, + database_name: resources ? text(db['name']) : name, database_id: text(db['uuid']), migrations_dir: 'migrations', }; @@ -305,8 +315,9 @@ export async function runOperator(argv: string[], io: OperatorIO): Promise if (prior && prior['repository_id'] !== repo.id) throw new Error('Use a new namespace for a different repository'); } - await ensurePrivateBucket(io, name); - config.r2_buckets[0] = { binding: 'ARTIFACTS', bucket_name: name }; + const bucketName = resources?.bucket.bucket_name ?? name; + await ensurePrivateBucket(io, bucketName, !resources); + config.r2_buckets[0] = { binding: 'ARTIFACTS', bucket_name: bucketName }; config.workers_dev = origin.hostname.endsWith('.workers.dev'); if (!config.workers_dev) config.routes = [{ pattern: origin.hostname, custom_domain: true }]; config.vars['GC_BATCH_SIZE'] = profile === 'free' ? '16' : '256'; diff --git a/packages/remote-cache/test/deploy.test.ts b/packages/remote-cache/test/deploy.test.ts new file mode 100644 index 000000000..ffe684f7f --- /dev/null +++ b/packages/remote-cache/test/deploy.test.ts @@ -0,0 +1,198 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { deploy, checkDeployment } from '../scripts/deploy.ts'; +import { ApiError, readTemplate, type OperatorIO } from '../scripts/operator.ts'; +import { harness } from './helpers.ts'; + +void test('button deployment uses provisioned bindings and preserves policies on a fresh-checkout retry', async () => { + const h = await harness({ + initializeDatabase: false, + namespaces: ['cache'], + deploymentId: 'button-test', + }); + const template = await readTemplate(); + template.name = 'custom-worker'; + template.vars['CACHE_REPOSITORY'] = 'owner/repo'; + template.d1_databases[0]!.database_id = '12345678-1234-1234-1234-123456789abc'; + template.d1_databases[0]!.database_name = 'custom-index'; + template.r2_buckets[0]!.bucket_name = 'custom-artifacts'; + let config = structuredClone(template); + let migrated = false; + let repositoryId = 123; + let bucketMissing = false; + let staleResponse = true; + const commands: string[][] = []; + const mutations: string[] = []; + const waits: number[] = []; + const io: OperatorIO = { + async api(path, method, body) { + if (path === '/workers/subdomain') return { subdomain: 'team' }; + if (path === `/d1/database/${template.d1_databases[0]!.database_id}`) + return { uuid: template.d1_databases[0]!.database_id, name: 'custom-index' }; + if (path.endsWith('/query')) { + assert.ok(path.includes(template.d1_databases[0]!.database_id)); + const { sql, params } = body as { sql: string; params: (string | number | null)[] }; + if (!sql.startsWith('SELECT')) mutations.push(sql); + return [ + await h.db + .prepare(sql) + .bind(...params) + .all(), + ]; + } + if (path === '/r2/buckets/custom-artifacts') { + if (bucketMissing) throw new ApiError(404); + return { storage_class: 'Standard' }; + } + if (path === '/r2/buckets/custom-artifacts/domains/custom') return { domains: [] }; + if (path === '/r2/buckets/custom-artifacts/domains/managed') { + assert.equal(method, 'PUT'); + assert.deepEqual(body, { enabled: false }); + mutations.push('disable public bucket'); + return {}; + } + throw new Error(`Unexpected API request: ${path}`); + }, + async github() { + return { + id: repositoryId, + full_name: 'owner/repo', + owner: { id: 456 }, + private: false, + visibility: 'public', + default_branch: 'main', + }; + }, + async wrangler(args) { + assert.ok(!args.includes('create'), 'Provisioned resources must not be recreated'); + commands.push(args); + if (args[1] === 'migrations' && !migrated) { + await h.migrate(); + migrated = true; + } + }, + async readConfig() { + return structuredClone(config); + }, + async writeConfig(value) { + config = structuredClone(value); + mutations.push('config'); + }, + async lifecycle(bucket) { + assert.equal(bucket, 'custom-artifacts'); + mutations.push('lifecycle'); + }, + print() {}, + }; + const request: typeof fetch = async (input, init) => { + if (staleResponse) { + staleResponse = false; + return new Response(null, { + status: 401, + headers: { 'X-Remote-Cache-Deployment': 'old', 'Cache-Control': 'no-store' }, + }); + } + const req = new Request(input, init); + const response = await h.mf.dispatchFetch(req.url, { + method: req.method, + headers: Object.fromEntries(req.headers), + body: await req.arrayBuffer(), + }); + return new Response(await response.arrayBuffer(), { + status: response.status, + headers: Object.fromEntries(response.headers), + }); + }; + const run = (profile = 'free') => + deploy( + template, + io, + { WORKERS_CI_BUILD_UUID: 'button-test', CACHE_PROFILE: profile }, + request, + async (ms) => { + waits.push(ms); + }, + ); + try { + await run(); + assert.deepEqual(waits, [3000]); + assert.equal(config.name, 'custom-worker'); + assert.equal(config.d1_databases[0]!.database_name, 'custom-index'); + assert.equal(config.r2_buckets[0]!.bucket_name, 'custom-artifacts'); + assert.equal(config.vars['GC_BATCH_SIZE'], '16'); + assert.equal(config.vars['DEPLOYMENT_ID'], 'button-test'); + assert.deepEqual(JSON.parse(config.vars['NAMESPACES']!), ['cache']); + assert.equal(commands.at(-1)![0], 'deploy'); + const scope = await h.db.prepare('SELECT * FROM scopes').first(); + assert.equal(scope!.endpoint, 'https://custom-worker.team.workers.dev/projects/cache'); + assert.equal(scope!.repository_id, '123'); + assert.equal(scope!.branch, 'refs/heads/main'); + + await h.db + .prepare( + "UPDATE scopes SET enabled = 0, writes_enabled = 0, byte_limit = 12345 WHERE scope_id = 'cache'", + ) + .run(); + const withdrawn = await h.db.prepare('SELECT * FROM scopes').first(); + config = structuredClone(template); // A new Workers Build has no wrangler.operator.json. + await run('paid'); + assert.equal(config.vars['GC_BATCH_SIZE'], '256'); + assert.deepEqual(await h.db.prepare('SELECT * FROM scopes').first(), withdrawn); + + mutations.length = 0; + repositoryId = 999; + await assert.rejects(run(), /different repository/); + assert.deepEqual(mutations, []); + repositoryId = 123; + bucketMissing = true; + await assert.rejects(run(), /Cloudflare API failed/); + assert.deepEqual(mutations, []); + } finally { + await h.close(); + } +}); + +void test('button deployment rejects incomplete configuration before contacting Cloudflare', async () => { + const config = await readTemplate(); + const unexpected = async () => { + throw new Error('Unexpected operator call'); + }; + const io: OperatorIO = { + api: unexpected, + github: unexpected, + wrangler: unexpected, + readConfig: unexpected, + writeConfig: unexpected, + lifecycle: unexpected, + print() { + throw new Error('Unexpected output'); + }, + }; + await assert.rejects(deploy(config, io, {}), /CACHE_REPOSITORY/); + config.vars['CACHE_REPOSITORY'] = 'owner/repo'; + await assert.rejects(deploy(config, io, {}), /provisioned/); + config.d1_databases[0]!.database_id = '12345678-1234-1234-1234-123456789abc'; + await assert.rejects( + deploy(config, io, { WRANGLER_CI_OVERRIDE_NAME: 'different-worker' }), + /Worker name/, + ); +}); + +void test('deployment checks reject a stale revision, unexpected status, or cached response', async () => { + for (const response of [ + new Response(null, { + status: 401, + headers: { 'X-Remote-Cache-Deployment': 'old', 'Cache-Control': 'no-store' }, + }), + new Response(null, { + status: 200, + headers: { 'X-Remote-Cache-Deployment': 'new', 'Cache-Control': 'no-store' }, + }), + new Response(null, { status: 401, headers: { 'X-Remote-Cache-Deployment': 'new' } }), + ]) { + await assert.rejects( + checkDeployment('https://cache.example.com/projects/test', 'new', true, async () => response), + /Deployment check failed/, + ); + } +}); diff --git a/packages/remote-cache/test/index.test.ts b/packages/remote-cache/test/index.test.ts index 1c37c3520..cf2cbdfe8 100644 --- a/packages/remote-cache/test/index.test.ts +++ b/packages/remote-cache/test/index.test.ts @@ -1,5 +1,6 @@ import './protocol.test.ts'; import './service.test.ts'; import './operator.test.ts'; +import './deploy.test.ts'; import './failures.test.ts'; import './deployed.test.ts'; diff --git a/packages/remote-cache/wrangler.jsonc b/packages/remote-cache/wrangler.jsonc index 33f239c85..df77d4f24 100644 --- a/packages/remote-cache/wrangler.jsonc +++ b/packages/remote-cache/wrangler.jsonc @@ -23,6 +23,10 @@ { "name": "STORE_LIMITER", "namespace_id": "1002", "simple": { "limit": 30, "period": 60 } }, ], "vars": { + // Deploy to Cloudflare inputs. Set the repository before running pnpm deploy. + "CACHE_REPOSITORY": "", + "CACHE_NAMESPACE": "cache", + "CACHE_PROFILE": "free", "DEPLOYMENT_ID": "local", "NAMESPACES": "[]", "LIMITS": "{}", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a3c960387..b6c34cf26 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -92,8 +92,11 @@ importers: specifier: 6.2.12 version: 6.2.12 devDependencies: + '@cloudflare/workers-types': + specifier: 5.20260911.1 + version: 5.20260911.1 '@types/node': - specifier: 'catalog:' + specifier: 25.0.3 version: 25.0.3 cborg: specifier: 6.1.2 @@ -111,11 +114,11 @@ importers: specifier: 4.23.13 version: 4.23.13 typescript: - specifier: 'catalog:' + specifier: 6.0.3 version: 6.0.3 wrangler: specifier: 4.131.1 - version: 4.131.1(@types/node@25.0.3) + version: 4.131.1(@cloudflare/workers-types@5.20260911.1)(@types/node@25.0.3) packages/tools: dependencies: @@ -245,6 +248,9 @@ packages: cpu: [x64] os: [win32] + '@cloudflare/workers-types@5.20260911.1': + resolution: {integrity: sha512-yiAvknjulcU85B3yB4aKOn9+l+garWP+AbHgsdCFckeRYDdhZ1rPULi64BDf52R3VTaKqxi47kF+o9ZbjsCt5g==} + '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -2398,6 +2404,8 @@ snapshots: '@cloudflare/workerd-windows-64@1.20260911.1': optional: true + '@cloudflare/workers-types@5.20260911.1': {} + '@cspotcode/source-map-support@0.8.1': dependencies: '@jridgewell/trace-mapping': 0.3.9 @@ -4162,7 +4170,7 @@ snapshots: '@cloudflare/workerd-linux-arm64': 1.20260911.1 '@cloudflare/workerd-windows-64': 1.20260911.1 - wrangler@4.131.1(@types/node@25.0.3): + wrangler@4.131.1(@cloudflare/workers-types@5.20260911.1)(@types/node@25.0.3): dependencies: '@cloudflare/kv-asset-handler': 0.5.0 '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260911.1) @@ -4173,6 +4181,7 @@ snapshots: unenv: 2.0.0-rc.24 workerd: 1.20260911.1 optionalDependencies: + '@cloudflare/workers-types': 5.20260911.1 fsevents: 2.3.3 transitivePeerDependencies: - '@types/node' From 1b47bfe4b12808533b7b3a464f0367f135013dc7 Mon Sep 17 00:00:00 2001 From: MK Date: Wed, 16 Sep 2026 16:47:19 +0800 Subject: [PATCH 15/17] ci(remote-cache): guard temporary deployment button URLs Co-authored-by: GPT-6 Codex --- .github/workflows/ci.yml | 32 +++++++++++++++++++++- README.md | 2 +- packages/remote-cache/README.md | 2 +- packages/remote-cache/docs/self-hosting.md | 4 +-- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74859233a..5800ecd3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ permissions: on: workflow_dispatch: pull_request: - types: [opened, synchronize] + types: [opened, synchronize, ready_for_review, converted_to_draft] push: branches: - main @@ -23,6 +23,35 @@ defaults: shell: bash jobs: + deploy-button-links: + name: Deploy button links + runs-on: namespace-profile-linux-x64-default + steps: + - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { readFile } = require('node:fs/promises'); + const main = 'https://github.com/voidzero-dev/vite-task/tree/main/packages/remote-cache'; + const pr = context.payload.pull_request; + const temporary = pr?.draft + ? `https://github.com/${pr.head.repo.full_name}/tree/${pr.head.ref}/packages/remote-cache` + : undefined; + for (const file of ['README.md', 'packages/remote-cache/README.md', 'packages/remote-cache/docs/self-hosting.md']) { + const content = await readFile(file, 'utf8'); + const buttons = [...content.matchAll(/\]\((https:\/\/deploy\.workers\.cloudflare\.com\/\?url=[^)]+)\)/g)]; + if (!buttons.length) core.setFailed(`${file}: Deploy to Cloudflare button is missing.`); + for (const [, button] of buttons) { + const source = new URL(button).searchParams.get('url'); + if (source === main) continue; + if (temporary && source === temporary) { + core.warning(`${file}: Temporary PR branch URL. Restore the button to main before marking this PR ready for review.`); + } else { + core.setFailed(`${file}: Restore the Deploy to Cloudflare source URL to ${main} before merge.`); + } + } + } + detect-changes: runs-on: namespace-profile-linux-x64-default permissions: @@ -405,6 +434,7 @@ jobs: runs-on: namespace-profile-linux-x64-default if: always() needs: + - deploy-button-links - clippy - test - test-musl diff --git a/README.md b/README.md index ed373a24e..51802f28b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ vp run --cache build # run with caching enabled ## Remote cache service -[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https%3A%2F%2Fgithub.com%2Fvoidzero-dev%2Fvite-task%2Ftree%2Fmain%2Fpackages%2Fremote-cache) +[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https%3A%2F%2Fgithub.com%2Fvoidzero-dev%2Fvite-task%2Ftree%2Ffeat%2Fpublic-remote-cache%2Fpackages%2Fremote-cache) To deploy the public remote cache service in your own Cloudflare account and bind it to a GitHub repository, follow the [self-hosting guide](packages/remote-cache/docs/self-hosting.md). The service is available in this source tree; the `vp run` remote-cache client adapter is not yet implemented here. diff --git a/packages/remote-cache/README.md b/packages/remote-cache/README.md index a256cf3d8..68f1122ce 100644 --- a/packages/remote-cache/README.md +++ b/packages/remote-cache/README.md @@ -4,7 +4,7 @@ This package implements the server in the [remote cache RFC](rfcs/0001-remote-ca The package contains no `vp run` client adapter. Cache keys, values, and blobs remain opaque. A successful lookup does not prove that a result is reusable; a client must validate its inputs and output archive. -[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https%3A%2F%2Fgithub.com%2Fvoidzero-dev%2Fvite-task%2Ftree%2Fmain%2Fpackages%2Fremote-cache) +[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https%3A%2F%2Fgithub.com%2Fvoidzero-dev%2Fvite-task%2Ftree%2Ffeat%2Fpublic-remote-cache%2Fpackages%2Fremote-cache) **Start here:** [Deploy the service and bind your repository](docs/self-hosting.md#quick-start-deploy-to-cloudflare). The button creates a standalone repository and provisions storage. Set `CACHE_REPOSITORY` to your public GitHub repository and keep the default `free` profile. The guide covers build-token permissions, deployment checks, custom domains, and the current client-integration limitation. diff --git a/packages/remote-cache/docs/self-hosting.md b/packages/remote-cache/docs/self-hosting.md index 49c2a82ca..9658fdd1c 100644 --- a/packages/remote-cache/docs/self-hosting.md +++ b/packages/remote-cache/docs/self-hosting.md @@ -6,7 +6,7 @@ This guide deploys the service in your Cloudflare account and gives your reposit ## Quick start: Deploy to Cloudflare -[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https%3A%2F%2Fgithub.com%2Fvoidzero-dev%2Fvite-task%2Ftree%2Fmain%2Fpackages%2Fremote-cache) +[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https%3A%2F%2Fgithub.com%2Fvoidzero-dev%2Fvite-task%2Ftree%2Ffeat%2Fpublic-remote-cache%2Fpackages%2Fremote-cache) The button copies this package into a new GitHub repository in your account and connects it to Workers Builds. It provisions a Worker, a D1 database, and an R2 bucket. You do not need a local checkout or GitHub Actions secrets. The repository that hosts the Worker can differ from the application repository that uses its cache. See [Cloudflare's deployment-button guide](https://developers.cloudflare.com/workers/platform/deploy-buttons/). @@ -38,7 +38,7 @@ Keep the generated D1 ID and R2 name in `wrangler.jsonc`. Each build reconstruct The automatic checks verify the new deployment ID, disabled HTTP caching, rejected anonymous uploads, invalid requests, and cache misses. They do not upload data or prove authorized GitHub OIDC writes. The [e2e plan](e2e-plan.md#deploy-to-cloudflare-button) covers the full acceptance checks. -The button targets the package on `main`. Before this change is merged, test the flow by supplying the package URL at the reviewed PR commit to Cloudflare's deployment page. +The button temporarily targets `feat/public-remote-cache` for PR #718's live e2e test. Before marking the PR ready for review, restore the buttons in the root README, package README, and this guide to `main`, and remove this note. The CI button check rejects temporary branch URLs on a PR that is ready for review. ## CLI deployment From a069da21eeeb821279cc096b9b36082a5cec03b4 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Mon, 28 Sep 2026 00:31:02 +0800 Subject: [PATCH 16/17] ci(remote-cache): run the standalone check with a native pnpm executable `npm_execpath` points to pnpm's standalone executable on the Linux and Windows runners, so running it through `node` failed. Run it directly unless it's a JavaScript entry point. The remote cache package also brought esbuild and workerd into the workspace, and a root `pnpm install` without `--ignore-scripts` failed on their unapproved build scripts. Allow them, as the standalone package already does. Co-authored-by: Claude Opus 5.5 --- .github/scripts/check-remote-cache-standalone.mjs | 4 +++- pnpm-workspace.yaml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/scripts/check-remote-cache-standalone.mjs b/.github/scripts/check-remote-cache-standalone.mjs index 8e272f438..14268b0cb 100644 --- a/.github/scripts/check-remote-cache-standalone.mjs +++ b/.github/scripts/check-remote-cache-standalone.mjs @@ -5,6 +5,8 @@ import { basename, join } from 'node:path'; const pnpm = process.env.npm_execpath; if (!pnpm) throw new Error('Run this check through pnpm check-remote-cache-standalone'); +// pnpm can be a JavaScript entry point or a standalone executable. +const [command, prefix] = /\.[cm]?js$/.test(pnpm) ? [process.execPath, [pnpm]] : [pnpm, []]; const directory = await mkdtemp(join(tmpdir(), 'remote-cache-standalone-')); const excluded = new Set([ 'node_modules', @@ -25,7 +27,7 @@ try { }); for (const args of [['install', '--frozen-lockfile'], ['check'], ['build']]) { await new Promise((resolve, reject) => { - const child = spawn(process.execPath, [pnpm, ...args], { + const child = spawn(command, [...prefix, ...args], { cwd: directory, stdio: 'inherit', shell: false, diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index cb08fb5f9..626d1f3b4 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,6 +8,8 @@ allowBuilds: '@playwright/browser-chromium': true bun: true deno: true + esbuild: true + workerd: true minimumReleaseAgeExclude: - vite-plus From 0c55376f703fa7456db036ea15996bdf4834c523 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Mon, 28 Sep 2026 00:42:27 +0800 Subject: [PATCH 17/17] test(cache): run the remote cache e2e tests against the public cache service `remote-cache-server` now runs the Worker from `packages/remote-cache` in workerd through Miniflare instead of the Node test backend, so the e2e tests exercise `vp run` against the real service. The wrapper proxies requests to the service. It signs an upload token for each store with a key the service fetches in place of GitHub's. It numbers the service's random blob IDs in upload order for the request log, and copies each blob to `remote-cache/blobs/`, storing a changed copy on the next run so tests can still corrupt an archive. The only snapshot change is the request log for a fetch miss, which the service answers with a 404. The cases are skipped on musl because workerd's prebuilt binaries require glibc. The `remote_cache_backend` fixture only tested the old backend, and the service has its own protocol tests, so it's removed along with `cbor-http` and its EDN formatting. Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 12 +- .../fixtures/remote_cache/snapshots.toml | 15 +- .../remote_cache_backend/package.json | 1 - .../remote_cache_backend/snapshots.toml | 479 ------------------ .../snapshots/blob_replacement.md | 105 ---- .../snapshots/entries_and_associations.md | 84 --- .../snapshots/invalid_requests.md | 138 ----- .../snapshots/opaque_binary_data.md | 43 -- .../snapshots/state_across_invocations.md | 40 -- packages/tools/README.md | 40 +- packages/tools/package.json | 9 +- packages/tools/src/cbor-http.ts | 105 ---- packages/tools/src/edn.test.ts | 17 - packages/tools/src/edn.ts | 24 - packages/tools/src/remote-cache/cli.ts | 15 +- packages/tools/src/remote-cache/server.ts | 443 +++++++++------- pnpm-lock.yaml | 41 +- pnpm-workspace.yaml | 2 - 18 files changed, 309 insertions(+), 1304 deletions(-) delete mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/package.json delete mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml delete mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md delete mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md delete mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md delete mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md delete mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md delete mode 100755 packages/tools/src/cbor-http.ts delete mode 100644 packages/tools/src/edn.test.ts delete mode 100644 packages/tools/src/edn.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5800ecd3f..137213b5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,9 +177,7 @@ jobs: run: pnpm --filter vite-task-tools exec playwright install-deps chromium - name: Run ignored tests - run: | - pnpm --filter vite-task-tools test - ${{ matrix.cargo_cmd }} test --target ${{ matrix.build_target }} -- --ignored + run: ${{ matrix.cargo_cmd }} test --target ${{ matrix.build_target }} -- --ignored # Windows tests are cross-compiled on a fast Linux runner with cargo-xwin # (clang-cl + lld-link against the xwin-downloaded MSVC CRT/Windows SDK) @@ -312,9 +310,7 @@ jobs: - name: Run ignored tests if: matrix.mode == 'ignored' - run: | - pnpm --filter vite-task-tools test - cargo-nextest nextest run --archive-file windows-tests.tar.zst --workspace-remap . --run-ignored ignored-only + run: cargo-nextest nextest run --archive-file windows-tests.tar.zst --workspace-remap . --run-ignored ignored-only test-musl: needs: detect-changes @@ -368,9 +364,7 @@ jobs: - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 - name: Run ignored tests - run: | - pnpm --filter vite-task-tools test - cargo test -- --ignored + run: cargo test -- --ignored fmt: name: Format and Check Deps diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml index ba61d0b67..b0d174f5b 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml @@ -1,9 +1,10 @@ # Cases that use the remote cache backend are ignored because it runs on # Node.js. Windows is skipped because the PTY launcher cannot execute pnpm -# command shims. +# command shims. musl is skipped because the backend runs the public cache +# service in workerd, whose prebuilt binaries require glibc. [[e2e]] name = "read_write" -cfg = "not(windows)" +cfg = 'not(any(windows, target_env = "musl"))' ignore = true steps = [ { argv = [ @@ -32,7 +33,7 @@ steps = [ [[e2e]] name = "read" -cfg = "not(windows)" +cfg = 'not(any(windows, target_env = "musl"))' ignore = true steps = [ { argv = [ @@ -62,7 +63,7 @@ steps = [ [[e2e]] name = "restore" -cfg = "not(windows)" +cfg = 'not(any(windows, target_env = "musl"))' ignore = true steps = [ { argv = [ @@ -130,7 +131,7 @@ steps = [ [[e2e]] name = "local_and_remote_hits" -cfg = "not(windows)" +cfg = 'not(any(windows, target_env = "musl"))' ignore = true steps = [ { argv = [ @@ -169,7 +170,7 @@ steps = [ [[e2e]] name = "fallback" -cfg = "not(windows)" +cfg = 'not(any(windows, target_env = "musl"))' ignore = true steps = [ { argv = [ @@ -205,7 +206,7 @@ steps = [ [[e2e]] name = "corrupt_archive" -cfg = "not(windows)" +cfg = 'not(any(windows, target_env = "musl"))' ignore = true steps = [ { argv = [ diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/package.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/package.json deleted file mode 100644 index 0967ef424..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/package.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml deleted file mode 100644 index 7fbf89fcc..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml +++ /dev/null @@ -1,479 +0,0 @@ -# Windows is skipped because the PTY launcher cannot execute pnpm command shims. -[[e2e]] -name = "entries_and_associations" -cfg = "not(windows)" -ignore = true -steps = [ - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'VA'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'U', \"value\": 'VA'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'B', \"secondary_key\": 'S', \"value\": 'VB'}", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], comment = "The exact entry survives reassignment of S to B." }, - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'C', \"secondary_key\": 'S'}", - ], comment = "The previous fetch did not change S, so fallback returns B's entry." }, - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'VA2'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'missing'}", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'C', \"secondary_key\": 'U'}", - ], comment = "An older association to A still resolves after A is replaced." }, - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'C', \"secondary_key\": 'T'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'C', \"secondary_key\": 'S'}", - ], -] - -[[e2e]] -name = "blob_replacement" -cfg = "not(windows)" -ignore = true -steps = [ - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/missing", - ], - [ - "vtt", - "write-file", - "archive.txt", - "first archive", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'first'}", - "--form-file", - "blob=archive.txt", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/1", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-data", - "blob='second archive'", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'second'}", - ], comment = "Accept blob before metadata, with neither part supplying a filename." }, - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/2", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/1", - ], comment = "Previously returned IDs retain their original bytes." }, - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'no archive'}", - ], comment = "Omitting blob clears the association." }, - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'empty archive'}", - "--form-data", - "blob=''", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/3", - ], comment = "An empty blob has an ID and downloads as an empty byte string." }, -] - -[[e2e]] -name = "opaque_binary_data" -cfg = "not(windows)" -ignore = true -steps = [ - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": b64'AP+A', \"secondary_key\": '', \"value\": b64'AP+A'}", - "--form-data", - "blob=b64'AP+A'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": b64'AP+A', \"secondary_key\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": '', \"secondary_key\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/1", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": '', \"secondary_key\": '', \"value\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": '', \"secondary_key\": ''}", - ], -] - -[[e2e]] -name = "invalid_requests" -cfg = "not(windows)" -ignore = true -steps = [ - [ - "remote-cache-server", - "cbor-http", - "GET", - "/missing", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--data", - "'bad'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--data", - "'bad'", - "--content-type", - "application/cbor", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": \"text\", \"secondary_key\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "[]", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": '', \"key\": 'duplicate', \"secondary_key\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'original'}", - "--form-data", - "blob='original archive'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--cbor", - "{}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-data", - "blob='missing metadata'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": \"wrong type\"}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'replacement'}", - "--form-cbor", - "metadata={}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'replacement'}", - "--form-data", - "blob='one'", - "--form-data", - "blob='two'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={}", - "--form-data", - "unexpected='part'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-data", - "metadata='wrong content type'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--data", - "'--boundary\\r\\nContent-Disposition: form-data; name=\"metadata\"\\r\\nContent-Type: application/cbor\\r\\n\\r\\n'", - "--content-type", - "multipart/form-data; boundary=boundary", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], comment = "Failed stores did not replace the value or blob." }, - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/1", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'B', \"secondary_key\": 'T'}", - ], comment = "Failed stores did not publish a secondary association." }, -] - -[[e2e]] -name = "state_across_invocations" -cfg = "not(windows)" -ignore = true -steps = [ - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'VA'}", - "--form-data", - "blob='first archive'", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'C', \"secondary_key\": 'S'}", - ], comment = "A later invocation reads the stored entry and association." }, - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'B', \"secondary_key\": 'T', \"value\": 'VB'}", - "--form-data", - "blob='second archive'", - ], comment = "Blob numbering continues across invocations." }, - { argv = [ - "vtt", - "write-file", - "remote-cache/blobs/1", - "replaced archive", - ], comment = "Each blob is a file named by its ID." }, - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/1", - ], -] diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md deleted file mode 100644 index ecf1cae0f..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md +++ /dev/null @@ -1,105 +0,0 @@ -# blob_replacement - -## `remote-cache-server cbor-http GET /blob/missing` - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Blob not found"} -[remote-cache] GET /blob/missing 404 -``` - -## `vtt write-file archive.txt 'first archive'` - -``` -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''first'\''}' --form-file blob=archive.txt` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'first', "blob_id": "1"}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http GET /blob/1` - -``` -{"status": 200, "content_type": "application/octet-stream", "body": 'first archive'} -[remote-cache] GET /blob/1 200 -``` - -## `remote-cache-server cbor-http POST /store --form-data 'blob='\''second archive'\''' --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''second'\''}'` - -Accept blob before metadata, with neither part supplying a filename. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "2"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'second', "blob_id": "2"}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http GET /blob/2` - -``` -{"status": 200, "content_type": "application/octet-stream", "body": 'second archive'} -[remote-cache] GET /blob/2 200 -``` - -## `remote-cache-server cbor-http GET /blob/1` - -Previously returned IDs retain their original bytes. - -``` -{"status": 200, "content_type": "application/octet-stream", "body": 'first archive'} -[remote-cache] GET /blob/1 200 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''no archive'\''}'` - -Omitting blob clears the association. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'no archive', "blob_id": null}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''empty archive'\''}' --form-data blob=''` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "3"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'empty archive', "blob_id": "3"}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http GET /blob/3` - -An empty blob has an ID and downloads as an empty byte string. - -``` -{"status": 200, "content_type": "application/octet-stream", "body": ''} -[remote-cache] GET /blob/3 200 -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md deleted file mode 100644 index 9b5ad0152..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md +++ /dev/null @@ -1,84 +0,0 @@ -# entries_and_associations - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Not found"} -[remote-cache] POST /fetch 404 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''VA'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''U'\'', "value": '\''VA'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''B'\'', "secondary_key": '\''S'\'', "value": '\''VB'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -The exact entry survives reassignment of S to B. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'VA', "blob_id": null}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` - -The previous fetch did not change S, so fallback returns B's entry. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'B', "value": 'VB', "blob_id": null}} -[remote-cache] POST /fetch 200 fallback -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''VA2'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''missing'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'VA2', "blob_id": null}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''U'\''}'` - -An older association to A still resolves after A is replaced. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A', "value": 'VA2', "blob_id": null}} -[remote-cache] POST /fetch 200 fallback -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''T'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A', "value": 'VA2', "blob_id": null}} -[remote-cache] POST /fetch 200 fallback -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'B', "value": 'VB', "blob_id": null}} -[remote-cache] POST /fetch 200 fallback -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md deleted file mode 100644 index 9d468db74..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md +++ /dev/null @@ -1,138 +0,0 @@ -# invalid_requests - -## `remote-cache-server cbor-http GET /missing` - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Route not found"} -[remote-cache] GET /missing 404 -``` - -## `remote-cache-server cbor-http POST /fetch --data 'bad'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected application/cbor"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /fetch --data 'bad' --content-type application/cbor` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": "text", "secondary_key": '\'''\''}'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\''}'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor []` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\'', "key": '\''duplicate'\'', "secondary_key": '\'''\''}'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid CBOR"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''original'\''}' --form-data 'blob='\''original archive'\'''` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /store --cbor {}` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected multipart/form-data"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-data 'blob='\''missing metadata'\'''` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Missing metadata"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": "wrong type"}'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key, value"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-cbor metadata={}` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-data blob='one' --form-data blob='two'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor metadata={} --form-data unexpected='part'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-data 'metadata='\''wrong content type'\'''` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --data ''\''--boundary\r\nContent-Disposition: form-data; name="metadata"\r\nContent-Type: application/cbor\r\n\r\n'\''' --content-type 'multipart/form-data; boundary=boundary'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -Failed stores did not replace the value or blob. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'original', "blob_id": "1"}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http GET /blob/1` - -``` -{"status": 200, "content_type": "application/octet-stream", "body": 'original archive'} -[remote-cache] GET /blob/1 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''B'\'', "secondary_key": '\''T'\''}'` - -Failed stores did not publish a secondary association. - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Not found"} -[remote-cache] POST /fetch 404 -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md deleted file mode 100644 index b9bb2cd5e..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md +++ /dev/null @@ -1,43 +0,0 @@ -# opaque_binary_data - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": b64'\''AP+A'\'', "secondary_key": '\'''\'', "value": b64'\''AP+A'\''}' --form-data blob=b64'AP+A'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": b64'\''AP+A'\'', "secondary_key": '\'''\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": b64'AP+A', "blob_id": "1"}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\'', "secondary_key": '\'''\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": b64'AP+A', "value": b64'AP+A', "blob_id": "1"}} -[remote-cache] POST /fetch 200 fallback -``` - -## `remote-cache-server cbor-http GET /blob/1` - -``` -{"status": 200, "content_type": "application/octet-stream", "body": b64'AP+A'} -[remote-cache] GET /blob/1 200 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\'''\'', "secondary_key": '\'''\'', "value": '\'''\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\'', "secondary_key": '\'''\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": '', "blob_id": null}} -[remote-cache] POST /fetch 200 exact -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md deleted file mode 100644 index 4eb0f38fd..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md +++ /dev/null @@ -1,40 +0,0 @@ -# state_across_invocations - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''VA'\''}' --form-data 'blob='\''first archive'\'''` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` - -A later invocation reads the stored entry and association. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A', "value": 'VA', "blob_id": "1"}} -[remote-cache] POST /fetch 200 fallback -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''B'\'', "secondary_key": '\''T'\'', "value": '\''VB'\''}' --form-data 'blob='\''second archive'\'''` - -Blob numbering continues across invocations. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "2"}} -[remote-cache] POST /store 200 -``` - -## `vtt write-file remote-cache/blobs/1 'replaced archive'` - -Each blob is a file named by its ID. - -``` -``` - -## `remote-cache-server cbor-http GET /blob/1` - -``` -{"status": 200, "content_type": "application/octet-stream", "body": 'replaced archive'} -[remote-cache] GET /blob/1 200 -``` diff --git a/packages/tools/README.md b/packages/tools/README.md index 6d815e236..71421faf6 100644 --- a/packages/tools/README.md +++ b/packages/tools/README.md @@ -2,16 +2,17 @@ This package provides Node.js dependencies and commands for the test suites. ## Remote cache backend -Run `pnpm install` at the repository root to install `remote-cache-server` and `cbor-http` into this package's `node_modules/.bin`. The workspace self-dependency makes pnpm link the package's own commands there. The E2E harness already includes that directory in PATH. Both commands run TypeScript directly using the Node version in `.node-version`. +Run `pnpm install` at the repository root to install `remote-cache-server` into this package's `node_modules/.bin`. The workspace self-dependency makes pnpm link the package's own commands there. The E2E harness already includes that directory in PATH. The command runs TypeScript directly using the Node version in `.node-version`. ```sh -remote-cache-server cbor-http POST /store --form-cbor "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'record'}" -remote-cache-server cbor-http POST /fetch --cbor "{\"key\": 'A', \"secondary_key\": 'S'}" +VP_REMOTE_CACHE=read-write remote-cache-server vt run build ``` -`remote-cache-server COMMAND [ARGS...]` starts the backend on a free loopback port and runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint, `http://127.0.0.1:/projects/test`. The fixed base path gives every endpoint a namespace path. The wrapper takes no options and passes all arguments to the command unchanged. The command inherits stdio. When it exits, the server stops and the wrapper exits with the command's exit code. +`remote-cache-server COMMAND [ARGS...]` starts the [public cache service](../remote-cache/README.md) in workerd through Miniflare, serves it on a free loopback port, and runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint, `http://127.0.0.1:/projects/test`. The wrapper takes no options and passes all arguments to the command unchanged. The command inherits stdio. When it exits, the server stops and the wrapper exits with the command's exit code. -After the command exits, the wrapper prints one line to stderr for each request it served, in the order of the responses. Each line has the method, the path below the base path, and the status. Successful fetch responses add their kind: +The service accepts uploads only with a GitHub Actions token. The wrapper signs one for each store request with a key it generates, and the service fetches that key in place of GitHub's. Everything else reaches the service unchanged, so responses follow [its protocol](../remote-cache/README.md#protocol): for example, a fetch that matches no entry is a `404`. + +After the command exits, the wrapper prints one line to stderr for each request it served, in the order of the responses. Each line has the method, the path below the endpoint, and the status. Successful fetches add their kind: ```text [remote-cache] POST /fetch 404 @@ -20,31 +21,10 @@ After the command exits, the wrapper prints one line to stderr for each request [remote-cache] GET /blob/1 200 ``` -State persists in `remote-cache/` in the current directory, so consecutive commands share it. Each E2E case has its own directory and state. `state.json` holds the entries, associations, and next blob ID, with keys and values hex-encoded. Each blob is a file in `remote-cache/blobs/` named by its blob ID. Blob IDs are sequential strings and continue across invocations, keeping snapshots deterministic. - -The backend implements `POST /fetch`, `POST /store`, and `GET /blob/{blob_id}` from the [remote cache server API](https://github.com/voidzero-dev/vite-task/pull/713). A fetch that matches neither key gets a `404` with the plain-text body `Not found`. Keys, values, and blobs are opaque bytes without length limits. There is no authentication. - -## CBOR HTTP client - -`cbor-http METHOD PATH` appends the path to `VP_REMOTE_CACHE_URL`, so run it through `remote-cache-server`. Request bodies use [CBOR extended diagnostic notation (EDN)](https://www.rfc-editor.org/rfc/rfc8610.html#appendix-G): `"text"` is text, `'bytes'` is a UTF-8 byte string, and `b64'AP+A'` contains arbitrary binary bytes. - -| Option | Body | -| ----------------------- | --------------------------------------------------------- | -| `--cbor EDN` | Encode EDN as CBOR with `Content-Type: application/cbor`. | -| `--data EDN` | Send an EDN byte string as raw bytes. | -| `--form-cbor NAME=EDN` | Add a CBOR multipart part. | -| `--form-data NAME=EDN` | Add an EDN byte string as a raw multipart part. | -| `--form-file NAME=FILE` | Add file contents as a raw multipart part. | -| `--content-type TYPE` | Override the request content type. | +State persists in `remote-cache/` in the current directory, so consecutive commands share it. Each E2E case has its own directory and state. The service keeps its D1 database and R2 bucket in `remote-cache/state/`. -Multipart options can repeat and retain their order. Raw bodies and parts default to `application/octet-stream`. Parts have no filename, including binary metadata. - -Every response prints one EDN map containing `status`, `content_type`, and `body`. CBOR is decoded, text becomes a text string, and other bodies become byte strings. Byte strings use readable UTF-8 when possible and base64 EDN otherwise. HTTP error responses exit successfully so snapshots can assert their status and body. Argument errors, transport failures, and invalid CBOR responses exit unsuccessfully. - -```text -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'record', "blob_id": null}} -``` +Blob IDs are random, so the wrapper numbers blobs in upload order, and request lines show the number in place of the ID. The numbers continue across invocations, keeping snapshots deterministic, and `remote-cache/blobs.json` lists the IDs in order. When the wrapper exits, it copies each blob to `remote-cache/blobs/`. If a copy has changed by the next invocation, the wrapper stores its contents as the blob, so a test can corrupt an archive by overwriting the file. -Run `pnpm --filter vite-task-tools check` for type checking and `pnpm --filter vite-task-tools test` for the EDN-formatting unit test. Run `cargo test -p vt_bin --test e2e_snapshots -- remote_cache_backend --ignored` for the backend snapshots. +Run `pnpm --filter vite-task-tools check` for type checking. Run `cargo test -p vt_bin --test e2e_snapshots -- remote_cache --ignored` for the snapshots that use the backend. -The backend snapshots are skipped on Windows because the PTY launcher cannot execute pnpm command shims. +Those snapshots are skipped on Windows because the PTY launcher cannot execute pnpm command shims, and on musl because workerd's prebuilt binaries require glibc. diff --git a/packages/tools/package.json b/packages/tools/package.json index 767398052..5414a0d44 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -2,26 +2,25 @@ "name": "vite-task-tools", "private": true, "bin": { - "cbor-http": "./src/cbor-http.ts", "remote-cache-server": "./src/remote-cache/cli.ts" }, "type": "module", "scripts": { - "check": "tsc --noEmit -p tsconfig.json", - "test": "node --test src/*.test.ts" + "check": "tsc --noEmit -p tsconfig.json" }, "dependencies": { "@anthropic-ai/sandbox-runtime": "catalog:", - "@fastify/busboy": "catalog:", "@openai/codex": "catalog:", "@playwright/browser-chromium": "catalog:", "@vitest/browser-playwright": "catalog:", + "@voidzero-dev/remote-cache": "workspace:*", "@voidzero-dev/vite-task-client": "workspace:*", "bun": "catalog:", - "cbor-edn": "catalog:", "cbor2": "catalog:", "cross-env": "catalog:", "deno": "catalog:", + "esbuild": "0.28.1", + "miniflare": "5.20260911.0-alpha", "oxfmt": "catalog:", "oxlint": "catalog:", "oxlint-tsgolint": "catalog:", diff --git a/packages/tools/src/cbor-http.ts b/packages/tools/src/cbor-http.ts deleted file mode 100755 index 30535fd77..000000000 --- a/packages/tools/src/cbor-http.ts +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env node -import { parseEDN } from 'cbor-edn'; -import { decode } from 'cbor2/decoder'; -import { encode } from 'cbor2/encoder'; -import { randomUUID } from 'node:crypto'; -import { readFile } from 'node:fs/promises'; -import { parseArgs } from 'node:util'; -import { toEDN } from './edn.ts'; - -function bytes(edn: string): Uint8Array { - const value: unknown = decode(parseEDN(edn, {})); - if (!(value instanceof Uint8Array)) throw new Error('Expected an EDN byte string'); - return value; -} - -async function main(): Promise { - const { values, positionals, tokens } = parseArgs({ - allowPositionals: true, - tokens: true, - options: { - cbor: { type: 'string' }, - data: { type: 'string' }, - 'content-type': { type: 'string' }, - 'form-cbor': { type: 'string', multiple: true }, - 'form-data': { type: 'string', multiple: true }, - 'form-file': { type: 'string', multiple: true }, - }, - }); - const [method, path] = positionals; - if (positionals.length !== 2 || !method || !path) { - throw new Error( - 'Usage: cbor-http METHOD PATH [--cbor EDN | --data EDN | --form-cbor NAME=EDN | --form-data NAME=EDN | --form-file NAME=FILE]', - ); - } - let body: Buffer | undefined; - let contentType: string | undefined; - const formTokens = tokens.filter( - (token) => token.kind === 'option' && token.name.startsWith('form-'), - ); - if ( - [values.cbor !== undefined, values.data !== undefined, formTokens.length > 0].filter(Boolean) - .length > 1 - ) { - throw new Error('Choose one request body format'); - } - if (values.cbor !== undefined) { - body = Buffer.from(parseEDN(values.cbor, {})); - contentType = 'application/cbor'; - } else if (values.data !== undefined) { - body = Buffer.from(bytes(values.data)); - contentType = 'application/octet-stream'; - } else if (formTokens.length) { - const boundary = randomUUID(); - const chunks: Buffer[] = []; - for (const token of formTokens) { - if (token.kind !== 'option' || token.value === undefined) continue; - const equal = token.value.indexOf('='); - const name = token.value.slice(0, equal); - const value = token.value.slice(equal + 1); - if (equal < 1 || /["\r\n\\]/.test(name)) throw new Error('Expected multipart NAME=VALUE'); - const isCbor = token.name === 'form-cbor'; - const data = isCbor - ? parseEDN(value, {}) - : token.name === 'form-file' - ? await readFile(value) - : bytes(value); - // Do not add filenames: binary metadata is valid without one. - chunks.push( - Buffer.from( - `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\nContent-Type: ${isCbor ? 'application/cbor' : 'application/octet-stream'}\r\n\r\n`, - ), - ); - chunks.push(Buffer.from(data), Buffer.from('\r\n')); - } - chunks.push(Buffer.from(`--${boundary}--\r\n`)); - body = Buffer.concat(chunks); - contentType = `multipart/form-data; boundary=${boundary}`; - } - contentType = values['content-type'] ?? contentType; - const response = await fetch(`${process.env['VP_REMOTE_CACHE_URL']}${path}`, { - method, - ...(body === undefined ? {} : { body }), - ...(contentType === undefined ? {} : { headers: { 'content-type': contentType } }), - }); - const responseType = response.headers.get('content-type') ?? ''; - const raw = new Uint8Array(await response.arrayBuffer()); - const type = responseType.split(';')[0]?.trim(); - let printedBody: string; - if (type === 'application/cbor') { - decode(raw); // diagnose alone does not reject every incomplete CBOR item. - printedBody = toEDN(raw); - } else { - printedBody = toEDN(encode(type?.startsWith('text/') ? new TextDecoder().decode(raw) : raw)); - } - console.log( - `{"status": ${response.status}, "content_type": ${JSON.stringify(responseType)}, "body": ${printedBody}}`, - ); -} - -try { - await main(); -} catch (error) { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; -} diff --git a/packages/tools/src/edn.test.ts b/packages/tools/src/edn.test.ts deleted file mode 100644 index 50f5d5721..000000000 --- a/packages/tools/src/edn.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import assert from 'node:assert/strict'; -import test from 'node:test'; -import { parseEDN } from 'cbor-edn'; -import { decode } from 'cbor2/decoder'; -import { encode } from 'cbor2/encoder'; -import { toEDN } from './edn.ts'; - -await test('EDN formatting preserves escapes, a UTF-8 BOM, and text resembling byte strings', () => { - for (const value of [ - new TextEncoder().encode("quotes: '\"\\\n\t\0 and h'ff'"), - new Uint8Array([239, 187, 191, 65]), - { text: "h'ff'", bytes: new TextEncoder().encode('hello') }, - ]) { - const edn = toEDN(encode(value)); - assert.deepEqual(decode(parseEDN(edn, {})), value); - } -}); diff --git a/packages/tools/src/edn.ts b/packages/tools/src/edn.ts deleted file mode 100644 index 44919d221..000000000 --- a/packages/tools/src/edn.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { diagnose } from 'cbor2/diagnostic'; - -/** Keep byte strings readable without mistaking text containing h'...' for bytes. */ -export function toEDN(bytes: Uint8Array): string { - return diagnose(bytes).replace( - /"(?:[^"\\]|\\.)*"|h'([0-9a-f]*)'/gi, - (token, hex: string | undefined) => { - if (hex === undefined) return token; - const binary = Buffer.from(hex, 'hex'); - try { - const text = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(binary); - const escaped = JSON.stringify(text) - .slice(1, -1) - .replace(/\\.|'/g, (part) => { - if (part === '\\"') return '"'; - return part === "'" ? "\\'" : part; - }); - return `'${escaped}'`; - } catch { - return `b64'${binary.toString('base64')}'`; - } - }, - ); -} diff --git a/packages/tools/src/remote-cache/cli.ts b/packages/tools/src/remote-cache/cli.ts index b118e8b18..561e8ed49 100755 --- a/packages/tools/src/remote-cache/cli.ts +++ b/packages/tools/src/remote-cache/cli.ts @@ -1,26 +1,19 @@ #!/usr/bin/env node import { spawn } from 'node:child_process'; import { once } from 'node:events'; -import type { AddressInfo } from 'node:net'; -import { createCacheServer } from './server.ts'; - -const basePath = '/projects/test'; +import { startCacheServer } from './server.ts'; const [command, ...args] = process.argv.slice(2); const requests: string[] = []; -const server = createCacheServer({ - basePath, +const server = await startCacheServer({ directory: 'remote-cache', logRequest: (line) => requests.push(line), }); -server.listen(0, '127.0.0.1'); -await once(server, 'listening'); -const { port } = server.address() as AddressInfo; const child = spawn(command!, args, { stdio: 'inherit', - env: { ...process.env, VP_REMOTE_CACHE_URL: `http://127.0.0.1:${port}${basePath}` }, + env: { ...process.env, VP_REMOTE_CACHE_URL: server.url }, }); const [code] = (await once(child, 'exit')) as [number | null]; -server.close(); +await server.close(); for (const line of requests) console.error(`[remote-cache] ${line}`); process.exitCode = code; diff --git a/packages/tools/src/remote-cache/server.ts b/packages/tools/src/remote-cache/server.ts index 76a14e33a..6211f6efa 100644 --- a/packages/tools/src/remote-cache/server.ts +++ b/packages/tools/src/remote-cache/server.ts @@ -1,56 +1,50 @@ -import { Busboy } from '@fastify/busboy'; import { decode } from 'cbor2/decoder'; -import { encode } from 'cbor2/encoder'; +import { build } from 'esbuild'; +import { Miniflare, convertV4MiniflareOptions } from 'miniflare'; +import { generateKeyPairSync, sign, type KeyObject } from 'node:crypto'; +import { once } from 'node:events'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { createServer, type IncomingMessage } from 'node:http'; +import type { AddressInfo } from 'node:net'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; -interface Entry { - value: string; - blob_id: string | null; -} - -/** The contents of `state.json`. Keys and values are hex-encoded. */ -interface State { - next_blob_id: number; - entries: Record; - associations: Record; -} +const serviceDirectory = new URL( + './', + import.meta.resolve('@voidzero-dev/remote-cache/package.json'), +); -class RequestError extends Error { - status: number; +/** The token issuer the service trusts for uploads: GitHub Actions. */ +const issuer = 'https://token.actions.githubusercontent.com'; - constructor(status: number, message: string) { - super(message); - this.status = status; - } -} +/** + * Requests reach the service at this origin, whatever address the proxy + * listens on. `endpoint` is also the audience of upload tokens. + */ +const origin = 'https://cache.example'; +const namespace = 'test'; +const basePath = `/projects/${namespace}`; +const endpoint = `${origin}${basePath}`; -function mediaType(value: string | undefined): string { - return value?.split(';')[0]?.trim().toLowerCase() ?? ''; -} +/** Upload token claims that satisfy the namespace's write policy. */ +const writerClaims = { + repository_id: '1', + repository_owner_id: '1', + repository_visibility: 'public', + ref: 'refs/heads/main', + ref_type: 'branch', + event_name: 'push', +}; -function toHex(bytes: Uint8Array): string { - return Buffer.from(bytes).toString('hex'); -} +/** Request headers that describe the connection to the proxy. */ +const connectionHeaders = new Set(['connection', 'content-length', 'host', 'transfer-encoding']); -function fromHex(hex: string): Uint8Array { - // Encode Uint8Array, not a Node Buffer, whose toJSON method would otherwise - // turn it into a map. - return new Uint8Array(Buffer.from(hex, 'hex')); -} +type Database = Awaited>; +type Bucket = Awaited>; -function byteFields(body: Uint8Array, names: string[]): Map { - let value: unknown; - try { - value = decode(body, { preferMap: true, rejectDuplicateKeys: true }); - } catch { - throw new RequestError(400, 'Invalid CBOR'); - } - if (!(value instanceof Map) || names.some((name) => !(value.get(name) instanceof Uint8Array))) { - throw new RequestError(400, `Expected byte strings: ${names.join(', ')}`); - } - return value; +interface ResponseFields { + kind?: unknown; + blob_id?: unknown; } function readBody(request: IncomingMessage): Promise { @@ -59,154 +53,259 @@ function readBody(request: IncomingMessage): Promise { request.on('data', (chunk: Buffer) => chunks.push(chunk)); request.on('end', () => resolve(Buffer.concat(chunks))); request.on('error', reject); - request.on('aborted', () => reject(new RequestError(400, 'Incomplete request'))); }); } -async function readParts(body: Buffer, contentType: string): Promise> { - return new Promise((resolve, reject) => { - const parts = new Map(); - const names = new Set(); - // Both parts are binary, including metadata without a filename. - const parser = new Busboy({ - headers: { 'content-type': contentType }, - isPartAFile: () => true, - }); - parser.on('file', (name, stream, _filename, _encoding, type) => { - const expected = name === 'metadata' ? 'application/cbor' : 'application/octet-stream'; - if (!['metadata', 'blob'].includes(name) || names.has(name) || type !== expected) { - reject(new Error('Invalid multipart part')); - } - names.add(name); - const chunks: Buffer[] = []; - stream.on('data', (chunk: Buffer) => chunks.push(chunk)); - stream.on('end', () => parts.set(name, Buffer.concat(chunks))); - stream.on('error', reject); - }); - parser.on('error', reject); - parser.on('finish', () => resolve(parts)); - parser.end(body); +/** The fields of a CBOR response, or none if it isn't a CBOR map. */ +function cborFields(contentType: string | null, body: Uint8Array): ResponseFields { + if (contentType !== 'application/cbor') return {}; + try { + const value = decode(body); + return typeof value === 'object' && value !== null ? value : {}; + } catch { + return {}; + } +} + +async function bundleService(): Promise { + const bundle = await build({ + entryPoints: [fileURLToPath(new URL('src/index.ts', serviceDirectory))], + bundle: true, + write: false, + format: 'esm', + platform: 'neutral', + target: 'es2022', + external: ['node:*', 'cloudflare:*'], }); + return bundle.outputFiles[0]!.text; } -function cbor(response: ServerResponse, value: unknown): void { - response.writeHead(200, { 'content-type': 'application/cbor' }); - response.end(encode(value)); +/** Apply the service's schema and register the namespace, unless done before. */ +async function initializeDatabase(database: Database): Promise { + const initialized = await database + .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'scopes'") + .first(); + if (initialized) return; + const migration = readFileSync(new URL('migrations/0001_cache.sql', serviceDirectory), 'utf8'); + // Keep each trigger body in one statement. + const statements = migration.match( + /CREATE TRIGGER[\s\S]*?\nEND;|(?:CREATE TABLE|CREATE (?:UNIQUE )?INDEX|INSERT INTO)[\s\S]*?;/g, + )!; + await database.batch(statements.map((sql) => database.prepare(sql))); + await database + .prepare( + `INSERT INTO scopes (scope_id, endpoint, repository, repository_id, repository_owner_id, branch) + VALUES (?, ?, 'owner/repository', ?, ?, ?)`, + ) + .bind( + namespace, + endpoint, + writerClaims.repository_id, + writerClaims.repository_owner_id, + writerClaims.ref, + ) + .run(); +} + +/** An upload token signed with `privateKey`, whose public key is `kid`. */ +function uploadToken(privateKey: KeyObject, kid: string): string { + const now = Math.floor(Date.now() / 1000); + const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url'); + const header = encode({ alg: 'RS256', kid, typ: 'JWT' }); + const claims = encode({ + ...writerClaims, + iss: issuer, + aud: endpoint, + iat: now, + nbf: now, + exp: now + 300, + }); + const signature = sign('sha256', Buffer.from(`${header}.${claims}`), privateKey); + return `${header}.${claims}.${signature.toString('base64url')}`; } /** - * A test backend that keeps its state in `directory`: entries and associations - * in `state.json`, and each blob in `blobs/` under its ID. Keys, values, and - * blobs remain opaque bytes. A fetch that matches neither key gets a plain-text - * 404. After each response, `logRequest` receives a line with the method, the - * route below `basePath`, the status, and for successful fetch responses, the - * kind. + * Blob IDs are random, so blobs are numbered in upload order. The numbers are + * saved in `blobs.json`, and each blob has a copy in `blobs/{number}`. */ -export function createCacheServer({ - basePath, +class Blobs { + readonly #file: string; + readonly #directory: string; + readonly #ids: string[]; + readonly #database: Database; + readonly #bucket: Bucket; + + constructor(directory: string, database: Database, bucket: Bucket) { + this.#file = join(directory, 'blobs.json'); + this.#directory = join(directory, 'blobs'); + this.#ids = existsSync(this.#file) ? JSON.parse(readFileSync(this.#file, 'utf8')) : []; + this.#database = database; + this.#bucket = bucket; + } + + /** Number the blob `id` if it's new. */ + add(id: string): void { + if (!this.#ids.includes(id)) this.#ids.push(id); + } + + /** The number of the blob `id`, or `undefined` if it has none. */ + find(id: string): number | undefined { + const index = this.#ids.indexOf(id); + return index === -1 ? undefined : index + 1; + } + + /** Store the contents of each copy that differs from its blob. */ + async replaceEdited(): Promise { + for (const [id, file, object] of await this.#entries()) { + if (!existsSync(file)) continue; + const bytes = readFileSync(file); + const stored = await this.#bucket.get(object); + if (stored && Buffer.from(await stored.arrayBuffer()).equals(bytes)) continue; + await this.#bucket.put(object, bytes); + // The service checks the size of each blob it serves. + await this.#database + .prepare('UPDATE generations SET blob_size = ? WHERE blob_id = ?') + .bind(bytes.length, id) + .run(); + } + } + + /** Save the numbers and a copy of each blob. */ + async save(): Promise { + mkdirSync(this.#directory, { recursive: true }); + writeFileSync(this.#file, `${JSON.stringify(this.#ids, null, 2)}\n`); + for (const [, file, object] of await this.#entries()) { + const stored = await this.#bucket.get(object); + if (stored) writeFileSync(file, Buffer.from(await stored.arrayBuffer())); + } + } + + /** The ID, copy, and storage object of each numbered blob still stored. */ + async #entries(): Promise<[string, string, string][]> { + const entries: [string, string, string][] = []; + for (const [index, id] of this.#ids.entries()) { + const object = await this.#database + .prepare('SELECT blob_object FROM generations WHERE blob_id = ?') + .bind(id) + .first('blob_object'); + if (object !== null) entries.push([id, join(this.#directory, String(index + 1)), object]); + } + return entries; + } +} + +/** + * Run the public cache service from `packages/remote-cache` in workerd and + * serve it through a proxy on a free loopback port. The service keeps its D1 + * database and R2 bucket in `directory/state`, so consecutive servers share + * them. + * + * The proxy adds a signed upload token to each store request, and the service + * verifies it with a key served in place of GitHub's. After each response, + * `logRequest` receives a line with the method, the route below the endpoint, + * the status, and for successful fetches, the kind. Blob routes show the + * blob's number instead of its ID. + * + * When the server closes, each blob is copied to `directory/blobs/{number}`. + * The next server stores a copy that has changed as that blob's contents. + */ +export async function startCacheServer({ directory, logRequest, }: { - basePath: string; directory: string; logRequest: (line: string) => void; -}) { - const stateFile = join(directory, 'state.json'); - const blobDirectory = join(directory, 'blobs'); - const state: State = existsSync(stateFile) - ? JSON.parse(readFileSync(stateFile, 'utf8')) - : { next_blob_id: 1, entries: {}, associations: {} }; - const entries = new Map(Object.entries(state.entries)); - const associations = new Map(Object.entries(state.associations)); - let nextBlobId = state.next_blob_id; - - /** Respond to `request`, returning the kind of a successful fetch response. */ - async function handle( - request: IncomingMessage, - response: ServerResponse, - path: string, - ): Promise { - if (request.method === 'GET' && path.startsWith(`${basePath}/blob/`)) { - const file = join(blobDirectory, path.slice(`${basePath}/blob/`.length)); - if (!existsSync(file)) throw new RequestError(404, 'Blob not found'); - response.writeHead(200, { 'content-type': 'application/octet-stream' }); - response.end(readFileSync(file)); - return undefined; - } - if (request.method !== 'POST' || ![`${basePath}/fetch`, `${basePath}/store`].includes(path)) { - throw new RequestError(404, 'Route not found'); - } +}): Promise<{ url: string; close: () => Promise }> { + const { publicKey, privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + const jwk = { ...publicKey.export({ format: 'jwk' }), kid: 'test', alg: 'RS256', use: 'sig' }; + const miniflare = new Miniflare( + convertV4MiniflareOptions({ + // Don't discover or register other local Wrangler/Miniflare sessions. + unsafeDevRegistryPath: '', + unsafeRegisterWorker: false, + resourcePersistencePath: join(directory, 'state'), + modules: true, + script: await bundleService(), + compatibilityDate: '2026-09-11', + compatibilityFlags: ['nodejs_compat'], + d1Databases: ['INDEX'], + r2Buckets: ['ARTIFACTS'], + bindings: { + DEPLOYMENT_ID: 'local', + NAMESPACES: JSON.stringify([namespace]), + LIMITS: '{}', + GC_BATCH_SIZE: '16', + LOG_SAMPLE_RATE: '0', + }, + ratelimits: { + READ_LIMITER: { namespace_id: '1001', simple: { limit: 10000, period: 60 } }, + STORE_LIMITER: { namespace_id: '1002', simple: { limit: 10000, period: 60 } }, + }, + outboundService: (request) => { + if (request.url !== `${issuer}/.well-known/jwks`) { + throw new Error(`Unexpected outbound request to ${request.url}`); + } + return Response.json({ keys: [jwk] }); + }, + }), + ); - const contentType = request.headers['content-type'] ?? ''; - const body = await readBody(request); - if (path === `${basePath}/fetch`) { - if (mediaType(contentType) !== 'application/cbor') { - throw new RequestError(400, 'Expected application/cbor'); - } - const fields = byteFields(body, ['key', 'secondary_key']); - const exact = entries.get(toHex(fields.get('key')!)); - const associatedKey = associations.get(toHex(fields.get('secondary_key')!)); - const fallback = associatedKey === undefined ? undefined : entries.get(associatedKey); - if (exact) { - cbor(response, { kind: 'exact', value: fromHex(exact.value), blob_id: exact.blob_id }); - return 'exact'; - } - if (fallback) { - cbor(response, { - kind: 'fallback', - key: fromHex(associatedKey!), - value: fromHex(fallback.value), - blob_id: fallback.blob_id, + try { + const database = await miniflare.getD1Database('INDEX'); + await initializeDatabase(database); + const blobs = new Blobs(directory, database, await miniflare.getR2Bucket('ARTIFACTS')); + await blobs.replaceEdited(); + + const server = createServer(async (request, response) => { + const path = new URL(request.url ?? '/', 'http://localhost').pathname; + const route = path.startsWith(basePath) ? path.slice(basePath.length) : path; + let status = 500; + let fields: ResponseFields = {}; + try { + const headers: Record = {}; + for (const [name, value] of Object.entries(request.headers)) { + if (value !== undefined && !connectionHeaders.has(name)) headers[name] = String(value); + } + if (route === '/store') + headers['authorization'] = `Bearer ${uploadToken(privateKey, jwk.kid)}`; + const body = await readBody(request); + const upstream = await miniflare.dispatchFetch(`${origin}${request.url}`, { + method: request.method!, + headers, + ...(body.length > 0 ? { body } : {}), }); - return 'fallback'; + const bytes = new Uint8Array(await upstream.arrayBuffer()); + status = upstream.status; + response.writeHead(status, Object.fromEntries(upstream.headers)); + response.end(bytes); + if (status === 200) fields = cborFields(upstream.headers.get('content-type'), bytes); + } catch (error) { + console.error(error); + response.writeHead(status, { 'content-type': 'text/plain; charset=utf-8' }); + response.end('Internal server error'); } - throw new RequestError(404, 'Not found'); - } + if (route === '/store' && typeof fields.blob_id === 'string') blobs.add(fields.blob_id); + const blobId = /^\/blob\/(.+)$/.exec(route)?.[1]; + const blobNumber = blobId === undefined ? undefined : blobs.find(blobId); + const kind = route === '/fetch' && typeof fields.kind === 'string' ? fields.kind : undefined; + const parts = [request.method, blobNumber ? `/blob/${blobNumber}` : route, status, kind]; + logRequest(parts.filter((part) => part !== undefined).join(' ')); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const { port } = server.address() as AddressInfo; - if (mediaType(contentType) !== 'multipart/form-data') { - throw new RequestError(400, 'Expected multipart/form-data'); - } - let parts: Map; - try { - parts = await readParts(body, contentType); - } catch { - throw new RequestError(400, 'Invalid multipart body'); - } - const metadata = parts.get('metadata'); - if (metadata === undefined) throw new RequestError(400, 'Missing metadata'); - const fields = byteFields(metadata, ['key', 'secondary_key', 'value']); - const key = toHex(fields.get('key')!); - const blob = parts.get('blob'); - mkdirSync(blobDirectory, { recursive: true }); - const blobId = blob === undefined ? null : String(nextBlobId++); - if (blobId !== null) writeFileSync(join(blobDirectory, blobId), blob!); - entries.set(key, { value: toHex(fields.get('value')!), blob_id: blobId }); - associations.set(toHex(fields.get('secondary_key')!), key); - const saved: State = { - next_blob_id: nextBlobId, - entries: Object.fromEntries(entries), - associations: Object.fromEntries(associations), + return { + url: `http://127.0.0.1:${port}${basePath}`, + async close() { + server.close(); + server.closeAllConnections(); + await blobs.save(); + await miniflare.dispose(); + }, }; - writeFileSync(stateFile, `${JSON.stringify(saved, null, 2)}\n`); - cbor(response, { blob_id: blobId }); - return undefined; + } catch (error) { + await miniflare.dispose(); + throw error; } - - return createServer((request, response) => { - const path = new URL(request.url ?? '/', 'http://localhost').pathname; - const log = (kind?: string) => { - const parts = [request.method, path.slice(basePath.length), response.statusCode, kind]; - logRequest(parts.filter((part) => part !== undefined).join(' ')); - }; - void handle(request, response, path).then(log, (error: unknown) => { - const known = error instanceof RequestError; - if (!known) console.error(error); - response.writeHead(known ? error.status : 500, { - 'content-type': 'text/plain; charset=utf-8', - }); - response.end(known ? error.message : 'Internal server error'); - request.resume(); - log(); - }); - }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b6c34cf26..ffb7770c8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,9 +9,6 @@ catalogs: '@anthropic-ai/sandbox-runtime': specifier: 0.0.77 version: 0.0.77 - '@fastify/busboy': - specifier: 3.2.2 - version: 3.2.2 '@openai/codex': specifier: 0.156.1 version: 0.156.1 @@ -30,9 +27,6 @@ catalogs: bun: specifier: 1.4.2 version: 1.4.2 - cbor-edn: - specifier: 0.2.2 - version: 0.2.2 cbor2: specifier: 1.12.0 version: 1.12.0 @@ -125,9 +119,6 @@ importers: '@anthropic-ai/sandbox-runtime': specifier: 'catalog:' version: 0.0.77 - '@fastify/busboy': - specifier: 'catalog:' - version: 3.2.2 '@openai/codex': specifier: 'catalog:' version: 0.156.1 @@ -137,15 +128,15 @@ importers: '@vitest/browser-playwright': specifier: 'catalog:' version: 4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@25.0.3)(esbuild@0.28.1)(tsx@4.23.13))(vitest@4.1.11) + '@voidzero-dev/remote-cache': + specifier: workspace:* + version: link:../remote-cache '@voidzero-dev/vite-task-client': specifier: workspace:* version: link:../vite-task-client bun: specifier: 'catalog:' version: 1.4.2 - cbor-edn: - specifier: 'catalog:' - version: 0.2.2 cbor2: specifier: 'catalog:' version: 1.12.0 @@ -155,6 +146,12 @@ importers: deno: specifier: 'catalog:' version: 2.9.6 + esbuild: + specifier: 0.28.1 + version: 0.28.1 + miniflare: + specifier: 5.20260911.0-alpha + version: 5.20260911.0-alpha(@types/node@25.0.3) oxfmt: specifier: 'catalog:' version: 0.70.0(vite-plus@0.3.3(@types/node@25.0.3)(@vitest/browser-playwright@4.1.11)(esbuild@0.28.1)(tsx@4.23.13)(typescript@7.1.0-dev.20260924.1)) @@ -449,9 +446,6 @@ packages: cpu: [x64] os: [win32] - '@fastify/busboy@3.2.2': - resolution: {integrity: sha512-yXSS27qPExaXeuLvMRMXOLtpipzfQYNjG3FkunDWKGfMYjKuhFXko9CVzqxm8jcF+lmtS9Fd89QNdh9XDjnbNg==} - '@img/colour@1.1.0': resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} engines: {node: '>=18'} @@ -1802,15 +1796,6 @@ packages: os: [darwin, linux, android, freebsd, win32] hasBin: true - cbor-edn@0.2.2: - resolution: {integrity: sha512-/zu3hPQmrkHtHoW1Bp73pYRlIg1R34rNiP4L87FMI2EbBf7l0cO3B2W/rBx3W8mvqDh3Qc4cJy3DcAnVaamQPQ==} - engines: {node: '>=18.7'} - hasBin: true - - cbor2@1.11.0: - resolution: {integrity: sha512-teHnpDGcDzwouFJgCJLpDY2zMaEOYvYDryd6sln67wPrCuObmwriG1NAA1w7rUIf/l++bNdts4U7WeVdYTGwLg==} - engines: {node: '>=18.7'} - cbor2@1.12.0: resolution: {integrity: sha512-3Cco8XQhi27DogSp9Ri6LYNZLi/TBY/JVnDe+mj06NkBjW/ZYOtekaEU4wZ4xcRMNrFkDv8KNtOAqHyDfz3lYg==} engines: {node: '>=18.7'} @@ -2513,8 +2498,6 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true - '@fastify/busboy@3.2.2': {} - '@img/colour@1.1.0': {} '@img/sharp-darwin-arm64@0.35.4': @@ -3438,12 +3421,6 @@ snapshots: '@oven/bun-windows-aarch64': 1.4.2 '@oven/bun-windows-x64': 1.4.2 - cbor-edn@0.2.2: - dependencies: - cbor2: 1.11.0 - - cbor2@1.11.0: {} - cbor2@1.12.0: {} cborg@6.1.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 626d1f3b4..43966e4e8 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -26,14 +26,12 @@ minimumReleaseAgeExclude: catalog: '@anthropic-ai/sandbox-runtime': 0.0.77 - '@fastify/busboy': 3.2.2 '@openai/codex': 0.156.1 '@tsconfig/strictest': ^2.0.8 '@types/node': 25.0.3 '@playwright/browser-chromium': 1.63.0 '@vitest/browser-playwright': 4.1.11 bun: 1.4.2 - cbor-edn: 0.2.2 cbor2: 1.12.0 cross-env: ^10.1.0 deno: 2.9.6