From bc2c6a3be8840aeb45c3158e7a7b7f41e036849f Mon Sep 17 00:00:00 2001 From: Jimmy Ho Date: Tue, 29 Sep 2026 10:36:57 -0500 Subject: [PATCH] feat(store): declared non-overlapping intervals and retry-safe host transactions (#902) - intervals: {start, end, within?, scope?, when?} on a collection refuses any create, PUT, PATCH or transition whose half-open [start, end) overlaps another constrained record's in its scope with 409 interval_conflict, inside the write transaction, through a partial expression index (one descending index step; 2.4 us median at 10k records vs 3.2 ms without the index). error.conflict.id only for a record the caller may read, so another owner's booking blocks without being named. Bounds are numbers or UTC (Z) date-times with at most millisecond precision. Activation refuses stored overlaps and drops stale interval indexes; reassign and ownerless-assign refuse moves that would overlap under scope: owner. - StoreExports.transaction(work, {idempotencyKey, fingerprint}) keeps hashed key/fingerprint and the JSON result (<= 16 KiB) in the same transaction and replays it without running work; different fingerprint is 422 idempotency_key_reused. Schema version 4 adds store_transaction_results. - bench:store --intervals, OpenAPI 409/conflict, STORE.md, README, llms, CHANGELOG; STORE.md "What is not covered" points at #902 for items 2, 4-6. Co-Authored-By: Claude Opus 5.5 --- artifacts/store-schema/schemas/config.json | 61 +++++ docs/EXTENSION-REFERENCE.md | 2 +- docs/STORE.md | 179 ++++++++++++-- llms-full.txt | 2 +- llms.txt | 2 +- packages/store/CHANGELOG.md | 4 + packages/store/README.md | 30 ++- packages/store/bench/store-throughput.ts | 83 ++++++- packages/store/llms.txt | 10 +- packages/store/src/authoring.ts | 2 + packages/store/src/collection.ts | 210 +++++++++++++++- packages/store/src/database.ts | 11 +- packages/store/src/index.ts | 5 +- packages/store/src/openapi.ts | 12 +- packages/store/src/ownership.ts | 20 +- packages/store/src/records.ts | 76 +++++- packages/store/src/store.ts | 18 +- packages/store/test/intervals.test.ts | 228 ++++++++++++++++++ packages/store/test/race-worker.ts | 11 + packages/store/test/sqlite.test.ts | 2 +- .../store/test/transaction-retries.test.ts | 101 ++++++++ packages/store/urlcode.json | 89 +++++++ scripts/package-audit.ts | 8 +- 23 files changed, 1108 insertions(+), 58 deletions(-) create mode 100644 packages/store/test/intervals.test.ts create mode 100644 packages/store/test/transaction-retries.test.ts diff --git a/artifacts/store-schema/schemas/config.json b/artifacts/store-schema/schemas/config.json index b95e97f0..60cd6a5a 100644 --- a/artifacts/store-schema/schemas/config.json +++ b/artifacts/store-schema/schemas/config.json @@ -357,6 +357,67 @@ "description": "true: every record this mount answers carries _owner, the opaque principal id of the owner (for auth, the user id; never an email or name), so a member can tell requesters apart. Only this mount shows it: the owner's mount, transitions and StoreExports never do." } } + }, + "intervals": { + "description": "A non-overlap constraint for scheduling: among the records it applies to, no two in the same scope (and with equal within values) may hold overlapping half-open [start, end) intervals, so an interval ending where another starts is allowed. Checked inside the write transaction of every create, PUT, PATCH and transition against an index, never a scan of the collection; an overlap answers 409 interval_conflict and nothing is written, so a move that would overlap keeps the record where it was. Activation refuses stored records that already overlap. Not on a membership collection.", + "type": "object", + "additionalProperties": false, + "required": [ + "start", + "end" + ], + "properties": { + "start": { + "type": "string", + "pattern": "^[a-z][A-Za-z0-9_]{0,63}$", + "description": "A required property holding the start: a string with format: date-time, whose values must be UTC (ending in Z) with at most millisecond precision and compare as instants, or an integer or number." + }, + "end": { + "type": "string", + "pattern": "^[a-z][A-Za-z0-9_]{0,63}$", + "description": "A required property of the same kind as start holding the end; a record whose end is not after its start answers 422 invalid_record." + }, + "within": { + "type": "array", + "maxItems": 4, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[a-z][A-Za-z0-9_]{0,63}$" + }, + "description": "Required properties that partition the constraint (a room, a resource): two intervals conflict only when every one of these is equal." + }, + "scope": { + "enum": [ + "collection", + "owner" + ], + "description": "collection (default): every record blocks every other, across owners on an owned collection (another owner's conflicting record is never named). owner: with ownership: owner only, each owner's records are constrained among themselves." + }, + "when": { + "type": "object", + "minProperties": 1, + "maxProperties": 8, + "propertyNames": { + "pattern": "^[a-z][A-Za-z0-9_]{0,63}$" + }, + "additionalProperties": { + "oneOf": [ + { + "type": "string", + "maxLength": 256 + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "description": "Only records holding exactly these values take part, for example {status: booked} so a cancelled booking frees its slot; each value must satisfy its property's schema. Without it every record takes part." + } + } } } }, diff --git a/docs/EXTENSION-REFERENCE.md b/docs/EXTENSION-REFERENCE.md index 8a840f15..ef443ccf 100644 --- a/docs/EXTENSION-REFERENCE.md +++ b/docs/EXTENSION-REFERENCE.md @@ -32,7 +32,7 @@ Two distinctions hold throughout: | `audit` | Durable, bounded audit log other extensions record privileged actions into | — | config 1 | [@jimhoyd/urlcode-audit](../packages/audit/README.md#field-reference) | | `auth` | Accounts and sessions from Better Auth on one mount; protected routes receive the signed-in user id | — | config 0 | [@jimhoyd/urlcode-auth](../packages/auth/README.md#field-reference) | | `mcp` | Declarative MCP (Model Context Protocol) server: tools, resources and prompts backed by trusted project handlers | — | config 43 | [@jimhoyd/urlcode-mcp](../packages/mcp/README.md#field-reference) | -| `store` | SQLite-backed collections served as a bounded CRUD API, declared in YAML with no handler code | uses audit | config 44 | [@jimhoyd/urlcode-store](../packages/store/README.md#field-reference) | +| `store` | SQLite-backed collections served as a bounded CRUD API, declared in YAML with no handler code | uses audit | config 50 | [@jimhoyd/urlcode-store](../packages/store/README.md#field-reference) | ## Capability to reference diff --git a/docs/STORE.md b/docs/STORE.md index 2dd083d0..a3ea8930 100644 --- a/docs/STORE.md +++ b/docs/STORE.md @@ -124,7 +124,7 @@ no principal is `401`. Writes need `Content-Type: application/json` (`415` otherwise); an `Origin` header on a write that is neither `--origin` nor an operator [alias origin](EXTENSIONS.md#site-origins-and-same-origin-checks) is refused -(`403`). Errors are `{error: {code, message, issues?, fields?}}`; submitted +(`403`). Errors are `{error: {code, message, issues?, fields?, conflict?}}`; submitted values are never echoed. A route's own body-schema `422` has the same `{error: {code, message, issues}}` shape only when the site sets `errors: {format: json}` for its path (code `UNPROCESSABLE_CONTENT`); otherwise @@ -138,8 +138,9 @@ mapping each offending parameter to a fixed message. Status codes: `400` malformed JSON, header or query, `403` `own_record_refused` (a `by: others` transition on the caller's own record), `404`, `405` with `Allow`, `409` `collection_full` (or `owner_quota_exceeded` on an -[owned collection with a per-owner limit](#per-owner-record-limit), or -`transition_conflict`), `412` stale `If-Match`, `413` body or record too +[owned collection with a per-owner limit](#per-owner-record-limit), +`transition_conflict`, or `interval_conflict` with +[declared intervals](#non-overlapping-intervals)), `412` stale `If-Match`, `413` body or record too large, `415`, `422` `invalid_record` or `idempotency_key_reused`, `503` `storage_unavailable` when the database write failed or another process held its lock past the busy timeout (nothing is written), `500` for anything unexpected @@ -602,8 +603,43 @@ store.transaction(tx => { that requires the store, never from a route `function`, `middleware` or a `sandbox: true` route. `work` runs unsandboxed with full Node access, and the principals it passes are taken as given. -- It takes no `Idempotency-Key`: a host extension that serves retries keeps its - own key, or uses a declared transition. +- **Retries** ([#902](https://github.com/jimhoyd-com/urlcode/issues/902)). + `transaction(work, {idempotencyKey, fingerprint})` makes it retry-safe, the + way `Idempotency-Key` makes an HTTP write retry-safe: + + ```js + // POST /transfers with an Idempotency-Key header, served by a host extension. + let ran = false; + const result = store.transaction(tx => { ran = true; /* the transfer, as above */ return { from: fromId, to: toId, amount }; }, + { idempotencyKey: `transfers:${principal.id}:${headerKey}`, fingerprint: canonicalBody }); + // ran === false: a replay; answer it with Idempotency-Replayed: true. + ``` + + - The claim is read under the transaction's write lock, so of racing calls + with one key exactly one runs `work`, in one process and across + connections to the same file. + - A committed call keeps, in the same transaction, the SHA-256 of the key, + the SHA-256 of the fingerprint (absent is the empty string) and `work`'s + return value as JSON. A later call with the key and the same fingerprint + returns a copy of that value without running `work` or writing anything + (no record, no audit event); a different fingerprint is + `422 idempotency_key_reused`. + - The value is a **snapshot**, unlike the HTTP replay's current record: a + host transaction can return anything, so there is no one record to read + again. It must be a JSON value (or `undefined`) that JSON keeps exactly (a + `Date`, a `Map`, `NaN` or a class instance is refused) and at most 16 KiB + serialized (`TRANSACTION_RETRIES.resultBytes`); otherwise the call throws + and nothing it did is committed. Return ids and the values the caller + must see again, not whole records: a kept result outlives a later change + or delete of the records it names. + - A call that throws (a `StoreError`, the caller's own error, a `503`) + keeps nothing, so its retry runs again. + - Keys are one namespace for the whole store database, not per collection: + prefix a key with the caller's own scope (the extension, the principal), + as the HTTP API scopes a header value to the principal. The newest 1000 + keys are kept (`TRANSACTION_RETRIES.keys`), evicted by count; an evicted + key's retry runs again. Claims survive a restart and are carried by a + backup (schema version 4, table `store_transaction_results`). ### Design decisions @@ -619,7 +655,9 @@ membership check (`403 membership_required`); the retained key (`422` or a replay); the record in the caller's scope (`404`, so another owner's record is a missing one); for `by: others` the owner check (`403 own_record_refused`); `If-Match` (`412`); the transition's `from` values -(`409 transition_conflict`); the record schema (`422 invalid_record`); quotas (`409`); then +(`409 transition_conflict`); the record schema and the interval rules +(`422 invalid_record`); quotas (`409`); the +[interval check](#non-overlapping-intervals) (`409 interval_conflict`); then the write, the claim and the audit event. **Conflict and no-mutation behavior.** Every refusal is thrown inside the @@ -670,10 +708,112 @@ The #835 counterexamples, and what serves each: | Contract | Served by | Not built | |---|---|---| | Approval of another owner's pending request ([#843](https://github.com/jimhoyd-com/urlcode/issues/843)) | a `by: others` transition with `readOnly` state, gated by a membership collection (maintained with `urlcode-store members`, audited with `audit: true`); a readers mount for the pending list across owners, showing the requester's id with `showOwner` ([the proof](../proofs/private-requests/README.md) has no application code) | a requester reference other than the opaque principal id (a display name stays an application field) | -| Scheduling: exclusive half-open intervals, expected revision, rejected move keeps its slot | a host transaction (list, check overlap, create or `update` with `ifMatch`) | a declarative non-overlap constraint; an interval index (the check reads the caller's records, at most `maxRecords`) | -| Simulated credits: hold, commit, cancel across records, conserving the total | a host transaction (`minimum: 0` refuses an overdraft and rolls the whole transfer back) | a declarative transfer; `Idempotency-Key` on host transactions | +| Scheduling: exclusive half-open intervals, expected revision, rejected move keeps its slot | a declared [`intervals`](#non-overlapping-intervals) constraint checked through an index, across owners on an owned collection, with `If-Match` and transitions (cancel, reopen); no application code | recurring intervals, capacity above one per slot | +| Simulated credits: hold, commit, cancel across records, conserving the total | a host transaction (`minimum: 0` refuses an overdraft and rolls the whole transfer back), retry-safe with an [idempotency key](#host-transactions) | a declarative transfer | | Consent/capture coordination | a host transaction | cancelling pending records on a membership change declaratively | +[#902](https://github.com/jimhoyd-com/urlcode/issues/902) tracks what is left +of this contract: a declarative transfer between records, supported +multi-process serving (with crash and disk-full evidence beyond injected +failures), sorted lists in SQL, and measuring the plumbing and edit effort the +scheduling and credit counterexamples save. + + +## Non-overlapping intervals + +A collection may declare that its records hold intervals that must not overlap +([#902](https://github.com/jimhoyd-com/urlcode/issues/902)): room bookings, +appointment slots, shifts. The store checks it inside every write's +transaction, through an index, with no application code. It replaces the host +transaction the scheduling counterexample of #835 needed, which could only +see the caller's own records. + +```yaml +collections: + bookings: + mount: /api/bookings + ownership: owner + schema: + type: object + additionalProperties: false + required: [room, start, end] + properties: + room: {type: string, maxLength: 40} + start: {type: string, format: date-time} + end: {type: string, format: date-time} + status: {type: string, enum: [booked, cancelled]} + defaults: {status: booked} + readOnlyProperties: [status] + intervals: + start: start + end: end + within: [room] # a room's bookings conflict with each other only + when: {status: booked} # a cancelled booking frees its slot + transitions: + cancel: {from: {status: booked}, set: {status: cancelled}} + reopen: {from: {status: cancelled}, set: {status: booked}} +``` + +- **The rule.** Among the records holding every `when` value (all records + without `when`), no two in the same scope with equal `within` values may + hold overlapping half-open intervals `[start, end)`. An interval that ends + exactly where another starts does not overlap it. A create, `PUT`, `PATCH` + or transition that would break it answers `409 interval_conflict` and writes + nothing, no `Idempotency-Key` claim or audit event either. A move that would + overlap therefore keeps the record where it was, and a record may move over + its own old interval. A transition is checked like any write, so `reopen` + above is refused while another booking holds the slot. +- **Scope.** `scope: collection` (the default) constrains every record against + every other, **across owners** on an owned collection: another owner's + booking blocks the slot. `scope: owner` (owned collections only) constrains + each owner's records among themselves, such as a personal calendar. +- **Privacy.** The `409` body is `{error: {code: "interval_conflict", message, + conflict?: {id}}}`. `conflict.id` names the conflicting record only when the + caller may read it: any record on a shared collection, and on an owned one + only the caller's own. Another owner's booking is never named, and nothing + about it (its owner, its interval, its other fields) is returned. The caller + still learns that the slot is taken, which is the point of the constraint; + on an owned collection with `scope: collection`, that is what one owner learns + about another's records. The same rule applies to `StoreExports`, by the + principal the caller passes. +- **Bounds.** `start` and `end` are required properties, both + `type: string` with `format: date-time` or both `integer`/`number`; the + `within` properties are required too, and none of them may be an increment. + A date-time bound must be UTC, written with `Z`, with at most millisecond + precision (`2026-10-01T09:00:00Z`, `2026-10-01T09:00:00.250Z`), and bounds + compare as instants, so `10:00:00Z` and `10:00:00.000Z` are the same. + Anything else (an offset such as `+01:00`, a lower-case `z`, microseconds) + and an `end` not after its `start` are `422 invalid_record` with the keyword + `intervals`. *Why require `Z` rather than normalize:* the store never + rewrites a value the caller sent, and a UTC-only bound sorts and filters + the same way everywhere; converting a local time is the client's job, where + the time zone is known. +- **Cost.** The declaration builds one partial SQLite index over the records + it applies to, keyed by owner (with `scope: owner`), the `within` values and + the start instant. Because stored intervals in one scope never overlap, the + only record that can conflict with a new interval is the one in its scope + that starts last before the new one ends, so the check is one descending + step of that index (`ORDER BY start DESC LIMIT 1`), not a scan. Measured with + `npm run bench:store -- --intervals` (Apple M4 Pro, Node 26.10, SQLite + 3.53.4, 20 rooms of back-to-back bookings owned by 50 principals): at 10,000 + records the check query takes 2.4 µs at the median (1.8 µs at 1,000), the + same query without the index 3.2 ms, and reading one room's records to + compare them in JavaScript, as the host transaction did, 197 µs. A whole + accepted create with `synchronous=FULL` took 184 µs with the constraint and + 180 µs without it; the check is not what a write pays for. +- **Activation and operator commands.** Activation refuses stored records that + break the rules or overlap, naming the two record ids, rather than serving + them. With `scope: owner`, `urlcode-store ownerless-assign` and + `urlcode-store reassign` (dry runs included) refuse, before anything is + written, a move that would give the target principal overlapping intervals. + Changing the declaration changes the index: activation builds the new one + and drops the one nobody declares any more. +- **Races.** The check reads committed state under the write lock, so of + concurrent bookings of one slot exactly one commits, in one process and + across connections to the same file (the tests race both). +- **Not covered.** One booking per slot (no capacity above one), no recurring + intervals, no open-ended interval (both bounds are required), and no + suggestion of a free slot. Not on a membership collection. ## Record schema @@ -1232,10 +1372,15 @@ operator says made the change, not proof of it. Commands that change nothing `store_idempotency` (retained `Idempotency-Key` claims: the scoped key hash, the request fingerprint, the status and the record id, never record values) and `store_audit_outbox` (undelivered audit events), plus the one-row - `store_audit_drain` (when the audit drain last kept up; schema version 3). + `store_audit_drain` (when the audit drain last kept up; schema version 3) + and `store_transaction_results` (retained + [host transaction](#host-transactions) keys and results; schema version 4). Collections are rows, not tables, so declaring, changing or removing a collection never changes the - schema; the rows of a collection that is no longer declared stay untouched. + tables; the rows of a collection that is no longer declared stay untouched. + The one derived object is the partial index a declared + [`intervals`](#non-overlapping-intervals) reads through: activation builds + it, and drops an interval index no live activation declares any more. - The schema only moves forward. An empty file is initialized in one transaction; opening an up-to-date database changes nothing; a later release that changes the schema adds a step, and each step runs in its own @@ -1253,8 +1398,8 @@ operator says made the change, not proof of it. Commands that change nothing and writes everything it changes: the record, its unique key, the `Idempotency-Key` claim and eviction, and the audit event. Any failure rolls all of it back. `If-Match`, key uniqueness, `maxRecords`, - `maxRecordsPerOwner`, increment bounds, retained keys and the audit backlog - are therefore checked against committed state and cannot be overshot by + `maxRecordsPerOwner`, increment bounds, declared intervals, retained keys + and the audit backlog are therefore checked against committed state and cannot be overshot by concurrent requests. Statements are synchronous: within the process no other request runs between a transaction's checks and its commit, and each commit's fsync blocks the event loop while it runs. [Capacity](CAPACITY.md#measured-the-sqlite-store) @@ -1502,11 +1647,11 @@ operations as one database transaction: see ## Not built yet -SQL ordering for sorted lists, a declarative interval (non-overlap) -constraint, a declarative multi-record transfer and roles beyond a -[membership collection](#membership-gates-and-cross-owner-reads) are not built -([#835](https://github.com/jimhoyd-com/urlcode/issues/835); the -[transition design](#what-is-not-covered) lists what each needs). Recorded in +SQL ordering for sorted lists, a declarative multi-record transfer and +supported multi-process serving are not built +([#902](https://github.com/jimhoyd-com/urlcode/issues/902); the +[transition design](#what-is-not-covered) lists what each needs), nor are roles +beyond a [membership collection](#membership-gates-and-cross-owner-reads). Recorded in [open decisions](OPEN-DECISIONS.md): ranges and text search. Owned collections ([#331](https://github.com/jimhoyd-com/urlcode/issues/331)) are owner-only apart from [readers mounts](#membership-gates-and-cross-owner-reads): sharing one record with chosen principals and write access for managers or support are not diff --git a/llms-full.txt b/llms-full.txt index 032b2392..f092659b 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -6490,7 +6490,7 @@ Two distinctions hold throughout: | `audit` | Durable, bounded audit log other extensions record privileged actions into | — | config 1 | [@jimhoyd/urlcode-audit](https://github.com/jimhoyd-com/urlcode/blob/main/packages/audit/README.md#field-reference) | | `auth` | Accounts and sessions from Better Auth on one mount; protected routes receive the signed-in user id | — | config 0 | [@jimhoyd/urlcode-auth](https://github.com/jimhoyd-com/urlcode/blob/main/packages/auth/README.md#field-reference) | | `mcp` | Declarative MCP (Model Context Protocol) server: tools, resources and prompts backed by trusted project handlers | — | config 43 | [@jimhoyd/urlcode-mcp](https://github.com/jimhoyd-com/urlcode/blob/main/packages/mcp/README.md#field-reference) | -| `store` | SQLite-backed collections served as a bounded CRUD API, declared in YAML with no handler code | uses audit | config 44 | [@jimhoyd/urlcode-store](https://github.com/jimhoyd-com/urlcode/blob/main/packages/store/README.md#field-reference) | +| `store` | SQLite-backed collections served as a bounded CRUD API, declared in YAML with no handler code | uses audit | config 50 | [@jimhoyd/urlcode-store](https://github.com/jimhoyd-com/urlcode/blob/main/packages/store/README.md#field-reference) | ### Capability to reference diff --git a/llms.txt b/llms.txt index a1cecae1..8cb34293 100644 --- a/llms.txt +++ b/llms.txt @@ -78,7 +78,7 @@ or bypass target limits or operator grants ([design principle](https://github.co | Sign-in, sign-out, protected routes | `extensions.auth: {version: "1", config: {}}` (a thin Better Auth adapter; `urlcode extensions add auth`, then `npx urlcode-auth migrate`), an `/api/auth/*` mount with `extension: auth` and `methods: [GET, POST]`, then `auth: true` on each protected route (its function reads `context.capabilities.auth.identity.userId`; there is no role, permission or CSRF key); browsers sign in with `createAuthClient({basePath: '/api/auth'})` from `better-auth/client`; roles and permissions are application data ([auth](https://github.com/jimhoyd-com/urlcode/blob/main/packages/auth/README.md), [the `auth` short form](https://github.com/jimhoyd-com/urlcode/blob/main/docs/EXTENSIONS.md#protecting-a-route-the-auth-short-form)) | | Application rules needing the signed-in caller | first the store: owned collections, declared transitions and membership gates (end-to-end example with no application code: [proofs/private-requests](https://github.com/jimhoyd-com/urlcode/blob/main/proofs/private-requests/README.md)); otherwise ordinary YAML function routes with `auth: true`, explicit methods and bounded request bodies, where the trusted function reads the verified user id from `context.capabilities.auth.identity.userId` (not available to `sandbox: true` routes) ([request-bound capabilities](https://github.com/jimhoyd-com/urlcode/blob/main/docs/EXTENSIONS.md#request-bound-capabilities), recipe `authenticated-json-api`) | | Audit log | `audit` extension: no route; `audit: true` on a store collection ([nesting](https://github.com/jimhoyd-com/urlcode/blob/main/docs/EXTENSIONS.md#nesting)) | -| Persistence | operator-installed store extension, recipe `store-crud`, each collection's record `schema` a JSON Schema 2020-12 object schema in the request body profile, inline or a named project schema, with store behavior (`defaults`, `readOnlyProperties`) beside it on the collection (422 with the body-schema issues; [record schema](https://github.com/jimhoyd-com/urlcode/blob/main/docs/STORE.md#record-schema)), described by `urlcode openapi --host-file`, per-record ownership (`ownership: owner` on a collection behind `auth: true`, [store ownership](https://github.com/jimhoyd-com/urlcode/blob/main/docs/STORE.md#per-record-ownership)), declared single-record `transitions` and `Idempotency-Key` replay ([transitions](https://github.com/jimhoyd-com/urlcode/blob/main/docs/STORE.md#conditional-transitions-and-result-aware-retries)), permissions as data: a `membership: true` collection of principal ids gating a transition (`members`) and read-only cross-owner `readers` mounts ([membership gates](https://github.com/jimhoyd-com/urlcode/blob/main/docs/STORE.md#membership-gates-and-cross-owner-reads)); declarative multi-record or interval constraints and filtering beyond equality are gaps | +| Persistence | operator-installed store extension, recipe `store-crud`, each collection's record `schema` a JSON Schema 2020-12 object schema in the request body profile, inline or a named project schema, with store behavior (`defaults`, `readOnlyProperties`) beside it on the collection (422 with the body-schema issues; [record schema](https://github.com/jimhoyd-com/urlcode/blob/main/docs/STORE.md#record-schema)), described by `urlcode openapi --host-file`, per-record ownership (`ownership: owner` on a collection behind `auth: true`, [store ownership](https://github.com/jimhoyd-com/urlcode/blob/main/docs/STORE.md#per-record-ownership)), declared single-record `transitions` and `Idempotency-Key` replay ([transitions](https://github.com/jimhoyd-com/urlcode/blob/main/docs/STORE.md#conditional-transitions-and-result-aware-retries)), permissions as data: a `membership: true` collection of principal ids gating a transition (`members`) and read-only cross-owner `readers` mounts ([membership gates](https://github.com/jimhoyd-com/urlcode/blob/main/docs/STORE.md#membership-gates-and-cross-owner-reads)), a declared non-overlap `intervals` constraint for scheduling (`409 interval_conflict`, [intervals](https://github.com/jimhoyd-com/urlcode/blob/main/docs/STORE.md#non-overlapping-intervals)) and retry-safe host transactions (`StoreExports.transaction(work, {idempotencyKey})`); declarative multi-record transfers and filtering beyond equality are gaps | | A form: validate a submission, email it, rate-limit it | a frontend posting JSON to a route whose `request.body.POST.schema` validates it before code runs (`format: email` and the other standard formats); `respond` answers and `signals` notifies a hook with no project code (recipe `contact-form`); to email it, a trusted `function` calls the provider's SDK, HTTP API or nodemailer with its credentials in granted `secrets`; `policies.throttle` limits the route | | A frontend over store collections | the application's own code: `fetch` to the JSON mounts, reading a list's `etags` (send `If-Match`) and `may` (offer only allowed transitions); URLCode ships no component kit or screens. Reference: [proofs/private-requests/client](https://github.com/jimhoyd-com/urlcode/blob/main/proofs/private-requests/client/main.js); for shadcn/ui use the official tooling and `urlcode artifacts stage` | | Frontend at the site root beside `/api/...` extension mounts | a root `/*` `static` mount (`index: index.html`) plus the extension mounts: the longest mount prefix wins, so each extension keeps its whole mount; a route inside an extension's mount, or an extension mount enclosing another, fails `urlcode validate` naming both routes ([site-root frontend](https://github.com/jimhoyd-com/urlcode/blob/main/docs/ROUTING.md#a-site-root-frontend-beside-api-extension-mounts)) | diff --git a/packages/store/CHANGELOG.md b/packages/store/CHANGELOG.md index 3d131bdc..3e3bd3d2 100644 --- a/packages/store/CHANGELOG.md +++ b/packages/store/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +**Declared non-overlapping intervals (#902).** A collection may declare `intervals: {start, end, within?, scope?, when?}`: among the records holding every `when` value, no two in the same scope with equal `within` values may hold overlapping half-open `[start, end)` intervals (adjacent ones are allowed). Every create, `PUT`, `PATCH` and transition is checked inside its own write transaction and refused with `409 interval_conflict`, writing nothing (a refused move keeps its slot; a transition back into `when` is checked too). `scope: collection` (the default) constrains every record across owners on an owned collection, and `scope: owner` each owner's records alone. The body's `error.conflict.id` names the conflicting record only when the caller may read it (any record of a shared collection, only the caller's own on an owned one), so another owner's booking blocks without being disclosed. Bounds are required, both `integer`/`number` or both `format: date-time` strings, which must be UTC (`Z`) with at most millisecond precision (anything else, and an end not after its start, is `422 invalid_record` with keyword `intervals`) and compare as instants. The check reads one step of a partial expression index the declaration builds (activation creates it, and drops interval indexes no live activation declares): 2.4 µs at the median over 10,000 records, against 3.2 ms for the same query without the index (`npm run bench:store -- --intervals`, which this adds). Activation refuses stored records that overlap or break the rules, naming both ids, and `reassign` and `ownerless-assign` refuse a move that would give a principal overlapping intervals under `scope: owner`. The OpenAPI description lists `interval_conflict` and the `conflict` member of `StoreError`. + +**Retry-safe host transactions (#902).** `StoreExports.transaction(work, {idempotencyKey, fingerprint?})` keeps the SHA-256 of the key and of the fingerprint with `work`'s return value (JSON that round-trips exactly, or `undefined`; at most 16 KiB) in the same transaction, and returns a copy of that value to a later call with the key and the same fingerprint without running `work` or writing anything; a different fingerprint is `422 idempotency_key_reused`. Racing calls with one key run `work` once, across connections too; a transaction that throws keeps nothing. The newest 1000 keys are kept, store-wide, and survive a restart. Schema version 4 adds the `store_transaction_results` table (the upgrade adds it and changes nothing else). `TRANSACTION_RETRIES`, `StoreTransactionOptions`, `StoreTransaction`, `StoreTransactionRecords`, `IntervalSpec` and `IntervalScope` are exported. + **Breaking: named record schemas, and defaults and read-only properties on the collection (#908).** A collection's `schema` may name one of the project's named schemas (top-level `schemas:`, `ExtensionActivation.schemas`) instead of writing one inline, so one schema is a collection's record shape, a route's request body and an MCP tool's arguments, and all three refuse the same invalid input at the same pointer with the same issue. The named schema must satisfy the inline restrictions (a flat object of scalar properties, `additionalProperties: false`, no `$defs`); activation refuses one that does not, naming the collection, the schema and the pointer, and an unknown name. **Breaking:** a record schema, inline or named, is value shape only: `default` and `readOnly` on a property are refused at activation, and the collection carries them instead as `defaults: {: }` and `readOnlyProperties: []`, with the same behavior as before (a default must satisfy its property's schema; a read-only property changes only through a transition). `StoreRecords` adds `defaults` and `readOnlyProperties`; `PropertySchema` loses `default`/`readOnly`; `CollectionSpec.schema` is `RecordSchema | string`; `normalize`, `compileRecordSchema` and the `Collection` constructor take the project's named schemas; `OwnerlessOptions`, `ReassignOptions` and `MembershipOptions` take `schemas` (the CLI passes the project's). The OpenAPI description references a named schema's component (`#/components/schemas//properties/`, with the collection's `default`/`readOnly` beside it) instead of copying it, and `StoreCreate` is the component itself when the collection takes it unchanged as a create body. **Commit durability is an operator setting (#859).** `store({durability: 'full' | 'normal'})` in `host.mjs`, else `STORE_DURABILITY`, else `full`, sets the connection's SQLite `synchronous` level: `full` (the default and the previous behavior) fsyncs every commit; `normal` fsyncs only at checkpoints, so commits are faster and the last ones before a power loss or OS crash can be lost (a process crash loses nothing; the database never corrupts). Any other value, `off` and `extra` included, is refused by `host()` before anything is served. An activation with `normal` writes one `extension_warning` to the operator log. The `urlcode-store` operator commands always commit with `full`. `StoreExtensionOptions.durability`, `STORE_DURABILITIES` and the `StoreDurability` type are exported, and `createStore` returns the resolved `durability`. docs/STORE.md "Durability" documents the trade-off with the docs/CAPACITY.md measurements. diff --git a/packages/store/README.md b/packages/store/README.md index 515abcba..5597b350 100644 --- a/packages/store/README.md +++ b/packages/store/README.md @@ -237,7 +237,11 @@ partial `update` (which clears a property given `null`, like `PATCH`) and a paginated `list`, each scoped to the request principal exactly as the JSON API is, a declared `transition`, and `transaction(work)`, which runs several of those operations synchronously as one database transaction (trusted host code only, -never sandboxed). See [using a collection from another extension][store-using-a-collection-from-another-extension]. +never sandboxed). `transaction(work, {idempotencyKey, fingerprint})` is +retry-safe: the key, the fingerprint and `work`'s JSON result (at most 16 KiB) +are kept in the same transaction, a retry with the same fingerprint returns +that result without running `work`, and a different one is +`422 idempotency_key_reused`. See [using a collection from another extension][store-using-a-collection-from-another-extension]. ## Transitions @@ -248,9 +252,21 @@ moves one record from the exact `from` values to the constant `set` values writes nothing. On an owned collection `by: others` lets any principal except the owner run it, on a separate mount whose route policy decides who may; a property in `readOnlyProperties` can only change through a transition. -Transitions are not an expression language: interval constraints and -multi-record transfers use a host transaction. See +Transitions are not an expression language: multi-record transfers use a host +transaction. See [conditional transitions and result-aware retries][store-conditional-transitions-and-result-aware-retries]. + +A scheduling collection declares `intervals: {start, end, within?, scope?, +when?}` (#902): among the records holding the `when` values, no two in one +scope with equal `within` values (a room) may hold overlapping half-open +`[start, end)` intervals. Every create, `PUT`, `PATCH` and transition is +checked in its own transaction through a partial index (one index step, not a +scan) and an overlap answers `409 interval_conflict`, writing nothing, so a +refused move keeps its slot. On an owned collection the default +`scope: collection` lets another owner's booking block a slot without naming +it: `error.conflict.id` appears only for a record the caller may read. Bounds +are both numbers or both UTC date-times (`Z`, at most millisecond precision). +See [non-overlapping intervals][store-non-overlapping-intervals]. A list carries each listed record's `ETag` in `etags` and the transitions the caller may run on it now in `may`, both keyed by id (one record's answer has them as the `ETag` and `Allow-Transitions` headers), so a client sends @@ -356,6 +372,7 @@ version it describes; `npm run release:bump` moves them and scripts/check-local- [store-what-the-caller-may-run]: https://github.com/jimhoyd-com/urlcode/blob/v0.6.5/docs/STORE.md#what-the-caller-may-run [store-membership-gates-and-cross-owner-reads]: https://github.com/jimhoyd-com/urlcode/blob/v0.6.5/docs/STORE.md#membership-gates-and-cross-owner-reads [store-bounded-keyed-transitions]: https://github.com/jimhoyd-com/urlcode/blob/v0.6.5/docs/STORE.md#bounded-keyed-transitions +[store-non-overlapping-intervals]: https://github.com/jimhoyd-com/urlcode/blob/v0.6.5/docs/STORE.md#non-overlapping-intervals [extensions-artifacts]: https://github.com/jimhoyd-com/urlcode/blob/v0.6.5/docs/EXTENSIONS.md#artifacts @@ -413,6 +430,12 @@ Every key `store` accepts, rendered from this package's `urlcode.json` (the sche | `extensions.store.config.collections.*.readers.mount` | string | yes | maxLength: 256; pattern: "^/[A-Za-z0-9._~/-]*[A-Za-z0-9._~-]$" | A separate mount: a route `/*` with extension: store (GET, HEAD) and a principal-providing policy. | | `extensions.store.config.collections.*.readers.members` | string | yes | pattern: "^[a-z][a-z0-9_-]{0,63}$" | A membership collection: anyone it does not list gets 403 membership_required before any record is read. | | `extensions.store.config.collections.*.readers.showOwner` | boolean | no | — | true: every record this mount answers carries _owner, the opaque principal id of the owner (for auth, the user id; never an email or name), so a member can tell requesters apart. Only this mount shows it: the owner's mount, transitions and StoreExports never do. | +| `extensions.store.config.collections.*.intervals` | object | no | unknown keys rejected | A non-overlap constraint for scheduling: among the records it applies to, no two in the same scope (and with equal within values) may hold overlapping half-open [start, end) intervals, so an interval ending where another starts is allowed. Checked inside the write transaction of every create, PUT, PATCH and transition against an index, never a scan of the collection; an overlap answers 409 interval_conflict and nothing is written, so a move that would overlap keeps the record where it was. Activation refuses stored records that already overlap. Not on a membership collection. | +| `extensions.store.config.collections.*.intervals.start` | string | yes | pattern: "^[a-z][A-Za-z0-9_]{0,63}$" | A required property holding the start: a string with format: date-time, whose values must be UTC (ending in Z) with at most millisecond precision and compare as instants, or an integer or number. | +| `extensions.store.config.collections.*.intervals.end` | string | yes | pattern: "^[a-z][A-Za-z0-9_]{0,63}$" | A required property of the same kind as start holding the end; a record whose end is not after its start answers 422 invalid_record. | +| `extensions.store.config.collections.*.intervals.within` | array | no | maxItems: 4; uniqueItems: true; items: string (pattern: "^[a-z][A-Za-z0-9_]{0,63}$") | Required properties that partition the constraint (a room, a resource): two intervals conflict only when every one of these is equal. | +| `extensions.store.config.collections.*.intervals.scope` | string | no | enum: ["collection","owner"] | collection (default): every record blocks every other, across owners on an owned collection (another owner's conflicting record is never named). owner: with ownership: owner only, each owner's records are constrained among themselves. | +| `extensions.store.config.collections.*.intervals.when` | object | no | minProperties: 1; maxProperties: 8; keys: "^[a-z][A-Za-z0-9_]{0,63}$"; values: string / number / boolean (one of: string (maxLength: 256); number; boolean) | Only records holding exactly these values take part, for example {status: booked} so a cancelled booking frees its slot; each value must satisfy its property's schema. Without it every record takes part. | | `extensions.store.config.shortLinks` | object | no | maxProperties: 32; keys: "^[a-z][a-z0-9_-]{0,63}$" | Public redirect mounts by name: GET `/` atomically increments a counter and answers 302 to the record's stored destination; HEAD answers the same 302 without counting; an unknown key is 404. Each needs a route `/*` with extension: store (GET, HEAD). | | `extensions.store.config.shortLinks.*.mount` | string | yes | maxLength: 256; pattern: "^/[A-Za-z0-9._~/-]*[A-Za-z0-9._~-]$" | URL path of the redirect mount, separate from the collection's CRUD mount. | | `extensions.store.config.shortLinks.*.collection` | string | yes | pattern: "^[a-z][a-z0-9_-]{0,63}$" | A declared shared collection with a key; the key value is the path segment after the mount. | @@ -426,6 +449,7 @@ Declare collections under extensions.store.config.collections and mount each on - **collections** (configuration, `urlcode.yaml`): Per-collection mount, a record `schema` (a JSON Schema 2020-12 object schema in the request body profile: flat scalar properties, `additionalProperties: false`; inline, or the name of a project schema under top-level `schemas:` shared with route bodies and MCP tools; a record that breaks it answers `422 invalid_record` with the body-schema issue list), `defaults` (values a create stores for omitted properties) and `readOnlyProperties` (changed only by a transition), bounded unique `key`, numeric `increments`, durable bounded `idempotency`, maxRecords, maxRecordBytes, pageSize, readOnly, `sortable` / `filterable` property lists, and `audit: true` (every write recorded in the audit log with property names and the principal, never values; needs the audit extension, and writes answer `503 audit_backlog` while 1000 events wait to drain). Create, list, read, replace, update and delete need no handler. - **ownership** (configuration, `urlcode.yaml`): `ownership: owner` on a collection: each signed-in principal creates, lists, reads, changes and deletes only its own records: every read and write is scoped to the principal that created the record. Not combined with a unique `key`. The mount must be guarded by a principal-providing policy such as `auth: true`. An optional `maxRecordsPerOwner` (at most maxRecords) answers `409 owner_quota_exceeded` at the limit. - **transitions** (configuration, `urlcode.yaml`): Declared `transitions` move one record from exact `from` values to constant `set` values (with `stamp: {property: actor\|now}`) in one transaction as `POST //`; a record not in the `from` state answers `409 transition_conflict`. On an owned collection `by: others` serves the transition on its own `mount` to any principal except the owner (an approval or review step), and `members: ` admits only that list's members. Not an expression language. +- **intervals** (configuration, `urlcode.yaml`): `intervals: {start, end, within?, scope?, when?}` on a collection: no two records it applies to (those holding the `when` values, such as `{status: booked}`) in one scope with equal `within` values (a room) may hold overlapping half-open `[start, end)` intervals; any create, PUT, PATCH or transition that would answers `409 interval_conflict` and writes nothing. Checked through an index in the write transaction, across owners on an owned collection without naming another owner's record. Bounds are both numbers or both `format: date-time` strings in UTC (`Z`). - **membership** (configuration, `urlcode.yaml`): Permissions as data keyed by the principal id, never roles in auth: a `membership: true` collection with a `key` lists principal ids and has no mount (the operator maintains it with `urlcode-store members` or `addMember`/`removeMember`). A transition's `members` and a collection's `readers.members` name it. - **readers** (configuration, `urlcode.yaml`): An owned collection's `readers: {mount, members, showOwner?}` lets the members of a membership collection list (with the collection's limit, cursor, sort and filters) and read every owner's records read-only on a separate mount; owners keep their own view on the collection mount. - **shortLinks** (configuration, `urlcode.yaml`): Optional public GET redirect mounts that look up a collection key, use a declared `format: uri` destination property that takes only HTTP(S) URLs, and atomically increment a declared counter. diff --git a/packages/store/bench/store-throughput.ts b/packages/store/bench/store-throughput.ts index 1806f69b..04b2c1e6 100644 --- a/packages/store/bench/store-throughput.ts +++ b/packages/store/bench/store-throughput.ts @@ -4,7 +4,7 @@ // npm run bench:store -- --quick # smaller counts, for checking the script itself // npm run bench:store -- --json out.json // -// Three parts, each printed as a table: +// Four parts, each printed as a table (`--intervals` runs only the fourth): // 1. HTTP: a real server (a child process running startServer with the store, audit and a header principal) driven // by a bounded keep-alive node:http client at fixed concurrency. Creates (Idempotency-Key), PATCH with If-Match // and declared transitions on an owned collection with audit off and on, then list latency at page sizes 20 and @@ -15,6 +15,9 @@ // 3. List micro-benchmark: the store's sorted/filtered list path (projection query, JS ordering via runList, page // fetch) at 1k/10k/50k records, beside an ORDER BY ... LIMIT in SQL with and without an expression index. 50k is // above the configurable maxRecords and is measured only to show the curve. +// 4. Interval micro-benchmark (#902): the declared non-overlap check's query through its index at 1k and 10k +// records, beside the same query without the index and the host-transaction scan it replaces, and whole creates +// (refused and accepted) with and without `intervals`. import { fork } from 'node:child_process'; import type { ChildProcess } from 'node:child_process'; import { randomUUID } from 'node:crypto'; @@ -340,11 +343,89 @@ async function main(): Promise { for (const size of [1000, 10_000, 50_000]) micro.push(...await listMicro(dir, size)); table(micro as unknown as Record[]); report.listMicro = micro; + await intervalPart(root, report); if (jsonOut) await writeFile(jsonOut, JSON.stringify(report, null, 2)); } finally { await rm(root, { recursive: true, force: true }); } } +// --------------------------------------------------------------------------------------------------------------------- +// Interval micro-benchmark (#902 item 1): the declared non-overlap check at 1k and 10k records (the configurable +// maximum), in process. The check's own query through the interval index, the same query with the index dropped, and +// the host-transaction check it replaces (read every record in the calendar, compare in JS); then whole writes (one +// synchronous=FULL commit each) through Collection.create with and without `intervals`. The records are 20 rooms of +// back-to-back one-hour bookings, owned by 50 principals, `scope: collection`. +interface IntervalMicro { size: number; shape: string; p50us: number; p95us: number; p99us: number } +async function intervalMicro(dir: string, size: number): Promise { + const { openStoreDatabase } = await import('../src/database.ts'); + const { Collection } = await import('../src/collection.ts'); + const rooms = 20, perRoom = size / rooms, base = Date.UTC(2026, 0, 1), hourMs = 3_600_000; + const bookingSchema = { type: 'object', additionalProperties: false, required: ['room', 'start', 'end'], properties: { room: { type: 'string', maxLength: 20 }, start: { type: 'string', format: 'date-time' }, end: { type: 'string', format: 'date-time' } } }; + const spec = (intervals: boolean) => ({ mount: '/api/bookings', ownership: 'owner', maxRecords: 10_000, schema: bookingSchema, ...(intervals ? { intervals: { start: 'start', end: 'end', within: ['room'] } } : {}) }) as unknown as import('../src/collection.ts').CollectionSpec; + const db = await openStoreDatabase(join(dir, `intervals-${size}.sqlite`)); + const now = new Date().toISOString(), iso = (ms: number) => new Date(ms).toISOString(); + db.transaction(() => { + for (let r = 0; r < rooms; r++) for (let h = 0; h < perRoom; h++) + db.run('INSERT INTO store_records(collection, id, owner, key, created_at, updated_at, data) VALUES (?, ?, ?, NULL, ?, ?, ?)', 'bookings', randomUUID(), `p${(r * perRoom + h) % 50}`, now, now, JSON.stringify({ room: `room-${r}`, start: iso(base + h * hourMs), end: iso(base + (h + 1) * hourMs) })); + }); + const checked = new Collection('bookings', spec(true)); + checked.open(db); + const intervals = checked.spec.intervals!; + const time = (shape: string, iterations: number, fn: (i: number) => unknown): IntervalMicro => { + for (let i = 0; i < 5; i++) fn(i); + const times: number[] = []; + for (let i = 0; i < iterations; i++) { const t0 = performance.now(); fn(i); times.push((performance.now() - t0) * 1000); } + times.sort((a, b) => a - b); + return { size, shape, p50us: round(pct(times, 50), 1), p95us: round(pct(times, 95), 1), p99us: round(pct(times, 99), 1) }; + }; + // A probe inside the booked range, so both a hit and a miss have to find their neighbour. + const probe = (i: number) => { const h = (i * 7919) % perRoom; return { room: `room-${i % rooms}`, start: base + h * hourMs + 1_800_000 }; }; + const iterations = quick ? 200 : 2000; + const out: IntervalMicro[] = []; + const query = (label: string) => time(`check query, ${label}`, iterations, i => { const p = probe(i); return db.get(intervals.latest, p.room, p.start + hourMs, 'none'); }); + out.push(query('interval index')); + // What a host transaction did before (#835's scheduling proof): read the calendar's records and compare in JS. + out.push(time('host-transaction scan (read the room, compare in JS)', quick ? 20 : 200, i => { + const p = probe(i); + return db.all<{ data: string }>("SELECT data FROM store_records WHERE collection = ? AND data ->> '$.room' = ?", 'bookings', p.room).some(row => { const b = JSON.parse(row.data) as { start: string; end: string }; return p.start < Date.parse(b.end) && Date.parse(b.start) < p.start + hourMs; }); + })); + db.run(`DROP INDEX "${intervals.index}"`); + out.push(time('check query, index dropped', quick ? 20 : 200, i => { const p = probe(i); return db.get(intervals.latest, p.room, p.start + hourMs, 'none'); })); + db.run(intervals.create); + // Whole writes: refused (conflict) and accepted (a fresh room each, so every create is a new booking), with and + // without the declaration. Accepted creates stop below maxRecords. + const writes = quick ? 50 : 300; + out.push(time('create refused (409 interval_conflict), intervals', quick ? 50 : 500, i => { const p = probe(i); try { checked.create({ room: p.room, start: iso(p.start), end: iso(p.start + hourMs) }, undefined, 'p1', 'p1'); } catch { /* expected */ } })); + // Room for the accepted writes under maxRecords: the newest seeded bookings go, and the writes bring the size back. + const room = () => db.run('DELETE FROM store_records WHERE seq IN (SELECT seq FROM store_records WHERE collection = ? ORDER BY seq DESC LIMIT ?)', 'bookings', writes + 10); + room(); + let fresh = 0; + out.push(time('create accepted, intervals', writes, () => checked.create({ room: `new-${fresh++}`, start: iso(base), end: iso(base + hourMs) }, undefined, 'p1', 'p1'))); + room(); + db.run(`DROP INDEX "${intervals.index}"`); + const plain = new Collection('bookings', spec(false)); + plain.open(db); + out.push(time('create accepted, no intervals (no index)', writes, () => plain.create({ room: `plain-${fresh++}`, start: iso(base), end: iso(base + hourMs) }, undefined, 'p1', 'p1'))); + db.close(); + return out; +} +async function intervalPart(root: string, report: Record): Promise { + console.log('## Interval micro-benchmark (in process, no HTTP)\n'); + const dir = join(root, 'intervals'); + await mkdir(dir, { recursive: true }); + const rows: IntervalMicro[] = []; + for (const size of [1000, 10_000]) rows.push(...await intervalMicro(dir, size)); + table(rows as unknown as Record[]); + report.intervals = rows; +} + if (args.includes('--server')) await serve(); +else if (args.includes('--intervals')) { + // Only the interval part: npm run bench:store -- --intervals + const root = await mkdtemp(join(tmpdir(), 'store-bench-')); + const report: Record = { machine: { cpu: cpus()[0]?.model, cores: cpus().length, platform: `${platform()} ${release()}` }, node: process.version, sqlite: process.versions.sqlite, quick }; + try { await intervalPart(root, report); if (jsonOut) await writeFile(jsonOut, JSON.stringify(report, null, 2)); } + finally { await rm(root, { recursive: true, force: true }); } +} else await main(); diff --git a/packages/store/llms.txt b/packages/store/llms.txt index fcc04f25..93424723 100644 --- a/packages/store/llms.txt +++ b/packages/store/llms.txt @@ -33,8 +33,14 @@ Part of the URLCode framework, in the same repository: docs/FRAMEWORK.md stamps, `If-Match`, `Idempotency-Key`; `409 transition_conflict` otherwise). `by: others` on an owned collection lets any principal but the owner act, on its own guarded mount; properties listed in the collection's `readOnlyProperties` change only through a transition. `idempotency: {maxKeys}` replays a retried key (same request: first - status, current record; different request: `422 idempotency_key_reused`). Interval constraints and - multi-record transfers are not declarative: a host extension uses `StoreExports.transaction(work)`. + status, current record; different request: `422 idempotency_key_reused`). Multi-record transfers + are not declarative: a host extension uses `StoreExports.transaction(work, {idempotencyKey?, fingerprint?})`, + which with a key keeps `work`'s JSON result (at most 16 KiB) and returns it to a retry without running `work`. +- Scheduling: `intervals: {start, end, within?, scope?: collection|owner, when?}` on a collection refuses any + write whose half-open `[start, end)` overlaps another record's in its scope with `409 interval_conflict` + (checked in the write's transaction through a partial index; `error.conflict.id` only for a record the + caller may read, so another owner's booking blocks without being named). Bounds are both numbers or both + `format: date-time` strings in UTC (`Z`, at most millisecond precision). - Stored short links: `extensions.store.config.shortLinks.: {mount, collection, destination, clicks}` over a collection with a `key`, a required `format: uri` destination property (HTTP(S) only) and one `increments` counter, plus a route `/*: {extension: store, methods: [GET, HEAD]}`; no diff --git a/packages/store/src/authoring.ts b/packages/store/src/authoring.ts index 9b51d74e..272ff9c3 100644 --- a/packages/store/src/authoring.ts +++ b/packages/store/src/authoring.ts @@ -10,6 +10,8 @@ export const storeAuthoring: ExtensionAuthoringContract = { goals: ['own', 'owner', 'owners', 'owned', 'ownership', 'their', 'mine', 'per-user', 'personal'] }, { kind: 'configuration', name: 'transitions', description: 'Declared `transitions` move one record from exact `from` values to constant `set` values (with `stamp: {property: actor|now}`) in one transaction as `POST //`; a record not in the `from` state answers `409 transition_conflict`. On an owned collection `by: others` serves the transition on its own `mount` to any principal except the owner (an approval or review step), and `members: ` admits only that list\'s members. Not an expression language.', path: 'urlcode.yaml', goals: ['approve', 'approves', 'approved', 'approval', 'approvals', 'reject', 'rejects', 'rejected', 'pending', 'submit', 'submits', 'submitted', 'transition', 'transitions', 'cancel', 'cancelled', 'publish', 'published', 'archive', 'archived', 'claim', 'workflow', 'review', 'reviews', 'reviewed'] }, + { kind: 'configuration', name: 'intervals', description: '`intervals: {start, end, within?, scope?, when?}` on a collection: no two records it applies to (those holding the `when` values, such as `{status: booked}`) in one scope with equal `within` values (a room) may hold overlapping half-open `[start, end)` intervals; any create, PUT, PATCH or transition that would answers `409 interval_conflict` and writes nothing. Checked through an index in the write transaction, across owners on an owned collection without naming another owner\'s record. Bounds are both numbers or both `format: date-time` strings in UTC (`Z`).', path: 'urlcode.yaml', + goals: ['book', 'booking', 'bookings', 'schedule', 'scheduling', 'slot', 'slots', 'appointment', 'appointments', 'reservation', 'reservations', 'reserve', 'calendar', 'overlap', 'overlapping', 'availability', 'shift', 'shifts', 'interval', 'intervals'] }, { kind: 'configuration', name: 'membership', description: 'Permissions as data keyed by the principal id, never roles in auth: a `membership: true` collection with a `key` lists principal ids and has no mount (the operator maintains it with `urlcode-store members` or `addMember`/`removeMember`). A transition\'s `members` and a collection\'s `readers.members` name it.', path: 'urlcode.yaml', goals: ['member', 'members', 'membership', 'reviewer', 'reviewers', 'approver', 'approvers', 'moderator', 'moderators', 'staff', 'team', 'admin', 'admins', 'role', 'roles', 'permission', 'permissions'] }, { kind: 'configuration', name: 'readers', description: 'An owned collection\'s `readers: {mount, members, showOwner?}` lets the members of a membership collection list (with the collection\'s limit, cursor, sort and filters) and read every owner\'s records read-only on a separate mount; owners keep their own view on the collection mount.', path: 'urlcode.yaml', diff --git a/packages/store/src/collection.ts b/packages/store/src/collection.ts index 8101965a..37f63dfe 100644 --- a/packages/store/src/collection.ts +++ b/packages/store/src/collection.ts @@ -32,6 +32,11 @@ export type PropertyType = 'string' | 'integer' | 'number' | 'boolean'; export type Scalar = string | number | boolean; const PROPERTY_TYPES: readonly PropertyType[] = ['string', 'integer', 'number', 'boolean']; const TRANSITION_LIMITS = { transitions: 16, fields: 8, stamps: 4 } as const; +const INTERVAL_LIMITS = { within: 4, when: 8 } as const; +/** A collection name, as the store's configuration schema admits it; an interval constraint embeds it in its index. */ +const COLLECTION_NAME = /^[a-z][a-z0-9_-]{0,63}$/; +/** The only date-time an interval bound takes: RFC 3339 in UTC (`Z`) with at most millisecond precision. */ +const UTC_INSTANT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/; const MOUNT = { type: 'string', pattern: '^/[A-Za-z0-9._~/-]*[A-Za-z0-9._~-]$', maxLength: 256 } as const; const FIELD_NAME = '^[a-z][A-Za-z0-9_]{0,63}$'; const SCALAR = { oneOf: [{ type: 'string', maxLength: 256 }, { type: 'number' }, { type: 'boolean' }] } as const; @@ -132,11 +137,35 @@ export interface CollectionSpec { membership?: boolean; /** On an owned collection: who may list and read every owner's records, and where. */ readers?: ReadersSpec; + /** A non-overlap constraint (#902): no two records in one scope hold overlapping half-open `[start, end)` intervals. */ + intervals?: IntervalSpec; +} +/** Where an interval constraint applies: every record of the collection, or each owner's records on their own. */ +export type IntervalScope = 'collection' | 'owner'; +/** + * A declared non-overlap constraint (#902): among the records it applies to (those holding every `when` value), no two + * in the same scope with equal `within` values may hold overlapping half-open `[start, end)` intervals. Checked inside + * every write's transaction against an index, so the check never reads the whole collection. + */ +export interface IntervalSpec { + /** A required property holding the interval's start: a UTC date-time string (`format: date-time`) or a number. */ + start: string; + /** A required property of the same kind holding its end, which must be after `start`. */ + end: string; + /** Required properties that partition the constraint (a room, a calendar): intervals conflict only when all are equal. */ + within?: string[]; + /** `collection` (default): every record blocks every other, across owners. `owner`: each owner's records only. */ + scope?: IntervalScope; + /** Only records holding exactly these values take part (for example `{status: booked}`, so a cancelled one frees its slot). */ + when?: Record; } export type StoredRecord = Record; type FieldErrors = Record; -/** What a refusal carries besides its code: query parameter names with fixed messages, or record schema issues. */ -export interface StoreErrorDetails { fields?: FieldErrors; issues?: readonly BodySchemaIssue[] } +/** + * What a refusal carries besides its code: query parameter names with fixed messages, record schema issues, or (on a + * 409 `interval_conflict`) the conflicting record's id, only when the caller may read that record. + */ +export interface StoreErrorDetails { fields?: FieldErrors; issues?: readonly BodySchemaIssue[]; conflict?: { id: string } } /** * Thrown for caller mistakes; carries names and fixed messages only, never a submitted value. A record that breaks @@ -144,8 +173,8 @@ export interface StoreErrorDetails { fields?: FieldErrors; issues?: readonly Bod * 400 `invalid_query` with `fields` naming the offending parameters. */ export class StoreError extends Error { - readonly status: number; readonly code: string; readonly fields: FieldErrors | undefined; readonly issues: readonly BodySchemaIssue[] | undefined; - constructor(status: number, code: string, message: string, details: StoreErrorDetails = {}) { super(message); this.status = status; this.code = code; this.fields = details.fields; this.issues = details.issues; } + readonly status: number; readonly code: string; readonly fields: FieldErrors | undefined; readonly issues: readonly BodySchemaIssue[] | undefined; readonly conflict: { id: string } | undefined; + constructor(status: number, code: string, message: string, details: StoreErrorDetails = {}) { super(message); this.status = status; this.code = code; this.fields = details.fields; this.issues = details.issues; this.conflict = details.conflict; } } /** The 422 for a record that breaks the collection schema (or a store rule on a named property). */ export const invalidRecord = (issues: readonly BodySchemaIssue[]): StoreError => new StoreError(422, 'invalid_record', 'Record does not match the collection schema', { issues }); @@ -210,6 +239,13 @@ export const collectionSchema = { members: { type: 'string', pattern: '^[a-z][a-z0-9_-]{0,63}$', description: 'A membership collection: anyone it does not list gets 403 membership_required before any record is read.' }, showOwner: { type: 'boolean', description: 'true: every record this mount answers carries _owner, the opaque principal id of the owner (for auth, the user id; never an email or name), so a member can tell requesters apart. Only this mount shows it: the owner\'s mount, transitions and StoreExports never do.' }, } }, + intervals: { description: 'A non-overlap constraint for scheduling: among the records it applies to, no two in the same scope (and with equal within values) may hold overlapping half-open [start, end) intervals, so an interval ending where another starts is allowed. Checked inside the write transaction of every create, PUT, PATCH and transition against an index, never a scan of the collection; an overlap answers 409 interval_conflict and nothing is written, so a move that would overlap keeps the record where it was. Activation refuses stored records that already overlap. Not on a membership collection.', type: 'object', additionalProperties: false, required: ['start', 'end'], properties: { + start: { type: 'string', pattern: FIELD_NAME, description: 'A required property holding the start: a string with format: date-time, whose values must be UTC (ending in Z) with at most millisecond precision and compare as instants, or an integer or number.' }, + end: { type: 'string', pattern: FIELD_NAME, description: 'A required property of the same kind as start holding the end; a record whose end is not after its start answers 422 invalid_record.' }, + within: { type: 'array', maxItems: INTERVAL_LIMITS.within, uniqueItems: true, items: { type: 'string', pattern: FIELD_NAME }, description: 'Required properties that partition the constraint (a room, a resource): two intervals conflict only when every one of these is equal.' }, + scope: { enum: ['collection', 'owner'], description: 'collection (default): every record blocks every other, across owners on an owned collection (another owner\'s conflicting record is never named). owner: with ownership: owner only, each owner\'s records are constrained among themselves.' }, + when: { type: 'object', minProperties: 1, maxProperties: INTERVAL_LIMITS.when, propertyNames: { pattern: FIELD_NAME }, additionalProperties: SCALAR, description: 'Only records holding exactly these values take part, for example {status: booked} so a cancelled booking frees its slot; each value must satisfy its property\'s schema. Without it every record takes part.' }, + } }, }, } as const; @@ -298,7 +334,18 @@ export interface NormalizedSpec { mount?: string; records: CompiledRecordSchema; maxRecords: number; maxRecordBytes: number; pageSize: number; readOnly: boolean; key?: string; increments: string[]; idempotency?: IdempotencySpec; sortable: string[]; filterable: string[]; ownership: Ownership; maxRecordsPerOwner?: number; audit: boolean; transitions: Record; - membership: boolean; readers?: Required; + membership: boolean; readers?: Required; intervals?: NormalizedIntervals; +} +/** + * A validated interval constraint and its SQL, built once from the declaration. `kind` says how a bound compares + * (`date-time` as UTC epoch milliseconds, `number` as itself). `create` builds the partial expression index `index` + * over the records the constraint applies to; `latest` finds, through it, the record in the same scope whose interval + * starts last before a given end; `scan` reads every constrained interval in index order (activation, operator + * commands). Every name embedded in the SQL passed FIELD_NAME or COLLECTION_NAME; every `when` value is a quoted literal. + */ +export interface NormalizedIntervals { + start: string; end: string; within: string[]; scope: IntervalScope; when: Record; kind: 'date-time' | 'number'; + index: string; create: string; latest: string; scan: string; } /** A validated transition. `by` is `any` on a shared collection, whose records have no owner to compare. */ export interface NormalizedTransition { from: Record; set: Record; stamp: Record; by: TransitionActor | 'any'; mount?: string; members?: string } @@ -391,12 +438,119 @@ export function normalize(name: string, spec: CollectionSpec, schemas: Readonly< if (readers.mount === spec.mount) throw new Error(`Collection ${name}: the readers mount must differ from the collection mount`); if (Object.values(transitions).some(transition => transition.mount === readers.mount)) throw new Error(`Collection ${name}: the readers mount must differ from every transition mount`); } + const intervals = spec.intervals === undefined ? undefined : intervalsOf(name, spec.intervals, records, ownership, membership, increments); for (const field of records.readOnly) { // A create never carries a readOnly property, so a required one is satisfiable only through its default. if (records.required.includes(field) && !hasOwn(records.defaults, field)) throw new Error(`Collection ${name}: property ${field} is required and readOnly, so it needs a default`); if (!Object.values(transitions).some(transition => hasOwn(transition.set, field) || hasOwn(transition.stamp, field))) throw new Error(`Collection ${name}: property ${field} is readOnly but no transition sets or stamps it`); } - return { ...(spec.mount === undefined ? {} : { mount: spec.mount }), records, ...(key === undefined ? {} : { key }), increments, ...(spec.idempotency === undefined ? {} : { idempotency: spec.idempotency }), sortable: queryable('sortable'), filterable: queryable('filterable'), ownership, ...(perOwner === undefined ? {} : { maxRecordsPerOwner: perOwner }), maxRecords, maxRecordBytes: spec.maxRecordBytes ?? 4096, pageSize: spec.pageSize ?? 50, readOnly: spec.readOnly ?? false, audit: spec.audit ?? false, transitions, membership, ...(readers === undefined ? {} : { readers: { mount: readers.mount, members: readers.members, showOwner: readers.showOwner === true } }) }; + return { ...(spec.mount === undefined ? {} : { mount: spec.mount }), records, ...(key === undefined ? {} : { key }), increments, ...(spec.idempotency === undefined ? {} : { idempotency: spec.idempotency }), sortable: queryable('sortable'), filterable: queryable('filterable'), ownership, ...(perOwner === undefined ? {} : { maxRecordsPerOwner: perOwner }), maxRecords, maxRecordBytes: spec.maxRecordBytes ?? 4096, pageSize: spec.pageSize ?? 50, readOnly: spec.readOnly ?? false, audit: spec.audit ?? false, transitions, membership, ...(readers === undefined ? {} : { readers: { mount: readers.mount, members: readers.members, showOwner: readers.showOwner === true } }), ...(intervals === undefined ? {} : { intervals }) }; +} + +/** + * Validates an interval constraint and builds its SQL. The bounds are both UTC date-times or both numbers, and they + * and the `within` properties are required, so every record the constraint applies to has an interval to compare. + * No property it names may be an increment, which would change an interval without the check. + */ +function intervalsOf(name: string, declared: IntervalSpec, records: CompiledRecordSchema, ownership: Ownership, membership: boolean, increments: readonly string[]): NormalizedIntervals { + const where = `Collection ${name}: intervals`; + if (!COLLECTION_NAME.test(name)) throw new Error(`${where}: the collection name is not valid`); + if (membership) throw new Error(`${where}: a membership collection takes no intervals`); + const property = (field: string): PropertySchema => { + if (typeof field !== 'string' || !new RegExp(FIELD_NAME).test(field) || !hasOwn(records.properties, field)) throw new Error(`${where}: ${String(field).slice(0, 64)} is not a declared property`); + if (increments.includes(field)) throw new Error(`${where}: ${field} is an increment property, which would change an interval without the check`); + return records.properties[field]!; + }; + const kindOf = (field: string): 'date-time' | 'number' => { + const schema = property(field); + if (!records.required.includes(field)) throw new Error(`${where}: ${field} must be required`); + if (schema.type === 'integer' || schema.type === 'number') return 'number'; + if (schema.type === 'string' && schema.format === 'date-time') return 'date-time'; + throw new Error(`${where}: ${field} must be a string with format: date-time, an integer or a number`); + }; + const { start, end } = declared, kind = kindOf(start); + if (start === end) throw new Error(`${where}: start and end must be different properties`); + if (kindOf(end) !== kind) throw new Error(`${where}: start and end must both be date-times or both be numbers`); + const within = [...declared.within ?? []]; + for (const field of within) { + property(field); + if (field === start || field === end) throw new Error(`${where}: within cannot name start or end`); + if (!records.required.includes(field)) throw new Error(`${where}: within property ${field} must be required`); + } + const scope = declared.scope ?? 'collection'; + if (scope === 'owner' && ownership !== 'owner') throw new Error(`${where}: scope: owner needs ownership: owner`); + const when: Record = {}; + for (const [field, value] of Object.entries(declared.when ?? {})) { + property(field); + const issue = propertyIssue(records, field, value); + if (issue) throw new Error(`${where}: when value for ${field} ${issue.message}`); + if (typeof value === 'string' && value.includes('\u0000')) throw new Error(`${where}: when value for ${field} cannot contain NUL`); + when[field] = value; + } + // `->>` yields SQL text for a JSON string, the number for a number and 1 or 0 for a boolean. + const path = (field: string): string => `(data ->> '$.${field}')`; + const instant = (field: string): string => kind === 'date-time' ? `CAST(round(unixepoch(${path(field)}, 'subsec') * 1000) AS INTEGER)` : path(field); + const literal = (value: Scalar): string => typeof value === 'boolean' ? (value ? '1' : '0') : typeof value === 'number' ? String(value) : `'${value.replaceAll("'", "''")}'`; + // The collection and the `when` terms are literals, so SQLite can prove that a query repeating them may use the partial index. + const filter = [`collection = '${name}'`, ...Object.entries(when).map(([field, value]) => `${path(field)} = ${literal(value)}`)].join(' AND '); + const columns = [...scope === 'owner' ? ['owner'] : [], ...within.map(path), instant(start)]; + const index = `store_intervals_${createHash('sha256').update(`${columns.join(', ')} WHERE ${filter}`).digest('hex').slice(0, 24)}`; + return { + start, end, within, scope, when, kind, index, + create: `CREATE INDEX IF NOT EXISTS "${index}" ON store_records(${columns.join(', ')}) WHERE ${filter}`, + latest: `SELECT id, owner, ${instant(end)} AS until FROM store_records WHERE ${filter}${scope === 'owner' ? ' AND owner = ?' : ''}${within.map(field => ` AND ${path(field)} = ?`).join('')} AND ${instant(start)} < ? AND id <> ? ORDER BY ${instant(start)} DESC LIMIT 1`, + scan: `SELECT id, owner, ${[...within.map((field, at) => `${path(field)} AS w${at}`), `${instant(start)} AS since`, `${instant(end)} AS until`].join(', ')} FROM store_records WHERE ${filter} ORDER BY ${columns.join(', ')}`, + }; +} +/** An interval bound as the constraint compares it: UTC epoch milliseconds for a date-time, the number itself otherwise. */ +function instantOf(kind: NormalizedIntervals['kind'], value: unknown): number | undefined { + if (kind === 'number') return typeof value === 'number' && Number.isFinite(value) ? value : undefined; + if (typeof value !== 'string' || !UTC_INSTANT.test(value)) return undefined; + const ms = Date.parse(value); + // The round trip refuses what Date.parse would roll over or reject (a 30 February, a leap second). + return Number.isFinite(ms) && new Date(ms).toISOString().slice(0, 19) === value.slice(0, 19) ? ms : undefined; +} +/** Whether the constraint applies to `record`: it holds every `when` value. */ +const constrained = (intervals: NormalizedIntervals, record: Readonly>): boolean => Object.entries(intervals.when).every(([field, value]) => record[field] === value); +/** The record-level interval rules, as schema issues: each present bound is a comparable value, and `end` is after `start`. */ +function intervalIssues(intervals: NormalizedIntervals, values: Readonly>): BodySchemaIssue[] { + const start = instantOf(intervals.kind, values[intervals.start]), end = instantOf(intervals.kind, values[intervals.end]); + const utc = 'must be a UTC date-time ending in Z, with at most millisecond precision'; + const issues: BodySchemaIssue[] = []; + // A missing bound is the schema's own `required` issue. + if (start === undefined && values[intervals.start] !== undefined) issues.push({ pointer: `/${intervals.start}`, keyword: 'intervals', message: utc }); + if (end === undefined && values[intervals.end] !== undefined) issues.push({ pointer: `/${intervals.end}`, keyword: 'intervals', message: utc }); + if (start !== undefined && end !== undefined && end <= start) issues.push({ pointer: `/${intervals.end}`, keyword: 'intervals', message: `must be after ${intervals.start}` }); + return issues; +} +/** A property value as SQLite compares it through `->>`: a boolean is 1 or 0. */ +const bound = (value: Scalar | undefined): string | number | null => value === undefined ? null : typeof value === 'boolean' ? (value ? 1 : 0) : value; +/** + * The first pair of constrained records in one scope whose intervals overlap, or `undefined`: one pass in index order + * (scope, then start), keeping the latest end seen in the current scope. Records with no owner are nobody's under + * `scope: owner` and are skipped there, as the write check never matches them. With `moving` (an operator command about + * to give `from`'s records to `to`, or with `from` null the ownerless ones), only the two principals' records are + * judged, as if the move had happened, before anything is written. + */ +export function overlapping(db: StoreDatabase, intervals: NormalizedIntervals, moving?: { from: string | null; to: string }): [string, string] | undefined { + type Row = { id: string; group: string; since: number | null; until: number | null }; + let rows: Row[] = db.all>(intervals.scan).flatMap(row => { + let owner = intervals.scope === 'owner' ? row.owner as string | null : null; + if (moving && intervals.scope === 'owner') { + if (owner !== moving.from && owner !== moving.to) return []; + owner = moving.to; + } + if (intervals.scope === 'owner' && owner === null) return []; + return [{ id: row.id as string, group: JSON.stringify([owner, ...intervals.within.map((_, at) => row[`w${at}`])]), since: row.since as number | null, until: row.until as number | null }]; + }); + // The scan is in index order already; merging two owners' records needs ordering again. + if (moving) rows = rows.sort((a, b) => a.group < b.group ? -1 : a.group > b.group ? 1 : (a.since ?? 0) - (b.since ?? 0)); + let previous: { id: string; group: string; until: number } | undefined; + for (const row of rows) { + if (previous?.group === row.group && row.since !== null && previous.until > row.since) return [previous.id, row.id]; + if (row.until !== null && (previous?.group !== row.group || row.until > previous.until)) previous = { id: row.id, group: row.group, until: row.until }; + } + return undefined; } /** @@ -487,7 +641,7 @@ function fieldsOf(record: StoredRecord): StoredRecord { /** * One collection: a view of its rows in the store database (`store_records` where `collection` is its name). Nothing * is cached in memory: every read queries the database and every write is one BEGIN IMMEDIATE transaction that reads - * what it checks (the ETag, the key, the quotas, the retained Idempotency-Keys, the audit backlog) and writes the + * what it checks (the ETag, the key, the quotas, the declared intervals, the retained Idempotency-Keys, the audit backlog) and writes the * record, the key claim and the audit event together, or rolls all of it back. Statements are synchronous, so within * this process no other request runs between a transaction's check and its write. */ @@ -512,6 +666,7 @@ export class Collection { for (const row of rows) { let record: StoredRecord; try { record = this.parse(row); } catch (error) { throw new Error(error instanceof RowError ? error.message : `Collection ${this.name}: the store holds an invalid record`, { cause: error }); } + if (this.spec.intervals && intervalIssues(this.spec.intervals, record).length) throw new Error(`Collection ${this.name}: record ${row.id} holds an interval that intervals refuses (an end not after its start, or a date-time that is not UTC with at most millisecond precision)`); const key = this.spec.key === undefined ? null : record[this.spec.key]; if (key !== null && (typeof key !== 'string' || keys.has(key))) throw new Error(`Collection ${this.name}: the store holds an invalid record key`); if (key !== null) keys.add(key); @@ -522,8 +677,17 @@ export class Collection { db.run('UPDATE store_records SET key = NULL WHERE collection = ?', this.name); for (const [id, key] of derived) if (key !== null) db.run('UPDATE store_records SET key = ? WHERE collection = ? AND id = ?', key, this.name, id); }); + const intervals = this.spec.intervals; + if (intervals) { + // The index is derived from the declaration and built once; the check below and every write's check read through it. + db.run(intervals.create); + const pair = overlapping(db, intervals); + if (pair) throw new Error(`Collection ${this.name}: records ${pair[0]} and ${pair[1]} hold overlapping intervals, which intervals refuses; move or delete one of them first`); + } this.db = db; } + /** The name of the interval index this declaration reads through, if it declares `intervals` (store.ts drops stale ones). */ + get intervalIndex(): string | undefined { return this.spec.intervals?.index; } /** Stops serving from the database; the registration closes it with its last activation. Idempotent. */ close(): void { this.db = undefined; } @@ -598,7 +762,9 @@ export class Collection { * short link's destination is `redirectable`. Returned in declaration order. */ private validated(values: Record): StoredRecord { - const issues = bodyIssues(this.spec.records.record, values); + // The interval rules judge a record the schema accepts, so a bound is already of its declared type. + const issues = [...bodyIssues(this.spec.records.record, values)]; + if (!issues.length && this.spec.intervals) issues.push(...intervalIssues(this.spec.intervals, values)); if (issues.length) throw invalidRecord(issues); const key = this.spec.key; if (this.spec.membership && key !== undefined && !principalIdPattern.test(values[key] as string)) throw invalidRecord([{ pointer: `/${key}`, keyword: 'membership', message: 'must be a principal id' }]); @@ -608,6 +774,26 @@ export class Collection { private sized(record: StoredRecord): void { if (Buffer.byteLength(JSON.stringify(record)) > this.spec.maxRecordBytes) throw new StoreError(413, 'record_too_large', `Record exceeds ${this.spec.maxRecordBytes} bytes`); } + /** + * The interval check (#902), inside the write's transaction and before its row is written: when the constraint + * applies to `record`, the record in its scope whose interval starts last before `record` ends (the record itself + * excluded, so a move is judged against the others) must end by the time `record` starts. Since the stored intervals + * of a scope never overlap, that one record is the only candidate, found by one descending step of the index. + * Otherwise 409 `interval_conflict`, naming the conflicting record only when `reader` may read it: on a shared + * collection anyone who reaches the mount may; on an owned one only its owner, so another owner's booking blocks + * the slot without being disclosed. + */ + private fits(db: StoreDatabase, record: StoredRecord, reader: string | undefined): void { + const intervals = this.spec.intervals; + if (!intervals || !constrained(intervals, record)) return; + const start = instantOf(intervals.kind, record[intervals.start])!, end = instantOf(intervals.kind, record[intervals.end])!; + const scope = intervals.scope === 'owner' ? [(record[OWNER_FIELD] as string | undefined) ?? null] : []; + const latest = db.get<{ id: string; owner: string | null; until: number | null }>(intervals.latest, ...scope, ...intervals.within.map(field => bound(record[field])), end, record.id as string); + if (latest && latest.until !== null && latest.until > start) { + const visible = !this.owned || (reader !== undefined && latest.owner === reader); + throw new StoreError(409, 'interval_conflict', 'The interval overlaps another record', visible ? { conflict: { id: latest.id } } : {}); + } + } private keyOf(record: StoredRecord): string | null { return this.spec.key === undefined ? null : record[this.spec.key] as string; } private insert(db: StoreDatabase, record: StoredRecord): void { db.run('INSERT INTO store_records(collection, id, owner, key, created_at, updated_at, data) VALUES (?, ?, ?, ?, ?, ?, ?)', @@ -903,6 +1089,7 @@ export class Collection { if (db.get<{ n: number }>('SELECT count(*) AS n FROM store_records WHERE collection = ?', this.name)!.n >= this.spec.maxRecords) throw new StoreError(409, 'collection_full', `Collection holds its maximum of ${this.spec.maxRecords} records`); const now = stamp(), record: StoredRecord = { id: randomUUID(), createdAt: now, updatedAt: now, ...(scope === undefined ? {} : { [OWNER_FIELD]: scope }), ...clean }; this.sized(record); + this.fits(db, record, scope); this.insert(db, record); return { record, audited: this.audited(db, 'created', record.id as string, this.changed(undefined, record), actor, undefined, record) }; } @@ -938,6 +1125,7 @@ export class Collection { if (this.keyTaken(db, record[this.spec.key] as string)) throw new StoreError(409, 'key_exists', 'A record already uses this key'); } this.sized(record); + this.fits(db, record, scope); this.replaceRow(db, record); return { record, audited: this.audited(db, replace ? 'replaced' : 'updated', id, this.changed(current, record), actor) }; } @@ -971,6 +1159,12 @@ export class Collection { const stamped = Object.fromEntries(Object.entries(transition.stamp).map(([field, source]) => [field, source === 'now' ? updatedAt : actor ?? 'anonymous'])); const record: StoredRecord = { ...current, updatedAt, ...transition.set, ...stamped }; this.sized(record); + // A transition can bring a record under the constraint (a `reopen` back to `when`), so it is checked like any write. + if (this.spec.intervals) { + const issues = intervalIssues(this.spec.intervals, record); + if (issues.length) throw invalidRecord(issues); + this.fits(db, record, caller); + } this.replaceRow(db, record); return { record, audited: this.audited(db, 'transitioned', id, this.changed(current, record), actor, name) }; } diff --git a/packages/store/src/database.ts b/packages/store/src/database.ts index ca452edd..0d50f016 100644 --- a/packages/store/src/database.ts +++ b/packages/store/src/database.ts @@ -1,6 +1,6 @@ -// The store's one SQLite database per site (#835): every collection's records, retained Idempotency-Key claims and -// undelivered audit events are rows in three shared tables, so a record write, its claim and its audit event commit -// in one transaction. Direct parameterized SQL through node:sqlite; no query builder. The schema only ever moves +// The store's one SQLite database per site (#835): every collection's records, retained Idempotency-Key claims +// (of HTTP writes and of host transactions) and undelivered audit events are rows in shared tables, so a record +// write, its claim and its audit event commit in one transaction. Direct parameterized SQL through node:sqlite; no query builder. The schema only ever moves // forward: an empty file is initialized to the newest version, an older store schema is upgraded step by step in // one transaction each, and a newer or foreign one is refused before anything is served. import { lstat, mkdir, open, realpath } from 'node:fs/promises'; @@ -42,6 +42,11 @@ const MIGRATIONS: readonly string[] = [ // milliseconds of its last ack or empty peek. An operator command reads it to warn that the events it just wrote // wait for a drain that is not running. A database no drain has touched has no row. `CREATE TABLE store_audit_drain(id INTEGER PRIMARY KEY CHECK (id = 1), drained_at INTEGER NOT NULL);`, + // 3 -> 4. Retries for host transactions (#902): one row per retained `StoreExports.transaction` idempotency key, the + // SHA-256 of the key and of the caller's fingerprint, and the transaction's JSON result (NULL for `undefined`), + // written in the transaction it records. Keys are store-wide, not per collection: a transaction spans collections. + `CREATE TABLE store_transaction_results(seq INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT NOT NULL UNIQUE, + fingerprint TEXT NOT NULL, result TEXT CHECK (result IS NULL OR json_valid(result)), claimed_at INTEGER NOT NULL);`, ]; export const STORE_SCHEMA_VERSION = MIGRATIONS.length; /** diff --git a/packages/store/src/index.ts b/packages/store/src/index.ts index ef1a8430..0c679c57 100644 --- a/packages/store/src/index.ts +++ b/packages/store/src/index.ts @@ -4,11 +4,12 @@ export type { StoreExtensionOptions } from './store.ts'; export { STORE_APPLICATION_ID, STORE_DURABILITIES, STORE_SCHEMA_VERSION } from './database.ts'; export type { StoreDurability } from './database.ts'; export { AUDIT_BACKLOG, Collection, StoreError, collectionSchema, compileRecordSchema, normalize, LIMITS, OWNER_FIELD, RESERVED_FIELDS } from './collection.ts'; -export type { CollectionAuditor, CollectionSpec, Ownership, Page, PropertySchema, PropertyType, ReadersSpec, RecordSchema, Scalar, Shown, StoredRecord, TransitionSpec, Viewer } from './collection.ts'; +export type { CollectionAuditor, CollectionSpec, IntervalScope, IntervalSpec, Ownership, Page, PropertySchema, PropertyType, ReadersSpec, RecordSchema, Scalar, Shown, StoredRecord, TransitionSpec, Viewer } from './collection.ts'; export { assignOwnerless, deleteOwnerless, reassignOwner, reportOwnerless } from './ownership.ts'; export { backupStore } from './backup.ts'; export type { StoreBackupOptions, StoreBackupResult } from './backup.ts'; export type { OwnerlessReport, ReassignCollectionReport, ReassignMembershipReport, ReassignOptions, ReassignReport } from './ownership.ts'; -export type { StoreExports, StoreListResult, StorePrincipal, StoreRecordResult, StoreRecords } from './records.ts'; +export { TRANSACTION_RETRIES } from './records.ts'; +export type { StoreExports, StoreListResult, StorePrincipal, StoreRecordResult, StoreRecords, StoreTransaction, StoreTransactionOptions, StoreTransactionRecords } from './records.ts'; export { addMember, listMembers, removeMember } from './membership.ts'; export type { MemberOptions, MemberReport, MembershipOptions } from './membership.ts'; diff --git a/packages/store/src/openapi.ts b/packages/store/src/openapi.ts index 2f43ee54..d4194897 100644 --- a/packages/store/src/openapi.ts +++ b/packages/store/src/openapi.ts @@ -14,7 +14,7 @@ const ref = (name: string): Json => ({ $ref: `#/components/schemas/${name}` }); const json = (schema: Json): Json => ({ 'application/json': { schema } }); const ID_PATTERN = '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; -/** The store's error envelope: `fields` on a 400 invalid_query, `issues` (core's body-validation issues) on a 422 invalid_record or a 409 increment_limit. */ +/** The store's error envelope: `fields` on a 400 invalid_query, `issues` (core's body-validation issues) on a 422 invalid_record or a 409 increment_limit, `conflict` on a 409 interval_conflict. */ const errorSchema: Json = { description: 'Every error the store writes: a fixed code and message, and never a submitted value.', type: 'object', required: ['error'], additionalProperties: false, @@ -22,6 +22,7 @@ const errorSchema: Json = { code: { type: 'string' }, message: { type: 'string' }, fields: { type: 'object', additionalProperties: { type: 'string' }, description: 'invalid_query: the offending query parameters, each with a fixed message.' }, issues: { type: 'array', items: ref('UrlcodeBodyValidationIssue'), description: 'invalid_record and increment_limit: the schema issues, in the shape a body-schema route answers.' }, + conflict: { type: 'object', required: ['id'], additionalProperties: false, properties: { id: { type: 'string', pattern: ID_PATTERN } }, description: 'interval_conflict: the record whose interval overlaps, only when the caller may read it (on an owned collection, only the caller\'s own record; another owner\'s blocks the slot without being named).' }, } } }, }; const failure = (description: string): Json => ({ description, content: json(ref('StoreError')) }); @@ -124,11 +125,12 @@ function collectionPaths(mount: string, name: string, spec: NormalizedSpec, name ...(conflict ? { '409': failure(conflict) } : {}), '413': failure('record_too_large: the body or the resulting record exceeds maxRecordBytes.'), '415': failure('Send Content-Type: application/json.'), - '422': failure(`invalid_record: the record does not satisfy the collection schema${spec.idempotency ? '; or idempotency_key_reused' : ''}.`), + '422': failure(`invalid_record: the record does not satisfy the collection schema${spec.intervals ? ' or its intervals (a bound that is not a UTC date-time, or an end not after its start)' : ''}${spec.idempotency ? '; or idempotency_key_reused' : ''}.`), '503': unavailable, }); const listed = { parameters: listParameters(spec), responses: { '200': { description: 'One page of the caller\'s records.', content: json(ref(names.list)) }, '400': failure('invalid_query: an undeclared, repeated or invalid list parameter.'), '503': unavailable } }; - const conflicts = [spec.key ? 'key_exists' : '', 'collection_full', spec.maxRecordsPerOwner ? 'owner_quota_exceeded' : ''].filter(Boolean).join(', '); + const overlap = spec.intervals ? 'interval_conflict (the interval overlaps another record\'s; see error.conflict)' : ''; + const conflicts = [spec.key ? 'key_exists' : '', 'collection_full', spec.maxRecordsPerOwner ? 'owner_quota_exceeded' : '', overlap].filter(Boolean).join(', '); const paths: Record = { [mount]: { summary: `The ${name} collection`, @@ -145,7 +147,7 @@ function collectionPaths(mount: string, name: string, spec: NormalizedSpec, name if (!spec.readOnly) { const update = (kind: string, schema: string): Json => ({ summary: `${kind} a ${name} record`, parameters: [ifMatch, ...retry], requestBody: body(schema), - responses: { ...recordAnswer('200', 'Updated.'), '404': notFound, '412': failure('precondition_failed: the record changed since that ETag.'), ...writeErrors(spec.key ? 'key_exists' : undefined) }, + responses: { ...recordAnswer('200', 'Updated.'), '404': notFound, '412': failure('precondition_failed: the record changed since that ETag.'), ...writeErrors([spec.key ? 'key_exists' : '', overlap].filter(Boolean).join(', ') || undefined) }, }); item.put = update('Replace', names.create); item.patch = update('Update', names.patch); @@ -176,7 +178,7 @@ function transitionOperation(name: string, transition: string, declared: Normali ...recordAnswer('200', 'Transitioned.'), '400': badRequest, '403': failure(['forbidden_origin: a cross-origin write', ...declared.members ? ['membership_required: the caller is not a member'] : [], ...declared.by === 'others' ? ['own_record_refused: the caller owns the record'] : []].join('; ') + '.'), - '404': notFound, '409': failure('transition_conflict: the record is not in the from state.'), '412': failure('precondition_failed: the record changed since that ETag.'), + '404': notFound, '409': failure(`transition_conflict: the record is not in the from state${spec.intervals ? '; or interval_conflict: the record as the transition leaves it would overlap another record\'s interval (see error.conflict)' : ''}.`), '412': failure('precondition_failed: the record changed since that ETag.'), ...(spec.idempotency ? { '422': failure('idempotency_key_reused.') } : {}), '503': unavailable, }, }; diff --git a/packages/store/src/ownership.ts b/packages/store/src/ownership.ts index ef247442..320f431b 100644 --- a/packages/store/src/ownership.ts +++ b/packages/store/src/ownership.ts @@ -28,7 +28,7 @@ import { isAbsolute } from 'node:path'; import type { AuditEvent } from '@jimhoyd/urlcode-audit'; import { principalIdPattern } from '@jimhoyd/urlcode/extensions'; -import { AUDIT_BACKLOG, StoreError, membershipEvent, normalize, stamp, writeAuditEvent } from './collection.ts'; +import { AUDIT_BACKLOG, StoreError, membershipEvent, normalize, overlapping, stamp, writeAuditEvent } from './collection.ts'; import type { CollectionSpec, NormalizedSpec } from './collection.ts'; import { auditValidator, operatorActor } from './membership.ts'; import { auditDelivery, openStoreDatabase } from './database.ts'; @@ -61,6 +61,16 @@ const ownerlessIds = (db: StoreDatabase, collection: string): string[] => db.all const ownedIds = (db: StoreDatabase, collection: string, owner: string): string[] => db.all<{ id: string }>('SELECT id FROM store_records WHERE collection = ? AND owner = ? ORDER BY seq', collection, owner).map(row => row.id); type Validate = (value: unknown) => AuditEvent; +/** + * Refuses, before anything is written, giving `from`'s records (the ownerless ones when `from` is null) to `to` on a + * collection whose `intervals` constrain each owner's records (`scope: owner`) when that would leave `to` holding two + * overlapping intervals (#902). A `scope: collection` constraint does not depend on the owner, so a move keeps it. + */ +function refuseOverlap(db: StoreDatabase, collection: string, spec: NormalizedSpec, from: string | null, to: string): void { + if (spec.intervals?.scope !== 'owner') return; + const pair = overlapping(db, spec.intervals, { from, to }); + if (pair) throw new Error(`Nothing was moved: collection ${collection} would give ${to} records ${pair[0]} and ${pair[1]}, whose intervals overlap, which its intervals refuse. Move or delete one of them first.`); +} /** * Refuses, before anything is written, a change that would record `planned` events in a collection past its backlog: * the same 503 `audit_backlog` a write at the cap answers, by the same count `writeAuditEvent` checks per event. @@ -78,13 +88,13 @@ const reassignedEvent = (collection: string, id: string, from: string | undefine ({ action: 'store.record.reassigned', actor, subject: `${collection}/${id}`, metadata: { collection, ...(from === undefined ? {} : { from }), to } }); /** The declared `ownership: owner` collection an ownerless command names, and audit's validator when it is audited. */ -async function ownedCollection(options: OwnerlessOptions): Promise<{ collection: string; validate: Validate | undefined; actor: string }> { +async function ownedCollection(options: OwnerlessOptions): Promise<{ collection: string; spec: NormalizedSpec; validate: Validate | undefined; actor: string }> { if (!options?.collections || typeof options.collections !== 'object') throw new Error('The project declares no store collections'); const collection = named(options.collection), actor = operatorActor(options.actor); if (!Object.hasOwn(options.collections, collection)) throw new Error(`Collection ${collection} is not declared`); const spec = normalize(collection, options.collections[collection]!, options.schemas); if (spec.ownership !== 'owner') throw new Error(`Collection ${collection} is not declared with ownership: owner`); - return { collection, validate: spec.audit ? await auditValidator(collection) : undefined, actor }; + return { collection, spec, validate: spec.audit ? await auditValidator(collection) : undefined, actor }; } /** Counts (and lists the ids of) a collection's records that carry no owner. Changes nothing. */ @@ -99,10 +109,11 @@ export async function reportOwnerless(database: string, collection: string): Pro export async function assignOwnerless(database: string, options: OwnerlessOptions & { owner: string }): Promise { const owner = options?.owner; if (typeof owner !== 'string' || !principalIdPattern.test(owner)) throw new Error('Owner must be a principal id: 1 to 128 ASCII letters, digits, ".", "_", ":" or "-", starting with a letter or digit'); - const { collection, validate, actor } = await ownedCollection(options); + const { collection, spec, validate, actor } = await ownedCollection(options); return transaction(database, db => { const ids = ownerlessIds(db, collection); if (validate) assertBacklog(db, new Map([[collection, ids.length]])); + refuseOverlap(db, collection, spec, null, owner); db.run('UPDATE store_records SET owner = ? WHERE collection = ? AND owner IS NULL', owner, collection); if (validate) for (const id of ids) writeAuditEvent(db, collection, validate, reassignedEvent(collection, id, undefined, owner, actor)); return { collection, records: total(db, collection), ownerless: 0, ids, ...(validate ? auditDelivery(db, [collection], Date.now()) : {}) }; @@ -188,6 +199,7 @@ export async function reassignOwner(database: string, options: ReassignOptions): return { collection: name, moved, toBefore, toAfter: toBefore + moved, maxRecordsPerOwner: spec.maxRecordsPerOwner ?? null }; }); const over = collections.filter(report => report.moved > 0 && report.maxRecordsPerOwner !== null && report.toAfter > report.maxRecordsPerOwner); + for (const report of collections) if (report.moved > 0) refuseOverlap(db, report.collection, specOf(report.collection), from, to); if (over.length) throw new Error(`Nothing was moved: ${over.map(report => `collection ${report.collection} would give ${to} ${report.toAfter} records, over its maxRecordsPerOwner of ${report.maxRecordsPerOwner}`).join('; ')}. Delete or reassign some of its records first, or raise the limit.`); // Every audit event the move records, per collection, is counted before anything is written. const planned = new Map(); diff --git a/packages/store/src/records.ts b/packages/store/src/records.ts index d915ab2b..d5835e6a 100644 --- a/packages/store/src/records.ts +++ b/packages/store/src/records.ts @@ -9,6 +9,8 @@ * `transaction(work)` (#835) runs several of those operations, across the declared collections, as one database * transaction: trusted host code only (never sandboxed, never a route function), synchronous, one database. */ +import { createHash } from 'node:crypto'; +import { isDeepStrictEqual } from 'node:util'; import type { ExtensionPrincipal } from '@jimhoyd/urlcode/extensions'; import { OWNER_FIELD, StoreError, etagOf, storageFailure } from './collection.ts'; import type { Collection, Ownership, RecordSchema, Scalar, Step, StoredRecord } from './collection.ts'; @@ -29,9 +31,31 @@ export interface StoreExports { * and rolled back, and `tx` refuses every call once `work` has returned. Transactions do not nest, and calling * `records()` inside `work` is refused because it would open a second one. Trusted host code only: this is not a * sandbox, `work` runs with full Node access, and the principals it passes are taken as given. + * + * With `options.idempotencyKey` (#902) the transaction is retry-safe: the key's claim is read under the write lock, + * so of racing calls with one key exactly one runs `work`. Its return value, which must then be a JSON value (or + * `undefined`) of at most `TRANSACTION_RETRIES.resultBytes` bytes serialized, is kept with the claim in the same + * transaction, and a later call with the key and the same `fingerprint` returns a copy of it without running `work` + * or writing anything. A different fingerprint is a 422 `idempotency_key_reused` `StoreError`. A transaction that + * throws keeps nothing, so its retry runs again. Keys are store-wide: prefix them with the caller's own scope. */ - transaction(work: (tx: StoreTransaction) => T): T; + transaction(work: (tx: StoreTransaction) => T, options?: StoreTransactionOptions): T; } +/** Makes a host transaction retry-safe (`StoreExports.transaction`). */ +export interface StoreTransactionOptions { + /** The caller's key for this logical operation, 1 to `TRANSACTION_RETRIES.keyLength` characters; stored only as a hash. */ + idempotencyKey: string; + /** + * What the operation is (for example the canonical request it serves), at most `TRANSACTION_RETRIES.fingerprintLength` + * characters; stored only as a hash. A retry with the key must carry the same one (absent counts as the empty string). + */ + fingerprint?: string; +} +/** + * Host transaction retry bounds: the key and fingerprint lengths a caller may pass, the largest result kept (its JSON + * in bytes), and how many keys the store retains, newest first; an evicted key's retry runs again. + */ +export const TRANSACTION_RETRIES = { keyLength: 256, fingerprintLength: 4096, resultBytes: 16_384, keys: 1_000 } as const; /** The operations of one host transaction (`StoreExports.transaction`). */ export interface StoreTransaction { /** One declared collection, inside this transaction. Throws an `Error` for an undeclared one. */ @@ -114,13 +138,39 @@ function pageOf(collection: Collection, options: { limit?: number; cursor?: stri }); } +const sha256 = (text: string): string => createHash('sha256').update(text).digest('hex'); +/** The hashed key and fingerprint of an idempotent host transaction, or `undefined` without a key. Misuse is a TypeError. */ +function retryOf(options: StoreTransactionOptions | undefined): { key: string; fingerprint: string } | undefined { + if (options === undefined) return undefined; + if (options === null || typeof options !== 'object') throw new TypeError('transaction options must be an object'); + const { idempotencyKey, fingerprint = '' } = options; + if (idempotencyKey === undefined) return undefined; + if (typeof idempotencyKey !== 'string' || idempotencyKey.length < 1 || idempotencyKey.length > TRANSACTION_RETRIES.keyLength) throw new TypeError(`idempotencyKey must be a string of 1 to ${TRANSACTION_RETRIES.keyLength} characters`); + if (typeof fingerprint !== 'string' || fingerprint.length > TRANSACTION_RETRIES.fingerprintLength) throw new TypeError(`fingerprint must be a string of at most ${TRANSACTION_RETRIES.fingerprintLength} characters`); + return { key: sha256(idempotencyKey), fingerprint: sha256(fingerprint) }; +} +/** + * An idempotent transaction's result as it is kept: its JSON (null for `undefined`). A value JSON would change (a + * Date, a Map, NaN, a class instance) or one over the size bound throws, which rolls the transaction back, so a + * replay always returns exactly what the first call returned. + */ +function kept(value: unknown): string | null { + if (value === undefined) return null; + let text: string | undefined; + try { text = JSON.stringify(value); } catch { text = undefined; } + if (text === undefined || !isDeepStrictEqual(JSON.parse(text), value)) throw new TypeError('An idempotent store transaction must return a JSON value or undefined: its result is kept for replay, so nothing it did was committed'); + if (Buffer.byteLength(text) > TRANSACTION_RETRIES.resultBytes) throw new RangeError(`An idempotent store transaction's result must serialize to at most ${TRANSACTION_RETRIES.resultBytes} bytes of JSON, so nothing it did was committed; return ids rather than records`); + return text; +} + /** * One host transaction over the activation's collections. Every operation is a write step from collection.ts run on * the one open database; audited collections are woken after the commit. `open` turns false when `work` returns, so a * handle kept past the transaction (for example across an `await`) refuses instead of writing outside it. */ -function runTransaction(byName: Map, work: (tx: StoreTransaction) => T): T { +function runTransaction(byName: Map, work: (tx: StoreTransaction) => T, options?: StoreTransactionOptions): T { if (typeof work !== 'function') throw new TypeError('transaction needs a synchronous function'); + const retry = retryOf(options); const first = byName.values().next().value as Collection | undefined; if (!first) throw new Error('store declares no collections'); const db: StoreDatabase = first.database(); @@ -153,11 +203,25 @@ function runTransaction(byName: Map, work: (tx: StoreTran let value: T; try { value = db.transaction(() => { + if (retry) { + // Read under the write lock: of racing calls with one key, exactly one gets past here without a claim. + const claimed = db.get<{ fingerprint: string; result: string | null }>('SELECT fingerprint, result FROM store_transaction_results WHERE key = ?', retry.key); + if (claimed) { + open = false; + if (claimed.fingerprint !== retry.fingerprint) throw new StoreError(422, 'idempotency_key_reused', 'This idempotency key was already used for a different operation'); + return (claimed.result === null ? undefined : JSON.parse(claimed.result)) as T; + } + } + let returned: T; try { - const returned = work(tx); + returned = work(tx); if (returned !== null && typeof returned === 'object' && typeof (returned as { then?: unknown }).then === 'function') throw new TypeError('A store transaction function must be synchronous: it returned a promise, so nothing it did was committed'); - return returned; } finally { open = false; } + if (retry) { + db.run('INSERT INTO store_transaction_results(key, fingerprint, result, claimed_at) VALUES (?, ?, ?, ?)', retry.key, retry.fingerprint, kept(returned), Date.now()); + db.run('DELETE FROM store_transaction_results WHERE seq <= (SELECT seq FROM store_transaction_results ORDER BY seq DESC LIMIT 1 OFFSET ?)', TRANSACTION_RETRIES.keys); + } + return returned; }); } catch (error) { return storageFailure(error, true); } for (const collection of audited) collection.notifyAudit(); @@ -201,9 +265,9 @@ export function storeExports(): { exports: StoreExports; attach(collections: rea if (!found) throw new Error(`store declares no collection ${String(name).slice(0, 64)}`); return found; }, - transaction(work: (tx: StoreTransaction) => T): T { + transaction(work: (tx: StoreTransaction) => T, options?: StoreTransactionOptions): T { if (!current) throw new Error('store is not active yet: run transactions from activate or a request, not from host()'); - return runTransaction(current.collections, work); + return runTransaction(current.collections, work, options); }, }); return { diff --git a/packages/store/src/store.ts b/packages/store/src/store.ts index 6d074fd4..375b1504 100644 --- a/packages/store/src/store.ts +++ b/packages/store/src/store.ts @@ -42,7 +42,7 @@ const NAME = /^[a-z][a-z0-9_-]{0,63}$/; const FIELD = /^[a-z][A-Za-z0-9_]{0,63}$/; const json = (status: number, value: unknown, extra: [string, string][] = []): HandlerResult => jsonResponse(status, value, extra); const failure = (error: StoreError, extra: [string, string][] = []): HandlerResult => - json(error.status, { error: { code: error.code, message: error.message, ...(error.fields ? { fields: error.fields } : {}), ...(error.issues ? { issues: error.issues } : {}) } }, extra); + json(error.status, { error: { code: error.code, message: error.message, ...(error.fields ? { fields: error.fields } : {}), ...(error.issues ? { issues: error.issues } : {}), ...(error.conflict ? { conflict: error.conflict } : {}) } }, extra); /** What a caller sees of a record: everything but the stored owner, which only a readers mount with `showOwner` shows. */ const view = (record: StoredRecord): StoredRecord => { if (!Object.hasOwn(record, OWNER_FIELD)) return record; const { [OWNER_FIELD]: _owner, ...rest } = record; return rest; }; /** @@ -117,6 +117,8 @@ export function createStore(options: StoreExtensionOptions): { registration: Run const shared = storeExports(), audit = options.audit; // The live activations, oldest first. The newest is the one being served; the producer drains only while one is live. const live: symbol[] = []; + // The interval indexes each live activation reads through (#902), so a reload drops only indexes nobody declares. + const intervalIndexes = new Map(); // The one connection and how many live activations hold it. let connection: Connection | undefined; const acquire = async (): Promise<{ db: StoreDatabase; release(): Promise }> => { @@ -246,6 +248,8 @@ export function createStore(options: StoreExtensionOptions): { registration: Run } catch (error) { await held.release(); throw error; } const exported = shared.attach(collections); live.push(exported); + intervalIndexes.set(exported, collections.flatMap(collection => collection.intervalIndex ?? [])); + dropStaleIntervalIndexes(held.db, new Set([...intervalIndexes.values()].flat())); // Events a previous run left in the outbox drain now rather than at the next write or poll. if (pending > 0) attachment?.notify(); let closed = false; @@ -255,6 +259,7 @@ export function createStore(options: StoreExtensionOptions): { registration: Run if (closed) return; closed = true; shared.detach(exported); + intervalIndexes.delete(exported); const index = live.indexOf(exported); if (index >= 0) live.splice(index, 1); for (const collection of collections) collection.close(); @@ -266,6 +271,17 @@ export function createStore(options: StoreExtensionOptions): { registration: Run return { registration, exports: shared.exports, durability, close: async () => { await attachment?.close(); } }; } +/** + * Drops the interval indexes (#902) that no live activation declares: a changed or removed `intervals` would otherwise + * leave an index every write keeps paying for. Housekeeping only: an index is never needed for a check to be correct, + * so a lock another process holds just leaves the drop to the next activation. + */ +function dropStaleIntervalIndexes(db: StoreDatabase, wanted: ReadonlySet): void { + try { + for (const { name } of db.all<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'index' AND name GLOB 'store_intervals_*'")) + if (!wanted.has(name) && /^store_intervals_[0-9a-f]{24}$/.test(name)) db.run(`DROP INDEX IF EXISTS "${name}"`); + } catch { /* Retried by the next activation. */ } +} /** The registration's one connection while any activation holds it. */ interface Connection { readonly opening: Promise; db?: StoreDatabase; refs: number } function opener(database: string, durability: StoreDurability): Connection { diff --git a/packages/store/test/intervals.test.ts b/packages/store/test/intervals.test.ts new file mode 100644 index 00000000..2ccd84e9 --- /dev/null +++ b/packages/store/test/intervals.test.ts @@ -0,0 +1,228 @@ +// The declarative non-overlap constraint (#902 item 1): `intervals` refuses a write whose half-open [start, end) +// interval overlaps another record's in its scope, inside the write's transaction and through an index. Proved for +// adjacency, moves that keep their old slot, transitions (a cancelled booking frees its slot), owned-collection +// privacy (another owner's booking blocks without being named), rollback, races in one process and across +// connections, activation over stored overlaps, the operator's reassign, and the UTC date-time rule. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { DatabaseSync } from 'node:sqlite'; +import { createStore, normalize, reassignOwner } from '../src/index.ts'; +import type { CollectionSpec } from '../src/index.ts'; +import { describeStore } from '../src/openapi.ts'; +import { direct, pin, race } from './direct.ts'; +import { counts, execute, records, seed } from './rows.ts'; + +const hour = (h: number, suffix = ':00:00Z') => `2026-10-01T${String(h).padStart(2, '0')}${suffix}`; +const rooms = { + mount: '/api/rooms', idempotency: { maxKeys: 100 }, + schema: { type: 'object', additionalProperties: false, required: ['room', 'start', 'end'], properties: { room: { type: 'string', maxLength: 20 }, start: { type: 'string', format: 'date-time', maxLength: 40 }, end: { type: 'string', format: 'date-time', maxLength: 40 }, status: { type: 'string', enum: ['booked', 'cancelled'] }, note: { type: 'string', maxLength: 40 } } }, + defaults: { status: 'booked' }, readOnlyProperties: ['status'], + intervals: { start: 'start', end: 'end', within: ['room'], when: { status: 'booked' } }, + transitions: { cancel: { from: { status: 'booked' }, set: { status: 'cancelled' } }, reopen: { from: { status: 'cancelled' }, set: { status: 'booked' } } }, +}; +const slots = { type: 'object', additionalProperties: false, required: ['desk', 'start', 'end'], properties: { desk: { type: 'string', maxLength: 20 }, start: { type: 'integer', minimum: 0 }, end: { type: 'integer', minimum: 1 } } }; +const desks = { mount: '/api/desks', ownership: 'owner', schema: slots, intervals: { start: 'start', end: 'end', within: ['desk'] } }; +const diaries = { mount: '/api/diaries', ownership: 'owner', schema: slots, intervals: { start: 'start', end: 'end', scope: 'owner' } }; +const config = { collections: { rooms, desks, diaries } }; +const mounts = ['/api/rooms', '/api/desks', '/api/diaries']; +const code = (answer: { body: Record | undefined }) => (answer.body?.error as { code?: string } | undefined)?.code; + +test('adjacent intervals are allowed, overlapping ones are 409 interval_conflict naming the record, other rooms are independent', async t => { + const store = await direct(t, config, { mounts, principalMounts: ['/api/desks', '/api/diaries'] }); + const nine = await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(9), end: hour(10) } }); + assert.equal(nine.status, 201); + // Half-open: [9, 10) and [10, 11) share only the instant 10:00, which belongs to the second. The same instant + // written with milliseconds compares equal. + assert.equal((await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(10, ':00:00.000Z'), end: hour(11) } })).status, 201); + assert.equal((await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(8), end: hour(9) } })).status, 201); + const before = counts(store.database); + for (const [start, end] of [[hour(9, ':30:00Z'), hour(9, ':45:00Z')], [hour(7), hour(12)], [hour(9, ':59:59.999Z'), hour(10, ':30:00Z')], [hour(8, ':00:00.001Z'), hour(8, ':00:00.002Z')]] as const) { + const refused = await store.call('POST', '/api/rooms', { body: { room: 'a', start, end }, headers: { 'idempotency-key': `k-${start}` } }); + assert.equal(refused.status, 409, `${start} to ${end}`); assert.equal(code(refused), 'interval_conflict'); + assert.equal(typeof (refused.body!.error as { conflict?: { id?: string } }).conflict?.id, 'string', 'a shared collection names the conflicting record'); + } + const named = await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(9, ':15:00Z'), end: hour(9, ':20:00Z') } }); + assert.deepEqual((named.body!.error as { conflict: unknown }).conflict, { id: nine.body!.id }); + assert.deepEqual(counts(store.database), before, 'a refusal writes nothing, not even an Idempotency-Key claim'); + assert.equal((await store.call('POST', '/api/rooms', { body: { room: 'b', start: hour(9), end: hour(10) } })).status, 201, 'another room is another scope'); +}); + +test('bounds are UTC date-times with at most millisecond precision, and end must be after start (422)', async t => { + const store = await direct(t, config, { mounts, principalMounts: ['/api/desks', '/api/diaries'] }); + const issues = async (start: string, end: string) => { const answer = await store.call('POST', '/api/rooms', { body: { room: 'a', start, end } }); assert.equal(answer.status, 422, `${start} ${end}`); return (answer.body!.error as { issues: { pointer: string; keyword: string }[] }).issues.map(issue => `${issue.pointer} ${issue.keyword}`); }; + assert.deepEqual(await issues(hour(10), hour(10)), ['/end intervals']); + assert.deepEqual(await issues(hour(11), hour(10)), ['/end intervals']); + assert.deepEqual(await issues('2026-10-01T10:00:00+01:00', hour(12)), ['/start intervals'], 'an offset is refused: compare in UTC, write Z'); + assert.deepEqual(await issues(hour(9), '2026-10-01T10:00:00.0001Z'), ['/end intervals'], 'sub-millisecond precision is refused'); + assert.deepEqual(await issues(hour(9), '2026-10-01t10:00:00z'), ['/end intervals'], 'lower-case t and z are refused'); + assert.equal(records(store.database, 'rooms').length, 0); + const numeric = await store.call('POST', '/api/desks', { who: 'ann', body: { desk: 'd', start: 5, end: 5 } }); + assert.equal(numeric.status, 422); assert.equal(code(numeric), 'invalid_record'); +}); + +test('a move that would overlap is refused and keeps its old slot; a move over its own old slot is allowed', async t => { + const store = await direct(t, config, { mounts, principalMounts: ['/api/desks', '/api/diaries'] }); + const nine = await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(9), end: hour(10) } }); + const ten = await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(10), end: hour(11) } }); + const id = ten.body!.id as string; + for (const [method, body] of [['PATCH', { start: hour(9, ':30:00Z') }], ['PUT', { room: 'a', start: hour(9, ':30:00Z'), end: hour(10, ':30:00Z') }]] as const) { + const moved = await store.call(method, `/api/rooms/${id}`, { body, headers: { 'if-match': ten.header('etag')! } }); + assert.equal(moved.status, 409, method); assert.equal(code(moved), 'interval_conflict'); + const kept = await store.call('GET', `/api/rooms/${id}`); + assert.deepEqual([kept.body!.start, kept.body!.end, kept.header('etag')], [hour(10), hour(11), ten.header('etag')], 'the record keeps its old slot and revision'); + } + // Overlapping its own current interval is not a conflict: the record itself is excluded. + const shifted = await store.call('PATCH', `/api/rooms/${id}`, { body: { start: hour(10, ':30:00Z'), end: hour(11, ':30:00Z') }, headers: { 'if-match': ten.header('etag')! } }); + assert.equal(shifted.status, 200); + // A change that leaves the interval alone still passes through the check and is accepted. + assert.equal((await store.call('PATCH', `/api/rooms/${nine.body!.id as string}`, { body: { note: 'standup' } })).status, 200); + // The slot the move left is free again. + assert.equal((await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(10), end: hour(10, ':30:00Z') } })).status, 201); +}); + +test('only records holding the when values take part: cancelling frees the slot, and reopening into a taken slot is refused', async t => { + const store = await direct(t, config, { mounts, principalMounts: ['/api/desks', '/api/diaries'] }); + const first = await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(9), end: hour(10) } }); + const id = first.body!.id as string; + assert.equal((await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(9), end: hour(10) } })).status, 409); + assert.equal((await store.call('POST', `/api/rooms/${id}/cancel`)).status, 200); + const second = await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(9), end: hour(10) } }); + assert.equal(second.status, 201, 'a cancelled booking no longer holds its slot'); + const reopened = await store.call('POST', `/api/rooms/${id}/reopen`); + assert.equal(reopened.status, 409); assert.equal(code(reopened), 'interval_conflict'); + assert.deepEqual((reopened.body!.error as { conflict: unknown }).conflict, { id: second.body!.id }); + assert.equal((await store.call('GET', `/api/rooms/${id}`)).body!.status, 'cancelled', 'the refused transition changed nothing'); + assert.equal((await store.call('POST', `/api/rooms/${second.body!.id as string}/cancel`)).status, 200); + assert.equal((await store.call('POST', `/api/rooms/${id}/reopen`)).status, 200, 'once the slot is free again the transition applies'); +}); + +test('owned collections: another owner\'s booking blocks the slot without being named; scope: owner constrains each owner alone', async t => { + const store = await direct(t, config, { mounts, principalMounts: ['/api/desks', '/api/diaries'] }); + const ann = await store.call('POST', '/api/desks', { who: 'ann', body: { desk: 'd1', start: 10, end: 20 } }); + const bob = await store.call('POST', '/api/desks', { who: 'bob', body: { desk: 'd1', start: 15, end: 25 } }); + assert.equal(bob.status, 409); assert.equal(code(bob), 'interval_conflict'); + assert.equal((bob.body!.error as { conflict?: unknown }).conflict, undefined, 'bob cannot read ann\'s booking, so it is not named'); + assert.ok(!JSON.stringify(bob.body).includes(ann.body!.id as string)); + const own = await store.call('POST', '/api/desks', { who: 'ann', body: { desk: 'd1', start: 15, end: 25 } }); + assert.deepEqual((own.body!.error as { conflict: unknown }).conflict, { id: ann.body!.id }, 'ann may read her own booking, so it is named'); + // Through a host transaction the same rule applies to the principal passed. + assert.throws(() => store.exports.transaction(tx => tx.records('desks').create({ id: 'bob' }, { desk: 'd1', start: 12, end: 13 })), (error: unknown) => (error as { code?: string; conflict?: unknown }).code === 'interval_conflict' && (error as { conflict?: unknown }).conflict === undefined); + assert.equal((await store.call('POST', '/api/desks', { who: 'bob', body: { desk: 'd1', start: 20, end: 25 } })).status, 201); + // scope: owner, and no within: each owner's diary is its own timeline. + assert.equal((await store.call('POST', '/api/diaries', { who: 'ann', body: { desk: 'x', start: 1, end: 5 } })).status, 201); + assert.equal((await store.call('POST', '/api/diaries', { who: 'bob', body: { desk: 'y', start: 1, end: 5 } })).status, 201, 'another owner\'s interval never conflicts'); + assert.equal((await store.call('POST', '/api/diaries', { who: 'ann', body: { desk: 'z', start: 4, end: 6 } })).status, 409, 'ann\'s own intervals still may not overlap, in any desk'); +}); + +test('the operator\'s reassign refuses a move that would give the target overlapping intervals under scope: owner', async t => { + const store = await direct(t, config, { mounts, principalMounts: ['/api/desks', '/api/diaries'] }); + await store.call('POST', '/api/diaries', { who: 'ann', body: { desk: 'x', start: 1, end: 5 } }); + await store.call('POST', '/api/diaries', { who: 'bob', body: { desk: 'x', start: 3, end: 8 } }); + const before = records(store.database, 'diaries'); + for (const dryRun of [true, false]) await assert.rejects(reassignOwner(store.database, { from: 'ann', to: 'bob', collections: config.collections as Record, dryRun }), /Nothing was moved: collection diaries would give bob records .* whose intervals overlap/); + assert.deepEqual(records(store.database, 'diaries'), before); + assert.equal((await reassignOwner(store.database, { from: 'ann', to: 'cy', collections: config.collections as Record })).moved, 1, 'a principal with no overlapping interval takes them'); +}); + +test('a failure after the check rolls everything back and the slot stays free', async t => { + const store = await direct(t, config, { mounts, principalMounts: ['/api/desks', '/api/diaries'] }); + execute(store.database, "CREATE TRIGGER fail_insert BEFORE INSERT ON store_records WHEN NEW.collection = 'rooms' BEGIN SELECT RAISE(ABORT, 'injected'); END;"); + const failed = await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(9), end: hour(10) }, headers: { 'idempotency-key': 'once' } }); + assert.equal(failed.status, 503); + assert.deepEqual(counts(store.database), { records: 0, idempotency: 0, outbox: 0 }); + execute(store.database, 'DROP TRIGGER fail_insert'); + assert.equal((await store.call('POST', '/api/rooms', { body: { room: 'a', start: hour(9), end: hour(10) }, headers: { 'idempotency-key': 'once' } })).status, 201, 'the retry finds the slot free'); + // A host transaction that books and then fails leaves nothing behind either. + assert.throws(() => store.exports.transaction(tx => { tx.records('rooms').create(null, { room: 'b', start: hour(9), end: hour(10) }); throw new Error('later step failed'); }), /later step failed/); + assert.equal((await store.call('POST', '/api/rooms', { body: { room: 'b', start: hour(9), end: hour(10) } })).status, 201); +}); + +test('racing overlapping bookings: exactly one commits, in one process and across connections', async t => { + const store = await direct(t, config, { mounts, principalMounts: ['/api/desks', '/api/diaries'] }); + const local = await Promise.all(Array.from({ length: 12 }, (_, index) => store.call('POST', '/api/desks', { who: `p${index % 3}`, body: { desk: 'hot', start: 100 + index, end: 200 } }))); + assert.equal(local.filter(answer => answer.status === 201).length, 1); + assert.ok(local.every(answer => answer.status === 201 || code(answer) === 'interval_conflict')); + await store.close(); + // Four threads, each its own connection as a second process would have, book overlapping slots at once. + const racers = Array.from({ length: 4 }, (_, worker) => Array.from({ length: 3 }, (_, index) => ({ method: 'POST', path: '/api/desks', init: { who: `w${worker}`, body: { desk: 'cold', start: 10 * index + worker, end: 10 * index + worker + 15 } } }))); + const answers = (await race(t, store.database, config, store.activation, racers)).flat(); + assert.ok(answers.every(answer => answer.status === 201 || answer.status === 409), JSON.stringify(answers.map(answer => answer.status))); + const kept = records(store.database, 'desks').filter(record => record.desk === 'cold').sort((a, b) => (a.start as number) - (b.start as number)); + assert.equal(kept.length, answers.filter(answer => answer.status === 201).length); + assert.ok(kept.length >= 1); + for (let index = 1; index < kept.length; index++) assert.ok((kept[index - 1]!.end as number) <= (kept[index]!.start as number), 'no two committed bookings overlap'); +}); + +test('activation refuses stored records that overlap or break the interval rules, and reads through its index', async t => { + const store = await direct(t, config, { mounts, principalMounts: ['/api/desks', '/api/diaries'] }); + await store.close(); + const at = '2026-10-01T00:00:00.000Z'; + const row = (id: string, fields: Record) => ({ id: `00000000-0000-4000-8000-00000000000${id}`, createdAt: at, updatedAt: at, ...fields }); + await seed(store.database, 'rooms', [row('1', { room: 'a', start: hour(9), end: hour(10), status: 'booked' }), row('2', { room: 'a', start: hour(9, ':30:00Z'), end: hour(11), status: 'cancelled' })]); + await store.open(); + // The index covers only the booked rows (the when filter) and the check's query uses it: never a scan of the collection. + const spec = normalize('rooms', rooms as unknown as CollectionSpec).intervals!; + const db = new DatabaseSync(store.database); + try { + assert.deepEqual(db.prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name GLOB 'store_intervals_*' ORDER BY name").all().map(entry => entry.name).includes(spec.index), true); + const plan = db.prepare(`EXPLAIN QUERY PLAN ${spec.latest}`).all('a', 0, 'x').map(step => String(step.detail)); + assert.ok(plan.some(detail => detail.includes(`USING INDEX ${spec.index}`)), plan.join('; ')); + assert.ok(!plan.some(detail => detail.includes('TEMP B-TREE')), 'the order comes from the index'); + } finally { db.close(); } + await store.close(); + execute(store.database, "UPDATE store_records SET data = json_set(data, '$.status', 'booked') WHERE id LIKE '%2'"); + await assert.rejects(store.open(), /records 00000000-0000-4000-8000-000000000001 and 00000000-0000-4000-8000-000000000002 hold overlapping intervals/); + execute(store.database, "UPDATE store_records SET data = json_set(data, '$.start', '2026-10-01T12:00:00+02:00') WHERE id LIKE '%2'"); + await assert.rejects(store.open(), /record 00000000-0000-4000-8000-000000000002 holds an interval that intervals refuses/); +}); + +test('a changed or removed declaration drops the index nobody declares any more', async t => { + const store = await direct(t, config, { mounts, principalMounts: ['/api/desks', '/api/diaries'] }); + await store.close(); + const indexes = () => { const db = new DatabaseSync(store.database); try { return db.prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name GLOB 'store_intervals_*' ORDER BY name").all().map(entry => String(entry.name)); } finally { db.close(); } }; + assert.equal(indexes().length, 3); + const { intervals: _dropped, ...plain } = desks; + const running = createStore({ database: store.database, projectSha256: pin }); + const instance = await running.registration.activate({ collections: { rooms, desks: plain, diaries: { ...diaries, intervals: { ...diaries.intervals, scope: 'collection' } } } }, store.activation); + try { assert.deepEqual(indexes().sort(), [normalize('rooms', rooms as unknown as CollectionSpec).intervals!.index, normalize('diaries', { ...diaries, intervals: { ...diaries.intervals, scope: 'collection' } } as unknown as CollectionSpec).intervals!.index].sort()); } + finally { await instance.close?.(); await running.close(); } +}); + +test('declarations the constraint cannot enforce are refused at activation', () => { + const refuse = (spec: Record, message: RegExp) => assert.throws(() => normalize('c', { ...desks, ...spec } as unknown as CollectionSpec), message); + const optional = { ...slots, required: ['desk'] }; + refuse({ schema: optional }, /intervals: start must be required/); + refuse({ schema: { ...slots, properties: { ...slots.properties, start: { type: 'string', maxLength: 30 } } } }, /start must be a string with format: date-time, an integer or a number/); + refuse({ schema: { ...slots, properties: { ...slots.properties, end: { type: 'string', format: 'date-time' } } } }, /start and end must both be date-times or both be numbers/); + refuse({ intervals: { start: 'start', end: 'start' } }, /start and end must be different properties/); + refuse({ intervals: { start: 'start', end: 'end', within: ['nope'] } }, /nope is not a declared property/); + refuse({ intervals: { start: 'start', end: 'end', within: ['end'] } }, /within cannot name start or end/); + refuse({ ownership: 'shared', intervals: { start: 'start', end: 'end', scope: 'owner' } }, /scope: owner needs ownership: owner/); + refuse({ schema: slots, defaults: { start: 0 }, increments: ['start'] }, /start is an increment property/); + refuse({ intervals: { start: 'start', end: 'end', when: { desk: 'x'.repeat(30) } } }, /when value for desk/); + assert.throws(() => normalize('m', { membership: true, key: 'desk', schema: slots, intervals: { start: 'start', end: 'end' } } as unknown as CollectionSpec), /a membership collection takes no intervals/); +}); + +test('the OpenAPI description names interval_conflict and its bounded conflict body', () => { + const described = describeStore({ config, mount: '/api/rooms', schemas: {} } as unknown as Parameters[0])!; + const paths = described.paths as Record }>>; + assert.match(paths['/api/rooms']!.post!.responses['409']!.description, /interval_conflict/); + assert.match(paths['/api/rooms/{id}']!.patch!.responses['409']!.description, /interval_conflict/); + assert.match(paths['/api/rooms/{id}/reopen']!.post!.responses['409']!.description, /transition_conflict.*interval_conflict/); + const error = (described.schemas as Record } } }>).StoreError!; + assert.deepEqual(Object.keys(error.properties.error.properties), ['code', 'message', 'fields', 'issues', 'conflict']); + const plain = describeStore({ config: { collections: { desks: { ...desks, intervals: undefined } } }, mount: '/api/desks', schemas: {} } as unknown as Parameters[0])!; + assert.doesNotMatch(JSON.stringify(plain.paths), /interval_conflict/, 'a collection without intervals never answers it'); +}); + +test('a UTC date-time compares in SQL exactly as it does in JavaScript, to the millisecond', () => { + const db = new DatabaseSync(':memory:'); + try { + const sql = db.prepare("SELECT CAST(round(unixepoch(?, 'subsec') * 1000) AS INTEGER) AS ms"); + for (let index = 0; index < 5000; index++) { + const ms = Math.floor((Math.random() * 2 - 0.5) * 4e12), iso = new Date(ms).toISOString(); + const digits = index % 4, text = digits === 3 ? iso : iso.replace(/\.\d{3}Z$/, digits === 0 ? 'Z' : `.${iso.slice(20, 20 + digits)}Z`); + assert.equal(Number(sql.get(text)!.ms), Date.parse(text), text); + } + } finally { db.close(); } +}); diff --git a/packages/store/test/race-worker.ts b/packages/store/test/race-worker.ts index 6b773ada..0a04af4a 100644 --- a/packages/store/test/race-worker.ts +++ b/packages/store/test/race-worker.ts @@ -16,6 +16,17 @@ Atomics.wait(flag, 0, 0); const results: { status: number; replayed: string | null; body: unknown }[] = []; try { for (const request of requests) { + if (request.method === 'TRANSACTION') { + // A host transaction (#902) instead of a request: `path` names the collection, the body the record and the key. + const { values, key, fingerprint } = request.init.body as { values: Record; key: string; fingerprint?: string }; + const principal = request.init.who ? { id: request.init.who } : null; + let ran = false; + try { + const id = store.exports.transaction(tx => { ran = true; return tx.records(request.path).create(principal, values).record.id; }, { idempotencyKey: key, ...(fingerprint === undefined ? {} : { fingerprint }) }); + results.push({ status: 200, replayed: ran ? null : 'true', body: { id } }); + } catch (error) { results.push({ status: (error as { status?: number }).status ?? 500, replayed: null, body: { error: { code: (error as { code?: string }).code } } }); } + continue; + } const result = answer(await instance.handle!(requestFor(activation.mounts, request.method, request.path, request.init))); results.push({ status: result.status, replayed: result.header('idempotency-replayed') ?? null, body: result.body }); } diff --git a/packages/store/test/sqlite.test.ts b/packages/store/test/sqlite.test.ts index 74862b4b..c7994fc1 100644 --- a/packages/store/test/sqlite.test.ts +++ b/packages/store/test/sqlite.test.ts @@ -88,7 +88,7 @@ test('schema initialization is forward-only and idempotent: reopening changes no (await openStoreDatabase(path)).close(); const initial = schema(); assert.equal(initial.version, STORE_SCHEMA_VERSION); assert.equal(initial.application, STORE_APPLICATION_ID); - assert.deepEqual(initial.objects.map(object => object.name), ['store_audit_drain', 'store_audit_outbox', 'store_audit_outbox_collection', 'store_audit_outbox_order', 'store_idempotency', 'store_idempotency_order', 'store_records', 'store_records_order', 'store_records_owner']); + assert.deepEqual(initial.objects.map(object => object.name), ['store_audit_drain', 'store_audit_outbox', 'store_audit_outbox_collection', 'store_audit_outbox_order', 'store_idempotency', 'store_idempotency_order', 'store_records', 'store_records_order', 'store_records_owner', 'store_transaction_results']); const { todos: api, close } = await activate(t, root, path); await api.create(null, { title: 'kept' }); await close(); diff --git a/packages/store/test/transaction-retries.test.ts b/packages/store/test/transaction-retries.test.ts new file mode 100644 index 00000000..80d6beab --- /dev/null +++ b/packages/store/test/transaction-retries.test.ts @@ -0,0 +1,101 @@ +// Retries for host transactions (#902 item 3): `StoreExports.transaction(work, {idempotencyKey, fingerprint})` keeps +// the transaction's JSON result with its key in the same transaction and replays it to a retry without running +// `work`; a different fingerprint is 422; a failed transaction keeps nothing; the claims survive a restart; racing +// calls with one key run `work` once, in one process and across connections; the result is bounded. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { DatabaseSync } from 'node:sqlite'; +import { StoreError, TRANSACTION_RETRIES } from '../src/index.ts'; +import type { StoreExports, StoreTransactionOptions } from '../src/index.ts'; +import { direct, race } from './direct.ts'; +import { counts, records } from './rows.ts'; + +const accounts = { mount: '/api/accounts', maxRecords: 2000, schema: { type: 'object', additionalProperties: false, required: ['name'], properties: { name: { type: 'string', maxLength: 40 }, available: { type: 'integer', minimum: 0 } } }, defaults: { available: 0 } }; +const config = { collections: { accounts } }; +const mounts = ['/api/accounts']; +const claims = (database: string): number => { const db = new DatabaseSync(database); try { return Number(db.prepare('SELECT count(*) AS n FROM store_transaction_results').get()!.n); } finally { db.close(); } }; + +/** Opens an account and reports whether `work` ran, as a caller that sets a replay header would. */ +function open(store: StoreExports, name: string, options?: StoreTransactionOptions) { + let ran = false; + const value = store.transaction(tx => { ran = true; const created = tx.records('accounts').create(null, { name, available: 10 }); return { id: created.record.id as string, etag: created.etag }; }, options); + return { value, ran }; +} + +test('a retry with the key and fingerprint replays the first result without running work or writing', async t => { + const store = await direct(t, config, { mounts, principalMounts: [] }); + const first = open(store.exports, 'ann', { idempotencyKey: 'open:ann', fingerprint: 'POST /open {"name":"ann"}' }); + assert.equal(first.ran, true); + const before = counts(store.database); + const replay = open(store.exports, 'ann', { idempotencyKey: 'open:ann', fingerprint: 'POST /open {"name":"ann"}' }); + assert.equal(replay.ran, false, 'work did not run'); + assert.deepEqual(replay.value, first.value, 'the first result, as it was returned'); + assert.deepEqual(counts(store.database), before, 'nothing was written'); + assert.equal(records(store.database, 'accounts').length, 1); + // The result is a snapshot: a later change does not alter what the key replays. + store.exports.transaction(tx => tx.records('accounts').update(null, first.value.id, { available: 3 })); + assert.deepEqual(open(store.exports, 'ann', { idempotencyKey: 'open:ann', fingerprint: 'POST /open {"name":"ann"}' }).value, first.value); + // Without a fingerprint the empty one is used, and `undefined` is a result too. + let runs = 0; + const nothing = () => store.exports.transaction(() => { runs++; }, { idempotencyKey: 'no-result' }); + assert.equal(nothing(), undefined); assert.equal(nothing(), undefined); assert.equal(runs, 1); + // Without a key (or with an undefined one) every call runs. + assert.equal(open(store.exports, 'bob').ran, true); + assert.equal(open(store.exports, 'bob', { idempotencyKey: undefined } as unknown as StoreTransactionOptions).ran, true); + assert.equal(records(store.database, 'accounts').length, 3); +}); + +test('the same key with a different fingerprint is 422 idempotency_key_reused and writes nothing', async t => { + const store = await direct(t, config, { mounts, principalMounts: [] }); + open(store.exports, 'ann', { idempotencyKey: 'k', fingerprint: 'one' }); + const before = counts(store.database); + for (const options of [{ idempotencyKey: 'k', fingerprint: 'two' }, { idempotencyKey: 'k' }]) { + assert.throws(() => open(store.exports, 'ann', options), (error: unknown) => error instanceof StoreError && error.status === 422 && error.code === 'idempotency_key_reused'); + } + assert.deepEqual(counts(store.database), before); +}); + +test('a transaction that throws or returns what JSON cannot keep retains nothing, and its retry runs again', async t => { + const store = await direct(t, config, { mounts, principalMounts: [] }); + const options = { idempotencyKey: 'retry-me', fingerprint: 'f' }; + assert.throws(() => store.exports.transaction(tx => { tx.records('accounts').create(null, { name: 'x' }); throw new Error('declined'); }, options), /declined/); + assert.throws(() => store.exports.transaction(tx => tx.records('accounts').update(null, '00000000-0000-4000-8000-000000000000', { available: 1 }), options), (error: unknown) => error instanceof StoreError && error.status === 404); + assert.throws(() => store.exports.transaction(tx => ({ at: new Date(), id: tx.records('accounts').create(null, { name: 'x' }).record.id }), options), /must return a JSON value or undefined/); + assert.throws(() => store.exports.transaction(tx => { tx.records('accounts').create(null, { name: 'x' }); return 'y'.repeat(TRANSACTION_RETRIES.resultBytes); }, options), RangeError); + assert.deepEqual(counts(store.database), { records: 0, idempotency: 0, outbox: 0 }); + assert.equal(claims(store.database), 0); + assert.equal(open(store.exports, 'x', options).ran, true, 'the retry runs'); + assert.throws(() => store.exports.transaction(() => 1, { idempotencyKey: '' }), TypeError); + assert.throws(() => store.exports.transaction(() => 1, { idempotencyKey: 'k'.repeat(TRANSACTION_RETRIES.keyLength + 1) }), TypeError); + assert.throws(() => store.exports.transaction(() => 1, { idempotencyKey: 'k', fingerprint: 7 as unknown as string }), TypeError); +}); + +test('the retry history survives a restart, stores hashes only, and keeps the newest keys', async t => { + const store = await direct(t, config, { mounts, principalMounts: [] }); + const first = open(store.exports, 'ann', { idempotencyKey: 'secret-key-value', fingerprint: 'secret-fingerprint' }); + await store.open(); + const replay = open(store.exports, 'ann', { idempotencyKey: 'secret-key-value', fingerprint: 'secret-fingerprint' }); + assert.equal(replay.ran, false); assert.deepEqual(replay.value, first.value); + const db = new DatabaseSync(store.database); + try { + const row = db.prepare('SELECT key, fingerprint FROM store_transaction_results').get()!; + assert.match(String(row.key), /^[0-9a-f]{64}$/); assert.match(String(row.fingerprint), /^[0-9a-f]{64}$/); + } finally { db.close(); } + for (let index = 0; index < TRANSACTION_RETRIES.keys; index++) store.exports.transaction(() => index, { idempotencyKey: `bulk-${index}` }); + assert.equal(claims(store.database), TRANSACTION_RETRIES.keys); + assert.equal(open(store.exports, 'ann', { idempotencyKey: 'secret-key-value', fingerprint: 'secret-fingerprint' }).ran, true, 'the oldest key was evicted, so its retry runs again'); +}); + +test('racing calls with one key run work once, in one process and across connections', async t => { + const store = await direct(t, config, { mounts, principalMounts: [] }); + const local = await Promise.all(Array.from({ length: 10 }, async () => { await new Promise(resolve => setImmediate(resolve)); return open(store.exports, 'local', { idempotencyKey: 'race-local' }); })); + assert.equal(local.filter(call => call.ran).length, 1); + assert.equal(new Set(local.map(call => call.value.id)).size, 1); + await store.close(); + const racer = [{ method: 'TRANSACTION', path: 'accounts', init: { body: { values: { name: 'remote' }, key: 'race-remote', fingerprint: 'same' } } }]; + const answers = (await race(t, store.database, config, store.activation, Array.from({ length: 4 }, () => racer))).flat(); + assert.ok(answers.every(answer => answer.status === 200), JSON.stringify(answers)); + assert.equal(answers.filter(answer => answer.replayed === null).length, 1, 'exactly one ran'); + assert.equal(new Set(answers.map(answer => answer.body!.id)).size, 1, 'and every other call got its result'); + assert.equal(records(store.database, 'accounts').filter(record => record.name === 'remote').length, 1); +}); diff --git a/packages/store/urlcode.json b/packages/store/urlcode.json index 3511f28f..6c4c68d6 100644 --- a/packages/store/urlcode.json +++ b/packages/store/urlcode.json @@ -369,6 +369,67 @@ "description": "true: every record this mount answers carries _owner, the opaque principal id of the owner (for auth, the user id; never an email or name), so a member can tell requesters apart. Only this mount shows it: the owner's mount, transitions and StoreExports never do." } } + }, + "intervals": { + "description": "A non-overlap constraint for scheduling: among the records it applies to, no two in the same scope (and with equal within values) may hold overlapping half-open [start, end) intervals, so an interval ending where another starts is allowed. Checked inside the write transaction of every create, PUT, PATCH and transition against an index, never a scan of the collection; an overlap answers 409 interval_conflict and nothing is written, so a move that would overlap keeps the record where it was. Activation refuses stored records that already overlap. Not on a membership collection.", + "type": "object", + "additionalProperties": false, + "required": [ + "start", + "end" + ], + "properties": { + "start": { + "type": "string", + "pattern": "^[a-z][A-Za-z0-9_]{0,63}$", + "description": "A required property holding the start: a string with format: date-time, whose values must be UTC (ending in Z) with at most millisecond precision and compare as instants, or an integer or number." + }, + "end": { + "type": "string", + "pattern": "^[a-z][A-Za-z0-9_]{0,63}$", + "description": "A required property of the same kind as start holding the end; a record whose end is not after its start answers 422 invalid_record." + }, + "within": { + "type": "array", + "maxItems": 4, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[a-z][A-Za-z0-9_]{0,63}$" + }, + "description": "Required properties that partition the constraint (a room, a resource): two intervals conflict only when every one of these is equal." + }, + "scope": { + "enum": [ + "collection", + "owner" + ], + "description": "collection (default): every record blocks every other, across owners on an owned collection (another owner's conflicting record is never named). owner: with ownership: owner only, each owner's records are constrained among themselves." + }, + "when": { + "type": "object", + "minProperties": 1, + "maxProperties": 8, + "propertyNames": { + "pattern": "^[a-z][A-Za-z0-9_]{0,63}$" + }, + "additionalProperties": { + "oneOf": [ + { + "type": "string", + "maxLength": 256 + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ] + }, + "description": "Only records holding exactly these values take part, for example {status: booked} so a cancelled booking frees its slot; each value must satisfy its property's schema. Without it every record takes part." + } + } } } }, @@ -495,6 +556,34 @@ "reviewed" ] }, + { + "kind": "configuration", + "name": "intervals", + "description": "`intervals: {start, end, within?, scope?, when?}` on a collection: no two records it applies to (those holding the `when` values, such as `{status: booked}`) in one scope with equal `within` values (a room) may hold overlapping half-open `[start, end)` intervals; any create, PUT, PATCH or transition that would answers `409 interval_conflict` and writes nothing. Checked through an index in the write transaction, across owners on an owned collection without naming another owner's record. Bounds are both numbers or both `format: date-time` strings in UTC (`Z`).", + "path": "urlcode.yaml", + "goals": [ + "book", + "booking", + "bookings", + "schedule", + "scheduling", + "slot", + "slots", + "appointment", + "appointments", + "reservation", + "reservations", + "reserve", + "calendar", + "overlap", + "overlapping", + "availability", + "shift", + "shifts", + "interval", + "intervals" + ] + }, { "kind": "configuration", "name": "membership", diff --git a/scripts/package-audit.ts b/scripts/package-audit.ts index acbcde5f..31b92057 100644 --- a/scripts/package-audit.ts +++ b/scripts/package-audit.ts @@ -360,7 +360,10 @@ export const budgets: Record = { // entries (Node 26), keeping about 3.4 KiB and 4.2 KiB of headroom. // #908 named record schemas, the collection's defaults/readOnlyProperties and the README "Named schemas" section, // on top of #920's main: 103298 packed / 399594 unpacked bytes, 32 entries (Node 26); ~3 KiB headroom on each. - packed: 104 * 1024, + // #902 declared intervals (their config schema again in urlcode.json and the README field reference, the index + // and check in dist/collection.js, the OpenAPI 409) and retry-safe host transactions (dist/records.js), with + // the README/CHANGELOG contract: 114883 packed / 445013 unpacked bytes, 32 entries (Node 26); +3 KiB headroom. + packed: 116 * 1024, // Unpacked raised from 120 to 140 KiB: per-record ownership (#331) adds // the owner scoping in dist/collection.js and dist/store.js, the operator // step for legacy records (dist/ownership.js, the urlcode-store bin @@ -388,7 +391,8 @@ export const budgets: Record = { // #861/#881 record schema and OpenAPI description: see the packed note above. // With #913 authoring goals and #916 README responses together: 386402 unpacked bytes (Node 26). // #908 named record schemas: see the packed note above. - unpacked: 394 * 1024, + // #902 intervals and host transaction retries: see the packed note above. + unpacked: 438 * 1024, // #859 online backup (dist/backup.js and dist/backup.d.ts, CLI usage, README) on top of #863 measures // 78786 packed and 317431 unpacked bytes in 32 entries: inside 80/315 KiB, one more entry. entries: 32,