diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 316b3d24..01f4de87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,28 @@ jobs: - name: Dependency audit run: npm audit --production --audit-level=high + onlion: + name: Service backend (onlion) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + # Two manifests: the service builds its prompt tables with a root-level + # tool, so both dependency trees are installed. + cache-dependency-path: | + package-lock.json + onlion/package-lock.json + - run: npm ci + - name: Install service dependencies + run: npm ci + working-directory: onlion + - name: Service verification contract + run: npm run verify:onlion + tests: name: Product tests and coverage runs-on: ubuntu-latest diff --git a/docs/decisions/implemented/2026-10-28-add-log-accept-default.md b/docs/decisions/implemented/2026-10-28-add-log-accept-default.md new file mode 100644 index 00000000..d75fc17a --- /dev/null +++ b/docs/decisions/implemented/2026-10-28-add-log-accept-default.md @@ -0,0 +1,95 @@ +# Add is accepted into the log by default + +[简体中文](2026-10-28-add-log-accept-default.zh-CN.md) · [Terms](2026-10-28-add-log-accept-default.terms.yaml) + +**Status:** implemented +**Approved:** explicit + +## Problem + +`Add` was a read-modify-write of the whole user partition under a per-user manifest CAS. +Sixteen concurrent Adds on one user serialize into sixteen rounds, each moving megabytes, +and the harness saw the result as `/add` rising 36 s → 95 s → a 120 s platform timeout +while `/health` and `/search` stayed in milliseconds. + +The append-log path was built for this (see +[the design](../../design/add-append-log-and-fold-design.md)): accept writes one immutable +entry, fold absorbs up to `MERGE_BATCH` entries per partition write, and the read path +sees entries that have not been folded. It shipped behind `NMG_ADD_MODE` and stayed +`sync` until a measurement justified the switch. + +## Decision + +`log` is the default. `NMG_ADD_MODE=sync` restores the old path without a code change, so +the switch is configuration and is reversible in one deployment. + +The gate was "no regression" rather than an absolute latency target: the base cost of a +request on this path is dominated by the connection and the embedding call, so the +question was whether accepting through the log made an Add slower, not whether it made +the link fast. + +## Measured result + +16 concurrent Adds of 8 messages to one user, same client, same endpoint +(`ap-hongkong`), one warmup Add first, `concurrency=16` therefore one round per run. +Rounds that crossed the fold trigger are excluded, and with `rounds=1` none did. + +| mode | runs | responses | p50 | max | failures | +| ------ | -------------- | ------------------------ | ----------------------------- | ----------------------------- | -------- | +| `sync` | 1 clean round | 16 x 200 | 21.8 s | 30.7 s | 0 | +| `log` | 4 clean rounds | 16 x 200 each (64 total) | 3.8 s / 4.3 s / 4.3 s / 3.8 s | 5.9 s / 5.2 s / 7.3 s / 4.9 s | 0 | + +The accept path is roughly five times cheaper at the median and six times cheaper at the +worst case in this burst. Nothing regressed. + +### Instability is part of the result + +Roughly half the runs taken during the same hour could not establish a connection for 5 to +20 of the 16 requests (`UND_ERR_CONNECT_TIMEOUT`), in **both** modes, while other requests +in the same run answered in milliseconds. Those runs are not "noise to be discarded": a +client that cannot reach the endpoint is the experience the leaderboard reports, and the +log switch neither causes nor cures it. What matters for this decision is that the clean +runs in both modes were taken an hour apart on the same path, and the difference between +them is far larger than the run-to-run spread within a mode. + +## What this does and does not establish + +- **The fold cost is not measured.** With `rounds=1` and a fresh user, pending peaked at + 17 and never reached `FOLD_TRIGGER` (24), so no request in these runs paid for folding. + What is measured is the accept path. The fold's cost is one partition write per batch and + is bounded by `MERGE_BATCH`, but its latency in production is unknown. +- One region, one user, one payload shape (8 messages). A different burst size, a longer + running user, or a different region can differ. +- The instability above means individual runs are not comparable; only runs with zero + client-side failures were used, and there were four of those for `log` and one for `sync`. +- No answer-quality or retrieval effect is claimed: this changes when an Add becomes part of + a partition, not what is stored. + +## Alternatives considered + +- **Keep `sync` and lower the retry budget.** The 503s come from twelve CAS attempts + running out under sixteen writers; a shorter budget turns them into faster 503s, and a + longer one into slower successes. Neither changes how many partition rewrites a burst + costs. +- **Additive delta writes for `Add`.** Measured and rejected earlier: an 8-message Add + changes 72–86 pages of 8 KiB against a 32-page (`PART_DELTA_LIMIT`) budget, 0 of 13 adds + fit, and the overlay lives inline in the manifest every writer contends for. +- **Pool `Add` requests server-side.** Functions are stateless across instances, so + "collect concurrent Adds and write once" needs a coordinator the deployment does not have + and a queue this runtime cannot provide. +- **Wait for a stable-client measurement before switching.** The base cost here is the + connection and the model call, both of which are shared by the two paths; a five-fold + difference in the accept path is far larger than the run-to-run spread, and the switch is + one environment variable away from being reverted. + +## Consequences + +- An accepted Add is now readable through `pendingForSearch` until a fold moves it into a + partition, capped at `MAX_PENDING_READ` entries per search. +- `/capacity` reports `pending_entries`; `sqlite_bytes` stays zero until the first fold, + which is why the EdgeOne smoke had to stop asserting `sqlite_bytes > 0` after an Add. +- Rolling the **code** back after pending entries exist requires draining first: + `POST /admin/fold?user_id=` answers 409 until the log is empty, and a v4 manifest is + unreadable to an older deployment. +- Folding happens on the `Add` path when pending reaches `FOLD_TRIGGER`; it never happens + on the read path, so search latency does not inherit write contention. diff --git a/docs/decisions/implemented/2026-10-28-add-log-accept-default.terms.yaml b/docs/decisions/implemented/2026-10-28-add-log-accept-default.terms.yaml new file mode 100644 index 00000000..c9c1a952 --- /dev/null +++ b/docs/decisions/implemented/2026-10-28-add-log-accept-default.terms.yaml @@ -0,0 +1,3 @@ +version: 2 +terms: [] +references: [] diff --git a/docs/decisions/implemented/2026-10-28-add-log-accept-default.zh-CN.md b/docs/decisions/implemented/2026-10-28-add-log-accept-default.zh-CN.md new file mode 100644 index 00000000..ccb31bfa --- /dev/null +++ b/docs/decisions/implemented/2026-10-28-add-log-accept-default.zh-CN.md @@ -0,0 +1,75 @@ +# Add 默认走日志受理 + +[English](2026-10-28-add-log-accept-default.md) · [Terms](2026-10-28-add-log-accept-default.terms.yaml) + +**Status:** implemented +**Approved:** explicit + +## 问题 + +`Add` 原本是对整个用户分区的读-改-写,由每用户的 manifest CAS 串行化。同一用户 16 路并发 +会被拉成 16 轮,每轮搬动兆字节;评测方看到的结果是 `/add` 从 36 s 升到 95 s、最后 120 s +平台超时,而 `/health` 与 `/search` 一直是毫秒级。 + +日志受理路径就是为此而建(见[设计](../../design/add-append-log-and-fold-design.zh-CN.md)): +受理只写一条不可变条目,折叠每次分区写吸收最多 `MERGE_BATCH` 条,读路径能看见尚未折叠的条目。 +它一直藏在 `NMG_ADD_MODE` 后面,默认 `sync`,直到有测量能支撑切换。 + +## 决策 + +默认改为 `log`。`NMG_ADD_MODE=sync` 无需改代码即可回到旧路径,所以切换是配置、一次部署可逆。 + +门槛是"**无回退**"而不是某个绝对延迟:这条路径上的基础成本主要由连接与 embedding 决定, +要回答的是"经日志受理会不会让 Add 变慢",而不是"能不能让链路变快"。 + +## 实测结果 + +同一用户、每次 8 条消息、16 路并发,同一客户端、同一端点(`ap-hongkong`),先做一次预热 +Add,因此每次运行只有一轮。跨过折叠阈值的轮被排除;`rounds=1` 时没有任何一轮跨过。 + +| 模式 | 运行 | 响应 | p50 | max | 失败 | +| ------ | ---------- | ------------------------ | ----------------------- | ----------------------- | ---- | +| `sync` | 1 个干净轮 | 16 × 200 | 21.8 s | 30.7 s | 0 | +| `log` | 4 个干净轮 | 每轮 16 × 200(合计 64) | 3.8 / 4.3 / 4.3 / 3.8 s | 5.9 / 5.2 / 7.3 / 4.9 s | 0 | + +这次突发里,受理路径的中位数便宜约 5 倍、最坏情况便宜约 6 倍。没有回退。 + +### 抖动本身也是结果 + +同一小时内约有半数的运行里,16 个请求中有 5–20 个**无法建立连接** +(`UND_ERR_CONNECT_TIMEOUT`),两种模式都出现,而同一次运行里另一些请求是毫秒级返回。 +这些运行不是"可以丢掉的噪声":客户端连不上端点,正是评测方报告的那种体验,而日志切换既 +不制造它也不治愈它。对这个决策而言重要的是:两种模式的干净运行取自同一路径、相隔约一小时, +而它们之间的差异远大于同一模式内的运行间波动。 + +## 这确立了什么、没有确立什么 + +- **折叠成本未测量。** `rounds=1` 且是新用户,pending 峰值只有 17,从未到达 + `FOLD_TRIGGER`(24),所以这些运行里没有任何请求为折叠付过代价。测量的是受理路径。 + 折叠的成本是"每批一次分区写"、由 `MERGE_BATCH` 封顶,但它在生产中的延迟未知。 +- 一个地区、一个用户、一种 payload 形状(8 条消息)。不同的突发规模、长期存在的用户或别的 + 地区都可能不同。 +- 上面的抖动意味着单次运行之间不可横比;只有**零客户端失败**的运行被采用,其中 `log` 有四个、 + `sync` 只有一个。 +- 不主张任何答案质量或检索效果的变化:这次改变的是"Add 何时成为分区的一部分",不是"存了什么"。 + +## 考虑过的替代方案 + +- **保留 `sync`,只调重试预算。** 503 来自 16 个写者打满 12 次 CAS;预算调小只是把 503 提前, + 调大只是把成功推后,都不改变一次突发要重写多少次分区。 +- **`Add` 走增量页写。** 此前实测否决:8 条消息的 Add 改 72–86 页(每页 8 KiB),而预算是 + 32 页(`PART_DELTA_LIMIT`),13 次里 0 次命中;且 overlay 内联在所有写者争用的 manifest 里。 +- **服务端合并并发的 `Add`。** 函数实例之间无共享状态,"收集并发写一次性落盘"需要一个部署里 + 没有的协调者,以及这个运行时提供不了的队列。 +- **等稳定客户端测完再切。** 这条路径的基础成本是连接与模型调用,两条路径共享;受理路径上 + 5 倍的差异远大于运行间波动,而切回只差一个环境变量。 + +## 后果 + +- 受理后的 Add 在折叠之前通过 `pendingForSearch` 可读,每次检索最多 `MAX_PENDING_READ` 条。 +- `/capacity` 上报 `pending_entries`;`sqlite_bytes` 在首次折叠前保持 0——这就是 EdgeOne smoke + 不能再断言"Add 之后 `sqlite_bytes > 0`"的原因。 +- 存在 pending 条目时回滚**代码**需要先排空:`POST /admin/fold?user_id=` 在日志清空前返回 409, + 而 v4 的 manifest 对旧部署不可读。 +- 折叠只发生在 `Add` 路径(pending 达到 `FOLD_TRIGGER` 时),读路径永不折叠,因此检索延迟不 + 继承写竞争。 diff --git a/docs/decisions/implemented/2026-10-28-query-window-plan.md b/docs/decisions/implemented/2026-10-28-query-window-plan.md new file mode 100644 index 00000000..7d91c3cf --- /dev/null +++ b/docs/decisions/implemented/2026-10-28-query-window-plan.md @@ -0,0 +1,172 @@ +# A long query is searched by its own clause and planned windows + +[简体中文](2026-10-28-query-window-plan.zh-CN.md) · [Terms](2026-10-28-query-window-plan.terms.yaml) + +**Status:** implemented +**Approved:** explicit + +## Problem + +Payload caps are sized to the platform body, so a search may legitimately carry a +query far longer than an embedding model's token window: an entry can be up to +1,000,000 characters and the model sees 512 tokens. Long queries were therefore +split, and the split decided two things at once — which parts of the query reach +the model, and how many embedding calls a search spends. + +The shipped split (`splitQueryChunks`) tiled windows forward from the start with +boundary-snapped cuts, then appended one window anchored at the end. For a +36,833-character query that is eight windows covering `[0, 14000)` and +`[34833, 36833)`: the opening is dense and the middle is a single contiguous blind +block of about 20,800 characters. The first feature was also the raw clause, up to +1,000,000 characters, which the provider truncates on its own — a head-only +embedding paid for at megabyte request sizes. + +The measured consequence, on 281 LoCoMo multi-hop questions, was that a question +placed in the middle of the text scored **what it scored with no splitting at +all**: 2.14% full-page coverage, the same number as searching one unsplit string, +and worse than the 5.34% obtained when the question sat at the end. The extra +windows were not covering the middle, and at 36,000 characters the shipped layout +did not beat the no-split baseline on that condition at all (0W/0L). + +## Decision + +Plan the windows geometrically and spend a counted feature budget on them. + +`planQueryWindows` (in `onlion/backend/query-chunks.mjs`) chooses one of two +layouts from `n = min(ceil(L / W), MAX_QUERY_WINDOWS)` where `W = +QUERY_CHUNK_CHARS`: + +- `L <= n*W` — the budget covers the text: cut it into `n` equal, contiguous, + disjoint segments that share one boundary-snapped cut point per seam, so nothing + is dropped and nothing is embedded twice. +- `L > n*W` — budget-bound: `n` windows of `W`, the first at the start and the + last anchored at the end, with the uncovered text spread evenly between them. + +`selectQueryFeatures` then builds each clause's feature list as the **clause itself +first**, followed by the end-anchored tail and the interior windows. The planned +first window is **dropped**: it re-covers the opening, which the clause feature +already covers with a longer, token-based window. + +The feature budget is counted, not timed. `MAX_QUERY_FEATURES = 24` is shared +across clauses round-robin, so every clause gets both ends before any clause gets +a third window and a request fills the budget instead of stopping at a fixed +count. `MAX_QUERY_WINDOWS = 10` is a per-clause ceiling, reported as +`limits.max_query_chunks`. No environment override is exposed: two ways to change +the same bound is one way too many, and the measured configuration ships as a +constant. The previous extra-call budget (`NMG_QUERY_INTERIOR_BUDGET`) is removed +for the same reason. + +`src`-side embedding clients are unaffected: the whole query is never a single +input any more, so the truncation the provider used to perform silently no longer +decides what is searchable. + +## Measured result + +Zero LLM cost: gold-evidence recall on the 281 LoCoMo multi-hop questions of +`evals/locomo/results/locomo-eligible-manifest.json`, page 20, fresh local BGE +(`BAAI/bge-small-en-v1.5`), the same record vectors as the product path. Full page +coverage (`all%`) of the four long-form conditions, plus the `short` control which +must be identical across arms because a short query is one feature everywhere. + +Arms: **E** = shipped (clause + forward tiling + one end-anchored window), **F** = +planned windows only, **G** = clause + all planned windows, **H** = clause + +planned windows minus the first window. + +6,000 characters: + +| arm | long | mixed | tail | mid | total | +| ---------------- | --------- | ----- | ---- | ---- | --------- | +| E | **11.74** | 7.83 | 7.12 | 3.91 | 30.60 | +| F (adaptive n=3) | 9.61 | 7.83 | 5.69 | 5.34 | 28.47 | +| G (n=10) | 11.74 | 7.83 | 7.47 | 5.69 | 32.73 | +| H (n=10) | 11.74 | 7.83 | 7.47 | 5.69 | **32.73** | + +36,000 characters: + +| arm | long | mixed | tail | mid | total | +| ---------------- | --------- | ----- | ---- | ---- | --------- | +| E | **11.74** | 7.83 | 5.34 | 1.78 | 26.69 | +| F (adaptive n=8) | 11.39 | 7.83 | 6.05 | 4.63 | 29.90 | +| G (n=10) | 11.03 | 7.83 | 6.05 | 4.63 | 29.54 | +| H (n=10) | 11.39 | 7.83 | 6.05 | 4.63 | **29.90** | + +Paired significance is computed on the per-question all-or-nothing indicator: + +- **H > E, mid, 36,000 characters: 8W/0L, p=0.0078.** E's `mid` is not merely + weaker than H's, it equals the no-split baseline (1.78 against 2.14), so the + windows E spends there move the reference answer _out_ of the page. +- **H > E, mid, 6,000 characters: 6W/1L** (not significant at this length). +- **H vs G: no difference on any condition at either length** (0W/0L everywhere + except 36,000 `long`, where H is 1W/0L ahead). Dropping the redundant opening + window costs nothing and saves one embedding call per clause. +- **E vs H, `long`:** within noise at 36,000 (0W/2L, p=0.5) and identical at 6,000, + because both keep the clause itself as a feature. +- `short` is identical across all arms, as the control requires. + +Live verification after deploying `dpedvhr1ryme` to `nmg-memory-api-overseas` +(`ap-hongkong`): `limits.max_query_chunks = 10`, `max_query_features = 24`, +`npm run smoke:edgeone` 9/9, and a 36,000-character query with the question at the +head, the middle and the tail each returned the planted record at `top_k=100` +(3.1 s, 4.3 s, 2.8 s). + +## What this does and does not establish + +- The padding in this probe is **filler text**, so the middle is uninformative by + construction. What was measured is that a layout without a middle window cannot + find a question placed there; it is not a measurement of how often real long + queries carry information in the middle. Real transcripts make that more likely, + which would favour the planned layout further — that is reasoning, not a number. +- Only retrieval is measured: gold-evidence recall at a 20-record page. No answer + or judge model was involved, and no end-to-end score was produced for this + change. The change is therefore not additive with any earlier benchmark result. +- `W = 2000` **characters** is calibrated for English (≈512 tokens for this model). + For CJK text 512 tokens is roughly 500 characters, so a window may still exceed + the model's token window; the layout is not wrong but `W` is not language-neutral. +- The stores, questions and vectors are fixed, so the numbers are comparable across + arms but not to a different corpus, page size or embedding model. + +## Alternatives considered + +- **Keep the shipped forward tiling** — rejected: it leaves one contiguous blind + block, and on the middle condition it matched the no-split baseline. +- **Planned windows only, without the clause feature (F)** — rejected: it loses + head-aligned recall (6,000 characters: 9.61 vs 11.74, 7 losses) because the + model's own token window gives the opening more context than a character window. +- **Keep the clause feature and all planned windows (G)** — rejected as H's + superset with no gain: identical on every condition, one extra embedding call. +- **A wall-clock budget** ("stop splitting when time runs out") — rejected: the + same query would resolve different windows on a fast and a slow machine, which + makes every recorded retrieval number unreproducible. +- **Volume pooling of windows** — not revisited here; mean pooling was already + measured as losing to per-feature maximum when a query's parts point in + different directions. Nothing in the product pools windows, so the helper was + removed from the service module and lives on only in the probe that measured it. + +## Rejected alongside this change: additive delta writes for `Add` + +The same investigation asked whether an Add could upload only its changed pages +instead of the whole partition snapshot, using the existing `makePageDelta` +machinery. Measured with the built worker and realistic 384-dimension vectors on +a ~3 MB partition: an 8-message Add changes **72–86 pages of 8 KiB** (576–688 KB), +20-message 80–121 pages, 30-message 94–102 pages, against a `PART_DELTA_LIMIT` of +256 KiB (32 pages). **0 of 13 measured Adds fit.** The overlay lives inline in the +manifest, so raising the limit would grow the object that every concurrent writer +competes for, and `USER_DELTA_LIMIT` is 1 MiB — about one Add. The change was +dropped rather than tuned. + +## Consequences + +- A long clause costs at most **10** embedding calls (the clause plus nine + windows); before this change the same clause cost one unbounded input plus up to + eight windows. Feature count is bounded by `MAX_QUERY_FEATURES` for every request, + so the ceiling does not depend on how many clauses the caller sends. +- Retrieval is deterministic: the same query text resolves the same windows + regardless of machine speed or load, which keeps recorded benchmarks comparable. +- The product keeps three constants (`QUERY_CHUNK_CHARS`, `MAX_QUERY_WINDOWS`, + `MAX_QUERY_FEATURES`) and two functions (`planQueryWindows`, + `selectQueryFeatures`). `NMG_QUERY_INTERIOR_BUDGET` never shipped, no override + was added, and the pooling helper left the service module. +- The probe keeps every arm that produced this record, the previous layout + included, and imports the product function only for the arms that are meant to + track the product. Importing it for the baseline would make the baseline move + with the product. diff --git a/docs/decisions/implemented/2026-10-28-query-window-plan.terms.yaml b/docs/decisions/implemented/2026-10-28-query-window-plan.terms.yaml new file mode 100644 index 00000000..c9c1a952 --- /dev/null +++ b/docs/decisions/implemented/2026-10-28-query-window-plan.terms.yaml @@ -0,0 +1,3 @@ +version: 2 +terms: [] +references: [] diff --git a/docs/decisions/implemented/2026-10-28-query-window-plan.zh-CN.md b/docs/decisions/implemented/2026-10-28-query-window-plan.zh-CN.md new file mode 100644 index 00000000..27244237 --- /dev/null +++ b/docs/decisions/implemented/2026-10-28-query-window-plan.zh-CN.md @@ -0,0 +1,143 @@ +# 长 query 用「clause 本身 + 规划窗」检索 + +[English](2026-10-28-query-window-plan.md) · [Terms](2026-10-28-query-window-plan.terms.yaml) + +**Status:** implemented +**Approved:** explicit + +## 问题 + +payload 上限按平台请求体设定,所以一次 Search 携带的 query 可以合法地远长于 +embedding 模型的 token 窗:单条 query 最长 1,000,000 字符,而模型只看 512 token。 +长 query 因此要被切分,而切分同时决定了两件事——query 的哪些部分进入模型,以及 +一次检索花掉多少次 embedding 调用。 + +此前的切分(`splitQueryChunks`)从开头向前铺窗、按边界吸附,末尾再补一个锚定结尾 +的窗。对 36,833 字符的 query,那是 8 个窗覆盖 `[0, 14000)` 与 `[34833, 36833)`: +前部密集,中间是一整块约 20,800 字符的盲区。第一个特征还是整条原始 clause(最长 +1,000,000 字符),由 provider 自己截断——等于付了兆字节的请求体,只换来一个头部 +向量。 + +在 281 条 LoCoMo multi-hop 上量到的直接后果是:把问题放在文本中段时,得分**与完全 +不切分完全相同**——整页覆盖 2.14%,而不切分也是 2.14%,还不如问题在结尾时的 5.34%。 +多出来的窗并没有覆盖中段;在 36,000 字符下,此前的布局在该条件上完全没能超过 +"不切分"基线(0W/0L)。 + +## 决策 + +用几何方式规划窗,并把特征预算变成可数的配额。 + +`planQueryWindows`(`onlion/backend/query-chunks.mjs`)按 `n = min(ceil(L / W), +MAX_QUERY_WINDOWS)`(`W = QUERY_CHUNK_CHARS`)在两套布局中选一套: + +- `L <= n*W`:预算覆盖全文 → 切成 `n` 段等长、连续、互不重叠的整段,接缝共用一个 + 吸附后的切点,既不丢字也不重复嵌入; +- `L > n*W`:预算不足 → `n` 个 `W` 窗,首窗从开头起、末窗锚定结尾,未被覆盖的文本 + 在它们之间均匀分布。 + +`selectQueryFeatures` 随后把每条 clause 的特征列表构造成:**clause 本身放第一位**, +其后是尾锚窗与中间窗。规划出的**首窗被丢弃**——它只是重新覆盖开头,而 clause 特征 +用更长(按 token 计)的窗口已经覆盖了同一区域。 + +预算是**按次数**计的,不是按时间。`MAX_QUERY_FEATURES = 24` 跨 clause 轮转共享, +因此任何一句拿到第三个窗之前,所有句都先拿到两端,一次请求会把预算填满而不是停在 +固定条数。`MAX_QUERY_WINDOWS = 10` 是每句天花板,以 `limits.max_query_chunks` 上报。 +不开放环境变量覆盖:同一个限制有两种改法,就等于多了一种没必要的改法,实测配置以 +常量形式发布。此前的"内部窗额外调用预算"(`NMG_QUERY_INTERIOR_BUDGET`)出于同样理由 +被删除。 + +`src` 侧的 embedding 客户端不受影响:整条 query 不再是单个输入,因此 provider 过去 +那种静默截断不再决定什么内容可被检索。 + +## 实测结果 + +零 LLM 成本:在 `evals/locomo/results/locomo-eligible-manifest.json` 的 281 条 +LoCoMo multi-hop 上量 gold 证据召回,page 20,本地 BGE +(`BAAI/bge-small-en-v1.5`),记录向量与产品路径一致。下表是四种长文本条件的整页 +覆盖(`all%`),外加 `short` 对照——短 query 在每个 arm 里都是单个特征,因此必须 +完全相同。 + +arm 定义:**E** = 现状(clause + 前向铺窗 + 一个尾锚窗);**F** = 只有规划窗; +**G** = clause + 全部规划窗;**H** = clause + 规划窗但去掉首窗。 + +6,000 字符: + +| arm | long | mixed | tail | mid | 合计 | +| --------------- | --------- | ----- | ---- | ---- | --------- | +| E | **11.74** | 7.83 | 7.12 | 3.91 | 30.60 | +| F(自适应 n=3) | 9.61 | 7.83 | 5.69 | 5.34 | 28.47 | +| G(n=10) | 11.74 | 7.83 | 7.47 | 5.69 | 32.73 | +| H(n=10) | 11.74 | 7.83 | 7.47 | 5.69 | **32.73** | + +36,000 字符: + +| arm | long | mixed | tail | mid | 合计 | +| --------------- | --------- | ----- | ---- | ---- | --------- | +| E | **11.74** | 7.83 | 5.34 | 1.78 | 26.69 | +| F(自适应 n=8) | 11.39 | 7.83 | 6.05 | 4.63 | 29.90 | +| G(n=10) | 11.03 | 7.83 | 6.05 | 4.63 | 29.54 | +| H(n=10) | 11.39 | 7.83 | 6.05 | 4.63 | **29.90** | + +成对显著性用逐题的"整页命中与否"指标计算: + +- **H > E,mid,36,000 字符:8W/0L,p=0.0078。** E 的 `mid` 不只是更弱,它等于 + "不切分"基线(1.78 对 2.14),说明 E 在那里花掉的窗是把参考答案挤出了页面。 +- **H > E,mid,6,000 字符:6W/1L**(该长度下不显著)。 +- **H 与 G:两个长度下所有条件都没有差异**(唯一例外是 36,000 的 `long`,H 1W/0L + 略好)。丢掉冗余首窗零损失,且每句省一次 embedding 调用。 +- **E 与 H 在 `long` 上**:36,000 字符下在噪声内(0W/2L,p=0.5),6,000 字符下完全 + 相同——因为两者都保留 clause 本身作为特征。 +- `short` 在所有 arm 中完全一致,符合对照要求。 + +部署到 `nmg-memory-api-overseas`(`ap-hongkong`,部署 `dpedvhr1ryme`)后的在线验证: +`limits.max_query_chunks = 10`、`max_query_features = 24`、`npm run smoke:edgeone` +9/9;36,000 字符、问题分别落在头部/中段/结尾的 query 在 `top_k=100` 下都召回了植入 +的记录(3.1 s / 4.3 s / 2.8 s)。 + +## 这确立了什么、没有确立什么 + +- 探针里的填充文本是**无信息的人工文本**,所以中段按构造不含信号。这里量到的是 + "没有中段窗的布局找不到落在中段的问题",并不是"真实长 query 里有多少信息落在 + 中段"。真实转写稿更可能有,那会让规划窗布局更占优——但那是推理,不是数字。 +- 只量了检索:20 条页面的 gold 证据召回。没有 answer/judge 模型参与,本次改动没有 + 任何端到端分数。因此它与此前任何 benchmark 结果**不可相加**。 +- `W = 2000` 是**字符**,按英文标定(对该模型约 512 token)。中文里 512 token 大约 + 只有 500 字符,所以一个窗仍可能超出模型 token 窗;布局本身不算错,但 `W` 不是 + 语言中立的。 +- 语料、问题、向量都是固定的,所以各 arm 之间可比,但不可与另一个语料、page 大小或 + embedding 模型的结果横比。 + +## 考虑过的替代方案 + +- **保留原有的前向铺窗** — 否决:留下一整块连续盲区,且在 mid 条件上与"不切分"持平。 +- **只用规划窗、不要 clause 特征(F)** — 否决:丢掉头部对齐的召回(6,000 字符下 + 9.61 对 11.74,7 次失利),因为模型自己的 token 窗给开头的上下文比字符窗更多。 +- **保留 clause 特征 + 全部规划窗(G)** — 否决:它是 H 的超集且无收益,所有条件 + 相同却多花一次 embedding 调用。 +- **按墙钟预算**("时间不够就停止切分")— 否决:同一 query 在快机器与慢机器上会解析 + 出不同窗口,使所有已记录的检索数字不可复现。 +- **窗口向量池化** — 本次未重新评估;此前已量到,当 query 各部分方向不同时均值池化 + 劣于按特征取最大。产品里没有任何地方池化窗口,因此该 helper 已从服务模块移除, + 只留在做过测量的探针里。 + +## 与本次一并被否决的:Add 的增量页写 + +同一轮排查还问了:Add 能否只上传变化的页,而不是整个分区快照(复用现有 +`makePageDelta`)。用构建后的 worker 加真实的 384 维向量,在约 3 MB 的分区上实测: +8 条消息的 Add 改动 **72–86 页(每页 8 KiB,计 576–688 KB)**,20 条约 80–121 页, +30 条约 94–102 页,而 `PART_DELTA_LIMIT` 只有 256 KiB(32 页)。**13 次实测 0 次 +命中。** overlay 是内联在 manifest 里的,抬高上限会让所有并发写者争用的那个对象变大, +而 `USER_DELTA_LIMIT` 是 1 MiB,约等于一次 Add。因此直接放弃,未做调参。 + +## 后果 + +- 一条长 clause 最多花 **10 次** embedding 调用(clause + 9 个窗);改前是"一次无界 + 输入 + 最多 8 个窗"。每次请求的特征数由 `MAX_QUERY_FEATURES` 兜底,所以上限不随 + 调用方发送多少条 clause 而膨胀。 +- 检索是确定性的:同一段 query 文本解析出的窗口与机器快慢、负载无关,已记录的 + benchmark 数字因此保持可比。 +- 产品只保留三个常量(`QUERY_CHUNK_CHARS`、`MAX_QUERY_WINDOWS`、`MAX_QUERY_FEATURES`) + 和两个函数(`planQueryWindows`、`selectQueryFeatures`)。`NMG_QUERY_INTERIOR_BUDGET` + 从未发布、也没有新增覆盖变量,池化 helper 已离开服务模块。 +- 探针保留了产出本记录的全部 arm(含旧布局)。只有"要跟随产品"的 arm 才 import 产品 + 函数;基线若也 import,基线就会随产品一起漂移。 diff --git a/docs/design/add-append-log-and-fold-design.md b/docs/design/add-append-log-and-fold-design.md new file mode 100644 index 00000000..e39f2107 --- /dev/null +++ b/docs/design/add-append-log-and-fold-design.md @@ -0,0 +1,287 @@ +# Add as an immutable log entry, folded into partitions afterwards + +**Status:** draft +**Created:** 2026-10-28 + +## Problem + +`Add` is a read-modify-write of the whole user partition under a per-user manifest +CAS. Every attempt costs a manifest read, a partition download, a SQLite export, a +partition upload, and a conditional manifest write. Concurrent Adds on one user +serialize through that CAS: a 16-way burst is sixteen rounds, and each round moves +megabytes. + +Measured on the deployed service (`ap-hongkong`), 16 concurrent Adds to one user, +8 messages per Add: + +| quantity | value | +| ------------------------------------ | --------------------------- | +| success | 16/16, zero failures | +| p50 | 17.7 s | +| p95 | 27.1 s | +| snapshot per partition (probe store) | 3.0–3.9 MB | +| pages changed per 8-message Add | 72–86 of 8 KiB (576–688 KB) | + +The harness reported the same shape from its side: `/add` rising 36 s → 95 s → a +120 s platform timeout, while `/health` and `/search` stayed in the millisecond +range. Delta writing was measured and rejected: an 8-message Add changes 72–86 +pages against a 32-page (`PART_DELTA_LIMIT` = 256 KiB) budget, 0 of 13 Adds fit, and +the overlay lives inline in the manifest that every writer contends for. + +Two things do not apply here and were verified rather than assumed: this runs as an +EdgeOne **cloud function** (`Nodejs20.19`, no edge-middleware stage), and there is +**no database or queue client** anywhere in the service — storage is COS over +HTTPS only (no VPC, no TCP). The runtime exposes no `waitUntil`, so a "return now, +finish later" background write cannot be made durable either. + +## Goal + +Make acceptance cheap and independent of burst size, without weakening what `Add` +has already promised. + +The contract requires that a 200 means the data is durable and that a later +`Search` by the same user can retrieve it, with `request_id` idempotent on retry. +The goal is therefore not "async write" — it is **durable accept with deferred +compaction**: one immutable HTTPS write to accept, bounded folding afterwards, and +a read path that can see what has not been folded yet. + +Non-goals: no change to embedding, ranking, payload caps, or the storage engine; no +new managed service; no relaxation of the idempotency or durability contract. + +## Design + +### 1. Log entry + +`users//log/-.json`, written with +`x-cos-forbid-overwrite` so the name is the idempotency key and an existing object +cannot be overwritten: + +``` +{ v: 1, generation, user_id, session_id, request_id, request_hash, + fingerprint, received_at, messages: [{ role, content, indexText? }] } +``` + +The name sorts lexicographically in arrival order, which the fold relies on. +`indexText` for structured (image) content has to be prepared before the PUT — that +step is CPU and possibly a vision call, and it stays on the request path. + +### 2. Accept + +Validate → look for `-` in the log (replay returns the stored +response; a different fingerprint is `409 request_id_reused`) → prepare messages → +PUT the entry → 200 with the same echo as today plus `task_id` and +`status: "complete"`. + +`complete` describes persistence, not compaction: after the PUT the entry is durable +and immediately readable through the log-aware read path. If the harness interprets +`/add/status/` more strictly, that is an open question below. + +### 3. Read path + +`Search` merges two sources: the partition search it already performs, and the +pending log entries for that user, bounded by count (`PENDING_READ_LIMIT`) and +bytes. Pending entries are scored the same way partition records are, and fused by +the existing identity/highest-score rule, so a pending record can neither duplicate +a folded one (identity) nor outrank it spuriously. + +### 4. Fold + +After an accept — or at the start of a request when pending already exceeds the +threshold — fold up to `MERGE_BATCH` entries **in one partition write**: read the +entries in name order, apply them in a single SQLite session, write their receipts +into the manifest, commit the partition and the manifest atomically, then +batch-delete the folded entries. + +Cost per fold is one partition round trip regardless of how many entries it +absorbs, which is the whole point: a 16-way burst needs two folds, not sixteen +partition rewrites. Work is bounded by count and bytes so one oversized entry +cannot make a fold unbounded. + +### 5. Idempotency and receipts + +The entry name is the idempotency key while pending; the partition receipt is the +key after folding. Fold skips an entry whose receipt already exists, so a fold that +committed and then crashed before deleting its entries is harmless. Deletion always +happens after the commit, never before. + +### 6. Quotas + +`USER_LIMIT` (1 GiB) and `MAX_RECEIPTS` (16 384) count partition bytes plus pending +bytes/entries. `/capacity` reports both. When pending would exceed +`PENDING_BYTES_LIMIT`, `Add` folds instead of accepting, so the log cannot grow +without bound; the caller sees a slower but successful Add, not a rejection. + +### 7. Storage format + +The manifest gains `pending` (count) and `format: "nmg-shards-v4"`. `loadManifest` +already rejects unknown formats, so an older deployment fails loudly instead of +silently ignoring pending data — which is the correct failure mode, but it means a +rollback must drain the log first. + +### 8. Reset + +`Reset` bumps the generation and deletes the log prefix. Entries carry the +generation they were written under, and `mutateUser`'s existing `user_reset` guard +stays, so a fold that races a reset cannot resurrect old data. + +### 9. Observability + +`/health limits` reports `add_mode`, `pending_read_limit`, `merge_batch`; +`/capacity` reports `pending_entries`, `pending_bytes`, `folded_total`. `Add` +`POST /admin/fold?user_id=` drains a log in bounded rounds and answers **409** until it +is empty, so the pre-rollback check cannot pass half-drained. `Add` responses gain phase +timings (`accept`, `fold`) so the next latency question can be +answered from a response instead of a log hunt. + +## Frozen interfaces (slice 1) + +Everything below is fixed before parallel work starts, so three writers cannot +disagree about the seams. Changing any of it is a design change, not an +implementation detail. + +### Log entry + +``` +name: users//log/-.json + written with x-cos-forbid-overwrite; the name is the idempotency key +body: { v: 1, generation, user_id, session_id, request_id, request_hash, + fingerprint, received_at, messages: [{ role, content, indexText? }] } +``` + +`userKey` is `keyFor(userId)`, `request_hash` is the existing `hash(requestId)`, +`fingerprint` is the existing `hash(JSON.stringify({ sessionId, messages }))`. +`messages[].content` and `indexText` are stored exactly as prepared for +`rememberBatch`, so a folded record's statement is the same text a pending record +is scored on. + +### `onlion/backend/append-log.mjs` (new; injected bucket only) + +```js +export const LOG_ENTRY_VERSION = 1; // must equal HTTP payloads' `v` +export const MAX_PENDING_READ = 8; // pending entries scored per search +export function logPrefix(userKey) // "users//log/" +export function logEntryName(receivedAt, requestHash) // padded, sortable +export async function putLogEntry(bucket, userKey, entry) // { name, created } +export async function readLogEntry(bucket, userKey, name) // entry | null +export async function findLogEntry(bucket, userKey, requestHash) // { name, entry } | null +export async function listPending(bucket, userKey, options) // [{ name, entry }] arrival order +export async function deleteLogEntries(bucket, userKey, names) // batch, idempotent +export function foldPlan(entries, limits) // pure: { fold, keep } +``` + +`bucket` is the existing COS contract (`get`/`head`/`put`/`delete`/`list`). This +module must not import `worker.mjs`, `storage.mjs`, or SQLite. + +### `onlion/backend/storage.mjs` additions + +```js +export const STORAGE_FORMAT_V4 = "nmg-shards-v4"; +export function manifestFormat(manifest) // pending > 0 ? v4 : (parts have delta ? v3 : v2) +export function pendingCount(manifest) // number written by the fold/accept path +export function withPending(manifest, count) +export async function clearLog(bucket, userKey) // reset path; prefix must equal logPrefix() +``` + +`commitManifest` writes `manifestFormat(manifest)`; `loadManifest` accepts v4. The +format only becomes v4 once `pending > 0`, which is what keeps a flag-off deployment +byte-identical on disk. + +`clearLog` takes its prefix from `append-log.mjs` instead of redeclaring the string. +That runtime dependency was deliberately avoided while the two halves were being +written in parallel, so that either could run alone; once they ship together, a +prefix that cannot drift is worth more than a module that loads on its own. + +### Worker seams (single writer) + +```js +const addMode = (env) => (env.NMG_ADD_MODE === "log" ? "log" : "sync"); // default until measured +async function acceptAdd(env, input) // 200 body, durable log entry written +async function foldPending(env, userId, limits) // { folded, remaining } +async function pendingForSearch(env, userId, limit) // provisional records, max MAX_PENDING_READ +``` + +`pendingForSearch` returns page-shaped items with `pending:`-prefixed ids so the +existing fusion applies unchanged; each item costs one embedding call, capped at +`MAX_PENDING_READ`. + +## Invariants (each one becomes a test) + +- **I1** After a 200, a later `Search` by the same user retrieves the data, whether + it is still pending or already folded. +- **I2** A retry of the same `request_id` returns a byte-identical response; a + different fingerprint returns 409. +- **I3** Folding is idempotent: any crash point followed by a retry produces no + duplicate memory, guarded by receipt presence. +- **I4** No partition write crosses a generation; nothing written before a `Reset` + can reappear after it. +- **I5** The read path does not fold. Folding happens on `Add` (or an explicit + maintenance route) only, so read latency does not inherit write contention. +- **I6** Pending reads are bounded in entries and bytes, and `Search` cost does not + grow with the size of the log. + +## Failure matrix + +| failure point | state | next request | +| -------------------------------- | ---------------------------- | -------------------------------------- | +| before the log PUT | nothing written | retry accepts normally | +| after the PUT, response lost | entry durable | retry finds the entry, same echo | +| fold crashes before commit | entries intact | next fold redoes it (no receipt) | +| fold commits, dies before delete | entry + receipt both present | fold skips it (I3), delete retried | +| two folds race | manifest CAS serializes | loser retries; entries still valid | +| reset races a fold | generation changed | fold aborts; `user_reset` 409 as today | + +## Migration + +No backfill: the first `Add` after the upgrade writes a log entry, and existing +partitions keep serving. `NMG_ADD_MODE=sync` restores today's path and still reads +and folds pending entries, so the switch is reversible without data loss. Rolling +the **code** back after the manifest becomes v4 requires draining pending first — +that is a documented operational step, not an automatic downgrade. + +## Work breakdown + +| piece | file | net lines | +| -------------------------------------------------------- | ------------------------------------- | --------- | +| append-log module (entry shape, put/read/list, receipts) | `onlion/backend/append-log.mjs` (new) | ~150 | +| accept path | `onlion/backend/worker.mjs` | ~70 | +| pending-aware read path | `worker.mjs` (+ ranking fusion) | ~70 | +| fold | `worker.mjs`, reusing `addOneBatch` | ~60 | +| quota and health surface | `worker.mjs` | ~30 | +| manifest `pending`, format v4, log cleanup | `onlion/backend/storage.mjs` | ~50 | +| tests (invariants, failure injection, 16-way burst) | `onlion/scripts/test-backend.mjs` | ~250 | +| **total** | | **~680** | + +Slices that can be verified separately: + +1. **Plumbing, no behaviour change** — manifest `pending`, format v4, the log + module, `/capacity` fields. `Add` still synchronous. +2. **Accept + fold + pending-aware read** — must land together, behind + `NMG_ADD_MODE=log`; verified by a 16-way burst against the synchronous path. +3. **Default on, decision record, benchmark note.** + +## Open questions + +1. **How pending entries are scored at read time.** Embedding them per read costs up + to `PENDING_READ_LIMIT` extra embedding calls on every search; scoring them + lexically only is cheap but weaker. Bounded either way, but the choice changes + what I1 means in practice. +2. **What `status: "complete"` may claim.** The service already returns `task_id` + and serves `/add/status/`. If the harness treats that endpoint as + evidence of processing rather than persistence, the status value needs agreement + before this ships. +3. **`indexText` cost for structured content.** If it calls a vision model, it stays + a synchronous step in the accept path and sets a floor on Add latency that no + amount of folding removes. + +## Rejected alternatives + +- **Fire-and-forget background write.** The runtime has no `waitUntil`; a background + task is not guaranteed to run, so a 200 would sometimes mean "lost". +- **Delta/overlay writes for `Add`.** Measured: 0 of 13 Adds fit the 32-page budget; + raising the limit grows the manifest every writer contends for. +- **Larger retry budget or shorter backoff.** Only converts a 503 into a longer wait; + the 120 s platform cap still applies. +- **Folding on the search path.** Would move write contention into read latency, + which is the one thing currently fast. +- **A queue or database service.** Not reachable from this runtime, and it would add + a dependency the deployment does not have today. diff --git a/docs/design/add-append-log-and-fold-design.terms.yaml b/docs/design/add-append-log-and-fold-design.terms.yaml new file mode 100644 index 00000000..c9c1a952 --- /dev/null +++ b/docs/design/add-append-log-and-fold-design.terms.yaml @@ -0,0 +1,3 @@ +version: 2 +terms: [] +references: [] diff --git a/docs/design/add-append-log-and-fold-design.zh-CN.md b/docs/design/add-append-log-and-fold-design.zh-CN.md new file mode 100644 index 00000000..cc19b042 --- /dev/null +++ b/docs/design/add-append-log-and-fold-design.zh-CN.md @@ -0,0 +1,245 @@ +# Add 写不可变日志条目,之后再折叠进分区 + +**Status:** draft +**Created:** 2026-10-28 + +## 问题 + +`Add` 现在是对整个用户分区的"读-改-写",并由每用户的 manifest CAS 串行化。每一次尝试 +都要付出:读 manifest、下载分区、导出 SQLite、上传分区、条件写 manifest。同一用户的并发 +Add 会在 CAS 上排队:16 路并发就是 16 轮,而每轮搬动的是兆字节。 + +在已部署服务(`ap-hongkong`)上实测:同一用户 16 并发、每次 8 条消息: + +| 指标 | 数值 | +| -------------------------- | ------------------------------ | +| 成功率 | 16/16,零失败 | +| p50 | 17.7 s | +| p95 | 27.1 s | +| 探针分区快照 | 3.0–3.9 MB | +| 一次 8 条消息 Add 改动的页 | 8 KiB × 72–86 页(576–688 KB) | + +评测方从另一侧看到的是同一形状:`/add` 从 36 s 升到 95 s、最后 120 s 平台超时,而 +`/health`、`/search` 一直是毫秒级。增量页写已经实测过并被否决:一次 8 条消息的 Add 改 +72–86 页,而预算是 32 页(`PART_DELTA_LIMIT` = 256 KiB),13 次里 0 次命中;且 overlay +是内联在**所有写者都在争用**的那个 manifest 里的。 + +有两条建议不适用于我们的部署形态,是核实过的而非假设:这里跑的是 EdgeOne **云函数** +(`Nodejs20.19`,没有边缘中间件阶段);服务里**没有任何数据库或队列客户端**,存储只有 +COS over HTTPS(无 VPC、无 TCP)。运行时也不提供 `waitUntil`,所以"先返回、后台再写" +无法做到持久化保证。 + +## 目标 + +让"受理"变便宜,且与突发规模无关,同时不削弱 `Add` 已经承诺过的东西。 + +契约要求:返回 200 就意味着数据已持久化,且同一用户后续 `Search` 能检索到;`request_id` +在重试下幂等。因此目标不是"异步写",而是**持久受理 + 延迟压实**:受理只需一次不可变的 +HTTPS 写,之后有界折叠,读路径能看见尚未折叠的内容。 + +非目标:不改 embedding、排序、payload 上限、存储引擎;不引入任何托管服务;不放宽幂等与 +持久化契约。 + +## 设计 + +### 1. 日志条目 + +`users//log/-.json`,用 +`x-cos-forbid-overwrite` 写入,让名字本身就是幂等键,且已存在的对象不能被覆盖: + +``` +{ v: 1, generation, user_id, session_id, request_id, request_hash, + fingerprint, received_at, messages: [{ role, content, indexText? }] } +``` + +名字按字典序即到达顺序,折叠依赖这一点。结构化(图片)内容的 `indexText` 必须在 PUT 之前 +准备好——那一步是 CPU、可能还是视觉模型调用,仍留在请求路径上。 + +### 2. 受理 + +校验 → 在日志里找 `-`(重放返回已存响应;fingerprint 不同则 +`409 request_id_reused`)→ 准备消息 → PUT 条目 → 200,回显与今天一致,外加 `task_id` 与 +`status: "complete"`。 + +`complete` 描述的是**持久化**而非压实:PUT 之后条目已持久化,并通过日志感知的读路径立刻 +可读。如果评测方对 `/add/status/` 的语义理解更严格,那就属于下面的未决问题。 + +### 3. 读路径 + +`Search` 合并两个来源:它本来就要做的分区检索,以及该用户的 pending 日志条目(按条目数与 +字节数双重有界,`PENDING_READ_LIMIT`)。pending 条目用与分区记录相同的方式打分,并用既有的 +"按身份取最高分"规则融合,因此 pending 记录既不会与已折叠的重复(身份相同),也不会凭空 +压过它。 + +### 4. 折叠 + +在一次受理之后——或在请求开始时发现 pending 已超阈值——把**最多 `MERGE_BATCH` 条**在一个 +分区写里折叠:按名字顺序读条目,在一个 SQLite 会话里依次应用,把它们的回执写进 manifest, +原子提交分区与 manifest,然后批量删除已折叠的条目。 + +一次折叠只付一次分区往返,与吸收多少条无关,这正是关键:16 路突发只需要两次折叠,而不是 +十六次分区重写。工作量按条数与字节数双重封顶,避免单条超大条目把折叠拖成无界。 + +### 5. 幂等与回执 + +pending 期间条目的名字就是幂等键;折叠之后由分区里的回执接替。折叠会跳过回执已存在的条目, +因此"已提交但删除前崩溃"是无害的。删除永远发生在提交之后,绝不提前。 + +### 6. 配额 + +`USER_LIMIT`(1 GiB)与 `MAX_RECEIPTS`(16 384)按分区 + pending 合计计算。`/capacity` +分别上报两者。当 pending 将超过 `PENDING_BYTES_LIMIT` 时,`Add` 改为**执行折叠**而不是 +继续受理,日志因此不会无界增长;调用方看到的是"慢一点但成功",不是被拒绝。 + +### 7. 存储格式 + +manifest 增加 `pending`(计数)与 `format: "nmg-shards-v4"`。`loadManifest` 已经会拒绝 +未知格式,因此旧部署会**明确失败**而不是静默忽略 pending 数据——这是正确的失败方式,但也 +意味着回滚前必须先把日志排空。 + +### 8. Reset + +`Reset` 提升 generation 并删除日志前缀。条目携带写入时的 generation,且 `mutateUser` 现有 +的 `user_reset` 守卫保留,因此与 reset 竞争的折叠无法复活旧数据。 + +### 9. 可观测 + +`/health limits` 上报 `add_mode`、`pending_read_limit`、`merge_batch`;`/capacity` 上报 +`pending_entries`、`pending_bytes`、`folded_total`。`Add` 响应增加分阶段耗时 +(`accept`、`fold`),让下一个延迟问题能从响应回答,而不是翻日志。 + +## 冻结接口(切片 1) + +以下内容在并行开工前固定,避免三个写者对"接缝"的理解不一致。改这里属于设计变更,不是 +实现细节。 + +### 日志条目 + +``` +name: users//log/-.json + 用 x-cos-forbid-overwrite 写入;名字即幂等键 +body: { v: 1, generation, user_id, session_id, request_id, request_hash, + fingerprint, received_at, messages: [{ role, content, indexText? }] } +``` + +`userKey` 是 `keyFor(userId)`,`request_hash` 是现有的 `hash(requestId)`, +`fingerprint` 是现有的 `hash(JSON.stringify({ sessionId, messages }))`。 +`messages[].content` 与 `indexText` 按 **为 `rememberBatch` 准备好的同一份**存储, +这样折叠后记录的 statement 与 pending 记录打分用的文本完全一致。 + +### `onlion/backend/append-log.mjs`(新文件;只接收注入的 bucket) + +```js +export const LOG_ENTRY_VERSION = 1; // 必须等于条目里的 `v` +export const MAX_PENDING_READ = 8; // 每次检索最多打分的 pending 条数 +export function logPrefix(userKey) // "users//log/" +export function logEntryName(receivedAt, requestHash) // 定长、可按字典序排序 +export async function putLogEntry(bucket, userKey, entry) // { name, created } +export async function readLogEntry(bucket, userKey, name) // entry | null +export async function findLogEntry(bucket, userKey, requestHash) // { name, entry } | null +export async function listPending(bucket, userKey, options) // [{ name, entry }] 按到达顺序 +export async function deleteLogEntries(bucket, userKey, names) // 批量,幂等 +export function foldPlan(entries, limits) // 纯函数:{ fold, keep } +``` + +`bucket` 就是现有的 COS 契约(`get`/`head`/`put`/`delete`/`list`)。本模块**不得** +import `worker.mjs`、`storage.mjs` 或 SQLite。 + +### `onlion/backend/storage.mjs` 增补 + +```js +export const STORAGE_FORMAT_V4 = "nmg-shards-v4"; +export function manifestFormat(manifest) // pending > 0 ? v4 : (存在 delta ? v3 : v2) +export function pendingCount(manifest) +export function withPending(manifest, count) +export async function clearLog(bucket, userKey) // reset 路径;前缀必须等于 logPrefix() +``` + +`commitManifest` 写 `manifestFormat(manifest)`;`loadManifest` 接受 v4。只有 +`pending > 0` 时格式才变成 v4——这正是"开关关闭时盘上字节不变"的保证所在。 + +`clearLog` 的前缀取自 `append-log.mjs`,不在本地重复声明字符串。两半并行开发期间刻意避免 +这个运行时依赖,为的是任一半能单独跑起来;两者一起发布之后,"前缀不可能漂移"比"模块能 +单独加载"更值钱。 + +### worker 接缝(单写者) + +```js +const addMode = (env) => (env.NMG_ADD_MODE === "log" ? "log" : "sync"); // 实测前默认同步 +async function acceptAdd(env, input) // 返回 200 响应体,日志条目已持久化 +async function foldPending(env, userId, limits) // { folded, remaining } +async function pendingForSearch(env, userId, limit) // 临时记录,最多 MAX_PENDING_READ 条 +``` + +`pendingForSearch` 返回与页面同形的条目(`id`/`content`/`score`/`stored_at`,`id` 前缀 +`pending:`),这样现有的"按身份取最高分"融合逻辑无需改动即可复用。每条 pending 在检索时 +消耗一次 embedding 调用,按 `MAX_PENDING_READ` 封顶。 + +## 不变量(每条都要变成测试) + +- **I1** 返回 200 之后,同一用户后续 `Search` 必能检索到该数据,无论它仍在 pending 还是 + 已折叠。 +- **I2** 同一 `request_id` 的重试返回逐字节相同的响应;fingerprint 不同则 409。 +- **I3** 折叠幂等:任何崩溃点之后重试都不会产生重复记忆,以回执存在性为守卫。 +- **I4** 任何分区写不跨越 generation;reset 之前写入的内容不能在之后重新出现。 +- **I5** 读路径不做折叠。折叠只发生在 `Add`(或显式维护路由)上,读延迟因此不继承写竞争。 +- **I6** pending 读取按条目数与字节数有界,`Search` 开销不随日志规模线性增长。 + +## 失败矩阵 + +| 崩溃点 | 状态 | 下一个请求 | +| ---------------------- | ------------------- | ------------------------------------- | +| 日志 PUT 之前 | 什么都没写 | 重试正常受理 | +| PUT 之后、响应丢失 | 条目已持久化 | 重试命中条目,返回相同回显 | +| 折叠提交前崩溃 | 条目完好 | 下次折叠重做(无回执) | +| 折叠已提交、删除前崩溃 | 条目与回执同时存在 | 折叠跳过(I3),删除重试 | +| 两个折叠竞争 | manifest CAS 串行化 | 落败者重试,条目仍有效 | +| reset 与折叠竞争 | generation 已变更 | 折叠放弃;按现有行为 `user_reset` 409 | + +## 迁移 + +无需回填:升级后第一次 `Add` 写一个日志条目,现存分区继续服务。`NMG_ADD_MODE=sync` 可恢复 +今天的路径,且仍会读取与折叠 pending 条目,因此开关是可逆的、不丢数据。**代码**回滚则不同: +manifest 变成 v4 之后必须先排空 pending,这是有文档的操作步骤,不是自动降级。 + +## 工作量分解 + +| 部分 | 文件 | 净行数 | +| ----------------------------------------- | ------------------------------------- | -------- | +| 日志模块(条目形状、put/read/list、回执) | `onlion/backend/append-log.mjs`(新) | ~150 | +| 受理路径 | `onlion/backend/worker.mjs` | ~70 | +| 读路径合并 pending | `worker.mjs`(+ 排序融合) | ~70 | +| 折叠 | `worker.mjs`,复用 `addOneBatch` | ~60 | +| 配额与健康面 | `worker.mjs` | ~30 | +| manifest `pending`、格式 v4、日志清理 | `onlion/backend/storage.mjs` | ~50 | +| 测试(不变量、故障注入、16 路突发) | `onlion/scripts/test-backend.mjs` | ~250 | +| **合计** | | **~680** | + +可分别验收的切片: + +1. **管道,不改行为** —— manifest `pending`、格式 v4、日志模块、`/capacity` 字段;`Add` + 仍同步。 +2. **受理 + 折叠 + 感知 pending 的读路径** —— 必须一起上,由 `NMG_ADD_MODE=log` 控制; + 用 16 路突发与同步路径对照验证。 +3. **默认打开、决策记录、benchmark 说明。** + +## 未决问题 + +1. **pending 条目在读时如何打分。** 每次检索为它们做 embedding 会多出最多 + `PENDING_READ_LIMIT` 次调用;只做词法打分便宜但更弱。两种都有界,但选择会改变 I1 在 + 实践中的含义。 +2. **`status: "complete"` 可以主张什么。** 服务已经返回 `task_id` 并提供 + `/add/status/`。如果评测方把该端点当作"已处理"的证据而非"已持久化",这个取值 + 需要先对齐。 +3. **结构化内容的 `indexText` 成本。** 若它调用视觉模型,它就是受理路径上的同步步骤,会 + 给 Add 延迟设一个折叠无法消除的下限。 + +## 被否决的替代方案 + +- **发射后不管的后台写。** 运行时没有 `waitUntil`,后台任务不保证执行,200 有时就等于 + "丢了"。 +- **`Add` 走增量/overlay 写。** 实测 13 次 0 次命中 32 页预算;抬高上限会让 manifest + 变得更大,而 manifest 是所有写者争用的对象。 +- **加大重试预算或缩短退避。** 只是把 503 换成更长的等待,120 s 平台上限照样生效。 +- **在检索路径上折叠。** 会把写竞争搬进读延迟,而读是当前唯一快的东西。 +- **引入队列或数据库服务。** 这个运行时到不了它们,而且会给当前部署引入并不存在的依赖。 diff --git a/docs/design/add-append-log-and-fold-stopping-rules.md b/docs/design/add-append-log-and-fold-stopping-rules.md new file mode 100644 index 00000000..c443d1a9 --- /dev/null +++ b/docs/design/add-append-log-and-fold-stopping-rules.md @@ -0,0 +1,72 @@ +# Stopping rules for the append-log write path + +**Status:** draft +**Created:** 2026-10-28 + +Companion to [Add as an immutable log entry](add-append-log-and-fold-design.md) and +[the pending counter's states](add-log-pending-counter-state.md). Those own the design; +this page owns **when the work stops**. + +## Why this page exists + +The first four review rounds each produced a real defect, and each was treated as +blocking. That loop had no termination: a reviewer cannot know what counts as +"finished" unless the owner writes it down, and an owner who accepts every finding as +blocking has no answer to "how much is enough". The cost is not the fixes; it is that +a class-A correctness issue and a missing metric queued in the same line. + +## Defect classes + +**A — blocking.** The only class that stops a slice. It can make already-accepted data +unsearchable, duplicated or lost; make a `200` claim something untrue; or make a +rollback unsafe. Every defect found so far is class A: the fold that iterated objects as +strings, the decrement that ignored retired entries, the zero that never reached the +manifest, the double subtraction, the entry written before its counter, the failed write +read as "did not land", the reset hand-off that dropped the count, and the window that +updated after truncation. + +**B — recorded, not blocking.** Performance, observability, error wording, missing +fields: an extra listing per fold, `/capacity` byte accounting, per-phase timings. It +gets a written follow-up and an owner, not a slice. + +**C — decided as out of scope.** Recorded so nobody re-proposes it: + +- fault injection for a physically failing object write, beyond the fake bucket that + already proves the three-state selection; +- a formal proof of the manifest CAS under concurrency (the design relies on retry); +- stale entries above 32 on the read path (bounded by `SCAN_FACTOR`, and self-healing + because the reset hand-off carries their count into the fold trigger); +- latency tuning across regions. + +## Slice 1 — the code lands, `NMG_ADD_MODE` stays `sync` + +All five must hold: + +1. the `onlion` CI job is green; +2. `verify:static` is green, including the complexity cap on the files touched here; +3. the state table covers every transition and names the function performing it; +4. the most recent review round found no class A; +5. production behaviour is unchanged — `sync` remains the default. + +## Slice 2 — the default becomes `log` + +1. a 16-way concurrent Add burst is measured, and the `log` p95 is below the `sync` + p95; both numbers are recorded in the decision record, not asserted from theory; +2. `/admin/fold` answers 200 for the user before a code rollback, so the runbook step is + executable rather than aspirational; +3. the switch is one setting and is reversible; +4. one review round after the switch finds no class A. + +## Round cap + +Two consecutive review rounds that find only B or C finish the slice. Findings after +that are follow-ups with an owner and do not block. A finding that claims to block must +name the class-A property it breaks — data visibility, honesty of a `200`, or rollback +safety — because "it could be better" is class B by definition. + +## Accepted, known risks at slice 1 + +- a write that fails on a real store is only exercised with a fake bucket; +- concurrent folds are serialized by the manifest CAS, assumed rather than proven; +- the read window scans `4 × 8` keys, so a large stale generation delays but does not + permanently hide current entries. diff --git a/docs/design/add-append-log-and-fold-stopping-rules.terms.yaml b/docs/design/add-append-log-and-fold-stopping-rules.terms.yaml new file mode 100644 index 00000000..c9c1a952 --- /dev/null +++ b/docs/design/add-append-log-and-fold-stopping-rules.terms.yaml @@ -0,0 +1,3 @@ +version: 2 +terms: [] +references: [] diff --git a/docs/design/add-append-log-and-fold-stopping-rules.zh-CN.md b/docs/design/add-append-log-and-fold-stopping-rules.zh-CN.md new file mode 100644 index 00000000..e4b21dcc --- /dev/null +++ b/docs/design/add-append-log-and-fold-stopping-rules.zh-CN.md @@ -0,0 +1,62 @@ +# 日志写入路径的停止规则 + +**Status:** draft +**Created:** 2026-10-28 + +与 [Add 写不可变日志条目](add-append-log-and-fold-design.zh-CN.md) 和 +[pending 计数器状态](add-log-pending-counter-state.zh-CN.md) 配套。那两篇负责设计,本页负责 +**什么时候停**。 + +## 为什么需要这一页 + +前四轮复核每轮都发现真缺陷,而每一条都被当成阻断项处理。这个循环没有终止条件:复核方在 +负责人写下"什么是完成"之前无从判断,而把每条发现都当阻断的负责人也无法回答"多少才算够"。 +代价不是修本身,而是**A 类正确性问题与"少个字段"排在同一行**。 + +## 缺陷分级 + +**A —— 阻断。** 唯一能停下某一片的类别。它会让已受理数据不可检索、重复或丢失;让 `200` +声称不成立的事;或让回滚不安全。迄今所有缺陷都是 A:把对象当字符串迭代的折叠、忽略退休条目 +的递减、从未写进 manifest 的零、二次扣减、先写条目后提交计数、把失败写读成"没落地"、丢掉 +计数的 reset 交接、以及先截断后过滤的读取窗口。 + +**B —— 记录,不阻断。** 性能、可观测性、错误措辞、缺字段:每次折叠多一次列举、 +`/capacity` 的字节口径、分阶段耗时。它得到一条带负责人的跟进项,而不是一片工作。 + +**C —— 明确判定不做。** 写下来以免有人重复提议: + +- 真实对象写失败的故障注入(假 bucket 已能证明三态选择); +- 并发下 manifest CAS 的形式化证明(设计依赖重试); +- 读取路径遗留条目超过 32 的情况(受 `SCAN_FACTOR` 界定,且因 reset 交接把数量带进折叠 + 触发器而自愈); +- 跨区域延迟调优。 + +## 片 1 —— 代码入库,`NMG_ADD_MODE` 保持 `sync` + +五条同时成立: + +1. `onlion` 的 CI job 绿; +2. `verify:static` 绿,含本次触及文件的复杂度上限; +3. 状态表覆盖每个迁移并点名执行它的函数; +4. 最近一轮复核没有 A 类; +5. 生产行为不变——默认仍是 `sync`。 + +## 片 2 —— 默认切到 `log` + +1. 实测 16 路并发 Add 突发,且 `log` 的 p95 低于 `sync` 的 p95;两个数字都写进决策记录, + 不是从理论上断言; +2. 回滚前对用户调用 `/admin/fold` 能返回 200,使 runbook 那一步可执行而非愿望; +3. 切换是一个设置项,且可逆; +4. 切换后**一轮**复核没有 A 类。 + +## 回合上限 + +连续**两轮**复核只发现 B 或 C,该片即完成。此后的发现一律记为带负责人的跟进项,不阻断。 +声称阻断的发现必须**点名它破坏了哪条 A 类性质**——数据可见性、`200` 的诚实性、回滚安全—— +因为"还能更好"按定义属于 B。 + +## 片 1 已接受并记录的已知风险 + +- 真实存储上的写失败只用假 bucket 覆盖过; +- 并发折叠由 manifest CAS 串行化,是假设而非证明; +- 读取窗口扫描 `4 × 8` 个键,大规模遗留代际只会延迟、不会永久遮蔽当前条目。 diff --git a/docs/design/add-log-pending-counter-state.md b/docs/design/add-log-pending-counter-state.md new file mode 100644 index 00000000..fd33c7e8 --- /dev/null +++ b/docs/design/add-log-pending-counter-state.md @@ -0,0 +1,136 @@ +# The pending counter: states, transitions and invariants + +**Status:** draft +**Created:** 2026-10-28 + +Part of [Add as an immutable log entry](add-append-log-and-fold-design.md), which owns +the design. This page owns one component: `manifest.pending`, the count that decides +whether a search has to look at the log and whether a request has to fold. + +## Why this needs a table + +The log path shipped with the counter as an increment-and-hope value. Four review +rounds found six defects in it, every one in the same component and every one a +transition nobody had written down: + +| defect | shape | +| -------------------------------------------------------- | ---------------------------------------------------------- | +| fold iterated `{name, entry}` as strings | entry never applied, nothing deleted | +| counter decremented by applied entries only | receipt-present and cross-generation entries never retired | +| `Object.assign(manifest, withPending(m, 0))` | the zero case never reached the manifest | +| delete failure while the counter was already decremented | the same batch subtracted twice | +| entry written before the counter was committed | an entry no counter accounted for, hidden from search | +| a throwing write treated as "did not land" | a reservation released for an entry on disk | + +None of these is subtle once the states are on paper. That is the point of this page. + +## States + +`pending` and the log are two observations of one thing, so states are pairs: + +| state | `manifest.pending` | log | reachable by | +| ---------------------- | ------------------ | -------------------------------- | -------------------------------------- | +| **C** clean | absent | empty | every path converges here | +| **P** productive | `n > 0` | `m` entries, `n >= m` | accepts outrunning folds | +| **D** draining | `n > 0` | `m` entries being folded | a fold in flight | +| **R** residue-high | `n > 0` | empty | a reservation whose write never landed | +| **G** stale-generation | `n > 0` | entries from an older generation | reset racing a fold | + +There is deliberately no "residue-low" state (`pending` absent, log non-empty). It was +reachable when the entry was written before the counter; reserving first makes it +impossible, which is the reason for that ordering. **Seeing it means a bug**, not a +state: a search would skip the log and hide data. + +## Transitions + +`E` is the set of readable entries under the current generation; `pending` is the +stored count. Every row states what the transition must leave behind. + +| from | event | to | must leave | +| ------------ | ----------------------------------------- | --------- | ----------------------------------------------------------------------------- | +| C | accept | P | counter committed **before** the entry is durable (`acceptAdd`) | +| P/D | accept | P/D | counter +1, reservation recorded (`acceptAdd`) | +| P (reserved) | write fails, `logEntryLanded` = **true** | P | entry exists, so answer success (`acceptAdd`) | +| P (reserved) | write fails, `logEntryLanded` = **false** | P | counter −1 (`releasePending`) | +| P (reserved) | write fails, `logEntryLanded` = **null** | P | counter untouched, error surfaced | +| P/D | fold applies an entry | D | receipt written in the same partition commit | +| D | fold absorbs a batch | P/C | retire `S` = applied ∪ receipt-present ∪ cross-generation; counter − | S | ; delete `S` **after** commit | +| D | delete fails | P | counter re-read from the log (`reconcilePending`), never by subtraction again | +| R | any fold or drain pass | C | counter −1 per pass (`releasePending`), reported as progress | +| C/R | drain pass with counter 0 | C | no write, no CAS | +| G | fold | C | cross-generation entries retire with the batch, are never applied | +| G | search | G | entries from another generation are filtered out (`pendingForSearch`) | +| any | reset | C | generation barrier first, then `clearLog` (I4) | +| any | capacity read | unchanged | reports the counter, writes nothing | + +## Cross-generation shadowing (state G plus a zero count) + +State **G** is only safe while the count is above zero. Two independent mechanisms +break that, and fixing either alone is not enough: + +- **The read window.** A filtered list must be filtered _while reading_. Listing keys + first and filtering after truncation lets a handful of entries from the old + generation consume the whole window, so `pendingForSearch` returns nothing while + current entries sit in the log. `listPending` therefore lists every key when a + `generation` is given (and stays bounded when it is not). +- **The reset hand-off.** The empty manifest a reset commits used to drop `pending`, + so after a failed log cleanup the surviving stale entries had **objects but no + count**. A fold then retired and subtracted them - entries that were never added to + the count - driving it to zero, where the short-circuit hides everything accepted + after the reset. The reset now carries the count forward and zeroes it only after + `clearLog` succeeded, guarded by the count not having moved so it cannot eat a + concurrent reservation. + +With only the first fix the count still drifts to zero; with only the second the read +window is still starved. The table's T1 is what both are serving: the count may never +fall below what the current generation can read. + +### The read window is bounded, not eliminated + +A filtered listing scans `SCAN_FACTOR * limit` keys (4 × 8 = **32** on the read path), so +more than 32 stale entries can still push current ones out of one window. That window +self-heals rather than staying broken: the reset hand-off carries the stale count, so +the fold trigger sees the backlog and retires those entries, after which the window +fills from current entries again. + +## Invariants + +- **T1 — never below the truth.** `manifest.pending >= |readable entries|` at every + commit. The reverse hides an accepted entry from every search, because + `pendingForSearch` short-circuits on a zero count. +- **T2 — every over-count has a path back.** Over-count is the safe direction but not + a terminal state: a failed write releases (`releasePending`), a failed delete + re-reads (`reconcilePending`), and a drain releases one slot per pass (`foldPending` + empty-log branch). Without T2 an over-count below `FOLD_TRIGGER` has no trigger that + would ever notice it. +- **T3 — bounded convergence.** Repeated `POST /admin/fold` reaches `C` in + `ceil(pending / FOLD_BATCH) + 1` calls, and answers 409 for every call before the + last one. +- **T4 — absence means zero.** `pending` is deleted, never stored as 0; the format is + v4 only while an entry is pending, so a drained store is byte-identical to one that + never had a log. +- **T5 — idempotency is carried by names, not counts.** Replaying an accept is guarded + by the entry name and the receipt, never by the counter, so a wrong count can cost a + listing but cannot duplicate a write. + +Every transition above names the function that performs it, and every invariant names +the assertion that fails when it is broken: + +| invariant | enforced by | failing assertion | +| --------- | ------------------------------------------------------- | --------------------------------------------------- | +| T1 | reserve-before-write, three-state landed check | `an accepted but unfolded entry must be searchable` | +| T2 | `releasePending`, `reconcilePending`, empty-log release | `a single residue slot must report clean` | +| T3 | `foldRoute` 8-round bound + 409 | `one slot is left after a full pass` | +| T4 | `setPending` deletes at zero; `manifestFormat` | byte-identity test in `test-manifest-pending.mjs` | +| T5 | entry name + receipt | `replay must survive folding` | + +## What this table does not cover + +- **A physically failing write.** The three-state answer is tested with a fake bucket; + the condition that selects it (an object PUT throwing on a real store) is not + exercised end to end. +- **Concurrent folds on one user.** Manifest CAS serializes them, so the table assumes + one commit at a time; a lost CAS re-runs the transition from the top, which the + table relies on rather than proves. +- **A count corrupted outside this code.** `loadManifest` rejects a non-integer count + as `storage_corrupt`; anything else writable by hand is out of scope. diff --git a/docs/design/add-log-pending-counter-state.terms.yaml b/docs/design/add-log-pending-counter-state.terms.yaml new file mode 100644 index 00000000..c9c1a952 --- /dev/null +++ b/docs/design/add-log-pending-counter-state.terms.yaml @@ -0,0 +1,3 @@ +version: 2 +terms: [] +references: [] diff --git a/docs/design/add-log-pending-counter-state.zh-CN.md b/docs/design/add-log-pending-counter-state.zh-CN.md new file mode 100644 index 00000000..8c7124f6 --- /dev/null +++ b/docs/design/add-log-pending-counter-state.zh-CN.md @@ -0,0 +1,115 @@ +# pending 计数器:状态、迁移与不变量 + +**Status:** draft +**Created:** 2026-10-28 + +属于 [Add 写不可变日志条目](add-append-log-and-fold-design.zh-CN.md) 的一部分,设计由那篇负责。 +本页只负责一个部件:`manifest.pending`——决定"一次检索要不要看日志""一次请求要不要折叠"的那个计数。 + +## 为什么需要一张表 + +日志路径上线时,计数器是"加一、然后祈祷"的值。四轮复核在同一部件里找出六个缺陷,每一个都是 +**没有人写下来的那次迁移**: + +| 缺陷 | 形状 | +| -------------------------------------------- | ---------------------------------------- | +| 折叠把 `{name, entry}` 当字符串迭代 | 条目从不被应用,删除也不发生 | +| 计数只减"已应用"的条数 | 回执已存在、跨代的条目永不退休 | +| `Object.assign(manifest, withPending(m, 0))` | 归零永远写不进 manifest | +| 删除失败时计数已扣 | 同一批被扣两次 | +| 先写条目、后提交计数 | 盘上有条目而计数不含它,被搜索短路藏起来 | +| 把"抛错"当成"没落盘" | 为一条已在盘上的条目释放了预留 | + +这些一旦画在纸上都不算难。这一页存在的意义就是这个。 + +## 状态 + +`pending` 与日志是同一件事的两个观测,所以状态是"对": + +| 状态 | `manifest.pending` | 日志 | 由什么到达 | +| ------------------ | ------------------ | ---------------- | -------------------- | +| **C** 干净 | 缺键 | 空 | 所有路径最终汇合于此 | +| **P** 正常积压 | `n > 0` | `m` 条,`n >= m` | 受理快于折叠 | +| **D** 折叠中 | `n > 0` | `m` 条正在被折 | 一次折叠在途 | +| **R** 计数偏高残留 | `n > 0` | 空 | 预留了但写入从未落地 | +| **G** 跨代残留 | `n > 0` | 上一代的条目 | reset 与折叠竞争 | + +**刻意不存在"计数偏低"状态**(`pending` 缺键而日志非空)。在"先写条目、后提交计数"的顺序下它 +可到达;改成**先留位、再写**之后它不可到达——这正是那个顺序的理由。**看到它就说明有 bug**: +检索会跳过日志并藏起数据。 + +## 迁移 + +`E` = 当前代可读条目集合,`pending` = 存下的计数。每行都写明这次迁移必须留下什么。 + +| 从 | 事件 | 到 | 必须留下 | +| ----------- | ------------------------------------- | ---- | ----------------------------------------------------------------------- | +| C | 受理 | P | 计数**先于**条目持久化提交(`acceptAdd`) | +| P/D | 受理 | P/D | 计数 +1,登记预留(`acceptAdd`) | +| P(已预留) | 写失败且 `logEntryLanded` = **true** | P | 条目已在盘上 → 按成功返回 | +| P(已预留) | 写失败且 `logEntryLanded` = **false** | P | 计数 −1(`releasePending`) | +| P(已预留) | 写失败且 `logEntryLanded` = **null** | P | 计数不动,抛出原错误 | +| P/D | 折叠应用一条 | D | 回执与分区在同一次提交里写入 | +| D | 折叠吸收一批 | P/C | 退休 `S` = 已应用 ∪ 回执已存在 ∪ 跨代;计数 − \|S\|;**提交之后**删 `S` | +| D | 删除失败 | P | 回读日志重算(`reconcilePending`),不再用减法 | +| R | 任意折叠/排空一轮 | C | 每轮释放一格(`releasePending`),并作为进展上报 | +| C/R | 计数为 0 的排空轮 | C | 不写、不 CAS | +| G | 折叠 | C | 跨代条目随批次退休,绝不应用 | +| G | 检索 | G | 其他代的条目被过滤掉(`pendingForSearch`) | +| 任意 | reset | C | 先立代际屏障,再 `clearLog`(I4) | +| 任意 | 容量读取 | 不变 | 只报告计数,不写 | + +## 跨代遮蔽(状态 G 且计数为零) + +状态 **G** 只在计数大于零时才安全。两条独立机制会破坏它,只修任一条都不成立: + +- **读取窗口。** 带过滤的列举必须在**读取时**过滤。先按键截断再过滤,会让少量旧代 + 条目占满整个窗口,`pendingForSearch` 返回空,而当前代的条目明明躺在日志里。 + 因此 `listPending` 在给定 `generation` 时列举全部键(不给时仍保持有界)。 +- **reset 的交接。** reset 提交的空清单原本会丢掉 `pending`,于是一次失败的日志清理之后, + 遗留的旧代条目**有对象、无计数**。随后折叠会把这些从未计入的条目退休并从计数里减掉, + 把计数推向零;一旦到零,`pendingForSearch` 的短路就会把 reset 之后受理的一切藏起来。 + 现在 reset 把计数**带过去**,并且只在 `clearLog` 成功之后才归零,且以"计数未变动"为条件, + 因此不会吃掉并发受理的预留。 + +只落第一条,计数仍会漂到零;只落第二条,读取窗口仍被饿死。两条都在服务同一条 T1: +计数不得低于当前代可读的条目数。 + +### 读取窗口是"有界"而不是"消除" + +带过滤的列举会扫描 `SCAN_FACTOR * limit` 个键(读取路径是 4 × 8 = **32**),所以遗留条目 +超过 32 条时,当前代的条目仍可能被挤出单个窗口。它会**自愈**而不是永久失效:reset 交接把 +遗留条目的数量带进了计数,折叠触发器因此看到积压并退休这些条目,之后窗口重新由当前代条目填满。 + +## 不变量 + +- **T1 —— 绝不低于真值。** 每次提交都满足 `manifest.pending >= |可读条目|`。反向意味着 + 一条已受理的条目被每次检索挡住,因为 `pendingForSearch` 在计数为 0 时直接短路。 +- **T2 —— 每个偏高都有回程。** 偏高是安全方向,但不是终态:写失败释放(`releasePending`)、 + 删除失败回读(`reconcilePending`)、排空时空日志每轮释放一格。没有 T2,低于 + `FOLD_TRIGGER` 的偏高没有任何触发器会注意到它。 +- **T3 —— 有界收敛。** 反复调用 `POST /admin/fold` 能在 `ceil(pending / FOLD_BATCH) + 1` + 次内到达 `C`,且最后一次之前的每次调用都返回 409。 +- **T4 —— 缺失即零。** `pending` 用删键表达,不存 0;格式仅在有条目待处理时是 v4,因此排空后 + 的存储与"从未有过日志"的部署逐字节相同。 +- **T5 —— 幂等由名字承载,不由计数承载。** 重放靠条目名与回执守卫,不当依赖计数,所以计数错了 + 最多多花一次列举,不可能重复写入。 + +上表每行都点名了执行它的函数,每条不变量都点名了破坏它时会失败的那条断言: + +| 不变量 | 由谁保证 | 失败时红的断言 | +| ------ | -------------------------------------------------- | --------------------------------------------------- | +| T1 | 先留位后写、落地三态判定 | `an accepted but unfolded entry must be searchable` | +| T2 | `releasePending` / `reconcilePending` / 空日志释放 | `a single residue slot must report clean` | +| T3 | `foldRoute` 的 8 轮上界 + 409 | `one slot is left after a full pass` | +| T4 | `setPending` 归零删键;`manifestFormat` | `test-manifest-pending.mjs` 的字节一致性断言 | +| T5 | 条目名 + 回执 | `replay must survive folding` | + +## 这张表没有覆盖什么 + +- **真实的写失败。** 三态判定用假 bucket 测过;让它被选中的条件(真实存储上对象 PUT 抛错) + 没有端到端跑过。 +- **同一用户的并发折叠。** manifest CAS 把它们串行化,所以表里假设一次只有一个提交;CAS 落败 + 时整次迁移从头重跑——表依赖这一点,而不是证明它。 +- **代码之外被改坏的计数。** `loadManifest` 会把非整数计数判为 `storage_corrupt`,其余 + "手工写成别的值"不在范围内。 diff --git a/evals/omnimemeval/research/probes/query-chunk-pooling-ab.ts b/evals/omnimemeval/research/probes/query-chunk-pooling-ab.ts index 9097fe6b..ed06419b 100644 --- a/evals/omnimemeval/research/probes/query-chunk-pooling-ab.ts +++ b/evals/omnimemeval/research/probes/query-chunk-pooling-ab.ts @@ -20,13 +20,95 @@ import { DatabaseSync } from "node:sqlite"; import { parseArgs } from "node:util"; import { cosineSimilarity } from "../../../../src/core/vector.ts"; -import { poolQueryVectors, splitQueryChunks } from "../../../../onlion/backend/query-chunks.mjs"; /** The bge-en query template: chunk vectors must come from the same view as the * baked record vectors or the comparison would measure a template change. */ const QUERY_PREFIX = "Represent this sentence for searching relevant passages: "; -const CONDITIONS = ["short", "long", "mixed", "tail"] as const; -type Condition = (typeof CONDITIONS)[number]; + +/** Unit-length mean of window vectors. Kept here, not imported: the product no + * longer pools windows, and this arm exists only as the record of what was + * compared. */ +function poolQueryVectors(vectors: Float32Array[]): Float32Array { + const first = vectors[0]; + if (!first) throw new Error("poolQueryVectors requires at least one vector"); + const pooled = new Float32Array(first.length); + for (const vector of vectors) for (let i = 0; i < first.length; i += 1) pooled[i] += vector[i]; + let norm = 0; + for (let i = 0; i < pooled.length; i += 1) { + pooled[i] /= vectors.length; + norm += pooled[i] * pooled[i]; + } + norm = Math.sqrt(norm); + if (norm) for (let i = 0; i < pooled.length; i += 1) pooled[i] /= norm; + return pooled; +} + +/** Arm E's layout, kept here as the baseline it measures against: tile forward from + * the start with boundary-snapped cuts, then append one end-anchored window. This is + * the shape the product shipped before the planned layout replaced it, so the probe + * must keep its own copy rather than importing whatever the product does today. */ +const LEGACY_CHUNK_CHARS = 2000; +const LEGACY_MAX_CHUNKS = 8; +function legacyBoundaryBefore(window: string): number { + const candidates = [ + window.lastIndexOf("\n"), + window.lastIndexOf(". "), + window.lastIndexOf("? "), + window.lastIndexOf("! "), + window.lastIndexOf("。"), + window.lastIndexOf(";"), + window.lastIndexOf(","), + ].filter((index) => index > 0); + const best = Math.max(-1, ...candidates); + return best > window.length / 2 ? best + 1 : window.length; +} +function legacyFirstBoundary(window: string): number { + const candidates = [ + window.indexOf("\n"), + window.indexOf(". "), + window.indexOf("? "), + window.indexOf("! "), + window.indexOf("。"), + window.indexOf(";"), + window.indexOf(","), + ].filter((index) => index > 0); + const best = candidates.length === 0 ? -1 : Math.min(...candidates); + return best > 0 && best < window.length / 2 ? best + 1 : 0; +} +function legacyForwardWindows(text: string): string[] { + const size = LEGACY_CHUNK_CHARS; + const limit = LEGACY_MAX_CHUNKS; + if (text.length <= size) return [text]; + const chunks: string[] = []; + let start = 0; + while (start < text.length && chunks.length < Math.max(1, limit - 1)) { + if (chunks.length > 0 && text.length - start <= size / 4) { + chunks[chunks.length - 1] += text.slice(start); + break; + } + const window = text.slice(start, Math.min(start + size, text.length)); + const cut = legacyBoundaryBefore(window); + chunks.push(text.slice(start, start + cut)); + start += cut; + } + if (start >= text.length) return chunks; + if (text.length - start < size / 4) { + chunks[chunks.length - 1] += text.slice(start); + return chunks; + } + const tail = text.slice(Math.max(0, text.length - size)); + const from = legacyFirstBoundary(tail); + chunks.push(from > 0 ? tail.slice(from) : tail); + return chunks; +} + +const CONDITIONS_BASE = ["short", "long", "mixed", "tail", "mid"] as const; +/** Built in `main`: the base conditions, or a position sweep when `--offsets` is + * given. A sweep is what separates the two layouts: E tiles forward from the start, + * so how well it does depends on *where* the informative part sits, while a plan + * that spreads its windows should be nearly flat. */ +let CONDITIONS: string[] = [...CONDITIONS_BASE]; +type Condition = string; interface Qa { category?: number | string; @@ -50,16 +132,27 @@ interface Row { cHits?: number; dHits?: number; eHits?: number; + fHits?: number; + gHits?: number; + hHits?: number; aAll?: number; cAll?: number; dAll?: number; eAll?: number; + fAll?: number; + gAll?: number; + hAll?: number; identical?: boolean; } interface Options { page: number; candidates: number; longChars: number; + /** Force a segment count for arm F instead of `ceil(L / W)`; 0 keeps the + * adaptive count. Sweeping it answers "how long should one window be". */ + planCount: number; + /** Percentage offsets for the position sweep; empty uses the base conditions. */ + offsets: number[]; /** Which vector strategies to compute: `a` mean-pool, `c` max-rerank over a * pooled candidate pool, `d` no splitting, `e` search with the head vector and * every chunk as separate features, reduced by max. Only the requested arms are @@ -213,6 +306,55 @@ function rankByChunkMax( .map((row) => row.id); } +/** Last measurement arm: the window layout itself, independent of pooling. + * + * F keeps the head and the tail by construction and never overlaps: with + * `n = min(ceil(L / W), maxWindows)` windows of budget `W`, + * - `L <= n*W` (the budget covers the text) cuts the text into `n` equal, + * contiguous, disjoint segments that share one boundary-snapped cut point per + * seam, so nothing is dropped and nothing is embedded twice; + * - `L > n*W` (budget-bound) uses `n` windows of `W`, the first at 0 and the last + * anchored at the end, with the uncovered text spread evenly between them + * instead of leaving one block at the far end unsearched. + * + * E, by contrast, tiles forward from the start and then appends the end-anchored + * window, so its blind region is a single contiguous block in the middle. */ +const BOUNDARIES = ["\n", ". ", "? ", "! "]; +function snapCut(text: string, ideal: number, previous: number, end: number): number { + const ceiling = Math.min(end - 1, ideal + 400); + for (let at = Math.max(ideal, previous + 1); at <= ceiling; at += 1) { + const tail = text.slice(at, at + 2); + if (BOUNDARIES.some((boundary) => tail.startsWith(boundary))) return at + 1; + } + return Math.min(Math.max(ideal, previous + 1), ceiling); +} +function planQueryWindows( + text: string, + windowChars = 2000, + maxWindows = 9, + forcedCount = 0, +): string[] { + const length = text.length; + if (length <= windowChars) return [text]; + const count = + forcedCount > 0 + ? Math.min(forcedCount, Math.max(2, maxWindows)) + : Math.min(Math.ceil(length / windowChars), Math.max(2, maxWindows)); + if (length <= count * windowChars) { + const cuts = [0]; + for (let i = 1; i < count; i += 1) + cuts.push(snapCut(text, Math.round((i * length) / count), cuts[i - 1]!, length)); + cuts.push(length); + const segments = cuts.slice(1).map((end, i) => text.slice(cuts[i]!, end)); + return segments.filter((segment) => segment.length > 0); + } + const stride = (length - count * windowChars) / (count - 1) + windowChars; + return Array.from({ length: count }, (_unused, i) => { + const start = i === 0 ? 0 : i === count - 1 ? length - windowChars : Math.round(i * stride); + return text.slice(start, start + windowChars); + }); +} + function recallOf(gold: string[][], page: string[]): { hits: number; total: number; all: number } { const set = new Set(page); let hits = 0; @@ -252,7 +394,8 @@ function queryTexts( question: string, others: string[], longChars: number, -): Record { + offsets: number[], +): Record { const padding = fillerText(Math.max(0, longChars - question.length)); const mixed = [question, ...others.filter((text) => text !== question).slice(0, 6)] .join("\n") @@ -261,79 +404,149 @@ function queryTexts( // `tail` puts the question after the padding: with no splitting the model only // sees its head window, so this condition separates "truncation works" from // "truncation worked because the question happened to be first". - return { + const texts: Record = { short: question, long: `${question}\n${padding}`, mixed, tail: `${padding}\n${question}`, + // A percentage condition: the question sits at that fraction of the text. The + // padding is filler either way, so the only thing that moves is the offset. + mid: midText(question, longChars, 0.5), }; + for (const offset of offsets) + texts[`pos-${String(offset).padStart(3, "0")}`] = midText(question, longChars, offset / 100); + return texts; } -/** Three arms per condition: - * A = mean-pooled chunk vector, B rounds the page (shipped); - * C = same pooled vector for a candidate pool, reranked by per-chunk max; - * D = no chunking at all — the whole query as one text, so the embedding model - * applies its own token window. D is the baseline for "is chunking worth it". */ -/** A = mean-pooled chunk vector (shipped); C = pooled candidate pool reranked by - * per-chunk max; D = no chunking at all — the whole query as one text, so the - * embedding model applies its own token window. D is the baseline for "is - * chunking worth it". */ -async function evaluateQuestion( - question: string, - gold: string[][], - others: string[], - docs: Doc[], +/** Question at `fraction` of the padded text. */ +function midText(question: string, longChars: number, fraction: number): string { + if (longChars <= question.length + 2) return question; + const before = Math.round(Math.min(Math.max(fraction, 0), 1) * (longChars - question.length - 1)); + const after = Math.max(0, longChars - question.length - 1 - before); + return `${fillerText(before)}\n${question}\n${fillerText(after)}`; +} + +/** Feature texts for every condition, and their vectors. + * + * Each arm needs a different subset, so the sets are gathered once per question and + * sliced back in the same order. Keeping the gathering here leaves the per-condition + * loop reading as a list of arms rather than a list of bookkeeping. */ +async function prepareVectors( + texts: Record, embedBase: string, options: Options, -): Promise { - const texts = queryTexts(question, others, options.longChars); +) { const wantChunks = options.arms.has("a") || options.arms.has("c") || options.arms.has("e"); - const wantWhole = options.arms.has("d") || options.arms.has("e"); + const wantWhole = + options.arms.has("d") || + options.arms.has("e") || + options.arms.has("g") || + options.arms.has("h"); + const wantPlan = options.arms.has("f") || options.arms.has("g") || options.arms.has("h"); const chunksBy = Object.fromEntries( CONDITIONS.map((condition) => [ condition, - wantChunks ? (splitQueryChunks(texts[condition]) as string[]) : [], + wantChunks ? legacyForwardWindows(texts[condition]) : [], ]), ) as Record; - const chunkTexts = wantChunks - ? CONDITIONS.flatMap((condition) => chunksBy[condition]) - : ([] as string[]); + const planBy = Object.fromEntries( + CONDITIONS.map((condition) => [ + condition, + wantPlan ? planQueryWindows(texts[condition], 2000, 9, options.planCount) : [], + ]), + ) as Record; + const chunkTexts = CONDITIONS.flatMap((condition) => chunksBy[condition]); const wholeTexts = wantWhole ? CONDITIONS.map((condition) => texts[condition]) : []; - const vectors = await embed(embedBase, [...chunkTexts, ...wholeTexts]); - const whole = vectors.slice(chunkTexts.length); + const planTexts = wantPlan ? CONDITIONS.flatMap((condition) => planBy[condition]) : []; + const vectors = await embed(embedBase, [...chunkTexts, ...wholeTexts, ...planTexts]); + return { + chunksBy, + planBy, + chunks: vectors.slice(0, chunkTexts.length), + whole: vectors.slice(chunkTexts.length, chunkTexts.length + wholeTexts.length), + plans: vectors.slice(chunkTexts.length + wholeTexts.length), + wantChunks, + wantWhole, + wantPlan, + }; +} + +interface ArmInput { + gold: string[][]; + docs: Doc[]; + options: Options; + whole?: Float32Array; + chunks: Float32Array[]; + plans: Float32Array[]; + arms: Set; +} + +/** Every arm's page for one condition. One function per arm would be six functions of + * four lines; the branches below are the arms themselves. */ +function scoreCondition(input: ArmInput): Partial { + const { gold, docs, options, whole, chunks, plans } = input; + const { page, arms } = options; + const row: Partial = {}; + const record = (name: string, recall: ReturnType) => { + row[`${name}Hits` as keyof Row] = recall.hits as never; + row[`${name}All` as keyof Row] = recall.all as never; + }; + if (arms.has("a") || arms.has("c") || arms.has("e")) { + const pooled = chunks.length === 1 ? chunks[0]! : poolQueryVectors(chunks); + const aPage = rankByVector(pooled, docs, page); + record("a", recallOf(gold, aPage)); + if (arms.has("c")) { + const cPage = rankByChunkMax(pooled, chunks, docs, options); + record("c", recallOf(gold, cPage)); + row.identical = aPage.join(",") === cPage.join(","); + } + } + if (arms.has("d") && whole) record("d", recallOf(gold, rankByVector(whole, docs, page))); + if (arms.has("e") && whole) + record("e", recallOf(gold, rankByFeatureMax([whole, ...chunks], docs, page).page)); + if (arms.has("f")) record("f", recallOf(gold, rankByFeatureMax(plans, docs, page).page)); + if (arms.has("g") && whole) + record("g", recallOf(gold, rankByFeatureMax([whole, ...plans], docs, page).page)); + // H drops the first planned window: the clause itself already covers the opening + // with the model's own token window, which is longer than one character window. + if (arms.has("h") && whole) + record("h", recallOf(gold, rankByFeatureMax([whole, ...plans.slice(1)], docs, page).page)); + return row; +} + +async function evaluateQuestion( + question: string, + gold: string[][], + others: string[], + docs: Doc[], + embedBase: string, + options: Options, +): Promise { + const texts = queryTexts(question, others, options.longChars, options.offsets); + const prepared = await prepareVectors(texts, embedBase, options); const rows: Row[] = []; - let offset = 0; + let chunkOffset = 0; + let planOffset = 0; for (const [index, condition] of CONDITIONS.entries()) { - const count = chunksBy[condition].length; - const chunks = vectors.slice(offset, offset + count); - offset += count; - const row: Row = { condition, total: gold.length }; - if (wantChunks) { - const pooled = count === 1 ? chunks[0]! : poolQueryVectors(chunks); - const aPage = rankByVector(pooled, docs, options.page); - const a = recallOf(gold, aPage); - row.aHits = a.hits; - row.aAll = a.all; - if (options.arms.has("c")) { - const cPage = rankByChunkMax(pooled, chunks, docs, options); - const c = recallOf(gold, cPage); - row.cHits = c.hits; - row.cAll = c.all; - row.identical = aPage.join(",") === cPage.join(","); - } - } - if (wantWhole) { - const d = recallOf(gold, rankByVector(whole[index]!, docs, options.page)); - row.dHits = d.hits; - row.dAll = d.all; - } - if (options.arms.has("e")) { - const { page } = rankByFeatureMax([whole[index]!, ...chunks], docs, options.page); - const e = recallOf(gold, page); - row.eHits = e.hits; - row.eAll = e.all; - } - rows.push(row); + const chunkCount = prepared.chunksBy[condition].length; + const planCount = prepared.planBy[condition].length; + const chunks = prepared.chunks.slice(chunkOffset, chunkOffset + chunkCount); + const plans = prepared.plans.slice(planOffset, planOffset + planCount); + chunkOffset += chunkCount; + planOffset += planCount; + rows.push({ + condition, + total: gold.length, + ...scoreCondition({ + gold, + docs, + options, + whole: prepared.whole[index], + chunks, + plans, + arms: options.arms, + }), + }); } return rows; } @@ -369,20 +582,23 @@ function summarize(rows: Row[]): Array> { 1, slice.reduce((sum, row) => sum + row.total, 0), ); - const pct = (key: "aHits" | "cHits" | "dHits" | "eHits") => { + const pct = (key: "aHits" | "cHits" | "dHits" | "eHits" | "fHits" | "gHits" | "hHits") => { const present = slice.filter((row) => row[key] !== undefined); if (present.length === 0) return null; const hits = present.reduce((sum, row) => sum + Number(row[key]), 0); return Number(((100 * hits) / totalLabels).toFixed(2)); }; - const share = (key: "aAll" | "cAll" | "dAll" | "eAll") => { + const share = (key: "aAll" | "cAll" | "dAll" | "eAll" | "fAll" | "gAll" | "hAll") => { const present = slice.filter((row) => row[key] !== undefined); if (present.length === 0) return null; return Number( ((100 * present.reduce((sum, row) => sum + Number(row[key]), 0)) / n).toFixed(2), ); }; - const paired = (left: "aAll" | "cAll" | "eAll", right: "dAll" | "aAll" | "cAll") => { + const paired = ( + left: "aAll" | "cAll" | "eAll" | "fAll" | "gAll" | "hAll" | "dAll", + right: "dAll" | "aAll" | "cAll" | "eAll" | "fAll" | "gAll" | "hAll", + ) => { const usable = slice.filter((row) => row[left] !== undefined && row[right] !== undefined); if (usable.length === 0) return "-"; const wins = usable.filter((row) => Number(row[left]) > Number(row[right])).length; @@ -394,6 +610,12 @@ function summarize(rows: Row[]): Array> { const eOverD = paired("eAll", "dAll"); const eOverA = paired("eAll", "aAll"); const eOverC = paired("eAll", "cAll"); + const fOverD = paired("fAll", "dAll"); + const fOverE = paired("fAll", "eAll"); + const gOverE = paired("gAll", "eAll"); + const gOverF = paired("gAll", "fAll"); + const hOverG = paired("hAll", "gAll"); + const hOverE = paired("hAll", "eAll"); return { condition, n: slice.length, @@ -401,16 +623,28 @@ function summarize(rows: Row[]): Array> { cRecall: pct("cHits"), dRecall: pct("dHits"), eRecall: pct("eHits"), + fRecall: pct("fHits"), + gRecall: pct("gHits"), + hRecall: pct("hHits"), aAll: share("aAll"), cAll: share("cAll"), dAll: share("dAll"), eAll: share("eAll"), + fAll: share("fAll"), + gAll: share("gAll"), + hAll: share("hAll"), identical: slice.filter((row) => row.identical).length, cOverD, aOverD, eOverD, eOverA, eOverC, + fOverD, + fOverE, + gOverE, + gOverF, + hOverG, + hOverE, }; }); } @@ -426,6 +660,8 @@ async function main(): Promise { page: { type: "string", default: "20" }, candidates: { type: "string", default: "100" }, "long-chars": { type: "string", default: "6000" }, + offsets: { type: "string", default: "" }, + "plan-count": { type: "string", default: "0" }, arms: { type: "string", default: "a,c,d" }, }, }); @@ -435,6 +671,12 @@ async function main(): Promise { page: Number(values.page), candidates: Number(values.candidates), longChars: Number(values["long-chars"]), + planCount: Number(values["plan-count"]), + offsets: values.offsets + .split(",") + .filter((value) => value.trim().length > 0) + .map((value) => Number(value.trim())) + .filter((value) => Number.isFinite(value) && value >= 0 && value <= 100), arms: new Set( values.arms .split(",") @@ -448,6 +690,13 @@ async function main(): Promise { embed: values.embed.replace(/\/$/u, ""), }; const samples = JSON.parse(fs.readFileSync(values.data, "utf8")) as Sample[]; + // A position sweep replaces the long-form conditions: the offsets themselves are + // the variable, and the short control stays so the sweep can be sanity-checked. + if (options.offsets.length > 0) + CONDITIONS = [ + "short", + ...options.offsets.map((offset) => `pos-${String(offset).padStart(3, "0")}`), + ]; const rows: Row[] = []; for (const sample of samples) rows.push(...(await runSample(sample, paths, options))); const summary = summarize(rows); @@ -463,7 +712,10 @@ async function main(): Promise { `A(pool) all=${row.aAll}% rec=${row.aRecall}% | ` + `C(max-rerank) all=${row.cAll}% rec=${row.cRecall}% | ` + `E(head+chunks max) all=${row.eAll}% rec=${row.eRecall}% | ` + - `E>D ${row.eOverD} | E>A ${row.eOverA} | E>C ${row.eOverC}`, + `F(plan windows) all=${row.fAll}% rec=${row.fRecall}% | ` + + `G(whole+plan) all=${row.gAll}% rec=${row.gRecall}% | ` + + `H(whole+plan-no-first) all=${row.hAll}% rec=${row.hRecall}% | ` + + `E>D ${row.eOverD} | G>E ${row.gOverE} | H>G ${row.hOverG}`, ); console.log(`wrote ${values.out}`); } diff --git a/onlion/backend/append-log.mjs b/onlion/backend/append-log.mjs new file mode 100644 index 00000000..0fd7a1f1 --- /dev/null +++ b/onlion/backend/append-log.mjs @@ -0,0 +1,345 @@ +// Append-only Add log: one immutable object per accepted Add, folded into +// partitions afterwards (docs/design/add-append-log-and-fold-design.md). +// +// The object name is the idempotency key, so an entry is written with +// x-cos-forbid-overwrite and can never be replaced: a retry of the same +// request_id finds the existing entry instead of overwriting it. Names sort +// lexicographically in arrival order, which is the order folding applies them. +// +// This module only uses the injected bucket contract (get/head/put/delete/list) +// and must never import worker.mjs, storage.mjs or SQLite: it is the seam that +// lets the accept path, the fold and the read path be built in parallel. +// +// Errors: a value outside its allowed range throws RangeError, a wrong type throws +// TypeError, and an object inside the log prefix whose name is not a log entry is +// treated as corruption (`log_entry_name`) rather than skipped - hiding it would make +// a naming change look like "Add answers 200 but nothing is searchable". +// +// Because every reader validates that name, the `log/` prefix is reserved for entries +// named by logEntryName and nothing else: a sidecar file (metadata, markers, compaction +// state) placed inside it would fail accept, search and fold for that user at once, so +// future additions belong beside the prefix (`users//log-meta.json`) instead. The +// only way to clear a stray object is a reset, which deletes the prefix wholesale. +import { Buffer } from 'node:buffer'; + +export const LOG_ENTRY_VERSION = 1; // must equal the HTTP payloads' `v` +export const MAX_PENDING_READ = 8; // pending entries scored per search + +// received_ms is padded to a fixed 13 digits (enough for any millisecond +// timestamp until year 2286), so plain string order is arrival order. +const SEQ_DIGITS = 13; +const MAX_RECEIVED_MS = 10 ** SEQ_DIGITS - 1; +const REQUEST_HASH = /^[0-9a-f]{64}$/; +const LIST_PAGE = 1000; // COS list page size +const READ_CONCURRENCY = 4; +// A generation-filtered read may look past the first `limit` entries, but never +// past a multiple of it: a log stuffed with entries from an older generation must +// cost bounded reads instead of a scan of the whole prefix. +const SCAN_FACTOR = 4; + +export function logPrefix(userKey) { + if (!isPathSegment(userKey)) throw new RangeError(`Invalid user key: ${String(userKey)}`); + return `users/${userKey}/log/`; +} + +export function logEntryName(receivedAt, requestHash) { + const sequence = String(receivedMs(receivedAt)).padStart(SEQ_DIGITS, '0'); + return `${sequence}-${requestHashSuffix(requestHash)}.json`; +} + +/** Durably append one entry. `created: false` means an entry with the same + * request_hash was already accepted, so the caller replays instead of writing. */ +export async function putLogEntry(bucket, userKey, entry) { + if (entry?.v !== LOG_ENTRY_VERSION) + throw new RangeError(`Unsupported log entry version: ${String(entry?.v)}`); + const name = logEntryName(entry.received_at, entry.request_hash); + const written = await bucket.put( + logPrefix(userKey) + name, + JSON.stringify(entry), + { onlyIf: { etagDoesNotMatch: '*' }, httpMetadata: { contentType: 'application/json' } }, + ); + return { name, created: !!written }; +} + +/** `null` means the entry is gone (folded, reset or never written). A missing + * entry is an ordinary answer; a corrupt or newer-version one is not. */ +export async function readLogEntry(bucket, userKey, name) { + const object = await bucket.get(logPrefix(userKey) + stripPrefix(userKey, name)); + if (!object) return null; + const entry = await object.json(); + if (entry?.v !== LOG_ENTRY_VERSION) + throw logError('log_entry_version', `Unsupported log entry version in ${name}`, name); + if (!REQUEST_HASH.test(entry.request_hash ?? '')) + throw logError('log_entry_corrupt', `Log entry ${name} has no valid request_hash`, name); + return entry; +} + +/** Did this entry reach the bucket? + * + * Three states on purpose: `null` means the question could not be answered, and a + * caller that reads "unknown" as "absent" releases a reservation for an entry that + * is already on disk - a counter below the truth, which is the one direction that + * hides accepted data from search. Existence is asked with HEAD, not by reading the + * entry: `readLogEntry` throws on an entry it cannot parse, and that error says + * nothing about whether the object was written. + * + * Exported beyond the frozen set on purpose: the write path needs it to decide + * whether a failed write actually failed. */ +export async function logEntryLanded(bucket, userKey, entry) { + try { + const name = logEntryName(entry.received_at, entry.request_hash); + const object = await bucket.head(logPrefix(userKey) + name); + return !!object; + } catch { + // A failed HEAD, or an entry that cannot be named, is "unknown", never "absent". + return null; + } +} + +/** Finds the oldest entry carrying one request_hash. The oldest match wins so a + * retry that recomputed received_at still resolves to the accepted entry. + * + * `generation`, when given, skips entries written under another generation: after a + * reset the old echo must not be replayed to a client whose data was wiped. The + * listing is deliberately unbounded - a replay lookup has to be able to report a + * real miss, so it cannot stop at the first page. + * + * Skipping a match and continuing is required, not defensive: one request_hash can + * legitimately have an entry in two generations (accepted before a reset, replayed + * after it, with the old object still present while `clearLog` finishes). A filter + * that stopped at the first match would drop the old entry and then report a miss, + * so a later retry would be accepted a third time instead of resolving to the entry + * the current generation already holds. The same loop covers a match whose entry + * vanished between list and read; that case only avoids manufacturing a redundant + * object, because a receipt written by a fold is checked before this lookup and + * stays the authoritative duplicate guard. */ +export async function findLogEntry(bucket, userKey, requestHash, options = {}) { + const wanted = requestHashSuffix(requestHash); + const generation = options?.generation; + const prefix = logPrefix(userKey); + for (const object of await listObjects(bucket, userKey)) { + const name = object.key.slice(prefix.length); + // The whole name, not just the hash suffix: anything that logEntryName could not + // have produced is corpus corruption and must not be reported later as a content + // error by `readLogEntry`. + const hash = requestHashFromName(name); + if (hash === null) + throw logError( + 'log_entry_name', + `Object in the log prefix is not named like a log entry: ${name}`, + name, + ); + if (hash !== wanted) continue; + const entry = await readLogEntry(bucket, userKey, name); + if (!entry) continue; + if (generation !== undefined && (entry.generation ?? null) !== (generation ?? null)) continue; + return { name, entry }; + } + return null; +} + +/** Pending entries in arrival order. `limit` defaults to MAX_PENDING_READ (the + * read path's cap); pass Infinity for the fold path, which wants everything. + * + * Two filters decide which entries the caller can actually use, and both run + * *while reading* rather than after the truncation: + * - `generation`: drops entries taken under another generation. + * - `skipRequestHash(hash)`: drops entries the caller is finished with. The search + * path passes a receipt lookup here, so an entry that is folded but not yet + * deleted costs one name inspection and zero body reads. + * Truncating first would let a handful of such entries consume the whole window + * and hide the current ones, so any filter widens the scan to + * max(MAX_PENDING_READ, SCAN_FACTOR * limit) keys; past that a log full of + * filtered-out entries returns fewer matches instead of paging through the whole + * prefix. Callers that need the log's truth (the fold batch, and the reconcile + * count) must pass neither filter. + * + * Every listed name is validated, so a stray object under the log prefix fails every + * caller with `log_entry_name` rather than being ignored or reported as a content + * error. That is deliberate fail-loud behaviour, and it does not clear itself: the + * log prefix is only emptied by a reset (`clearLog`). */ +export async function listPending(bucket, userKey, options = {}) { + const limit = readLimit(options?.limit); + const generation = options?.generation; + const skipRequestHash = options?.skipRequestHash; + if (skipRequestHash !== undefined && typeof skipRequestHash !== 'function') + throw new TypeError('skipRequestHash must be a function'); + const prefix = logPrefix(userKey); + const filtered = generation !== undefined || skipRequestHash !== undefined; + const scan = + limit === Infinity + ? Infinity + : filtered + ? Math.max(MAX_PENDING_READ, limit * SCAN_FACTOR) + : limit; + const objects = await listObjects(bucket, userKey, scan); + const pending = []; + for (let index = 0; index < objects.length && pending.length < limit; index += READ_CONCURRENCY) { + const chunk = objects.slice(index, index + READ_CONCURRENCY); + const read = await Promise.all( + chunk.map(async (object) => { + const name = object.key.slice(prefix.length); + // Every caller parses the name, not just the one with a predicate: an object in + // this prefix that is not named like an entry is corruption, and reporting it as + // `log_entry_name` here keeps the fold and the reconcile read from failing with a + // raw JSON SyntaxError that says nothing about naming. Skipping it instead would + // turn a rename of logEntryName into "Add answers 200, nothing is searchable, and + // the log drains normally". + const hash = requestHashFromName(name); + if (hash === null) + throw logError( + 'log_entry_name', + `Object in the log prefix is not named like a log entry: ${name}`, + name, + ); + // The hash is in the name, so an entry the caller is finished with is skipped + // before its body is fetched: no read, no parse, no embedding downstream. + if (skipRequestHash && skipRequestHash(hash)) return null; + // A concurrent fold or reset may delete an entry between list and get. + const entry = await readLogEntry(bucket, userKey, name); + if (!entry) return null; + if (generation !== undefined && (entry.generation ?? null) !== (generation ?? null)) + return null; + return { name, entry }; + }), + ); + pending.push(...read.filter(Boolean)); + } + return pending.slice(0, limit); +} + +/** Batch delete. Deletion is always after the partition commit, so repeating it + * after a crash must be harmless: deleting a missing key is not an error. */ +export async function deleteLogEntries(bucket, userKey, names) { + const prefix = logPrefix(userKey); + const keys = [ + ...new Set( + (names ?? []) + .filter((name) => typeof name === 'string' && name.length > 0) + .map((name) => (name.startsWith(prefix) ? name : prefix + name)), + ), + ]; + if (keys.length) await bucket.delete(keys); +} + +/** Pure fold budget: the longest arrival-ordered prefix of `entries` that fits + * `limits.maxEntries` and `limits.maxBytes`, measured as the UTF-8 size of the + * entry as it would be re-serialized. The first entry always folds, so one + * oversized entry cannot stall the log forever (it is bounded by the Add + * payload cap instead). Both `{name, entry}` items and bare entries are + * accepted; `keep` preserves whatever shape was passed in. */ +export function foldPlan(entries, limits = {}) { + const list = Array.isArray(entries) ? entries : []; + const maxEntries = budget(limits.maxEntries ?? limits.count, list.length); + const maxBytes = budget(limits.maxBytes ?? limits.bytes, Infinity); + const fold = []; + let bytes = 0; + for (const item of list) { + if (fold.length >= maxEntries) break; + const size = entryBytes(item); + if (fold.length > 0 && bytes + size > maxBytes) break; + fold.push(item); + bytes += size; + } + return { fold, keep: list.slice(fold.length) }; +} + +function receivedMs(value) { + const ms = + typeof value === 'number' + ? value + : typeof value === 'string' + ? Date.parse(value) + : value instanceof Date + ? value.getTime() + : NaN; + if (!Number.isInteger(ms) || ms < 0 || ms > MAX_RECEIVED_MS) + throw new RangeError(`Invalid received_at: ${String(value)}`); + return ms; +} + +function requestHashSuffix(requestHash) { + if (!REQUEST_HASH.test(requestHash ?? '')) + throw new RangeError(`Invalid request_hash: ${String(requestHash)}`); + return requestHash; +} + +// The name has exactly one shape, the one `logEntryName` produces: 13 zero-padded +// millisecond digits, the request hash, and `.json`. A caller that only needs the hash +// (a receipt lookup) reads it from there, and every reader validates the full shape - +// a looser match would let `foo-.json` through and report a naming problem as a +// content problem. +const LOG_NAME = new RegExp(`^(\\d{${SEQ_DIGITS}})-([0-9a-f]{64})\\.json$`); +function requestHashFromName(name) { + const match = LOG_NAME.exec(name); + return match ? match[2] : null; +} + +function readLimit(limit) { + if (limit === undefined) return MAX_PENDING_READ; + if (limit === Infinity) return Infinity; + if (!Number.isInteger(limit) || limit < 1) + throw new RangeError(`Invalid pending limit: ${String(limit)}`); + return limit; +} + +function budget(value, fallback) { + if (value === undefined) return fallback; + if (value === Infinity) return Infinity; + if (!Number.isInteger(value) || value < 0) + throw new RangeError(`Invalid fold budget: ${String(value)}`); + return value; +} + +function entryBytes(item) { + return Buffer.byteLength(JSON.stringify(item?.entry ?? item), 'utf8'); +} + +function isPathSegment(userKey) { + return ( + typeof userKey === 'string' && + userKey.length > 0 && + !/[\\/]/.test(userKey) && + userKey !== '.' && + userKey !== '..' + ); +} + +function stripPrefix(userKey, name) { + const prefix = logPrefix(userKey); + if (typeof name !== 'string' || name.length === 0) + throw new RangeError(`Invalid log entry name: ${String(name)}`); + return name.startsWith(prefix) ? name.slice(prefix.length) : name; +} + +// Pages through the log prefix and returns objects in arrival (name) order. +// The bucket lists keys in ascending order (COS and R2 both do), so once `limit` +// names are collected a later page can only hold later ones: the read path stops +// there, which keeps its cost bounded by MAX_PENDING_READ instead of by the log. +// The fold path passes Infinity and still follows the cursor to the last page. +async function listObjects(bucket, userKey, limit = Infinity) { + const prefix = logPrefix(userKey); + const objects = []; + let cursor; + for (;;) { + const page = await bucket.list({ prefix, cursor, limit: LIST_PAGE }); + const matching = (page?.objects ?? []).filter( + (object) => + typeof object?.key === 'string' && + object.key.startsWith(prefix) && + object.key.length > prefix.length, + ); + objects.push(...matching); + if (objects.length >= limit || !page?.truncated || !page.cursor) break; + cursor = page.cursor; + } + objects.sort((left, right) => (left.key < right.key ? -1 : left.key > right.key ? 1 : 0)); + return objects.slice(0, limit); +} + +function logError(code, message, key) { + const error = new Error(message); + error.code = code; + error.key = key; + return error; +} diff --git a/onlion/backend/query-chunks.mjs b/onlion/backend/query-chunks.mjs index acd1767e..84b7c487 100644 --- a/onlion/backend/query-chunks.mjs +++ b/onlion/backend/query-chunks.mjs @@ -1,112 +1,120 @@ -/** Query chunking for embedding. +/** Query windowing for embedding. * - * An embedding model only sees its own token window, so a very long query - * embedded as one string keeps the opening and silently drops the rest — the - * request would succeed while ranking on a fraction of the question. Payload caps - * are sized to the platform body (see `MAX_QUERY_CHARS` in `worker.mjs`), so a - * long query is legitimate input that still has to be searchable. + * An embedding model only sees its own token window, so a long query embedded as + * one string keeps the opening and silently drops the rest — the request would + * succeed while ranking on a fraction of the question. Payload caps are sized to + * the platform body (see `MAX_QUERY_CHARS` in `worker.mjs`), so a long query is + * legitimate input that still has to be searchable. * - * `splitQueryChunks` returns bounded windows that always cover both ends of the - * text: the first `limit - 1` windows walk forward from the start, and the final - * window is anchored at the end. Without that anchor the windows only reach - * `limit * size` characters (8 x 2,000 = 16,000) and anything past it was never - * searched at all, so a query whose informative part sat at the very end was - * invisible regardless of how the feature budget was spent. The middle of a very - * long query is still skipped — that is the bounded cost, and it is the least - * informative of the three regions. + * Two layouts, chosen by whether the budget covers the text: * - * `poolQueryVectors` remains for callers that need one vector per query. The - * worker does not use it: measuring mean pooling against per-feature maximum - * showed pooling loses recall whenever the query's parts point in different - * directions, so the worker searches head and windows as separate features. + * n = min(ceil(L / W), MAX_QUERY_WINDOWS), W = QUERY_CHUNK_CHARS + * + * L <= n*W the budget covers the text: cut it into n equal, contiguous, + * disjoint segments that share one boundary-snapped cut point per + * seam. Nothing is dropped and nothing is embedded twice. + * L > n*W budget-bound: n windows of W, the first at the start and the last + * anchored at the end, with the uncovered text spread evenly between + * them. The middle is then sampled, not contiguous. + * + * Measured on 281 LoCoMo multi-hop questions against the previous layout (tile + * forward from the start, then one end-anchored window): the previous layout left + * a single contiguous blind block, and a question placed in the middle of the text + * scored exactly what it scored with no splitting at all, while this layout found + * it (36,000 characters: 4.63% vs 2.14% full-page coverage, 8W/0L, p=0.0078). + * Equal segments also beat the forward tiling on the end-anchored condition once + * the text was long enough, and never lost the opening. + * + * Windows are searched as separate features and fused by maximum. Mean pooling was + * measured as losing recall whenever a query's parts point in different directions, + * so nothing in the product pools them. */ export const QUERY_CHUNK_CHARS = 2000; -export const MAX_QUERY_CHUNKS = 8; +/** Windows one clause may be searched by. This is a ceiling, not an allocation: + * `selectQueryFeatures` still shares `MAX_QUERY_FEATURES` across clauses. */ +export const MAX_QUERY_WINDOWS = 10; +/** Features one request may embed, across every clause. */ +export const MAX_QUERY_FEATURES = 24; -/** Prefer a paragraph, sentence, or clause boundary in the second half of the - * window, so a chunk does not end mid-sentence when a boundary is available. */ -function boundaryBefore(window) { - const candidates = [ - window.lastIndexOf("\n"), - window.lastIndexOf(". "), - window.lastIndexOf("? "), - window.lastIndexOf("! "), - window.lastIndexOf("。"), - window.lastIndexOf(";"), - window.lastIndexOf(","), - ].filter((index) => index > 0); - const best = Math.max(-1, ...candidates); - return best > window.length / 2 ? best + 1 : window.length; -} +const BOUNDARIES = ["\n", ". ", "? ", "! ", "。", ";", ","]; -/** First boundary inside a window, so an end-anchored window does not begin - * mid-sentence when a boundary appears early in it. */ -function firstBoundary(window) { - const candidates = [ - window.indexOf("\n"), - window.indexOf(". "), - window.indexOf("? "), - window.indexOf("! "), - window.indexOf("。"), - window.indexOf(";"), - window.indexOf(","), - ].filter((index) => index > 0); - const best = candidates.length === 0 ? -1 : Math.min(...candidates); - return best > 0 && best < window.length / 2 ? best + 1 : 0; +/** Nearest boundary at or after `ideal`, bounded so a segment cannot swallow its + * neighbour. Shared between the two segments it separates, which is what keeps + * equal segmentation gap-free. */ +function snapCut(text, ideal, previous, end) { + const ceiling = Math.min(end - 1, ideal + 400); + for (let at = Math.max(ideal, previous + 1); at <= ceiling; at += 1) { + const slice = text.slice(at, at + 2); + if (BOUNDARIES.some((boundary) => slice.startsWith(boundary))) return at + 1; + } + return Math.min(Math.max(ideal, previous + 1), ceiling); } -/** Bounded, non-overlapping windows covering the head and the tail of the text. */ -export function splitQueryChunks(text, options = {}) { - const limit = options.maxChunks ?? MAX_QUERY_CHUNKS; - const size = options.chunkChars ?? QUERY_CHUNK_CHARS; - if (text.length <= size) return [text]; - const chunks = []; - let start = 0; - while (start < text.length && chunks.length < Math.max(1, limit - 1)) { - // A sliver is not worth its own retrieval pass: fold it into the window just - // closed rather than spending a feature on a few dozen characters. - if (chunks.length > 0 && text.length - start <= size / 4) { - chunks[chunks.length - 1] += text.slice(start); - break; - } - const window = text.slice(start, Math.min(start + size, text.length)); - const cut = boundaryBefore(window); - chunks.push(text.slice(start, start + cut)); - start += cut; +/** Bounded, non-overlapping windows covering both ends of the text. */ +export function planQueryWindows(text, options = {}) { + const windowChars = options.windowChars ?? QUERY_CHUNK_CHARS; + const maxWindows = options.maxWindows ?? MAX_QUERY_WINDOWS; + const length = text.length; + if (length <= windowChars) return [text]; + const count = Math.min(Math.ceil(length / windowChars), Math.max(2, maxWindows)); + if (length <= count * windowChars) { + const cuts = [0]; + for (let index = 1; index < count; index += 1) + cuts.push(snapCut(text, Math.round((index * length) / count), cuts[index - 1], length)); + cuts.push(length); + return cuts + .slice(1) + .map((end, index) => text.slice(cuts[index], end)) + .filter((window) => window.length > 0); } - if (start >= text.length) return chunks; - // A sliver is not worth its own retrieval pass: fold it into the previous - // window instead of spending a feature on a few dozen characters. - if (text.length - start < size / 4) { - chunks[chunks.length - 1] += text.slice(start); - return chunks; - } - const tail = text.slice(Math.max(0, text.length - size)); - const from = firstBoundary(tail); - chunks.push(from > 0 ? tail.slice(from) : tail); - return chunks; + const stride = (length - count * windowChars) / (count - 1) + windowChars; + // Pinned ends: the opening is what the question usually states, and the tail is + // where a question appended after a transcript sits. + const startAt = (index) => + index === 0 ? 0 : index === count - 1 ? length - windowChars : Math.round(index * stride); + return Array.from({ length: count }, (_unused, index) => + text.slice(startAt(index), startAt(index) + windowChars), + ); } -/** Unit-length mean of the chunk vectors. Embedding vectors are stored - * normalized, so a pooled query must be normalized too or cosine ranking would - * shift with the number of chunks. */ -export function poolQueryVectors(vectors) { - const first = vectors[0]; - if (!first) throw new Error("poolQueryVectors requires at least one vector"); - const dims = first.length; - const pooled = new Array(dims).fill(0); - for (const vector of vectors) { - if (vector.length !== dims) throw new Error("chunk vectors must share a dimension"); - for (let i = 0; i < dims; i += 1) pooled[i] += vector[i]; - } - let norm = 0; - for (let i = 0; i < dims; i += 1) { - pooled[i] /= vectors.length; - norm += pooled[i] * pooled[i]; +/** The features to embed for each clause, in the order they should be spent. + * + * The first feature is the clause itself: the model's own token window sees the + * opening with more context than any character window can, and measured + * head-aligned recall depends on it. The first planned window is therefore + * dropped — it covers the opening again, and dropping it measured identical to + * keeping it on every condition at 6,000 and 36,000 characters (36,000: + * 11.39% vs 11.03% on the head-aligned condition, 0 losses) while saving one + * embedding call per clause. + * + * What remains is the end-anchored tail first, then the interior windows, shared + * round-robin across clauses: a scarce feature budget buys both ends for every + * clause before any clause gets a third window, and one very long clause cannot + * starve the others. A clause that fits one window contributes one feature. */ +export function selectQueryFeatures( + queries, + maxFeatures = MAX_QUERY_FEATURES, + maxWindows = MAX_QUERY_WINDOWS, +) { + const plans = queries.map((text) => planQueryWindows(text, { maxWindows })); + const features = queries.map((text) => [text]); + const pending = plans.map((windows) => + windows.length <= 1 ? [] : [windows[windows.length - 1], ...windows.slice(1, -1)], + ); + const cursors = pending.map(() => 0); + let remaining = Math.max(0, maxFeatures - features.length); + let progressed = true; + while (remaining > 0 && progressed) { + progressed = false; + for (let index = 0; index < pending.length && remaining > 0; index += 1) { + const next = cursors[index]; + if (next >= pending[index].length) continue; + cursors[index] = next + 1; + features[index].push(pending[index][next]); + remaining -= 1; + progressed = true; + } } - norm = Math.sqrt(norm); - if (!norm) return pooled; - for (let i = 0; i < dims; i += 1) pooled[i] /= norm; - return pooled; + return features; } diff --git a/onlion/backend/reclamation.mjs b/onlion/backend/reclamation.mjs index cae4c697..b8a4800c 100644 --- a/onlion/backend/reclamation.mjs +++ b/onlion/backend/reclamation.mjs @@ -48,6 +48,19 @@ export async function memoryBucket(bucket){ return result; }, delete:keys=>bucket.delete(Array.isArray(keys)?keys.map(scoped):scoped(keys)), + // Enumeration is scoped the same way: the caller sees logical keys, and a + // listing can never cross into another generation's objects. Cursors are + // passed through untouched because they are only valid for one scoped prefix, + // which is the only prefix this wrapper can produce. + async list(options={}){ + const target=options.prefix===undefined?prefix:scoped(options.prefix); + const page=await bucket.list({...options,prefix:target}); + const objects=(page?.objects??[]).map(object=>({ + ...object, + key:prefix&&String(object.key).startsWith(prefix)?String(object.key).slice(prefix.length):object.key, + })); + return {...page,objects}; + }, }; } diff --git a/onlion/backend/storage.mjs b/onlion/backend/storage.mjs index 7e1c4d80..5ac342de 100644 --- a/onlion/backend/storage.mjs +++ b/onlion/backend/storage.mjs @@ -3,6 +3,7 @@ import {applyPageDelta, makePageDelta, deltaBytes} from './page-delta.mjs'; import {createHash} from 'node:crypto'; import {DatabaseSync, withDatabases} from './sqlite.mjs'; import {EmbeddingPreparation} from './embedding-session.mjs'; +import {logPrefix} from './append-log.mjs'; export const USER_LIMIT=1024*1024*1024; export const PART_LIMIT=8*1024*1024; @@ -10,17 +11,57 @@ export const PART_TARGET=6*1024*1024; export const MAX_PARTS=128; export const MAX_RECEIPTS=16384; export const MAX_MANIFEST_BYTES=16*1024*1024; +// v4 is v2/v3 plus a pending log. It is only written while `pending` is non-zero, +// so a user whose log is empty keeps the exact bytes an older deployment wrote. +export const STORAGE_FORMAT_V4='nmg-shards-v4'; +const STORAGE_FORMATS=['nmg-shards-v2','nmg-shards-v3',STORAGE_FORMAT_V4]; export const keyFor=text=>createHash('sha256').update(text).digest('hex').slice(0,24); export class StorageError extends Error { constructor(code,message,status=413){super(message);this.code=code;this.status=status;} } const manifestKey=key=>`users/${key}/manifest.json`; +/** Entries still in the log. Absent or zero means nothing is pending. */ +export function pendingCount(manifest){ + const count=manifest?.pending??0; + return Number.isSafeInteger(count)&&count>0?count:0; +} +/** Shallow copy of `manifest` carrying `count` pending entries; `receipts` is + * shared, as everywhere else in this module. Delegates the rule to `setPending` + * so "zero deletes the key" has one implementation: two copies of that rule is + * how the drain once committed `pending: 0` and stayed v4. */ +export function withPending(manifest,count){ + const next={...manifest}; + setPending(next,count); + return next; +} +/** Sets the pending count on the manifest **in place**. + * + * Zero deletes the key instead of storing `pending: 0`: the lazy format bump and + * the "a drained log stores the same bytes as a deployment that never had one" + * invariant both depend on absence, not on a zero value. In place because the + * caller is holding the clone it is about to commit - and because + * `Object.assign(manifest, withPending(manifest, 0))` silently keeps the old key, + * which is a bug this service already shipped once. Returns the stored count. */ +export function setPending(manifest,count){ + if(!Number.isSafeInteger(count)||count<0)throw new TypeError('pending count must be a non-negative integer'); + if(count)manifest.pending=count;else delete manifest.pending; + return count; +} +/** Format to store. Pending entries make the manifest unreadable to a + * deployment that predates the log, which is the correct failure mode. */ +export function manifestFormat(manifest){ + if(pendingCount(manifest)>0)return STORAGE_FORMAT_V4; + return manifest.parts.some(p=>p.delta)?'nmg-shards-v3':'nmg-shards-v2'; +} export async function loadManifest(bucket,key,userId){ const object=await bucket.get(manifestKey(key)); if(object){ if(object.size>MAX_MANIFEST_BYTES)throw new StorageError('user_storage_limit','User manifest exceeds 16 MiB'); const manifest=await object.json(); - if(!['nmg-shards-v2','nmg-shards-v3'].includes(manifest.format)||!Array.isArray(manifest.parts))throw new StorageError('storage_corrupt','Unsupported user storage format',503); + if(!STORAGE_FORMATS.includes(manifest.format)||!Array.isArray(manifest.parts))throw new StorageError('storage_corrupt','Unsupported user storage format',503); + // A forged or truncated count would silently skip folding, so treat it as + // corruption instead of reading it as zero. + if(manifest.pending!==undefined&&(!Number.isSafeInteger(manifest.pending)||manifest.pending<0))throw new StorageError('storage_corrupt','Invalid pending count in user manifest',503); return {object,manifest}; } const legacy=await bucket.head(`users/${key}.sqlite`); @@ -67,9 +108,20 @@ async function openPart(bucket,part,sqlite,operation,options){ return {...result,snapshot,legacyReceipts,previousImage:image}; }); } -export async function stagePart(bucket,userKey,part,snapshot){ +/** Uploads one partition snapshot. + * + * `reuse` lets a single request skip an upload it already made for + * byte-identical content. A manifest CAS conflict retries the whole operation, so + * without it every retry paid for another multi-megabyte snapshot upload, which + * was the dominant cost of concurrent Adds on one user. Content is compared, not + * assumed deterministic. */ +export async function stagePart(bucket,userKey,part,snapshot,reuse){ + const fingerprint=reuse?`${part.scope}\n${createHash('sha256').update(snapshot).digest('hex')}`:null; + const prior=fingerprint?reuse.get(fingerprint):null; + if(prior)return {...part,key:prior,bytes:snapshot.byteLength}; const key=`users/${userKey}/parts/${crypto.randomUUID()}.sqlite`; await measure('partition_save',()=>bucket.put(key,snapshot,{httpMetadata:{contentType:'application/vnd.sqlite3'},customMetadata:{format:'nmg-part-v2'}})); + if(fingerprint)reuse.set(fingerprint,key); const base={...part}; delete base.delta; return {...base,key,bytes:snapshot.byteLength}; @@ -83,7 +135,9 @@ export async function stageSearchPart(bucket,userKey,part,result,userBudget,full export async function commitManifest(bucket,key,previous,manifest){ // Older deployments must reject overlay manifests rather than silently load // the stale base snapshot. Fully compacted manifests remain v2-compatible. - manifest.format=manifest.parts.some(p=>p.delta)?'nmg-shards-v3':'nmg-shards-v2'; + // The same laziness keeps a drained log byte-identical to v2/v3. + if(!pendingCount(manifest))delete manifest.pending; + manifest.format=manifestFormat(manifest); if(manifest.parts.length>MAX_PARTS||manifest.parts.reduce((sum,p)=>sum+p.bytes,0)>USER_LIMIT) throw new StorageError('user_storage_limit','User exceeds 1 GiB or 128 storage partitions'); if(Object.keys(manifest.receipts).length>MAX_RECEIPTS)throw new StorageError('request_receipt_limit','User exceeds 16384 distinct Add requests'); @@ -91,6 +145,24 @@ export async function commitManifest(bucket,key,previous,manifest){ if(new TextEncoder().encode(body).length>MAX_MANIFEST_BYTES)throw new StorageError('user_storage_limit','User manifest exceeds 16 MiB'); return bucket.put(manifestKey(key),body,{onlyIf:previous?{etagMatches:previous.etag}:{etagDoesNotMatch:'*'},httpMetadata:{contentType:'application/json'}}); } +/** Deletes every pending log entry for one user. Reset is the only caller and + * runs it after the generation barrier is committed, so a fold racing the reset + * cannot fold these entries anyway (I4). The prefix comes from append-log.mjs so + * the two modules cannot drift apart. Returns the number of objects removed; a + * second call removes nothing, which is what makes retrying a reset safe. */ +export async function clearLog(bucket,userKey){ + const prefix=logPrefix(userKey); + let removed=0, cursor; + for(;;){ + const page=await bucket.list({prefix,cursor,limit:1000}); + const keys=(page?.objects??[]).map(object=>object.key).filter(key=>typeof key==='string'&&key.startsWith(prefix)&&key.length>prefix.length); + if(keys.length){await bucket.delete(keys);removed+=keys.length;} + // A cursor that does not advance would loop forever inside a reset. + if(!page?.truncated||!page.cursor||page.cursor===cursor)break; + cursor=page.cursor; + } + return removed; +} export async function removeStaged(bucket,keys){ if(keys.length)await bucket.delete(keys); } @@ -120,11 +192,14 @@ export async function userCapacity(bucket,userId){ return {user_id:userId,checked_at:new Date().toISOString(),source:legacy?'legacy_object_head':'manifest', exists:!!object||legacy,sqlite_bytes:sqlite,partitions,add_requests:requests,manifest_bytes:metadata, inline_delta_bytes:manifest.parts.reduce((sum,p)=>sum+deltaBytes(p),0), + pending_entries:pendingCount(manifest), active_partition:{bytes:last?.bytes??0,target_bytes:PART_TARGET,max_add_bytes:PART_LIMIT, target_headroom_bytes:last?Math.max(0,PART_TARGET-last.bytes):PART_TARGET}, warnings,notes:[ 'SQLite bytes describe current logical images, including restored overlays; not total billed bucket usage.', 'No exact memory count is collected. Add requests count distinct durable receipts, not messages or memories.', + 'pending_entries counts accepted Adds that are durable in the log but not yet folded into a partition.', + 'Remaining byte quota does not guarantee another Add fits; rollover, request size and metadata limits also apply.', ...(legacy?['Legacy receipt count is unknown until the existing image is imported by a normal operation.']:[])]}; } @@ -137,6 +212,11 @@ function resetManifestState(manifest,resetId,key){ const empty={format:'nmg-shards-v2',generation:crypto.randomUUID(),parts:[],receipts:{}, reset:{id:resetId,at:new Date().toISOString(),pending_keys:keys}, reset_receipts:{...manifest.reset_receipts,[receiptKey]:new Date().toISOString()}}; + // Carry the count instead of clearing it. If the log cleanup below fails, entries + // from the old generation are still on disk; a zero here would let a fold retire + // and subtract them until the clamp hits 0, and the short-circuit would then hide + // everything accepted after the reset. + setPending(empty,pendingCount(manifest)); return {receiptKey,empty}; } async function clearResetPendingKeys(bucket,key,userId,current){ @@ -167,7 +247,23 @@ export async function resetUser(bucket,userId,resetId){ if(await commitManifest(bucket,key,object,empty)){current=empty;break;} } if(!current)throw new StorageError('reset_conflict','Concurrent changes detected; pause writes and retry the same reset_id',409); - const cleanupComplete=await clearResetPendingKeys(bucket,key,userId,current); + // Part cleanup and log cleanup are independent: a failure in one must not + // skip the other, and repeating the same reset_id retries both. + const partsCleared=await clearResetPendingKeys(bucket,key,userId,current); + const logCleared=await clearLog(bucket,key).then(()=>true,()=>false); + if(logCleared){ + // Only now is the count provably zero. Metadata-only, and gated on the count not + // having moved: an equal count means nothing was accepted between the read and + // the clear, so this cannot eat a concurrent accept's reservation. + try{ + const {object,manifest}=await loadManifest(bucket,key,userId); + if(pendingCount(manifest)===pendingCount(current)){ + setPending(manifest,0); + await commitManifest(bucket,key,object,manifest); + } + }catch{/* A repeat of the same reset_id retries the whole cleanup. */} + } + const cleanupComplete=partsCleared&&logCleared; return {success:true,user_id:userId,reset_id:resetId,reset_at:current.reset_receipts?.[createHash('sha256').update(resetId).digest('hex')]??current.reset.at, cleanup_complete:cleanupComplete, message:cleanupComplete?'User memory and Add receipts reset; previous referenced snapshots removed.':'User memory and Add receipts reset; retry the same reset_id to finish snapshot cleanup.'}; diff --git a/onlion/backend/worker.mjs b/onlion/backend/worker.mjs index a5222d14..bcbddf28 100644 --- a/onlion/backend/worker.mjs +++ b/onlion/backend/worker.mjs @@ -54,11 +54,30 @@ import { retireParts, userCapacity, resetUser, + manifestFormat, + pendingCount, + setPending, } from "./storage.mjs"; import { deltaBytes, USER_DELTA_LIMIT } from "./page-delta.mjs"; import { deferredDocuments } from "./embedding-session.mjs"; import { queryVector, rememberQueryVector } from "./query-cache.mjs"; -import { MAX_QUERY_CHUNKS, QUERY_CHUNK_CHARS, splitQueryChunks } from "./query-chunks.mjs"; +import { + LOG_ENTRY_VERSION, + MAX_PENDING_READ, + deleteLogEntries, + findLogEntry, + foldPlan, + listPending, + logEntryLanded, + putLogEntry, +} from "./append-log.mjs"; +import { cosineSimilarity } from "../../src/core/vector.ts"; +import { + MAX_QUERY_FEATURES, + MAX_QUERY_WINDOWS, + QUERY_CHUNK_CHARS, + selectQueryFeatures, +} from "./query-chunks.mjs"; const MAX_DATABASE_BYTES = USER_LIMIT; const MAX_BODY_BYTES = 6 * 1024 * 1024; /** Payload caps are sized against the platform request body, not against a single @@ -79,14 +98,13 @@ const MAX_QUERY_CHARS = 1_000_000; const MAX_MESSAGE_CHARS = 1_000_000; const MAX_COMBINED_MESSAGE_CHARS = 1_500_000; const MAX_ORDERED_CONTENT_BYTES = 4 * 1024 * 1024; -const MAX_QUERY_FEATURES = 24; const TEXT_LIMITS = { max_query_chars: MAX_QUERY_CHARS, max_message_chars: MAX_MESSAGE_CHARS, max_combined_message_chars: MAX_COMBINED_MESSAGE_CHARS, max_ordered_content_bytes: MAX_ORDERED_CONTENT_BYTES, query_chunk_chars: QUERY_CHUNK_CHARS, - max_query_chunks: MAX_QUERY_CHUNKS, + max_query_chunks: MAX_QUERY_WINDOWS, max_query_features: MAX_QUERY_FEATURES, }; const SOURCE_REVISION = @@ -210,7 +228,55 @@ async function readJson(request) { throw new HttpError(400, "invalid_json", "Body must be a JSON object"); } } +/** Retry delay for a manifest CAS conflict: exponential with full jitter. The + * previous fixed 15-100 ms window made sixteen writers re-collide in lockstep, so + * each of them spent its whole attempt budget on the same round trips. */ +const WRITE_BACKOFF_BASE_MS = 20; +const WRITE_BACKOFF_CAP_MS = 600; +const writeBackoffMs = (attempt) => + Math.random() * Math.min(WRITE_BACKOFF_CAP_MS, WRITE_BACKOFF_BASE_MS * 2 ** (attempt - 1)); // Atomic manifest commits preserve request idempotency across all partitions. +/** One attempt's manifest read, with the guard that turns a reset during a retry into + * a clear 409 instead of a write against a replaced dataset. */ +async function readAttempt(env, key, userId, attempt, generation) { + const { object, manifest: before } = await measure("manifest_read", () => + loadManifest(env.MEMORY, key, userId), + ); + const observed = before.generation ?? null; + if (attempt === 0) return { object, before, generation: observed }; + if (observed !== generation) + throw new HttpError( + 409, + "user_reset", + "User memory was reset during this operation; submit only the new dataset", + ); + return { object, before, generation }; +} + +/** Runs the operation once and commits it, or reports that the CAS rejected the write. + * + * The outcome carries an explicit `retry` flag rather than signalling a lost CAS with + * `response: null`. `null` is a legal JSON payload, so an operation that legitimately + * returned it would be retried until the attempt budget ran out and then answered 503 - + * a failure mode no test would notice. Read-only outcomes are `committed: false` and + * must not be retried either. */ +async function commitAttempt(env, { sqlite, key, object, before, operation, reuse, staged }) { + const manifest = structuredClone(before); + const result = await operation({ sqlite, key, manifest, staged, reuse }); + if (result.readOnly) return { response: result.response, committed: false, retry: false }; + const saved = await measure("manifest_commit", () => + commitManifest(env.MEMORY, key, object, manifest), + ); + if (!saved) return { retry: true }; + const live = new Set(manifest.parts.map((p) => p.key)); + await removeStaged( + env.MEMORY, + staged.filter((k) => !live.has(k)), + ).catch(() => {}); + await measure("retired_cleanup", () => retireParts(env.MEMORY, before, manifest)); + return { response: result.response, committed: true, retry: false }; +} + async function mutateUser(env, userId, operation) { const sqlite = await measure("sqlite_init", () => sqliteReady()), key = keyFor(userId); @@ -219,50 +285,50 @@ async function mutateUser(env, userId, operation) { // benchmark harness can hit one user with 16 writers at once, where every // round trip is cheap (~1 s) and giving up surfaces as a spurious 503. const attempts = Number(env.NMG_WRITE_ATTEMPTS) > 0 ? Number(env.NMG_WRITE_ATTEMPTS) : 12; + // Partition snapshots this request already uploaded, keyed by content. A retry + // recomputes the snapshot, and a competing Add that rewrote a different + // partition leaves this one byte-identical, so re-uploading it was pure cost. + const reuse = new Map(); + let committed = false; let generation; - for (let attempt = 0; attempt < attempts; attempt++) { - // Jitter keeps retrying writers from colliding again in lockstep. - if (attempt > 0) await new Promise((resolve) => setTimeout(resolve, 15 + Math.random() * 85)); - const { object, manifest: before } = await measure("manifest_read", () => - loadManifest(env.MEMORY, key, userId), - ); - const observed = before.generation ?? null; - if (attempt === 0) generation = observed; - else if (observed !== generation) - throw new HttpError( - 409, - "user_reset", - "User memory was reset during this operation; submit only the new dataset", - ); - const manifest = structuredClone(before), - staged = []; - try { - const result = await operation({ sqlite, key, manifest, staged }); - if (result.readOnly) return result.response; - const saved = await measure("manifest_commit", () => - commitManifest(env.MEMORY, key, object, manifest), - ); - if (saved) { - const live = new Set(manifest.parts.map((p) => p.key)); - await removeStaged( - env.MEMORY, - staged.filter((k) => !live.has(k)), - ).catch(() => {}); - await measure("retired_cleanup", () => retireParts(env.MEMORY, before, manifest)); - return result.response; + try { + for (let attempt = 0; attempt < attempts; attempt++) { + if (attempt > 0) await new Promise((resolve) => setTimeout(resolve, writeBackoffMs(attempt))); + const attemptState = await readAttempt(env, key, userId, attempt, generation); + generation = attemptState.generation; + // Staged objects survive a conflict retry so the next attempt can reuse an + // identical upload; they are removed once, after the last attempt. + const staged = []; + try { + const outcome = await commitAttempt(env, { + sqlite, + key, + object: attemptState.object, + before: attemptState.before, + operation, + reuse, + staged, + }); + if (!outcome.retry) { + // Only the ending attempt may claim the commit: a lost CAS must not touch + // `committed`, or the `finally` cleanup below would run for a write that a + // rival did commit. + committed ||= outcome.committed; + return outcome.response; + } + } catch (error) { + if (error.code === "snapshot_changed") continue; + throw error; } - } catch (error) { - await removeStaged(env.MEMORY, staged).catch(() => {}); - if (error.code === "snapshot_changed") continue; - throw error; } - await removeStaged(env.MEMORY, staged).catch(() => {}); + throw new HttpError( + 503, + "write_conflict", + "Concurrent changes detected; retry the same request_id", + ); + } finally { + if (!committed) await removeStaged(env.MEMORY, [...reuse.values()]).catch(() => {}); } - throw new HttpError( - 503, - "write_conflict", - "Concurrent changes detected; retry the same request_id", - ); } function embeddingClient(env) { const client = createEmbeddingClientFromEnv({ @@ -412,12 +478,20 @@ async function addOneBatch(env, ctx, requestId, batch, manifest, staged) { "One Add batch exceeds 8 MiB; submit fewer messages", ); copyLegacyReceipts(manifest, result.legacyReceipts); - const part = await stagePart(env.MEMORY, ctx.key, manifest.parts[index], result.snapshot); + const part = await stagePart( + env.MEMORY, + ctx.key, + manifest.parts[index], + result.snapshot, + ctx.reuse, + ); manifest.parts[index] = part; staged.push(part.key); return result.response.added; } -async function add(env, input) { +/** Validation shared by both accept paths: the log and the synchronous write must + * reject exactly the same requests, or a mode switch changes what is legal. */ +function readAddRequest(input) { const userId = textField(input.user_id ?? input.userId, "user_id"); requestIdentity(userId, input.request_id); const requestId = textField(input.request_id, "request_id"); @@ -436,10 +510,383 @@ async function add(env, input) { "request_too_large", `Combined message content limit is ${MAX_COMBINED_MESSAGE_CHARS} characters`, ); - const fingerprint = hash(JSON.stringify({ sessionId, messages })); + return { + userId, + requestId, + sessionId, + messages, + requestHash: hash(requestId), + fingerprint: hash(JSON.stringify({ sessionId, messages })), + }; +} +/** Body returned by both accept paths. `completed` describes persistence, not + * compaction: after the log write the data is durable and readable through + * `pendingForSearch`, which is what the contract asks a 200 to mean. */ +function addResponse(key, requestId, userId, sessionId, requestHash, added) { + return { + success: true, + request_id: requestId, + user_id: userId, + session_id: sessionId, + task_id: `${key}.${requestHash}`, + status: "completed", + added, + }; +} +/** Accepts an Add as one immutable log entry. + * + * The cost is a manifest read, one log write, and a manifest CAS that carries the + * pending counter - no partition download, no SQLite export, no snapshot upload, + * which is what made a 16-way burst on one user sixteen multi-megabyte rounds. + * The retry loop re-writes the same entry (a replay, no write) and re-commits only + * the manifest, so a conflict is cheap here. */ +async function acceptAdd(env, input) { + const { userId, requestId, sessionId, messages, requestHash, fingerprint } = + readAddRequest(input); + const contentView = createContentView(); + let accepted = 0; + let prepared = null; + let reservation = null; + const response = await mutateUser(env, userId, async (ctx) => { + const reused = await existingReceipt(env, ctx, requestHash, fingerprint); + if (reused) return reused; + // Scoped to this generation: an entry written before a reset has the same name, + // and replaying its stored response would swallow the new Add - the request would + // answer 200 while its data was never recorded (class A, data lost). + const pending = await findLogEntry(env.MEMORY, ctx.key, requestHash, { + generation: ctx.manifest.generation ?? null, + }); + if (pending) { + if (pending.entry.fingerprint !== fingerprint) + throw new HttpError( + 409, + "request_id_reused", + "request_id was already used for different content", + ); + return { readOnly: true, response: pending.entry.response }; + } + // Structured content is prepared once per request, not once per retry: a + // caption may come from a vision model and a CAS conflict must not pay for it + // twice. It is prepared before the entry is written, so a fold never has to call + // a vision model while holding a partition. + if (!prepared) { + prepared = []; + for (const message of messages) + prepared.push( + typeof message.content === "string" + ? message + : { ...message, indexText: await contentView(message.content) }, + ); + } + const response = addResponse( + ctx.key, + requestId, + userId, + sessionId, + requestHash, + prepared.length, + ); + // Reserve the counter *before* the entry is durable. The order is the safety + // argument: a failed commit leaves no entry behind, while a failed write leaves + // the counter one too high - and an over-count only costs a log listing. The + // other order strands an entry no counter accounts for, and `pendingCount === 0` + // then hides it from every search (I1). + accepted = setPending(ctx.manifest, pendingCount(ctx.manifest) + 1); + reservation = { + key: ctx.key, + entry: { + v: LOG_ENTRY_VERSION, + generation: ctx.manifest.generation ?? null, + user_id: userId, + session_id: sessionId, + request_id: requestId, + request_hash: requestHash, + fingerprint, + received_at: Date.now(), + messages: prepared, + response, + }, + }; + return { response }; + }); + // The reservation is committed; only now must the entry become durable. A failure + // answers 5xx with the counter one too high, which capacity or the next fold + // corrects by reading the log back. + if (reservation) { + try { + await putLogEntry(env.MEMORY, reservation.key, reservation.entry); + } catch (error) { + // A throwing write is not proof that nothing reached the bucket: the storage + // wrapper writes first and then verifies the generation, so the throw can + // happen after the object landed. Ask instead of assuming, and only give the + // slot back when the answer is a definite "no" - releasing for an entry that + // is on disk leaves a counter below the truth, which hides accepted data. + const landed = await logEntryLanded(env.MEMORY, reservation.key, reservation.entry).catch( + () => null, + ); + if (landed === true) return { response, pending: accepted }; + if (landed === false) await releasePending(env, userId).catch(() => {}); + throw error; + } + } + return { response, pending: accepted }; +} +/** Folds waiting entries into the partition, one partition write for the batch. + * + * Entries whose receipt already exists are skipped, which is what makes a fold + * that committed and then crashed before deleting its entries safe to repeat (I3). + * Their objects are removed only after the commit, never before. */ +async function foldPending(env, userId) { + const client = embeddingClient(env); + const key = keyFor(userId); + const waiting = await listPending(env.MEMORY, key, { limit: FOLD_BATCH }); + if (waiting.length === 0) { + // An empty log with a non-zero counter is the residue of a reservation whose + // write never landed. Nothing else notices it: the fold trigger is the counter + // itself, so it never fires. Give one slot back per call and report the release + // as progress, which is what keeps `/admin/fold` iterating until it is clean. + const manifest = (await loadManifest(env.MEMORY, key, userId)).manifest; + if (pendingCount(manifest) === 0) return { folded: 0, remaining: 0, released: false }; + const released = await releasePending(env, userId).catch(() => null); + // `remaining` must be the count *after* the release. Reporting the pre-release + // number made a log that needed exactly the round budget answer 409 on its last + // call: safe, but it looked like the drain had failed to make progress. + return { folded: 0, remaining: released ?? pendingCount(manifest), released: true }; + } + const plan = foldPlan(waiting, { maxEntries: FOLD_BATCH, maxBytes: MAX_BODY_BYTES }); + let drop = []; + let remaining = 0; + const result = await mutateUser(env, userId, async (ctx) => { + // A name leaves the counter whether the entry is applied or skipped: a receipt + // that already exists (crash recovery) and a cross-generation entry both mean + // this name is finished with. Counting only the applied ones leaves the counter + // above zero forever, which disables the search short-circuit in + // `pendingForSearch` and reports a backlog that does not exist. + const names = new Set(); + let folded = 0; + const generation = ctx.manifest.generation ?? null; + for (const { name, entry } of plan.fold) { + names.add(name); + if (ctx.manifest.receipts[entry.request_hash]) continue; + if ((entry.generation ?? null) !== generation) continue; + const batch = entry.messages.map((message, index) => ({ + ...message, + source_message_id: `aml:${entry.request_hash}:${index}`, + })); + await addOneBatch( + env, + { ...ctx, userId, sessionId: entry.session_id, client }, + entry.request_id, + batch, + ctx.manifest, + ctx.staged, + ); + ctx.manifest.receipts[entry.request_hash] = { + fingerprint: entry.fingerprint, + response: entry.response, + completed_at: new Date().toISOString(), + }; + folded += 1; + } + drop = [...names]; + if (names.size === 0) return { readOnly: true, response: { folded: 0 } }; + // The counter is deliberately not touched here. Subtracting what this batch + // retired is only correct for the attempt that computed the plan: `mutateUser` + // re-runs this callback with the *same* plan after a CAS conflict, and by then a + // rival fold has usually retired those names already, so the subtraction would + // happen twice and drive the count below the log. The count is reconciled from + // the log after the delete instead - derived, never accumulated. + return { response: { folded } }; + }); + // Only after the commit: an entry present without a receipt is safe to repeat, + // an entry deleted without a receipt would be lost (I3). + if (drop.length > 0) { + await deleteLogEntries(env.MEMORY, key, drop).catch(() => {}); + // One read of the log's own truth settles every way the count could have drifted: + // a stale plan after a CAS retry, entries whose receipt already existed, entries + // from another generation, and a failed delete. Reading it after the delete is + // what makes the counter a derivation instead of a history. + const left = await listPending(env.MEMORY, key, { limit: Infinity }).then( + (list) => list.length, + () => null, + ); + if (left !== null) { + remaining = left; + await reconcilePending(env, userId, left).catch(() => {}); + } + } + return { folded: result.folded ?? 0, remaining }; +} +/** Gives one reserved slot back after a write that never became durable. + * + * Relative, not absolute: a concurrent accept may have reserved in between, and + * writing a computed absolute value would erase its increment (a lost update). + * The read-modify-write is inside the manifest CAS like every other count change. */ +async function releasePending(env, userId) { + const response = await mutateUser(env, userId, async (ctx) => { + const count = pendingCount(ctx.manifest); + if (count === 0) return { readOnly: true, response: { pending: 0 } }; + setPending(ctx.manifest, count - 1); + return { response: { pending: count - 1 } }; + }); + // The count itself, not the response body: callers that report `remaining` need + // the value after the release, and `mutateUser` already unwraps `response`. + return response.pending; +} +/** Writes the observed number of waiting entries back to the manifest. Used after a + * failed delete, where the decremented counter no longer matches the log. */ +async function reconcilePending(env, userId, count) { + return mutateUser(env, userId, async (ctx) => { + if (pendingCount(ctx.manifest) === count) + return { readOnly: true, response: { pending: count } }; + setPending(ctx.manifest, count); + return { response: { pending: count } }; + }); +} +/** Terms the lexical fallback scores against: the query features, split the way the + * store's own lexical channel splits them (lowercase, alphanumeric runs, stopword-ish + * short tokens dropped). */ +function queryTerms(features) { + const seen = new Set(); + for (const feature of features) + for (const term of String(feature) + .toLowerCase() + .split(/[^a-z0-9一-鿿]+/u)) + if (term.length > 2) seen.add(term); + return [...seen]; +} + +/** How many query terms a pending entry mentions. Zero means it is not a match, which is + * what keeps an unmatched query answering with an empty page in a lexical-only + * deployment. */ +function pendingTermScore(content, terms) { + const text = String(content).toLowerCase(); + let score = 0; + for (const term of terms) if (text.includes(term)) score += 1; + return score; +} +/** Fuses the entries still in the log into the page, scored against the same clauses the + * partitions were searched with. + * + * Each entry costs one embedding call, capped by `MAX_PENDING_READ`, because a search has + * to see an Add that was just accepted. It is also attributed to the clause it matches + * best: the per-clause counts must keep covering every returned memory, which is the + * multi-clause contract, and a pending record that skipped attribution made the sum fall + * short of the page size. + * + * Without a vector channel the same fusion runs on term overlap instead, because a + * deployment that has no embeddings still has to return what it accepted - and a flat + * zero score would return every pending entry for every query, including one that + * matches nothing. + */ +async function fusePendingRecords({ + env, + ctx, + features, + candidates, + client, + modes, + topK, + contributions, + appendOnly, + appendedById, +}) { + if (pendingCount(ctx.manifest) === 0) return candidates; + const waiting = await pendingForSearch(env, ctx, Math.min(topK, MAX_PENDING_READ)); + if (waiting.length === 0) return candidates; + const attribute = (clauseIndex) => { + const total = contributions?.[clauseIndex]; + if (total && clauseIndex >= 0) total.contributed += 1; + }; + const merge = (scored) => { + const page = []; + for (const item of scored) { + attribute(item.clause); + // An append-only clause never competes for the page: its matches are the supplement, + // exactly as they are for a folded record. + if (appendOnly?.[item.clause]) { + const prior = appendedById?.get(item.record.id); + if (prior && prior.score > item.record.score) continue; + appendedById?.set(item.record.id, item.record); + continue; + } + page.push(item.record); + } + return [...candidates, ...page] + .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)) + .slice(0, topK); + }; + + if (!client) { + const termsPerClause = features.map((list) => queryTerms(list)); + const scored = []; + for (const record of waiting) { + const perClause = termsPerClause.map((terms) => pendingTermScore(record.content, terms)); + const best = Math.max(...perClause); + if (best > 0) + scored.push({ record: { ...record, score: best }, clause: perClause.indexOf(best) }); + } + return merge(scored); + } + + const vectorsPerClause = []; + for (const list of features) { + const vectors = []; + for (const text of list) { + const [vector] = await client.embedQueries([text]).catch(() => []); + if (vector) vectors.push(vector); + } + vectorsPerClause.push(vectors); + } + const scored = []; + for (const record of waiting) { + const [vector] = await client.embedQueries([record.content]).catch(() => []); + if (!vector) continue; + const perClause = vectorsPerClause.map((vectors) => + vectors.reduce( + (best, feature) => Math.max(best, cosineSimilarity(vector, feature)), + -Infinity, + ), + ); + const best = Math.max(...perClause); + if (!Number.isFinite(best)) continue; + modes.add("records"); + scored.push({ record: { ...record, score: best }, clause: perClause.indexOf(best) }); + } + return merge(scored); +} + +/** Records that are still in the log, shaped like page items so the existing + * fusion by identity and highest score applies unchanged. + * + * Scoring happens in `search`: one embedding call per entry, capped at + * `MAX_PENDING_READ`, because a search must see an Add that was just accepted. */ +async function pendingForSearch(env, ctx, limit) { + if (pendingCount(ctx.manifest) === 0) return []; + const waiting = await listPending(env.MEMORY, ctx.key, { + limit: Math.min(limit, MAX_PENDING_READ), + generation: ctx.manifest.generation ?? null, + }); + return waiting.flatMap(({ entry }) => + entry.messages.map((message, index) => ({ + id: `pending:${entry.request_hash}:${index}`, + content: message.indexText ?? (typeof message.content === "string" ? message.content : ""), + score: 0, + // Message times, not the accept time: a pending entry must look like the record it + // will become, or the page changes shape the moment a fold runs. `created_at` and + // `event_time` both come from the message, which is what the partition would store. + event_time: message.chat_time ?? null, + created_at: message.chat_time ?? new Date(entry.received_at).toISOString(), + stored_at: new Date(entry.received_at).toISOString(), + pending: true, + })), + ); +} +async function add(env, input) { + const { userId, requestId, sessionId, messages, requestHash, fingerprint } = + readAddRequest(input); const client = embeddingClient(env); const contentView = createContentView(); - const requestHash = hash(requestId); const serviceContext = { userId, sessionId, client }; return mutateUser(env, userId, async (ctx) => { const { key, manifest, staged } = ctx; @@ -494,40 +941,24 @@ async function add(env, input) { * of the coverage gain of twenty at roughly a third of the text. */ const APPENDED_MAX_RECORDS = 5; -/** Whole-clause texts plus their split windows, bounded by `MAX_QUERY_FEATURES`. +/** Where an Add is accepted: `log` writes one immutable entry and folds it later, + * `sync` is the original read-modify-write. `log` is the default; `NMG_ADD_MODE=sync` + * is the escape hatch, so a deployment can go back without a code change. */ +const addMode = (env) => (env.NMG_ADD_MODE === "sync" ? "sync" : "log"); +/** Entries folded into a partition by one request. A fold is one partition write + * regardless of how many entries it absorbs, so a 16-way burst needs two folds. */ +const FOLD_BATCH = 8; +/** Accept stops accepting and folds instead once this many entries are waiting. */ +const FOLD_TRIGGER = 24; + +/** Features to embed and search per clause, bounded by `MAX_QUERY_FEATURES`. * - * A clause is searched by several features: its whole text (what the embedding - * model's own window sees) and each window of `splitQueryChunks`. Results fuse by - * identity on the highest score, so a record matches a clause by its *best* - * feature. Measured on LoCoMo multi-hop, that never loses to the whole text alone - * and recovers a query whose informative part sits past the model's window, where - * pooling the windows into one vector lost most of the recall. Windows are handed - * out round-robin so one very long clause cannot starve the others. A clause that - * does not split contributes exactly one feature, which is the previous behaviour. */ -function queryFeatures(queries) { - const windows = queries.map((text) => splitQueryChunks(text)); - const features = queries.map((text) => [text]); - let budget = MAX_QUERY_FEATURES - queries.length; - // Head is already in. Give the end-anchored window next, so the far end of a - // long clause survives budget pressure, then the middle windows in order. - const pending = windows.map((list) => - list.length <= 1 ? [] : [list[list.length - 1], ...list.slice(1, -1)], - ); - const cursors = pending.map(() => 0); - let progressed = true; - while (budget > 0 && progressed) { - progressed = false; - for (let i = 0; i < windows.length && budget > 0; i += 1) { - const next = cursors[i]; - if (next >= pending[i].length) continue; - cursors[i] = next + 1; - features[i].push(pending[i][next]); - budget -= 1; - progressed = true; - } - } - return features; -} + * The raw clause is never embedded: `selectQueryFeatures` returns its head window, + * its end-anchored tail window, then the interior windows, so both ends of a long + * clause reach the model instead of a megabyte string the model silently truncated. + * Results fuse by identity on the highest score, so a record matches a clause by + * its *best* feature. `warmClauseVectors` trims the tail end of this list when the + * embedding budget runs out, which is why the order matters. */ const MAX_SEARCH_CLAUSES = 8; /** One request may carry independent retrieval clauses: each is plain text plus * an optional label. The legacy `query`(+`options`) shape is one clause. */ @@ -597,11 +1028,33 @@ function addAppended(appended, memories) { if (!prior || (memory.score ?? 0) >= (prior.score ?? 0)) appended.set(memory.id, memory); } } -/** Resolve every clause vector once, before SQLite is opened: the per-request - * cache then supplies every partition without holding a DB during remote I/O. */ -async function warmClauseVectors(client, queries, parts) { +/** Resolve every feature before SQLite is opened, so no partition holds a database + * across remote I/O. + * + * `selectQueryFeatures` already spent the feature budget, so every feature here is + * one the search will use: nothing is trimmed. Features are resolved round by + * round — round 0 across all clauses, then round 1, and so on — which keeps the + * clauses balanced instead of finishing one clause's windows before starting the + * next. Best-effort: a failed embedding is simply not cached, and the feature falls + * back to whatever the retrieval layer manages, so the request still answers. */ +const WARM_CONCURRENCY = 4; +async function warmClauseVectors(client, features, parts) { if (!client || parts.length === 0) return; - for (const clause of queries) await client.embedQueries([clause]).catch(() => undefined); + const depth = Math.max(0, ...features.map((list) => list.length)); + for (let round = 0; round < depth; round += 1) { + const batch = []; + features.forEach((list) => { + if (round < list.length) batch.push(list[round]); + }); + let next = 0; + const warm = async () => { + while (next < batch.length) { + const text = batch[next++]; + await client.embedQueries([text]).catch(() => undefined); + } + }; + await Promise.all(Array.from({ length: Math.min(WARM_CONCURRENCY, batch.length) }, warm)); + } } /** Fold one partition's persistence mode into the request totals. */ function tallyPartPersistence(persistence, saved) { @@ -639,7 +1092,7 @@ async function search(env, input, options = {}) { const optionTexts = choices.map((choice) => textField(choice, "option", 4096)); const clauses = await readClauses(input, optionTexts); const queries = clauses.map((clause) => clause.text); - const features = queryFeatures(queries); + const features = selectQueryFeatures(queries); const appendOnly = clauses.map((clause) => clause.appendOnly === true); const hasAppendOnly = appendOnly.some(Boolean); const appendedById = new Map(); @@ -650,7 +1103,7 @@ async function search(env, input, options = {}) { let candidates = []; const modes = new Set(); let changed = false; - await warmClauseVectors(client, queries, manifest.parts); + await warmClauseVectors(client, features, manifest.parts); const persistence = { inline_delta_saves: 0, full_snapshot_saves: 0, changed_pages: 0 }; for (let i = 0; i < manifest.parts.length; i++) { const result = await runPart( @@ -700,6 +1153,19 @@ async function search(env, input, options = {}) { candidates.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); candidates = candidates.slice(0, topK); } + const fused = await fusePendingRecords({ + env, + ctx: { key, manifest }, + features, + candidates, + client, + modes, + topK, + contributions, + appendOnly, + appendedById, + }); + candidates = fused; persistence.inline_bytes = manifest.parts.reduce((sum, p) => sum + deltaBytes(p), 0); const appended = collectAppended( appendedById, @@ -725,7 +1191,7 @@ async function search(env, input, options = {}) { })), ...(hasAppendOnly ? { appended } : {}), persistence, - storage_format: manifest.parts.some((p) => p.delta) ? "nmg-shards-v3" : "nmg-shards-v2", + storage_format: manifestFormat(manifest), }, }; }); @@ -750,6 +1216,21 @@ async function status(env, taskId) { const { manifest } = await loadManifest(env.MEMORY, key, ""); const receipt = manifest.receipts[requestHash]; if (receipt) return { ...receipt.response, completed_at: receipt.completed_at }; + // An Add accepted into the log has no partition receipt yet, but it is durable and the + // caller is entitled to ask about it. The entry is scoped to the current generation, the + // same rule the accept-side replay lookup follows. + const pending = await findLogEntry(env.MEMORY, key, requestHash, { + generation: manifest.generation ?? null, + }).then( + (found) => found?.entry ?? null, + () => null, + ); + if (pending) + return { + ...pending.response, + completed_at: new Date(pending.received_at).toISOString(), + pending: true, + }; const legacy = manifest.parts.find((p) => p.key === objectKey(key)); if (legacy) { // Status uses the original file name and never needs the original user_id. @@ -893,8 +1374,12 @@ async function healthResponse(env, path) { max_partitions: MAX_PARTS, graph_scope: "partition", max_request_bytes: MAX_BODY_BYTES, - limits: TEXT_LIMITS, - add_mode: "synchronous", + limits: { + ...TEXT_LIMITS, + add_mode: addMode(env), + pending_read_limit: MAX_PENDING_READ, + merge_batch: FOLD_BATCH, + }, upstream_commit: SOURCE_REVISION.split("+")[0], source_revision: SOURCE_REVISION, core_source: CORE_SOURCE, @@ -1218,9 +1703,46 @@ async function capacityRoute(request, env, url, path) { await userCapacity(env.MEMORY, textField(url.searchParams.get("user_id"), "user_id")), ); } +/** Drains one user's log. Needed before rolling the code back - a v4 manifest is + * unreadable to an older deployment - and by the log-mode test, which otherwise has + * no way to observe a fully drained counter. Bounded so one request cannot fold an + * unbounded log. */ +async function foldRoute(request, env, url, path) { + if (path !== "/admin/fold") return undefined; + requireOwner(request, env); + if (request.method !== "POST") + throw new HttpError(405, "method_not_allowed", "Use POST to fold one user's log"); + const userId = textField(url.searchParams.get("user_id"), "user_id"); + let folded = 0; + let remaining = 0; + let released = 0; + for (let round = 0; round < 8; round += 1) { + const step = await foldPending(env, userId); + folded += step.folded; + remaining = step.remaining; + if (step.released) released += 1; + if (step.folded === 0 && !step.released) break; + } + // A drain is a verification, not an attempt: the runbook uses it before rolling + // the code back, and a v4 manifest is unreadable to an older deployment. Answering + // 200 with entries left would make that check a false green, so anything not fully + // drained is a 409 the caller has to retry. + const drained = remaining === 0; + return json( + { success: drained, user_id: userId, folded, released, remaining }, + drained ? 200 : 409, + ); +} async function addRoute(request, env, path) { if ((path !== "/add" && path !== "/remember") || request.method !== "POST") return undefined; - return json(await add(env, await readJson(request))); + const input = await readJson(request); + if (addMode(env) !== "log") return json(await add(env, input)); + const accepted = await acceptAdd(env, input); + // A burst leaves entries waiting; fold a bounded batch here rather than on the + // read path, so accepted latency stays flat while the log cannot grow forever. + if (accepted.pending >= FOLD_TRIGGER) + await foldPending(env, accepted.response.user_id).catch(() => {}); + return json(accepted.response); } async function handle(request, env) { const url = new URL(request.url); @@ -1228,7 +1750,8 @@ async function handle(request, env) { const early = pageResponse(request, env, path) ?? (await healthResponse(env, path)) ?? - (await reclaimAllRoute(request, env, path)); + (await reclaimAllRoute(request, env, path)) ?? + (await foldRoute(request, env, url, path)); if (early) return early; if (isMemoryRoute(path)) { if (path.startsWith("/admin/")) requireOwner(request, env); diff --git a/onlion/package-lock.json b/onlion/package-lock.json index b4cb7cf0..7d888914 100644 --- a/onlion/package-lock.json +++ b/onlion/package-lock.json @@ -13,6 +13,7 @@ "devDependencies": { "@types/node": "24.12.4", "esbuild": "0.28.1", + "miniflare": "4.20260515.0", "typescript": "5.9.3", "wrangler": "4.153.0" }, @@ -608,9 +609,9 @@ } }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.5.tgz", - "integrity": "sha512-QRUlFQ0WxvdWyqqG/WtI3iupfD5rBzmCHXSdPsY91sAtVtTo7Q4cb6zOccZ3gqEqkr0f1As1ehLqmEpDsRf+lg==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", "cpu": [ "arm64" ], @@ -621,19 +622,19 @@ "darwin" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.3.4" + "@img/sharp-libvips-darwin-arm64": "1.2.4" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.5.tgz", - "integrity": "sha512-+BR255RhDlpygUpOc/Jdt1nT6DQ3XG/ERo5wbcdOf5Q320dKtPCKPLR1LJs9VGXRaMa8l1uUa0tkCNOXiAxZUw==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", "cpu": [ "x64" ], @@ -644,13 +645,13 @@ "darwin" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.3.4" + "@img/sharp-libvips-darwin-x64": "1.2.4" } }, "node_modules/@img/sharp-freebsd-wasm32": { @@ -673,10 +674,27 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-freebsd-wasm32/node_modules/@img/sharp-wasm32": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.5.tgz", + "integrity": "sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==", + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.4.tgz", - "integrity": "sha512-5R89nBYiRdUlSWJxPhO+GVtaXzXSxKnRu/xqMn3KTA3L9EB9Oy/P+Nn2f2vlhPuUdy/Zusb2DarbyTpGCfEDuw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", "cpu": [ "arm64" ], @@ -691,9 +709,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.4.tgz", - "integrity": "sha512-iR2OKH80yi0U+dUplyh3/xdpFvps6YkCwsXenIJxqxR1v9o+xtKTGbS9H7cps+2Vxjc8B1j96p75NmTGjIhtpQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", "cpu": [ "x64" ], @@ -708,9 +726,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.4.tgz", - "integrity": "sha512-LmRtTsOHuvM2+wlO2Db37dx5MiZhB0FvSunciw48YjdOkZz9KAiRbm8ujeMOA1INqmei5NapFxYEK1D1ZSidmw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", "cpu": [ "arm" ], @@ -728,9 +746,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.4.tgz", - "integrity": "sha512-Y3dgX/6lE2QhQb+Gxy0WZxfg9MEm/JBjamZpS2IklP7xIQoKN4hzAm7KcMVGtaVDt3neE9OKBC7vAfonA/Lr1A==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", "cpu": [ "arm64" ], @@ -748,9 +766,9 @@ } }, "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.4.tgz", - "integrity": "sha512-Le6boB8Tai0Nis+gIxIpKx68UDVVIqdR8Tin5Yf1z2LJJQLDJvCDRqRu+jC2qCoD+eIomonmOwB4smBRxfVpYQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", "cpu": [ "ppc64" ], @@ -768,9 +786,9 @@ } }, "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.4.tgz", - "integrity": "sha512-aHkkIEHPRdQEegJN20MLmGtxYD9R2wQr3Cwpddnu5+YKMt6Uzax7S9h5gpZTo8wyrGuZSlfQ63OevL5mTyOC7Q==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", "cpu": [ "riscv64" ], @@ -788,9 +806,9 @@ } }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.4.tgz", - "integrity": "sha512-ra/mB6MikESDUO7Yg+Mi95bFBb9GsObURuhnOv3OqknjGe9sZrG8tCe9q0xSIGrtLgvgw0gKnFWcK4blSgQOuQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", "cpu": [ "s390x" ], @@ -808,9 +826,9 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.4.tgz", - "integrity": "sha512-GJ//SSXbnwSDes02umB3nDJLFcQzw8a18V8fyhqr6tV515tOEMdImjjxj1AoafMRz56F3PHgftnj1QEKSU1zkw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", "cpu": [ "x64" ], @@ -828,9 +846,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.4.tgz", - "integrity": "sha512-hvulFwtjUcagsis6BBxHwGFwWoNZjgYmULGVrZcyfNbjA8hKILbRxGg15/7w5HDyXHXUos/j6baAWqnCyQ2DWA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", "cpu": [ "arm64" ], @@ -848,9 +866,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.4.tgz", - "integrity": "sha512-6zXKeE/p39I1AmA3cJG35eyBGNqNddLnUXjhwBnsGjFPWqf5VKkDBEqaEkPDoTEtkxwi2vv8Tcr2mDyP4So7Fg==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", "cpu": [ "x64" ], @@ -868,9 +886,9 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.5.tgz", - "integrity": "sha512-LEaXK2WdXVK5ykcw0buWyPMsmLLL2vpHLD6yrNSW+JGEL3BZPA4tpKN6iaMc4AxTTAoaX/sU1rOL51lcIz48ZQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", "cpu": [ "arm" ], @@ -884,19 +902,19 @@ "linux" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.3.4" + "@img/sharp-libvips-linux-arm": "1.2.4" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.5.tgz", - "integrity": "sha512-LYVx5JTsOM2CBzmxreh+nl64/3H6Xb09iSLknqH47z2T2DFFxDeFLP5y4dJwe6H7uGQlHPyEEtIqyo3DYsRwdQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", "cpu": [ "arm64" ], @@ -910,19 +928,19 @@ "linux" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.3.4" + "@img/sharp-libvips-linux-arm64": "1.2.4" } }, "node_modules/@img/sharp-linux-ppc64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.5.tgz", - "integrity": "sha512-QVxAAq8evVRI9ia2vqgwrmWucn5Dfv+JdWzj75pD8omHLPSP7f8p20O8jxzjCcuCEQEOtYOZUmX1hkiZ0kdevA==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", "cpu": [ "ppc64" ], @@ -936,19 +954,19 @@ "linux" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.3.4" + "@img/sharp-libvips-linux-ppc64": "1.2.4" } }, "node_modules/@img/sharp-linux-riscv64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.5.tgz", - "integrity": "sha512-LtdreXguaavKODPIfzJ4kffx7UNt1omwtK0rch4EBbbSTXPnxWmYSayXdLJw0fJzQ97kHt1gL/yh4tvU+nCyRQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", "cpu": [ "riscv64" ], @@ -962,19 +980,19 @@ "linux" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.3.4" + "@img/sharp-libvips-linux-riscv64": "1.2.4" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.5.tgz", - "integrity": "sha512-UZasTOFiYzotTsGOCu42BfUzP6Tu6Do/947iRm1RsLKvlllxwGcn4RN27LibGWceix4Y+Pmw3jsnTcCQIgWjqA==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", "cpu": [ "s390x" ], @@ -988,19 +1006,19 @@ "linux" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.3.4" + "@img/sharp-libvips-linux-s390x": "1.2.4" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.5.tgz", - "integrity": "sha512-SxFtLTeJInhAA9Q836kux2vZNeOBQEx658qvbboZScr0wIARym3IcGmW7KpVD5sbVg0Ojy+udFQdayYIZyoNog==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", "cpu": [ "x64" ], @@ -1014,19 +1032,19 @@ "linux" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.3.4" + "@img/sharp-libvips-linux-x64": "1.2.4" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.5.tgz", - "integrity": "sha512-9HbMclmI1zlNkFRs3z9/eBtDjfD0sGlrX1z6b1qwmiFY5ElDLh4BC0LPBdVp7z1DXFiKlIcznf+ZlsuZzLxQqg==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", "cpu": [ "arm64" ], @@ -1040,19 +1058,19 @@ "linux" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.3.4" + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.5.tgz", - "integrity": "sha512-4KOphqB035HrVdqLZfCgMzzERrQkkzOwRhl4OAkRO1YCldbaFjySXMaK534Mo0V+LndnlJk+sbUyLeU0ULyD1A==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", "cpu": [ "x64" ], @@ -1066,27 +1084,30 @@ "linux" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.3.4" + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.5.tgz", - "integrity": "sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { - "@emnapi/runtime": "^1.11.3" + "@emnapi/runtime": "^1.7.0" }, "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" @@ -1112,10 +1133,27 @@ "url": "https://opencollective.com/libvips" } }, - "node_modules/@img/sharp-win32-arm64": { + "node_modules/@img/sharp-webcontainers-wasm32/node_modules/@img/sharp-wasm32": { "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.5.tgz", - "integrity": "sha512-X4t7g+7ZA5DKblCBEXGjUqqemj4vczING/5viFwAL8h4N3qYeyjwdCvRLHi4EdOUI+2Z7UFlp1VM+p/AuEtm6Q==", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.5.tgz", + "integrity": "sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==", + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", "cpu": [ "arm64" ], @@ -1126,16 +1164,16 @@ "win32" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.5.tgz", - "integrity": "sha512-5Zm82LoBc43nhwNybZlG7Y1KO//Zhsn306fQl29ZOuStHLGTo3BWL83q3cznX0poxSAMuYL1On/BHBxkBeKr6A==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", "cpu": [ "ia32" ], @@ -1146,16 +1184,16 @@ "win32" ], "engines": { - "node": "^20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.5.tgz", - "integrity": "sha512-x76eH0vEiHlcMQu8Y8IenntaACtddpT6W0wmXtWrnKcnKI7ME5DdgqhAD6SEWOEl1v2zDvkZDhFA9KnURwpfqg==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", "cpu": [ "x64" ], @@ -1166,7 +1204,7 @@ "win32" ], "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" @@ -1368,23 +1406,132 @@ } }, "node_modules/miniflare": { - "version": "5.20261023.0-beta", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20261023.0-beta.tgz", - "integrity": "sha512-SHY8A98gCvmrnBLqABHRTSh3duAL1z+tHPRZu/tqxHFqVARrZ4oQis/ALYwp3XvzHvs6B/ff/CdcCafqP+cOFA==", + "version": "4.20260515.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260515.0.tgz", + "integrity": "sha512-2j0oQWizk1Eu4Cm8tDX7Z+Nsjd0nebIj1TQcQ+Oy1QKeo0Ay9+bdn8wfLAtOj9znDCybDCUlnS1+nYvKXEdfNg==", "dev": true, "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "0.8.1", - "sharp": "0.35.5", - "undici": "7.30.1", - "workerd": "1.20261023.1", - "ws": "8.21.0", + "sharp": "^0.34.5", + "undici": "7.24.8", + "workerd": "1.20260515.1", + "ws": "8.18.0", "youch": "4.1.0-beta.10" }, + "bin": { + "miniflare": "bootstrap.js" + }, "engines": { "node": ">=22.0.0" } }, + "node_modules/miniflare/node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20260515.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260515.1.tgz", + "integrity": "sha512-Wtw44el2pNbzixvTkWdfeBDTrQwQbJRz7/JUvPKV27I0pQWXbhNJPpM8cstq/pbrU5AGcA/HjFH6yPMRTIRKig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/miniflare/node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20260515.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260515.1.tgz", + "integrity": "sha512-X8EqkZej6FfmhF9AVAQ3FhyQRr9acS4RcDunMU2YiuxKHF1IU8zzH3vY30/POaG+rUu9vGDp/VgUl49VPenHJQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/miniflare/node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20260515.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260515.1.tgz", + "integrity": "sha512-CDC89QxQ7Y7t7RG1Jd9vj/qolE1sQRkI2OSEuV5BMJi0vW/gV4OVG6xjpdK3b1OYnSWDzF7NpvlR5Yg86q7k4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/miniflare/node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20260515.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260515.1.tgz", + "integrity": "sha512-WxbW/PToYES4fvHXzsr/5qOiETQs/Z9iZ0mjSZAiEwq5cMLZemzGN0COx+uFb9OvQwzh6Pg159qPFnw3+i9FuA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/miniflare/node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20260515.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260515.1.tgz", + "integrity": "sha512-WmV/iv+MHjYsvkcMVzpM2B5/mf06UUkdpVhZrtMfV9graWjBGPYFvE/eab8748RPVGKh1Xe1vXofLzDSwc08lA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/miniflare/node_modules/workerd": { + "version": "1.20260515.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260515.1.tgz", + "integrity": "sha512-MjKOJLcvU45xXedQowvuiHtJTxu4WTHYQeIlF7YmjuqhiI6dImTFxWCEoRQHiskztxuVSNEmdO7/0UfDu6OMnQ==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20260515.1", + "@cloudflare/workerd-darwin-arm64": "1.20260515.1", + "@cloudflare/workerd-linux-64": "1.20260515.1", + "@cloudflare/workerd-linux-arm64": "1.20260515.1", + "@cloudflare/workerd-windows-64": "1.20260515.1" + } + }, "node_modules/path-to-regexp": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", @@ -1419,53 +1566,48 @@ } }, "node_modules/sharp": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.5.tgz", - "integrity": "sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", "dev": true, + "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { - "@img/colour": "^1.1.0", + "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", - "semver": "^7.8.5" + "semver": "^7.7.3" }, "engines": { - "node": ">=20.9.0" + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.35.5", - "@img/sharp-darwin-x64": "0.35.5", - "@img/sharp-freebsd-wasm32": "0.35.5", - "@img/sharp-libvips-darwin-arm64": "1.3.4", - "@img/sharp-libvips-darwin-x64": "1.3.4", - "@img/sharp-libvips-linux-arm": "1.3.4", - "@img/sharp-libvips-linux-arm64": "1.3.4", - "@img/sharp-libvips-linux-ppc64": "1.3.4", - "@img/sharp-libvips-linux-riscv64": "1.3.4", - "@img/sharp-libvips-linux-s390x": "1.3.4", - "@img/sharp-libvips-linux-x64": "1.3.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.3.4", - "@img/sharp-libvips-linuxmusl-x64": "1.3.4", - "@img/sharp-linux-arm": "0.35.5", - "@img/sharp-linux-arm64": "0.35.5", - "@img/sharp-linux-ppc64": "0.35.5", - "@img/sharp-linux-riscv64": "0.35.5", - "@img/sharp-linux-s390x": "0.35.5", - "@img/sharp-linux-x64": "0.35.5", - "@img/sharp-linuxmusl-arm64": "0.35.5", - "@img/sharp-linuxmusl-x64": "0.35.5", - "@img/sharp-webcontainers-wasm32": "0.35.5", - "@img/sharp-win32-arm64": "0.35.5", - "@img/sharp-win32-ia32": "0.35.5", - "@img/sharp-win32-x64": "0.35.5" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" } }, "node_modules/supports-color": { @@ -1504,9 +1646,9 @@ } }, "node_modules/undici": { - "version": "7.30.1", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", - "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", + "version": "7.24.8", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.8.tgz", + "integrity": "sha512-6KQ/+QxK49Z/p3HO6E5ZCZWNnCasyZLa5ExaVYyvPxUwKtbCPMKELJOqh7EqOle0t9cH/7d2TaaTRRa6Nhs4YQ==", "dev": true, "license": "MIT", "engines": { @@ -2029,52 +2171,660 @@ "node": ">=18" } }, - "node_modules/wrangler/node_modules/esbuild": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", - "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "node_modules/wrangler/node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.5.tgz", + "integrity": "sha512-QRUlFQ0WxvdWyqqG/WtI3iupfD5rBzmCHXSdPsY91sAtVtTo7Q4cb6zOccZ3gqEqkr0f1As1ehLqmEpDsRf+lg==", + "cpu": [ + "arm64" + ], "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.4" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-darwin-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.5.tgz", + "integrity": "sha512-+BR255RhDlpygUpOc/Jdt1nT6DQ3XG/ERo5wbcdOf5Q320dKtPCKPLR1LJs9VGXRaMa8l1uUa0tkCNOXiAxZUw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], "engines": { - "node": ">=18" + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.28.2", - "@esbuild/android-arm": "0.28.2", - "@esbuild/android-arm64": "0.28.2", - "@esbuild/android-x64": "0.28.2", - "@esbuild/darwin-arm64": "0.28.2", - "@esbuild/darwin-x64": "0.28.2", - "@esbuild/freebsd-arm64": "0.28.2", - "@esbuild/freebsd-x64": "0.28.2", - "@esbuild/linux-arm": "0.28.2", - "@esbuild/linux-arm64": "0.28.2", - "@esbuild/linux-ia32": "0.28.2", - "@esbuild/linux-loong64": "0.28.2", - "@esbuild/linux-mips64el": "0.28.2", - "@esbuild/linux-ppc64": "0.28.2", - "@esbuild/linux-riscv64": "0.28.2", - "@esbuild/linux-s390x": "0.28.2", - "@esbuild/linux-x64": "0.28.2", - "@esbuild/netbsd-arm64": "0.28.2", - "@esbuild/netbsd-x64": "0.28.2", - "@esbuild/openbsd-arm64": "0.28.2", - "@esbuild/openbsd-x64": "0.28.2", - "@esbuild/openharmony-arm64": "0.28.2", - "@esbuild/sunos-x64": "0.28.2", - "@esbuild/win32-arm64": "0.28.2", - "@esbuild/win32-ia32": "0.28.2", - "@esbuild/win32-x64": "0.28.2" + "@img/sharp-libvips-darwin-x64": "1.3.4" } }, - "node_modules/ws": { - "version": "8.21.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", - "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "node_modules/wrangler/node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.4.tgz", + "integrity": "sha512-5R89nBYiRdUlSWJxPhO+GVtaXzXSxKnRu/xqMn3KTA3L9EB9Oy/P+Nn2f2vlhPuUdy/Zusb2DarbyTpGCfEDuw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.4.tgz", + "integrity": "sha512-iR2OKH80yi0U+dUplyh3/xdpFvps6YkCwsXenIJxqxR1v9o+xtKTGbS9H7cps+2Vxjc8B1j96p75NmTGjIhtpQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.4.tgz", + "integrity": "sha512-LmRtTsOHuvM2+wlO2Db37dx5MiZhB0FvSunciw48YjdOkZz9KAiRbm8ujeMOA1INqmei5NapFxYEK1D1ZSidmw==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.4.tgz", + "integrity": "sha512-Y3dgX/6lE2QhQb+Gxy0WZxfg9MEm/JBjamZpS2IklP7xIQoKN4hzAm7KcMVGtaVDt3neE9OKBC7vAfonA/Lr1A==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.4.tgz", + "integrity": "sha512-Le6boB8Tai0Nis+gIxIpKx68UDVVIqdR8Tin5Yf1z2LJJQLDJvCDRqRu+jC2qCoD+eIomonmOwB4smBRxfVpYQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.4.tgz", + "integrity": "sha512-aHkkIEHPRdQEegJN20MLmGtxYD9R2wQr3Cwpddnu5+YKMt6Uzax7S9h5gpZTo8wyrGuZSlfQ63OevL5mTyOC7Q==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.4.tgz", + "integrity": "sha512-ra/mB6MikESDUO7Yg+Mi95bFBb9GsObURuhnOv3OqknjGe9sZrG8tCe9q0xSIGrtLgvgw0gKnFWcK4blSgQOuQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.4.tgz", + "integrity": "sha512-GJ//SSXbnwSDes02umB3nDJLFcQzw8a18V8fyhqr6tV515tOEMdImjjxj1AoafMRz56F3PHgftnj1QEKSU1zkw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.4.tgz", + "integrity": "sha512-hvulFwtjUcagsis6BBxHwGFwWoNZjgYmULGVrZcyfNbjA8hKILbRxGg15/7w5HDyXHXUos/j6baAWqnCyQ2DWA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.4.tgz", + "integrity": "sha512-6zXKeE/p39I1AmA3cJG35eyBGNqNddLnUXjhwBnsGjFPWqf5VKkDBEqaEkPDoTEtkxwi2vv8Tcr2mDyP4So7Fg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-linux-arm": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.5.tgz", + "integrity": "sha512-LEaXK2WdXVK5ykcw0buWyPMsmLLL2vpHLD6yrNSW+JGEL3BZPA4tpKN6iaMc4AxTTAoaX/sU1rOL51lcIz48ZQ==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.4" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-linux-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.5.tgz", + "integrity": "sha512-LYVx5JTsOM2CBzmxreh+nl64/3H6Xb09iSLknqH47z2T2DFFxDeFLP5y4dJwe6H7uGQlHPyEEtIqyo3DYsRwdQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.4" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.5.tgz", + "integrity": "sha512-QVxAAq8evVRI9ia2vqgwrmWucn5Dfv+JdWzj75pD8omHLPSP7f8p20O8jxzjCcuCEQEOtYOZUmX1hkiZ0kdevA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.4" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.5.tgz", + "integrity": "sha512-LtdreXguaavKODPIfzJ4kffx7UNt1omwtK0rch4EBbbSTXPnxWmYSayXdLJw0fJzQ97kHt1gL/yh4tvU+nCyRQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.4" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-linux-s390x": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.5.tgz", + "integrity": "sha512-UZasTOFiYzotTsGOCu42BfUzP6Tu6Do/947iRm1RsLKvlllxwGcn4RN27LibGWceix4Y+Pmw3jsnTcCQIgWjqA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.4" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-linux-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.5.tgz", + "integrity": "sha512-SxFtLTeJInhAA9Q836kux2vZNeOBQEx658qvbboZScr0wIARym3IcGmW7KpVD5sbVg0Ojy+udFQdayYIZyoNog==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.4" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.5.tgz", + "integrity": "sha512-9HbMclmI1zlNkFRs3z9/eBtDjfD0sGlrX1z6b1qwmiFY5ElDLh4BC0LPBdVp7z1DXFiKlIcznf+ZlsuZzLxQqg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.4" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.5.tgz", + "integrity": "sha512-4KOphqB035HrVdqLZfCgMzzERrQkkzOwRhl4OAkRO1YCldbaFjySXMaK534Mo0V+LndnlJk+sbUyLeU0ULyD1A==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.4" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-win32-arm64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.5.tgz", + "integrity": "sha512-X4t7g+7ZA5DKblCBEXGjUqqemj4vczING/5viFwAL8h4N3qYeyjwdCvRLHi4EdOUI+2Z7UFlp1VM+p/AuEtm6Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-win32-ia32": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.5.tgz", + "integrity": "sha512-5Zm82LoBc43nhwNybZlG7Y1KO//Zhsn306fQl29ZOuStHLGTo3BWL83q3cznX0poxSAMuYL1On/BHBxkBeKr6A==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/@img/sharp-win32-x64": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.5.tgz", + "integrity": "sha512-x76eH0vEiHlcMQu8Y8IenntaACtddpT6W0wmXtWrnKcnKI7ME5DdgqhAD6SEWOEl1v2zDvkZDhFA9KnURwpfqg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/wrangler/node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "node_modules/wrangler/node_modules/miniflare": { + "version": "5.20261023.0-beta", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20261023.0-beta.tgz", + "integrity": "sha512-SHY8A98gCvmrnBLqABHRTSh3duAL1z+tHPRZu/tqxHFqVARrZ4oQis/ALYwp3XvzHvs6B/ff/CdcCafqP+cOFA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "0.35.5", + "undici": "7.30.1", + "workerd": "1.20261023.1", + "ws": "8.21.0", + "youch": "4.1.0-beta.10" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/wrangler/node_modules/sharp": { + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.5.tgz", + "integrity": "sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.5", + "@img/sharp-darwin-x64": "0.35.5", + "@img/sharp-freebsd-wasm32": "0.35.5", + "@img/sharp-libvips-darwin-arm64": "1.3.4", + "@img/sharp-libvips-darwin-x64": "1.3.4", + "@img/sharp-libvips-linux-arm": "1.3.4", + "@img/sharp-libvips-linux-arm64": "1.3.4", + "@img/sharp-libvips-linux-ppc64": "1.3.4", + "@img/sharp-libvips-linux-riscv64": "1.3.4", + "@img/sharp-libvips-linux-s390x": "1.3.4", + "@img/sharp-libvips-linux-x64": "1.3.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.4", + "@img/sharp-libvips-linuxmusl-x64": "1.3.4", + "@img/sharp-linux-arm": "0.35.5", + "@img/sharp-linux-arm64": "0.35.5", + "@img/sharp-linux-ppc64": "0.35.5", + "@img/sharp-linux-riscv64": "0.35.5", + "@img/sharp-linux-s390x": "0.35.5", + "@img/sharp-linux-x64": "0.35.5", + "@img/sharp-linuxmusl-arm64": "0.35.5", + "@img/sharp-linuxmusl-x64": "0.35.5", + "@img/sharp-webcontainers-wasm32": "0.35.5", + "@img/sharp-win32-arm64": "0.35.5", + "@img/sharp-win32-ia32": "0.35.5", + "@img/sharp-win32-x64": "0.35.5" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/wrangler/node_modules/undici": { + "version": "7.30.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", + "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/wrangler/node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "dev": true, "license": "MIT", "engines": { diff --git a/onlion/package.json b/onlion/package.json index 793b1142..954bc1b3 100644 --- a/onlion/package.json +++ b/onlion/package.json @@ -18,13 +18,19 @@ "test:worker-media": "node scripts/test-multimodal-worker.mjs", "test:cos": "node scripts/test-cos-bucket.mjs", "test:edgeone": "node scripts/test-edgeone-worker.mjs", - "verify": "npm run build && npm run check && npm run test:temporal && npm run test:media && npm run test:backend && npm run test:worker-media", + "verify": "npm run build && npm run check && npm run test:temporal && npm run test:media && npm run test:backend && npm run test:append-log && npm run test:manifest-pending && npm run test:add-log && npm run test:page-delta && npm run test:extractive && npm run test:worker-media", "generate:callbacks": "node scripts/generate-callbacks.mjs", - "smoke:edgeone": "node scripts/smoke-edgeone.mjs" + "smoke:edgeone": "node scripts/smoke-edgeone.mjs", + "test:add-log": "node scripts/test-add-log-mode.mjs", + "test:append-log": "node scripts/test-append-log.mjs", + "test:manifest-pending": "node scripts/test-manifest-pending.mjs", + "test:page-delta": "node scripts/test-page-delta.mjs", + "test:extractive": "node --experimental-transform-types scripts/test-extractive-summaries.mjs" }, "devDependencies": { "@types/node": "24.12.4", "esbuild": "0.28.1", + "miniflare": "4.20260515.0", "typescript": "5.9.3", "wrangler": "4.153.0" }, diff --git a/onlion/scripts/bench-add-concurrency.mjs b/onlion/scripts/bench-add-concurrency.mjs new file mode 100644 index 00000000..f4ec984f --- /dev/null +++ b/onlion/scripts/bench-add-concurrency.mjs @@ -0,0 +1,420 @@ +/** + * Does the log accept path actually beat the synchronous one under a burst? + * + * The stopping rules for the append-log write path put one number behind the default: + * a 16-way concurrent Add burst must show a lower p95 with the log accept path than with + * `sync`. This produces both numbers from the same client against the same endpoint. + * + * Three things it deliberately does differently from a naive burst loop: + * + * - **one user per run, not per round.** A fresh user per round peaks at `concurrency` + * pending entries, below `FOLD_TRIGGER`, so a burst benchmark would never fold at all + * and would measure accept while calling it the write path. + * - **folds are derived, not read.** The service exposes no fold counter, so each round + * samples `pending_entries` before and after and derives how many folds must have run. + * - **the fold cost is isolated.** `--mode fold` warms pending to one below the trigger + * and then times a single Add, which is the request that must pay both. + * + * A percentile needs enough samples to exist: with fewer than twenty the 0.95 quantile is + * the maximum, so per-round lines are labelled `p95(=max)` and only the pooled figure - + * `concurrency x rounds` - is quoted as a p95. + * + * Rounds are grouped by `crossedTrigger` (`pendingBefore + accepted >= FOLD_TRIGGER`): + * a round that crosses contains the request that had to fold. It is `null` when the + * counter could not be read, and `null` is excluded from the clean group rather than + * counted as clean. + * + * Failed *connections* are reported apart from server failures, and a run where most + * requests never connected says so instead of publishing percentiles about this host. + * + * node onlion/scripts/bench-add-concurrency.mjs \ + * --endpoint https://api.nmgmemory.dpdns.org --key "$KEY" \ + * --concurrency 16 --messages 8 --rounds 3 --warmup 1 \ + * --owner owner@example.test --add-mode log + */ +import { parseArgs } from "node:util"; + +const FOLD_BATCH = 8; +const FOLD_TRIGGER = 24; + +function percentile(sorted, fraction) { + if (sorted.length === 0) return 0; + const index = Math.min(sorted.length - 1, Math.ceil(fraction * sorted.length) - 1); + return sorted[Math.max(0, index)]; +} + +const turn = (tag, index) => + `Burst ${tag} turn ${index}: the spare key is under the blue planter and the ` + + `invoice number ${tag}-${index} was paid on the fifth.`; + +function addBody(user, requestId, messages) { + return { + user_id: user, + session_id: "bench", + request_id: requestId, + messages: Array.from({ length: messages }, (_unused, index) => ({ + role: index % 2 === 0 ? "user" : "assistant", + content: turn(requestId, index), + })), + }; +} + +async function request(url, options) { + const started = performance.now(); + try { + const response = await fetch(url, options); + const text = await response.text(); + // The parsed body is kept for the capacity reads and dropped before serialisation: a + // page of records per request would dominate the JSON and say nothing about timing. + let body = null; + try { + body = JSON.parse(text); + } catch { + body = null; + } + return { + ms: performance.now() - started, + status: response.status, + code: body?.error?.code ?? null, + body, + }; + } catch (error) { + return { + ms: performance.now() - started, + status: 0, + code: error?.cause?.code ?? error?.name ?? "network_error", + body: null, + }; + } +} + +const add = (endpoint, key, user, requestId, messages, timeout) => + request(`${endpoint}/add`, { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${key}` }, + body: JSON.stringify(addBody(user, requestId, messages)), + signal: AbortSignal.timeout(timeout), + }); + +/** Waiting entries for one user, or null when the endpoint cannot say. */ +async function pendingOf(endpoint, key, user, timeout) { + const result = await request(`${endpoint}/capacity?user_id=${encodeURIComponent(user)}`, { + headers: { authorization: `Bearer ${key}` }, + signal: AbortSignal.timeout(timeout), + }); + if (result.status !== 200) return null; + return result.body?.pending_entries ?? null; +} + +/** Drains the log so a measurement can start from zero. Best effort: without an owner + * credential this is a 401, and the run continues with whatever pending it found. */ +async function drain(endpoint, key, user, owner, timeout) { + if (!owner) return { drained: false, reason: "no --owner, skipped" }; + const result = await request(`${endpoint}/admin/fold?user_id=${encodeURIComponent(user)}`, { + method: "POST", + headers: { + authorization: `Bearer ${key}`, + "oai-authenticated-user-email": owner, + origin: endpoint, + }, + signal: AbortSignal.timeout(timeout), + }); + return { drained: result.status === 200, status: result.status, code: result.code }; +} + +/** Runs `count` requests with `concurrency` in flight, never more. */ +async function burst(tasks, concurrency) { + const results = []; + let next = 0; + const worker = async () => { + while (next < tasks.length) { + const index = next++; + results[index] = await tasks[index](); + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, tasks.length) }, worker)); + return results; +} + +/** Timing fields only: the parsed body exists for the capacity reads, not for the report. */ +const timing = ({ ms, status, code }) => ({ ms: Math.round(ms), status, code }); + +function summarise(label, results) { + const times = results.map((result) => result.ms).sort((left, right) => left - right); + const byStatus = new Map(); + for (const result of results) { + const key = result.status === 0 ? `err:${result.code}` : String(result.status); + byStatus.set(key, (byStatus.get(key) ?? 0) + 1); + } + const failures = results.filter((result) => result.status < 200 || result.status >= 300); + return { + label, + count: results.length, + p50: Math.round(percentile(times, 0.5)), + p95: Math.round(percentile(times, 0.95)), + // Below 20 samples the 0.95 quantile lands on the last sample, so calling it a p95 + // would invite a decision on a maximum. + p95IsMax: times.length < 20, + max: Math.round(times.at(-1) ?? 0), + statuses: Object.fromEntries([...byStatus].sort()), + failures: failures.length, + clientErrors: results.filter((result) => result.status === 0).length, + serverErrors: results.filter((result) => result.status >= 500).length, + codes: [...new Set(failures.map((result) => result.code ?? `http_${result.status}`))], + }; +} + +const stats = (times) => ({ + n: times.length, + p50: Math.round(percentile(times, 0.5)), + p95: Math.round(percentile(times, 0.95)), + p95IsMax: times.length < 20, +}); + +/** How many folds must have run: pending grew by the accepts that succeeded and shrank by + * what each fold absorbed. Derived, because the service exposes no fold counter. + * + * Only meaningful with the log accept path: in `sync` mode an accepted Add never becomes a + * pending entry, so the same arithmetic would invent folds that cannot exist. */ +function deriveFolds(addMode, before, accepted, after) { + if (addMode !== "log") return 0; + if (before === null || after === null) return null; + const absorbed = before + accepted - after; + if (absorbed < 0) return 0; + return Math.round(absorbed / FOLD_BATCH); +} + +function parseOptions() { + const { values } = parseArgs({ + options: { + endpoint: { type: "string" }, + key: { type: "string" }, + concurrency: { type: "string", default: "16" }, + messages: { type: "string", default: "8" }, + rounds: { type: "string", default: "3" }, + warmup: { type: "string", default: "1" }, + timeout: { type: "string", default: "120000" }, + mode: { type: "string", default: "burst" }, + "add-mode": { type: "string", default: "sync" }, + owner: { type: "string" }, + json: { type: "boolean", default: false }, + label: { type: "string", default: "unlabelled" }, + }, + }); + const endpoint = (values.endpoint ?? "").replace(/\/$/u, ""); + if (!endpoint || !values.key) { + process.stderr.write("usage: bench-add-concurrency --endpoint URL --key KEY\n"); + process.exitCode = 2; + return null; + } + if (values.mode !== "burst" && values.mode !== "fold") { + process.stderr.write(`unknown --mode: ${values.mode} (burst | fold)\n`); + process.exitCode = 2; + return null; + } + return { + endpoint, + key: values.key, + concurrency: Number(values.concurrency), + messages: Number(values.messages), + rounds: Number(values.rounds), + warmup: Number(values.warmup), + timeout: Number(values.timeout), + mode: values.mode, + addMode: values["add-mode"], + owner: values.owner, + json: values.json, + label: values.label, + }; +} + +/** Three attempts: one stall is how an ordinary host behaves on a bad minute, but a + * benchmark against a dead endpoint must not report zeros. */ +async function waitForEndpoint(endpoint) { + let server = null; + for (let attempt = 0; attempt < 3 && server === null; attempt += 1) { + try { + const health = await fetch(`${endpoint}/health`, { signal: AbortSignal.timeout(20_000) }); + if (health.ok) server = await health.json(); + else process.stderr.write(`health answered ${health.status}\n`); + } catch (error) { + process.stderr.write(`health attempt ${attempt + 1} failed: ${error.message}\n`); + } + } + return server; +} + +async function runFoldMode(config, summary) { + // One below the trigger, untimed, so the single timed Add is the request that pays a + // fold. Its cost is an upper bound on what folding adds to one Add. + for (let index = 0; index < FOLD_TRIGGER - 1; index += 1) + await add(config.endpoint, config.key, summary.user, `${summary.stamp}-pre-${index}`, config.messages, config.timeout); + const before = await pendingOf(config.endpoint, config.key, summary.user, config.timeout); + const timed = await add(config.endpoint, config.key, summary.user, `${summary.stamp}-boundary`, config.messages, config.timeout); + const after = await pendingOf(config.endpoint, config.key, summary.user, config.timeout); + summary.boundary = { + ...summarise("fold boundary", [timed]), + sample: timing(timed), + pendingBefore: before, + pendingAfter: after, + foldsDerived: deriveFolds(summary.server.add_mode ?? config.addMode, before, timed.status === 200 ? 1 : 0, after), + }; + // One sample: there is no percentile to report, only that measurement, and it is an + // upper bound on what folding adds to one Add. + if (!config.json) + process.stdout.write( + `fold boundary (single sample, upper bound): status ${summary.boundary.sample.status} ` + + `in ${summary.boundary.sample.ms}ms | pending ${before} -> ${after} | ` + + `folds derived=${summary.boundary.foldsDerived}\n`, + ); +} + +async function runBurstMode(config, summary) { + for (let round = 0; round < config.rounds; round += 1) { + // Drain before each round when a credential allows it: otherwise only the first round + // sits below the fold trigger, the clean group collapses to one round (n=16, a maximum + // rather than a p95), and the gate has nothing to quote. + if (config.owner) await drain(config.endpoint, config.key, summary.user, config.owner, config.timeout); + const before = await pendingOf(config.endpoint, config.key, summary.user, config.timeout); + const tasks = Array.from( + { length: config.concurrency }, + (_unused, slot) => () => + add(config.endpoint, config.key, summary.user, `${summary.stamp}-${round}-${slot}`, config.messages, config.timeout), + ); + const results = await burst(tasks, config.concurrency); + const after = await pendingOf(config.endpoint, config.key, summary.user, config.timeout); + const roundSummary = summarise(`round ${round + 1}`, results); + roundSummary.samples = results.map(timing); + roundSummary.pendingBefore = before; + roundSummary.pendingAfter = after; + roundSummary.accepted = results.filter((result) => result.status === 200).length; + roundSummary.crossedTrigger = + before === null ? null : before + roundSummary.accepted >= FOLD_TRIGGER; + roundSummary.foldsDerived = deriveFolds( + summary.server.add_mode ?? config.addMode, + before, + roundSummary.accepted, + after, + ); + if (summary.server.add_mode === "sync" && (before !== 0 || after !== 0)) + roundSummary.modeMismatch = + `declared sync but pending moved ${before}->${after}; the deployment may be running the log path`; + summary.rounds.push(roundSummary); + if (!config.json) + process.stdout.write( + `${roundSummary.label}: n=${roundSummary.count} p50=${roundSummary.p50}ms ` + + `${roundSummary.p95IsMax ? "p95(=max)" : "p95"}=${roundSummary.p95}ms ` + + `max=${roundSummary.max}ms failures=${roundSummary.failures} ` + + `(client=${roundSummary.clientErrors} server=${roundSummary.serverErrors}) ` + + `pending ${before}->${after} folds~${roundSummary.foldsDerived} ` + + `crossed=${roundSummary.crossedTrigger} statuses=${JSON.stringify(roundSummary.statuses)}\n`, + ); + } + summariseBurst(summary, config); +} + +function summariseBurst(summary, config) { + const pooledTimes = summary.rounds + .flatMap((round) => round.samples.map((sample) => sample.ms)) + .sort((left, right) => left - right); + // "clean" = rounds that provably never crossed the fold trigger. A round whose pending + // counter could not be read is not clean, it is unknown. + const cleanTimes = summary.rounds + .filter((round) => round.crossedTrigger === false) + .flatMap((round) => round.samples.map((sample) => sample.ms)) + .sort((left, right) => left - right); + const measured = summary.rounds.reduce((sum, round) => sum + round.count, 0); + const clientErrors = summary.rounds.reduce((sum, round) => sum + round.clientErrors, 0); + summary.addMode = config.addMode; + summary.pooled = { + all: stats(pooledTimes), + clean: cleanTimes.length === 0 ? null : stats(cleanTimes), + failures: summary.rounds.reduce((sum, round) => sum + round.failures, 0), + foldsDerived: summary.rounds.reduce((sum, round) => sum + (round.foldsDerived ?? 0), 0), + }; + summary.headline = + `${summary.label}: mode=burst concurrency=${config.concurrency} messages=${config.messages} ` + + `rounds=${config.rounds} n=${summary.pooled.all.n} | p50 ${summary.pooled.all.p50}ms | ` + + `${summary.pooled.all.p95IsMax ? "p95(=max)" : "p95"} ${summary.pooled.all.p95}ms | ` + + `failures ${summary.pooled.failures} (client ${clientErrors}) ` + + (summary.pooled.clean + ? `| clean n=${summary.pooled.clean.n} ` + + `${summary.pooled.clean.p95IsMax ? "p95(=max)" : "p95"} ${summary.pooled.clean.p95}ms` + : "| no round avoided the fold trigger") + + (config.addMode === "log" + ? ` | folds derived=${summary.pooled.foldsDerived} (derived, not reported by the service)` + : ` | add mode=${config.addMode} (log inactive, pending must stay 0)`); + // Refuse to present a measurement that is mostly this host failing to connect. + if (clientErrors * 2 >= measured) + summary.warning = + "more than half the requests never established a connection; these percentiles " + + "measure this host, not the endpoint - run the burst from a stable vantage point"; +} + +function report(summary, json) { + if (json) { + process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); + return; + } + if (summary.headline) process.stdout.write(`${summary.headline}\n`); + if (summary.warning) process.stdout.write(`WARNING: ${summary.warning}\n`); +} + +async function main() { + const config = parseOptions(); + if (!config) return; + const server = await waitForEndpoint(config.endpoint); + if (!server) { + process.stderr.write(`endpoint unreachable: ${config.endpoint}\n`); + process.exitCode = 2; + return; + } + // The deployment decides which path is being measured, not the flag: declaring `log` + // against a `sync` deployment would otherwise produce "folds 0" and read like a + // legitimate log measurement with nothing folded - the exact symptom of a forgotten + // environment variable. Refuse to run, with the same exit code as an unreachable + // endpoint: both mean the setup is wrong, not the service. + const serverMode = server.limits?.add_mode ?? null; + if (serverMode !== null && serverMode !== config.addMode) { + process.stderr.write( + `deployment runs add_mode=${serverMode} but this run declares ${config.addMode}: ` + + "set NMG_ADD_MODE on the project and redeploy, or pass the matching --add-mode\n", + ); + process.exitCode = 2; + return; + } + + const stamp = Date.now().toString(36); + const summary = { + label: config.label, + mode: config.mode, + endpoint: config.endpoint, + user: `bench-${stamp}`, + stamp, + concurrency: config.concurrency, + messages: config.messages, + roundCount: config.rounds, + warmup: config.warmup, + server: { + add_mode: serverMode, + source_revision: server.source_revision ?? null, + upstream_commit: server.upstream_commit ?? null, + }, + rounds: [], + }; + + // Warm the cold path, then empty the log so the measured rounds start from zero: a + // warmup Add leaves one pending entry, and the first round would otherwise start at 1. + if (config.warmup > 0) { + await add(config.endpoint, config.key, summary.user, `${stamp}-warmup`, config.messages, config.timeout); + summary.drain = await drain(config.endpoint, config.key, summary.user, config.owner, config.timeout); + if (!summary.drain.drained) + process.stderr.write(`drain skipped or failed: ${JSON.stringify(summary.drain)}\n`); + } + + if (config.mode === "fold") await runFoldMode(config, summary); + else await runBurstMode(config, summary); + report(summary, config.json); +} + +await main(); diff --git a/onlion/scripts/smoke-edgeone.mjs b/onlion/scripts/smoke-edgeone.mjs index d5ec6c0f..4bf53a9a 100644 --- a/onlion/scripts/smoke-edgeone.mjs +++ b/onlion/scripts/smoke-edgeone.mjs @@ -93,7 +93,16 @@ try { check("retrieval mode is reported", typeof search.body?.retrieval_mode === "string", JSON.stringify(search.body?.retrieval_mode)); const capacity = await call(`/api/capacity?user_id=${encodeURIComponent(userId)}`); - check("/api/capacity answers", capacity.status === 200 && capacity.body?.sqlite_bytes?.used > 0, `${capacity.status} ${capacity.raw?.slice(0, 200)}`); + // Either the Add reached a partition or it is still waiting in the log: which one + // depends on the accept path, and asserting only the first made this smoke a test of + // the sync path rather than of the service. + const stored = + (capacity.body?.sqlite_bytes?.used ?? 0) > 0 || (capacity.body?.pending_entries ?? 0) > 0; + check( + "/api/capacity answers", + capacity.status === 200 && stored, + `${capacity.status} sqlite=${capacity.body?.sqlite_bytes?.used} pending=${capacity.body?.pending_entries}`, + ); const reset = await call("/api/reset", { user_id: userId, confirm_user_id: userId, reset_id: "smoke-reset" }); check("/api/reset commits the barrier", reset.status === 200, `${reset.status} ${reset.raw?.slice(0, 200)}`); diff --git a/onlion/scripts/test-add-log-mode.mjs b/onlion/scripts/test-add-log-mode.mjs new file mode 100644 index 00000000..6d91661b --- /dev/null +++ b/onlion/scripts/test-add-log-mode.mjs @@ -0,0 +1,267 @@ +/* eslint-disable no-console -- deliberate test/diagnostic output */ +/* Verifies the log accept path end to end: durable accept, replay, fold, and that a + * search sees an entry that has not been folded yet. Runs against the built worker + * with a stub embedding endpoint, so no model or network is involved. */ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { createWorkerFixture } from "./worker-fixture.mjs"; + +// Per-process directory: on Windows a previous run's Miniflare can still hold a handle +// on the shared path and the cleanup fails with EPERM before any test runs. The retries +// cover the case where the handle is released a moment later. +const storage = path.join(process.cwd(), ".runtime/tests/log-mode-r2-" + process.pid); +fs.rmSync(storage, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +const DIMENSIONS = 8; +/** Deterministic unit vector per text, so a record and the query that names it + * produce the same direction. */ +function vector(text) { + const raw = new Array(DIMENSIONS).fill(0); + for (const [index, char] of [...String(text)].entries()) + raw[index % DIMENSIONS] += (char.charCodeAt(0) % 17) - 8; + const norm = Math.hypot(...raw) || 1; + return raw.map((value) => Number((value / norm).toFixed(6))); +} +const outboundService = async (request) => { + const url = new URL(request.url); + if (!url.pathname.endsWith("/embeddings")) return new Response("not found", { status: 404 }); + const body = await request.json(); + const inputs = Array.isArray(body.input) ? body.input : [body.input]; + return new Response( + JSON.stringify({ data: inputs.map((text, index) => ({ index, embedding: vector(text) })) }), + { headers: { "content-type": "application/json" } }, + ); +}; + +const mf = createWorkerFixture({ + storage, + outboundService, + bindings: { + NMG_ADD_MODE: "log", + NMG_DEBUG_ERRORS: "1", + NMG_EMBED_BASE_URL: "https://embed.test/v1", + NMG_EMBED_MODEL: "BAAI/bge-small-en-v1.5", + NMG_EMBED_DIMENSIONS: String(DIMENSIONS), + }, +}); +const worker = await mf.getWorker("a"); +let counter = 0; +async function call(route, body, headers = {}) { + const response = await worker.fetch("http://localhost" + route, { + method: body ? "POST" : "GET", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + ...headers, + }, + ...(body ? { body: JSON.stringify(body) } : {}), + }); + const text = await response.text(); + let parsed; + try { + parsed = JSON.parse(text); + } catch { + parsed = { raw: text }; + } + return { status: response.status, body: parsed }; +} +function addBody(userId, requestId, content, sessionId = "s") { + return { + user_id: userId, + session_id: sessionId, + request_id: requestId, + messages: [ + { role: "user", content }, + { role: "assistant", content: "noted" }, + ], + }; +} + +try { + const health = await call("/health"); + // The fixture overrides the mode, so the code default is pinned at the source instead: a + // future edit must not quietly put production back on the synchronous path. + { + const source = fs.readFileSync( + path.join(path.dirname(fileURLToPath(import.meta.url)), "../backend/worker.mjs"), + "utf8", + ); + assert.match( + source, + /const addMode = \(env\) => \(env\.NMG_ADD_MODE === "sync" \? "sync" : "log"\)/, + "the code default must be the log path, with NMG_ADD_MODE=sync as the escape hatch", + ); + } + assert.equal(health.body.limits.add_mode, "log", "the fixture must run the log path"); + + // I1: accepted Add is searchable before any fold. + const first = await call( + "/add", + addBody("alice", "req-1", "the spare key is under the blue planter"), + ); + assert.equal(first.status, 200, JSON.stringify(first.body).slice(0, 900)); + assert.equal(first.body.status, "completed"); + const seen = await call("/search", { user_id: "alice", query: "spare key", top_k: 20 }); + assert.equal(seen.status, 200); + assert.ok( + seen.body.data.some((item) => String(item.content).includes("spare key")), + "an accepted but unfolded entry must be searchable", + ); + assert.ok( + seen.body.data.some((item) => String(item.id).startsWith("pending:")), + "the entry is served from the log, not the partition", + ); + + // I2: replay returns the same body; different content for the same id is 409. + const replay = await call( + "/add", + addBody("alice", "req-1", "the spare key is under the blue planter"), + ); + assert.deepEqual(replay.body, first.body, "a replay must return the stored response"); + const conflict = await call("/add", addBody("alice", "req-1", "different content entirely")); + assert.equal(conflict.status, 409); + assert.equal(conflict.body.error.code, "request_id_reused"); + + // Fold: enough accepts trip the trigger, and everything stays searchable. + for (let index = 0; index < 30; index += 1) + await call( + "/add", + addBody("alice", `bulk-${index}`, `filler turn number ${index} about the garden`), + ); + const afterFold = await call("/search", { + user_id: "alice", + query: "spare key under the blue planter", + top_k: 50, + }); + assert.equal(afterFold.status, 200); + const keyRecord = afterFold.body.data.find((item) => String(item.content).includes("spare key")); + assert.ok(keyRecord, "a folded entry must remain searchable"); + // The first Add is the oldest entry, so the first fold that ran during the burst + // absorbed it. Serving it from the partition rather than from the log is the only + // thing that proves folding happened at all - a run where foldPending silently + // did nothing still answered every other assertion in this file. + assert.ok( + !String(keyRecord.id).startsWith("pending:"), + `the oldest entry must have been folded, got ${keyRecord.id}`, + ); + + // I2 after folding: the receipt now lives in the partition. + const replayAfterFold = await call( + "/add", + addBody("alice", "req-1", "the spare key is under the blue planter"), + ); + assert.deepEqual(replayAfterFold.body, first.body, "replay must survive folding"); + + // The counter is observable, and draining it is an operation, not a side effect: + // without this the burst would settle above zero forever and a search would list + // the log on every request. + const waiting = await call("/capacity?user_id=alice"); + assert.ok(waiting.body.pending_entries > 0, "the burst must leave entries waiting"); + // The counter is a derivation, not a history: after the burst's folds it must equal + // the number of entries actually in the log. Subtraction would drift here on a CAS + // retry, and a count below the log hides accepted data. + { + const key = createHash("sha256").update("alice").digest("hex").slice(0, 24); + const liveBucket = await mf.getR2Bucket("MEMORY", "a"); + const listed = await liveBucket.list({ prefix: `users/${key}/log/` }); + assert.equal( + waiting.body.pending_entries, + listed.objects.length, + "the counter must equal the number of log entries", + ); + } + const drain = await call( + "/admin/fold?user_id=alice", + {}, + { + "oai-authenticated-user-email": "owner@example.test", + origin: "http://localhost", + }, + ); + assert.equal(drain.status, 200, JSON.stringify(drain.body)); + assert.ok(drain.body.folded > 0, "draining must fold something"); + const drained = await call("/capacity?user_id=alice"); + assert.equal( + drained.body.pending_entries, + 0, + `draining must zero the counter: drain=${JSON.stringify(drain.body)} before=${waiting.body.pending_entries} after=${drained.body.pending_entries}`, + ); + // A drain is a verification, not an attempt: the runbook gates a rollback on it, so + // "clean" must be distinguishable from "ran out of rounds". + assert.equal(drain.body.success, true, "a drained log must report success"); + const again = await call( + "/admin/fold?user_id=alice", + {}, + { + "oai-authenticated-user-email": "owner@example.test", + origin: "http://localhost", + }, + ); + assert.equal(again.status, 200, "draining an already clean log stays 200"); + assert.equal(again.body.remaining, 0); + assert.equal(again.body.folded, 0); + const afterDrain = await call("/search", { + user_id: "alice", + query: "spare key under the blue planter", + top_k: 50, + }); + const drainedRecord = afterDrain.body.data.find((item) => + String(item.content).includes("spare key"), + ); + assert.ok(drainedRecord, "a drained entry must still be searchable"); + assert.ok( + !String(drainedRecord.id).startsWith("pending:"), + "a drained entry must be served by the partition", + ); + + // Residue: an empty log with a non-zero counter is what a reservation whose write + // never landed leaves behind. Nothing else notices it, because the fold is only + // triggered by that counter, so the drain has to release it - and report the count + // *after* the release, or the last call of a full round budget answers 409. + const bucket = await mf.getR2Bucket("MEMORY", "a"); + const owner = { + "oai-authenticated-user-email": "owner@example.test", + origin: "http://localhost", + }; + async function seedPendingCounter(userId, pending) { + const key = createHash("sha256").update(userId).digest("hex").slice(0, 24); + await bucket.put( + `users/${key}/manifest.json`, + JSON.stringify({ format: "nmg-shards-v4", parts: [], receipts: {}, pending }), + ); + } + await seedPendingCounter("residue-one", 1); + const residueOne = await call("/admin/fold?user_id=residue-one", {}, owner); + assert.equal(residueOne.status, 200, JSON.stringify(residueOne.body)); + assert.equal(residueOne.body.remaining, 0, "a single residue slot must report clean"); + assert.equal(residueOne.body.released, 1); + + // Nine slots exceed one drain pass, so the first call must say "not clean" rather + // than claim success it cannot back up. + await seedPendingCounter("residue-nine", 9); + const partial = await call("/admin/fold?user_id=residue-nine", {}, owner); + assert.equal(partial.status, 409, JSON.stringify(partial.body)); + assert.equal(partial.body.success, false); + assert.equal(partial.body.remaining, 1, "one slot is left after a full pass"); + const finished = await call("/admin/fold?user_id=residue-nine", {}, owner); + assert.equal(finished.status, 200, JSON.stringify(finished.body)); + assert.equal(finished.body.remaining, 0); + + // Reset drains the log as well as the partitions. + const reset = await call("/reset", { + user_id: "alice", + confirm_user_id: "alice", + reset_id: `reset-${++counter}`, + }); + assert.equal(reset.status, 200, JSON.stringify(reset.body)); + const afterReset = await call("/search", { user_id: "alice", query: "spare key", top_k: 20 }); + assert.equal(afterReset.body.data.length, 0, "reset must remove pending and folded data"); + + console.log( + "PASS: an Add accepted into the log is searchable before folding, replays byte-identically, survives the fold, conflicts on different content, and reset drains it.", + ); +} finally { + await mf.dispose(); +} diff --git a/onlion/scripts/test-append-log.mjs b/onlion/scripts/test-append-log.mjs new file mode 100644 index 00000000..5d3ce986 --- /dev/null +++ b/onlion/scripts/test-append-log.mjs @@ -0,0 +1,530 @@ +/* eslint-disable no-console -- deliberate test/diagnostic output */ +// Verifies the frozen append-log seam (docs/design/add-append-log-and-fold-design.md +// "Frozen interfaces (slice 1)") against an in-memory bucket that copies the COS +// contract: forbid-overwrite is a falsy PUT, delete of a missing key is a no-op, +// and list pages by cursor. +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + LOG_ENTRY_VERSION, + MAX_PENDING_READ, + logPrefix, + logEntryName, + putLogEntry, + readLogEntry, + findLogEntry, + listPending, + deleteLogEntries, + foldPlan, + logEntryLanded, +} from '../backend/append-log.mjs'; + +const HASH_A = 'a'.repeat(64); +const HASH_B = 'b'.repeat(64); +const USER = 'userkey0123456789abcdef'; + +function memoryBucket({ pageSize = 1000 } = {}) { + const records = new Map(); + const stats = { puts: 0, deletes: 0, lists: 0, gets: 0 }; + let sequence = 0; + const bodyOf = (record) => record.body; + const view = (key, record) => ({ + key, + size: Buffer.byteLength(bodyOf(record), 'utf8'), + etag: record.etag, + async json() { + return JSON.parse(bodyOf(record)); + }, + async arrayBuffer() { + return new TextEncoder().encode(bodyOf(record)).buffer; + }, + }); + return { + records, + stats, + async get(key) { + stats.gets += 1; + const record = records.get(key); + return record ? view(key, record) : null; + }, + async head(key) { + const record = records.get(key); + return record ? { key, size: Buffer.byteLength(bodyOf(record), 'utf8'), etag: record.etag } : null; + }, + async put(key, body, options = {}) { + if (options.onlyIf?.etagDoesNotMatch === '*' && records.has(key)) return null; + stats.puts += 1; + const text = typeof body === 'string' ? body : JSON.stringify(body); + const record = { body: text, etag: (++sequence).toString(16).padStart(8, '0') }; + records.set(key, record); + return { key, size: Buffer.byteLength(text, 'utf8'), etag: record.etag }; + }, + async delete(keys) { + for (const key of Array.isArray(keys) ? keys : [keys]) { + stats.deletes += 1; + records.delete(key); + } + }, + async list({ prefix = '', limit = pageSize, cursor } = {}) { + stats.lists += 1; + const matching = [...records.keys()].filter((key) => key.startsWith(prefix)).sort(); + const start = cursor ? Number(cursor) : 0; + const page = matching.slice(start, start + Math.min(limit, pageSize)); + const next = start + page.length; + return { + objects: page.map((key) => ({ key, size: Buffer.byteLength(bodyOf(records.get(key)), 'utf8') })), + truncated: next < matching.length, + cursor: next < matching.length ? String(next) : null, + }; + }, + }; +} + +const entry = (receivedAt, requestHash, marker, messages = [{ role: 'user', content: marker }]) => ({ + v: LOG_ENTRY_VERSION, + generation: 'gen-1', + user_id: 'user-a', + session_id: 'session-1', + request_id: `request-${marker}`, + request_hash: requestHash, + fingerprint: 'f'.repeat(64), + received_at: receivedAt, + messages, +}); + +// --- frozen surface ------------------------------------------------------- +assert.equal(LOG_ENTRY_VERSION, 1); +assert.equal(MAX_PENDING_READ, 8); +const moduleSource = fs.readFileSync( + path.join(path.dirname(fileURLToPath(import.meta.url)), '../backend/append-log.mjs'), + 'utf8', +); +const imports = [...moduleSource.matchAll(/^import\s.*?from\s+['"]([^'"]+)['"];?/gm)].map( + (match) => match[1], +); +assert.deepEqual(imports, ['node:buffer'], 'append-log must not import worker.mjs, storage.mjs or SQLite'); + +// --- naming -------------------------------------------------------------- +assert.equal(logPrefix(USER), `users/${USER}/log/`); +assert.throws(() => logPrefix(''), RangeError); +assert.throws(() => logPrefix('a/b'), RangeError); +assert.throws(() => logPrefix('..'), RangeError); + +assert.equal(logEntryName(1700000000000, HASH_A), `1700000000000-${HASH_A}.json`); +assert.equal(logEntryName(42, HASH_A), `0000000000042-${HASH_A}.json`); +assert.ok(logEntryName(999, HASH_A) < logEntryName(1000, HASH_B), '13-digit padding sorts as arrival order'); +assert.equal(logEntryName(new Date(1700000000000), HASH_A), logEntryName(1700000000000, HASH_A)); +assert.equal(logEntryName('2023-11-14T22:13:20.000Z', HASH_A), logEntryName(1700000000000, HASH_A)); +assert.throws(() => logEntryName(1.5, HASH_A), RangeError); +assert.throws(() => logEntryName(-1, HASH_A), RangeError); +assert.throws(() => logEntryName(10 ** 13, HASH_A), RangeError); +assert.throws(() => logEntryName(1700000000000, 'not-a-hash'), RangeError); +assert.throws(() => logEntryName(1700000000000, `${'A'.repeat(64)}`), RangeError); + +// --- put / read / find --------------------------------------------------- +{ + const bucket = memoryBucket(); + const first = entry(1700000000000, HASH_A, 'first'); + assert.deepEqual(await putLogEntry(bucket, USER, first), { name: `1700000000000-${HASH_A}.json`, created: true }); + assert.equal(bucket.records.size, 1); + assert.deepEqual(await readLogEntry(bucket, USER, `1700000000000-${HASH_A}.json`), first); + assert.deepEqual(await findLogEntry(bucket, USER, HASH_A), { + name: `1700000000000-${HASH_A}.json`, + entry: first, + }); + assert.equal(await findLogEntry(bucket, USER, HASH_B), null); + assert.equal(await readLogEntry(bucket, USER, `1800000000000-${HASH_A}.json`), null); + + // The name is the idempotency key: the second writer cannot replace the body. + const replay = await putLogEntry(bucket, USER, first); + assert.deepEqual(replay, { name: `1700000000000-${HASH_A}.json`, created: false }); + const impostor = await putLogEntry(bucket, USER, { ...first, messages: [{ role: 'user', content: 'lost' }] }); + assert.equal(impostor.created, false); + assert.deepEqual(await readLogEntry(bucket, USER, impostor.name), first); + assert.equal(bucket.stats.puts, 1, 'a replay must not cost a write'); + assert.equal(bucket.stats.deletes, 0); + + // The oldest entry wins when a retry recomputed received_at for the same request. + const retried = entry(1700000009999, HASH_A, 'first'); + assert.deepEqual(await putLogEntry(bucket, USER, retried), { name: `1700000009999-${HASH_A}.json`, created: true }); + assert.equal((await findLogEntry(bucket, USER, HASH_A)).name, `1700000000000-${HASH_A}.json`); + + // After a reset the old echo must not be replayed to a client whose data was + // wiped: the generation filter skips the older entry and finds the current one. + const generations = memoryBucket(); + await putLogEntry(generations, USER, { ...entry(1700000000000, HASH_A, 'old'), generation: 'g1' }); + await putLogEntry(generations, USER, { ...entry(1700000600000, HASH_A, 'new'), generation: 'g2' }); + assert.equal((await findLogEntry(generations, USER, HASH_A)).entry.messages[0].content, 'old'); + assert.equal( + (await findLogEntry(generations, USER, HASH_A, { generation: 'g2' })).entry.messages[0].content, + 'new', + ); + assert.equal(await findLogEntry(generations, USER, HASH_A, { generation: 'gone' }), null); + + // A match that vanishes between the list and the read is skipped, not reported as + // a miss: the replay lookup must keep looking at the next candidate. + const vanishing = memoryBucket(); + await putLogEntry(vanishing, USER, entry(1700000000000, HASH_A, 'first')); + await putLogEntry(vanishing, USER, entry(1700000000001, HASH_A, 'second')); + const vanishingKey = `${logPrefix(USER)}1700000000000-${HASH_A}.json`; + const racingLookup = { + ...vanishing, + async get(key) { + if (key === vanishingKey) { + vanishing.records.delete(key); + return null; + } + return vanishing.get(key); + }, + }; + assert.equal( + (await findLogEntry(racingLookup, USER, HASH_A)).entry.messages[0].content, + 'second', + ); + + // The replay lookup validates the whole name too: a key that only looks like an entry + // (`foo-.json`, no 13-digit prefix) is reported as a naming problem, not later as + // a JSON parse error. + const strayLookup = memoryBucket(); + await putLogEntry(strayLookup, USER, entry(1700000000000, HASH_B, 'kept')); + await strayLookup.put(`${logPrefix(USER)}foo-${HASH_A}.json`, '{}'); + await assert.rejects( + () => findLogEntry(strayLookup, USER, HASH_A), + (error) => error.code === 'log_entry_name', + 'a lookalike name must be reported as corruption, not as a parse error', + ); + + await assert.rejects(putLogEntry(bucket, USER, { ...first, v: 2 }), RangeError); + const wrongVersion = memoryBucket(); + await wrongVersion.put(`${logPrefix(USER)}1700000000000-${HASH_A}.json`, JSON.stringify({ ...first, v: 99 })); + await assert.rejects(readLogEntry(wrongVersion, USER, `1700000000000-${HASH_A}.json`), /Unsupported log entry version/); + const noHash = memoryBucket(); + await noHash.put(`${logPrefix(USER)}1700000000000-${HASH_A}.json`, JSON.stringify({ ...first, request_hash: null })); + await assert.rejects(readLogEntry(noHash, USER, `1700000000000-${HASH_A}.json`), /request_hash/); +} + +// --- listPending --------------------------------------------------------- +{ + const bucket = memoryBucket(); + const hashes = ['1', '2', '3', '4', '5'].map((digit) => digit.repeat(64)); + // Written out of arrival order: the listing must still be name-ascending. + for (const index of [2, 0, 4, 1, 3]) + await putLogEntry(bucket, USER, entry(1700000000000 + index, hashes[index], `m${index}`)); + await bucket.put('users/other/log/1700000000000-' + HASH_B + '.json', 'not json', {}); + + const all = await listPending(bucket, USER, { limit: Infinity }); + assert.deepEqual( + all.map((item) => item.entry.messages[0].content), + ['m0', 'm1', 'm2', 'm3', 'm4'], + ); + assert.deepEqual( + all.map((item) => item.name), + [...all.map((item) => item.name)].sort(), + ); + assert.equal((await listPending(bucket, USER, { limit: 2 })).length, 2); + assert.equal((await listPending(bucket, USER)).length, 5, 'default limit still returns everything under the cap'); + assert.equal((await listPending(bucket, USER, { limit: 1 }))[0].entry.messages[0].content, 'm0'); + await assert.rejects(listPending(bucket, USER, { limit: 0 }), RangeError); + + // Paging by cursor: five entries read across three list pages. + const paged = memoryBucket({ pageSize: 2 }); + for (const index of [0, 1, 2, 3, 4]) + await putLogEntry(paged, USER, entry(1700000000000 + index, hashes[index], `m${index}`)); + assert.deepEqual( + (await listPending(paged, USER, { limit: Infinity })).map((item) => item.entry.messages[0].content), + ['m0', 'm1', 'm2', 'm3', 'm4'], + ); + assert.ok(paged.stats.lists >= 3, 'the cursor must be followed to the last page'); + + // The read path must not pay for the whole log: with one key per page, a + // two-entry read stops after two pages, and the fold path still sees all five. + const slow = memoryBucket({ pageSize: 1 }); + for (const index of [0, 1, 2, 3, 4]) + await putLogEntry(slow, USER, entry(1700000000000 + index, hashes[index], `m${index}`)); + slow.stats.lists = 0; + assert.equal((await listPending(slow, USER, { limit: 2 })).length, 2); + assert.equal(slow.stats.lists, 2, 'a bounded read must stop listing once it has enough'); + slow.stats.lists = 0; + assert.equal((await listPending(slow, USER, { limit: Infinity })).length, 5); + assert.equal(slow.stats.lists, 5, 'the fold path follows the cursor to the last page'); + + // A generation filter must not be starved by older entries: the current ones sit + // behind the foreign ones, and a key-truncated read would never reach them. + const mixed = memoryBucket(); + const serial = (index) => String(index).padStart(64, '0'); + for (const index of [0, 1, 2]) + await putLogEntry(mixed, USER, { + ...entry(1700000000000 + index, serial(index), `old${index}`), + generation: 'g1', + }); + for (const index of [3, 4]) + await putLogEntry(mixed, USER, { + ...entry(1700000000000 + index, serial(index), `new${index}`), + generation: 'g2', + }); + assert.deepEqual( + (await listPending(mixed, USER, { limit: 2, generation: 'g2' })).map( + (item) => item.entry.messages[0].content, + ), + ['new3', 'new4'], + ); + assert.equal((await listPending(mixed, USER, { limit: 2, generation: 'g1' })).length, 2); + assert.deepEqual(await listPending(mixed, USER, { limit: 2, generation: 'gone' }), []); + + // The scan behind a filter is bounded: a log stuffed with foreign entries returns + // fewer matches instead of reading the whole prefix (I6), so a search cannot be + // made to pay for a log it will never use. + const buried = memoryBucket(); + for (let index = 0; index < 40; index += 1) + await putLogEntry(buried, USER, { + ...entry(1700000000000 + index, serial(index), `old${index}`), + generation: 'g1', + }); + await putLogEntry(buried, USER, { + ...entry(1700000000500, HASH_A, 'current'), + generation: 'g2', + }); + buried.stats.gets = 0; + const buriedRead = await listPending(buried, USER, { limit: 2, generation: 'g2' }); + assert.deepEqual(buriedRead, [], 'a bounded scan may return fewer matches than the limit'); + assert.ok( + buried.stats.gets <= 8, + `a filtered read must not read the whole prefix (read ${buried.stats.gets} entries)`, + ); + // The documented bound, in code rather than prose: search asks for at most + // MAX_PENDING_READ, so a filter scans at most max(MAX_PENDING_READ, 4 * limit) keys. + // Beyond that the current generation stays out of the window until folds retire the + // leftovers - which is why the counter has to include them (reset hand-off). + buried.stats.gets = 0; + assert.deepEqual(await listPending(buried, USER, { limit: 8, generation: 'g2' }), []); + assert.equal(buried.stats.gets, 32, 'a filtered read scans at most 32 keys for limit 8'); + + // An entry the caller is finished with (its receipt is already written, the object + // is not deleted yet) must be skipped before its body is read: that saves one GET + // here and one embedding call per ghost on the search path. + const ghosts = memoryBucket(); + const receipts = new Set([0, 1, 2, 3].map((index) => serial(index))); + for (let index = 0; index < 4; index += 1) + await putLogEntry(ghosts, USER, { + ...entry(1700000000000 + index, serial(index), `ghost${index}`), + generation: 'g', + }); + for (const index of [4, 5]) + await putLogEntry(ghosts, USER, { + ...entry(1700000000000 + index, serial(index), `live${index}`), + generation: 'g', + }); + ghosts.stats.gets = 0; + const live = await listPending(ghosts, USER, { + limit: 2, + skipRequestHash: (hash) => receipts.has(hash), + }); + assert.deepEqual( + live.map((item) => item.entry.messages[0].content), + ['live4', 'live5'], + ); + assert.equal(ghosts.stats.gets, 2, 'a skipped entry must not cost a body read'); + // Without the filter the ghosts still show up: the fold batch needs to see them to + // delete them, and the reconcile count needs the log's own truth. + assert.equal((await listPending(ghosts, USER, { limit: Infinity })).length, 6); + await assert.rejects(() => listPending(ghosts, USER, { skipRequestHash: 'nope' }), TypeError); + // A name in the log prefix that is not a log entry is corruption, not a reason to + // hide the entry: with a predicate the hash comes from the name, so the failure has + // to be loud instead of turning into a silent skip. + const stray = memoryBucket(); + await putLogEntry(stray, USER, entry(1700000000000, serial(0), 'kept')); + await stray.put(`${logPrefix(USER)}not-a-log-entry.json`, '{}'); + await assert.rejects( + () => listPending(stray, USER, { limit: 2, skipRequestHash: () => false }), + (error) => error.code === 'log_entry_name', + 'an unparseable name must fail loudly when a predicate needs the hash', + ); + // The fold batch and the reconcile read pass no predicate; they must report the same + // corpus problem with the same code instead of a raw JSON SyntaxError. + await assert.rejects( + () => listPending(stray, USER, { limit: Infinity }), + (error) => error.code === 'log_entry_name', + 'an unparseable name must fail loudly for callers without a predicate too', + ); + // A name that only looks like an entry (`foo-.json`) must not slip through the + // shape check and turn into a JSON parse error further down. + const lookalike = memoryBucket(); + await putLogEntry(lookalike, USER, entry(1700000000000, serial(0), 'kept')); + await lookalike.put(`${logPrefix(USER)}foo-${HASH_A}.json`, '{}'); + await assert.rejects( + () => listPending(lookalike, USER, { limit: 2, skipRequestHash: () => false }), + (error) => error.code === 'log_entry_name', + 'a lookalike name must be reported as corruption', + ); + console.log( + 'PASS: a receipt the caller already wrote skips the entry before its body is read, and is still visible to the fold path and to the reconcile count.', + ); + + // A fold or reset can delete an entry between the list and the read. + const racing = { + ...bucket, + async get() { + return null; + }, + }; + assert.deepEqual(await listPending(racing, USER, { limit: Infinity }), []); + assert.deepEqual(await listPending(bucket, 'nobody', { limit: Infinity }), []); +} + +// --- deleteLogEntries ---------------------------------------------------- +{ + const bucket = memoryBucket(); + const names = [ + `1700000000000-${HASH_A}.json`, + `1700000000001-${HASH_A}.json`, + `1700000000002-${HASH_A}.json`, + ]; + for (let index = 0; index < names.length; index += 1) + await putLogEntry(bucket, USER, entry(1700000000000 + index, HASH_A, `m${index}`)); + assert.equal(bucket.records.size, 3); + + await deleteLogEntries(bucket, USER, [names[0], `${logPrefix(USER)}${names[1]}`, names[0], '', null]); + assert.deepEqual([...bucket.records.keys()], [`${logPrefix(USER)}${names[2]}`]); + await deleteLogEntries(bucket, USER, [names[0], names[1]]); // idempotent: already gone + await deleteLogEntries(bucket, USER, []); + assert.deepEqual([...bucket.records.keys()], [`${logPrefix(USER)}${names[2]}`]); +} + +// --- foldPlan ------------------------------------------------------------ +{ + const small = entry(1700000000000, HASH_A, 'x', [{ role: 'user', content: 'y'.repeat(100) }]); + const big = entry(1700000000001, HASH_A, 'y', [{ role: 'user', content: 'z'.repeat(100) }]); + const items = [small, big, entry(1700000000002, HASH_A, 'w')]; + + assert.deepEqual(foldPlan([]), { fold: [], keep: [] }); + assert.deepEqual(foldPlan(undefined), { fold: [], keep: [] }); + const byCount = foldPlan(items, { maxEntries: 2 }); + assert.deepEqual(byCount.fold, [small, big]); + assert.deepEqual(byCount.keep, [items[2]]); + assert.deepEqual(foldPlan(items, { maxEntries: 0 }), { fold: [], keep: items }); + assert.deepEqual(foldPlan(items, { maxEntries: Infinity }).fold.length, 3); + + const size = Buffer.byteLength(JSON.stringify(small), 'utf8'); + assert.deepEqual(foldPlan(items, { maxBytes: size }).fold, [small]); + assert.deepEqual(foldPlan(items, { maxBytes: size * 2 }).fold, [small, big]); + assert.deepEqual(foldPlan(items, { maxEntries: 3, maxBytes: size * 2 }).fold, [small, big]); + + // One oversized entry still folds: progress beats the byte budget, otherwise it + // would be retried by every later fold and never absorbed. + assert.deepEqual(foldPlan(items, { maxBytes: 1 }).fold, [small]); + assert.deepEqual(foldPlan(items, { maxBytes: 0 }).fold, [small]); + + // {name, entry} items are measured by their entry and preserved as passed. + const named = items.map((item, index) => ({ name: `n${index}`, entry: item })); + assert.deepEqual(foldPlan(named, { maxEntries: 1 }).fold, [named[0]]); + assert.deepEqual(foldPlan(named, { maxBytes: size }).keep, named.slice(1)); + + assert.throws(() => foldPlan(items, { maxEntries: -1 }), RangeError); + assert.throws(() => foldPlan(items, { maxBytes: 1.5 }), RangeError); + assert.throws(() => foldPlan(items, { maxEntries: '2' }), RangeError); +} + +// --- accept / fold / read lifecycle -------------------------------------- +{ + const bucket = memoryBucket(); + const arrivals = [0, 1, 2, 3, 4, 5].map((index) => + entry(1700000000000 + index, String(index).repeat(64), `m${index}`), + ); + for (const item of arrivals) await putLogEntry(bucket, USER, item); + + const pending = await listPending(bucket, USER, { limit: Infinity }); + assert.equal(pending.length, 6); + const { fold, keep } = foldPlan(pending, { maxEntries: 4 }); + assert.deepEqual( + fold.map((item) => item.entry.messages[0].content), + ['m0', 'm1', 'm2', 'm3'], + ); + await deleteLogEntries(bucket, USER, fold.map((item) => item.name)); + assert.deepEqual( + (await listPending(bucket, USER, { limit: Infinity })).map((item) => item.name), + keep.map((item) => item.name), + ); + + // A crash after the delete retries cleanly: nothing is duplicated or lost. + await deleteLogEntries(bucket, USER, fold.map((item) => item.name)); + assert.equal((await listPending(bucket, USER, { limit: Infinity })).length, 2); + assert.equal(await findLogEntry(bucket, USER, String(0).repeat(64)), null); + assert.ok(await findLogEntry(bucket, USER, String(5).repeat(64))); +} + +console.log( + 'PASS: append-log names are padded, sortable idempotency keys; forbid-overwrite replays without a write; pending listing is cursor-paged and arrival-ordered; deletes are idempotent; foldPlan is count- and byte-bounded and always makes progress.', +); + +// The landed check has three answers and the caller must not collapse them: "no" +// releases a reserved slot, "unknown" must leave the counter alone. +{ + const entry = { v: 1, received_at: 1700000000000, request_hash: HASH_A }; + const bucketWith = (behaviour) => ({ + head: + behaviour === 'object' + ? async () => ({ size: 10, etag: 'x' }) + : behaviour === 'null' + ? async () => null + : async () => { + throw new Error('head unavailable'); + }, + }); + assert.equal(await logEntryLanded(bucketWith('object'), 'k'.repeat(24), entry), true); + assert.equal(await logEntryLanded(bucketWith('null'), 'k'.repeat(24), entry), false); + assert.equal( + await logEntryLanded(bucketWith('throw'), 'k'.repeat(24), entry), + null, + 'an unreadable answer is unknown, never false', + ); + console.log( + 'PASS: the landed check answers yes / no / unknown, and unknown is never collapsed to no.', + ); +} + +// A generation filter must be applied while reading, not after truncating keys: with +// stale entries first, a post-truncation filter returns nothing and the read path +// silently stops seeing the entries accepted after a reset. +{ + const bucket = memoryBucket(); + const stale = []; + for (let i = 0; i < 3; i += 1) { + const entry = { + v: 1, + generation: 'old', + user_id: 'u', + session_id: 's', + request_id: `stale-${i}`, + request_hash: createHash('sha256').update(`stale-${i}`).digest('hex'), + fingerprint: 'f', + received_at: 1700000000000 + i, + messages: [], + }; + await putLogEntry(bucket, 'user-key', entry); + stale.push(entry); + } + for (let i = 0; i < 2; i += 1) { + await putLogEntry(bucket, 'user-key', { + v: 1, + generation: 'new', + user_id: 'u', + session_id: 's', + request_id: `fresh-${i}`, + request_hash: createHash('sha256').update(`fresh-${i}`).digest('hex'), + fingerprint: 'f', + received_at: 1700000001000 + i, + messages: [], + }); + } + const fresh = await listPending(bucket, 'user-key', { limit: 2, generation: 'new' }); + assert.equal(fresh.length, 2, 'the window must be filled with current-generation entries'); + assert.ok(fresh.every(({ entry }) => entry.generation === 'new')); + const staleOnly = await listPending(bucket, 'user-key', { limit: 8, generation: 'old' }); + assert.equal(staleOnly.length, 3); + console.log( + 'PASS: a generation filter fills the window from current entries even when every older key sorts first.', + ); +} diff --git a/onlion/scripts/test-backend.mjs b/onlion/scripts/test-backend.mjs index c3e84c2c..18d6be94 100644 --- a/onlion/scripts/test-backend.mjs +++ b/onlion/scripts/test-backend.mjs @@ -6,7 +6,13 @@ import { createWorkerFixture } from "./worker-fixture.mjs"; import { createHash } from "node:crypto"; import { spawnSync } from "node:child_process"; import { applyPageDelta, makePageDelta } from "../backend/page-delta.mjs"; -import { MAX_QUERY_CHUNKS, QUERY_CHUNK_CHARS, poolQueryVectors, splitQueryChunks } from "../backend/query-chunks.mjs"; +import { + MAX_QUERY_FEATURES, + MAX_QUERY_WINDOWS, + QUERY_CHUNK_CHARS, + planQueryWindows, + selectQueryFeatures, +} from "../backend/query-chunks.mjs"; const root = process.cwd(); const storage = path.join(root, ".runtime/tests/backend-r2"); const python = process.env.NMG_TEST_PYTHON ?? "python"; @@ -189,6 +195,14 @@ try { assert.equal(largeResult.body.added, 70); for (let i = 0; i < 32; i++) assert.deepEqual(await call(i % 2 ? "a" : "b", "/add", large), largeResult); + // Fold the accepted entries into a partition before inspecting the snapshot: with the + // log accept path an Add is durable but still pending, so `parts` stays empty until a + // fold runs. This check is about the snapshot's contents, not about when it appears. + const foldBatch = await call("a", "/admin/fold?user_id=batch70", {}, undefined, { + "oai-authenticated-user-email": "owner@example.test", + origin: "http://localhost", + }); + assert.equal(foldBatch.status, 200, JSON.stringify(foldBatch.body)); const batchBucket = await mf.getR2Bucket("MEMORY", "a"); const batchManifest = JSON.parse( await ( @@ -878,8 +892,11 @@ print(json.dumps(r))`, max_combined_message_chars: 1_500_000, max_ordered_content_bytes: 4 * 1024 * 1024, query_chunk_chars: QUERY_CHUNK_CHARS, - max_query_chunks: MAX_QUERY_CHUNKS, + max_query_chunks: MAX_QUERY_WINDOWS, max_query_features: 24, + add_mode: "sync", + pending_read_limit: 8, + merge_batch: 8, }); const longQuery = await call("a", "/search", { user_id: "payload-limits", @@ -900,50 +917,84 @@ print(json.dumps(r))`, 413, "a message longer than the old 128000-character cap must not be rejected as too large", ); - // A long query must still produce one usable vector: the text is split into - // bounded windows and the chunk vectors are pooled, because an embedding model - // only sees its own token window. - assert.deepEqual(splitQueryChunks("short question"), ["short question"]); - const longText = ("memory question about the trip. ".repeat(400)).trim(); - const windows = splitQueryChunks(longText); + // A long query is searched by bounded windows plus the clause itself: an + // embedding model only sees its own token window. + assert.deepEqual(planQueryWindows("short question"), ["short question"]); + const longText = "memory question about the trip. ".repeat(400).trim(); + const windows = planQueryWindows(longText); assert.ok(windows.length > 1, "a long query must split"); assert.ok( - windows.length <= MAX_QUERY_CHUNKS, - `chunk count must stay bounded, got ${windows.length}`, + windows.length <= MAX_QUERY_WINDOWS, + `window count must stay bounded, got ${windows.length}`, ); for (const window of windows) assert.ok(window.length <= QUERY_CHUNK_CHARS, `chunk exceeds the window: ${window.length}`); assert.ok(windows.join("").length > 0); - const unit = [1, 0, 0]; - const pooled = poolQueryVectors([unit, unit]); - assert.deepEqual( - pooled.map((value) => Math.round(value * 1e6) / 1e6), - unit, - "pooling identical unit vectors returns the same unit vector", - ); - const mixed = poolQueryVectors([ - [1, 0], - [0, 1], - ]); - const length = Math.hypot(...mixed); - assert.ok(Math.abs(length - 1) < 1e-9, `pooled vector must stay unit length, got ${length}`); - assert.ok(mixed[0] > 0 && mixed[1] > 0, "pooling averages the chunk directions"); // The windows must always cover the tail: without it, anything past // limit*size characters was never searched at all. const NL = String.fromCharCode(10); const huge = "filler sentence about weather. ".repeat(800).trim() + NL + "WHERE IS THE SPARE KEY"; - const hugeWindows = splitQueryChunks(huge); + const hugeWindows = planQueryWindows(huge); assert.ok( hugeWindows.at(-1).includes("SPARE KEY"), "the end-anchored window must contain the tail of a long query", ); - assert.ok(huge.length > QUERY_CHUNK_CHARS * MAX_QUERY_CHUNKS, "precondition: longer than the window budget"); + assert.ok( + huge.length > QUERY_CHUNK_CHARS * MAX_QUERY_WINDOWS, + "precondition: longer than the window budget", + ); + // The feature list is the clause itself (the model's own token window sees the + // opening with more context than a character window) followed by bounded + // windows, opening first and the end-anchored tail second. + const features = selectQueryFeatures([huge]); + assert.equal(features[0][0], huge, "the first feature is the clause itself"); + for (const feature of features[0].slice(1)) + assert.ok( + feature.length <= QUERY_CHUNK_CHARS, + `feature exceeds the model window: ${feature.length}`, + ); + assert.ok( + features[0].length <= MAX_QUERY_FEATURES, + `the feature budget must hold, got ${features[0].length}`, + ); + assert.ok( + features[0].length <= MAX_QUERY_FEATURES, + `one clause cannot exceed the request budget, got ${features[0].length}`, + ); + assert.ok( + features[0][0].startsWith("filler"), + "the clause is its own feature, so its opening is always searched", + ); + assert.ok( + !features[0][1].startsWith(features[0][0].slice(0, 40)), + "the opening window is dropped: the clause itself already covers it", + ); + assert.ok( + features[0][1].includes("SPARE KEY"), + "the end-anchored tail comes first, so budget pressure costs the middle", + ); + // A clause that does not split contributes exactly one feature, which is the + // previous behaviour, and short clauses are never windowed. + assert.deepEqual(selectQueryFeatures(["short question"]), [["short question"]]); + // The budget is shared round-robin: one very long clause cannot starve another. + const twoClauses = selectQueryFeatures([huge, huge, "short question"], 3); + assert.equal(twoClauses.length, 3); + assert.ok( + twoClauses.every((list) => list.length >= 1), + "every clause keeps its head when the budget is tight", + ); + assert.ok( + twoClauses[0].length >= twoClauses[2].length, + "extra budget goes to the clause that can still spend it", + ); console.log( - "PASS: a long query splits into bounded windows and always covers its tail, and the pool helper keeps a unit vector for callers that need one.", + "PASS: a long query plans bounded windows that always cover its tail, and mean pooling is no longer part of the service.", + ); + console.log( + "PASS: the clause is its own feature, the opening window is dropped as redundant, the end-anchored tail is spent first, and a short clause contributes exactly one feature.", ); } - } finally { await mf.dispose(); } diff --git a/onlion/scripts/test-capacity.mjs b/onlion/scripts/test-capacity.mjs index 9113510a..27c191d8 100644 --- a/onlion/scripts/test-capacity.mjs +++ b/onlion/scripts/test-capacity.mjs @@ -6,7 +6,9 @@ import {createRequire} from 'node:module'; import {createHash} from 'node:crypto'; import {spawnSync} from 'node:child_process'; const require=createRequire(import.meta.url); -const {Miniflare}=createRequire(require.resolve('wrangler/package.json'))('miniflare'); +// Resolved from this package: wrangler's own range tracks a newer Miniflare whose +// constructor rejects this fixture's options, and a transitive dependency is not a pin. +const {Miniflare}=require('miniflare'); const root=process.cwd(),storage=path.join(root,'.sites-runtime/capacity-r2'); fs.rmSync(storage,{recursive:true,force:true}); const modules=[{type:'ESModule',path:root+'/dist/server/index.js'}, diff --git a/onlion/scripts/test-manifest-pending.mjs b/onlion/scripts/test-manifest-pending.mjs new file mode 100644 index 00000000..526cad16 --- /dev/null +++ b/onlion/scripts/test-manifest-pending.mjs @@ -0,0 +1,369 @@ +/* eslint-disable no-console -- deliberate test/diagnostic output */ +/** + * Slice 1 of docs/design/add-append-log-and-fold-design.md, storage side: + * the manifest's `pending` count, the lazy `nmg-shards-v4` bump, and log cleanup. + * + * The invariant worth protecting is negative: a user whose log is empty must + * store exactly the bytes today's writer stores, so a deployment with the log + * disabled (or drained) is indistinguishable on disk from the current one. + * A second invariant is that the log prefix has exactly one definition. storage.mjs + * takes it from append-log.mjs's logPrefix() (see the design's clearLog note), so + * this test spells the frozen name out literally and requires both the entries it + * seeds and the prefix that function returns to agree with it. + * + * No build, no Miniflare, no COS credentials: a bucket with the same five + * operations (get/head/put/delete/list) plus the conditional-write contract is + * enough. Usage: node scripts/test-manifest-pending.mjs + */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { logPrefix } from "../backend/append-log.mjs"; +import { + STORAGE_FORMAT_V4, + clearLog, + commitManifest, + keyFor, + loadManifest, + manifestFormat, + pendingCount, + resetUser, + setPending, + withPending, +} from "../backend/storage.mjs"; + +/** + * In-memory bucket. `onlyIf.etagMatches` compares tags, `onlyIf.etagDoesNotMatch:'*'` + * is first-create-only, and `list` pages with a stable key cursor so deleting + * while paging behaves like COS/R2 (an index cursor would silently skip keys). + */ +function createMemoryBucket() { + const objects = new Map(); // key -> { body, etag } + const stats = { puts: 0, deletes: 0, lists: 0 }; + const etagOf = (body) => createHash("md5").update(body).digest("hex"); + return { + objects, + stats, + async get(key) { + const object = objects.get(key); + if (!object) return null; + const buffer = Buffer.from(object.body, "utf8"); + return { + key, + size: buffer.length, + etag: object.etag, + arrayBuffer: async () => + buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength), + json: async () => JSON.parse(object.body), + }; + }, + async head(key) { + const object = objects.get(key); + return object + ? { key, size: Buffer.byteLength(object.body), etag: object.etag } + : null; + }, + async put(key, body, options = {}) { + const text = typeof body === "string" ? body : Buffer.from(body).toString("utf8"); + const onlyIf = options.onlyIf; + if (onlyIf?.etagMatches && objects.get(key)?.etag !== onlyIf.etagMatches) return null; + if (onlyIf?.etagDoesNotMatch === "*" && objects.has(key)) return null; + const etag = etagOf(text); + objects.set(key, { body: text, etag }); + stats.puts++; + return { key, etag, size: Buffer.byteLength(text) }; + }, + async delete(keys) { + for (const key of Array.isArray(keys) ? keys : [keys]) if (objects.delete(key)) stats.deletes++; + }, + async list({ prefix = "", limit = 1000, cursor } = {}) { + stats.lists++; + const keys = [...objects.keys()].filter((key) => key.startsWith(prefix)).sort(); + const rest = cursor ? keys.filter((key) => key > cursor) : keys; + const page = rest.slice(0, limit); + const truncated = rest.length > page.length; + return { + objects: page.map((key) => ({ key, size: Buffer.byteLength(objects.get(key).body) })), + truncated, + cursor: truncated ? page.at(-1) : null, + }; + }, + }; +} + +const bodyOf = (bucket, key) => bucket.objects.get(key)?.body; +const keysUnder = (bucket, prefix) => [...bucket.objects.keys()].filter((key) => key.startsWith(prefix)); +const entryName = (sequence, hashChar) => `${String(sequence).padStart(13, "0")}-${hashChar.repeat(64)}.json`; +// The frozen name from the design, spelled out rather than derived: clearLog() +// must delete what the accept path wrote even if append-log.mjs renames its +// prefix, so the test seeds with the literal and checks the function against it. +const frozenPrefix = (userKey) => `users/${userKey}/log/`; +const flat = () => ({ format: "nmg-shards-v2", parts: [], receipts: {} }); + +assert.equal(logPrefix(keyFor("prefix-contract")), frozenPrefix(keyFor("prefix-contract"))); +assert.equal(logPrefix("abc123"), "users/abc123/log/"); + +// 1. Format selection: pending decides v4, otherwise the existing v2/v3 rule. +{ + assert.equal(manifestFormat(flat()), "nmg-shards-v2"); + assert.equal( + manifestFormat({ ...flat(), parts: [{ scope: "s", key: "k", bytes: 1, delta: { pages: {} } }] }), + "nmg-shards-v3", + ); + assert.equal(manifestFormat(withPending(flat(), 2)), STORAGE_FORMAT_V4); + assert.equal(manifestFormat({ ...flat(), pending: 0 }), "nmg-shards-v2"); + // Pending dominates: a reader that cannot see the log must refuse the manifest. + assert.equal( + manifestFormat(withPending({ format: "nmg-shards-v3", parts: [{ delta: {} }], receipts: {} }, 1)), + STORAGE_FORMAT_V4, + ); + console.log("PASS: manifestFormat is v2/v3 without pending and v4 as soon as one entry is pending."); +} + +// 2. pendingCount / withPending: pure, non-mutating, and zero means "no key". +{ + assert.equal(pendingCount(undefined), 0); + assert.equal(pendingCount(flat()), 0); + assert.equal(pendingCount({ ...flat(), pending: 0 }), 0); + assert.equal(pendingCount({ ...flat(), pending: 7 }), 7); + // loadManifest() rejects these as corruption; the counter itself stays total. + assert.equal(pendingCount({ ...flat(), pending: -1 }), 0); + assert.equal(pendingCount({ ...flat(), pending: "3" }), 0); + + const base = flat(); + const counted = withPending(base, 4); + assert.equal(counted.pending, 4); + assert.equal(base.pending, undefined, "withPending must not mutate its input"); + assert.equal(JSON.stringify(withPending(counted, 0)), JSON.stringify(base)); + assert.equal(JSON.stringify(withPending(withPending(base, 3), 0)), JSON.stringify(base)); + + for (const bad of [-1, 1.5, Number.NaN, "2", undefined]) + assert.throws(() => withPending(base, bad), TypeError, `withPending(${String(bad)})`); + console.log("PASS: withPending copies, drops the key at zero, and rejects non-counts."); +} + +// 2b. setPending is the in-place form the worker actually calls, and the reason it +// exists: Object.assign(manifest, withPending(manifest, 0)) keeps the old key, so a +// drain would never delete it and the manifest would stay v4 forever. +{ + const v4 = () => ({ format: STORAGE_FORMAT_V4, parts: [], receipts: {}, pending: 7 }); + const manifest = v4(); + assert.equal(setPending(manifest, 3), 3); + assert.equal(manifest.pending, 3, "setPending must work in place"); + assert.equal(setPending(manifest, 0), 0); + assert.ok(!("pending" in manifest), "zero must delete the key, not store 0"); + // Byte equality with the same manifest that never carried a pending key; the + // format string is commitManifest's job, not setPending's. + assert.equal( + JSON.stringify(manifest), + JSON.stringify({ format: STORAGE_FORMAT_V4, parts: [], receipts: {} }), + ); + + // The bug that shipped once: assigning a key-less copy cannot delete a key, so a + // drain left `pending: 7` behind and the manifest stayed v4 forever. + const trapped = v4(); + Object.assign(trapped, withPending(trapped, 0)); + assert.equal(trapped.pending, 7, "documents why callers must use setPending, not Object.assign"); + setPending(trapped, 0); + assert.ok(!("pending" in trapped)); + + // Both forms must agree on the stored bytes: if only one of them is ever changed, + // the drift comes back through the other one. + const copied = withPending(v4(), 0); + const inPlace = v4(); + setPending(inPlace, 0); + assert.equal(JSON.stringify(copied), JSON.stringify(inPlace)); + const two = v4(); + assert.equal(setPending(two, 2), 2, "setPending returns the stored count, not the manifest"); + assert.equal(JSON.stringify(two), JSON.stringify(withPending(v4(), 2))); + + for (const bad of [-1, 1.5, Number.NaN, "2", undefined]) + assert.throws(() => setPending(v4(), bad), TypeError, `setPending(${String(bad)})`); + console.log("PASS: setPending writes in place, deletes the key at zero, and refuses non-counts."); +} + +// 3. commitManifest: lazy bytes, v4 while pending, CAS still enforces one writer. +{ + const bucket = createMemoryBucket(); + const userKey = keyFor("pending-user"); + const manifestKey = `users/${userKey}/manifest.json`; + const manifest = flat(); + + assert.ok(await commitManifest(bucket, userKey, null, manifest)); + const withoutLog = bodyOf(bucket, manifestKey); + assert.equal(withoutLog, JSON.stringify(flat()), "no pending entry must leave today's bytes"); + + const current = await bucket.head(manifestKey); + assert.ok(await commitManifest(bucket, userKey, current, withPending(manifest, 3))); + const withLog = JSON.parse(bodyOf(bucket, manifestKey)); + assert.equal(withLog.format, STORAGE_FORMAT_V4); + assert.equal(withLog.pending, 3); + + // A stale etag is still refused: the pending count rides the same CAS. + assert.equal(await commitManifest(bucket, userKey, current, withPending(manifest, 1)), null); + assert.equal(JSON.parse(bodyOf(bucket, manifestKey)).pending, 3); + + // A caller that passes `pending: 0` explicitly must not change the bytes. + const stale = await bucket.head(manifestKey); + assert.ok(await commitManifest(bucket, userKey, stale, { ...withLog, pending: 0 })); + assert.equal(JSON.parse(bodyOf(bucket, manifestKey)).format, "nmg-shards-v2"); + assert.ok(!bodyOf(bucket, manifestKey).includes('"pending"')); + + // Folding the log drains it: the manifest returns to the exact v2 bytes. + const drained = await bucket.head(manifestKey); + assert.ok(await commitManifest(bucket, userKey, drained, withPending(JSON.parse(bodyOf(bucket, manifestKey)), 0))); + assert.equal(bodyOf(bucket, manifestKey), withoutLog, "a drained log restores the v2 bytes"); + console.log("PASS: v4 only while pending is non-zero; a drained log round-trips to the v2 bytes."); +} + +// 4. loadManifest accepts v4 and rejects anything a newer deployment wrote. +{ + const bucket = createMemoryBucket(); + const userKey = keyFor("load-user"); + const manifestKey = `users/${userKey}/manifest.json`; + + // Written by the real writer and read back: the pair cannot disagree. + assert.ok(await commitManifest(bucket, userKey, null, withPending({ ...flat(), generation: "g" }, 2))); + const loaded = await loadManifest(bucket, userKey, "load-user"); + assert.equal(loaded.manifest.format, STORAGE_FORMAT_V4); + assert.equal(pendingCount(loaded.manifest), 2); + assert.equal(loaded.manifest.generation, "g"); + + await bucket.put(manifestKey, JSON.stringify(flat())); + assert.equal((await loadManifest(bucket, userKey, "load-user")).manifest.format, "nmg-shards-v2"); + await bucket.put(manifestKey, JSON.stringify({ ...flat(), format: "nmg-shards-v3" })); + assert.equal((await loadManifest(bucket, userKey, "load-user")).manifest.format, "nmg-shards-v3"); + // A v4 manifest written by a newer deployment loads even without a local writer. + await bucket.put(manifestKey, JSON.stringify({ format: STORAGE_FORMAT_V4, parts: [], receipts: {}, pending: 1 })); + assert.equal(pendingCount((await loadManifest(bucket, userKey, "load-user")).manifest), 1); + + const rejects = async (manifest, code) => { + await bucket.put(manifestKey, JSON.stringify(manifest)); + await assert.rejects(loadManifest(bucket, userKey, "load-user"), (error) => { + assert.equal(error.code, code); + assert.equal(error.status, 503); + return true; + }); + }; + // Rolling back to a build that does not know v4 must fail loudly, not serve a + // partition set that is missing pending data. + await rejects({ format: "nmg-shards-v5", parts: [], receipts: {} }, "storage_corrupt"); + await rejects({ format: STORAGE_FORMAT_V4, parts: [], receipts: {}, pending: -1 }, "storage_corrupt"); + await rejects({ format: STORAGE_FORMAT_V4, parts: [], receipts: {}, pending: 1.5 }, "storage_corrupt"); + // A missing manifest still resolves through the legacy image head. + assert.deepEqual((await loadManifest(createMemoryBucket(), keyFor("absent"), "absent")).manifest, { + format: "nmg-shards-v2", + parts: [], + receipts: {}, + }); + console.log("PASS: v4 loads, unknown formats and invalid pending counts fail as storage_corrupt."); +} + +// 5. clearLog: prefix-scoped, paged, idempotent, and blind to neighbouring keys. +{ + const bucket = createMemoryBucket(); + const owner = keyFor("log-owner"); + const other = keyFor("log-other"); + const prefix = frozenPrefix(owner); + const names = [entryName(1, "a"), entryName(2, "b")]; + for (const name of names) await bucket.put(prefix + name, JSON.stringify({ v: 1 })); + await bucket.put(frozenPrefix(other) + names[0], JSON.stringify({ v: 1 })); + for (const neighbour of [ + `users/${owner}/manifest.json`, + `users/${owner}/login.json`, + `users/${owner}/logx/keep.json`, + `users/${owner}/parts/keep.sqlite`, + ]) + await bucket.put(neighbour, "keep"); + + assert.deepEqual(keysUnder(bucket, prefix).sort(), names.map((name) => prefix + name).sort()); + assert.equal(await clearLog(bucket, owner), names.length); + assert.deepEqual(keysUnder(bucket, prefix), []); + for (const neighbour of [ + `users/${owner}/manifest.json`, + `users/${owner}/login.json`, + `users/${owner}/logx/keep.json`, + `users/${owner}/parts/keep.sqlite`, + frozenPrefix(other) + names[0], + ]) + assert.ok(bucket.objects.has(neighbour), `${neighbour} must survive`); + assert.equal(await clearLog(bucket, owner), 0, "a repeated clear removes nothing"); + + // A key that could step outside its own tree must be refused before anything is + // listed. String equality alone would not catch a later divergence in validation. + const listsBefore = bucket.stats.lists; + for (const key of ["a/b", "a\\b", "..", ".", ""]) + await assert.rejects(clearLog(bucket, key), RangeError, `clearLog(${JSON.stringify(key)})`); + assert.equal(bucket.stats.lists, listsBefore, "a rejected key must not reach the bucket"); + assert.throws(() => logPrefix("a/b"), RangeError); + console.log("PASS: clearLog deletes only this user's log entries, repeats as a no-op, and refuses escaping keys."); +} + +// 6. clearLog pages through a log larger than one list response. +{ + const bucket = createMemoryBucket(); + const owner = keyFor("log-bulk"); + const prefix = frozenPrefix(owner); + const total = 1205; + for (let sequence = 1; sequence <= total; sequence++) + bucket.objects.set(prefix + entryName(sequence, "c"), { body: "{}", etag: "e" }); + + assert.equal(await clearLog(bucket, owner), total); + assert.deepEqual(keysUnder(bucket, prefix), []); + assert.equal(bucket.stats.lists, 2, "1205 entries need two list pages"); + console.log("PASS: clearLog pages with a stable cursor and removes every page."); +} + +// 7. resetUser clears the log after the generation barrier is committed. +{ + const bucket = createMemoryBucket(); + const userId = "reset-user"; + const userKey = keyFor(userId); + const prefix = frozenPrefix(userKey); + await commitManifest(bucket, userKey, null, withPending(flat(), 2)); + const names = [entryName(1, "d"), entryName(2, "e")]; + for (const name of names) await bucket.put(prefix + name, JSON.stringify({ v: 1, generation: "g1" })); + const before = (await loadManifest(bucket, userKey, userId)).manifest; + + const reset = await resetUser(bucket, userId, "reset-1"); + assert.equal(reset.success, true); + assert.equal(reset.cleanup_complete, true); + assert.deepEqual(keysUnder(bucket, prefix), [], "the log prefix is emptied by reset"); + + const after = (await loadManifest(bucket, userKey, userId)).manifest; + assert.notEqual(after.generation, before.generation); + assert.equal(pendingCount(after), 0); + assert.equal(after.format, "nmg-shards-v2", "a reset manifest is v2, not v4"); + assert.ok(!bodyOf(bucket, `users/${userKey}/manifest.json`).includes('"pending"')); + + const replay = await resetUser(bucket, userId, "reset-1"); + assert.equal(replay.success, true); + assert.equal(replay.cleanup_complete, true); + console.log("PASS: reset empties the log, writes a v2 manifest, and replays idempotently."); +} + +// 8. A reset whose log cleanup fails still succeeds; the same reset_id retries it. +{ + const bucket = createMemoryBucket(); + const userId = "reset-log-failure"; + const userKey = keyFor(userId); + const prefix = frozenPrefix(userKey); + for (const name of [entryName(3, "f"), entryName(4, "0")]) + await bucket.put(prefix + name, JSON.stringify({ v: 1, generation: "g1" })); + const unavailable = { ...bucket, delete: async () => Promise.reject(new Error("COS unavailable")) }; + + const reset = await resetUser(unavailable, userId, "reset-2"); + assert.equal(reset.success, true, "the manifest commit is the barrier; cleanup is retryable"); + assert.equal(reset.cleanup_complete, false); + assert.equal(keysUnder(bucket, prefix).length, 2, "failed cleanup leaves the entries for the retry"); + + const retried = await resetUser(bucket, userId, "reset-2"); + assert.equal(retried.success, true); + assert.equal(retried.cleanup_complete, true); + assert.deepEqual(keysUnder(bucket, prefix), []); + console.log("PASS: a failed log cleanup reports incomplete and the same reset_id finishes it."); +} + +console.log( + "PASS: manifest pending/lazy-v4/clearLog verified without a build — " + + "empty log keeps today's bytes, pending log is v4, reset drains the log.", +); diff --git a/onlion/scripts/test-request-resume.mjs b/onlion/scripts/test-request-resume.mjs index e2ade45b..74f5833f 100644 --- a/onlion/scripts/test-request-resume.mjs +++ b/onlion/scripts/test-request-resume.mjs @@ -4,7 +4,9 @@ import fs from 'node:fs'; import path from 'node:path'; import {createRequire} from 'node:module'; const require=createRequire(import.meta.url); -const {Miniflare}=createRequire(require.resolve('wrangler/package.json'))('miniflare'); +// Resolved from this package: wrangler's own range tracks a newer Miniflare whose +// constructor rejects this fixture's options, and a transitive dependency is not a pin. +const {Miniflare}=require('miniflare'); const root=process.cwd(), storage=path.join(root,'.sites-runtime/resume-r2'); fs.rmSync(storage,{recursive:true,force:true}); const modules=[{type:'ESModule',path:root+'/dist/server/index.js'}, diff --git a/onlion/scripts/worker-fixture.mjs b/onlion/scripts/worker-fixture.mjs index f451187c..a72e8754 100644 --- a/onlion/scripts/worker-fixture.mjs +++ b/onlion/scripts/worker-fixture.mjs @@ -1,30 +1,42 @@ -import path from 'node:path'; -import { createRequire } from 'node:module'; +import path from "node:path"; +import { createRequire } from "node:module"; const require = createRequire(import.meta.url); -const { Miniflare } = createRequire(require.resolve('wrangler/package.json'))('miniflare'); -const root = path.resolve(import.meta.dirname, '..'); +// Resolved from this package, not through wrangler: wrangler's own range tracks a newer +// Miniflare whose constructor rejects this fixture's options, and a transitive dependency +// is not a pin. CI installs from the lockfile, so the two have to agree. +const { Miniflare } = require("miniflare"); +const root = path.resolve(import.meta.dirname, ".."); const modules = [ - { type: 'ESModule', path: path.join(root, 'dist/server/index.js') }, - { type: 'ESModule', path: path.join(root, 'dist/server/sqlite/worker-loader.mjs') }, - { type: 'CompiledWasm', path: path.join(root, 'dist/server/sqlite/sqlite3.wasm') }, - { type: 'CompiledWasm', path: path.join(root, 'dist/server/sqlite/callbacks.wasm') }, + { type: "ESModule", path: path.join(root, "dist/server/index.js") }, + { type: "ESModule", path: path.join(root, "dist/server/sqlite/worker-loader.mjs") }, + { type: "CompiledWasm", path: path.join(root, "dist/server/sqlite/sqlite3.wasm") }, + { type: "CompiledWasm", path: path.join(root, "dist/server/sqlite/callbacks.wasm") }, ]; /** Same built Worker, WASM modules and compatibility target for every API test. */ export function createWorkerFixture({ storage = false, bindings = {}, outboundService } = {}) { return new Miniflare({ r2Persist: storage, - workers: ['a', 'b'].map(name => ({ - name, modules, modulesRoot: path.join(root, 'dist/server'), - compatibilityDate: '2026-05-15', compatibilityFlags: ['nodejs_compat'], + workers: ["a", "b"].map((name) => ({ + name, + modules, + modulesRoot: path.join(root, "dist/server"), + compatibilityDate: "2026-05-15", + compatibilityFlags: ["nodejs_compat"], bindings: { - NMG_API_KEY: 'test-key', NMG_OWNER_EMAIL: 'owner@example.test', - NMG_SELFTEST_TOKEN: 'self-test-key', - NMG_SELFTEST_EXPIRES_AT: String(name === 'b' ? 1 : Date.now() + 3600000), + // The API suites assert partition-write timing (a partition after an Add, receipts in + // the manifest, delta compaction). Those are properties of the synchronous accept + // path, which remains supported as the escape hatch, so they pin it explicitly. The + // log path has its own suite, which also pins that `log` is the code default. + NMG_ADD_MODE: "sync", + NMG_API_KEY: "test-key", + NMG_OWNER_EMAIL: "owner@example.test", + NMG_SELFTEST_TOKEN: "self-test-key", + NMG_SELFTEST_EXPIRES_AT: String(name === "b" ? 1 : Date.now() + 3600000), ...bindings, }, - r2Buckets: { MEMORY: 'memory' }, + r2Buckets: { MEMORY: "memory" }, ...(outboundService ? { outboundService } : {}), })), }); diff --git a/package.json b/package.json index 4e5c633b..667c3836 100644 --- a/package.json +++ b/package.json @@ -123,7 +123,8 @@ "pi:state": "node --experimental-strip-types scripts/pi-control.ts state", "skill:nmg:sync": "node --experimental-strip-types scripts/sync-nmg-skill.ts", "skill:nmg:check": "node --experimental-strip-types scripts/sync-nmg-skill.ts --check", - "docs:check": "node --experimental-strip-types scripts/verify-docs.mts && node --experimental-strip-types scripts/verify-document-groups.ts", "agent:context": "node --experimental-strip-types tools/repo-context.ts", + "docs:check": "node --experimental-strip-types scripts/verify-docs.mts && node --experimental-strip-types scripts/verify-document-groups.ts", + "agent:context": "node --experimental-strip-types tools/repo-context.ts", "agent:context:check": "node --experimental-strip-types tools/repo-context.ts --check", "agent:verify": "node --experimental-strip-types tools/agent-verify-watchdog.ts", "ci:uncovered-tests": "node --experimental-strip-types tools/ci-uncovered-tests.ts --check", @@ -145,7 +146,8 @@ "test:cg": "node --experimental-strip-types --test tests/core/autodiff.test.ts tests/core/differentiable-controller.test.ts tests/core/hierarchical-activation.test.ts tests/core/memory-graph-reasoner.test.ts tests/core/fork-merge.test.ts tests/core/robustness.test.ts", "glossary:check": "node --experimental-strip-types tools/glossary-check.ts", "check:policy-words": "node --experimental-strip-types tools/policy-word-check.ts", - "rtm:check": "node --experimental-strip-types tools/rtm-check.ts" + "rtm:check": "node --experimental-strip-types tools/rtm-check.ts", + "verify:onlion": "cd onlion && npm run verify" }, "devDependencies": { "@deepseek-ai/cordis": "4.0.1", diff --git a/tests/tools/ci-uncovered-tests.test.ts b/tests/tools/ci-uncovered-tests.test.ts index 5747fafc..b24619f3 100644 --- a/tests/tools/ci-uncovered-tests.test.ts +++ b/tests/tools/ci-uncovered-tests.test.ts @@ -32,6 +32,9 @@ const scripts = (coverage: string) => ({ "test:coverage": `node --test ${coverage}`, "test:research": 'node --test "tests/evals/**/*.test.ts"', "test:chaos": 'node --test "tests/chaos/*.test.ts"', + // The service entry point switches into its own manifest; a fixture has no `onlion/`, and + // the gate treats a missing manifest as "no scripts" rather than an error. + "verify:onlion": "cd onlion && npm run verify", }); test("it names the suites no CI job reaches, and passes when they are acknowledged", (t) => { diff --git a/tools/ci-uncovered-tests.ts b/tools/ci-uncovered-tests.ts index 84c2353c..d5b1f8b1 100644 --- a/tools/ci-uncovered-tests.ts +++ b/tools/ci-uncovered-tests.ts @@ -15,11 +15,18 @@ // evidence of that run, not as suites anyone maintains, so scanning them is a false positive. // Anything *else* that falls out of CI has to be acknowledged here on purpose. import { execFileSync } from "node:child_process"; -import { globSync, readFileSync } from "node:fs"; +import { existsSync, globSync, readFileSync } from "node:fs"; import { join } from "node:path"; /** The entry points of the CI jobs, in .github/workflows/ci.yml. */ -const CI_ENTRY_POINTS = ["verify:product-ci", "verify:research", "verify:chaos"]; +const CI_ENTRY_POINTS = ["verify:product-ci", "verify:research", "verify:chaos", "verify:onlion"]; + +/** A second manifest whose suites live in its own script set. Service tests are + * `scripts/test-*.mjs` invoked by name rather than globbed `.test.ts` files, so both + * the universe and the resolution need to know about them - otherwise a whole + * production surface sits outside every CI job and outside this gate, which is + * exactly how six defects reached a green commit. */ +const SERVICE_ROOT = "onlion"; /** Specific roots precede their parent. Acknowledgement does not imply standalone execution. */ const ACKNOWLEDGED_ROOTS = [ @@ -29,6 +36,26 @@ const ACKNOWLEDGED_ROOTS = [ reason: "fixture checks run through their harness", }, { prefix: "evals/", onDemand: true, reason: "on-demand only" }, + { + prefix: "onlion/scripts/test-cos-bucket.mjs", + onDemand: true, + reason: "needs a real COS bucket and credentials", + }, + { + prefix: "onlion/scripts/test-edgeone-worker.mjs", + onDemand: true, + reason: "needs a deployed endpoint and credentials", + }, + { + prefix: "onlion/scripts/test-capacity.mjs", + onDemand: true, + reason: "asserts the synchronous accept path; not yet ported to the log default", + }, + { + prefix: "onlion/scripts/test-request-resume.mjs", + onDemand: true, + reason: "asserts the synchronous accept path; not yet ported to the log default", + }, { prefix: "docs/experiments/execution/archive/", onDemand: false, @@ -42,16 +69,44 @@ function acknowledgedRoot(file: string) { type Scripts = Record; -/** Follows `npm run ` chains so a job's script can be read for its own globs. */ -function scriptGlobs(name: string, scripts: Scripts, seen = new Set()): string[] { - if (seen.has(name)) return []; - seen.add(name); +/** Follows `npm run ` chains so a job's script can be read for its own globs. + * A `cd onlion && ...` segment switches to that manifest; a `node ` segment + * names the file it runs, which is how the service suites are invoked. */ +function scriptGlobs( + name: string, + scripts: Scripts, + manifests: Record, + prefix = "", + seen = new Set(), +): string[] { + const key = prefix + name; + if (seen.has(key)) return []; + seen.add(key); const command = scripts[name]; - if (command === undefined) throw new Error(`no such script: ${name}`); + if (command === undefined) { + // A root entry point that names a script nobody defines is drift and must be loud. A + // subpackage script that is absent contributes nothing: the job that runs that package + // would fail at `npm run ` on its own, so this gate does not duplicate it. + if (prefix) return []; + throw new Error(`no such script: ${key}`); + } const globs: string[] = []; - for (const match of command.matchAll(/npm run ([a-zA-Z0-9:_-]+)/g)) - globs.push(...scriptGlobs(match[1]!, scripts, seen)); - for (const match of command.matchAll(/"([^"]*\*[^"]*)"/g)) globs.push(match[1]!); + for (const segment of command.split("&&")) { + const part = segment.trim(); + const nested = /^cd\s+([\w./-]+)$/.exec(part); + if (nested) { + prefix = nested[1]!.replace(/\/$/u, "") + "/"; + continue; + } + const run = /^npm run ([a-zA-Z0-9:_-]+)$/.exec(part); + if (run) { + globs.push(...scriptGlobs(run[1]!, manifests[prefix] ?? {}, manifests, prefix, seen)); + continue; + } + const node = /^node\s+(?:--\S+\s+)*(\S+\.(?:mjs|cjs|js|ts|mts))$/.exec(part); + if (node) globs.push(prefix + node[1]!); + } + for (const match of command.matchAll(/"([^"]*\*[^"]*)"/g)) globs.push(prefix + match[1]!); return globs; } @@ -65,16 +120,32 @@ export interface Coverage { /** Every test file in the repository, and which of them the CI jobs reach. */ export function testCoverage(root: string): Coverage { - const scripts = ( - JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { scripts: Scripts } - ).scripts; - const globs = [...new Set(CI_ENTRY_POINTS.flatMap((name) => scriptGlobs(name, scripts)))]; + // A manifest that is not there contributes no scripts: this gate is also run against + // synthetic repositories in its own tests, which carry a root manifest and nothing else. + const read = (dir: string) => { + const file = join(root, dir, "package.json"); + if (!existsSync(file)) return {}; + return (JSON.parse(readFileSync(file, "utf8")) as { scripts: Scripts }).scripts; + }; + const manifests: Record = { + "": read(""), + [`${SERVICE_ROOT}/`]: read(SERVICE_ROOT), + }; + const globs = [ + ...new Set(CI_ENTRY_POINTS.flatMap((name) => scriptGlobs(name, manifests[""]!, manifests))), + ]; const covered = new Set(); for (const glob of globs) for (const file of globSync(glob, { cwd: root })) covered.add(file.replaceAll("\\", "/")); // Only files the repository actually contains: another agent's untracked work in progress is // not part of it yet, and judging it here would report a gap nobody has committed. - const all = execFileSync("git", ["ls-files", "*.test.ts"], { cwd: root, encoding: "utf8" }) + // The service suites are `scripts/test-*.mjs`, invoked by name rather than globbed, so + // they are part of the universe too - otherwise a whole production surface sits outside + // every CI job and outside this gate. + const all = execFileSync("git", ["ls-files", "*.test.ts", `${SERVICE_ROOT}/scripts/test-*.mjs`], { + cwd: root, + encoding: "utf8", + }) .split("\n") .map((file) => file.trim()) .filter(Boolean);