From 586f73016fc134c09c2e19b02e45c94b390e8c98 Mon Sep 17 00:00:00 2001 From: focaxisdev <263514337+focaxisdev@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:12:27 +0800 Subject: [PATCH 1/6] fix: publish verified recall efficiency and lifecycle snapshot Atomic connector publication of the exact tracked tree from local commit a66a52d4fceaeba3dac4a24c8e7cc82d834a4331. Source tree: 7365590fdcc4ea07a3513e9695ecf53a0e80e759 Verified remote main parent: 188d7e8c3aed224e45071d041cd140509887b29f Original local commits bad6b527409a44a693562fbe60c41be74126e573, fa479335dbc5acbad46a352d1ee8bd6086c62cd7, 4c05a77605f6cf63a10261dc394c0824208313ea and later checkpoint commits remain preserved locally. This new remote commit intentionally has a different SHA. No release, merge or package publication. Branch-only acceptance must pass independently. --- .github/workflows/recall-acceptance.yml | 80 + AGENTS.md | 13 + CHANGELOG.md | 13 + README.md | 7 + docs/actions-acceptance.md | 125 ++ docs/actions-publication-evidence.json | 314 +++ docs/engine/recall-benchmark.json | 335 ++++ .../recall-cache-ownership-benchmark.json | 32 + docs/engine/recall-efficiency.md | 187 ++ docs/engine/recall-evaluation.json | 1675 +++++++++++++++++ docs/engine/recall-lifecycle.md | 189 ++ docs/engine/recall-round2-benchmark.json | 1013 ++++++++++ docs/engine/semantic-engine.md | 10 +- docs/release-recall-efficiency.md | 117 ++ docs/release-recall-lifecycle.md | 161 ++ docs/social-recall-lifecycle-draft.md | 24 + docs/verification-loop-evidence.json | 95 + docs/verification-loop.md | 153 ++ package.json | 4 +- scripts/benchmark-cache-ownership.mjs | 24 + scripts/benchmark-recall.mjs | 80 + scripts/evaluate-recall.mjs | 53 + scripts/fixtures/recall-evaluation.json | 425 +++++ src/core/semantic-recall-engine.ts | 163 +- src/index.ts | 3 + src/memory/default-chunker.ts | 5 +- src/plugins/cached-embedding-provider.ts | 105 ++ src/plugins/create-in-memory-engine.ts | 4 + src/retrieval/cue-aliases.ts | 26 + src/retrieval/prompt-projection.ts | 102 + src/storage/in-memory-storage.ts | 50 +- src/storage/in-memory-vector-store.ts | 5 + src/types/memory.ts | 3 + src/types/plugins.ts | 16 + src/utils/limits.ts | 9 + tests/cli-errors.test.ts | 68 + tests/helpers/cli-errors.ts | 34 + tests/memory-cli.test.ts | 6 +- tests/package-pack.test.ts | 4 +- tests/recall-efficiency.test.ts | 347 ++++ tests/recall-lifecycle.test.ts | 295 +++ tests/reliability.test.ts | 4 +- 42 files changed, 6326 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/recall-acceptance.yml create mode 100644 docs/actions-acceptance.md create mode 100644 docs/actions-publication-evidence.json create mode 100644 docs/engine/recall-benchmark.json create mode 100644 docs/engine/recall-cache-ownership-benchmark.json create mode 100644 docs/engine/recall-efficiency.md create mode 100644 docs/engine/recall-evaluation.json create mode 100644 docs/engine/recall-lifecycle.md create mode 100644 docs/engine/recall-round2-benchmark.json create mode 100644 docs/release-recall-efficiency.md create mode 100644 docs/release-recall-lifecycle.md create mode 100644 docs/social-recall-lifecycle-draft.md create mode 100644 docs/verification-loop-evidence.json create mode 100644 docs/verification-loop.md create mode 100644 scripts/benchmark-cache-ownership.mjs create mode 100644 scripts/benchmark-recall.mjs create mode 100644 scripts/evaluate-recall.mjs create mode 100644 scripts/fixtures/recall-evaluation.json create mode 100644 src/plugins/cached-embedding-provider.ts create mode 100644 src/retrieval/cue-aliases.ts create mode 100644 src/retrieval/prompt-projection.ts create mode 100644 src/utils/limits.ts create mode 100644 tests/cli-errors.test.ts create mode 100644 tests/helpers/cli-errors.ts create mode 100644 tests/recall-efficiency.test.ts create mode 100644 tests/recall-lifecycle.test.ts diff --git a/.github/workflows/recall-acceptance.yml b/.github/workflows/recall-acceptance.yml new file mode 100644 index 0000000..8b1ed48 --- /dev/null +++ b/.github/workflows/recall-acceptance.yml @@ -0,0 +1,80 @@ +name: Recall acceptance + +on: + push: + branches: + - fix/recall-efficiency-and-budgets + +permissions: + contents: read + +jobs: + acceptance: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + defaults: + run: + shell: bash + working-directory: ${{ runner.temp }}/deja-vu-public + steps: + - name: Checkout exact push revision + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Export public source and synthetic fixtures + working-directory: ${{ github.workspace }} + run: | + git rev-parse HEAD + mkdir -p "$RUNNER_TEMP/deja-vu-public" + git archive HEAD AGENTS.md CHANGELOG.md CONTRIBUTING.md LICENSE README.md package.json package-lock.json tsconfig.json src tests scripts docs examples starter-kit .github .gitignore encoding-status.md llms.txt | tar -x -C "$RUNNER_TEMP/deja-vu-public" + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install locked dependencies + run: npm ci + + - name: Type check + run: node node_modules/typescript/bin/tsc --noEmit -p tsconfig.json + + - name: Complete source, links, compiled and installed-package acceptance + run: npm test + + - name: Frozen recall quality and cache lifecycle counts + run: | + node scripts/evaluate-recall.mjs dist/src/index.js dist/src/index.js > "$RUNNER_TEMP/recall-quality.json" + node scripts/benchmark-cache-ownership.mjs dist/src/index.js dist/src/index.js > "$RUNNER_TEMP/recall-cache.json" + node --input-type=module <<'NODE' + import assert from 'node:assert/strict'; + import { readFileSync } from 'node:fs'; + import { join } from 'node:path'; + const read = path => JSON.parse(readFileSync(path, 'utf8')); + const quality = read(join(process.env.RUNNER_TEMP, 'recall-quality.json')); + const expected = read('docs/engine/recall-evaluation.json'); + // Runtime metadata is recorded, not a recall quality assertion. + console.log('Evaluation runtime:', quality.node, 'recorded runtime:', expected.node); + const { node: actualNode, ...actualQuality } = quality; + const { node: expectedNode, ...expectedQuality } = expected; + assert.deepEqual(actualQuality, expectedQuality); + const cache = read(join(process.env.RUNNER_TEMP, 'recall-cache.json')); + const recorded = read('docs/engine/recall-cache-ownership-benchmark.json'); + for (const mode of ['after-unscoped', 'after-owned']) { + assert.deepEqual(cache.modes[mode], recorded.modes[mode]); + } + assert.deepEqual(cache.modes['before-unscoped'], cache.modes['after-unscoped']); + console.log('Frozen development/held-out IDs and metrics, cache call counts and retained entries: passed'); + NODE + + - name: Synthetic protocol fixture checks + run: | + node scripts/dejavu-lint-memory.mjs --memory-root starter-kit/memory + node scripts/dejavu-lint-memory.mjs --memory-root examples/protocol-project/memory + node scripts/dejavu-lint-memory.mjs --memory-root docs/examples/settings-project/memory + npm run report:feedback diff --git a/AGENTS.md b/AGENTS.md index bb73fc6..33270d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,3 +30,16 @@ Write back only durable project memory: - safety or privacy constraints Never store secrets, full transcripts, PII, raw logs, low-signal chatter, or disposable exploration noise. + +## Verification Loop + +For implementation or repair work: + +1. Define the scope, exclusions, baseline revision and acceptance checklist before changing code. Preserve unrelated user changes and original requirements. +2. Make acceptance observable: include relevant source and compiled tests, types, package/install/use checks, regression coverage, behavior/quality checks and accurate documentation. Distinguish case counts from file-level summaries. +3. Keep evidence for each failure: command, environment, revision, exit status, original error/cause and the smallest reproducible example. Do not replace an operational error with a secondary parsing error. +4. Follow failure -> diagnosis -> focused correction -> affected regression tests -> full acceptance. State the hypothesis and use a check that can distinguish it from alternatives. +5. After two iterations without new evidence or improvement, change the hypothesis or strategy. Do not endlessly retry the same failure or spend the loop on unrelated polish. +6. Never weaken assertions, skip failing cases, hide errors or change security controls to obtain a green result. Use runtime/execution alternatives only when authorized and established not to bypass restrictions. +7. Finish only when acceptance passes or a genuine external blocker is demonstrated. For a blocker, preserve a reproducible checkpoint, identify the blocked gates and exact resource/permission needed, and report it as blocked rather than passed. +8. Review the final diff, document measured effects and limitations, and verify repository status. Commit, push, merge or publish only within the user's authorization. Task-specific data/privacy constraints take precedence over a generic verification command. diff --git a/CHANGELOG.md b/CHANGELOG.md index 846cc80..4ef58f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file. ## Unreleased +- Preserve original CLI subprocess failures in test helpers, retain execution/parse causes for invalid error responses, and add ten diagnostic regressions without weakening product assertions. +- Append general acceptance/evidence/repair-loop rules to AGENTS.md and record an [external-blocker checkpoint](docs/verification-loop.md); complete subprocess validation remains unverified. + +- Add opt-in exact query aliases, managed per-memory/revision embedding scopes, and serialized reads; digest cache keys avoid retained plaintext input keys. +- Compensate failed deletions, add explicit recoverable `supersedeMemory`, and return defensive summary/chunk copies from the in-memory adapter. +- Add opt-in prompt provenance and qualifier-loss warnings, plus explicit raw-content retrieval. +- Record development/held-out synthetic recall evaluation, ownership costs, repeated performance comparisons, and direct offline installed-package verification in the [round-two notes](docs/release-recall-lifecycle.md). Subprocess tests still report EPERM; no release is claimed. + +- Fix optional-engine false-none recall when an ineligible hybrid winner hides an eligible candidate; keep existing thresholds. +- Add opt-in candidate search breadth, bounded prompt projection, metadata-only embedding reuse, and a content/model-keyed bounded embedding cache. +- Cache lexical token sets with update/delete invalidation and validate recall/chunking limits before expensive work. +- Add synthetic regression tests, a reproducible comparison benchmark, and [unreleased notes](docs/release-recall-efficiency.md) with migration guidance and measurement limits. + ## [0.7.1] - Unreleased Release preparation only; not yet published. See the [release checklist](docs/release-v0.7.1.md). diff --git a/README.md b/README.md index 9df7a17..e2320bb 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,13 @@ const engine = createInMemorySemanticRecallEngine({ Read the [engine guide](./docs/engine/semantic-engine.md) or run the [basic example](./examples/basic/index.ts). The engine is an optional acceleration layer, not the default adoption path. +Unreleased engine work adds eligible-candidate selection, bounded prompt output, +and opt-in embedding reuse. The three-file setup is unchanged. See the +[local release-note draft](docs/release-recall-efficiency.md) and +[usage examples](docs/engine/recall-efficiency.md). +Further unreleased work adds opt-in [explicit aliases, cache ownership and provenance](docs/engine/recall-lifecycle.md). +The [round-two notes](docs/release-recall-lifecycle.md) report recall gains from authored aliases, measured costs and remaining verification blockers. + ## Contributing Issues and focused pull requests are welcome. Read [CONTRIBUTING.md](./CONTRIBUTING.md) for the project boundaries and verification commands. diff --git a/docs/actions-acceptance.md b/docs/actions-acceptance.md new file mode 100644 index 0000000..15a1a9c --- /dev/null +++ b/docs/actions-acceptance.md @@ -0,0 +1,125 @@ +# Branch acceptance on GitHub Actions + +Unreleased verification setup, continuing the [Cloud checkpoint](verification-loop.md). +Starting revision: `4c05a77605f6cf63a10261dc394c0824208313ea`. + +## Scope and acceptance defined before execution + +The dedicated `Recall acceptance` workflow runs only on pushes to +`fix/recall-efficiency-and-budgets`, with `contents: read`, Node 22, and the +existing Ubuntu/Windows runner matrix. It does not publish, deploy, create a +release, access secrets, or require a pull request. The existing main/PR CI is +unchanged. Checkout does not persist credentials. + +Both jobs must finish successfully at the exact pushed SHA: + +| Gate | Command and required evidence | +| --- | --- | +| Types | Existing TypeScript compiler, `--noEmit -p tsconfig.json` | +| Complete source suite | `npm test` invokes the unchanged complete source selection, including helper regressions | +| Complete compiled suite | `npm test` builds and runs all six compiled test files | +| Pack/install/use | `npm test` invokes the package test, including prepack, offline local installation, installed CLI and engine API assertions | +| Regressions, lifecycle and projection | All existing product assertions in the complete selections remain enabled | +| Recall quality | Exact frozen fixture digest, returned IDs, development/held-out metrics and interpretation; runtime version is recorded separately | +| Cache measurements | Current unscoped/owned provider-call and retained-entry counts match the frozen record; these are not latency measurements | +| Documentation and protocol | Existing link checker plus public synthetic fixture lint and feedback checks | + +The workflow exports the committed public source and synthetic fixtures to the +runner's temporary directory. It excludes the project's root `memory/` tree, +retains every test, and runs the unmodified link checker over that public snapshot. +Package tests initialize their own synthetic project. No private project-memory +read is needed. Dependencies come from the existing lockfile; no dependency is +added. Hosted runner results establish behavior on those runtimes, not the exact +cause of the Cloud Node 24 native EPERM. + +For a failed run, retain its SHA, job/log URL, original error and failed gate; +diagnose before making a focused correction. Rerun the complete acceptance after +each correction. Do not turn a blocked or skipped gate into a pass. Follow the +repository's verification loop and stop on an actual access denial. + +## Initial transport checkpoint + +The initial local HEAD and clean worktree were verified. The remote repository is +public; its main SHA is `188d7e8c3aed224e45071d041cd140509887b29f`. The only +existing workflow is `ci.yml`, triggered by main pushes and pull requests, with +read-only contents permission and no release/publish/deploy step. + +The ordinary read-only transport check +`git ls-remote origin refs/heads/main refs/heads/fix/recall-efficiency-and-budgets` +failed with exit 128: + +```text +fatal: unable to access 'https://github.com/focaxisdev/deja-vu.git/': Failed to connect to proxy port 8080 after 0 ms: Could not connect to server +``` + +The existing GitHub connector can read this repository and reports push access, +but its branch listing has no fix branch and the Git commit endpoint returns 404 +for `4c05a77605f6cf63a10261dc394c0824208313ea`. Its commit-creation interface does +not accept original author/committer metadata or raw Git objects, so recreating +commits through it cannot preserve the existing commit SHAs. No substitute history +or remote branch was created. No push or Actions run is claimed. + +An ordinary push would require a working connection to the Cloud's configured +Git proxy. No new secret, broader repository permission, direct connection, or +security-setting change was attempted. At this checkpoint, all GitHub Actions +gates were **not run / transport blocked**. + +Local preparation checks passed: workflow YAML parsing and trigger/permission/gate +assertions, Bash syntax checking of all run blocks, TypeScript no-emit checking, +and the exact frozen-quality/cache assertion block using the previously compiled +unchanged engine. The unmodified link checker passed over the public snapshot +(69 Markdown files). These checks do not execute a hosted job, `npm ci`, or the +blocked complete subprocess suites and cannot establish Actions acceptance. + +## Authorized connector publication + +The subsequent Cloud UI push also reported failure. A fresh remote branch listing +still showed no fix branch and main remained at the SHA above. The user authorized +an alternative: preserve all original local commits, but create a new remote +commit on that verified main with the exact final local tracked tree. Remote and +local commit SHAs are intentionally different; this is not a force push or a +claim that the original local commit objects were transferred. + +Before creating the branch, compare the remote tree SHA and every recursive +entry's path, type, mode and object SHA against the local tree. Transfer only +reviewed changed blobs, retaining unchanged public baseline objects by reference. +Include additions, modifications and deletions in the comparison. Do not read or +upload private memory, credentials or untracked files. Record the source local +commit and tree in the remote commit message, then record the returned remote SHA +in the verification checkpoint. If the connector rejects access or cannot preserve +the tree, stop without substituting an incomplete tree. Hosted acceptance remains +unverified until both jobs reach successful terminal states at the published SHA. + +## Connector result: permission blocked + +The reviewed source snapshot is local commit +`c2a9bb103e4f502d23a66a6c067f00c8b0b10467`, tree +`66a2c3949b8ad087289f438b451c47e608413b68`. All 122 baseline file entries +matched the remote main tree by path, type, mode and object SHA. Its final tree +has 147 files: 25 additions, 16 modifications, no deletions or mode changes. +Changed UTF-8 bytes and blob hashes were checked locally; credential-pattern +checks had no alerts. Only reviewed changed files were submitted, without reading +private memory or credential files. Unchanged baseline blobs were referenced. + +The first write, `POST /repos/focaxisdev/deja-vu/git/trees` through the existing +connector's `create_tree`, returned HTTP 403: + +```text +Resource not accessible by integration +``` + +No remote commit or branch was created, no alternative write endpoint was tried, +and no permission/security configuration was changed. A subsequent read confirmed +main unchanged and the target branch still absent. Repository metadata reporting +`push: true` did not establish integration access to this write endpoint. The exact +missing permission is not exposed by this error and is not assumed to be a +specific workflow or contents scope. + +All hosted gates remain **not run / connector permission blocked**. The final +remote tree could not be created or verified; there is no remote SHA or Actions +run/log URL. A maintainer must restore ordinary approved Git transport or provide +an already-authorized connection that can perform this Git tree write before +publication can continue. This task does not expand permissions or add secrets. +The [publication evidence](actions-publication-evidence.json) records every changed +path, mode and blob SHA, the source mapping and the denied action. The original +local history is retained; later local documentation commits record this blocker. diff --git a/docs/actions-publication-evidence.json b/docs/actions-publication-evidence.json new file mode 100644 index 0000000..0a0bd76 --- /dev/null +++ b/docs/actions-publication-evidence.json @@ -0,0 +1,314 @@ +{ + "sourceLocalCommit": "c2a9bb103e4f502d23a66a6c067f00c8b0b10467", + "sourceLocalTree": "66a2c3949b8ad087289f438b451c47e608413b68", + "remoteBase": "188d7e8c3aed224e45071d041cd140509887b29f", + "remoteBaseTree": "cb95df23b058b72a75b365b4d352e737f376241f", + "baselineEntriesVerified": 122, + "finalLocalEntries": 147, + "changes": [ + { + "path": ".github/workflows/recall-acceptance.yml", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "8b1ed489e6415c500c4023169695c006dd89b085" + }, + { + "path": "AGENTS.md", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "33270d556f5c1be0d02ca54c32dc8e09f584a7ba" + }, + { + "path": "CHANGELOG.md", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "4ef58f21f543ca5961e95ecf2fa20b215ed962da" + }, + { + "path": "README.md", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "e2320bbbf83644662fe84808b76207c782ab2e3f" + }, + { + "path": "docs/actions-acceptance.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "0b0307a9b95bf570c186cf8219e6932b68f59e8d" + }, + { + "path": "docs/engine/recall-benchmark.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "95d93c5bb453d61eebf5f5cb231afcea2a00bd8e" + }, + { + "path": "docs/engine/recall-cache-ownership-benchmark.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "f92893f3cd63bf547945baa69d16455cbaa45955" + }, + { + "path": "docs/engine/recall-efficiency.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "c80a43583a6a55530dd51b68129aebf2416baf91" + }, + { + "path": "docs/engine/recall-evaluation.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "e95e87e688358f55caf086f203d071ebd6c9c729" + }, + { + "path": "docs/engine/recall-lifecycle.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "2591379befb4825968837b52c04f385173c80366" + }, + { + "path": "docs/engine/recall-round2-benchmark.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "861faee413e75d501f15ea69c75eb77cd8a98584" + }, + { + "path": "docs/engine/semantic-engine.md", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "42ceb868d1acca6ce8671a38171acdc3e3bd2537" + }, + { + "path": "docs/release-recall-efficiency.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "d5b44490488e97637f678d4b9a11ea3b2c1b2bfa" + }, + { + "path": "docs/release-recall-lifecycle.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "0bf5db999b1d1725322653c501b2e6743fecb40d" + }, + { + "path": "docs/social-recall-lifecycle-draft.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "1da8f6e526cad1c92d048d74f26cad8995cf7f2a" + }, + { + "path": "docs/verification-loop-evidence.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "e7e5de1ebf19f10b575ceb062ac5b8a4e9d5871c" + }, + { + "path": "docs/verification-loop.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "0e55770b76e82b314d5d1dcbaf9071210587c716" + }, + { + "path": "package.json", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "0a50d3ab17808c647384733fbc9f7493ef69b701" + }, + { + "path": "scripts/benchmark-cache-ownership.mjs", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "1d8a23e7f9941c3c5c4f9fb6cc569818ac661f59" + }, + { + "path": "scripts/benchmark-recall.mjs", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "004f08a2ad406deb0659e9fea6cfda79ede5cc57" + }, + { + "path": "scripts/evaluate-recall.mjs", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "a78f6102ba5d7aa8820ae869365605e3ed6295eb" + }, + { + "path": "scripts/fixtures/recall-evaluation.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "3e3aa9cc7fa9d2cef41b3b1035520f158e357831" + }, + { + "path": "src/core/semantic-recall-engine.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "14392eae632bbb459e54499a97135729275e8b9c" + }, + { + "path": "src/index.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "a161f611bf081328e0c408d71f82403288a914b2" + }, + { + "path": "src/memory/default-chunker.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "8eea21770ca802a4b686193b52089d5f2d4c0bd5" + }, + { + "path": "src/plugins/cached-embedding-provider.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "b022f6d4b9e054093782b11df0b8bd015931aec6" + }, + { + "path": "src/plugins/create-in-memory-engine.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "ab0566ed641cec6bd7bc8505fdf95e9491d4319f" + }, + { + "path": "src/retrieval/cue-aliases.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "58088a6571f8c1a392e31a28cf515ec9b35e792e" + }, + { + "path": "src/retrieval/prompt-projection.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "55b8e694749dafba4099f4bc93652b5b7dfa251b" + }, + { + "path": "src/storage/in-memory-storage.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "fe57cd4bab44fe33766ee8e3568160be6029a199" + }, + { + "path": "src/storage/in-memory-vector-store.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "8f2f74767246be6a93bb8ba4fc3d456b5af2bfb8" + }, + { + "path": "src/types/memory.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "29d17a0b8752ac4e4ad2ab142d33b901a5a2808a" + }, + { + "path": "src/types/plugins.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "893f997f8c2b9976592d7a69feb009b589507deb" + }, + { + "path": "src/utils/limits.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "b15ca1ff1f6f5b923129968e7fb7bc70c79c4475" + }, + { + "path": "tests/cli-errors.test.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "7447239aa5e8ada5238f339040556ac6c1ec8537" + }, + { + "path": "tests/helpers/cli-errors.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "35d6b9300479f6429acac2c31af8142409f5267b" + }, + { + "path": "tests/memory-cli.test.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "18d736b13a1cf5b6ef8a4c3eadf8f6579bb3de2c" + }, + { + "path": "tests/package-pack.test.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "15c3800262e09e0e4edfaf4716786679cf32280a" + }, + { + "path": "tests/recall-efficiency.test.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "a37c0afd8634ccc029b551fad4cfceaea8cc9bed" + }, + { + "path": "tests/recall-lifecycle.test.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "5e944848c81426f1556cce594455d36b22c2342b" + }, + { + "path": "tests/reliability.test.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "c52772bdbed7671760abaa97f2264965b2fdba48" + } + ], + "verification": { + "remoteBaselineTree": "exact match", + "changedBlobEncodingAndHashes": "verified locally", + "secretPatternAlerts": 0, + "finalRemoteTree": "not created or verified; API denied", + "remoteBranch": "absent", + "actions": "not run" + }, + "blockedAction": { + "tool": "github_create_tree", + "method": "POST", + "endpoint": "https://api.github.com/repos/focaxisdev/deja-vu/git/trees", + "status": 403, + "message": "Resource not accessible by integration" + }, + "remoteCommit": null, + "remoteTree": null +} diff --git a/docs/engine/recall-benchmark.json b/docs/engine/recall-benchmark.json new file mode 100644 index 0000000..95d93c5 --- /dev/null +++ b/docs/engine/recall-benchmark.json @@ -0,0 +1,335 @@ +{ + "node": "v24.19.0", + "samples": 25, + "warmups": 5, + "units": "milliseconds", + "notes": [ + "Micro: storage.searchImpressions only. Scan: engine.scanImpressions.", + "API end-to-end: resident recall(loadChunks=true) plus JSON.stringify, including query embedding and chunk retrieval.", + "Excludes process start, index construction, persistence, network, model inference and host/LLM execution.", + "Baseline/current alternate measurement order. Medians/p95 are descriptive of this small synthetic run." + ], + "workloads": [ + { + "n": 100, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0083, + "p95": 0.0254 + }, + "current": { + "median": 0.0042, + "p95": 0.0172 + } + }, + "scan": { + "baseline": { + "median": 0.0129, + "p95": 0.0435 + }, + "current": { + "median": 0.0098, + "p95": 0.0535 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0694, + "p95": 1.6628 + }, + "current": { + "median": 0.0654, + "p95": 0.155 + } + } + }, + { + "n": 100, + "kind": "common", + "micro": { + "baseline": { + "median": 0.282, + "p95": 0.5696 + }, + "current": { + "median": 0.0236, + "p95": 0.1202 + } + }, + "scan": { + "baseline": { + "median": 0.2596, + "p95": 1.6196 + }, + "current": { + "median": 0.0304, + "p95": 0.1273 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.3136, + "p95": 1.9662 + }, + "current": { + "median": 0.0906, + "p95": 1.273 + } + } + }, + { + "n": 100, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0006, + "p95": 0.001 + }, + "current": { + "median": 0.0005, + "p95": 0.0007 + } + }, + "scan": { + "baseline": { + "median": 0.0014, + "p95": 0.0039 + }, + "current": { + "median": 0.0019, + "p95": 0.0024 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0027, + "p95": 0.0031 + }, + "current": { + "median": 0.0032, + "p95": 0.005 + } + } + }, + { + "n": 1000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0053, + "p95": 0.0058 + }, + "current": { + "median": 0.003, + "p95": 0.0046 + } + }, + "scan": { + "baseline": { + "median": 0.0085, + "p95": 0.0104 + }, + "current": { + "median": 0.0073, + "p95": 0.0085 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0506, + "p95": 0.2037 + }, + "current": { + "median": 0.0518, + "p95": 0.5262 + } + } + }, + { + "n": 1000, + "kind": "common", + "micro": { + "baseline": { + "median": 3.1107, + "p95": 4.956 + }, + "current": { + "median": 0.1643, + "p95": 0.4436 + } + }, + "scan": { + "baseline": { + "median": 2.4218, + "p95": 4.4501 + }, + "current": { + "median": 0.1747, + "p95": 0.3611 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 2.4995, + "p95": 3.9804 + }, + "current": { + "median": 0.2487, + "p95": 0.6969 + } + } + }, + { + "n": 1000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0004, + "p95": 0.0005 + }, + "current": { + "median": 0.0004, + "p95": 0.0005 + } + }, + "scan": { + "baseline": { + "median": 0.0012, + "p95": 0.0016 + }, + "current": { + "median": 0.0015, + "p95": 0.002 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0026, + "p95": 0.0028 + }, + "current": { + "median": 0.0028, + "p95": 0.0032 + } + } + }, + { + "n": 10000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0035, + "p95": 0.0077 + }, + "current": { + "median": 0.0014, + "p95": 0.0027 + } + }, + "scan": { + "baseline": { + "median": 0.0056, + "p95": 0.0079 + }, + "current": { + "median": 0.004, + "p95": 0.0066 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0503, + "p95": 0.133 + }, + "current": { + "median": 0.0478, + "p95": 0.1606 + } + } + }, + { + "n": 10000, + "kind": "common", + "micro": { + "baseline": { + "median": 30.7427, + "p95": 42.9272 + }, + "current": { + "median": 2.6053, + "p95": 6.1685 + } + }, + "scan": { + "baseline": { + "median": 30.5848, + "p95": 44.0301 + }, + "current": { + "median": 2.528, + "p95": 4.9677 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 30.8083, + "p95": 39.9615 + }, + "current": { + "median": 2.7437, + "p95": 4.7024 + } + } + }, + { + "n": 10000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0009, + "p95": 0.0015 + }, + "current": { + "median": 0.0004, + "p95": 0.0009 + } + }, + "scan": { + "baseline": { + "median": 0.0024, + "p95": 0.0033 + }, + "current": { + "median": 0.0024, + "p95": 0.0027 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0032, + "p95": 0.0046 + }, + "current": { + "median": 0.003, + "p95": 0.005 + } + } + } + ], + "operations": { + "baseline": { + "addCalls": 12, + "repeatedRecallCalls": 5, + "metadataUpdateCalls": 12, + "rawRecallBytes": 10253 + }, + "currentOptIn": { + "addCalls": 4, + "repeatedRecallCalls": 1, + "metadataUpdateCalls": 0, + "rawRecallBytes": 10253, + "projectedBytes": 2301 + } + } +} diff --git a/docs/engine/recall-cache-ownership-benchmark.json b/docs/engine/recall-cache-ownership-benchmark.json new file mode 100644 index 0000000..f92893f --- /dev/null +++ b/docs/engine/recall-cache-ownership-benchmark.json @@ -0,0 +1,32 @@ +{ + "note": "Deterministic provider-call counts, not latency or token costs. Sixteen memories share ten identical chunks; unique per-memory routing tags. Cache capacity 256.", + "modes": { + "before-unscoped": { + "addCalls": 18, + "twoRecallPassesCalls": 16, + "metadataUpdateCalls": 0, + "contentUpdateCalls": 2, + "retainedAfterDelete": "No public stats API in baseline" + }, + "after-unscoped": { + "addCalls": 18, + "twoRecallPassesCalls": 16, + "metadataUpdateCalls": 0, + "contentUpdateCalls": 2, + "retainedAfterDelete": { + "entries": 36, + "accountedBytes": 5184 + } + }, + "after-owned": { + "addCalls": 48, + "twoRecallPassesCalls": 16, + "metadataUpdateCalls": 0, + "contentUpdateCalls": 3, + "retainedAfterDelete": { + "entries": 0, + "accountedBytes": 0 + } + } + } +} diff --git a/docs/engine/recall-efficiency.md b/docs/engine/recall-efficiency.md new file mode 100644 index 0000000..c80a435 --- /dev/null +++ b/docs/engine/recall-efficiency.md @@ -0,0 +1,187 @@ +# Bounded recall for the optional engine + +These changes are unreleased. The three-file protocol, Markdown source of truth, +and CLI workflow do not need a migration. The optional engine now prevents an +ineligible hybrid winner from hiding an eligible match, reuses token sets, and +provides explicit controls for search breadth, prompt output and embedding reuse. + +## Find candidates before choosing output + +`topK` still defaults to five and limits returned candidates. The new +`candidateLimit` defaults to `topK`; set it explicitly to search more records +before ranking. It must be at least `topK`. Both are integers from 0 to 1000. +`topK: 0` returns no candidates without calling storage search. + +Candidates that meet the existing weak threshold come before candidates that +meet neither threshold. Hybrid ordering is preserved within both groups, +including between weak and strong matches. Scores are therefore not necessarily +globally descending across the eligibility boundary. With no eligible candidates, +`matched` remains false and no summary or detail is loaded. + +The default thresholds remain strong 0.85 and weak 0.75. A configured threshold +must be finite and satisfy `0 <= weak <= strong <= 1`. Increasing search breadth +can surface a better candidate, but also costs more ranking work and may expose +more irrelevant matches. It does not change the lexical candidate generator or +recover synonyms that have no shared tokens. + +## Opt in to bounded prompt output + +```ts +import { + createInMemorySemanticRecallEngine, + projectRecallForPrompt, +} from "@focaxisdev/deja-vu"; + +const engine = createInMemorySemanticRecallEngine(); +await engine.addMemory({ + id: "settings", + title: "Settings sync", + content: "Decision: settings sync stays server-backed.", +}); +const recall = await engine.recall({ + text: "settings sync", + candidateLimit: 20, + topK: 5, + loadChunks: true, + chunkLimit: 3, +}); +const context = projectRecallForPrompt(recall, { maxBytes: 4096 }); +// Put context.text in the host's memory-data section of the prompt. +// Check context.truncated before relying on it as complete evidence. +console.log(context.bytes, context.truncated); +``` + +The projection returns JSONL containing only summary text, recent updates, chunk +text, and their identifying fields. It excludes vectors, arbitrary metadata, +candidate lists, scores and internal budget explanations. The existing +`RecallResult` is unchanged; serializing it directly still includes vectors. + +The byte budget covers the entire `context.text`, including JSON framing and +newlines. It defaults to 8192 UTF-8 bytes, accepts 0 through 1,048,576, and returns +empty text if even a record header cannot fit. Text is added in description, +context, intent, recent-updates, then chunk order. The last included field may be +cut at a Unicode code-point boundary. JSON escaping keeps each included record +parseable, but truncation can remove a qualifier or omit an important chunk. + +For an actual token budget, supply the target model's tokenizer. No tokenizer is +installed or guessed by the library: + +```ts +import { projectRecallForPrompt, type RecallResult } from "@focaxisdev/deja-vu"; + +function promptMemory(result: RecallResult, countModelTokens: (text: string) => number) { + return projectRecallForPrompt(result, { + maxBytes: 4096, + maxTokens: 800, + countTokens: countModelTokens, + }); +} +``` + +`maxTokens` accepts 0 through 1,048,576 and requires a deterministic counter that +returns a non-negative safe integer. Each accepted prefix is checked as a whole; +a non-monotonic tokenizer can leave unused budget. `tokens` is reported only when +a counter is supplied. Reserve separate space for the host's wrappers and other +prompt sections. The projection controls size and fields; it does not make memory +text trusted instructions or sanitize prompt injection. + +## Opt in to embedding reuse + +```ts +import { + CachedEmbeddingProvider, + MockEmbeddingProvider, + createInMemorySemanticRecallEngine, +} from "@focaxisdev/deja-vu"; + +// The mock is deterministic local test data, not a semantic language model. +const embeddings = new CachedEmbeddingProvider(new MockEmbeddingProvider(256), { + modelKey: "mock:256:v1", + maxEntries: 128, + maxBytes: 8 * 1024 * 1024, +}); +const engine = createInMemorySemanticRecallEngine({ + embeddingProvider: embeddings, + preserveDerivedOnMetadataUpdate: true, +}); +await engine.addMemory({ id: "settings", title: "Settings sync", content: "Settings sync stays server-backed." }); +await engine.updateMemory("settings", { importance: 0.9, metadata: { reviewed: "yes" } }); +// No summaries, chunks or embeddings are regenerated for this update. +embeddings.clear(); // Release this wrapper's retained digest/vector entries when needed. +``` + +`preserveDerivedOnMetadataUpdate` defaults to false. With it enabled, an update +whose keys are limited to `importance` and `metadata` preserves summary text, +chunk IDs, chunk importance/timestamps, embeddings, raw content, creation time, +and last-access time. It updates the memory's modification time, familiarity +importance, and merged summary metadata. Title and tags remain authoritative. +The existing compensated write path still runs; this is not a zero-I/O update. + +This mode is a contract to preserve derived content, including for custom +summary generators and chunkers. Do not enable it if changing metadata must +regenerate those fields. Any other update key uses the normal preparation path. +The default behavior still invokes custom generators on metadata updates. + +The embedding cache is optional, local to one wrapper instance, and stores only +completed embeddings. Its key is now a keyed digest of exact input text and `modelKey`, without +text normalization; see the [ownership guide](recall-lifecycle.md). Include provider, model revision, dimensions and preprocessing +options in that identifier. A callback may supply the current identifier when +configuration changes. Keep provider configuration stable for each in-flight +call: the key must identify the model that actually produced its vector. +Model changes also require the host to rebuild stored +corpus vectors; cache invalidation alone does not make mixed-model vectors valid. + +The defaults are 128 entries and 8 MiB of accounted key/vector storage. Limits +are 0–10,000 entries and 0–64 MiB. Setting either limit to zero disables retention. +Accounted storage is 128 bytes per digest plus vector float64 bytes, not a measurement +of the JavaScript heap. Entry count also bounds per-entry overhead. Oversized +entries bypass the cache, failures are not cached, and returned vectors are copies. +Concurrent misses may compute separately; there is no unbounded pending-promise +map. Clearing the cache prevents already-pending requests from repopulating it. + +Use a separate cache instance for each project/security scope. Without managed +ownership, deleting a memory does not selectively release shared cache entries; +use `clear()` for the wrapper or opt in to +[per-memory/revision scopes](recall-lifecycle.md). Scopes prevent late insertion +and avoid shared-entry ownership ambiguity, at the cost of cross-memory reuse. +Reference release is not verified secure erasure. There is no recall-result cache, +but default unsynchronized reads can overlap writes; enable managed ownership or +`consistentReads` for ordering within one engine instance. + +## Migration checks + +- `topK`, `candidateLimit` and `chunkLimit` reject negative, fractional, non-finite + or greater-than-1000 values with `RangeError`, before engine adapter calls. + `chunkLimit` still defaults to three; zero skips query embedding and detail reads. + Built-in storage/vector search methods validate their limits too. +- Default chunking accepts `maxCharacters` from 1 to 1,000,000 and overlap from + zero to `maxCharacters - 1`. The default remains 280/40; for a smaller custom + maximum, omitted overlap is capped to fit. Previously invalid sizes now reject. +- Familiarity getters/search results from the built-in storage return copies. + Use `saveFamiliarity`, `updateMemory` and `deleteMemory` to change indexed data; + mutating a returned object no longer edits the store. +- Token sets are created on save and invalidated on replacement/deletion. The + lexical score and posting-order tie behavior are retained, including Unicode + behavior. There is no automatic language segmentation or stop-word policy. Exact host-authored + query aliases are available as a separate opt-in. +- Leave the new opt-ins disabled to retain the previous search breadth, raw + result shape and metadata regeneration behavior. Eligibility correction, + numeric validation and token-set reuse apply by default. + +## Measurements and deferred work + +See the [unreleased notes](../release-recall-efficiency.md) for measured results +and validation limits. The [benchmark script](../../scripts/benchmark-recall.mjs) +uses synthetic fixtures only. Token caching increases resident index memory and +moves work to writes; defensive copies also allocate memory. Heap usage and +index construction are outside the hot-path benchmark. + +CLI index persistence, route/frontmatter caching and supersession graph changes +are deferred. This batch leaves CLI implementation and the three-file protocol +unchanged. Also deferred: automatic language-aware candidate generation, semantic fallback, +ANN, chunk-by-ID adapter reads, batch embedding APIs and durable transactions. +No paid service, runtime dependency or daemon is introduced. The built-in +in-memory adapters remain nonpersistent, and the default mock embedding model +still does not provide semantic or multilingual understanding. + +For newer lifecycle opt-ins, digest-key byte-budget migration, raw retrieval and inline truncation warnings, read the [round-two guide](recall-lifecycle.md). diff --git a/docs/engine/recall-evaluation.json b/docs/engine/recall-evaluation.json new file mode 100644 index 0000000..e95e87e --- /dev/null +++ b/docs/engine/recall-evaluation.json @@ -0,0 +1,1675 @@ +{ + "fixtureSha256": "4e1b70b5520e1d3cfb85489aad053825e339938016b20e659f2a4eb4677aee5d", + "node": "v24.19.0", + "interpretation": "Recall@K is mean recall over positive queries, considering only eligible candidates. False recalls include any non-gold eligible ID in the first K; only the first candidate is loaded by recall(). Exact authored aliases are routing rules, not learned multilingual semantics. Held-out topics are disjoint; this small author-created split is not independent validation.", + "splits": { + "development": { + "before": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 0.5, + "missedQueries": 2, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "d-01", + "category": "chinese", + "query": "快取失效規則", + "relevant": [ + "d-cache" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-cache" + ] + }, + { + "id": "d-02", + "category": "chinese", + "query": "登入重試", + "relevant": [ + "d-login" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-login" + ] + }, + { + "id": "d-03", + "category": "chinese", + "query": "更新後快取怎麼辦", + "relevant": [ + "d-cache" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "d-04", + "category": "chinese", + "query": "無法登入怎麼處理", + "relevant": [ + "d-login" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "d-05", + "category": "synonym", + "query": "restore backup", + "relevant": [ + "d-backup" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "d-06", + "category": "synonym", + "query": "night mode", + "relevant": [ + "d-theme" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "d-07", + "category": "cross-language", + "query": "CACHE EVICTION", + "relevant": [ + "d-cache" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "d-08", + "category": "cross-language", + "query": "SIGN IN RETRY", + "relevant": [ + "d-login" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "d-09", + "category": "cross-language", + "query": " recover archived data ", + "relevant": [ + "d-backup" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [] + }, + { + "id": "d-10", + "category": "paraphrase", + "query": "重新整理存下來的線索以免拿到舊版", + "relevant": [ + "d-cache" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-11", + "category": "paraphrase", + "query": "進不去帳號時應該再試幾次", + "relevant": [ + "d-login" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-12", + "category": "paraphrase", + "query": "希望找回昨日保存的內容", + "relevant": [ + "d-backup" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-13", + "category": "synonym", + "query": "dark appearance", + "relevant": [ + "d-theme" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [] + }, + { + "id": "d-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "d-15", + "category": "unrelated", + "query": "do not restore backup", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "d-16", + "category": "unrelated", + "query": "sign in retry price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "d-17", + "category": "stale", + "query": "dretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "d-18", + "category": "stale", + "query": "ddeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + }, + "after-default": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 0.5, + "missedQueries": 2, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "d-01", + "category": "chinese", + "query": "快取失效規則", + "relevant": [ + "d-cache" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-cache" + ] + }, + { + "id": "d-02", + "category": "chinese", + "query": "登入重試", + "relevant": [ + "d-login" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-login" + ] + }, + { + "id": "d-03", + "category": "chinese", + "query": "更新後快取怎麼辦", + "relevant": [ + "d-cache" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "d-04", + "category": "chinese", + "query": "無法登入怎麼處理", + "relevant": [ + "d-login" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "d-05", + "category": "synonym", + "query": "restore backup", + "relevant": [ + "d-backup" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "d-06", + "category": "synonym", + "query": "night mode", + "relevant": [ + "d-theme" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "d-07", + "category": "cross-language", + "query": "CACHE EVICTION", + "relevant": [ + "d-cache" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "d-08", + "category": "cross-language", + "query": "SIGN IN RETRY", + "relevant": [ + "d-login" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "d-09", + "category": "cross-language", + "query": " recover archived data ", + "relevant": [ + "d-backup" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [] + }, + { + "id": "d-10", + "category": "paraphrase", + "query": "重新整理存下來的線索以免拿到舊版", + "relevant": [ + "d-cache" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-11", + "category": "paraphrase", + "query": "進不去帳號時應該再試幾次", + "relevant": [ + "d-login" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-12", + "category": "paraphrase", + "query": "希望找回昨日保存的內容", + "relevant": [ + "d-backup" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-13", + "category": "synonym", + "query": "dark appearance", + "relevant": [ + "d-theme" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [] + }, + { + "id": "d-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "d-15", + "category": "unrelated", + "query": "do not restore backup", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "d-16", + "category": "unrelated", + "query": "sign in retry price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "d-17", + "category": "stale", + "query": "dretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "d-18", + "category": "stale", + "query": "ddeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + }, + "after-aliases": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "d-01", + "category": "chinese", + "query": "快取失效規則", + "relevant": [ + "d-cache" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-cache" + ] + }, + { + "id": "d-02", + "category": "chinese", + "query": "登入重試", + "relevant": [ + "d-login" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-login" + ] + }, + { + "id": "d-03", + "category": "chinese", + "query": "更新後快取怎麼辦", + "relevant": [ + "d-cache" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [ + "d-cache" + ] + }, + { + "id": "d-04", + "category": "chinese", + "query": "無法登入怎麼處理", + "relevant": [ + "d-login" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [ + "d-login" + ] + }, + { + "id": "d-05", + "category": "synonym", + "query": "restore backup", + "relevant": [ + "d-backup" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [ + "d-backup" + ] + }, + { + "id": "d-06", + "category": "synonym", + "query": "night mode", + "relevant": [ + "d-theme" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [ + "d-theme" + ] + }, + { + "id": "d-07", + "category": "cross-language", + "query": "CACHE EVICTION", + "relevant": [ + "d-cache" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [ + "d-cache" + ] + }, + { + "id": "d-08", + "category": "cross-language", + "query": "SIGN IN RETRY", + "relevant": [ + "d-login" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [ + "d-login" + ] + }, + { + "id": "d-09", + "category": "cross-language", + "query": " recover archived data ", + "relevant": [ + "d-backup" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [ + "d-backup" + ] + }, + { + "id": "d-10", + "category": "paraphrase", + "query": "重新整理存下來的線索以免拿到舊版", + "relevant": [ + "d-cache" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-11", + "category": "paraphrase", + "query": "進不去帳號時應該再試幾次", + "relevant": [ + "d-login" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-12", + "category": "paraphrase", + "query": "希望找回昨日保存的內容", + "relevant": [ + "d-backup" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-13", + "category": "synonym", + "query": "dark appearance", + "relevant": [ + "d-theme" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [ + "d-theme" + ] + }, + { + "id": "d-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "d-15", + "category": "unrelated", + "query": "do not restore backup", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "d-16", + "category": "unrelated", + "query": "sign in retry price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "d-17", + "category": "stale", + "query": "dretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "d-18", + "category": "stale", + "query": "ddeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + } + }, + "held-out": { + "before": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 0.5, + "missedQueries": 2, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "h-01", + "category": "chinese", + "query": "發票修正", + "relevant": [ + "h-billing" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-billing" + ] + }, + { + "id": "h-02", + "category": "chinese", + "query": "連線逾時", + "relevant": [ + "h-timeout" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-timeout" + ] + }, + { + "id": "h-03", + "category": "chinese", + "query": "發票寫錯怎麼辦", + "relevant": [ + "h-billing" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "h-04", + "category": "chinese", + "query": "連不上伺服器", + "relevant": [ + "h-timeout" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "h-05", + "category": "synonym", + "query": "download report", + "relevant": [ + "h-export" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "h-06", + "category": "synonym", + "query": "scheduled job timezone", + "relevant": [ + "h-schedule" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "h-07", + "category": "cross-language", + "query": "CORRECT INVOICE", + "relevant": [ + "h-billing" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "h-08", + "category": "cross-language", + "query": "CONNECTION DEADLINE", + "relevant": [ + "h-timeout" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "h-09", + "category": "cross-language", + "query": " export records ", + "relevant": [ + "h-export" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [] + }, + { + "id": "h-10", + "category": "paraphrase", + "query": "帳單的金額填錯要如何更正", + "relevant": [ + "h-billing" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-11", + "category": "paraphrase", + "query": "伺服器等太久了需要重新連線嗎", + "relevant": [ + "h-timeout" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-12", + "category": "paraphrase", + "query": "把表格存成外部檔案需要注意什麼", + "relevant": [ + "h-export" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-13", + "category": "synonym", + "query": "timer zone", + "relevant": [ + "h-schedule" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [] + }, + { + "id": "h-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "h-15", + "category": "unrelated", + "query": "do not download report", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "h-16", + "category": "unrelated", + "query": "connection deadline price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "h-17", + "category": "stale", + "query": "hretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "h-18", + "category": "stale", + "query": "hdeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + }, + "after-default": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 0.5, + "missedQueries": 2, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "h-01", + "category": "chinese", + "query": "發票修正", + "relevant": [ + "h-billing" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-billing" + ] + }, + { + "id": "h-02", + "category": "chinese", + "query": "連線逾時", + "relevant": [ + "h-timeout" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-timeout" + ] + }, + { + "id": "h-03", + "category": "chinese", + "query": "發票寫錯怎麼辦", + "relevant": [ + "h-billing" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "h-04", + "category": "chinese", + "query": "連不上伺服器", + "relevant": [ + "h-timeout" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "h-05", + "category": "synonym", + "query": "download report", + "relevant": [ + "h-export" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "h-06", + "category": "synonym", + "query": "scheduled job timezone", + "relevant": [ + "h-schedule" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "h-07", + "category": "cross-language", + "query": "CORRECT INVOICE", + "relevant": [ + "h-billing" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "h-08", + "category": "cross-language", + "query": "CONNECTION DEADLINE", + "relevant": [ + "h-timeout" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "h-09", + "category": "cross-language", + "query": " export records ", + "relevant": [ + "h-export" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [] + }, + { + "id": "h-10", + "category": "paraphrase", + "query": "帳單的金額填錯要如何更正", + "relevant": [ + "h-billing" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-11", + "category": "paraphrase", + "query": "伺服器等太久了需要重新連線嗎", + "relevant": [ + "h-timeout" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-12", + "category": "paraphrase", + "query": "把表格存成外部檔案需要注意什麼", + "relevant": [ + "h-export" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-13", + "category": "synonym", + "query": "timer zone", + "relevant": [ + "h-schedule" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [] + }, + { + "id": "h-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "h-15", + "category": "unrelated", + "query": "do not download report", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "h-16", + "category": "unrelated", + "query": "connection deadline price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "h-17", + "category": "stale", + "query": "hretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "h-18", + "category": "stale", + "query": "hdeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + }, + "after-aliases": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "h-01", + "category": "chinese", + "query": "發票修正", + "relevant": [ + "h-billing" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-billing" + ] + }, + { + "id": "h-02", + "category": "chinese", + "query": "連線逾時", + "relevant": [ + "h-timeout" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-timeout" + ] + }, + { + "id": "h-03", + "category": "chinese", + "query": "發票寫錯怎麼辦", + "relevant": [ + "h-billing" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [ + "h-billing" + ] + }, + { + "id": "h-04", + "category": "chinese", + "query": "連不上伺服器", + "relevant": [ + "h-timeout" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [ + "h-timeout" + ] + }, + { + "id": "h-05", + "category": "synonym", + "query": "download report", + "relevant": [ + "h-export" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [ + "h-export" + ] + }, + { + "id": "h-06", + "category": "synonym", + "query": "scheduled job timezone", + "relevant": [ + "h-schedule" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [ + "h-schedule" + ] + }, + { + "id": "h-07", + "category": "cross-language", + "query": "CORRECT INVOICE", + "relevant": [ + "h-billing" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [ + "h-billing" + ] + }, + { + "id": "h-08", + "category": "cross-language", + "query": "CONNECTION DEADLINE", + "relevant": [ + "h-timeout" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [ + "h-timeout" + ] + }, + { + "id": "h-09", + "category": "cross-language", + "query": " export records ", + "relevant": [ + "h-export" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [ + "h-export" + ] + }, + { + "id": "h-10", + "category": "paraphrase", + "query": "帳單的金額填錯要如何更正", + "relevant": [ + "h-billing" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-11", + "category": "paraphrase", + "query": "伺服器等太久了需要重新連線嗎", + "relevant": [ + "h-timeout" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-12", + "category": "paraphrase", + "query": "把表格存成外部檔案需要注意什麼", + "relevant": [ + "h-export" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-13", + "category": "synonym", + "query": "timer zone", + "relevant": [ + "h-schedule" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [ + "h-schedule" + ] + }, + { + "id": "h-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "h-15", + "category": "unrelated", + "query": "do not download report", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "h-16", + "category": "unrelated", + "query": "connection deadline price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "h-17", + "category": "stale", + "query": "hretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "h-18", + "category": "stale", + "query": "hdeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + } + } + } +} diff --git a/docs/engine/recall-lifecycle.md b/docs/engine/recall-lifecycle.md new file mode 100644 index 0000000..2591379 --- /dev/null +++ b/docs/engine/recall-lifecycle.md @@ -0,0 +1,189 @@ +# Explicit cues, cache ownership and recall provenance + +Unreleased additions to the optional TypeScript engine. The three-file protocol, +CLI commands and default familiarity thresholds are unchanged. Start with the +[bounded recall guide](recall-efficiency.md) for candidate and output budgets. + +## Opt in to explicit query aliases + +```ts +import { createInMemorySemanticRecallEngine } from "@focaxisdev/deja-vu"; + +const engine = createInMemorySemanticRecallEngine({ + cueAliases: [ + { alias: "更新後快取怎麼辦", cue: "cachepolicy" }, + { alias: "cache eviction", cue: "cachepolicy" }, + ], +}); +await engine.addMemory({ + id: "cache-policy", + title: "Cache policy", + tags: ["cachepolicy"], + content: "Invalidate cached data after an update or deletion.", +}); +const result = await engine.recall("更新後快取怎麼辦"); +console.log(result.topMatch?.id); // cache-policy +``` + +Aliases match the **entire query** after lowercase and whitespace normalization. +They do not match substrings, strip negation, translate text, segment Chinese, +expand unknown synonyms or change thresholds. For example, `do not cache eviction` +does not activate `cache eviction`. If no alias matches, ordinary lexical recall +still runs; it can have its own false positives. Use distinctive existing tags as +cues. Duplicate cues across memories still use the existing candidate/ranking +rules, so an alias is not a guaranteed memory-ID lookup. + +At most 128 entries are accepted. Each alias and cue must be a non-empty string +of at most 256 UTF-16 code units. Conflicting normalized aliases reject at +construction. The engine copies the routing map. No corpus rewrite is required +when aliases point to existing tokens. Adding tags to existing memories requires +an explicit update. Audit aliases when a policy changes: a stale alias can still +route to a live memory if its cue remains indexed. Exact matching deliberately +misses unlisted paraphrases and qualified variants. + +## Opt in to managed embedding ownership + +```ts +import { + CachedEmbeddingProvider, MockEmbeddingProvider, + createInMemorySemanticRecallEngine, +} from "@focaxisdev/deja-vu"; + +const embeddings = new CachedEmbeddingProvider(new MockEmbeddingProvider(256), { + modelKey: "mock:256:v1", + maxEntries: 128, + maxBytes: 8 * 1024 * 1024, +}); +const engine = createInMemorySemanticRecallEngine({ + embeddingProvider: embeddings, + manageEmbeddingCache: true, + preserveDerivedOnMetadataUpdate: true, +}); +await engine.addMemory({ id: "old", title: "Old policy", content: "Synthetic old policy." }); +await engine.updateMemory("old", { metadata: { reviewed: true } }); +await engine.supersedeMemory("old", { + id: "current", title: "Current policy", content: "Synthetic replacement policy.", +}); +await engine.deleteMemory("current"); +console.log(embeddings.getStats()); // { entries: 0, accountedBytes: 0 } +``` + +The mock remains a deterministic testing provider, not a semantic or multilingual +model. Managed ownership requires an embedding provider with `createScope()`; +`CachedEmbeddingProvider` supplies it. A scope belongs to one memory revision. +Repeated text within that scope reuses vectors. Identical text in another memory +or revision has a separate entry: there is no shared-reference deletion ambiguity. +This conservative policy reduces cache hits and can increase provider calls. + +Successful content updates dispose the previous scope only after commit. +Failed preparation/commit disposes the new scope and retains the previous one. +Opt-in metadata-only preservation retains the scope. Successful deletion and +supersession release the old scope; failures preserve ownership for recovery. +A disposed scope rejects new calls, and already-running calls cannot repopulate +it. Those calls can still finish and return their vector: disposal is not +cancellation. Root `clear()` also blocks late insertion across all scopes. + +Managed ownership automatically serializes reads with writes in this engine +instance, including query embedding, so a late recall cannot recreate a deleted +memory's scope. `consistentReads: true` enables the same read ordering without +requiring a cache provider. A read queued before deletion may finish before the +delete completes; reads queued after deletion observe the resulting state. +Slow providers hold up later operations. Do not re-enter the engine from an +adapter, summary generator or embedding callback while this mode is enabled; +that would wait on its own operation. Other engine instances, direct adapter +writes, process crashes and remote storage transactions are outside this lock. + +Cache keys now use per-instance/per-scope keyed SHA-256 digests of exact UTF-16 +input and a length-framed model identifier. Plaintext input is not retained as a +cache key. The configured model key must describe the actual provider/model, +revision, dimensions and options; keep it stable throughout a write and rebuild +stored vectors after changing models. Digest keys do not validate model identity. +The cache uses Node's built-in crypto module and no new package dependency. + +`maxBytes` counts 128 bytes per hex digest plus eight bytes per vector value. +This excludes object/Map overhead, scope objects and their keys. The global +entry limit bounds retained cache entries; the engine also holds one small scope +object per live owned memory. A budget below 136 bytes cannot hold even a +one-value vector. Long inputs can now be cached without retaining long text keys; +hashing still takes time proportional to input length. `getStats()` reports entry +count and these accounted bytes, not heap usage, tokens or provider costs. + +Unmanaged `embed()` calls retain a separate cache namespace. Deleting an engine +memory does not clear those entries, another engine's scopes, an underlying +provider's caches, copies held by callers, storage snapshots or backups. Use +separate wrappers for separate projects. Reference release and keyed digests are +**not verified secure erasure**. Private raw text is still passed to the selected +provider when a calculation is needed. + +## Deletion and supersession recovery + +Deletion now snapshots the memory before removing its routes and data. If an +adapter fails, it attempts to restore summary, chunks, raw content, familiarity +and both vector indexes. `MemoryWriteError` includes a detached recovery snapshot +and any recovery errors. Do not log the snapshot: it contains memory content. +Incomplete existing storage rejects before mutation and needs adapter repair. + +`supersedeMemory(oldId, replacement)` is an explicit operation: the old ID must +exist and the replacement ID must be new. It prepares and saves the replacement, +then removes the old memory. On failure it restores the old memory and attempts +to remove the replacement. `MemorySupersedeError` identifies both IDs and reports +cleanup/recovery errors; both records can remain when compensation fails. The +cause may contain a `MemoryWriteError` recovery snapshot. Inspect and repair +rather than blindly retrying. Nothing is crash-atomic or a distributed transaction. + +The optional engine stores no historical redirects or protocol supersession +frontmatter. Old IDs become unavailable after success. Hosts that need audit +history must retain it separately. The file-first CLI lifecycle is unchanged. + +## Mark incomplete prompt evidence + +```ts +import { projectRecallForPrompt, type RecallResult } from "@focaxisdev/deja-vu"; + +function project(result: RecallResult) { + return projectRecallForPrompt(result, { + maxBytes: 4096, + includeProvenance: true, + }); +} +``` + +With provenance enabled, each record includes `source: "getSummary"` or +`source: "getChunks"`, `rawSource: "getRawContent"`, and its memory/field/chunk +identifiers. Recent updates also include their array index. These are API lookup +instructions for the host, not automatically executed calls. Retrieve a complete +stored summary with `engine.getSummary(memoryId)` or the original stored +`AddMemoryInput.content` with `engine.getRawContent(memoryId)`. + +Every projected summary warns that it is derived and may omit original +constraints. A field clipped by projection, or a summary already clipped by weak +recall, includes `incomplete: true` and a warning that qualifiers may be missing. +Warnings and references count toward the same whole-output byte/token budget. +They reduce room for content. If even an annotated record cannot fit, it is +omitted; `truncated` is true and text can be empty. Check that flag even when the +last included record is complete, because later fields may have been omitted. + +This marks loss rather than preserving sentence meaning. `getSummary()` cannot +restore text already omitted by the summary generator. `getRawContent()` returns +only the originally stored content, not separately supplied context/metadata +fields; place durable constraints in content if they must remain retrievable. +Projection does not fetch raw content, sanitize prompt injection, guarantee +source availability after deletion or reserve budget for the host's wrapper. +Bytes are not tokens; use the host's real tokenizer for a token budget. + +## Migration and validation + +All new routing, ownership, read serialization and provenance options default +to off. Existing result shapes, thresholds, search breadth and three-file formats +remain compatible. The default cache key representation changes to digests, so +small custom byte budgets may now cache fewer entries. The cache is in-memory; +there is no persisted key migration. Deletion now requires a complete snapshot +(or the existing legacy-adapter fallback reads) and reports recovery failures. +Built-in summary/chunk reads and writes now copy arrays and metadata so callers +cannot silently mutate stored text or vectors through returned references. + +See the [round-two validation and measurements](../release-recall-lifecycle.md), +[labeled evaluation](recall-evaluation.json), and +[fixture](../../scripts/fixtures/recall-evaluation.json). General language-aware +segmentation, implicit semantic fallback, ANN, CLI rewrites, cancellation, +transactional persistence and secure erasure remain outside this change. diff --git a/docs/engine/recall-round2-benchmark.json b/docs/engine/recall-round2-benchmark.json new file mode 100644 index 0000000..861faee --- /dev/null +++ b/docs/engine/recall-round2-benchmark.json @@ -0,0 +1,1013 @@ +{ + "baselineCommit": "bad6b527409a44a693562fbe60c41be74126e573", + "notes": [ + "Three complete repetitions retained. No timing-based acceptance threshold. Default engine configuration; owned-cache serialization measured separately by call counts. Median of per-run medians is descriptive, not a confidence interval." + ], + "runs": [ + { + "node": "v24.19.0", + "samples": 25, + "warmups": 5, + "units": "milliseconds", + "notes": [ + "Micro: storage.searchImpressions only. Scan: engine.scanImpressions.", + "API end-to-end: resident recall(loadChunks=true) plus JSON.stringify, including query embedding and chunk retrieval.", + "Excludes process start, index construction, persistence, network, model inference and host/LLM execution.", + "Baseline/current alternate measurement order. Medians/p95 are descriptive of this small synthetic run." + ], + "workloads": [ + { + "n": 100, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.004, + "p95": 0.0248 + }, + "current": { + "median": 0.004, + "p95": 0.0278 + } + }, + "scan": { + "baseline": { + "median": 0.0084, + "p95": 0.0443 + }, + "current": { + "median": 0.0089, + "p95": 0.0433 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0666, + "p95": 0.1077 + }, + "current": { + "median": 0.0793, + "p95": 0.2151 + } + } + }, + { + "n": 100, + "kind": "common", + "micro": { + "baseline": { + "median": 0.02, + "p95": 0.2031 + }, + "current": { + "median": 0.0204, + "p95": 0.1166 + } + }, + "scan": { + "baseline": { + "median": 0.0304, + "p95": 0.1166 + }, + "current": { + "median": 0.0303, + "p95": 0.1513 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0789, + "p95": 0.1246 + }, + "current": { + "median": 0.0793, + "p95": 0.167 + } + } + }, + { + "n": 100, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0006, + "p95": 0.001 + }, + "current": { + "median": 0.0006, + "p95": 0.0007 + } + }, + "scan": { + "baseline": { + "median": 0.0018, + "p95": 0.0023 + }, + "current": { + "median": 0.0021, + "p95": 0.0022 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0032, + "p95": 0.0039 + }, + "current": { + "median": 0.0035, + "p95": 0.0036 + } + } + }, + { + "n": 1000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0032, + "p95": 0.0035 + }, + "current": { + "median": 0.0032, + "p95": 0.0036 + } + }, + "scan": { + "baseline": { + "median": 0.0075, + "p95": 0.008 + }, + "current": { + "median": 0.0077, + "p95": 0.0094 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.06, + "p95": 0.1113 + }, + "current": { + "median": 0.0736, + "p95": 0.0941 + } + } + }, + { + "n": 1000, + "kind": "common", + "micro": { + "baseline": { + "median": 0.2235, + "p95": 0.344 + }, + "current": { + "median": 0.2589, + "p95": 0.3885 + } + }, + "scan": { + "baseline": { + "median": 0.1531, + "p95": 0.3711 + }, + "current": { + "median": 0.2059, + "p95": 0.7824 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.2801, + "p95": 0.395 + }, + "current": { + "median": 0.3552, + "p95": 0.4837 + } + } + }, + { + "n": 1000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0004, + "p95": 0.002 + }, + "current": { + "median": 0.0004, + "p95": 0.0019 + } + }, + "scan": { + "baseline": { + "median": 0.003, + "p95": 0.0034 + }, + "current": { + "median": 0.0033, + "p95": 0.0045 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0046, + "p95": 0.0061 + }, + "current": { + "median": 0.005, + "p95": 0.0065 + } + } + }, + { + "n": 10000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0018, + "p95": 0.0047 + }, + "current": { + "median": 0.0019, + "p95": 0.0077 + } + }, + "scan": { + "baseline": { + "median": 0.0044, + "p95": 0.0065 + }, + "current": { + "median": 0.0044, + "p95": 0.0885 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0477, + "p95": 0.1411 + }, + "current": { + "median": 0.0502, + "p95": 0.1252 + } + } + }, + { + "n": 10000, + "kind": "common", + "micro": { + "baseline": { + "median": 2.2983, + "p95": 3.369 + }, + "current": { + "median": 2.6976, + "p95": 4.1453 + } + }, + "scan": { + "baseline": { + "median": 2.4837, + "p95": 3.8567 + }, + "current": { + "median": 2.7058, + "p95": 3.9417 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 2.5853, + "p95": 4.2817 + }, + "current": { + "median": 2.9508, + "p95": 4.0031 + } + } + }, + { + "n": 10000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0008, + "p95": 0.0015 + }, + "current": { + "median": 0.0008, + "p95": 0.0015 + } + }, + "scan": { + "baseline": { + "median": 0.0029, + "p95": 0.0034 + }, + "current": { + "median": 0.0033, + "p95": 0.037 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0029, + "p95": 0.0037 + }, + "current": { + "median": 0.0032, + "p95": 0.0039 + } + } + } + ], + "operations": { + "baseline": { + "addCalls": 12, + "repeatedRecallCalls": 5, + "metadataUpdateCalls": 12, + "rawRecallBytes": 10253 + }, + "currentOptIn": { + "addCalls": 4, + "repeatedRecallCalls": 1, + "metadataUpdateCalls": 0, + "rawRecallBytes": 10253, + "projectedBytes": 2301 + } + } + }, + { + "node": "v24.19.0", + "samples": 25, + "warmups": 5, + "units": "milliseconds", + "notes": [ + "Micro: storage.searchImpressions only. Scan: engine.scanImpressions.", + "API end-to-end: resident recall(loadChunks=true) plus JSON.stringify, including query embedding and chunk retrieval.", + "Excludes process start, index construction, persistence, network, model inference and host/LLM execution.", + "Baseline/current alternate measurement order. Medians/p95 are descriptive of this small synthetic run." + ], + "workloads": [ + { + "n": 100, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0058, + "p95": 0.0278 + }, + "current": { + "median": 0.0061, + "p95": 0.0157 + } + }, + "scan": { + "baseline": { + "median": 0.0082, + "p95": 0.0504 + }, + "current": { + "median": 0.0086, + "p95": 0.0491 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.1062, + "p95": 0.3244 + }, + "current": { + "median": 0.1098, + "p95": 0.3654 + } + } + }, + { + "n": 100, + "kind": "common", + "micro": { + "baseline": { + "median": 0.024, + "p95": 0.0778 + }, + "current": { + "median": 0.0366, + "p95": 0.1213 + } + }, + "scan": { + "baseline": { + "median": 0.0276, + "p95": 0.0721 + }, + "current": { + "median": 0.0273, + "p95": 0.0925 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0826, + "p95": 0.1748 + }, + "current": { + "median": 0.0817, + "p95": 0.3494 + } + } + }, + { + "n": 100, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0005, + "p95": 0.0008 + }, + "current": { + "median": 0.0005, + "p95": 0.0006 + } + }, + "scan": { + "baseline": { + "median": 0.0019, + "p95": 0.0022 + }, + "current": { + "median": 0.002, + "p95": 0.0022 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0033, + "p95": 0.0108 + }, + "current": { + "median": 0.0033, + "p95": 0.0058 + } + } + }, + { + "n": 1000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0032, + "p95": 0.004 + }, + "current": { + "median": 0.0032, + "p95": 0.0035 + } + }, + "scan": { + "baseline": { + "median": 0.0073, + "p95": 0.0077 + }, + "current": { + "median": 0.0076, + "p95": 0.0093 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0627, + "p95": 0.0932 + }, + "current": { + "median": 0.0726, + "p95": 0.2009 + } + } + }, + { + "n": 1000, + "kind": "common", + "micro": { + "baseline": { + "median": 0.2434, + "p95": 0.5814 + }, + "current": { + "median": 0.377, + "p95": 0.8127 + } + }, + "scan": { + "baseline": { + "median": 0.1672, + "p95": 0.5347 + }, + "current": { + "median": 0.23, + "p95": 1.3395 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.2723, + "p95": 0.3916 + }, + "current": { + "median": 0.3713, + "p95": 0.5661 + } + } + }, + { + "n": 1000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0004, + "p95": 0.002 + }, + "current": { + "median": 0.0004, + "p95": 0.002 + } + }, + "scan": { + "baseline": { + "median": 0.0031, + "p95": 0.0404 + }, + "current": { + "median": 0.0032, + "p95": 0.0046 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0047, + "p95": 0.0367 + }, + "current": { + "median": 0.0047, + "p95": 0.0064 + } + } + }, + { + "n": 10000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0016, + "p95": 0.002 + }, + "current": { + "median": 0.0016, + "p95": 0.002 + } + }, + "scan": { + "baseline": { + "median": 0.004, + "p95": 0.005 + }, + "current": { + "median": 0.0042, + "p95": 0.0048 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0507, + "p95": 0.1416 + }, + "current": { + "median": 0.0535, + "p95": 0.0964 + } + } + }, + { + "n": 10000, + "kind": "common", + "micro": { + "baseline": { + "median": 2.899, + "p95": 4.1866 + }, + "current": { + "median": 2.7897, + "p95": 4.4533 + } + }, + "scan": { + "baseline": { + "median": 2.5471, + "p95": 7.5842 + }, + "current": { + "median": 2.7542, + "p95": 5.178 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 2.5329, + "p95": 3.6354 + }, + "current": { + "median": 2.8307, + "p95": 4.9149 + } + } + }, + { + "n": 10000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0008, + "p95": 0.0011 + }, + "current": { + "median": 0.0008, + "p95": 0.0011 + } + }, + "scan": { + "baseline": { + "median": 0.0018, + "p95": 0.0032 + }, + "current": { + "median": 0.0021, + "p95": 0.004 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0031, + "p95": 0.0049 + }, + "current": { + "median": 0.0033, + "p95": 0.0057 + } + } + } + ], + "operations": { + "baseline": { + "addCalls": 12, + "repeatedRecallCalls": 5, + "metadataUpdateCalls": 12, + "rawRecallBytes": 10253 + }, + "currentOptIn": { + "addCalls": 4, + "repeatedRecallCalls": 1, + "metadataUpdateCalls": 0, + "rawRecallBytes": 10253, + "projectedBytes": 2301 + } + } + }, + { + "node": "v24.19.0", + "samples": 25, + "warmups": 5, + "units": "milliseconds", + "notes": [ + "Micro: storage.searchImpressions only. Scan: engine.scanImpressions.", + "API end-to-end: resident recall(loadChunks=true) plus JSON.stringify, including query embedding and chunk retrieval.", + "Excludes process start, index construction, persistence, network, model inference and host/LLM execution.", + "Baseline/current alternate measurement order. Medians/p95 are descriptive of this small synthetic run." + ], + "workloads": [ + { + "n": 100, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0056, + "p95": 0.0912 + }, + "current": { + "median": 0.0059, + "p95": 0.014 + } + }, + "scan": { + "baseline": { + "median": 0.0083, + "p95": 0.0547 + }, + "current": { + "median": 0.0088, + "p95": 0.0649 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0655, + "p95": 0.1486 + }, + "current": { + "median": 0.0752, + "p95": 0.3647 + } + } + }, + { + "n": 100, + "kind": "common", + "micro": { + "baseline": { + "median": 0.0208, + "p95": 0.0795 + }, + "current": { + "median": 0.021, + "p95": 0.2328 + } + }, + "scan": { + "baseline": { + "median": 0.0293, + "p95": 0.0958 + }, + "current": { + "median": 0.0321, + "p95": 0.0989 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0929, + "p95": 0.2033 + }, + "current": { + "median": 0.0882, + "p95": 0.3483 + } + } + }, + { + "n": 100, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0006, + "p95": 0.0009 + }, + "current": { + "median": 0.0006, + "p95": 0.0008 + } + }, + "scan": { + "baseline": { + "median": 0.0018, + "p95": 0.0024 + }, + "current": { + "median": 0.002, + "p95": 0.0023 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0034, + "p95": 0.0069 + }, + "current": { + "median": 0.0036, + "p95": 0.0054 + } + } + }, + { + "n": 1000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.003, + "p95": 0.0052 + }, + "current": { + "median": 0.0031, + "p95": 0.0647 + } + }, + "scan": { + "baseline": { + "median": 0.0072, + "p95": 0.0096 + }, + "current": { + "median": 0.0074, + "p95": 0.0684 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0612, + "p95": 0.1556 + }, + "current": { + "median": 0.0781, + "p95": 0.2021 + } + } + }, + { + "n": 1000, + "kind": "common", + "micro": { + "baseline": { + "median": 0.25, + "p95": 0.4187 + }, + "current": { + "median": 0.2708, + "p95": 0.4305 + } + }, + "scan": { + "baseline": { + "median": 0.2103, + "p95": 0.8576 + }, + "current": { + "median": 0.2244, + "p95": 1.0381 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.2708, + "p95": 0.3856 + }, + "current": { + "median": 0.3494, + "p95": 0.5329 + } + } + }, + { + "n": 1000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0004, + "p95": 0.002 + }, + "current": { + "median": 0.0004, + "p95": 0.0019 + } + }, + "scan": { + "baseline": { + "median": 0.003, + "p95": 0.0048 + }, + "current": { + "median": 0.0032, + "p95": 0.0044 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0046, + "p95": 0.0063 + }, + "current": { + "median": 0.0049, + "p95": 0.0063 + } + } + }, + { + "n": 10000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0025, + "p95": 0.0033 + }, + "current": { + "median": 0.0026, + "p95": 0.0031 + } + }, + "scan": { + "baseline": { + "median": 0.0039, + "p95": 0.0044 + }, + "current": { + "median": 0.0042, + "p95": 0.0046 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0464, + "p95": 0.1183 + }, + "current": { + "median": 0.0503, + "p95": 0.0763 + } + } + }, + { + "n": 10000, + "kind": "common", + "micro": { + "baseline": { + "median": 2.3091, + "p95": 4.9869 + }, + "current": { + "median": 2.5937, + "p95": 3.4029 + } + }, + "scan": { + "baseline": { + "median": 2.4826, + "p95": 5.0033 + }, + "current": { + "median": 2.7656, + "p95": 3.8497 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 2.6475, + "p95": 4.7129 + }, + "current": { + "median": 2.9064, + "p95": 3.605 + } + } + }, + { + "n": 10000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0007, + "p95": 0.001 + }, + "current": { + "median": 0.0007, + "p95": 0.0011 + } + }, + "scan": { + "baseline": { + "median": 0.0017, + "p95": 0.0024 + }, + "current": { + "median": 0.0021, + "p95": 0.0285 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0029, + "p95": 0.0031 + }, + "current": { + "median": 0.0031, + "p95": 0.0033 + } + } + } + ], + "operations": { + "baseline": { + "addCalls": 12, + "repeatedRecallCalls": 5, + "metadataUpdateCalls": 12, + "rawRecallBytes": 10253 + }, + "currentOptIn": { + "addCalls": 4, + "repeatedRecallCalls": 1, + "metadataUpdateCalls": 0, + "rawRecallBytes": 10253, + "projectedBytes": 2301 + } + } + } + ] +} diff --git a/docs/engine/semantic-engine.md b/docs/engine/semantic-engine.md index 0e6315f..42ceb86 100644 --- a/docs/engine/semantic-engine.md +++ b/docs/engine/semantic-engine.md @@ -50,9 +50,10 @@ await engine.deleteMemory(id); ## Write failure contract -Adds and updates prepare summaries, chunks, and all embeddings before changing -storage. Updates retain importance, tags, and creation time unless explicitly -changed where supported. Adding an existing ID is rejected; use `updateMemory`. +Adds and normal updates prepare summaries, chunks, and all embeddings before changing +storage. Opt-in metadata-only updates may preserve those derived values; see +[bounded recall and reuse](recall-efficiency.md). Updates retain importance, tags, +and creation time unless explicitly changed where supported. Adding an existing ID is rejected; use `updateMemory`. Mutations are serialized within one engine instance. If a storage or vector write fails, the engine attempts compensating cleanup @@ -90,7 +91,10 @@ If you adopt the engine, the host still owns: ## Related references +- [Bounded recall, opt-in reuse and migration](recall-efficiency.md) - [docs/agent-handshake.md](../agent-handshake.md) - [docs/engine/protocol-to-engine.md](./protocol-to-engine.md) - [src/types/plugins.ts](../../src/types/plugins.ts) - [src/types/memory.ts](../../src/types/memory.ts) + +See [explicit cues, lifecycle recovery and provenance](recall-lifecycle.md) for the additional opt-ins and their concurrency/migration limits. diff --git a/docs/release-recall-efficiency.md b/docs/release-recall-efficiency.md new file mode 100644 index 0000000..d5b4449 --- /dev/null +++ b/docs/release-recall-efficiency.md @@ -0,0 +1,117 @@ +# Unreleased: more reliable, bounded engine recall + +Local release-note draft only. No new package version, release date, tag or +publication is assigned by this change. + +These measurements and validation counts describe the first implementation batch +at `bad6b527409a44a693562fbe60c41be74126e573`. See the +[round-two notes](release-recall-lifecycle.md) for current cache ownership, +provenance, quality evaluation, performance costs and installed-package results. + +## User introduction + +Deja Vu's optional engine can now find an eligible memory even when a recent, +less relevant candidate has a higher hybrid score. You can search more candidates +without expanding the returned list, project recalled text into an explicit +prompt budget, and opt in to bounded embedding reuse. Common-cue searches also +avoid rebuilding each record's token set on every request. + +The three-file memory protocol stays the same. No file migration, database, +paid embedding service or daemon is needed. See the +[usage and migration guide](engine/recall-efficiency.md) for opt-in examples. + +## What changes + +- Eligible candidates precede ineligible hybrid winners. Hybrid preference among + eligible candidates is retained; semantic thresholds are unchanged. +- `candidateLimit` separates search breadth from `topK`, with previous breadth + retained by default. Invalid numeric limits now reject instead of producing + surprising array slices or excessive work. +- `projectRecallForPrompt()` provides allowlisted JSONL with an exact UTF-8 byte + cap and optional host-tokenizer budget. Raw API results remain compatible. +- The in-memory lexical scorer caches normalized record token sets, replaces them + on updates, and removes them on deletion. Returned familiarity records are copies. +- `preserveDerivedOnMetadataUpdate` skips regeneration for metadata/importance-only + edits when explicitly enabled. Custom-generator behavior remains the default. +- `CachedEmbeddingProvider` offers opt-in content/model-keyed LRU retention with + entry and accounted-storage bounds, explicit clearing, and defensive vector copies. + +## Measured effects + +Baseline: repository main at `188d7e8c3aed224e45071d041cd140509887b29f`. +Both versions were compiled with existing dependencies and measured in the same +Linux x64 / Node v24.19.0 process, alternating measurement order. Each case used +five warmups and 25 samples. Fixtures contain the same five routing terms per +record used in the preceding audit; detail fixtures add ten synthetic chunks per +record. No real user memory, network model or paid API was used. + +| Synthetic common-cue workload | Baseline median | Current median | +| --- | ---: | ---: | +| 100 records, storage search microbenchmark | 0.2820 ms | 0.0236 ms | +| 1,000 records, storage search microbenchmark | 3.1107 ms | 0.1643 ms | +| 10,000 records, storage search microbenchmark | 30.7427 ms | 2.6053 ms | +| 10,000 records, resident API end-to-end | 30.8083 ms | 2.7437 ms | + +The last row covers `recall(loadChunks=true)` plus JSON serialization, including +mock query embedding, summary/chunk retrieval and access-time updates. It is +**not** end-to-end agent or product latency: startup, index construction, +persistence, external model inference, host prompt processing and LLM work are +excluded. API timing tests use default options, not the embedding cache opt-in. +Rare-cue and no-match costs were already small; some smaller cases became slightly +slower. These measurements do not establish an application-wide speedup or recall +quality improvement. Full samples' median/p95 summaries are in the +[measurement record](engine/recall-benchmark.json). + +In a separate ten-chunk repeated-content fixture, enabling the embedding cache +reduced actual mock-provider calls on add from 12 to 4 and on five repeated detail +recalls from 5 to 1. With metadata preservation also enabled, an importance-only +update used zero embedding calls instead of 12. Counts depend on repeated content +and cache capacity; they are not measured paid-service cost savings. + +The raw recall payload stayed at 10,253 bytes. Its optional prompt projection was +2,301 UTF-8 bytes in this fixture, about 77.6% smaller, excluding vectors and other +non-projected fields. **Bytes are not model tokens.** This is a representation +comparison, not proof of equal task quality; omitted fields may matter to a host. + +## Validation and limitations + +- TypeScript no-emit validation and compilation passed with existing dependencies. +- All 31 source engine/regression tests passed in Node's supported in-process test mode. +- All 31 compiled engine/regression tests and 26 existing compiled engine + rollback/concurrency tests passed (57 distinct tests, with the first 31 also + exercised from source). +- The standard source test command failed under this Cloud environment's process + isolation. An isolated existing CLI test confirmed `spawnSync EPERM`. + At that point, full CLI/process and installed-package verification still needed an environment + that permits those subprocesses; this draft does not claim full-suite success. +- No new dependency was installed. Repository-wide memory scans and link checks + that read private memory were excluded; changed documentation links were checked + separately. Full `npm test` and CI remain required before a future release. + +Regression coverage includes the false-none counterexample, lexical prefilter +truncation, invalid and zero limits, multilingual score/tie equivalence, Unicode +prompt clipping, update/delete invalidation, opt-in rollback, model changes, +cache eviction and pending-request clearing. Threshold quality, semantic synonym +recall, real tokenizer costs, concurrent production traffic and cold starts were +not measured. Prompt projection remains untrusted memory data, not an instruction +sanitizer. Read the migration guide before enabling metadata preservation or +sharing embedding caches. + +CLI optimizations, language-aware routing, semantic fallback, ANN and persistent +storage changes are deferred. This draft is not a release announcement or a +claim that those features are implemented. + +## Reproduce the synthetic comparison + +Compile the baseline source and current source into separate directories with +the existing TypeScript compiler. Then run: + +```bash +node scripts/benchmark-recall.mjs /path/to/baseline/dist/src/index.js /path/to/current/dist/src/index.js +``` + +Use compiled modules so TypeScript loader overhead is outside the measurement. +The script asserts unchanged candidate IDs/scores for its performance fixtures +and writes JSON to stdout. It neither installs packages nor touches memory files. +Use the regression tests, rather than these speed fixtures, to check changed +eligibility behavior. diff --git a/docs/release-recall-lifecycle.md b/docs/release-recall-lifecycle.md new file mode 100644 index 0000000..0bf5db9 --- /dev/null +++ b/docs/release-recall-lifecycle.md @@ -0,0 +1,161 @@ +# Unreleased: explicit recall cues and recoverable memory lifecycle + +Local release-note draft. No new version, release date, tag, release or social +publication is assigned. This follows the [first recall-efficiency changes](release-recall-efficiency.md) +and uses `bad6b527409a44a693562fbe60c41be74126e573` as its comparison baseline. + +These results describe the second implementation batch. The subsequent +[verification-loop checkpoint](verification-loop.md) fixes test-helper diagnostic +loss and records current validation plus the unresolved native-execution blocker. + +## User introduction + +The optional engine now supports explicit aliases for familiar wording, managed +embedding-cache ownership, recoverable deletion and supersession, and prompt +excerpts that say when constraints may be missing. Enable only what your host +needs using the [examples and migration guide](engine/recall-lifecycle.md). + +The three-file protocol remains unchanged. These additions do not provide an +automatic Chinese segmenter, a multilingual semantic model, a vector database or +a paid service. The CLI has no implementation changes in this batch. + +## Labeled retrieval evaluation + +The [fixture](../scripts/fixtures/recall-evaluation.json) contains 36 synthetic +queries: 18 development and 18 held-out, each with 13 positive and five negative +queries. Categories cover Chinese, synonyms, paraphrases, explicit bilingual +aliases, unrelated requests and deleted/superseded memories. Topics are disjoint +between splits. Corpus aliases and both query splits were frozen before the first +evaluation; development results were inspected first and rules were not tuned +after held-out scoring. Cases and aliases share an author, so this is not an +independent or representative language benchmark. No private memories were used. + +| Split | Baseline / new default Recall@1 | Aliases Recall@1 | Missed positive queries, before → aliases | False-recall queries, before → aliases | +| --- | ---: | ---: | ---: | ---: | +| Development | 2/13 (15.4%) | 10/13 (76.9%) | 11 → 3 | 0 → 0 | +| Held-out | 2/13 (15.4%) | 10/13 (76.9%) | 11 → 3 | 0 → 0 | + +Recall@3 and Recall@5 are identical here: widening K cannot recover a query with +no eligible lexical candidate. Recall is averaged over positive queries. A false +recall means a non-gold eligible ID in the first K candidates; only the first +candidate is actually loaded by recall. Negative-query false positives were 0/5 +in each split and mode. Zero observed errors in this small set is not a guarantee. +Per-category results and exact returned IDs are in the +[evaluation record](engine/recall-evaluation.json). + +All six unlisted paraphrases still miss. Improvement comes from exact, +host-authored aliases, including bilingual pairs, not learned translation or +Chinese segmentation. No threshold was lowered. Broad substring expansion and +automatic segmentation were deferred because these tests do not establish a +safe precision/recall tradeoff for them. Stale cases test successful removal; +separate fault-injection tests cover failed lifecycle operations. + +## Performance and ownership costs + +The same synthetic performance script used in the first round compares the +previous commit with this implementation. Three complete repetitions are retained +in the [measurement record](engine/recall-round2-benchmark.json). Each uses five +warmups and 25 samples, alternating baseline/current order, in Node v24.19.0. +Numbers below are medians of the three per-run medians, in milliseconds. + +| Common-cue workload | Previous | Current | +| --- | ---: | ---: | +| 100 records, storage-search microbenchmark | 0.0208 | 0.0210 | +| 1,000 records, storage-search microbenchmark | 0.2434 | 0.2708 | +| 10,000 records, storage-search microbenchmark | 2.3091 | 2.6976 | +| 100 records, resident API end-to-end | 0.0826 | 0.0817 | +| 1,000 records, resident API end-to-end | 0.2723 | 0.3552 | +| 10,000 records, resident API end-to-end | 2.5853 | 2.9064 | + +This round does **not** establish a speedup. The 10,000-record resident API median +rose about 12.4%. Defensive summary/chunk copies add allocation, but timing alone +does not isolate the causes; storage-search logic itself is unchanged. Noise, +garbage collection and memory pressure remain confounders. No timing threshold +was used to select a favorable run. These default-mode measurements exclude +owned-cache serialization and concurrent traffic. + +The resident API measurement covers recall with mock query embedding, summary +and chunk reads, access-time updates and JSON serialization. It excludes startup, +ingestion/index construction, persistence, external model inference, prompt +processing and LLM work. It is not whole-product or agent end-to-end latency. +The first round's main-to-baseline acceleration remains historical evidence, +not a new speedup attributable to this batch. + +A separate [ownership fixture](engine/recall-cache-ownership-benchmark.json) uses +16 memories with repeated content and ten identical chunks each: + +| Actual provider calls | Previous unscoped cache | Current unscoped cache | Current owned cache | +| --- | ---: | ---: | ---: | +| Add all memories | 18 | 18 | 48 | +| Two complete detail-recall passes | 16 | 16 | 16 | +| Metadata-only update, preservation enabled | 0 | 0 | 0 | +| Content update | 2 | 2 | 3 | + +After deleting all memories, current unscoped mode retains 36 entries (5,184 +accounted bytes); owned mode retains zero. Baseline has no public statistics API. +Separate ownership trades cross-memory reuse for predictable invalidation. Counts +are deterministic mock calls, not paid-service costs, tokens or throughput. + +## Validation and blockers + +- TypeScript checking and compilation pass with existing dependencies. +- Source and compiled engine regression suites pass: 50 distinct cases, including + aliases, scope isolation, late completion, model/content keys, lifecycle + compensation, queued reads, mutable-result isolation and qualifier warnings. +- The complete compiled test selection was executed in Node's supported + in-process mode: **77 passed, 32 failed, 0 skipped**. Failures are in CLI/process + scenarios: 27 preserve EPERM evidence, while five lose the original cause in + the error-response helper and report JSON parsing errors. Those five cannot + be conclusively attributed to EPERM from the saved log alone. + The existing engine reliability subset passes all 26 cases; one additional + initialization rollback test also passes in the full selection. +- The standard isolated source and package test commands fail in this Cloud. + A dependency-free `spawnSync(process.execPath, ["-e", "process.stdout.write('ok')"])` + reproduces `EPERM`; the returned `status: 0` does not override the error. + The isolated package-test diagnostic fails at its `npm pack` child call. + Node's permission API is absent; seccomp filtering and `NoNewPrivs` are active. + The exact denied syscall or interception layer is **not established**. +- Direct `npm pack --offline` succeeds in the same Cloud, including unchanged + prepack clean/build. Direct offline installation of that tarball and installed + CLI/API smoke checks pass. This validates packaging separately; it does not + turn the failing automated package test into a pass. No restriction, test, + lifecycle script or assertion was disabled to make the runner pass. +- Full `npm test` is not complete. Its repository-wide Markdown scan would read + private memory, so it was not run; public documentation links were checked + separately. No private memory, network service or new dependency was used. + +The failed subprocess scenarios still need an authorized subprocess-capable Cloud +configuration before release. No alternate environment was used. All files are +local preparation; this is not a release-readiness claim. + +## Remaining limits + +Managed ownership serializes this engine's reads and writes; slow providers can +block later calls, and plugins must not re-enter the engine during those calls. +It does not coordinate other engines or direct adapter writes. Compensation can +fail and leave both supersession IDs present; errors expose recovery information. +There are no crash-safe transactions, historical redirects or verified secure +erasure. External providers and caller-held copies may retain data. + +Provenance warnings consume prompt budget and may force an entire field to be +omitted. They mark possible loss; they do not preserve semantic qualifiers or +sanitize instructions. Raw lookup returns stored content, not original separate +context fields. Bytes are not tokens. Real tokenizer/model quality, high-load +queueing, heap bounds and production traffic remain unmeasured. CLI optimization, +ANN, semantic fallback and automatic Chinese segmentation are deferred. + +## Reproduce + +Compile the baseline and current source with the existing compiler into separate +directories. The following scripts use only synthetic data and local modules: + +```bash +node scripts/evaluate-recall.mjs /path/to/before/src/index.js /path/to/after/src/index.js development +node scripts/evaluate-recall.mjs /path/to/before/src/index.js /path/to/after/src/index.js held-out +node scripts/benchmark-recall.mjs /path/to/before/src/index.js /path/to/after/src/index.js +node scripts/benchmark-cache-ownership.mjs /path/to/before/src/index.js /path/to/after/src/index.js +``` + +The evaluation fixture is packaged with its script. Results include the fixture's +SHA-256 digest. Timing scripts assert comparable candidate IDs/scores; quality +and failure behavior belong to the labeled evaluation and regression tests. diff --git a/docs/social-recall-lifecycle-draft.md b/docs/social-recall-lifecycle-draft.md new file mode 100644 index 0000000..1da8f6e --- /dev/null +++ b/docs/social-recall-lifecycle-draft.md @@ -0,0 +1,24 @@ +# Local social draft: recall cues and lifecycle + +Unpublished draft only. No release date or version is assigned. Do not use this +copy as a release announcement until the remaining verification blockers are +resolved. See the [evidence and limitations](release-recall-lifecycle.md). + +## Short introduction + +We're preparing optional-engine improvements for Deja Vu: explicit query aliases, +embedding-cache ownership that follows memory updates and deletions, recoverable +supersession, and prompt excerpts that warn when constraints may be missing. +The three-file memory protocol stays the same. + +## Technical follow-up + +In a small synthetic held-out set, authored aliases raised Recall@1 from 2/13 to +10/13, with no false recalls observed. Unlisted paraphrases still missed. This is +explicit routing, not a new multilingual semantic model. + +The tradeoffs matter: separate cache ownership means less cross-memory reuse, +and this round did not improve measured latency. Direct offline packaging and +installed smoke checks pass, while subprocess-based tests still report EPERM in +the current Cloud. These changes remain local and unreleased. Usage, migration, +full measurements and remaining limits are in the release-note draft. diff --git a/docs/verification-loop-evidence.json b/docs/verification-loop-evidence.json new file mode 100644 index 0000000..e7e5de1 --- /dev/null +++ b/docs/verification-loop-evidence.json @@ -0,0 +1,95 @@ +{ + "baselineCommit": "fa479335dbc5acbad46a352d1ee8bd6086c62cd7", + "scope": "Same Cloud, public-source baseline snapshot; private memory excluded; existing dependencies linked, no alternate runtime.", + "runtime": { + "node": "v24.19.0", + "libuv": "1.52.1", + "permissionAPI": "undefined", + "NODE_OPTIONS_present": false, + "LD_PRELOAD_present": false, + "NoNewPrivs": 1, + "Seccomp": 2, + "Seccomp_filters": 1, + "CapEff": "0000000000000000" + }, + "historicalCompleteCompiledRun": { + "commit": "fa479335dbc5acbad46a352d1ee8bd6086c62cd7", + "log": "/tmp/dejavu-round2/full-compiled.log", + "tests": 109, + "pass": 77, + "fail": 32, + "directEperm": 9, + "assertionOnEperm": 18, + "originalCauseMaskedByJsonParse": 5, + "note": "Historical result, not a rerun of this revision. Five original error codes remain unknown." + }, + "diagnosticTrace": { + "exitCode": 1, + "stderr": [ + "strace: test_ptrace_get_syscall_info: PTRACE_TRACEME: Operation not permitted", + "strace: ptrace(PTRACE_TRACEME, ...): Operation not permitted", + "strace: PTRACE_SETOPTIONS: Operation not permitted", + "strace: cleanup: waitpid(-1, __WALL): No child processes" + ], + "probeOutputBytes": 0, + "traceBytes": 0, + "interpretation": "Tracing itself is denied. No successful syscall trace or new Node probe result was obtained. No retry, fallback tracer or security change.", + "argv": [ + "strace", + "-f", + "-qq", + "-e", + "trace=process,epoll_ctl,epoll_wait,epoll_pwait,epoll_pwait2,io_uring_setup,io_uring_enter,pipe2,socketpair,dup2,dup3,fcntl", + "-o", + "/tmp/dejavu-round3/spawn-sync.trace", + "node", + "-e", + "const r=require(\"node:child_process\").spawnSync(process.execPath,[\"-e\",\"process.stdout.write(\\\"synthetic-ok\\\")\"],{encoding:\"utf8\"});console.log(JSON.stringify({error:r.error?.code,syscall:r.error?.syscall,status:r.status,signal:r.signal,stdout:r.stdout,stderr:r.stderr,pid:r.pid}));" + ] + }, + "nativeBoundary": "Installed internal/child_process.spawnSync calls spawn_sync.spawn and wraps returned result.error in ErrnoException. This narrows the error boundary, not the underlying syscall or policy decision.", + "freshSubsetResults": { + "baseline-source-engine.log": { + "tests": 50, + "pass": 50, + "fail": 0, + "skipped": 0 + }, + "baseline-compiled-engine.log": { + "tests": 50, + "pass": 50, + "fail": 0, + "skipped": 0 + }, + "current-source-engine-harness.log": { + "tests": 60, + "pass": 60, + "fail": 0, + "skipped": 0 + }, + "current-compiled-engine-harness.log": { + "tests": 60, + "pass": 60, + "fail": 0, + "skipped": 0 + } + }, + "gates": { + "completeSourceSuite": "BLOCKED_NOT_RERUN: known subprocess failures; no bypass or skipped-test success claim", + "completeCompiledSuite": "BLOCKED_NOT_RERUN: known subprocess failures; partial subsets do not satisfy this gate", + "automatedPackageTest": "BLOCKED_NOT_RERUN: execFileSync pack call previously failed; direct checks are separate", + "baselineAndCurrentTypecheckAndBuild": "PASS", + "baselineAndCurrentDirectOfflinePackInstallUse": "PASS: unchanged prepack lifecycle, local tarball install, help/init/doctor/lint/scan/report and old/new engine API", + "productAssertions": "PASS: 70 memory-cli, 80 reliability and 27 package-pack assertions unchanged; infrastructure undefined guard replaced by equivalent strict rethrow", + "qualityAndCache": "PASS: frozen evaluation JSON unchanged; current cache lifecycle counts unchanged; baseline/current unscoped counts equal", + "privateMemoryAndCredentials": "NOT_USED", + "publicDocumentation": "PASS: unmodified checker on public snapshots; baseline 67 Markdown files, current 68. Private-memory documentation is outside this evidence.", + "originalAgentsPrefix": "PASS: original instructions retained byte-for-byte", + "helperBaselineComparison": "PASS: exact baseline helper loses synthetic EPERM cause; repaired helpers retain original identity without subprocess calls" + }, + "fixtureSha256": "4e1b70b5520e1d3cfb85489aad053825e339938016b20e659f2a4eb4677aee5d", + "checkpointDirectory": "/tmp/dejavu-round3", + "requiredResource": "Environment-owner-provided denied-syscall audit/trace for the synthetic probe, or narrowly approved tracing capability for that probe; then a confirmed policy-compliant runtime/execution configuration for complete subprocess tests.", + "unresolved": "Native spawn_sync EPERM has not been proven to be a Node/libuv bug or a specific sandbox rule. Ptrace denial is independently confirmed. No alternative spawn API/runtime/security option was tried.", + "unchangedEngineSourceTreeSha256": "b63b7595c48900bf94e59f4ceebff4260d22bee420b9fe40d01c3b414a11fc4a" +} diff --git a/docs/verification-loop.md b/docs/verification-loop.md new file mode 100644 index 0000000..0e55770 --- /dev/null +++ b/docs/verification-loop.md @@ -0,0 +1,153 @@ +# Verification loop: harness diagnostics and Cloud execution + +Unreleased engineering checkpoint. Starting commit: +`fa479335dbc5acbad46a352d1ee8bd6086c62cd7`, on the existing local fix branch. + +## Scope and acceptance, defined before changes + +This iteration fixes loss of subprocess error evidence in test helpers, adds +regression coverage, and appends general verification-loop rules to AGENTS.md. +Engine behavior, thresholds, public protocol, dependencies and runtime security +configuration are outside the planned changes unless a reproducible regression +requires a focused correction. Preserve previous commits and unrelated edits. + +| Gate | Required evidence | +| --- | --- | +| Source suite | All existing cases plus new harness cases run; file-level summaries are not case counts. No skips or weakened assertions. | +| Compiled suite | The same complete selection runs after a fresh build. Reconcile each failure, including original cause, with the source selection. | +| Package/install/use | Unchanged prepack clean/build, package file assertions, offline installation of the local tarball, installed CLI and engine API assertions. | +| Types | TypeScript no-emit checking and compilation with existing dependencies. | +| Regression | Existing engine, CLI and recovery assertions retained; helper cases cover operational failures, expected nonzero exits, invalid JSON, signals and diagnostic cause preservation. | +| Recall quality | The frozen development/held-out fixture yields the previously recorded IDs and metrics; no tuning to held-out labels. | +| Cache/projection | Ownership, model/content keys, pending invalidation, update/delete/supersede recovery and projection budget/provenance cases remain passing. | +| Documentation | Claims distinguish tested results, inference and blocked gates; new links resolve; original AGENTS instructions remain an exact prefix. | +| Completion | All applicable gates pass, or an external blocker has exact evidence, a reproducible checkpoint and the required resource/permission. A blocked gate is never a pass. | + +Use only synthetic fixtures and existing dependencies. Do not read private +project memory. If a full-tree documentation check is needed, use a public-source +snapshot with only public/synthetic fixtures and describe that scope explicitly. +Do not disable tests, change security controls, drop inherited guard variables, +install another runtime, use paid services, or publish anything. + +## Loop protocol + +For each failure, record the command, baseline/current revision, environment, +exit/error evidence, hypothesis and next discriminating check. Fix the smallest +supported cause, rerun affected regressions, then the full acceptance gates. +After two iterations without new evidence or improvement, change hypothesis or +strategy; do not repeat an unchanged denied operation. A diagnostic trace is for +observation only and must stop if tracing itself is denied. Any runtime option +must be established as ordinary, authorized execution rather than an evasion of +policy before it is used. Preserve failures and blockers in the final record. + +## Initial evidence and hypotheses + +The preceding compiled run had 109 cases: 77 passed and 32 failed. Of the failures, +27 retain explicit EPERM evidence; five lose their original cause when a helper +parses empty stdout. The two affected test files and their CLI dependencies are +unchanged from the preceding baseline. Existing direct offline packaging and +installed smoke checks passed in the same Cloud. These observations do not yet +identify the syscall or prove a Node bug versus a security-policy denial. + +Hypotheses to distinguish: helper-only diagnostic loss; Node/libuv synchronous +child-I/O failure; test-runner-specific behavior; or an enforced environment +restriction. Fix diagnostic loss first. Compare unchanged baseline code without +weakening it, and use a bounded observation only where it adds evidence. + +## Changes and observed results + +The error-response helper now accepts only ordinary nonzero exits with string +stdout and no operational error/code/signal. EPERM, EACCES, ENOENT, timeouts, +buffer failures, signals and invalid execution results preserve the original +thrown object. Invalid JSON from an ordinary nonzero exit becomes +`CliResponseError`, with the execution error as `cause` and the parse error +available separately. The message does not copy arbitrary stdout/stderr. + +The reliability helper now rethrows `result.error` directly, retaining the same +strict `!== undefined` guard. It no longer obscures an operational error behind +`ERR_ASSERTION`. AST comparison confirms that all 70 memory-cli, 80 reliability +and 27 package-pack product assertions are unchanged. The package test now uses +npm's ordinary `--offline` option for pack/install; this restricts network use +and does not disable prepack or any assertion. Ten new helper cases cover these +boundaries, including a misleading `status: 0` accompanying EPERM. + +The original AGENTS.md is an exact byte-for-byte prefix of the updated file. +Only general verification-loop rules were appended. Engine implementation, +thresholds, frozen fixtures and dependency lockfile are unchanged. + +| Acceptance gate | This iteration's result | +| --- | --- | +| Complete source suite | **BLOCKED / not rerun** after known subprocess failures. The new revision has not passed this gate. | +| Complete compiled suite | **BLOCKED / not rerun** for the same reason. Do not reuse the historical 77/32 as a new result. | +| Automated package test | **BLOCKED / not rerun**; its execFileSync path previously reported EPERM. | +| Unaffected source subset | Baseline 50/50; current 60/60 (same 50 engine cases plus ten helper cases). This is not the complete suite. | +| Unaffected compiled subset | Baseline 50/50; current 60/60. These repeat the source cases, not additional distinct cases. | +| Type checking/build | Both baseline and current passed with existing dependencies. | +| Direct offline package/install/use | Both baseline and current passed unchanged prepack clean/build, tarball installation, installed metadata, help/init/doctor/lint/scan/report, and old/new engine API smoke assertions. Separate evidence, not an automated package-test pass. | +| Recall quality | Exact evaluation JSON matches the frozen record, including returned IDs and development/held-out metrics. | +| Cache/projection | Existing lifecycle/budget/provenance cases pass within the 50 engine cases. Provider-call and retained-entry measurements match the prior record. | +| Public documentation | Unmodified checker passes on public-source snapshots; private memory is excluded from the snapshot, not silently treated as verified. | +| Completion | **External diagnostic/runtime blocker checkpoint; full acceptance is not green.** | + +The complete selection contains the previous 109 cases plus ten new helper cases; +the automated package case is separate. This is an expected inventory, not a new +119-case execution result. The earlier full compiled run had 77 passes and 32 +failures: nine direct EPERM, 18 infrastructure assertions containing EPERM, and +five JSON parse failures with lost original causes. The five historical causes +cannot be recovered by fixing the helper now. + +## Evidence loop and stopping boundary + +1. **Hypothesis: helper parsing hides operational errors.** The unchanged baseline + helper reproduces the loss with synthetic errors in one process. The focused + fix preserves error identity and expected CLI JSON behavior; ten regressions + pass. No product assertion was weakened. +2. **Hypothesis: the native error boundary can be distinguished with a syscall + observation.** One bounded trace attempt failed before yielding a Node probe + result: `PTRACE_TRACEME` and `PTRACE_SETOPTIONS` returned `Operation not + permitted`. The trace and probe-output files are empty. Tracing was not retried + or replaced with another mechanism, and no policy was changed. +3. **Changed strategy: unchanged public-source baseline and safe observations.** + The same Node v24.19.0 / libuv 1.52.1 runtime passes pure engine tests and direct + offline pack/install/use for both versions. Its built-in JavaScript wrapper + shows the error comes back from `spawn_sync.spawn`, before the helper or test + runner handles it. Existing dependency-free reproduction outside the test + runner means the runner alone does not explain the failure. This does not + identify the failing native syscall or prove Node/libuv is defective. + +`Seccomp: 2`, `NoNewPrivs: 1`, and zero effective capabilities are observed. +Node's permission API is absent; NODE_OPTIONS and LD_PRELOAD are not present. +None of these observations proves which rule caused the original spawnSync +error. Direct prepack children succeeding also means it would be inaccurate to +claim all child creation is forbidden. The **tracing restriction is confirmed**; +Node/libuv versus a specific spawn/I/O policy interaction remains unresolved. + +No alternative spawn API, alternate runtime, inherited-guard removal, security +setting change or same-denial retry was used. In-process test selection runs only +unaffected cases and remains explicitly partial. It is not a way to count the +blocked CLI scenarios as passed. Two iterations without new evidence must change +strategy; once further discrimination requires unavailable privileges, the loop +stops at this checkpoint instead of manufacturing a green result. + +## Checkpoint and required resource + +The compact [evidence record](verification-loop-evidence.json) preserves counts, +trace denial, runtime observations, fixture digest and gate states. Detailed logs, +public baseline/current snapshots, local tarballs and synthetic installed projects +are under `/tmp/dejavu-round3` in the current Cloud. Temporary files may not survive +environment replacement; the committed evidence and source remain reproducible. + +An environment maintainer must provide either a denied-syscall audit/trace for the +synthetic spawnSync probe, or narrowly approved tracing capability for that probe. +That evidence is needed to identify the native error and choose a confirmed, +policy-compliant runtime/execution configuration. This is **not** a request to +turn off seccomp or broadly allow process access. No repository change can grant +that observation capability, and a different runtime has not been assumed safe. + +Once that prerequisite is satisfied, run the unchanged baseline and repaired +revision under the approved configuration. Capture the original error for any +failure, rerun the affected regression, and then run full `npm test` on the public +source snapshot. It retains every source/compiled/pack assertion and includes the +new helper cases. Compare the frozen evaluation and cache measurements again. +Only all passing gates can replace the blocked status. Any new failure restarts +failure -> diagnosis -> correction -> affected tests -> full acceptance. diff --git a/package.json b/package.json index f47710b..0a50d3a 100644 --- a/package.json +++ b/package.json @@ -32,9 +32,9 @@ "clean": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"", "prepack": "npm run clean && npm run build", "prepublishOnly": "npm test", - "test": "npm run test:src:readonly && npm run check:links && npm run build && node --test dist/tests/semantic-recall-engine.test.js dist/tests/memory-cli.test.js dist/tests/reliability.test.js && npm run test:pack", + "test": "npm run test:src:readonly && npm run check:links && npm run build && node --test dist/tests/semantic-recall-engine.test.js dist/tests/recall-efficiency.test.js dist/tests/recall-lifecycle.test.js dist/tests/cli-errors.test.js dist/tests/memory-cli.test.js dist/tests/reliability.test.js && npm run test:pack", "test:src": "npm run test:src:readonly", - "test:src:readonly": "node --import \"data:text/javascript,import { register } from 'node:module'; import { pathToFileURL } from 'node:url'; register('ts-node/esm', pathToFileURL('./'));\" --test tests/semantic-recall-engine.test.ts tests/memory-cli.test.ts tests/reliability.test.ts", + "test:src:readonly": "node --import \"data:text/javascript,import { register } from 'node:module'; import { pathToFileURL } from 'node:url'; register('ts-node/esm', pathToFileURL('./'));\" --test tests/semantic-recall-engine.test.ts tests/recall-efficiency.test.ts tests/recall-lifecycle.test.ts tests/cli-errors.test.ts tests/memory-cli.test.ts tests/reliability.test.ts", "test:pack": "node --import \"data:text/javascript,import { register } from 'node:module'; import { pathToFileURL } from 'node:url'; register('ts-node/esm', pathToFileURL('./'));\" --test tests/package-pack.test.ts", "check:links": "node scripts/check-markdown-links.mjs", "scan:memory": "node scripts/dejavu-scan-memory.mjs", diff --git a/scripts/benchmark-cache-ownership.mjs b/scripts/benchmark-cache-ownership.mjs new file mode 100644 index 0000000..1d8a23e --- /dev/null +++ b/scripts/benchmark-cache-ownership.mjs @@ -0,0 +1,24 @@ +#!/usr/bin/env node +import { pathToFileURL } from 'node:url'; +import { resolve } from 'node:path'; +const [beforePath, afterPath] = process.argv.slice(2); +if (!beforePath || !afterPath) throw new Error('Usage: node scripts/benchmark-cache-ownership.mjs BEFORE_INDEX AFTER_INDEX'); +const before = await import(pathToFileURL(resolve(beforePath))); +const after = await import(pathToFileURL(resolve(afterPath))); +const report = { note: 'Deterministic provider-call counts, not latency or token costs. Sixteen memories share ten identical chunks; unique per-memory routing tags. Cache capacity 256.', modes: {} }; +for (const [mode, api, managed] of [['before-unscoped', before, false], ['after-unscoped', after, false], ['after-owned', after, true]]) { + let calls = 0; + const cache = new api.CachedEmbeddingProvider({ async embed(text) { calls++; return [text.length, 1]; } }, { modelKey: 'synthetic:v1', maxEntries: 256 }); + const engine = api.createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: managed, + preserveDerivedOnMetadataUpdate: true, chunker: { chunk: () => Array.from({ length: 10 }, () => ({ type: 'note', content: 'shared chunk' })) } }); + for (let i = 0; i < 16; i++) await engine.addMemory({ id: `memory${i}`, title: 'Shared policy', content: 'shared content', tags: [`route${i}`] }); + const result = { addCalls: calls }; calls = 0; + for (let pass = 0; pass < 2; pass++) for (let i = 0; i < 16; i++) await engine.recall({ text: `route${i}`, loadChunks: true }); + result.twoRecallPassesCalls = calls; calls = 0; + await engine.updateMemory('memory0', { importance: 0.9 }); result.metadataUpdateCalls = calls; calls = 0; + await engine.updateMemory('memory0', { content: 'replacement content' }); result.contentUpdateCalls = calls; + for (let i = 0; i < 16; i++) await engine.deleteMemory(`memory${i}`); + result.retainedAfterDelete = cache.getStats?.() ?? 'No public stats API in baseline'; + report.modes[mode] = result; +} +console.log(JSON.stringify(report, null, 2)); diff --git a/scripts/benchmark-recall.mjs b/scripts/benchmark-recall.mjs new file mode 100644 index 0000000..004f08a --- /dev/null +++ b/scripts/benchmark-recall.mjs @@ -0,0 +1,80 @@ +#!/usr/bin/env node +// Synthetic data only. Run against compiled modules; no network or dependencies. +// node scripts/benchmark-recall.mjs /path/to/base/dist/src/index.js ./dist/src/index.js +import { performance } from "node:perf_hooks"; +import { pathToFileURL } from "node:url"; +import { resolve } from "node:path"; +import assert from "node:assert/strict"; + +const paths = process.argv.slice(2); +if (paths.length !== 2) throw new Error("Usage: benchmark-recall.mjs BASELINE_MODULE CURRENT_MODULE"); +const [baseline, current] = await Promise.all(paths.map((path) => import(pathToFileURL(resolve(path)).href))); +const samples = 25; +const report = { node: process.version, samples, warmups: 5, units: "milliseconds", notes: [ + "Micro: storage.searchImpressions only. Scan: engine.scanImpressions.", + "API end-to-end: resident recall(loadChunks=true) plus JSON.stringify, including query embedding and chunk retrieval.", + "Excludes process start, index construction, persistence, network, model inference and host/LLM execution.", + "Baseline/current alternate measurement order. Medians/p95 are descriptive of this small synthetic run.", +], workloads: [], operations: {} }; +const stats = (values) => { + const sorted = values.toSorted((a, b) => a - b); + return { median: +sorted[Math.floor(sorted.length / 2)].toFixed(4), p95: +sorted[Math.ceil(sorted.length * .95) - 1].toFixed(4) }; +}; +async function paired(left, right) { + for (let i = 0; i < 5; i++) { await left(); await right(); } + const times = [[], []]; + const fns = [left, right]; + for (let i = 0; i < samples; i++) { + for (const index of (i % 2 ? [1, 0] : [0, 1])) { + const start = performance.now(); await fns[index](); times[index].push(performance.now() - start); + } + } + return { baseline: stats(times[0]), current: stats(times[1]) }; +} +async function fixture(api, n) { + const storage = new api.InMemoryStorage(); + const chunkVectorStore = new api.InMemoryChunkVectorStore(); + const vector = await new api.MockEmbeddingProvider().embed("shared routing synthetic benchmark"); + for (let i = 0; i < n; i++) { + const id = `item${i}`; + await storage.saveFamiliarity({ id, title: id, shortSummary: id, tags: [], + impressionTokens: ["shared", `topic${i}`, "routing", "synthetic", "benchmark"], + importance: .5, lastAccessedAt: "2026-01-01T00:00:00Z", embeddingVector: [1] }); + await storage.saveSummary({ id, description: "Synthetic decision", context: "Synthetic rationale", architectureOrIntent: "Synthetic intent", recentUpdates: [], metadata: {}, embeddingVector: vector }); + const chunks = Array.from({ length: 10 }, (_, j) => ({ chunkId: `${id}-${j}`, memoryId: id, type: "note", content: "shared routing synthetic benchmark", createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", importance: .5, embeddingVector: vector })); + await storage.saveChunks(chunks); await chunkVectorStore.upsert(chunks); + } + return { storage, engine: api.createInMemorySemanticRecallEngine({ storage, chunkVectorStore }) }; +} +for (const n of [100, 1000, 10000]) { + const fixtures = [await fixture(baseline, n), await fixture(current, n)]; + for (const [kind, query] of [["rare", `topic${n - 1}`], ["common", "shared"], ["none", "absentword"]]) { + const tokens = baseline.extractKeywords(query, 200); + const hits = await Promise.all(fixtures.map(({ storage }) => storage.searchImpressions(tokens, 5))); + assert.deepEqual(hits[0].map(h => [h.record.id, h.similarity]), hits[1].map(h => [h.record.id, h.similarity])); + const micro = await paired(...fixtures.map(({ storage }) => () => storage.searchImpressions(tokens, 5))); + const scan = await paired(...fixtures.map(({ engine }) => () => engine.scanImpressions(query))); + const apiEndToEnd = await paired(...fixtures.map(({ engine }) => async () => JSON.stringify(await engine.recall({ text: query, loadChunks: true })))); + report.workloads.push({ n, kind, micro, scan, apiEndToEnd }); + } +} +async function operations(api, optimized) { + let calls = 0; + const mock = new api.MockEmbeddingProvider(); + const counting = { async embed(text) { calls++; return mock.embed(text); } }; + const embeddingProvider = optimized ? new api.CachedEmbeddingProvider(counting, { modelKey: "mock:256:v1" }) : counting; + const engine = api.createInMemorySemanticRecallEngine({ embeddingProvider, preserveDerivedOnMetadataUpdate: optimized }); + const content = ("Decision: settings sync preserves preferences. " + "Synthetic explanation. ".repeat(14) + "\n\n").repeat(5); + await engine.addMemory({ id: "synthetic", title: "settings sync", tags: ["preferences"], content }); + const addCalls = calls; calls = 0; + let result; + for (let i = 0; i < 5; i++) result = await engine.recall({ text: "settings sync", loadChunks: true }); + const repeatedRecallCalls = calls; calls = 0; + await engine.updateMemory("synthetic", { importance: .9 }); + return { addCalls, repeatedRecallCalls, metadataUpdateCalls: calls, + rawRecallBytes: Buffer.byteLength(JSON.stringify(result)), + ...(optimized ? { projectedBytes: api.projectRecallForPrompt(result).bytes } : {}) }; +} +report.operations.baseline = await operations(baseline, false); +report.operations.currentOptIn = await operations(current, true); +console.log(JSON.stringify(report, null, 2)); diff --git a/scripts/evaluate-recall.mjs b/scripts/evaluate-recall.mjs new file mode 100644 index 0000000..a78f610 --- /dev/null +++ b/scripts/evaluate-recall.mjs @@ -0,0 +1,53 @@ +#!/usr/bin/env node +// Offline, labeled synthetic evaluation. No ingestion or query uses external services. +import { readFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { pathToFileURL } from 'node:url'; +import { resolve } from 'node:path'; +const [beforePath, afterPath, split = 'all'] = process.argv.slice(2); +if (!beforePath || !afterPath || !['all', 'development', 'held-out'].includes(split)) { + throw new Error('Usage: node scripts/evaluate-recall.mjs BEFORE_INDEX AFTER_INDEX [development|held-out|all]'); +} +const source = readFileSync(new URL('./fixtures/recall-evaluation.json', import.meta.url)); +const fixture = JSON.parse(source); +const before = await import(pathToFileURL(resolve(beforePath))); +const after = await import(pathToFileURL(resolve(afterPath))); +function metrics(rows, k) { + const positives = rows.filter(r => r.relevant.length); + const negatives = rows.filter(r => !r.relevant.length); + const hits = r => r.retrieved.slice(0, k); + return { + k, queries: rows.length, positiveQueries: positives.length, negativeQueries: negatives.length, + recallAtK: positives.length ? positives.reduce((n, r) => n + hits(r).filter(id => r.relevant.includes(id)).length / r.relevant.length, 0) / positives.length : null, + missedQueries: positives.filter(r => !hits(r).some(id => r.relevant.includes(id))).length, + falseRecallQueries: rows.filter(r => hits(r).some(id => !r.relevant.includes(id))).length, + falsePositiveNegativeQueries: negatives.filter(r => hits(r).length).length, + }; +} +const report = { fixtureSha256: createHash('sha256').update(source).digest('hex'), node: process.version, + interpretation: 'Recall@K is mean recall over positive queries, considering only eligible candidates. False recalls include any non-gold eligible ID in the first K; only the first candidate is loaded by recall(). Exact authored aliases are routing rules, not learned multilingual semantics. Held-out topics are disjoint; this small author-created split is not independent validation.', splits: {} }; +for (const [name, data] of Object.entries(fixture.splits)) { + if (split !== 'all' && split !== name) continue; + const modes = {}; + for (const [mode, api, enabled] of [['before', before, false], ['after-default', after, false], ['after-aliases', after, true]]) { + const aliases = data.memories.flatMap(m => m.aliases.map(alias => ({ alias, cue: m.id }))); + const engine = api.createInMemorySemanticRecallEngine(enabled ? { cueAliases: aliases } : {}); + for (const { aliases: _, ...memory } of data.memories) await engine.addMemory(memory); + const prefix = name === 'development' ? 'd' : 'h'; + await engine.addMemory({ id: 'retired', title: prefix + 'retiredobsolete', content: prefix + 'retiredobsolete' }); + const replacement = { id: 'successor', title: prefix + 'currentpolicy', content: prefix + 'currentpolicy' }; + if (engine.supersedeMemory) await engine.supersedeMemory('retired', replacement); + else { await engine.addMemory(replacement); await engine.deleteMemory('retired'); } + await engine.addMemory({ id: 'deleted', title: prefix + 'deletedobsolete', content: prefix + 'deletedobsolete' }); + await engine.deleteMemory('deleted'); + const rows = []; + for (const item of data.cases) { + const result = await engine.scanImpressions({ text: item.query, topK: 5, candidateLimit: 32 }); + rows.push({ ...item, retrieved: result.candidates.filter(c => c.familiarityLevel !== 'none').map(c => c.id) }); + } + modes[mode] = { overall: [1, 3, 5].map(k => metrics(rows, k)), + byCategory: Object.fromEntries([...new Set(rows.map(r => r.category))].map(category => [category, metrics(rows.filter(r => r.category === category), 1)])), rows }; + } + report.splits[name] = modes; +} +console.log(JSON.stringify(report, null, 2)); diff --git a/scripts/fixtures/recall-evaluation.json b/scripts/fixtures/recall-evaluation.json new file mode 100644 index 0000000..3e3aa9c --- /dev/null +++ b/scripts/fixtures/recall-evaluation.json @@ -0,0 +1,425 @@ +{ + "description": "Synthetic, small diagnostic fixture; not a semantic model benchmark. Corpus aliases and both topic-disjoint query splits frozen before the first evaluation. Held-out scoring follows development-only inspection; cases share an author and are not independent validation. No real memories.", + "splits": { + "development": { + "memories": [ + { + "id": "d-cache", + "title": "快取失效規則", + "content": "更新或刪除後釋放快取。只有測試環境可以停用驗證。", + "tags": [ + "d-cache" + ], + "aliases": [ + "更新後快取怎麼辦", + "cache eviction", + "刷新記憶快取" + ] + }, + { + "id": "d-login", + "title": "登入重試", + "content": "帳號驗證失敗最多重試三次。", + "tags": [ + "d-login" + ], + "aliases": [ + "無法登入怎麼處理", + "sign in retry", + "authentication retries" + ] + }, + { + "id": "d-backup", + "title": "備份還原", + "content": "每日離線備份；還原前驗證校驗碼。", + "tags": [ + "d-backup" + ], + "aliases": [ + "如何救回資料", + "restore backup", + "recover archived data" + ] + }, + { + "id": "d-theme", + "title": "顏色主題", + "content": "無障礙深色模式需要足夠對比。", + "tags": [ + "d-theme" + ], + "aliases": [ + "深色介面", + "night mode", + "dark appearance" + ] + } + ], + "cases": [ + { + "id": "d-01", + "category": "chinese", + "query": "快取失效規則", + "relevant": [ + "d-cache" + ], + "rationale": "Exact title cue remains a valid control." + }, + { + "id": "d-02", + "category": "chinese", + "query": "登入重試", + "relevant": [ + "d-login" + ], + "rationale": "Exact title cue remains a valid control." + }, + { + "id": "d-03", + "category": "chinese", + "query": "更新後快取怎麼辦", + "relevant": [ + "d-cache" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token." + }, + { + "id": "d-04", + "category": "chinese", + "query": "無法登入怎麼處理", + "relevant": [ + "d-login" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token." + }, + { + "id": "d-05", + "category": "synonym", + "query": "restore backup", + "relevant": [ + "d-backup" + ], + "rationale": "Authored English synonym routes to the corpus tag." + }, + { + "id": "d-06", + "category": "synonym", + "query": "night mode", + "relevant": [ + "d-theme" + ], + "rationale": "Authored English synonym routes to the corpus tag." + }, + { + "id": "d-07", + "category": "cross-language", + "query": "CACHE EVICTION", + "relevant": [ + "d-cache" + ], + "rationale": "Explicit bilingual alias; case normalization only." + }, + { + "id": "d-08", + "category": "cross-language", + "query": "SIGN IN RETRY", + "relevant": [ + "d-login" + ], + "rationale": "Explicit bilingual alias; case normalization only." + }, + { + "id": "d-09", + "category": "cross-language", + "query": " recover archived data ", + "relevant": [ + "d-backup" + ], + "rationale": "Explicit bilingual alias with whitespace normalization." + }, + { + "id": "d-10", + "category": "paraphrase", + "query": "重新整理存下來的線索以免拿到舊版", + "relevant": [ + "d-cache" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "d-11", + "category": "paraphrase", + "query": "進不去帳號時應該再試幾次", + "relevant": [ + "d-login" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "d-12", + "category": "paraphrase", + "query": "希望找回昨日保存的內容", + "relevant": [ + "d-backup" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "d-13", + "category": "synonym", + "query": "dark appearance", + "relevant": [ + "d-theme" + ], + "rationale": "Explicit alternate terminology." + }, + { + "id": "d-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus." + }, + { + "id": "d-15", + "category": "unrelated", + "query": "do not restore backup", + "relevant": [], + "rationale": "Negated query must not activate a substring alias." + }, + { + "id": "d-16", + "category": "unrelated", + "query": "sign in retry price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten." + }, + { + "id": "d-17", + "category": "stale", + "query": "dretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded." + }, + { + "id": "d-18", + "category": "stale", + "query": "ddeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted." + } + ] + }, + "held-out": { + "memories": [ + { + "id": "h-billing", + "title": "發票修正", + "content": "已結帳的發票只能以折讓單修正。", + "tags": [ + "h-billing" + ], + "aliases": [ + "發票寫錯怎麼辦", + "correct invoice", + "billing correction" + ] + }, + { + "id": "h-timeout", + "title": "連線逾時", + "content": "連線逾時只允許重試冪等請求。", + "tags": [ + "h-timeout" + ], + "aliases": [ + "連不上伺服器", + "connection deadline", + "network timeout" + ] + }, + { + "id": "h-export", + "title": "資料匯出", + "content": "匯出報表前必須遮蔽私人欄位。", + "tags": [ + "h-export" + ], + "aliases": [ + "怎樣下載報表", + "download report", + "export records" + ] + }, + { + "id": "h-schedule", + "title": "排程時區", + "content": "排程使用明確時區並處理日光節約時間。", + "tags": [ + "h-schedule" + ], + "aliases": [ + "定時工作用哪個時區", + "scheduled job timezone", + "timer zone" + ] + } + ], + "cases": [ + { + "id": "h-01", + "category": "chinese", + "query": "發票修正", + "relevant": [ + "h-billing" + ], + "rationale": "Exact title cue remains a valid control." + }, + { + "id": "h-02", + "category": "chinese", + "query": "連線逾時", + "relevant": [ + "h-timeout" + ], + "rationale": "Exact title cue remains a valid control." + }, + { + "id": "h-03", + "category": "chinese", + "query": "發票寫錯怎麼辦", + "relevant": [ + "h-billing" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token." + }, + { + "id": "h-04", + "category": "chinese", + "query": "連不上伺服器", + "relevant": [ + "h-timeout" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token." + }, + { + "id": "h-05", + "category": "synonym", + "query": "download report", + "relevant": [ + "h-export" + ], + "rationale": "Authored English synonym routes to the corpus tag." + }, + { + "id": "h-06", + "category": "synonym", + "query": "scheduled job timezone", + "relevant": [ + "h-schedule" + ], + "rationale": "Authored English synonym routes to the corpus tag." + }, + { + "id": "h-07", + "category": "cross-language", + "query": "CORRECT INVOICE", + "relevant": [ + "h-billing" + ], + "rationale": "Explicit bilingual alias; case normalization only." + }, + { + "id": "h-08", + "category": "cross-language", + "query": "CONNECTION DEADLINE", + "relevant": [ + "h-timeout" + ], + "rationale": "Explicit bilingual alias; case normalization only." + }, + { + "id": "h-09", + "category": "cross-language", + "query": " export records ", + "relevant": [ + "h-export" + ], + "rationale": "Explicit bilingual alias with whitespace normalization." + }, + { + "id": "h-10", + "category": "paraphrase", + "query": "帳單的金額填錯要如何更正", + "relevant": [ + "h-billing" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "h-11", + "category": "paraphrase", + "query": "伺服器等太久了需要重新連線嗎", + "relevant": [ + "h-timeout" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "h-12", + "category": "paraphrase", + "query": "把表格存成外部檔案需要注意什麼", + "relevant": [ + "h-export" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "h-13", + "category": "synonym", + "query": "timer zone", + "relevant": [ + "h-schedule" + ], + "rationale": "Explicit alternate terminology." + }, + { + "id": "h-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus." + }, + { + "id": "h-15", + "category": "unrelated", + "query": "do not download report", + "relevant": [], + "rationale": "Negated query must not activate a substring alias." + }, + { + "id": "h-16", + "category": "unrelated", + "query": "connection deadline price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten." + }, + { + "id": "h-17", + "category": "stale", + "query": "hretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded." + }, + { + "id": "h-18", + "category": "stale", + "query": "hdeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted." + } + ] + } + } +} diff --git a/src/core/semantic-recall-engine.ts b/src/core/semantic-recall-engine.ts index 159ae29..14392ea 100644 --- a/src/core/semantic-recall-engine.ts +++ b/src/core/semantic-recall-engine.ts @@ -17,9 +17,11 @@ import type { StoredMemory, UpdateMemoryInput, } from "../types/memory.js"; -import type { SemanticRecallEngineConfig } from "../types/plugins.js"; +import type { EmbeddingProvider, EmbeddingScope, SemanticRecallEngineConfig } from "../types/plugins.js"; import { createId } from "../utils/id.js"; import { extractKeywords } from "../utils/text.js"; +import { boundedInteger, MAX_RECALL_LIMIT } from "../utils/limits.js"; +import { compileCueAliases, normalizeCue } from "../retrieval/cue-aliases.js"; import { isoNow } from "../utils/time.js"; /** Includes a recovery copy if an adapter fails during compensation. Do not log its contents. */ @@ -36,6 +38,15 @@ export class MemoryWriteError extends Error { } } +/** A successor cleanup failure is explicit; inspect both IDs before retrying. */ +export class MemorySupersedeError extends Error { + constructor(cause: unknown, readonly memoryId: string, readonly replacementId: string, + readonly recoveryErrors: unknown[]) { + super(recoveryErrors.length ? "Supersede failed; recovery requires adapter repair" : "Supersede failed; previous state restored", { cause }); + this.name = "MemorySupersedeError"; + } +} + export class SemanticRecallEngine { private readonly familiarityLayer: FamiliarityLayer; private readonly summaryLayer: SummaryLayer; @@ -44,6 +55,8 @@ export class SemanticRecallEngine { private readonly summaryGenerator; private readonly thresholds; private readonly scoringStrategy; + private readonly cueAliases; + private readonly embeddingScopes = new Map(); private mutationQueue: Promise = Promise.resolve(); private mutate(operation: () => Promise): Promise { @@ -53,12 +66,20 @@ export class SemanticRecallEngine { } constructor(private readonly config: SemanticRecallEngineConfig) { + this.cueAliases = compileCueAliases(config.cueAliases); + if (config.manageEmbeddingCache && typeof config.embeddingProvider.createScope !== "function") { + throw new TypeError("manageEmbeddingCache requires embeddingProvider.createScope"); + } this.familiarityLayer = new FamiliarityLayer(config.storage, config.familiarityVectorStore); this.summaryLayer = new SummaryLayer(config.storage); this.chunkLayer = new ChunkLayer(config.storage, config.chunkVectorStore); this.chunker = config.chunker ?? new DefaultChunker(); this.summaryGenerator = config.summaryGenerator ?? new DefaultSummaryGenerator(); - this.thresholds = config.thresholds ?? { strong: 0.85, weak: 0.75 }; + this.thresholds = { ...(config.thresholds ?? { strong: 0.85, weak: 0.75 }) }; + const { weak, strong } = this.thresholds; + if (!Number.isFinite(weak) || !Number.isFinite(strong) || weak < 0 || strong > 1 || weak > strong) { + throw new RangeError("thresholds must satisfy 0 <= weak <= strong <= 1"); + } this.scoringStrategy = config.scoringStrategy ?? new HybridScoringStrategy(); } @@ -66,23 +87,36 @@ export class SemanticRecallEngine { return this.mutate(async () => { const id = input.id ?? createId("memory"); if (await this.snapshot(id)) throw new Error(`Memory already exists: ${id}; use updateMemory`); - const prepared = await this.prepareMemory({ ...input, id }); - await this.commitMemory(prepared, null); + await this.writeMemory({ ...input, id }, null); return { id }; }); } - private async prepareMemory(input: AddMemoryInput): Promise { + private async writeMemory(input: AddMemoryInput & { id: string }, previous: StoredMemory | null): Promise { + const scope = this.config.manageEmbeddingCache ? this.config.embeddingProvider.createScope!() : undefined; + try { + const prepared = await this.prepareMemory(input, scope ?? this.config.embeddingProvider); + if (previous) prepared.createdAt = previous.createdAt; + await this.commitMemory(prepared, previous); + } catch (error) { + scope?.dispose(); + throw error; + } + this.embeddingScopes.get(input.id)?.dispose(); + if (scope) this.embeddingScopes.set(input.id, scope); + } + + private async prepareMemory(input: AddMemoryInput, provider: Pick): Promise { const id = input.id ?? createId("memory"); const now = isoNow(); const shortSummary = await this.summaryGenerator.generateShortSummary(input); const structuredSummary = await this.summaryGenerator.generateStructuredSummary(input); const chunkSeeds = this.chunker.chunk(input, now); - const familiarityVector = await this.config.embeddingProvider.embed( + const familiarityVector = await provider.embed( [input.title, shortSummary, ...(input.tags ?? [])].join("\n"), ); - const summaryVector = await this.config.embeddingProvider.embed( + const summaryVector = await provider.embed( `${structuredSummary.description}\n${structuredSummary.context}\n${structuredSummary.architectureOrIntent}`, ); @@ -96,7 +130,7 @@ export class SemanticRecallEngine { createdAt: now, updatedAt: now, importance: seed.importance ?? input.importance ?? 0.5, - embeddingVector: await this.config.embeddingProvider.embed(seed.content), + embeddingVector: await provider.embed(seed.content), }); } @@ -180,9 +214,13 @@ export class SemanticRecallEngine { } async recall(query: RecallInput | string): Promise { + return this.config.consistentReads || this.config.manageEmbeddingCache ? this.mutate(() => this.recallInternal(query)) : this.recallInternal(query); + } + + private async recallInternal(query: RecallInput | string): Promise { const normalized: RecallInput = typeof query === "string" ? { text: query } : query; - const chunkLimit = normalized.chunkLimit ?? 3; - const scan = await this.scanImpressions(normalized); + const chunkLimit = boundedInteger("chunkLimit", normalized.chunkLimit ?? 3, MAX_RECALL_LIMIT); + const scan = await this.scanImpressionsInternal(normalized); const topMatch = scan.topMatch; const familiarityLevel = scan.familiarityLevel; const budget: RecallBudgetReport = { @@ -213,8 +251,13 @@ export class SemanticRecallEngine { budget.whyLoaded.push("weak familiarity loaded a clipped summary"); } - if (familiarityLevel === "strong" && normalized.loadChunks) { - const vector = await this.config.embeddingProvider.embed(normalized.text); + if (familiarityLevel === "strong" && normalized.loadChunks && chunkLimit > 0) { + let scope = this.embeddingScopes.get(topMatch.id); + if (this.config.manageEmbeddingCache && !scope) { + scope = this.config.embeddingProvider.createScope!(); + this.embeddingScopes.set(topMatch.id, scope); + } + const vector = await (scope ?? this.config.embeddingProvider).embed(normalized.text); const chunkHits = await this.chunkLayer.search(topMatch.id, vector, chunkLimit); const allChunks = await this.chunkLayer.get(topMatch.id); const chunkMap = new Map(allChunks.map((chunk) => [chunk.chunkId, chunk])); @@ -244,12 +287,20 @@ export class SemanticRecallEngine { } async scanImpressions(query: RecallInput | string): Promise { + return this.config.consistentReads || this.config.manageEmbeddingCache ? this.mutate(() => this.scanImpressionsInternal(query)) : this.scanImpressionsInternal(query); + } + + private async scanImpressionsInternal(query: RecallInput | string): Promise { const normalized: RecallInput = typeof query === "string" ? { text: query } : query; - const topK = normalized.topK ?? 5; - const queryTokens = extractKeywords(normalized.text, 200); - const hits = await this.config.storage.searchImpressions(queryTokens, topK); + if (typeof normalized.text !== "string") throw new TypeError("text must be a string"); + const topK = boundedInteger("topK", normalized.topK ?? 5, MAX_RECALL_LIMIT); + const candidateLimit = boundedInteger("candidateLimit", normalized.candidateLimit ?? topK, MAX_RECALL_LIMIT, topK); + boundedInteger("chunkLimit", normalized.chunkLimit ?? 3, MAX_RECALL_LIMIT); + const cue = this.cueAliases.size ? this.cueAliases.get(normalizeCue(normalized.text)) ?? normalized.text : normalized.text; + const queryTokens = extractKeywords(cue, 200); + const hits = topK === 0 ? [] : await this.config.storage.searchImpressions(queryTokens, candidateLimit); - const candidates = this.scoringStrategy.rank( + const ranked = this.scoringStrategy.rank( hits.map((hit) => ({ record: hit.record, similarity: hit.similarity, @@ -259,6 +310,12 @@ export class SemanticRecallEngine { familiarityLevel: resolveFamiliarityLevel(candidate.semanticSimilarity, this.thresholds), })); + // Eligibility precedes hybrid preference. Keep hybrid ordering within each group, + // including weak vs strong, and retain ineligible candidates for diagnostics. + const candidates = [ + ...ranked.filter((candidate) => candidate.familiarityLevel !== "none"), + ...ranked.filter((candidate) => candidate.familiarityLevel === "none"), + ].slice(0, topK); const topMatch = candidates[0] ?? null; const familiarityLevel = topMatch ? resolveFamiliarityLevel(topMatch.semanticSimilarity, this.thresholds) @@ -284,19 +341,38 @@ export class SemanticRecallEngine { } async getSummary(id: string): Promise { - return this.summaryLayer.get(id); + return this.config.consistentReads || this.config.manageEmbeddingCache ? this.mutate(() => this.summaryLayer.get(id)) : this.summaryLayer.get(id); } async getChunks(id: string): Promise { - return this.chunkLayer.get(id); + return this.config.consistentReads || this.config.manageEmbeddingCache ? this.mutate(() => this.chunkLayer.get(id)) : this.chunkLayer.get(id); + } + + /** Explicit full raw-content retrieval; never loaded by prompt projection automatically. */ + async getRawContent(id: string): Promise { + return this.config.consistentReads || this.config.manageEmbeddingCache ? this.mutate(() => this.config.storage.getRawContent(id)) : this.config.storage.getRawContent(id); } async updateMemory(id: string, input: UpdateMemoryInput): Promise<{ id: string }> { return this.mutate(async () => { const previous = await this.snapshot(id); if (!previous) throw new Error(`Memory not found: ${id}`); + if (this.config.preserveDerivedOnMetadataUpdate && + Object.keys(input).every((key) => key === "importance" || key === "metadata")) { + // Explicitly preserve derived fields even for custom generators/chunkers. + // Reuse the existing compensated write path and detached snapshot. + const next = structuredClone(previous); + next.updatedAt = isoNow(); + next.familiarity.importance = input.importance ?? previous.familiarity.importance; + next.summary.metadata = { + ...previous.summary.metadata, ...input.metadata, + title: previous.familiarity.title, tags: [...previous.familiarity.tags], + }; + await this.commitMemory(next, previous); + return { id }; + } const existingSummary = previous.summary; - const prepared = await this.prepareMemory({ + await this.writeMemory({ id, title: input.title ?? previous.familiarity.title, content: input.content ?? previous.rawContent, @@ -312,18 +388,57 @@ export class SemanticRecallEngine { tags: input.tags ?? previous.familiarity.tags, }, chunkStrategy: input.chunkStrategy, - }); - prepared.createdAt = previous.createdAt; - await this.commitMemory(prepared, previous); + }, previous); return { id }; }); } - async deleteMemory(id: string): Promise { + /** Replace a live memory with a new ID. No historical redirect is stored by this optional engine. */ + async supersedeMemory(id: string, replacement: AddMemoryInput): Promise<{ id: string; supersededId: string }> { return this.mutate(async () => { + const previous = await this.snapshot(id); + if (!previous) throw new Error(`Memory not found: ${id}`); + const nextId = replacement.id ?? createId("memory"); + if (nextId === id || await this.snapshot(nextId)) throw new Error(`Replacement memory already exists: ${nextId}`); + await this.writeMemory({ ...replacement, id: nextId }, null); + try { + await this.deleteInternal(id, previous); + } catch (cause) { + const recoveryErrors: unknown[] = cause instanceof MemoryWriteError ? [...cause.recoveryErrors] : [cause]; + try { await this.deleteInternal(nextId, await this.snapshot(nextId)); } + catch (error) { recoveryErrors.push(error); } + throw new MemorySupersedeError(cause, id, nextId, recoveryErrors); + } + return { id: nextId, supersededId: id }; + }); + } + + async deleteMemory(id: string): Promise { + return this.mutate(async () => this.deleteInternal(id, await this.snapshot(id))); + } + + private async deleteInternal(id: string, previous: StoredMemory | null): Promise { + try { await this.familiarityLayer.remove(id); await this.chunkLayer.remove(id); await this.config.storage.deleteMemory(id); - }); + } catch (cause) { + const recoveryErrors: unknown[] = []; + const attempt = async (operation: () => Promise) => { + try { await operation(); } catch (error) { recoveryErrors.push(error); } + }; + if (previous) { + await attempt(() => this.config.storage.saveSummary(previous.summary)); + await attempt(() => this.config.storage.saveChunks(previous.chunks)); + await attempt(() => this.config.storage.saveRawContent(id, previous.rawContent, previous.createdAt, previous.updatedAt)); + await attempt(() => this.config.storage.saveFamiliarity(previous.familiarity)); + await attempt(() => this.config.familiarityVectorStore.upsert(previous.familiarity)); + await attempt(() => this.config.chunkVectorStore.upsert(previous.chunks)); + } + throw new MemoryWriteError(recoveryErrors.length ? "Memory delete failed; recovery requires adapter repair" : "Memory delete failed; previous state restored", + cause, previous, recoveryErrors, id); + } + this.embeddingScopes.get(id)?.dispose(); + this.embeddingScopes.delete(id); } } diff --git a/src/index.ts b/src/index.ts index f327f51..a161f61 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,3 +9,6 @@ export * from "./storage/in-memory-vector-store.js"; export * from "./types/memory.js"; export * from "./types/plugins.js"; export * from "./utils/text.js"; +export * from "./retrieval/prompt-projection.js"; +export * from "./plugins/cached-embedding-provider.js"; +export type { CueAlias } from "./retrieval/cue-aliases.js"; diff --git a/src/memory/default-chunker.ts b/src/memory/default-chunker.ts index 9e16a65..8eea217 100644 --- a/src/memory/default-chunker.ts +++ b/src/memory/default-chunker.ts @@ -1,5 +1,6 @@ import type { AddMemoryInput, MemoryChunkType } from "../types/memory.js"; import type { Chunker, MemoryChunkSeed } from "../types/plugins.js"; +import { boundedInteger } from "../utils/limits.js"; const DEFAULT_MAX = 280; const DEFAULT_OVERLAP = 40; @@ -78,8 +79,8 @@ function packBoundaryChunks(blocks: string[], maxCharacters: number, overlapChar export class DefaultChunker implements Chunker { chunk(input: AddMemoryInput): MemoryChunkSeed[] { - const maxCharacters = input.chunkStrategy?.maxCharacters ?? DEFAULT_MAX; - const overlapCharacters = input.chunkStrategy?.overlapCharacters ?? DEFAULT_OVERLAP; + const maxCharacters = boundedInteger("maxCharacters", input.chunkStrategy?.maxCharacters ?? DEFAULT_MAX, 1_000_000, 1); + const overlapCharacters = boundedInteger("overlapCharacters", input.chunkStrategy?.overlapCharacters ?? Math.min(DEFAULT_OVERLAP, maxCharacters - 1), maxCharacters - 1); const fallbackType = input.chunkStrategy?.preferredTypes?.[0] ?? "note"; const text = input.content.trim(); diff --git a/src/plugins/cached-embedding-provider.ts b/src/plugins/cached-embedding-provider.ts new file mode 100644 index 0000000..b022f6d --- /dev/null +++ b/src/plugins/cached-embedding-provider.ts @@ -0,0 +1,105 @@ +import { createHmac, randomBytes } from "node:crypto"; +import type { EmbeddingProvider, EmbeddingScope } from "../types/plugins.js"; +import { boundedInteger } from "../utils/limits.js"; + +export interface EmbeddingCacheOptions { + /** Include provider, model, revision and any embedding options. Change on model changes. */ + modelKey: string | (() => string); + maxEntries?: number; + /** Bound digest UTF-16 bytes + vector float64 bytes, not JS heap overhead. */ + maxBytes?: number; +} + +interface Scope { active: boolean; key: Buffer } +interface Entry { vector: number[]; bytes: number; scope?: Scope } + +/** Opt-in, instance-local LRU. Digests avoid retained plaintext content keys, not secure erasure. */ +export class CachedEmbeddingProvider implements EmbeddingProvider { + private readonly entries = new Map(); + private readonly secret = randomBytes(32); + private readonly maxEntries: number; + private readonly maxBytes: number; + private readonly modelKey: EmbeddingCacheOptions["modelKey"]; + private bytes = 0; + private generation = 0; + + constructor(private readonly provider: EmbeddingProvider, options: EmbeddingCacheOptions) { + this.modelKey = options.modelKey; + this.maxEntries = boundedInteger("maxEntries", options.maxEntries ?? 128, 10_000); + this.maxBytes = boundedInteger("maxBytes", options.maxBytes ?? 8 * 1024 * 1024, 64 * 1024 * 1024); + this.currentModelKey(); + } + + private currentModelKey(): string { + const key = typeof this.modelKey === "function" ? this.modelKey() : this.modelKey; + if (typeof key !== "string" || !key.trim() || key.length > 4096) { + throw new TypeError("modelKey must be a non-empty model/version identifier of at most 4096 characters"); + } + return key; + } + + /** Release references; already running provider work is not cancelled or securely erased. */ + clear(): void { + this.generation++; + this.entries.clear(); + this.bytes = 0; + } + + getStats(): { entries: number; accountedBytes: number } { + return { entries: this.entries.size, accountedBytes: this.bytes }; + } + + /** Separate ownership: identical inputs in different scopes never share retained entries. */ + createScope(): EmbeddingScope { + const scope: Scope = { active: true, key: randomBytes(32) }; + return { + embed: (text) => this.embedInScope(text, scope), + dispose: () => { + scope.active = false; + for (const [key, entry] of this.entries) { + if (entry.scope === scope) { this.bytes -= entry.bytes; this.entries.delete(key); } + } + }, + }; + } + + embed(text: string): Promise { return this.embedInScope(text); } + + private async embedInScope(text: string, scope?: Scope): Promise { + if (scope && !scope.active) throw new Error("Embedding scope has been disposed"); + const model = this.currentModelKey(); + // Length framing and UTF-16 preserve exact JS strings, including lone surrogates. + // Each owner has a different HMAC key; no plaintext input is retained as a key. + const key = this.maxEntries && this.maxBytes >= 136 + ? createHmac("sha256", scope?.key ?? this.secret).update(`${model.length}:`) + .update(model, "utf16le").update(text, "utf16le").digest("hex") : undefined; + const hit = key === undefined ? undefined : this.entries.get(key); + if (hit && key !== undefined) { + this.entries.delete(key); + this.entries.set(key, hit); + return [...hit.vector]; + } + const generation = this.generation; + // Independent misses: there is no unbounded in-flight promise registry. + const vector = await this.provider.embed(text); + if (!Array.isArray(vector) || !vector.length) throw new TypeError("embedding must be a non-empty finite vector"); + for (const value of vector) { + if (!Number.isFinite(value)) throw new TypeError("embedding must be a non-empty finite vector"); + } + if (key !== undefined && generation === this.generation && (!scope || scope.active) && model === this.currentModelKey()) { + const bytes = key.length * 2 + vector.length * 8; + if (bytes <= this.maxBytes) { + const previous = this.entries.get(key); + if (previous) { this.bytes -= previous.bytes; this.entries.delete(key); } + while (this.entries.size >= this.maxEntries || this.bytes + bytes > this.maxBytes) { + const oldest = this.entries.keys().next().value!; + this.bytes -= this.entries.get(oldest)!.bytes; + this.entries.delete(oldest); + } + this.entries.set(key, { vector: [...vector], bytes, scope }); + this.bytes += bytes; + } + } + return [...vector]; + } +} diff --git a/src/plugins/create-in-memory-engine.ts b/src/plugins/create-in-memory-engine.ts index c96f30d..ab0566e 100644 --- a/src/plugins/create-in-memory-engine.ts +++ b/src/plugins/create-in-memory-engine.ts @@ -15,9 +15,13 @@ export function createInMemorySemanticRecallEngine( familiarityVectorStore: config.familiarityVectorStore ?? new InMemoryFamiliarityVectorStore(), chunkVectorStore: config.chunkVectorStore ?? new InMemoryChunkVectorStore(), storage: config.storage ?? new InMemoryStorage(), + cueAliases: config.cueAliases, + manageEmbeddingCache: config.manageEmbeddingCache, + consistentReads: config.consistentReads, scoringStrategy: config.scoringStrategy, chunker: config.chunker, summaryGenerator: config.summaryGenerator, thresholds: config.thresholds, + preserveDerivedOnMetadataUpdate: config.preserveDerivedOnMetadataUpdate, }); } diff --git a/src/retrieval/cue-aliases.ts b/src/retrieval/cue-aliases.ts new file mode 100644 index 0000000..58088a6 --- /dev/null +++ b/src/retrieval/cue-aliases.ts @@ -0,0 +1,26 @@ +export interface CueAlias { + /** An exact complete query, compared after lowercase and whitespace normalization. */ + alias: string; + /** Existing lexical cue(s), usually a distinctive tag on the intended memory. */ + cue: string; +} + +export function normalizeCue(text: string): string { + return text.trim().toLowerCase().replace(/\s+/gu, " "); +} + +/** Explicit routing, not translation, substring expansion, or a semantic model. */ +export function compileCueAliases(aliases: readonly CueAlias[] = []): ReadonlyMap { + if (!Array.isArray(aliases) || aliases.length > 128) throw new RangeError("cueAliases must contain at most 128 entries"); + const map = new Map(); + for (const entry of aliases) { + if (!entry || typeof entry.alias !== "string" || typeof entry.cue !== "string" || + !entry.alias.trim() || !entry.cue.trim() || entry.alias.length > 256 || entry.cue.length > 256) { + throw new TypeError("alias and cue must be non-empty strings of at most 256 characters"); + } + const alias = normalizeCue(entry.alias); + if (map.has(alias) && map.get(alias) !== entry.cue) throw new Error(`Conflicting cue alias: ${alias}`); + map.set(alias, entry.cue); + } + return map; +} diff --git a/src/retrieval/prompt-projection.ts b/src/retrieval/prompt-projection.ts new file mode 100644 index 0000000..55b8e69 --- /dev/null +++ b/src/retrieval/prompt-projection.ts @@ -0,0 +1,102 @@ +import type { RecallResult } from "../types/memory.js"; +import { boundedInteger } from "../utils/limits.js"; + +export interface PromptProjectionOptions { + /** UTF-8 budget for the entire returned text, including JSON framing. Default 8192. */ + maxBytes?: number; + /** Add source API names and inline warnings when a field may have lost qualifiers. */ + includeProvenance?: boolean; + /** Requires the host's tokenizer; no character-to-token approximation is used. */ + maxTokens?: number; + countTokens?: (text: string) => number; +} + +export interface PromptProjection { + /** JSONL data, not instructions. Inject this text, not the original RecallResult. */ + text: string; + bytes: number; + tokens?: number; + truncated: boolean; +} + +type PromptRecord = { rawSource?: "getRawContent"; source?: "getSummary" | "getChunks"; updateIndex?: number; incomplete?: boolean; warning?: string; kind: "summary" | "chunk"; memoryId: string; text: string; field?: string; chunkId?: string; type?: string }; +const encoder = new TextEncoder(); + +const incompleteWarning = "Incomplete excerpt; qualifiers may be missing. Retrieve the source before relying on it."; +function* records(result: RecallResult, provenance: boolean): Generator { + if (!result.matched || !result.topMatch) return; + const summary = result.summaryIfLoaded; + if (summary) { + const source = provenance ? { source: "getSummary" as const, rawSource: "getRawContent" as const, + warning: "Derived summary; original constraints may be missing. Consult raw content.", + ...(result.familiarityLevel === "weak" ? { incomplete: true, warning: incompleteWarning } : {}) } : {}; + for (const field of ["description", "context", "architectureOrIntent"] as const) { + if (summary[field]) yield { ...source, kind: "summary", memoryId: summary.id, field, text: summary[field] }; + } + for (const [updateIndex, text] of summary.recentUpdates.entries()) { + if (text) yield { ...source, kind: "summary", memoryId: summary.id, field: "recentUpdate", ...(provenance ? { updateIndex } : {}), text }; + } + } + for (const chunk of result.chunksIfLoaded) { + if (chunk.content) yield { ...(provenance ? { source: "getChunks" as const, rawSource: "getRawContent" as const } : {}), kind: "chunk", memoryId: chunk.memoryId, chunkId: chunk.chunkId, type: chunk.type, text: chunk.content }; + } +} + +/** Allowlisted, bounded data projection. It does not sanitize prompt injection in memory text. */ +export function projectRecallForPrompt(result: RecallResult, options: PromptProjectionOptions = {}): PromptProjection { + const maxBytes = boundedInteger("maxBytes", options.maxBytes ?? 8192, 1_048_576); + const maxTokens = options.maxTokens === undefined ? undefined : boundedInteger("maxTokens", options.maxTokens, 1_048_576); + if (maxTokens !== undefined && typeof options.countTokens !== "function") { + throw new TypeError("maxTokens requires countTokens for the target model"); + } + const count = (text: string) => { + if (!options.countTokens) return undefined; + const tokens = options.countTokens(text); + if (!Number.isSafeInteger(tokens) || tokens < 0) throw new RangeError("countTokens must return a non-negative safe integer"); + return tokens; + }; + const fits = (text: string) => encoder.encode(text).length <= maxBytes && + (maxTokens === undefined || count(text)! <= maxTokens); + let text = ""; + let truncated = Boolean(options.includeProvenance && result.summaryIfLoaded && result.familiarityLevel === "weak"); + if (!fits(text)) throw new RangeError("token budget cannot represent empty text"); + for (const record of records(result, options.includeProvenance === true)) { + // Bound temporary allocation too, even for oversized adapter-supplied fields. + const { text: content, ...header } = record; + if (Object.values(header).some((value) => typeof value === "string" && value.length > maxBytes)) { truncated = true; break; } + const points: string[] = []; + let prefixLength = 0; + for (const point of content) { + if (points.length >= maxBytes) break; + points.push(point); + prefixLength += point.length; + } + const render = (length: number) => text + (text ? "\n" : "") + JSON.stringify({ ...header, + ...(options.includeProvenance && (length < points.length || prefixLength < content.length) + ? { incomplete: true, warning: incompleteWarning } : {}), text: points.slice(0, length).join("") }); + const full = render(points.length); + if (fits(full)) { + text = full; + if (prefixLength < content.length) { truncated = true; break; } + continue; + } + // Tokenizers need not be monotonic: this chooses a fitting prefix, not + // necessarily the largest one. Every accepted candidate is checked in full. + let low = 1; + let high = points.length; + let accepted = ""; + while (low <= high) { + const mid = Math.floor((low + high) / 2); + const candidate = render(mid); + if (fits(candidate)) { accepted = candidate; low = mid + 1; } + else high = mid - 1; + } + if (accepted) text = accepted; + truncated = true; + break; + } + const tokens = count(text); + // Also guards against a stateful or inconsistent host tokenizer. + if (maxTokens !== undefined && tokens! > maxTokens) throw new RangeError("countTokens returned inconsistent results"); + return { text, bytes: encoder.encode(text).length, ...(tokens === undefined ? {} : { tokens }), truncated }; +} diff --git a/src/storage/in-memory-storage.ts b/src/storage/in-memory-storage.ts index 087f8db..fe57cd4 100644 --- a/src/storage/in-memory-storage.ts +++ b/src/storage/in-memory-storage.ts @@ -1,6 +1,19 @@ import type { FamiliarityRecord, MemoryChunk, SummaryRecord, StoredMemory } from "../types/memory.js"; import type { ImpressionSearchHit, MemoryStorage } from "../types/plugins.js"; -import { lexicalSimilarity } from "../utils/text.js"; +import { extractKeywords } from "../utils/text.js"; +import { boundedInteger, MAX_RECALL_LIMIT } from "../utils/limits.js"; + +function copyFamiliarity(record: FamiliarityRecord): FamiliarityRecord { + return { ...record, impressionTokens: [...record.impressionTokens], tags: [...record.tags], embeddingVector: [...record.embeddingVector] }; +} + +function copySummary(record: SummaryRecord): SummaryRecord { + return { ...record, embeddingVector: [...record.embeddingVector], + metadata: Object.fromEntries(Object.entries(record.metadata).map(([key, value]) => [key, Array.isArray(value) ? [...value] : value])) }; +} +function copyChunks(chunks: MemoryChunk[]): MemoryChunk[] { + return chunks.map(chunk => ({ ...chunk, embeddingVector: [...chunk.embeddingVector] })); +} interface RawMemoryRecord { content: string; @@ -11,6 +24,7 @@ interface RawMemoryRecord { export class InMemoryStorage implements MemoryStorage { private readonly familiarities = new Map(); private readonly impressionTokenIndex = new Map>(); + private readonly scoringTokens = new Map>(); private readonly summaries = new Map(); private readonly chunks = new Map(); private readonly raw = new Map(); @@ -30,7 +44,8 @@ export class InMemoryStorage implements MemoryStorage { async saveFamiliarity(record: FamiliarityRecord): Promise { this.removeFromImpressionIndex(record.id); - this.familiarities.set(record.id, record); + this.familiarities.set(record.id, copyFamiliarity(record)); + this.scoringTokens.set(record.id, new Set(extractKeywords(record.impressionTokens.join(" "), 200))); for (const token of record.impressionTokens) { const ids = this.impressionTokenIndex.get(token) ?? new Set(); ids.add(record.id); @@ -39,12 +54,12 @@ export class InMemoryStorage implements MemoryStorage { } async saveSummary(record: SummaryRecord): Promise { - this.summaries.set(record.id, record); + this.summaries.set(record.id, copySummary(record)); } async saveChunks(chunks: MemoryChunk[]): Promise { if (chunks.length === 0) return; - this.chunks.set(chunks[0].memoryId, chunks); + this.chunks.set(chunks[0].memoryId, copyChunks(chunks)); } async saveRawContent(memoryId: string, content: string, createdAt: string, updatedAt: string): Promise { @@ -52,15 +67,17 @@ export class InMemoryStorage implements MemoryStorage { } async getFamiliarity(id: string): Promise { - return this.familiarities.get(id) ?? null; + const record = this.familiarities.get(id); + return record ? copyFamiliarity(record) : null; } async getSummary(id: string): Promise { - return this.summaries.get(id) ?? null; + const record = this.summaries.get(id); + return record ? copySummary(record) : null; } async getChunks(id: string): Promise { - return this.chunks.get(id) ?? []; + return copyChunks(this.chunks.get(id) ?? []); } async getRawContent(id: string): Promise { @@ -76,10 +93,12 @@ export class InMemoryStorage implements MemoryStorage { } async listFamiliarities(): Promise { - return [...this.familiarities.values()]; + return [...this.familiarities.values()].map(copyFamiliarity); } async searchImpressions(tokens: string[], topK: number): Promise { + boundedInteger("topK", topK, MAX_RECALL_LIMIT); + if (topK === 0) return []; const candidateIds = new Set(); for (const token of tokens) { for (const id of this.impressionTokenIndex.get(token) ?? []) { @@ -87,18 +106,22 @@ export class InMemoryStorage implements MemoryStorage { } } + if (candidateIds.size === 0) return []; + const queryTokens = new Set(extractKeywords(tokens.join(" "), 200)); return [...candidateIds] .map((id) => { const record = this.familiarities.get(id); if (!record) return null; - return { - record, - similarity: lexicalSimilarity(tokens.join(" "), record.impressionTokens.join(" ")), - }; + const recordTokens = this.scoringTokens.get(id)!; + let overlap = 0; + for (const token of queryTokens) if (recordTokens.has(token)) overlap++; + const denominator = Math.min(queryTokens.size, recordTokens.size); + return { record, similarity: denominator ? overlap / denominator : 0 }; }) .filter((hit): hit is ImpressionSearchHit => hit !== null) .sort((a, b) => b.similarity - a.similarity) - .slice(0, topK); + .slice(0, topK) + .map((hit) => ({ ...hit, record: copyFamiliarity(hit.record) })); } async touchMemory(id: string, lastAccessedAt: string): Promise { @@ -108,6 +131,7 @@ export class InMemoryStorage implements MemoryStorage { } private removeFromImpressionIndex(id: string): void { + this.scoringTokens.delete(id); const existing = this.familiarities.get(id); if (!existing) return; for (const token of existing.impressionTokens) { diff --git a/src/storage/in-memory-vector-store.ts b/src/storage/in-memory-vector-store.ts index 15f23e7..8f2f747 100644 --- a/src/storage/in-memory-vector-store.ts +++ b/src/storage/in-memory-vector-store.ts @@ -1,5 +1,6 @@ import type { FamiliarityRecord, MemoryChunk } from "../types/memory.js"; import type { ChunkVectorStore, FamiliarityVectorStore, VectorSearchHit } from "../types/plugins.js"; +import { boundedInteger, MAX_RECALL_LIMIT } from "../utils/limits.js"; import { cosineSimilarity } from "../utils/math.js"; export class InMemoryFamiliarityVectorStore implements FamiliarityVectorStore { @@ -14,6 +15,8 @@ export class InMemoryFamiliarityVectorStore implements FamiliarityVectorStore { } async search(vector: number[], topK: number): Promise { + boundedInteger("topK", topK, MAX_RECALL_LIMIT); + if (topK === 0) return []; return [...this.records.values()] .map((record) => ({ id: record.id, @@ -37,6 +40,8 @@ export class InMemoryChunkVectorStore implements ChunkVectorStore { } async searchByMemoryId(memoryId: string, vector: number[], topK: number): Promise { + boundedInteger("topK", topK, MAX_RECALL_LIMIT); + if (topK === 0) return []; return (this.chunksByMemoryId.get(memoryId) ?? []) .map((chunk) => ({ id: chunk.chunkId, diff --git a/src/types/memory.ts b/src/types/memory.ts index 8e2d40c..29d17a0 100644 --- a/src/types/memory.ts +++ b/src/types/memory.ts @@ -108,7 +108,10 @@ export interface ImpressionScanResult { export interface RecallInput { text: string; + /** Final candidate output limit (default 5, maximum 1000). */ topK?: number; + /** Opt-in search breadth before ranking; defaults to topK and must be >= topK. */ + candidateLimit?: number; loadChunks?: boolean; chunkLimit?: number; } diff --git a/src/types/plugins.ts b/src/types/plugins.ts index 2932a5c..893f997 100644 --- a/src/types/plugins.ts +++ b/src/types/plugins.ts @@ -8,8 +8,16 @@ import type { StoredMemory, } from "./memory.js"; +export interface EmbeddingScope { + embed(text: string): Promise; + /** Must synchronously release this scope and prevent late cache insertion; must not throw. */ + dispose(): void; +} + export interface EmbeddingProvider { embed(text: string): Promise; + /** Optional independent cache ownership; implementations must not share entries across scopes. */ + createScope?(): EmbeddingScope; } export interface Chunker { @@ -92,6 +100,14 @@ export interface SemanticRecallEngineConfig { scoringStrategy?: ScoringStrategy; chunker?: Chunker; summaryGenerator?: SummaryGenerator; + /** Opt in to preserving summaries/chunks/vectors for importance/metadata-only updates. */ + preserveDerivedOnMetadataUpdate?: boolean; + /** Exact, bounded, host-authored query routing. No automatic translation. */ + cueAliases?: readonly import("../retrieval/cue-aliases.js").CueAlias[]; + /** Opt in to per-memory/revision cache ownership and serialized reads; requires provider.createScope. */ + manageEmbeddingCache?: boolean; + /** Serialize this engine instance's reads with writes; adapters shared by other engines need transactions. */ + consistentReads?: boolean; thresholds?: { strong: number; weak: number; diff --git a/src/utils/limits.ts b/src/utils/limits.ts new file mode 100644 index 0000000..b15ca1f --- /dev/null +++ b/src/utils/limits.ts @@ -0,0 +1,9 @@ +/** Resource limits are validated before adapters are called; zero means no results. */ +export const MAX_RECALL_LIMIT = 1000; + +export function boundedInteger(name: string, value: number, max: number, min = 0): number { + if (!Number.isSafeInteger(value) || value < min || value > max) { + throw new RangeError(`${name} must be an integer between ${min} and ${max}`); + } + return value; +} diff --git a/tests/cli-errors.test.ts b/tests/cli-errors.test.ts new file mode 100644 index 0000000..7447239 --- /dev/null +++ b/tests/cli-errors.test.ts @@ -0,0 +1,68 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { CliResponseError, parseExpectedCliFailure, requireSpawnResult } from "./helpers/cli-errors.js"; + +for (const code of ["EPERM", "EACCES", "ENOENT", "ETIMEDOUT", "ENOBUFS"]) { + test(`expected-failure helper preserves original ${code}, even with misleading status/output`, () => { + for (const status of [0, 1, null]) { + const cause = new Error("original cause"); + const original = Object.assign(new Error("spawn failed", { cause }), { + code, status, stdout: status === 1 ? '{"ok":false}' : "", stderr: "synthetic stderr", + path: "synthetic-node", syscall: "spawnSync", spawnargs: ["synthetic-cli"], + }); + assert.throws(() => parseExpectedCliFailure(original), error => error === original); + assert.equal(original.cause, cause); + } + }); +} + +test("ordinary nonzero CLI JSON remains available for unchanged product assertions", () => { + for (const status of [1, 2]) { + const error = Object.assign(new Error("nonzero exit"), { status, signal: null, stdout: '{"ok":false,"diagnostics":[{"code":"invalid"}]}' }); + assert.deepEqual(parseExpectedCliFailure(error), { ok: false, diagnostics: [{ code: "invalid" }] }); + } +}); + +test("invalid or empty CLI JSON preserves both execution and parse errors", () => { + for (const stdout of ["", " \n", "{broken", "synthetic-sensitive-output"]) { + const original = Object.assign(new Error("nonzero exit"), { status: 1, signal: null, stdout, stderr: "synthetic stderr" }); + assert.throws(() => parseExpectedCliFailure(original), error => { + assert.ok(error instanceof CliResponseError); + assert.equal(error.cause, original); assert.equal(error.exitStatus, 1); + assert.ok(error.parseError instanceof SyntaxError); + assert.ok(!error.message.includes("synthetic-sensitive-output")); return true; + }); + } +}); + +test("signals, nested operational errors and invalid statuses are not accepted as CLI responses", () => { + for (const extra of [{ signal: "SIGTERM" }, { error: new Error("transport") }, + ...[undefined, null, 0, -1, 1.5, NaN, Infinity].map(status => ({ status })), { stdout: undefined }]) { + const error = Object.assign(new Error("failed"), { status: 1, stdout: '{"ok":false}', ...extra }); + assert.throws(() => parseExpectedCliFailure(error), value => value === error); + } +}); + +test("non-execution and non-object errors retain their identity", () => { + for (const error of [new SyntaxError("original parse failure"), null, undefined, "failure", 42]) { + let caught: unknown = Symbol("not thrown"); + try { parseExpectedCliFailure(error); } catch (value) { caught = value; } + assert.equal(caught, error); + } +}); + +test("spawn-result guard preserves native errors regardless of status and leaves product output unchanged", () => { + const original = Object.assign(new Error("native spawn error"), { code: "EPERM", syscall: "spawnSync", path: "synthetic-node" }); + for (const status of [0, 1, null]) { + assert.throws(() => requireSpawnResult({ error: original, status }), error => error === original); + } + // Nonzero exit status is a product response, not a spawn error. Callers still + // assert its status and parsed output in their existing product tests. + for (const status of [0, 1]) { + const result = { error: undefined, status, stdout: '{"ok":false}' }; + assert.equal(requireSpawnResult(result), result); + } + let caught: unknown = "not thrown"; + try { requireSpawnResult({ error: null }); } catch (error) { caught = error; } + assert.equal(caught, null); // Preserve the old strict undefined guard. +}); diff --git a/tests/helpers/cli-errors.ts b/tests/helpers/cli-errors.ts new file mode 100644 index 0000000..35d6b93 --- /dev/null +++ b/tests/helpers/cli-errors.ts @@ -0,0 +1,34 @@ +/** Parse an expected CLI error response without hiding spawn/transport failures. */ +export function parseExpectedCliFailure(error: unknown): any { + if (typeof error !== "object" || error === null) throw error; + const failure = error as { status?: unknown; signal?: unknown; code?: unknown; error?: unknown; stdout?: unknown }; + // execFileSync throws for both operational errors and ordinary nonzero exits. + // Only the latter can be an expected JSON response. Keep the original object + // (code, syscall, path, args, stderr and cause) for every operational failure. + if (failure.code !== undefined || failure.error !== undefined || + (failure.signal !== null && failure.signal !== undefined) || + typeof failure.status !== "number" || !Number.isInteger(failure.status) || failure.status <= 0 || + typeof failure.stdout !== "string") { + throw error; + } + try { + return JSON.parse(failure.stdout); + } catch (parseError) { + throw new CliResponseError(failure.status, error, parseError); + } +} + +export class CliResponseError extends Error { + constructor(readonly exitStatus: number, cause: unknown, readonly parseError: unknown) { + // Do not copy arbitrary CLI stdout/stderr into the message. The original + // execution error remains available to the test runner through cause. + super(`CLI exited with status ${exitStatus} without a valid JSON response`, { cause }); + this.name = "CliResponseError"; + } +} + +/** Preserve a native spawn error instead of replacing it with an assertion error. */ +export function requireSpawnResult(result: T): T { + if (result.error !== undefined) throw result.error; + return result; +} diff --git a/tests/memory-cli.test.ts b/tests/memory-cli.test.ts index 5550cab..18d736b 100644 --- a/tests/memory-cli.test.ts +++ b/tests/memory-cli.test.ts @@ -4,6 +4,7 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { parseExpectedCliFailure } from "./helpers/cli-errors.js"; const root = process.cwd(); const unifiedCli = join(root, "scripts", "deja-vu.mjs"); @@ -19,10 +20,7 @@ function runJsonAllowFailure(script: string, args: string[], cwd: string) { try { return runJson(script, args, cwd); } catch (error) { - if (typeof error === "object" && error !== null && "stdout" in error) { - return JSON.parse(String((error as { stdout: string }).stdout)); - } - throw error; + return parseExpectedCliFailure(error); } } diff --git a/tests/package-pack.test.ts b/tests/package-pack.test.ts index 6ebfadc..15c3800 100644 --- a/tests/package-pack.test.ts +++ b/tests/package-pack.test.ts @@ -17,7 +17,7 @@ test("package metadata exposes memory CLI binaries and starter kit", () => { const packRoot = mkdtempSync(join(tmpdir(), "dejavu-pack-")); const cache = join(packRoot, "npm-cache"); const result = JSON.parse( - execFileSync(process.execPath, [npmExecPath, "pack", "--json", "--pack-destination", packRoot], { + execFileSync(process.execPath, [npmExecPath, "pack", "--offline", "--json", "--pack-destination", packRoot], { cwd: root, encoding: "utf8", env: { ...process.env, npm_config_cache: cache }, @@ -46,7 +46,7 @@ test("package metadata exposes memory CLI binaries and starter kit", () => { const tarball = join(packRoot, result[0].filename); execFileSync( process.execPath, - [npmExecPath, "install", "--prefix", installRoot, "--ignore-scripts", "--no-audit", "--no-fund", tarball], + [npmExecPath, "install", "--offline", "--prefix", installRoot, "--ignore-scripts", "--no-audit", "--no-fund", tarball], { cwd: packRoot, encoding: "utf8", env: { ...process.env, npm_config_cache: cache } }, ); diff --git a/tests/recall-efficiency.test.ts b/tests/recall-efficiency.test.ts new file mode 100644 index 0000000..a37c0af --- /dev/null +++ b/tests/recall-efficiency.test.ts @@ -0,0 +1,347 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + createInMemorySemanticRecallEngine, InMemoryStorage, InMemoryChunkVectorStore, + InMemoryFamiliarityVectorStore, CachedEmbeddingProvider, MockEmbeddingProvider, + DefaultChunker, DefaultSummaryGenerator, projectRecallForPrompt, lexicalSimilarity, + type FamiliarityRecord, type EmbeddingProvider, type RecallResult, type RecallInput, +} from "../src/index.js"; + +const now = () => new Date().toISOString(); +function familiarity(id: string, tokens: string[], importance = 0.5, lastAccessedAt = now()): FamiliarityRecord { + return { id, title: id, shortSummary: id, tags: [], impressionTokens: tokens, importance, lastAccessedAt, embeddingVector: [1] }; +} +class CountingProvider extends MockEmbeddingProvider { + calls = 0; + async embed(text: string): Promise { this.calls++; return super.embed(text); } +} + +test("eligible lexical match cannot be hidden by a recent ineligible hybrid winner", async () => { + const storage = new InMemoryStorage(); + const query = Array.from({ length: 10 }, (_, i) => `aa${i}`); + await storage.saveFamiliarity(familiarity("perfect-old", query, 0, "2000-01-01T00:00:00Z")); + await storage.saveFamiliarity(familiarity("partial-new", [...query.slice(0, 7), "xx1", "xx2", "xx3"], 1)); + const engine = createInMemorySemanticRecallEngine({ storage }); + const scan = await engine.scanImpressions(query.join(" ")); + assert.equal(scan.familiarityLevel, "strong"); + assert.equal(scan.topMatch?.id, "perfect-old"); + assert.equal(scan.candidates[1].familiarityLevel, "none"); + assert.ok(scan.candidates[1].score > scan.candidates[0].score); + assert.equal((await engine.recall(query.join(" "))).matched, true); +}); + +test("candidateLimit widens search without widening output; default breadth stays compatible", async () => { + const storage = new InMemoryStorage(); + for (let i = 0; i < 6; i++) await storage.saveFamiliarity(familiarity(`tie${i}`, ["alpha"], i === 5 ? 1 : 0)); + const engine = createInMemorySemanticRecallEngine({ storage }); + assert.equal((await engine.scanImpressions("alpha")).topMatch?.id, "tie0"); + const wider = await engine.scanImpressions({ text: "alpha", topK: 1, candidateLimit: 6 }); + assert.equal(wider.topMatch?.id, "tie5"); + assert.equal(wider.candidates.length, 1); + const none = await engine.scanImpressions("absent"); + assert.equal(none.matched, false); + assert.equal(none.topMatch, null); +}); + +test("hybrid order remains intact among eligible weak and strong candidates", async () => { + const storage = new InMemoryStorage(); + const query = Array.from({ length: 10 }, (_, i) => `aa${i}`); + await storage.saveFamiliarity(familiarity("strong", query, 0, "2000-01-01T00:00:00Z")); + await storage.saveFamiliarity(familiarity("weak", [...query.slice(0, 8), "xx1", "xx2"], 1)); + const scan = await createInMemorySemanticRecallEngine({ storage }).scanImpressions(query.join(" ")); + assert.equal(scan.topMatch?.id, "weak"); + assert.equal(scan.familiarityLevel, "weak"); +}); + +for (const limit of [-1, NaN, Infinity, -Infinity, 1.5, 1001, Number.MAX_SAFE_INTEGER]) { + test(`rejects invalid recall limits before I/O: ${limit}`, async () => { + const engine = createInMemorySemanticRecallEngine({ storage: new Proxy(new InMemoryStorage(), { + get() { throw new Error("unexpected storage access"); }, + }) }); + for (const key of ["topK", "candidateLimit", "chunkLimit"] as const) { + const query: RecallInput = { text: "alpha", [key]: limit }; + await assert.rejects(engine.scanImpressions(query), RangeError); + await assert.rejects(engine.recall(query), RangeError); + } + await assert.rejects(new InMemoryStorage().searchImpressions([], limit), RangeError); + await assert.rejects(new InMemoryFamiliarityVectorStore().search([1], limit), RangeError); + await assert.rejects(new InMemoryChunkVectorStore().searchByMemoryId("id", [1], limit), RangeError); + }); +} + +test("zero limits are empty and avoid unnecessary embedding; inconsistent breadth rejects", async () => { + const provider = new CountingProvider(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: provider }); + await engine.addMemory({ id: "alpha", title: "alpha", content: "alpha details" }); + provider.calls = 0; + const empty = await engine.recall({ text: "alpha", topK: 0, loadChunks: true }); + assert.equal(empty.matched, false); + assert.deepEqual(empty.candidates, []); + const summary = await engine.recall({ text: "alpha", loadChunks: true, chunkLimit: 0 }); + assert.ok(summary.summaryIfLoaded); + assert.deepEqual(summary.chunksIfLoaded, []); + assert.equal(provider.calls, 0); + await assert.rejects(engine.scanImpressions({ text: "alpha", topK: 5, candidateLimit: 4 }), RangeError); + await assert.rejects(engine.scanImpressions({ text: 42 } as unknown as RecallInput), TypeError); +}); + +test("thresholds and chunk sizes reject unsafe numeric inputs without retuning defaults", () => { + for (const thresholds of [{ weak: NaN, strong: 1 }, { weak: 0.9, strong: 0.8 }, { weak: -1, strong: 1 }, { weak: 0, strong: Infinity }]) { + assert.throws(() => createInMemorySemanticRecallEngine({ thresholds }), RangeError); + } + const chunker = new DefaultChunker(); + for (const value of [-1, 0, Infinity, NaN, 1.5, 1_000_001]) { + assert.throws(() => chunker.chunk({ title: "test", content: "alpha", chunkStrategy: { maxCharacters: value } }), RangeError); + } + assert.throws(() => chunker.chunk({ title: "test", content: "alpha", chunkStrategy: { maxCharacters: 10, overlapCharacters: 10 } }), RangeError); +}); + +test("cached token scoring equals legacy lexical scoring across multilingual inputs", async () => { + const storage = new InMemoryStorage(); + const tokenSets = [ + ["alpha", "beta", "beta"], ["設定同步", "資料"], ["café", "naïve"], + ["東京", "sync"], ["Über", "STRASSE", "sync"], ["e\u0301", "é", "sync"], + ["a", "b", "sync"], ["alpha-beta", "sync"], + ]; + for (const [i, tokens] of tokenSets.entries()) await storage.saveFamiliarity(familiarity(`r${i}`, tokens)); + for (const query of [...tokenSets, ["sync"], ["設定同步", "sync"], ["missing"], []]) { + const expected = tokenSets.map((tokens, i) => ({ id: `r${i}`, tokens })) + .filter((r) => query.some((t) => r.tokens.includes(t))) + .map((r) => ({ id: r.id, similarity: lexicalSimilarity(query.join(" "), r.tokens.join(" ")) })) + .sort((a, b) => b.similarity - a.similarity); + const actual = await storage.searchImpressions(query, 100); + // For equal scores, postings order (first query token, then insertion) is preserved. + assert.deepEqual(actual.map((h) => ({ id: h.record.id, similarity: h.similarity })).sort((a, b) => a.id.localeCompare(b.id)), + expected.sort((a, b) => a.id.localeCompare(b.id))); + } +}); + +test("token cache is replaced on save, removed on delete, and isolated from returned mutations", async () => { + const storage = new InMemoryStorage(); + const original = familiarity("id", ["old", "設定同步"]); + await storage.saveFamiliarity(original); + original.impressionTokens.push("external"); + const returned = (await storage.searchImpressions(["old"], 5))[0].record; + returned.impressionTokens.push("mutated"); + (await storage.getFamiliarity("id"))!.impressionTokens.push("getter"); + (await storage.listFamiliarities())[0].impressionTokens.push("list"); + assert.deepEqual((await storage.getFamiliarity("id"))!.impressionTokens, ["old", "設定同步"]); + await storage.saveFamiliarity(familiarity("id", ["new", "東京"])); + assert.deepEqual(await storage.searchImpressions(["old", "設定同步"], 5), []); + assert.equal((await storage.searchImpressions(["new", "東京"], 5))[0].similarity, 1); + await storage.deleteMemory("id"); + assert.deepEqual(await storage.searchImpressions(["new", "東京"], 5), []); + await storage.saveFamiliarity(familiarity("id", ["replacement"])); + assert.deepEqual(await storage.searchImpressions(["new"], 5), []); +}); + +async function richRecall(): Promise { + const engine = createInMemorySemanticRecallEngine(); + await engine.addMemory({ id: "synthetic", title: "settings sync", content: "Decision: settings sync. " + "設定同步 café 😀資料 ".repeat(150), + metadata: { privateExtra: "DO_NOT_PROJECT" }, recentUpdates: ["更新😀".repeat(1000)] }); + return engine.recall({ text: "settings sync", loadChunks: true }); +} + +test("prompt projection allowlists data, enforces whole-output bytes and preserves Unicode", async () => { + const result = await richRecall(); + const original = JSON.stringify(result); + const projection = projectRecallForPrompt(result, { maxBytes: 512 }); + assert.ok(projection.truncated); + assert.ok(projection.bytes <= 512); + assert.equal(projection.bytes, Buffer.byteLength(projection.text)); + assert.ok(!projection.text.includes("embeddingVector")); + assert.ok(!projection.text.includes("DO_NOT_PROJECT")); + assert.ok(!projection.text.includes("candidates")); + for (const line of projection.text.split("\n")) { + const record = JSON.parse(line); + assert.ok(["summary", "chunk"].includes(record.kind)); + assert.ok(!/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + const result = await richRecall(); + const countTokens = (text: string) => Array.from(text).length; + for (const maxTokens of [0, 1, 80, 250]) { + const projected = projectRecallForPrompt(result, { maxBytes: 500, maxTokens, countTokens }); + assert.ok(projected.tokens! <= maxTokens); + assert.equal(projected.tokens, countTokens(projected.text)); + assert.ok(Buffer.byteLength(projected.text) <= 500); + } + assert.equal(projectRecallForPrompt(result, { maxBytes: 0 }).text, ""); + assert.equal(projectRecallForPrompt(result, { maxBytes: 1 }).text, ""); + result.summaryIfLoaded!.id = "x".repeat(100_000); + assert.equal(projectRecallForPrompt(result, { maxBytes: 100 }).text, ""); + assert.throws(() => projectRecallForPrompt(result, { maxTokens: 10 }), TypeError); + for (const value of [-1, NaN, Infinity, 0.5, 1_048_577]) { + assert.throws(() => projectRecallForPrompt(result, { maxBytes: value }), RangeError); + assert.throws(() => projectRecallForPrompt(result, { maxTokens: value, countTokens }), RangeError); + } + assert.throws(() => projectRecallForPrompt(result, { countTokens: () => NaN }), RangeError); + const empty = await createInMemorySemanticRecallEngine().recall("absent"); + assert.deepEqual(projectRecallForPrompt(empty), { text: "", bytes: 0, truncated: false }); +}); + +test("opt-in metadata updates preserve vectors/chunk IDs and use zero generation calls", async () => { + const provider = new CountingProvider(); + const storage = new InMemoryStorage(); + const engine = createInMemorySemanticRecallEngine({ storage, embeddingProvider: provider, preserveDerivedOnMetadataUpdate: true }); + await engine.addMemory({ id: "id", title: "alpha", content: "Decision: alpha. " + "details ".repeat(100) }); + const before = await storage.getMemorySnapshot("id"); + provider.calls = 0; + await engine.updateMemory("id", { importance: 0.9, metadata: { source: "synthetic", title: "cannot override" } }); + const after = await storage.getMemorySnapshot("id"); + assert.equal(provider.calls, 0); + assert.deepEqual(after!.chunks, before!.chunks); + assert.deepEqual(after!.summary.embeddingVector, before!.summary.embeddingVector); + assert.equal(after!.createdAt, before!.createdAt); + assert.equal(after!.familiarity.importance, 0.9); + assert.equal(after!.summary.metadata.title, "alpha"); + assert.equal(after!.summary.metadata.source, "synthetic"); + await engine.updateMemory("id", { content: "beta replacement", title: "beta" }); + assert.ok(provider.calls > 0); + assert.equal((await engine.recall("alpha")).matched, false); + assert.equal((await engine.recall("beta")).matched, true); + await engine.deleteMemory("id"); + assert.equal((await engine.recall("beta")).matched, false); +}); + +test("default metadata updates still invoke custom generators", async () => { + const provider = new CountingProvider(); + const generator = new DefaultSummaryGenerator(); + let calls = 0; + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: provider, summaryGenerator: { + generateShortSummary: (input) => generator.generateShortSummary(input), + generateStructuredSummary: async (input) => { calls++; return generator.generateStructuredSummary(input); }, + } }); + await engine.addMemory({ id: "id", title: "alpha", content: "alpha" }); + provider.calls = 0; calls = 0; + await engine.updateMemory("id", { metadata: { value: "new" } }); + assert.equal(calls, 1); + assert.ok(provider.calls > 0); +}); + +test("metadata fast path retains compensated rollback", async () => { + const storage = new InMemoryStorage(); + let armed = false; + const adapter = new Proxy(storage, { get(target, key) { + const value = Reflect.get(target, key); + if (key === "saveSummary") return async (...args: Parameters) => { + if (armed) { armed = false; throw new Error("injected write failure"); } + return target.saveSummary(...args); + }; + return typeof value === "function" ? value.bind(target) : value; + } }); + const engine = createInMemorySemanticRecallEngine({ storage: adapter, preserveDerivedOnMetadataUpdate: true }); + await engine.addMemory({ id: "id", title: "alpha", content: "alpha" }); + const before = await storage.getMemorySnapshot("id"); + armed = true; + await assert.rejects(engine.updateMemory("id", { importance: 0.9 }), /previous state restored/); + assert.deepEqual(await storage.getMemorySnapshot("id"), before); + assert.equal((await engine.recall("alpha")).matched, true); +}); + +test("embedding cache keys exact content and model version, isolates vector mutation and evicts LRU", async () => { + let version = "provider/model:v1"; + let calls = 0; + const provider: EmbeddingProvider = { async embed(text) { calls++; return [text.length, calls]; } }; + const cache = new CachedEmbeddingProvider(provider, { modelKey: () => version, maxEntries: 2 }); + const first = await cache.embed("設定同步"); first[0] = 999; + assert.notEqual((await cache.embed("設定同步"))[0], 999); + assert.equal(calls, 1); + await cache.embed("alpha"); await cache.embed("設定同步"); await cache.embed("beta"); + await cache.embed("alpha"); assert.equal(calls, 4); // alpha was LRU + version = "provider/model:v2"; + await cache.embed("alpha"); assert.equal(calls, 5); + await cache.embed("Alpha"); assert.equal(calls, 6); + cache.clear(); await cache.embed("alpha"); assert.equal(calls, 7); +}); + +test("embedding cache bounds bytes, bypasses oversized entries and does not cache failures", async () => { + let calls = 0; + // SHA-256 hex digest accounts for 128 bytes; two vector values add 16. + // One entry fits exactly. Oversized vectors still bypass retention. + const cache = new CachedEmbeddingProvider({ async embed(text) { calls++; return text.length > 10 ? Array(100).fill(1) : [1, 2]; } }, { modelKey: "v1", maxBytes: 144 }); + await cache.embed("a"); await cache.embed("a"); assert.equal(calls, 1); + await cache.embed("b"); await cache.embed("a"); assert.equal(calls, 3); + await cache.embed("x".repeat(100)); await cache.embed("x".repeat(100)); assert.equal(calls, 5); + let attempt = 0; + const failing = new CachedEmbeddingProvider({ async embed() { if (++attempt === 1) throw new Error("retry"); return [1]; } }, { modelKey: "v1" }); + await assert.rejects(failing.embed("same"), /retry/); + await failing.embed("same"); await failing.embed("same"); assert.equal(attempt, 2); + for (const vector of [[], [NaN], [Infinity], new Array(3)]) { + const invalid = new CachedEmbeddingProvider({ async embed() { return vector; } }, { modelKey: "v1" }); + await assert.rejects(invalid.embed("x"), TypeError); + } + for (const value of [-1, NaN, Infinity, 1.5, Number.MAX_SAFE_INTEGER]) { + assert.throws(() => new CachedEmbeddingProvider(new MockEmbeddingProvider(), { modelKey: "v1", maxEntries: value }), RangeError); + assert.throws(() => new CachedEmbeddingProvider(new MockEmbeddingProvider(), { modelKey: "v1", maxBytes: value }), RangeError); + } + assert.throws(() => new CachedEmbeddingProvider(new MockEmbeddingProvider(), { modelKey: "" }), TypeError); +}); + +test("clear or model changes while embedding is pending cannot populate an obsolete cache", async () => { + let complete!: (vector: number[]) => void; + let calls = 0; + let model = "v1"; + const cache = new CachedEmbeddingProvider({ embed: async () => { + calls++; + if (calls <= 2) return new Promise((resolve) => { complete = resolve; }); + return [calls]; + } }, { modelKey: () => model }); + const pending = cache.embed("text"); cache.clear(); complete([1]); await pending; + const pending2 = cache.embed("text"); model = "v2"; complete([2]); await pending2; + model = "v1"; await cache.embed("text"); assert.equal(calls, 3); +}); + +test("cached embeddings do not resurrect updated or deleted memory", async () => { + const provider = new CountingProvider(); + const cache = new CachedEmbeddingProvider(provider, { modelKey: "mock:v1", maxEntries: 32 }); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache }); + await engine.addMemory({ id: "id", title: "alpha", content: "alpha" }); + provider.calls = 0; + await engine.recall({ text: "alpha", loadChunks: true }); + await engine.recall({ text: "alpha", loadChunks: true }); + assert.ok(provider.calls <= 1); + await engine.updateMemory("id", { title: "beta", content: "beta" }); + assert.equal((await engine.recall("alpha")).matched, false); + assert.equal((await engine.recall({ text: "beta", loadChunks: true })).chunksIfLoaded[0].content, "beta"); + await engine.deleteMemory("id"); + assert.equal((await engine.recall("beta")).matched, false); +}); + +test("cached scoring preserves exact scores and posting tie order on seeded mixed-language queries", async () => { + let seed = 42; + const pick = (n: number) => { seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; return seed % n; }; + const terms = ["alpha", "beta", "café", "設定同步", "東京", "данные", "sync", "c++", "a", "e\u0301"]; + const storage = new InMemoryStorage(); + const rows = Array.from({ length: 40 }, (_, i) => familiarity(`id${i}`, Array.from({ length: 12 }, () => terms[pick(terms.length)]))); + for (const row of rows) await storage.saveFamiliarity(row); + for (let i = 0; i < 100; i++) { + const query = Array.from({ length: 6 }, () => terms[pick(terms.length)]); + const ids = new Set(); + for (const token of query) for (const row of rows) if (row.impressionTokens.includes(token)) ids.add(row.id); + const expected = [...ids].map((id) => { + const row = rows.find((r) => r.id === id)!; + return [id, lexicalSimilarity(query.join(" "), row.impressionTokens.join(" "))] as const; + }).sort((a, b) => b[1] - a[1]).slice(0, 5); + const actual = (await storage.searchImpressions(query, 5)).map((hit) => [hit.record.id, hit.similarity]); + assert.deepEqual(actual, expected); + } +}); + +test("concurrent embedding misses remain bounded and disabled caches never retain entries", async () => { + let calls = 0; + const provider = { async embed(text: string) { calls++; await Promise.resolve(); return [text.length]; } }; + const disabled = new CachedEmbeddingProvider(provider, { modelKey: "v1", maxEntries: 0 }); + await disabled.embed("a"); await disabled.embed("a"); assert.equal(calls, 2); + const cache = new CachedEmbeddingProvider(provider, { modelKey: "v1", maxEntries: 1 }); + await Promise.all([cache.embed("a"), cache.embed("a"), cache.embed("b")]); + const before = calls; + await cache.embed("b"); assert.equal(calls, before); + await cache.embed("a"); assert.equal(calls, before + 1); +}); diff --git a/tests/recall-lifecycle.test.ts b/tests/recall-lifecycle.test.ts new file mode 100644 index 0000000..5e94484 --- /dev/null +++ b/tests/recall-lifecycle.test.ts @@ -0,0 +1,295 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { + CachedEmbeddingProvider, createInMemorySemanticRecallEngine, InMemoryStorage, + InMemoryChunkVectorStore, InMemoryFamiliarityVectorStore, MemoryWriteError, + MemorySupersedeError, projectRecallForPrompt, type EmbeddingProvider, type RecallResult, +} from "../src/index.js"; + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(r => { resolve = r; }); + return { promise, resolve }; +} +function counted() { + let calls = 0; + const cache = new CachedEmbeddingProvider({ async embed(text) { calls++; return [text.length, 1]; } }, { modelKey: "synthetic:v1" }); + return { cache, calls: () => calls }; +} +function fault(target: T, method: keyof T, shouldFail: () => boolean): T { + return new Proxy(target, { get(object, key) { + const value = Reflect.get(object, key); + if (key === method) return async (...args: unknown[]) => { + if (shouldFail()) throw new Error(`injected ${String(method)}`); + if (typeof value !== "function") throw new TypeError("fault target must be callable"); + return value.apply(object, args); + }; + return typeof value === "function" ? value.bind(object) : value; + } }); +} + +test("aliases are bounded exact queries, copied at construction, and opt-in", async () => { + const aliases = [{ alias: "還原備份", cue: "restoreroute" }, { alias: "restore archive", cue: "restoreroute" }]; + const engine = createInMemorySemanticRecallEngine({ cueAliases: aliases }); + const plain = createInMemorySemanticRecallEngine(); + for (const e of [engine, plain]) await e.addMemory({ id: "id", title: "restoreroute", content: "synthetic" }); + aliases[0].cue = "changed"; + assert.equal((await engine.recall("還原備份")).topMatch?.id, "id"); + assert.equal((await engine.recall(" RESTORE archive ")).topMatch?.id, "id"); + assert.equal((await plain.recall("還原備份")).matched, false); + for (const query of ["不要還原備份", "do not restore archive", "restore archive price"]) { + assert.equal((await engine.recall(query)).matched, false); + } + await engine.updateMemory("id", { title: "otherroute" }); + assert.equal((await engine.recall("還原備份")).matched, false); + await engine.deleteMemory("id"); + assert.equal((await engine.recall("還原備份")).matched, false); + for (const cueAliases of [Array(129).fill({ alias: "a", cue: "b" }), [{ alias: "", cue: "b" }], + [{ alias: "a", cue: "x".repeat(257) }], [{ alias: "A", cue: "one" }, { alias: " a ", cue: "two" }]]) { + assert.throws(() => createInMemorySemanticRecallEngine({ cueAliases })); + } +}); + +test("scopes isolate identical content; disposing one owner preserves the other and the root", async () => { + const { cache, calls } = counted(); + const a = cache.createScope(), b = cache.createScope(); + await a.embed("identical"); await b.embed("identical"); await cache.embed("identical"); + assert.equal(calls(), 3); assert.equal(cache.getStats().entries, 3); + a.dispose(); a.dispose(); + assert.equal(cache.getStats().entries, 2); + await b.embed("identical"); await cache.embed("identical"); assert.equal(calls(), 3); + await assert.rejects(a.embed("identical"), /disposed/); + b.dispose(); cache.clear(); assert.deepEqual(cache.getStats(), { entries: 0, accountedBytes: 0 }); +}); + +test("disposed scopes and global clearing invalidate in-flight retention without pretending to cancel work", async () => { + const pending = deferred(); + const cache = new CachedEmbeddingProvider({ embed: () => pending.promise }, { modelKey: "v1" }); + const scope = cache.createScope(); + const call = scope.embed("pending"); scope.dispose(); pending.resolve([1]); + assert.deepEqual(await call, [1]); assert.equal(cache.getStats().entries, 0); + const pending2 = deferred(); + const second = new CachedEmbeddingProvider({ embed: () => pending2.promise }, { modelKey: "v1" }); + const scoped = second.createScope(); const call2 = scoped.embed("pending"); second.clear(); pending2.resolve([1]); + await call2; assert.equal(second.getStats().entries, 0); +}); + +test("digest keys preserve exact surrogate content and model framing; accounted bounds include digests", async () => { + let calls = 0; let model = "ab"; + const cache = new CachedEmbeddingProvider({ async embed() { return [++calls]; } }, { modelKey: () => model, maxBytes: 136 }); + await cache.embed("c"); await cache.embed("c"); assert.equal(calls, 1); + assert.equal(cache.getStats().accountedBytes, 136); + model = "a"; await cache.embed("bc"); assert.equal(calls, 2); + await cache.embed("\ud800"); await cache.embed("\ufffd"); assert.equal(calls, 4); + await cache.embed("\ufffd"); assert.equal(calls, 4); + assert.ok(cache.getStats().entries <= 1); + const disabled = new CachedEmbeddingProvider({ async embed() { return [++calls]; } }, { modelKey: "v1", maxBytes: 135 }); + await disabled.embed("large".repeat(10_000)); await disabled.embed("large".repeat(10_000)); + assert.equal(calls, 6); assert.equal(disabled.getStats().entries, 0); +}); + +test("engine ownership releases update/delete revisions without clearing another memory", async () => { + const { cache, calls } = counted(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, preserveDerivedOnMetadataUpdate: true }); + for (const id of ["a", "b"]) await engine.addMemory({ id, title: "identical", content: "identical" }); + const entries = cache.getStats().entries; + assert.ok(entries > 0 && entries % 2 === 0); + const before = calls(); await engine.updateMemory("a", { importance: 0.9 }); + assert.equal(calls(), before); assert.equal(cache.getStats().entries, entries); + await engine.updateMemory("a", { title: "changed", content: "changed" }); + assert.ok(calls() > before); assert.equal(cache.getStats().entries, entries); + await engine.deleteMemory("a"); assert.equal(cache.getStats().entries, entries / 2); + const beforeB = calls(); await engine.recall({ text: "identical", loadChunks: true }); + assert.equal(calls(), beforeB); // The chunk text is already retained in b's scope. + await engine.deleteMemory("b"); assert.equal(cache.getStats().entries, 0); + assert.equal((await engine.recall("identical")).matched, false); + assert.throws(() => createInMemorySemanticRecallEngine({ embeddingProvider: { async embed() { return [1]; } }, manageEmbeddingCache: true }), /createScope/); +}); + +test("failed preparation and failed commit dispose new scopes but retain the previous revision", async () => { + const storage = new InMemoryStorage(); let fail = false; + const { cache } = counted(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, + storage: fault(storage, "saveSummary", () => { const value = fail; fail = false; return value; }) }); + await engine.addMemory({ id: "id", title: "original", content: "original" }); + const entries = cache.getStats().entries, snapshot = await storage.getMemorySnapshot("id"); + fail = true; await assert.rejects(engine.updateMemory("id", { title: "new", content: "new" }), MemoryWriteError); + assert.equal(cache.getStats().entries, entries); + assert.deepEqual(await storage.getMemorySnapshot("id"), snapshot); + let attempts = 0; + const failureCache = new CachedEmbeddingProvider({ async embed() { if (++attempts === 2) throw new Error("provider failure"); return [1]; } }, { modelKey: "v1" }); + const other = createInMemorySemanticRecallEngine({ embeddingProvider: failureCache, manageEmbeddingCache: true }); + await assert.rejects(other.addMemory({ title: "new", content: "distinct" }), /provider failure/); + assert.equal(failureCache.getStats().entries, 0); +}); + +for (const stage of ["familiarity", "chunks", "storage"] as const) { + test(`delete compensates a ${stage} failure and retains valid cache ownership`, async () => { + const storage = new InMemoryStorage(), familiarity = new InMemoryFamiliarityVectorStore(), chunks = new InMemoryChunkVectorStore(); + const { cache } = counted(); let armed = false; + const failOnce = () => { const value = armed; armed = false; return value; }; + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, + storage: stage === "storage" ? fault(storage, "deleteMemory", failOnce) : storage, + familiarityVectorStore: stage === "familiarity" ? fault(familiarity, "remove", failOnce) : familiarity, + chunkVectorStore: stage === "chunks" ? fault(chunks, "removeByMemoryId", failOnce) : chunks }); + await engine.addMemory({ id: "id", title: "route", content: "route" }); + const snapshot = await storage.getMemorySnapshot("id"), stats = cache.getStats(); + armed = true; await assert.rejects(engine.deleteMemory("id"), (error: unknown) => { + assert.ok(error instanceof MemoryWriteError); assert.equal(error.recoveryErrors.length, 0); return true; + }); + assert.deepEqual(await storage.getMemorySnapshot("id"), snapshot); + assert.deepEqual(cache.getStats(), stats); + assert.equal((await familiarity.search(snapshot!.familiarity.embeddingVector, 5))[0].id, "id"); + assert.equal((await chunks.searchByMemoryId("id", snapshot!.chunks[0].embeddingVector, 5)).length, snapshot!.chunks.length); + assert.equal((await engine.recall({ text: "route", loadChunks: true })).chunksIfLoaded[0].content, "route"); + }); +} + +test("delete compensation errors expose a detached recovery snapshot", async () => { + const storage = new InMemoryStorage(); let armed = false; + const adapted = fault(fault(storage, "deleteMemory", () => armed), "saveSummary", () => armed); + const engine = createInMemorySemanticRecallEngine({ storage: adapted }); + await engine.addMemory({ id: "id", title: "route", content: "route" }); armed = true; + await assert.rejects(engine.deleteMemory("id"), (error: unknown) => { + assert.ok(error instanceof MemoryWriteError); assert.equal(error.recoveryErrors.length, 1); + assert.equal(error.recoverySnapshot?.rawContent, "route"); return true; + }); +}); + +test("supersede removes old cues and scopes, rejects ID collisions, and can delete the successor", async () => { + const { cache } = counted(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, consistentReads: true }); + await engine.addMemory({ id: "old", title: "obsoletecue", content: "old content" }); + await assert.rejects(engine.supersedeMemory("old", { id: "old", title: "x", content: "x" }), /exists/); + assert.deepEqual(await engine.supersedeMemory("old", { id: "new", title: "currentcue", content: "new content" }), { id: "new", supersededId: "old" }); + assert.equal((await engine.recall("obsoletecue")).matched, false); + assert.equal((await engine.recall("currentcue")).topMatch?.id, "new"); + assert.equal(await engine.getSummary("old"), null); + await engine.deleteMemory("new"); assert.equal(cache.getStats().entries, 0); +}); + +test("failed supersede restores old memory and removes the uncommitted successor", async () => { + const storage = new InMemoryStorage(); let armed = false; + const { cache } = counted(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, + storage: fault(storage, "deleteMemory", () => { const value = armed; armed = false; return value; }) }); + await engine.addMemory({ id: "old", title: "obsoletecue", content: "old content" }); + const snapshot = await storage.getMemorySnapshot("old"), entries = cache.getStats().entries; + armed = true; + await assert.rejects(engine.supersedeMemory("old", { id: "new", title: "currentcue", content: "new content" }), (error: unknown) => { + assert.ok(error instanceof MemorySupersedeError); assert.deepEqual(error.recoveryErrors, []); return true; + }); + assert.deepEqual(await storage.getMemorySnapshot("old"), snapshot); + assert.equal(await storage.getMemorySnapshot("new"), null); + assert.equal(cache.getStats().entries, entries); +}); + +test("managed ownership serializes pending recall before delete, then never returns a stale queued match", async () => { + const started = deferred(), finish = deferred(); let armed = false; + const provider: EmbeddingProvider = { async embed(text) { + if (armed && text === "route") { started.resolve(); return finish.promise; } + return [text.length, 1]; + } }; + const cache = new CachedEmbeddingProvider(provider, { modelKey: "v1" }); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true }); + await engine.addMemory({ id: "id", title: "route", content: "different content" }); + armed = true; const reading = engine.recall({ text: "route", loadChunks: true }); await started.promise; + let deleted = false; const deletion = engine.deleteMemory("id").then(() => { deleted = true; }); + const after = engine.recall("route"); await Promise.resolve(); assert.equal(deleted, false); + finish.resolve([1, 1]); assert.equal((await reading).matched, true); + await deletion; assert.equal((await after).matched, false); assert.equal(cache.getStats().entries, 0); +}); + +test("consistentReads hides an update's intermediate storage state", async () => { + const storage = new InMemoryStorage(), started = deferred(), finish = deferred(); let armed = false; + const adapted = new Proxy(storage, { get(target, key) { + if (key === "saveSummary") return async (...args: Parameters) => { + if (armed) { started.resolve(); await finish.promise; } return target.saveSummary(...args); + }; + const value = Reflect.get(target, key); return typeof value === "function" ? value.bind(target) : value; + } }); + const engine = createInMemorySemanticRecallEngine({ storage: adapted, consistentReads: true }); + await engine.addMemory({ id: "id", title: "oldcue", content: "old" }); + armed = true; const update = engine.updateMemory("id", { title: "newcue", content: "new" }); await started.promise; + const read = engine.recall("newcue"); finish.resolve(); await update; + assert.equal((await read).summaryIfLoaded?.description, "Decision: new"); +}); + +test("in-memory summaries and chunks do not expose mutable stored vectors or text", async () => { + const engine = createInMemorySemanticRecallEngine(); + await engine.addMemory({ id: "id", title: "cue", content: "original" }); + const summary = (await engine.getSummary("id"))!; summary.description = "mutated"; summary.embeddingVector[0] = 99; + const chunks = await engine.getChunks("id"); chunks[0].content = "mutated"; chunks[0].embeddingVector[0] = 99; + assert.equal((await engine.getSummary("id"))?.description, "Decision: original"); + assert.equal((await engine.getChunks("id"))[0].content, "original"); + assert.notEqual((await engine.getSummary("id"))?.embeddingVector[0], 99); +}); + +test("provenance marks clipped qualifiers inside the byte/token budget and points to full source", async () => { + const engine = createInMemorySemanticRecallEngine({ summaryGenerator: { + async generateShortSummary() { return "route"; }, + async generateStructuredSummary(input) { return { description: "Short content", context: input.context!, architectureOrIntent: "", recentUpdates: [], metadata: {} }; }, + } }); + const context = "Allowed for synthetic experiments. ".repeat(30) + "NEVER use production secrets."; + await engine.addMemory({ id: "id", title: "route", content: "Short content", context }); + const result = await engine.recall("route"); + const projected = projectRecallForPrompt(result, { maxBytes: 900, maxTokens: 900, countTokens: text => [...text].length, includeProvenance: true }); + assert.equal(projected.truncated, true); assert.ok(projected.bytes <= 900 && projected.tokens! <= 900); + const records = projected.text.split("\n").map(line => JSON.parse(line)); + const clipped = records.find(r => r.incomplete); + assert.ok(clipped); assert.match(clipped.warning, /qualifiers may be missing/); + assert.equal(clipped.rawSource, "getRawContent"); assert.equal(clipped.source, "getSummary"); assert.equal(clipped.memoryId, "id"); assert.equal(clipped.field, "context"); + assert.ok(!clipped.text.includes("NEVER")); assert.ok((await engine.getSummary(clipped.memoryId))?.context.endsWith("NEVER use production secrets.")); + assert.equal(projectRecallForPrompt(result, { maxBytes: 1, includeProvenance: true }).text, ""); + const weak = { ...result, familiarityLevel: "weak" } as RecallResult; + const weakProjection = projectRecallForPrompt(weak, { maxBytes: 8192, includeProvenance: true }); + assert.equal(weakProjection.truncated, true); + for (const row of weakProjection.text.split("\n").map(line => JSON.parse(line))) assert.equal(row.incomplete, true); +}); + +test("frozen synthetic development and held-out splits retain defaults and expose unseen paraphrase misses", async () => { + const fixture = JSON.parse(readFileSync("scripts/fixtures/recall-evaluation.json", "utf8")); + for (const split of Object.values(fixture.splits) as any[]) { + for (const enabled of [false, true]) { + const engine = createInMemorySemanticRecallEngine(enabled ? { cueAliases: split.memories.flatMap((m: any) => m.aliases.map((alias: string) => ({ alias, cue: m.id }))) } : {}); + for (const { aliases: _, ...memory } of split.memories) await engine.addMemory(memory); + let hits = 0, falseRecalls = 0; + for (const row of split.cases) { + const result = await engine.scanImpressions({ text: row.query, topK: 5, candidateLimit: 32 }); + const id = result.matched ? result.topMatch?.id : undefined; + if (id && row.relevant.includes(id)) hits++; + else if (id) falseRecalls++; + } + assert.equal(hits, enabled ? 10 : 2); assert.equal(falseRecalls, 0); + } + } +}); + +test("derived-summary provenance does not claim that an uncut summary preserves raw qualifiers", async () => { + const engine = createInMemorySemanticRecallEngine(); + const content = "Allowed experiments. ".repeat(50) + "Never use production secrets."; + await engine.addMemory({ id: "id", title: "route", content }); + const result = await engine.recall("route"); + assert.ok(!result.summaryIfLoaded!.description.includes("Never use production")); + const projection = projectRecallForPrompt(result, { includeProvenance: true }); + const record = JSON.parse(projection.text.split("\n")[0]); + assert.match(record.warning, /Derived summary/); + assert.equal(record.rawSource, "getRawContent"); + assert.equal(await engine.getRawContent(record.memoryId), content); +}); + +test("supersede cleanup failures identify both IDs and never claim a completed rollback", async () => { + const storage = new InMemoryStorage(); let failures = 0; + const { cache } = counted(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, + storage: fault(storage, "deleteMemory", () => failures-- > 0) }); + await engine.addMemory({ id: "old", title: "oldcue", content: "original" }); failures = 2; + await assert.rejects(engine.supersedeMemory("old", { id: "new", title: "newcue", content: "replacement" }), (error: unknown) => { + assert.ok(error instanceof MemorySupersedeError); + assert.equal(error.memoryId, "old"); assert.equal(error.replacementId, "new"); + assert.equal(error.recoveryErrors.length, 1); assert.match(error.message, /adapter repair/); return true; + }); + assert.ok(await storage.getMemorySnapshot("old")); assert.ok(await storage.getMemorySnapshot("new")); + await engine.deleteMemory("old"); await engine.deleteMemory("new"); assert.equal(cache.getStats().entries, 0); +}); diff --git a/tests/reliability.test.ts b/tests/reliability.test.ts index ed4d47d..c52772b 100644 --- a/tests/reliability.test.ts +++ b/tests/reliability.test.ts @@ -1,3 +1,4 @@ +import { requireSpawnResult } from "./helpers/cli-errors.js"; import test from "node:test"; import assert from "node:assert/strict"; import * as fs from "node:fs"; @@ -180,8 +181,7 @@ test("concurrent updates retain each other's fields", async () => { const root = process.cwd(); function cli(script: string, args: string[], cwd: string) { - const result = spawnSync(process.execPath, [join(root, "scripts", script), ...args], { cwd, encoding: "utf8" }); - assert.equal(result.error, undefined); + const result = requireSpawnResult(spawnSync(process.execPath, [join(root, "scripts", script), ...args], { cwd, encoding: "utf8" })); return { status: result.status, data: JSON.parse(result.stdout) }; } function project() { From 57aaf0d9d15733aa1c3c955ed09ac275d123b601 Mon Sep 17 00:00:00 2001 From: focaxisdev <263514337+focaxisdev@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:13:59 +0800 Subject: [PATCH 2/6] ci: resolve runner paths at step execution time Exact source local commit: f880ca2d67b14e939164bb7fc594800598e8fcdd Verified tracked tree: da94ba9b8c80a7cfd0d79a114a8dcdbb4fb5b97e Original local history retained; remote SHA intentionally differs. --- .github/workflows/recall-acceptance.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/recall-acceptance.yml b/.github/workflows/recall-acceptance.yml index 8b1ed48..612eb2e 100644 --- a/.github/workflows/recall-acceptance.yml +++ b/.github/workflows/recall-acceptance.yml @@ -19,7 +19,6 @@ jobs: defaults: run: shell: bash - working-directory: ${{ runner.temp }}/deja-vu-public steps: - name: Checkout exact push revision uses: actions/checkout@v4 @@ -39,15 +38,19 @@ jobs: node-version: 22 - name: Install locked dependencies + working-directory: ${{ runner.temp }}/deja-vu-public run: npm ci - name: Type check + working-directory: ${{ runner.temp }}/deja-vu-public run: node node_modules/typescript/bin/tsc --noEmit -p tsconfig.json - name: Complete source, links, compiled and installed-package acceptance + working-directory: ${{ runner.temp }}/deja-vu-public run: npm test - name: Frozen recall quality and cache lifecycle counts + working-directory: ${{ runner.temp }}/deja-vu-public run: | node scripts/evaluate-recall.mjs dist/src/index.js dist/src/index.js > "$RUNNER_TEMP/recall-quality.json" node scripts/benchmark-cache-ownership.mjs dist/src/index.js dist/src/index.js > "$RUNNER_TEMP/recall-cache.json" @@ -73,6 +76,7 @@ jobs: NODE - name: Synthetic protocol fixture checks + working-directory: ${{ runner.temp }}/deja-vu-public run: | node scripts/dejavu-lint-memory.mjs --memory-root starter-kit/memory node scripts/dejavu-lint-memory.mjs --memory-root examples/protocol-project/memory From 262e82660c23b9cc008c9d944b58b57148f77331 Mon Sep 17 00:00:00 2001 From: focaxisdev <263514337+focaxisdev@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:16:27 +0800 Subject: [PATCH 3/6] test: make recall tie fixture deterministic and normalize Windows export paths Exact source local commit: bb2b64e486e8530c8eeca09ed3d8fe6fc71f8c67 Verified tracked tree: c574e262740b3ec25a212c752ba1a14eb3953e0d Original local history retained; remote SHA intentionally differs. --- .github/workflows/recall-acceptance.yml | 8 ++++++-- tests/recall-efficiency.test.ts | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/recall-acceptance.yml b/.github/workflows/recall-acceptance.yml index 612eb2e..1043989 100644 --- a/.github/workflows/recall-acceptance.yml +++ b/.github/workflows/recall-acceptance.yml @@ -29,8 +29,12 @@ jobs: working-directory: ${{ github.workspace }} run: | git rev-parse HEAD - mkdir -p "$RUNNER_TEMP/deja-vu-public" - git archive HEAD AGENTS.md CHANGELOG.md CONTRIBUTING.md LICENSE README.md package.json package-lock.json tsconfig.json src tests scripts docs examples starter-kit .github .gitignore encoding-status.md llms.txt | tar -x -C "$RUNNER_TEMP/deja-vu-public" + public_root="$RUNNER_TEMP/deja-vu-public" + if [[ "$RUNNER_OS" == "Windows" ]]; then + public_root="$(cygpath -u "$RUNNER_TEMP")/deja-vu-public" + fi + mkdir -p "$public_root" + git archive HEAD AGENTS.md CHANGELOG.md CONTRIBUTING.md LICENSE README.md package.json package-lock.json tsconfig.json src tests scripts docs examples starter-kit .github .gitignore encoding-status.md llms.txt | tar -x -C "$public_root" - name: Setup Node uses: actions/setup-node@v4 diff --git a/tests/recall-efficiency.test.ts b/tests/recall-efficiency.test.ts index a37c0af..0a68399 100644 --- a/tests/recall-efficiency.test.ts +++ b/tests/recall-efficiency.test.ts @@ -30,7 +30,10 @@ test("eligible lexical match cannot be hidden by a recent ineligible hybrid winn assert.equal((await engine.recall(query.join(" "))).matched, true); }); -test("candidateLimit widens search without widening output; default breadth stays compatible", async () => { +test("candidateLimit widens search without widening output; default breadth stays compatible", async (t) => { + // These candidates must tie on recency as well as lexical score. Wall-clock + // changes during setup otherwise make the tie0 assertion timing-dependent. + t.mock.timers.enable({ apis: ["Date"], now: new Date("2026-01-01T00:00:00Z") }); const storage = new InMemoryStorage(); for (let i = 0; i < 6; i++) await storage.saveFamiliarity(familiarity(`tie${i}`, ["alpha"], i === 5 ? 1 : 0)); const engine = createInMemorySemanticRecallEngine({ storage }); From db56925d789482ba1f2a850b5833e73e7f031de4 Mon Sep 17 00:00:00 2001 From: focaxisdev <263514337+focaxisdev@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:18:40 +0800 Subject: [PATCH 4/6] ci: preserve committed fixture bytes in Windows public snapshot Exact source local commit: b139302c87b891cdc4e45ed735b5e2b7461b941f Verified tracked tree: 0cbc219a67320f375f3685656e2e30ecf424baa2 Original local history retained; remote SHA intentionally differs. --- .github/workflows/recall-acceptance.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/recall-acceptance.yml b/.github/workflows/recall-acceptance.yml index 1043989..b697de2 100644 --- a/.github/workflows/recall-acceptance.yml +++ b/.github/workflows/recall-acceptance.yml @@ -34,7 +34,8 @@ jobs: public_root="$(cygpath -u "$RUNNER_TEMP")/deja-vu-public" fi mkdir -p "$public_root" - git archive HEAD AGENTS.md CHANGELOG.md CONTRIBUTING.md LICENSE README.md package.json package-lock.json tsconfig.json src tests scripts docs examples starter-kit .github .gitignore encoding-status.md llms.txt | tar -x -C "$public_root" + # Export committed bytes rather than runner-specific CRLF conversion. + git -c core.autocrlf=false archive HEAD AGENTS.md CHANGELOG.md CONTRIBUTING.md LICENSE README.md package.json package-lock.json tsconfig.json src tests scripts docs examples starter-kit .github .gitignore encoding-status.md llms.txt | tar -x -C "$public_root" - name: Setup Node uses: actions/setup-node@v4 From 6f7b07f002f6ac9bf4d51cdb805cde1a8cca0409 Mon Sep 17 00:00:00 2001 From: focaxisdev <263514337+focaxisdev@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:22:12 +0800 Subject: [PATCH 5/6] docs: record passing hosted acceptance and preserved publication history Exact source local commit: 7ccb05ceb29cd67940dfee64f182a875f1326a9d Verified tracked tree: ea4f5e480a6c4646f6aac23afea865e6369ec43e Original local history retained; remote SHA intentionally differs. --- CHANGELOG.md | 5 +- docs/actions-acceptance-results.json | 168 ++++++++++++++++++++++++++ docs/actions-acceptance-results.md | 106 ++++++++++++++++ docs/actions-acceptance.md | 9 +- docs/release-recall-efficiency.md | 2 + docs/release-recall-lifecycle.md | 9 +- docs/social-recall-lifecycle-draft.md | 12 +- docs/verification-loop.md | 4 + 8 files changed, 304 insertions(+), 11 deletions(-) create mode 100644 docs/actions-acceptance-results.json create mode 100644 docs/actions-acceptance-results.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ef58f2..c15513b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,13 @@ All notable changes to this project will be documented in this file. ## Unreleased - Preserve original CLI subprocess failures in test helpers, retain execution/parse causes for invalid error responses, and add ten diagnostic regressions without weakening product assertions. -- Append general acceptance/evidence/repair-loop rules to AGENTS.md and record an [external-blocker checkpoint](docs/verification-loop.md); complete subprocess validation remains unverified. +- Append general acceptance/evidence/repair-loop rules to AGENTS.md and preserve the historical [Cloud blocker checkpoint](docs/verification-loop.md). +- Verify complete source, compiled, installed-package, type, recall-quality and cache gates on Ubuntu and Windows in [branch-only Actions](docs/actions-acceptance-results.md). Fix workflow context/path/newline handling and freeze the recall tie fixture's clock without changing product assertions. These changes remain unreleased. - Add opt-in exact query aliases, managed per-memory/revision embedding scopes, and serialized reads; digest cache keys avoid retained plaintext input keys. - Compensate failed deletions, add explicit recoverable `supersedeMemory`, and return defensive summary/chunk copies from the in-memory adapter. - Add opt-in prompt provenance and qualifier-loss warnings, plus explicit raw-content retrieval. -- Record development/held-out synthetic recall evaluation, ownership costs, repeated performance comparisons, and direct offline installed-package verification in the [round-two notes](docs/release-recall-lifecycle.md). Subprocess tests still report EPERM; no release is claimed. +- Record development/held-out synthetic recall evaluation, ownership costs, repeated performance comparisons, and direct offline installed-package verification in the [round-two notes](docs/release-recall-lifecycle.md). Hosted validation now passes; the native cause of the historical Cloud EPERM remains unresolved. - Fix optional-engine false-none recall when an ineligible hybrid winner hides an eligible candidate; keep existing thresholds. - Add opt-in candidate search breadth, bounded prompt projection, metadata-only embedding reuse, and a content/model-keyed bounded embedding cache. diff --git a/docs/actions-acceptance-results.json b/docs/actions-acceptance-results.json new file mode 100644 index 0000000..c10819a --- /dev/null +++ b/docs/actions-acceptance-results.json @@ -0,0 +1,168 @@ +{ + "status": "hosted acceptance passed; unreleased", + "verifiedRun": { + "id": 39089631583, + "url": "https://github.com/focaxisdev/deja-vu/actions/runs/39089631583", + "sha": "db56925d789482ba1f2a850b5833e73e7f031de4", + "status": "completed", + "conclusion": "success" + }, + "publicationMappings": [ + { + "local": "a66a52d4fceaeba3dac4a24c8e7cc82d834a4331", + "remote": "586f73016fc134c09c2e19b02e45c94b390e8c98", + "tree": "7365590fdcc4ea07a3513e9695ecf53a0e80e759", + "run": 39089273873 + }, + { + "local": "f880ca2d67b14e939164bb7fc594800598e8fcdd", + "remote": "57aaf0d9d15733aa1c3c955ed09ac275d123b601", + "tree": "da94ba9b8c80a7cfd0d79a114a8dcdbb4fb5b97e", + "parent": "586f73016fc134c09c2e19b02e45c94b390e8c98", + "files": 148, + "changed": 1, + "run": 39089353842 + }, + { + "local": "bb2b64e486e8530c8eeca09ed3d8fe6fc71f8c67", + "remote": "262e82660c23b9cc008c9d944b58b57148f77331", + "tree": "c574e262740b3ec25a212c752ba1a14eb3953e0d", + "parent": "57aaf0d9d15733aa1c3c955ed09ac275d123b601", + "files": 148, + "changed": 2, + "run": 39089505184 + }, + { + "local": "b139302c87b891cdc4e45ed735b5e2b7461b941f", + "remote": "db56925d789482ba1f2a850b5833e73e7f031de4", + "tree": "0cbc219a67320f375f3685656e2e30ecf424baa2", + "parent": "262e82660c23b9cc008c9d944b58b57148f77331", + "files": 148, + "changed": 1, + "run": 39089631583 + } + ], + "verification": { + "initialCompleteTreeFileCount": 148, + "initialDeltaFromMain": { + "added": 26, + "modified": 16, + "deleted": 0, + "modeChanges": 0 + }, + "everyPublication": "root tree SHA and each recursive file path/type/mode/blob SHA equal the source local tree; remote commit parent and branch ref verified", + "permissions": "contents: read; no release/deploy/publish job; no force updates", + "localHistory": "all original local commits retained" + }, + "jobs": [ + { + "os": "Ubuntu 24.04.5", + "id": 117563410807, + "url": "https://github.com/focaxisdev/deja-vu/actions/runs/39089631583/job/117563410807", + "node": "v22.23.3", + "npm": "10.9.9", + "source": { + "tests": 119, + "pass": 119, + "fail": 0, + "skipped": 0 + }, + "compiled": { + "tests": 119, + "pass": 119, + "fail": 0, + "skipped": 0 + }, + "packageInstallUse": { + "tests": 1, + "pass": 1, + "fail": 0, + "skipped": 0 + }, + "types": "passed", + "frozenRecallQuality": "passed", + "cacheLifecycleCounts": "passed", + "protocolFixtures": "passed", + "publicMarkdownLinks": { + "files": 69, + "status": "passed" + } + }, + { + "os": "Windows Server 2025", + "id": 117563410610, + "url": "https://github.com/focaxisdev/deja-vu/actions/runs/39089631583/job/117563410610", + "node": "v22.23.3", + "npm": "10.9.9", + "source": { + "tests": 119, + "pass": 119, + "fail": 0, + "skipped": 0 + }, + "compiled": { + "tests": 119, + "pass": 119, + "fail": 0, + "skipped": 0 + }, + "packageInstallUse": { + "tests": 1, + "pass": 1, + "fail": 0, + "skipped": 0 + }, + "types": "passed", + "frozenRecallQuality": "passed", + "cacheLifecycleCounts": "passed", + "protocolFixtures": "passed", + "publicMarkdownLinks": { + "files": 69, + "status": "passed" + } + } + ], + "failuresPreserved": [ + { + "run": 39089273873, + "kind": "workflow planning", + "jobs": 0, + "detail": "runner.temp context was used under job defaults; moved workdir resolution to steps. API exposed no validation annotation or runner log." + }, + { + "run": 39089353842, + "job": 117562559666, + "kind": "nondeterministic test fixture", + "source": { + "tests": 119, + "pass": 118, + "fail": 1, + "skipped": 0 + }, + "error": "ERR_ASSERTION: 'tie2' !== 'tie0'", + "fix": "Freeze Date in the lexical-tie/candidateLimit test; all 95 efficiency assertions retained." + }, + { + "run": 39089353842, + "job": 117562559864, + "kind": "Windows archive path", + "error": "tar: Cannot open: No such file or directory", + "fix": "Convert Windows RUNNER_TEMP to a Bash path using cygpath." + }, + { + "run": 39089505184, + "job": 117563017821, + "kind": "frozen fixture byte mismatch", + "expectedFixtureSha256": "4e1b70b5520e1d3cfb85489aad053825e339938016b20e659f2a4eb4677aee5d", + "actualFixtureSha256": "21552541a3f35ae72ba055b265d2fd99a03f67c768bd9ca9b4f07b95a9373a7e", + "fix": "Command-scoped core.autocrlf=false on git archive preserves committed LF bytes. Hash/content assertions and frozen answers unchanged." + } + ], + "limits": [ + "Hosted Node 22 results do not identify the native syscall/policy cause of Cloud Node 24 spawnSync EPERM.", + "Five historical masked errors cannot be retrospectively attributed.", + "No new latency benchmark; provider-call/retained-entry checks are not performance or token claims.", + "Source/compiled and OS counts repeat the same cases; 120 distinct selected tests, not 478 distinct tests.", + "Documentation follow-up changes require another complete run at their own published SHA." + ] +} diff --git a/docs/actions-acceptance-results.md b/docs/actions-acceptance-results.md new file mode 100644 index 0000000..c09cad0 --- /dev/null +++ b/docs/actions-acceptance-results.md @@ -0,0 +1,106 @@ +# Hosted acceptance results + +Unreleased engineering evidence. The branch is published for verification only; +no main merge, release, npm publication or deployment is included. + +## Verified result + +[Run 39089631583](https://github.com/focaxisdev/deja-vu/actions/runs/39089631583) +completed successfully at remote commit +`db56925d789482ba1f2a850b5833e73e7f031de4`. Its tree +`0cbc219a67320f375f3685656e2e30ecf424baa2` exactly matches local commit +`b139302c87b891cdc4e45ed735b5e2b7461b941f`. All original local commits remain in +the local fix branch's ancestry; remote commits intentionally have different SHAs. + +| Gate | Ubuntu | Windows | +| --- | --- | --- | +| TypeScript no-emit check | Passed | Passed | +| Complete source tests | 119 passed, 0 failed, 0 skipped | 119 passed, 0 failed, 0 skipped | +| Fresh build and complete compiled tests | 119 passed, 0 failed, 0 skipped | 119 passed, 0 failed, 0 skipped | +| Package, offline installation, installed CLI/API use | 1 passed, 0 failed, 0 skipped | 1 passed, 0 failed, 0 skipped | +| Frozen recall IDs, metrics and fixture digest | Passed | Passed | +| Cache provider-call and retained-entry counts | Passed | Passed | +| Synthetic protocol fixtures and feedback | Passed | Passed | +| Public Markdown links | 69 files, passed | 69 files, passed | + +Both jobs used Node v22.23.3 and npm 10.9.9. Logs: +[Ubuntu](https://github.com/focaxisdev/deja-vu/actions/runs/39089631583/job/117563410807), +[Windows](https://github.com/focaxisdev/deja-vu/actions/runs/39089631583/job/117563410610). +The source and compiled counts repeat the same 119 cases on two operating +systems; the package test is a separate case, not hundreds of distinct new tests. +The 50 engine cases and ten diagnostic-helper cases are included, alongside all +CLI/reliability cases. No failing assertion or test selection was removed. + +The [machine-readable record](actions-acceptance-results.json) includes every +publication mapping and earlier failures. This report records the run above; +documentation follow-up commits must receive their own successful complete run. +Their source local SHA/tree are recorded in each remote commit message. + +## Evidence-driven corrections + +1. [Initial run](https://github.com/focaxisdev/deja-vu/actions/runs/39089273873): + workflow failure with zero jobs. The API exposed no validation annotation or + runner log. Inspection found `runner.temp` used under job defaults, where that + context is unavailable. Moving unchanged working directories to the steps + allowed both jobs to start. This was a workflow defect, not a test result. +2. [Next run](https://github.com/focaxisdev/deja-vu/actions/runs/39089353842): + Ubuntu executed all 119 source tests: 118 passed, one failed with + `ERR_ASSERTION: 'tie2' !== 'tie0'`. The candidate-breadth test generated each + record's timestamp independently, so lexical ties were not necessarily hybrid + ties. A synthetic timing example reproduced the recency distinction. Freeze + Date for that fixture; retain all 95 efficiency assertions, including the + original `tie0` and widened-search `tie5` checks. Its local source/compiled + 25-case suites passed before the complete hosted rerun. +3. In that same run Windows failed before tests: `tar` could not open the native + Windows temporary path from Git Bash. Convert the export destination with + `cygpath`, without changing security controls or test behavior. +4. [Third run](https://github.com/focaxisdev/deja-vu/actions/runs/39089505184): + both platforms passed all source/compiled/package tests; Windows then failed + the frozen fixture hash assertion. Its observed SHA256 was exactly the hash + obtained by converting the fixture's LF bytes to CRLF. Reproducing this with + `git -c core.autocrlf=true archive` confirmed the cause. Use command-scoped + `core.autocrlf=false` for export. Keep the original fixture hash, labels, + returned IDs and metric assertions. No persistent Git configuration changed. +5. The next complete run passed all gates on both platforms. Product source, + thresholds, dependency lockfile and AGENTS loop rules were unchanged throughout + these hosted-test corrections. + +## Publication integrity and scope + +After the user added the repository to the existing GitHub installation, the +authorized retry of the same previously denied `create_tree` operation succeeded. +The initial published tree exactly matched local +`a66a52d4fceaeba3dac4a24c8e7cc82d834a4331`; remote commit +`586f73016fc134c09c2e19b02e45c94b390e8c98` has the verified main commit +`188d7e8c3aed224e45071d041cd140509887b29f` as its parent. The initial 148-file +tree added 26 files and modified 16, without deletions or mode changes. + +Every publication compares the complete recursive path/type/mode/blob-SHA list +and root tree SHA, checks the commit parent, and verifies the branch ref. Later +updates are non-force descendants of the remote branch. Changed blobs are reviewed +for private paths and credential patterns; unchanged baseline blobs are retained +by reference. Root project memory and credentials are not read or uploaded anew. +Hosted tests export public source and synthetic fixtures, excluding root memory. + +The dedicated workflow triggers only for this fix branch, has `contents: read`, +does not persist checkout credentials, and has no release or deployment step. +The main/PR workflow is unchanged. The original +[permission-denial checkpoint](actions-publication-evidence.json) remains historical +evidence, not the current publication status. + +## Limits + +- Hosted Node 22 passes establish the tested package behavior on these two runner + images. They do not establish Node 20 coverage or identify the native syscall + responsible for the earlier Cloud Node 24 `spawnSync EPERM`. No Cloud security + setting, proxy or credential was changed, and denied calls were not blindly retried. +- The earlier 32 failures remain nine direct EPERM, 18 infrastructure assertions + containing EPERM, and five parse failures whose original causes were lost. New + passing runs cannot reconstruct those five historical causes. +- The synthetic language evaluation and authored aliases retain their previous + limitations. Aliases are explicit routing, not a multilingual semantic model. +- No latency benchmark was added in this CI round. Cache call counts are not + end-to-end speedups; accounted bytes are not tokens. The prior performance and + ownership tradeoffs remain in the [release notes](release-recall-lifecycle.md). +- A green verification branch is not a release. Publishing or merging requires + separate authorization. diff --git a/docs/actions-acceptance.md b/docs/actions-acceptance.md index 15a1a9c..ae5420f 100644 --- a/docs/actions-acceptance.md +++ b/docs/actions-acceptance.md @@ -3,6 +3,11 @@ Unreleased verification setup, continuing the [Cloud checkpoint](verification-loop.md). Starting revision: `4c05a77605f6cf63a10261dc394c0824208313ea`. +Current outcome: the authorized connector publication succeeded after installation +access was corrected, and both hosted jobs passed all gates. See the +[verified results, logs and SHA mappings](actions-acceptance-results.md). The +transport and permission sections below preserve earlier checkpoints. + ## Scope and acceptance defined before execution The dedicated `Recall acceptance` workflow runs only on pushes to @@ -90,7 +95,7 @@ in the verification checkpoint. If the connector rejects access or cannot preser the tree, stop without substituting an incomplete tree. Hosted acceptance remains unverified until both jobs reach successful terminal states at the published SHA. -## Connector result: permission blocked +## Historical connector result: permission blocked The reviewed source snapshot is local commit `c2a9bb103e4f502d23a66a6c067f00c8b0b10467`, tree @@ -115,7 +120,7 @@ main unchanged and the target branch still absent. Repository metadata reporting missing permission is not exposed by this error and is not assumed to be a specific workflow or contents scope. -All hosted gates remain **not run / connector permission blocked**. The final +At that checkpoint all hosted gates were **not run / connector permission blocked**. The final remote tree could not be created or verified; there is no remote SHA or Actions run/log URL. A maintainer must restore ordinary approved Git transport or provide an already-authorized connection that can perform this Git tree write before diff --git a/docs/release-recall-efficiency.md b/docs/release-recall-efficiency.md index d5b4449..a665380 100644 --- a/docs/release-recall-efficiency.md +++ b/docs/release-recall-efficiency.md @@ -7,6 +7,8 @@ These measurements and validation counts describe the first implementation batch at `bad6b527409a44a693562fbe60c41be74126e573`. See the [round-two notes](release-recall-lifecycle.md) for current cache ownership, provenance, quality evaluation, performance costs and installed-package results. +The later [hosted acceptance report](actions-acceptance-results.md) records full +Ubuntu/Windows validation; these first-batch counts remain historical. ## User introduction diff --git a/docs/release-recall-lifecycle.md b/docs/release-recall-lifecycle.md index 0bf5db9..d124e15 100644 --- a/docs/release-recall-lifecycle.md +++ b/docs/release-recall-lifecycle.md @@ -1,12 +1,17 @@ # Unreleased: explicit recall cues and recoverable memory lifecycle -Local release-note draft. No new version, release date, tag, release or social +Unreleased release-note draft. No new version, release date, tag, release or social publication is assigned. This follows the [first recall-efficiency changes](release-recall-efficiency.md) and uses `bad6b527409a44a693562fbe60c41be74126e573` as its comparison baseline. These results describe the second implementation batch. The subsequent [verification-loop checkpoint](verification-loop.md) fixes test-helper diagnostic -loss and records current validation plus the unresolved native-execution blocker. +loss and records the historical native-execution blocker. Complete hosted +[acceptance now passes on Ubuntu and Windows](actions-acceptance-results.md), +including 119 source cases, 119 compiled cases and one package/install/use case +per platform, with no skipped tests. The verification branch is published; the +package and social draft remain unreleased. The original Cloud EPERM cause is +still unresolved, and the results below retain their original measurement scope. ## User introduction diff --git a/docs/social-recall-lifecycle-draft.md b/docs/social-recall-lifecycle-draft.md index 1da8f6e..a05192b 100644 --- a/docs/social-recall-lifecycle-draft.md +++ b/docs/social-recall-lifecycle-draft.md @@ -1,8 +1,8 @@ # Local social draft: recall cues and lifecycle -Unpublished draft only. No release date or version is assigned. Do not use this -copy as a release announcement until the remaining verification blockers are -resolved. See the [evidence and limitations](release-recall-lifecycle.md). +Unpublished draft only. No release date or version is assigned. Hosted acceptance +has passed; publishing a release or social announcement remains a separate action. +See the [evidence and limitations](release-recall-lifecycle.md). ## Short introduction @@ -19,6 +19,8 @@ explicit routing, not a new multilingual semantic model. The tradeoffs matter: separate cache ownership means less cross-memory reuse, and this round did not improve measured latency. Direct offline packaging and -installed smoke checks pass, while subprocess-based tests still report EPERM in -the current Cloud. These changes remain local and unreleased. Usage, migration, +installed smoke checks pass. Full source, compiled and installed-package tests +also pass on Ubuntu and Windows in the verification branch's GitHub Actions. +That does not explain the earlier Cloud runtime's native EPERM. The branch is +public; the package and this social draft remain unreleased. Usage, migration, full measurements and remaining limits are in the release-note draft. diff --git a/docs/verification-loop.md b/docs/verification-loop.md index 0e55770..1922bec 100644 --- a/docs/verification-loop.md +++ b/docs/verification-loop.md @@ -3,6 +3,10 @@ Unreleased engineering checkpoint. Starting commit: `fa479335dbc5acbad46a352d1ee8bd6086c62cd7`, on the existing local fix branch. +This is a historical Cloud checkpoint. Subsequent authorized +[GitHub Actions acceptance](actions-acceptance-results.md) passes the complete +suite on Ubuntu and Windows; the Cloud native EPERM cause remains unresolved. + ## Scope and acceptance, defined before changes This iteration fixes loss of subprocess error evidence in test helpers, adds From 93525c0be02e366fef21c4950683850feac7bbb1 Mon Sep 17 00:00:00 2001 From: focaxisdev <263514337+focaxisdev@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:36:10 +0800 Subject: [PATCH 6/6] docs: feature verified recall improvements on the repository homepage Exact source local commit: 30d32751613557d7b3d252040b07aa23317b2171 Verified tracked tree: 4147bece082ecb5a44ae48c21aee9da852999bbe Original local history retained; remote SHA intentionally differs. --- .github/workflows/ci.yml | 30 +------- .github/workflows/recall-acceptance.yml | 12 ++++ CHANGELOG.md | 2 + README.md | 29 ++++++-- docs/actions-acceptance-results.md | 18 +++-- docs/actions-acceptance.md | 12 ++-- docs/recall-update.md | 95 +++++++++++++++++++++++++ docs/release-recall-efficiency.md | 10 +-- docs/release-recall-lifecycle.md | 21 +++--- docs/social-recall-lifecycle-draft.md | 6 +- 10 files changed, 173 insertions(+), 62 deletions(-) create mode 100644 docs/recall-update.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f283f0d..afb5e75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,32 +11,4 @@ permissions: jobs: test: - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: npm - - - name: Install - run: npm ci - - - name: Source, compiled, links and installed-package tests - run: npm test - - - name: Memory lint - run: node scripts/dejavu-lint-memory.mjs && node scripts/dejavu-lint-memory.mjs --memory-root starter-kit/memory && node scripts/dejavu-lint-memory.mjs --memory-root examples/protocol-project/memory && node scripts/dejavu-lint-memory.mjs --memory-root docs/examples/settings-project/memory - - - name: Project readiness - run: node scripts/deja-vu.mjs doctor --json - - - name: Feedback report - run: npm run report:feedback + uses: ./.github/workflows/recall-acceptance.yml diff --git a/.github/workflows/recall-acceptance.yml b/.github/workflows/recall-acceptance.yml index b697de2..1896624 100644 --- a/.github/workflows/recall-acceptance.yml +++ b/.github/workflows/recall-acceptance.yml @@ -1,6 +1,7 @@ name: Recall acceptance on: + workflow_call: push: branches: - fix/recall-efficiency-and-budgets @@ -87,3 +88,14 @@ jobs: node scripts/dejavu-lint-memory.mjs --memory-root examples/protocol-project/memory node scripts/dejavu-lint-memory.mjs --memory-root docs/examples/settings-project/memory npm run report:feedback + + - name: Fresh synthetic project readiness + working-directory: ${{ runner.temp }}/deja-vu-public + run: | + project_root="$RUNNER_TEMP/deja-vu-readiness" + if [[ "$RUNNER_OS" == "Windows" ]]; then + project_root="$(cygpath -u "$RUNNER_TEMP")/deja-vu-readiness" + fi + node scripts/deja-vu.mjs init --cwd "$project_root" --project-id project:ci --json + node scripts/dejavu-lint-memory.mjs --memory-root "$project_root/memory" + node scripts/deja-vu.mjs doctor --cwd "$project_root" --json diff --git a/CHANGELOG.md b/CHANGELOG.md index c15513b..b25960c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. ## Unreleased +- Introduce the source update on the repository homepage, with [opt-in examples, migration guidance and scoped measurements](docs/recall-update.md). No new npm package or GitHub Release is published. +- Reuse the complete synthetic-data acceptance workflow for main and pull requests, including fresh-project lint/readiness checks, without reading project memory. - Preserve original CLI subprocess failures in test helpers, retain execution/parse causes for invalid error responses, and add ten diagnostic regressions without weakening product assertions. - Append general acceptance/evidence/repair-loop rules to AGENTS.md and preserve the historical [Cloud blocker checkpoint](docs/verification-loop.md). - Verify complete source, compiled, installed-package, type, recall-quality and cache gates on Ubuntu and Windows in [branch-only Actions](docs/actions-acceptance-results.md). Fix workflow context/path/newline handling and freeze the recall tie fixture's clock without changing product assertions. These changes remain unreleased. diff --git a/README.md b/README.md index e2320bb..3ea49e4 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,23 @@ Deja Vu is a three-file, repo-local memory system that any capable coding agent No database. No vector store. No embeddings. No SaaS. No daemon. No required npm install. +## Latest update: more reliable, bounded recall + +**Available in repository source; a new npm package release has not been published.** + +The optional TypeScript engine now keeps eligible memories from being hidden by +less relevant ranking winners. It also adds bounded prompt excerpts, recoverable +deletion and supersession, and opt-in query aliases and embedding-cache reuse. +The three-file protocol and [60-second setup](#start-in-60-seconds) stay the same. + +Efficiency depends on the workload and enabled options: these changes do not +make every request faster. Explicit aliases are routing rules, not a new +multilingual semantic model. + +**[Read the full update, opt-in examples and migration notes](./docs/recall-update.md)** +· [Measurements and tradeoffs](./docs/release-recall-lifecycle.md) +· [Complete Ubuntu/Windows acceptance evidence](./docs/actions-acceptance-results.md) + ![Deja Vu turns tiny repo-local cues into minimal, relevant recall](./docs/assets/deja-vu-preview.svg) ## Start in 60 seconds @@ -196,12 +213,12 @@ const engine = createInMemorySemanticRecallEngine({ Read the [engine guide](./docs/engine/semantic-engine.md) or run the [basic example](./examples/basic/index.ts). The engine is an optional acceleration layer, not the default adoption path. -Unreleased engine work adds eligible-candidate selection, bounded prompt output, -and opt-in embedding reuse. The three-file setup is unchanged. See the -[local release-note draft](docs/release-recall-efficiency.md) and -[usage examples](docs/engine/recall-efficiency.md). -Further unreleased work adds opt-in [explicit aliases, cache ownership and provenance](docs/engine/recall-lifecycle.md). -The [round-two notes](docs/release-recall-lifecycle.md) report recall gains from authored aliases, measured costs and remaining verification blockers. +Repository source includes eligible-candidate selection, bounded prompt output, +and opt-in embedding reuse, [explicit aliases, cache ownership and provenance](docs/engine/recall-lifecycle.md). +These additions are not yet published in a new npm package. See the +[update and migration overview](docs/recall-update.md), +[usage examples](docs/engine/recall-efficiency.md), and +[complete hosted acceptance results](docs/actions-acceptance-results.md). ## Contributing diff --git a/docs/actions-acceptance-results.md b/docs/actions-acceptance-results.md index c09cad0..4cf4cce 100644 --- a/docs/actions-acceptance-results.md +++ b/docs/actions-acceptance-results.md @@ -1,7 +1,9 @@ # Hosted acceptance results -Unreleased engineering evidence. The branch is published for verification only; -no main merge, release, npm publication or deployment is included. +Engineering evidence for the public repository-source update. This report records +pre-merge verification; current main results are available in +[main CI](https://github.com/focaxisdev/deja-vu/actions/workflows/ci.yml). +An npm package or GitHub Release is a separate publication, not implied by CI. ## Verified result @@ -82,9 +84,11 @@ for private paths and credential patterns; unchanged baseline blobs are retained by reference. Root project memory and credentials are not read or uploaded anew. Hosted tests export public source and synthetic fixtures, excluding root memory. -The dedicated workflow triggers only for this fix branch, has `contents: read`, -does not persist checkout credentials, and has no release or deployment step. -The main/PR workflow is unchanged. The original +At the recorded runs, the dedicated workflow triggered only for the fix branch. +It is now also reused by main/PR CI, retains `contents: read`, does not persist +checkout credentials, and has no release or deployment step. Main/PR verification +uses the same complete tests, adding fresh synthetic-project lint/readiness checks +instead of reading project memory. The original [permission-denial checkpoint](actions-publication-evidence.json) remains historical evidence, not the current publication status. @@ -102,5 +106,5 @@ evidence, not the current publication status. - No latency benchmark was added in this CI round. Cache call counts are not end-to-end speedups; accounted bytes are not tokens. The prior performance and ownership tradeoffs remain in the [release notes](release-recall-lifecycle.md). -- A green verification branch is not a release. Publishing or merging requires - separate authorization. +- A green CI result or source merge is not an npm package or GitHub Release. + Package/release publication requires separate authorization. diff --git a/docs/actions-acceptance.md b/docs/actions-acceptance.md index ae5420f..dd104d9 100644 --- a/docs/actions-acceptance.md +++ b/docs/actions-acceptance.md @@ -10,11 +10,13 @@ transport and permission sections below preserve earlier checkpoints. ## Scope and acceptance defined before execution -The dedicated `Recall acceptance` workflow runs only on pushes to -`fix/recall-efficiency-and-budgets`, with `contents: read`, Node 22, and the -existing Ubuntu/Windows runner matrix. It does not publish, deploy, create a -release, access secrets, or require a pull request. The existing main/PR CI is -unchanged. Checkout does not persist credentials. +The `Recall acceptance` workflow runs directly on pushes to +`fix/recall-efficiency-and-budgets` and is reused by main/PR CI through +`workflow_call`, with `contents: read`, Node 22, and the existing Ubuntu/Windows +runner matrix. It does not publish, deploy, create a release or access configured +secrets. Checkout does not persist credentials. Main/PR runs use the same complete +test selection and frozen evaluation assertions as branch acceptance; fresh +synthetic-project lint/readiness checks replace inspection of project memory. Both jobs must finish successfully at the exact pushed SHA: diff --git a/docs/recall-update.md b/docs/recall-update.md new file mode 100644 index 0000000..8310d2e --- /dev/null +++ b/docs/recall-update.md @@ -0,0 +1,95 @@ +# More reliable recall, with explicit budgets and opt-ins + +These changes are available in repository source. No new package version, npm +publication, GitHub Release or release date is assigned. The existing npm setup +instructions still describe the three-file protocol; do not assume an npm install +already includes every source addition below. + +## What improves + +The optional TypeScript engine selects eligible memories before applying hybrid +ranking preference, avoiding false "no memory" results when an ineligible record +has a higher hybrid score. Search breadth can grow without expanding final output. +Cached token sets avoid repeated lexical preprocessing, with update/delete +invalidation. Familiarity thresholds and the public three-file protocol are unchanged. + +Hosts can opt in to bounded prompt projection, metadata-only embedding reuse, +content/model-keyed embedding caches, exact query aliases, and per-memory cache +ownership. Deletion and supersession attempt recovery on adapter failures; +projection can mark missing qualifiers and identify where to retrieve stored text. +These are optional engine features, not new requirements for file-first adoption. + +## Enable only what your host needs + +With the repository build, exact aliases can route familiar wording to an existing +distinctive cue, while candidate search and prompt output have separate budgets: + +```ts +import { + createInMemorySemanticRecallEngine, projectRecallForPrompt, +} from "@focaxisdev/deja-vu"; + +const engine = createInMemorySemanticRecallEngine({ + cueAliases: [{ alias: "更新後快取怎麼辦", cue: "cachepolicy" }], +}); +await engine.addMemory({ + id: "cache-policy", title: "Cache policy", tags: ["cachepolicy"], + content: "Invalidate cached data after an update or deletion.", +}); +const recalled = await engine.recall({ + text: "更新後快取怎麼辦", candidateLimit: 32, topK: 3, +}); +const prompt = projectRecallForPrompt(recalled, { + maxBytes: 4096, includeProvenance: true, +}); +``` + +Aliases match the normalized entire query. They do not translate, segment Chinese +or recognize unlisted paraphrases. The byte cap is not a token cap; use a host +tokenizer for token budgets. Projection is untrusted memory data, not an instruction +sanitizer. See the [budget/cache guide](engine/recall-efficiency.md) and +[ownership, aliases and provenance guide](engine/recall-lifecycle.md) for full +options, examples and failure handling. + +## Benefits and limits, measured separately + +- **Lexical preprocessing:** 30.74 → 2.61 ms was the median of one synthetic + 10,000-record shared-cue storage-search microbenchmark. It is not a whole-product + speedup, does not cover all requests, and excludes external models and agent work. + The [first-batch measurements](release-recall-efficiency.md) distinguish this + microbenchmark from the resident API workload. +- **Recall quality:** authored aliases raised Recall@1 from 2/13 to 10/13 in + each split of a small same-author synthetic dataset. No false recalls were + observed there, but unlisted paraphrases still missed. This is not independent + multilingual-model validation or a general recall-rate guarantee. +- **Latency and ownership:** the second batch's 10,000-record default-mode + resident API median rose about 12.4%; that measurement did not isolate ownership + mode. Managed ownership separately increased mock provider calls on ingest + from 18 to 48 and on a content update from 2 to 3 in the shared-content fixture, + while retained entries after deletion fell from 36 to zero. It also serializes + reads/writes within one engine, so slow providers can delay later calls. + +The [second-batch report](release-recall-lifecycle.md) retains workloads, raw +measurement links and caveats. Cache counts are not paid-service savings; +accounted bytes are not tokens or measured heap use. These changes do not establish +that every request is faster. + +## Migration and verification + +No migration of `AGENTS.md`, `memory/summary.md` or `memory/impressions.jsonl` is +required. Existing API result shapes, default search breadth and thresholds are +retained. Aliases, ownership, read serialization, metadata preservation and +projection provenance need explicit opt-in. Invalid limits now reject; small +custom cache byte budgets can hold fewer entries with digest keys. Review custom +adapter recovery and generator assumptions before enabling these options. + +Recovery is not a crash-safe transaction. Cache release is not verified secure +erasure. Prompt warnings mark possible qualifier loss rather than restoring +meaning; use the documented source lookup when complete evidence matters. + +Complete Ubuntu/Windows acceptance passed: 119 source cases, the same 119 cases +compiled, and one package/install/use case per platform, with zero skipped tests. +Type checks, frozen recall results, cache lifecycle counts and public documentation +checks also passed. See the [logs and failure-to-fix record](actions-acceptance-results.md) +and [current main CI](https://github.com/focaxisdev/deja-vu/actions/workflows/ci.yml). +Hosted Node 22 results do not identify the earlier Cloud Node 24 native EPERM cause. diff --git a/docs/release-recall-efficiency.md b/docs/release-recall-efficiency.md index a665380..ae78d53 100644 --- a/docs/release-recall-efficiency.md +++ b/docs/release-recall-efficiency.md @@ -1,7 +1,8 @@ # Unreleased: more reliable, bounded engine recall -Local release-note draft only. No new package version, release date, tag or -publication is assigned by this change. +Source update notes; package release pending. No new package version, release +date, tag or npm publication is assigned by this change. Start with the +[combined update and opt-in overview](recall-update.md). These measurements and validation counts describe the first implementation batch at `bad6b527409a44a693562fbe60c41be74126e573`. See the @@ -75,7 +76,7 @@ The raw recall payload stayed at 10,253 bytes. Its optional prompt projection wa non-projected fields. **Bytes are not model tokens.** This is a representation comparison, not proof of equal task quality; omitted fields may matter to a host. -## Validation and limitations +## Historical first-batch validation and limitations - TypeScript no-emit validation and compilation passed with existing dependencies. - All 31 source engine/regression tests passed in Node's supported in-process test mode. @@ -88,7 +89,8 @@ comparison, not proof of equal task quality; omitted fields may matter to a host that permits those subprocesses; this draft does not claim full-suite success. - No new dependency was installed. Repository-wide memory scans and link checks that read private memory were excluded; changed documentation links were checked - separately. Full `npm test` and CI remain required before a future release. + separately. Full `npm test` and CI were still outstanding at that checkpoint; + the later hosted acceptance linked above passed them. Regression coverage includes the false-none counterexample, lexical prefilter truncation, invalid and zero limits, multilingual score/tie equivalence, Unicode diff --git a/docs/release-recall-lifecycle.md b/docs/release-recall-lifecycle.md index d124e15..edabe56 100644 --- a/docs/release-recall-lifecycle.md +++ b/docs/release-recall-lifecycle.md @@ -1,7 +1,8 @@ # Unreleased: explicit recall cues and recoverable memory lifecycle -Unreleased release-note draft. No new version, release date, tag, release or social -publication is assigned. This follows the [first recall-efficiency changes](release-recall-efficiency.md) +Source update notes; package release pending. No new version, release date, tag, +npm package, GitHub Release or social post is published by these notes. See the +[combined introduction and opt-ins](recall-update.md). This follows the [first recall-efficiency changes](release-recall-efficiency.md) and uses `bad6b527409a44a693562fbe60c41be74126e573` as its comparison baseline. These results describe the second implementation batch. The subsequent @@ -9,8 +10,8 @@ These results describe the second implementation batch. The subsequent loss and records the historical native-execution blocker. Complete hosted [acceptance now passes on Ubuntu and Windows](actions-acceptance-results.md), including 119 source cases, 119 compiled cases and one package/install/use case -per platform, with no skipped tests. The verification branch is published; the -package and social draft remain unreleased. The original Cloud EPERM cause is +per platform, with no skipped tests. The source and these notes are public; the +package update and social draft remain unreleased. The original Cloud EPERM cause is still unresolved, and the results below retain their original measurement scope. ## User introduction @@ -101,7 +102,10 @@ accounted bytes); owned mode retains zero. Baseline has no public statistics API Separate ownership trades cross-memory reuse for predictable invalidation. Counts are deterministic mock calls, not paid-service costs, tokens or throughput. -## Validation and blockers +## Historical second-batch validation and blockers + +The following records the original Cloud checkpoint, before the later hosted +acceptance linked above. It does not describe the current CI status. - TypeScript checking and compilation pass with existing dependencies. - Source and compiled engine regression suites pass: 50 distinct cases, including @@ -129,9 +133,10 @@ are deterministic mock calls, not paid-service costs, tokens or throughput. private memory, so it was not run; public documentation links were checked separately. No private memory, network service or new dependency was used. -The failed subprocess scenarios still need an authorized subprocess-capable Cloud -configuration before release. No alternate environment was used. All files are -local preparation; this is not a release-readiness claim. +At that checkpoint the failed subprocess scenarios still needed an authorized +environment. Subsequent GitHub Actions runs passed the complete source, compiled +and installed-package selections on Ubuntu and Windows. The native Cloud EPERM +cause remains unresolved; hosted success is not an npm or GitHub Release. ## Remaining limits diff --git a/docs/social-recall-lifecycle-draft.md b/docs/social-recall-lifecycle-draft.md index a05192b..fc442e3 100644 --- a/docs/social-recall-lifecycle-draft.md +++ b/docs/social-recall-lifecycle-draft.md @@ -6,7 +6,7 @@ See the [evidence and limitations](release-recall-lifecycle.md). ## Short introduction -We're preparing optional-engine improvements for Deja Vu: explicit query aliases, +Deja Vu's repository source now includes optional-engine improvements: explicit query aliases, embedding-cache ownership that follows memory updates and deletions, recoverable supersession, and prompt excerpts that warn when constraints may be missing. The three-file memory protocol stays the same. @@ -21,6 +21,6 @@ The tradeoffs matter: separate cache ownership means less cross-memory reuse, and this round did not improve measured latency. Direct offline packaging and installed smoke checks pass. Full source, compiled and installed-package tests also pass on Ubuntu and Windows in the verification branch's GitHub Actions. -That does not explain the earlier Cloud runtime's native EPERM. The branch is -public; the package and this social draft remain unreleased. Usage, migration, +That does not explain the earlier Cloud runtime's native EPERM. The source is +public; the package update and this social draft remain unreleased. Usage, migration, full measurements and remaining limits are in the release-note draft.