diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f283f0d..afb5e75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,32 +11,4 @@ permissions: jobs: test: - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: npm - - - name: Install - run: npm ci - - - name: Source, compiled, links and installed-package tests - run: npm test - - - name: Memory lint - run: node scripts/dejavu-lint-memory.mjs && node scripts/dejavu-lint-memory.mjs --memory-root starter-kit/memory && node scripts/dejavu-lint-memory.mjs --memory-root examples/protocol-project/memory && node scripts/dejavu-lint-memory.mjs --memory-root docs/examples/settings-project/memory - - - name: Project readiness - run: node scripts/deja-vu.mjs doctor --json - - - name: Feedback report - run: npm run report:feedback + uses: ./.github/workflows/recall-acceptance.yml diff --git a/.github/workflows/recall-acceptance.yml b/.github/workflows/recall-acceptance.yml new file mode 100644 index 0000000..1896624 --- /dev/null +++ b/.github/workflows/recall-acceptance.yml @@ -0,0 +1,101 @@ +name: Recall acceptance + +on: + workflow_call: + push: + branches: + - fix/recall-efficiency-and-budgets + +permissions: + contents: read + +jobs: + acceptance: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + defaults: + run: + shell: bash + steps: + - name: Checkout exact push revision + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Export public source and synthetic fixtures + working-directory: ${{ github.workspace }} + run: | + git rev-parse HEAD + public_root="$RUNNER_TEMP/deja-vu-public" + if [[ "$RUNNER_OS" == "Windows" ]]; then + public_root="$(cygpath -u "$RUNNER_TEMP")/deja-vu-public" + fi + mkdir -p "$public_root" + # Export committed bytes rather than runner-specific CRLF conversion. + git -c core.autocrlf=false archive HEAD AGENTS.md CHANGELOG.md CONTRIBUTING.md LICENSE README.md package.json package-lock.json tsconfig.json src tests scripts docs examples starter-kit .github .gitignore encoding-status.md llms.txt | tar -x -C "$public_root" + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install locked dependencies + working-directory: ${{ runner.temp }}/deja-vu-public + run: npm ci + + - name: Type check + working-directory: ${{ runner.temp }}/deja-vu-public + run: node node_modules/typescript/bin/tsc --noEmit -p tsconfig.json + + - name: Complete source, links, compiled and installed-package acceptance + working-directory: ${{ runner.temp }}/deja-vu-public + run: npm test + + - name: Frozen recall quality and cache lifecycle counts + working-directory: ${{ runner.temp }}/deja-vu-public + run: | + node scripts/evaluate-recall.mjs dist/src/index.js dist/src/index.js > "$RUNNER_TEMP/recall-quality.json" + node scripts/benchmark-cache-ownership.mjs dist/src/index.js dist/src/index.js > "$RUNNER_TEMP/recall-cache.json" + node --input-type=module <<'NODE' + import assert from 'node:assert/strict'; + import { readFileSync } from 'node:fs'; + import { join } from 'node:path'; + const read = path => JSON.parse(readFileSync(path, 'utf8')); + const quality = read(join(process.env.RUNNER_TEMP, 'recall-quality.json')); + const expected = read('docs/engine/recall-evaluation.json'); + // Runtime metadata is recorded, not a recall quality assertion. + console.log('Evaluation runtime:', quality.node, 'recorded runtime:', expected.node); + const { node: actualNode, ...actualQuality } = quality; + const { node: expectedNode, ...expectedQuality } = expected; + assert.deepEqual(actualQuality, expectedQuality); + const cache = read(join(process.env.RUNNER_TEMP, 'recall-cache.json')); + const recorded = read('docs/engine/recall-cache-ownership-benchmark.json'); + for (const mode of ['after-unscoped', 'after-owned']) { + assert.deepEqual(cache.modes[mode], recorded.modes[mode]); + } + assert.deepEqual(cache.modes['before-unscoped'], cache.modes['after-unscoped']); + console.log('Frozen development/held-out IDs and metrics, cache call counts and retained entries: passed'); + NODE + + - name: Synthetic protocol fixture checks + working-directory: ${{ runner.temp }}/deja-vu-public + run: | + node scripts/dejavu-lint-memory.mjs --memory-root starter-kit/memory + node scripts/dejavu-lint-memory.mjs --memory-root examples/protocol-project/memory + node scripts/dejavu-lint-memory.mjs --memory-root docs/examples/settings-project/memory + npm run report:feedback + + - name: Fresh synthetic project readiness + working-directory: ${{ runner.temp }}/deja-vu-public + run: | + project_root="$RUNNER_TEMP/deja-vu-readiness" + if [[ "$RUNNER_OS" == "Windows" ]]; then + project_root="$(cygpath -u "$RUNNER_TEMP")/deja-vu-readiness" + fi + node scripts/deja-vu.mjs init --cwd "$project_root" --project-id project:ci --json + node scripts/dejavu-lint-memory.mjs --memory-root "$project_root/memory" + node scripts/deja-vu.mjs doctor --cwd "$project_root" --json diff --git a/AGENTS.md b/AGENTS.md index bb73fc6..33270d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,3 +30,16 @@ Write back only durable project memory: - safety or privacy constraints Never store secrets, full transcripts, PII, raw logs, low-signal chatter, or disposable exploration noise. + +## Verification Loop + +For implementation or repair work: + +1. Define the scope, exclusions, baseline revision and acceptance checklist before changing code. Preserve unrelated user changes and original requirements. +2. Make acceptance observable: include relevant source and compiled tests, types, package/install/use checks, regression coverage, behavior/quality checks and accurate documentation. Distinguish case counts from file-level summaries. +3. Keep evidence for each failure: command, environment, revision, exit status, original error/cause and the smallest reproducible example. Do not replace an operational error with a secondary parsing error. +4. Follow failure -> diagnosis -> focused correction -> affected regression tests -> full acceptance. State the hypothesis and use a check that can distinguish it from alternatives. +5. After two iterations without new evidence or improvement, change the hypothesis or strategy. Do not endlessly retry the same failure or spend the loop on unrelated polish. +6. Never weaken assertions, skip failing cases, hide errors or change security controls to obtain a green result. Use runtime/execution alternatives only when authorized and established not to bypass restrictions. +7. Finish only when acceptance passes or a genuine external blocker is demonstrated. For a blocker, preserve a reproducible checkpoint, identify the blocked gates and exact resource/permission needed, and report it as blocked rather than passed. +8. Review the final diff, document measured effects and limitations, and verify repository status. Commit, push, merge or publish only within the user's authorization. Task-specific data/privacy constraints take precedence over a generic verification command. diff --git a/CHANGELOG.md b/CHANGELOG.md index 846cc80..b25960c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file. ## Unreleased +- Introduce the source update on the repository homepage, with [opt-in examples, migration guidance and scoped measurements](docs/recall-update.md). No new npm package or GitHub Release is published. +- Reuse the complete synthetic-data acceptance workflow for main and pull requests, including fresh-project lint/readiness checks, without reading project memory. +- Preserve original CLI subprocess failures in test helpers, retain execution/parse causes for invalid error responses, and add ten diagnostic regressions without weakening product assertions. +- Append general acceptance/evidence/repair-loop rules to AGENTS.md and preserve the historical [Cloud blocker checkpoint](docs/verification-loop.md). +- Verify complete source, compiled, installed-package, type, recall-quality and cache gates on Ubuntu and Windows in [branch-only Actions](docs/actions-acceptance-results.md). Fix workflow context/path/newline handling and freeze the recall tie fixture's clock without changing product assertions. These changes remain unreleased. + +- Add opt-in exact query aliases, managed per-memory/revision embedding scopes, and serialized reads; digest cache keys avoid retained plaintext input keys. +- Compensate failed deletions, add explicit recoverable `supersedeMemory`, and return defensive summary/chunk copies from the in-memory adapter. +- Add opt-in prompt provenance and qualifier-loss warnings, plus explicit raw-content retrieval. +- Record development/held-out synthetic recall evaluation, ownership costs, repeated performance comparisons, and direct offline installed-package verification in the [round-two notes](docs/release-recall-lifecycle.md). Hosted validation now passes; the native cause of the historical Cloud EPERM remains unresolved. + +- Fix optional-engine false-none recall when an ineligible hybrid winner hides an eligible candidate; keep existing thresholds. +- Add opt-in candidate search breadth, bounded prompt projection, metadata-only embedding reuse, and a content/model-keyed bounded embedding cache. +- Cache lexical token sets with update/delete invalidation and validate recall/chunking limits before expensive work. +- Add synthetic regression tests, a reproducible comparison benchmark, and [unreleased notes](docs/release-recall-efficiency.md) with migration guidance and measurement limits. + ## [0.7.1] - Unreleased Release preparation only; not yet published. See the [release checklist](docs/release-v0.7.1.md). diff --git a/README.md b/README.md index 9df7a17..3ea49e4 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,23 @@ Deja Vu is a three-file, repo-local memory system that any capable coding agent No database. No vector store. No embeddings. No SaaS. No daemon. No required npm install. +## Latest update: more reliable, bounded recall + +**Available in repository source; a new npm package release has not been published.** + +The optional TypeScript engine now keeps eligible memories from being hidden by +less relevant ranking winners. It also adds bounded prompt excerpts, recoverable +deletion and supersession, and opt-in query aliases and embedding-cache reuse. +The three-file protocol and [60-second setup](#start-in-60-seconds) stay the same. + +Efficiency depends on the workload and enabled options: these changes do not +make every request faster. Explicit aliases are routing rules, not a new +multilingual semantic model. + +**[Read the full update, opt-in examples and migration notes](./docs/recall-update.md)** +· [Measurements and tradeoffs](./docs/release-recall-lifecycle.md) +· [Complete Ubuntu/Windows acceptance evidence](./docs/actions-acceptance-results.md) + ![Deja Vu turns tiny repo-local cues into minimal, relevant recall](./docs/assets/deja-vu-preview.svg) ## Start in 60 seconds @@ -196,6 +213,13 @@ const engine = createInMemorySemanticRecallEngine({ Read the [engine guide](./docs/engine/semantic-engine.md) or run the [basic example](./examples/basic/index.ts). The engine is an optional acceleration layer, not the default adoption path. +Repository source includes eligible-candidate selection, bounded prompt output, +and opt-in embedding reuse, [explicit aliases, cache ownership and provenance](docs/engine/recall-lifecycle.md). +These additions are not yet published in a new npm package. See the +[update and migration overview](docs/recall-update.md), +[usage examples](docs/engine/recall-efficiency.md), and +[complete hosted acceptance results](docs/actions-acceptance-results.md). + ## Contributing Issues and focused pull requests are welcome. Read [CONTRIBUTING.md](./CONTRIBUTING.md) for the project boundaries and verification commands. diff --git a/docs/actions-acceptance-results.json b/docs/actions-acceptance-results.json new file mode 100644 index 0000000..c10819a --- /dev/null +++ b/docs/actions-acceptance-results.json @@ -0,0 +1,168 @@ +{ + "status": "hosted acceptance passed; unreleased", + "verifiedRun": { + "id": 39089631583, + "url": "https://github.com/focaxisdev/deja-vu/actions/runs/39089631583", + "sha": "db56925d789482ba1f2a850b5833e73e7f031de4", + "status": "completed", + "conclusion": "success" + }, + "publicationMappings": [ + { + "local": "a66a52d4fceaeba3dac4a24c8e7cc82d834a4331", + "remote": "586f73016fc134c09c2e19b02e45c94b390e8c98", + "tree": "7365590fdcc4ea07a3513e9695ecf53a0e80e759", + "run": 39089273873 + }, + { + "local": "f880ca2d67b14e939164bb7fc594800598e8fcdd", + "remote": "57aaf0d9d15733aa1c3c955ed09ac275d123b601", + "tree": "da94ba9b8c80a7cfd0d79a114a8dcdbb4fb5b97e", + "parent": "586f73016fc134c09c2e19b02e45c94b390e8c98", + "files": 148, + "changed": 1, + "run": 39089353842 + }, + { + "local": "bb2b64e486e8530c8eeca09ed3d8fe6fc71f8c67", + "remote": "262e82660c23b9cc008c9d944b58b57148f77331", + "tree": "c574e262740b3ec25a212c752ba1a14eb3953e0d", + "parent": "57aaf0d9d15733aa1c3c955ed09ac275d123b601", + "files": 148, + "changed": 2, + "run": 39089505184 + }, + { + "local": "b139302c87b891cdc4e45ed735b5e2b7461b941f", + "remote": "db56925d789482ba1f2a850b5833e73e7f031de4", + "tree": "0cbc219a67320f375f3685656e2e30ecf424baa2", + "parent": "262e82660c23b9cc008c9d944b58b57148f77331", + "files": 148, + "changed": 1, + "run": 39089631583 + } + ], + "verification": { + "initialCompleteTreeFileCount": 148, + "initialDeltaFromMain": { + "added": 26, + "modified": 16, + "deleted": 0, + "modeChanges": 0 + }, + "everyPublication": "root tree SHA and each recursive file path/type/mode/blob SHA equal the source local tree; remote commit parent and branch ref verified", + "permissions": "contents: read; no release/deploy/publish job; no force updates", + "localHistory": "all original local commits retained" + }, + "jobs": [ + { + "os": "Ubuntu 24.04.5", + "id": 117563410807, + "url": "https://github.com/focaxisdev/deja-vu/actions/runs/39089631583/job/117563410807", + "node": "v22.23.3", + "npm": "10.9.9", + "source": { + "tests": 119, + "pass": 119, + "fail": 0, + "skipped": 0 + }, + "compiled": { + "tests": 119, + "pass": 119, + "fail": 0, + "skipped": 0 + }, + "packageInstallUse": { + "tests": 1, + "pass": 1, + "fail": 0, + "skipped": 0 + }, + "types": "passed", + "frozenRecallQuality": "passed", + "cacheLifecycleCounts": "passed", + "protocolFixtures": "passed", + "publicMarkdownLinks": { + "files": 69, + "status": "passed" + } + }, + { + "os": "Windows Server 2025", + "id": 117563410610, + "url": "https://github.com/focaxisdev/deja-vu/actions/runs/39089631583/job/117563410610", + "node": "v22.23.3", + "npm": "10.9.9", + "source": { + "tests": 119, + "pass": 119, + "fail": 0, + "skipped": 0 + }, + "compiled": { + "tests": 119, + "pass": 119, + "fail": 0, + "skipped": 0 + }, + "packageInstallUse": { + "tests": 1, + "pass": 1, + "fail": 0, + "skipped": 0 + }, + "types": "passed", + "frozenRecallQuality": "passed", + "cacheLifecycleCounts": "passed", + "protocolFixtures": "passed", + "publicMarkdownLinks": { + "files": 69, + "status": "passed" + } + } + ], + "failuresPreserved": [ + { + "run": 39089273873, + "kind": "workflow planning", + "jobs": 0, + "detail": "runner.temp context was used under job defaults; moved workdir resolution to steps. API exposed no validation annotation or runner log." + }, + { + "run": 39089353842, + "job": 117562559666, + "kind": "nondeterministic test fixture", + "source": { + "tests": 119, + "pass": 118, + "fail": 1, + "skipped": 0 + }, + "error": "ERR_ASSERTION: 'tie2' !== 'tie0'", + "fix": "Freeze Date in the lexical-tie/candidateLimit test; all 95 efficiency assertions retained." + }, + { + "run": 39089353842, + "job": 117562559864, + "kind": "Windows archive path", + "error": "tar: Cannot open: No such file or directory", + "fix": "Convert Windows RUNNER_TEMP to a Bash path using cygpath." + }, + { + "run": 39089505184, + "job": 117563017821, + "kind": "frozen fixture byte mismatch", + "expectedFixtureSha256": "4e1b70b5520e1d3cfb85489aad053825e339938016b20e659f2a4eb4677aee5d", + "actualFixtureSha256": "21552541a3f35ae72ba055b265d2fd99a03f67c768bd9ca9b4f07b95a9373a7e", + "fix": "Command-scoped core.autocrlf=false on git archive preserves committed LF bytes. Hash/content assertions and frozen answers unchanged." + } + ], + "limits": [ + "Hosted Node 22 results do not identify the native syscall/policy cause of Cloud Node 24 spawnSync EPERM.", + "Five historical masked errors cannot be retrospectively attributed.", + "No new latency benchmark; provider-call/retained-entry checks are not performance or token claims.", + "Source/compiled and OS counts repeat the same cases; 120 distinct selected tests, not 478 distinct tests.", + "Documentation follow-up changes require another complete run at their own published SHA." + ] +} diff --git a/docs/actions-acceptance-results.md b/docs/actions-acceptance-results.md new file mode 100644 index 0000000..4cf4cce --- /dev/null +++ b/docs/actions-acceptance-results.md @@ -0,0 +1,110 @@ +# Hosted acceptance results + +Engineering evidence for the public repository-source update. This report records +pre-merge verification; current main results are available in +[main CI](https://github.com/focaxisdev/deja-vu/actions/workflows/ci.yml). +An npm package or GitHub Release is a separate publication, not implied by CI. + +## Verified result + +[Run 39089631583](https://github.com/focaxisdev/deja-vu/actions/runs/39089631583) +completed successfully at remote commit +`db56925d789482ba1f2a850b5833e73e7f031de4`. Its tree +`0cbc219a67320f375f3685656e2e30ecf424baa2` exactly matches local commit +`b139302c87b891cdc4e45ed735b5e2b7461b941f`. All original local commits remain in +the local fix branch's ancestry; remote commits intentionally have different SHAs. + +| Gate | Ubuntu | Windows | +| --- | --- | --- | +| TypeScript no-emit check | Passed | Passed | +| Complete source tests | 119 passed, 0 failed, 0 skipped | 119 passed, 0 failed, 0 skipped | +| Fresh build and complete compiled tests | 119 passed, 0 failed, 0 skipped | 119 passed, 0 failed, 0 skipped | +| Package, offline installation, installed CLI/API use | 1 passed, 0 failed, 0 skipped | 1 passed, 0 failed, 0 skipped | +| Frozen recall IDs, metrics and fixture digest | Passed | Passed | +| Cache provider-call and retained-entry counts | Passed | Passed | +| Synthetic protocol fixtures and feedback | Passed | Passed | +| Public Markdown links | 69 files, passed | 69 files, passed | + +Both jobs used Node v22.23.3 and npm 10.9.9. Logs: +[Ubuntu](https://github.com/focaxisdev/deja-vu/actions/runs/39089631583/job/117563410807), +[Windows](https://github.com/focaxisdev/deja-vu/actions/runs/39089631583/job/117563410610). +The source and compiled counts repeat the same 119 cases on two operating +systems; the package test is a separate case, not hundreds of distinct new tests. +The 50 engine cases and ten diagnostic-helper cases are included, alongside all +CLI/reliability cases. No failing assertion or test selection was removed. + +The [machine-readable record](actions-acceptance-results.json) includes every +publication mapping and earlier failures. This report records the run above; +documentation follow-up commits must receive their own successful complete run. +Their source local SHA/tree are recorded in each remote commit message. + +## Evidence-driven corrections + +1. [Initial run](https://github.com/focaxisdev/deja-vu/actions/runs/39089273873): + workflow failure with zero jobs. The API exposed no validation annotation or + runner log. Inspection found `runner.temp` used under job defaults, where that + context is unavailable. Moving unchanged working directories to the steps + allowed both jobs to start. This was a workflow defect, not a test result. +2. [Next run](https://github.com/focaxisdev/deja-vu/actions/runs/39089353842): + Ubuntu executed all 119 source tests: 118 passed, one failed with + `ERR_ASSERTION: 'tie2' !== 'tie0'`. The candidate-breadth test generated each + record's timestamp independently, so lexical ties were not necessarily hybrid + ties. A synthetic timing example reproduced the recency distinction. Freeze + Date for that fixture; retain all 95 efficiency assertions, including the + original `tie0` and widened-search `tie5` checks. Its local source/compiled + 25-case suites passed before the complete hosted rerun. +3. In that same run Windows failed before tests: `tar` could not open the native + Windows temporary path from Git Bash. Convert the export destination with + `cygpath`, without changing security controls or test behavior. +4. [Third run](https://github.com/focaxisdev/deja-vu/actions/runs/39089505184): + both platforms passed all source/compiled/package tests; Windows then failed + the frozen fixture hash assertion. Its observed SHA256 was exactly the hash + obtained by converting the fixture's LF bytes to CRLF. Reproducing this with + `git -c core.autocrlf=true archive` confirmed the cause. Use command-scoped + `core.autocrlf=false` for export. Keep the original fixture hash, labels, + returned IDs and metric assertions. No persistent Git configuration changed. +5. The next complete run passed all gates on both platforms. Product source, + thresholds, dependency lockfile and AGENTS loop rules were unchanged throughout + these hosted-test corrections. + +## Publication integrity and scope + +After the user added the repository to the existing GitHub installation, the +authorized retry of the same previously denied `create_tree` operation succeeded. +The initial published tree exactly matched local +`a66a52d4fceaeba3dac4a24c8e7cc82d834a4331`; remote commit +`586f73016fc134c09c2e19b02e45c94b390e8c98` has the verified main commit +`188d7e8c3aed224e45071d041cd140509887b29f` as its parent. The initial 148-file +tree added 26 files and modified 16, without deletions or mode changes. + +Every publication compares the complete recursive path/type/mode/blob-SHA list +and root tree SHA, checks the commit parent, and verifies the branch ref. Later +updates are non-force descendants of the remote branch. Changed blobs are reviewed +for private paths and credential patterns; unchanged baseline blobs are retained +by reference. Root project memory and credentials are not read or uploaded anew. +Hosted tests export public source and synthetic fixtures, excluding root memory. + +At the recorded runs, the dedicated workflow triggered only for the fix branch. +It is now also reused by main/PR CI, retains `contents: read`, does not persist +checkout credentials, and has no release or deployment step. Main/PR verification +uses the same complete tests, adding fresh synthetic-project lint/readiness checks +instead of reading project memory. The original +[permission-denial checkpoint](actions-publication-evidence.json) remains historical +evidence, not the current publication status. + +## Limits + +- Hosted Node 22 passes establish the tested package behavior on these two runner + images. They do not establish Node 20 coverage or identify the native syscall + responsible for the earlier Cloud Node 24 `spawnSync EPERM`. No Cloud security + setting, proxy or credential was changed, and denied calls were not blindly retried. +- The earlier 32 failures remain nine direct EPERM, 18 infrastructure assertions + containing EPERM, and five parse failures whose original causes were lost. New + passing runs cannot reconstruct those five historical causes. +- The synthetic language evaluation and authored aliases retain their previous + limitations. Aliases are explicit routing, not a multilingual semantic model. +- No latency benchmark was added in this CI round. Cache call counts are not + end-to-end speedups; accounted bytes are not tokens. The prior performance and + ownership tradeoffs remain in the [release notes](release-recall-lifecycle.md). +- A green CI result or source merge is not an npm package or GitHub Release. + Package/release publication requires separate authorization. diff --git a/docs/actions-acceptance.md b/docs/actions-acceptance.md new file mode 100644 index 0000000..dd104d9 --- /dev/null +++ b/docs/actions-acceptance.md @@ -0,0 +1,132 @@ +# Branch acceptance on GitHub Actions + +Unreleased verification setup, continuing the [Cloud checkpoint](verification-loop.md). +Starting revision: `4c05a77605f6cf63a10261dc394c0824208313ea`. + +Current outcome: the authorized connector publication succeeded after installation +access was corrected, and both hosted jobs passed all gates. See the +[verified results, logs and SHA mappings](actions-acceptance-results.md). The +transport and permission sections below preserve earlier checkpoints. + +## Scope and acceptance defined before execution + +The `Recall acceptance` workflow runs directly on pushes to +`fix/recall-efficiency-and-budgets` and is reused by main/PR CI through +`workflow_call`, with `contents: read`, Node 22, and the existing Ubuntu/Windows +runner matrix. It does not publish, deploy, create a release or access configured +secrets. Checkout does not persist credentials. Main/PR runs use the same complete +test selection and frozen evaluation assertions as branch acceptance; fresh +synthetic-project lint/readiness checks replace inspection of project memory. + +Both jobs must finish successfully at the exact pushed SHA: + +| Gate | Command and required evidence | +| --- | --- | +| Types | Existing TypeScript compiler, `--noEmit -p tsconfig.json` | +| Complete source suite | `npm test` invokes the unchanged complete source selection, including helper regressions | +| Complete compiled suite | `npm test` builds and runs all six compiled test files | +| Pack/install/use | `npm test` invokes the package test, including prepack, offline local installation, installed CLI and engine API assertions | +| Regressions, lifecycle and projection | All existing product assertions in the complete selections remain enabled | +| Recall quality | Exact frozen fixture digest, returned IDs, development/held-out metrics and interpretation; runtime version is recorded separately | +| Cache measurements | Current unscoped/owned provider-call and retained-entry counts match the frozen record; these are not latency measurements | +| Documentation and protocol | Existing link checker plus public synthetic fixture lint and feedback checks | + +The workflow exports the committed public source and synthetic fixtures to the +runner's temporary directory. It excludes the project's root `memory/` tree, +retains every test, and runs the unmodified link checker over that public snapshot. +Package tests initialize their own synthetic project. No private project-memory +read is needed. Dependencies come from the existing lockfile; no dependency is +added. Hosted runner results establish behavior on those runtimes, not the exact +cause of the Cloud Node 24 native EPERM. + +For a failed run, retain its SHA, job/log URL, original error and failed gate; +diagnose before making a focused correction. Rerun the complete acceptance after +each correction. Do not turn a blocked or skipped gate into a pass. Follow the +repository's verification loop and stop on an actual access denial. + +## Initial transport checkpoint + +The initial local HEAD and clean worktree were verified. The remote repository is +public; its main SHA is `188d7e8c3aed224e45071d041cd140509887b29f`. The only +existing workflow is `ci.yml`, triggered by main pushes and pull requests, with +read-only contents permission and no release/publish/deploy step. + +The ordinary read-only transport check +`git ls-remote origin refs/heads/main refs/heads/fix/recall-efficiency-and-budgets` +failed with exit 128: + +```text +fatal: unable to access 'https://github.com/focaxisdev/deja-vu.git/': Failed to connect to proxy port 8080 after 0 ms: Could not connect to server +``` + +The existing GitHub connector can read this repository and reports push access, +but its branch listing has no fix branch and the Git commit endpoint returns 404 +for `4c05a77605f6cf63a10261dc394c0824208313ea`. Its commit-creation interface does +not accept original author/committer metadata or raw Git objects, so recreating +commits through it cannot preserve the existing commit SHAs. No substitute history +or remote branch was created. No push or Actions run is claimed. + +An ordinary push would require a working connection to the Cloud's configured +Git proxy. No new secret, broader repository permission, direct connection, or +security-setting change was attempted. At this checkpoint, all GitHub Actions +gates were **not run / transport blocked**. + +Local preparation checks passed: workflow YAML parsing and trigger/permission/gate +assertions, Bash syntax checking of all run blocks, TypeScript no-emit checking, +and the exact frozen-quality/cache assertion block using the previously compiled +unchanged engine. The unmodified link checker passed over the public snapshot +(69 Markdown files). These checks do not execute a hosted job, `npm ci`, or the +blocked complete subprocess suites and cannot establish Actions acceptance. + +## Authorized connector publication + +The subsequent Cloud UI push also reported failure. A fresh remote branch listing +still showed no fix branch and main remained at the SHA above. The user authorized +an alternative: preserve all original local commits, but create a new remote +commit on that verified main with the exact final local tracked tree. Remote and +local commit SHAs are intentionally different; this is not a force push or a +claim that the original local commit objects were transferred. + +Before creating the branch, compare the remote tree SHA and every recursive +entry's path, type, mode and object SHA against the local tree. Transfer only +reviewed changed blobs, retaining unchanged public baseline objects by reference. +Include additions, modifications and deletions in the comparison. Do not read or +upload private memory, credentials or untracked files. Record the source local +commit and tree in the remote commit message, then record the returned remote SHA +in the verification checkpoint. If the connector rejects access or cannot preserve +the tree, stop without substituting an incomplete tree. Hosted acceptance remains +unverified until both jobs reach successful terminal states at the published SHA. + +## Historical connector result: permission blocked + +The reviewed source snapshot is local commit +`c2a9bb103e4f502d23a66a6c067f00c8b0b10467`, tree +`66a2c3949b8ad087289f438b451c47e608413b68`. All 122 baseline file entries +matched the remote main tree by path, type, mode and object SHA. Its final tree +has 147 files: 25 additions, 16 modifications, no deletions or mode changes. +Changed UTF-8 bytes and blob hashes were checked locally; credential-pattern +checks had no alerts. Only reviewed changed files were submitted, without reading +private memory or credential files. Unchanged baseline blobs were referenced. + +The first write, `POST /repos/focaxisdev/deja-vu/git/trees` through the existing +connector's `create_tree`, returned HTTP 403: + +```text +Resource not accessible by integration +``` + +No remote commit or branch was created, no alternative write endpoint was tried, +and no permission/security configuration was changed. A subsequent read confirmed +main unchanged and the target branch still absent. Repository metadata reporting +`push: true` did not establish integration access to this write endpoint. The exact +missing permission is not exposed by this error and is not assumed to be a +specific workflow or contents scope. + +At that checkpoint all hosted gates were **not run / connector permission blocked**. The final +remote tree could not be created or verified; there is no remote SHA or Actions +run/log URL. A maintainer must restore ordinary approved Git transport or provide +an already-authorized connection that can perform this Git tree write before +publication can continue. This task does not expand permissions or add secrets. +The [publication evidence](actions-publication-evidence.json) records every changed +path, mode and blob SHA, the source mapping and the denied action. The original +local history is retained; later local documentation commits record this blocker. diff --git a/docs/actions-publication-evidence.json b/docs/actions-publication-evidence.json new file mode 100644 index 0000000..0a0bd76 --- /dev/null +++ b/docs/actions-publication-evidence.json @@ -0,0 +1,314 @@ +{ + "sourceLocalCommit": "c2a9bb103e4f502d23a66a6c067f00c8b0b10467", + "sourceLocalTree": "66a2c3949b8ad087289f438b451c47e608413b68", + "remoteBase": "188d7e8c3aed224e45071d041cd140509887b29f", + "remoteBaseTree": "cb95df23b058b72a75b365b4d352e737f376241f", + "baselineEntriesVerified": 122, + "finalLocalEntries": 147, + "changes": [ + { + "path": ".github/workflows/recall-acceptance.yml", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "8b1ed489e6415c500c4023169695c006dd89b085" + }, + { + "path": "AGENTS.md", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "33270d556f5c1be0d02ca54c32dc8e09f584a7ba" + }, + { + "path": "CHANGELOG.md", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "4ef58f21f543ca5961e95ecf2fa20b215ed962da" + }, + { + "path": "README.md", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "e2320bbbf83644662fe84808b76207c782ab2e3f" + }, + { + "path": "docs/actions-acceptance.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "0b0307a9b95bf570c186cf8219e6932b68f59e8d" + }, + { + "path": "docs/engine/recall-benchmark.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "95d93c5bb453d61eebf5f5cb231afcea2a00bd8e" + }, + { + "path": "docs/engine/recall-cache-ownership-benchmark.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "f92893f3cd63bf547945baa69d16455cbaa45955" + }, + { + "path": "docs/engine/recall-efficiency.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "c80a43583a6a55530dd51b68129aebf2416baf91" + }, + { + "path": "docs/engine/recall-evaluation.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "e95e87e688358f55caf086f203d071ebd6c9c729" + }, + { + "path": "docs/engine/recall-lifecycle.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "2591379befb4825968837b52c04f385173c80366" + }, + { + "path": "docs/engine/recall-round2-benchmark.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "861faee413e75d501f15ea69c75eb77cd8a98584" + }, + { + "path": "docs/engine/semantic-engine.md", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "42ceb868d1acca6ce8671a38171acdc3e3bd2537" + }, + { + "path": "docs/release-recall-efficiency.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "d5b44490488e97637f678d4b9a11ea3b2c1b2bfa" + }, + { + "path": "docs/release-recall-lifecycle.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "0bf5db999b1d1725322653c501b2e6743fecb40d" + }, + { + "path": "docs/social-recall-lifecycle-draft.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "1da8f6e526cad1c92d048d74f26cad8995cf7f2a" + }, + { + "path": "docs/verification-loop-evidence.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "e7e5de1ebf19f10b575ceb062ac5b8a4e9d5871c" + }, + { + "path": "docs/verification-loop.md", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "0e55770b76e82b314d5d1dcbaf9071210587c716" + }, + { + "path": "package.json", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "0a50d3ab17808c647384733fbc9f7493ef69b701" + }, + { + "path": "scripts/benchmark-cache-ownership.mjs", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "1d8a23e7f9941c3c5c4f9fb6cc569818ac661f59" + }, + { + "path": "scripts/benchmark-recall.mjs", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "004f08a2ad406deb0659e9fea6cfda79ede5cc57" + }, + { + "path": "scripts/evaluate-recall.mjs", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "a78f6102ba5d7aa8820ae869365605e3ed6295eb" + }, + { + "path": "scripts/fixtures/recall-evaluation.json", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "3e3aa9cc7fa9d2cef41b3b1035520f158e357831" + }, + { + "path": "src/core/semantic-recall-engine.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "14392eae632bbb459e54499a97135729275e8b9c" + }, + { + "path": "src/index.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "a161f611bf081328e0c408d71f82403288a914b2" + }, + { + "path": "src/memory/default-chunker.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "8eea21770ca802a4b686193b52089d5f2d4c0bd5" + }, + { + "path": "src/plugins/cached-embedding-provider.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "b022f6d4b9e054093782b11df0b8bd015931aec6" + }, + { + "path": "src/plugins/create-in-memory-engine.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "ab0566ed641cec6bd7bc8505fdf95e9491d4319f" + }, + { + "path": "src/retrieval/cue-aliases.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "58088a6571f8c1a392e31a28cf515ec9b35e792e" + }, + { + "path": "src/retrieval/prompt-projection.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "55b8e694749dafba4099f4bc93652b5b7dfa251b" + }, + { + "path": "src/storage/in-memory-storage.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "fe57cd4bab44fe33766ee8e3568160be6029a199" + }, + { + "path": "src/storage/in-memory-vector-store.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "8f2f74767246be6a93bb8ba4fc3d456b5af2bfb8" + }, + { + "path": "src/types/memory.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "29d17a0b8752ac4e4ad2ab142d33b901a5a2808a" + }, + { + "path": "src/types/plugins.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "893f997f8c2b9976592d7a69feb009b589507deb" + }, + { + "path": "src/utils/limits.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "b15ca1ff1f6f5b923129968e7fb7bc70c79c4475" + }, + { + "path": "tests/cli-errors.test.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "7447239aa5e8ada5238f339040556ac6c1ec8537" + }, + { + "path": "tests/helpers/cli-errors.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "35d6b9300479f6429acac2c31af8142409f5267b" + }, + { + "path": "tests/memory-cli.test.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "18d736b13a1cf5b6ef8a4c3eadf8f6579bb3de2c" + }, + { + "path": "tests/package-pack.test.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "15c3800262e09e0e4edfaf4716786679cf32280a" + }, + { + "path": "tests/recall-efficiency.test.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "a37c0afd8634ccc029b551fad4cfceaea8cc9bed" + }, + { + "path": "tests/recall-lifecycle.test.ts", + "mode": "100644", + "type": "blob", + "action": "add", + "sha": "5e944848c81426f1556cce594455d36b22c2342b" + }, + { + "path": "tests/reliability.test.ts", + "mode": "100644", + "type": "blob", + "action": "modify", + "sha": "c52772bdbed7671760abaa97f2264965b2fdba48" + } + ], + "verification": { + "remoteBaselineTree": "exact match", + "changedBlobEncodingAndHashes": "verified locally", + "secretPatternAlerts": 0, + "finalRemoteTree": "not created or verified; API denied", + "remoteBranch": "absent", + "actions": "not run" + }, + "blockedAction": { + "tool": "github_create_tree", + "method": "POST", + "endpoint": "https://api.github.com/repos/focaxisdev/deja-vu/git/trees", + "status": 403, + "message": "Resource not accessible by integration" + }, + "remoteCommit": null, + "remoteTree": null +} diff --git a/docs/engine/recall-benchmark.json b/docs/engine/recall-benchmark.json new file mode 100644 index 0000000..95d93c5 --- /dev/null +++ b/docs/engine/recall-benchmark.json @@ -0,0 +1,335 @@ +{ + "node": "v24.19.0", + "samples": 25, + "warmups": 5, + "units": "milliseconds", + "notes": [ + "Micro: storage.searchImpressions only. Scan: engine.scanImpressions.", + "API end-to-end: resident recall(loadChunks=true) plus JSON.stringify, including query embedding and chunk retrieval.", + "Excludes process start, index construction, persistence, network, model inference and host/LLM execution.", + "Baseline/current alternate measurement order. Medians/p95 are descriptive of this small synthetic run." + ], + "workloads": [ + { + "n": 100, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0083, + "p95": 0.0254 + }, + "current": { + "median": 0.0042, + "p95": 0.0172 + } + }, + "scan": { + "baseline": { + "median": 0.0129, + "p95": 0.0435 + }, + "current": { + "median": 0.0098, + "p95": 0.0535 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0694, + "p95": 1.6628 + }, + "current": { + "median": 0.0654, + "p95": 0.155 + } + } + }, + { + "n": 100, + "kind": "common", + "micro": { + "baseline": { + "median": 0.282, + "p95": 0.5696 + }, + "current": { + "median": 0.0236, + "p95": 0.1202 + } + }, + "scan": { + "baseline": { + "median": 0.2596, + "p95": 1.6196 + }, + "current": { + "median": 0.0304, + "p95": 0.1273 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.3136, + "p95": 1.9662 + }, + "current": { + "median": 0.0906, + "p95": 1.273 + } + } + }, + { + "n": 100, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0006, + "p95": 0.001 + }, + "current": { + "median": 0.0005, + "p95": 0.0007 + } + }, + "scan": { + "baseline": { + "median": 0.0014, + "p95": 0.0039 + }, + "current": { + "median": 0.0019, + "p95": 0.0024 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0027, + "p95": 0.0031 + }, + "current": { + "median": 0.0032, + "p95": 0.005 + } + } + }, + { + "n": 1000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0053, + "p95": 0.0058 + }, + "current": { + "median": 0.003, + "p95": 0.0046 + } + }, + "scan": { + "baseline": { + "median": 0.0085, + "p95": 0.0104 + }, + "current": { + "median": 0.0073, + "p95": 0.0085 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0506, + "p95": 0.2037 + }, + "current": { + "median": 0.0518, + "p95": 0.5262 + } + } + }, + { + "n": 1000, + "kind": "common", + "micro": { + "baseline": { + "median": 3.1107, + "p95": 4.956 + }, + "current": { + "median": 0.1643, + "p95": 0.4436 + } + }, + "scan": { + "baseline": { + "median": 2.4218, + "p95": 4.4501 + }, + "current": { + "median": 0.1747, + "p95": 0.3611 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 2.4995, + "p95": 3.9804 + }, + "current": { + "median": 0.2487, + "p95": 0.6969 + } + } + }, + { + "n": 1000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0004, + "p95": 0.0005 + }, + "current": { + "median": 0.0004, + "p95": 0.0005 + } + }, + "scan": { + "baseline": { + "median": 0.0012, + "p95": 0.0016 + }, + "current": { + "median": 0.0015, + "p95": 0.002 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0026, + "p95": 0.0028 + }, + "current": { + "median": 0.0028, + "p95": 0.0032 + } + } + }, + { + "n": 10000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0035, + "p95": 0.0077 + }, + "current": { + "median": 0.0014, + "p95": 0.0027 + } + }, + "scan": { + "baseline": { + "median": 0.0056, + "p95": 0.0079 + }, + "current": { + "median": 0.004, + "p95": 0.0066 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0503, + "p95": 0.133 + }, + "current": { + "median": 0.0478, + "p95": 0.1606 + } + } + }, + { + "n": 10000, + "kind": "common", + "micro": { + "baseline": { + "median": 30.7427, + "p95": 42.9272 + }, + "current": { + "median": 2.6053, + "p95": 6.1685 + } + }, + "scan": { + "baseline": { + "median": 30.5848, + "p95": 44.0301 + }, + "current": { + "median": 2.528, + "p95": 4.9677 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 30.8083, + "p95": 39.9615 + }, + "current": { + "median": 2.7437, + "p95": 4.7024 + } + } + }, + { + "n": 10000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0009, + "p95": 0.0015 + }, + "current": { + "median": 0.0004, + "p95": 0.0009 + } + }, + "scan": { + "baseline": { + "median": 0.0024, + "p95": 0.0033 + }, + "current": { + "median": 0.0024, + "p95": 0.0027 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0032, + "p95": 0.0046 + }, + "current": { + "median": 0.003, + "p95": 0.005 + } + } + } + ], + "operations": { + "baseline": { + "addCalls": 12, + "repeatedRecallCalls": 5, + "metadataUpdateCalls": 12, + "rawRecallBytes": 10253 + }, + "currentOptIn": { + "addCalls": 4, + "repeatedRecallCalls": 1, + "metadataUpdateCalls": 0, + "rawRecallBytes": 10253, + "projectedBytes": 2301 + } + } +} diff --git a/docs/engine/recall-cache-ownership-benchmark.json b/docs/engine/recall-cache-ownership-benchmark.json new file mode 100644 index 0000000..f92893f --- /dev/null +++ b/docs/engine/recall-cache-ownership-benchmark.json @@ -0,0 +1,32 @@ +{ + "note": "Deterministic provider-call counts, not latency or token costs. Sixteen memories share ten identical chunks; unique per-memory routing tags. Cache capacity 256.", + "modes": { + "before-unscoped": { + "addCalls": 18, + "twoRecallPassesCalls": 16, + "metadataUpdateCalls": 0, + "contentUpdateCalls": 2, + "retainedAfterDelete": "No public stats API in baseline" + }, + "after-unscoped": { + "addCalls": 18, + "twoRecallPassesCalls": 16, + "metadataUpdateCalls": 0, + "contentUpdateCalls": 2, + "retainedAfterDelete": { + "entries": 36, + "accountedBytes": 5184 + } + }, + "after-owned": { + "addCalls": 48, + "twoRecallPassesCalls": 16, + "metadataUpdateCalls": 0, + "contentUpdateCalls": 3, + "retainedAfterDelete": { + "entries": 0, + "accountedBytes": 0 + } + } + } +} diff --git a/docs/engine/recall-efficiency.md b/docs/engine/recall-efficiency.md new file mode 100644 index 0000000..c80a435 --- /dev/null +++ b/docs/engine/recall-efficiency.md @@ -0,0 +1,187 @@ +# Bounded recall for the optional engine + +These changes are unreleased. The three-file protocol, Markdown source of truth, +and CLI workflow do not need a migration. The optional engine now prevents an +ineligible hybrid winner from hiding an eligible match, reuses token sets, and +provides explicit controls for search breadth, prompt output and embedding reuse. + +## Find candidates before choosing output + +`topK` still defaults to five and limits returned candidates. The new +`candidateLimit` defaults to `topK`; set it explicitly to search more records +before ranking. It must be at least `topK`. Both are integers from 0 to 1000. +`topK: 0` returns no candidates without calling storage search. + +Candidates that meet the existing weak threshold come before candidates that +meet neither threshold. Hybrid ordering is preserved within both groups, +including between weak and strong matches. Scores are therefore not necessarily +globally descending across the eligibility boundary. With no eligible candidates, +`matched` remains false and no summary or detail is loaded. + +The default thresholds remain strong 0.85 and weak 0.75. A configured threshold +must be finite and satisfy `0 <= weak <= strong <= 1`. Increasing search breadth +can surface a better candidate, but also costs more ranking work and may expose +more irrelevant matches. It does not change the lexical candidate generator or +recover synonyms that have no shared tokens. + +## Opt in to bounded prompt output + +```ts +import { + createInMemorySemanticRecallEngine, + projectRecallForPrompt, +} from "@focaxisdev/deja-vu"; + +const engine = createInMemorySemanticRecallEngine(); +await engine.addMemory({ + id: "settings", + title: "Settings sync", + content: "Decision: settings sync stays server-backed.", +}); +const recall = await engine.recall({ + text: "settings sync", + candidateLimit: 20, + topK: 5, + loadChunks: true, + chunkLimit: 3, +}); +const context = projectRecallForPrompt(recall, { maxBytes: 4096 }); +// Put context.text in the host's memory-data section of the prompt. +// Check context.truncated before relying on it as complete evidence. +console.log(context.bytes, context.truncated); +``` + +The projection returns JSONL containing only summary text, recent updates, chunk +text, and their identifying fields. It excludes vectors, arbitrary metadata, +candidate lists, scores and internal budget explanations. The existing +`RecallResult` is unchanged; serializing it directly still includes vectors. + +The byte budget covers the entire `context.text`, including JSON framing and +newlines. It defaults to 8192 UTF-8 bytes, accepts 0 through 1,048,576, and returns +empty text if even a record header cannot fit. Text is added in description, +context, intent, recent-updates, then chunk order. The last included field may be +cut at a Unicode code-point boundary. JSON escaping keeps each included record +parseable, but truncation can remove a qualifier or omit an important chunk. + +For an actual token budget, supply the target model's tokenizer. No tokenizer is +installed or guessed by the library: + +```ts +import { projectRecallForPrompt, type RecallResult } from "@focaxisdev/deja-vu"; + +function promptMemory(result: RecallResult, countModelTokens: (text: string) => number) { + return projectRecallForPrompt(result, { + maxBytes: 4096, + maxTokens: 800, + countTokens: countModelTokens, + }); +} +``` + +`maxTokens` accepts 0 through 1,048,576 and requires a deterministic counter that +returns a non-negative safe integer. Each accepted prefix is checked as a whole; +a non-monotonic tokenizer can leave unused budget. `tokens` is reported only when +a counter is supplied. Reserve separate space for the host's wrappers and other +prompt sections. The projection controls size and fields; it does not make memory +text trusted instructions or sanitize prompt injection. + +## Opt in to embedding reuse + +```ts +import { + CachedEmbeddingProvider, + MockEmbeddingProvider, + createInMemorySemanticRecallEngine, +} from "@focaxisdev/deja-vu"; + +// The mock is deterministic local test data, not a semantic language model. +const embeddings = new CachedEmbeddingProvider(new MockEmbeddingProvider(256), { + modelKey: "mock:256:v1", + maxEntries: 128, + maxBytes: 8 * 1024 * 1024, +}); +const engine = createInMemorySemanticRecallEngine({ + embeddingProvider: embeddings, + preserveDerivedOnMetadataUpdate: true, +}); +await engine.addMemory({ id: "settings", title: "Settings sync", content: "Settings sync stays server-backed." }); +await engine.updateMemory("settings", { importance: 0.9, metadata: { reviewed: "yes" } }); +// No summaries, chunks or embeddings are regenerated for this update. +embeddings.clear(); // Release this wrapper's retained digest/vector entries when needed. +``` + +`preserveDerivedOnMetadataUpdate` defaults to false. With it enabled, an update +whose keys are limited to `importance` and `metadata` preserves summary text, +chunk IDs, chunk importance/timestamps, embeddings, raw content, creation time, +and last-access time. It updates the memory's modification time, familiarity +importance, and merged summary metadata. Title and tags remain authoritative. +The existing compensated write path still runs; this is not a zero-I/O update. + +This mode is a contract to preserve derived content, including for custom +summary generators and chunkers. Do not enable it if changing metadata must +regenerate those fields. Any other update key uses the normal preparation path. +The default behavior still invokes custom generators on metadata updates. + +The embedding cache is optional, local to one wrapper instance, and stores only +completed embeddings. Its key is now a keyed digest of exact input text and `modelKey`, without +text normalization; see the [ownership guide](recall-lifecycle.md). Include provider, model revision, dimensions and preprocessing +options in that identifier. A callback may supply the current identifier when +configuration changes. Keep provider configuration stable for each in-flight +call: the key must identify the model that actually produced its vector. +Model changes also require the host to rebuild stored +corpus vectors; cache invalidation alone does not make mixed-model vectors valid. + +The defaults are 128 entries and 8 MiB of accounted key/vector storage. Limits +are 0–10,000 entries and 0–64 MiB. Setting either limit to zero disables retention. +Accounted storage is 128 bytes per digest plus vector float64 bytes, not a measurement +of the JavaScript heap. Entry count also bounds per-entry overhead. Oversized +entries bypass the cache, failures are not cached, and returned vectors are copies. +Concurrent misses may compute separately; there is no unbounded pending-promise +map. Clearing the cache prevents already-pending requests from repopulating it. + +Use a separate cache instance for each project/security scope. Without managed +ownership, deleting a memory does not selectively release shared cache entries; +use `clear()` for the wrapper or opt in to +[per-memory/revision scopes](recall-lifecycle.md). Scopes prevent late insertion +and avoid shared-entry ownership ambiguity, at the cost of cross-memory reuse. +Reference release is not verified secure erasure. There is no recall-result cache, +but default unsynchronized reads can overlap writes; enable managed ownership or +`consistentReads` for ordering within one engine instance. + +## Migration checks + +- `topK`, `candidateLimit` and `chunkLimit` reject negative, fractional, non-finite + or greater-than-1000 values with `RangeError`, before engine adapter calls. + `chunkLimit` still defaults to three; zero skips query embedding and detail reads. + Built-in storage/vector search methods validate their limits too. +- Default chunking accepts `maxCharacters` from 1 to 1,000,000 and overlap from + zero to `maxCharacters - 1`. The default remains 280/40; for a smaller custom + maximum, omitted overlap is capped to fit. Previously invalid sizes now reject. +- Familiarity getters/search results from the built-in storage return copies. + Use `saveFamiliarity`, `updateMemory` and `deleteMemory` to change indexed data; + mutating a returned object no longer edits the store. +- Token sets are created on save and invalidated on replacement/deletion. The + lexical score and posting-order tie behavior are retained, including Unicode + behavior. There is no automatic language segmentation or stop-word policy. Exact host-authored + query aliases are available as a separate opt-in. +- Leave the new opt-ins disabled to retain the previous search breadth, raw + result shape and metadata regeneration behavior. Eligibility correction, + numeric validation and token-set reuse apply by default. + +## Measurements and deferred work + +See the [unreleased notes](../release-recall-efficiency.md) for measured results +and validation limits. The [benchmark script](../../scripts/benchmark-recall.mjs) +uses synthetic fixtures only. Token caching increases resident index memory and +moves work to writes; defensive copies also allocate memory. Heap usage and +index construction are outside the hot-path benchmark. + +CLI index persistence, route/frontmatter caching and supersession graph changes +are deferred. This batch leaves CLI implementation and the three-file protocol +unchanged. Also deferred: automatic language-aware candidate generation, semantic fallback, +ANN, chunk-by-ID adapter reads, batch embedding APIs and durable transactions. +No paid service, runtime dependency or daemon is introduced. The built-in +in-memory adapters remain nonpersistent, and the default mock embedding model +still does not provide semantic or multilingual understanding. + +For newer lifecycle opt-ins, digest-key byte-budget migration, raw retrieval and inline truncation warnings, read the [round-two guide](recall-lifecycle.md). diff --git a/docs/engine/recall-evaluation.json b/docs/engine/recall-evaluation.json new file mode 100644 index 0000000..e95e87e --- /dev/null +++ b/docs/engine/recall-evaluation.json @@ -0,0 +1,1675 @@ +{ + "fixtureSha256": "4e1b70b5520e1d3cfb85489aad053825e339938016b20e659f2a4eb4677aee5d", + "node": "v24.19.0", + "interpretation": "Recall@K is mean recall over positive queries, considering only eligible candidates. False recalls include any non-gold eligible ID in the first K; only the first candidate is loaded by recall(). Exact authored aliases are routing rules, not learned multilingual semantics. Held-out topics are disjoint; this small author-created split is not independent validation.", + "splits": { + "development": { + "before": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 0.5, + "missedQueries": 2, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "d-01", + "category": "chinese", + "query": "快取失效規則", + "relevant": [ + "d-cache" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-cache" + ] + }, + { + "id": "d-02", + "category": "chinese", + "query": "登入重試", + "relevant": [ + "d-login" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-login" + ] + }, + { + "id": "d-03", + "category": "chinese", + "query": "更新後快取怎麼辦", + "relevant": [ + "d-cache" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "d-04", + "category": "chinese", + "query": "無法登入怎麼處理", + "relevant": [ + "d-login" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "d-05", + "category": "synonym", + "query": "restore backup", + "relevant": [ + "d-backup" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "d-06", + "category": "synonym", + "query": "night mode", + "relevant": [ + "d-theme" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "d-07", + "category": "cross-language", + "query": "CACHE EVICTION", + "relevant": [ + "d-cache" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "d-08", + "category": "cross-language", + "query": "SIGN IN RETRY", + "relevant": [ + "d-login" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "d-09", + "category": "cross-language", + "query": " recover archived data ", + "relevant": [ + "d-backup" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [] + }, + { + "id": "d-10", + "category": "paraphrase", + "query": "重新整理存下來的線索以免拿到舊版", + "relevant": [ + "d-cache" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-11", + "category": "paraphrase", + "query": "進不去帳號時應該再試幾次", + "relevant": [ + "d-login" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-12", + "category": "paraphrase", + "query": "希望找回昨日保存的內容", + "relevant": [ + "d-backup" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-13", + "category": "synonym", + "query": "dark appearance", + "relevant": [ + "d-theme" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [] + }, + { + "id": "d-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "d-15", + "category": "unrelated", + "query": "do not restore backup", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "d-16", + "category": "unrelated", + "query": "sign in retry price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "d-17", + "category": "stale", + "query": "dretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "d-18", + "category": "stale", + "query": "ddeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + }, + "after-default": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 0.5, + "missedQueries": 2, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "d-01", + "category": "chinese", + "query": "快取失效規則", + "relevant": [ + "d-cache" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-cache" + ] + }, + { + "id": "d-02", + "category": "chinese", + "query": "登入重試", + "relevant": [ + "d-login" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-login" + ] + }, + { + "id": "d-03", + "category": "chinese", + "query": "更新後快取怎麼辦", + "relevant": [ + "d-cache" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "d-04", + "category": "chinese", + "query": "無法登入怎麼處理", + "relevant": [ + "d-login" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "d-05", + "category": "synonym", + "query": "restore backup", + "relevant": [ + "d-backup" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "d-06", + "category": "synonym", + "query": "night mode", + "relevant": [ + "d-theme" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "d-07", + "category": "cross-language", + "query": "CACHE EVICTION", + "relevant": [ + "d-cache" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "d-08", + "category": "cross-language", + "query": "SIGN IN RETRY", + "relevant": [ + "d-login" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "d-09", + "category": "cross-language", + "query": " recover archived data ", + "relevant": [ + "d-backup" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [] + }, + { + "id": "d-10", + "category": "paraphrase", + "query": "重新整理存下來的線索以免拿到舊版", + "relevant": [ + "d-cache" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-11", + "category": "paraphrase", + "query": "進不去帳號時應該再試幾次", + "relevant": [ + "d-login" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-12", + "category": "paraphrase", + "query": "希望找回昨日保存的內容", + "relevant": [ + "d-backup" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-13", + "category": "synonym", + "query": "dark appearance", + "relevant": [ + "d-theme" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [] + }, + { + "id": "d-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "d-15", + "category": "unrelated", + "query": "do not restore backup", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "d-16", + "category": "unrelated", + "query": "sign in retry price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "d-17", + "category": "stale", + "query": "dretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "d-18", + "category": "stale", + "query": "ddeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + }, + "after-aliases": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "d-01", + "category": "chinese", + "query": "快取失效規則", + "relevant": [ + "d-cache" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-cache" + ] + }, + { + "id": "d-02", + "category": "chinese", + "query": "登入重試", + "relevant": [ + "d-login" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "d-login" + ] + }, + { + "id": "d-03", + "category": "chinese", + "query": "更新後快取怎麼辦", + "relevant": [ + "d-cache" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [ + "d-cache" + ] + }, + { + "id": "d-04", + "category": "chinese", + "query": "無法登入怎麼處理", + "relevant": [ + "d-login" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [ + "d-login" + ] + }, + { + "id": "d-05", + "category": "synonym", + "query": "restore backup", + "relevant": [ + "d-backup" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [ + "d-backup" + ] + }, + { + "id": "d-06", + "category": "synonym", + "query": "night mode", + "relevant": [ + "d-theme" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [ + "d-theme" + ] + }, + { + "id": "d-07", + "category": "cross-language", + "query": "CACHE EVICTION", + "relevant": [ + "d-cache" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [ + "d-cache" + ] + }, + { + "id": "d-08", + "category": "cross-language", + "query": "SIGN IN RETRY", + "relevant": [ + "d-login" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [ + "d-login" + ] + }, + { + "id": "d-09", + "category": "cross-language", + "query": " recover archived data ", + "relevant": [ + "d-backup" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [ + "d-backup" + ] + }, + { + "id": "d-10", + "category": "paraphrase", + "query": "重新整理存下來的線索以免拿到舊版", + "relevant": [ + "d-cache" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-11", + "category": "paraphrase", + "query": "進不去帳號時應該再試幾次", + "relevant": [ + "d-login" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-12", + "category": "paraphrase", + "query": "希望找回昨日保存的內容", + "relevant": [ + "d-backup" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "d-13", + "category": "synonym", + "query": "dark appearance", + "relevant": [ + "d-theme" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [ + "d-theme" + ] + }, + { + "id": "d-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "d-15", + "category": "unrelated", + "query": "do not restore backup", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "d-16", + "category": "unrelated", + "query": "sign in retry price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "d-17", + "category": "stale", + "query": "dretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "d-18", + "category": "stale", + "query": "ddeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + } + }, + "held-out": { + "before": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 0.5, + "missedQueries": 2, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "h-01", + "category": "chinese", + "query": "發票修正", + "relevant": [ + "h-billing" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-billing" + ] + }, + { + "id": "h-02", + "category": "chinese", + "query": "連線逾時", + "relevant": [ + "h-timeout" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-timeout" + ] + }, + { + "id": "h-03", + "category": "chinese", + "query": "發票寫錯怎麼辦", + "relevant": [ + "h-billing" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "h-04", + "category": "chinese", + "query": "連不上伺服器", + "relevant": [ + "h-timeout" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "h-05", + "category": "synonym", + "query": "download report", + "relevant": [ + "h-export" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "h-06", + "category": "synonym", + "query": "scheduled job timezone", + "relevant": [ + "h-schedule" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "h-07", + "category": "cross-language", + "query": "CORRECT INVOICE", + "relevant": [ + "h-billing" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "h-08", + "category": "cross-language", + "query": "CONNECTION DEADLINE", + "relevant": [ + "h-timeout" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "h-09", + "category": "cross-language", + "query": " export records ", + "relevant": [ + "h-export" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [] + }, + { + "id": "h-10", + "category": "paraphrase", + "query": "帳單的金額填錯要如何更正", + "relevant": [ + "h-billing" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-11", + "category": "paraphrase", + "query": "伺服器等太久了需要重新連線嗎", + "relevant": [ + "h-timeout" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-12", + "category": "paraphrase", + "query": "把表格存成外部檔案需要注意什麼", + "relevant": [ + "h-export" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-13", + "category": "synonym", + "query": "timer zone", + "relevant": [ + "h-schedule" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [] + }, + { + "id": "h-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "h-15", + "category": "unrelated", + "query": "do not download report", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "h-16", + "category": "unrelated", + "query": "connection deadline price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "h-17", + "category": "stale", + "query": "hretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "h-18", + "category": "stale", + "query": "hdeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + }, + "after-default": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.15384615384615385, + "missedQueries": 11, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 0.5, + "missedQueries": 2, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "h-01", + "category": "chinese", + "query": "發票修正", + "relevant": [ + "h-billing" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-billing" + ] + }, + { + "id": "h-02", + "category": "chinese", + "query": "連線逾時", + "relevant": [ + "h-timeout" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-timeout" + ] + }, + { + "id": "h-03", + "category": "chinese", + "query": "發票寫錯怎麼辦", + "relevant": [ + "h-billing" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "h-04", + "category": "chinese", + "query": "連不上伺服器", + "relevant": [ + "h-timeout" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [] + }, + { + "id": "h-05", + "category": "synonym", + "query": "download report", + "relevant": [ + "h-export" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "h-06", + "category": "synonym", + "query": "scheduled job timezone", + "relevant": [ + "h-schedule" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [] + }, + { + "id": "h-07", + "category": "cross-language", + "query": "CORRECT INVOICE", + "relevant": [ + "h-billing" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "h-08", + "category": "cross-language", + "query": "CONNECTION DEADLINE", + "relevant": [ + "h-timeout" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [] + }, + { + "id": "h-09", + "category": "cross-language", + "query": " export records ", + "relevant": [ + "h-export" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [] + }, + { + "id": "h-10", + "category": "paraphrase", + "query": "帳單的金額填錯要如何更正", + "relevant": [ + "h-billing" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-11", + "category": "paraphrase", + "query": "伺服器等太久了需要重新連線嗎", + "relevant": [ + "h-timeout" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-12", + "category": "paraphrase", + "query": "把表格存成外部檔案需要注意什麼", + "relevant": [ + "h-export" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-13", + "category": "synonym", + "query": "timer zone", + "relevant": [ + "h-schedule" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [] + }, + { + "id": "h-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "h-15", + "category": "unrelated", + "query": "do not download report", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "h-16", + "category": "unrelated", + "query": "connection deadline price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "h-17", + "category": "stale", + "query": "hretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "h-18", + "category": "stale", + "query": "hdeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + }, + "after-aliases": { + "overall": [ + { + "k": 1, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 3, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + { + "k": 5, + "queries": 18, + "positiveQueries": 13, + "negativeQueries": 5, + "recallAtK": 0.7692307692307693, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + ], + "byCategory": { + "chinese": { + "k": 1, + "queries": 4, + "positiveQueries": 4, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "synonym": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "cross-language": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 1, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "paraphrase": { + "k": 1, + "queries": 3, + "positiveQueries": 3, + "negativeQueries": 0, + "recallAtK": 0, + "missedQueries": 3, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "unrelated": { + "k": 1, + "queries": 3, + "positiveQueries": 0, + "negativeQueries": 3, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + }, + "stale": { + "k": 1, + "queries": 2, + "positiveQueries": 0, + "negativeQueries": 2, + "recallAtK": null, + "missedQueries": 0, + "falseRecallQueries": 0, + "falsePositiveNegativeQueries": 0 + } + }, + "rows": [ + { + "id": "h-01", + "category": "chinese", + "query": "發票修正", + "relevant": [ + "h-billing" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-billing" + ] + }, + { + "id": "h-02", + "category": "chinese", + "query": "連線逾時", + "relevant": [ + "h-timeout" + ], + "rationale": "Exact title cue remains a valid control.", + "retrieved": [ + "h-timeout" + ] + }, + { + "id": "h-03", + "category": "chinese", + "query": "發票寫錯怎麼辦", + "relevant": [ + "h-billing" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [ + "h-billing" + ] + }, + { + "id": "h-04", + "category": "chinese", + "query": "連不上伺服器", + "relevant": [ + "h-timeout" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token.", + "retrieved": [ + "h-timeout" + ] + }, + { + "id": "h-05", + "category": "synonym", + "query": "download report", + "relevant": [ + "h-export" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [ + "h-export" + ] + }, + { + "id": "h-06", + "category": "synonym", + "query": "scheduled job timezone", + "relevant": [ + "h-schedule" + ], + "rationale": "Authored English synonym routes to the corpus tag.", + "retrieved": [ + "h-schedule" + ] + }, + { + "id": "h-07", + "category": "cross-language", + "query": "CORRECT INVOICE", + "relevant": [ + "h-billing" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [ + "h-billing" + ] + }, + { + "id": "h-08", + "category": "cross-language", + "query": "CONNECTION DEADLINE", + "relevant": [ + "h-timeout" + ], + "rationale": "Explicit bilingual alias; case normalization only.", + "retrieved": [ + "h-timeout" + ] + }, + { + "id": "h-09", + "category": "cross-language", + "query": " export records ", + "relevant": [ + "h-export" + ], + "rationale": "Explicit bilingual alias with whitespace normalization.", + "retrieved": [ + "h-export" + ] + }, + { + "id": "h-10", + "category": "paraphrase", + "query": "帳單的金額填錯要如何更正", + "relevant": [ + "h-billing" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-11", + "category": "paraphrase", + "query": "伺服器等太久了需要重新連線嗎", + "relevant": [ + "h-timeout" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-12", + "category": "paraphrase", + "query": "把表格存成外部檔案需要注意什麼", + "relevant": [ + "h-export" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias.", + "retrieved": [] + }, + { + "id": "h-13", + "category": "synonym", + "query": "timer zone", + "relevant": [ + "h-schedule" + ], + "rationale": "Explicit alternate terminology.", + "retrieved": [ + "h-schedule" + ] + }, + { + "id": "h-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus.", + "retrieved": [] + }, + { + "id": "h-15", + "category": "unrelated", + "query": "do not download report", + "relevant": [], + "rationale": "Negated query must not activate a substring alias.", + "retrieved": [] + }, + { + "id": "h-16", + "category": "unrelated", + "query": "connection deadline price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten.", + "retrieved": [] + }, + { + "id": "h-17", + "category": "stale", + "query": "hretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded.", + "retrieved": [] + }, + { + "id": "h-18", + "category": "stale", + "query": "hdeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted.", + "retrieved": [] + } + ] + } + } + } +} diff --git a/docs/engine/recall-lifecycle.md b/docs/engine/recall-lifecycle.md new file mode 100644 index 0000000..2591379 --- /dev/null +++ b/docs/engine/recall-lifecycle.md @@ -0,0 +1,189 @@ +# Explicit cues, cache ownership and recall provenance + +Unreleased additions to the optional TypeScript engine. The three-file protocol, +CLI commands and default familiarity thresholds are unchanged. Start with the +[bounded recall guide](recall-efficiency.md) for candidate and output budgets. + +## Opt in to explicit query aliases + +```ts +import { createInMemorySemanticRecallEngine } from "@focaxisdev/deja-vu"; + +const engine = createInMemorySemanticRecallEngine({ + cueAliases: [ + { alias: "更新後快取怎麼辦", cue: "cachepolicy" }, + { alias: "cache eviction", cue: "cachepolicy" }, + ], +}); +await engine.addMemory({ + id: "cache-policy", + title: "Cache policy", + tags: ["cachepolicy"], + content: "Invalidate cached data after an update or deletion.", +}); +const result = await engine.recall("更新後快取怎麼辦"); +console.log(result.topMatch?.id); // cache-policy +``` + +Aliases match the **entire query** after lowercase and whitespace normalization. +They do not match substrings, strip negation, translate text, segment Chinese, +expand unknown synonyms or change thresholds. For example, `do not cache eviction` +does not activate `cache eviction`. If no alias matches, ordinary lexical recall +still runs; it can have its own false positives. Use distinctive existing tags as +cues. Duplicate cues across memories still use the existing candidate/ranking +rules, so an alias is not a guaranteed memory-ID lookup. + +At most 128 entries are accepted. Each alias and cue must be a non-empty string +of at most 256 UTF-16 code units. Conflicting normalized aliases reject at +construction. The engine copies the routing map. No corpus rewrite is required +when aliases point to existing tokens. Adding tags to existing memories requires +an explicit update. Audit aliases when a policy changes: a stale alias can still +route to a live memory if its cue remains indexed. Exact matching deliberately +misses unlisted paraphrases and qualified variants. + +## Opt in to managed embedding ownership + +```ts +import { + CachedEmbeddingProvider, MockEmbeddingProvider, + createInMemorySemanticRecallEngine, +} from "@focaxisdev/deja-vu"; + +const embeddings = new CachedEmbeddingProvider(new MockEmbeddingProvider(256), { + modelKey: "mock:256:v1", + maxEntries: 128, + maxBytes: 8 * 1024 * 1024, +}); +const engine = createInMemorySemanticRecallEngine({ + embeddingProvider: embeddings, + manageEmbeddingCache: true, + preserveDerivedOnMetadataUpdate: true, +}); +await engine.addMemory({ id: "old", title: "Old policy", content: "Synthetic old policy." }); +await engine.updateMemory("old", { metadata: { reviewed: true } }); +await engine.supersedeMemory("old", { + id: "current", title: "Current policy", content: "Synthetic replacement policy.", +}); +await engine.deleteMemory("current"); +console.log(embeddings.getStats()); // { entries: 0, accountedBytes: 0 } +``` + +The mock remains a deterministic testing provider, not a semantic or multilingual +model. Managed ownership requires an embedding provider with `createScope()`; +`CachedEmbeddingProvider` supplies it. A scope belongs to one memory revision. +Repeated text within that scope reuses vectors. Identical text in another memory +or revision has a separate entry: there is no shared-reference deletion ambiguity. +This conservative policy reduces cache hits and can increase provider calls. + +Successful content updates dispose the previous scope only after commit. +Failed preparation/commit disposes the new scope and retains the previous one. +Opt-in metadata-only preservation retains the scope. Successful deletion and +supersession release the old scope; failures preserve ownership for recovery. +A disposed scope rejects new calls, and already-running calls cannot repopulate +it. Those calls can still finish and return their vector: disposal is not +cancellation. Root `clear()` also blocks late insertion across all scopes. + +Managed ownership automatically serializes reads with writes in this engine +instance, including query embedding, so a late recall cannot recreate a deleted +memory's scope. `consistentReads: true` enables the same read ordering without +requiring a cache provider. A read queued before deletion may finish before the +delete completes; reads queued after deletion observe the resulting state. +Slow providers hold up later operations. Do not re-enter the engine from an +adapter, summary generator or embedding callback while this mode is enabled; +that would wait on its own operation. Other engine instances, direct adapter +writes, process crashes and remote storage transactions are outside this lock. + +Cache keys now use per-instance/per-scope keyed SHA-256 digests of exact UTF-16 +input and a length-framed model identifier. Plaintext input is not retained as a +cache key. The configured model key must describe the actual provider/model, +revision, dimensions and options; keep it stable throughout a write and rebuild +stored vectors after changing models. Digest keys do not validate model identity. +The cache uses Node's built-in crypto module and no new package dependency. + +`maxBytes` counts 128 bytes per hex digest plus eight bytes per vector value. +This excludes object/Map overhead, scope objects and their keys. The global +entry limit bounds retained cache entries; the engine also holds one small scope +object per live owned memory. A budget below 136 bytes cannot hold even a +one-value vector. Long inputs can now be cached without retaining long text keys; +hashing still takes time proportional to input length. `getStats()` reports entry +count and these accounted bytes, not heap usage, tokens or provider costs. + +Unmanaged `embed()` calls retain a separate cache namespace. Deleting an engine +memory does not clear those entries, another engine's scopes, an underlying +provider's caches, copies held by callers, storage snapshots or backups. Use +separate wrappers for separate projects. Reference release and keyed digests are +**not verified secure erasure**. Private raw text is still passed to the selected +provider when a calculation is needed. + +## Deletion and supersession recovery + +Deletion now snapshots the memory before removing its routes and data. If an +adapter fails, it attempts to restore summary, chunks, raw content, familiarity +and both vector indexes. `MemoryWriteError` includes a detached recovery snapshot +and any recovery errors. Do not log the snapshot: it contains memory content. +Incomplete existing storage rejects before mutation and needs adapter repair. + +`supersedeMemory(oldId, replacement)` is an explicit operation: the old ID must +exist and the replacement ID must be new. It prepares and saves the replacement, +then removes the old memory. On failure it restores the old memory and attempts +to remove the replacement. `MemorySupersedeError` identifies both IDs and reports +cleanup/recovery errors; both records can remain when compensation fails. The +cause may contain a `MemoryWriteError` recovery snapshot. Inspect and repair +rather than blindly retrying. Nothing is crash-atomic or a distributed transaction. + +The optional engine stores no historical redirects or protocol supersession +frontmatter. Old IDs become unavailable after success. Hosts that need audit +history must retain it separately. The file-first CLI lifecycle is unchanged. + +## Mark incomplete prompt evidence + +```ts +import { projectRecallForPrompt, type RecallResult } from "@focaxisdev/deja-vu"; + +function project(result: RecallResult) { + return projectRecallForPrompt(result, { + maxBytes: 4096, + includeProvenance: true, + }); +} +``` + +With provenance enabled, each record includes `source: "getSummary"` or +`source: "getChunks"`, `rawSource: "getRawContent"`, and its memory/field/chunk +identifiers. Recent updates also include their array index. These are API lookup +instructions for the host, not automatically executed calls. Retrieve a complete +stored summary with `engine.getSummary(memoryId)` or the original stored +`AddMemoryInput.content` with `engine.getRawContent(memoryId)`. + +Every projected summary warns that it is derived and may omit original +constraints. A field clipped by projection, or a summary already clipped by weak +recall, includes `incomplete: true` and a warning that qualifiers may be missing. +Warnings and references count toward the same whole-output byte/token budget. +They reduce room for content. If even an annotated record cannot fit, it is +omitted; `truncated` is true and text can be empty. Check that flag even when the +last included record is complete, because later fields may have been omitted. + +This marks loss rather than preserving sentence meaning. `getSummary()` cannot +restore text already omitted by the summary generator. `getRawContent()` returns +only the originally stored content, not separately supplied context/metadata +fields; place durable constraints in content if they must remain retrievable. +Projection does not fetch raw content, sanitize prompt injection, guarantee +source availability after deletion or reserve budget for the host's wrapper. +Bytes are not tokens; use the host's real tokenizer for a token budget. + +## Migration and validation + +All new routing, ownership, read serialization and provenance options default +to off. Existing result shapes, thresholds, search breadth and three-file formats +remain compatible. The default cache key representation changes to digests, so +small custom byte budgets may now cache fewer entries. The cache is in-memory; +there is no persisted key migration. Deletion now requires a complete snapshot +(or the existing legacy-adapter fallback reads) and reports recovery failures. +Built-in summary/chunk reads and writes now copy arrays and metadata so callers +cannot silently mutate stored text or vectors through returned references. + +See the [round-two validation and measurements](../release-recall-lifecycle.md), +[labeled evaluation](recall-evaluation.json), and +[fixture](../../scripts/fixtures/recall-evaluation.json). General language-aware +segmentation, implicit semantic fallback, ANN, CLI rewrites, cancellation, +transactional persistence and secure erasure remain outside this change. diff --git a/docs/engine/recall-round2-benchmark.json b/docs/engine/recall-round2-benchmark.json new file mode 100644 index 0000000..861faee --- /dev/null +++ b/docs/engine/recall-round2-benchmark.json @@ -0,0 +1,1013 @@ +{ + "baselineCommit": "bad6b527409a44a693562fbe60c41be74126e573", + "notes": [ + "Three complete repetitions retained. No timing-based acceptance threshold. Default engine configuration; owned-cache serialization measured separately by call counts. Median of per-run medians is descriptive, not a confidence interval." + ], + "runs": [ + { + "node": "v24.19.0", + "samples": 25, + "warmups": 5, + "units": "milliseconds", + "notes": [ + "Micro: storage.searchImpressions only. Scan: engine.scanImpressions.", + "API end-to-end: resident recall(loadChunks=true) plus JSON.stringify, including query embedding and chunk retrieval.", + "Excludes process start, index construction, persistence, network, model inference and host/LLM execution.", + "Baseline/current alternate measurement order. Medians/p95 are descriptive of this small synthetic run." + ], + "workloads": [ + { + "n": 100, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.004, + "p95": 0.0248 + }, + "current": { + "median": 0.004, + "p95": 0.0278 + } + }, + "scan": { + "baseline": { + "median": 0.0084, + "p95": 0.0443 + }, + "current": { + "median": 0.0089, + "p95": 0.0433 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0666, + "p95": 0.1077 + }, + "current": { + "median": 0.0793, + "p95": 0.2151 + } + } + }, + { + "n": 100, + "kind": "common", + "micro": { + "baseline": { + "median": 0.02, + "p95": 0.2031 + }, + "current": { + "median": 0.0204, + "p95": 0.1166 + } + }, + "scan": { + "baseline": { + "median": 0.0304, + "p95": 0.1166 + }, + "current": { + "median": 0.0303, + "p95": 0.1513 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0789, + "p95": 0.1246 + }, + "current": { + "median": 0.0793, + "p95": 0.167 + } + } + }, + { + "n": 100, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0006, + "p95": 0.001 + }, + "current": { + "median": 0.0006, + "p95": 0.0007 + } + }, + "scan": { + "baseline": { + "median": 0.0018, + "p95": 0.0023 + }, + "current": { + "median": 0.0021, + "p95": 0.0022 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0032, + "p95": 0.0039 + }, + "current": { + "median": 0.0035, + "p95": 0.0036 + } + } + }, + { + "n": 1000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0032, + "p95": 0.0035 + }, + "current": { + "median": 0.0032, + "p95": 0.0036 + } + }, + "scan": { + "baseline": { + "median": 0.0075, + "p95": 0.008 + }, + "current": { + "median": 0.0077, + "p95": 0.0094 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.06, + "p95": 0.1113 + }, + "current": { + "median": 0.0736, + "p95": 0.0941 + } + } + }, + { + "n": 1000, + "kind": "common", + "micro": { + "baseline": { + "median": 0.2235, + "p95": 0.344 + }, + "current": { + "median": 0.2589, + "p95": 0.3885 + } + }, + "scan": { + "baseline": { + "median": 0.1531, + "p95": 0.3711 + }, + "current": { + "median": 0.2059, + "p95": 0.7824 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.2801, + "p95": 0.395 + }, + "current": { + "median": 0.3552, + "p95": 0.4837 + } + } + }, + { + "n": 1000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0004, + "p95": 0.002 + }, + "current": { + "median": 0.0004, + "p95": 0.0019 + } + }, + "scan": { + "baseline": { + "median": 0.003, + "p95": 0.0034 + }, + "current": { + "median": 0.0033, + "p95": 0.0045 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0046, + "p95": 0.0061 + }, + "current": { + "median": 0.005, + "p95": 0.0065 + } + } + }, + { + "n": 10000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0018, + "p95": 0.0047 + }, + "current": { + "median": 0.0019, + "p95": 0.0077 + } + }, + "scan": { + "baseline": { + "median": 0.0044, + "p95": 0.0065 + }, + "current": { + "median": 0.0044, + "p95": 0.0885 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0477, + "p95": 0.1411 + }, + "current": { + "median": 0.0502, + "p95": 0.1252 + } + } + }, + { + "n": 10000, + "kind": "common", + "micro": { + "baseline": { + "median": 2.2983, + "p95": 3.369 + }, + "current": { + "median": 2.6976, + "p95": 4.1453 + } + }, + "scan": { + "baseline": { + "median": 2.4837, + "p95": 3.8567 + }, + "current": { + "median": 2.7058, + "p95": 3.9417 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 2.5853, + "p95": 4.2817 + }, + "current": { + "median": 2.9508, + "p95": 4.0031 + } + } + }, + { + "n": 10000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0008, + "p95": 0.0015 + }, + "current": { + "median": 0.0008, + "p95": 0.0015 + } + }, + "scan": { + "baseline": { + "median": 0.0029, + "p95": 0.0034 + }, + "current": { + "median": 0.0033, + "p95": 0.037 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0029, + "p95": 0.0037 + }, + "current": { + "median": 0.0032, + "p95": 0.0039 + } + } + } + ], + "operations": { + "baseline": { + "addCalls": 12, + "repeatedRecallCalls": 5, + "metadataUpdateCalls": 12, + "rawRecallBytes": 10253 + }, + "currentOptIn": { + "addCalls": 4, + "repeatedRecallCalls": 1, + "metadataUpdateCalls": 0, + "rawRecallBytes": 10253, + "projectedBytes": 2301 + } + } + }, + { + "node": "v24.19.0", + "samples": 25, + "warmups": 5, + "units": "milliseconds", + "notes": [ + "Micro: storage.searchImpressions only. Scan: engine.scanImpressions.", + "API end-to-end: resident recall(loadChunks=true) plus JSON.stringify, including query embedding and chunk retrieval.", + "Excludes process start, index construction, persistence, network, model inference and host/LLM execution.", + "Baseline/current alternate measurement order. Medians/p95 are descriptive of this small synthetic run." + ], + "workloads": [ + { + "n": 100, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0058, + "p95": 0.0278 + }, + "current": { + "median": 0.0061, + "p95": 0.0157 + } + }, + "scan": { + "baseline": { + "median": 0.0082, + "p95": 0.0504 + }, + "current": { + "median": 0.0086, + "p95": 0.0491 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.1062, + "p95": 0.3244 + }, + "current": { + "median": 0.1098, + "p95": 0.3654 + } + } + }, + { + "n": 100, + "kind": "common", + "micro": { + "baseline": { + "median": 0.024, + "p95": 0.0778 + }, + "current": { + "median": 0.0366, + "p95": 0.1213 + } + }, + "scan": { + "baseline": { + "median": 0.0276, + "p95": 0.0721 + }, + "current": { + "median": 0.0273, + "p95": 0.0925 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0826, + "p95": 0.1748 + }, + "current": { + "median": 0.0817, + "p95": 0.3494 + } + } + }, + { + "n": 100, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0005, + "p95": 0.0008 + }, + "current": { + "median": 0.0005, + "p95": 0.0006 + } + }, + "scan": { + "baseline": { + "median": 0.0019, + "p95": 0.0022 + }, + "current": { + "median": 0.002, + "p95": 0.0022 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0033, + "p95": 0.0108 + }, + "current": { + "median": 0.0033, + "p95": 0.0058 + } + } + }, + { + "n": 1000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0032, + "p95": 0.004 + }, + "current": { + "median": 0.0032, + "p95": 0.0035 + } + }, + "scan": { + "baseline": { + "median": 0.0073, + "p95": 0.0077 + }, + "current": { + "median": 0.0076, + "p95": 0.0093 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0627, + "p95": 0.0932 + }, + "current": { + "median": 0.0726, + "p95": 0.2009 + } + } + }, + { + "n": 1000, + "kind": "common", + "micro": { + "baseline": { + "median": 0.2434, + "p95": 0.5814 + }, + "current": { + "median": 0.377, + "p95": 0.8127 + } + }, + "scan": { + "baseline": { + "median": 0.1672, + "p95": 0.5347 + }, + "current": { + "median": 0.23, + "p95": 1.3395 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.2723, + "p95": 0.3916 + }, + "current": { + "median": 0.3713, + "p95": 0.5661 + } + } + }, + { + "n": 1000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0004, + "p95": 0.002 + }, + "current": { + "median": 0.0004, + "p95": 0.002 + } + }, + "scan": { + "baseline": { + "median": 0.0031, + "p95": 0.0404 + }, + "current": { + "median": 0.0032, + "p95": 0.0046 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0047, + "p95": 0.0367 + }, + "current": { + "median": 0.0047, + "p95": 0.0064 + } + } + }, + { + "n": 10000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0016, + "p95": 0.002 + }, + "current": { + "median": 0.0016, + "p95": 0.002 + } + }, + "scan": { + "baseline": { + "median": 0.004, + "p95": 0.005 + }, + "current": { + "median": 0.0042, + "p95": 0.0048 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0507, + "p95": 0.1416 + }, + "current": { + "median": 0.0535, + "p95": 0.0964 + } + } + }, + { + "n": 10000, + "kind": "common", + "micro": { + "baseline": { + "median": 2.899, + "p95": 4.1866 + }, + "current": { + "median": 2.7897, + "p95": 4.4533 + } + }, + "scan": { + "baseline": { + "median": 2.5471, + "p95": 7.5842 + }, + "current": { + "median": 2.7542, + "p95": 5.178 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 2.5329, + "p95": 3.6354 + }, + "current": { + "median": 2.8307, + "p95": 4.9149 + } + } + }, + { + "n": 10000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0008, + "p95": 0.0011 + }, + "current": { + "median": 0.0008, + "p95": 0.0011 + } + }, + "scan": { + "baseline": { + "median": 0.0018, + "p95": 0.0032 + }, + "current": { + "median": 0.0021, + "p95": 0.004 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0031, + "p95": 0.0049 + }, + "current": { + "median": 0.0033, + "p95": 0.0057 + } + } + } + ], + "operations": { + "baseline": { + "addCalls": 12, + "repeatedRecallCalls": 5, + "metadataUpdateCalls": 12, + "rawRecallBytes": 10253 + }, + "currentOptIn": { + "addCalls": 4, + "repeatedRecallCalls": 1, + "metadataUpdateCalls": 0, + "rawRecallBytes": 10253, + "projectedBytes": 2301 + } + } + }, + { + "node": "v24.19.0", + "samples": 25, + "warmups": 5, + "units": "milliseconds", + "notes": [ + "Micro: storage.searchImpressions only. Scan: engine.scanImpressions.", + "API end-to-end: resident recall(loadChunks=true) plus JSON.stringify, including query embedding and chunk retrieval.", + "Excludes process start, index construction, persistence, network, model inference and host/LLM execution.", + "Baseline/current alternate measurement order. Medians/p95 are descriptive of this small synthetic run." + ], + "workloads": [ + { + "n": 100, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0056, + "p95": 0.0912 + }, + "current": { + "median": 0.0059, + "p95": 0.014 + } + }, + "scan": { + "baseline": { + "median": 0.0083, + "p95": 0.0547 + }, + "current": { + "median": 0.0088, + "p95": 0.0649 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0655, + "p95": 0.1486 + }, + "current": { + "median": 0.0752, + "p95": 0.3647 + } + } + }, + { + "n": 100, + "kind": "common", + "micro": { + "baseline": { + "median": 0.0208, + "p95": 0.0795 + }, + "current": { + "median": 0.021, + "p95": 0.2328 + } + }, + "scan": { + "baseline": { + "median": 0.0293, + "p95": 0.0958 + }, + "current": { + "median": 0.0321, + "p95": 0.0989 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0929, + "p95": 0.2033 + }, + "current": { + "median": 0.0882, + "p95": 0.3483 + } + } + }, + { + "n": 100, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0006, + "p95": 0.0009 + }, + "current": { + "median": 0.0006, + "p95": 0.0008 + } + }, + "scan": { + "baseline": { + "median": 0.0018, + "p95": 0.0024 + }, + "current": { + "median": 0.002, + "p95": 0.0023 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0034, + "p95": 0.0069 + }, + "current": { + "median": 0.0036, + "p95": 0.0054 + } + } + }, + { + "n": 1000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.003, + "p95": 0.0052 + }, + "current": { + "median": 0.0031, + "p95": 0.0647 + } + }, + "scan": { + "baseline": { + "median": 0.0072, + "p95": 0.0096 + }, + "current": { + "median": 0.0074, + "p95": 0.0684 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0612, + "p95": 0.1556 + }, + "current": { + "median": 0.0781, + "p95": 0.2021 + } + } + }, + { + "n": 1000, + "kind": "common", + "micro": { + "baseline": { + "median": 0.25, + "p95": 0.4187 + }, + "current": { + "median": 0.2708, + "p95": 0.4305 + } + }, + "scan": { + "baseline": { + "median": 0.2103, + "p95": 0.8576 + }, + "current": { + "median": 0.2244, + "p95": 1.0381 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.2708, + "p95": 0.3856 + }, + "current": { + "median": 0.3494, + "p95": 0.5329 + } + } + }, + { + "n": 1000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0004, + "p95": 0.002 + }, + "current": { + "median": 0.0004, + "p95": 0.0019 + } + }, + "scan": { + "baseline": { + "median": 0.003, + "p95": 0.0048 + }, + "current": { + "median": 0.0032, + "p95": 0.0044 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0046, + "p95": 0.0063 + }, + "current": { + "median": 0.0049, + "p95": 0.0063 + } + } + }, + { + "n": 10000, + "kind": "rare", + "micro": { + "baseline": { + "median": 0.0025, + "p95": 0.0033 + }, + "current": { + "median": 0.0026, + "p95": 0.0031 + } + }, + "scan": { + "baseline": { + "median": 0.0039, + "p95": 0.0044 + }, + "current": { + "median": 0.0042, + "p95": 0.0046 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0464, + "p95": 0.1183 + }, + "current": { + "median": 0.0503, + "p95": 0.0763 + } + } + }, + { + "n": 10000, + "kind": "common", + "micro": { + "baseline": { + "median": 2.3091, + "p95": 4.9869 + }, + "current": { + "median": 2.5937, + "p95": 3.4029 + } + }, + "scan": { + "baseline": { + "median": 2.4826, + "p95": 5.0033 + }, + "current": { + "median": 2.7656, + "p95": 3.8497 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 2.6475, + "p95": 4.7129 + }, + "current": { + "median": 2.9064, + "p95": 3.605 + } + } + }, + { + "n": 10000, + "kind": "none", + "micro": { + "baseline": { + "median": 0.0007, + "p95": 0.001 + }, + "current": { + "median": 0.0007, + "p95": 0.0011 + } + }, + "scan": { + "baseline": { + "median": 0.0017, + "p95": 0.0024 + }, + "current": { + "median": 0.0021, + "p95": 0.0285 + } + }, + "apiEndToEnd": { + "baseline": { + "median": 0.0029, + "p95": 0.0031 + }, + "current": { + "median": 0.0031, + "p95": 0.0033 + } + } + } + ], + "operations": { + "baseline": { + "addCalls": 12, + "repeatedRecallCalls": 5, + "metadataUpdateCalls": 12, + "rawRecallBytes": 10253 + }, + "currentOptIn": { + "addCalls": 4, + "repeatedRecallCalls": 1, + "metadataUpdateCalls": 0, + "rawRecallBytes": 10253, + "projectedBytes": 2301 + } + } + } + ] +} diff --git a/docs/engine/semantic-engine.md b/docs/engine/semantic-engine.md index 0e6315f..42ceb86 100644 --- a/docs/engine/semantic-engine.md +++ b/docs/engine/semantic-engine.md @@ -50,9 +50,10 @@ await engine.deleteMemory(id); ## Write failure contract -Adds and updates prepare summaries, chunks, and all embeddings before changing -storage. Updates retain importance, tags, and creation time unless explicitly -changed where supported. Adding an existing ID is rejected; use `updateMemory`. +Adds and normal updates prepare summaries, chunks, and all embeddings before changing +storage. Opt-in metadata-only updates may preserve those derived values; see +[bounded recall and reuse](recall-efficiency.md). Updates retain importance, tags, +and creation time unless explicitly changed where supported. Adding an existing ID is rejected; use `updateMemory`. Mutations are serialized within one engine instance. If a storage or vector write fails, the engine attempts compensating cleanup @@ -90,7 +91,10 @@ If you adopt the engine, the host still owns: ## Related references +- [Bounded recall, opt-in reuse and migration](recall-efficiency.md) - [docs/agent-handshake.md](../agent-handshake.md) - [docs/engine/protocol-to-engine.md](./protocol-to-engine.md) - [src/types/plugins.ts](../../src/types/plugins.ts) - [src/types/memory.ts](../../src/types/memory.ts) + +See [explicit cues, lifecycle recovery and provenance](recall-lifecycle.md) for the additional opt-ins and their concurrency/migration limits. diff --git a/docs/recall-update.md b/docs/recall-update.md new file mode 100644 index 0000000..8310d2e --- /dev/null +++ b/docs/recall-update.md @@ -0,0 +1,95 @@ +# More reliable recall, with explicit budgets and opt-ins + +These changes are available in repository source. No new package version, npm +publication, GitHub Release or release date is assigned. The existing npm setup +instructions still describe the three-file protocol; do not assume an npm install +already includes every source addition below. + +## What improves + +The optional TypeScript engine selects eligible memories before applying hybrid +ranking preference, avoiding false "no memory" results when an ineligible record +has a higher hybrid score. Search breadth can grow without expanding final output. +Cached token sets avoid repeated lexical preprocessing, with update/delete +invalidation. Familiarity thresholds and the public three-file protocol are unchanged. + +Hosts can opt in to bounded prompt projection, metadata-only embedding reuse, +content/model-keyed embedding caches, exact query aliases, and per-memory cache +ownership. Deletion and supersession attempt recovery on adapter failures; +projection can mark missing qualifiers and identify where to retrieve stored text. +These are optional engine features, not new requirements for file-first adoption. + +## Enable only what your host needs + +With the repository build, exact aliases can route familiar wording to an existing +distinctive cue, while candidate search and prompt output have separate budgets: + +```ts +import { + createInMemorySemanticRecallEngine, projectRecallForPrompt, +} from "@focaxisdev/deja-vu"; + +const engine = createInMemorySemanticRecallEngine({ + cueAliases: [{ alias: "更新後快取怎麼辦", cue: "cachepolicy" }], +}); +await engine.addMemory({ + id: "cache-policy", title: "Cache policy", tags: ["cachepolicy"], + content: "Invalidate cached data after an update or deletion.", +}); +const recalled = await engine.recall({ + text: "更新後快取怎麼辦", candidateLimit: 32, topK: 3, +}); +const prompt = projectRecallForPrompt(recalled, { + maxBytes: 4096, includeProvenance: true, +}); +``` + +Aliases match the normalized entire query. They do not translate, segment Chinese +or recognize unlisted paraphrases. The byte cap is not a token cap; use a host +tokenizer for token budgets. Projection is untrusted memory data, not an instruction +sanitizer. See the [budget/cache guide](engine/recall-efficiency.md) and +[ownership, aliases and provenance guide](engine/recall-lifecycle.md) for full +options, examples and failure handling. + +## Benefits and limits, measured separately + +- **Lexical preprocessing:** 30.74 → 2.61 ms was the median of one synthetic + 10,000-record shared-cue storage-search microbenchmark. It is not a whole-product + speedup, does not cover all requests, and excludes external models and agent work. + The [first-batch measurements](release-recall-efficiency.md) distinguish this + microbenchmark from the resident API workload. +- **Recall quality:** authored aliases raised Recall@1 from 2/13 to 10/13 in + each split of a small same-author synthetic dataset. No false recalls were + observed there, but unlisted paraphrases still missed. This is not independent + multilingual-model validation or a general recall-rate guarantee. +- **Latency and ownership:** the second batch's 10,000-record default-mode + resident API median rose about 12.4%; that measurement did not isolate ownership + mode. Managed ownership separately increased mock provider calls on ingest + from 18 to 48 and on a content update from 2 to 3 in the shared-content fixture, + while retained entries after deletion fell from 36 to zero. It also serializes + reads/writes within one engine, so slow providers can delay later calls. + +The [second-batch report](release-recall-lifecycle.md) retains workloads, raw +measurement links and caveats. Cache counts are not paid-service savings; +accounted bytes are not tokens or measured heap use. These changes do not establish +that every request is faster. + +## Migration and verification + +No migration of `AGENTS.md`, `memory/summary.md` or `memory/impressions.jsonl` is +required. Existing API result shapes, default search breadth and thresholds are +retained. Aliases, ownership, read serialization, metadata preservation and +projection provenance need explicit opt-in. Invalid limits now reject; small +custom cache byte budgets can hold fewer entries with digest keys. Review custom +adapter recovery and generator assumptions before enabling these options. + +Recovery is not a crash-safe transaction. Cache release is not verified secure +erasure. Prompt warnings mark possible qualifier loss rather than restoring +meaning; use the documented source lookup when complete evidence matters. + +Complete Ubuntu/Windows acceptance passed: 119 source cases, the same 119 cases +compiled, and one package/install/use case per platform, with zero skipped tests. +Type checks, frozen recall results, cache lifecycle counts and public documentation +checks also passed. See the [logs and failure-to-fix record](actions-acceptance-results.md) +and [current main CI](https://github.com/focaxisdev/deja-vu/actions/workflows/ci.yml). +Hosted Node 22 results do not identify the earlier Cloud Node 24 native EPERM cause. diff --git a/docs/release-recall-efficiency.md b/docs/release-recall-efficiency.md new file mode 100644 index 0000000..ae78d53 --- /dev/null +++ b/docs/release-recall-efficiency.md @@ -0,0 +1,121 @@ +# Unreleased: more reliable, bounded engine recall + +Source update notes; package release pending. No new package version, release +date, tag or npm publication is assigned by this change. Start with the +[combined update and opt-in overview](recall-update.md). + +These measurements and validation counts describe the first implementation batch +at `bad6b527409a44a693562fbe60c41be74126e573`. See the +[round-two notes](release-recall-lifecycle.md) for current cache ownership, +provenance, quality evaluation, performance costs and installed-package results. +The later [hosted acceptance report](actions-acceptance-results.md) records full +Ubuntu/Windows validation; these first-batch counts remain historical. + +## User introduction + +Deja Vu's optional engine can now find an eligible memory even when a recent, +less relevant candidate has a higher hybrid score. You can search more candidates +without expanding the returned list, project recalled text into an explicit +prompt budget, and opt in to bounded embedding reuse. Common-cue searches also +avoid rebuilding each record's token set on every request. + +The three-file memory protocol stays the same. No file migration, database, +paid embedding service or daemon is needed. See the +[usage and migration guide](engine/recall-efficiency.md) for opt-in examples. + +## What changes + +- Eligible candidates precede ineligible hybrid winners. Hybrid preference among + eligible candidates is retained; semantic thresholds are unchanged. +- `candidateLimit` separates search breadth from `topK`, with previous breadth + retained by default. Invalid numeric limits now reject instead of producing + surprising array slices or excessive work. +- `projectRecallForPrompt()` provides allowlisted JSONL with an exact UTF-8 byte + cap and optional host-tokenizer budget. Raw API results remain compatible. +- The in-memory lexical scorer caches normalized record token sets, replaces them + on updates, and removes them on deletion. Returned familiarity records are copies. +- `preserveDerivedOnMetadataUpdate` skips regeneration for metadata/importance-only + edits when explicitly enabled. Custom-generator behavior remains the default. +- `CachedEmbeddingProvider` offers opt-in content/model-keyed LRU retention with + entry and accounted-storage bounds, explicit clearing, and defensive vector copies. + +## Measured effects + +Baseline: repository main at `188d7e8c3aed224e45071d041cd140509887b29f`. +Both versions were compiled with existing dependencies and measured in the same +Linux x64 / Node v24.19.0 process, alternating measurement order. Each case used +five warmups and 25 samples. Fixtures contain the same five routing terms per +record used in the preceding audit; detail fixtures add ten synthetic chunks per +record. No real user memory, network model or paid API was used. + +| Synthetic common-cue workload | Baseline median | Current median | +| --- | ---: | ---: | +| 100 records, storage search microbenchmark | 0.2820 ms | 0.0236 ms | +| 1,000 records, storage search microbenchmark | 3.1107 ms | 0.1643 ms | +| 10,000 records, storage search microbenchmark | 30.7427 ms | 2.6053 ms | +| 10,000 records, resident API end-to-end | 30.8083 ms | 2.7437 ms | + +The last row covers `recall(loadChunks=true)` plus JSON serialization, including +mock query embedding, summary/chunk retrieval and access-time updates. It is +**not** end-to-end agent or product latency: startup, index construction, +persistence, external model inference, host prompt processing and LLM work are +excluded. API timing tests use default options, not the embedding cache opt-in. +Rare-cue and no-match costs were already small; some smaller cases became slightly +slower. These measurements do not establish an application-wide speedup or recall +quality improvement. Full samples' median/p95 summaries are in the +[measurement record](engine/recall-benchmark.json). + +In a separate ten-chunk repeated-content fixture, enabling the embedding cache +reduced actual mock-provider calls on add from 12 to 4 and on five repeated detail +recalls from 5 to 1. With metadata preservation also enabled, an importance-only +update used zero embedding calls instead of 12. Counts depend on repeated content +and cache capacity; they are not measured paid-service cost savings. + +The raw recall payload stayed at 10,253 bytes. Its optional prompt projection was +2,301 UTF-8 bytes in this fixture, about 77.6% smaller, excluding vectors and other +non-projected fields. **Bytes are not model tokens.** This is a representation +comparison, not proof of equal task quality; omitted fields may matter to a host. + +## Historical first-batch validation and limitations + +- TypeScript no-emit validation and compilation passed with existing dependencies. +- All 31 source engine/regression tests passed in Node's supported in-process test mode. +- All 31 compiled engine/regression tests and 26 existing compiled engine + rollback/concurrency tests passed (57 distinct tests, with the first 31 also + exercised from source). +- The standard source test command failed under this Cloud environment's process + isolation. An isolated existing CLI test confirmed `spawnSync EPERM`. + At that point, full CLI/process and installed-package verification still needed an environment + that permits those subprocesses; this draft does not claim full-suite success. +- No new dependency was installed. Repository-wide memory scans and link checks + that read private memory were excluded; changed documentation links were checked + separately. Full `npm test` and CI were still outstanding at that checkpoint; + the later hosted acceptance linked above passed them. + +Regression coverage includes the false-none counterexample, lexical prefilter +truncation, invalid and zero limits, multilingual score/tie equivalence, Unicode +prompt clipping, update/delete invalidation, opt-in rollback, model changes, +cache eviction and pending-request clearing. Threshold quality, semantic synonym +recall, real tokenizer costs, concurrent production traffic and cold starts were +not measured. Prompt projection remains untrusted memory data, not an instruction +sanitizer. Read the migration guide before enabling metadata preservation or +sharing embedding caches. + +CLI optimizations, language-aware routing, semantic fallback, ANN and persistent +storage changes are deferred. This draft is not a release announcement or a +claim that those features are implemented. + +## Reproduce the synthetic comparison + +Compile the baseline source and current source into separate directories with +the existing TypeScript compiler. Then run: + +```bash +node scripts/benchmark-recall.mjs /path/to/baseline/dist/src/index.js /path/to/current/dist/src/index.js +``` + +Use compiled modules so TypeScript loader overhead is outside the measurement. +The script asserts unchanged candidate IDs/scores for its performance fixtures +and writes JSON to stdout. It neither installs packages nor touches memory files. +Use the regression tests, rather than these speed fixtures, to check changed +eligibility behavior. diff --git a/docs/release-recall-lifecycle.md b/docs/release-recall-lifecycle.md new file mode 100644 index 0000000..edabe56 --- /dev/null +++ b/docs/release-recall-lifecycle.md @@ -0,0 +1,171 @@ +# Unreleased: explicit recall cues and recoverable memory lifecycle + +Source update notes; package release pending. No new version, release date, tag, +npm package, GitHub Release or social post is published by these notes. See the +[combined introduction and opt-ins](recall-update.md). This follows the [first recall-efficiency changes](release-recall-efficiency.md) +and uses `bad6b527409a44a693562fbe60c41be74126e573` as its comparison baseline. + +These results describe the second implementation batch. The subsequent +[verification-loop checkpoint](verification-loop.md) fixes test-helper diagnostic +loss and records the historical native-execution blocker. Complete hosted +[acceptance now passes on Ubuntu and Windows](actions-acceptance-results.md), +including 119 source cases, 119 compiled cases and one package/install/use case +per platform, with no skipped tests. The source and these notes are public; the +package update and social draft remain unreleased. The original Cloud EPERM cause is +still unresolved, and the results below retain their original measurement scope. + +## User introduction + +The optional engine now supports explicit aliases for familiar wording, managed +embedding-cache ownership, recoverable deletion and supersession, and prompt +excerpts that say when constraints may be missing. Enable only what your host +needs using the [examples and migration guide](engine/recall-lifecycle.md). + +The three-file protocol remains unchanged. These additions do not provide an +automatic Chinese segmenter, a multilingual semantic model, a vector database or +a paid service. The CLI has no implementation changes in this batch. + +## Labeled retrieval evaluation + +The [fixture](../scripts/fixtures/recall-evaluation.json) contains 36 synthetic +queries: 18 development and 18 held-out, each with 13 positive and five negative +queries. Categories cover Chinese, synonyms, paraphrases, explicit bilingual +aliases, unrelated requests and deleted/superseded memories. Topics are disjoint +between splits. Corpus aliases and both query splits were frozen before the first +evaluation; development results were inspected first and rules were not tuned +after held-out scoring. Cases and aliases share an author, so this is not an +independent or representative language benchmark. No private memories were used. + +| Split | Baseline / new default Recall@1 | Aliases Recall@1 | Missed positive queries, before → aliases | False-recall queries, before → aliases | +| --- | ---: | ---: | ---: | ---: | +| Development | 2/13 (15.4%) | 10/13 (76.9%) | 11 → 3 | 0 → 0 | +| Held-out | 2/13 (15.4%) | 10/13 (76.9%) | 11 → 3 | 0 → 0 | + +Recall@3 and Recall@5 are identical here: widening K cannot recover a query with +no eligible lexical candidate. Recall is averaged over positive queries. A false +recall means a non-gold eligible ID in the first K candidates; only the first +candidate is actually loaded by recall. Negative-query false positives were 0/5 +in each split and mode. Zero observed errors in this small set is not a guarantee. +Per-category results and exact returned IDs are in the +[evaluation record](engine/recall-evaluation.json). + +All six unlisted paraphrases still miss. Improvement comes from exact, +host-authored aliases, including bilingual pairs, not learned translation or +Chinese segmentation. No threshold was lowered. Broad substring expansion and +automatic segmentation were deferred because these tests do not establish a +safe precision/recall tradeoff for them. Stale cases test successful removal; +separate fault-injection tests cover failed lifecycle operations. + +## Performance and ownership costs + +The same synthetic performance script used in the first round compares the +previous commit with this implementation. Three complete repetitions are retained +in the [measurement record](engine/recall-round2-benchmark.json). Each uses five +warmups and 25 samples, alternating baseline/current order, in Node v24.19.0. +Numbers below are medians of the three per-run medians, in milliseconds. + +| Common-cue workload | Previous | Current | +| --- | ---: | ---: | +| 100 records, storage-search microbenchmark | 0.0208 | 0.0210 | +| 1,000 records, storage-search microbenchmark | 0.2434 | 0.2708 | +| 10,000 records, storage-search microbenchmark | 2.3091 | 2.6976 | +| 100 records, resident API end-to-end | 0.0826 | 0.0817 | +| 1,000 records, resident API end-to-end | 0.2723 | 0.3552 | +| 10,000 records, resident API end-to-end | 2.5853 | 2.9064 | + +This round does **not** establish a speedup. The 10,000-record resident API median +rose about 12.4%. Defensive summary/chunk copies add allocation, but timing alone +does not isolate the causes; storage-search logic itself is unchanged. Noise, +garbage collection and memory pressure remain confounders. No timing threshold +was used to select a favorable run. These default-mode measurements exclude +owned-cache serialization and concurrent traffic. + +The resident API measurement covers recall with mock query embedding, summary +and chunk reads, access-time updates and JSON serialization. It excludes startup, +ingestion/index construction, persistence, external model inference, prompt +processing and LLM work. It is not whole-product or agent end-to-end latency. +The first round's main-to-baseline acceleration remains historical evidence, +not a new speedup attributable to this batch. + +A separate [ownership fixture](engine/recall-cache-ownership-benchmark.json) uses +16 memories with repeated content and ten identical chunks each: + +| Actual provider calls | Previous unscoped cache | Current unscoped cache | Current owned cache | +| --- | ---: | ---: | ---: | +| Add all memories | 18 | 18 | 48 | +| Two complete detail-recall passes | 16 | 16 | 16 | +| Metadata-only update, preservation enabled | 0 | 0 | 0 | +| Content update | 2 | 2 | 3 | + +After deleting all memories, current unscoped mode retains 36 entries (5,184 +accounted bytes); owned mode retains zero. Baseline has no public statistics API. +Separate ownership trades cross-memory reuse for predictable invalidation. Counts +are deterministic mock calls, not paid-service costs, tokens or throughput. + +## Historical second-batch validation and blockers + +The following records the original Cloud checkpoint, before the later hosted +acceptance linked above. It does not describe the current CI status. + +- TypeScript checking and compilation pass with existing dependencies. +- Source and compiled engine regression suites pass: 50 distinct cases, including + aliases, scope isolation, late completion, model/content keys, lifecycle + compensation, queued reads, mutable-result isolation and qualifier warnings. +- The complete compiled test selection was executed in Node's supported + in-process mode: **77 passed, 32 failed, 0 skipped**. Failures are in CLI/process + scenarios: 27 preserve EPERM evidence, while five lose the original cause in + the error-response helper and report JSON parsing errors. Those five cannot + be conclusively attributed to EPERM from the saved log alone. + The existing engine reliability subset passes all 26 cases; one additional + initialization rollback test also passes in the full selection. +- The standard isolated source and package test commands fail in this Cloud. + A dependency-free `spawnSync(process.execPath, ["-e", "process.stdout.write('ok')"])` + reproduces `EPERM`; the returned `status: 0` does not override the error. + The isolated package-test diagnostic fails at its `npm pack` child call. + Node's permission API is absent; seccomp filtering and `NoNewPrivs` are active. + The exact denied syscall or interception layer is **not established**. +- Direct `npm pack --offline` succeeds in the same Cloud, including unchanged + prepack clean/build. Direct offline installation of that tarball and installed + CLI/API smoke checks pass. This validates packaging separately; it does not + turn the failing automated package test into a pass. No restriction, test, + lifecycle script or assertion was disabled to make the runner pass. +- Full `npm test` is not complete. Its repository-wide Markdown scan would read + private memory, so it was not run; public documentation links were checked + separately. No private memory, network service or new dependency was used. + +At that checkpoint the failed subprocess scenarios still needed an authorized +environment. Subsequent GitHub Actions runs passed the complete source, compiled +and installed-package selections on Ubuntu and Windows. The native Cloud EPERM +cause remains unresolved; hosted success is not an npm or GitHub Release. + +## Remaining limits + +Managed ownership serializes this engine's reads and writes; slow providers can +block later calls, and plugins must not re-enter the engine during those calls. +It does not coordinate other engines or direct adapter writes. Compensation can +fail and leave both supersession IDs present; errors expose recovery information. +There are no crash-safe transactions, historical redirects or verified secure +erasure. External providers and caller-held copies may retain data. + +Provenance warnings consume prompt budget and may force an entire field to be +omitted. They mark possible loss; they do not preserve semantic qualifiers or +sanitize instructions. Raw lookup returns stored content, not original separate +context fields. Bytes are not tokens. Real tokenizer/model quality, high-load +queueing, heap bounds and production traffic remain unmeasured. CLI optimization, +ANN, semantic fallback and automatic Chinese segmentation are deferred. + +## Reproduce + +Compile the baseline and current source with the existing compiler into separate +directories. The following scripts use only synthetic data and local modules: + +```bash +node scripts/evaluate-recall.mjs /path/to/before/src/index.js /path/to/after/src/index.js development +node scripts/evaluate-recall.mjs /path/to/before/src/index.js /path/to/after/src/index.js held-out +node scripts/benchmark-recall.mjs /path/to/before/src/index.js /path/to/after/src/index.js +node scripts/benchmark-cache-ownership.mjs /path/to/before/src/index.js /path/to/after/src/index.js +``` + +The evaluation fixture is packaged with its script. Results include the fixture's +SHA-256 digest. Timing scripts assert comparable candidate IDs/scores; quality +and failure behavior belong to the labeled evaluation and regression tests. diff --git a/docs/social-recall-lifecycle-draft.md b/docs/social-recall-lifecycle-draft.md new file mode 100644 index 0000000..fc442e3 --- /dev/null +++ b/docs/social-recall-lifecycle-draft.md @@ -0,0 +1,26 @@ +# Local social draft: recall cues and lifecycle + +Unpublished draft only. No release date or version is assigned. Hosted acceptance +has passed; publishing a release or social announcement remains a separate action. +See the [evidence and limitations](release-recall-lifecycle.md). + +## Short introduction + +Deja Vu's repository source now includes optional-engine improvements: explicit query aliases, +embedding-cache ownership that follows memory updates and deletions, recoverable +supersession, and prompt excerpts that warn when constraints may be missing. +The three-file memory protocol stays the same. + +## Technical follow-up + +In a small synthetic held-out set, authored aliases raised Recall@1 from 2/13 to +10/13, with no false recalls observed. Unlisted paraphrases still missed. This is +explicit routing, not a new multilingual semantic model. + +The tradeoffs matter: separate cache ownership means less cross-memory reuse, +and this round did not improve measured latency. Direct offline packaging and +installed smoke checks pass. Full source, compiled and installed-package tests +also pass on Ubuntu and Windows in the verification branch's GitHub Actions. +That does not explain the earlier Cloud runtime's native EPERM. The source is +public; the package update and this social draft remain unreleased. Usage, migration, +full measurements and remaining limits are in the release-note draft. diff --git a/docs/verification-loop-evidence.json b/docs/verification-loop-evidence.json new file mode 100644 index 0000000..e7e5de1 --- /dev/null +++ b/docs/verification-loop-evidence.json @@ -0,0 +1,95 @@ +{ + "baselineCommit": "fa479335dbc5acbad46a352d1ee8bd6086c62cd7", + "scope": "Same Cloud, public-source baseline snapshot; private memory excluded; existing dependencies linked, no alternate runtime.", + "runtime": { + "node": "v24.19.0", + "libuv": "1.52.1", + "permissionAPI": "undefined", + "NODE_OPTIONS_present": false, + "LD_PRELOAD_present": false, + "NoNewPrivs": 1, + "Seccomp": 2, + "Seccomp_filters": 1, + "CapEff": "0000000000000000" + }, + "historicalCompleteCompiledRun": { + "commit": "fa479335dbc5acbad46a352d1ee8bd6086c62cd7", + "log": "/tmp/dejavu-round2/full-compiled.log", + "tests": 109, + "pass": 77, + "fail": 32, + "directEperm": 9, + "assertionOnEperm": 18, + "originalCauseMaskedByJsonParse": 5, + "note": "Historical result, not a rerun of this revision. Five original error codes remain unknown." + }, + "diagnosticTrace": { + "exitCode": 1, + "stderr": [ + "strace: test_ptrace_get_syscall_info: PTRACE_TRACEME: Operation not permitted", + "strace: ptrace(PTRACE_TRACEME, ...): Operation not permitted", + "strace: PTRACE_SETOPTIONS: Operation not permitted", + "strace: cleanup: waitpid(-1, __WALL): No child processes" + ], + "probeOutputBytes": 0, + "traceBytes": 0, + "interpretation": "Tracing itself is denied. No successful syscall trace or new Node probe result was obtained. No retry, fallback tracer or security change.", + "argv": [ + "strace", + "-f", + "-qq", + "-e", + "trace=process,epoll_ctl,epoll_wait,epoll_pwait,epoll_pwait2,io_uring_setup,io_uring_enter,pipe2,socketpair,dup2,dup3,fcntl", + "-o", + "/tmp/dejavu-round3/spawn-sync.trace", + "node", + "-e", + "const r=require(\"node:child_process\").spawnSync(process.execPath,[\"-e\",\"process.stdout.write(\\\"synthetic-ok\\\")\"],{encoding:\"utf8\"});console.log(JSON.stringify({error:r.error?.code,syscall:r.error?.syscall,status:r.status,signal:r.signal,stdout:r.stdout,stderr:r.stderr,pid:r.pid}));" + ] + }, + "nativeBoundary": "Installed internal/child_process.spawnSync calls spawn_sync.spawn and wraps returned result.error in ErrnoException. This narrows the error boundary, not the underlying syscall or policy decision.", + "freshSubsetResults": { + "baseline-source-engine.log": { + "tests": 50, + "pass": 50, + "fail": 0, + "skipped": 0 + }, + "baseline-compiled-engine.log": { + "tests": 50, + "pass": 50, + "fail": 0, + "skipped": 0 + }, + "current-source-engine-harness.log": { + "tests": 60, + "pass": 60, + "fail": 0, + "skipped": 0 + }, + "current-compiled-engine-harness.log": { + "tests": 60, + "pass": 60, + "fail": 0, + "skipped": 0 + } + }, + "gates": { + "completeSourceSuite": "BLOCKED_NOT_RERUN: known subprocess failures; no bypass or skipped-test success claim", + "completeCompiledSuite": "BLOCKED_NOT_RERUN: known subprocess failures; partial subsets do not satisfy this gate", + "automatedPackageTest": "BLOCKED_NOT_RERUN: execFileSync pack call previously failed; direct checks are separate", + "baselineAndCurrentTypecheckAndBuild": "PASS", + "baselineAndCurrentDirectOfflinePackInstallUse": "PASS: unchanged prepack lifecycle, local tarball install, help/init/doctor/lint/scan/report and old/new engine API", + "productAssertions": "PASS: 70 memory-cli, 80 reliability and 27 package-pack assertions unchanged; infrastructure undefined guard replaced by equivalent strict rethrow", + "qualityAndCache": "PASS: frozen evaluation JSON unchanged; current cache lifecycle counts unchanged; baseline/current unscoped counts equal", + "privateMemoryAndCredentials": "NOT_USED", + "publicDocumentation": "PASS: unmodified checker on public snapshots; baseline 67 Markdown files, current 68. Private-memory documentation is outside this evidence.", + "originalAgentsPrefix": "PASS: original instructions retained byte-for-byte", + "helperBaselineComparison": "PASS: exact baseline helper loses synthetic EPERM cause; repaired helpers retain original identity without subprocess calls" + }, + "fixtureSha256": "4e1b70b5520e1d3cfb85489aad053825e339938016b20e659f2a4eb4677aee5d", + "checkpointDirectory": "/tmp/dejavu-round3", + "requiredResource": "Environment-owner-provided denied-syscall audit/trace for the synthetic probe, or narrowly approved tracing capability for that probe; then a confirmed policy-compliant runtime/execution configuration for complete subprocess tests.", + "unresolved": "Native spawn_sync EPERM has not been proven to be a Node/libuv bug or a specific sandbox rule. Ptrace denial is independently confirmed. No alternative spawn API/runtime/security option was tried.", + "unchangedEngineSourceTreeSha256": "b63b7595c48900bf94e59f4ceebff4260d22bee420b9fe40d01c3b414a11fc4a" +} diff --git a/docs/verification-loop.md b/docs/verification-loop.md new file mode 100644 index 0000000..1922bec --- /dev/null +++ b/docs/verification-loop.md @@ -0,0 +1,157 @@ +# Verification loop: harness diagnostics and Cloud execution + +Unreleased engineering checkpoint. Starting commit: +`fa479335dbc5acbad46a352d1ee8bd6086c62cd7`, on the existing local fix branch. + +This is a historical Cloud checkpoint. Subsequent authorized +[GitHub Actions acceptance](actions-acceptance-results.md) passes the complete +suite on Ubuntu and Windows; the Cloud native EPERM cause remains unresolved. + +## Scope and acceptance, defined before changes + +This iteration fixes loss of subprocess error evidence in test helpers, adds +regression coverage, and appends general verification-loop rules to AGENTS.md. +Engine behavior, thresholds, public protocol, dependencies and runtime security +configuration are outside the planned changes unless a reproducible regression +requires a focused correction. Preserve previous commits and unrelated edits. + +| Gate | Required evidence | +| --- | --- | +| Source suite | All existing cases plus new harness cases run; file-level summaries are not case counts. No skips or weakened assertions. | +| Compiled suite | The same complete selection runs after a fresh build. Reconcile each failure, including original cause, with the source selection. | +| Package/install/use | Unchanged prepack clean/build, package file assertions, offline installation of the local tarball, installed CLI and engine API assertions. | +| Types | TypeScript no-emit checking and compilation with existing dependencies. | +| Regression | Existing engine, CLI and recovery assertions retained; helper cases cover operational failures, expected nonzero exits, invalid JSON, signals and diagnostic cause preservation. | +| Recall quality | The frozen development/held-out fixture yields the previously recorded IDs and metrics; no tuning to held-out labels. | +| Cache/projection | Ownership, model/content keys, pending invalidation, update/delete/supersede recovery and projection budget/provenance cases remain passing. | +| Documentation | Claims distinguish tested results, inference and blocked gates; new links resolve; original AGENTS instructions remain an exact prefix. | +| Completion | All applicable gates pass, or an external blocker has exact evidence, a reproducible checkpoint and the required resource/permission. A blocked gate is never a pass. | + +Use only synthetic fixtures and existing dependencies. Do not read private +project memory. If a full-tree documentation check is needed, use a public-source +snapshot with only public/synthetic fixtures and describe that scope explicitly. +Do not disable tests, change security controls, drop inherited guard variables, +install another runtime, use paid services, or publish anything. + +## Loop protocol + +For each failure, record the command, baseline/current revision, environment, +exit/error evidence, hypothesis and next discriminating check. Fix the smallest +supported cause, rerun affected regressions, then the full acceptance gates. +After two iterations without new evidence or improvement, change hypothesis or +strategy; do not repeat an unchanged denied operation. A diagnostic trace is for +observation only and must stop if tracing itself is denied. Any runtime option +must be established as ordinary, authorized execution rather than an evasion of +policy before it is used. Preserve failures and blockers in the final record. + +## Initial evidence and hypotheses + +The preceding compiled run had 109 cases: 77 passed and 32 failed. Of the failures, +27 retain explicit EPERM evidence; five lose their original cause when a helper +parses empty stdout. The two affected test files and their CLI dependencies are +unchanged from the preceding baseline. Existing direct offline packaging and +installed smoke checks passed in the same Cloud. These observations do not yet +identify the syscall or prove a Node bug versus a security-policy denial. + +Hypotheses to distinguish: helper-only diagnostic loss; Node/libuv synchronous +child-I/O failure; test-runner-specific behavior; or an enforced environment +restriction. Fix diagnostic loss first. Compare unchanged baseline code without +weakening it, and use a bounded observation only where it adds evidence. + +## Changes and observed results + +The error-response helper now accepts only ordinary nonzero exits with string +stdout and no operational error/code/signal. EPERM, EACCES, ENOENT, timeouts, +buffer failures, signals and invalid execution results preserve the original +thrown object. Invalid JSON from an ordinary nonzero exit becomes +`CliResponseError`, with the execution error as `cause` and the parse error +available separately. The message does not copy arbitrary stdout/stderr. + +The reliability helper now rethrows `result.error` directly, retaining the same +strict `!== undefined` guard. It no longer obscures an operational error behind +`ERR_ASSERTION`. AST comparison confirms that all 70 memory-cli, 80 reliability +and 27 package-pack product assertions are unchanged. The package test now uses +npm's ordinary `--offline` option for pack/install; this restricts network use +and does not disable prepack or any assertion. Ten new helper cases cover these +boundaries, including a misleading `status: 0` accompanying EPERM. + +The original AGENTS.md is an exact byte-for-byte prefix of the updated file. +Only general verification-loop rules were appended. Engine implementation, +thresholds, frozen fixtures and dependency lockfile are unchanged. + +| Acceptance gate | This iteration's result | +| --- | --- | +| Complete source suite | **BLOCKED / not rerun** after known subprocess failures. The new revision has not passed this gate. | +| Complete compiled suite | **BLOCKED / not rerun** for the same reason. Do not reuse the historical 77/32 as a new result. | +| Automated package test | **BLOCKED / not rerun**; its execFileSync path previously reported EPERM. | +| Unaffected source subset | Baseline 50/50; current 60/60 (same 50 engine cases plus ten helper cases). This is not the complete suite. | +| Unaffected compiled subset | Baseline 50/50; current 60/60. These repeat the source cases, not additional distinct cases. | +| Type checking/build | Both baseline and current passed with existing dependencies. | +| Direct offline package/install/use | Both baseline and current passed unchanged prepack clean/build, tarball installation, installed metadata, help/init/doctor/lint/scan/report, and old/new engine API smoke assertions. Separate evidence, not an automated package-test pass. | +| Recall quality | Exact evaluation JSON matches the frozen record, including returned IDs and development/held-out metrics. | +| Cache/projection | Existing lifecycle/budget/provenance cases pass within the 50 engine cases. Provider-call and retained-entry measurements match the prior record. | +| Public documentation | Unmodified checker passes on public-source snapshots; private memory is excluded from the snapshot, not silently treated as verified. | +| Completion | **External diagnostic/runtime blocker checkpoint; full acceptance is not green.** | + +The complete selection contains the previous 109 cases plus ten new helper cases; +the automated package case is separate. This is an expected inventory, not a new +119-case execution result. The earlier full compiled run had 77 passes and 32 +failures: nine direct EPERM, 18 infrastructure assertions containing EPERM, and +five JSON parse failures with lost original causes. The five historical causes +cannot be recovered by fixing the helper now. + +## Evidence loop and stopping boundary + +1. **Hypothesis: helper parsing hides operational errors.** The unchanged baseline + helper reproduces the loss with synthetic errors in one process. The focused + fix preserves error identity and expected CLI JSON behavior; ten regressions + pass. No product assertion was weakened. +2. **Hypothesis: the native error boundary can be distinguished with a syscall + observation.** One bounded trace attempt failed before yielding a Node probe + result: `PTRACE_TRACEME` and `PTRACE_SETOPTIONS` returned `Operation not + permitted`. The trace and probe-output files are empty. Tracing was not retried + or replaced with another mechanism, and no policy was changed. +3. **Changed strategy: unchanged public-source baseline and safe observations.** + The same Node v24.19.0 / libuv 1.52.1 runtime passes pure engine tests and direct + offline pack/install/use for both versions. Its built-in JavaScript wrapper + shows the error comes back from `spawn_sync.spawn`, before the helper or test + runner handles it. Existing dependency-free reproduction outside the test + runner means the runner alone does not explain the failure. This does not + identify the failing native syscall or prove Node/libuv is defective. + +`Seccomp: 2`, `NoNewPrivs: 1`, and zero effective capabilities are observed. +Node's permission API is absent; NODE_OPTIONS and LD_PRELOAD are not present. +None of these observations proves which rule caused the original spawnSync +error. Direct prepack children succeeding also means it would be inaccurate to +claim all child creation is forbidden. The **tracing restriction is confirmed**; +Node/libuv versus a specific spawn/I/O policy interaction remains unresolved. + +No alternative spawn API, alternate runtime, inherited-guard removal, security +setting change or same-denial retry was used. In-process test selection runs only +unaffected cases and remains explicitly partial. It is not a way to count the +blocked CLI scenarios as passed. Two iterations without new evidence must change +strategy; once further discrimination requires unavailable privileges, the loop +stops at this checkpoint instead of manufacturing a green result. + +## Checkpoint and required resource + +The compact [evidence record](verification-loop-evidence.json) preserves counts, +trace denial, runtime observations, fixture digest and gate states. Detailed logs, +public baseline/current snapshots, local tarballs and synthetic installed projects +are under `/tmp/dejavu-round3` in the current Cloud. Temporary files may not survive +environment replacement; the committed evidence and source remain reproducible. + +An environment maintainer must provide either a denied-syscall audit/trace for the +synthetic spawnSync probe, or narrowly approved tracing capability for that probe. +That evidence is needed to identify the native error and choose a confirmed, +policy-compliant runtime/execution configuration. This is **not** a request to +turn off seccomp or broadly allow process access. No repository change can grant +that observation capability, and a different runtime has not been assumed safe. + +Once that prerequisite is satisfied, run the unchanged baseline and repaired +revision under the approved configuration. Capture the original error for any +failure, rerun the affected regression, and then run full `npm test` on the public +source snapshot. It retains every source/compiled/pack assertion and includes the +new helper cases. Compare the frozen evaluation and cache measurements again. +Only all passing gates can replace the blocked status. Any new failure restarts +failure -> diagnosis -> correction -> affected tests -> full acceptance. diff --git a/package.json b/package.json index f47710b..0a50d3a 100644 --- a/package.json +++ b/package.json @@ -32,9 +32,9 @@ "clean": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"", "prepack": "npm run clean && npm run build", "prepublishOnly": "npm test", - "test": "npm run test:src:readonly && npm run check:links && npm run build && node --test dist/tests/semantic-recall-engine.test.js dist/tests/memory-cli.test.js dist/tests/reliability.test.js && npm run test:pack", + "test": "npm run test:src:readonly && npm run check:links && npm run build && node --test dist/tests/semantic-recall-engine.test.js dist/tests/recall-efficiency.test.js dist/tests/recall-lifecycle.test.js dist/tests/cli-errors.test.js dist/tests/memory-cli.test.js dist/tests/reliability.test.js && npm run test:pack", "test:src": "npm run test:src:readonly", - "test:src:readonly": "node --import \"data:text/javascript,import { register } from 'node:module'; import { pathToFileURL } from 'node:url'; register('ts-node/esm', pathToFileURL('./'));\" --test tests/semantic-recall-engine.test.ts tests/memory-cli.test.ts tests/reliability.test.ts", + "test:src:readonly": "node --import \"data:text/javascript,import { register } from 'node:module'; import { pathToFileURL } from 'node:url'; register('ts-node/esm', pathToFileURL('./'));\" --test tests/semantic-recall-engine.test.ts tests/recall-efficiency.test.ts tests/recall-lifecycle.test.ts tests/cli-errors.test.ts tests/memory-cli.test.ts tests/reliability.test.ts", "test:pack": "node --import \"data:text/javascript,import { register } from 'node:module'; import { pathToFileURL } from 'node:url'; register('ts-node/esm', pathToFileURL('./'));\" --test tests/package-pack.test.ts", "check:links": "node scripts/check-markdown-links.mjs", "scan:memory": "node scripts/dejavu-scan-memory.mjs", diff --git a/scripts/benchmark-cache-ownership.mjs b/scripts/benchmark-cache-ownership.mjs new file mode 100644 index 0000000..1d8a23e --- /dev/null +++ b/scripts/benchmark-cache-ownership.mjs @@ -0,0 +1,24 @@ +#!/usr/bin/env node +import { pathToFileURL } from 'node:url'; +import { resolve } from 'node:path'; +const [beforePath, afterPath] = process.argv.slice(2); +if (!beforePath || !afterPath) throw new Error('Usage: node scripts/benchmark-cache-ownership.mjs BEFORE_INDEX AFTER_INDEX'); +const before = await import(pathToFileURL(resolve(beforePath))); +const after = await import(pathToFileURL(resolve(afterPath))); +const report = { note: 'Deterministic provider-call counts, not latency or token costs. Sixteen memories share ten identical chunks; unique per-memory routing tags. Cache capacity 256.', modes: {} }; +for (const [mode, api, managed] of [['before-unscoped', before, false], ['after-unscoped', after, false], ['after-owned', after, true]]) { + let calls = 0; + const cache = new api.CachedEmbeddingProvider({ async embed(text) { calls++; return [text.length, 1]; } }, { modelKey: 'synthetic:v1', maxEntries: 256 }); + const engine = api.createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: managed, + preserveDerivedOnMetadataUpdate: true, chunker: { chunk: () => Array.from({ length: 10 }, () => ({ type: 'note', content: 'shared chunk' })) } }); + for (let i = 0; i < 16; i++) await engine.addMemory({ id: `memory${i}`, title: 'Shared policy', content: 'shared content', tags: [`route${i}`] }); + const result = { addCalls: calls }; calls = 0; + for (let pass = 0; pass < 2; pass++) for (let i = 0; i < 16; i++) await engine.recall({ text: `route${i}`, loadChunks: true }); + result.twoRecallPassesCalls = calls; calls = 0; + await engine.updateMemory('memory0', { importance: 0.9 }); result.metadataUpdateCalls = calls; calls = 0; + await engine.updateMemory('memory0', { content: 'replacement content' }); result.contentUpdateCalls = calls; + for (let i = 0; i < 16; i++) await engine.deleteMemory(`memory${i}`); + result.retainedAfterDelete = cache.getStats?.() ?? 'No public stats API in baseline'; + report.modes[mode] = result; +} +console.log(JSON.stringify(report, null, 2)); diff --git a/scripts/benchmark-recall.mjs b/scripts/benchmark-recall.mjs new file mode 100644 index 0000000..004f08a --- /dev/null +++ b/scripts/benchmark-recall.mjs @@ -0,0 +1,80 @@ +#!/usr/bin/env node +// Synthetic data only. Run against compiled modules; no network or dependencies. +// node scripts/benchmark-recall.mjs /path/to/base/dist/src/index.js ./dist/src/index.js +import { performance } from "node:perf_hooks"; +import { pathToFileURL } from "node:url"; +import { resolve } from "node:path"; +import assert from "node:assert/strict"; + +const paths = process.argv.slice(2); +if (paths.length !== 2) throw new Error("Usage: benchmark-recall.mjs BASELINE_MODULE CURRENT_MODULE"); +const [baseline, current] = await Promise.all(paths.map((path) => import(pathToFileURL(resolve(path)).href))); +const samples = 25; +const report = { node: process.version, samples, warmups: 5, units: "milliseconds", notes: [ + "Micro: storage.searchImpressions only. Scan: engine.scanImpressions.", + "API end-to-end: resident recall(loadChunks=true) plus JSON.stringify, including query embedding and chunk retrieval.", + "Excludes process start, index construction, persistence, network, model inference and host/LLM execution.", + "Baseline/current alternate measurement order. Medians/p95 are descriptive of this small synthetic run.", +], workloads: [], operations: {} }; +const stats = (values) => { + const sorted = values.toSorted((a, b) => a - b); + return { median: +sorted[Math.floor(sorted.length / 2)].toFixed(4), p95: +sorted[Math.ceil(sorted.length * .95) - 1].toFixed(4) }; +}; +async function paired(left, right) { + for (let i = 0; i < 5; i++) { await left(); await right(); } + const times = [[], []]; + const fns = [left, right]; + for (let i = 0; i < samples; i++) { + for (const index of (i % 2 ? [1, 0] : [0, 1])) { + const start = performance.now(); await fns[index](); times[index].push(performance.now() - start); + } + } + return { baseline: stats(times[0]), current: stats(times[1]) }; +} +async function fixture(api, n) { + const storage = new api.InMemoryStorage(); + const chunkVectorStore = new api.InMemoryChunkVectorStore(); + const vector = await new api.MockEmbeddingProvider().embed("shared routing synthetic benchmark"); + for (let i = 0; i < n; i++) { + const id = `item${i}`; + await storage.saveFamiliarity({ id, title: id, shortSummary: id, tags: [], + impressionTokens: ["shared", `topic${i}`, "routing", "synthetic", "benchmark"], + importance: .5, lastAccessedAt: "2026-01-01T00:00:00Z", embeddingVector: [1] }); + await storage.saveSummary({ id, description: "Synthetic decision", context: "Synthetic rationale", architectureOrIntent: "Synthetic intent", recentUpdates: [], metadata: {}, embeddingVector: vector }); + const chunks = Array.from({ length: 10 }, (_, j) => ({ chunkId: `${id}-${j}`, memoryId: id, type: "note", content: "shared routing synthetic benchmark", createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", importance: .5, embeddingVector: vector })); + await storage.saveChunks(chunks); await chunkVectorStore.upsert(chunks); + } + return { storage, engine: api.createInMemorySemanticRecallEngine({ storage, chunkVectorStore }) }; +} +for (const n of [100, 1000, 10000]) { + const fixtures = [await fixture(baseline, n), await fixture(current, n)]; + for (const [kind, query] of [["rare", `topic${n - 1}`], ["common", "shared"], ["none", "absentword"]]) { + const tokens = baseline.extractKeywords(query, 200); + const hits = await Promise.all(fixtures.map(({ storage }) => storage.searchImpressions(tokens, 5))); + assert.deepEqual(hits[0].map(h => [h.record.id, h.similarity]), hits[1].map(h => [h.record.id, h.similarity])); + const micro = await paired(...fixtures.map(({ storage }) => () => storage.searchImpressions(tokens, 5))); + const scan = await paired(...fixtures.map(({ engine }) => () => engine.scanImpressions(query))); + const apiEndToEnd = await paired(...fixtures.map(({ engine }) => async () => JSON.stringify(await engine.recall({ text: query, loadChunks: true })))); + report.workloads.push({ n, kind, micro, scan, apiEndToEnd }); + } +} +async function operations(api, optimized) { + let calls = 0; + const mock = new api.MockEmbeddingProvider(); + const counting = { async embed(text) { calls++; return mock.embed(text); } }; + const embeddingProvider = optimized ? new api.CachedEmbeddingProvider(counting, { modelKey: "mock:256:v1" }) : counting; + const engine = api.createInMemorySemanticRecallEngine({ embeddingProvider, preserveDerivedOnMetadataUpdate: optimized }); + const content = ("Decision: settings sync preserves preferences. " + "Synthetic explanation. ".repeat(14) + "\n\n").repeat(5); + await engine.addMemory({ id: "synthetic", title: "settings sync", tags: ["preferences"], content }); + const addCalls = calls; calls = 0; + let result; + for (let i = 0; i < 5; i++) result = await engine.recall({ text: "settings sync", loadChunks: true }); + const repeatedRecallCalls = calls; calls = 0; + await engine.updateMemory("synthetic", { importance: .9 }); + return { addCalls, repeatedRecallCalls, metadataUpdateCalls: calls, + rawRecallBytes: Buffer.byteLength(JSON.stringify(result)), + ...(optimized ? { projectedBytes: api.projectRecallForPrompt(result).bytes } : {}) }; +} +report.operations.baseline = await operations(baseline, false); +report.operations.currentOptIn = await operations(current, true); +console.log(JSON.stringify(report, null, 2)); diff --git a/scripts/evaluate-recall.mjs b/scripts/evaluate-recall.mjs new file mode 100644 index 0000000..a78f610 --- /dev/null +++ b/scripts/evaluate-recall.mjs @@ -0,0 +1,53 @@ +#!/usr/bin/env node +// Offline, labeled synthetic evaluation. No ingestion or query uses external services. +import { readFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { pathToFileURL } from 'node:url'; +import { resolve } from 'node:path'; +const [beforePath, afterPath, split = 'all'] = process.argv.slice(2); +if (!beforePath || !afterPath || !['all', 'development', 'held-out'].includes(split)) { + throw new Error('Usage: node scripts/evaluate-recall.mjs BEFORE_INDEX AFTER_INDEX [development|held-out|all]'); +} +const source = readFileSync(new URL('./fixtures/recall-evaluation.json', import.meta.url)); +const fixture = JSON.parse(source); +const before = await import(pathToFileURL(resolve(beforePath))); +const after = await import(pathToFileURL(resolve(afterPath))); +function metrics(rows, k) { + const positives = rows.filter(r => r.relevant.length); + const negatives = rows.filter(r => !r.relevant.length); + const hits = r => r.retrieved.slice(0, k); + return { + k, queries: rows.length, positiveQueries: positives.length, negativeQueries: negatives.length, + recallAtK: positives.length ? positives.reduce((n, r) => n + hits(r).filter(id => r.relevant.includes(id)).length / r.relevant.length, 0) / positives.length : null, + missedQueries: positives.filter(r => !hits(r).some(id => r.relevant.includes(id))).length, + falseRecallQueries: rows.filter(r => hits(r).some(id => !r.relevant.includes(id))).length, + falsePositiveNegativeQueries: negatives.filter(r => hits(r).length).length, + }; +} +const report = { fixtureSha256: createHash('sha256').update(source).digest('hex'), node: process.version, + interpretation: 'Recall@K is mean recall over positive queries, considering only eligible candidates. False recalls include any non-gold eligible ID in the first K; only the first candidate is loaded by recall(). Exact authored aliases are routing rules, not learned multilingual semantics. Held-out topics are disjoint; this small author-created split is not independent validation.', splits: {} }; +for (const [name, data] of Object.entries(fixture.splits)) { + if (split !== 'all' && split !== name) continue; + const modes = {}; + for (const [mode, api, enabled] of [['before', before, false], ['after-default', after, false], ['after-aliases', after, true]]) { + const aliases = data.memories.flatMap(m => m.aliases.map(alias => ({ alias, cue: m.id }))); + const engine = api.createInMemorySemanticRecallEngine(enabled ? { cueAliases: aliases } : {}); + for (const { aliases: _, ...memory } of data.memories) await engine.addMemory(memory); + const prefix = name === 'development' ? 'd' : 'h'; + await engine.addMemory({ id: 'retired', title: prefix + 'retiredobsolete', content: prefix + 'retiredobsolete' }); + const replacement = { id: 'successor', title: prefix + 'currentpolicy', content: prefix + 'currentpolicy' }; + if (engine.supersedeMemory) await engine.supersedeMemory('retired', replacement); + else { await engine.addMemory(replacement); await engine.deleteMemory('retired'); } + await engine.addMemory({ id: 'deleted', title: prefix + 'deletedobsolete', content: prefix + 'deletedobsolete' }); + await engine.deleteMemory('deleted'); + const rows = []; + for (const item of data.cases) { + const result = await engine.scanImpressions({ text: item.query, topK: 5, candidateLimit: 32 }); + rows.push({ ...item, retrieved: result.candidates.filter(c => c.familiarityLevel !== 'none').map(c => c.id) }); + } + modes[mode] = { overall: [1, 3, 5].map(k => metrics(rows, k)), + byCategory: Object.fromEntries([...new Set(rows.map(r => r.category))].map(category => [category, metrics(rows.filter(r => r.category === category), 1)])), rows }; + } + report.splits[name] = modes; +} +console.log(JSON.stringify(report, null, 2)); diff --git a/scripts/fixtures/recall-evaluation.json b/scripts/fixtures/recall-evaluation.json new file mode 100644 index 0000000..3e3aa9c --- /dev/null +++ b/scripts/fixtures/recall-evaluation.json @@ -0,0 +1,425 @@ +{ + "description": "Synthetic, small diagnostic fixture; not a semantic model benchmark. Corpus aliases and both topic-disjoint query splits frozen before the first evaluation. Held-out scoring follows development-only inspection; cases share an author and are not independent validation. No real memories.", + "splits": { + "development": { + "memories": [ + { + "id": "d-cache", + "title": "快取失效規則", + "content": "更新或刪除後釋放快取。只有測試環境可以停用驗證。", + "tags": [ + "d-cache" + ], + "aliases": [ + "更新後快取怎麼辦", + "cache eviction", + "刷新記憶快取" + ] + }, + { + "id": "d-login", + "title": "登入重試", + "content": "帳號驗證失敗最多重試三次。", + "tags": [ + "d-login" + ], + "aliases": [ + "無法登入怎麼處理", + "sign in retry", + "authentication retries" + ] + }, + { + "id": "d-backup", + "title": "備份還原", + "content": "每日離線備份；還原前驗證校驗碼。", + "tags": [ + "d-backup" + ], + "aliases": [ + "如何救回資料", + "restore backup", + "recover archived data" + ] + }, + { + "id": "d-theme", + "title": "顏色主題", + "content": "無障礙深色模式需要足夠對比。", + "tags": [ + "d-theme" + ], + "aliases": [ + "深色介面", + "night mode", + "dark appearance" + ] + } + ], + "cases": [ + { + "id": "d-01", + "category": "chinese", + "query": "快取失效規則", + "relevant": [ + "d-cache" + ], + "rationale": "Exact title cue remains a valid control." + }, + { + "id": "d-02", + "category": "chinese", + "query": "登入重試", + "relevant": [ + "d-login" + ], + "rationale": "Exact title cue remains a valid control." + }, + { + "id": "d-03", + "category": "chinese", + "query": "更新後快取怎麼辦", + "relevant": [ + "d-cache" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token." + }, + { + "id": "d-04", + "category": "chinese", + "query": "無法登入怎麼處理", + "relevant": [ + "d-login" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token." + }, + { + "id": "d-05", + "category": "synonym", + "query": "restore backup", + "relevant": [ + "d-backup" + ], + "rationale": "Authored English synonym routes to the corpus tag." + }, + { + "id": "d-06", + "category": "synonym", + "query": "night mode", + "relevant": [ + "d-theme" + ], + "rationale": "Authored English synonym routes to the corpus tag." + }, + { + "id": "d-07", + "category": "cross-language", + "query": "CACHE EVICTION", + "relevant": [ + "d-cache" + ], + "rationale": "Explicit bilingual alias; case normalization only." + }, + { + "id": "d-08", + "category": "cross-language", + "query": "SIGN IN RETRY", + "relevant": [ + "d-login" + ], + "rationale": "Explicit bilingual alias; case normalization only." + }, + { + "id": "d-09", + "category": "cross-language", + "query": " recover archived data ", + "relevant": [ + "d-backup" + ], + "rationale": "Explicit bilingual alias with whitespace normalization." + }, + { + "id": "d-10", + "category": "paraphrase", + "query": "重新整理存下來的線索以免拿到舊版", + "relevant": [ + "d-cache" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "d-11", + "category": "paraphrase", + "query": "進不去帳號時應該再試幾次", + "relevant": [ + "d-login" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "d-12", + "category": "paraphrase", + "query": "希望找回昨日保存的內容", + "relevant": [ + "d-backup" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "d-13", + "category": "synonym", + "query": "dark appearance", + "relevant": [ + "d-theme" + ], + "rationale": "Explicit alternate terminology." + }, + { + "id": "d-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus." + }, + { + "id": "d-15", + "category": "unrelated", + "query": "do not restore backup", + "relevant": [], + "rationale": "Negated query must not activate a substring alias." + }, + { + "id": "d-16", + "category": "unrelated", + "query": "sign in retry price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten." + }, + { + "id": "d-17", + "category": "stale", + "query": "dretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded." + }, + { + "id": "d-18", + "category": "stale", + "query": "ddeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted." + } + ] + }, + "held-out": { + "memories": [ + { + "id": "h-billing", + "title": "發票修正", + "content": "已結帳的發票只能以折讓單修正。", + "tags": [ + "h-billing" + ], + "aliases": [ + "發票寫錯怎麼辦", + "correct invoice", + "billing correction" + ] + }, + { + "id": "h-timeout", + "title": "連線逾時", + "content": "連線逾時只允許重試冪等請求。", + "tags": [ + "h-timeout" + ], + "aliases": [ + "連不上伺服器", + "connection deadline", + "network timeout" + ] + }, + { + "id": "h-export", + "title": "資料匯出", + "content": "匯出報表前必須遮蔽私人欄位。", + "tags": [ + "h-export" + ], + "aliases": [ + "怎樣下載報表", + "download report", + "export records" + ] + }, + { + "id": "h-schedule", + "title": "排程時區", + "content": "排程使用明確時區並處理日光節約時間。", + "tags": [ + "h-schedule" + ], + "aliases": [ + "定時工作用哪個時區", + "scheduled job timezone", + "timer zone" + ] + } + ], + "cases": [ + { + "id": "h-01", + "category": "chinese", + "query": "發票修正", + "relevant": [ + "h-billing" + ], + "rationale": "Exact title cue remains a valid control." + }, + { + "id": "h-02", + "category": "chinese", + "query": "連線逾時", + "relevant": [ + "h-timeout" + ], + "rationale": "Exact title cue remains a valid control." + }, + { + "id": "h-03", + "category": "chinese", + "query": "發票寫錯怎麼辦", + "relevant": [ + "h-billing" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token." + }, + { + "id": "h-04", + "category": "chinese", + "query": "連不上伺服器", + "relevant": [ + "h-timeout" + ], + "rationale": "Authored Chinese alias; baseline treats the whole phrase as one token." + }, + { + "id": "h-05", + "category": "synonym", + "query": "download report", + "relevant": [ + "h-export" + ], + "rationale": "Authored English synonym routes to the corpus tag." + }, + { + "id": "h-06", + "category": "synonym", + "query": "scheduled job timezone", + "relevant": [ + "h-schedule" + ], + "rationale": "Authored English synonym routes to the corpus tag." + }, + { + "id": "h-07", + "category": "cross-language", + "query": "CORRECT INVOICE", + "relevant": [ + "h-billing" + ], + "rationale": "Explicit bilingual alias; case normalization only." + }, + { + "id": "h-08", + "category": "cross-language", + "query": "CONNECTION DEADLINE", + "relevant": [ + "h-timeout" + ], + "rationale": "Explicit bilingual alias; case normalization only." + }, + { + "id": "h-09", + "category": "cross-language", + "query": " export records ", + "relevant": [ + "h-export" + ], + "rationale": "Explicit bilingual alias with whitespace normalization." + }, + { + "id": "h-10", + "category": "paraphrase", + "query": "帳單的金額填錯要如何更正", + "relevant": [ + "h-billing" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "h-11", + "category": "paraphrase", + "query": "伺服器等太久了需要重新連線嗎", + "relevant": [ + "h-timeout" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "h-12", + "category": "paraphrase", + "query": "把表格存成外部檔案需要注意什麼", + "relevant": [ + "h-export" + ], + "rationale": "Unseen paraphrase deliberately has no exact alias." + }, + { + "id": "h-13", + "category": "synonym", + "query": "timer zone", + "relevant": [ + "h-schedule" + ], + "rationale": "Explicit alternate terminology." + }, + { + "id": "h-14", + "category": "unrelated", + "query": "海底火山的溫度", + "relevant": [], + "rationale": "Outside the corpus." + }, + { + "id": "h-15", + "category": "unrelated", + "query": "do not download report", + "relevant": [], + "rationale": "Negated query must not activate a substring alias." + }, + { + "id": "h-16", + "category": "unrelated", + "query": "connection deadline price prediction", + "relevant": [], + "rationale": "Alias plus unrelated qualification must not be rewritten." + }, + { + "id": "h-17", + "category": "stale", + "query": "hretiredobsolete", + "relevant": [], + "rationale": "The only matching record has been superseded." + }, + { + "id": "h-18", + "category": "stale", + "query": "hdeletedobsolete", + "relevant": [], + "rationale": "The only matching record has been deleted." + } + ] + } + } +} diff --git a/src/core/semantic-recall-engine.ts b/src/core/semantic-recall-engine.ts index 159ae29..14392ea 100644 --- a/src/core/semantic-recall-engine.ts +++ b/src/core/semantic-recall-engine.ts @@ -17,9 +17,11 @@ import type { StoredMemory, UpdateMemoryInput, } from "../types/memory.js"; -import type { SemanticRecallEngineConfig } from "../types/plugins.js"; +import type { EmbeddingProvider, EmbeddingScope, SemanticRecallEngineConfig } from "../types/plugins.js"; import { createId } from "../utils/id.js"; import { extractKeywords } from "../utils/text.js"; +import { boundedInteger, MAX_RECALL_LIMIT } from "../utils/limits.js"; +import { compileCueAliases, normalizeCue } from "../retrieval/cue-aliases.js"; import { isoNow } from "../utils/time.js"; /** Includes a recovery copy if an adapter fails during compensation. Do not log its contents. */ @@ -36,6 +38,15 @@ export class MemoryWriteError extends Error { } } +/** A successor cleanup failure is explicit; inspect both IDs before retrying. */ +export class MemorySupersedeError extends Error { + constructor(cause: unknown, readonly memoryId: string, readonly replacementId: string, + readonly recoveryErrors: unknown[]) { + super(recoveryErrors.length ? "Supersede failed; recovery requires adapter repair" : "Supersede failed; previous state restored", { cause }); + this.name = "MemorySupersedeError"; + } +} + export class SemanticRecallEngine { private readonly familiarityLayer: FamiliarityLayer; private readonly summaryLayer: SummaryLayer; @@ -44,6 +55,8 @@ export class SemanticRecallEngine { private readonly summaryGenerator; private readonly thresholds; private readonly scoringStrategy; + private readonly cueAliases; + private readonly embeddingScopes = new Map(); private mutationQueue: Promise = Promise.resolve(); private mutate(operation: () => Promise): Promise { @@ -53,12 +66,20 @@ export class SemanticRecallEngine { } constructor(private readonly config: SemanticRecallEngineConfig) { + this.cueAliases = compileCueAliases(config.cueAliases); + if (config.manageEmbeddingCache && typeof config.embeddingProvider.createScope !== "function") { + throw new TypeError("manageEmbeddingCache requires embeddingProvider.createScope"); + } this.familiarityLayer = new FamiliarityLayer(config.storage, config.familiarityVectorStore); this.summaryLayer = new SummaryLayer(config.storage); this.chunkLayer = new ChunkLayer(config.storage, config.chunkVectorStore); this.chunker = config.chunker ?? new DefaultChunker(); this.summaryGenerator = config.summaryGenerator ?? new DefaultSummaryGenerator(); - this.thresholds = config.thresholds ?? { strong: 0.85, weak: 0.75 }; + this.thresholds = { ...(config.thresholds ?? { strong: 0.85, weak: 0.75 }) }; + const { weak, strong } = this.thresholds; + if (!Number.isFinite(weak) || !Number.isFinite(strong) || weak < 0 || strong > 1 || weak > strong) { + throw new RangeError("thresholds must satisfy 0 <= weak <= strong <= 1"); + } this.scoringStrategy = config.scoringStrategy ?? new HybridScoringStrategy(); } @@ -66,23 +87,36 @@ export class SemanticRecallEngine { return this.mutate(async () => { const id = input.id ?? createId("memory"); if (await this.snapshot(id)) throw new Error(`Memory already exists: ${id}; use updateMemory`); - const prepared = await this.prepareMemory({ ...input, id }); - await this.commitMemory(prepared, null); + await this.writeMemory({ ...input, id }, null); return { id }; }); } - private async prepareMemory(input: AddMemoryInput): Promise { + private async writeMemory(input: AddMemoryInput & { id: string }, previous: StoredMemory | null): Promise { + const scope = this.config.manageEmbeddingCache ? this.config.embeddingProvider.createScope!() : undefined; + try { + const prepared = await this.prepareMemory(input, scope ?? this.config.embeddingProvider); + if (previous) prepared.createdAt = previous.createdAt; + await this.commitMemory(prepared, previous); + } catch (error) { + scope?.dispose(); + throw error; + } + this.embeddingScopes.get(input.id)?.dispose(); + if (scope) this.embeddingScopes.set(input.id, scope); + } + + private async prepareMemory(input: AddMemoryInput, provider: Pick): Promise { const id = input.id ?? createId("memory"); const now = isoNow(); const shortSummary = await this.summaryGenerator.generateShortSummary(input); const structuredSummary = await this.summaryGenerator.generateStructuredSummary(input); const chunkSeeds = this.chunker.chunk(input, now); - const familiarityVector = await this.config.embeddingProvider.embed( + const familiarityVector = await provider.embed( [input.title, shortSummary, ...(input.tags ?? [])].join("\n"), ); - const summaryVector = await this.config.embeddingProvider.embed( + const summaryVector = await provider.embed( `${structuredSummary.description}\n${structuredSummary.context}\n${structuredSummary.architectureOrIntent}`, ); @@ -96,7 +130,7 @@ export class SemanticRecallEngine { createdAt: now, updatedAt: now, importance: seed.importance ?? input.importance ?? 0.5, - embeddingVector: await this.config.embeddingProvider.embed(seed.content), + embeddingVector: await provider.embed(seed.content), }); } @@ -180,9 +214,13 @@ export class SemanticRecallEngine { } async recall(query: RecallInput | string): Promise { + return this.config.consistentReads || this.config.manageEmbeddingCache ? this.mutate(() => this.recallInternal(query)) : this.recallInternal(query); + } + + private async recallInternal(query: RecallInput | string): Promise { const normalized: RecallInput = typeof query === "string" ? { text: query } : query; - const chunkLimit = normalized.chunkLimit ?? 3; - const scan = await this.scanImpressions(normalized); + const chunkLimit = boundedInteger("chunkLimit", normalized.chunkLimit ?? 3, MAX_RECALL_LIMIT); + const scan = await this.scanImpressionsInternal(normalized); const topMatch = scan.topMatch; const familiarityLevel = scan.familiarityLevel; const budget: RecallBudgetReport = { @@ -213,8 +251,13 @@ export class SemanticRecallEngine { budget.whyLoaded.push("weak familiarity loaded a clipped summary"); } - if (familiarityLevel === "strong" && normalized.loadChunks) { - const vector = await this.config.embeddingProvider.embed(normalized.text); + if (familiarityLevel === "strong" && normalized.loadChunks && chunkLimit > 0) { + let scope = this.embeddingScopes.get(topMatch.id); + if (this.config.manageEmbeddingCache && !scope) { + scope = this.config.embeddingProvider.createScope!(); + this.embeddingScopes.set(topMatch.id, scope); + } + const vector = await (scope ?? this.config.embeddingProvider).embed(normalized.text); const chunkHits = await this.chunkLayer.search(topMatch.id, vector, chunkLimit); const allChunks = await this.chunkLayer.get(topMatch.id); const chunkMap = new Map(allChunks.map((chunk) => [chunk.chunkId, chunk])); @@ -244,12 +287,20 @@ export class SemanticRecallEngine { } async scanImpressions(query: RecallInput | string): Promise { + return this.config.consistentReads || this.config.manageEmbeddingCache ? this.mutate(() => this.scanImpressionsInternal(query)) : this.scanImpressionsInternal(query); + } + + private async scanImpressionsInternal(query: RecallInput | string): Promise { const normalized: RecallInput = typeof query === "string" ? { text: query } : query; - const topK = normalized.topK ?? 5; - const queryTokens = extractKeywords(normalized.text, 200); - const hits = await this.config.storage.searchImpressions(queryTokens, topK); + if (typeof normalized.text !== "string") throw new TypeError("text must be a string"); + const topK = boundedInteger("topK", normalized.topK ?? 5, MAX_RECALL_LIMIT); + const candidateLimit = boundedInteger("candidateLimit", normalized.candidateLimit ?? topK, MAX_RECALL_LIMIT, topK); + boundedInteger("chunkLimit", normalized.chunkLimit ?? 3, MAX_RECALL_LIMIT); + const cue = this.cueAliases.size ? this.cueAliases.get(normalizeCue(normalized.text)) ?? normalized.text : normalized.text; + const queryTokens = extractKeywords(cue, 200); + const hits = topK === 0 ? [] : await this.config.storage.searchImpressions(queryTokens, candidateLimit); - const candidates = this.scoringStrategy.rank( + const ranked = this.scoringStrategy.rank( hits.map((hit) => ({ record: hit.record, similarity: hit.similarity, @@ -259,6 +310,12 @@ export class SemanticRecallEngine { familiarityLevel: resolveFamiliarityLevel(candidate.semanticSimilarity, this.thresholds), })); + // Eligibility precedes hybrid preference. Keep hybrid ordering within each group, + // including weak vs strong, and retain ineligible candidates for diagnostics. + const candidates = [ + ...ranked.filter((candidate) => candidate.familiarityLevel !== "none"), + ...ranked.filter((candidate) => candidate.familiarityLevel === "none"), + ].slice(0, topK); const topMatch = candidates[0] ?? null; const familiarityLevel = topMatch ? resolveFamiliarityLevel(topMatch.semanticSimilarity, this.thresholds) @@ -284,19 +341,38 @@ export class SemanticRecallEngine { } async getSummary(id: string): Promise { - return this.summaryLayer.get(id); + return this.config.consistentReads || this.config.manageEmbeddingCache ? this.mutate(() => this.summaryLayer.get(id)) : this.summaryLayer.get(id); } async getChunks(id: string): Promise { - return this.chunkLayer.get(id); + return this.config.consistentReads || this.config.manageEmbeddingCache ? this.mutate(() => this.chunkLayer.get(id)) : this.chunkLayer.get(id); + } + + /** Explicit full raw-content retrieval; never loaded by prompt projection automatically. */ + async getRawContent(id: string): Promise { + return this.config.consistentReads || this.config.manageEmbeddingCache ? this.mutate(() => this.config.storage.getRawContent(id)) : this.config.storage.getRawContent(id); } async updateMemory(id: string, input: UpdateMemoryInput): Promise<{ id: string }> { return this.mutate(async () => { const previous = await this.snapshot(id); if (!previous) throw new Error(`Memory not found: ${id}`); + if (this.config.preserveDerivedOnMetadataUpdate && + Object.keys(input).every((key) => key === "importance" || key === "metadata")) { + // Explicitly preserve derived fields even for custom generators/chunkers. + // Reuse the existing compensated write path and detached snapshot. + const next = structuredClone(previous); + next.updatedAt = isoNow(); + next.familiarity.importance = input.importance ?? previous.familiarity.importance; + next.summary.metadata = { + ...previous.summary.metadata, ...input.metadata, + title: previous.familiarity.title, tags: [...previous.familiarity.tags], + }; + await this.commitMemory(next, previous); + return { id }; + } const existingSummary = previous.summary; - const prepared = await this.prepareMemory({ + await this.writeMemory({ id, title: input.title ?? previous.familiarity.title, content: input.content ?? previous.rawContent, @@ -312,18 +388,57 @@ export class SemanticRecallEngine { tags: input.tags ?? previous.familiarity.tags, }, chunkStrategy: input.chunkStrategy, - }); - prepared.createdAt = previous.createdAt; - await this.commitMemory(prepared, previous); + }, previous); return { id }; }); } - async deleteMemory(id: string): Promise { + /** Replace a live memory with a new ID. No historical redirect is stored by this optional engine. */ + async supersedeMemory(id: string, replacement: AddMemoryInput): Promise<{ id: string; supersededId: string }> { return this.mutate(async () => { + const previous = await this.snapshot(id); + if (!previous) throw new Error(`Memory not found: ${id}`); + const nextId = replacement.id ?? createId("memory"); + if (nextId === id || await this.snapshot(nextId)) throw new Error(`Replacement memory already exists: ${nextId}`); + await this.writeMemory({ ...replacement, id: nextId }, null); + try { + await this.deleteInternal(id, previous); + } catch (cause) { + const recoveryErrors: unknown[] = cause instanceof MemoryWriteError ? [...cause.recoveryErrors] : [cause]; + try { await this.deleteInternal(nextId, await this.snapshot(nextId)); } + catch (error) { recoveryErrors.push(error); } + throw new MemorySupersedeError(cause, id, nextId, recoveryErrors); + } + return { id: nextId, supersededId: id }; + }); + } + + async deleteMemory(id: string): Promise { + return this.mutate(async () => this.deleteInternal(id, await this.snapshot(id))); + } + + private async deleteInternal(id: string, previous: StoredMemory | null): Promise { + try { await this.familiarityLayer.remove(id); await this.chunkLayer.remove(id); await this.config.storage.deleteMemory(id); - }); + } catch (cause) { + const recoveryErrors: unknown[] = []; + const attempt = async (operation: () => Promise) => { + try { await operation(); } catch (error) { recoveryErrors.push(error); } + }; + if (previous) { + await attempt(() => this.config.storage.saveSummary(previous.summary)); + await attempt(() => this.config.storage.saveChunks(previous.chunks)); + await attempt(() => this.config.storage.saveRawContent(id, previous.rawContent, previous.createdAt, previous.updatedAt)); + await attempt(() => this.config.storage.saveFamiliarity(previous.familiarity)); + await attempt(() => this.config.familiarityVectorStore.upsert(previous.familiarity)); + await attempt(() => this.config.chunkVectorStore.upsert(previous.chunks)); + } + throw new MemoryWriteError(recoveryErrors.length ? "Memory delete failed; recovery requires adapter repair" : "Memory delete failed; previous state restored", + cause, previous, recoveryErrors, id); + } + this.embeddingScopes.get(id)?.dispose(); + this.embeddingScopes.delete(id); } } diff --git a/src/index.ts b/src/index.ts index f327f51..a161f61 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,3 +9,6 @@ export * from "./storage/in-memory-vector-store.js"; export * from "./types/memory.js"; export * from "./types/plugins.js"; export * from "./utils/text.js"; +export * from "./retrieval/prompt-projection.js"; +export * from "./plugins/cached-embedding-provider.js"; +export type { CueAlias } from "./retrieval/cue-aliases.js"; diff --git a/src/memory/default-chunker.ts b/src/memory/default-chunker.ts index 9e16a65..8eea217 100644 --- a/src/memory/default-chunker.ts +++ b/src/memory/default-chunker.ts @@ -1,5 +1,6 @@ import type { AddMemoryInput, MemoryChunkType } from "../types/memory.js"; import type { Chunker, MemoryChunkSeed } from "../types/plugins.js"; +import { boundedInteger } from "../utils/limits.js"; const DEFAULT_MAX = 280; const DEFAULT_OVERLAP = 40; @@ -78,8 +79,8 @@ function packBoundaryChunks(blocks: string[], maxCharacters: number, overlapChar export class DefaultChunker implements Chunker { chunk(input: AddMemoryInput): MemoryChunkSeed[] { - const maxCharacters = input.chunkStrategy?.maxCharacters ?? DEFAULT_MAX; - const overlapCharacters = input.chunkStrategy?.overlapCharacters ?? DEFAULT_OVERLAP; + const maxCharacters = boundedInteger("maxCharacters", input.chunkStrategy?.maxCharacters ?? DEFAULT_MAX, 1_000_000, 1); + const overlapCharacters = boundedInteger("overlapCharacters", input.chunkStrategy?.overlapCharacters ?? Math.min(DEFAULT_OVERLAP, maxCharacters - 1), maxCharacters - 1); const fallbackType = input.chunkStrategy?.preferredTypes?.[0] ?? "note"; const text = input.content.trim(); diff --git a/src/plugins/cached-embedding-provider.ts b/src/plugins/cached-embedding-provider.ts new file mode 100644 index 0000000..b022f6d --- /dev/null +++ b/src/plugins/cached-embedding-provider.ts @@ -0,0 +1,105 @@ +import { createHmac, randomBytes } from "node:crypto"; +import type { EmbeddingProvider, EmbeddingScope } from "../types/plugins.js"; +import { boundedInteger } from "../utils/limits.js"; + +export interface EmbeddingCacheOptions { + /** Include provider, model, revision and any embedding options. Change on model changes. */ + modelKey: string | (() => string); + maxEntries?: number; + /** Bound digest UTF-16 bytes + vector float64 bytes, not JS heap overhead. */ + maxBytes?: number; +} + +interface Scope { active: boolean; key: Buffer } +interface Entry { vector: number[]; bytes: number; scope?: Scope } + +/** Opt-in, instance-local LRU. Digests avoid retained plaintext content keys, not secure erasure. */ +export class CachedEmbeddingProvider implements EmbeddingProvider { + private readonly entries = new Map(); + private readonly secret = randomBytes(32); + private readonly maxEntries: number; + private readonly maxBytes: number; + private readonly modelKey: EmbeddingCacheOptions["modelKey"]; + private bytes = 0; + private generation = 0; + + constructor(private readonly provider: EmbeddingProvider, options: EmbeddingCacheOptions) { + this.modelKey = options.modelKey; + this.maxEntries = boundedInteger("maxEntries", options.maxEntries ?? 128, 10_000); + this.maxBytes = boundedInteger("maxBytes", options.maxBytes ?? 8 * 1024 * 1024, 64 * 1024 * 1024); + this.currentModelKey(); + } + + private currentModelKey(): string { + const key = typeof this.modelKey === "function" ? this.modelKey() : this.modelKey; + if (typeof key !== "string" || !key.trim() || key.length > 4096) { + throw new TypeError("modelKey must be a non-empty model/version identifier of at most 4096 characters"); + } + return key; + } + + /** Release references; already running provider work is not cancelled or securely erased. */ + clear(): void { + this.generation++; + this.entries.clear(); + this.bytes = 0; + } + + getStats(): { entries: number; accountedBytes: number } { + return { entries: this.entries.size, accountedBytes: this.bytes }; + } + + /** Separate ownership: identical inputs in different scopes never share retained entries. */ + createScope(): EmbeddingScope { + const scope: Scope = { active: true, key: randomBytes(32) }; + return { + embed: (text) => this.embedInScope(text, scope), + dispose: () => { + scope.active = false; + for (const [key, entry] of this.entries) { + if (entry.scope === scope) { this.bytes -= entry.bytes; this.entries.delete(key); } + } + }, + }; + } + + embed(text: string): Promise { return this.embedInScope(text); } + + private async embedInScope(text: string, scope?: Scope): Promise { + if (scope && !scope.active) throw new Error("Embedding scope has been disposed"); + const model = this.currentModelKey(); + // Length framing and UTF-16 preserve exact JS strings, including lone surrogates. + // Each owner has a different HMAC key; no plaintext input is retained as a key. + const key = this.maxEntries && this.maxBytes >= 136 + ? createHmac("sha256", scope?.key ?? this.secret).update(`${model.length}:`) + .update(model, "utf16le").update(text, "utf16le").digest("hex") : undefined; + const hit = key === undefined ? undefined : this.entries.get(key); + if (hit && key !== undefined) { + this.entries.delete(key); + this.entries.set(key, hit); + return [...hit.vector]; + } + const generation = this.generation; + // Independent misses: there is no unbounded in-flight promise registry. + const vector = await this.provider.embed(text); + if (!Array.isArray(vector) || !vector.length) throw new TypeError("embedding must be a non-empty finite vector"); + for (const value of vector) { + if (!Number.isFinite(value)) throw new TypeError("embedding must be a non-empty finite vector"); + } + if (key !== undefined && generation === this.generation && (!scope || scope.active) && model === this.currentModelKey()) { + const bytes = key.length * 2 + vector.length * 8; + if (bytes <= this.maxBytes) { + const previous = this.entries.get(key); + if (previous) { this.bytes -= previous.bytes; this.entries.delete(key); } + while (this.entries.size >= this.maxEntries || this.bytes + bytes > this.maxBytes) { + const oldest = this.entries.keys().next().value!; + this.bytes -= this.entries.get(oldest)!.bytes; + this.entries.delete(oldest); + } + this.entries.set(key, { vector: [...vector], bytes, scope }); + this.bytes += bytes; + } + } + return [...vector]; + } +} diff --git a/src/plugins/create-in-memory-engine.ts b/src/plugins/create-in-memory-engine.ts index c96f30d..ab0566e 100644 --- a/src/plugins/create-in-memory-engine.ts +++ b/src/plugins/create-in-memory-engine.ts @@ -15,9 +15,13 @@ export function createInMemorySemanticRecallEngine( familiarityVectorStore: config.familiarityVectorStore ?? new InMemoryFamiliarityVectorStore(), chunkVectorStore: config.chunkVectorStore ?? new InMemoryChunkVectorStore(), storage: config.storage ?? new InMemoryStorage(), + cueAliases: config.cueAliases, + manageEmbeddingCache: config.manageEmbeddingCache, + consistentReads: config.consistentReads, scoringStrategy: config.scoringStrategy, chunker: config.chunker, summaryGenerator: config.summaryGenerator, thresholds: config.thresholds, + preserveDerivedOnMetadataUpdate: config.preserveDerivedOnMetadataUpdate, }); } diff --git a/src/retrieval/cue-aliases.ts b/src/retrieval/cue-aliases.ts new file mode 100644 index 0000000..58088a6 --- /dev/null +++ b/src/retrieval/cue-aliases.ts @@ -0,0 +1,26 @@ +export interface CueAlias { + /** An exact complete query, compared after lowercase and whitespace normalization. */ + alias: string; + /** Existing lexical cue(s), usually a distinctive tag on the intended memory. */ + cue: string; +} + +export function normalizeCue(text: string): string { + return text.trim().toLowerCase().replace(/\s+/gu, " "); +} + +/** Explicit routing, not translation, substring expansion, or a semantic model. */ +export function compileCueAliases(aliases: readonly CueAlias[] = []): ReadonlyMap { + if (!Array.isArray(aliases) || aliases.length > 128) throw new RangeError("cueAliases must contain at most 128 entries"); + const map = new Map(); + for (const entry of aliases) { + if (!entry || typeof entry.alias !== "string" || typeof entry.cue !== "string" || + !entry.alias.trim() || !entry.cue.trim() || entry.alias.length > 256 || entry.cue.length > 256) { + throw new TypeError("alias and cue must be non-empty strings of at most 256 characters"); + } + const alias = normalizeCue(entry.alias); + if (map.has(alias) && map.get(alias) !== entry.cue) throw new Error(`Conflicting cue alias: ${alias}`); + map.set(alias, entry.cue); + } + return map; +} diff --git a/src/retrieval/prompt-projection.ts b/src/retrieval/prompt-projection.ts new file mode 100644 index 0000000..55b8e69 --- /dev/null +++ b/src/retrieval/prompt-projection.ts @@ -0,0 +1,102 @@ +import type { RecallResult } from "../types/memory.js"; +import { boundedInteger } from "../utils/limits.js"; + +export interface PromptProjectionOptions { + /** UTF-8 budget for the entire returned text, including JSON framing. Default 8192. */ + maxBytes?: number; + /** Add source API names and inline warnings when a field may have lost qualifiers. */ + includeProvenance?: boolean; + /** Requires the host's tokenizer; no character-to-token approximation is used. */ + maxTokens?: number; + countTokens?: (text: string) => number; +} + +export interface PromptProjection { + /** JSONL data, not instructions. Inject this text, not the original RecallResult. */ + text: string; + bytes: number; + tokens?: number; + truncated: boolean; +} + +type PromptRecord = { rawSource?: "getRawContent"; source?: "getSummary" | "getChunks"; updateIndex?: number; incomplete?: boolean; warning?: string; kind: "summary" | "chunk"; memoryId: string; text: string; field?: string; chunkId?: string; type?: string }; +const encoder = new TextEncoder(); + +const incompleteWarning = "Incomplete excerpt; qualifiers may be missing. Retrieve the source before relying on it."; +function* records(result: RecallResult, provenance: boolean): Generator { + if (!result.matched || !result.topMatch) return; + const summary = result.summaryIfLoaded; + if (summary) { + const source = provenance ? { source: "getSummary" as const, rawSource: "getRawContent" as const, + warning: "Derived summary; original constraints may be missing. Consult raw content.", + ...(result.familiarityLevel === "weak" ? { incomplete: true, warning: incompleteWarning } : {}) } : {}; + for (const field of ["description", "context", "architectureOrIntent"] as const) { + if (summary[field]) yield { ...source, kind: "summary", memoryId: summary.id, field, text: summary[field] }; + } + for (const [updateIndex, text] of summary.recentUpdates.entries()) { + if (text) yield { ...source, kind: "summary", memoryId: summary.id, field: "recentUpdate", ...(provenance ? { updateIndex } : {}), text }; + } + } + for (const chunk of result.chunksIfLoaded) { + if (chunk.content) yield { ...(provenance ? { source: "getChunks" as const, rawSource: "getRawContent" as const } : {}), kind: "chunk", memoryId: chunk.memoryId, chunkId: chunk.chunkId, type: chunk.type, text: chunk.content }; + } +} + +/** Allowlisted, bounded data projection. It does not sanitize prompt injection in memory text. */ +export function projectRecallForPrompt(result: RecallResult, options: PromptProjectionOptions = {}): PromptProjection { + const maxBytes = boundedInteger("maxBytes", options.maxBytes ?? 8192, 1_048_576); + const maxTokens = options.maxTokens === undefined ? undefined : boundedInteger("maxTokens", options.maxTokens, 1_048_576); + if (maxTokens !== undefined && typeof options.countTokens !== "function") { + throw new TypeError("maxTokens requires countTokens for the target model"); + } + const count = (text: string) => { + if (!options.countTokens) return undefined; + const tokens = options.countTokens(text); + if (!Number.isSafeInteger(tokens) || tokens < 0) throw new RangeError("countTokens must return a non-negative safe integer"); + return tokens; + }; + const fits = (text: string) => encoder.encode(text).length <= maxBytes && + (maxTokens === undefined || count(text)! <= maxTokens); + let text = ""; + let truncated = Boolean(options.includeProvenance && result.summaryIfLoaded && result.familiarityLevel === "weak"); + if (!fits(text)) throw new RangeError("token budget cannot represent empty text"); + for (const record of records(result, options.includeProvenance === true)) { + // Bound temporary allocation too, even for oversized adapter-supplied fields. + const { text: content, ...header } = record; + if (Object.values(header).some((value) => typeof value === "string" && value.length > maxBytes)) { truncated = true; break; } + const points: string[] = []; + let prefixLength = 0; + for (const point of content) { + if (points.length >= maxBytes) break; + points.push(point); + prefixLength += point.length; + } + const render = (length: number) => text + (text ? "\n" : "") + JSON.stringify({ ...header, + ...(options.includeProvenance && (length < points.length || prefixLength < content.length) + ? { incomplete: true, warning: incompleteWarning } : {}), text: points.slice(0, length).join("") }); + const full = render(points.length); + if (fits(full)) { + text = full; + if (prefixLength < content.length) { truncated = true; break; } + continue; + } + // Tokenizers need not be monotonic: this chooses a fitting prefix, not + // necessarily the largest one. Every accepted candidate is checked in full. + let low = 1; + let high = points.length; + let accepted = ""; + while (low <= high) { + const mid = Math.floor((low + high) / 2); + const candidate = render(mid); + if (fits(candidate)) { accepted = candidate; low = mid + 1; } + else high = mid - 1; + } + if (accepted) text = accepted; + truncated = true; + break; + } + const tokens = count(text); + // Also guards against a stateful or inconsistent host tokenizer. + if (maxTokens !== undefined && tokens! > maxTokens) throw new RangeError("countTokens returned inconsistent results"); + return { text, bytes: encoder.encode(text).length, ...(tokens === undefined ? {} : { tokens }), truncated }; +} diff --git a/src/storage/in-memory-storage.ts b/src/storage/in-memory-storage.ts index 087f8db..fe57cd4 100644 --- a/src/storage/in-memory-storage.ts +++ b/src/storage/in-memory-storage.ts @@ -1,6 +1,19 @@ import type { FamiliarityRecord, MemoryChunk, SummaryRecord, StoredMemory } from "../types/memory.js"; import type { ImpressionSearchHit, MemoryStorage } from "../types/plugins.js"; -import { lexicalSimilarity } from "../utils/text.js"; +import { extractKeywords } from "../utils/text.js"; +import { boundedInteger, MAX_RECALL_LIMIT } from "../utils/limits.js"; + +function copyFamiliarity(record: FamiliarityRecord): FamiliarityRecord { + return { ...record, impressionTokens: [...record.impressionTokens], tags: [...record.tags], embeddingVector: [...record.embeddingVector] }; +} + +function copySummary(record: SummaryRecord): SummaryRecord { + return { ...record, embeddingVector: [...record.embeddingVector], + metadata: Object.fromEntries(Object.entries(record.metadata).map(([key, value]) => [key, Array.isArray(value) ? [...value] : value])) }; +} +function copyChunks(chunks: MemoryChunk[]): MemoryChunk[] { + return chunks.map(chunk => ({ ...chunk, embeddingVector: [...chunk.embeddingVector] })); +} interface RawMemoryRecord { content: string; @@ -11,6 +24,7 @@ interface RawMemoryRecord { export class InMemoryStorage implements MemoryStorage { private readonly familiarities = new Map(); private readonly impressionTokenIndex = new Map>(); + private readonly scoringTokens = new Map>(); private readonly summaries = new Map(); private readonly chunks = new Map(); private readonly raw = new Map(); @@ -30,7 +44,8 @@ export class InMemoryStorage implements MemoryStorage { async saveFamiliarity(record: FamiliarityRecord): Promise { this.removeFromImpressionIndex(record.id); - this.familiarities.set(record.id, record); + this.familiarities.set(record.id, copyFamiliarity(record)); + this.scoringTokens.set(record.id, new Set(extractKeywords(record.impressionTokens.join(" "), 200))); for (const token of record.impressionTokens) { const ids = this.impressionTokenIndex.get(token) ?? new Set(); ids.add(record.id); @@ -39,12 +54,12 @@ export class InMemoryStorage implements MemoryStorage { } async saveSummary(record: SummaryRecord): Promise { - this.summaries.set(record.id, record); + this.summaries.set(record.id, copySummary(record)); } async saveChunks(chunks: MemoryChunk[]): Promise { if (chunks.length === 0) return; - this.chunks.set(chunks[0].memoryId, chunks); + this.chunks.set(chunks[0].memoryId, copyChunks(chunks)); } async saveRawContent(memoryId: string, content: string, createdAt: string, updatedAt: string): Promise { @@ -52,15 +67,17 @@ export class InMemoryStorage implements MemoryStorage { } async getFamiliarity(id: string): Promise { - return this.familiarities.get(id) ?? null; + const record = this.familiarities.get(id); + return record ? copyFamiliarity(record) : null; } async getSummary(id: string): Promise { - return this.summaries.get(id) ?? null; + const record = this.summaries.get(id); + return record ? copySummary(record) : null; } async getChunks(id: string): Promise { - return this.chunks.get(id) ?? []; + return copyChunks(this.chunks.get(id) ?? []); } async getRawContent(id: string): Promise { @@ -76,10 +93,12 @@ export class InMemoryStorage implements MemoryStorage { } async listFamiliarities(): Promise { - return [...this.familiarities.values()]; + return [...this.familiarities.values()].map(copyFamiliarity); } async searchImpressions(tokens: string[], topK: number): Promise { + boundedInteger("topK", topK, MAX_RECALL_LIMIT); + if (topK === 0) return []; const candidateIds = new Set(); for (const token of tokens) { for (const id of this.impressionTokenIndex.get(token) ?? []) { @@ -87,18 +106,22 @@ export class InMemoryStorage implements MemoryStorage { } } + if (candidateIds.size === 0) return []; + const queryTokens = new Set(extractKeywords(tokens.join(" "), 200)); return [...candidateIds] .map((id) => { const record = this.familiarities.get(id); if (!record) return null; - return { - record, - similarity: lexicalSimilarity(tokens.join(" "), record.impressionTokens.join(" ")), - }; + const recordTokens = this.scoringTokens.get(id)!; + let overlap = 0; + for (const token of queryTokens) if (recordTokens.has(token)) overlap++; + const denominator = Math.min(queryTokens.size, recordTokens.size); + return { record, similarity: denominator ? overlap / denominator : 0 }; }) .filter((hit): hit is ImpressionSearchHit => hit !== null) .sort((a, b) => b.similarity - a.similarity) - .slice(0, topK); + .slice(0, topK) + .map((hit) => ({ ...hit, record: copyFamiliarity(hit.record) })); } async touchMemory(id: string, lastAccessedAt: string): Promise { @@ -108,6 +131,7 @@ export class InMemoryStorage implements MemoryStorage { } private removeFromImpressionIndex(id: string): void { + this.scoringTokens.delete(id); const existing = this.familiarities.get(id); if (!existing) return; for (const token of existing.impressionTokens) { diff --git a/src/storage/in-memory-vector-store.ts b/src/storage/in-memory-vector-store.ts index 15f23e7..8f2f747 100644 --- a/src/storage/in-memory-vector-store.ts +++ b/src/storage/in-memory-vector-store.ts @@ -1,5 +1,6 @@ import type { FamiliarityRecord, MemoryChunk } from "../types/memory.js"; import type { ChunkVectorStore, FamiliarityVectorStore, VectorSearchHit } from "../types/plugins.js"; +import { boundedInteger, MAX_RECALL_LIMIT } from "../utils/limits.js"; import { cosineSimilarity } from "../utils/math.js"; export class InMemoryFamiliarityVectorStore implements FamiliarityVectorStore { @@ -14,6 +15,8 @@ export class InMemoryFamiliarityVectorStore implements FamiliarityVectorStore { } async search(vector: number[], topK: number): Promise { + boundedInteger("topK", topK, MAX_RECALL_LIMIT); + if (topK === 0) return []; return [...this.records.values()] .map((record) => ({ id: record.id, @@ -37,6 +40,8 @@ export class InMemoryChunkVectorStore implements ChunkVectorStore { } async searchByMemoryId(memoryId: string, vector: number[], topK: number): Promise { + boundedInteger("topK", topK, MAX_RECALL_LIMIT); + if (topK === 0) return []; return (this.chunksByMemoryId.get(memoryId) ?? []) .map((chunk) => ({ id: chunk.chunkId, diff --git a/src/types/memory.ts b/src/types/memory.ts index 8e2d40c..29d17a0 100644 --- a/src/types/memory.ts +++ b/src/types/memory.ts @@ -108,7 +108,10 @@ export interface ImpressionScanResult { export interface RecallInput { text: string; + /** Final candidate output limit (default 5, maximum 1000). */ topK?: number; + /** Opt-in search breadth before ranking; defaults to topK and must be >= topK. */ + candidateLimit?: number; loadChunks?: boolean; chunkLimit?: number; } diff --git a/src/types/plugins.ts b/src/types/plugins.ts index 2932a5c..893f997 100644 --- a/src/types/plugins.ts +++ b/src/types/plugins.ts @@ -8,8 +8,16 @@ import type { StoredMemory, } from "./memory.js"; +export interface EmbeddingScope { + embed(text: string): Promise; + /** Must synchronously release this scope and prevent late cache insertion; must not throw. */ + dispose(): void; +} + export interface EmbeddingProvider { embed(text: string): Promise; + /** Optional independent cache ownership; implementations must not share entries across scopes. */ + createScope?(): EmbeddingScope; } export interface Chunker { @@ -92,6 +100,14 @@ export interface SemanticRecallEngineConfig { scoringStrategy?: ScoringStrategy; chunker?: Chunker; summaryGenerator?: SummaryGenerator; + /** Opt in to preserving summaries/chunks/vectors for importance/metadata-only updates. */ + preserveDerivedOnMetadataUpdate?: boolean; + /** Exact, bounded, host-authored query routing. No automatic translation. */ + cueAliases?: readonly import("../retrieval/cue-aliases.js").CueAlias[]; + /** Opt in to per-memory/revision cache ownership and serialized reads; requires provider.createScope. */ + manageEmbeddingCache?: boolean; + /** Serialize this engine instance's reads with writes; adapters shared by other engines need transactions. */ + consistentReads?: boolean; thresholds?: { strong: number; weak: number; diff --git a/src/utils/limits.ts b/src/utils/limits.ts new file mode 100644 index 0000000..b15ca1f --- /dev/null +++ b/src/utils/limits.ts @@ -0,0 +1,9 @@ +/** Resource limits are validated before adapters are called; zero means no results. */ +export const MAX_RECALL_LIMIT = 1000; + +export function boundedInteger(name: string, value: number, max: number, min = 0): number { + if (!Number.isSafeInteger(value) || value < min || value > max) { + throw new RangeError(`${name} must be an integer between ${min} and ${max}`); + } + return value; +} diff --git a/tests/cli-errors.test.ts b/tests/cli-errors.test.ts new file mode 100644 index 0000000..7447239 --- /dev/null +++ b/tests/cli-errors.test.ts @@ -0,0 +1,68 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { CliResponseError, parseExpectedCliFailure, requireSpawnResult } from "./helpers/cli-errors.js"; + +for (const code of ["EPERM", "EACCES", "ENOENT", "ETIMEDOUT", "ENOBUFS"]) { + test(`expected-failure helper preserves original ${code}, even with misleading status/output`, () => { + for (const status of [0, 1, null]) { + const cause = new Error("original cause"); + const original = Object.assign(new Error("spawn failed", { cause }), { + code, status, stdout: status === 1 ? '{"ok":false}' : "", stderr: "synthetic stderr", + path: "synthetic-node", syscall: "spawnSync", spawnargs: ["synthetic-cli"], + }); + assert.throws(() => parseExpectedCliFailure(original), error => error === original); + assert.equal(original.cause, cause); + } + }); +} + +test("ordinary nonzero CLI JSON remains available for unchanged product assertions", () => { + for (const status of [1, 2]) { + const error = Object.assign(new Error("nonzero exit"), { status, signal: null, stdout: '{"ok":false,"diagnostics":[{"code":"invalid"}]}' }); + assert.deepEqual(parseExpectedCliFailure(error), { ok: false, diagnostics: [{ code: "invalid" }] }); + } +}); + +test("invalid or empty CLI JSON preserves both execution and parse errors", () => { + for (const stdout of ["", " \n", "{broken", "synthetic-sensitive-output"]) { + const original = Object.assign(new Error("nonzero exit"), { status: 1, signal: null, stdout, stderr: "synthetic stderr" }); + assert.throws(() => parseExpectedCliFailure(original), error => { + assert.ok(error instanceof CliResponseError); + assert.equal(error.cause, original); assert.equal(error.exitStatus, 1); + assert.ok(error.parseError instanceof SyntaxError); + assert.ok(!error.message.includes("synthetic-sensitive-output")); return true; + }); + } +}); + +test("signals, nested operational errors and invalid statuses are not accepted as CLI responses", () => { + for (const extra of [{ signal: "SIGTERM" }, { error: new Error("transport") }, + ...[undefined, null, 0, -1, 1.5, NaN, Infinity].map(status => ({ status })), { stdout: undefined }]) { + const error = Object.assign(new Error("failed"), { status: 1, stdout: '{"ok":false}', ...extra }); + assert.throws(() => parseExpectedCliFailure(error), value => value === error); + } +}); + +test("non-execution and non-object errors retain their identity", () => { + for (const error of [new SyntaxError("original parse failure"), null, undefined, "failure", 42]) { + let caught: unknown = Symbol("not thrown"); + try { parseExpectedCliFailure(error); } catch (value) { caught = value; } + assert.equal(caught, error); + } +}); + +test("spawn-result guard preserves native errors regardless of status and leaves product output unchanged", () => { + const original = Object.assign(new Error("native spawn error"), { code: "EPERM", syscall: "spawnSync", path: "synthetic-node" }); + for (const status of [0, 1, null]) { + assert.throws(() => requireSpawnResult({ error: original, status }), error => error === original); + } + // Nonzero exit status is a product response, not a spawn error. Callers still + // assert its status and parsed output in their existing product tests. + for (const status of [0, 1]) { + const result = { error: undefined, status, stdout: '{"ok":false}' }; + assert.equal(requireSpawnResult(result), result); + } + let caught: unknown = "not thrown"; + try { requireSpawnResult({ error: null }); } catch (error) { caught = error; } + assert.equal(caught, null); // Preserve the old strict undefined guard. +}); diff --git a/tests/helpers/cli-errors.ts b/tests/helpers/cli-errors.ts new file mode 100644 index 0000000..35d6b93 --- /dev/null +++ b/tests/helpers/cli-errors.ts @@ -0,0 +1,34 @@ +/** Parse an expected CLI error response without hiding spawn/transport failures. */ +export function parseExpectedCliFailure(error: unknown): any { + if (typeof error !== "object" || error === null) throw error; + const failure = error as { status?: unknown; signal?: unknown; code?: unknown; error?: unknown; stdout?: unknown }; + // execFileSync throws for both operational errors and ordinary nonzero exits. + // Only the latter can be an expected JSON response. Keep the original object + // (code, syscall, path, args, stderr and cause) for every operational failure. + if (failure.code !== undefined || failure.error !== undefined || + (failure.signal !== null && failure.signal !== undefined) || + typeof failure.status !== "number" || !Number.isInteger(failure.status) || failure.status <= 0 || + typeof failure.stdout !== "string") { + throw error; + } + try { + return JSON.parse(failure.stdout); + } catch (parseError) { + throw new CliResponseError(failure.status, error, parseError); + } +} + +export class CliResponseError extends Error { + constructor(readonly exitStatus: number, cause: unknown, readonly parseError: unknown) { + // Do not copy arbitrary CLI stdout/stderr into the message. The original + // execution error remains available to the test runner through cause. + super(`CLI exited with status ${exitStatus} without a valid JSON response`, { cause }); + this.name = "CliResponseError"; + } +} + +/** Preserve a native spawn error instead of replacing it with an assertion error. */ +export function requireSpawnResult(result: T): T { + if (result.error !== undefined) throw result.error; + return result; +} diff --git a/tests/memory-cli.test.ts b/tests/memory-cli.test.ts index 5550cab..18d736b 100644 --- a/tests/memory-cli.test.ts +++ b/tests/memory-cli.test.ts @@ -4,6 +4,7 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { parseExpectedCliFailure } from "./helpers/cli-errors.js"; const root = process.cwd(); const unifiedCli = join(root, "scripts", "deja-vu.mjs"); @@ -19,10 +20,7 @@ function runJsonAllowFailure(script: string, args: string[], cwd: string) { try { return runJson(script, args, cwd); } catch (error) { - if (typeof error === "object" && error !== null && "stdout" in error) { - return JSON.parse(String((error as { stdout: string }).stdout)); - } - throw error; + return parseExpectedCliFailure(error); } } diff --git a/tests/package-pack.test.ts b/tests/package-pack.test.ts index 6ebfadc..15c3800 100644 --- a/tests/package-pack.test.ts +++ b/tests/package-pack.test.ts @@ -17,7 +17,7 @@ test("package metadata exposes memory CLI binaries and starter kit", () => { const packRoot = mkdtempSync(join(tmpdir(), "dejavu-pack-")); const cache = join(packRoot, "npm-cache"); const result = JSON.parse( - execFileSync(process.execPath, [npmExecPath, "pack", "--json", "--pack-destination", packRoot], { + execFileSync(process.execPath, [npmExecPath, "pack", "--offline", "--json", "--pack-destination", packRoot], { cwd: root, encoding: "utf8", env: { ...process.env, npm_config_cache: cache }, @@ -46,7 +46,7 @@ test("package metadata exposes memory CLI binaries and starter kit", () => { const tarball = join(packRoot, result[0].filename); execFileSync( process.execPath, - [npmExecPath, "install", "--prefix", installRoot, "--ignore-scripts", "--no-audit", "--no-fund", tarball], + [npmExecPath, "install", "--offline", "--prefix", installRoot, "--ignore-scripts", "--no-audit", "--no-fund", tarball], { cwd: packRoot, encoding: "utf8", env: { ...process.env, npm_config_cache: cache } }, ); diff --git a/tests/recall-efficiency.test.ts b/tests/recall-efficiency.test.ts new file mode 100644 index 0000000..0a68399 --- /dev/null +++ b/tests/recall-efficiency.test.ts @@ -0,0 +1,350 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + createInMemorySemanticRecallEngine, InMemoryStorage, InMemoryChunkVectorStore, + InMemoryFamiliarityVectorStore, CachedEmbeddingProvider, MockEmbeddingProvider, + DefaultChunker, DefaultSummaryGenerator, projectRecallForPrompt, lexicalSimilarity, + type FamiliarityRecord, type EmbeddingProvider, type RecallResult, type RecallInput, +} from "../src/index.js"; + +const now = () => new Date().toISOString(); +function familiarity(id: string, tokens: string[], importance = 0.5, lastAccessedAt = now()): FamiliarityRecord { + return { id, title: id, shortSummary: id, tags: [], impressionTokens: tokens, importance, lastAccessedAt, embeddingVector: [1] }; +} +class CountingProvider extends MockEmbeddingProvider { + calls = 0; + async embed(text: string): Promise { this.calls++; return super.embed(text); } +} + +test("eligible lexical match cannot be hidden by a recent ineligible hybrid winner", async () => { + const storage = new InMemoryStorage(); + const query = Array.from({ length: 10 }, (_, i) => `aa${i}`); + await storage.saveFamiliarity(familiarity("perfect-old", query, 0, "2000-01-01T00:00:00Z")); + await storage.saveFamiliarity(familiarity("partial-new", [...query.slice(0, 7), "xx1", "xx2", "xx3"], 1)); + const engine = createInMemorySemanticRecallEngine({ storage }); + const scan = await engine.scanImpressions(query.join(" ")); + assert.equal(scan.familiarityLevel, "strong"); + assert.equal(scan.topMatch?.id, "perfect-old"); + assert.equal(scan.candidates[1].familiarityLevel, "none"); + assert.ok(scan.candidates[1].score > scan.candidates[0].score); + assert.equal((await engine.recall(query.join(" "))).matched, true); +}); + +test("candidateLimit widens search without widening output; default breadth stays compatible", async (t) => { + // These candidates must tie on recency as well as lexical score. Wall-clock + // changes during setup otherwise make the tie0 assertion timing-dependent. + t.mock.timers.enable({ apis: ["Date"], now: new Date("2026-01-01T00:00:00Z") }); + const storage = new InMemoryStorage(); + for (let i = 0; i < 6; i++) await storage.saveFamiliarity(familiarity(`tie${i}`, ["alpha"], i === 5 ? 1 : 0)); + const engine = createInMemorySemanticRecallEngine({ storage }); + assert.equal((await engine.scanImpressions("alpha")).topMatch?.id, "tie0"); + const wider = await engine.scanImpressions({ text: "alpha", topK: 1, candidateLimit: 6 }); + assert.equal(wider.topMatch?.id, "tie5"); + assert.equal(wider.candidates.length, 1); + const none = await engine.scanImpressions("absent"); + assert.equal(none.matched, false); + assert.equal(none.topMatch, null); +}); + +test("hybrid order remains intact among eligible weak and strong candidates", async () => { + const storage = new InMemoryStorage(); + const query = Array.from({ length: 10 }, (_, i) => `aa${i}`); + await storage.saveFamiliarity(familiarity("strong", query, 0, "2000-01-01T00:00:00Z")); + await storage.saveFamiliarity(familiarity("weak", [...query.slice(0, 8), "xx1", "xx2"], 1)); + const scan = await createInMemorySemanticRecallEngine({ storage }).scanImpressions(query.join(" ")); + assert.equal(scan.topMatch?.id, "weak"); + assert.equal(scan.familiarityLevel, "weak"); +}); + +for (const limit of [-1, NaN, Infinity, -Infinity, 1.5, 1001, Number.MAX_SAFE_INTEGER]) { + test(`rejects invalid recall limits before I/O: ${limit}`, async () => { + const engine = createInMemorySemanticRecallEngine({ storage: new Proxy(new InMemoryStorage(), { + get() { throw new Error("unexpected storage access"); }, + }) }); + for (const key of ["topK", "candidateLimit", "chunkLimit"] as const) { + const query: RecallInput = { text: "alpha", [key]: limit }; + await assert.rejects(engine.scanImpressions(query), RangeError); + await assert.rejects(engine.recall(query), RangeError); + } + await assert.rejects(new InMemoryStorage().searchImpressions([], limit), RangeError); + await assert.rejects(new InMemoryFamiliarityVectorStore().search([1], limit), RangeError); + await assert.rejects(new InMemoryChunkVectorStore().searchByMemoryId("id", [1], limit), RangeError); + }); +} + +test("zero limits are empty and avoid unnecessary embedding; inconsistent breadth rejects", async () => { + const provider = new CountingProvider(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: provider }); + await engine.addMemory({ id: "alpha", title: "alpha", content: "alpha details" }); + provider.calls = 0; + const empty = await engine.recall({ text: "alpha", topK: 0, loadChunks: true }); + assert.equal(empty.matched, false); + assert.deepEqual(empty.candidates, []); + const summary = await engine.recall({ text: "alpha", loadChunks: true, chunkLimit: 0 }); + assert.ok(summary.summaryIfLoaded); + assert.deepEqual(summary.chunksIfLoaded, []); + assert.equal(provider.calls, 0); + await assert.rejects(engine.scanImpressions({ text: "alpha", topK: 5, candidateLimit: 4 }), RangeError); + await assert.rejects(engine.scanImpressions({ text: 42 } as unknown as RecallInput), TypeError); +}); + +test("thresholds and chunk sizes reject unsafe numeric inputs without retuning defaults", () => { + for (const thresholds of [{ weak: NaN, strong: 1 }, { weak: 0.9, strong: 0.8 }, { weak: -1, strong: 1 }, { weak: 0, strong: Infinity }]) { + assert.throws(() => createInMemorySemanticRecallEngine({ thresholds }), RangeError); + } + const chunker = new DefaultChunker(); + for (const value of [-1, 0, Infinity, NaN, 1.5, 1_000_001]) { + assert.throws(() => chunker.chunk({ title: "test", content: "alpha", chunkStrategy: { maxCharacters: value } }), RangeError); + } + assert.throws(() => chunker.chunk({ title: "test", content: "alpha", chunkStrategy: { maxCharacters: 10, overlapCharacters: 10 } }), RangeError); +}); + +test("cached token scoring equals legacy lexical scoring across multilingual inputs", async () => { + const storage = new InMemoryStorage(); + const tokenSets = [ + ["alpha", "beta", "beta"], ["設定同步", "資料"], ["café", "naïve"], + ["東京", "sync"], ["Über", "STRASSE", "sync"], ["e\u0301", "é", "sync"], + ["a", "b", "sync"], ["alpha-beta", "sync"], + ]; + for (const [i, tokens] of tokenSets.entries()) await storage.saveFamiliarity(familiarity(`r${i}`, tokens)); + for (const query of [...tokenSets, ["sync"], ["設定同步", "sync"], ["missing"], []]) { + const expected = tokenSets.map((tokens, i) => ({ id: `r${i}`, tokens })) + .filter((r) => query.some((t) => r.tokens.includes(t))) + .map((r) => ({ id: r.id, similarity: lexicalSimilarity(query.join(" "), r.tokens.join(" ")) })) + .sort((a, b) => b.similarity - a.similarity); + const actual = await storage.searchImpressions(query, 100); + // For equal scores, postings order (first query token, then insertion) is preserved. + assert.deepEqual(actual.map((h) => ({ id: h.record.id, similarity: h.similarity })).sort((a, b) => a.id.localeCompare(b.id)), + expected.sort((a, b) => a.id.localeCompare(b.id))); + } +}); + +test("token cache is replaced on save, removed on delete, and isolated from returned mutations", async () => { + const storage = new InMemoryStorage(); + const original = familiarity("id", ["old", "設定同步"]); + await storage.saveFamiliarity(original); + original.impressionTokens.push("external"); + const returned = (await storage.searchImpressions(["old"], 5))[0].record; + returned.impressionTokens.push("mutated"); + (await storage.getFamiliarity("id"))!.impressionTokens.push("getter"); + (await storage.listFamiliarities())[0].impressionTokens.push("list"); + assert.deepEqual((await storage.getFamiliarity("id"))!.impressionTokens, ["old", "設定同步"]); + await storage.saveFamiliarity(familiarity("id", ["new", "東京"])); + assert.deepEqual(await storage.searchImpressions(["old", "設定同步"], 5), []); + assert.equal((await storage.searchImpressions(["new", "東京"], 5))[0].similarity, 1); + await storage.deleteMemory("id"); + assert.deepEqual(await storage.searchImpressions(["new", "東京"], 5), []); + await storage.saveFamiliarity(familiarity("id", ["replacement"])); + assert.deepEqual(await storage.searchImpressions(["new"], 5), []); +}); + +async function richRecall(): Promise { + const engine = createInMemorySemanticRecallEngine(); + await engine.addMemory({ id: "synthetic", title: "settings sync", content: "Decision: settings sync. " + "設定同步 café 😀資料 ".repeat(150), + metadata: { privateExtra: "DO_NOT_PROJECT" }, recentUpdates: ["更新😀".repeat(1000)] }); + return engine.recall({ text: "settings sync", loadChunks: true }); +} + +test("prompt projection allowlists data, enforces whole-output bytes and preserves Unicode", async () => { + const result = await richRecall(); + const original = JSON.stringify(result); + const projection = projectRecallForPrompt(result, { maxBytes: 512 }); + assert.ok(projection.truncated); + assert.ok(projection.bytes <= 512); + assert.equal(projection.bytes, Buffer.byteLength(projection.text)); + assert.ok(!projection.text.includes("embeddingVector")); + assert.ok(!projection.text.includes("DO_NOT_PROJECT")); + assert.ok(!projection.text.includes("candidates")); + for (const line of projection.text.split("\n")) { + const record = JSON.parse(line); + assert.ok(["summary", "chunk"].includes(record.kind)); + assert.ok(!/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + const result = await richRecall(); + const countTokens = (text: string) => Array.from(text).length; + for (const maxTokens of [0, 1, 80, 250]) { + const projected = projectRecallForPrompt(result, { maxBytes: 500, maxTokens, countTokens }); + assert.ok(projected.tokens! <= maxTokens); + assert.equal(projected.tokens, countTokens(projected.text)); + assert.ok(Buffer.byteLength(projected.text) <= 500); + } + assert.equal(projectRecallForPrompt(result, { maxBytes: 0 }).text, ""); + assert.equal(projectRecallForPrompt(result, { maxBytes: 1 }).text, ""); + result.summaryIfLoaded!.id = "x".repeat(100_000); + assert.equal(projectRecallForPrompt(result, { maxBytes: 100 }).text, ""); + assert.throws(() => projectRecallForPrompt(result, { maxTokens: 10 }), TypeError); + for (const value of [-1, NaN, Infinity, 0.5, 1_048_577]) { + assert.throws(() => projectRecallForPrompt(result, { maxBytes: value }), RangeError); + assert.throws(() => projectRecallForPrompt(result, { maxTokens: value, countTokens }), RangeError); + } + assert.throws(() => projectRecallForPrompt(result, { countTokens: () => NaN }), RangeError); + const empty = await createInMemorySemanticRecallEngine().recall("absent"); + assert.deepEqual(projectRecallForPrompt(empty), { text: "", bytes: 0, truncated: false }); +}); + +test("opt-in metadata updates preserve vectors/chunk IDs and use zero generation calls", async () => { + const provider = new CountingProvider(); + const storage = new InMemoryStorage(); + const engine = createInMemorySemanticRecallEngine({ storage, embeddingProvider: provider, preserveDerivedOnMetadataUpdate: true }); + await engine.addMemory({ id: "id", title: "alpha", content: "Decision: alpha. " + "details ".repeat(100) }); + const before = await storage.getMemorySnapshot("id"); + provider.calls = 0; + await engine.updateMemory("id", { importance: 0.9, metadata: { source: "synthetic", title: "cannot override" } }); + const after = await storage.getMemorySnapshot("id"); + assert.equal(provider.calls, 0); + assert.deepEqual(after!.chunks, before!.chunks); + assert.deepEqual(after!.summary.embeddingVector, before!.summary.embeddingVector); + assert.equal(after!.createdAt, before!.createdAt); + assert.equal(after!.familiarity.importance, 0.9); + assert.equal(after!.summary.metadata.title, "alpha"); + assert.equal(after!.summary.metadata.source, "synthetic"); + await engine.updateMemory("id", { content: "beta replacement", title: "beta" }); + assert.ok(provider.calls > 0); + assert.equal((await engine.recall("alpha")).matched, false); + assert.equal((await engine.recall("beta")).matched, true); + await engine.deleteMemory("id"); + assert.equal((await engine.recall("beta")).matched, false); +}); + +test("default metadata updates still invoke custom generators", async () => { + const provider = new CountingProvider(); + const generator = new DefaultSummaryGenerator(); + let calls = 0; + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: provider, summaryGenerator: { + generateShortSummary: (input) => generator.generateShortSummary(input), + generateStructuredSummary: async (input) => { calls++; return generator.generateStructuredSummary(input); }, + } }); + await engine.addMemory({ id: "id", title: "alpha", content: "alpha" }); + provider.calls = 0; calls = 0; + await engine.updateMemory("id", { metadata: { value: "new" } }); + assert.equal(calls, 1); + assert.ok(provider.calls > 0); +}); + +test("metadata fast path retains compensated rollback", async () => { + const storage = new InMemoryStorage(); + let armed = false; + const adapter = new Proxy(storage, { get(target, key) { + const value = Reflect.get(target, key); + if (key === "saveSummary") return async (...args: Parameters) => { + if (armed) { armed = false; throw new Error("injected write failure"); } + return target.saveSummary(...args); + }; + return typeof value === "function" ? value.bind(target) : value; + } }); + const engine = createInMemorySemanticRecallEngine({ storage: adapter, preserveDerivedOnMetadataUpdate: true }); + await engine.addMemory({ id: "id", title: "alpha", content: "alpha" }); + const before = await storage.getMemorySnapshot("id"); + armed = true; + await assert.rejects(engine.updateMemory("id", { importance: 0.9 }), /previous state restored/); + assert.deepEqual(await storage.getMemorySnapshot("id"), before); + assert.equal((await engine.recall("alpha")).matched, true); +}); + +test("embedding cache keys exact content and model version, isolates vector mutation and evicts LRU", async () => { + let version = "provider/model:v1"; + let calls = 0; + const provider: EmbeddingProvider = { async embed(text) { calls++; return [text.length, calls]; } }; + const cache = new CachedEmbeddingProvider(provider, { modelKey: () => version, maxEntries: 2 }); + const first = await cache.embed("設定同步"); first[0] = 999; + assert.notEqual((await cache.embed("設定同步"))[0], 999); + assert.equal(calls, 1); + await cache.embed("alpha"); await cache.embed("設定同步"); await cache.embed("beta"); + await cache.embed("alpha"); assert.equal(calls, 4); // alpha was LRU + version = "provider/model:v2"; + await cache.embed("alpha"); assert.equal(calls, 5); + await cache.embed("Alpha"); assert.equal(calls, 6); + cache.clear(); await cache.embed("alpha"); assert.equal(calls, 7); +}); + +test("embedding cache bounds bytes, bypasses oversized entries and does not cache failures", async () => { + let calls = 0; + // SHA-256 hex digest accounts for 128 bytes; two vector values add 16. + // One entry fits exactly. Oversized vectors still bypass retention. + const cache = new CachedEmbeddingProvider({ async embed(text) { calls++; return text.length > 10 ? Array(100).fill(1) : [1, 2]; } }, { modelKey: "v1", maxBytes: 144 }); + await cache.embed("a"); await cache.embed("a"); assert.equal(calls, 1); + await cache.embed("b"); await cache.embed("a"); assert.equal(calls, 3); + await cache.embed("x".repeat(100)); await cache.embed("x".repeat(100)); assert.equal(calls, 5); + let attempt = 0; + const failing = new CachedEmbeddingProvider({ async embed() { if (++attempt === 1) throw new Error("retry"); return [1]; } }, { modelKey: "v1" }); + await assert.rejects(failing.embed("same"), /retry/); + await failing.embed("same"); await failing.embed("same"); assert.equal(attempt, 2); + for (const vector of [[], [NaN], [Infinity], new Array(3)]) { + const invalid = new CachedEmbeddingProvider({ async embed() { return vector; } }, { modelKey: "v1" }); + await assert.rejects(invalid.embed("x"), TypeError); + } + for (const value of [-1, NaN, Infinity, 1.5, Number.MAX_SAFE_INTEGER]) { + assert.throws(() => new CachedEmbeddingProvider(new MockEmbeddingProvider(), { modelKey: "v1", maxEntries: value }), RangeError); + assert.throws(() => new CachedEmbeddingProvider(new MockEmbeddingProvider(), { modelKey: "v1", maxBytes: value }), RangeError); + } + assert.throws(() => new CachedEmbeddingProvider(new MockEmbeddingProvider(), { modelKey: "" }), TypeError); +}); + +test("clear or model changes while embedding is pending cannot populate an obsolete cache", async () => { + let complete!: (vector: number[]) => void; + let calls = 0; + let model = "v1"; + const cache = new CachedEmbeddingProvider({ embed: async () => { + calls++; + if (calls <= 2) return new Promise((resolve) => { complete = resolve; }); + return [calls]; + } }, { modelKey: () => model }); + const pending = cache.embed("text"); cache.clear(); complete([1]); await pending; + const pending2 = cache.embed("text"); model = "v2"; complete([2]); await pending2; + model = "v1"; await cache.embed("text"); assert.equal(calls, 3); +}); + +test("cached embeddings do not resurrect updated or deleted memory", async () => { + const provider = new CountingProvider(); + const cache = new CachedEmbeddingProvider(provider, { modelKey: "mock:v1", maxEntries: 32 }); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache }); + await engine.addMemory({ id: "id", title: "alpha", content: "alpha" }); + provider.calls = 0; + await engine.recall({ text: "alpha", loadChunks: true }); + await engine.recall({ text: "alpha", loadChunks: true }); + assert.ok(provider.calls <= 1); + await engine.updateMemory("id", { title: "beta", content: "beta" }); + assert.equal((await engine.recall("alpha")).matched, false); + assert.equal((await engine.recall({ text: "beta", loadChunks: true })).chunksIfLoaded[0].content, "beta"); + await engine.deleteMemory("id"); + assert.equal((await engine.recall("beta")).matched, false); +}); + +test("cached scoring preserves exact scores and posting tie order on seeded mixed-language queries", async () => { + let seed = 42; + const pick = (n: number) => { seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; return seed % n; }; + const terms = ["alpha", "beta", "café", "設定同步", "東京", "данные", "sync", "c++", "a", "e\u0301"]; + const storage = new InMemoryStorage(); + const rows = Array.from({ length: 40 }, (_, i) => familiarity(`id${i}`, Array.from({ length: 12 }, () => terms[pick(terms.length)]))); + for (const row of rows) await storage.saveFamiliarity(row); + for (let i = 0; i < 100; i++) { + const query = Array.from({ length: 6 }, () => terms[pick(terms.length)]); + const ids = new Set(); + for (const token of query) for (const row of rows) if (row.impressionTokens.includes(token)) ids.add(row.id); + const expected = [...ids].map((id) => { + const row = rows.find((r) => r.id === id)!; + return [id, lexicalSimilarity(query.join(" "), row.impressionTokens.join(" "))] as const; + }).sort((a, b) => b[1] - a[1]).slice(0, 5); + const actual = (await storage.searchImpressions(query, 5)).map((hit) => [hit.record.id, hit.similarity]); + assert.deepEqual(actual, expected); + } +}); + +test("concurrent embedding misses remain bounded and disabled caches never retain entries", async () => { + let calls = 0; + const provider = { async embed(text: string) { calls++; await Promise.resolve(); return [text.length]; } }; + const disabled = new CachedEmbeddingProvider(provider, { modelKey: "v1", maxEntries: 0 }); + await disabled.embed("a"); await disabled.embed("a"); assert.equal(calls, 2); + const cache = new CachedEmbeddingProvider(provider, { modelKey: "v1", maxEntries: 1 }); + await Promise.all([cache.embed("a"), cache.embed("a"), cache.embed("b")]); + const before = calls; + await cache.embed("b"); assert.equal(calls, before); + await cache.embed("a"); assert.equal(calls, before + 1); +}); diff --git a/tests/recall-lifecycle.test.ts b/tests/recall-lifecycle.test.ts new file mode 100644 index 0000000..5e94484 --- /dev/null +++ b/tests/recall-lifecycle.test.ts @@ -0,0 +1,295 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { + CachedEmbeddingProvider, createInMemorySemanticRecallEngine, InMemoryStorage, + InMemoryChunkVectorStore, InMemoryFamiliarityVectorStore, MemoryWriteError, + MemorySupersedeError, projectRecallForPrompt, type EmbeddingProvider, type RecallResult, +} from "../src/index.js"; + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(r => { resolve = r; }); + return { promise, resolve }; +} +function counted() { + let calls = 0; + const cache = new CachedEmbeddingProvider({ async embed(text) { calls++; return [text.length, 1]; } }, { modelKey: "synthetic:v1" }); + return { cache, calls: () => calls }; +} +function fault(target: T, method: keyof T, shouldFail: () => boolean): T { + return new Proxy(target, { get(object, key) { + const value = Reflect.get(object, key); + if (key === method) return async (...args: unknown[]) => { + if (shouldFail()) throw new Error(`injected ${String(method)}`); + if (typeof value !== "function") throw new TypeError("fault target must be callable"); + return value.apply(object, args); + }; + return typeof value === "function" ? value.bind(object) : value; + } }); +} + +test("aliases are bounded exact queries, copied at construction, and opt-in", async () => { + const aliases = [{ alias: "還原備份", cue: "restoreroute" }, { alias: "restore archive", cue: "restoreroute" }]; + const engine = createInMemorySemanticRecallEngine({ cueAliases: aliases }); + const plain = createInMemorySemanticRecallEngine(); + for (const e of [engine, plain]) await e.addMemory({ id: "id", title: "restoreroute", content: "synthetic" }); + aliases[0].cue = "changed"; + assert.equal((await engine.recall("還原備份")).topMatch?.id, "id"); + assert.equal((await engine.recall(" RESTORE archive ")).topMatch?.id, "id"); + assert.equal((await plain.recall("還原備份")).matched, false); + for (const query of ["不要還原備份", "do not restore archive", "restore archive price"]) { + assert.equal((await engine.recall(query)).matched, false); + } + await engine.updateMemory("id", { title: "otherroute" }); + assert.equal((await engine.recall("還原備份")).matched, false); + await engine.deleteMemory("id"); + assert.equal((await engine.recall("還原備份")).matched, false); + for (const cueAliases of [Array(129).fill({ alias: "a", cue: "b" }), [{ alias: "", cue: "b" }], + [{ alias: "a", cue: "x".repeat(257) }], [{ alias: "A", cue: "one" }, { alias: " a ", cue: "two" }]]) { + assert.throws(() => createInMemorySemanticRecallEngine({ cueAliases })); + } +}); + +test("scopes isolate identical content; disposing one owner preserves the other and the root", async () => { + const { cache, calls } = counted(); + const a = cache.createScope(), b = cache.createScope(); + await a.embed("identical"); await b.embed("identical"); await cache.embed("identical"); + assert.equal(calls(), 3); assert.equal(cache.getStats().entries, 3); + a.dispose(); a.dispose(); + assert.equal(cache.getStats().entries, 2); + await b.embed("identical"); await cache.embed("identical"); assert.equal(calls(), 3); + await assert.rejects(a.embed("identical"), /disposed/); + b.dispose(); cache.clear(); assert.deepEqual(cache.getStats(), { entries: 0, accountedBytes: 0 }); +}); + +test("disposed scopes and global clearing invalidate in-flight retention without pretending to cancel work", async () => { + const pending = deferred(); + const cache = new CachedEmbeddingProvider({ embed: () => pending.promise }, { modelKey: "v1" }); + const scope = cache.createScope(); + const call = scope.embed("pending"); scope.dispose(); pending.resolve([1]); + assert.deepEqual(await call, [1]); assert.equal(cache.getStats().entries, 0); + const pending2 = deferred(); + const second = new CachedEmbeddingProvider({ embed: () => pending2.promise }, { modelKey: "v1" }); + const scoped = second.createScope(); const call2 = scoped.embed("pending"); second.clear(); pending2.resolve([1]); + await call2; assert.equal(second.getStats().entries, 0); +}); + +test("digest keys preserve exact surrogate content and model framing; accounted bounds include digests", async () => { + let calls = 0; let model = "ab"; + const cache = new CachedEmbeddingProvider({ async embed() { return [++calls]; } }, { modelKey: () => model, maxBytes: 136 }); + await cache.embed("c"); await cache.embed("c"); assert.equal(calls, 1); + assert.equal(cache.getStats().accountedBytes, 136); + model = "a"; await cache.embed("bc"); assert.equal(calls, 2); + await cache.embed("\ud800"); await cache.embed("\ufffd"); assert.equal(calls, 4); + await cache.embed("\ufffd"); assert.equal(calls, 4); + assert.ok(cache.getStats().entries <= 1); + const disabled = new CachedEmbeddingProvider({ async embed() { return [++calls]; } }, { modelKey: "v1", maxBytes: 135 }); + await disabled.embed("large".repeat(10_000)); await disabled.embed("large".repeat(10_000)); + assert.equal(calls, 6); assert.equal(disabled.getStats().entries, 0); +}); + +test("engine ownership releases update/delete revisions without clearing another memory", async () => { + const { cache, calls } = counted(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, preserveDerivedOnMetadataUpdate: true }); + for (const id of ["a", "b"]) await engine.addMemory({ id, title: "identical", content: "identical" }); + const entries = cache.getStats().entries; + assert.ok(entries > 0 && entries % 2 === 0); + const before = calls(); await engine.updateMemory("a", { importance: 0.9 }); + assert.equal(calls(), before); assert.equal(cache.getStats().entries, entries); + await engine.updateMemory("a", { title: "changed", content: "changed" }); + assert.ok(calls() > before); assert.equal(cache.getStats().entries, entries); + await engine.deleteMemory("a"); assert.equal(cache.getStats().entries, entries / 2); + const beforeB = calls(); await engine.recall({ text: "identical", loadChunks: true }); + assert.equal(calls(), beforeB); // The chunk text is already retained in b's scope. + await engine.deleteMemory("b"); assert.equal(cache.getStats().entries, 0); + assert.equal((await engine.recall("identical")).matched, false); + assert.throws(() => createInMemorySemanticRecallEngine({ embeddingProvider: { async embed() { return [1]; } }, manageEmbeddingCache: true }), /createScope/); +}); + +test("failed preparation and failed commit dispose new scopes but retain the previous revision", async () => { + const storage = new InMemoryStorage(); let fail = false; + const { cache } = counted(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, + storage: fault(storage, "saveSummary", () => { const value = fail; fail = false; return value; }) }); + await engine.addMemory({ id: "id", title: "original", content: "original" }); + const entries = cache.getStats().entries, snapshot = await storage.getMemorySnapshot("id"); + fail = true; await assert.rejects(engine.updateMemory("id", { title: "new", content: "new" }), MemoryWriteError); + assert.equal(cache.getStats().entries, entries); + assert.deepEqual(await storage.getMemorySnapshot("id"), snapshot); + let attempts = 0; + const failureCache = new CachedEmbeddingProvider({ async embed() { if (++attempts === 2) throw new Error("provider failure"); return [1]; } }, { modelKey: "v1" }); + const other = createInMemorySemanticRecallEngine({ embeddingProvider: failureCache, manageEmbeddingCache: true }); + await assert.rejects(other.addMemory({ title: "new", content: "distinct" }), /provider failure/); + assert.equal(failureCache.getStats().entries, 0); +}); + +for (const stage of ["familiarity", "chunks", "storage"] as const) { + test(`delete compensates a ${stage} failure and retains valid cache ownership`, async () => { + const storage = new InMemoryStorage(), familiarity = new InMemoryFamiliarityVectorStore(), chunks = new InMemoryChunkVectorStore(); + const { cache } = counted(); let armed = false; + const failOnce = () => { const value = armed; armed = false; return value; }; + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, + storage: stage === "storage" ? fault(storage, "deleteMemory", failOnce) : storage, + familiarityVectorStore: stage === "familiarity" ? fault(familiarity, "remove", failOnce) : familiarity, + chunkVectorStore: stage === "chunks" ? fault(chunks, "removeByMemoryId", failOnce) : chunks }); + await engine.addMemory({ id: "id", title: "route", content: "route" }); + const snapshot = await storage.getMemorySnapshot("id"), stats = cache.getStats(); + armed = true; await assert.rejects(engine.deleteMemory("id"), (error: unknown) => { + assert.ok(error instanceof MemoryWriteError); assert.equal(error.recoveryErrors.length, 0); return true; + }); + assert.deepEqual(await storage.getMemorySnapshot("id"), snapshot); + assert.deepEqual(cache.getStats(), stats); + assert.equal((await familiarity.search(snapshot!.familiarity.embeddingVector, 5))[0].id, "id"); + assert.equal((await chunks.searchByMemoryId("id", snapshot!.chunks[0].embeddingVector, 5)).length, snapshot!.chunks.length); + assert.equal((await engine.recall({ text: "route", loadChunks: true })).chunksIfLoaded[0].content, "route"); + }); +} + +test("delete compensation errors expose a detached recovery snapshot", async () => { + const storage = new InMemoryStorage(); let armed = false; + const adapted = fault(fault(storage, "deleteMemory", () => armed), "saveSummary", () => armed); + const engine = createInMemorySemanticRecallEngine({ storage: adapted }); + await engine.addMemory({ id: "id", title: "route", content: "route" }); armed = true; + await assert.rejects(engine.deleteMemory("id"), (error: unknown) => { + assert.ok(error instanceof MemoryWriteError); assert.equal(error.recoveryErrors.length, 1); + assert.equal(error.recoverySnapshot?.rawContent, "route"); return true; + }); +}); + +test("supersede removes old cues and scopes, rejects ID collisions, and can delete the successor", async () => { + const { cache } = counted(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, consistentReads: true }); + await engine.addMemory({ id: "old", title: "obsoletecue", content: "old content" }); + await assert.rejects(engine.supersedeMemory("old", { id: "old", title: "x", content: "x" }), /exists/); + assert.deepEqual(await engine.supersedeMemory("old", { id: "new", title: "currentcue", content: "new content" }), { id: "new", supersededId: "old" }); + assert.equal((await engine.recall("obsoletecue")).matched, false); + assert.equal((await engine.recall("currentcue")).topMatch?.id, "new"); + assert.equal(await engine.getSummary("old"), null); + await engine.deleteMemory("new"); assert.equal(cache.getStats().entries, 0); +}); + +test("failed supersede restores old memory and removes the uncommitted successor", async () => { + const storage = new InMemoryStorage(); let armed = false; + const { cache } = counted(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, + storage: fault(storage, "deleteMemory", () => { const value = armed; armed = false; return value; }) }); + await engine.addMemory({ id: "old", title: "obsoletecue", content: "old content" }); + const snapshot = await storage.getMemorySnapshot("old"), entries = cache.getStats().entries; + armed = true; + await assert.rejects(engine.supersedeMemory("old", { id: "new", title: "currentcue", content: "new content" }), (error: unknown) => { + assert.ok(error instanceof MemorySupersedeError); assert.deepEqual(error.recoveryErrors, []); return true; + }); + assert.deepEqual(await storage.getMemorySnapshot("old"), snapshot); + assert.equal(await storage.getMemorySnapshot("new"), null); + assert.equal(cache.getStats().entries, entries); +}); + +test("managed ownership serializes pending recall before delete, then never returns a stale queued match", async () => { + const started = deferred(), finish = deferred(); let armed = false; + const provider: EmbeddingProvider = { async embed(text) { + if (armed && text === "route") { started.resolve(); return finish.promise; } + return [text.length, 1]; + } }; + const cache = new CachedEmbeddingProvider(provider, { modelKey: "v1" }); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true }); + await engine.addMemory({ id: "id", title: "route", content: "different content" }); + armed = true; const reading = engine.recall({ text: "route", loadChunks: true }); await started.promise; + let deleted = false; const deletion = engine.deleteMemory("id").then(() => { deleted = true; }); + const after = engine.recall("route"); await Promise.resolve(); assert.equal(deleted, false); + finish.resolve([1, 1]); assert.equal((await reading).matched, true); + await deletion; assert.equal((await after).matched, false); assert.equal(cache.getStats().entries, 0); +}); + +test("consistentReads hides an update's intermediate storage state", async () => { + const storage = new InMemoryStorage(), started = deferred(), finish = deferred(); let armed = false; + const adapted = new Proxy(storage, { get(target, key) { + if (key === "saveSummary") return async (...args: Parameters) => { + if (armed) { started.resolve(); await finish.promise; } return target.saveSummary(...args); + }; + const value = Reflect.get(target, key); return typeof value === "function" ? value.bind(target) : value; + } }); + const engine = createInMemorySemanticRecallEngine({ storage: adapted, consistentReads: true }); + await engine.addMemory({ id: "id", title: "oldcue", content: "old" }); + armed = true; const update = engine.updateMemory("id", { title: "newcue", content: "new" }); await started.promise; + const read = engine.recall("newcue"); finish.resolve(); await update; + assert.equal((await read).summaryIfLoaded?.description, "Decision: new"); +}); + +test("in-memory summaries and chunks do not expose mutable stored vectors or text", async () => { + const engine = createInMemorySemanticRecallEngine(); + await engine.addMemory({ id: "id", title: "cue", content: "original" }); + const summary = (await engine.getSummary("id"))!; summary.description = "mutated"; summary.embeddingVector[0] = 99; + const chunks = await engine.getChunks("id"); chunks[0].content = "mutated"; chunks[0].embeddingVector[0] = 99; + assert.equal((await engine.getSummary("id"))?.description, "Decision: original"); + assert.equal((await engine.getChunks("id"))[0].content, "original"); + assert.notEqual((await engine.getSummary("id"))?.embeddingVector[0], 99); +}); + +test("provenance marks clipped qualifiers inside the byte/token budget and points to full source", async () => { + const engine = createInMemorySemanticRecallEngine({ summaryGenerator: { + async generateShortSummary() { return "route"; }, + async generateStructuredSummary(input) { return { description: "Short content", context: input.context!, architectureOrIntent: "", recentUpdates: [], metadata: {} }; }, + } }); + const context = "Allowed for synthetic experiments. ".repeat(30) + "NEVER use production secrets."; + await engine.addMemory({ id: "id", title: "route", content: "Short content", context }); + const result = await engine.recall("route"); + const projected = projectRecallForPrompt(result, { maxBytes: 900, maxTokens: 900, countTokens: text => [...text].length, includeProvenance: true }); + assert.equal(projected.truncated, true); assert.ok(projected.bytes <= 900 && projected.tokens! <= 900); + const records = projected.text.split("\n").map(line => JSON.parse(line)); + const clipped = records.find(r => r.incomplete); + assert.ok(clipped); assert.match(clipped.warning, /qualifiers may be missing/); + assert.equal(clipped.rawSource, "getRawContent"); assert.equal(clipped.source, "getSummary"); assert.equal(clipped.memoryId, "id"); assert.equal(clipped.field, "context"); + assert.ok(!clipped.text.includes("NEVER")); assert.ok((await engine.getSummary(clipped.memoryId))?.context.endsWith("NEVER use production secrets.")); + assert.equal(projectRecallForPrompt(result, { maxBytes: 1, includeProvenance: true }).text, ""); + const weak = { ...result, familiarityLevel: "weak" } as RecallResult; + const weakProjection = projectRecallForPrompt(weak, { maxBytes: 8192, includeProvenance: true }); + assert.equal(weakProjection.truncated, true); + for (const row of weakProjection.text.split("\n").map(line => JSON.parse(line))) assert.equal(row.incomplete, true); +}); + +test("frozen synthetic development and held-out splits retain defaults and expose unseen paraphrase misses", async () => { + const fixture = JSON.parse(readFileSync("scripts/fixtures/recall-evaluation.json", "utf8")); + for (const split of Object.values(fixture.splits) as any[]) { + for (const enabled of [false, true]) { + const engine = createInMemorySemanticRecallEngine(enabled ? { cueAliases: split.memories.flatMap((m: any) => m.aliases.map((alias: string) => ({ alias, cue: m.id }))) } : {}); + for (const { aliases: _, ...memory } of split.memories) await engine.addMemory(memory); + let hits = 0, falseRecalls = 0; + for (const row of split.cases) { + const result = await engine.scanImpressions({ text: row.query, topK: 5, candidateLimit: 32 }); + const id = result.matched ? result.topMatch?.id : undefined; + if (id && row.relevant.includes(id)) hits++; + else if (id) falseRecalls++; + } + assert.equal(hits, enabled ? 10 : 2); assert.equal(falseRecalls, 0); + } + } +}); + +test("derived-summary provenance does not claim that an uncut summary preserves raw qualifiers", async () => { + const engine = createInMemorySemanticRecallEngine(); + const content = "Allowed experiments. ".repeat(50) + "Never use production secrets."; + await engine.addMemory({ id: "id", title: "route", content }); + const result = await engine.recall("route"); + assert.ok(!result.summaryIfLoaded!.description.includes("Never use production")); + const projection = projectRecallForPrompt(result, { includeProvenance: true }); + const record = JSON.parse(projection.text.split("\n")[0]); + assert.match(record.warning, /Derived summary/); + assert.equal(record.rawSource, "getRawContent"); + assert.equal(await engine.getRawContent(record.memoryId), content); +}); + +test("supersede cleanup failures identify both IDs and never claim a completed rollback", async () => { + const storage = new InMemoryStorage(); let failures = 0; + const { cache } = counted(); + const engine = createInMemorySemanticRecallEngine({ embeddingProvider: cache, manageEmbeddingCache: true, + storage: fault(storage, "deleteMemory", () => failures-- > 0) }); + await engine.addMemory({ id: "old", title: "oldcue", content: "original" }); failures = 2; + await assert.rejects(engine.supersedeMemory("old", { id: "new", title: "newcue", content: "replacement" }), (error: unknown) => { + assert.ok(error instanceof MemorySupersedeError); + assert.equal(error.memoryId, "old"); assert.equal(error.replacementId, "new"); + assert.equal(error.recoveryErrors.length, 1); assert.match(error.message, /adapter repair/); return true; + }); + assert.ok(await storage.getMemorySnapshot("old")); assert.ok(await storage.getMemorySnapshot("new")); + await engine.deleteMemory("old"); await engine.deleteMemory("new"); assert.equal(cache.getStats().entries, 0); +}); diff --git a/tests/reliability.test.ts b/tests/reliability.test.ts index ed4d47d..c52772b 100644 --- a/tests/reliability.test.ts +++ b/tests/reliability.test.ts @@ -1,3 +1,4 @@ +import { requireSpawnResult } from "./helpers/cli-errors.js"; import test from "node:test"; import assert from "node:assert/strict"; import * as fs from "node:fs"; @@ -180,8 +181,7 @@ test("concurrent updates retain each other's fields", async () => { const root = process.cwd(); function cli(script: string, args: string[], cwd: string) { - const result = spawnSync(process.execPath, [join(root, "scripts", script), ...args], { cwd, encoding: "utf8" }); - assert.equal(result.error, undefined); + const result = requireSpawnResult(spawnSync(process.execPath, [join(root, "scripts", script), ...args], { cwd, encoding: "utf8" })); return { status: result.status, data: JSON.parse(result.stdout) }; } function project() {