From 41505fa5ab5cd95d69f75e99b47adcf3b63c84cc Mon Sep 17 00:00:00 2001 From: forrestlinfeng Date: Wed, 21 Oct 2026 13:21:53 +0800 Subject: [PATCH 1/6] Rewrite the recall query and review candidates before injecting them With EVOMAP_API_KEY set, recall asks gemini-3.5-flash on the EvoMap gateway to rewrite the whole task (up to 6000 characters) into one English search query, fetches the Hub's top ten for it, and has the same model judge each candidate against the task (up to 4000 characters), one request per candidate sent together. Up to five accepted candidates are injected; none are injected when none is accepted. A failed rewrite searches with the 400-character prompt prefix, a failed review injects the best candidate unreviewed, and both are reported in the recall outcome and logged. Without the key recall is unchanged. Offline over the 499-task bank this cut misleading injections from 715 to 13 while keeping 384 of 406 fitting ones. Co-Authored-By: Claude Opus 5.5 --- src/index.js | 15 +++- src/prime.js | 168 ++++++++++++++++++++++++++------------ src/recall-judge.js | 117 ++++++++++++++++++++++++++ test/prime.test.js | 100 +++++++++++++++++++++++ test/recall-judge.test.js | 85 +++++++++++++++++++ 5 files changed, 428 insertions(+), 57 deletions(-) create mode 100644 src/recall-judge.js create mode 100644 test/recall-judge.test.js diff --git a/src/index.js b/src/index.js index c18b290..b50040c 100644 --- a/src/index.js +++ b/src/index.js @@ -17,6 +17,7 @@ import { noticeDue, pendingClaimUrl } from './onboarding.js'; import { hubGene, promptTextOf } from './prime.js'; import { resolveProjectDir, sessionDir } from './project-dir.js'; import { createProxyClient } from './proxy.js'; +import { API_KEY_ENV, TASK_MAX_CHARS, createRecallJudge } from './recall-judge.js'; import { createReportQueue } from './report-queue.js'; import { rememberForRetention, retentionReportsOf, takeDueRetentions } from './retention-pending.js'; import { correctionReportOf, immediateReportOf, reportedReuseOf } from './reuse.js'; @@ -85,7 +86,7 @@ function afterWait(ms) { // that carries both. Workspace memory is a session fact and seeds once; the Hub // is re-queried per turn, because each prompt is a different task — bounded by // the ids already listed, so a repeat search adds nothing the model has seen. -function primeSteps(ctx, config, primeFetch, engineSetup, recallLog, onPrompt) { +function primeSteps(ctx, config, primeFetch, judge, engineSetup, recallLog, onPrompt) { const seeded = new WeakSet(); const searchedTurn = new WeakMap(); const listedAssets = new WeakMap(); @@ -115,11 +116,15 @@ function primeSteps(ctx, config, primeFetch, engineSetup, recallLog, onPrompt) { const listed = listedFor(agent); const startedAt = Date.now(); - const search = hubGene(primeFetch, promptTextOf(claimed), { + const search = hubGene(primeFetch, promptTextOf(claimed, TASK_MAX_CHARS), { signal, listedIds: listed, minSimilarity: config.assetPrimeMinSimilarity, - onOutcome: (outcome) => recallLog.record(outcome, startedAt), + judge, + onOutcome: (outcome) => { + recallLog.record(outcome, startedAt); + if (outcome.fallbacks) ctx.logger?.warn?.(`evolver recall judge fell back: ${JSON.stringify(outcome.fallbacks)}`); + }, }); const inline = await Promise.race([search, afterWait(config.assetPrimeWaitMs ?? DEFAULT_PRIME_WAIT_MS)]); if (inline) { @@ -245,6 +250,8 @@ export function apply(ctx, config = {}) { const projectDirOf = (session) => sessionDir(session?.header?.cwd, fallbackDir); const proxyFetch = createProxyClient({ port: config.proxyPort, timeoutMs: config.proxyTimeoutMs }); const primeFetch = createProxyClient({ port: config.proxyPort, timeoutMs: config.assetPrimeTimeoutMs ?? 8_000 }); + const judge = createRecallJudge(); + if (!judge) ctx.logger?.info?.(`evolver recall judge is off: ${API_KEY_ENV} is not set, so recall searches with the prompt prefix unreviewed`); const engineSetup = createEngineSetup(() => prepareEngine({ probeVersion: installedEvolverVersion, @@ -267,7 +274,7 @@ export function apply(ctx, config = {}) { const records = createTurnRecords(); const labelable = new Map(); - primeSteps(ctx, config, primeFetch, engineSetup, recallLog, followUpReporter(deliver, labelable)); + primeSteps(ctx, config, primeFetch, judge, engineSetup, recallLog, followUpReporter(deliver, labelable)); observeTools(ctx, config.editToolNames ?? EDIT_TOOL_NAMES, records, projectDirOf); reportTurns(ctx, records, deliver, labelable, projectDirOf); onSessionStart(ctx, engineSetup, deliver, projectDirOf); diff --git a/src/prime.js b/src/prime.js index d81da7a..1f63e07 100644 --- a/src/prime.js +++ b/src/prime.js @@ -5,10 +5,15 @@ import { withoutSecrets } from './redact.js'; // One round trip, not two: since evolver 2.0.39 `/asset/fetch` recalls by text // when it is given no ids, so selecting a candidate and materialising it are the -// same call. No `limit` is sent, so the Hub's own default decides how many come -// back (two at present). +// same call. Without a judge no `limit` is sent, so the Hub's own default +// decides how many come back (two at present). const MIN_PROMPT_CHARS = 8; const PROMPT_MAX_CHARS = 400; +// Offline over 499 tasks, reviewing the Hub's top ten against the whole task +// and keeping at most five cut misleading injections from 715 to 13 while +// keeping 384 of 406 fitting ones. +const REVIEWED_CANDIDATES = 10; +const MAX_INJECTED = 5; const STEP_MAX_CHARS = 400; const TITLE_MAX_CHARS = 80; const DEFAULT_MIN_SIMILARITY = 0.3; @@ -42,7 +47,7 @@ function strategyChars(steps) { // A step's batch also carries dsh's own injections — the runtime-context // snapshot and the skill catalog — and searching with those drowns the task in // boilerplate, so only what the person typed is matched against. -export function promptTextOf(messages) { +export function promptTextOf(messages, maxChars = PROMPT_MAX_CHARS) { const parts = []; for (const message of messages ?? []) { if (message?.source?.kind !== 'user') continue; @@ -50,7 +55,7 @@ export function promptTextOf(messages) { if (block?.type === 'text' && typeof block.text === 'string' && block.text.trim()) parts.push(block.text.trim()); } } - return withoutSecrets(parts.join('\n')).trim().slice(0, PROMPT_MAX_CHARS); + return withoutSecrets(parts.join('\n')).trim().slice(0, maxChars); } function recalledAssets(data) { @@ -114,76 +119,133 @@ async function proxyJson(proxyFetch, path, body, signal) { } } + +function errorText(error) { + return String(error?.message ?? error); +} + +// A judge that fails falls back to today's recall step by step: a failed +// rewrite searches with the prompt prefix, a failed review injects the best +// candidate unreviewed. Either way the reason is reported, never swallowed. +async function searchQueryOf(judge, taskText, signal, fallbacks) { + const prefix = taskText.slice(0, PROMPT_MAX_CHARS); + if (!judge) return { query: prefix, source: 'prompt_prefix' }; + try { + return { query: await judge.searchQueryOf(taskText, signal), source: 'rewritten' }; + } catch (error) { + fallbacks.rewrite = errorText(error); + return { query: prefix, source: 'prompt_prefix' }; + } +} + +async function reviewedAssets(judge, taskText, eligible, signal, fallbacks) { + if (!judge) return { assets: eligible.slice(0, 1), review: null }; + const candidates = eligible.slice(0, REVIEWED_CANDIDATES); + try { + const { usable, failed } = await judge.usableOf(taskText, candidates.map(({ asset }) => asset), signal); + const accepted = candidates.filter(({ asset }) => usable.includes(asset)); + return { assets: accepted.slice(0, MAX_INJECTED), review: { judged: candidates.length, usable: accepted.length, failed } }; + } catch (error) { + fallbacks.review = errorText(error); + return { assets: eligible.slice(0, 1), review: null }; + } +} + +function eligibleAssets(assets, listedIds, minSimilarity, drop) { + return assets + .filter((asset) => { + const reason = rankingRejection(asset, listedIds, minSimilarity); + if (reason) drop(reason); + return !reason; + }) + .sort((left, right) => similarityOf(right) - similarityOf(left)) + .map((asset) => ({ asset, steps: strategySteps(asset) })) + .filter(({ steps }) => { + const reason = strategyRejection(steps); + if (reason) drop(reason); + return !reason; + }); +} + +function memoryText(picked) { + const ids = picked.map(({ asset }) => asset.asset_id); + const sections = picked.map(({ asset, steps }) => [ + `[Evolution Memory] ${readableNameOf(asset)} (EvoMap network):`, + ...steps.map((step, index) => `${index + 1}. ${step}`), + '', + ]); + return [ + ...sections.flat(), + 'These are the complete steps already recalled from the network, so there is nothing more to search or fetch for this task.', + `Apply them where they fit, then report the outcome with evolver_asset_reuse_result for ${ids.join(', ')}.`, + ].join('\n'); +} + // Priming sits on the critical path of every answer: an unreachable Proxy, a -// slow Hub, or a malformed body must cost the turn nothing but the deadline. -// One asset's strategy is injected and nothing else — a summary only tells the -// model that something exists, while the steps are what it can actually reuse. -// Every lookup reports why it ended through `onOutcome`, because an empty match -// alone cannot tell a Proxy error from a recall that found nothing usable. -export async function hubGene(proxyFetch, text, { +// slow Hub, a failing judge, or a malformed body must cost the turn nothing but +// the deadline. Only strategies are injected — a summary only tells the model +// that something exists, while the steps are what it can actually reuse. +// Without a judge one asset is injected; with one, up to five that it accepted +// for the whole task, or none when it accepted none. Every lookup reports why +// it ended through `onOutcome`, because an empty match alone cannot tell a +// Proxy error from a recall that found nothing usable. +export async function hubGene(proxyFetch, taskText, { signal, listedIds = new Set(), minSimilarity = DEFAULT_MIN_SIMILARITY, + judge = null, onOutcome = () => {}, } = {}) { - if (text.length < MIN_PROMPT_CHARS) { + if (taskText.length < MIN_PROMPT_CHARS) { onOutcome({ status: 'skipped', reason: 'prompt_too_short' }); return EMPTY_MATCH; } - const recalled = await proxyJson(proxyFetch, '/asset/fetch', { text }, signal); + const fallbacks = {}; + const { query, source } = await searchQueryOf(judge, taskText, signal, fallbacks); + const searched = judge ? { query_source: source, ...(source === 'rewritten' ? { query } : {}) } : {}; + const report = (outcome) => onOutcome({ ...outcome, ...searched, ...(Object.keys(fallbacks).length ? { fallbacks } : {}) }); + + const request = judge ? { text: query, limit: REVIEWED_CANDIDATES } : { text: query }; + const recalled = await proxyJson(proxyFetch, '/asset/fetch', request, signal); if (recalled.error) { - onOutcome({ status: 'proxy_error', error: recalled.error }); + report({ status: 'proxy_error', error: recalled.error }); return EMPTY_MATCH; } const assets = recalledAssets(recalled.data); if (assets.length === 0) { - onOutcome({ status: 'no_candidates' }); + report({ status: 'no_candidates' }); return EMPTY_MATCH; } const dropped = {}; - const drop = (reason) => { + const eligible = eligibleAssets(assets, listedIds, minSimilarity, (reason) => { dropped[reason] = (dropped[reason] ?? 0) + 1; - }; - const ranked = assets - .filter((asset) => { - const reason = rankingRejection(asset, listedIds, minSimilarity); - if (reason) drop(reason); - return !reason; - }) - .sort((left, right) => similarityOf(right) - similarityOf(left)); - - for (const candidate of ranked) { - const steps = strategySteps(candidate); - const reason = strategyRejection(steps); - if (reason) { - drop(reason); - continue; - } + }); + if (eligible.length === 0) { + report({ status: 'all_filtered', candidates: assets.length, dropped }); + return EMPTY_MATCH; + } - onOutcome({ - status: 'injected', - candidates: assets.length, - dropped, - asset_id: candidate.asset_id, - title: readableNameOf(candidate), - steps: steps.length, - chars: strategyChars(steps), - similarity: typeof candidate.similarity === 'number' ? candidate.similarity : null, - }); - return { - ids: [candidate.asset_id], - text: [ - `[Evolution Memory] ${readableNameOf(candidate)} (EvoMap network):`, - ...steps.map((step, index) => `${index + 1}. ${step}`), - '', - 'These are the complete steps already recalled from the network, so there is nothing more to search or fetch for this task.', - `Apply them where they fit, then report the outcome with evolver_asset_reuse_result for ${candidate.asset_id}.`, - ].join('\n'), - }; + const { assets: picked, review } = await reviewedAssets(judge, taskText, eligible, signal, fallbacks); + if (picked.length === 0) { + report({ status: 'none_usable', candidates: assets.length, dropped, review }); + return EMPTY_MATCH; } - onOutcome({ status: 'all_filtered', candidates: assets.length, dropped }); - return EMPTY_MATCH; + + const [first] = picked; + report({ + status: 'injected', + candidates: assets.length, + dropped, + review, + asset_id: first.asset.asset_id, + asset_ids: picked.map(({ asset }) => asset.asset_id), + title: readableNameOf(first.asset), + steps: first.steps.length, + chars: picked.reduce((total, { steps }) => total + strategyChars(steps), 0), + similarity: typeof first.asset.similarity === 'number' ? first.asset.similarity : null, + }); + return { ids: picked.map(({ asset }) => asset.asset_id), text: memoryText(picked) }; } diff --git a/src/recall-judge.js b/src/recall-judge.js new file mode 100644 index 0000000..4f10536 --- /dev/null +++ b/src/recall-judge.js @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 EvoMap + +const GATEWAY_URL = 'https://api.evomap.ai/v1'; +// Not the agent's own model: in the benchmark that is deepseek-flash, which +// also grades the injected assets, so the judge here must be a different one. +const MODEL = 'gemini-3.5-flash'; +export const API_KEY_ENV = 'EVOMAP_API_KEY'; +export const TASK_MAX_CHARS = 6_000; +const VERIFY_TASK_MAX_CHARS = 4_000; +const QUERY_MAX_CHARS = 400; +const SUMMARY_MAX_CHARS = 600; +const TRIGGERS_MAX_CHARS = 400; +const MAX_TOKENS = 2_000; +const CALL_TIMEOUT_MS = 12_000; + +// Both prompts are the ones the offline replay over the 499-task bank was +// measured with; rewording them invalidates that measurement. +const REWRITE_PROMPT = `Rewrite the task below into ONE search query for a library of reusable problem-solving playbooks. +Keep only what the task is about: the concrete problem, the domain, and the kind of data or system involved. +Drop role statements, file paths, tool or environment instructions, output-format and saving rules. +Write it in English, at most 200 characters. Answer with the query only. + +## Task +{task}`; + +const VERIFY_PROMPT = `An AI agent is about to work on the task below. A memory library offers it one stored playbook. +Decide whether the agent should read this playbook before starting. + +Say yes only if the playbook addresses the same kind of problem with the same kind of data, system or deliverable, +so its steps would shorten the work. Say no if it is about a different tool, platform, environment or deliverable, +if it would push the agent toward extra steps the task does not need, or if it is generic advice. + +## Task +{task} + +## Playbook +Title: {title} +Summary: {summary} +Category: {category} +Triggers: {triggers} + +Answer with JSON only: {"use": true|false}`; + +function filled(template, values) { + return template.replace(/\{(task|title|summary|category|triggers)\}/g, (_, key) => values[key]); +} + +// A recalled asset is the GEP body itself, without the Hub's short_title: its +// local id stands in for the title, and a Capsule's trigger for the signals a +// Gene carries. +function playbookCardOf(asset) { + const payload = asset?.payload && typeof asset.payload === 'object' ? asset.payload : asset; + const signals = payload?.signals_match ?? asset?.signals_match ?? payload?.trigger ?? asset?.trigger; + return { + title: asset?.short_title || asset?.local_id || asset?.id || '', + summary: String(payload?.summary || asset?.nl_summary || asset?.summary || '').slice(0, SUMMARY_MAX_CHARS), + category: payload?.category || asset?.category || '', + triggers: (Array.isArray(signals) ? signals.join('; ') : '').slice(0, TRIGGERS_MAX_CHARS), + }; +} + +function useVerdictOf(answer) { + const start = answer.indexOf('{'); + const end = answer.lastIndexOf('}'); + if (start < 0 || end < start) throw new Error(`verdict is not JSON: ${answer.slice(0, 120)}`); + const use = JSON.parse(answer.slice(start, end + 1))?.use; + if (typeof use !== 'boolean') throw new Error(`verdict has no boolean "use": ${answer.slice(0, 120)}`); + return use; +} + +// Returns null without a gateway key, so the caller keeps today's recall +// exactly: the prompt prefix as the query and no verdicts. +export function createRecallJudge({ apiKey = process.env[API_KEY_ENV], fetchImpl = fetch, timeoutMs = CALL_TIMEOUT_MS } = {}) { + if (!apiKey) return null; + + const completion = async (content, signal, extra = {}) => { + const timeout = AbortSignal.timeout(timeoutMs); + const response = await fetchImpl(`${GATEWAY_URL}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: MODEL, messages: [{ role: 'user', content }], max_tokens: MAX_TOKENS, ...extra }), + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, + }); + if (!response.ok) throw new Error(`gateway returned HTTP ${response.status}`); + const data = await response.json(); + return String(data?.choices?.[0]?.message?.content ?? ''); + }; + + const verdictOf = async (task, asset, signal) => { + const prompt = filled(VERIFY_PROMPT, { task: task.slice(0, VERIFY_TASK_MAX_CHARS), ...playbookCardOf(asset) }); + return useVerdictOf(await completion(prompt, signal, { response_format: { type: 'json_object' } })); + }; + + return { + async searchQueryOf(task, signal) { + const answer = await completion(filled(REWRITE_PROMPT, { task: task.slice(0, TASK_MAX_CHARS) }), signal); + const query = answer.trim().replace(/^"+|"+$/g, '').trim(); + if (!query) throw new Error('rewrite came back empty'); + return query.slice(0, QUERY_MAX_CHARS); + }, + + // One request per candidate, sent together: as fast as one batched call, + // and each verdict sees the same single-playbook prompt as offline. A + // candidate whose verdict failed is not used; only when every verdict + // failed is the review itself treated as failed. + async usableOf(task, assets, signal) { + const settled = await Promise.allSettled(assets.map((asset) => verdictOf(task, asset, signal))); + const failures = settled.filter((result) => result.status === 'rejected'); + if (assets.length > 0 && failures.length === assets.length) throw failures[0].reason; + return { + usable: assets.filter((_, index) => settled[index].status === 'fulfilled' && settled[index].value === true), + failed: failures.length, + }; + }, + }; +} diff --git a/test/prime.test.js b/test/prime.test.js index 57c0863..1489254 100644 --- a/test/prime.test.js +++ b/test/prime.test.js @@ -280,3 +280,103 @@ test('a broad prompt whose genes all run past eight steps still gets one injecte assert.deepEqual(ids, ['sha256:fifteen']); assert.match(text, /^15\. Step 14: /m); }); + +function geneOf(id, similarity = 0.9) { + return { asset_id: id, asset_type: 'Gene', similarity, short_title: `Playbook ${id}`, strategy: [`${id} one.`, 'Two.', 'Three.', 'Four.'] }; +} + +function stubJudge({ query = 'retry uploads with backoff', usable = () => true, rewriteError, reviewError } = {}) { + const calls = { rewrite: [], review: [] }; + return { + calls, + judge: { + async searchQueryOf(task) { + calls.rewrite.push(task); + if (rewriteError) throw rewriteError; + return query; + }, + async usableOf(task, assets) { + calls.review.push({ task, ids: assets.map((asset) => asset.asset_id) }); + if (reviewError) throw reviewError; + return { usable: assets.filter(usable), failed: 0 }; + }, + }, + }; +} + +const LONG_TASK = `You are a careful analyst. ${'Reconcile the ledger against the bank export. '.repeat(40)}`; + +test('with a judge, the rewritten query asks the Hub for ten and the whole task is reviewed', async () => { + const assets = Array.from({ length: 12 }, (_, index) => geneOf(`sha256:g${index}`, 0.9 - index * 0.01)); + const { proxyFetch, calls } = stubProxy({ assets }); + const { judge, calls: judged } = stubJudge(); + const outcomes = []; + + const { ids, text } = await hubGene(proxyFetch, LONG_TASK, { judge, onOutcome: (outcome) => outcomes.push(outcome) }); + + assert.deepEqual(judged.rewrite, [LONG_TASK]); + assert.deepEqual(calls[0].body, { text: 'retry uploads with backoff', limit: 10 }); + assert.equal(judged.review[0].task, LONG_TASK, 'the review sees the task, not the query'); + assert.deepEqual(judged.review[0].ids, assets.slice(0, 10).map((asset) => asset.asset_id)); + assert.deepEqual(ids, assets.slice(0, 5).map((asset) => asset.asset_id), 'at most five are injected'); + assert.equal(text.match(/^\[Evolution Memory\]/gm).length, 5); + assert.match(text, /evolver_asset_reuse_result for sha256:g0, sha256:g1, sha256:g2, sha256:g3, sha256:g4\.$/); + assert.equal(outcomes[0].status, 'injected'); + assert.equal(outcomes[0].query_source, 'rewritten'); + assert.equal(outcomes[0].query, 'retry uploads with backoff'); + assert.deepEqual(outcomes[0].review, { judged: 10, usable: 10, failed: 0 }); +}); + +test('only the candidates the judge accepted are injected, in rank order', async () => { + const { proxyFetch } = stubProxy({ assets: [geneOf('sha256:a', 0.9), geneOf('sha256:b', 0.8), geneOf('sha256:c', 0.7)] }); + const { judge } = stubJudge({ usable: (asset) => asset.asset_id !== 'sha256:a' }); + + const { ids, text } = await hubGene(proxyFetch, LONG_TASK, { judge }); + assert.deepEqual(ids, ['sha256:b', 'sha256:c']); + assert.doesNotMatch(text, /sha256:a one/); +}); + +test('when the judge accepts nothing, nothing is injected', async () => { + const { proxyFetch } = stubProxy({ assets: [geneOf('sha256:a'), geneOf('sha256:b')] }); + const { judge } = stubJudge({ usable: () => false }); + const outcomes = []; + + assert.deepEqual(await hubGene(proxyFetch, LONG_TASK, { judge, onOutcome: (outcome) => outcomes.push(outcome) }), { ids: [], text: '' }); + assert.equal(outcomes[0].status, 'none_usable'); + assert.deepEqual(outcomes[0].review, { judged: 2, usable: 0, failed: 0 }); +}); + +test('a failed rewrite searches with the prompt prefix and says why', async () => { + const { proxyFetch, calls } = stubProxy({ assets: [geneOf('sha256:a')] }); + const { judge } = stubJudge({ rewriteError: new Error('gateway returned HTTP 503') }); + const outcomes = []; + + const { ids } = await hubGene(proxyFetch, LONG_TASK, { judge, onOutcome: (outcome) => outcomes.push(outcome) }); + assert.equal(calls[0].body.text, LONG_TASK.slice(0, 400)); + assert.deepEqual(ids, ['sha256:a'], 'the review still runs'); + assert.equal(outcomes[0].query_source, 'prompt_prefix'); + assert.deepEqual(outcomes[0].fallbacks, { rewrite: 'gateway returned HTTP 503' }); +}); + +test('a failed review injects the best candidate unreviewed, as without a judge, and says why', async () => { + const { proxyFetch } = stubProxy({ assets: [geneOf('sha256:a', 0.7), geneOf('sha256:b', 0.9)] }); + const { judge } = stubJudge({ reviewError: new Error('verdict is not JSON: maybe') }); + const outcomes = []; + + const { ids } = await hubGene(proxyFetch, LONG_TASK, { judge, onOutcome: (outcome) => outcomes.push(outcome) }); + assert.deepEqual(ids, ['sha256:b']); + assert.equal(outcomes[0].review, null); + assert.deepEqual(outcomes[0].fallbacks, { review: 'verdict is not JSON: maybe' }); +}); + +test('without a judge the prompt prefix is the query and the Hub default decides how many', async () => { + const { proxyFetch, calls } = stubProxy({ assets: [geneOf('sha256:a')] }); + await hubGene(proxyFetch, LONG_TASK); + assert.deepEqual(calls[0].body, { text: LONG_TASK.slice(0, 400) }); +}); + +test('the task text for recall keeps up to six thousand characters, the default still four hundred', () => { + const messages = [{ content: [{ type: 'text', text: 'y'.repeat(7_000) }], source: { kind: 'user' } }]; + assert.equal(promptTextOf(messages).length, 400); + assert.equal(promptTextOf(messages, 6_000).length, 6_000); +}); diff --git a/test/recall-judge.test.js b/test/recall-judge.test.js new file mode 100644 index 0000000..f56ee7f --- /dev/null +++ b/test/recall-judge.test.js @@ -0,0 +1,85 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { createRecallJudge } from '../src/recall-judge.js'; + +function gateway(answer) { + const requests = []; + const fetchImpl = async (url, init) => { + const body = JSON.parse(init.body); + requests.push({ url, headers: init.headers, body }); + const reply = typeof answer === 'function' ? answer(body) : answer; + if (reply instanceof Error) throw reply; + if (typeof reply === 'number') return { ok: false, status: reply, json: async () => ({}) }; + return { ok: true, status: 200, json: async () => ({ choices: [{ message: { content: reply } }] }) }; + }; + return { fetchImpl, requests }; +} + +const ASSET = { + asset_id: 'sha256:a', + short_title: 'Reconcile a ledger', + payload: { summary: 's'.repeat(900), category: 'finance', signals_match: ['ledger', 'bank export'] }, +}; + +test('no gateway key means no judge', () => { + assert.equal(createRecallJudge({ apiKey: '' }), null); +}); + +test('the rewrite sends the bounded task to gemini-3.5-flash and returns the bare query', async () => { + const { fetchImpl, requests } = gateway(' "reconcile ledger against bank export"\n'); + const judge = createRecallJudge({ apiKey: 'k', fetchImpl }); + + assert.equal(await judge.searchQueryOf('t'.repeat(9_000)), 'reconcile ledger against bank export'); + assert.equal(requests[0].url, 'https://api.evomap.ai/v1/chat/completions'); + assert.equal(requests[0].headers.Authorization, 'Bearer k'); + assert.equal(requests[0].body.model, 'gemini-3.5-flash'); + const prompt = requests[0].body.messages[0].content; + assert.match(prompt, /^Rewrite the task below into ONE search query/); + assert.ok(prompt.endsWith(`## Task\n${'t'.repeat(6_000)}`)); +}); + +test('an empty or failed rewrite throws, so the caller can fall back', async () => { + await assert.rejects(createRecallJudge({ apiKey: 'k', fetchImpl: gateway(' ').fetchImpl }).searchQueryOf('task'), /empty/); + await assert.rejects(createRecallJudge({ apiKey: 'k', fetchImpl: gateway(503).fetchImpl }).searchQueryOf('task'), /HTTP 503/); +}); + +test('each candidate is judged on its own card against the bounded task', async () => { + const { fetchImpl, requests } = gateway((body) => (body.messages[0].content.includes('Title: keep') ? '{"use": true}' : '{"use": false}')); + const judge = createRecallJudge({ apiKey: 'k', fetchImpl }); + const keep = { ...ASSET, asset_id: 'sha256:keep', short_title: 'keep' }; + + const { usable, failed } = await judge.usableOf('t'.repeat(5_000), [ASSET, keep]); + assert.deepEqual(usable.map((asset) => asset.asset_id), ['sha256:keep']); + assert.equal(failed, 0); + assert.equal(requests.length, 2); + assert.deepEqual(requests[0].body.response_format, { type: 'json_object' }); + const prompt = requests[0].body.messages[0].content; + assert.ok(prompt.includes(`## Task\n${'t'.repeat(4_000)}\n\n## Playbook`)); + assert.match(prompt, /^Title: Reconcile a ledger$/m); + assert.match(prompt, new RegExp(`^Summary: ${'s'.repeat(600)}$`, 'm')); + assert.match(prompt, /^Category: finance$/m); + assert.match(prompt, /^Triggers: ledger; bank export$/m); + assert.match(prompt, /Answer with JSON only: \{"use": true\|false\}$/); +}); + +test('a candidate whose verdict failed is not used; the review fails only when every verdict did', async () => { + const mixed = gateway((body) => (body.messages[0].content.includes('Title: keep') ? '{"use": true}' : 'not json')); + const { usable, failed } = await createRecallJudge({ apiKey: 'k', fetchImpl: mixed.fetchImpl }) + .usableOf('task', [ASSET, { ...ASSET, short_title: 'keep' }]); + assert.deepEqual(usable.map((asset) => asset.short_title), ['keep']); + assert.equal(failed, 1); + + await assert.rejects(createRecallJudge({ apiKey: 'k', fetchImpl: gateway(new Error('timeout')).fetchImpl }).usableOf('task', [ASSET]), /timeout/); +}); + +test('a recalled GEP body without a short title is judged by its local id and trigger', async () => { + const { fetchImpl, requests } = gateway('{"use": true}'); + await createRecallJudge({ apiKey: 'k', fetchImpl }).usableOf('task', [{ + id: 'capsule_legal_writer', asset_id: 'sha256:c', type: 'Capsule', summary: 'Draft legal documents.', trigger: ['draft a contract', 'write a legal memo'], + }]); + const prompt = requests[0].body.messages[0].content; + assert.match(prompt, /^Title: capsule_legal_writer$/m); + assert.match(prompt, /^Summary: Draft legal documents\.$/m); + assert.match(prompt, /^Triggers: draft a contract; write a legal memo$/m); +}); From 827a2c59b3294e920504b88102b50e50c6ad2ffe Mon Sep 17 00:00:00 2001 From: forrestlinfeng Date: Wed, 21 Oct 2026 14:03:59 +0800 Subject: [PATCH 2/6] Judge a follow-up turn with the opening prompt in front of it A follow-up such as "Continue with the task from where you stopped." says nothing about the work. Judged alone, its rewrite became a query about resuming interrupted workflows and the review accepted three checkpointing playbooks for a legal drafting task. The rewrite and the review now read the session's opening prompt followed by this turn's; a fallback still searches with this turn's prompt prefix, as before. Co-Authored-By: Claude Opus 5.5 --- src/index.js | 16 ++++++++++++++-- src/prime.js | 22 ++++++++++++++++------ test/prime.test.js | 21 ++++++++++++++++++++- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/index.js b/src/index.js index b50040c..5cce96f 100644 --- a/src/index.js +++ b/src/index.js @@ -14,7 +14,7 @@ import { feedbackLabelOf } from './feedback-label.js'; import { forgetSession, rememberInjected, takeUnreported } from './injected-assets.js'; import { recordReuseLocally } from './local-ledger.js'; import { noticeDue, pendingClaimUrl } from './onboarding.js'; -import { hubGene, promptTextOf } from './prime.js'; +import { hubGene, promptTextOf, taskWithOpening } from './prime.js'; import { resolveProjectDir, sessionDir } from './project-dir.js'; import { createProxyClient } from './proxy.js'; import { API_KEY_ENV, TASK_MAX_CHARS, createRecallJudge } from './recall-judge.js'; @@ -90,6 +90,16 @@ function primeSteps(ctx, config, primeFetch, judge, engineSetup, recallLog, onPr const seeded = new WeakSet(); const searchedTurn = new WeakMap(); const listedAssets = new WeakMap(); + const openingPrompts = new WeakMap(); + + const taskOf = (agent, prompt) => { + const opening = openingPrompts.get(agent); + if (!opening) { + openingPrompts.set(agent, prompt); + return prompt; + } + return taskWithOpening(opening, prompt); + }; const listedFor = (agent) => { let listed = listedAssets.get(agent); @@ -116,7 +126,9 @@ function primeSteps(ctx, config, primeFetch, judge, engineSetup, recallLog, onPr const listed = listedFor(agent); const startedAt = Date.now(); - const search = hubGene(primeFetch, promptTextOf(claimed, TASK_MAX_CHARS), { + const prompt = promptTextOf(claimed, TASK_MAX_CHARS); + const search = hubGene(primeFetch, prompt, { + task: taskOf(agent, prompt), signal, listedIds: listed, minSimilarity: config.assetPrimeMinSimilarity, diff --git a/src/prime.js b/src/prime.js index 1f63e07..8fe9e29 100644 --- a/src/prime.js +++ b/src/prime.js @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap +import { TASK_MAX_CHARS } from './recall-judge.js'; import { withoutSecrets } from './redact.js'; // One round trip, not two: since evolver 2.0.39 `/asset/fetch` recalls by text @@ -120,6 +121,11 @@ async function proxyJson(proxyFetch, path, body, signal) { } +export function taskWithOpening(opening, prompt, maxChars = TASK_MAX_CHARS) { + if (opening === prompt) return prompt; + return `${opening.slice(0, Math.max(0, maxChars - prompt.length - 2))}\n\n${prompt}`.slice(0, maxChars); +} + function errorText(error) { return String(error?.message ?? error); } @@ -127,8 +133,8 @@ function errorText(error) { // A judge that fails falls back to today's recall step by step: a failed // rewrite searches with the prompt prefix, a failed review injects the best // candidate unreviewed. Either way the reason is reported, never swallowed. -async function searchQueryOf(judge, taskText, signal, fallbacks) { - const prefix = taskText.slice(0, PROMPT_MAX_CHARS); +async function searchQueryOf(judge, promptText, taskText, signal, fallbacks) { + const prefix = promptText.slice(0, PROMPT_MAX_CHARS); if (!judge) return { query: prefix, source: 'prompt_prefix' }; try { return { query: await judge.searchQueryOf(taskText, signal), source: 'rewritten' }; @@ -188,21 +194,25 @@ function memoryText(picked) { // Without a judge one asset is injected; with one, up to five that it accepted // for the whole task, or none when it accepted none. Every lookup reports why // it ended through `onOutcome`, because an empty match alone cannot tell a -// Proxy error from a recall that found nothing usable. -export async function hubGene(proxyFetch, taskText, { +// Proxy error from a recall that found nothing usable. The judge reads `task`, +// which a caller widens beyond this turn's prompt: a follow-up such as +// "continue" says nothing about the work, and judged alone it let through +// playbooks about resuming interrupted workflows. +export async function hubGene(proxyFetch, promptText, { + task: taskText = promptText, signal, listedIds = new Set(), minSimilarity = DEFAULT_MIN_SIMILARITY, judge = null, onOutcome = () => {}, } = {}) { - if (taskText.length < MIN_PROMPT_CHARS) { + if (promptText.length < MIN_PROMPT_CHARS) { onOutcome({ status: 'skipped', reason: 'prompt_too_short' }); return EMPTY_MATCH; } const fallbacks = {}; - const { query, source } = await searchQueryOf(judge, taskText, signal, fallbacks); + const { query, source } = await searchQueryOf(judge, promptText, taskText, signal, fallbacks); const searched = judge ? { query_source: source, ...(source === 'rewritten' ? { query } : {}) } : {}; const report = (outcome) => onOutcome({ ...outcome, ...searched, ...(Object.keys(fallbacks).length ? { fallbacks } : {}) }); diff --git a/test/prime.test.js b/test/prime.test.js index 1489254..c71d4d7 100644 --- a/test/prime.test.js +++ b/test/prime.test.js @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import { hubGene, promptTextOf } from '../src/prime.js'; +import { hubGene, promptTextOf, taskWithOpening } from '../src/prime.js'; function stubProxy(recalled) { const calls = []; @@ -380,3 +380,22 @@ test('the task text for recall keeps up to six thousand characters, the default assert.equal(promptTextOf(messages).length, 400); assert.equal(promptTextOf(messages, 6_000).length, 6_000); }); + +test('the judge reads the widened task, while a fallback searches with this turn\'s prompt alone', async () => { + const { proxyFetch, calls } = stubProxy({ assets: [geneOf('sha256:a')] }); + const { judge, calls: judged } = stubJudge({ rewriteError: new Error('down') }); + const task = `${LONG_TASK}\n\nContinue with the task from where you stopped.`; + + await hubGene(proxyFetch, 'Continue with the task from where you stopped.', { task, judge }); + assert.deepEqual(judged.rewrite, [task]); + assert.equal(judged.review[0].task, task); + assert.equal(calls[0].body.text, 'Continue with the task from where you stopped.'); +}); + +test('a follow-up turn is judged with the opening prompt in front of it, and the follow-up always fits', () => { + assert.equal(taskWithOpening('fix the ledger', 'fix the ledger'), 'fix the ledger'); + assert.equal(taskWithOpening('fix the ledger', 'continue'), 'fix the ledger\n\ncontinue'); + const widened = taskWithOpening('x'.repeat(6_000), 'continue'); + assert.equal(widened.length, 6_000); + assert.ok(widened.endsWith('\n\ncontinue')); +}); From a3cf6e022a75baf93616610726dbb28891ea015a Mon Sep 17 00:00:00 2001 From: forrestlinfeng Date: Wed, 21 Oct 2026 15:56:56 +0800 Subject: [PATCH 3/6] Retry the recall judge on a busy or failing gateway The gateway caps concurrent requests per key and answers 429 concurrency_limit; a review sends ten at once beside the agent's own calls on the same key, and a lost verdict counts as a rejection. A 429 or 5xx is retried twice, after 1 s and 3 s or the Retry-After it names (at most 5 s); any other refusal fails at once. Co-Authored-By: Claude Opus 5.5 --- src/recall-judge.js | 48 ++++++++++++++++++++++++++++++++++----- test/recall-judge.test.js | 17 ++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/src/recall-judge.js b/src/recall-judge.js index 4f10536..f936747 100644 --- a/src/recall-judge.js +++ b/src/recall-judge.js @@ -13,6 +13,10 @@ const SUMMARY_MAX_CHARS = 600; const TRIGGERS_MAX_CHARS = 400; const MAX_TOKENS = 2_000; const CALL_TIMEOUT_MS = 12_000; +// The gateway caps concurrent requests per key (429 concurrency_limit), and a +// review sends ten at once beside the agent's own calls on the same key. +const RETRY_DELAYS_MS = [1_000, 3_000]; +const RETRY_AFTER_MAX_MS = 5_000; // Both prompts are the ones the offline replay over the 499-task bank was // measured with; rewording them invalidates that measurement. @@ -71,20 +75,52 @@ function useVerdictOf(answer) { // Returns null without a gateway key, so the caller keeps today's recall // exactly: the prompt prefix as the query and no verdicts. -export function createRecallJudge({ apiKey = process.env[API_KEY_ENV], fetchImpl = fetch, timeoutMs = CALL_TIMEOUT_MS } = {}) { +function isRetryable(status) { + return status === 429 || status >= 500; +} + +function retryDelayMs(response, attempt) { + const seconds = Number(response.headers?.get?.('retry-after')); + if (Number.isFinite(seconds) && seconds > 0) return Math.min(seconds * 1_000, RETRY_AFTER_MAX_MS); + return RETRY_DELAYS_MS[attempt]; +} + +function pause(ms, signal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(signal.reason); + const timer = setTimeout(resolve, ms); + timer.unref?.(); + signal?.addEventListener('abort', () => { clearTimeout(timer); reject(signal.reason); }, { once: true }); + }); +} + +export function createRecallJudge({ + apiKey = process.env[API_KEY_ENV], fetchImpl = fetch, timeoutMs = CALL_TIMEOUT_MS, sleep = pause, +} = {}) { if (!apiKey) return null; - const completion = async (content, signal, extra = {}) => { + const request = (content, signal, extra) => { const timeout = AbortSignal.timeout(timeoutMs); - const response = await fetchImpl(`${GATEWAY_URL}/chat/completions`, { + return fetchImpl(`${GATEWAY_URL}/chat/completions`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ model: MODEL, messages: [{ role: 'user', content }], max_tokens: MAX_TOKENS, ...extra }), signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }); - if (!response.ok) throw new Error(`gateway returned HTTP ${response.status}`); - const data = await response.json(); - return String(data?.choices?.[0]?.message?.content ?? ''); + }; + + const completion = async (content, signal, extra = {}) => { + for (let attempt = 0; ; attempt++) { + const response = await request(content, signal, extra); + if (response.ok) { + const data = await response.json(); + return String(data?.choices?.[0]?.message?.content ?? ''); + } + if (!isRetryable(response.status) || attempt >= RETRY_DELAYS_MS.length) { + throw new Error(`gateway returned HTTP ${response.status}`); + } + await sleep(retryDelayMs(response, attempt), signal); + } }; const verdictOf = async (task, asset, signal) => { diff --git a/test/recall-judge.test.js b/test/recall-judge.test.js index f56ee7f..1813eb9 100644 --- a/test/recall-judge.test.js +++ b/test/recall-judge.test.js @@ -83,3 +83,20 @@ test('a recalled GEP body without a short title is judged by its local id and tr assert.match(prompt, /^Summary: Draft legal documents\.$/m); assert.match(prompt, /^Triggers: draft a contract; write a legal memo$/m); }); + +test('a busy or failing gateway is retried twice with backoff, a refusal is not', async () => { + const waits = []; + const sleep = async (ms) => { waits.push(ms); }; + let calls = 0; + const flaky = gateway(() => (++calls < 3 ? 429 : 'ledger reconciliation')); + assert.equal(await createRecallJudge({ apiKey: 'k', fetchImpl: flaky.fetchImpl, sleep }).searchQueryOf('task'), 'ledger reconciliation'); + assert.deepEqual(waits, [1_000, 3_000]); + + const busy = gateway(503); + await assert.rejects(createRecallJudge({ apiKey: 'k', fetchImpl: busy.fetchImpl, sleep }).searchQueryOf('task'), /HTTP 503/); + assert.equal(busy.requests.length, 3); + + const refused = gateway(401); + await assert.rejects(createRecallJudge({ apiKey: 'k', fetchImpl: refused.fetchImpl, sleep }).searchQueryOf('task'), /HTTP 401/); + assert.equal(refused.requests.length, 1); +}); From 05b06a2b37568cd2f4ae2fb43b660d31a4b204dc Mon Sep 17 00:00:00 2001 From: forrestlinfeng Date: Fri, 23 Oct 2026 15:46:55 +0800 Subject: [PATCH 4/6] Rewrite the recall query on the session's own model and drop the review The query rewrite no longer calls a hard-coded gateway and model: it runs on the provider and model the session already uses, through dsh's llm service, so it needs no key of its own. Without that route, or when the rewrite fails, recall searches with the prompt prefix as before. The pre-injection review is gone, with the ten-candidate request, the five-asset injection and the gateway retries: recall injects the single best asset again, as on main. The review added up to ten model calls and about ten seconds before the first step, and up to five strategies to the context. Co-Authored-By: Claude Opus 5.5 --- src/index.js | 22 +++-- src/prime.js | 175 ++++++++++++++------------------------ src/recall-judge.js | 153 --------------------------------- src/recall-query.js | 47 ++++++++++ test/prime.test.js | 119 ++++++++------------------ test/recall-judge.test.js | 102 ---------------------- test/recall-query.test.js | 47 ++++++++++ 7 files changed, 209 insertions(+), 456 deletions(-) delete mode 100644 src/recall-judge.js create mode 100644 src/recall-query.js delete mode 100644 test/recall-judge.test.js create mode 100644 test/recall-query.test.js diff --git a/src/index.js b/src/index.js index 5cce96f..cac74a3 100644 --- a/src/index.js +++ b/src/index.js @@ -17,7 +17,7 @@ import { noticeDue, pendingClaimUrl } from './onboarding.js'; import { hubGene, promptTextOf, taskWithOpening } from './prime.js'; import { resolveProjectDir, sessionDir } from './project-dir.js'; import { createProxyClient } from './proxy.js'; -import { API_KEY_ENV, TASK_MAX_CHARS, createRecallJudge } from './recall-judge.js'; +import { TASK_MAX_CHARS, rewrittenQueryOf } from './recall-query.js'; import { createReportQueue } from './report-queue.js'; import { rememberForRetention, retentionReportsOf, takeDueRetentions } from './retention-pending.js'; import { correctionReportOf, immediateReportOf, reportedReuseOf } from './reuse.js'; @@ -86,11 +86,21 @@ function afterWait(ms) { // that carries both. Workspace memory is a session fact and seeds once; the Hub // is re-queried per turn, because each prompt is a different task — bounded by // the ids already listed, so a repeat search adds nothing the model has seen. -function primeSteps(ctx, config, primeFetch, judge, engineSetup, recallLog, onPrompt) { +function primeSteps(ctx, config, primeFetch, engineSetup, recallLog, onPrompt) { const seeded = new WeakSet(); const searchedTurn = new WeakMap(); const listedAssets = new WeakMap(); const openingPrompts = new WeakMap(); + let llm = null; + ctx.inject(['llm'], (scoped) => { + llm = scoped.llm; + }); + + const rewriteFor = (agent) => { + const { provider, model } = agent.options ?? {}; + if (!llm || !provider || !model) return null; + return (task, signal) => rewrittenQueryOf(llm, { provider, model }, task, signal); + }; const taskOf = (agent, prompt) => { const opening = openingPrompts.get(agent); @@ -132,10 +142,10 @@ function primeSteps(ctx, config, primeFetch, judge, engineSetup, recallLog, onPr signal, listedIds: listed, minSimilarity: config.assetPrimeMinSimilarity, - judge, + rewrite: rewriteFor(agent), onOutcome: (outcome) => { recallLog.record(outcome, startedAt); - if (outcome.fallbacks) ctx.logger?.warn?.(`evolver recall judge fell back: ${JSON.stringify(outcome.fallbacks)}`); + if (outcome.rewrite_error) ctx.logger?.warn?.(`evolver recall searched with the prompt prefix: the query rewrite failed: ${outcome.rewrite_error}`); }, }); const inline = await Promise.race([search, afterWait(config.assetPrimeWaitMs ?? DEFAULT_PRIME_WAIT_MS)]); @@ -262,8 +272,6 @@ export function apply(ctx, config = {}) { const projectDirOf = (session) => sessionDir(session?.header?.cwd, fallbackDir); const proxyFetch = createProxyClient({ port: config.proxyPort, timeoutMs: config.proxyTimeoutMs }); const primeFetch = createProxyClient({ port: config.proxyPort, timeoutMs: config.assetPrimeTimeoutMs ?? 8_000 }); - const judge = createRecallJudge(); - if (!judge) ctx.logger?.info?.(`evolver recall judge is off: ${API_KEY_ENV} is not set, so recall searches with the prompt prefix unreviewed`); const engineSetup = createEngineSetup(() => prepareEngine({ probeVersion: installedEvolverVersion, @@ -286,7 +294,7 @@ export function apply(ctx, config = {}) { const records = createTurnRecords(); const labelable = new Map(); - primeSteps(ctx, config, primeFetch, judge, engineSetup, recallLog, followUpReporter(deliver, labelable)); + primeSteps(ctx, config, primeFetch, engineSetup, recallLog, followUpReporter(deliver, labelable)); observeTools(ctx, config.editToolNames ?? EDIT_TOOL_NAMES, records, projectDirOf); reportTurns(ctx, records, deliver, labelable, projectDirOf); onSessionStart(ctx, engineSetup, deliver, projectDirOf); diff --git a/src/prime.js b/src/prime.js index 8fe9e29..6bec139 100644 --- a/src/prime.js +++ b/src/prime.js @@ -1,20 +1,15 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 EvoMap -import { TASK_MAX_CHARS } from './recall-judge.js'; +import { TASK_MAX_CHARS } from './recall-query.js'; import { withoutSecrets } from './redact.js'; // One round trip, not two: since evolver 2.0.39 `/asset/fetch` recalls by text // when it is given no ids, so selecting a candidate and materialising it are the -// same call. Without a judge no `limit` is sent, so the Hub's own default -// decides how many come back (two at present). +// same call. No `limit` is sent, so the Hub's own default decides how many come +// back (two at present). const MIN_PROMPT_CHARS = 8; const PROMPT_MAX_CHARS = 400; -// Offline over 499 tasks, reviewing the Hub's top ten against the whole task -// and keeping at most five cut misleading injections from 715 to 13 while -// keeping 384 of 406 fitting ones. -const REVIEWED_CANDIDATES = 10; -const MAX_INJECTED = 5; const STEP_MAX_CHARS = 400; const TITLE_MAX_CHARS = 80; const DEFAULT_MIN_SIMILARITY = 0.3; @@ -120,142 +115,100 @@ async function proxyJson(proxyFetch, path, body, signal) { } } - export function taskWithOpening(opening, prompt, maxChars = TASK_MAX_CHARS) { if (opening === prompt) return prompt; return `${opening.slice(0, Math.max(0, maxChars - prompt.length - 2))}\n\n${prompt}`.slice(0, maxChars); } -function errorText(error) { - return String(error?.message ?? error); -} - -// A judge that fails falls back to today's recall step by step: a failed -// rewrite searches with the prompt prefix, a failed review injects the best -// candidate unreviewed. Either way the reason is reported, never swallowed. -async function searchQueryOf(judge, promptText, taskText, signal, fallbacks) { +// The prompt prefix is the query whenever there is no rewrite, or the rewrite +// fails; a failure is reported with the outcome, never swallowed. +async function searchQueryOf(rewrite, promptText, taskText, signal) { const prefix = promptText.slice(0, PROMPT_MAX_CHARS); - if (!judge) return { query: prefix, source: 'prompt_prefix' }; - try { - return { query: await judge.searchQueryOf(taskText, signal), source: 'rewritten' }; - } catch (error) { - fallbacks.rewrite = errorText(error); - return { query: prefix, source: 'prompt_prefix' }; - } -} - -async function reviewedAssets(judge, taskText, eligible, signal, fallbacks) { - if (!judge) return { assets: eligible.slice(0, 1), review: null }; - const candidates = eligible.slice(0, REVIEWED_CANDIDATES); + if (!rewrite) return { query: prefix, logged: {} }; try { - const { usable, failed } = await judge.usableOf(taskText, candidates.map(({ asset }) => asset), signal); - const accepted = candidates.filter(({ asset }) => usable.includes(asset)); - return { assets: accepted.slice(0, MAX_INJECTED), review: { judged: candidates.length, usable: accepted.length, failed } }; + const query = await rewrite(taskText, signal); + return { query, logged: { query_source: 'rewritten', query } }; } catch (error) { - fallbacks.review = errorText(error); - return { assets: eligible.slice(0, 1), review: null }; + return { query: prefix, logged: { query_source: 'prompt_prefix', rewrite_error: String(error?.message ?? error) } }; } } -function eligibleAssets(assets, listedIds, minSimilarity, drop) { - return assets - .filter((asset) => { - const reason = rankingRejection(asset, listedIds, minSimilarity); - if (reason) drop(reason); - return !reason; - }) - .sort((left, right) => similarityOf(right) - similarityOf(left)) - .map((asset) => ({ asset, steps: strategySteps(asset) })) - .filter(({ steps }) => { - const reason = strategyRejection(steps); - if (reason) drop(reason); - return !reason; - }); -} - -function memoryText(picked) { - const ids = picked.map(({ asset }) => asset.asset_id); - const sections = picked.map(({ asset, steps }) => [ - `[Evolution Memory] ${readableNameOf(asset)} (EvoMap network):`, - ...steps.map((step, index) => `${index + 1}. ${step}`), - '', - ]); - return [ - ...sections.flat(), - 'These are the complete steps already recalled from the network, so there is nothing more to search or fetch for this task.', - `Apply them where they fit, then report the outcome with evolver_asset_reuse_result for ${ids.join(', ')}.`, - ].join('\n'); -} - // Priming sits on the critical path of every answer: an unreachable Proxy, a -// slow Hub, a failing judge, or a malformed body must cost the turn nothing but -// the deadline. Only strategies are injected — a summary only tells the model -// that something exists, while the steps are what it can actually reuse. -// Without a judge one asset is injected; with one, up to five that it accepted -// for the whole task, or none when it accepted none. Every lookup reports why -// it ended through `onOutcome`, because an empty match alone cannot tell a -// Proxy error from a recall that found nothing usable. The judge reads `task`, -// which a caller widens beyond this turn's prompt: a follow-up such as -// "continue" says nothing about the work, and judged alone it let through -// playbooks about resuming interrupted workflows. +// slow Hub, a failed rewrite, or a malformed body must cost the turn nothing +// but the deadline. One asset's strategy is injected and nothing else — a +// summary only tells the model that something exists, while the steps are what +// it can actually reuse. Every lookup reports why it ended through `onOutcome`, +// because an empty match alone cannot tell a Proxy error from a recall that +// found nothing usable. The rewrite reads `task`, which a caller widens beyond +// this turn's prompt: a follow-up such as "continue" says nothing about the work. export async function hubGene(proxyFetch, promptText, { task: taskText = promptText, signal, listedIds = new Set(), minSimilarity = DEFAULT_MIN_SIMILARITY, - judge = null, - onOutcome = () => {}, + rewrite = null, + onOutcome: onSearchOutcome = () => {}, } = {}) { if (promptText.length < MIN_PROMPT_CHARS) { - onOutcome({ status: 'skipped', reason: 'prompt_too_short' }); + onSearchOutcome({ status: 'skipped', reason: 'prompt_too_short' }); return EMPTY_MATCH; } - const fallbacks = {}; - const { query, source } = await searchQueryOf(judge, promptText, taskText, signal, fallbacks); - const searched = judge ? { query_source: source, ...(source === 'rewritten' ? { query } : {}) } : {}; - const report = (outcome) => onOutcome({ ...outcome, ...searched, ...(Object.keys(fallbacks).length ? { fallbacks } : {}) }); - - const request = judge ? { text: query, limit: REVIEWED_CANDIDATES } : { text: query }; - const recalled = await proxyJson(proxyFetch, '/asset/fetch', request, signal); + const { query, logged } = await searchQueryOf(rewrite, promptText, taskText, signal); + const onOutcome = (outcome) => onSearchOutcome({ ...outcome, ...logged }); + const recalled = await proxyJson(proxyFetch, '/asset/fetch', { text: query }, signal); if (recalled.error) { - report({ status: 'proxy_error', error: recalled.error }); + onOutcome({ status: 'proxy_error', error: recalled.error }); return EMPTY_MATCH; } const assets = recalledAssets(recalled.data); if (assets.length === 0) { - report({ status: 'no_candidates' }); + onOutcome({ status: 'no_candidates' }); return EMPTY_MATCH; } const dropped = {}; - const eligible = eligibleAssets(assets, listedIds, minSimilarity, (reason) => { + const drop = (reason) => { dropped[reason] = (dropped[reason] ?? 0) + 1; - }); - if (eligible.length === 0) { - report({ status: 'all_filtered', candidates: assets.length, dropped }); - return EMPTY_MATCH; - } + }; + const ranked = assets + .filter((asset) => { + const reason = rankingRejection(asset, listedIds, minSimilarity); + if (reason) drop(reason); + return !reason; + }) + .sort((left, right) => similarityOf(right) - similarityOf(left)); + + for (const candidate of ranked) { + const steps = strategySteps(candidate); + const reason = strategyRejection(steps); + if (reason) { + drop(reason); + continue; + } - const { assets: picked, review } = await reviewedAssets(judge, taskText, eligible, signal, fallbacks); - if (picked.length === 0) { - report({ status: 'none_usable', candidates: assets.length, dropped, review }); - return EMPTY_MATCH; + onOutcome({ + status: 'injected', + candidates: assets.length, + dropped, + asset_id: candidate.asset_id, + title: readableNameOf(candidate), + steps: steps.length, + chars: strategyChars(steps), + similarity: typeof candidate.similarity === 'number' ? candidate.similarity : null, + }); + return { + ids: [candidate.asset_id], + text: [ + `[Evolution Memory] ${readableNameOf(candidate)} (EvoMap network):`, + ...steps.map((step, index) => `${index + 1}. ${step}`), + '', + 'These are the complete steps already recalled from the network, so there is nothing more to search or fetch for this task.', + `Apply them where they fit, then report the outcome with evolver_asset_reuse_result for ${candidate.asset_id}.`, + ].join('\n'), + }; } - - const [first] = picked; - report({ - status: 'injected', - candidates: assets.length, - dropped, - review, - asset_id: first.asset.asset_id, - asset_ids: picked.map(({ asset }) => asset.asset_id), - title: readableNameOf(first.asset), - steps: first.steps.length, - chars: picked.reduce((total, { steps }) => total + strategyChars(steps), 0), - similarity: typeof first.asset.similarity === 'number' ? first.asset.similarity : null, - }); - return { ids: picked.map(({ asset }) => asset.asset_id), text: memoryText(picked) }; + onOutcome({ status: 'all_filtered', candidates: assets.length, dropped }); + return EMPTY_MATCH; } diff --git a/src/recall-judge.js b/src/recall-judge.js deleted file mode 100644 index f936747..0000000 --- a/src/recall-judge.js +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2026 EvoMap - -const GATEWAY_URL = 'https://api.evomap.ai/v1'; -// Not the agent's own model: in the benchmark that is deepseek-flash, which -// also grades the injected assets, so the judge here must be a different one. -const MODEL = 'gemini-3.5-flash'; -export const API_KEY_ENV = 'EVOMAP_API_KEY'; -export const TASK_MAX_CHARS = 6_000; -const VERIFY_TASK_MAX_CHARS = 4_000; -const QUERY_MAX_CHARS = 400; -const SUMMARY_MAX_CHARS = 600; -const TRIGGERS_MAX_CHARS = 400; -const MAX_TOKENS = 2_000; -const CALL_TIMEOUT_MS = 12_000; -// The gateway caps concurrent requests per key (429 concurrency_limit), and a -// review sends ten at once beside the agent's own calls on the same key. -const RETRY_DELAYS_MS = [1_000, 3_000]; -const RETRY_AFTER_MAX_MS = 5_000; - -// Both prompts are the ones the offline replay over the 499-task bank was -// measured with; rewording them invalidates that measurement. -const REWRITE_PROMPT = `Rewrite the task below into ONE search query for a library of reusable problem-solving playbooks. -Keep only what the task is about: the concrete problem, the domain, and the kind of data or system involved. -Drop role statements, file paths, tool or environment instructions, output-format and saving rules. -Write it in English, at most 200 characters. Answer with the query only. - -## Task -{task}`; - -const VERIFY_PROMPT = `An AI agent is about to work on the task below. A memory library offers it one stored playbook. -Decide whether the agent should read this playbook before starting. - -Say yes only if the playbook addresses the same kind of problem with the same kind of data, system or deliverable, -so its steps would shorten the work. Say no if it is about a different tool, platform, environment or deliverable, -if it would push the agent toward extra steps the task does not need, or if it is generic advice. - -## Task -{task} - -## Playbook -Title: {title} -Summary: {summary} -Category: {category} -Triggers: {triggers} - -Answer with JSON only: {"use": true|false}`; - -function filled(template, values) { - return template.replace(/\{(task|title|summary|category|triggers)\}/g, (_, key) => values[key]); -} - -// A recalled asset is the GEP body itself, without the Hub's short_title: its -// local id stands in for the title, and a Capsule's trigger for the signals a -// Gene carries. -function playbookCardOf(asset) { - const payload = asset?.payload && typeof asset.payload === 'object' ? asset.payload : asset; - const signals = payload?.signals_match ?? asset?.signals_match ?? payload?.trigger ?? asset?.trigger; - return { - title: asset?.short_title || asset?.local_id || asset?.id || '', - summary: String(payload?.summary || asset?.nl_summary || asset?.summary || '').slice(0, SUMMARY_MAX_CHARS), - category: payload?.category || asset?.category || '', - triggers: (Array.isArray(signals) ? signals.join('; ') : '').slice(0, TRIGGERS_MAX_CHARS), - }; -} - -function useVerdictOf(answer) { - const start = answer.indexOf('{'); - const end = answer.lastIndexOf('}'); - if (start < 0 || end < start) throw new Error(`verdict is not JSON: ${answer.slice(0, 120)}`); - const use = JSON.parse(answer.slice(start, end + 1))?.use; - if (typeof use !== 'boolean') throw new Error(`verdict has no boolean "use": ${answer.slice(0, 120)}`); - return use; -} - -// Returns null without a gateway key, so the caller keeps today's recall -// exactly: the prompt prefix as the query and no verdicts. -function isRetryable(status) { - return status === 429 || status >= 500; -} - -function retryDelayMs(response, attempt) { - const seconds = Number(response.headers?.get?.('retry-after')); - if (Number.isFinite(seconds) && seconds > 0) return Math.min(seconds * 1_000, RETRY_AFTER_MAX_MS); - return RETRY_DELAYS_MS[attempt]; -} - -function pause(ms, signal) { - return new Promise((resolve, reject) => { - if (signal?.aborted) return reject(signal.reason); - const timer = setTimeout(resolve, ms); - timer.unref?.(); - signal?.addEventListener('abort', () => { clearTimeout(timer); reject(signal.reason); }, { once: true }); - }); -} - -export function createRecallJudge({ - apiKey = process.env[API_KEY_ENV], fetchImpl = fetch, timeoutMs = CALL_TIMEOUT_MS, sleep = pause, -} = {}) { - if (!apiKey) return null; - - const request = (content, signal, extra) => { - const timeout = AbortSignal.timeout(timeoutMs); - return fetchImpl(`${GATEWAY_URL}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model: MODEL, messages: [{ role: 'user', content }], max_tokens: MAX_TOKENS, ...extra }), - signal: signal ? AbortSignal.any([signal, timeout]) : timeout, - }); - }; - - const completion = async (content, signal, extra = {}) => { - for (let attempt = 0; ; attempt++) { - const response = await request(content, signal, extra); - if (response.ok) { - const data = await response.json(); - return String(data?.choices?.[0]?.message?.content ?? ''); - } - if (!isRetryable(response.status) || attempt >= RETRY_DELAYS_MS.length) { - throw new Error(`gateway returned HTTP ${response.status}`); - } - await sleep(retryDelayMs(response, attempt), signal); - } - }; - - const verdictOf = async (task, asset, signal) => { - const prompt = filled(VERIFY_PROMPT, { task: task.slice(0, VERIFY_TASK_MAX_CHARS), ...playbookCardOf(asset) }); - return useVerdictOf(await completion(prompt, signal, { response_format: { type: 'json_object' } })); - }; - - return { - async searchQueryOf(task, signal) { - const answer = await completion(filled(REWRITE_PROMPT, { task: task.slice(0, TASK_MAX_CHARS) }), signal); - const query = answer.trim().replace(/^"+|"+$/g, '').trim(); - if (!query) throw new Error('rewrite came back empty'); - return query.slice(0, QUERY_MAX_CHARS); - }, - - // One request per candidate, sent together: as fast as one batched call, - // and each verdict sees the same single-playbook prompt as offline. A - // candidate whose verdict failed is not used; only when every verdict - // failed is the review itself treated as failed. - async usableOf(task, assets, signal) { - const settled = await Promise.allSettled(assets.map((asset) => verdictOf(task, asset, signal))); - const failures = settled.filter((result) => result.status === 'rejected'); - if (assets.length > 0 && failures.length === assets.length) throw failures[0].reason; - return { - usable: assets.filter((_, index) => settled[index].status === 'fulfilled' && settled[index].value === true), - failed: failures.length, - }; - }, - }; -} diff --git a/src/recall-query.js b/src/recall-query.js new file mode 100644 index 0000000..8067e6a --- /dev/null +++ b/src/recall-query.js @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 EvoMap + +import { createUserMessage } from '@deepseek-ai/dsh-llm'; + +export const TASK_MAX_CHARS = 6_000; +const QUERY_MAX_CHARS = 400; +// Room for a reasoning model to think before it answers with one short line. +const REWRITE_MAX_TOKENS = 2_000; +const REWRITE_TIMEOUT_MS = 12_000; + +// The prompt the offline replay over the 499-task bank was measured with; +// rewording it invalidates that measurement. +const REWRITE_PROMPT = `Rewrite the task below into ONE search query for a library of reusable problem-solving playbooks. +Keep only what the task is about: the concrete problem, the domain, and the kind of data or system involved. +Drop role statements, file paths, tool or environment instructions, output-format and saving rules. +Write it in English, at most 200 characters. Answer with the query only. + +## Task +{task}`; + +async function answerOf(llm, options) { + let text = ''; + for await (const chunk of llm.stream(options)) { + if (chunk.type === 'text-delta') text += chunk.text; + if (chunk.type === 'finish' && chunk.reason?.failure) throw new Error(chunk.reason.failure.message); + } + return text; +} + +// The session's own provider and model write the query, so recall needs no +// key, endpoint or model of its own and costs one short call on the route the +// person already chose. +export async function rewrittenQueryOf(llm, { provider, model }, task, signal) { + const timeout = AbortSignal.timeout(REWRITE_TIMEOUT_MS); + const prompt = REWRITE_PROMPT.replace('{task}', () => task.slice(0, TASK_MAX_CHARS)); + const answer = await answerOf(llm, { + provider, + model, + messages: [createUserMessage({ content: [{ type: 'text', text: prompt }] })], + maxTokens: REWRITE_MAX_TOKENS, + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, + }); + const query = answer.trim().replace(/^"+|"+$/g, '').trim(); + if (!query) throw new Error('rewrite came back empty'); + return query.slice(0, QUERY_MAX_CHARS); +} diff --git a/test/prime.test.js b/test/prime.test.js index c71d4d7..83b7390 100644 --- a/test/prime.test.js +++ b/test/prime.test.js @@ -281,98 +281,62 @@ test('a broad prompt whose genes all run past eight steps still gets one injecte assert.match(text, /^15\. Step 14: /m); }); -function geneOf(id, similarity = 0.9) { - return { asset_id: id, asset_type: 'Gene', similarity, short_title: `Playbook ${id}`, strategy: [`${id} one.`, 'Two.', 'Three.', 'Four.'] }; -} +const LONG_TASK = `You are a careful analyst. ${'Reconcile the ledger against the bank export. '.repeat(40)}`; -function stubJudge({ query = 'retry uploads with backoff', usable = () => true, rewriteError, reviewError } = {}) { - const calls = { rewrite: [], review: [] }; - return { - calls, - judge: { - async searchQueryOf(task) { - calls.rewrite.push(task); - if (rewriteError) throw rewriteError; - return query; - }, - async usableOf(task, assets) { - calls.review.push({ task, ids: assets.map((asset) => asset.asset_id) }); - if (reviewError) throw reviewError; - return { usable: assets.filter(usable), failed: 0 }; - }, - }, +function stubRewrite({ query = 'reconcile ledger against bank export', error } = {}) { + const tasks = []; + const rewrite = async (task) => { + tasks.push(task); + if (error) throw error; + return query; }; + return { rewrite, tasks }; } -const LONG_TASK = `You are a careful analyst. ${'Reconcile the ledger against the bank export. '.repeat(40)}`; - -test('with a judge, the rewritten query asks the Hub for ten and the whole task is reviewed', async () => { - const assets = Array.from({ length: 12 }, (_, index) => geneOf(`sha256:g${index}`, 0.9 - index * 0.01)); - const { proxyFetch, calls } = stubProxy({ assets }); - const { judge, calls: judged } = stubJudge(); +test('with a rewrite, the Hub is searched with the rewritten query and one asset is still injected', async () => { + const { proxyFetch, calls } = stubProxy({ assets: [ASSET, { ...ASSET, asset_id: 'sha256:def' }] }); + const { rewrite, tasks } = stubRewrite(); const outcomes = []; - const { ids, text } = await hubGene(proxyFetch, LONG_TASK, { judge, onOutcome: (outcome) => outcomes.push(outcome) }); + const { ids } = await hubGene(proxyFetch, LONG_TASK, { rewrite, onOutcome: (outcome) => outcomes.push(outcome) }); - assert.deepEqual(judged.rewrite, [LONG_TASK]); - assert.deepEqual(calls[0].body, { text: 'retry uploads with backoff', limit: 10 }); - assert.equal(judged.review[0].task, LONG_TASK, 'the review sees the task, not the query'); - assert.deepEqual(judged.review[0].ids, assets.slice(0, 10).map((asset) => asset.asset_id)); - assert.deepEqual(ids, assets.slice(0, 5).map((asset) => asset.asset_id), 'at most five are injected'); - assert.equal(text.match(/^\[Evolution Memory\]/gm).length, 5); - assert.match(text, /evolver_asset_reuse_result for sha256:g0, sha256:g1, sha256:g2, sha256:g3, sha256:g4\.$/); + assert.deepEqual(tasks, [LONG_TASK]); + assert.deepEqual(calls[0].body, { text: 'reconcile ledger against bank export' }); + assert.equal(ids.length, 1); assert.equal(outcomes[0].status, 'injected'); assert.equal(outcomes[0].query_source, 'rewritten'); - assert.equal(outcomes[0].query, 'retry uploads with backoff'); - assert.deepEqual(outcomes[0].review, { judged: 10, usable: 10, failed: 0 }); -}); - -test('only the candidates the judge accepted are injected, in rank order', async () => { - const { proxyFetch } = stubProxy({ assets: [geneOf('sha256:a', 0.9), geneOf('sha256:b', 0.8), geneOf('sha256:c', 0.7)] }); - const { judge } = stubJudge({ usable: (asset) => asset.asset_id !== 'sha256:a' }); - - const { ids, text } = await hubGene(proxyFetch, LONG_TASK, { judge }); - assert.deepEqual(ids, ['sha256:b', 'sha256:c']); - assert.doesNotMatch(text, /sha256:a one/); -}); - -test('when the judge accepts nothing, nothing is injected', async () => { - const { proxyFetch } = stubProxy({ assets: [geneOf('sha256:a'), geneOf('sha256:b')] }); - const { judge } = stubJudge({ usable: () => false }); - const outcomes = []; - - assert.deepEqual(await hubGene(proxyFetch, LONG_TASK, { judge, onOutcome: (outcome) => outcomes.push(outcome) }), { ids: [], text: '' }); - assert.equal(outcomes[0].status, 'none_usable'); - assert.deepEqual(outcomes[0].review, { judged: 2, usable: 0, failed: 0 }); + assert.equal(outcomes[0].query, 'reconcile ledger against bank export'); }); test('a failed rewrite searches with the prompt prefix and says why', async () => { - const { proxyFetch, calls } = stubProxy({ assets: [geneOf('sha256:a')] }); - const { judge } = stubJudge({ rewriteError: new Error('gateway returned HTTP 503') }); + const { proxyFetch, calls } = stubProxy({ assets: [ASSET] }); + const { rewrite } = stubRewrite({ error: new Error('provider returned 503') }); const outcomes = []; - const { ids } = await hubGene(proxyFetch, LONG_TASK, { judge, onOutcome: (outcome) => outcomes.push(outcome) }); + const { ids } = await hubGene(proxyFetch, LONG_TASK, { rewrite, onOutcome: (outcome) => outcomes.push(outcome) }); + assert.equal(calls[0].body.text, LONG_TASK.slice(0, 400)); - assert.deepEqual(ids, ['sha256:a'], 'the review still runs'); + assert.deepEqual(ids, ['sha256:abc']); assert.equal(outcomes[0].query_source, 'prompt_prefix'); - assert.deepEqual(outcomes[0].fallbacks, { rewrite: 'gateway returned HTTP 503' }); + assert.equal(outcomes[0].rewrite_error, 'provider returned 503'); }); -test('a failed review injects the best candidate unreviewed, as without a judge, and says why', async () => { - const { proxyFetch } = stubProxy({ assets: [geneOf('sha256:a', 0.7), geneOf('sha256:b', 0.9)] }); - const { judge } = stubJudge({ reviewError: new Error('verdict is not JSON: maybe') }); +test('without a rewrite the prompt prefix is the query and nothing extra is logged', async () => { + const { proxyFetch, calls } = stubProxy({ assets: [ASSET] }); const outcomes = []; - - const { ids } = await hubGene(proxyFetch, LONG_TASK, { judge, onOutcome: (outcome) => outcomes.push(outcome) }); - assert.deepEqual(ids, ['sha256:b']); - assert.equal(outcomes[0].review, null); - assert.deepEqual(outcomes[0].fallbacks, { review: 'verdict is not JSON: maybe' }); + await hubGene(proxyFetch, LONG_TASK, { onOutcome: (outcome) => outcomes.push(outcome) }); + assert.deepEqual(calls[0].body, { text: LONG_TASK.slice(0, 400) }); + assert.equal('query_source' in outcomes[0], false); }); -test('without a judge the prompt prefix is the query and the Hub default decides how many', async () => { - const { proxyFetch, calls } = stubProxy({ assets: [geneOf('sha256:a')] }); - await hubGene(proxyFetch, LONG_TASK); - assert.deepEqual(calls[0].body, { text: LONG_TASK.slice(0, 400) }); +test('the rewrite reads the widened task, while a fallback searches with this turn\'s prompt alone', async () => { + const { proxyFetch, calls } = stubProxy({ assets: [ASSET] }); + const { rewrite, tasks } = stubRewrite({ error: new Error('down') }); + const task = `${LONG_TASK}\n\nContinue with the task from where you stopped.`; + + await hubGene(proxyFetch, 'Continue with the task from where you stopped.', { task, rewrite }); + assert.deepEqual(tasks, [task]); + assert.equal(calls[0].body.text, 'Continue with the task from where you stopped.'); }); test('the task text for recall keeps up to six thousand characters, the default still four hundred', () => { @@ -381,18 +345,7 @@ test('the task text for recall keeps up to six thousand characters, the default assert.equal(promptTextOf(messages, 6_000).length, 6_000); }); -test('the judge reads the widened task, while a fallback searches with this turn\'s prompt alone', async () => { - const { proxyFetch, calls } = stubProxy({ assets: [geneOf('sha256:a')] }); - const { judge, calls: judged } = stubJudge({ rewriteError: new Error('down') }); - const task = `${LONG_TASK}\n\nContinue with the task from where you stopped.`; - - await hubGene(proxyFetch, 'Continue with the task from where you stopped.', { task, judge }); - assert.deepEqual(judged.rewrite, [task]); - assert.equal(judged.review[0].task, task); - assert.equal(calls[0].body.text, 'Continue with the task from where you stopped.'); -}); - -test('a follow-up turn is judged with the opening prompt in front of it, and the follow-up always fits', () => { +test('a follow-up turn is rewritten with the opening prompt in front of it, and the follow-up always fits', () => { assert.equal(taskWithOpening('fix the ledger', 'fix the ledger'), 'fix the ledger'); assert.equal(taskWithOpening('fix the ledger', 'continue'), 'fix the ledger\n\ncontinue'); const widened = taskWithOpening('x'.repeat(6_000), 'continue'); diff --git a/test/recall-judge.test.js b/test/recall-judge.test.js deleted file mode 100644 index 1813eb9..0000000 --- a/test/recall-judge.test.js +++ /dev/null @@ -1,102 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'node:test'; - -import { createRecallJudge } from '../src/recall-judge.js'; - -function gateway(answer) { - const requests = []; - const fetchImpl = async (url, init) => { - const body = JSON.parse(init.body); - requests.push({ url, headers: init.headers, body }); - const reply = typeof answer === 'function' ? answer(body) : answer; - if (reply instanceof Error) throw reply; - if (typeof reply === 'number') return { ok: false, status: reply, json: async () => ({}) }; - return { ok: true, status: 200, json: async () => ({ choices: [{ message: { content: reply } }] }) }; - }; - return { fetchImpl, requests }; -} - -const ASSET = { - asset_id: 'sha256:a', - short_title: 'Reconcile a ledger', - payload: { summary: 's'.repeat(900), category: 'finance', signals_match: ['ledger', 'bank export'] }, -}; - -test('no gateway key means no judge', () => { - assert.equal(createRecallJudge({ apiKey: '' }), null); -}); - -test('the rewrite sends the bounded task to gemini-3.5-flash and returns the bare query', async () => { - const { fetchImpl, requests } = gateway(' "reconcile ledger against bank export"\n'); - const judge = createRecallJudge({ apiKey: 'k', fetchImpl }); - - assert.equal(await judge.searchQueryOf('t'.repeat(9_000)), 'reconcile ledger against bank export'); - assert.equal(requests[0].url, 'https://api.evomap.ai/v1/chat/completions'); - assert.equal(requests[0].headers.Authorization, 'Bearer k'); - assert.equal(requests[0].body.model, 'gemini-3.5-flash'); - const prompt = requests[0].body.messages[0].content; - assert.match(prompt, /^Rewrite the task below into ONE search query/); - assert.ok(prompt.endsWith(`## Task\n${'t'.repeat(6_000)}`)); -}); - -test('an empty or failed rewrite throws, so the caller can fall back', async () => { - await assert.rejects(createRecallJudge({ apiKey: 'k', fetchImpl: gateway(' ').fetchImpl }).searchQueryOf('task'), /empty/); - await assert.rejects(createRecallJudge({ apiKey: 'k', fetchImpl: gateway(503).fetchImpl }).searchQueryOf('task'), /HTTP 503/); -}); - -test('each candidate is judged on its own card against the bounded task', async () => { - const { fetchImpl, requests } = gateway((body) => (body.messages[0].content.includes('Title: keep') ? '{"use": true}' : '{"use": false}')); - const judge = createRecallJudge({ apiKey: 'k', fetchImpl }); - const keep = { ...ASSET, asset_id: 'sha256:keep', short_title: 'keep' }; - - const { usable, failed } = await judge.usableOf('t'.repeat(5_000), [ASSET, keep]); - assert.deepEqual(usable.map((asset) => asset.asset_id), ['sha256:keep']); - assert.equal(failed, 0); - assert.equal(requests.length, 2); - assert.deepEqual(requests[0].body.response_format, { type: 'json_object' }); - const prompt = requests[0].body.messages[0].content; - assert.ok(prompt.includes(`## Task\n${'t'.repeat(4_000)}\n\n## Playbook`)); - assert.match(prompt, /^Title: Reconcile a ledger$/m); - assert.match(prompt, new RegExp(`^Summary: ${'s'.repeat(600)}$`, 'm')); - assert.match(prompt, /^Category: finance$/m); - assert.match(prompt, /^Triggers: ledger; bank export$/m); - assert.match(prompt, /Answer with JSON only: \{"use": true\|false\}$/); -}); - -test('a candidate whose verdict failed is not used; the review fails only when every verdict did', async () => { - const mixed = gateway((body) => (body.messages[0].content.includes('Title: keep') ? '{"use": true}' : 'not json')); - const { usable, failed } = await createRecallJudge({ apiKey: 'k', fetchImpl: mixed.fetchImpl }) - .usableOf('task', [ASSET, { ...ASSET, short_title: 'keep' }]); - assert.deepEqual(usable.map((asset) => asset.short_title), ['keep']); - assert.equal(failed, 1); - - await assert.rejects(createRecallJudge({ apiKey: 'k', fetchImpl: gateway(new Error('timeout')).fetchImpl }).usableOf('task', [ASSET]), /timeout/); -}); - -test('a recalled GEP body without a short title is judged by its local id and trigger', async () => { - const { fetchImpl, requests } = gateway('{"use": true}'); - await createRecallJudge({ apiKey: 'k', fetchImpl }).usableOf('task', [{ - id: 'capsule_legal_writer', asset_id: 'sha256:c', type: 'Capsule', summary: 'Draft legal documents.', trigger: ['draft a contract', 'write a legal memo'], - }]); - const prompt = requests[0].body.messages[0].content; - assert.match(prompt, /^Title: capsule_legal_writer$/m); - assert.match(prompt, /^Summary: Draft legal documents\.$/m); - assert.match(prompt, /^Triggers: draft a contract; write a legal memo$/m); -}); - -test('a busy or failing gateway is retried twice with backoff, a refusal is not', async () => { - const waits = []; - const sleep = async (ms) => { waits.push(ms); }; - let calls = 0; - const flaky = gateway(() => (++calls < 3 ? 429 : 'ledger reconciliation')); - assert.equal(await createRecallJudge({ apiKey: 'k', fetchImpl: flaky.fetchImpl, sleep }).searchQueryOf('task'), 'ledger reconciliation'); - assert.deepEqual(waits, [1_000, 3_000]); - - const busy = gateway(503); - await assert.rejects(createRecallJudge({ apiKey: 'k', fetchImpl: busy.fetchImpl, sleep }).searchQueryOf('task'), /HTTP 503/); - assert.equal(busy.requests.length, 3); - - const refused = gateway(401); - await assert.rejects(createRecallJudge({ apiKey: 'k', fetchImpl: refused.fetchImpl, sleep }).searchQueryOf('task'), /HTTP 401/); - assert.equal(refused.requests.length, 1); -}); diff --git a/test/recall-query.test.js b/test/recall-query.test.js new file mode 100644 index 0000000..655cf1e --- /dev/null +++ b/test/recall-query.test.js @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { rewrittenQueryOf } from '../src/recall-query.js'; + +function stubLlm(chunks) { + const calls = []; + const llm = { + async *stream(options) { + calls.push(options); + yield* chunks; + }, + }; + return { llm, calls }; +} + +const ROUTE = { provider: 'evomap', model: 'deepseek-flash' }; + +test('the rewrite runs on the session\'s own provider and model and returns the bare query', async () => { + const { llm, calls } = stubLlm([ + { type: 'reasoning-delta', index: 0, text: 'thinking' }, + { type: 'text-delta', index: 1, text: ' "reconcile ledger ' }, + { type: 'text-delta', index: 1, text: 'against bank export"\n' }, + { type: 'finish', reason: { kind: 'stop' } }, + ]); + + assert.equal(await rewrittenQueryOf(llm, ROUTE, 't'.repeat(9_000)), 'reconcile ledger against bank export'); + assert.equal(calls[0].provider, 'evomap'); + assert.equal(calls[0].model, 'deepseek-flash'); + const prompt = calls[0].messages[0].content[0].text; + assert.match(prompt, /^Rewrite the task below into ONE search query/); + assert.ok(prompt.endsWith(`## Task\n${'t'.repeat(6_000)}`)); +}); + +test('a task with dollar signs is placed into the prompt verbatim', async () => { + const { llm, calls } = stubLlm([{ type: 'text-delta', index: 0, text: 'q' }, { type: 'finish', reason: { kind: 'stop' } }]); + await rewrittenQueryOf(llm, ROUTE, 'cost is $& and $1'); + assert.ok(calls[0].messages[0].content[0].text.endsWith('## Task\ncost is $& and $1')); +}); + +test('a failed or empty rewrite throws, so the caller can fall back to the prompt prefix', async () => { + const failed = stubLlm([{ type: 'finish', reason: { kind: 'error', failure: { message: 'provider returned 503', code: 'http' } } }]); + await assert.rejects(rewrittenQueryOf(failed.llm, ROUTE, 'task'), /provider returned 503/); + + const empty = stubLlm([{ type: 'text-delta', index: 0, text: ' "" ' }, { type: 'finish', reason: { kind: 'stop' } }]); + await assert.rejects(rewrittenQueryOf(empty.llm, ROUTE, 'task'), /came back empty/); +}); From edb78db1962c996df53c5987b0f4fa3f0dbc08e5 Mon Sep 17 00:00:00 2001 From: forrestlinfeng Date: Fri, 23 Oct 2026 17:17:28 +0800 Subject: [PATCH 5/6] Name the rewrite test's route as the session's own, not a real provider Co-Authored-By: Claude Opus 5.5 --- test/recall-query.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/recall-query.test.js b/test/recall-query.test.js index 655cf1e..984fc42 100644 --- a/test/recall-query.test.js +++ b/test/recall-query.test.js @@ -14,7 +14,7 @@ function stubLlm(chunks) { return { llm, calls }; } -const ROUTE = { provider: 'evomap', model: 'deepseek-flash' }; +const ROUTE = { provider: 'session-provider', model: 'session-model' }; test('the rewrite runs on the session\'s own provider and model and returns the bare query', async () => { const { llm, calls } = stubLlm([ @@ -25,8 +25,8 @@ test('the rewrite runs on the session\'s own provider and model and returns the ]); assert.equal(await rewrittenQueryOf(llm, ROUTE, 't'.repeat(9_000)), 'reconcile ledger against bank export'); - assert.equal(calls[0].provider, 'evomap'); - assert.equal(calls[0].model, 'deepseek-flash'); + assert.equal(calls[0].provider, 'session-provider'); + assert.equal(calls[0].model, 'session-model'); const prompt = calls[0].messages[0].content[0].text; assert.match(prompt, /^Rewrite the task below into ONE search query/); assert.ok(prompt.endsWith(`## Task\n${'t'.repeat(6_000)}`)); From ce51ef3d9a59e9d874720a41c19b22ae5ee81b61 Mon Sep 17 00:00:00 2001 From: forrestlinfeng Date: Fri, 23 Oct 2026 17:21:35 +0800 Subject: [PATCH 6/6] Test that recall rewrites only with an llm service and a model route Co-Authored-By: Claude Opus 5.5 --- test/apply.test.js | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/apply.test.js b/test/apply.test.js index d9a3a47..fd16eba 100644 --- a/test/apply.test.js +++ b/test/apply.test.js @@ -328,3 +328,37 @@ test('a fetched asset renders as reusable prose, not the raw envelope', async () assert.doesNotMatch(text, /signals_match|source_node_id|gdi_score/); }); + +test('a turn is searched with a rewritten query only when the session has an llm service and a model route', async () => { + const projectDir = projectDirectory('evolver-rewrite-'); + const proxy = await startProxy(() => ({ assets: [] })); + const restoreHome = freshHome('evolver-rewrite-home-'); + + try { + const routes = []; + const { ctx, listeners } = fakeContext(); + ctx.llm = { + async *stream({ provider, model }) { + routes.push({ provider, model }); + yield { type: 'text-delta', index: 0, text: 'retry uploads with backoff' }; + yield { type: 'finish', reason: { kind: 'stop' } }; + }, + }; + apply(ctx, Config({ projectDir, proxyPort: proxy.port })); + + const { agent: routed } = fakeAgent({ id: 'routed', cwd: projectDir }); + routed.options = { provider: 'session-provider', model: 'session-model' }; + await primedBy(listeners, routed, 'add a retry to the uploader', 1); + const { agent: unrouted } = fakeAgent({ id: 'unrouted', cwd: projectDir }); + await primedBy(listeners, unrouted, 'add a retry to the uploader', 1); + + assert.deepEqual(routes, [{ provider: 'session-provider', model: 'session-model' }]); + assert.deepEqual( + proxy.requests.filter((request) => request.path === '/asset/fetch').map((request) => request.body), + [{ text: 'retry uploads with backoff' }, { text: 'add a retry to the uploader' }], + ); + } finally { + restoreHome(); + proxy.close(); + } +});