From 2c665fe606a479f2cda8a41d0af031680de9f1e1 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Wed, 14 Oct 2026 16:08:21 +0530 Subject: [PATCH 1/4] Phase 7 tasks 1 to 4: the detail variant and a session source for each fact Tasks 1 and 2 read the 19 misses of sample A that no stored fact answers. In 14 of them, the user stated the fact in passing, in a turn about another topic. In 5, the answer is a value that the assistant said. Task 2 found no reply at the token limit, no reply that fails to parse, and no near fact in "Facts already in memory". plan.md records the findings. The variant `detail` asks the extractor to read every turn, to keep a fact that the user states in passing, to keep each item of a list, and to write absolute dates. Each fact now carries the source `session:`, with 12 hex digits of the SHA-256 of the session id. The block shows sources to the reader, and an evidence session id holds `answer`, so the source does not show the id. `--no-fact-sources` runs the memory arm as phases 5 and 6 did. The diagnosis gives the session of each fact that holds the answer. --- bench/README.md | 9 +++++-- bench/diagnose.py | 33 ++++++++++++++++++++--- bench/e2e.py | 40 +++++++++++++++++++++++----- bench/prompts.py | 43 ++++++++++++++++++++++++++++-- bench/tests/test_diagnose.py | 16 +++++++++-- bench/tests/test_e2e.py | 51 ++++++++++++++++++++++++++++++++++++ docs/internal/plan.md | 36 ++++++++++++++++++++++++- 7 files changed, 211 insertions(+), 17 deletions(-) diff --git a/bench/README.md b/bench/README.md index 84751ce..34ad0b4 100644 --- a/bench/README.md +++ b/bench/README.md @@ -86,6 +86,7 @@ The options are these: | `--recall-k` | the `k` of the memory recall | `30` | | `--recall-budget` | the `budget` of the memory recall, in characters | `8000` | | `--extractor` | a variant name in `VARIANTS` of `bench/prompts.py` | `baseline` | +| `--no-fact-sources` | write the facts with no session source, as phases 5 and 6 did | off | | `--replay` | call no model, and stop at the first reply that the cache does not hold | off | | `--workers` | the number of processes that run the questions | `1` | | `--tag` | a suffix for the name of the result file | none | @@ -94,7 +95,10 @@ The options are these: Each worker gets the questions in turn, and runs them over a new local database of its own. Each table then holds one question at a time, as in a run with one worker. BM25 ranks over the whole table before the scope filter, so the rows of a second question in a table -can change the ranking. An extractor variant names a prompt and a model. The result file is +can change the ranking. An extractor variant names a prompt and a model. Each fact carries +the source `session:`, where the hash is the first 12 hex digits of the SHA-256 of +the session id. The block shows the sources to the reader, and the id of an evidence +session holds `answer`, so the source does not show the id. The result file is `bench/results/e2e-sample-[-].json`, and it records each option. ## The checks of the results @@ -128,4 +132,5 @@ Each miss then gets one of four causes: For the last cause, GPT-4o tells whether the marked facts in the block hold every value that the answer needs ("all present") or only some of them ("some present"). The command prints the groups for each question type, and writes them next to the result file, as -`-diagnosis.json`. +`-diagnosis.json`. For each fact that holds the answer, the file gives its session +id, if the fact has a source. \ No newline at end of file diff --git a/bench/diagnose.py b/bench/diagnose.py index b4bef3e..3d3a423 100644 --- a/bench/diagnose.py +++ b/bench/diagnose.py @@ -24,7 +24,16 @@ from bench import prompts from bench.data import DATA_DIR, DATA_FILE, QUESTION_TYPES, load -from bench.e2e import ACTOR, BASELINE, RECALL_BUDGET, RECALL_K, Config, extractor, local_store +from bench.e2e import ( + ACTOR, + BASELINE, + RECALL_BUDGET, + RECALL_K, + Config, + extractor, + fact_source, + local_store, +) from bench.embed import CACHE_DIR from bench.models import JUDGE, CachedModels, Offline, Remote from bench.retrieval import RANKINGS, RESULTS_DIR, _commit @@ -103,7 +112,8 @@ def verdict(holding: set[str], best_rank: int | None, block: set[str], k: int = def memory_config(result: dict[str, Any]) -> Config: """Return the configuration of the memory arm that a result file records. - A result file of phase 5 records no `memory` section, and its run used `BASELINE`. + A result file of phase 5 records no `memory` section, and its run used `BASELINE`. A + result file of phase 6 records no `fact_sources`, and its facts carried no source. """ found = result["config"].get("memory") if found is None: @@ -113,6 +123,7 @@ def memory_config(result: dict[str, Any]) -> Config: k=found["k"], budget=found["budget"], extractor=found["extractor"], + fact_sources=found.get("fact_sources", False), ) @@ -154,15 +165,16 @@ def main(argv: Sequence[str] | None = None) -> Path: run_id = f"diagnose-{datetime.now(UTC):%Y%m%d%H%M%S}-{uuid.uuid4().hex[:6]}" with local_store(args.url, run_id) as (stores, empty): memory = Memory(stores[config.memory_ranking]) - extract = extractor(replay, config.variant) for count, question_id in enumerate(missed, start=1): question = questions[question_id] namespace = ("bench", "lme_s", run_id, question.id) + sessions = {fact_source(session.id): session.id for session in question.sessions} for session in question.sessions: + source = fact_source(session.id) if config.fact_sources else "" memory.capture( prompts.session_text(session), namespace, - extract, + extractor(replay, config.variant, source), actor=ACTOR, observed_at=session.date, ) @@ -189,6 +201,7 @@ def main(argv: Sequence[str] | None = None) -> Path: "type": question.type, "facts": len(facts), "holding": [facts[n - 1].content for n in marked], + "holding_sessions": [session_of_fact(facts[n - 1], sessions) for n in marked], "best_rank": ranks, "cause": cause, "present": present, @@ -207,6 +220,7 @@ def main(argv: Sequence[str] | None = None) -> Path: "k": config.k, "budget": config.budget, "extractor": config.extractor, + "fact_sources": config.fact_sources, }, "marker": JUDGE, "hotmemory_commit": commit, @@ -224,6 +238,17 @@ def main(argv: Sequence[str] | None = None) -> Path: return path +def session_of_fact(fact: Record, sessions: dict[str, str]) -> str | None: + """Return the session id of `fact` from its sources, or None if no source names one. + + `sessions` maps the result of `fact_source` to the session id. + """ + for source in fact.sources: + if source in sessions: + return sessions[source] + return None + + def _mark(marker: CachedModels, question: str, answer: str, facts: Sequence[Record]) -> list[int]: """Return the numbers, from 1, of the facts that the marker says hold the answer.""" numbered = "\n".join(f"{n}. {fact.content}" for n, fact in enumerate(facts, 1)) diff --git a/bench/e2e.py b/bench/e2e.py index 2c9d6e2..2f16052 100644 --- a/bench/e2e.py +++ b/bench/e2e.py @@ -14,6 +14,7 @@ from __future__ import annotations import argparse +import hashlib import json import multiprocessing import random @@ -71,6 +72,7 @@ class Config: `retrieval_ranking` ranks the sessions of the retrieval arm. The memory arm captures and recalls with `memory_ranking`, recalls `k` records into a block of at most `budget` characters, and extracts with the variant `extractor` of `prompts.VARIANTS`. + With `fact_sources`, each fact carries the source of its session from `fact_source`. """ retrieval_ranking: str = "fused" @@ -78,6 +80,7 @@ class Config: k: int = RECALL_K budget: int = RECALL_BUDGET extractor: str = "baseline" + fact_sources: bool = True def __post_init__(self) -> None: for ranking in (self.retrieval_ranking, self.memory_ranking): @@ -94,7 +97,7 @@ def variant(self) -> prompts.Variant: return prompts.VARIANTS[self.extractor] -BASELINE = Config() +BASELINE = Config(fact_sources=False) """The configuration of the phase 5 run.""" @@ -147,10 +150,23 @@ def sample_ids(name: str, questions: Sequence[Question], root: Path = HERE.paren return ids +def fact_source(session_id: str) -> str: + """Return the source of the facts of a session: `session:` and 12 hex digits of its SHA-256. + + The reader sees the sources in the block, and an evidence session id holds `answer`, so + the source does not show the id. + """ + return "session:" + hashlib.sha256(session_id.encode()).hexdigest()[:12] + + def extractor( - models: Models, variant: prompts.Variant = prompts.VARIANTS["baseline"] + models: Models, variant: prompts.Variant = prompts.VARIANTS["baseline"], source: str = "" ) -> Callable[[str, datetime | None, list[Record]], list[Fact]]: - """Return the extractor callable of `capture`: one call to the variant's model per session.""" + """Return the extractor callable of `capture`: one call to the variant's model per session. + + With `source`, each fact carries it in `sources`. + """ + sources = (source,) if source else () def extract(text: str, observed_at: datetime | None, current: list[Record]) -> list[Fact]: date = observed_at.strftime("%Y/%m/%d (%a) %H:%M") if observed_at else "unknown" @@ -159,7 +175,12 @@ def extract(text: str, observed_at: datetime | None, current: list[Record]) -> l reply = models(variant.model, prompt, EXTRACTOR_MAX_TOKENS, prompts.EXTRACT_SCHEMA) facts = json.loads(reply.text)["facts"] return [ - Fact(kind="fact", subject=item["subject"].strip(), content=item["content"].strip()) + Fact( + kind="fact", + subject=item["subject"].strip(), + content=item["content"].strip(), + sources=sources, + ) for item in facts if item["content"].strip() ] @@ -220,14 +241,14 @@ def run_question( if "memory" in arms: namespace = ("bench", "lme_s", run_id, "memory", question.id) memory = Memory(stores[config.memory_ranking]) - extract = extractor(models, config.variant) written = 0 for session in question.sessions: + source = fact_source(session.id) if config.fact_sources else "" written += len( memory.capture( prompts.session_text(session), namespace, - extract, + extractor(models, config.variant, source), actor=ACTOR, observed_at=session.date, ) @@ -454,6 +475,11 @@ def main(argv: Sequence[str] | None = None) -> Path | None: "--recall-budget", type=int, default=RECALL_BUDGET, help="characters of the memory block" ) parser.add_argument("--extractor", choices=tuple(prompts.VARIANTS), default="baseline") + parser.add_argument( + "--no-fact-sources", + action="store_true", + help="write the facts with no session source, as phases 5 and 6 did", + ) parser.add_argument( "--replay", action="store_true", help="call no model: use cached replies, stop at a miss" ) @@ -478,6 +504,7 @@ def main(argv: Sequence[str] | None = None) -> Path | None: k=args.recall_k, budget=args.recall_budget, extractor=args.extractor, + fact_sources=not args.no_fact_sources, ) except ValueError as error: parser.error(str(error)) @@ -560,6 +587,7 @@ def main(argv: Sequence[str] | None = None) -> Path | None: "k": config.k, "budget": config.budget, "extractor": config.extractor, + "fact_sources": config.fact_sources, }, "arms": { "full": "every session of the question", diff --git a/bench/prompts.py b/bench/prompts.py index 1cd452b..69974e6 100644 --- a/bench/prompts.py +++ b/bench/prompts.py @@ -86,6 +86,38 @@ Session date: {date} +Session: +{session}""" +EXTRACT_DETAIL = """You read one chat session between a user and an assistant, and you write \ +down the facts that a personal assistant must remember about the user for later \ +conversations. + +Read every turn, from the first turn to the last turn. The user often states a fact about \ +their own life in passing, in a turn about another topic, for example "By the way, I just \ +...". Such a fact counts as much as the main topic of the session. Keep each one: what the \ +user did, bought, owns, likes, plans, or feels, with its time, place, and amount. + +If the user can ask later about what the assistant said, keep it: a recommendation, a \ +name, a number, a step, or a result. Keep each item of a list as its own fact. If the list \ +is numbered, keep the number of the item. + +Write each fact as one short sentence that stands alone. Name the user as "the user". Keep \ +names, numbers, places, and amounts exactly as the session gives them. Write each date as \ +an absolute date, from the session date: for "yesterday" in a session of 2023/05/21, write \ +"on 2023/05/20". Leave out small talk, and leave out general knowledge that does not concern \ +the user and that the assistant did not tell the user. + +The memory already holds the facts below. Do not repeat one of them. If the session changes \ +one of them, write the new fact in full. + +Facts already in memory: +{known} + +Give `subject` as a short label of what the fact is about, such as "car" or "sister's \ +wedding". Return an empty list if the session holds no fact to remember. + +Session date: {date} + Session: {session}""" EXTRACT_SCHEMA = { @@ -114,8 +146,15 @@ class Variant: model: str -VARIANTS = {"baseline": Variant(EXTRACT, EXTRACTOR)} -"""The extractor variants by name. `baseline` is the extractor of phase 5.""" +VARIANTS = { + "baseline": Variant(EXTRACT, EXTRACTOR), + "detail": Variant(EXTRACT_DETAIL, EXTRACTOR), +} +"""The extractor variants by name. `baseline` is the extractor of phase 5. + +`detail` asks for each fact that the user states in passing, each item of a list that the +assistant gives, and absolute dates. +""" def history(sessions: Sequence[Session]) -> str: diff --git a/bench/tests/test_diagnose.py b/bench/tests/test_diagnose.py index 2f621b7..2cfe33f 100644 --- a/bench/tests/test_diagnose.py +++ b/bench/tests/test_diagnose.py @@ -1,11 +1,12 @@ +from dataclasses import replace from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import pytest -from bench.diagnose import block_ids, group, memory_config, table, verdict -from bench.e2e import BASELINE +from bench.diagnose import block_ids, group, memory_config, session_of_fact, table, verdict +from bench.e2e import BASELINE, fact_source from bench.models import READER, CachedModels, CacheMiss, Offline, Reply from hotmemory import Record @@ -55,6 +56,17 @@ def test_memory_config_reads_the_result_file() -> None: } ) assert (found.memory_ranking, found.k, found.budget) == ("vector", 50, 900) + assert not found.fact_sources + memory = {"ranking": "fused", "k": 30, "budget": 8000, "extractor": "detail"} + sourced = memory_config({"config": {"memory": {**memory, "fact_sources": True}}}) + assert (sourced.extractor, sourced.fact_sources) == ("detail", True) + + +def test_session_of_fact_reads_the_source() -> None: + sessions = {fact_source("answer_a_1"): "answer_a_1"} + sourced = replace(fact("k"), sources=(fact_source("answer_a_1"),)) + assert session_of_fact(sourced, sessions) == "answer_a_1" + assert session_of_fact(fact("k"), sessions) is None def test_table_splits_the_recalled_group() -> None: diff --git a/bench/tests/test_e2e.py b/bench/tests/test_e2e.py index 3c143ff..7038012 100644 --- a/bench/tests/test_e2e.py +++ b/bench/tests/test_e2e.py @@ -13,6 +13,8 @@ Config, choose_sample, estimate, + extractor, + fact_source, run_question, sample_ids, shards_of, @@ -211,3 +213,52 @@ def test_shards_deal_every_question_once(questions: dict[str, Question]) -> None assert [q.id for q in shards[1].questions] == ["q1", "q5", "q9"] assert [shard.run_id for shard in shards_of(pool, 1, "r1")] == ["r1"] assert len(shards_of(pool[:2], 4, "r1")) == 2 + + +def test_each_variant_formats_and_keeps_the_baseline() -> None: + assert prompts.VARIANTS["baseline"] == prompts.Variant(prompts.EXTRACT, EXTRACTOR) + for name, variant in prompts.VARIANTS.items(): + text = prompts.extract("user: hi", "2023/05/21 (Sun) 10:00", ["- old fact"], variant.prompt) + assert text.endswith("Session date: 2023/05/21 (Sun) 10:00\n\nSession:\nuser: hi"), name + assert "Facts already in memory:\n- old fact\n" in text, name + detail = prompts.VARIANTS["detail"].prompt + assert "in passing" in detail + assert "Keep each item of a list as its own fact" in detail + assert "absolute date" in detail + + +def test_fact_source_hides_the_session_id() -> None: + source = fact_source("answer_a_1") + assert source == fact_source("answer_a_1") + assert source != fact_source("answer_a_2") + assert source.startswith("session:") + assert len(source) == len("session:") + 12 + assert "answer" not in source + + +def test_extractor_adds_the_source(questions: dict[str, Question]) -> None: + session = questions["q_one"].sessions[0] + text = prompts.session_text(session) + plain = extractor(FakeModels())(text, session.date, []) + sourced = extractor(FakeModels(), source="session:abc")(text, session.date, []) + assert plain and all(fact.sources == () for fact in plain) + assert [fact.sources for fact in sourced] == [("session:abc",)] * len(plain) + + +def test_run_question_gives_each_fact_its_session(questions: dict[str, Question]) -> None: + question = questions["q_one"] + store = MemoryStore(embedder=fake_embedder, clock=FakeClock()) + models = FakeModels() + + def empty(namespace: Sequence[str]) -> None: + for record in store.list(namespace, limit=1000): + store.delete(record.namespace, record.key) + + run_question( + question, models, {"fused": store, "vector": store}, empty, "r1", ("memory",), Config() + ) + block = [p for m, p in models.prompts if m == READER and "several facts" in p[:60]][-1] + late = next(s for s in question.sessions if s.id == "answer_a_late") + assert f"[sources: {fact_source(late.id)}]" in block + assert "[sources: none]" not in block + assert "answer_" not in block diff --git a/docs/internal/plan.md b/docs/internal/plan.md index be7ed60..a400274 100644 --- a/docs/internal/plan.md +++ b/docs/internal/plan.md @@ -127,7 +127,7 @@ Tasks: Read from `bench/models.py` and `bench/e2e.py`, not observed: a reply that reaches the limit raises before the cache stores it. A reply that fails to parse is stored in the cache, but it stops the run. So no completed result file holds either fault, and this - task confirms it. + task makes sure that this is true. 3. Write a new prompt variant with these rules: - If the user can ask later about what the assistant said, keep it. - Keep each item of a list as its own fact. @@ -156,6 +156,40 @@ Stop and ask if: - A variant writes more than three times the facts of the baseline. The block then fills faster, and the owner decides the trade. +What tasks 1 and 2 found, observed on 2026-10-14 by a replay of `capture` from the cache +of the sample A run: + +- The phase 5 finding holds for 5 of the 19 misses. Each of the 5 is of the type + single-session-assistant, and the answer is a value that the assistant said. The + baseline prompt already asks for such facts. +- The other 14 misses are a fact that the user stated in passing, in a turn about another + topic, such as "By the way, I just repainted my bedroom walls a lighter shade of gray". +- The first evidence turn is turn 0 or turn 2 in 21 of the 26 evidence sessions. The + extractor wrote 126 facts from these sessions, and about 21 percent of them come from + the first third of the session. A word-overlap match between each fact and each turn + gave this share, so it is approximate. +- Task 2 found no fault. No cached extractor reply fails to parse, and the largest has + 2,602 output tokens, under the limit of 4,000. No list of "Facts already in memory" for + the 26 evidence sessions holds a fact near the answer. + +Decisions taken during the work. The owner can change any of them in review. + +- The variant `detail` adds the rules of task 3 to the baseline prompt. It also tells the + extractor to read every turn, and to keep a fact that the user states in passing. +- The source of a fact is `session:` and the first 12 hex digits of the SHA-256 of the + session id, not `session:` (agreed with the owner on 2026-10-14). The block shows + the sources to the reader, and LongMemEval names each evidence session `answer_...`. + The plain id tells the reader which facts hold the answer. The diagnosis maps the + source back to the session id. The episodes of the retrieval arm keep `session:`, + because the reader does not see their sources. +- The sources make each line of the block longer and change the reader prompt. They + also change the extractor prompt. `capture` gets "Facts already in memory" from + `recall` with its default budget of 2,000 characters, and a longer line lets fewer + facts fit (observed: 9 in place of 10 for question `66f24dbb`). So the extractor cache + of phase 6 does not replay with sources. The baseline runs again with sources, at the + full price, as the comparison point of this phase. `--no-fact-sources` runs the arm as phases 5 and 6 did, and a + result file records `fact_sources`. + ## Phase 8. Episodes behind facts Goal: `recall` can give the reader the source text of its facts. The retrieval arm shows From a9d87c0105ba2fc5cd11c6d7350e7151c2eb6fbe Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Wed, 14 Oct 2026 18:00:44 +0530 Subject: [PATCH 2/4] Give no facts for a session whose extractor reply reaches the token limit The first runs with sources stopped on question a3332713. The extractor reply for one session reached the limit of 4,000 output tokens. In that session the user pastes a list of survey answers from other people. A reply with a schema that reaches its limit now comes back with `stopped` set, and the cache keeps it. The extractor writes no facts for that session, so a replay and the diagnosis skip the same session. Each result row and the result file count these sessions in `sessions_at_limit`. The marker calls of the diagnosis still stop the run at the limit. --- bench/README.md | 7 +++++-- bench/diagnose.py | 6 +++++- bench/e2e.py | 33 +++++++++++++++++++++++++++------ bench/models.py | 13 +++++++++---- bench/tests/test_e2e.py | 33 ++++++++++++++++++++++++++++++++- docs/internal/plan.md | 17 +++++++++++++---- 6 files changed, 91 insertions(+), 18 deletions(-) diff --git a/bench/README.md b/bench/README.md index 34ad0b4..38c218c 100644 --- a/bench/README.md +++ b/bench/README.md @@ -98,8 +98,11 @@ over the whole table before the scope filter, so the rows of a second question i can change the ranking. An extractor variant names a prompt and a model. Each fact carries the source `session:`, where the hash is the first 12 hex digits of the SHA-256 of the session id. The block shows the sources to the reader, and the id of an evidence -session holds `answer`, so the source does not show the id. The result file is -`bench/results/e2e-sample-[-].json`, and it records each option. +session holds `answer`, so the source does not show the id. If an extractor reply reaches +its limit of 4,000 output tokens, the session writes no facts. The cache keeps that reply, +so a replay skips the same session. The result file counts these sessions in +`sessions_at_limit`. The result file is `bench/results/e2e-sample-[-].json`, +and it records each option. ## The checks of the results diff --git a/bench/diagnose.py b/bench/diagnose.py index 3d3a423..0e688a7 100644 --- a/bench/diagnose.py +++ b/bench/diagnose.py @@ -174,7 +174,7 @@ def main(argv: Sequence[str] | None = None) -> Path: memory.capture( prompts.session_text(session), namespace, - extractor(replay, config.variant, source), + extractor(replay, config.variant, source, []), actor=ACTOR, observed_at=session.date, ) @@ -254,6 +254,8 @@ def _mark(marker: CachedModels, question: str, answer: str, facts: Sequence[Reco numbered = "\n".join(f"{n}. {fact.content}" for n, fact in enumerate(facts, 1)) prompt = MARK.format(question=question, answer=answer, facts=numbered) reply = marker(JUDGE, prompt, 1000, MARK_SCHEMA) + if reply.stopped: + raise RuntimeError(f"{JUDGE} stopped at 1000 tokens") return sorted({n for n in json.loads(reply.text)["facts"] if 1 <= n <= len(facts)}) @@ -262,6 +264,8 @@ def _complete(marker: CachedModels, question: str, answer: str, facts: Sequence[ listed = "\n".join(f"- {fact}" for fact in facts) prompt = COMPLETE.format(question=question, answer=answer, facts=listed) reply = marker(JUDGE, prompt, 100, COMPLETE_SCHEMA) + if reply.stopped: + raise RuntimeError(f"{JUDGE} stopped at 100 tokens") return bool(json.loads(reply.text)["complete"]) diff --git a/bench/e2e.py b/bench/e2e.py index 2f16052..838b78e 100644 --- a/bench/e2e.py +++ b/bench/e2e.py @@ -160,11 +160,15 @@ def fact_source(session_id: str) -> str: def extractor( - models: Models, variant: prompts.Variant = prompts.VARIANTS["baseline"], source: str = "" + models: Models, + variant: prompts.Variant = prompts.VARIANTS["baseline"], + source: str = "", + at_limit: list[str] | None = None, ) -> Callable[[str, datetime | None, list[Record]], list[Fact]]: """Return the extractor callable of `capture`: one call to the variant's model per session. - With `source`, each fact carries it in `sources`. + With `source`, each fact carries it in `sources`. A reply that reached the token limit + gives no facts, and `source` is appended to `at_limit`. """ sources = (source,) if source else () @@ -173,6 +177,10 @@ def extract(text: str, observed_at: datetime | None, current: list[Record]) -> l known = [f"- {record.content}" for record in current] prompt = prompts.extract(text, date, known, variant.prompt) reply = models(variant.model, prompt, EXTRACTOR_MAX_TOKENS, prompts.EXTRACT_SCHEMA) + if reply.stopped: + if at_limit is not None: + at_limit.append(source) + return [] facts = json.loads(reply.text)["facts"] return [ Fact( @@ -196,6 +204,7 @@ class Arm: answer: Reply verdict: Reply extracted: int = 0 + at_limit: int = 0 @property def correct(self) -> bool: @@ -203,12 +212,17 @@ def correct(self) -> bool: def answer( - models: Models, question: Question, prompt: str, context_chars: int, extracted: int = 0 + models: Models, + question: Question, + prompt: str, + context_chars: int, + extracted: int = 0, + at_limit: int = 0, ) -> Arm: """Read `prompt` with READER, judge the answer with JUDGE, and return the arm.""" reply = models(READER, prompt, READER_MAX_TOKENS) verdict = models(JUDGE, prompts.judge(question, reply.text), JUDGE_MAX_TOKENS) - return Arm(context_chars, reply, verdict, extracted) + return Arm(context_chars, reply, verdict, extracted, at_limit) def run_question( @@ -242,13 +256,14 @@ def run_question( namespace = ("bench", "lme_s", run_id, "memory", question.id) memory = Memory(stores[config.memory_ranking]) written = 0 + at_limit: list[str] = [] for session in question.sessions: source = fact_source(session.id) if config.fact_sources else "" written += len( memory.capture( prompts.session_text(session), namespace, - extractor(models, config.variant, source), + extractor(models, config.variant, source, at_limit), actor=ACTOR, observed_at=session.date, ) @@ -258,7 +273,7 @@ def run_question( ) empty(namespace) prompt = prompts.read_facts(question, block) - result["memory"] = answer(models, question, prompt, len(block), written) + result["memory"] = answer(models, question, prompt, len(block), written, len(at_limit)) return result @@ -417,6 +432,7 @@ def row_of(question: Question, result: dict[str, Arm]) -> dict[str, Any]: "correct": found.correct, "context_chars": found.context_chars, "facts_written": found.extracted, + "sessions_at_limit": found.at_limit, "reader_input_tokens": found.answer.input_tokens, "answer": found.answer.text, "verdict": found.verdict.text, @@ -608,6 +624,11 @@ def main(argv: Sequence[str] | None = None) -> Path | None: "workers": len(shards), "spent_usd": round(spent, 2), "model_calls": calls, + "sessions_at_limit": sum( + row["arms"]["memory"]["sessions_at_limit"] + for row in rows + if "memory" in row["arms"] + ), "embedder_misses": misses, "hotmemory_commit": commit, "run_id": run_id, diff --git a/bench/models.py b/bench/models.py index 197f576..7848589 100644 --- a/bench/models.py +++ b/bench/models.py @@ -25,11 +25,16 @@ @dataclass(frozen=True) class Reply: - """The text of one model reply and its token counts.""" + """The text of one model reply and its token counts. + + `stopped` is True if a reply with a schema reached its token limit, so its text is not + whole JSON. + """ text: str input_tokens: int output_tokens: int + stopped: bool = False def cost(self, model: str) -> float: """Return the price of this reply in US dollars.""" @@ -106,7 +111,8 @@ class Remote: """`Complete` over the OpenAI chat API, at temperature 0, with 120 seconds for each try. The client reads the key from `OPENAI_MODEL_KEY`. `OPENAI_API_KEY` stays the key of the - embedder. With a schema, the reply is JSON that the schema allows. + embedder. With a schema, the reply is JSON that the schema allows, or a reply with + `stopped` set if it reached `max_tokens`. """ def __init__(self) -> None: @@ -134,11 +140,10 @@ def __call__( **extra, ) choice = completion.choices[0] - if choice.finish_reason == "length" and schema is not None: - raise RuntimeError(f"{model} stopped at {max_tokens} tokens") usage = completion.usage return Reply( text=(choice.message.content or "").strip(), input_tokens=usage.prompt_tokens if usage else 0, output_tokens=usage.completion_tokens if usage else 0, + stopped=choice.finish_reason == "length" and schema is not None, ) diff --git a/bench/tests/test_e2e.py b/bench/tests/test_e2e.py index 7038012..edd1a79 100644 --- a/bench/tests/test_e2e.py +++ b/bench/tests/test_e2e.py @@ -20,7 +20,7 @@ shards_of, summarize, ) -from bench.models import EXTRACTOR, JUDGE, READER, CachedModels, Reply +from bench.models import EXTRACTOR, JUDGE, READER, CachedModels, Offline, Reply from hotmemory import MemoryStore from tests.conftest import FakeClock, fake_embedder @@ -262,3 +262,34 @@ def empty(namespace: Sequence[str]) -> None: assert f"[sources: {fact_source(late.id)}]" in block assert "[sources: none]" not in block assert "answer_" not in block + + +def test_a_reply_at_the_limit_gives_no_facts(questions: dict[str, Question]) -> None: + session = questions["q_one"].sessions[0] + + def stopped(model: str, prompt: str, max_tokens: int, schema: Any = None) -> Reply: + return Reply('{"facts": [{"subject": "a", "con', 10, max_tokens, stopped=True) + + at_limit: list[str] = [] + extract = extractor(stopped, source="session:abc", at_limit=at_limit) + assert extract(prompts.session_text(session), session.date, []) == [] + assert at_limit == ["session:abc"] + + +def test_cached_models_keep_a_stopped_reply(tmp_path: Path) -> None: + def complete(model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None) -> Reply: + return Reply("{", 1, max_tokens, stopped=True) + + path = tmp_path / "replies.sqlite" + models = CachedModels(complete, path) + assert models(EXTRACTOR, "p", 10, {}).stopped + models.close() + replay = CachedModels(Offline(), path) + assert replay(EXTRACTOR, "p", 10, {}).stopped + with replay._db: + replay._db.execute( + "INSERT INTO replies VALUES (?, ?)", + ("old", json.dumps({"text": "yes", "input_tokens": 1, "output_tokens": 1})), + ) + found = replay._db.execute("SELECT reply FROM replies WHERE key = 'old'").fetchone()[0] + assert not Reply(**json.loads(found)).stopped diff --git a/docs/internal/plan.md b/docs/internal/plan.md index a400274..4684ecd 100644 --- a/docs/internal/plan.md +++ b/docs/internal/plan.md @@ -168,9 +168,13 @@ of the sample A run: extractor wrote 126 facts from these sessions, and about 21 percent of them come from the first third of the session. A word-overlap match between each fact and each turn gave this share, so it is approximate. -- Task 2 found no fault. No cached extractor reply fails to parse, and the largest has - 2,602 output tokens, under the limit of 4,000. No list of "Facts already in memory" for - the 26 evidence sessions holds a fact near the answer. +- Task 2 found no fault in the cache. No cached extractor reply fails to parse, and the + largest has 2,602 output tokens, under the limit of 4,000. No list of "Facts already in + memory" for the 26 evidence sessions holds a fact near the answer. +- The first runs with sources found fault 1. On question `a3332713`, both the baseline and + `detail` reached the limit of 4,000 output tokens on session `sharegpt_7uSzi6w_2`, and + both runs stopped. In that session, the user pastes a list of survey answers from other + people, and the extractor wrote each line as a fact about the user. Decisions taken during the work. The owner can change any of them in review. @@ -187,7 +191,12 @@ Decisions taken during the work. The owner can change any of them in review. `recall` with its default budget of 2,000 characters, and a longer line lets fewer facts fit (observed: 9 in place of 10 for question `66f24dbb`). So the extractor cache of phase 6 does not replay with sources. The baseline runs again with sources, at the - full price, as the comparison point of this phase. `--no-fact-sources` runs the arm as phases 5 and 6 did, and a + full price, as the comparison point of this phase. +- A reply that reaches the token limit gives no facts for its session (agreed with the + owner on 2026-10-14). The cache keeps the reply, marked as stopped, so a replay and the + diagnosis skip the same session. A result file counts these sessions in + `sessions_at_limit`. The other choice was a second call with a higher limit. It would + store a fact about the user for each line of a pasted document. `--no-fact-sources` runs the arm as phases 5 and 6 did, and a result file records `fact_sources`. ## Phase 8. Episodes behind facts From db2fe634701a50aa263407a761a178d2ce6acfc4 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Wed, 14 Oct 2026 21:16:43 +0530 Subject: [PATCH 3/4] Phase 7 tasks 5 to 8: the results of the extractor variants On sample A, the baseline with sources scored 22 of 54 and `detail` scored 31. On sample B, `detail` scored 32, against 27 for the phase 6 baseline. The group with no stored fact for the answer went from 19 to 16 on sample A and from 16 to 14 on sample B. docs/benchmarks.md reports each run with its prompt, model, cost, and diagnosis. It also records that the marker of the diagnosis can miss stored facts. plan.md records the owner's acceptance of `detail` below the stop condition, that task 6 did not run, and the spend of the phase. --- docs/benchmarks.md | 82 +++++++++++++++++++++++++++++++++++++++++-- docs/internal/plan.md | 19 +++++++--- 2 files changed, 93 insertions(+), 8 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 1d93e1f..c789a1d 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -144,7 +144,7 @@ sample. |---|---| | Reader | `gpt-4o-2024-08-06`, temperature 0, up to 800 output tokens, the chain-of-thought prompt of the reference code | | Judge | `gpt-4o-2024-08-06`, temperature 0, the prompt of the reference code for each question type | -| Extractor of the memory arm | `gpt-4o-mini-2024-07-18`, with the `baseline` prompt in `bench/prompts.py` | +| Extractor of the memory arm | `gpt-4o-mini-2024-07-18`, with the `baseline` prompt in `bench/prompts.py`, except in the section on extractor variants | | Driver | `HotdataStore` with the fused ranking, on the local stack | The arms: @@ -256,6 +256,80 @@ The misses of each question type, on sample B: | knowledge-update | 4 | 1 | 0 | 1 | 2 | | temporal-reasoning | 2 | 1 | 0 | 0 | 1 | +### The extractor variants + +Phase 7 tried two extractor prompts, each with `gpt-4o-mini-2024-07-18`, on the memory arm +only. Both are in `bench/prompts.py`. + +| Variant | What the prompt asks for | +|---|---| +| `baseline` | The prompt of phases 5 and 6. | +| `detail` | The `baseline` rules, with four more. Read every turn. Keep a fact that the user states in passing. Keep each item of a list as its own fact. Write each date as an absolute date. | + +From phase 7, each fact carries the source `session:`, with the first 12 hex digits +of the SHA-256 of the session id. The block shows the sources to the reader, so each line +is longer than in phases 5 and 6. The runs of phases 5 and 6 had no sources. The baseline +therefore ran again with sources, as the point of comparison. + +Observed on 2026-10-14, at commit `a9d87c0`, with 4 workers. + +| Run | Sample | Correct of 54 | Facts per session | Reader tokens at the median | Sessions at the token limit | Cost | File | +|---|---|---|---|---|---|---|---| +| `baseline`, no sources (phase 6) | A | 19 | 5.34 | 1,462 | not recorded | see above | `e2e-sample-a.json` | +| `baseline` | A | 22 | 5.36 | 1,698 | 0 | at least $0.92 | `e2e-sample-a-baseline-sources.json` | +| `detail` | A | 31 | 5.39 | 1,655 | 1 | at least $1.73 | `e2e-sample-a-detail.json` | +| `baseline`, no sources (phase 6) | B | 27 | 5.20 | 1,466 | not recorded | see above | `e2e-sample-b.json` | +| `detail` | B | 32 | 5.00 | 1,664 | 0 | $1.71 | `e2e-sample-b-detail.json` | + +The first try of each run on sample A stopped at question `a3332713`. One extractor reply +reached the limit of 4,000 output tokens on a session where the user pastes a list of +survey answers. From commit `a9d87c0`, such a reply gives no facts for its session, and +the run goes on. The second try took the replies of the first from the cache. The cost of +each run on sample A is the sum of both tries. The first try printed its cost only after +each question, so its last calls are not in the sum. + +Correct answers by question type: + +| Question type | `baseline`, A | `detail`, A | `baseline` no sources, B | `detail`, B | +|---|---|---|---|---| +| single-session-user | 4 | 4 | 6 | 6 | +| single-session-assistant | 3 | 2 | 4 | 4 | +| single-session-preference | 3 | 4 | 0 | 1 | +| multi-session | 3 | 6 | 5 | 6 | +| knowledge-update | 4 | 8 | 5 | 7 | +| temporal-reasoning | 5 | 7 | 7 | 8 | +| All | 22 | 31 | 27 | 32 | + +The diagnosis of each run, with the same commands as above. The diagnoses cost $0.31, +$0.29, and $0.27. + +| Cause | `baseline`, A | `detail`, A | `baseline` no sources, B | `detail`, B | +|---|---|---|---|---| +| No stored fact states the answer or a value that it needs | 19 | 16 | 16 | 14 | +| A stored fact holds it, and the fused search ranked it below 30 | 1 | 1 | 0 | 0 | +| A marked fact was in the top 30, and `recall` left it out | 0 | 0 | 0 | 0 | +| The block holds every needed fact, and the reader missed | 0 | 0 | 1 | 1 | +| The block holds some of the needed facts | 12 | 6 | 10 | 7 | +| Misses | 32 | 23 | 27 | 22 | + +`detail` gained 13 questions on sample A over the `baseline` with sources, and lost 4. Of +the 13, 9 were misses where the block of the baseline held some of the needed facts. So +most of the gain comes from facts that are more complete, with exact values and absolute +dates. The group with no stored fact got smaller by 3 on sample A and by 2 on sample B. + +Phase 7 read the misses and found these facts: + +- Phase 5 said that the extractor dropped what the assistant said. On sample A, that + holds for 5 of the 19 misses with no stored fact. The other 14 are a fact that the user + states in passing, in a turn about another topic. +- With `detail`, the extractor still drops what the assistant said. On sample A, 7 misses + of the type single-session-assistant have no stored fact. For 3 of their sessions, + `detail` wrote no fact at all. +- The marker of the diagnosis can miss facts. On sample A, `detail` stored the facts that + hold the answer for `2ce6a0f2`, `92a0aa75`, and `71017277`, and the marker marked none of + them. The marker reads all facts of a question in one call, 226 to 300 facts here. So + the group with no stored fact can be smaller than the table shows, in each run. + ## Known limits - The end-to-end numbers come from samples of 54 questions, so one question moves an arm @@ -263,7 +337,9 @@ The misses of each question type, on sample B: run on all 500 questions waits for the owner's agreement. - The full-context arm gives the reader every session. The reference code keeps the last 50 sessions, so the two differ for histories longer than 50 sessions. -- The extractor of the memory arm is the only one tried. A stronger extractor or another - prompt was not measured. +- The memory arm tried one extractor model, `gpt-4o-mini-2024-07-18`, with two prompts. A + stronger extractor model was not measured. +- The diagnosis marks the facts of a question in one model call, and it can miss facts in + a long list. Its count of misses with no stored fact can be too high. - `recall` keeps the records that are valid at `as_of`, and a search over history does not exist yet. A question about a value before it changed can need a superseded record. diff --git a/docs/internal/plan.md b/docs/internal/plan.md index 4684ecd..098b017 100644 --- a/docs/internal/plan.md +++ b/docs/internal/plan.md @@ -174,7 +174,7 @@ of the sample A run: - The first runs with sources found fault 1. On question `a3332713`, both the baseline and `detail` reached the limit of 4,000 output tokens on session `sharegpt_7uSzi6w_2`, and both runs stopped. In that session, the user pastes a list of survey answers from other - people, and the extractor wrote each line as a fact about the user. + people. The extractor wrote each line as a fact about the user. Decisions taken during the work. The owner can change any of them in review. @@ -191,13 +191,22 @@ Decisions taken during the work. The owner can change any of them in review. `recall` with its default budget of 2,000 characters, and a longer line lets fewer facts fit (observed: 9 in place of 10 for question `66f24dbb`). So the extractor cache of phase 6 does not replay with sources. The baseline runs again with sources, at the - full price, as the comparison point of this phase. + full price, as the comparison point of this phase. `--no-fact-sources` runs the arm as + phases 5 and 6 did, and a result file records `fact_sources`. - A reply that reaches the token limit gives no facts for its session (agreed with the owner on 2026-10-14). The cache keeps the reply, marked as stopped, so a replay and the diagnosis skip the same session. A result file counts these sessions in - `sessions_at_limit`. The other choice was a second call with a higher limit. It would - store a fact about the user for each line of a pasted document. `--no-fact-sources` runs the arm as phases 5 and 6 did, and a - result file records `fact_sources`. + `sessions_at_limit`. The other choice was a second call with a higher limit. That call + stores a fact about the user for each line of a pasted document. +- `detail` met a stop condition on sample A. It cut the group "no stored fact held the + answer" from 19 to 16, and the condition asks for 12 or fewer. The owner accepted + `detail` on 2026-10-14 and asked for its run on sample B. More prompt variants wait for + later work. +- Task 6 did not run. The owner named no second extractor model. +- The phase spent at least $5.23 on models, from the result files and the run logs: + $0.92 and $1.73 for the `baseline` and `detail` runs on sample A, $1.71 for `detail` on + sample B, and $0.87 for the three diagnoses. The first try of each run on sample A + printed its cost only after each question, so its last calls are not in the sum. ## Phase 8. Episodes behind facts From 7670f01278066617c94761eddd3a5e21b8eef2e1 Mon Sep 17 00:00:00 2001 From: Rohan Dsouza Date: Wed, 14 Oct 2026 21:27:23 +0530 Subject: [PATCH 4/4] Report the detail scores in the README, and the replay without sources The README gave only the baseline scores of the memory arm. It now also gives the scores of the `detail` prompt on both samples. A replay of sample A with `--no-fact-sources` gave the same 54 answers and marks as phase 6, with no model call. docs/benchmarks.md records it. --- README.md | 9 +++++---- docs/benchmarks.md | 5 ++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c6fa595..0318ec9 100644 --- a/README.md +++ b/README.md @@ -145,11 +145,12 @@ sample B on 2026-10-14. The reader and the judge were `gpt-4o-2024-08-06`. |---|---|---| | Every session of the question, about 114,000 tokens | 39 of 54 (72.2%) | 43 of 54 (79.6%) | | The 10 sessions that the search ranked first | 45 of 54 (83.3%) | 48 of 54 (88.9%) | -| The block of `recall`, after `capture` with a `gpt-4o-mini` extractor | 19 of 54 (35.2%) | 27 of 54 (50.0%) | +| The block of `recall`, after `capture` with a `gpt-4o-mini` extractor and the `baseline` prompt | 19 of 54 (35.2%) | 27 of 54 (50.0%) | +| The same, on 2026-10-14, with the `detail` prompt and a session source on each fact | 31 of 54 (57.4%) | 32 of 54 (59.3%) | -The memory contract, with this extractor, scores below the full history. The extractor -did not store the answer in 19 of the 35 misses on sample A, and in 16 of the 27 misses on -sample B. With 54 questions, each score can be about 10 points off, and the memory arm +The memory contract scores below the full history. With the `baseline` prompt, the +extractor did not store the answer in 19 of the 35 misses on sample A, and in 16 of the 27 +misses on sample B. With the `detail` prompt, these counts are 16 of 23 and 14 of 22. With 54 questions, each score can be about 10 points off, and the memory arm differs by 14.8 points between the two samples. In the cloud, a search took 361 ms at p50 with the vector ranking, and 998 ms with the fused ranking. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index c789a1d..c1b0b4c 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -269,7 +269,10 @@ only. Both are in `bench/prompts.py`. From phase 7, each fact carries the source `session:`, with the first 12 hex digits of the SHA-256 of the session id. The block shows the sources to the reader, so each line is longer than in phases 5 and 6. The runs of phases 5 and 6 had no sources. The baseline -therefore ran again with sources, as the point of comparison. +therefore ran again with sources, as the point of comparison. On 2026-10-14, at commit +`db2fe63`, a replay of sample A with `--no-fact-sources` gave the same answer and the same +mark on each of the 54 questions as phase 6, with no model call. File: +`e2e-sample-a-replay-no-sources.json`. Observed on 2026-10-14, at commit `a9d87c0`, with 4 workers.