Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions docs/reference/trajectory-assessment.md
Original file line number Diff line number Diff line change
Expand Up @@ -304,14 +304,18 @@ leading signs, precision and percent suffixes stay exact, while spelled-out unit
are explained in the output language. An adjacent topic cannot resolve an
ambiguous referent without supporting field-bound evidence. This remains model
guidance; the exact-literal qualifier and original-text fallback are unchanged.
The curator selects it in a new versioned input through the existing analysis
action. Earlier encodings retain their exact prompts, identities and artifacts;
reading or refreshing them never starts or upgrades analysis.
New CLI preparations and default pack registrations select it when a captured
display snapshot has no explicit encoding. The normalized input is saved with
that choice before analysis. Explicit encodings/configurations and frozen inputs
retain their exact prompts, identities and artifacts; reading or refreshing them
never starts or upgrades analysis. Translation quality belongs to the existing
analysis generation, with original text available for comparison.

显式数值编码 v3 增加生成时的逐项忠实性核对:正负号、精度和百分号按允许的原始写法
保留,文字单位用目标语言解释;不能仅凭邻近主题补全含糊所指。这仍是模型指导,
既有字面校验和原文回退不放宽。由原输入整理者通过既有分析入口明确生成新版本,
旧编码的提示、身份和产物保持,阅读与刷新不升级或执行分析。
既有字面校验和原文回退不放宽。新CLI准备和默认pack登记在快照未指定编码时采用v3,
先将选择保存到版本化输入,再由既有分析入口生成。显式编码/配置及历史输入的提示、
身份和产物保持,阅读与刷新不升级或执行分析。译文质量由默认生成承担,原文仍可核对。

Translated prose is applied to decoded narrative fields; it is never serialized
and reparsed as an Explore decision. Owner-format labels or list markers inside
Expand Down
2 changes: 2 additions & 0 deletions examples/personal-workspace-browser/research-history.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -719,6 +719,8 @@ export const researchHistoryScenario = {
await unavailablePanel.getByRole('status').filter({hasText: '分析来源未就绪'}).waitFor();
assert.equal(await unavailablePanel.locator('header p').count(), 0, 'An empty reader does not repeat analysis instructions');
await unavailablePanel.press('Escape');
await page.waitForFunction(element => element === document.activeElement,
await history.getByRole('button', {name: '深度分析', exact: true}).elementHandle());
assert(await history.getByRole('button', {name: '深度分析', exact: true}).evaluate(e => e === document.activeElement));
assert.equal(analysisWrites, 0, 'Unavailable analysis, help and recovery are reads');
await page.setViewportSize({width: 1280, height: 900});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ def prepare(
"benchmark.trajectory.prepare",
{
"input": json.loads(input_path.read_text(encoding="utf-8")),
"generation_defaults": output_schema_revision == 2,
"config": {"model": model, "effort": effort, "service_tier": service_tier,
**({"output_schema_revision": output_schema_revision} if output_schema_revision is not None else {}),
**({"research_synthesis": True, "research_synthesis_revision": 2} if research_synthesis else {})},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,7 @@ def register_pack(
{
"input": json.loads(input_path.read_text(encoding="utf-8")),
"config": config or DEFAULT_CONFIG,
"generation_defaults": not config,
},
)
record = {
Expand Down Expand Up @@ -571,7 +572,7 @@ def _continuation(runtime_root, owner_goal_id, record, prepared, result, languag
if base_id is None:
return None, None
try:
_, previous = _resolve(runtime_root, owner_goal_id, base_id)
owner, previous = _resolve(runtime_root, owner_goal_id, base_id)
base = inspect_pack(runtime_root=runtime_root, owner_goal_id=owner_goal_id,
pack_id=base_id, output_language=language, include_continuation=False)
old = _prepared(previous, language) if language is not None else _prepared(previous)
Expand All @@ -589,9 +590,24 @@ def _continuation(runtime_root, owner_goal_id, record, prepared, result, languag
except (OSError, ValueError):
readable = False
usable = readable and base["result"] is not None and base["reason_code"] != "method_changed"
prior = {**old, **({"output": base["result"]} if usable else {})}
# Readback has restored display literals. Requalify the exact successful
# attempt's provider response, including when a later retry failed.
output = None
if usable:
directory = _job(owner, old, old["analysis_key"])
if base.get("result_job"):
directory = directory / f"attempt-{base['result_job']['attempt']}"
try:
output = json.loads((directory / "model-output.json").read_text(encoding="utf-8"))
except (OSError, ValueError):
usable = False
prior = {**old, **({"output": output} if usable else {})}
# Compare analytical rows without resending the restored display sidecar;
# the raw response above supplies the versioned display data for reuse.
comparison_output = {key: result[key] for key in ("assessments", "limitations", "episode_questions", "synthesis")
if key in result} if result is not None else None
projection = effect_runtime_result("benchmark.trajectory.continuation", {
**prepared, "previous": prior, **({"output": result} if result is not None else {}),
**prepared, "previous": prior, **({"output": comparison_output} if result is not None else {}),
})
return {
**projection,
Expand Down
10 changes: 7 additions & 3 deletions loopx/control_plane/capabilities/trajectory_assessment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ function metric(v: unknown): TrajectoryMetric {
if (!TRAJECTORY_METRICS.includes(v as TrajectoryMetric)) throw new EffectRuntimeRequestError('unsupported metric');
return v as TrajectoryMetric;
}
export function parseTrajectoryInput(raw: unknown): TrajectoryInput {
export function parseTrajectoryInput(raw: unknown, generationDefaults = false): TrajectoryInput {
const x = object(raw, 'input');
if (x.schema_version !== 'trajectory_assessment_input_v1') throw new EffectRuntimeRequestError('unsupported input schema');
const s = object(x.scope, 'scope'), selection = object(x.selection, 'selection');
Expand Down Expand Up @@ -67,7 +67,9 @@ export function parseTrajectoryInput(raw: unknown): TrajectoryInput {
if (selection.exhaustive && population !== episodes.length) throw new EffectRuntimeRequestError('exhaustive requires a complete known population');
const display = x.display_snapshot === undefined ? undefined : object(x.display_snapshot, 'display_snapshot');
if (display && display.schema_version !== 'research_display_snapshot_v1') throw new EffectRuntimeRequestError('unsupported display snapshot');
const encoding = display?.translation_encoding;
const requestedEncoding = display?.translation_encoding;
const encoding = display && generationDefaults && requestedEncoding === undefined
? 'source_text_aliases_with_literal_values_v3' : requestedEncoding;
if (encoding !== undefined && encoding !== 'source_text_aliases_v1' && encoding !== 'source_text_aliases_with_literals_v1'
&& encoding !== 'source_text_aliases_with_literal_values_v1' && encoding !== 'source_text_aliases_with_literal_values_v2'
&& encoding !== 'source_text_aliases_with_literal_values_v3') {
Expand Down Expand Up @@ -242,7 +244,9 @@ const synthesisSchema = {type: 'object', additionalProperties: false,
}};

export function prepareTrajectoryAssessment(params: Record<string, unknown>) {
const input = parseTrajectoryInput(params.input);
// Only new writer requests select defaults. Frozen inputs are re-prepared on
// every read and must retain their original prompt and analysis identity.
const input = parseTrajectoryInput(params.input, params.generation_defaults === true);
const config = object(params.config, 'config');
if (config.research_synthesis !== undefined && typeof config.research_synthesis !== 'boolean') {
throw new EffectRuntimeRequestError('research_synthesis must be boolean');
Expand Down
39 changes: 33 additions & 6 deletions tests/capabilities/test_trajectory_assessment_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -583,30 +583,38 @@ def test_missing_source_blocks_new_execution_and_leaves_registration_visible(pac
execute_pack(**operation, codex="must-not-run")


@pytest.mark.parametrize("encoding", [None, "source_text_aliases_v1", "source_text_aliases_with_literals_v1", "source_text_aliases_with_literal_values_v1", "source_text_aliases_with_literal_values_v2", "source_text_aliases_with_literal_values_v3"])
@pytest.mark.parametrize("encoding", ["default", None, "source_text_aliases_v1", "source_text_aliases_with_literals_v1", "source_text_aliases_with_literal_values_v1", "source_text_aliases_with_literal_values_v2", "source_text_aliases_with_literal_values_v3"])
@pytest.mark.parametrize("context", [False, True])
def test_full_history_display_uses_the_same_job_and_preserves_original_versions(pack, tmp_path, encoding, context):
scope, _, _, input_path, sources, _ = pack
from loopx.capabilities.benchmark_toolkit.trajectory_assessment import prepare

scope, _, initial, input_path, sources, _ = pack
data = json.loads(input_path.read_text())
history = {"ok": True, "goal_id": "solver", "total": 2, "next_cursor": None, "items": [
dict(schema_version="loopx_explore_result_event_v0", event_id="question-old", result_id="question", goal_id="solver", event_kind="node",
node_kind="question", recorded_at="2026-01-01T00:00:00Z", title="Does V18 help?", summary="Not established.", status="open"),
dict(schema_version="loopx_explore_result_event_v0", event_id="question-new", result_id="question", goal_id="solver", event_kind="node",
node_kind="question", recorded_at="2026-01-02T00:00:00Z", title="Does V19 help?", summary="Still unknown.", status="blocked"),
]}
if encoding in {"source_text_aliases_with_literal_values_v1", "source_text_aliases_with_literal_values_v2", "source_text_aliases_with_literal_values_v3"}:
if encoding in {"default", "source_text_aliases_with_literal_values_v1", "source_text_aliases_with_literal_values_v2", "source_text_aliases_with_literal_values_v3"}:
history["items"][0]["summary"] = "Trial4failed2checks"
text = json.dumps(history, ensure_ascii=False)
(sources / "history.json").write_text(text)
data["sources"].append(dict(id="history", artifact="history.json", artifact_sha256=hashlib.sha256(text.encode()).hexdigest(), first_line=1, text=text))
data["display_snapshot"] = dict(schema_version="research_display_snapshot_v1", source_id="history", source_namespace="source-a")
if encoding is not None:
if encoding is not None and encoding != "default":
data["display_snapshot"]["translation_encoding"] = encoding
context_citation = dict(source_id="source", first_line=2, last_line=2)
if context:
data["display_snapshot"]["contexts"] = [dict(event_id="question-old", field="summary", citations=[context_citation])]
input_path.write_text(json.dumps(data))
registered = register_pack(**scope, input_path=input_path, source_root=sources, title="Whole history · synthetic")
registered = register_pack(**scope, input_path=input_path, source_root=sources, title="Whole history · synthetic",
config=None if encoding == "default" else initial["config"])
if encoding == "default":
prepared = prepare(input_path, model="gpt-6.1-sol", effort="max", service_tier="fast")
assert prepared["input"]["display_snapshot"]["translation_encoding"] == "source_text_aliases_with_literal_values_v3"
assert prepared["analysis_key"] == registered["analysis_key"]
assert json.loads(input_path.read_text()) == data
current = inspect_pack(**scope, pack_id=registered["pack_id"], output_language="zh")
assert current["display_request"]["eligible"] == 4 and current["job"] is None
provider = tmp_path / "display-provider"
Expand Down Expand Up @@ -640,7 +648,7 @@ def translation(field):
read_citation(**operation, source_id="history", first_line=1, last_line=1)
else:
assert not any("context_citations" in field for field in result["result"]["display"]["fields"])
if encoding in {"source_text_aliases_with_literal_values_v1", "source_text_aliases_with_literal_values_v2", "source_text_aliases_with_literal_values_v3"}:
if encoding in {"default", "source_text_aliases_with_literal_values_v1", "source_text_aliases_with_literal_values_v2", "source_text_aliases_with_literal_values_v3"}:
assert any(field["text"] == "第4次尝试有2项检查未通过(Trial4failed2checks)。"
for field in result["result"]["display"]["fields"])
assert result["job"]["usage"]["input_tokens"] == 19 # one shared attempt, not four new jobs
Expand All @@ -657,6 +665,25 @@ def translation(field):
assert prior["job"]["status"] == "failed" and prior["job"]["attempt"] == 2
assert prior["result_attempt"] == 1 and prior["result_job"]["usage"]["input_tokens"] == 19
assert prior["result"] == result["result"]
if encoding == "default":
# A failed retry keeps the prior successful response reusable. Readback
# contains restored literals, so it cannot substitute for raw output.
successor_input = json.loads(input_path.read_text())
successor_input["scope"]["revision"] = "successor-capture"
successor_path = tmp_path / "successor.json"
successor_path.write_text(json.dumps(successor_input))
successor = register_pack(**scope, input_path=successor_path, source_root=sources,
title="Same evidence, new capture", base_pack_id=registered["pack_id"])
successor_view = inspect_pack(**scope, pack_id=successor["pack_id"], output_language="zh")
assert successor_view["continuation"]["base_available"] and successor_view["continuation"]["can_reuse"]
assert successor_view["job"] is None
reuse = execute_pack(**scope, pack_id=successor["pack_id"], analysis_key=successor_view["analysis_key"],
output_language="zh", codex="must-not-run")
assert reuse["status"] == "succeeded" and reuse["reused"]
reused = inspect_pack(**scope, pack_id=successor["pack_id"], output_language="zh")
assert reused["result"]["display"] == prior["result"]["display"]
assert reused["result"]["metrics"] == prior["result"]["metrics"]
assert reused["continuation"]["verdict_changes"] == []
provider.write_text(successful_provider)
assert execute_pack(**operation, codex=str(provider), retry=True)["status"] == "succeeded"
result = inspect_pack(**scope, pack_id=registered["pack_id"], output_language="zh")
Expand Down
28 changes: 28 additions & 0 deletions tests/control_plane_ts/trajectory_assessment.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,34 @@ const contextInput = () => {
const translatedOutput = (p = displayPrepared()) => ({...output(), episode_questions: [{episode_id: 'e1', question: 'A 是否有帮助?'}, {episode_id: 'e2', question: 'B 是否有帮助?'}],
display_translations: JSON.parse(p.prompt.split('\n\ndisplay_fields:\n')[1]!).map((row: {field_id: string; text: string}) => ({field_id: row.field_id, text: `【译文】${row.text}`}))});

test('new generation selects faithful display defaults without upgrading frozen or explicit inputs', () => {
for (const language of ['zh', 'en']) {
const original = contextInput(), configForLanguage = {...config, output_language: language};
delete original.display_snapshot!.translation_encoding;
const before = structuredClone(original);
const frozen = prepareTrajectoryAssessment({input: original, config: configForLanguage});
const next = prepareTrajectoryAssessment({input: original, config: configForLanguage, generation_defaults: true});
assert.equal(next.input.display_snapshot!.translation_encoding, 'source_text_aliases_with_literal_values_v3');
assert.notEqual(next.analysis_key, frozen.analysis_key);
assert.equal(next.config.output_language, language);
assert.deepEqual(next.input.sources, original.sources);
assert.deepEqual(next.input.episodes, original.episodes);
assert.deepEqual(original, before, 'Writer normalization does not modify the captured input');
assert.deepEqual(prepareTrajectoryAssessment({input: next.input, config: next.config}), next);
assert.deepEqual(prepareTrajectoryAssessment({input: original, config: configForLanguage}), frozen);
for (const encoding of ['source_text_aliases_v1', 'source_text_aliases_with_literal_values_v2'] as const) {
const explicit = structuredClone(original);
explicit.display_snapshot!.translation_encoding = encoding;
assert.deepEqual(prepareTrajectoryAssessment({input: explicit, config: configForLanguage, generation_defaults: true}),
prepareTrajectoryAssessment({input: explicit, config: configForLanguage}));
}
const invalid = {...original, display_snapshot: {...original.display_snapshot, translation_encoding: null}};
assert.throws(() => prepareTrajectoryAssessment({input: invalid, config: configForLanguage, generation_defaults: true}),
/unsupported display translation encoding/);
}
assert.deepEqual(prepareTrajectoryAssessment({input: input(), config, generation_defaults: true}), prepared());
});

test('field-bound evidence disambiguates identical notes without sharing translations or changing source authority', () => {
const input = contextInput(), p = prepareTrajectoryAssessment({input, config: {...config, output_language: 'zh'}});
const rows = JSON.parse(p.prompt.split('\n\ndisplay_fields:\n')[1]!);
Expand Down
Loading