From 301c1e2e6ab974215a8db7be14c642de66753e58 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:05:26 +0800 Subject: [PATCH] fix(benchmark): default heartbeat and Turn sessions to resume Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- benchmark/LHTB/README.md | 6 ++++-- .../LHTB/configs/heartbeat-generic-cli.yaml | 2 +- benchmark/LHTB/run.sh | 8 +++++++- benchmark/LHTB/scripts/preflight.py | 2 +- benchmark/LHTB/scripts/render_config.py | 3 ++- benchmark/runtime/RUNTIME.md | 10 +++++----- benchmark/runtime/codex.py | 4 +++- benchmark/runtime/harbor.py | 2 +- benchmark/runtime/sessions.py | 1 + benchmark/runtime/worker.py | 2 +- benchmark/tests/test_lhtb_preflight.py | 7 +++++-- benchmark/tests/test_resume_sessions.py | 10 ++++++++-- benchmark/tests/test_shared_codex_runtime.py | 19 +++++++++++++++++++ 13 files changed, 58 insertions(+), 18 deletions(-) diff --git a/benchmark/LHTB/README.md b/benchmark/LHTB/README.md index c839376108..a2fca6b962 100644 --- a/benchmark/LHTB/README.md +++ b/benchmark/LHTB/README.md @@ -114,8 +114,10 @@ not replayed; bootstrap and Todo lifecycle follow the installed product version. ## Shared execution configuration `LOOPX_EXECUTION_MODE` selects `plain`, `native-goal`, `heartbeat` (default), -`turn` or `loopx-goal`. `LOOPX_ITERATION_CONTEXT` defaults to `fresh`; heartbeat and Turn -accept `resume`, sharing the same Goal/Agent session across planning, wakes and Todos. Turn also requires `LOOPX_VALIDATION_COMMAND_JSON`, +`turn` or `loopx-goal`. `LOOPX_ITERATION_CONTEXT` defaults to `resume` for heartbeat and Turn, +sharing the same Goal/Agent session across planning, wakes and Todos. Other modes +retain `fresh`. Set `LOOPX_ITERATION_CONTEXT=fresh` explicitly for an independent +conversation on each invocation; existing frozen jobs keep their recorded settings. Turn also requires `LOOPX_VALIDATION_COMMAND_JSON`, an argv array for the independently protected task validator. No generic benchmark scoring or hidden-verifier feedback is introduced. diff --git a/benchmark/LHTB/configs/heartbeat-generic-cli.yaml b/benchmark/LHTB/configs/heartbeat-generic-cli.yaml index 7a701590cc..8aa9ef8d29 100644 --- a/benchmark/LHTB/configs/heartbeat-generic-cli.yaml +++ b/benchmark/LHTB/configs/heartbeat-generic-cli.yaml @@ -22,7 +22,7 @@ agents: override_timeout_sec: 5400 kwargs: execution_mode: heartbeat - iteration_context: fresh + iteration_context: resume reasoning_effort: max goals: "false" web_search: disabled diff --git a/benchmark/LHTB/run.sh b/benchmark/LHTB/run.sh index e816f65739..7975863bca 100755 --- a/benchmark/LHTB/run.sh +++ b/benchmark/LHTB/run.sh @@ -49,8 +49,14 @@ export PYTHONPATH="$LOOPX_SRC_DIR${PYTHONPATH:+:$PYTHONPATH}" LOOPX_EXECUTION_MODE="${LOOPX_EXECUTION_MODE:-heartbeat}" LOOPX_TASK_ENTRY="${LOOPX_TASK_ENTRY:-}" [[ -z "${LOOPX_PLANNING_TIMEOUT_SEC:-}" ]] || die "Remove LOOPX_PLANNING_TIMEOUT_SEC; planning shares LOOPX_SCHEDULER_TIMEOUT_SEC with execution" -LOOPX_ITERATION_CONTEXT="${LOOPX_ITERATION_CONTEXT:-fresh}" LOOPX_VALIDATION_COMMAND_JSON="${LOOPX_VALIDATION_COMMAND_JSON:-[]}" +if [[ -z "${LOOPX_ITERATION_CONTEXT:-}" ]]; then + LOOPX_ITERATION_CONTEXT="$("$VENV/bin/python" -c ' +import json, sys +from benchmark.runtime.codex import Execution +print(Execution(mode=sys.argv[1], validation_command=json.loads(sys.argv[2])).context) +' "$LOOPX_EXECUTION_MODE" "$LOOPX_VALIDATION_COMMAND_JSON")" +fi SHARED_CODEX_AGENT_DIR="$LOOPX_SRC_DIR/benchmark/runtime" [[ -f "$SHARED_CODEX_AGENT_DIR/codex_offline.py" ]] || \ die "Shared offline Codex adapter not found: $SHARED_CODEX_AGENT_DIR/codex_offline.py" diff --git a/benchmark/LHTB/scripts/preflight.py b/benchmark/LHTB/scripts/preflight.py index d577642110..e22d36d261 100755 --- a/benchmark/LHTB/scripts/preflight.py +++ b/benchmark/LHTB/scripts/preflight.py @@ -137,7 +137,7 @@ def check(label: str, passed: bool, detail: str) -> None: execution = Execution( mode=kwargs.get("execution_mode", "heartbeat"), task_entry=kwargs.get("task_entry"), - context=kwargs.get("iteration_context", "fresh"), + context=kwargs.get("iteration_context"), validation_command=kwargs.get("validation_command", []), ) check( diff --git a/benchmark/LHTB/scripts/render_config.py b/benchmark/LHTB/scripts/render_config.py index f154632e5c..3392300265 100755 --- a/benchmark/LHTB/scripts/render_config.py +++ b/benchmark/LHTB/scripts/render_config.py @@ -24,7 +24,8 @@ def main() -> int: parser.add_argument("--timeout", type=int, required=True) parser.add_argument("--task", action="append", default=[]) parser.add_argument("--execution-mode", choices=MODES, default="heartbeat") - parser.add_argument("--iteration-context", choices=CONTEXTS, default="fresh") + parser.add_argument("--iteration-context", choices=CONTEXTS, + help="Default: resume for heartbeat/turn; fresh for other modes") parser.add_argument("--task-entry", choices=TASK_ENTRIES, help="LoopX modes default to loopx-planned; explicit seeded-todo retains legacy entry") parser.add_argument("--validation-command-json", default="[]") diff --git a/benchmark/runtime/RUNTIME.md b/benchmark/runtime/RUNTIME.md index fe37dac73c..1c96c2f37a 100644 --- a/benchmark/runtime/RUNTIME.md +++ b/benchmark/runtime/RUNTIME.md @@ -25,7 +25,7 @@ agents: kwargs: execution_mode: heartbeat task_entry: loopx-planned - iteration_context: fresh + iteration_context: resume reasoning_effort: max codex_sandbox: danger-full-access turn_timeout_sec: null @@ -144,12 +144,12 @@ receipts or trajectories as feature-off; qualify their actual input separately. | `turn` | Public Turn CLI, typed result, independent validation, settlement | Present | | `loopx-goal` | Product Goal body + installed native Goal transport | Present | -`heartbeat` and `turn` accept `iteration_context: resume`. The first invocation +New `heartbeat` and `turn` runs default to `iteration_context: resume`. The first invocation creates one Codex conversation; later planning checkpoints and execution wakes resume that exact native session ID for the trial's Goal and Agent, including when the selected Todo changes. Both drivers share the product's agent-scoped -Codex session store. `fresh` remains the runner default and starts a new session -on each invocation. The former context name is rejected, with no alias. +Codex session store. Explicit `fresh` starts a new session on each invocation. Other execution modes +retain `fresh`; this default change does not rewrite existing trial configurations. The former context name is rejected, with no alias. Resume never falls back to a new conversation when a binding is corrupt, the trial home/workspace/model/settings change, or Codex returns a different ID. @@ -352,7 +352,7 @@ effort, tools, feedback and total budget fixed, and use separate trials: kwargs: execution_mode: heartbeat task_entry: loopx-planned - iteration_context: fresh + iteration_context: resume turn_timeout_sec: null scheduler_timeout_sec: 5080 ``` diff --git a/benchmark/runtime/codex.py b/benchmark/runtime/codex.py index 3d60db2ed8..30acca4b5d 100644 --- a/benchmark/runtime/codex.py +++ b/benchmark/runtime/codex.py @@ -22,7 +22,7 @@ @dataclass(frozen=True) class Execution: mode: str = "heartbeat" - context: str = "fresh" + context: str | None = None sandbox: str = "danger-full-access" timeout_seconds: float | None = None validation_command: tuple[str, ...] = () @@ -31,6 +31,8 @@ class Execution: context_delivery: str = "off" def __post_init__(self) -> None: + if self.context is None: + object.__setattr__(self, "context", "resume" if self.mode in {"heartbeat", "turn"} else "fresh") if self.context_delivery not in ("off", "on") or (self.context_delivery != "off" and self.mode != "heartbeat"): raise ValueError("context_delivery supports only off or on with heartbeat execution") if not isinstance(self.turn_envelope, bool) or (self.turn_envelope and self.mode != "heartbeat"): diff --git a/benchmark/runtime/harbor.py b/benchmark/runtime/harbor.py index 568c0c2dfa..1dd183ae83 100644 --- a/benchmark/runtime/harbor.py +++ b/benchmark/runtime/harbor.py @@ -58,7 +58,7 @@ def __init__( self, *args, execution_mode="heartbeat", - iteration_context="fresh", + iteration_context=None, codex_sandbox="danger-full-access", validation_command=None, turn_timeout_sec=None, diff --git a/benchmark/runtime/sessions.py b/benchmark/runtime/sessions.py index a21f3e5446..36db945b22 100644 --- a/benchmark/runtime/sessions.py +++ b/benchmark/runtime/sessions.py @@ -7,4 +7,5 @@ class BenchmarkSessionWake(CodexAgentSession): def __init__(self, env: dict[str, str], execution: Execution, receipt: dict) -> None: + assert execution.context is not None # Execution resolves the mode's default. super().__init__(env, context=execution.context, sandbox=execution.sandbox, receipt=receipt) diff --git a/benchmark/runtime/worker.py b/benchmark/runtime/worker.py index fc24648b02..217b11e291 100644 --- a/benchmark/runtime/worker.py +++ b/benchmark/runtime/worker.py @@ -420,7 +420,7 @@ def run_once(env: dict[str, str]) -> dict: raise ValueError("LOOPX_TURN_ENVELOPE must be 0 or 1") execution = Execution( mode=env.get("LOOPX_EXECUTION_MODE", "heartbeat"), - context=env.get("LOOPX_ITERATION_CONTEXT", "fresh"), + context=env.get("LOOPX_ITERATION_CONTEXT"), sandbox=env.get("LOOPX_CODEX_SANDBOX", "danger-full-access"), timeout_seconds=(float(env["LOOPX_CODEX_TURN_TIMEOUT_SEC"]) if env.get("LOOPX_CODEX_TURN_TIMEOUT_SEC") else None), diff --git a/benchmark/tests/test_lhtb_preflight.py b/benchmark/tests/test_lhtb_preflight.py index ba4eb5b8bd..8c4f8e9e97 100644 --- a/benchmark/tests/test_lhtb_preflight.py +++ b/benchmark/tests/test_lhtb_preflight.py @@ -38,7 +38,8 @@ def test_current_cli_accepts_the_benchmark_default(): @pytest.mark.parametrize("legacy_limit", [None, 300]) -def test_config_renderer_removes_only_legacy_null_planning_limit(tmp_path, monkeypatch, legacy_limit): +@pytest.mark.parametrize("context", [None, "fresh", "resume"]) +def test_config_renderer_removes_only_legacy_null_planning_limit(tmp_path, monkeypatch, legacy_limit, context): import yaml from benchmark.LHTB.scripts.render_config import main @@ -48,7 +49,8 @@ def test_config_renderer_removes_only_legacy_null_planning_limit(tmp_path, monke output = tmp_path / "candidate.yaml" monkeypatch.setattr(sys, "argv", ["render_config", "--template", str(template), "--output", str(output), "--job-name", "fixture", "--jobs-dir", "runs", - "--concurrency", "1", "--model", "fixture", "--effort", "high", "--timeout", "1000"]) + "--concurrency", "1", "--model", "fixture", "--effort", "high", "--timeout", "1000", + *(["--iteration-context", context] if context else [])]) if legacy_limit is not None: with pytest.raises(SystemExit) as error: main() @@ -58,6 +60,7 @@ def test_config_renderer_removes_only_legacy_null_planning_limit(tmp_path, monke kwargs = yaml.safe_load(output.read_text())["agents"][0]["kwargs"] assert "planning_timeout_sec" not in kwargs assert kwargs["task_entry"] == "loopx-planned" + assert kwargs["iteration_context"] == (context or "resume") assert kwargs["scheduler_timeout_sec"] == 5080 diff --git a/benchmark/tests/test_resume_sessions.py b/benchmark/tests/test_resume_sessions.py index bf3333a4c1..45fa067eaf 100644 --- a/benchmark/tests/test_resume_sessions.py +++ b/benchmark/tests/test_resume_sessions.py @@ -201,10 +201,13 @@ def binding(env): @pytest.mark.parametrize("planning_mode", ["heartbeat", "turn"]) +@pytest.mark.parametrize("context", [None, "resume"]) def test_planning_heartbeat_and_turn_share_one_binding( - tmp_path, monkeypatch, planning_mode + tmp_path, monkeypatch, planning_mode, context ): env = resume_env(tmp_path, monkeypatch) + if context is None: + env.pop("LOOPX_ITERATION_CONTEXT") plan_env = plan_stage(env, monkeypatch) | {"LOOPX_EXECUTION_MODE": planning_mode} if planning_mode == "turn": plan_env["LOOPX_VALIDATION_COMMAND_JSON"] = '["true"]' @@ -369,7 +372,8 @@ def test_explicit_fresh_replaces_corrupt_shared_binding(tmp_path, monkeypatch): @pytest.mark.skipif( not os.environ.get("LOOPX_TEST_CODEX_BIN"), reason="native Codex binary required" ) -def test_native_codex_resume_keeps_planning_history(tmp_path, monkeypatch): +@pytest.mark.parametrize("context", [None, "resume"]) +def test_native_codex_resume_keeps_planning_history(tmp_path, monkeypatch, context): """Exercise installed Codex against a disposable local Responses endpoint.""" requests = [] @@ -439,6 +443,8 @@ def log_message(self, *args): "LOOPX_CODEX_TURN_TIMEOUT_SEC": "30", "LOOPX_PHASE_DEADLINE_EPOCH": str(time.time() + 600), } + if context is None: + env.pop("LOOPX_ITERATION_CONTEXT") first = run_once(plan_stage(env, monkeypatch)) second = run_once(env) assert first["ok"] and second["ok"] diff --git a/benchmark/tests/test_shared_codex_runtime.py b/benchmark/tests/test_shared_codex_runtime.py index 09ec53859e..cbafc3c3b0 100644 --- a/benchmark/tests/test_shared_codex_runtime.py +++ b/benchmark/tests/test_shared_codex_runtime.py @@ -54,6 +54,25 @@ def test_invalid_execution_rejected_before_host(kwargs): Execution(**kwargs) +@pytest.mark.parametrize("mode", ["plain", "native-goal", "heartbeat", "turn", "loopx-goal"]) +def test_mode_default_context_preserves_explicit_fresh_and_transport_opt_ins(tmp_path, mode): + extra = {"validation_command": ("true",)} if mode == "turn" else {} + expected = "resume" if mode in {"heartbeat", "turn"} else "fresh" + implicit = Execution(mode=mode, **extra) + assert implicit.context == expected + assert implicit.context_delivery == "off" and implicit.turn_envelope is False + assert Execution(mode=mode, context="fresh", **extra).context == "fresh" + if expected == "fresh": + with pytest.raises(ValueError, match="resume requires"): + Execution(mode=mode, context="resume", **extra) + pytest.importorskip("harbor") + from benchmark.runtime.harbor import BenchmarkCodex + agent = BenchmarkCodex(logs_dir=tmp_path, model_name="openai/fixture", + execution_mode=mode, **extra) + assert agent.execution.context == expected + assert agent._worker_env(cwd="/task")["LOOPX_ITERATION_CONTEXT"] == expected + + def test_trial_home_preserves_sessions_and_fixes_nonconversation_inputs(tmp_path): args = settings(tmp_path) prepare_codex_home(**args)