fix(criteria): search the whole command in command_executed and bump harness SDKs for Opus 5.5

.claude/notes/contracts.md

@@ -95,9 +95,12 @@ the raw text alone can stop counting.
 It is memoized because the early-stop watcher re-scans the whole accumulated trajectory on
 every tool-call event, normalizing the same command many times per run.
 
-The regex search window is capped to bound ReDoS on a large command string, and
-normalization runs over that same truncated window, so `shlex` never sees more than the cap
-and needs no separate guard.
+Each regex search is bounded to 2000 characters to limit ReDoS on a large command string,
+but the whole command is searched: the first 2000 characters, then the rest in windows that
+start and end on logical-line boundaries. Agents write long heredoc scripts that end in the
+command a task checks for (`cat > x <<EOF ... EOF` then `uip agent validate`), and a single
+leading window scored those as never run. Normalization runs per window, so `shlex` never
+sees more than the bound and needs no separate guard.
 
 ## Recording a CLI invocation
 

.github/workflows/harbor-e2e.yml

@@ -29,6 +29,8 @@ env:
   # Anthropic-credit spend off this path; DirectRoute is exercised elsewhere.
   API_BACKEND: "bedrock"
   CLAUDE_CODE_USE_BEDROCK: "1"
+  # Same expression as pr-checks.yml (see the comment there).
+  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }},claude-agent-sdk
 
 jobs:
   harbor-e2e:

.github/workflows/pr-checks.yml

@@ -38,11 +38,12 @@ env:
   TELEMETRY_ENABLED: "false"
   # The `uipath-*` pool enforces a minimum package-age safe-chain check on installs.
   # Workflow-level so every installing job inherits it; per-job copies are how some
-  # jobs previously ended up with no exclusions at all. The literal is the operative
-  # value — no secret of that name exists at repo or org level, so the bare `secrets.`
-  # reference this replaced resolved to an empty list. (Image builds carry their own
-  # list in docker/Dockerfile; deliberately not the same set.)
-  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }}
+  # jobs previously ended up with no exclusions at all. A repo secret of this name
+  # replaces the literal when set, so claude-agent-sdk sits outside the `||`: its wheel
+  # bundles the Claude Code CLI, and new-model support ships there faster than the
+  # gate window. (Image builds carry their own list in docker/Dockerfile; deliberately
+  # not the same set.)
+  SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }},claude-agent-sdk
 
 jobs:
   quality-gate:

.github/workflows/publish-testpypi.yml

@@ -38,7 +38,7 @@ jobs:
     timeout-minutes: 10
     env:
       # `uv build` resolves build deps under the pool's safe-chain gate.
-      SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }}
+      SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }},claude-agent-sdk
     environment:
       name: testpypi
       url: https://test.pypi.org/project/coder-eval/

.github/workflows/release.yml

@@ -75,7 +75,7 @@ jobs:
       # Load-bearing on the release path: the pool enforces a package-age safe-chain
       # check on uv installs. Same expression as pr-checks.yml (see the comment there),
       # so a package can't pass PR CI and then fail the release install.
-      SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }}
+      SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }},claude-agent-sdk
 
     steps:
       # Only a real release (main) needs the app token: semantic-release pushes the

docker/Dockerfile

@@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # tag and is bumped deliberately -- mirrors the codex CLI pin
 # (`openai-codex-cli-bin==…` in pyproject). Leaving it at `@latest` let Docker
 # layer caching freeze it nondeterministically across rebuilds.
-ARG CLAUDE_CODE_VERSION=2.1.177
+ARG CLAUDE_CODE_VERSION=2.1.281
 RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
     && rm -rf /var/lib/apt/lists/* \
@@ -39,7 +39,7 @@ RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
 # Pi Node CLI, pinned — same rationale as the Claude Code pin above (the agent
 # binary is a dominant non-model driver of results; @latest would freeze
 # nondeterministically under layer caching). Node 22 + npm are already present.
-ARG PI_VERSION=0.84.4
+ARG PI_VERSION=0.87.1
 RUN npm install -g @earendil-works/pi-coding-agent@${PI_VERSION}
 
 # uv: matches host sandbox.py's `uv venv` + `uv pip install` fast path
@@ -84,12 +84,12 @@ COPY experiments/default.yaml ./experiments/default.yaml
 # pulls the `uipath` SDK from public PyPI (no credentials required).
 ARG CODER_EVAL_UV_EXTRAS=""
 
-# safe-chain min-age exclusions for the codex cli-bin pin (see pyproject [tool.uv])
-# and the pinned Antigravity harness. Codex and Antigravity are always installed,
-# so the default excludes their pinned packages from the min-age gate (the
-# google-antigravity release tracks the rapidly-moving Gemini harness and may be
+# safe-chain min-age exclusions for the codex cli-bin pin (see pyproject [tool.uv]),
+# the pinned Antigravity harness, and claude-agent-sdk (its wheel bundles the Claude
+# Code CLI). All three are always installed, so the default excludes their pinned
+# packages from the min-age gate (each tracks a rapidly-moving harness and may be
 # newer than the gate window); callers may override.
-ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex,google-antigravity"
+ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex,google-antigravity,claude-agent-sdk"
 
 # All extras (codex, antigravity, litellm, and the opt-in uipath) resolve from
 # public PyPI per uv.lock, so the build needs no private-index credentials.

docker/Dockerfile.runtime

@@ -53,7 +53,7 @@ COPY pyproject.toml uv.lock README.md ./
 COPY src/ ./src/
 COPY experiments/default.yaml ./experiments/default.yaml
 
-ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex"
+ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex,claude-agent-sdk"
 # Install the locked deps, then coder_eval itself NON-editable. An editable/path
 # install would leave the package source at /src (outside the kit), so the
 # `COPY --from /opt/coder-eval` overlay would ship a venv that can't import
@@ -66,7 +66,7 @@ RUN export SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="${SAFE_CHAIN_MINIMUM_PACKA
 
 # --- 3. Node LTS + the Claude Code CLI, under the kit dir. -------------------
 ARG NODE_VERSION=22.14.0
-ARG CLAUDE_CODE_VERSION=2.1.177
+ARG CLAUDE_CODE_VERSION=2.1.281
 # Download to a file first so a bad URL fails the build loudly (a piped
 # `curl | tar` would mask a 404 and only surface later as a missing npm), and
 # verify the tarball against nodejs.org's published SHASUMS256 before extracting
@@ -109,6 +109,6 @@ RUN chmod +x /usr/local/bin/coder_eval_entrypoint.sh
 # label; labels don't survive `COPY --from`). The claude-code pin mirrors the
 # framework image — a parity test (tests/test_image_from_dockerfiles.py) enforces it.
 ARG CODER_EVAL_VERSION=unknown
-ARG CLAUDE_CODE_VERSION=2.1.177
+ARG CLAUDE_CODE_VERSION=2.1.281
 LABEL org.coder-eval.version="${CODER_EVAL_VERSION}"
 LABEL org.coder-eval.claude-code-version="${CLAUDE_CODE_VERSION}"

pyproject.toml

@@ -43,7 +43,7 @@ dependencies = [
     # explicitly since our code/tests construct httpx2 types directly. Capped to
     # mirror anthropic's own bound on it (`httpx2<3,>=2.0.0`).
     "httpx2>=2.12.0,<3.0.0",
-    "claude-agent-sdk>=0.2.124",
+    "claude-agent-sdk>=0.2.157",
     "anyio>=4.14.2",
     "radon>=6.0.1",
     "tqdm>=4.67.3",
@@ -112,7 +112,7 @@ litellm = [
 # Without this extra, the framework still installs and runs; Codex-dependent
 # code paths fail at dispatch with a clear hint pointing back here.
 codex = [
-    "openai-codex>=0.144.4",
+    "openai-codex>=0.155.1",
 ]
 # Optional extra that enables Antigravity agent support:
 #   - AntigravityAgent implementation using Google's official google-antigravity SDK
@@ -131,7 +131,7 @@ codex = [
 # Without this extra the framework still installs and runs; Antigravity-dependent
 # code paths fail at start() with a clear hint pointing back here.
 antigravity = [
-    "google-antigravity==0.1.8",
+    "google-antigravity==0.1.18",
 ]
 # Optional extra that enables OpenCode agent support.
 #
@@ -180,7 +180,7 @@ pi = []
 # back here (it is not needed to EXPORT a task to Harbor format, only to run coder-eval
 # itself as Harbor's agent).
 harbor = [
-    "harbor==0.22.0",
+    "harbor==0.23.0",
 ]
 
 [project.scripts]
@@ -212,14 +212,14 @@ packages = ["src/coder_eval"]
 allow-direct-references = true
 
 [tool.uv]
-# openai-codex 0.144.4 hardpins `openai-codex-cli-bin==0.144.4` (a stable
+# openai-codex 0.156.1 hardpins `openai-codex-cli-bin==0.156.1` (a stable
 # release; the SDK version now tracks the codex CLI version line). Naming it
 # here documents and holds the pinned cli-bin build — the harness binary is a
 # dominant non-model driver of eval results, so it travels with the coder_eval
 # release tag and is bumped deliberately (mirrors the antigravity localharness
-# and claude-code CLI pins). 0.144.4 publishes manylinux wheels (x86_64 +
+# and claude-code CLI pins). 0.156.1 publishes manylinux wheels (x86_64 +
 # aarch64), so CI/Linux installs work.
-override-dependencies = ["openai-codex-cli-bin==0.144.4"]
+override-dependencies = ["openai-codex-cli-bin==0.156.1"]
 
 constraint-dependencies = [
     # Fix known CVEs in transitive dependencies
@@ -443,6 +443,8 @@ filterwarnings = [
     "ignore::DeprecationWarning",        # Ignore deprecation warnings from dependencies
     "ignore::PendingDeprecationWarning",
     "ignore::ResourceWarning",           # Ignore unclosed resources (handled by cleanup)
+    # litellm>=1.102 TypedDicts mark fields ReadOnly; pydantic warns when it builds their schema
+    "ignore:Item '.*' on TypedDict class '.*' is using the `ReadOnly` qualifier:UserWarning",
 ]
 
 # Coverage integration (when using --cov)

src/coder_eval/criteria/command_executed.py

@@ -18,9 +18,14 @@
 
 logger = logging.getLogger(__name__)
 
-# HAZARD: bounds ReDoS on a large command string. Normalization runs over this
-# same truncated window, so shlex needs no separate size guard.
+# HAZARD: bounds ReDoS on a large command string. Every search window is at most
+# this long and normalization runs per window, so shlex needs no separate size guard.
 _MAX_PATTERN_SEARCH_LEN = 2000
+# Overlap between the pieces of one logical line longer than the bound, so a match up
+# to this long is never split across two pieces.
+_WINDOW_OVERLAP = 256
+# A newline not escaped by a trailing backslash: where one logical shell line ends.
+_LOGICAL_LINE_END = re.compile(r"(?<!\\)\n")
 
 
 def _is_shell_program(arg0: str) -> bool:
@@ -97,29 +102,57 @@ def _normalize_shell(cmd_text: str) -> str | None:
     return " ".join(tokens)
 
 
+def _search_windows(cmd_text: str) -> list[str]:
+    """Slices of ``cmd_text`` a pattern is searched in, each at most ``_MAX_PATTERN_SEARCH_LEN``.
+
+    The first window is the leading ``_MAX_PATTERN_SEARCH_LEN`` characters. The rest
+    of the command is packed into windows that start and end on logical-line
+    boundaries (a backslash-continued line stays whole), so a command on a line after
+    a long heredoc is still seen. A single logical line longer than the bound is cut
+    into bound-sized pieces overlapping by ``_WINDOW_OVERLAP``. Work stays linear in
+    the command length and each regex search stays within the ReDoS bound.
+    """
+    cap = _MAX_PATTERN_SEARCH_LEN
+    if len(cmd_text) <= cap:
+        return [cmd_text]
+    bounds: list[tuple[int, int]] = []
+    start = end = 0
+    for line_end in [m.end() for m in _LOGICAL_LINE_END.finditer(cmd_text)] + [len(cmd_text)]:
+        if line_end - start > cap:
+            if end > start:
+                bounds.append((start, end))
+                start = end
+            while line_end - start > cap:
+                bounds.append((start, start + cap))
+                start += cap - _WINDOW_OVERLAP
+        end = line_end
+    if end > start:
+        bounds.append((start, end))
+    return [cmd_text[:cap]] + [cmd_text[s:e] for s, e in bounds if s > 0]
+
+
 def _match_haystacks(cmd_text: str, *, is_shell: bool) -> list[str]:
     """Strings a pattern may match against for one command.
 
-    Always the raw ``cmd_text`` truncated to the ReDoS bound; when ``is_shell``,
-    additionally the quote-resolved, wrapper-stripped form of that **same
-    truncated window** (see :func:`_normalize_shell`). Normalizing the already-
-    truncated slice keeps both haystacks describing the same window, so quote-
-    stripping can never slide content from past the cap into the match, and
-    caps ``shlex`` input at ``_MAX_PATTERN_SEARCH_LEN`` for free. Matching is
-    "either" — a pattern hits the command if it matches ANY haystack.
+    Every search window of ``cmd_text`` (see :func:`_search_windows`); when
+    ``is_shell``, additionally the quote-resolved, wrapper-stripped form of each
+    window (see :func:`_normalize_shell`). Normalizing per window keeps every
+    haystack within the ReDoS bound and caps ``shlex`` input for free. Matching is
+    "either" -- a pattern hits the command if it matches ANY haystack.
 
     ``is_shell`` is decided once by the caller (a Bash tool whose ``command`` is
     a non-empty ``str``) and passed in, rather than re-derived here from
     ``tool_name`` alone: a Bash record with a missing/empty ``command`` serializes
     its params to JSON, where shell tokenization is meaningless, and must NOT be
     normalized (else stripped JSON quotes could newly satisfy an exclusion).
     """
-    window = cmd_text[:_MAX_PATTERN_SEARCH_LEN]
-    haystacks = [window]
-    if is_shell:
-        normalized = _normalize_shell(window)
-        if normalized is not None and normalized != window:
-            haystacks.append(normalized)
+    haystacks: list[str] = []
+    for window in _search_windows(cmd_text):
+        haystacks.append(window)
+        if is_shell:
+            normalized = _normalize_shell(window)
+            if normalized is not None and normalized != window:
+                haystacks.append(normalized)
     return haystacks
 
 

src/coder_eval/harbor/agent.py

@@ -20,7 +20,7 @@
 from __future__ import annotations
 
 import shlex
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ClassVar
 
 from coder_eval.harbor.agent_paths import AGENT_TASK_YAML_PATH
 
@@ -62,7 +62,7 @@ class CoderEvalAgent(BaseInstalledAgent):
     on PATH there exactly as ``tests/test.sh`` assumes it is for grading.
     """
 
-    SUPPORTS_ATIF: bool = True
+    SUPPORTS_ATIF: ClassVar[bool] = True
 
     @staticmethod
     def name() -> str:

tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json

@@ -1,5 +1,5 @@
 {
-  "agent_output": "{\n  \"completed_at\": null,\n  \"duration_ms\": 1500,\n  \"error\": null,\n  \"id\": \"turn_1\",\n  \"items\": [\n    {\n      \"id\": \"m1\",\n      \"memory_citation\": null,\n      \"phase\": null,\n      \"text\": \"rebuilt from items\",\n      \"type\": \"agentMessage\"\n    }\n  ],\n  \"items_view\": \"TurnItemsView.full\",\n  \"started_at\": null,\n  \"status\": \"TurnStatus.completed\"\n}",
+  "agent_output": "{\n  \"completed_at\": null,\n  \"duration_ms\": 1500,\n  \"error\": null,\n  \"id\": \"turn_1\",\n  \"items\": [\n    {\n      \"delivery\": null,\n      \"id\": \"m1\",\n      \"memory_citation\": null,\n      \"phase\": null,\n      \"questions\": null,\n      \"text\": \"rebuilt from items\",\n      \"type\": \"agentMessage\"\n    }\n  ],\n  \"items_view\": \"TurnItemsView.full\",\n  \"started_at\": null,\n  \"status\": \"TurnStatus.completed\"\n}",
   "assistant_turn_count": 1,
   "commands": [],
   "crash_reason": null,

tests/test_antigravity_agent.py

@@ -11,6 +11,7 @@
 from collections.abc import Callable
 from datetime import datetime, timedelta
 from itertools import pairwise
+from pathlib import Path
 from types import ModuleType, SimpleNamespace
 from typing import Any
 
@@ -146,31 +147,23 @@ def test_resolve_workspaces_includes_workdir_and_skill_roots(tmp_path):
     ]
 
 
-def test_workspace_only_permits_skill_reads_with_resolved_workspaces(tmp_path):
-    """Drives the harness's real ``workspace_only`` policy: a skill read is denied when
-    scoped to the workdir alone (the bug), but permitted once the resolved skill roots
-    join ``workspaces`` (the fix). ``skills_paths`` feeds discovery, not the file-tool
-    allowlist, so the roots must be in ``workspaces`` for the agent to read SKILL.md."""
-    policy = pytest.importorskip("google.antigravity.hooks.policy")
-    ag_types = pytest.importorskip("google.antigravity.types")
-
+def test_resolved_workspaces_cover_skill_reads(tmp_path):
+    """The harness confines file tools to ``workspaces`` (enforced inside localharness,
+    not by a Python-side predicate), and ``skills_paths`` feeds discovery, not that
+    allowlist, so the resolved skill roots must be in ``workspaces`` for the agent to
+    read SKILL.md."""
     repo = tmp_path / "skills-repo"
     _make_skill(repo / "skills", "uipath-sdd")
-    skill_md = repo / "skills" / "uipath-sdd" / "SKILL.md"
+    skill_md = (repo / "skills" / "uipath-sdd" / "SKILL.md").resolve()
     workdir = tmp_path / "work"
     workdir.mkdir()
 
     agent = AntigravityAgent(parse_agent_config(type="antigravity", plugins=[{"type": "local", "path": str(repo)}]))
     agent.working_directory = workdir
-    skills_paths = agent._resolve_skills_paths(None)
-
-    def read_denied(workspaces) -> bool:
-        policies = policy.workspace_only([str(w) for w in workspaces])
-        tc = ag_types.ToolCall(name="read_file", canonical_path=str(skill_md))
-        return any(p.when(tc) for p in policies if p.when is not None)
+    workspaces = [Path(w).resolve() for w in agent._resolve_workspaces(agent._resolve_skills_paths(None))]
 
-    assert read_denied([workdir]) is True  # workdir only: out-of-workspace → denied
-    assert read_denied(agent._resolve_workspaces(skills_paths)) is False  # resolved workspaces permit it
+    assert not skill_md.is_relative_to(workdir.resolve())
+    assert any(skill_md.is_relative_to(w) for w in workspaces)
 
 
 def test_to_token_usage_maps_gemini_buckets():