From 1483a1879561210e07f7f6dc4919e14bcdc8179b Mon Sep 17 00:00:00 2001
From: Edgar Arakelyan <edgar.arakelyan@scale.com>
Date: Wed, 14 Oct 2026 11:01:47 -0700
Subject: [PATCH 1/3] feat(run): name the criteria a task failed in the
 summary, and fail CI with --fail-under

Under each task that scored below 1, the summary lists the criteria its verifications recorded as failed, each with
the reason the verifier gave, cut to one line, five at most. --fail-under SCORE exits 1 when a task scored below
SCORE or recorded no score; without it the exit status is as before. A run whose stdout reader leaves, as when piped
into head, now finishes and tears down quietly instead of warning that progress stopped and exiting 1.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 src/agent_env/bundle/run.py            | 12 +++-
 src/agent_env/cli/run.py               | 90 +++++++++++++++++++++-----
 src/agent_env/task/task.py             | 53 ++++++++++++++-
 tst/integration/cli/run_bundle_test.py | 17 +++++
 tst/unit/bundle/installed_test.py      |  3 +-
 tst/unit/bundle/run_test.py            | 78 ++++++++++++++++++++--
 tst/unit/task/failed_criteria_test.py  | 48 ++++++++++++++
 7 files changed, 278 insertions(+), 23 deletions(-)
 create mode 100644 tst/unit/task/failed_criteria_test.py

diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py
index 8bcec78d..839df618 100644
--- a/src/agent_env/bundle/run.py
+++ b/src/agent_env/bundle/run.py
@@ -29,7 +29,7 @@
 from agent_env.store.routing import namespace_routing, run_scope
 from agent_env.task import Task, record_task_cancelled
 from agent_env.task.interrupts import Interrupts
-from agent_env.task.task import recorded_scores
+from agent_env.task.task import FailedCriterion, failed_criteria, recorded_scores
 from agent_env.task.teardown import TeardownReport, teardown_run
 from agent_env.task_step.context import TaskStepContext
 from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM
@@ -83,6 +83,11 @@ def scores(self) -> dict[str, float]:
         when that isn't a step's verifier id."""
         return recorded_scores(self.task.steps, self.context)
 
+    @property
+    def failed_criteria(self) -> list[FailedCriterion]:
+        """Each criterion a verification scoring below 1 recorded as failed, with the reason it gave."""
+        return failed_criteria(self.task.steps, self.context)
+
     @property
     def outcome(self) -> Outcome:
         if self.cancelled:
@@ -112,6 +117,11 @@ class BundleRun:
     def failed(self) -> bool:
         return any(run.outcome is Outcome.FAILED for run in self.runs)
 
+    def below(self, threshold: float) -> tuple[TaskRun, ...]:
+        """The runs that finished but scored below ``threshold``, or recorded no score, which can't show it passed."""
+        return tuple(run for run in self.runs if run.outcome in (Outcome.PASSED, Outcome.BELOW_ONE, Outcome.UNSCORED)
+                     and (not run.scores or min(run.scores.values()) < threshold))
+
     def path(self, entry: BundleEntry) -> str:
         """``entry``'s path in the bundle, as the summary names it (``tasks/hello.json``)."""
         return relative(self.materialization.plan.bundle.bundle.root, entry.path)
diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py
index a57137f7..b4ff3eca 100644
--- a/src/agent_env/cli/run.py
+++ b/src/agent_env/cli/run.py
@@ -4,8 +4,11 @@
 
 import os
 import shlex
+import sys
 import time
 import traceback
+from collections.abc import Iterator
+from contextlib import contextmanager
 from pathlib import Path
 
 import click
@@ -25,6 +28,10 @@
 _ENV_ENDPOINTS = (("mcp", "mcp_url"), ("gateway", "gateway_url"), ("pgweb", "db_web_url"), ("db-mcp", "db_mcp_url"),
                   ("vnc", "vnc_url"), ("expires", "expires_at_utc"))
 _DRY_RUN = "Dry run: nothing is written or run."
+# How many of a task's failed criteria the summary names before it counts the rest.
+_SHOWN_CRITERIA = 5
+# How much of a failed criterion's reason fits its summary line.
+_REASON_CHARS = 160
 
 
 @click.command()
@@ -39,59 +46,95 @@
               help="Keep the sandboxes up after the runs, print their endpoints, and tear them down on Ctrl-C.")
 @click.option("--dry-run", is_flag=True,
               help="Check the bundle as the run does and show what it would write and run; write and run nothing.")
+@click.option("--fail-under", type=click.FloatRange(0, 1), default=None, metavar="SCORE",
+              help="Exit 1 when a task scored below SCORE, or recorded no score; --fail-under 1 fails on any miss.")
 @click.pass_context
 def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: tuple[str, ...], model: str | None,
-        sandbox: str | None, keep: bool, dry_run: bool):
+        sandbox: str | None, keep: bool, dry_run: bool, fail_under: float | None):
     """Run a bundle's tasks and evals, or list the installed bundles.
 
     BUNDLE is a folder, or the name of a bundle an installed package provides (PACKAGE/NAME when several
     packages provide that name). Without BUNDLE, it lists the installed bundles.
 
     With no --task or --eval, it runs every eval or, in a bundle without evals, every task. Each task runs
-    once, at most four at a time, and the run exits 1 if any of them failed. With --verbose, a run that
-    raised prints its traceback.
+    once, at most four at a time, and the run exits 1 if any of them failed. A task that scored below 1 lists
+    the criteria that failed and why, and exits 1 only under --fail-under, which also fails a task that
+    recorded no score. With --verbose, a run that raised prints its traceback.
 
     --dry-run makes the checks the run makes before its first task, reading the stores as the run does, and
     prints what it would write and run, writing and running nothing. It exits 1 on a problem the run would
-    stop at, and 0 otherwise; --model and --keep change nothing it shows. It takes no lock, so another run
-    can write first and change what it shows.
+    stop at, and 0 otherwise; --model, --keep and --fail-under change nothing it shows. It takes no lock, so
+    another run can write first and change what it shows.
 
     Each run's sandboxes are torn down as it ends; its instance and outputs stay. --keep holds them up
     until Ctrl-C instead. Ctrl-C or SIGTERM mid-run cancels the runs, tears them down, prints what ran and
     exits 130 (143 for SIGTERM); a second one stops the teardown and prints what is still up.
     """
     if bundle is None:
-        if tasks or evals or model or sandbox or keep or dry_run:
-            raise click.UsageError("--task, --eval, --model, --sandbox, --keep and --dry-run need a BUNDLE to run")
+        if tasks or evals or model or sandbox or keep or dry_run or fail_under is not None:
+            raise click.UsageError("--task, --eval, --model, --sandbox, --keep, --dry-run and --fail-under need a "
+                                   "BUNDLE to run")
         _list_installed()
         return
     root, id_root = _locate(bundle)
     verbose = ctx.find_root().params.get("verbose")
     try:
         if dry_run:
-            click.echo(_DRY_RUN)
-            _report_dry_run(dry_run_bundle(root, tasks=tasks, evals=evals, sandbox=sandbox, on_progress=click.echo,
-                                           id_root=id_root))
+            _echo(_DRY_RUN)
+            dry = dry_run_bundle(root, tasks=tasks, evals=evals, sandbox=sandbox, on_progress=_echo, id_root=id_root)
+            with _reader_may_leave():
+                _report_dry_run(dry)
             return
-        result = run_bundle(root, tasks=tasks, evals=evals, model=model, sandbox=sandbox, on_progress=click.echo,
+        result = run_bundle(root, tasks=tasks, evals=evals, model=model, sandbox=sandbox, on_progress=_echo,
                             id_root=id_root, keep=keep)
     except BundleError as e:
         _note_an_installed_namesake(e, bundle, root, id_root)
         raise
     except RunInterrupted as stop:
-        with Interrupts():
+        with Interrupts(), _reader_may_leave():
             _summarize(stop.result, verbose)
             _report_teardown(stop.result)
         raise SystemExit(128 + stop.signum) from None
     with Interrupts() as interrupts:
-        _summarize(result, verbose)
+        with _reader_may_leave():
+            _summarize(result, verbose)
         if keep:
             result = _hold(result, interrupts)
-        _report_teardown(result)
-    if result.failed:
+        with _reader_may_leave():
+            _report_teardown(result)
+    below = result.below(fail_under) if fail_under is not None else ()
+    with _reader_may_leave():
+        _report_below(result, below, fail_under)
+    if result.failed or below:
         raise SystemExit(1)
 
 
+def _echo(line: str) -> None:
+    """``click.echo`` for the run's progress, which stops printing once stdout's reader has gone, as when piped
+    into head, so the run still finishes and tears down."""
+    with _reader_may_leave():
+        click.echo(line)
+
+
+@contextmanager
+def _reader_may_leave() -> Iterator[None]:
+    """Print the block's output, or, once stdout's reader has gone, send the rest of the output nowhere."""
+    try:
+        yield
+    except BrokenPipeError:
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, sys.stdout.fileno())
+        os.close(devnull)
+
+
+def _report_below(result: BundleRun, below: tuple[TaskRun, ...], threshold: float | None) -> None:
+    if not below:
+        return
+    shown = ", ".join(f"{result.path(run.entry)} ({min(run.scores.values()):g})" if run.scores
+                      else f"{result.path(run.entry)} (no score)" for run in below)
+    click.echo(f"\nBelow --fail-under {threshold:g}: {shown}")
+
+
 def _path_like(text: str) -> bool:
     """Text that names a folder even when there is none there."""
     return text.startswith((".", "/", "~"))
@@ -173,6 +216,8 @@ def _summarize(result: BundleRun, verbose: bool) -> None:
     click.echo("Tasks:")
     for run in result.runs:
         click.echo(click.style(f"  {_described(result, run)}", fg=_COLORS[run.outcome]))
+        for line in _failed_criteria(run):
+            click.echo(f"    {line}")
     if result.evals:
         click.echo("Evals:")
     for eval_run in result.evals:
@@ -188,6 +233,21 @@ def _summarize(result: BundleRun, verbose: bool) -> None:
         _print_tracebacks(result)
 
 
+def _failed_criteria(run: TaskRun) -> list[str]:
+    """A line for each criterion the run failed, its step, the criterion and the reason it gave, up to
+    ``_SHOWN_CRITERIA``, then a count of the rest."""
+    failed = run.failed_criteria
+    lines = []
+    for criterion in failed[:_SHOWN_CRITERIA]:
+        reason = " ".join(criterion.reason.split())
+        if len(reason) > _REASON_CHARS:
+            reason = reason[:_REASON_CHARS - 1].rstrip() + "…"
+        lines.append(f"{criterion.step}: {criterion.criterion}" + (f": {reason}" if reason else ""))
+    if len(failed) > _SHOWN_CRITERIA:
+        lines.append(f"and {len(failed) - _SHOWN_CRITERIA} more")
+    return lines
+
+
 def _report_dry_run(dry: DryRun) -> None:
     materialization = dry.materialization
     plan = materialization.plan
diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py
index 46960671..fd91b373 100644
--- a/src/agent_env/task/task.py
+++ b/src/agent_env/task/task.py
@@ -120,7 +120,7 @@ def _sub(value):
 def recorded_scores(steps: list["TaskStep"], context: TaskStepContext) -> dict[str, float]:
     """Every score a run of ``steps`` recorded, by the id of the step that recorded it, or by its verification's key
     when that isn't a step's verifier id."""
-    step_ids = {step.verifier_id: step.id for step in steps if getattr(step, "verifier_id", None)}
+    step_ids = _verifier_steps(steps)
     scores = {}
     for key, verification in (context.metadata.get("verifications") or {}).items():
         score = verification.get("score")
@@ -129,6 +129,57 @@ def recorded_scores(steps: list["TaskStep"], context: TaskStepContext) -> dict[s
     return scores
 
 
+@dataclass(frozen=True)
+class FailedCriterion:
+    """A criterion a verification recorded as failed."""
+
+    step: str  # the id of the step that recorded it, or its verification's key
+    criterion: str
+    reason: str  # the verifier's justification or message, "" when it gave none
+
+
+def failed_criteria(steps: list["TaskStep"], context: TaskStepContext) -> list[FailedCriterion]:
+    """Each criterion that failed in a verification scoring below 1: its result is false, or, recording no result,
+    its score is below 1. A skipped criterion didn't fail."""
+    step_ids = _verifier_steps(steps)
+    failed = []
+    for key, verification in (context.metadata.get("verifications") or {}).items():
+        score = verification.get("score")
+        if not isinstance(score, (int, float)) or score >= 1:
+            continue
+        for index, result in enumerate(verification.get("results") or []):
+            if not isinstance(result, dict) or result.get("skipped"):
+                continue
+            passed = bool(result["result"]) if "result" in result else not _below_one(result.get("score"))
+            if not passed:
+                failed.append(FailedCriterion(step_ids.get(key, key), _criterion(result, index), _reason(result)))
+    return failed
+
+
+def _verifier_steps(steps: list["TaskStep"]) -> dict[str, str]:
+    """The id of each step that records a verification, by its verifier id."""
+    return {step.verifier_id: step.id for step in steps if getattr(step, "verifier_id", None)}
+
+
+def _below_one(score: object) -> bool:
+    return isinstance(score, (int, float)) and score < 1
+
+
+def _criterion(result: dict, index: int) -> str:
+    """How a result names its criterion: verifiers name it differently, and some only by its type."""
+    for key in ("criterion", "id", "name"):
+        if isinstance(result.get(key), str) and result[key]:
+            return result[key]
+    return f"{result['type']} #{index}" if isinstance(result.get("type"), str) else f"#{index}"
+
+
+def _reason(result: dict) -> str:
+    for key in ("justification", "message", "reason", "description"):
+        if isinstance(result.get(key), str) and result[key].strip():
+            return result[key]
+    return ""
+
+
 def _dependency_ids(steps: list["TaskStep"]) -> dict[str, set[str]]:
     """Direct dependency ids per step; depends_on None = all prior (scheduler semantics)."""
     deps: dict[str, set[str]] = {}
diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py
index 7d95138c..cb85a139 100644
--- a/tst/integration/cli/run_bundle_test.py
+++ b/tst/integration/cli/run_bundle_test.py
@@ -120,11 +120,28 @@ def test_a_bundle_runs_from_its_folder_and_a_rerun_writes_only_what_changed(bund
     third = _run(bundle)
     assert "artifacts/greeting: v2 (files changed: hello.txt)" in third
     assert "tasks/hello.json v1: scored below 1 (hello: 0)" in third
+    assert "\n    hello: greets: does not contain 'hello'\n" in third
     (instance_id,) = re.findall(r"instance (\S+)", third)
     with namespace_routing():
         results = Task.get_instance(instance_id).context["metadata"]["verifications"]["hello"]["results"]
     assert [(result["criterion"], result["result"]) for result in results] == [("greets", False), ("check passes", False)]
 
+    gated = CliRunner().invoke(cli, ["run", str(bundle), "--fail-under", "1"])
+    assert gated.exit_code == 1, gated.output
+    assert gated.output.endswith("\nBelow --fail-under 1: tasks/hello.json (0)\n"), gated.output
+
+
+def test_a_run_whose_reader_leaves_finishes_quietly_and_tears_down(bundle):
+    """As when piped into head: the reader closes the pipe after one line, and the run goes on to its end."""
+    process = subprocess.Popen([sys.executable, "-m", "agent_env.cli", "run", str(bundle)], stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE, text=True)
+    process.stdout.readline()
+    process.stdout.close()
+    errors = process.stderr.read()
+
+    assert process.wait(timeout=120) == 0, errors
+    assert "BrokenPipeError" not in errors, errors
+
 
 @skip_without_gnu_stat()
 def test_a_bundle_task_collects_from_its_sandbox_into_a_universe_named_after_the_run(bundle, quiet_logs):
diff --git a/tst/unit/bundle/installed_test.py b/tst/unit/bundle/installed_test.py
index 9af92eb7..261c59b7 100644
--- a/tst/unit/bundle/installed_test.py
+++ b/tst/unit/bundle/installed_test.py
@@ -238,7 +238,8 @@ def test_the_cli_says_so_when_no_bundles_are_installed_and_needs_a_bundle_for_it
     for option in (["--task", "t"], ["--keep"], ["--dry-run"]):
         usage = CliRunner().invoke(cli, ["run", *option])
         assert usage.exit_code == 2
-        assert "--task, --eval, --model, --sandbox, --keep and --dry-run need a BUNDLE to run" in usage.output
+        options = "--task, --eval, --model, --sandbox, --keep, --dry-run and --fail-under"
+        assert f"{options} need a BUNDLE to run" in " ".join(usage.output.split())
 
 
 def test_the_cli_runs_an_installed_bundle_by_name_with_ids_rooted_at_its_package(site, monkeypatch, quiet_logs,
diff --git a/tst/unit/bundle/run_test.py b/tst/unit/bundle/run_test.py
index 56a472c9..47791b97 100644
--- a/tst/unit/bundle/run_test.py
+++ b/tst/unit/bundle/run_test.py
@@ -41,16 +41,17 @@ class _Scored(TaskStep):
     peak = 0
     seen: list[TaskStepContext] = []
 
-    def __init__(self, score=1.0, verifier_id=None, **base):
+    def __init__(self, score=1.0, verifier_id=None, results=None, **base):
         super().__init__(**base)
-        self.score, self.verifier_id = score, verifier_id
+        self.score, self.verifier_id, self.results = score, verifier_id, results
 
     @classmethod
     def from_dict(cls, data):
-        return cls(**cls._base_from_dict(data), score=data.get("score"), verifier_id=data.get("verifier_id"))
+        return cls(**cls._base_from_dict(data), score=data.get("score"), verifier_id=data.get("verifier_id"),
+                   results=data.get("results"))
 
     def to_dict(self):
-        return {**super().to_dict(), "score": self.score, "verifier_id": self.verifier_id}
+        return {**super().to_dict(), "score": self.score, "verifier_id": self.verifier_id, "results": self.results}
 
     async def execute(self, context):
         _Scored.seen.append(context)
@@ -61,7 +62,9 @@ async def execute(self, context):
         finally:
             _Scored.active -= 1
         if self.verifier_id is not None:
-            context.metadata.setdefault("verifications", {})[self.verifier_id] = {"score": self.score}
+            verification = {"score": self.score} if self.results is None else {"score": self.score,
+                                                                                "results": self.results}
+            context.metadata.setdefault("verifications", {})[self.verifier_id] = verification
         return context
 
 
@@ -408,6 +411,71 @@ def test_the_cli_exits_0_when_no_run_failed_even_if_one_scored_below_1(bundle_di
     assert "tasks/b.json v1: scored below 1" in result.output
 
 
+FAILED = [
+    {"criterion": "greets", "result": False, "justification": "does not contain 'hello'"},
+    {"criterion": "builds", "result": True, "justification": "exit=0"},
+    {"id": "loaded", "result": False, "description": "list_items lists ['a']: {'items': []}"},
+    {"type": "response_contains", "result": False, "justification": "missing:\n  'Echo: hi'"},
+    {"id": "row-1", "score": 0.0, "message": "x" * 300},
+    {"criterion": "slow", "result": False, "skipped": True, "justification": "not run"},
+    {"id": "exit_code", "result": False, "message": "exit_code=2 timed_out=False"},
+    {"id": "late", "result": False},
+]
+
+
+def _criteria_bundle(root):
+    layout(root, {"tasks/a.json": _scored(1.0), "tasks/t.json": json.dumps([
+        {"id": "check", "type": "scored_run_test", "verifier_id": "v", "score": 0.2, "results": FAILED}])})
+    return root
+
+
+def test_the_summary_names_each_failed_criterion_and_why_up_to_five(bundle_dir, quiet_logs):
+    result = CliRunner().invoke(cli, ["run", str(_criteria_bundle(bundle_dir)), "--task", "t"])
+
+    assert result.exit_code == 0, result.output
+    lines = result.output.split("tasks/t.json v1: scored below 1 (check: 0.2)", 1)[1].splitlines()[1:7]
+    assert lines == [
+        "    check: greets: does not contain 'hello'",
+        "    check: loaded: list_items lists ['a']: {'items': []}",
+        "    check: response_contains #3: missing: 'Echo: hi'",
+        f"    check: row-1: {'x' * 159}…",
+        "    check: exit_code: exit_code=2 timed_out=False",
+        "    and 1 more",
+    ]
+
+
+@pytest.mark.parametrize("args, code, below", [
+    (["--task", "a", "--task", "b", "--fail-under", "1"], 1, "Below --fail-under 1: tasks/b.json (0.5)"),
+    (["--task", "a", "--task", "b", "--fail-under", "0.5"], 0, None),
+    (["--task", "a", "--fail-under", "1"], 0, None),
+    (["--task", "a", "--task", "b"], 0, None),
+], ids=["below", "at the threshold", "all passed", "no threshold"])
+def test_fail_under_exits_1_when_a_task_scored_below_it(bundle_dir, quiet_logs, args, code, below):
+    result = CliRunner().invoke(cli, ["run", str(bundle_dir), *args])
+
+    assert result.exit_code == code, result.output
+    assert (result.output.rstrip().rsplit("\n", 1)[-1] == below) if below else "Below --fail-under" not in result.output
+
+
+def test_fail_under_fails_a_task_that_recorded_no_score(bundle_dir, quiet_logs):
+    layout(bundle_dir, {"tasks/quiet.json": json.dumps([{"id": "step", "type": "scored_run_test"}])})
+
+    result = CliRunner().invoke(cli, ["run", str(bundle_dir), "--task", "quiet", "--fail-under", "0"])
+
+    assert result.exit_code == 1, result.output
+    assert "tasks/quiet.json v1: unscored" in result.output
+    assert result.output.endswith("\nBelow --fail-under 0: tasks/quiet.json (no score)\n")
+
+
+def test_fail_under_needs_a_bundle_and_changes_nothing_a_dry_run_shows(bundle_dir, quiet_logs):
+    bare = CliRunner().invoke(cli, ["run", "--fail-under", "1"])
+    dry = CliRunner().invoke(cli, ["run", str(bundle_dir), "--task", "b", "--fail-under", "1", "--dry-run"])
+
+    assert bare.exit_code == 2 and "--fail-under need a BUNDLE to run" in bare.output
+    assert dry.exit_code == 0, dry.output
+    assert "Below --fail-under" not in dry.output
+
+
 def test_the_cli_prints_an_unknown_sandbox_as_one_line(bundle_dir, quiet_logs):
     result = CliRunner().invoke(cli, ["run", str(bundle_dir), "--sandbox", "nowhere"])
 
diff --git a/tst/unit/task/failed_criteria_test.py b/tst/unit/task/failed_criteria_test.py
new file mode 100644
index 00000000..70fe1896
--- /dev/null
+++ b/tst/unit/task/failed_criteria_test.py
@@ -0,0 +1,48 @@
+"""The criteria a run's verifications recorded as failed, read from its context the way its scores are."""
+
+from agent_env.task.task import FailedCriterion, failed_criteria
+from agent_env.task_step.context import TaskStepContext
+
+
+class _Step:
+    def __init__(self, id, verifier_id):
+        self.id, self.verifier_id = id, verifier_id
+
+
+def _failed(verifications, steps=()):
+    return failed_criteria(list(steps), TaskStepContext(metadata={"verifications": verifications}))
+
+
+def test_a_failed_criterion_is_named_by_the_step_that_recorded_it_or_else_by_its_key():
+    verifications = {"v": {"score": 0, "results": [{"criterion": "greets", "result": False}]},
+                     "v-trajectory-mistakes": {"score": 0.5, "results": [{"id": "m1", "score": 0.5}]}}
+
+    assert _failed(verifications, [_Step("check", "v")]) == [
+        FailedCriterion("check", "greets", ""), FailedCriterion("v-trajectory-mistakes", "m1", "")]
+
+
+def test_only_a_verification_scoring_below_1_has_failed_criteria():
+    verifications = {"full": {"score": 1, "results": [{"id": "a", "result": False}]},
+                     "unscored": {"results": [{"id": "b", "result": False}]},
+                     "partial": {"score": 0.5, "results": [{"id": "c", "result": True}, {"id": "d", "result": False}]}}
+
+    assert [criterion.criterion for criterion in _failed(verifications)] == ["d"]
+
+
+def test_a_result_without_a_result_fails_by_its_score_and_a_skipped_one_never_fails():
+    verifications = {"v": {"score": 0.4, "results": [
+        {"id": "low", "score": 0.4}, {"id": "full", "score": 1.0}, {"id": "unsure"},
+        {"id": "skipped", "result": False, "skipped": True}, "not a result"]}}
+
+    assert [criterion.criterion for criterion in _failed(verifications)] == ["low"]
+
+
+def test_the_reason_is_the_first_one_the_verifier_gave():
+    verifications = {"v": {"score": 0, "results": [
+        {"id": "a", "result": False, "justification": "why", "message": "what", "description": "about"},
+        {"id": "b", "result": False, "message": "what", "description": "about"},
+        {"id": "c", "result": False, "justification": "  ", "description": "about"},
+        {"type": "response_contains", "result": False}]}}
+
+    assert [(criterion.criterion, criterion.reason) for criterion in _failed(verifications)] == [
+        ("a", "why"), ("b", "what"), ("c", "about"), ("response_contains #3", "")]

From 6626abeca1da3ea70499523aa98c34058156ce23 Mon Sep 17 00:00:00 2001
From: Edgar Arakelyan <edgar.arakelyan@scale.com>
Date: Wed, 14 Oct 2026 11:23:00 -0700
Subject: [PATCH 2/3] fix(run): print each line through one guard, so a reader
 leaving never cuts a block short

Every line agent-env run prints goes through _echo, which drops the line once its stream's reader has gone
and leaves the other stream alone. A block no longer stops at its first broken write, so --keep still waits
and tears down after the reader leaves, and --verbose tracebacks still print when only stdout is piped away.
The "Still up" warning goes to stderr, which still reaches the user when stdout is piped away.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 src/agent_env/cli/run.py               | 119 +++++++++++--------------
 tst/integration/cli/run_bundle_test.py |  53 +++++++++--
 tst/unit/bundle/run_teardown_test.py   |   1 +
 3 files changed, 100 insertions(+), 73 deletions(-)

diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py
index b4ff3eca..38085a75 100644
--- a/src/agent_env/cli/run.py
+++ b/src/agent_env/cli/run.py
@@ -7,8 +7,6 @@
 import sys
 import time
 import traceback
-from collections.abc import Iterator
-from contextlib import contextmanager
 from pathlib import Path
 
 import click
@@ -82,8 +80,7 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t
         if dry_run:
             _echo(_DRY_RUN)
             dry = dry_run_bundle(root, tasks=tasks, evals=evals, sandbox=sandbox, on_progress=_echo, id_root=id_root)
-            with _reader_may_leave():
-                _report_dry_run(dry)
+            _report_dry_run(dry)
             return
         result = run_bundle(root, tasks=tasks, evals=evals, model=model, sandbox=sandbox, on_progress=_echo,
                             id_root=id_root, keep=keep)
@@ -91,39 +88,29 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t
         _note_an_installed_namesake(e, bundle, root, id_root)
         raise
     except RunInterrupted as stop:
-        with Interrupts(), _reader_may_leave():
+        with Interrupts():
             _summarize(stop.result, verbose)
             _report_teardown(stop.result)
         raise SystemExit(128 + stop.signum) from None
     with Interrupts() as interrupts:
-        with _reader_may_leave():
-            _summarize(result, verbose)
+        _summarize(result, verbose)
         if keep:
             result = _hold(result, interrupts)
-        with _reader_may_leave():
-            _report_teardown(result)
+        _report_teardown(result)
     below = result.below(fail_under) if fail_under is not None else ()
-    with _reader_may_leave():
-        _report_below(result, below, fail_under)
+    _report_below(result, below, fail_under)
     if result.failed or below:
         raise SystemExit(1)
 
 
-def _echo(line: str) -> None:
-    """``click.echo`` for the run's progress, which stops printing once stdout's reader has gone, as when piped
-    into head, so the run still finishes and tears down."""
-    with _reader_may_leave():
-        click.echo(line)
-
-
-@contextmanager
-def _reader_may_leave() -> Iterator[None]:
-    """Print the block's output, or, once stdout's reader has gone, send the rest of the output nowhere."""
+def _echo(message: str | None = None, err: bool = False) -> None:
+    """``click.echo``, which drops the line once its stream's reader has gone, as when piped into head, so the run
+    still finishes and tears down."""
     try:
-        yield
+        click.echo(message, err=err)
     except BrokenPipeError:
         devnull = os.open(os.devnull, os.O_WRONLY)
-        os.dup2(devnull, sys.stdout.fileno())
+        os.dup2(devnull, (sys.stderr if err else sys.stdout).fileno())
         os.close(devnull)
 
 
@@ -132,7 +119,7 @@ def _report_below(result: BundleRun, below: tuple[TaskRun, ...], threshold: floa
         return
     shown = ", ".join(f"{result.path(run.entry)} ({min(run.scores.values()):g})" if run.scores
                       else f"{result.path(run.entry)} (no score)" for run in below)
-    click.echo(f"\nBelow --fail-under {threshold:g}: {shown}")
+    _echo(f"\nBelow --fail-under {threshold:g}: {shown}")
 
 
 def _path_like(text: str) -> bool:
@@ -177,21 +164,21 @@ def _installed() -> tuple[InstalledBundle, ...]:
 def _list_installed() -> None:
     bundles = _installed()
     if not bundles:
-        click.echo("No bundles are installed. agent-env run PATH runs a folder.")
+        _echo("No bundles are installed. agent-env run PATH runs a folder.")
         return
     header = ("NAME", "PACKAGE", "CONTENTS", "DESCRIPTION")
     rows = [(run_name(bundle, bundles), f"{bundle.package} {bundle.version}", *_contents(bundle)) for bundle in bundles]
     widths = [max(len(row[i]) for row in (header, *rows)) for i in range(3)]
 
     def echo(name: str, package: str, contents: str, description: str) -> None:
-        click.echo(f"{name:<{widths[0]}}  {package:<{widths[1]}}  {contents:<{widths[2]}}  {description}".rstrip())
+        _echo(f"{name:<{widths[0]}}  {package:<{widths[1]}}  {contents:<{widths[2]}}  {description}".rstrip())
 
     echo(*header)
     for bundle, row in zip(bundles, rows):
         echo(*row)
         if bundle.root is not None:
-            click.echo(f"{'':<{widths[0]}}  {bundle.root}")
-    click.echo("\nRun one with agent-env run NAME. To change one, copy its folder and run the copy.")
+            _echo(f"{'':<{widths[0]}}  {bundle.root}")
+    _echo("\nRun one with agent-env run NAME. To change one, copy its folder and run the copy.")
 
 
 def _contents(bundle: InstalledBundle) -> tuple[str, str]:
@@ -212,23 +199,23 @@ def _count(entries, kind: BundleKind, noun: str) -> str:
 
 
 def _summarize(result: BundleRun, verbose: bool) -> None:
-    click.echo()
-    click.echo("Tasks:")
+    _echo()
+    _echo("Tasks:")
     for run in result.runs:
-        click.echo(click.style(f"  {_described(result, run)}", fg=_COLORS[run.outcome]))
+        _echo(click.style(f"  {_described(result, run)}", fg=_COLORS[run.outcome]))
         for line in _failed_criteria(run):
-            click.echo(f"    {line}")
+            _echo(f"    {line}")
     if result.evals:
-        click.echo("Evals:")
+        _echo("Evals:")
     for eval_run in result.evals:
         passed, failed, cancelled = (sum(run.outcome is outcome for run in eval_run.runs)
                                      for outcome in (Outcome.PASSED, Outcome.FAILED, Outcome.CANCELLED))
         line = f"  {result.path(eval_run.entry)} v{eval_run.version}: {passed} of {len(eval_run.runs)} passed"
         rest = [f"{failed} failed" if failed else "", f"{cancelled} cancelled" if cancelled else ""]
-        click.echo(", ".join([line, *filter(None, rest)]))
+        _echo(", ".join([line, *filter(None, rest)]))
     for entry in result.skipped:
-        click.echo(f"{result.path(entry)} isn't named by any eval, so it didn't run; run it with "
-                   f"--task {shlex.quote(entry.name)}")
+        _echo(f"{result.path(entry)} isn't named by any eval, so it didn't run; run it with "
+              f"--task {shlex.quote(entry.name)}")
     if verbose:
         _print_tracebacks(result)
 
@@ -251,45 +238,45 @@ def _failed_criteria(run: TaskRun) -> list[str]:
 def _report_dry_run(dry: DryRun) -> None:
     materialization = dry.materialization
     plan = materialization.plan
-    click.echo()
+    _echo()
     if plan.store_refs:
-        click.echo("Store refs:")
+        _echo("Store refs:")
     for ref in plan.store_refs:
         if ref.version is None:
-            click.echo(f"  {ref.kind} {ref.id} v{plan.store_latest[ref.kind, ref.id]}, the latest")
+            _echo(f"  {ref.kind} {ref.id} v{plan.store_latest[ref.kind, ref.id]}, the latest")
         else:
-            click.echo(f"  {ref.kind} {ref.id} v{ref.version}")
+            _echo(f"  {ref.kind} {ref.id} v{ref.version}")
     if images := dry.images():
-        click.echo("Images:")
+        _echo("Images:")
     for line in images:
-        click.echo(f"  {line}")
+        _echo(f"  {line}")
     if dry.infra:
-        click.echo("Would build first:")
+        _echo("Would build first:")
     for build in dry.infra:
-        click.echo(f"  {build}")
-    click.echo("Would run:")
+        _echo(f"  {build}")
+    _echo("Would run:")
     for entry in dry.runs:
-        click.echo(f"  {dry.path(entry)} v{materialization.version_of('task', entry.id)}")
+        _echo(f"  {dry.path(entry)} v{materialization.version_of('task', entry.id)}")
     if plan.evals:
-        click.echo("Evals:")
+        _echo("Evals:")
     for entry in plan.evals:
         named = ", ".join(dry.path(ref.local) for ref in entry.references)
-        click.echo(f"  {dry.path(entry.entry)} v{materialization.version_of('eval', entry.entry.id)}: {named}")
+        _echo(f"  {dry.path(entry.entry)} v{materialization.version_of('eval', entry.entry.id)}: {named}")
     for entry in dry.skipped:
-        click.echo(f"{dry.path(entry)} isn't named by any eval, so it wouldn't run; run it with "
-                   f"--task {shlex.quote(entry.name)}")
+        _echo(f"{dry.path(entry)} isn't named by any eval, so it wouldn't run; run it with "
+              f"--task {shlex.quote(entry.name)}")
     if materialization.not_preflighted:
-        click.echo("Not preflighted, since each reads what the run would write first:")
+        _echo("Not preflighted, since each reads what the run would write first:")
     for write, step in materialization.not_preflighted:
-        click.echo(f"  {dry.path(write.source.entry)}: step {step.id!r} ({step.type})")
-    click.echo(_DRY_RUN)
+        _echo(f"  {dry.path(write.source.entry)}: step {step.id!r} ({step.type})")
+    _echo(_DRY_RUN)
 
 
 def _print_tracebacks(result: BundleRun) -> None:
     for run in result.runs:
         if run.error is not None:
-            click.echo(f"\n{result.path(run.entry)} raised:", err=True)
-            click.echo("".join(traceback.format_exception(run.error)).rstrip(), err=True)
+            _echo(f"\n{result.path(run.entry)} raised:", err=True)
+            _echo("".join(traceback.format_exception(run.error)).rstrip(), err=True)
 
 
 def _described(result: BundleRun, run: TaskRun) -> str:
@@ -314,22 +301,22 @@ def _hold(result: BundleRun, interrupts: Interrupts) -> BundleRun:
     kept = [run for run in result.runs if run.torn_down is None]
     count = sum(len(TeardownReport.skipped(run.context).left) for run in kept)
     if not count:
-        click.echo("\nNothing to keep up: no run left a sandbox.")
+        _echo("\nNothing to keep up: no run left a sandbox.")
         return result
-    click.echo("\nKept up:")
+    _echo("\nKept up:")
     for run in kept:
         lines = _kept_up(run.context)
         if lines:
-            click.echo(f"  {result.path(run.entry)}")
+            _echo(f"  {result.path(run.entry)}")
             for line in lines:
-                click.echo(f"    {line}")
+                _echo(f"    {line}")
     them = "it" if count == 1 else "them"
-    click.echo(f"\nHolding {_sandboxes(count)} up; Ctrl-C tears {them} down.")
+    _echo(f"\nHolding {_sandboxes(count)} up; Ctrl-C tears {them} down.")
     while not interrupts.count:
         time.sleep(0.2)
     if interrupts.count > 1:
         return result
-    click.echo(f"\nTearing down {_sandboxes(count)} (Ctrl-C again to stop now)")
+    _echo(f"\nTearing down {_sandboxes(count)} (Ctrl-C again to stop now)")
     try:
         return result.teardown()
     except RunInterrupted as stopped:
@@ -378,13 +365,13 @@ def _report_teardown(result: BundleRun) -> None:
     still_up = [sandbox for report in reports for sandbox in report.still_up
                 if sandbox.sandbox_type != "local" or LocalSandbox.find_work_dir(sandbox.sandbox_id)]
     if terminated or still_up:
-        click.echo()
+        _echo()
     if terminated:
-        click.echo(f"Tore down {_sandboxes(terminated)}.")
-    if still_up:
-        click.echo(click.style(f"Still up, {_sandboxes(len(still_up))}:", fg="yellow"))
+        _echo(f"Tore down {_sandboxes(terminated)}.")
+    if still_up:  # on stderr, which still reaches the user when stdout is piped away
+        _echo(click.style(f"Still up, {_sandboxes(len(still_up))}:", fg="yellow"), err=True)
         for sandbox in still_up:
-            click.echo(f"  {_still_up(sandbox)}")
+            _echo(f"  {_still_up(sandbox)}", err=True)
 
 
 def _still_up(sandbox: RecordedSandbox) -> str:
diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py
index cb85a139..0f16ae94 100644
--- a/tst/integration/cli/run_bundle_test.py
+++ b/tst/integration/cli/run_bundle_test.py
@@ -124,7 +124,8 @@ def test_a_bundle_runs_from_its_folder_and_a_rerun_writes_only_what_changed(bund
     (instance_id,) = re.findall(r"instance (\S+)", third)
     with namespace_routing():
         results = Task.get_instance(instance_id).context["metadata"]["verifications"]["hello"]["results"]
-    assert [(result["criterion"], result["result"]) for result in results] == [("greets", False), ("check passes", False)]
+    assert [(result["criterion"], result["result"]) for result in results] == [
+        ("greets", False), ("check passes", False)]
 
     gated = CliRunner().invoke(cli, ["run", str(bundle), "--fail-under", "1"])
     assert gated.exit_code == 1, gated.output
@@ -133,16 +134,54 @@ def test_a_bundle_runs_from_its_folder_and_a_rerun_writes_only_what_changed(bund
 
 def test_a_run_whose_reader_leaves_finishes_quietly_and_tears_down(bundle):
     """As when piped into head: the reader closes the pipe after one line, and the run goes on to its end."""
-    process = subprocess.Popen([sys.executable, "-m", "agent_env.cli", "run", str(bundle)], stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE, text=True)
-    process.stdout.readline()
-    process.stdout.close()
-    errors = process.stderr.read()
+    code, errors = _reader_leaves([bundle], "")
 
-    assert process.wait(timeout=120) == 0, errors
+    assert code == 0, errors
     assert "BrokenPipeError" not in errors, errors
 
 
+def test_a_kept_run_whose_reader_leaves_still_tears_down_on_ctrl_c(bundle, sigint_handled):
+    """As when piped into tee, which Ctrl-C stops too: the reader leaves while the run holds its sandbox up."""
+    code, errors = _reader_leaves([bundle, "--keep"], "Holding 1 sandbox up", signal.SIGINT)
+
+    assert code == 0, errors
+    assert "BrokenPipeError" not in errors, errors
+
+
+def test_a_stream_whose_reader_has_gone_drops_its_lines_and_the_other_stream_still_prints():
+    """As with --verbose piped from, when only stderr's reader leaves."""
+    read, write = os.pipe()
+    os.close(read)
+    echo = "from agent_env.cli.run import _echo; _echo('a'); _echo('x', err=True); _echo('b'); _echo('y', err=True)"
+    try:
+        result = subprocess.run([sys.executable, "-c", echo], stdout=subprocess.PIPE, stderr=write, text=True,
+                                timeout=120)
+    finally:
+        os.close(write)
+
+    assert (result.returncode, result.stdout) == (0, "a\nb\n")
+
+
+def _reader_leaves(args, when, signum=None):
+    """Run ``agent-env run ARGS`` in its own process group, close its stdout once a line containing ``when`` is
+    printed, then send ``signum`` to the group, if given. Returns the exit code and what it printed on stderr."""
+    process = subprocess.Popen([sys.executable, "-m", "agent_env.cli", "run", *map(str, args)], stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE, text=True, start_new_session=True)
+    watchdog = threading.Timer(120, process.kill)
+    watchdog.start()
+    try:
+        for line in process.stdout:
+            if when in line:
+                break
+        process.stdout.close()
+        if signum is not None:
+            os.killpg(process.pid, signum)
+        _, errors = process.communicate()
+        return process.returncode, errors
+    finally:
+        watchdog.cancel()
+
+
 @skip_without_gnu_stat()
 def test_a_bundle_task_collects_from_its_sandbox_into_a_universe_named_after_the_run(bundle, quiet_logs):
     steps = json.loads((bundle / "tasks/hello.json").read_text())
diff --git a/tst/unit/bundle/run_teardown_test.py b/tst/unit/bundle/run_teardown_test.py
index 916dbaf2..442544fc 100644
--- a/tst/unit/bundle/run_teardown_test.py
+++ b/tst/unit/bundle/run_teardown_test.py
@@ -287,6 +287,7 @@ def test_the_cli_prints_a_sandbox_it_couldnt_tear_down_as_still_up(bundle_dir, t
     assert result.exit_code == 0, result.output
     assert "[tasks/ok.json] couldn't tear down sb-ok: RuntimeError: stuck\n" in result.output
     assert result.output.endswith("\nStill up, 1 sandbox:\n  sb-ok  fake\n")
+    assert result.stderr.endswith("Still up, 1 sandbox:\n  sb-ok  fake\n")
 
 
 def test_the_cli_keep_prints_the_endpoints_holds_and_tears_down_on_ctrl_c(bundle_dir, terminated, quiet_logs,

From 8b86e8d78ce23bfe64dcd0103551bc2f3ee8bad8 Mon Sep 17 00:00:00 2001
From: Edgar Arakelyan <edgar.arakelyan@scale.com>
Date: Wed, 14 Oct 2026 11:48:28 -0700
Subject: [PATCH 3/3] refactor(run): derive the BUNDLE usage error from the
 flags given, and read verifications in one loop

agent-env run without a BUNDLE names the flags it was given ("--fail-under needs a BUNDLE to run") instead
of a hand-kept list of every flag. recorded_scores and failed_criteria share one loop over the recorded
verifications, so both name a step the same way, and the failure rule and the text lookups are stated once.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 src/agent_env/cli/run.py              | 23 ++++++------
 src/agent_env/task/task.py            | 54 +++++++++++++--------------
 tst/unit/bundle/installed_test.py     |  6 ++-
 tst/unit/bundle/run_test.py           |  2 +-
 tst/unit/task/failed_criteria_test.py |  2 +-
 5 files changed, 44 insertions(+), 43 deletions(-)

diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py
index 38085a75..4f5c1bfb 100644
--- a/src/agent_env/cli/run.py
+++ b/src/agent_env/cli/run.py
@@ -10,6 +10,7 @@
 from pathlib import Path
 
 import click
+from click.core import ParameterSource
 
 from agent_env.bundle import (BundleError, BundleKind, BundleRun, DryRun, Outcome, RunInterrupted, TaskRun,
                               dry_run_bundle, run_bundle)
@@ -26,9 +27,7 @@
 _ENV_ENDPOINTS = (("mcp", "mcp_url"), ("gateway", "gateway_url"), ("pgweb", "db_web_url"), ("db-mcp", "db_mcp_url"),
                   ("vnc", "vnc_url"), ("expires", "expires_at_utc"))
 _DRY_RUN = "Dry run: nothing is written or run."
-# How many of a task's failed criteria the summary names before it counts the rest.
 _SHOWN_CRITERIA = 5
-# How much of a failed criterion's reason fits its summary line.
 _REASON_CHARS = 160
 
 
@@ -69,9 +68,12 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t
     exits 130 (143 for SIGTERM); a second one stops the teardown and prints what is still up.
     """
     if bundle is None:
-        if tasks or evals or model or sandbox or keep or dry_run or fail_under is not None:
-            raise click.UsageError("--task, --eval, --model, --sandbox, --keep, --dry-run and --fail-under need a "
-                                   "BUNDLE to run")
+        given = [param.opts[0] for param in ctx.command.params if isinstance(param, click.Option)
+                 and ctx.get_parameter_source(param.name) not in (None, ParameterSource.DEFAULT)]
+        if given:
+            *rest, last = given
+            raise click.UsageError(f"{', '.join(rest)} and {last} need a BUNDLE to run" if rest
+                                   else f"{last} needs a BUNDLE to run")
         _list_installed()
         return
     root, id_root = _locate(bundle)
@@ -98,7 +100,8 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t
             result = _hold(result, interrupts)
         _report_teardown(result)
     below = result.below(fail_under) if fail_under is not None else ()
-    _report_below(result, below, fail_under)
+    if below:
+        _report_below(result, below, fail_under)
     if result.failed or below:
         raise SystemExit(1)
 
@@ -114,9 +117,7 @@ def _echo(message: str | None = None, err: bool = False) -> None:
         os.close(devnull)
 
 
-def _report_below(result: BundleRun, below: tuple[TaskRun, ...], threshold: float | None) -> None:
-    if not below:
-        return
+def _report_below(result: BundleRun, below: tuple[TaskRun, ...], threshold: float) -> None:
     shown = ", ".join(f"{result.path(run.entry)} ({min(run.scores.values()):g})" if run.scores
                       else f"{result.path(run.entry)} (no score)" for run in below)
     _echo(f"\nBelow --fail-under {threshold:g}: {shown}")
@@ -203,7 +204,7 @@ def _summarize(result: BundleRun, verbose: bool) -> None:
     _echo("Tasks:")
     for run in result.runs:
         _echo(click.style(f"  {_described(result, run)}", fg=_COLORS[run.outcome]))
-        for line in _failed_criteria(run):
+        for line in _criteria_lines(run):
             _echo(f"    {line}")
     if result.evals:
         _echo("Evals:")
@@ -220,7 +221,7 @@ def _summarize(result: BundleRun, verbose: bool) -> None:
         _print_tracebacks(result)
 
 
-def _failed_criteria(run: TaskRun) -> list[str]:
+def _criteria_lines(run: TaskRun) -> list[str]:
     """A line for each criterion the run failed, its step, the criterion and the reason it gave, up to
     ``_SHOWN_CRITERIA``, then a count of the rest."""
     failed = run.failed_criteria
diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py
index fd91b373..5038e996 100644
--- a/src/agent_env/task/task.py
+++ b/src/agent_env/task/task.py
@@ -11,6 +11,7 @@
 import logging
 import time
 import uuid
+from collections.abc import Iterator
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Callable, ClassVar, Optional
@@ -120,13 +121,8 @@ def _sub(value):
 def recorded_scores(steps: list["TaskStep"], context: TaskStepContext) -> dict[str, float]:
     """Every score a run of ``steps`` recorded, by the id of the step that recorded it, or by its verification's key
     when that isn't a step's verifier id."""
-    step_ids = _verifier_steps(steps)
-    scores = {}
-    for key, verification in (context.metadata.get("verifications") or {}).items():
-        score = verification.get("score")
-        if isinstance(score, (int, float)):
-            scores[step_ids.get(key, key)] = float(score)
-    return scores
+    return {name: float(verification["score"]) for name, verification in _verifications(steps, context)
+            if isinstance(verification.get("score"), (int, float))}
 
 
 @dataclass(frozen=True)
@@ -139,45 +135,47 @@ class FailedCriterion:
 
 
 def failed_criteria(steps: list["TaskStep"], context: TaskStepContext) -> list[FailedCriterion]:
-    """Each criterion that failed in a verification scoring below 1: its result is false, or, recording no result,
-    its score is below 1. A skipped criterion didn't fail."""
-    step_ids = _verifier_steps(steps)
+    """Each criterion that failed in a verification scoring below 1, named as ``recorded_scores`` names its score."""
     failed = []
-    for key, verification in (context.metadata.get("verifications") or {}).items():
+    for name, verification in _verifications(steps, context):
         score = verification.get("score")
         if not isinstance(score, (int, float)) or score >= 1:
             continue
         for index, result in enumerate(verification.get("results") or []):
-            if not isinstance(result, dict) or result.get("skipped"):
-                continue
-            passed = bool(result["result"]) if "result" in result else not _below_one(result.get("score"))
-            if not passed:
-                failed.append(FailedCriterion(step_ids.get(key, key), _criterion(result, index), _reason(result)))
+            if isinstance(result, dict) and _failed(result):
+                reason = _first_text(result, ("justification", "message", "reason", "description"))
+                failed.append(FailedCriterion(name, _criterion(result, index), reason))
     return failed
 
 
-def _verifier_steps(steps: list["TaskStep"]) -> dict[str, str]:
-    """The id of each step that records a verification, by its verifier id."""
-    return {step.verifier_id: step.id for step in steps if getattr(step, "verifier_id", None)}
+def _verifications(steps: list["TaskStep"], context: TaskStepContext) -> Iterator[tuple[str, dict]]:
+    """Each verification a run of ``steps`` recorded, by the id of the step that recorded it, or by its key when
+    that isn't a step's verifier id."""
+    step_ids = {step.verifier_id: step.id for step in steps if getattr(step, "verifier_id", None)}
+    for key, verification in (context.metadata.get("verifications") or {}).items():
+        yield step_ids.get(key, key), verification
 
 
-def _below_one(score: object) -> bool:
+def _failed(result: dict) -> bool:
+    """A result fails when its result is false or, recording none, its score is below 1; a skipped one never does."""
+    if result.get("skipped"):
+        return False
+    if "result" in result:
+        return not result["result"]
+    score = result.get("score")
     return isinstance(score, (int, float)) and score < 1
 
 
 def _criterion(result: dict, index: int) -> str:
     """How a result names its criterion: verifiers name it differently, and some only by its type."""
-    for key in ("criterion", "id", "name"):
-        if isinstance(result.get(key), str) and result[key]:
-            return result[key]
+    if name := _first_text(result, ("criterion", "id", "name")):
+        return name
     return f"{result['type']} #{index}" if isinstance(result.get("type"), str) else f"#{index}"
 
 
-def _reason(result: dict) -> str:
-    for key in ("justification", "message", "reason", "description"):
-        if isinstance(result.get(key), str) and result[key].strip():
-            return result[key]
-    return ""
+def _first_text(result: dict, keys: tuple[str, ...]) -> str:
+    """The first of ``keys`` that holds text in ``result``, or "" when none does."""
+    return next((result[key] for key in keys if isinstance(result.get(key), str) and result[key].strip()), "")
 
 
 def _dependency_ids(steps: list["TaskStep"]) -> dict[str, set[str]]:
diff --git a/tst/unit/bundle/installed_test.py b/tst/unit/bundle/installed_test.py
index 261c59b7..bfd780ee 100644
--- a/tst/unit/bundle/installed_test.py
+++ b/tst/unit/bundle/installed_test.py
@@ -238,8 +238,10 @@ def test_the_cli_says_so_when_no_bundles_are_installed_and_needs_a_bundle_for_it
     for option in (["--task", "t"], ["--keep"], ["--dry-run"]):
         usage = CliRunner().invoke(cli, ["run", *option])
         assert usage.exit_code == 2
-        options = "--task, --eval, --model, --sandbox, --keep, --dry-run and --fail-under"
-        assert f"{options} need a BUNDLE to run" in " ".join(usage.output.split())
+        assert f"{option[0]} needs a BUNDLE to run" in usage.output
+    several = CliRunner().invoke(cli, ["run", "--task", "t", "--model", "", "--keep"])
+    assert several.exit_code == 2
+    assert "--task, --model and --keep need a BUNDLE to run" in several.output
 
 
 def test_the_cli_runs_an_installed_bundle_by_name_with_ids_rooted_at_its_package(site, monkeypatch, quiet_logs,
diff --git a/tst/unit/bundle/run_test.py b/tst/unit/bundle/run_test.py
index 47791b97..4815a70e 100644
--- a/tst/unit/bundle/run_test.py
+++ b/tst/unit/bundle/run_test.py
@@ -471,7 +471,7 @@ def test_fail_under_needs_a_bundle_and_changes_nothing_a_dry_run_shows(bundle_di
     bare = CliRunner().invoke(cli, ["run", "--fail-under", "1"])
     dry = CliRunner().invoke(cli, ["run", str(bundle_dir), "--task", "b", "--fail-under", "1", "--dry-run"])
 
-    assert bare.exit_code == 2 and "--fail-under need a BUNDLE to run" in bare.output
+    assert bare.exit_code == 2 and "--fail-under needs a BUNDLE to run" in bare.output
     assert dry.exit_code == 0, dry.output
     assert "Below --fail-under" not in dry.output
 
diff --git a/tst/unit/task/failed_criteria_test.py b/tst/unit/task/failed_criteria_test.py
index 70fe1896..66d5dd48 100644
--- a/tst/unit/task/failed_criteria_test.py
+++ b/tst/unit/task/failed_criteria_test.py
@@ -42,7 +42,7 @@ def test_the_reason_is_the_first_one_the_verifier_gave():
         {"id": "a", "result": False, "justification": "why", "message": "what", "description": "about"},
         {"id": "b", "result": False, "message": "what", "description": "about"},
         {"id": "c", "result": False, "justification": "  ", "description": "about"},
-        {"type": "response_contains", "result": False}]}}
+        {"type": "response_contains", "id": " ", "result": False}]}}
 
     assert [(criterion.criterion, criterion.reason) for criterion in _failed(verifications)] == [
         ("a", "why"), ("b", "what"), ("c", "about"), ("response_contains #3", "")]