fix: reject docs scores without scoring ground truth

.github/workflows/devdex-eval.yml

@@ -59,7 +59,7 @@ jobs:
       - name: Install
         run: |
           python -m pip install --upgrade pip
-          pip install pytest
+          pip install pytest numpy
       - name: Dataset + scorer tests
         # the DIRECTORY, not a hand-listed set -- a new test file must not need a CI edit to run
         run: python -m pytest devdex/tests -q

benchmark/run_benchmark.py

@@ -102,7 +102,7 @@ def _ci(values, rng):
 
 def report(engine, out=None):
     """Read the run records back and score them with the published scorer."""
-    from devdex.scorer.report_metrics import score_records
+    from devdex.scorer.report_metrics import check_docs_coverage, score_records
     rng = np.random.default_rng(0)
     rows, results = [], {}
     for t in TRACKS:
@@ -113,6 +113,8 @@ def report(engine, out=None):
         for f in files:
             recs += json.load(open(f))
         is_gold, pool_test = _scorer_for(t)
+        if t == "docs":
+            check_docs_coverage(recs, suite._docs_meta(), f"docs/{engine}")
         m = score_records(recs, t, is_gold, pool_test)
         if not m:
             continue

devdex/scorer/report_metrics.py

@@ -417,7 +417,7 @@ def _mean_cells(cells):
 
 def check_docs_coverage(records, docs_meta, cell):
     """Every docs qid must exist in the scoring GT, else its score is a silent zero."""
-    missing = sorted({r.get("qid") for r in records} - set(docs_meta)) if docs_meta else []
+    missing = sorted({r.get("qid") for r in records} - set(docs_meta))
     if missing:
         raise SystemExit(
             f"{cell}: {len(missing)} of {len(records)} items are absent from {DOCS_SCORING_GT} "

devdex/scorer/suite.py

@@ -62,7 +62,8 @@
 # its own moved fc-web groundedness 0.450 -> 0.802. Import the definitions so a change to
 # scoring cannot land in one scorer and not the other.
 from devdex.scorer.report_metrics import (      # noqa: E402
-    CAP, cite_ref as _cite_ref, engine_slots as _engine_slots, golds as _golds, norm as _norm,
+    CAP, check_docs_coverage, cite_ref as _cite_ref, engine_slots as _engine_slots,
+    golds as _golds, norm as _norm,
 )
 
 
@@ -229,6 +230,7 @@ def score_pass(rows, track):
 
     if track == "docs":
         meta = _docs_meta()
+        check_docs_coverage(rows, meta, "docs/pass")
         clean = [r for r in rows if not meta.get(r.get("qid"), {}).get("memory_answerable")]
         has_log = any(r.get("search_log") for r in rows)
         # DETERMINISTIC docs scoring. A cited source counts when it resolves to one of the

devdex/tests/test_docs_scoring_coverage.py

@@ -0,0 +1,62 @@
+"""Unavailable docs ground truth is an evaluation error, never a retrieval miss."""
+import json
+
+import pytest
+
+from devdex.scorer import report_metrics, suite
+
+
+def record(qid="unknown"):
+    return {"qid": qid, "sources": ["https://docs.example.test/guide"], "committed": True,
+            "search_log": [{"results": []}], "gt_file": "original.jsonl"}
+
+
+def test_empty_metadata_cannot_disable_coverage_validation():
+    with pytest.raises(SystemExit, match="absent from"):
+        report_metrics.check_docs_coverage([record()], {}, "docs/example")
+
+
+def test_no_records_needs_no_ground_truth():
+    report_metrics.check_docs_coverage([], {}, "docs/example")
+
+
+@pytest.mark.parametrize("meta", [{}, {"other": {}}])
+def test_direct_pass_scoring_rejects_missing_qids(meta, monkeypatch):
+    monkeypatch.setattr(suite, "_DOCS_META", meta)
+    with pytest.raises(SystemExit, match="absent from"):
+        suite.score_pass([record()], "docs")
+
+
+def test_public_report_rejects_missing_qids(tmp_path, monkeypatch):
+    from benchmark import run_benchmark as public
+
+    tasks = tmp_path / "devdex/runs/20261026T120000-ext-example-docs/run1/tasks"
+    tasks.mkdir(parents=True)
+    (tasks / "p1_docs_external.json").write_text(json.dumps([record()]))
+    monkeypatch.setattr(public, "ROOT", tmp_path)
+    monkeypatch.setattr(suite, "DATASETS", tmp_path / "missing-gt")
+    monkeypatch.setattr(suite, "_DOCS_META", None)
+    with pytest.raises(SystemExit, match="absent from"):
+        public.report("example", tmp_path / "results.json")
+    assert not (tmp_path / "results.json").exists()
+
+
+def test_existing_report_rejects_missing_gt_file(tmp_path, monkeypatch):
+    tasks = tmp_path / "20261026T120000-p1-opus-docs-fc-mcp/run1/tasks"
+    tasks.mkdir(parents=True)
+    (tasks / "p1_docs_fc-mcp.json").write_text(json.dumps([record()]))
+    monkeypatch.setattr(suite, "DATASETS", tmp_path / "missing-gt")
+    monkeypatch.setattr(suite, "_DOCS_META", None)
+    monkeypatch.setattr(suite, "TRACKS", dict(suite.TRACKS))
+    with pytest.raises(SystemExit, match="absent from"):
+        report_metrics.build(str(tmp_path), "p1")
+
+
+def test_available_gold_with_no_retrieval_hit_still_scores_zero(monkeypatch):
+    monkeypatch.setattr(suite, "_DOCS_META", {"known": {
+        "repo_files": set(), "site": "docs.example.test", "page_stems": {"guide"},
+        "memory_answerable": False}})
+    metrics = suite.score_pass([record("known")], "docs")
+    assert metrics["n"] == 1
+    assert metrics["precision"] == 0
+    assert metrics["doc_found@10"] == 0