fix: isolate report cohorts and hash scored inputs

.github/workflows/devdex-eval.yml

@@ -59,7 +59,7 @@ jobs:
       - name: Install
         run: |
           python -m pip install --upgrade pip
-          pip install pytest
+          pip install pytest numpy
       - name: Dataset + scorer tests
         # the DIRECTORY, not a hand-listed set -- a new test file must not need a CI edit to run
         run: python -m pytest devdex/tests -q

README.md

@@ -105,6 +105,11 @@ uv run python benchmark/run_benchmark.py --name yourco --report-only --out resul
 
 Your server is mounted exactly like every arm above: same agent, same tool gate, same scorer.
 
+Reports select the newest run directory for each track under the engine's exact label. Older
+smoke tests and runs against previous server versions are not pooled; repeated passes within
+the selected directory are retained. The JSON output records the selected directories, record
+counts, and SHA-256 hashes of the scored files so a result can be traced to its inputs.
+
 **What your server needs**
 
 | | |

benchmark/run_benchmark.py

@@ -29,7 +29,7 @@
     python3 benchmark/run_benchmark.py --name yourco --report-only                # re-print scores
 """
 import argparse
-import glob
+import hashlib
 import json
 import os
 import sys
@@ -101,17 +101,34 @@ def _ci(values, rng):
 
 
 def report(engine, out=None):
-    """Read the run records back and score them with the published scorer."""
+    """Score the newest run directory per track, preserving its repeated passes.
+
+    Historical smoke runs and runs against older server versions are separate experiments;
+    pooling them changes both the estimate and its uncertainty. Directory timestamps are the
+    runner's ordering contract. A newest but empty run must not revive an older result.
+    """
     from devdex.scorer.report_metrics import score_records
     rng = np.random.default_rng(0)
-    rows, results = [], {}
+    rows, results, inputs = [], {}, {}
     for t in TRACKS:
-        files = sorted(glob.glob(str(ROOT / f"devdex/runs/*ext-{engine}-{t}/run*/tasks/*.json")))
+        runs_root = ROOT / "devdex" / "runs"
+        candidates = sorted(d for d in runs_root.glob("*")
+                            if d.is_dir() and d.name.endswith(f"-ext-{engine}-{t}"))
+        if not candidates:
+            continue
+        selected = candidates[-1]
+        files = sorted(selected.glob("run*/tasks/*.json"))
+        print(f"  {t}: reading {selected.name}")
         if not files:
             continue
         recs = []
+        hashes = {}
         for f in files:
-            recs += json.load(open(f))
+            content = f.read_bytes()
+            recs += json.loads(content)
+            hashes[str(f.relative_to(selected))] = hashlib.sha256(content).hexdigest()
+        inputs[t] = {"run": selected.name, "records": len(recs),
+                     "files": hashes}
         is_gold, pool_test = _scorer_for(t)
         m = score_records(recs, t, is_gold, pool_test)
         if not m:
@@ -160,6 +177,8 @@ def report(engine, out=None):
     payload = {"engine": engine, "mode": "agent", "depth_cap": 10,
                "dataset": "public sample (stratified ~50% of each track)",
                "scorer": "devdex.scorer.report_metrics.score_records",
+               "run_selection": "newest directory per track; all passes within that directory",
+               "inputs": inputs,
                "tracks": results}
     if out:
         Path(out).write_text(json.dumps(payload, indent=2))

devdex/tests/test_report_cohorts.py

@@ -0,0 +1,53 @@
+"""Exercise the public reporter on saved synthetic runs, with no provider calls."""
+import hashlib
+import json
+
+from benchmark import run_benchmark as report
+
+
+def save(root, stamp, engine="example", hit=True, run=1):
+    directory = root / "devdex" / "runs" / f"{stamp}-ext-{engine}-repo"
+    tasks = directory / f"run{run}" / "tasks"
+    tasks.mkdir(parents=True, exist_ok=True)
+    row = {"qid": "synthetic-1", "gold_accept": ["owner/project"],
+           "citations": ["owner/project"] if hit else [], "search_log": []}
+    (tasks / "records.json").write_text(json.dumps([row]))
+    return directory
+
+
+def test_latest_run_does_not_pool_old_smoke_results(tmp_path, monkeypatch):
+    save(tmp_path, "20261001T120000", hit=True)
+    save(tmp_path, "20261002T120000", hit=False)
+    monkeypatch.setattr(report, "ROOT", tmp_path)
+    output = tmp_path / "report.json"
+    assert report.report("example", output) == 0
+    result = json.loads(output.read_text())
+    assert result["tracks"]["repo"]["recall@10"] == 0.0
+    source = tmp_path / "devdex/runs/20261002T120000-ext-example-repo/run1/tasks/records.json"
+    assert result["inputs"]["repo"] == {
+        "run": "20261002T120000-ext-example-repo", "records": 1,
+        "files": {"run1/tasks/records.json": hashlib.sha256(source.read_bytes()).hexdigest()},
+    }
+
+
+def test_engine_label_is_literal_not_a_glob(tmp_path, monkeypatch):
+    save(tmp_path, "20261001T120000", engine="other", hit=True)
+    monkeypatch.setattr(report, "ROOT", tmp_path)
+    assert report.report("*") == 1
+
+
+def test_latest_empty_run_does_not_fall_back_to_stale_success(tmp_path, monkeypatch):
+    save(tmp_path, "20261001T120000", hit=True)
+    latest = tmp_path / "devdex" / "runs" / "20261002T120000-ext-example-repo"
+    latest.mkdir()
+    monkeypatch.setattr(report, "ROOT", tmp_path)
+    assert report.report("example") == 1
+
+
+def test_repetitions_within_selected_run_are_preserved(tmp_path, monkeypatch):
+    save(tmp_path, "20261001T120000", hit=True, run=1)
+    save(tmp_path, "20261001T120000", hit=False, run=2)
+    monkeypatch.setattr(report, "ROOT", tmp_path)
+    output = tmp_path / "report.json"
+    assert report.report("example", output) == 0
+    assert json.loads(output.read_text())["tracks"]["repo"]["recall@10"] == 0.5