diff --git a/.github/workflows/devdex-eval.yml b/.github/workflows/devdex-eval.yml index d554749..35b4a5a 100644 --- a/.github/workflows/devdex-eval.yml +++ b/.github/workflows/devdex-eval.yml @@ -59,7 +59,7 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install pytest + pip install pytest numpy - name: Dataset + scorer tests # the DIRECTORY, not a hand-listed set -- a new test file must not need a CI edit to run run: python -m pytest devdex/tests -q diff --git a/README.md b/README.md index ce830bd..33d7267 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,11 @@ uv run python benchmark/run_benchmark.py --name yourco --report-only --out resul Your server is mounted exactly like every arm above: same agent, same tool gate, same scorer. +Reports select the newest run directory for each track under the engine's exact label. Older +smoke tests and runs against previous server versions are not pooled; repeated passes within +the selected directory are retained. The JSON output records the selected directories, record +counts, and SHA-256 hashes of the scored files so a result can be traced to its inputs. + **What your server needs** | | | diff --git a/benchmark/run_benchmark.py b/benchmark/run_benchmark.py index 07c55c7..659c767 100644 --- a/benchmark/run_benchmark.py +++ b/benchmark/run_benchmark.py @@ -29,7 +29,7 @@ python3 benchmark/run_benchmark.py --name yourco --report-only # re-print scores """ import argparse -import glob +import hashlib import json import os import sys @@ -101,17 +101,34 @@ def _ci(values, rng): def report(engine, out=None): - """Read the run records back and score them with the published scorer.""" + """Score the newest run directory per track, preserving its repeated passes. + + Historical smoke runs and runs against older server versions are separate experiments; + pooling them changes both the estimate and its uncertainty. Directory timestamps are the + runner's ordering contract. A newest but empty run must not revive an older result. + """ from devdex.scorer.report_metrics import score_records rng = np.random.default_rng(0) - rows, results = [], {} + rows, results, inputs = [], {}, {} for t in TRACKS: - files = sorted(glob.glob(str(ROOT / f"devdex/runs/*ext-{engine}-{t}/run*/tasks/*.json"))) + runs_root = ROOT / "devdex" / "runs" + candidates = sorted(d for d in runs_root.glob("*") + if d.is_dir() and d.name.endswith(f"-ext-{engine}-{t}")) + if not candidates: + continue + selected = candidates[-1] + files = sorted(selected.glob("run*/tasks/*.json")) + print(f" {t}: reading {selected.name}") if not files: continue recs = [] + hashes = {} for f in files: - recs += json.load(open(f)) + content = f.read_bytes() + recs += json.loads(content) + hashes[str(f.relative_to(selected))] = hashlib.sha256(content).hexdigest() + inputs[t] = {"run": selected.name, "records": len(recs), + "files": hashes} is_gold, pool_test = _scorer_for(t) m = score_records(recs, t, is_gold, pool_test) if not m: @@ -160,6 +177,8 @@ def report(engine, out=None): payload = {"engine": engine, "mode": "agent", "depth_cap": 10, "dataset": "public sample (stratified ~50% of each track)", "scorer": "devdex.scorer.report_metrics.score_records", + "run_selection": "newest directory per track; all passes within that directory", + "inputs": inputs, "tracks": results} if out: Path(out).write_text(json.dumps(payload, indent=2)) diff --git a/devdex/tests/test_report_cohorts.py b/devdex/tests/test_report_cohorts.py new file mode 100644 index 0000000..08e588b --- /dev/null +++ b/devdex/tests/test_report_cohorts.py @@ -0,0 +1,53 @@ +"""Exercise the public reporter on saved synthetic runs, with no provider calls.""" +import hashlib +import json + +from benchmark import run_benchmark as report + + +def save(root, stamp, engine="example", hit=True, run=1): + directory = root / "devdex" / "runs" / f"{stamp}-ext-{engine}-repo" + tasks = directory / f"run{run}" / "tasks" + tasks.mkdir(parents=True, exist_ok=True) + row = {"qid": "synthetic-1", "gold_accept": ["owner/project"], + "citations": ["owner/project"] if hit else [], "search_log": []} + (tasks / "records.json").write_text(json.dumps([row])) + return directory + + +def test_latest_run_does_not_pool_old_smoke_results(tmp_path, monkeypatch): + save(tmp_path, "20261001T120000", hit=True) + save(tmp_path, "20261002T120000", hit=False) + monkeypatch.setattr(report, "ROOT", tmp_path) + output = tmp_path / "report.json" + assert report.report("example", output) == 0 + result = json.loads(output.read_text()) + assert result["tracks"]["repo"]["recall@10"] == 0.0 + source = tmp_path / "devdex/runs/20261002T120000-ext-example-repo/run1/tasks/records.json" + assert result["inputs"]["repo"] == { + "run": "20261002T120000-ext-example-repo", "records": 1, + "files": {"run1/tasks/records.json": hashlib.sha256(source.read_bytes()).hexdigest()}, + } + + +def test_engine_label_is_literal_not_a_glob(tmp_path, monkeypatch): + save(tmp_path, "20261001T120000", engine="other", hit=True) + monkeypatch.setattr(report, "ROOT", tmp_path) + assert report.report("*") == 1 + + +def test_latest_empty_run_does_not_fall_back_to_stale_success(tmp_path, monkeypatch): + save(tmp_path, "20261001T120000", hit=True) + latest = tmp_path / "devdex" / "runs" / "20261002T120000-ext-example-repo" + latest.mkdir() + monkeypatch.setattr(report, "ROOT", tmp_path) + assert report.report("example") == 1 + + +def test_repetitions_within_selected_run_are_preserved(tmp_path, monkeypatch): + save(tmp_path, "20261001T120000", hit=True, run=1) + save(tmp_path, "20261001T120000", hit=False, run=2) + monkeypatch.setattr(report, "ROOT", tmp_path) + output = tmp_path / "report.json" + assert report.report("example", output) == 0 + assert json.loads(output.read_text())["tracks"]["repo"]["recall@10"] == 0.5