From 2414279c3cd5b7affbb4b07f0ce1ce08489f5164 Mon Sep 17 00:00:00 2001
From: Rudy Celekli <rudy@gradiahq.com>
Date: Mon, 26 Oct 2026 21:02:44 -0400
Subject: [PATCH] fix: include retrieval misses in engine rank denominators

---
 devdex/scorer/suite.py                        |  7 ++-
 .../tests/test_engine_ranking_denominator.py  | 51 +++++++++++++++++++
 2 files changed, 56 insertions(+), 2 deletions(-)
 create mode 100644 devdex/tests/test_engine_ranking_denominator.py

diff --git a/devdex/scorer/suite.py b/devdex/scorer/suite.py
index 71d5a18..480bba8 100644
--- a/devdex/scorer/suite.py
+++ b/devdex/scorer/suite.py
@@ -287,6 +287,7 @@ def _cited_rank(r):
     # list's size and we cannot request k=10, those rates are not comparable to arms that
     # return 10 -- suppress rather than print a depth-2 number in a depth-10 column.
     nd = _no_depth(rows)
+    has_log = any(r.get("search_log") for r in rows)
     return {
         "n": n,
         "precision": round(precision / n, 4),
@@ -294,8 +295,10 @@ def _cited_rank(r):
         "groundedness": round(sum(1 for r in rows
                                   if _golds(r) & set(_cites(r)) & _engine_pool(r)) / n, 4),
         "pool_hit": None if nd else round(sum(1 for r in rows if _golds(r) & _engine_pool(r)) / n, 4),
-        "engR1": None if nd or not ranks else round(sum(1 for x in ranks if x == 1) / len(ranks), 4),
-        "engMRR": None if nd or not ranks else round(sum(1 / x for x in ranks) / len(ranks), 4),
+        # Missing ranks contribute zero, just as in the docs track. Dividing by len(ranks)
+        # conditions on successful retrieval: a single rank-1 hit among 100 misses becomes 1.0.
+        "engR1": None if nd or not has_log else round(sum(1 for x in ranks if x == 1) / n, 4),
+        "engMRR": None if nd or not has_log else round(sum(1 / x for x in ranks) / n, 4),
         "no_depth": nd,
         "abstain": round(sum(1 for r in rows if not r.get("committed")) / n, 4),
         "dead_runs": round(dead / n, 4),
diff --git a/devdex/tests/test_engine_ranking_denominator.py b/devdex/tests/test_engine_ranking_denominator.py
new file mode 100644
index 0000000..5a81fea
--- /dev/null
+++ b/devdex/tests/test_engine_ranking_denominator.py
@@ -0,0 +1,51 @@
+"""Engine rank rates use all attempted items, including misses and failed runs."""
+import pytest
+
+from devdex.scorer.suite import score_pass
+
+
+def row(rank=None, **extra):
+    results = [{"rank": rank, "ref": "gold"}] if rank is not None else []
+    return {"gold_accept": ["gold"], "citations": [], "committed": False,
+            "search_log": [{"results": results}], **extra}
+
+
+@pytest.mark.parametrize("track", ["repo", "fix"])
+def test_misses_and_timeouts_remain_in_engine_rank_denominators(track):
+    # Reciprocal ranks are [1, 1/2, 0, 0]; rank-1 indicators are [1, 0, 0, 0].
+    metrics = score_pass([row(0), row(1), row(), row(error="timeout")], track)
+    assert metrics["engR1"] == 0.25
+    assert metrics["engMRR"] == 0.375
+    assert metrics["pool_hit"] == 0.5
+
+
+@pytest.mark.parametrize("track", ["repo", "fix"])
+def test_adding_a_miss_lowers_engine_rank_rates(track):
+    hit = score_pass([row(0)], track)
+    mixed = score_pass([row(0), row()], track)
+    assert mixed["engR1"] == hit["engR1"] / 2
+    assert mixed["engMRR"] == hit["engMRR"] / 2
+
+
+@pytest.mark.parametrize("track", ["repo", "fix"])
+def test_no_hits_within_the_depth_cap_is_zero_not_missing(track):
+    metrics = score_pass([row(), row(10), row(error="timeout")], track)
+    assert metrics["engR1"] == 0.0
+    assert metrics["engMRR"] == 0.0
+
+
+def test_noncomparable_depth_remains_undefined():
+    metrics = score_pass([row(0, config={"no_depth": True}), row()], "repo")
+    assert metrics["engR1"] is None
+    assert metrics["engMRR"] is None
+
+
+def test_empty_pass_remains_empty():
+    assert score_pass([], "repo") == {}
+
+
+def test_native_arms_without_observed_ranks_remain_undefined():
+    metrics = score_pass([{"gold_accept": ["gold"], "pool_prov": [["gold", "search_direct"]]}],
+                         "repo")
+    assert metrics["engR1"] is None
+    assert metrics["engMRR"] is None