fix: count retrieval misses in engine rank denominators

devdex/scorer/suite.py

@@ -287,15 +287,18 @@ def _cited_rank(r):
     # list's size and we cannot request k=10, those rates are not comparable to arms that
     # return 10 -- suppress rather than print a depth-2 number in a depth-10 column.
     nd = _no_depth(rows)
+    has_log = any(r.get("search_log") for r in rows)
     return {
         "n": n,
         "precision": round(precision / n, 4),
         "correctness": round(sum(1 for r in rows if _golds(r) & set(_cites(r))) / n, 4),
         "groundedness": round(sum(1 for r in rows
                                   if _golds(r) & set(_cites(r)) & _engine_pool(r)) / n, 4),
         "pool_hit": None if nd else round(sum(1 for r in rows if _golds(r) & _engine_pool(r)) / n, 4),
-        "engR1": None if nd or not ranks else round(sum(1 for x in ranks if x == 1) / len(ranks), 4),
-        "engMRR": None if nd or not ranks else round(sum(1 / x for x in ranks) / len(ranks), 4),
+        # Missing ranks contribute zero, just as in the docs track. Dividing by len(ranks)
+        # conditions on successful retrieval: a single rank-1 hit among 100 misses becomes 1.0.
+        "engR1": None if nd or not has_log else round(sum(1 for x in ranks if x == 1) / n, 4),
+        "engMRR": None if nd or not has_log else round(sum(1 / x for x in ranks) / n, 4),
         "no_depth": nd,
         "abstain": round(sum(1 for r in rows if not r.get("committed")) / n, 4),
         "dead_runs": round(dead / n, 4),

devdex/tests/test_engine_ranking_denominator.py

@@ -0,0 +1,51 @@
+"""Engine rank rates use all attempted items, including misses and failed runs."""
+import pytest
+
+from devdex.scorer.suite import score_pass
+
+
+def row(rank=None, **extra):
+    results = [{"rank": rank, "ref": "gold"}] if rank is not None else []
+    return {"gold_accept": ["gold"], "citations": [], "committed": False,
+            "search_log": [{"results": results}], **extra}
+
+
+@pytest.mark.parametrize("track", ["repo", "fix"])
+def test_misses_and_timeouts_remain_in_engine_rank_denominators(track):
+    # Reciprocal ranks are [1, 1/2, 0, 0]; rank-1 indicators are [1, 0, 0, 0].
+    metrics = score_pass([row(0), row(1), row(), row(error="timeout")], track)
+    assert metrics["engR1"] == 0.25
+    assert metrics["engMRR"] == 0.375
+    assert metrics["pool_hit"] == 0.5
+
+
+@pytest.mark.parametrize("track", ["repo", "fix"])
+def test_adding_a_miss_lowers_engine_rank_rates(track):
+    hit = score_pass([row(0)], track)
+    mixed = score_pass([row(0), row()], track)
+    assert mixed["engR1"] == hit["engR1"] / 2
+    assert mixed["engMRR"] == hit["engMRR"] / 2
+
+
+@pytest.mark.parametrize("track", ["repo", "fix"])
+def test_no_hits_within_the_depth_cap_is_zero_not_missing(track):
+    metrics = score_pass([row(), row(10), row(error="timeout")], track)
+    assert metrics["engR1"] == 0.0
+    assert metrics["engMRR"] == 0.0
+
+
+def test_noncomparable_depth_remains_undefined():
+    metrics = score_pass([row(0, config={"no_depth": True}), row()], "repo")
+    assert metrics["engR1"] is None
+    assert metrics["engMRR"] is None
+
+
+def test_empty_pass_remains_empty():
+    assert score_pass([], "repo") == {}
+
+
+def test_native_arms_without_observed_ranks_remain_undefined():
+    metrics = score_pass([{"gold_accept": ["gold"], "pool_prov": [["gold", "search_direct"]]}],
+                         "repo")
+    assert metrics["engR1"] is None
+    assert metrics["engMRR"] is None