fix: derive GitHub provenance from the actual URL origin

devdex/harness/provenance.py

@@ -17,8 +17,9 @@
 recorded and re-derivable from the saved run.
 """
 import json, re, time
+from urllib.parse import urlsplit
 
-GH_ARTIFACT = re.compile(r"github\.com/([\w.-]+/[\w.-]+)/(?:issues|pull)/(\d+)")
+GH_ARTIFACT = re.compile(r"github\.com/([\w.-]+/[\w.-]+)/(?:issues|pull)/(\d+)(?=/|$|#|\?|\s|\))")
 GH_REPO = re.compile(r"github\.com/([\w.-]+/[\w.-]+?)(?:/|$|#|\?|\s|\))")
 URL_RE = re.compile(r"https?://[^\s\"'<>)\]]+")
 
@@ -41,15 +42,38 @@
 
 
 def ref_from_url(url):
-    m = GH_ARTIFACT.search(url or "")
+    try:
+        parsed = urlsplit(url or "")
+        if parsed.scheme not in ("http", "https") or parsed.hostname not in ("github.com", "www.github.com"):
+            return None
+        if parsed.port not in (None, 443 if parsed.scheme == "https" else 80):
+            return None
+    except ValueError:
+        return None
+    # Parse the actual authority/path. Searching the entire string also matched lookalike
+    # hosts, external mirror paths and GitHub URLs merely embedded in a redirect parameter.
+    target = "github.com" + parsed.path
+    m = GH_ARTIFACT.match(target)
     if m:
         return f"{m.group(1)}#{m.group(2)}"
-    m = GH_REPO.search(url or "")
+    m = GH_REPO.match(target)
     if m and m.group(1).count("/") == 1:
         return m.group(1).removesuffix(".git")
     return None
 
 
+def github_refs_from_text(text):
+    """Native stream refs use the same URL-origin checks as structured MCP results."""
+    seen = set()
+    for match in URL_RE.finditer(text):
+        ref = ref_from_url(match.group(0).rstrip(".,);"))
+        if ref:
+            for item in (ref, ref.split("#")[0]):
+                if item not in seen:
+                    seen.add(item)
+                    yield item
+
+
 def _blocks(resp):
     """Flatten an MCP tool_response into its text blocks. Vendors differ: some return
     {'content':[{'type':'text','text':...}]}, some a bare list, some a dict already.

devdex/harness/runner_sdk.py

@@ -598,11 +598,8 @@ async def run_one(x):
                         is_search = (nm == "WebSearch")
                         src = "search_direct" if is_search else "reader"
                         s_ = str(getattr(b, "content", "") or "")
-                        for m in P.GH_ARTIFACT.finditer(s_):
-                            state.add(f"{m.group(1)}#{m.group(2)}", src)
-                        for m in P.GH_REPO.finditer(s_):
-                            if m.group(1).count("/") == 1:
-                                state.add(m.group(1), src)
+                        for ref in P.github_refs_from_text(s_):
+                            state.add(ref, src)
                         items, seen_u = [], set()
                         for m in P.URL_RE.finditer(s_):
                             u = m.group(0).rstrip(".,);")

devdex/tests/test_github_provenance_origin.py

@@ -0,0 +1,83 @@
+"""A URL mentioning GitHub is not necessarily a result returned from GitHub."""
+import ast
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from devdex.harness import control, provenance
+from devdex.scorer.report_metrics import golds, score_records
+
+
+@pytest.mark.parametrize("url", [
+    "https://notgithub.com/o/r/issues/12",
+    "https://evil.test/github.com/o/r/issues/12",
+    "https://evil.test/?redirect=https://github.com/o/r/issues/12",
+    "https://evil.test/#https://github.com/o/r/issues/12",
+    "https://github.com.evil.test/o/r/issues/12",
+    "https://github.com@evil.test/o/r/issues/12",
+    "https://github.com:8443/o/r/issues/12",
+    "https://github.com:invalid/o/r/issues/12",
+])
+def test_non_github_authorities_do_not_derive_github_refs(url):
+    assert provenance.ref_from_url(url) is None
+
+
+@pytest.mark.parametrize(("url", "expected"), [
+    ("https://github.com/o/r/issues/12#comment", "o/r#12"),
+    ("https://www.github.com/o/r/pull/12/files", "o/r#12"),
+    ("https://github.com/o/r.git", "o/r"),
+    ("https://github.com/o/r?redirect=https://github.com/other/fake/issues/99", "o/r"),
+    ("https://github.com/o/r/issues/12wrong", "o/r"),
+])
+def test_only_the_actual_github_path_determines_the_reference(url, expected):
+    assert provenance.ref_from_url(url) == expected
+
+
+def test_real_controller_cannot_turn_lookalike_url_into_grounded_gold():
+    state = control.State()
+    cfg = control.Cfg(k=10, restrict=False, max_search=None, max_read=None,
+                      agent_snippet=None, page_full=None)
+    controller = control.Controller(state, cfg, search_tools={"search"})
+    controller.prepare("search", {"query": "find the fix"}, "call1")
+    controller.record("search", {"content": [{"type": "text", "text": json.dumps({
+        "results": [{"url": "https://notgithub.com/o/r/issues/12", "title": "A mirror"}]})}]},
+                      "call1")
+    row = {"qid": "synthetic", "gold_accept": ["o/r#12"], "citations": ["o/r#12"],
+           "committed": True, "search_log": state.search_log}
+    metrics = score_records([row], "fix", lambda item, rec: item in golds(rec))
+    assert metrics["correctness"] == 1
+    assert metrics["groundedness"] == metrics["precision"] == metrics["pool_hit"] == 0
+
+
+def native_result(text):
+    # Execute the actual stream-result loop without importing its paid SDK/setup code.
+    source = Path(provenance.__file__).with_name("runner_sdk.py")
+    tree = ast.parse(source.read_text())
+    loops = [node for node in ast.walk(tree) if isinstance(node, ast.For)
+             and "msg.content" in ast.unparse(node.iter)
+             and "state.search_log.append" in ast.unparse(node)]
+    assert len(loops) == 1
+    block = type("ToolResultBlock", (), {})()
+    block.tool_use_id = "native1"
+    block.content = text
+    state = control.State()
+    namespace = {"P": provenance, "state": state, "K": 10,
+                 "msg": SimpleNamespace(content=[block]),
+                 "called": {"native1": ("WebSearch", {"query": "test"})}}
+    exec(compile(ast.Module(body=loops, type_ignores=[]), str(source), "exec"), namespace)
+    return state
+
+
+def test_native_stream_pool_rejects_embedded_github_links_on_other_hosts():
+    state = native_result("https://evil.test/?redirect=https://github.com/o/r/issues/12")
+    assert "o/r#12" not in state.engine_pool
+    assert "o/r" not in state.engine_pool
+    assert state.search_log[0]["results"][0]["ref"] is None
+
+
+def test_native_stream_preserves_real_artifact_and_repository_refs():
+    state = native_result("https://github.com/o/r/issues/12")
+    assert {"o/r#12", "o/r"} <= state.engine_pool
+    assert state.search_log[0]["results"][0]["ref"] == "o/r#12"