From 452a5073fea393083375c895447cdeca203c8cc0 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Mon, 26 Oct 2026 21:49:06 -0400 Subject: [PATCH] fix: derive GitHub provenance only from the actual URL origin --- devdex/harness/provenance.py | 30 ++++++- devdex/harness/runner_sdk.py | 7 +- devdex/tests/test_github_provenance_origin.py | 83 +++++++++++++++++++ 3 files changed, 112 insertions(+), 8 deletions(-) create mode 100644 devdex/tests/test_github_provenance_origin.py diff --git a/devdex/harness/provenance.py b/devdex/harness/provenance.py index 17c210a..652ff97 100644 --- a/devdex/harness/provenance.py +++ b/devdex/harness/provenance.py @@ -17,8 +17,9 @@ recorded and re-derivable from the saved run. """ import json, re, time +from urllib.parse import urlsplit -GH_ARTIFACT = re.compile(r"github\.com/([\w.-]+/[\w.-]+)/(?:issues|pull)/(\d+)") +GH_ARTIFACT = re.compile(r"github\.com/([\w.-]+/[\w.-]+)/(?:issues|pull)/(\d+)(?=/|$|#|\?|\s|\))") GH_REPO = re.compile(r"github\.com/([\w.-]+/[\w.-]+?)(?:/|$|#|\?|\s|\))") URL_RE = re.compile(r"https?://[^\s\"'<>)\]]+") @@ -41,15 +42,38 @@ def ref_from_url(url): - m = GH_ARTIFACT.search(url or "") + try: + parsed = urlsplit(url or "") + if parsed.scheme not in ("http", "https") or parsed.hostname not in ("github.com", "www.github.com"): + return None + if parsed.port not in (None, 443 if parsed.scheme == "https" else 80): + return None + except ValueError: + return None + # Parse the actual authority/path. Searching the entire string also matched lookalike + # hosts, external mirror paths and GitHub URLs merely embedded in a redirect parameter. + target = "github.com" + parsed.path + m = GH_ARTIFACT.match(target) if m: return f"{m.group(1)}#{m.group(2)}" - m = GH_REPO.search(url or "") + m = GH_REPO.match(target) if m and m.group(1).count("/") == 1: return m.group(1).removesuffix(".git") return None +def github_refs_from_text(text): + """Native stream refs use the same URL-origin checks as structured MCP results.""" + seen = set() + for match in URL_RE.finditer(text): + ref = ref_from_url(match.group(0).rstrip(".,);")) + if ref: + for item in (ref, ref.split("#")[0]): + if item not in seen: + seen.add(item) + yield item + + def _blocks(resp): """Flatten an MCP tool_response into its text blocks. Vendors differ: some return {'content':[{'type':'text','text':...}]}, some a bare list, some a dict already. diff --git a/devdex/harness/runner_sdk.py b/devdex/harness/runner_sdk.py index 82c7e9f..6b9b67a 100644 --- a/devdex/harness/runner_sdk.py +++ b/devdex/harness/runner_sdk.py @@ -598,11 +598,8 @@ async def run_one(x): is_search = (nm == "WebSearch") src = "search_direct" if is_search else "reader" s_ = str(getattr(b, "content", "") or "") - for m in P.GH_ARTIFACT.finditer(s_): - state.add(f"{m.group(1)}#{m.group(2)}", src) - for m in P.GH_REPO.finditer(s_): - if m.group(1).count("/") == 1: - state.add(m.group(1), src) + for ref in P.github_refs_from_text(s_): + state.add(ref, src) items, seen_u = [], set() for m in P.URL_RE.finditer(s_): u = m.group(0).rstrip(".,);") diff --git a/devdex/tests/test_github_provenance_origin.py b/devdex/tests/test_github_provenance_origin.py new file mode 100644 index 0000000..6ae7387 --- /dev/null +++ b/devdex/tests/test_github_provenance_origin.py @@ -0,0 +1,83 @@ +"""A URL mentioning GitHub is not necessarily a result returned from GitHub.""" +import ast +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from devdex.harness import control, provenance +from devdex.scorer.report_metrics import golds, score_records + + +@pytest.mark.parametrize("url", [ + "https://notgithub.com/o/r/issues/12", + "https://evil.test/github.com/o/r/issues/12", + "https://evil.test/?redirect=https://github.com/o/r/issues/12", + "https://evil.test/#https://github.com/o/r/issues/12", + "https://github.com.evil.test/o/r/issues/12", + "https://github.com@evil.test/o/r/issues/12", + "https://github.com:8443/o/r/issues/12", + "https://github.com:invalid/o/r/issues/12", +]) +def test_non_github_authorities_do_not_derive_github_refs(url): + assert provenance.ref_from_url(url) is None + + +@pytest.mark.parametrize(("url", "expected"), [ + ("https://github.com/o/r/issues/12#comment", "o/r#12"), + ("https://www.github.com/o/r/pull/12/files", "o/r#12"), + ("https://github.com/o/r.git", "o/r"), + ("https://github.com/o/r?redirect=https://github.com/other/fake/issues/99", "o/r"), + ("https://github.com/o/r/issues/12wrong", "o/r"), +]) +def test_only_the_actual_github_path_determines_the_reference(url, expected): + assert provenance.ref_from_url(url) == expected + + +def test_real_controller_cannot_turn_lookalike_url_into_grounded_gold(): + state = control.State() + cfg = control.Cfg(k=10, restrict=False, max_search=None, max_read=None, + agent_snippet=None, page_full=None) + controller = control.Controller(state, cfg, search_tools={"search"}) + controller.prepare("search", {"query": "find the fix"}, "call1") + controller.record("search", {"content": [{"type": "text", "text": json.dumps({ + "results": [{"url": "https://notgithub.com/o/r/issues/12", "title": "A mirror"}]})}]}, + "call1") + row = {"qid": "synthetic", "gold_accept": ["o/r#12"], "citations": ["o/r#12"], + "committed": True, "search_log": state.search_log} + metrics = score_records([row], "fix", lambda item, rec: item in golds(rec)) + assert metrics["correctness"] == 1 + assert metrics["groundedness"] == metrics["precision"] == metrics["pool_hit"] == 0 + + +def native_result(text): + # Execute the actual stream-result loop without importing its paid SDK/setup code. + source = Path(provenance.__file__).with_name("runner_sdk.py") + tree = ast.parse(source.read_text()) + loops = [node for node in ast.walk(tree) if isinstance(node, ast.For) + and "msg.content" in ast.unparse(node.iter) + and "state.search_log.append" in ast.unparse(node)] + assert len(loops) == 1 + block = type("ToolResultBlock", (), {})() + block.tool_use_id = "native1" + block.content = text + state = control.State() + namespace = {"P": provenance, "state": state, "K": 10, + "msg": SimpleNamespace(content=[block]), + "called": {"native1": ("WebSearch", {"query": "test"})}} + exec(compile(ast.Module(body=loops, type_ignores=[]), str(source), "exec"), namespace) + return state + + +def test_native_stream_pool_rejects_embedded_github_links_on_other_hosts(): + state = native_result("https://evil.test/?redirect=https://github.com/o/r/issues/12") + assert "o/r#12" not in state.engine_pool + assert "o/r" not in state.engine_pool + assert state.search_log[0]["results"][0]["ref"] is None + + +def test_native_stream_preserves_real_artifact_and_repository_refs(): + state = native_result("https://github.com/o/r/issues/12") + assert {"o/r#12", "o/r"} <= state.engine_pool + assert state.search_log[0]["results"][0]["ref"] == "o/r#12"