From c935a2bf43fe368721e9e056593365c7adbbe4cc Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Mon, 26 Oct 2026 21:03:18 -0400 Subject: [PATCH 1/2] fix: enforce canonical docs site and path boundaries --- devdex/scorer/suite.py | 4 +- .../tests/test_docs_canonical_boundaries.py | 76 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 devdex/tests/test_docs_canonical_boundaries.py diff --git a/devdex/scorer/suite.py b/devdex/scorer/suite.py index 71d5a18..79b875b 100644 --- a/devdex/scorer/suite.py +++ b/devdex/scorer/suite.py @@ -193,7 +193,9 @@ def _docs_rank(row, meta, cap=CAP): hit = f"{m.group(1).lower()}/{m.group(2).lower()}:{m.group(3).lower()}" in files if not hit and site and stems: u = _strip_host(url) - if u.startswith(site): + # The declared site may include a base path. Require a segment boundary: + # plain startswith also admits docs.example.com.evil.test and /docs-other. + if u == site or u.startswith(site + "/"): tail = u[len(site):].strip("/").split("?")[0].split("#")[0] last = tail.split("/")[-1] if tail else "" # Docs sites serve the same page under several extensions — vite.dev diff --git a/devdex/tests/test_docs_canonical_boundaries.py b/devdex/tests/test_docs_canonical_boundaries.py new file mode 100644 index 0000000..986776a --- /dev/null +++ b/devdex/tests/test_docs_canonical_boundaries.py @@ -0,0 +1,76 @@ +"""Canonical docs URLs must stay on the declared host and within its base path.""" +import json + +import pytest + +from devdex.scorer import report_metrics, suite + + +def metadata(site="docs.example.com"): + return {"repo_files": set(), "site": site, "page_stems": {"guide"}} + + +def record(url): + return {"qid": "docs-1", "sources": [url], "committed": True, + "search_log": [{"results": [{"rank": 0, "url": url}]}]} + + +@pytest.mark.parametrize("url", [ + "https://docs.example.com.evil.test/guide", + "https://docs.example.com-extra.test/guide", + "https://docs.example.com@evil.test/guide", + "https://docs.example.com:8443/guide", + "https://docs.example.com?redirect=/guide", + "https://docs.example.com#redirect/guide", +]) +def test_lookalike_authorities_and_nonpath_suffixes_are_not_canonical(url): + assert suite._docs_rank(record(url), metadata()) is None + + +@pytest.mark.parametrize("url", [ + "https://example.com/docs-other/guide", + "https://example.com/docs2/guide", + "https://example.com/docs?redirect=/guide", + "https://example.com/docs#redirect/guide", +]) +def test_base_path_must_end_at_a_path_segment(url): + assert suite._docs_rank(record(url), metadata("example.com/docs")) is None + + +@pytest.mark.parametrize("url", [ + "https://docs.example.com/guide", + "http://docs.example.com/guide", + "https://www.docs.example.com/guide", + "https://DOCS.EXAMPLE.COM/nested/guide.md?query=yes#section", +]) +def test_existing_canonical_url_forms_still_match(url): + assert suite._docs_rank(record(url), metadata()) == 1 + + +def test_declared_base_path_and_page_stem_are_preserved(): + assert suite._docs_rank(record("https://example.com/docs/nested/guide.html"), + metadata("example.com/docs")) == 1 + assert suite._docs_rank(record("https://docs.example.com/unrelated"), metadata()) is None + + +def test_published_report_does_not_credit_a_lookalike_site(tmp_path, monkeypatch): + # Exercise the real canonical-source loader, gold probe, pool match and scorer together. + gt = tmp_path / "gt" + gt.mkdir() + (gt / report_metrics.DOCS_SCORING_GT).write_text(json.dumps({ + "id": "docs-1", "canonical_sources": { + "site": "https://docs.example.com", "page_stems": ["guide"]}}) + "\n") + monkeypatch.setattr(suite, "DATASETS", gt) + monkeypatch.setattr(suite, "_DOCS_META", None) + monkeypatch.setattr(suite, "TRACKS", dict(suite.TRACKS)) + tasks = tmp_path / "20260101T000000-p1-opus-docs-fc-mcp" / "run1" / "tasks" + tasks.mkdir(parents=True) + (tasks / "p1_docs_fc-mcp.json").write_text(json.dumps([ + record("https://docs.example.com/guide"), + record("https://docs.example.com.evil.test/guide")])) + + cells, failures = report_metrics.build(str(tmp_path), "p1") + metrics = cells["docs"]["fc-mcp"] + assert not failures + for metric in ("precision", "recall", "amrr", "groundedness", "pool_hit"): + assert metrics[metric] == 0.5, metric From 822c19972c41aee6d66173baa024cd0f54a4f118 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Mon, 26 Oct 2026 21:50:15 -0400 Subject: [PATCH 2/2] test: give independent docs tasks distinct query identities --- devdex/tests/test_docs_canonical_boundaries.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/devdex/tests/test_docs_canonical_boundaries.py b/devdex/tests/test_docs_canonical_boundaries.py index 986776a..e4fc95b 100644 --- a/devdex/tests/test_docs_canonical_boundaries.py +++ b/devdex/tests/test_docs_canonical_boundaries.py @@ -57,17 +57,20 @@ def test_published_report_does_not_credit_a_lookalike_site(tmp_path, monkeypatch # Exercise the real canonical-source loader, gold probe, pool match and scorer together. gt = tmp_path / "gt" gt.mkdir() - (gt / report_metrics.DOCS_SCORING_GT).write_text(json.dumps({ - "id": "docs-1", "canonical_sources": { - "site": "https://docs.example.com", "page_stems": ["guide"]}}) + "\n") + (gt / report_metrics.DOCS_SCORING_GT).write_text("".join(json.dumps({ + "id": qid, "canonical_sources": { + "site": "https://docs.example.com", "page_stems": ["guide"]}}) + "\n" + for qid in ("docs-1", "docs-2"))) monkeypatch.setattr(suite, "DATASETS", gt) monkeypatch.setattr(suite, "_DOCS_META", None) monkeypatch.setattr(suite, "TRACKS", dict(suite.TRACKS)) tasks = tmp_path / "20260101T000000-p1-opus-docs-fc-mcp" / "run1" / "tasks" tasks.mkdir(parents=True) - (tasks / "p1_docs_fc-mcp.json").write_text(json.dumps([ + records = [ record("https://docs.example.com/guide"), - record("https://docs.example.com.evil.test/guide")])) + record("https://docs.example.com.evil.test/guide")] + records[1]["qid"] = "docs-2" + (tasks / "p1_docs_fc-mcp.json").write_text(json.dumps(records)) cells, failures = report_metrics.build(str(tmp_path), "p1") metrics = cells["docs"]["fc-mcp"]