From 88927bda07748de3659952415b3d0f7020f4ba2e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 00:21:31 +0000 Subject: [PATCH 1/7] Drop sub-page links that are only the bare root of the same site drop_redundant_link_fields already removed a cfp_link, sponsor or finaid that repeats the main link. It now also removes one that is only the bare root of the same site: an empty or "/" path with no query or fragment, on the same host as link (ignoring "www.") or on a parent domain of it. "cfp_link: https://pycon.de/" next to "link: https://2027.pycon.de/" says nothing about the CfP, so it goes. Sub-pages, #anchors and query strings on the parent domain stay, as do links to other sites and a year subdomain next to a series link, which is more specific than the link itself. The current data has no such links, so the sort leaves _data unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QzyF2ciB5jhvcTVpj4ErMr --- tests/test_redundant_links.py | 59 ++++++++++++++++++ utils/tidy_conf/links.py | 112 +++++++++++++++++++++++++++++++--- 2 files changed, 163 insertions(+), 8 deletions(-) diff --git a/tests/test_redundant_links.py b/tests/test_redundant_links.py index d47d7af1b8..5e4d6e342a 100644 --- a/tests/test_redundant_links.py +++ b/tests/test_redundant_links.py @@ -120,3 +120,62 @@ def test_entry_without_sub_fields_untouched(self): data = [{"conference": "Plain Conf", "year": 2026, "link": "https://plain.conf/"}] result = drop_redundant_link_fields(data) assert result[0] == {"conference": "Plain Conf", "year": 2026, "link": "https://plain.conf/"} + + +class TestDropBareSiteRoot: + """A sub-page link that is only the bare root of the main link's site is dropped.""" + + @staticmethod + def _entry(link: str, **fields: str) -> list: + return [{"conference": "PyCon DE & PyData", "year": 2027, "link": link, **fields}] + + def test_parent_domain_root_dropped(self): + """The series site root next to a year subdomain says nothing about the CfP.""" + result = drop_redundant_link_fields(self._entry("https://2027.pycon.de/", cfp_link="https://pycon.de/")) + assert "cfp_link" not in result[0] + assert result[0]["link"] == "https://2027.pycon.de/" + + def test_parent_domain_root_with_www_and_no_slash_dropped(self): + result = drop_redundant_link_fields( + self._entry("https://2027.pycon.de/", sponsor="http://www.pycon.de", finaid="https://pycon.de/"), + ) + assert "sponsor" not in result[0] + assert "finaid" not in result[0] + + def test_same_host_root_dropped_when_link_is_a_sub_page(self): + result = drop_redundant_link_fields(self._entry("https://pycon.org/2026/", cfp_link="https://www.pycon.org/")) + assert "cfp_link" not in result[0] + + def test_sub_page_on_parent_domain_kept(self): + result = drop_redundant_link_fields( + self._entry("https://2027.pycon.de/", sponsor="https://pycon.de/sponsoring/"), + ) + assert result[0]["sponsor"] == "https://pycon.de/sponsoring/" + + def test_other_site_kept(self): + result = drop_redundant_link_fields(self._entry("https://2027.pycon.de/", cfp_link="https://pretalx.com/x/cfp")) + assert result[0]["cfp_link"] == "https://pretalx.com/x/cfp" + + def test_other_site_root_kept(self): + result = drop_redundant_link_fields(self._entry("https://2027.pycon.de/", cfp_link="https://pretalx.com/")) + assert result[0]["cfp_link"] == "https://pretalx.com/" + + def test_year_subdomain_root_next_to_series_link_kept(self): + """The reverse direction is more specific than the series site, so it stays.""" + result = drop_redundant_link_fields(self._entry("https://pycon.de/", cfp_link="https://2027.pycon.de/")) + assert result[0]["cfp_link"] == "https://2027.pycon.de/" + + def test_anchor_on_parent_domain_kept(self): + result = drop_redundant_link_fields(self._entry("https://2027.pycon.de/", cfp_link="https://pycon.de/#cfp")) + assert result[0]["cfp_link"] == "https://pycon.de/#cfp" + + def test_query_on_parent_domain_kept(self): + result = drop_redundant_link_fields( + self._entry("https://2027.pycon.de/", cfp_link="https://pycon.de/?page=cfp"), + ) + assert result[0]["cfp_link"] == "https://pycon.de/?page=cfp" + + def test_lookalike_host_kept(self): + """A host that only ends in the same letters is not a parent domain.""" + result = drop_redundant_link_fields(self._entry("https://2027.pycon.de/", cfp_link="https://con.de/")) + assert result[0]["cfp_link"] == "https://con.de/" diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index efa88d9405..20a5caa48a 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -13,7 +13,9 @@ # A cfp_link/sponsor/finaid that is just the homepage again carries no # information (e.g. "cfp_link: https://2027.pycon.de/" on the PyCon DE entry # whose link is https://2027.pycon.de/) - it should be a different page, a -# subdomain, or at least a #anchor on the homepage. +# subdomain, or at least a #anchor on the homepage. The bare root of the +# series site (e.g. "cfp_link: https://pycon.de/" next to +# "link: https://2027.pycon.de/") carries no information either. REDUNDANT_LINK_FIELDS = ("cfp_link", "sponsor", "finaid") @@ -40,12 +42,102 @@ def normalize_url_pointer(url: str) -> tuple: return (netloc, parsed.path.rstrip("/"), parsed.params, parsed.query, parsed.fragment) +def site_host(url: str) -> str: + """Return the host of a URL in lower case, without port or a "www." prefix. + + Parameters + ---------- + url : str + URL to take the host from + + Returns + ------- + str + Host name, or an empty string when the URL has none + """ + host = urlparse(str(url).strip()).hostname or "" + return host.lower().removeprefix("www.") + + +def is_parent_domain(parent: str, host: str) -> bool: + """Check whether ``parent`` is a parent domain of ``host``. + + A single label such as ``de`` does not count as a parent. + + Parameters + ---------- + parent : str + Candidate parent host, e.g. ``pycon.de`` + host : str + Host to test, e.g. ``2027.pycon.de`` + + Returns + ------- + bool + True when ``host`` is a subdomain of ``parent`` + """ + if "." not in parent: + return False + return host.endswith("." + parent) + + +def is_bare_site_root(url: str) -> bool: + """Check whether a URL is the bare root of a site. + + A bare root has an empty or "/" path and no params, query or #fragment. + + Parameters + ---------- + url : str + URL to check + + Returns + ------- + bool + True when the URL points at nothing more specific than the site itself + """ + parsed = urlparse(str(url).strip()) + if not parsed.hostname or parsed.path not in {"", "/"}: + return False + return not (parsed.params or parsed.query or parsed.fragment) + + +def is_redundant_site_root(value: str, link: str) -> bool: + """Check whether a sub-page link is only the bare root of the main link's site. + + The site is the same when the hosts are equal after dropping "www.", or + when the sub-page host is a parent domain of the main link's host + (``pycon.de`` for ``2027.pycon.de``). The reverse is not redundant: a + year subdomain is more specific than the series site. + + Parameters + ---------- + value : str + Sub-page link (cfp_link, sponsor or finaid) + link : str + Main conference link + + Returns + ------- + bool + True when ``value`` adds nothing over ``link`` + """ + if not is_bare_site_root(value): + return False + value_host, link_host = site_host(value), site_host(link) + if not value_host or not link_host: + return False + return value_host == link_host or is_parent_domain(value_host, link_host) + + def drop_redundant_link_fields(data: list) -> list: """Drop sub-page link fields that are the same pointer as the main link. Goes through cfp_link/sponsor/finaid on each conference and removes any that just repeat the conference's main link. This catches - upstream sources that fill every URL column with the homepage. + upstream sources that fill every URL column with the homepage. A field + that is only the bare root of the same site (``https://pycon.de/`` next + to ``https://2027.pycon.de/``) is dropped as well. Parameters ---------- @@ -66,12 +158,16 @@ def drop_redundant_link_fields(data: list) -> list: base = normalize_url_pointer(link) for field in REDUNDANT_LINK_FIELDS: value = q.get(field) - if value and normalize_url_pointer(value) == base: - tqdm.write( - f"Dropping redundant '{field}' from {q.get('conference')} {q.get('year')}: " - f"'{value}' is the same pointer as the main link", - ) - del q[field] + if not value: + continue + if normalize_url_pointer(value) == base: + reason = "is the same pointer as the main link" + elif is_redundant_site_root(value, link): + reason = f"is only the root of the same site as the main link '{link}'" + else: + continue + tqdm.write(f"Dropping redundant '{field}' from {q.get('conference')} {q.get('year')}: '{value}' {reason}") + del q[field] return data From a59308cee7df6b3a6a2ebf887499ae2691da217d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 00:25:18 +0000 Subject: [PATCH 2/7] Keep links that redirect to a different site and record the redirect check_link_availability treated any redirect that ended on a 200 as a healthy link, even when it landed on another site: pythoncamp.de now redirects to barcamps.eu and pytennessee.org to a casino site. When the final URL is on a different site, the original URL is now returned as it is (no Wayback lookup or save, no replacement), the redirect is logged, and a record is kept for review. Different site means the hosts differ after dropping "www.", neither is a subdomain of the other, and their registrable domains differ. The registrable domain is the last two labels, or three under a known two-level public suffix such as co.uk or org.au, so ug.pycon.org to africa.pycon.org stays the same site. Same-site redirects behave as before. Records go to the module-level REDIRECT_RECORDS list and, when the DESK_QUEUE_OUT environment variable names a file, are appended to it as JSON lines shaped like a "link-check" "redirect" queue item: source, kind, received, title, url, year and a payload with conference, year, field, final_url and status. sort_yaml.check_links passes the conference, year and field as keyword arguments, which default to None, so other callers keep working. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QzyF2ciB5jhvcTVpj4ErMr --- tests/test_link_redirects.py | 319 +++++++++++++++++++++++++++++++++++ utils/sort_yaml.py | 10 +- utils/tidy_conf/links.py | 212 ++++++++++++++++++++++- 3 files changed, 536 insertions(+), 5 deletions(-) create mode 100644 tests/test_link_redirects.py diff --git a/tests/test_link_redirects.py b/tests/test_link_redirects.py new file mode 100644 index 0000000000..9ed765f19a --- /dev/null +++ b/tests/test_link_redirects.py @@ -0,0 +1,319 @@ +"""Tests for redirects to a different site during link checking. + +A conference link that now redirects to another site (a lapsed domain taken +over by a casino, a series that moved to a barcamp platform) must not be +treated as healthy, swapped for a Wayback copy or replaced by the target. +check_link_availability keeps the original URL and records the redirect for +a person to review. Redirects within the same site behave as before. +""" + +import json +import sys +from datetime import date +from datetime import datetime +from datetime import timedelta +from datetime import timezone +from pathlib import Path +from unittest.mock import patch + +import pytest +import responses +from freezegun import freeze_time + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +import sort_yaml +from tidy_conf import links + +# Recent enough to be checked (younger than five years) but in the past, so a +# healthy link does not trigger a Wayback save request. +TODAY = datetime.now(tz=timezone.utc).date() +PAST_START = TODAY - timedelta(days=60) + + +@pytest.fixture(autouse=True) +def _clean_redirect_records(monkeypatch): + """Start every test with no recorded redirects and no output file.""" + monkeypatch.delenv(links.REDIRECT_RECORDS_ENV, raising=False) + links.REDIRECT_RECORDS.clear() + yield + links.REDIRECT_RECORDS.clear() + + +def _redirect(source: str, target: str, status: int = 200) -> None: + responses.add(responses.GET, source, status=301, headers={"Location": target}) + responses.add(responses.GET, target, status=status) + + +def _check(url: str, start: date = PAST_START, **context) -> str: + return links.check_link_availability(url, start, cache=set(), cache_archived=set(), **context) + + +class TestRegistrableDomain: + """Registrable domain is the last two labels, or three under a two-level public suffix.""" + + @pytest.mark.parametrize( + ("host", "expected"), + [ + ("pycon.de", "pycon.de"), + ("2027.pycon.de", "pycon.de"), + ("ug.pycon.org", "pycon.org"), + ("pycon.org.au", "pycon.org.au"), + ("2026.pycon.org.au", "pycon.org.au"), + ("pyconuk.co.uk", "pyconuk.co.uk"), + ("www.pydata.co.uk", "pydata.co.uk"), + ("pycon.jp", "pycon.jp"), + ], + ) + def test_registrable_domain(self, host, expected): + assert links.registrable_domain(host) == expected + + def test_required_two_level_suffixes_present(self): + required = { + "co.uk", + "org.uk", + "ac.uk", + "gov.uk", + "com.au", + "org.au", + "net.au", + "com.br", + "org.br", + "co.jp", + "or.jp", + "ne.jp", + "com.ar", + "org.ar", + "co.za", + "org.za", + "co.nz", + "com.mx", + "com.tr", + "co.in", + "com.cn", + "com.sg", + } + assert required <= links.TWO_LEVEL_PUBLIC_SUFFIXES + + +class TestIsDifferentSite: + """Hosts on different sites: different, not nested, different registrable domain.""" + + @pytest.mark.parametrize( + ("url", "other"), + [ + ("https://pythoncamp.de/", "https://barcamps.eu/pythoncamp/"), + ("https://pytennessee.org/", "https://infojudislotonline.com/"), + ("https://example.co.uk/", "https://other.co.uk/"), + ("https://pycon.org.au/", "https://pyconau.org.au/"), + ], + ) + def test_different_sites(self, url, other): + assert links.is_different_site(url, other) + + @pytest.mark.parametrize( + ("url", "other"), + [ + ("https://2026.pycon.it/", "https://2026.pycon.it/en"), + ("https://www.pycon.de/", "https://pycon.de/"), + ("http://pycon.de/", "https://pycon.de:443/"), + ("https://pycon.de/", "https://2027.pycon.de/"), + ("https://2027.pycon.de/", "https://pycon.de/"), + ("https://ug.pycon.org/", "https://africa.pycon.org/"), + ("https://a.example.co.uk/", "https://b.example.co.uk/"), + ("https://2026.pycon.org.au/", "https://pycon.org.au/"), + ], + ) + def test_same_site(self, url, other): + assert not links.is_different_site(url, other) + + def test_missing_host_is_not_different(self): + assert not links.is_different_site("https://pycon.de/", "not a url") + + +class TestCrossSiteRedirect: + """A redirect to a different site keeps the original URL and is recorded.""" + + @responses.activate + def test_pythoncamp_to_barcamps_kept_and_recorded(self): + url = "https://pythoncamp.de/" + _redirect(url, "https://barcamps.eu/pythoncamp/") + + result = _check(url, conference="PythonCamp Germany", year=2026, field="link") + + assert result == url + assert len(links.REDIRECT_RECORDS) == 1 + record = links.REDIRECT_RECORDS[0] + assert record["source"] == "link-check" + assert record["kind"] == "redirect" + assert record["title"] == "PythonCamp Germany" + assert record["url"] == url + assert record["year"] == 2026 + assert record["payload"] == { + "conference": "PythonCamp Germany", + "year": 2026, + "field": "link", + "final_url": "https://barcamps.eu/pythoncamp/", + "status": 200, + } + + @responses.activate + def test_pytennessee_to_casino_kept_and_recorded(self): + url = "https://pytennessee.org/" + _redirect(url, "https://infojudislotonline.com/") + + result = _check(url, conference="PyTennessee", year=2025, field="link") + + assert result == url + assert [r["payload"]["final_url"] for r in links.REDIRECT_RECORDS] == ["https://infojudislotonline.com/"] + + @responses.activate + def test_no_wayback_lookup_or_save(self): + """Neither a future start (archive save) nor an error status (archive lookup) touches the Wayback Machine.""" + url = "https://pytennessee.org/" + _redirect(url, "https://infojudislotonline.com/", status=404) + + result = _check(url, start=TODAY + timedelta(days=60), field="cfp_link") + + assert result == url + assert [call.request.url for call in responses.calls] == [url, "https://infojudislotonline.com/"] + assert links.REDIRECT_RECORDS[0]["payload"]["status"] == 404 + assert links.REDIRECT_RECORDS[0]["payload"]["field"] == "cfp_link" + + @responses.activate + def test_without_context_uses_url_as_title(self): + url = "https://pythoncamp.de/" + _redirect(url, "https://barcamps.eu/pythoncamp/") + + assert _check(url) == url + record = links.REDIRECT_RECORDS[0] + assert record["title"] == url + assert record["year"] is None + assert record["payload"]["conference"] is None + assert record["payload"]["field"] is None + + +class TestSameSiteRedirect: + """Redirects within the same site keep today's behaviour and are not recorded.""" + + @responses.activate + def test_same_host_redirect_followed(self): + _redirect("https://2026.pycon.it/", "https://2026.pycon.it/en") + + result = _check("https://2026.pycon.it/", conference="PyCon Italia", year=2026, field="link") + + assert result == "https://2026.pycon.it/en" + assert links.REDIRECT_RECORDS == [] + + @responses.activate + def test_same_registrable_domain_kept_as_before(self): + """ug.pycon.org to africa.pycon.org is a healthy link, as it was before.""" + _redirect("https://ug.pycon.org/", "https://africa.pycon.org/") + + result = _check("https://ug.pycon.org/", conference="PyCon Uganda", year=2026, field="link") + + assert result == "https://ug.pycon.org/" + assert links.REDIRECT_RECORDS == [] + assert len(responses.calls) == 2 + + +class TestRedirectQueueFile: + """With DESK_QUEUE_OUT set, every record is appended to that file as one JSON line.""" + + @freeze_time("2026-10-14 06:00:00") + @responses.activate + def test_json_lines_written(self, tmp_path, monkeypatch): + out_file = tmp_path / "queue" / "link-check.jsonl" + monkeypatch.setenv("DESK_QUEUE_OUT", str(out_file)) + _redirect("https://pythoncamp.de/", "https://barcamps.eu/pythoncamp/") + _redirect("https://pytennessee.org/", "https://infojudislotonline.com/", status=403) + + _check("https://pythoncamp.de/", conference="PythonCamp Germany", year=2026, field="link") + _check("https://pytennessee.org/", conference="PyTennessee", year=2025, field="sponsor") + + lines = out_file.read_text(encoding="utf-8").splitlines() + assert [json.loads(line) for line in lines] == [ + { + "source": "link-check", + "kind": "redirect", + "received": "2026-10-14T06:00:00Z", + "title": "PythonCamp Germany", + "url": "https://pythoncamp.de/", + "year": 2026, + "payload": { + "conference": "PythonCamp Germany", + "year": 2026, + "field": "link", + "final_url": "https://barcamps.eu/pythoncamp/", + "status": 200, + }, + }, + { + "source": "link-check", + "kind": "redirect", + "received": "2026-10-14T06:00:00Z", + "title": "PyTennessee", + "url": "https://pytennessee.org/", + "year": 2025, + "payload": { + "conference": "PyTennessee", + "year": 2025, + "field": "sponsor", + "final_url": "https://infojudislotonline.com/", + "status": 403, + }, + }, + ] + assert [json.loads(line) for line in lines] == links.REDIRECT_RECORDS + + @responses.activate + def test_existing_file_appended(self, tmp_path, monkeypatch): + out_file = tmp_path / "redirects.jsonl" + out_file.write_text('{"earlier": true}\n', encoding="utf-8") + monkeypatch.setenv("DESK_QUEUE_OUT", str(out_file)) + _redirect("https://pythoncamp.de/", "https://barcamps.eu/pythoncamp/") + + _check("https://pythoncamp.de/") + + lines = out_file.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + assert json.loads(lines[0]) == {"earlier": True} + assert json.loads(lines[1])["url"] == "https://pythoncamp.de/" + + @responses.activate + def test_no_file_without_env(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _redirect("https://pythoncamp.de/", "https://barcamps.eu/pythoncamp/") + + _check("https://pythoncamp.de/") + + assert len(links.REDIRECT_RECORDS) == 1 + assert not list(tmp_path.glob("*.jsonl")) + + +class TestCheckLinksContext: + """sort_yaml.check_links passes the conference, year and field to the link check.""" + + @patch("sort_yaml.get_cache", return_value=(set(), set())) + @patch("sort_yaml.check_link_availability", side_effect=lambda url, *_args, **_kwargs: url) + def test_context_passed(self, mock_check_link, mock_get_cache): + data = [ + { + "conference": "PythonCamp Germany", + "year": 2026, + "link": "https://pythoncamp.de/", + "sponsor": "https://pythoncamp.de/sponsors/", + "start": PAST_START, + }, + ] + + sort_yaml.check_links(data) + + contexts = [ + (call.args[0], call.kwargs["conference"], call.kwargs["year"], call.kwargs["field"]) + for call in mock_check_link.call_args_list + ] + assert contexts == [ + ("https://pythoncamp.de/", "PythonCamp Germany", 2026, "link"), + ("https://pythoncamp.de/sponsors/", "PythonCamp Germany", 2026, "sponsor"), + ] diff --git a/utils/sort_yaml.py b/utils/sort_yaml.py index a6254b7bc7..b3bea93876 100644 --- a/utils/sort_yaml.py +++ b/utils/sort_yaml.py @@ -232,7 +232,15 @@ def check_links(data): for i, q in tqdm(enumerate(sorted(data, key=operator.itemgetter("year"), reverse=True)), total=len(data)): for key in ("link", "cfp_link", "sponsor", "finaid"): if key in q: - new_link = check_link_availability(q[key], q["start"], cache=cache, cache_archived=cache_archived) + new_link = check_link_availability( + q[key], + q["start"], + cache=cache, + cache_archived=cache_archived, + conference=q.get("conference"), + year=q.get("year"), + field=key, + ) parsed_url = urlparse(new_link) if q[key] != new_link and parsed_url.hostname and parsed_url.hostname.endswith(".archive.org"): time.sleep(0.5) diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index 20a5caa48a..6653a08eea 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -1,3 +1,5 @@ +import json +import os import re import time from datetime import datetime @@ -18,6 +20,46 @@ # "link: https://2027.pycon.de/") carries no information either. REDUNDANT_LINK_FIELDS = ("cfp_link", "sponsor", "finaid") +# Public suffixes with two labels, under which a registrable domain has three +# labels (pycon.org.au rather than org.au). This is not the full Public Suffix +# List, only the suffixes conference sites are likely to use. +TWO_LEVEL_PUBLIC_SUFFIXES = frozenset( + { + "ac.uk", + "co.uk", + "gov.uk", + "org.uk", + "com.au", + "net.au", + "org.au", + "com.br", + "org.br", + "co.jp", + "ne.jp", + "or.jp", + "com.ar", + "org.ar", + "co.za", + "org.za", + "co.nz", + "com.mx", + "com.tr", + "co.in", + "com.cn", + "com.sg", + }, +) + +# Redirects to a different site found by check_link_availability. A link that +# now lands on another site (a lapsed domain, a parked page, a new home for +# the series) needs a person to look at it, so the link is kept as it is and +# the redirect is recorded here instead. +REDIRECT_RECORDS: list[dict] = [] + +# When this environment variable names a file, every redirect record is also +# appended to it as one JSON object per line, for a later job to pick up. +REDIRECT_RECORDS_ENV = "DESK_QUEUE_OUT" + def normalize_url_pointer(url: str) -> tuple: """Normalize a URL for same-pointer comparison. @@ -62,7 +104,7 @@ def site_host(url: str) -> str: def is_parent_domain(parent: str, host: str) -> bool: """Check whether ``parent`` is a parent domain of ``host``. - A single label such as ``de`` does not count as a parent. + A public suffix such as ``de`` or ``co.uk`` does not count as a parent. Parameters ---------- @@ -76,11 +118,122 @@ def is_parent_domain(parent: str, host: str) -> bool: bool True when ``host`` is a subdomain of ``parent`` """ - if "." not in parent: + if "." not in parent or parent in TWO_LEVEL_PUBLIC_SUFFIXES: return False return host.endswith("." + parent) +def registrable_domain(host: str) -> str: + """Return the registrable domain of a host. + + This is the last two labels, or the last three when the last two are a + known two-level public suffix (see ``TWO_LEVEL_PUBLIC_SUFFIXES``). + + Parameters + ---------- + host : str + Host name, e.g. ``2027.pycon.de`` + + Returns + ------- + str + Registrable domain, e.g. ``pycon.de`` + """ + labels = host.lower().strip(".").split(".") + if len(labels) >= 3 and ".".join(labels[-2:]) in TWO_LEVEL_PUBLIC_SUFFIXES: + return ".".join(labels[-3:]) + return ".".join(labels[-2:]) + + +def is_different_site(url: str, other: str) -> bool: + """Check whether two URLs belong to different sites. + + They are different sites when the hosts differ after dropping "www.", + neither host is a subdomain of the other, and their registrable domains + differ. ``ug.pycon.org`` and ``africa.pycon.org`` are the same site; + ``pythoncamp.de`` and ``barcamps.eu`` are not. + + Parameters + ---------- + url : str + First URL + other : str + Second URL + + Returns + ------- + bool + True when the URLs are on different sites; False when either has no host + """ + host, other_host = site_host(url), site_host(other) + if not host or not other_host or host == other_host: + return False + if is_parent_domain(host, other_host) or is_parent_domain(other_host, host): + return False + return registrable_domain(host) != registrable_domain(other_host) + + +def record_redirect( + url: str, + final_url: str, + status: int, + conference: str | None = None, + year: int | None = None, + field: str | None = None, +) -> dict: + """Record a redirect to a different site for review. + + The record is appended to ``REDIRECT_RECORDS`` and, when the + ``DESK_QUEUE_OUT`` environment variable names a file, written to that + file as one JSON line. Its shape is a queue item with source + "link-check" and kind "redirect". + + Parameters + ---------- + url : str + URL as stored in the data + final_url : str + URL the redirect chain ended on + status : int + HTTP status code of the final response + conference : str | None + Conference name, when known + year : int | None + Conference year, when known + field : str | None + Data field holding the URL (link, cfp_link, sponsor or finaid), when known + + Returns + ------- + dict + The record that was stored + """ + record = { + "source": "link-check", + "kind": "redirect", + "received": datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "title": conference or url, + "url": url, + "year": year, + "payload": { + "conference": conference, + "year": year, + "field": field, + "final_url": final_url, + "status": status, + }, + } + REDIRECT_RECORDS.append(record) + + out_path = os.environ.get(REDIRECT_RECORDS_ENV) + if out_path: + out_file = Path(out_path) + out_file.parent.mkdir(parents=True, exist_ok=True) + with out_file.open("a", encoding="utf-8", newline="\n") as f: + f.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + return record + + def is_bare_site_root(url: str) -> bool: """Check whether a URL is the bare root of a site. @@ -194,7 +347,16 @@ def get_cache(): return cache, cache_archived -def check_link_availability(url, start, cache=None, cache_archived=None): +def check_link_availability( + url, + start, + cache=None, + cache_archived=None, + *, + conference: str | None = None, + year: int | None = None, + field: str | None = None, +) -> str: """Checks if a URL is available. If not, tries to retrieve an archived version from the Wayback Machine. @@ -202,9 +364,35 @@ def check_link_availability(url, start, cache=None, cache_archived=None): Automatically redirects to the final URL if it was redirected within the same domain. + A redirect that ends on a different site (see ``is_different_site``) is not + treated as healthy: the original URL is returned unchanged, with no Wayback + lookup, and the redirect is recorded with ``record_redirect`` for review. + Automatically caches old (5+ years) URLs. - Note: Some URLs may not be available or archived. These return None and are logged. + Note: Some URLs may not be available or archived. These return the original URL and are logged. + + Parameters + ---------- + url : str + URL to check + start : datetime.date + Start date of the conference the URL belongs to + cache : set | None + URLs known to have no archived version, read from disk when None + cache_archived : set | None + URLs already archived, read from disk when None + conference : str | None + Conference name, only used to describe a recorded redirect + year : int | None + Conference year, only used to describe a recorded redirect + field : str | None + Data field holding the URL, only used to describe a recorded redirect + + Returns + ------- + str + The URL to keep: the original, a same-domain redirect target, or an archived copy """ # If it's archived already, return the URL if url.startswith(("https://web.archive.org", "http://web.archive.org")): @@ -226,6 +414,22 @@ def check_link_availability(url, start, cache=None, cache_archived=None): try: response = requests.get(url, allow_redirects=True, timeout=10) final_url = response.url + # A redirect to another site is not a healthy link: keep the URL as it + # is and leave the decision to a person. + if is_different_site(url, final_url): + tqdm.write( + f"URL {url} was redirected to a different site: {final_url} " + f"(status code: {response.status_code}). Keeping the original URL for review.", + ) + record_redirect( + url, + final_url, + response.status_code, + conference=conference, + year=year, + field=field, + ) + return url # Check if the final URL is within the same domain as the original URL if urlparse(url).netloc == urlparse(final_url).netloc and final_url != url: # Don't cache the redirect if it suddenly has a query string From 1dc9a0f5089f4107ce0fbeae1b2fe3aa37e23055 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 00:26:50 +0000 Subject: [PATCH 3/7] Add an optional coc field for the Code of Conduct URL Conference entries can now carry "coc", a URL to the conference's Code of Conduct. It is validated and serialised like finaid and sponsor (an http or https URL, written back as a plain string) and sits right after finaid in key order: in utils/schema.yml, which order_keywords follows, and in the pydantic model, whose field order the YAML writer follows. It is also listed with the other optional columns in tidy_conf/validation.py and documented in README.md and CONTRIBUTING.md. No entry uses it yet, so the sort leaves _data unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QzyF2ciB5jhvcTVpj4ErMr --- CONTRIBUTING.md | 1 + README.md | 2 ++ tests/test_schema_validation.py | 45 +++++++++++++++++++++++++++++++++ utils/schema.yml | 1 + utils/tidy_conf/schema.py | 3 ++- utils/tidy_conf/validation.py | 1 + 6 files changed, 52 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6b3cc79bc..5769a28a57 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,6 +80,7 @@ You can enhance your entry with these optional fields: - `timezone`: IANA timezone (defaults to AoE if omitted) - `sponsor`: Sponsorship page URL - `finaid`: Financial aid information URL +- `coc`: Code of Conduct URL - `twitter`: Conference Twitter handle - `mastodon`: Conference Mastodon URL - `note`: Additional important information diff --git a/README.md b/README.md index bf5c34a34a..3617bb0bb3 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ To add or update a deadline: end: YYYY-MM-DD # End date of conference for calendar sponsor: link-to-sponsor.page # URL to Sponsorship page (Optional) finaid: link-to-finaid.page # URL to Financial Aid page (Optional) + coc: link-to-coc.page # URL to Code of Conduct (Optional) twitter: BestConfEver # Twitter handle of conference (Optional) mastodon: https://mastodon.social/@bconf # Mastodon handle of conference (Optional) sub: PY # Type of conference (see or add _data/types.yml) @@ -83,6 +84,7 @@ To add or update a deadline: | `start` | Start date of conference for calendar | `date` | ✔ | | `end` | End date of conference for calendar | `date` | ✔ | | `finaid` | URL to financial aid information | `str` | | +| `coc` | URL to the Code of Conduct | `str` | | | `sponsor` | URL to sponsorship opportunities | `str` | | | `twitter` | Twitter handle of conference | `str` | | | `mastodon` | Mastodon handle of conference | `str` | | diff --git a/tests/test_schema_validation.py b/tests/test_schema_validation.py index 5ada95ea0a..7f400ad93f 100644 --- a/tests/test_schema_validation.py +++ b/tests/test_schema_validation.py @@ -11,11 +11,13 @@ sys.path.insert(0, str(Path(__file__).parent)) sys.path.append(str(Path(__file__).parent.parent / "utils")) +import sort_yaml from hypothesis_strategies import HYPOTHESIS_AVAILABLE from hypothesis_strategies import valid_latitude from hypothesis_strategies import valid_longitude from tidy_conf.schema import Conference from tidy_conf.schema import Location +from tidy_conf.schema import get_schema class TestConferenceSchema: @@ -365,6 +367,49 @@ def test_timezone_accepted(self, sample_conference): assert conf.timezone == tz +class TestCodeOfConductField: + """The optional coc field holds a URL to the conference's Code of Conduct.""" + + def test_coc_optional(self, sample_conference): + conf = Conference(**sample_conference) + assert conf.coc is None + assert "coc" not in conf.model_dump(exclude_none=True) + + def test_coc_accepts_url(self, sample_conference): + sample_conference["coc"] = "https://test.pycon.org/code-of-conduct/" + conf = Conference(**sample_conference) + assert str(conf.coc) == "https://test.pycon.org/code-of-conduct/" + # Serialised as a plain string, like the other URL fields + assert conf.model_dump()["coc"] == "https://test.pycon.org/code-of-conduct/" + + @pytest.mark.parametrize("value", ["not-a-url", "test.pycon.org/coc", "ftp://test.pycon.org/coc"]) + def test_coc_rejects_non_url(self, sample_conference, value): + sample_conference["coc"] = value + with pytest.raises(ValidationError) as exc_info: + Conference(**sample_conference) + assert {error["loc"][0] for error in exc_info.value.errors()} == {"coc"} + + def test_coc_key_order_after_finaid(self, sample_conference): + """Key order comes from utils/schema.yml (order_keywords) and the model (YAML output).""" + schema_keys = get_schema().columns.tolist() + assert schema_keys[schema_keys.index("finaid") + 1] == "coc" + + model_keys = list(Conference.model_fields) + assert model_keys[model_keys.index("finaid") + 1] == "coc" + + entry = { + "coc": "https://test.pycon.org/coc/", + **sample_conference, + "finaid": "https://test.pycon.org/finaid/", + "twitter": "pycontest", + } + ordered = list(sort_yaml.order_keywords(entry)) + assert ordered[ordered.index("finaid") + 1] == "coc" + + dumped = list(Conference(**entry).model_dump(exclude_none=True)) + assert dumped[dumped.index("finaid") + 1] == "coc" + + class TestSchemaRegressions: """Regression tests for schema validation bugs.""" diff --git a/utils/schema.yml b/utils/schema.yml index 9443815202..287908603e 100644 --- a/utils/schema.yml +++ b/utils/schema.yml @@ -15,6 +15,7 @@ end: YYYY-MM-DD # End date of conference for calendar sponsor: link-to-sponsor.page # URL to Sponsorship page (Optional) finaid: link-to-finaid.page # URL to Financial Aid page (Optional) + coc: link-to-coc.page # URL to Code of Conduct (Optional) twitter: BestConfEver # Twitter handle of conference (Optional) mastodon: https://mastodon.social/@bconf # Mastodon handle of conference (Optional) bluesky: https://bsky.app/@bconf # Bluesky handle of conference (Optional) diff --git a/utils/tidy_conf/schema.py b/utils/tidy_conf/schema.py index a5b8fcd7ff..add24a4c1b 100644 --- a/utils/tidy_conf/schema.py +++ b/utils/tidy_conf/schema.py @@ -78,6 +78,7 @@ class Conference(BaseModel): end: PythonDate sponsor: HttpUrl | None = None finaid: HttpUrl | None = None + coc: HttpUrl | None = None twitter: str | None = None mastodon: HttpUrl | None = None bluesky: str | None = None @@ -154,7 +155,7 @@ def validate_title(cls, v): return re.sub(r"\b(19|20)\d{2}\b", "", v).strip() return v - @field_serializer("link", "cfp_link", "sponsor", "finaid", "mastodon", "youtube") + @field_serializer("link", "cfp_link", "sponsor", "finaid", "coc", "mastodon", "youtube") def ser_url(self, value): return str(value) diff --git a/utils/tidy_conf/validation.py b/utils/tidy_conf/validation.py index 46a9dd1ee1..28ce669394 100644 --- a/utils/tidy_conf/validation.py +++ b/utils/tidy_conf/validation.py @@ -25,6 +25,7 @@ "sub", "sponsor", "finaid", + "coc", "tutorial_deadline", "workshop_deadline", "timezone", From 66dad41e201040c05a13deb4fc334ed4dc49e96c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 00:27:41 +0000 Subject: [PATCH 4/7] Fail the branch sync on a merge conflict instead of keeping main's side When the accumulator branch could not be rebased or merged onto main cleanly, the enrich-tba-conferences and canonical-source-update jobs retried with "git merge origin/main -X theirs", which resolves every conflict in favour of main and silently throws away the data changes already waiting on the branch. Both jobs now collect the conflicted files, abort the merge, report them as "::error title=Branch Sync Conflict::" and exit 1, so a person resolves the conflict. check-conference-update.yml has the same fallback and is left alone here. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QzyF2ciB5jhvcTVpj4ErMr --- .github/workflows/canonical-source-update.yml | 13 ++++++------- .github/workflows/enrich-tba-conferences.yml | 13 ++++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/canonical-source-update.yml b/.github/workflows/canonical-source-update.yml index 756ad5fb68..665f74ac32 100644 --- a/.github/workflows/canonical-source-update.yml +++ b/.github/workflows/canonical-source-update.yml @@ -94,18 +94,17 @@ jobs: git fetch origin $UPDATE_BRANCH git checkout $UPDATE_BRANCH - # Sync with main - try rebase first, then merge + # Sync with main - try rebase first, then merge. A merge conflict + # fails the job: resolving it in favour of main would silently drop + # the changes already on this branch. if ! git rebase origin/main; then echo "::warning title=Rebase Failed::Attempting merge instead" git rebase --abort 2>/dev/null || true if ! git merge origin/main --no-edit; then + CONFLICTS=$(git diff --name-only --diff-filter=U | paste -sd ' ' -) git merge --abort 2>/dev/null || true - echo "::warning title=Merge Conflict::Auto-resolving conflicts in favor of main" - if ! git merge origin/main -X theirs --no-edit; then - git merge --abort 2>/dev/null || true - echo "::error title=Branch Sync Failed::Could not sync with main" - exit 1 - fi + echo "::error title=Branch Sync Conflict::${CONFLICTS:-git merge origin/main failed with no conflicted files}" + exit 1 fi fi else diff --git a/.github/workflows/enrich-tba-conferences.yml b/.github/workflows/enrich-tba-conferences.yml index 6f6e6303da..27ceee7d5c 100644 --- a/.github/workflows/enrich-tba-conferences.yml +++ b/.github/workflows/enrich-tba-conferences.yml @@ -81,18 +81,17 @@ jobs: git fetch origin $UPDATE_BRANCH git checkout $UPDATE_BRANCH - # Sync with main - try rebase first, then merge + # Sync with main - try rebase first, then merge. A merge conflict + # fails the job: resolving it in favour of main would silently drop + # the changes already on this branch. if ! git rebase origin/main; then echo "::warning title=Rebase Failed::Attempting merge instead" git rebase --abort 2>/dev/null || true if ! git merge origin/main --no-edit; then + CONFLICTS=$(git diff --name-only --diff-filter=U | paste -sd ' ' -) git merge --abort 2>/dev/null || true - echo "::warning title=Merge Conflict::Auto-resolving conflicts in favor of main" - if ! git merge origin/main -X theirs --no-edit; then - git merge --abort 2>/dev/null || true - echo "::error title=Branch Sync Failed::Could not sync with main" - exit 1 - fi + echo "::error title=Branch Sync Conflict::${CONFLICTS:-git merge origin/main failed with no conflicted files}" + exit 1 fi fi else From 00696a3b231b15eaf746f76f8423934eea775cdd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 01:07:49 +0000 Subject: [PATCH 5/7] Treat or.tz, org.il and co.il as two-level public suffixes registrable_domain only keeps three labels under a suffix listed in TWO_LEVEL_PUBLIC_SUFFIXES. PyCon Tanzania (pycon.or.tz) and PyCon Israel (pycon.org.il, cfp.pycon.org.il) use suffixes that were not listed, so their registrable domain collapsed to the bare suffix and a redirect to any other site under it, such as a parked page on another .org.il domain, counted as the same site and went unrecorded. or.tz and org.il are the two-level suffixes current entries use that were missing; co.il is added with org.il. Regression cases cover both hosts in TestIsDifferentSite and TestRegistrableDomain. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QzyF2ciB5jhvcTVpj4ErMr --- tests/test_link_redirects.py | 13 +++++++++++++ utils/tidy_conf/links.py | 3 +++ 2 files changed, 16 insertions(+) diff --git a/tests/test_link_redirects.py b/tests/test_link_redirects.py index 9ed765f19a..497f27a747 100644 --- a/tests/test_link_redirects.py +++ b/tests/test_link_redirects.py @@ -63,6 +63,9 @@ class TestRegistrableDomain: ("pyconuk.co.uk", "pyconuk.co.uk"), ("www.pydata.co.uk", "pydata.co.uk"), ("pycon.jp", "pycon.jp"), + ("pycon.or.tz", "pycon.or.tz"), + ("cfp.pycon.org.il", "pycon.org.il"), + ("example.co.il", "example.co.il"), ], ) def test_registrable_domain(self, host, expected): @@ -92,6 +95,9 @@ def test_required_two_level_suffixes_present(self): "co.in", "com.cn", "com.sg", + "co.il", + "org.il", + "or.tz", } assert required <= links.TWO_LEVEL_PUBLIC_SUFFIXES @@ -106,6 +112,12 @@ class TestIsDifferentSite: ("https://pytennessee.org/", "https://infojudislotonline.com/"), ("https://example.co.uk/", "https://other.co.uk/"), ("https://pycon.org.au/", "https://pyconau.org.au/"), + # Two-level suffixes used by current entries: a lapsed domain that + # lands on another site under the same suffix is a different site. + ("https://pycon.or.tz/", "https://squatter.or.tz/"), + ("https://pycon.org.il/", "https://parked.org.il/"), + ("https://cfp.pycon.org.il/", "https://parked.org.il/"), + ("https://example.co.il/", "https://other.co.il/"), ], ) def test_different_sites(self, url, other): @@ -122,6 +134,7 @@ def test_different_sites(self, url, other): ("https://ug.pycon.org/", "https://africa.pycon.org/"), ("https://a.example.co.uk/", "https://b.example.co.uk/"), ("https://2026.pycon.org.au/", "https://pycon.org.au/"), + ("https://cfp.pycon.org.il/", "https://pycon.org.il/"), ], ) def test_same_site(self, url, other): diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index 6653a08eea..cfd5082b6d 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -47,6 +47,9 @@ "co.in", "com.cn", "com.sg", + "co.il", + "org.il", + "or.tz", }, ) From 4f328f3b50040393e90dbedf167ae547213fb283 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 01:09:00 +0000 Subject: [PATCH 6/7] Do not record redirects from short links to their expected targets Every forms.gle link redirects to docs.google.com, so the cross-site check recorded the three forms.gle cfp_links in conferences.yml (and the ones on recent archive entries) as different-site redirects on every link check. A person would review the same known-good items again after each run, and upcoming forms.gle links lost the Wayback save that any other healthy link gets. is_expected_redirect now covers links that redirect to another site by design. EXPECTED_REDIRECT_TARGETS lists hosts with a known target: forms.gle must land on google.com and lu.ma on luma.com, so a forms.gle link that lands anywhere else is still recorded. URL_SHORTENERS lists general shorteners (bit.ly, goo.gl, tinyurl.com), whose links may land anywhere. Such a redirect is not recorded and the link is checked as it was before the cross-site check: on a 200 the URL is kept and, for an upcoming conference, saved to the Wayback Machine. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QzyF2ciB5jhvcTVpj4ErMr --- tests/test_link_redirects.py | 86 ++++++++++++++++++++++++++++++++++-- utils/tidy_conf/links.py | 50 ++++++++++++++++++++- 2 files changed, 131 insertions(+), 5 deletions(-) diff --git a/tests/test_link_redirects.py b/tests/test_link_redirects.py index 497f27a747..161d73da08 100644 --- a/tests/test_link_redirects.py +++ b/tests/test_link_redirects.py @@ -4,7 +4,8 @@ over by a casino, a series that moved to a barcamp platform) must not be treated as healthy, swapped for a Wayback copy or replaced by the target. check_link_availability keeps the original URL and records the redirect for -a person to review. Redirects within the same site behave as before. +a person to review. Redirects within the same site behave as before, and so +do short links such as forms.gle that redirect to another site by design. """ import json @@ -40,8 +41,8 @@ def _clean_redirect_records(monkeypatch): links.REDIRECT_RECORDS.clear() -def _redirect(source: str, target: str, status: int = 200) -> None: - responses.add(responses.GET, source, status=301, headers={"Location": target}) +def _redirect(source: str, target: str, status: int = 200, redirect_status: int = 301) -> None: + responses.add(responses.GET, source, status=redirect_status, headers={"Location": target}) responses.add(responses.GET, target, status=status) @@ -206,6 +207,85 @@ def test_without_context_uses_url_as_title(self): assert record["payload"]["field"] is None +class TestExpectedRedirect: + """Short links redirect to another site by design and are not recorded.""" + + FORMS_URL = "https://forms.gle/iiFnW8oZ8p9do4TR7" + FORMS_TARGET = "https://docs.google.com/forms/d/e/1FAIpQLSe6/viewform?usp=send_form" + + @pytest.mark.parametrize( + ("url", "final_url"), + [ + ("https://forms.gle/iiFnW8oZ8p9do4TR7", "https://docs.google.com/forms/d/e/1FAIpQLSe6/viewform"), + ("https://forms.gle/abc", "https://accounts.google.com/ServiceLogin"), + ("https://lu.ma/pycon", "https://luma.com/pycon"), + ("https://bit.ly/pycon-cfp", "https://pretalx.com/pycon-2026/cfp"), + ("https://goo.gl/forms/abc", "https://docs.google.com/forms/d/abc/viewform"), + ("https://maps.app.goo.gl/abc", "https://www.google.com/maps/place/x"), + ("https://tinyurl.com/pycon", "https://example.org/"), + ], + ) + def test_expected(self, url, final_url): + assert links.is_expected_redirect(url, final_url) + + @pytest.mark.parametrize( + ("url", "final_url"), + [ + ("https://forms.gle/abc", "https://infojudislotonline.com/"), + ("https://lu.ma/pycon", "https://parked.example/"), + ("https://pythoncamp.de/", "https://barcamps.eu/pythoncamp/"), + ("https://docs.google.com/forms/d/abc/viewform", "https://forms.gle/abc"), + ("not a url", "https://docs.google.com/"), + ], + ) + def test_not_expected(self, url, final_url): + assert not links.is_expected_redirect(url, final_url) + + @responses.activate + def test_forms_gle_kept_and_not_recorded(self): + _redirect(self.FORMS_URL, self.FORMS_TARGET, redirect_status=302) + + result = _check(self.FORMS_URL, conference="PyCon Netherlands", year=2026, field="cfp_link") + + assert result == self.FORMS_URL + assert links.REDIRECT_RECORDS == [] + assert len(responses.calls) == 2 + + @responses.activate + def test_forms_gle_upcoming_saved_to_wayback(self, tmp_path, monkeypatch): + """An upcoming short link is still saved to the Wayback Machine like any healthy link.""" + monkeypatch.chdir(tmp_path) + _redirect(self.FORMS_URL, self.FORMS_TARGET, redirect_status=302) + save_url = "https://web.archive.org/save/" + self.FORMS_URL + responses.add(responses.GET, save_url, status=200) + + result = _check(self.FORMS_URL, start=TODAY + timedelta(days=60), field="cfp_link") + + assert result == self.FORMS_URL + assert links.REDIRECT_RECORDS == [] + assert [call.request.url for call in responses.calls][-1] == save_url + + @responses.activate + def test_forms_gle_to_other_site_recorded(self): + _redirect("https://forms.gle/abc", "https://infojudislotonline.com/", redirect_status=302) + + assert _check("https://forms.gle/abc", field="cfp_link") == "https://forms.gle/abc" + assert [r["payload"]["final_url"] for r in links.REDIRECT_RECORDS] == ["https://infojudislotonline.com/"] + + @responses.activate + def test_no_queue_line_for_short_link(self, tmp_path, monkeypatch): + out_file = tmp_path / "redirects.jsonl" + monkeypatch.setenv("DESK_QUEUE_OUT", str(out_file)) + _redirect(self.FORMS_URL, self.FORMS_TARGET, redirect_status=302) + _redirect("https://bit.ly/pycon-cfp", "https://pretalx.com/pycon-2026/cfp") + + _check(self.FORMS_URL, field="cfp_link") + _check("https://bit.ly/pycon-cfp", field="cfp_link") + + assert links.REDIRECT_RECORDS == [] + assert not out_file.exists() + + class TestSameSiteRedirect: """Redirects within the same site keep today's behaviour and are not recorded.""" diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index cfd5082b6d..4b5ba000d1 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -53,6 +53,19 @@ }, ) +# Hosts whose links always redirect to another site, mapped to the registrable +# domains those redirects are expected to land on. forms.gle is the Google +# Forms short link (it lands on docs.google.com) and lu.ma is the old Luma +# domain, which now redirects to luma.com. +EXPECTED_REDIRECT_TARGETS = { + "forms.gle": frozenset({"google.com"}), + "lu.ma": frozenset({"luma.com"}), +} + +# General URL shorteners. A short link lands wherever it was made to point, so +# a redirect from one to another site says nothing about the link. +URL_SHORTENERS = frozenset({"bit.ly", "goo.gl", "tinyurl.com"}) + # Redirects to a different site found by check_link_availability. A link that # now lands on another site (a lapsed domain, a parked page, a new home for # the series) needs a person to look at it, so the link is kept as it is and @@ -176,6 +189,34 @@ def is_different_site(url: str, other: str) -> bool: return registrable_domain(host) != registrable_domain(other_host) +def is_expected_redirect(url: str, final_url: str) -> bool: + """Check whether a redirect to another site is what the link is for. + + A link on a general URL shortener (see ``URL_SHORTENERS``) may land + anywhere. A link on a host listed in ``EXPECTED_REDIRECT_TARGETS`` must + land on one of the registrable domains listed for it: ``forms.gle`` on + ``google.com``, ``lu.ma`` on ``luma.com``. Any other redirect is not + expected. + + Parameters + ---------- + url : str + URL as stored in the data + final_url : str + URL the redirect chain ended on + + Returns + ------- + bool + True when the redirect is one the link is expected to make + """ + origin = registrable_domain(site_host(url)) + if origin in URL_SHORTENERS: + return True + targets = EXPECTED_REDIRECT_TARGETS.get(origin, frozenset()) + return registrable_domain(site_host(final_url)) in targets + + def record_redirect( url: str, final_url: str, @@ -370,6 +411,10 @@ def check_link_availability( A redirect that ends on a different site (see ``is_different_site``) is not treated as healthy: the original URL is returned unchanged, with no Wayback lookup, and the redirect is recorded with ``record_redirect`` for review. + A short link that lands where it is expected to (see ``is_expected_redirect``), + such as forms.gle on docs.google.com, is not recorded: it is checked like + any other link, so on a 200 the original URL is kept and, when the + conference is still to come, saved to the Wayback Machine. Automatically caches old (5+ years) URLs. @@ -418,8 +463,9 @@ def check_link_availability( response = requests.get(url, allow_redirects=True, timeout=10) final_url = response.url # A redirect to another site is not a healthy link: keep the URL as it - # is and leave the decision to a person. - if is_different_site(url, final_url): + # is and leave the decision to a person. Short links such as + # forms.gle redirect to another site by design and are left out. + if is_different_site(url, final_url) and not is_expected_redirect(url, final_url): tqdm.write( f"URL {url} was redirected to a different site: {final_url} " f"(status code: {response.status_code}). Keeping the original URL for review.", From f0f561bb5367f4abcfebdb922539ca1180db5ca6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 01:09:18 +0000 Subject: [PATCH 7/7] Say that the queue writer assigns the id of a redirect record The record_redirect docstring and the earlier commit called each record a queue item, but a queue item also carries an "id" (a sha1 of source, kind, url and payload), which the records leave out. The docstring and the comment on REDIRECT_RECORDS_ENV now say that the JSON lines are not complete items: the queue writer assigns the id, and the series key of the file the item goes in, when it adds a line, so both are defined in one place. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QzyF2ciB5jhvcTVpj4ErMr --- utils/tidy_conf/links.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index 4b5ba000d1..b125f7e607 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -73,7 +73,10 @@ REDIRECT_RECORDS: list[dict] = [] # When this environment variable names a file, every redirect record is also -# appended to it as one JSON object per line, for a later job to pick up. +# appended to it as one JSON object per line, for a later job to pick up. The +# lines are not complete queue items: the queue writer assigns each item its +# "id" and the series key of the file it goes in when it adds the line, so +# both are defined in one place. REDIRECT_RECORDS_ENV = "DESK_QUEUE_OUT" @@ -229,8 +232,10 @@ def record_redirect( The record is appended to ``REDIRECT_RECORDS`` and, when the ``DESK_QUEUE_OUT`` environment variable names a file, written to that - file as one JSON line. Its shape is a queue item with source - "link-check" and kind "redirect". + file as one JSON line. It holds the fields of a queue item with source + "link-check" and kind "redirect", except "id": the queue writer assigns + the id, and the series key of the file the item goes in, when it adds + the line. Parameters ----------