diff --git a/.github/workflows/canonical-source-update.yml b/.github/workflows/canonical-source-update.yml index 756ad5fb68..665f74ac32 100644 --- a/.github/workflows/canonical-source-update.yml +++ b/.github/workflows/canonical-source-update.yml @@ -94,18 +94,17 @@ jobs: git fetch origin $UPDATE_BRANCH git checkout $UPDATE_BRANCH - # Sync with main - try rebase first, then merge + # Sync with main - try rebase first, then merge. A merge conflict + # fails the job: resolving it in favour of main would silently drop + # the changes already on this branch. if ! git rebase origin/main; then echo "::warning title=Rebase Failed::Attempting merge instead" git rebase --abort 2>/dev/null || true if ! git merge origin/main --no-edit; then + CONFLICTS=$(git diff --name-only --diff-filter=U | paste -sd ' ' -) git merge --abort 2>/dev/null || true - echo "::warning title=Merge Conflict::Auto-resolving conflicts in favor of main" - if ! git merge origin/main -X theirs --no-edit; then - git merge --abort 2>/dev/null || true - echo "::error title=Branch Sync Failed::Could not sync with main" - exit 1 - fi + echo "::error title=Branch Sync Conflict::${CONFLICTS:-git merge origin/main failed with no conflicted files}" + exit 1 fi fi else diff --git a/.github/workflows/enrich-tba-conferences.yml b/.github/workflows/enrich-tba-conferences.yml index 6f6e6303da..27ceee7d5c 100644 --- a/.github/workflows/enrich-tba-conferences.yml +++ b/.github/workflows/enrich-tba-conferences.yml @@ -81,18 +81,17 @@ jobs: git fetch origin $UPDATE_BRANCH git checkout $UPDATE_BRANCH - # Sync with main - try rebase first, then merge + # Sync with main - try rebase first, then merge. A merge conflict + # fails the job: resolving it in favour of main would silently drop + # the changes already on this branch. if ! git rebase origin/main; then echo "::warning title=Rebase Failed::Attempting merge instead" git rebase --abort 2>/dev/null || true if ! git merge origin/main --no-edit; then + CONFLICTS=$(git diff --name-only --diff-filter=U | paste -sd ' ' -) git merge --abort 2>/dev/null || true - echo "::warning title=Merge Conflict::Auto-resolving conflicts in favor of main" - if ! git merge origin/main -X theirs --no-edit; then - git merge --abort 2>/dev/null || true - echo "::error title=Branch Sync Failed::Could not sync with main" - exit 1 - fi + echo "::error title=Branch Sync Conflict::${CONFLICTS:-git merge origin/main failed with no conflicted files}" + exit 1 fi fi else diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6b3cc79bc..5769a28a57 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,6 +80,7 @@ You can enhance your entry with these optional fields: - `timezone`: IANA timezone (defaults to AoE if omitted) - `sponsor`: Sponsorship page URL - `finaid`: Financial aid information URL +- `coc`: Code of Conduct URL - `twitter`: Conference Twitter handle - `mastodon`: Conference Mastodon URL - `note`: Additional important information diff --git a/README.md b/README.md index bf5c34a34a..3617bb0bb3 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ To add or update a deadline: end: YYYY-MM-DD # End date of conference for calendar sponsor: link-to-sponsor.page # URL to Sponsorship page (Optional) finaid: link-to-finaid.page # URL to Financial Aid page (Optional) + coc: link-to-coc.page # URL to Code of Conduct (Optional) twitter: BestConfEver # Twitter handle of conference (Optional) mastodon: https://mastodon.social/@bconf # Mastodon handle of conference (Optional) sub: PY # Type of conference (see or add _data/types.yml) @@ -83,6 +84,7 @@ To add or update a deadline: | `start` | Start date of conference for calendar | `date` | ✔ | | `end` | End date of conference for calendar | `date` | ✔ | | `finaid` | URL to financial aid information | `str` | | +| `coc` | URL to the Code of Conduct | `str` | | | `sponsor` | URL to sponsorship opportunities | `str` | | | `twitter` | Twitter handle of conference | `str` | | | `mastodon` | Mastodon handle of conference | `str` | | diff --git a/tests/test_link_redirects.py b/tests/test_link_redirects.py new file mode 100644 index 0000000000..161d73da08 --- /dev/null +++ b/tests/test_link_redirects.py @@ -0,0 +1,412 @@ +"""Tests for redirects to a different site during link checking. + +A conference link that now redirects to another site (a lapsed domain taken +over by a casino, a series that moved to a barcamp platform) must not be +treated as healthy, swapped for a Wayback copy or replaced by the target. +check_link_availability keeps the original URL and records the redirect for +a person to review. Redirects within the same site behave as before, and so +do short links such as forms.gle that redirect to another site by design. +""" + +import json +import sys +from datetime import date +from datetime import datetime +from datetime import timedelta +from datetime import timezone +from pathlib import Path +from unittest.mock import patch + +import pytest +import responses +from freezegun import freeze_time + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +import sort_yaml +from tidy_conf import links + +# Recent enough to be checked (younger than five years) but in the past, so a +# healthy link does not trigger a Wayback save request. +TODAY = datetime.now(tz=timezone.utc).date() +PAST_START = TODAY - timedelta(days=60) + + +@pytest.fixture(autouse=True) +def _clean_redirect_records(monkeypatch): + """Start every test with no recorded redirects and no output file.""" + monkeypatch.delenv(links.REDIRECT_RECORDS_ENV, raising=False) + links.REDIRECT_RECORDS.clear() + yield + links.REDIRECT_RECORDS.clear() + + +def _redirect(source: str, target: str, status: int = 200, redirect_status: int = 301) -> None: + responses.add(responses.GET, source, status=redirect_status, headers={"Location": target}) + responses.add(responses.GET, target, status=status) + + +def _check(url: str, start: date = PAST_START, **context) -> str: + return links.check_link_availability(url, start, cache=set(), cache_archived=set(), **context) + + +class TestRegistrableDomain: + """Registrable domain is the last two labels, or three under a two-level public suffix.""" + + @pytest.mark.parametrize( + ("host", "expected"), + [ + ("pycon.de", "pycon.de"), + ("2027.pycon.de", "pycon.de"), + ("ug.pycon.org", "pycon.org"), + ("pycon.org.au", "pycon.org.au"), + ("2026.pycon.org.au", "pycon.org.au"), + ("pyconuk.co.uk", "pyconuk.co.uk"), + ("www.pydata.co.uk", "pydata.co.uk"), + ("pycon.jp", "pycon.jp"), + ("pycon.or.tz", "pycon.or.tz"), + ("cfp.pycon.org.il", "pycon.org.il"), + ("example.co.il", "example.co.il"), + ], + ) + def test_registrable_domain(self, host, expected): + assert links.registrable_domain(host) == expected + + def test_required_two_level_suffixes_present(self): + required = { + "co.uk", + "org.uk", + "ac.uk", + "gov.uk", + "com.au", + "org.au", + "net.au", + "com.br", + "org.br", + "co.jp", + "or.jp", + "ne.jp", + "com.ar", + "org.ar", + "co.za", + "org.za", + "co.nz", + "com.mx", + "com.tr", + "co.in", + "com.cn", + "com.sg", + "co.il", + "org.il", + "or.tz", + } + assert required <= links.TWO_LEVEL_PUBLIC_SUFFIXES + + +class TestIsDifferentSite: + """Hosts on different sites: different, not nested, different registrable domain.""" + + @pytest.mark.parametrize( + ("url", "other"), + [ + ("https://pythoncamp.de/", "https://barcamps.eu/pythoncamp/"), + ("https://pytennessee.org/", "https://infojudislotonline.com/"), + ("https://example.co.uk/", "https://other.co.uk/"), + ("https://pycon.org.au/", "https://pyconau.org.au/"), + # Two-level suffixes used by current entries: a lapsed domain that + # lands on another site under the same suffix is a different site. + ("https://pycon.or.tz/", "https://squatter.or.tz/"), + ("https://pycon.org.il/", "https://parked.org.il/"), + ("https://cfp.pycon.org.il/", "https://parked.org.il/"), + ("https://example.co.il/", "https://other.co.il/"), + ], + ) + def test_different_sites(self, url, other): + assert links.is_different_site(url, other) + + @pytest.mark.parametrize( + ("url", "other"), + [ + ("https://2026.pycon.it/", "https://2026.pycon.it/en"), + ("https://www.pycon.de/", "https://pycon.de/"), + ("http://pycon.de/", "https://pycon.de:443/"), + ("https://pycon.de/", "https://2027.pycon.de/"), + ("https://2027.pycon.de/", "https://pycon.de/"), + ("https://ug.pycon.org/", "https://africa.pycon.org/"), + ("https://a.example.co.uk/", "https://b.example.co.uk/"), + ("https://2026.pycon.org.au/", "https://pycon.org.au/"), + ("https://cfp.pycon.org.il/", "https://pycon.org.il/"), + ], + ) + def test_same_site(self, url, other): + assert not links.is_different_site(url, other) + + def test_missing_host_is_not_different(self): + assert not links.is_different_site("https://pycon.de/", "not a url") + + +class TestCrossSiteRedirect: + """A redirect to a different site keeps the original URL and is recorded.""" + + @responses.activate + def test_pythoncamp_to_barcamps_kept_and_recorded(self): + url = "https://pythoncamp.de/" + _redirect(url, "https://barcamps.eu/pythoncamp/") + + result = _check(url, conference="PythonCamp Germany", year=2026, field="link") + + assert result == url + assert len(links.REDIRECT_RECORDS) == 1 + record = links.REDIRECT_RECORDS[0] + assert record["source"] == "link-check" + assert record["kind"] == "redirect" + assert record["title"] == "PythonCamp Germany" + assert record["url"] == url + assert record["year"] == 2026 + assert record["payload"] == { + "conference": "PythonCamp Germany", + "year": 2026, + "field": "link", + "final_url": "https://barcamps.eu/pythoncamp/", + "status": 200, + } + + @responses.activate + def test_pytennessee_to_casino_kept_and_recorded(self): + url = "https://pytennessee.org/" + _redirect(url, "https://infojudislotonline.com/") + + result = _check(url, conference="PyTennessee", year=2025, field="link") + + assert result == url + assert [r["payload"]["final_url"] for r in links.REDIRECT_RECORDS] == ["https://infojudislotonline.com/"] + + @responses.activate + def test_no_wayback_lookup_or_save(self): + """Neither a future start (archive save) nor an error status (archive lookup) touches the Wayback Machine.""" + url = "https://pytennessee.org/" + _redirect(url, "https://infojudislotonline.com/", status=404) + + result = _check(url, start=TODAY + timedelta(days=60), field="cfp_link") + + assert result == url + assert [call.request.url for call in responses.calls] == [url, "https://infojudislotonline.com/"] + assert links.REDIRECT_RECORDS[0]["payload"]["status"] == 404 + assert links.REDIRECT_RECORDS[0]["payload"]["field"] == "cfp_link" + + @responses.activate + def test_without_context_uses_url_as_title(self): + url = "https://pythoncamp.de/" + _redirect(url, "https://barcamps.eu/pythoncamp/") + + assert _check(url) == url + record = links.REDIRECT_RECORDS[0] + assert record["title"] == url + assert record["year"] is None + assert record["payload"]["conference"] is None + assert record["payload"]["field"] is None + + +class TestExpectedRedirect: + """Short links redirect to another site by design and are not recorded.""" + + FORMS_URL = "https://forms.gle/iiFnW8oZ8p9do4TR7" + FORMS_TARGET = "https://docs.google.com/forms/d/e/1FAIpQLSe6/viewform?usp=send_form" + + @pytest.mark.parametrize( + ("url", "final_url"), + [ + ("https://forms.gle/iiFnW8oZ8p9do4TR7", "https://docs.google.com/forms/d/e/1FAIpQLSe6/viewform"), + ("https://forms.gle/abc", "https://accounts.google.com/ServiceLogin"), + ("https://lu.ma/pycon", "https://luma.com/pycon"), + ("https://bit.ly/pycon-cfp", "https://pretalx.com/pycon-2026/cfp"), + ("https://goo.gl/forms/abc", "https://docs.google.com/forms/d/abc/viewform"), + ("https://maps.app.goo.gl/abc", "https://www.google.com/maps/place/x"), + ("https://tinyurl.com/pycon", "https://example.org/"), + ], + ) + def test_expected(self, url, final_url): + assert links.is_expected_redirect(url, final_url) + + @pytest.mark.parametrize( + ("url", "final_url"), + [ + ("https://forms.gle/abc", "https://infojudislotonline.com/"), + ("https://lu.ma/pycon", "https://parked.example/"), + ("https://pythoncamp.de/", "https://barcamps.eu/pythoncamp/"), + ("https://docs.google.com/forms/d/abc/viewform", "https://forms.gle/abc"), + ("not a url", "https://docs.google.com/"), + ], + ) + def test_not_expected(self, url, final_url): + assert not links.is_expected_redirect(url, final_url) + + @responses.activate + def test_forms_gle_kept_and_not_recorded(self): + _redirect(self.FORMS_URL, self.FORMS_TARGET, redirect_status=302) + + result = _check(self.FORMS_URL, conference="PyCon Netherlands", year=2026, field="cfp_link") + + assert result == self.FORMS_URL + assert links.REDIRECT_RECORDS == [] + assert len(responses.calls) == 2 + + @responses.activate + def test_forms_gle_upcoming_saved_to_wayback(self, tmp_path, monkeypatch): + """An upcoming short link is still saved to the Wayback Machine like any healthy link.""" + monkeypatch.chdir(tmp_path) + _redirect(self.FORMS_URL, self.FORMS_TARGET, redirect_status=302) + save_url = "https://web.archive.org/save/" + self.FORMS_URL + responses.add(responses.GET, save_url, status=200) + + result = _check(self.FORMS_URL, start=TODAY + timedelta(days=60), field="cfp_link") + + assert result == self.FORMS_URL + assert links.REDIRECT_RECORDS == [] + assert [call.request.url for call in responses.calls][-1] == save_url + + @responses.activate + def test_forms_gle_to_other_site_recorded(self): + _redirect("https://forms.gle/abc", "https://infojudislotonline.com/", redirect_status=302) + + assert _check("https://forms.gle/abc", field="cfp_link") == "https://forms.gle/abc" + assert [r["payload"]["final_url"] for r in links.REDIRECT_RECORDS] == ["https://infojudislotonline.com/"] + + @responses.activate + def test_no_queue_line_for_short_link(self, tmp_path, monkeypatch): + out_file = tmp_path / "redirects.jsonl" + monkeypatch.setenv("DESK_QUEUE_OUT", str(out_file)) + _redirect(self.FORMS_URL, self.FORMS_TARGET, redirect_status=302) + _redirect("https://bit.ly/pycon-cfp", "https://pretalx.com/pycon-2026/cfp") + + _check(self.FORMS_URL, field="cfp_link") + _check("https://bit.ly/pycon-cfp", field="cfp_link") + + assert links.REDIRECT_RECORDS == [] + assert not out_file.exists() + + +class TestSameSiteRedirect: + """Redirects within the same site keep today's behaviour and are not recorded.""" + + @responses.activate + def test_same_host_redirect_followed(self): + _redirect("https://2026.pycon.it/", "https://2026.pycon.it/en") + + result = _check("https://2026.pycon.it/", conference="PyCon Italia", year=2026, field="link") + + assert result == "https://2026.pycon.it/en" + assert links.REDIRECT_RECORDS == [] + + @responses.activate + def test_same_registrable_domain_kept_as_before(self): + """ug.pycon.org to africa.pycon.org is a healthy link, as it was before.""" + _redirect("https://ug.pycon.org/", "https://africa.pycon.org/") + + result = _check("https://ug.pycon.org/", conference="PyCon Uganda", year=2026, field="link") + + assert result == "https://ug.pycon.org/" + assert links.REDIRECT_RECORDS == [] + assert len(responses.calls) == 2 + + +class TestRedirectQueueFile: + """With DESK_QUEUE_OUT set, every record is appended to that file as one JSON line.""" + + @freeze_time("2026-10-14 06:00:00") + @responses.activate + def test_json_lines_written(self, tmp_path, monkeypatch): + out_file = tmp_path / "queue" / "link-check.jsonl" + monkeypatch.setenv("DESK_QUEUE_OUT", str(out_file)) + _redirect("https://pythoncamp.de/", "https://barcamps.eu/pythoncamp/") + _redirect("https://pytennessee.org/", "https://infojudislotonline.com/", status=403) + + _check("https://pythoncamp.de/", conference="PythonCamp Germany", year=2026, field="link") + _check("https://pytennessee.org/", conference="PyTennessee", year=2025, field="sponsor") + + lines = out_file.read_text(encoding="utf-8").splitlines() + assert [json.loads(line) for line in lines] == [ + { + "source": "link-check", + "kind": "redirect", + "received": "2026-10-14T06:00:00Z", + "title": "PythonCamp Germany", + "url": "https://pythoncamp.de/", + "year": 2026, + "payload": { + "conference": "PythonCamp Germany", + "year": 2026, + "field": "link", + "final_url": "https://barcamps.eu/pythoncamp/", + "status": 200, + }, + }, + { + "source": "link-check", + "kind": "redirect", + "received": "2026-10-14T06:00:00Z", + "title": "PyTennessee", + "url": "https://pytennessee.org/", + "year": 2025, + "payload": { + "conference": "PyTennessee", + "year": 2025, + "field": "sponsor", + "final_url": "https://infojudislotonline.com/", + "status": 403, + }, + }, + ] + assert [json.loads(line) for line in lines] == links.REDIRECT_RECORDS + + @responses.activate + def test_existing_file_appended(self, tmp_path, monkeypatch): + out_file = tmp_path / "redirects.jsonl" + out_file.write_text('{"earlier": true}\n', encoding="utf-8") + monkeypatch.setenv("DESK_QUEUE_OUT", str(out_file)) + _redirect("https://pythoncamp.de/", "https://barcamps.eu/pythoncamp/") + + _check("https://pythoncamp.de/") + + lines = out_file.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + assert json.loads(lines[0]) == {"earlier": True} + assert json.loads(lines[1])["url"] == "https://pythoncamp.de/" + + @responses.activate + def test_no_file_without_env(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _redirect("https://pythoncamp.de/", "https://barcamps.eu/pythoncamp/") + + _check("https://pythoncamp.de/") + + assert len(links.REDIRECT_RECORDS) == 1 + assert not list(tmp_path.glob("*.jsonl")) + + +class TestCheckLinksContext: + """sort_yaml.check_links passes the conference, year and field to the link check.""" + + @patch("sort_yaml.get_cache", return_value=(set(), set())) + @patch("sort_yaml.check_link_availability", side_effect=lambda url, *_args, **_kwargs: url) + def test_context_passed(self, mock_check_link, mock_get_cache): + data = [ + { + "conference": "PythonCamp Germany", + "year": 2026, + "link": "https://pythoncamp.de/", + "sponsor": "https://pythoncamp.de/sponsors/", + "start": PAST_START, + }, + ] + + sort_yaml.check_links(data) + + contexts = [ + (call.args[0], call.kwargs["conference"], call.kwargs["year"], call.kwargs["field"]) + for call in mock_check_link.call_args_list + ] + assert contexts == [ + ("https://pythoncamp.de/", "PythonCamp Germany", 2026, "link"), + ("https://pythoncamp.de/sponsors/", "PythonCamp Germany", 2026, "sponsor"), + ] diff --git a/tests/test_redundant_links.py b/tests/test_redundant_links.py index d47d7af1b8..5e4d6e342a 100644 --- a/tests/test_redundant_links.py +++ b/tests/test_redundant_links.py @@ -120,3 +120,62 @@ def test_entry_without_sub_fields_untouched(self): data = [{"conference": "Plain Conf", "year": 2026, "link": "https://plain.conf/"}] result = drop_redundant_link_fields(data) assert result[0] == {"conference": "Plain Conf", "year": 2026, "link": "https://plain.conf/"} + + +class TestDropBareSiteRoot: + """A sub-page link that is only the bare root of the main link's site is dropped.""" + + @staticmethod + def _entry(link: str, **fields: str) -> list: + return [{"conference": "PyCon DE & PyData", "year": 2027, "link": link, **fields}] + + def test_parent_domain_root_dropped(self): + """The series site root next to a year subdomain says nothing about the CfP.""" + result = drop_redundant_link_fields(self._entry("https://2027.pycon.de/", cfp_link="https://pycon.de/")) + assert "cfp_link" not in result[0] + assert result[0]["link"] == "https://2027.pycon.de/" + + def test_parent_domain_root_with_www_and_no_slash_dropped(self): + result = drop_redundant_link_fields( + self._entry("https://2027.pycon.de/", sponsor="http://www.pycon.de", finaid="https://pycon.de/"), + ) + assert "sponsor" not in result[0] + assert "finaid" not in result[0] + + def test_same_host_root_dropped_when_link_is_a_sub_page(self): + result = drop_redundant_link_fields(self._entry("https://pycon.org/2026/", cfp_link="https://www.pycon.org/")) + assert "cfp_link" not in result[0] + + def test_sub_page_on_parent_domain_kept(self): + result = drop_redundant_link_fields( + self._entry("https://2027.pycon.de/", sponsor="https://pycon.de/sponsoring/"), + ) + assert result[0]["sponsor"] == "https://pycon.de/sponsoring/" + + def test_other_site_kept(self): + result = drop_redundant_link_fields(self._entry("https://2027.pycon.de/", cfp_link="https://pretalx.com/x/cfp")) + assert result[0]["cfp_link"] == "https://pretalx.com/x/cfp" + + def test_other_site_root_kept(self): + result = drop_redundant_link_fields(self._entry("https://2027.pycon.de/", cfp_link="https://pretalx.com/")) + assert result[0]["cfp_link"] == "https://pretalx.com/" + + def test_year_subdomain_root_next_to_series_link_kept(self): + """The reverse direction is more specific than the series site, so it stays.""" + result = drop_redundant_link_fields(self._entry("https://pycon.de/", cfp_link="https://2027.pycon.de/")) + assert result[0]["cfp_link"] == "https://2027.pycon.de/" + + def test_anchor_on_parent_domain_kept(self): + result = drop_redundant_link_fields(self._entry("https://2027.pycon.de/", cfp_link="https://pycon.de/#cfp")) + assert result[0]["cfp_link"] == "https://pycon.de/#cfp" + + def test_query_on_parent_domain_kept(self): + result = drop_redundant_link_fields( + self._entry("https://2027.pycon.de/", cfp_link="https://pycon.de/?page=cfp"), + ) + assert result[0]["cfp_link"] == "https://pycon.de/?page=cfp" + + def test_lookalike_host_kept(self): + """A host that only ends in the same letters is not a parent domain.""" + result = drop_redundant_link_fields(self._entry("https://2027.pycon.de/", cfp_link="https://con.de/")) + assert result[0]["cfp_link"] == "https://con.de/" diff --git a/tests/test_schema_validation.py b/tests/test_schema_validation.py index 5ada95ea0a..7f400ad93f 100644 --- a/tests/test_schema_validation.py +++ b/tests/test_schema_validation.py @@ -11,11 +11,13 @@ sys.path.insert(0, str(Path(__file__).parent)) sys.path.append(str(Path(__file__).parent.parent / "utils")) +import sort_yaml from hypothesis_strategies import HYPOTHESIS_AVAILABLE from hypothesis_strategies import valid_latitude from hypothesis_strategies import valid_longitude from tidy_conf.schema import Conference from tidy_conf.schema import Location +from tidy_conf.schema import get_schema class TestConferenceSchema: @@ -365,6 +367,49 @@ def test_timezone_accepted(self, sample_conference): assert conf.timezone == tz +class TestCodeOfConductField: + """The optional coc field holds a URL to the conference's Code of Conduct.""" + + def test_coc_optional(self, sample_conference): + conf = Conference(**sample_conference) + assert conf.coc is None + assert "coc" not in conf.model_dump(exclude_none=True) + + def test_coc_accepts_url(self, sample_conference): + sample_conference["coc"] = "https://test.pycon.org/code-of-conduct/" + conf = Conference(**sample_conference) + assert str(conf.coc) == "https://test.pycon.org/code-of-conduct/" + # Serialised as a plain string, like the other URL fields + assert conf.model_dump()["coc"] == "https://test.pycon.org/code-of-conduct/" + + @pytest.mark.parametrize("value", ["not-a-url", "test.pycon.org/coc", "ftp://test.pycon.org/coc"]) + def test_coc_rejects_non_url(self, sample_conference, value): + sample_conference["coc"] = value + with pytest.raises(ValidationError) as exc_info: + Conference(**sample_conference) + assert {error["loc"][0] for error in exc_info.value.errors()} == {"coc"} + + def test_coc_key_order_after_finaid(self, sample_conference): + """Key order comes from utils/schema.yml (order_keywords) and the model (YAML output).""" + schema_keys = get_schema().columns.tolist() + assert schema_keys[schema_keys.index("finaid") + 1] == "coc" + + model_keys = list(Conference.model_fields) + assert model_keys[model_keys.index("finaid") + 1] == "coc" + + entry = { + "coc": "https://test.pycon.org/coc/", + **sample_conference, + "finaid": "https://test.pycon.org/finaid/", + "twitter": "pycontest", + } + ordered = list(sort_yaml.order_keywords(entry)) + assert ordered[ordered.index("finaid") + 1] == "coc" + + dumped = list(Conference(**entry).model_dump(exclude_none=True)) + assert dumped[dumped.index("finaid") + 1] == "coc" + + class TestSchemaRegressions: """Regression tests for schema validation bugs.""" diff --git a/utils/schema.yml b/utils/schema.yml index 9443815202..287908603e 100644 --- a/utils/schema.yml +++ b/utils/schema.yml @@ -15,6 +15,7 @@ end: YYYY-MM-DD # End date of conference for calendar sponsor: link-to-sponsor.page # URL to Sponsorship page (Optional) finaid: link-to-finaid.page # URL to Financial Aid page (Optional) + coc: link-to-coc.page # URL to Code of Conduct (Optional) twitter: BestConfEver # Twitter handle of conference (Optional) mastodon: https://mastodon.social/@bconf # Mastodon handle of conference (Optional) bluesky: https://bsky.app/@bconf # Bluesky handle of conference (Optional) diff --git a/utils/sort_yaml.py b/utils/sort_yaml.py index a6254b7bc7..b3bea93876 100644 --- a/utils/sort_yaml.py +++ b/utils/sort_yaml.py @@ -232,7 +232,15 @@ def check_links(data): for i, q in tqdm(enumerate(sorted(data, key=operator.itemgetter("year"), reverse=True)), total=len(data)): for key in ("link", "cfp_link", "sponsor", "finaid"): if key in q: - new_link = check_link_availability(q[key], q["start"], cache=cache, cache_archived=cache_archived) + new_link = check_link_availability( + q[key], + q["start"], + cache=cache, + cache_archived=cache_archived, + conference=q.get("conference"), + year=q.get("year"), + field=key, + ) parsed_url = urlparse(new_link) if q[key] != new_link and parsed_url.hostname and parsed_url.hostname.endswith(".archive.org"): time.sleep(0.5) diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index efa88d9405..b125f7e607 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -1,3 +1,5 @@ +import json +import os import re import time from datetime import datetime @@ -13,9 +15,70 @@ # A cfp_link/sponsor/finaid that is just the homepage again carries no # information (e.g. "cfp_link: https://2027.pycon.de/" on the PyCon DE entry # whose link is https://2027.pycon.de/) - it should be a different page, a -# subdomain, or at least a #anchor on the homepage. +# subdomain, or at least a #anchor on the homepage. The bare root of the +# series site (e.g. "cfp_link: https://pycon.de/" next to +# "link: https://2027.pycon.de/") carries no information either. REDUNDANT_LINK_FIELDS = ("cfp_link", "sponsor", "finaid") +# Public suffixes with two labels, under which a registrable domain has three +# labels (pycon.org.au rather than org.au). This is not the full Public Suffix +# List, only the suffixes conference sites are likely to use. +TWO_LEVEL_PUBLIC_SUFFIXES = frozenset( + { + "ac.uk", + "co.uk", + "gov.uk", + "org.uk", + "com.au", + "net.au", + "org.au", + "com.br", + "org.br", + "co.jp", + "ne.jp", + "or.jp", + "com.ar", + "org.ar", + "co.za", + "org.za", + "co.nz", + "com.mx", + "com.tr", + "co.in", + "com.cn", + "com.sg", + "co.il", + "org.il", + "or.tz", + }, +) + +# Hosts whose links always redirect to another site, mapped to the registrable +# domains those redirects are expected to land on. forms.gle is the Google +# Forms short link (it lands on docs.google.com) and lu.ma is the old Luma +# domain, which now redirects to luma.com. +EXPECTED_REDIRECT_TARGETS = { + "forms.gle": frozenset({"google.com"}), + "lu.ma": frozenset({"luma.com"}), +} + +# General URL shorteners. A short link lands wherever it was made to point, so +# a redirect from one to another site says nothing about the link. +URL_SHORTENERS = frozenset({"bit.ly", "goo.gl", "tinyurl.com"}) + +# Redirects to a different site found by check_link_availability. A link that +# now lands on another site (a lapsed domain, a parked page, a new home for +# the series) needs a person to look at it, so the link is kept as it is and +# the redirect is recorded here instead. +REDIRECT_RECORDS: list[dict] = [] + +# When this environment variable names a file, every redirect record is also +# appended to it as one JSON object per line, for a later job to pick up. The +# lines are not complete queue items: the queue writer assigns each item its +# "id" and the series key of the file it goes in when it adds the line, so +# both are defined in one place. +REDIRECT_RECORDS_ENV = "DESK_QUEUE_OUT" + def normalize_url_pointer(url: str) -> tuple: """Normalize a URL for same-pointer comparison. @@ -40,12 +103,243 @@ def normalize_url_pointer(url: str) -> tuple: return (netloc, parsed.path.rstrip("/"), parsed.params, parsed.query, parsed.fragment) +def site_host(url: str) -> str: + """Return the host of a URL in lower case, without port or a "www." prefix. + + Parameters + ---------- + url : str + URL to take the host from + + Returns + ------- + str + Host name, or an empty string when the URL has none + """ + host = urlparse(str(url).strip()).hostname or "" + return host.lower().removeprefix("www.") + + +def is_parent_domain(parent: str, host: str) -> bool: + """Check whether ``parent`` is a parent domain of ``host``. + + A public suffix such as ``de`` or ``co.uk`` does not count as a parent. + + Parameters + ---------- + parent : str + Candidate parent host, e.g. ``pycon.de`` + host : str + Host to test, e.g. ``2027.pycon.de`` + + Returns + ------- + bool + True when ``host`` is a subdomain of ``parent`` + """ + if "." not in parent or parent in TWO_LEVEL_PUBLIC_SUFFIXES: + return False + return host.endswith("." + parent) + + +def registrable_domain(host: str) -> str: + """Return the registrable domain of a host. + + This is the last two labels, or the last three when the last two are a + known two-level public suffix (see ``TWO_LEVEL_PUBLIC_SUFFIXES``). + + Parameters + ---------- + host : str + Host name, e.g. ``2027.pycon.de`` + + Returns + ------- + str + Registrable domain, e.g. ``pycon.de`` + """ + labels = host.lower().strip(".").split(".") + if len(labels) >= 3 and ".".join(labels[-2:]) in TWO_LEVEL_PUBLIC_SUFFIXES: + return ".".join(labels[-3:]) + return ".".join(labels[-2:]) + + +def is_different_site(url: str, other: str) -> bool: + """Check whether two URLs belong to different sites. + + They are different sites when the hosts differ after dropping "www.", + neither host is a subdomain of the other, and their registrable domains + differ. ``ug.pycon.org`` and ``africa.pycon.org`` are the same site; + ``pythoncamp.de`` and ``barcamps.eu`` are not. + + Parameters + ---------- + url : str + First URL + other : str + Second URL + + Returns + ------- + bool + True when the URLs are on different sites; False when either has no host + """ + host, other_host = site_host(url), site_host(other) + if not host or not other_host or host == other_host: + return False + if is_parent_domain(host, other_host) or is_parent_domain(other_host, host): + return False + return registrable_domain(host) != registrable_domain(other_host) + + +def is_expected_redirect(url: str, final_url: str) -> bool: + """Check whether a redirect to another site is what the link is for. + + A link on a general URL shortener (see ``URL_SHORTENERS``) may land + anywhere. A link on a host listed in ``EXPECTED_REDIRECT_TARGETS`` must + land on one of the registrable domains listed for it: ``forms.gle`` on + ``google.com``, ``lu.ma`` on ``luma.com``. Any other redirect is not + expected. + + Parameters + ---------- + url : str + URL as stored in the data + final_url : str + URL the redirect chain ended on + + Returns + ------- + bool + True when the redirect is one the link is expected to make + """ + origin = registrable_domain(site_host(url)) + if origin in URL_SHORTENERS: + return True + targets = EXPECTED_REDIRECT_TARGETS.get(origin, frozenset()) + return registrable_domain(site_host(final_url)) in targets + + +def record_redirect( + url: str, + final_url: str, + status: int, + conference: str | None = None, + year: int | None = None, + field: str | None = None, +) -> dict: + """Record a redirect to a different site for review. + + The record is appended to ``REDIRECT_RECORDS`` and, when the + ``DESK_QUEUE_OUT`` environment variable names a file, written to that + file as one JSON line. It holds the fields of a queue item with source + "link-check" and kind "redirect", except "id": the queue writer assigns + the id, and the series key of the file the item goes in, when it adds + the line. + + Parameters + ---------- + url : str + URL as stored in the data + final_url : str + URL the redirect chain ended on + status : int + HTTP status code of the final response + conference : str | None + Conference name, when known + year : int | None + Conference year, when known + field : str | None + Data field holding the URL (link, cfp_link, sponsor or finaid), when known + + Returns + ------- + dict + The record that was stored + """ + record = { + "source": "link-check", + "kind": "redirect", + "received": datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "title": conference or url, + "url": url, + "year": year, + "payload": { + "conference": conference, + "year": year, + "field": field, + "final_url": final_url, + "status": status, + }, + } + REDIRECT_RECORDS.append(record) + + out_path = os.environ.get(REDIRECT_RECORDS_ENV) + if out_path: + out_file = Path(out_path) + out_file.parent.mkdir(parents=True, exist_ok=True) + with out_file.open("a", encoding="utf-8", newline="\n") as f: + f.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + return record + + +def is_bare_site_root(url: str) -> bool: + """Check whether a URL is the bare root of a site. + + A bare root has an empty or "/" path and no params, query or #fragment. + + Parameters + ---------- + url : str + URL to check + + Returns + ------- + bool + True when the URL points at nothing more specific than the site itself + """ + parsed = urlparse(str(url).strip()) + if not parsed.hostname or parsed.path not in {"", "/"}: + return False + return not (parsed.params or parsed.query or parsed.fragment) + + +def is_redundant_site_root(value: str, link: str) -> bool: + """Check whether a sub-page link is only the bare root of the main link's site. + + The site is the same when the hosts are equal after dropping "www.", or + when the sub-page host is a parent domain of the main link's host + (``pycon.de`` for ``2027.pycon.de``). The reverse is not redundant: a + year subdomain is more specific than the series site. + + Parameters + ---------- + value : str + Sub-page link (cfp_link, sponsor or finaid) + link : str + Main conference link + + Returns + ------- + bool + True when ``value`` adds nothing over ``link`` + """ + if not is_bare_site_root(value): + return False + value_host, link_host = site_host(value), site_host(link) + if not value_host or not link_host: + return False + return value_host == link_host or is_parent_domain(value_host, link_host) + + def drop_redundant_link_fields(data: list) -> list: """Drop sub-page link fields that are the same pointer as the main link. Goes through cfp_link/sponsor/finaid on each conference and removes any that just repeat the conference's main link. This catches - upstream sources that fill every URL column with the homepage. + upstream sources that fill every URL column with the homepage. A field + that is only the bare root of the same site (``https://pycon.de/`` next + to ``https://2027.pycon.de/``) is dropped as well. Parameters ---------- @@ -66,12 +360,16 @@ def drop_redundant_link_fields(data: list) -> list: base = normalize_url_pointer(link) for field in REDUNDANT_LINK_FIELDS: value = q.get(field) - if value and normalize_url_pointer(value) == base: - tqdm.write( - f"Dropping redundant '{field}' from {q.get('conference')} {q.get('year')}: " - f"'{value}' is the same pointer as the main link", - ) - del q[field] + if not value: + continue + if normalize_url_pointer(value) == base: + reason = "is the same pointer as the main link" + elif is_redundant_site_root(value, link): + reason = f"is only the root of the same site as the main link '{link}'" + else: + continue + tqdm.write(f"Dropping redundant '{field}' from {q.get('conference')} {q.get('year')}: '{value}' {reason}") + del q[field] return data @@ -98,7 +396,16 @@ def get_cache(): return cache, cache_archived -def check_link_availability(url, start, cache=None, cache_archived=None): +def check_link_availability( + url, + start, + cache=None, + cache_archived=None, + *, + conference: str | None = None, + year: int | None = None, + field: str | None = None, +) -> str: """Checks if a URL is available. If not, tries to retrieve an archived version from the Wayback Machine. @@ -106,9 +413,39 @@ def check_link_availability(url, start, cache=None, cache_archived=None): Automatically redirects to the final URL if it was redirected within the same domain. + A redirect that ends on a different site (see ``is_different_site``) is not + treated as healthy: the original URL is returned unchanged, with no Wayback + lookup, and the redirect is recorded with ``record_redirect`` for review. + A short link that lands where it is expected to (see ``is_expected_redirect``), + such as forms.gle on docs.google.com, is not recorded: it is checked like + any other link, so on a 200 the original URL is kept and, when the + conference is still to come, saved to the Wayback Machine. + Automatically caches old (5+ years) URLs. - Note: Some URLs may not be available or archived. These return None and are logged. + Note: Some URLs may not be available or archived. These return the original URL and are logged. + + Parameters + ---------- + url : str + URL to check + start : datetime.date + Start date of the conference the URL belongs to + cache : set | None + URLs known to have no archived version, read from disk when None + cache_archived : set | None + URLs already archived, read from disk when None + conference : str | None + Conference name, only used to describe a recorded redirect + year : int | None + Conference year, only used to describe a recorded redirect + field : str | None + Data field holding the URL, only used to describe a recorded redirect + + Returns + ------- + str + The URL to keep: the original, a same-domain redirect target, or an archived copy """ # If it's archived already, return the URL if url.startswith(("https://web.archive.org", "http://web.archive.org")): @@ -130,6 +467,23 @@ def check_link_availability(url, start, cache=None, cache_archived=None): try: response = requests.get(url, allow_redirects=True, timeout=10) final_url = response.url + # A redirect to another site is not a healthy link: keep the URL as it + # is and leave the decision to a person. Short links such as + # forms.gle redirect to another site by design and are left out. + if is_different_site(url, final_url) and not is_expected_redirect(url, final_url): + tqdm.write( + f"URL {url} was redirected to a different site: {final_url} " + f"(status code: {response.status_code}). Keeping the original URL for review.", + ) + record_redirect( + url, + final_url, + response.status_code, + conference=conference, + year=year, + field=field, + ) + return url # Check if the final URL is within the same domain as the original URL if urlparse(url).netloc == urlparse(final_url).netloc and final_url != url: # Don't cache the redirect if it suddenly has a query string diff --git a/utils/tidy_conf/schema.py b/utils/tidy_conf/schema.py index a5b8fcd7ff..add24a4c1b 100644 --- a/utils/tidy_conf/schema.py +++ b/utils/tidy_conf/schema.py @@ -78,6 +78,7 @@ class Conference(BaseModel): end: PythonDate sponsor: HttpUrl | None = None finaid: HttpUrl | None = None + coc: HttpUrl | None = None twitter: str | None = None mastodon: HttpUrl | None = None bluesky: str | None = None @@ -154,7 +155,7 @@ def validate_title(cls, v): return re.sub(r"\b(19|20)\d{2}\b", "", v).strip() return v - @field_serializer("link", "cfp_link", "sponsor", "finaid", "mastodon", "youtube") + @field_serializer("link", "cfp_link", "sponsor", "finaid", "coc", "mastodon", "youtube") def ser_url(self, value): return str(value) diff --git a/utils/tidy_conf/validation.py b/utils/tidy_conf/validation.py index 46a9dd1ee1..28ce669394 100644 --- a/utils/tidy_conf/validation.py +++ b/utils/tidy_conf/validation.py @@ -25,6 +25,7 @@ "sub", "sponsor", "finaid", + "coc", "tutorial_deadline", "workshop_deadline", "timezone",