diff --git a/tests/test_cfp_candidates.py b/tests/test_cfp_candidates.py new file mode 100644 index 0000000000..cf70fdcd89 --- /dev/null +++ b/tests/test_cfp_candidates.py @@ -0,0 +1,220 @@ +"""Tests for cfp_candidates: archived series that have no entry in conferences.yml. + +A series counts as tracked when its name or any alt_name matches the name or +alt_name of an entry in conferences.yml, after normalize_conference_name and +the titles.yml mappings, ignoring case, accents and whitespace. +""" + +import json +import sys +from datetime import date +from pathlib import Path + +import pytest +import yaml +from freezegun import freeze_time + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from cfp_candidates import entry_names +from cfp_candidates import find_candidates +from cfp_candidates import load_data +from cfp_candidates import main +from cfp_candidates import series_key + +# Real entries: archive.yml at 3d34de3, conferences.yml before 405fc07 and 219d0b0, +# when the upcoming editions were spelt differently from their archived ones. +ARCHIVE_XTREME_PYTHON = { + "conference": "Xtreme Python", + "year": 2025, + "link": "https://xtremepython.dev/2025/", + "cfp": "2025-09-20 23:59:00", + "place": "Online", + "start": date(2025, 11, 18), + "end": date(2025, 11, 18), + "sponsor": "https://xtremepython.dev/2025/sponsors/", + "sub": "PY", +} +CURRENT_XTREMEPYTHON = { + "conference": "XtremePython", + "year": 2026, + "link": "https://xtremepython.dev/", + "cfp": "TBA", + "place": "Online", + "start": date(2026, 11, 17), + "end": date(2026, 11, 17), + "sub": "PY", +} +ARCHIVE_PYHEP_DEV = { + "conference": "PyHEP.dev", + "alt_name": "PyHEP", + "year": 2025, + "link": "https://indico.cern.ch/event/1515852/", + "cfp": "2025-07-08 23:59:00", + "place": "Seattle, USA", + "start": date(2025, 7, 14), + "end": date(2025, 7, 17), + "sub": "SCIPY", +} +CURRENT_PYHEP = { + "conference": "PyHEP", + "year": 2026, + "link": "https://indico.cern.ch/category/7971/", + "cfp": "TBA", + "timezone": "Europe/Zurich", + "place": "Geneva, Switzerland", + "start": date(2026, 9, 7), + "end": date(2026, 9, 9), + "sub": "PY", +} + + +def edition(conference, year, **fields): + """Build a minimal archived edition.""" + return {"conference": conference, "year": year, "link": f"https://example.org/{year}/", **fields} + + +def names(candidates): + return [c["conference"] for c in candidates] + + +class TestSeriesKey: + @pytest.mark.parametrize( + ("left", "right"), + [ + ("Xtreme Python", "XtremePython"), + ("PyDay Copiapó", "PyDay Copiapo"), + ("PyCon mini Tokai", "PyCon Mini Tokai"), + ("PyCon NZ", "Kiwi PyCon"), + ("EuroPython Conference", "EuroPython"), + ("PyCon Greece 2027", "PyCon Greece"), + ], + ) + def test_same_series(self, left, right): + assert series_key(left) == series_key(right) + + @pytest.mark.parametrize(("left", "right"), [("PyHEP.dev", "PyHEP"), ("PyData Berlin", "PyData Paris")]) + def test_different_series(self, left, right): + assert series_key(left) != series_key(right) + + +class TestEntryNames: + def test_conference_then_alt_name(self): + assert entry_names({"conference": "PyHEP.dev", "alt_name": "PyHEP"}) == ["PyHEP.dev", "PyHEP"] + + def test_skips_missing_and_blank(self): + assert entry_names({"conference": "PyCon", "alt_name": " "}) == ["PyCon"] + + +class TestFindCandidates: + def test_xtreme_python_spelling_is_tracked(self): + """Archive "Xtreme Python" is tracked by an upcoming "XtremePython".""" + assert find_candidates([CURRENT_XTREMEPYTHON], [ARCHIVE_XTREME_PYTHON], current_year=2026) == [] + + def test_xtreme_python_spelling_reversed(self): + archive = [{**CURRENT_XTREMEPYTHON, "year": 2025}] + current = [{**ARCHIVE_XTREME_PYTHON, "year": 2026}] + assert find_candidates(current, archive, current_year=2026) == [] + + def test_pyhep_dev_alt_name_is_tracked(self): + """Archive "PyHEP.dev" (alt_name PyHEP) is tracked by an upcoming "PyHEP".""" + assert find_candidates([CURRENT_PYHEP], [ARCHIVE_PYHEP_DEV], current_year=2026) == [] + + def test_pyhep_tracked_through_upcoming_alt_name(self): + """Archive "PyHEP" is tracked by an upcoming "PyHEP.dev" whose alt_name is PyHEP.""" + current = [{**ARCHIVE_PYHEP_DEV, "year": 2026}] + archive = [{**CURRENT_PYHEP, "year": 2025}] + assert find_candidates(current, archive, current_year=2026) == [] + + def test_reported_without_the_other_spelling(self): + candidates = find_candidates([], [ARCHIVE_XTREME_PYTHON, ARCHIVE_PYHEP_DEV], current_year=2026) + assert names(candidates) == ["PyHEP.dev", "Xtreme Python"] + + def test_punctuation_is_not_folded(self): + """Without the alt_name, "PyHEP.dev" and "PyHEP" stay separate series.""" + archive = [{k: v for k, v in ARCHIVE_PYHEP_DEV.items() if k != "alt_name"}] + assert names(find_candidates([CURRENT_PYHEP], archive, current_year=2026)) == ["PyHEP.dev"] + + def test_accents_and_case(self): + current = [edition("PyDay Copiapo", 2026)] + archive = [edition("PyDay Copiapó", 2025), edition("pycon mini tokai", 2025)] + assert names(find_candidates(current, archive, current_year=2026)) == ["pycon mini tokai"] + + def test_titles_mapping(self): + """titles.yml maps "PyCon NZ" to "Kiwi PyCon".""" + assert find_candidates([edition("Kiwi PyCon", 2026)], [edition("PyCon NZ", 2025)], current_year=2026) == [] + + def test_any_archived_alt_name_counts(self): + """The alt_name of an older edition still tracks the series, so a rename is not lost.""" + archive = [edition("PyCon Old Name", 2024, alt_name="PyCon New Name"), edition("PyCon Old Name", 2025)] + current = [edition("PyCon New Name", 2026)] + assert find_candidates(current, archive, current_year=2026) == [] + + def test_host_alt_name_hides_the_series(self): + """Known cost of the rule above: an old edition named after its host hides the series. + + The 2023 edition was hosted by PyCon Japan and carries it as alt_name, so the + upcoming PyCon Japan tracks PyCon Asia Pacific although it has no entry of its own. + """ + archive = [ + edition("PyCon Asia Pacific", 2025, alt_name="PyCon Philippines"), + edition("PyCon Asia Pacific", 2023, alt_name="PyCon Japan"), + ] + current = [edition("PyCon Japan", 2027)] + assert find_candidates(current, archive, current_year=2026) == [] + assert names(find_candidates(current, archive[:1], current_year=2026)) == ["PyCon Asia Pacific"] + + def test_spellings_grouped_into_one_series(self): + archive = [edition("PyCon Mini Tokai", 2024), edition("PyCon mini Tokai", 2025)] + candidates = find_candidates([], archive, current_year=2026) + assert [(c["conference"], c["latest_year"]) for c in candidates] == [("PyCon mini Tokai", 2025)] + + def test_lookback(self): + archive = [edition("Old Conf", 2023), edition("Recent Conf", 2024)] + assert names(find_candidates([], archive, current_year=2026)) == ["Recent Conf"] + assert names(find_candidates([], archive, current_year=2026, lookback=3)) == ["Recent Conf", "Old Conf"] + + def test_carries_fields_of_latest_edition(self): + archive = [ + edition("PyCon Example", 2024, place="Old Town", sub="PY"), + edition("PyCon Example", 2025, place="New Town", sub="PY", cfp_link="https://example.org/cfp"), + ] + assert find_candidates([], archive, current_year=2026) == [ + { + "conference": "PyCon Example", + "latest_year": 2025, + "link": "https://example.org/2025/", + "place": "New Town", + "sub": "PY", + "cfp_link": "https://example.org/cfp", + }, + ] + + def test_order_newest_first_then_name(self): + archive = [edition("B Conf", 2025), edition("A Conf", 2025), edition("C Conf", 2026)] + assert names(find_candidates([], archive, current_year=2026)) == ["C Conf", "A Conf", "B Conf"] + + +class TestCli: + @freeze_time("2026-10-14") + def test_output_shape(self, tmp_path, monkeypatch, capsys): + data = tmp_path / "_data" + data.mkdir() + (data / "conferences.yml").write_text(yaml.safe_dump([CURRENT_XTREMEPYTHON]), encoding="utf-8") + (data / "archive.yml").write_text( + yaml.safe_dump([ARCHIVE_XTREME_PYTHON, edition("PyCon Example", 2025)]), + encoding="utf-8", + ) + monkeypatch.setattr(sys, "argv", ["cfp_candidates.py", "--base", str(tmp_path)]) + + main() + + output = json.loads(capsys.readouterr().out) + assert output == { + "generated": "2026", + "count": 1, + "candidates": [{"conference": "PyCon Example", "latest_year": 2025, "link": "https://example.org/2025/"}], + } + + def test_load_data_missing_files(self, tmp_path): + assert load_data(tmp_path) == {"conferences": [], "archive": [], "legacy": []} diff --git a/tests/test_desk_worklist.py b/tests/test_desk_worklist.py new file mode 100644 index 0000000000..cb4aa4d881 --- /dev/null +++ b/tests/test_desk_worklist.py @@ -0,0 +1,953 @@ +"""Tests for desk_worklist: the deadline desk's deterministic daily worklist.""" + +import hashlib +import itertools +import json +import socket +import sys +from datetime import date +from datetime import timedelta +from pathlib import Path + +import pytest +import yaml +from freezegun import freeze_time + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from desk_worklist import build_worklist +from desk_worklist import clean_watch_title +from desk_worklist import load_queue +from desk_worklist import main +from desk_worklist import rotation_slot + +TODAY = date(2026, 10, 14) # a Wednesday, weekday() == 2 +GENERATED = "2026-10-14T06:00:00Z" + + +def conf(name, year, **fields): + """Build an entry whose dated cfp is far from today, so no deadline check fires.""" + entry = { + "conference": name, + "year": year, + "link": "https://example.org/", + "cfp": f"{year}-12-01 23:59:00", + "place": "Online", + "start": date(year, 12, 10), + "end": date(year, 12, 11), + "sub": "PY", + } + entry.update(fields) + return entry + + +def dataset(conferences=(), archive=(), legacy=()): + return {"conferences": list(conferences), "archive": list(archive), "legacy": list(legacy)} + + +def queue_item( + kind="signal", + title="", + *, + source="changedetection", + year=None, + payload=None, + file="queue/example.json", + series=None, + item_id=None, + received="2026-10-14T06:00:00Z", +): + """Build a queue item the way load_queue returns it.""" + if item_id is None: + item_id = hashlib.sha1(f"{source}|{kind}|{title}|{file}".encode(), usedforsecurity=False).hexdigest() + return { + "id": item_id, + "source": source, + "kind": kind, + "received": received, + "title": title, + "url": "https://example.org/", + "year": year, + "payload": payload or {}, + "file": file, + "series": series, + } + + +def build(data, queue=(), today=TODAY, **kwargs): + return build_worklist(data, list(queue), today, generated=GENERATED, **kwargs) + + +def items(worklist, reason=None): + """Return the items without the scout item, optionally only those with a reason.""" + found = [item for item in worklist["items"] if item["reasons"] != ["scout"]] + if reason is not None: + found = [item for item in found if reason in item["reasons"]] + return found + + +def only(worklist): + found = items(worklist) + assert len(found) == 1, found + return found[0] + + +def name_in_slot(base, slot, year=None): + """Return a name whose rotation slot (with year, for TBA entries) is the given weekday.""" + for number in range(1, 200): + name = f"{base} {number}" + if rotation_slot(name if year is None else f"{name}|{year}") == slot: + return name + raise AssertionError("no name found") + + +def days_ago(days, time="11:00:00"): + """Return a deadline the given number of days before TODAY. + + 11:00 falls on the same UTC date in AoE, Tokyo, Los Angeles and UTC+5, so days_since is ``days``. + """ + return f"{TODAY - timedelta(days=days)} {time}" + + +class TestCleanWatchTitle: + @pytest.mark.parametrize( + ("title", "expected"), + [ + ("[Conference] PyCon Greece 2027 - Main", ("PyCon Greece", 2027)), + ("[conference] PyCon Greece 2027 - CFP", ("PyCon Greece", 2027)), + ("[CONFERENCE] PyCon Greece 2027 - cfp", ("PyCon Greece", 2027)), + ("[Conference] PyCon Greece - Main", ("PyCon Greece", None)), + ("PyCon Greece 2027 - Main", ("PyCon Greece", 2027)), + ("PyCon Greece", ("PyCon Greece", None)), + ("PyCon Greece 2027", ("PyCon Greece 2027", None)), + ("[Conference] Python - Main Track 2027 - Main", ("Python - Main Track", 2027)), + ], + ) + def test_strips_prefix_and_suffix(self, title, expected): + assert clean_watch_title(title) == expected + + +class TestRouting: + def test_upcoming_series_goes_to_enricher(self): + data = dataset([conf("PyCon Example", 2026)]) + signal = queue_item(title="[Conference] PyCon Example 2026 - Main") + + item = only(build(data, [signal])) + + assert item["skill"] == "enricher" + assert item["reasons"] == ["signal"] + assert item["conference"] == "PyCon Example" + assert item["year"] == 2026 + assert item["entry"]["conference"] == "PyCon Example" + assert item["entry"]["start"] == "2026-12-10" + assert item["latest_edition"] is None + assert item["queue_refs"] == [{"file": "queue/example.json", "id": signal["id"]}] + assert "ambiguous_matches" not in item + + def test_archive_only_series_goes_to_new_year_checker(self): + data = dataset(archive=[conf("PyCon Example", 2022), conf("PyCon Example", 2023)]) + + item = only(build(data, [queue_item(title="[Conference] PyCon Example 2027 - Main")])) + + assert item["skill"] == "new-year-checker" + assert item["year"] == 2027 + assert item["entry"] is None + assert item["latest_edition"]["year"] == 2023 + assert item["context"] == {} + + def test_legacy_only_series_goes_to_new_year_checker(self): + data = dataset(legacy=[conf("PyCon Example", 2015)]) + + item = only(build(data, [queue_item(title="PyCon Example")])) + + assert item["skill"] == "new-year-checker" + assert item["latest_edition"]["year"] == 2015 + + def test_unknown_series_goes_to_scouter(self): + data = dataset([conf("PyCon Example", 2026)]) + + item = only(build(data, [queue_item(title="[Conference] PyCon Nowhere 2027 - CFP")])) + + assert item["skill"] == "scouter" + assert item["conference"] == "PyCon Nowhere" + assert item["year"] == 2027 + assert item["entry"] is None + assert item["latest_edition"] is None + + @pytest.mark.parametrize( + ("existing", "title"), + [ + (conf("PyCon Example", 2026, alt_name="PyEx"), "[Conference] PyEx 2026 - Main"), + (conf("PyDay México", 2026), "[Conference] pyday mexico 2026 - Main"), + (conf("Xtreme Python", 2026), "[Conference] XtremePython 2026 - Main"), + (conf("Kiwi PyCon", 2026), "[Conference] PyCon NZ 2026 - Main"), + ], + ids=["alt_name", "accents-and-case", "whitespace", "titles-mapping"], + ) + def test_name_matching(self, existing, title): + item = only(build(dataset([existing]), [queue_item(title=title)])) + + assert item["skill"] == "enricher" + assert item["conference"] == existing["conference"] + + def test_entry_for_the_item_year(self): + data = dataset([conf("PyCon Example", 2026), conf("PyCon Example", 2027)]) + + with_year = only(build(data, [queue_item(title="PyCon Example 2027 - Main")])) + without_year = only(build(data, [queue_item(title="PyCon Example")])) + + assert (with_year["year"], with_year["entry"]["year"], with_year["context"]) == (2027, 2027, {}) + assert (without_year["year"], without_year["entry"]["year"], without_year["context"]) == (2026, 2026, {}) + + def test_year_from_item_field(self): + data = dataset([conf("PyCon Example", 2026), conf("PyCon Example", 2027)]) + + item = only(build(data, [queue_item(title="PyCon Example", year=2026)])) + + assert item["entry"]["year"] == 2026 + + def test_payload_conference_is_matched_before_title(self): + data = dataset([conf("PyCon Example", 2026)]) + redirect = queue_item( + "redirect", + "https://example.org/old", + source="link-check", + payload={"conference": "PyCon Example", "year": 2026, "field": "link", "final_url": "x", "status": 301}, + ) + + item = only(build(data, [redirect])) + + assert (item["skill"], item["conference"], item["year"]) == ("enricher", "PyCon Example", 2026) + + def test_file_series_is_used_when_title_does_not_match(self): + data = dataset([conf("PyCon Example", 2026)]) + + item = only(build(data, [queue_item(title="Some page title", series="PyCon Example")])) + + assert item["skill"] == "enricher" + + +class TestQueueYear: + """Watch titles keep the year of the edition they were created for, so a queue year can be stale.""" + + def test_stale_title_year_takes_the_entry_year(self): + data = dataset([conf("PyCon US", 2027, cfp=days_ago(1))], [conf("PyCon US", 2026)]) + signal = queue_item(title="[Conference] PyCon US 2026 - Main", year=2026) + + item = only(build(data, [signal])) + + assert item["skill"] == "enricher" + assert item["reasons"] == ["deadline-check", "signal"] + assert (item["conference"], item["year"], item["entry"]["year"]) == ("PyCon US", 2027, 2027) + assert item["context"]["queue_year"] == 2026 + assert item["context"]["days_since"] == 1 + assert item["queue_refs"] == [{"file": "queue/example.json", "id": signal["id"]}] + + def test_stale_title_year_merges_into_tba_rotation(self): + name = name_in_slot("PyCon TBA", TODAY.weekday(), 2027) + data = dataset([conf(name, 2027, cfp="TBA")], [conf(name, 2026)]) + + item = only(build(data, [queue_item(title=f"[Conference] {name} 2026 - Main", year=2026)])) + + assert item["reasons"] == ["signal", "tba-rotation"] + assert (item["year"], item["entry"]["year"], item["context"]) == (2027, 2027, {"queue_year": 2026}) + + def test_stale_title_year_takes_the_next_edition(self): + data = dataset([conf("PyCon Example", 2026), conf("PyCon Example", 2027)], [conf("PyCon Example", 2025)]) + + item = only(build(data, [queue_item(title="[Conference] PyCon Example 2025 - Main")])) + + assert (item["year"], item["entry"]["year"], item["context"]) == (2026, 2026, {"queue_year": 2025}) + + def test_yearless_item_takes_the_next_edition(self): + """An edition starting next week comes before the TBA entry for the year after.""" + data = dataset( + [ + conf("PyTorch Conference", 2026, start=TODAY + timedelta(days=6), end=TODAY + timedelta(days=7)), + conf("PyTorch Conference", 2027), + ], + ) + + item = only(build(data, [queue_item(title="[Conference] PyTorch Conference - Main")])) + + assert (item["year"], item["entry"]["year"], item["context"]) == (2026, 2026, {}) + + @pytest.mark.parametrize(("days_since_end", "expected"), [(1, 2027), (0, 2026)]) + def test_ended_edition_is_passed_over(self, days_since_end, expected): + """An edition that has ended but is not archived yet is not the next one; one ending today is.""" + end = TODAY - timedelta(days=days_since_end) + data = dataset( + [conf("PyCon Example", 2026, start=end - timedelta(days=2), end=end), conf("PyCon Example", 2027)], + ) + + item = only(build(data, [queue_item(title="PyCon Example")])) + + assert item["year"] == expected + + def test_newest_when_every_edition_has_ended(self): + data = dataset( + [ + conf("PyCon Example", 2025, start=TODAY - timedelta(days=40), end=TODAY - timedelta(days=38)), + conf("PyCon Example", 2026, start=TODAY - timedelta(days=5), end=TODAY - timedelta(days=3)), + ], + ) + + item = only(build(data, [queue_item(title="PyCon Example")])) + + assert item["year"] == 2026 + + def test_ambiguous_item_keeps_its_queue_year(self): + data = dataset([conf("PyHEP.dev", 2027, alt_name="PyHEP")], [conf("PyHEP", 2025)]) + + item = only(build(data, [queue_item(title="[Conference] PyHEP 2025 - Main")])) + + assert (item["year"], item["entry"], item["context"]) == (2025, None, {}) + + @pytest.mark.parametrize("title_year", [2024, 2025]) + def test_known_edition_year_merges_into_stale_series(self, title_year): + """For the new-year-checker, a year it already has is not the edition to look for.""" + name = name_in_slot("PyCon Stale", TODAY.weekday()) + data = dataset(archive=[conf(name, 2024), conf(name, 2025)]) + + item = only(build(data, [queue_item(title=f"[Conference] {name} {title_year} - Main", year=title_year)])) + + assert item["skill"] == "new-year-checker" + assert item["reasons"] == ["signal", "stale-series"] + assert item["year"] is None + assert item["latest_edition"]["year"] == 2025 + assert item["context"] == {"queue_year": title_year, "latest_year": 2025} + + def test_missing_edition_year_is_kept(self): + data = dataset(archive=[conf("PyCon Example", 2025)]) + candidate = queue_item( + "candidate", + "PyCon Example 2026", + source="pretalx", + payload={"classification": "MISSING_EDITION", "fields": {"conference": "PyCon Example", "year": 2026}}, + ) + + item = only(build(data, [candidate])) + + assert (item["skill"], item["year"], item["context"]) == ("new-year-checker", 2026, {}) + + +class TestKindOverrides: + @pytest.mark.parametrize("kind", ["question", "redirect", "drift"]) + def test_archive_only_series_still_goes_to_enricher(self, kind): + data = dataset(archive=[conf("PyCon Example", 2023)]) + + item = only(build(data, [queue_item(kind, "PyCon Example")])) + + assert item["skill"] == "enricher" + assert item["reasons"] == [kind] + assert item["conference"] == "PyCon Example" + assert item["entry"] is None + + @pytest.mark.parametrize("kind", ["question", "redirect", "drift"]) + def test_archive_only_series_names_why_entry_is_empty(self, kind): + data = dataset(archive=[conf("PyCon Example", 2023)]) + + item = only(build(data, [queue_item(kind, "PyCon Example")])) + + assert item["context"] == {"no_entry": "no data file holds an edition of this series for this year"} + + @pytest.mark.parametrize("kind", ["question", "redirect", "drift"]) + def test_archived_edition_for_the_year_is_bound(self, kind): + data = dataset(archive=[conf("PyCon Example", 2023), conf("PyCon Example", 2024)]) + + item = only(build(data, [queue_item(kind, "PyCon Example", year=2023)])) + + assert (item["skill"], item["year"], item["context"]) == ("enricher", 2023, {}) + assert (item["entry"]["conference"], item["entry"]["year"]) == ("PyCon Example", 2023) + + @pytest.mark.parametrize("kind", ["question", "redirect", "drift"]) + def test_unknown_series_still_goes_to_enricher(self, kind): + item = only(build(dataset(), [queue_item(kind, "PyCon Nowhere")])) + + assert item["skill"] == "enricher" + assert item["context"] == {"no_entry": "no data file holds this series"} + + def test_missing_edition_goes_to_new_year_checker(self): + data = dataset([conf("PyCon Example", 2026)], [conf("PyCon Example", 2023)]) + candidate = queue_item( + "candidate", + "PyCon Exmple 2027", + source="pretalx", + payload={ + "classification": "MISSING_EDITION", + "fields": {"conference": "PyCon Exmple", "year": 2027}, + "matched": {"conference": "PyCon Example", "file": "_data/conferences.yml", "year": 2026, "score": 92}, + "flags": [], + }, + ) + + item = only(build(data, [candidate])) + + assert item["skill"] == "new-year-checker" + assert item["reasons"] == ["candidate"] + assert item["conference"] == "PyCon Example" + assert item["year"] == 2027 + assert item["latest_edition"]["year"] == 2026 + + @pytest.mark.parametrize("classification", ["NEW_SERIES", "MAYBE"]) + def test_new_series_and_maybe_go_to_scouter(self, classification): + data = dataset([conf("PyCon Example", 2026)]) + candidate = queue_item( + "candidate", + "PyCon Example", + payload={"classification": classification, "fields": {"conference": "PyCon Example"}, "matched": None}, + ) + + item = only(build(data, [candidate])) + + assert item["skill"] == "scouter" + assert item["entry"] is None + + def test_registry_goes_to_scouter(self): + data = dataset([conf("PyCon Example", 2026)]) + registry = queue_item( + "registry", + "Registry change", + source="registry", + payload={"names": ["PyCon Example"], "change": "removed", "evidence_url": "https://example.org/"}, + ) + + item = only(build(data, [registry])) + + assert (item["skill"], item["reasons"], item["conference"]) == ("scouter", ["registry"], "PyCon Example") + + def test_unknown_kind_is_treated_as_signal(self): + item = only(build(dataset([conf("PyCon Example", 2026)]), [queue_item("mystery", "PyCon Example")])) + + assert (item["skill"], item["reasons"]) == ("enricher", ["signal"]) + + +class TestAmbiguousMatches: + def test_any_upcoming_match_goes_to_enricher(self): + data = dataset([conf("PyHEP.dev", 2026, alt_name="PyHEP")], [conf("PyHEP", 2023)]) + + item = only(build(data, [queue_item(title="[Conference] PyHEP 2026 - Main")])) + + assert item["skill"] == "enricher" + assert item["conference"] == "PyHEP" + assert item["year"] == 2026 + assert item["entry"] is None + assert item["ambiguous_matches"] == [ + {"conference": "PyHEP", "year": 2023, "file": "_data/archive.yml"}, + {"conference": "PyHEP.dev", "year": 2026, "file": "_data/conferences.yml"}, + ] + + def test_archive_only_matches_go_to_new_year_checker(self): + data = dataset(archive=[conf("PyCon Old", 2023, alt_name="PyCon Shared"), conf("PyCon Shared", 2022)]) + + item = only(build(data, [queue_item(title="PyCon Shared")])) + + assert item["skill"] == "new-year-checker" + assert [match["conference"] for match in item["ambiguous_matches"]] == ["PyCon Old", "PyCon Shared"] + + def test_unique_name_wins_over_ambiguous_one(self): + data = dataset([conf("PyHEP.dev", 2026, alt_name="PyHEP")], [conf("PyHEP", 2023)]) + + item = only(build(data, [queue_item(title="PyHEP", series="PyHEP.dev")])) + + assert item["conference"] == "PyHEP.dev" + assert item["entry"]["conference"] == "PyHEP.dev" + assert "ambiguous_matches" not in item + + +class TestDeadlineChecks: + @pytest.mark.parametrize("days", [1, 3, 7]) + def test_aoe(self, days): + data = dataset([conf("PyCon Example", 2026, cfp=days_ago(days))]) + + item = only(build(data)) + + assert item["skill"] == "enricher" + assert item["reasons"] == ["deadline-check"] + assert item["entry"]["conference"] == "PyCon Example" + assert item["context"] == { + "effective_deadline": f"{TODAY - timedelta(days=days)}T11:00:00-12:00", + "deadline_field": "cfp", + "timezone": "AoE", + "days_since": days, + } + + @pytest.mark.parametrize("days", [1, 3, 7]) + def test_explicit_timezone(self, days): + data = dataset([conf("PyCon Example", 2026, cfp=days_ago(days, "12:00:00"), timezone="Asia/Tokyo")]) + + item = only(build(data)) + + assert item["context"]["effective_deadline"] == f"{TODAY - timedelta(days=days)}T12:00:00+09:00" + assert item["context"]["timezone"] == "Asia/Tokyo" + assert item["context"]["days_since"] == days + + @pytest.mark.parametrize("days", [-1, 0, 2, 4, 5, 6, 8, 30]) + @pytest.mark.parametrize("zone", [None, "Asia/Tokyo", "America/Los_Angeles"]) + def test_other_days_are_skipped(self, days, zone): + fields = {"timezone": zone} if zone else {} + data = dataset([conf("PyCon Example", 2026, cfp=days_ago(days), **fields)]) + + assert items(build(data)) == [] + + def test_counts_from_the_utc_date(self): + """00:30 in Tokyo on the 14th is 15:30 UTC on the 13th, so a run on the 14th is one day after.""" + data = dataset([conf("PyCon Example", 2026, cfp="2026-10-14 00:30:00", timezone="Asia/Tokyo")]) + + assert only(build(data))["context"]["days_since"] == 1 + + def test_aoe_deadline_counts_from_its_utc_date(self): + """23:59 AoE on the 12th closes at 11:59 UTC on the 13th, one day before the 14th.""" + data = dataset([conf("PyCon Example", 2026, cfp="2026-10-12 23:59:00")]) + + assert only(build(data))["context"]["days_since"] == 1 + + def test_no_check_while_an_aoe_deadline_is_open(self): + """23:59 AoE on the 13th closes at 11:59 UTC on the 14th, after a desk run early that day.""" + data = dataset([conf("PyCon Example", 2026, cfp="2026-10-13 23:59:00")]) + + assert items(build(data)) == [] + + def test_cfp_ext_takes_precedence(self): + data = dataset([conf("PyCon Example", 2026, cfp=days_ago(7), cfp_ext=days_ago(3))]) + + item = only(build(data)) + + assert item["context"]["deadline_field"] == "cfp_ext" + assert item["context"]["days_since"] == 3 + + def test_cfp_ext_in_future_suppresses_cfp_check(self): + data = dataset([conf("PyCon Example", 2026, cfp=days_ago(1), cfp_ext=days_ago(-5))]) + + assert items(build(data)) == [] + + def test_tba_cfp_is_not_checked(self): + data = dataset([conf("PyCon Example", 2026, cfp="TBA", cfp_ext=days_ago(1), start=date(2026, 10, 1))]) + + assert items(build(data)) == [] + + @pytest.mark.parametrize("value", [str(TODAY - timedelta(days=2)), TODAY - timedelta(days=2)]) + def test_date_only_deadline(self, value): + """A date-only deadline is 23:59 AoE, which falls on the next UTC date.""" + item = only(build(dataset([conf("PyCon Example", 2026, cfp=value)]))) + + assert item["context"]["effective_deadline"] == f"{TODAY - timedelta(days=2)}T23:59:00-12:00" + assert item["context"]["days_since"] == 1 + + def test_utc_offset_timezone(self): + item = only(build(dataset([conf("PyCon Example", 2026, cfp=days_ago(1), timezone="UTC+5")]))) + + assert item["context"]["effective_deadline"].endswith("+05:00") + assert item["context"]["timezone"] == "UTC+5" + + def test_unknown_timezone_falls_back_to_aoe(self): + item = only(build(dataset([conf("PyCon Example", 2026, cfp=days_ago(1), timezone="Mars/Olympus")]))) + + assert item["context"]["timezone"] == "AoE" + assert item["context"]["unknown_timezone"] == "Mars/Olympus" + + def test_most_recent_deadline_first(self): + data = dataset( + [ + conf("A Conf", 2026, cfp=days_ago(7)), + conf("B Conf", 2026, cfp=days_ago(1)), + conf("C Conf", 2026, cfp=days_ago(3)), + conf("A Conf Two", 2026, cfp=days_ago(1)), + ], + ) + + assert [item["conference"] for item in items(build(data))] == ["A Conf Two", "B Conf", "C Conf", "A Conf"] + + +class TestTbaRotation: + def test_included_on_its_weekday(self): + name = name_in_slot("PyCon TBA", TODAY.weekday(), 2027) + data = dataset([conf(name, 2027, cfp="TBA")]) + + item = only(build(data)) + + assert (item["skill"], item["reasons"], item["conference"], item["year"]) == ( + "enricher", + ["tba-rotation"], + name, + 2027, + ) + assert item["entry"]["cfp"] == "TBA" + + def test_skipped_on_other_weekdays(self): + name = name_in_slot("PyCon TBA", (TODAY.weekday() + 1) % 7, 2027) + + assert items(build(dataset([conf(name, 2027, cfp="TBA")]))) == [] + + def test_once_a_week(self): + data = dataset([conf("PyCon Example", 2027, cfp="TBA")]) + days = [day for day in range(7) if items(build(data, today=TODAY + timedelta(days=day)))] + + assert len(days) == 1 + + @pytest.mark.parametrize("word", ["TBA", "tbd", " Tba "]) + def test_tba_words(self, word): + name = name_in_slot("PyCon TBA", TODAY.weekday(), 2027) + + assert items(build(dataset([conf(name, 2027, cfp=word)]))) + + @pytest.mark.parametrize("word", ["None", "Cancelled", None, "n/a"]) + def test_settled_cfp_is_not_rotated(self, word): + """None means no public CfP and Cancelled a cancelled edition: no deadline to look for.""" + name = name_in_slot("PyCon TBA", TODAY.weekday(), 2027) + data = dataset([conf(name, 2027, cfp=word)]) + + assert [item for day in range(7) for item in items(build(data, today=TODAY + timedelta(days=day)))] == [] + + @pytest.mark.parametrize("start", [TODAY, TODAY - timedelta(days=1)]) + def test_started_event_excluded(self, start): + name = name_in_slot("PyCon TBA", TODAY.weekday(), 2026) + data = dataset([conf(name, 2026, cfp="TBA", start=start, end=start + timedelta(days=2))]) + + assert items(build(data)) == [] + + def test_slot_rule(self): + expected = int(hashlib.sha1(b"PyCon Example|2027", usedforsecurity=False).hexdigest(), 16) % 7 + + assert rotation_slot("PyCon Example|2027") == expected + + +class TestStaleSeries: + def test_included_on_its_weekday(self): + name = name_in_slot("PyCon Stale", TODAY.weekday()) + data = dataset(archive=[conf(name, 2024), conf(name, 2025)]) + + item = only(build(data)) + + assert (item["skill"], item["reasons"], item["conference"], item["year"]) == ( + "new-year-checker", + ["stale-series"], + name, + None, + ) + assert item["latest_edition"]["year"] == 2025 + assert item["context"] == {"latest_year": 2025} + + def test_once_a_week(self): + data = dataset(archive=[conf("PyCon Stale", 2025)]) + days = [day for day in range(7) if items(build(data, today=TODAY + timedelta(days=day)))] + + assert len(days) == 1 + + def test_respelt_upcoming_edition_tracks_the_series(self): + data = dataset([conf("XtremePython", 2026)], [conf("Xtreme Python", 2025)]) + + for day in range(7): + assert items(build(data, today=TODAY + timedelta(days=day))) == [] + + def test_outside_lookback_skipped(self): + name = name_in_slot("PyCon Stale", TODAY.weekday()) + + assert items(build(dataset(archive=[conf(name, 2023)]))) == [] + + +class TestScout: + def test_exactly_one_scout_item_last(self): + worklist = build(dataset([conf("PyCon Example", 2026, cfp=days_ago(1))])) + + scouts = [item for item in worklist["items"] if "scout" in item["reasons"]] + assert scouts == [worklist["items"][-1]] + assert scouts[0] == { + "id": hashlib.sha1(b"scout", usedforsecurity=False).hexdigest(), + "skill": "scouter", + "reasons": ["scout"], + "conference": None, + "year": None, + "entry": None, + "latest_edition": None, + "queue_refs": [], + "context": {}, + } + + def test_no_scout(self): + assert build(dataset(), scout=False)["items"] == [] + + def test_scout_not_merged_with_scouter_queue_items(self): + worklist = build(dataset(), [queue_item(title="PyCon Nowhere")]) + + assert [item["reasons"] for item in worklist["items"]] == [["signal"], ["scout"]] + + +class TestMerging: + def test_queue_item_merges_into_deadline_check(self): + data = dataset([conf("PyCon Example", 2026, cfp=days_ago(1))]) + signal = queue_item(title="[Conference] PyCon Example 2026 - CFP") + + item = only(build(data, [signal])) + + assert item["reasons"] == ["deadline-check", "signal"] + assert item["queue_refs"] == [{"file": "queue/example.json", "id": signal["id"]}] + assert item["context"]["days_since"] == 1 + + def test_items_for_same_series_from_several_files(self): + first = queue_item(title="PyCon Nowhere", file="queue/a.json") + second = queue_item("question", "PyCon Nowhere", file="queue/b.json") + third = queue_item(title="[Conference] PyCon Nowhere - Main", file="queue/c.json") + + worklist = build(dataset(), [first, second, third]) + + assert [(item["skill"], item["reasons"], len(item["queue_refs"])) for item in items(worklist)] == [ + ("scouter", ["signal"], 2), + ("enricher", ["question"], 1), + ] + + def test_duplicate_refs_collapse(self): + signal = queue_item(title="PyCon Example") + + item = only(build(dataset([conf("PyCon Example", 2026)]), [signal, dict(signal)])) + + assert len(item["queue_refs"]) == 1 + + def test_tba_rotation_merges_into_queue_item(self): + name = name_in_slot("PyCon TBA", TODAY.weekday(), 2027) + data = dataset([conf(name, 2027, cfp="TBA")]) + + item = only(build(data, [queue_item(title=name)])) + + assert item["reasons"] == ["signal", "tba-rotation"] + + def test_stale_series_merges_with_queue_item(self): + name = name_in_slot("PyCon Stale", TODAY.weekday()) + data = dataset(archive=[conf(name, 2025)]) + + item = only(build(data, [queue_item(title=name)])) + + assert item["skill"] == "new-year-checker" + assert item["reasons"] == ["signal", "stale-series"] + assert item["latest_edition"]["year"] == 2025 + + def test_different_years_stay_apart(self): + data = dataset([conf("PyCon Example", 2026), conf("PyCon Example", 2027)]) + signals = [ + queue_item(title="PyCon Example 2027 - Main", received="2026-10-14T06:00:00Z"), + queue_item(title="PyCon Example 2026 - Main", received="2026-10-13T06:00:00Z"), + ] + + assert [item["year"] for item in items(build(data, signals))] == [2026, 2027] + + +class TestOrdering: + @pytest.fixture() + def mixed(self): + tba = name_in_slot("PyCon TBA", TODAY.weekday(), 2027) + stale = name_in_slot("PyCon Stale", TODAY.weekday()) + data = dataset( + [ + conf("PyCon Deadline", 2026, cfp=days_ago(3)), + conf("PyCon Queue", 2026), + conf(tba, 2027, cfp="TBA"), + ], + [conf(stale, 2025)], + ) + queue = [ + queue_item(title="PyCon Unknown", file="queue/pycon-unknown.json", received="2026-10-13T06:00:00Z"), + queue_item(title="PyCon Queue", file="queue/pycon-queue.json"), + queue_item("question", "PyCon Queue", file="queue/pycon-queue.json", received="2026-10-12T06:00:00Z"), + ] + return data, queue + + def test_group_order(self, mixed): + data, queue = mixed + + worklist = build(data, queue) + + assert [item["reasons"] for item in worklist["items"]] == [ + ["deadline-check"], + ["signal", "question"], + ["signal"], + ["tba-rotation"], + ["stale-series"], + ["scout"], + ] + assert [item["conference"] for item in worklist["items"][1:3]] == ["PyCon Queue", "PyCon Unknown"] + # Queue items are taken by file, then by time received. + assert [ref["id"] for ref in worklist["items"][1]["queue_refs"]] == [queue[2]["id"], queue[1]["id"]] + + def test_same_output_whatever_the_queue_order(self, mixed): + data, queue = mixed + expected = json.dumps(build(data, queue)) + + for order in itertools.permutations(queue): + assert json.dumps(build(data, order)) == expected + + def test_ids_are_stable_and_distinct(self, mixed): + data, queue = mixed + + ids = [item["id"] for item in build(data, queue)["items"]] + + assert ids == [item["id"] for item in build(data, queue)["items"]] + assert len(set(ids)) == len(ids) + deadline = build(data, queue)["items"][0] + assert deadline["id"] == hashlib.sha1(b"enricher|PyCon Deadline|2026", usedforsecurity=False).hexdigest() + + def test_document_fields(self, mixed): + data, queue = mixed + + worklist = build(data, queue) + + assert list(worklist) == ["generated", "today", "items"] + assert (worklist["generated"], worklist["today"]) == (GENERATED, "2026-10-14") + for item in worklist["items"]: + assert list(item)[:9] == [ + "id", + "skill", + "reasons", + "conference", + "year", + "entry", + "latest_edition", + "queue_refs", + "context", + ] + + def test_input_data_is_not_modified(self, mixed): + data, queue = mixed + before = json.dumps(data, default=str), json.dumps(queue) + + build(data, queue) + + assert (json.dumps(data, default=str), json.dumps(queue)) == before + + +def write_queue_file(directory, key, series, queue_items): + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{key}.json" + path.write_text(json.dumps({"items": queue_items, "series": series}, indent=2, sort_keys=True), encoding="utf-8") + return path + + +def raw_item(number, **fields): + item = { + "id": hashlib.sha1(str(number).encode(), usedforsecurity=False).hexdigest(), + "kind": "signal", + "payload": {"diff": f"change {number}", "trigger": "website_change", "watch_uuid": None}, + "received": f"2026-10-{1 + number % 14:02d}T06:00:{number % 60:02d}Z", + "source": "changedetection", + "title": "[Conference] PyCon Greece 2027 - Main", + "url": "https://pycon.gr/", + "year": 2027, + } + item.update(fields) + return item + + +class TestLoadQueue: + def test_fifty_items_and_truncated_payload(self, tmp_path): + fifty = [raw_item(number) for number in range(50)] + truncated = raw_item( + 99, + payload={"diff": "x" * 20000, "truncated": True, "trigger": "website_change", "watch_uuid": "abc"}, + ) + write_queue_file(tmp_path / "queue", "pycon-greece", "PyCon Greece", fifty) + write_queue_file(tmp_path / "queue", "pycon-example", "PyCon Example", [truncated]) + + loaded = load_queue(tmp_path) + + assert len(loaded) == 51 + assert loaded[0]["file"] == "queue/pycon-example.json" + assert loaded[0]["series"] == "PyCon Example" + assert len(loaded[0]["payload"]["diff"]) == 20000 + assert loaded[0]["payload"]["truncated"] is True + assert {item["file"] for item in loaded[1:]} == {"queue/pycon-greece.json"} + + item = only(build(dataset(archive=[conf("PyCon Greece", 2023)]), loaded[1:])) + assert item["skill"] == "new-year-checker" + assert len(item["queue_refs"]) == 50 + + def test_accepts_the_queue_directory_itself(self, tmp_path): + write_queue_file(tmp_path / "queue", "pycon-greece", "PyCon Greece", [raw_item(1)]) + + assert load_queue(tmp_path / "queue") == load_queue(tmp_path) + + def test_bad_files_are_skipped(self, tmp_path, capsys): + queue = tmp_path / "queue" + write_queue_file(queue, "good", "PyCon Greece", [raw_item(1), "not an item"]) + (queue / "broken.json").write_text('{"items": [', encoding="utf-8") + (queue / "list.json").write_text("[]", encoding="utf-8") + (queue / "README.md").write_text("not json", encoding="utf-8") + + loaded = load_queue(tmp_path) + + assert [item["file"] for item in loaded] == ["queue/good.json"] + errors = capsys.readouterr().err + assert "broken.json" in errors + assert "list.json" in errors + + def test_empty_directory(self, tmp_path): + assert load_queue(tmp_path) == [] + + +class TestCli: + @pytest.fixture() + def repo(self, tmp_path): + data = tmp_path / "repo" / "_data" + data.mkdir(parents=True) + (data / "conferences.yml").write_text( + yaml.safe_dump([conf("PyCon Example", 2026, cfp=days_ago(7))]), + encoding="utf-8", + ) + (data / "archive.yml").write_text(yaml.safe_dump([conf("PyCon Greece", 2023)]), encoding="utf-8") + write_queue_file(tmp_path / "desk-queue" / "queue", "pycon-greece", "PyCon Greece", [raw_item(1)]) + return tmp_path + + def run(self, capsys, *args): + main(list(args)) + return json.loads(capsys.readouterr().out) + + def test_full_run(self, repo, capsys): + worklist = self.run( + capsys, + "--base", + str(repo / "repo"), + "--queue-dir", + str(repo / "desk-queue"), + "--today", + "2026-10-14", + ) + + assert worklist["today"] == "2026-10-14" + assert [(item["skill"], item["reasons"]) for item in worklist["items"]] == [ + ("enricher", ["deadline-check"]), + ("new-year-checker", ["signal"]), + ("scouter", ["scout"]), + ] + assert worklist["items"][1]["queue_refs"] == [{"file": "queue/pycon-greece.json", "id": raw_item(1)["id"]}] + + def test_no_scout_and_no_queue(self, repo, capsys): + worklist = self.run(capsys, "--base", str(repo / "repo"), "--today", "2026-10-14", "--no-scout") + + assert [item["reasons"] for item in worklist["items"]] == [["deadline-check"]] + + @freeze_time("2026-10-14 06:00:00") + def test_defaults_to_utc_today(self, repo, capsys): + worklist = self.run(capsys, "--base", str(repo / "repo")) + + assert (worklist["generated"], worklist["today"]) == ("2026-10-14T06:00:00Z", "2026-10-14") + + def test_missing_queue_dir_is_an_error(self, repo): + with pytest.raises(SystemExit): + main(["--base", str(repo / "repo"), "--queue-dir", str(repo / "absent")]) + + def test_never_opens_a_socket(self, repo, capsys, monkeypatch): + def refuse(*args, **kwargs): + raise AssertionError("network access") + + monkeypatch.setattr(socket, "socket", refuse) + monkeypatch.setattr(socket, "create_connection", refuse) + + worklist = self.run( + capsys, + "--base", + str(repo / "repo"), + "--queue-dir", + str(repo / "desk-queue"), + "--today", + "2026-10-14", + ) + + assert len(worklist["items"]) == 3 diff --git a/utils/cfp_candidates.py b/utils/cfp_candidates.py index b5f160b3a1..78ed83a5a9 100644 --- a/utils/cfp_candidates.py +++ b/utils/cfp_candidates.py @@ -5,28 +5,214 @@ upcoming or TBA) whose latest archived edition is recent enough that the series is plausibly still alive. Output is JSON on stdout, newest first. +A series counts as tracked when its name or any of its alt_names matches the +name or alt_name of an entry in _data/conferences.yml. Names are compared +after normalize_conference_name (which applies the titles.yml mappings), +ignoring case, accents and whitespace, so "Xtreme Python" and "XtremePython" +are the same series. + Intended for the cfp-scout routine: run once at session start, then work -from the JSON instead of re-reading the data files. +from the JSON instead of re-reading the data files. It needs the same +packages as sort_yaml.py (the pixi or uv environment), not PyYAML alone: +the title normalisation in tidy_conf imports pandas, pydantic, thefuzz and +tqdm. """ import argparse import json +from collections.abc import Iterable from datetime import datetime from datetime import timezone from pathlib import Path import yaml +try: + from tidy_conf.titles import normalize_conference_name + from tidy_conf.utils import fold_name + from tidy_conf.yaml import load_title_mappings +except ImportError: + from .tidy_conf.titles import normalize_conference_name + from .tidy_conf.utils import fold_name + from .tidy_conf.yaml import load_title_mappings + CARRY_FIELDS = ("link", "place", "sub", "alt_name", "cfp_link") +DATA_FILES = ("conferences", "archive", "legacy") def load(path: Path) -> list[dict]: + """Load one conference data file. + + Parameters + ---------- + path : Path + Location of the YAML file. + + Returns + ------- + list[dict] + The entries in the file, or an empty list when the file is missing or empty. + """ if not path.exists(): return [] - with path.open() as fh: + with path.open(encoding="utf-8") as fh: return yaml.safe_load(fh) or [] +def load_data(base: Path) -> dict[str, list[dict]]: + """Load conferences.yml, archive.yml and legacy.yml from a repository checkout. + + Parameters + ---------- + base : Path + Repository root. + + Returns + ------- + dict[str, list[dict]] + Entries keyed by file stem: "conferences", "archive" and "legacy". + """ + return {name: load(base / "_data" / f"{name}.yml") for name in DATA_FILES} + + +def series_key(name: str, known_mappings: dict | None = None) -> str: + """Return the key two names of the same series share. + + Parameters + ---------- + name : str + A conference name, alt_name or raw title. + known_mappings : dict, optional + Reverse titles.yml mapping from load_title_mappings(reverse=True). Loaded when omitted. + + Returns + ------- + str + The normalised name with case, accents and whitespace folded away. + """ + if known_mappings is None: + _, known_mappings = load_title_mappings(reverse=True) + return "".join(fold_name(normalize_conference_name(name, known_mappings)).split()) + + +def entry_names(entry: dict) -> list[str]: + """Return the conference name and alt_names of one entry. + + Parameters + ---------- + entry : dict + A conference entry. + + Returns + ------- + list[str] + Non-empty names, conference first. + """ + alt_names = entry.get("alt_name") + if not isinstance(alt_names, list): + alt_names = [alt_names] + return [name for name in (entry.get("conference"), *alt_names) if isinstance(name, str) and name.strip()] + + +def entry_keys(entry: dict, known_mappings: dict) -> set[str]: + """Return the series keys of an entry's conference name and alt_names. + + Parameters + ---------- + entry : dict + A conference entry. + known_mappings : dict + Reverse titles.yml mapping from load_title_mappings(reverse=True). + + Returns + ------- + set[str] + One key per distinct name. + """ + return {series_key(name, known_mappings) for name in entry_names(entry)} + + +def tracked_keys(conferences: Iterable[dict], known_mappings: dict) -> set[str]: + """Return the series keys of every name and alt_name in conferences.yml. + + Parameters + ---------- + conferences : Iterable[dict] + Entries from conferences.yml. + known_mappings : dict + Reverse titles.yml mapping from load_title_mappings(reverse=True). + + Returns + ------- + set[str] + Keys that count as tracked. + """ + keys: set[str] = set() + for entry in conferences: + keys |= entry_keys(entry, known_mappings) + return keys + + +def find_candidates( + conferences: list[dict], + archive: list[dict], + current_year: int, + lookback: int = 2, + known_mappings: dict | None = None, +) -> list[dict]: + """Find archived series without an entry in conferences.yml. + + Parameters + ---------- + conferences : list[dict] + Entries from conferences.yml. + archive : list[dict] + Entries from archive.yml. + current_year : int + Year the lookback counts back from. + lookback : int + Only report series whose latest edition is at most this many years before current_year. + known_mappings : dict, optional + Reverse titles.yml mapping from load_title_mappings(reverse=True). Loaded when omitted. + + Returns + ------- + list[dict] + One dict per series with "conference", "latest_year" and the CARRY_FIELDS of the + latest edition, newest first, then by name. + """ + if known_mappings is None: + _, known_mappings = load_title_mappings(reverse=True) + tracked = tracked_keys(conferences, known_mappings) + cutoff = current_year - lookback + + series: dict[str, dict] = {} + for conf in archive: + name = conf.get("conference") + if not isinstance(name, str) or not name.strip(): + continue + info = series.setdefault(series_key(name, known_mappings), {"keys": set(), "latest": None}) + # The names of every archived edition count, not only the latest one's, so a + # renamed series stays tracked under its old name. The known cost: an edition + # whose alt_name is its host conference (a regional event run by one national + # PyCon) hides the whole series for as long as the host has an entry. + info["keys"] |= entry_keys(conf, known_mappings) + if info["latest"] is None or (conf.get("year") or 0) > (info["latest"].get("year") or 0): + info["latest"] = conf + + candidates = [] + for info in series.values(): + latest = info["latest"] + latest_year = latest.get("year") or 0 + if info["keys"] & tracked or latest_year < cutoff: + continue + candidate = {"conference": latest["conference"], "latest_year": latest_year} + candidate.update({k: latest[k] for k in CARRY_FIELDS if latest.get(k)}) + candidates.append(candidate) + + return sorted(candidates, key=lambda x: (-x["latest_year"], x["conference"])) + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -39,24 +225,11 @@ def main() -> None: args = parser.parse_args() current_year = datetime.now(tz=timezone.utc).year - cutoff = current_year - args.lookback - - current_names = {c["conference"] for c in load(args.base / "_data/conferences.yml")} - - latest: dict[str, dict] = {} - for conf in load(args.base / "_data/archive.yml"): - name = conf.get("conference") - year = conf.get("year") or 0 - if not name or name in current_names: - continue - if year > latest.get(name, {}).get("latest_year", 0): - entry = {"conference": name, "latest_year": year} - entry.update({k: conf[k] for k in CARRY_FIELDS if conf.get(k)}) - latest[name] = entry - - candidates = sorted( - (info for info in latest.values() if info["latest_year"] >= cutoff), - key=lambda x: (-x["latest_year"], x["conference"]), + candidates = find_candidates( + load(args.base / "_data/conferences.yml"), + load(args.base / "_data/archive.yml"), + current_year=current_year, + lookback=args.lookback, ) print(json.dumps({"generated": f"{current_year}", "count": len(candidates), "candidates": candidates}, indent=2)) diff --git a/utils/desk_worklist.py b/utils/desk_worklist.py new file mode 100644 index 0000000000..a985e9996c --- /dev/null +++ b/utils/desk_worklist.py @@ -0,0 +1,716 @@ +#!/usr/bin/env python3 +"""Build the deadline desk's daily worklist as JSON. + +Every piece of work is routed to one skill: ``enricher`` for a series with an +entry in _data/conferences.yml, ``new-year-checker`` for a series known only +from _data/archive.yml or _data/legacy.yml, and ``scouter`` for anything else. +Work comes from four places, in this order in the output: + +1. Deadline checks: entries whose effective deadline (cfp_ext, else cfp) fell + 1, 3 or 7 days before today. The deadline is read in the entry's timezone + (AoE, UTC-12, when unset) and counted from the UTC date it falls on, so a + check never comes before the deadline has passed: 23:59 AoE on the 12th is + 11:59 UTC on the 13th, and its 1-day check is on the 14th. +2. Queue items read from a checkout of the desk-queue branch, matched to a + series by name the way cfp_candidates.py matches: after + normalize_conference_name and the titles.yml mappings, ignoring case, + accents and whitespace. A matched item takes the year of the entry it is + bound to: the entry for its own year, else the next edition that has not + ended. A differing queue year is kept in context["queue_year"]. +3. Rotations: entries whose cfp is TBA or TBD (not None, which means no public + CfP, or Cancelled) and stale series from cfp_candidates.py, + each picked on one weekday in seven by a hash of its name. +4. One open-ended scouting item, unless --no-scout is given. + +Items for the same skill, conference and year are merged. The same data, +queue and date always give the same output. Nothing here touches the network. + +It needs the same packages as sort_yaml.py (the pixi or uv environment), +because the title normalisation in tidy_conf imports pandas, pydantic, +thefuzz and tqdm. + +Usage:: + + python utils/desk_worklist.py --queue-dir ../desk-queue --today 2026-10-14 +""" + +import argparse +import hashlib +import json +import re +import sys +import zoneinfo +from collections.abc import Iterable +from datetime import date +from datetime import datetime +from datetime import timedelta +from datetime import timezone +from datetime import tzinfo +from pathlib import Path +from typing import Any + +try: + from cfp_candidates import entry_keys + from cfp_candidates import find_candidates + from cfp_candidates import load_data + from cfp_candidates import series_key + from sort_yaml import DATEFORMAT + from sort_yaml import DEFAULT_CFP_TIME + from tidy_conf.titles import normalize_conference_name + from tidy_conf.yaml import load_title_mappings +except ImportError: + from .cfp_candidates import entry_keys + from .cfp_candidates import find_candidates + from .cfp_candidates import load_data + from .cfp_candidates import series_key + from .sort_yaml import DATEFORMAT + from .sort_yaml import DEFAULT_CFP_TIME + from .tidy_conf.titles import normalize_conference_name + from .tidy_conf.yaml import load_title_mappings + +DATA_PATHS = { + "conferences": "_data/conferences.yml", + "archive": "_data/archive.yml", + "legacy": "_data/legacy.yml", +} +REASONS = ( + "deadline-check", + "signal", + "question", + "redirect", + "drift", + "candidate", + "registry", + "tba-rotation", + "stale-series", + "scout", +) +QUEUE_KINDS = ("signal", "question", "redirect", "drift", "candidate", "registry") +ENRICHER_KINDS = ("question", "redirect", "drift") +CANDIDATE_SKILLS = {"MISSING_EDITION": "new-year-checker", "NEW_SERIES": "scouter", "MAYBE": "scouter"} +CHECK_DAYS = (1, 3, 7) +# cfp values that mean the deadline is still to come. None (no public CfP) and Cancelled are settled. +ROTATION_CFP = ("tba", "tbd") +ROTATION_DAYS = 7 +AOE = timezone(timedelta(hours=-12), "AoE") + +WATCH_PREFIX = re.compile(r"^\s*\[conference\]\s*", re.IGNORECASE) +WATCH_SUFFIX = re.compile(r"\s+(?:(?P\d{4})\s+)?-\s+(?:main|cfp)\s*$", re.IGNORECASE) +DATED = re.compile(r"^\d{4}-\d{2}-\d{2}( \d{2}:\d{2}:\d{2})?$") +UTC_OFFSET = re.compile(r"^UTC([+-])(\d{1,2})$") + + +def clean_watch_title(title: str) -> tuple[str, int | None]: + """Strip the decoration a changedetection watch title carries. + + Parameters + ---------- + title : str + Raw title, e.g. "[Conference] PyCon Greece 2027 - Main". + + Returns + ------- + tuple[str, int | None] + The bare name ("PyCon Greece") and the year from the suffix, if any. + """ + name = WATCH_PREFIX.sub("", title) + year = None + match = WATCH_SUFFIX.search(name) + if match: + name = name[: match.start()] + if match["year"]: + year = int(match["year"]) + return name.strip(), year + + +def rotation_slot(text: str) -> int: + """Return the weekday (Monday is 0) on which a rotating item is due. + + Parameters + ---------- + text : str + "|" for a TBA entry, the series name for a stale series. + + Returns + ------- + int + A number from 0 to 6, compared with date.weekday(). + """ + return int(hashlib.sha1(text.encode("utf-8"), usedforsecurity=False).hexdigest(), 16) % ROTATION_DAYS + + +def timezone_for(name: Any) -> tuple[str, tzinfo, bool]: + """Resolve an entry's timezone field. + + Parameters + ---------- + name : Any + The timezone value: an IANA name, "AoE", "UTC+N" or "UTC-N", or empty. + + Returns + ------- + tuple[str, tzinfo, bool] + The label used, the zone, and False when the value was set but not recognised + (AoE is used instead). + """ + if not isinstance(name, str) or not name.strip() or name.strip() == "AoE": + return "AoE", AOE, True + label = name.strip() + iana = UTC_OFFSET.sub(lambda m: f"Etc/GMT{'-' if m[1] == '+' else '+'}{int(m[2])}", label) + try: + return label, zoneinfo.ZoneInfo(iana), True + except (zoneinfo.ZoneInfoNotFoundError, ValueError): + return "AoE", AOE, False + + +def _deadline_text(value: Any) -> str | None: + """Return a dated deadline as "YYYY-MM-DD HH:MM:SS", or None for TBA and the like.""" + if isinstance(value, datetime): + return value.strftime(DATEFORMAT) + if isinstance(value, date): + return value.isoformat() + DEFAULT_CFP_TIME + if isinstance(value, str) and DATED.match(value.strip()): + text = value.strip() + return text if " " in text else text + DEFAULT_CFP_TIME + return None + + +def effective_deadline(entry: dict) -> dict | None: + """Return the deadline a deadline check counts from. + + Parameters + ---------- + entry : dict + A conferences.yml entry. + + Returns + ------- + dict | None + "moment" (aware datetime), "field" ("cfp_ext" or "cfp"), "timezone" (label) and + "timezone_valid", or None when cfp is not a date. + """ + if _deadline_text(entry.get("cfp")) is None: + return None + label, zone, valid = timezone_for(entry.get("timezone")) + for field in ("cfp_ext", "cfp"): + text = _deadline_text(entry.get(field)) + if text is None: + continue + try: + moment = datetime.strptime(text, DATEFORMAT).replace(tzinfo=zone) + except ValueError: + continue + return {"moment": moment, "field": field, "timezone": label, "timezone_valid": valid} + return None + + +def _as_date(value: Any) -> date | None: + """Return a YAML date value as a date, or None.""" + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + if isinstance(value, str): + try: + return date.fromisoformat(value.strip()[:10]) + except ValueError: + return None + return None + + +def _as_year(value: Any) -> int | None: + """Return a year value as an int, or None.""" + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, str) and re.fullmatch(r"\d{4}", value.strip()): + return int(value) + return None + + +def _jsonable(value: Any) -> Any: + """Return a copy of a YAML value that json.dumps can write, dates as strings.""" + if isinstance(value, dict): + return {str(k): _jsonable(v) for k, v in value.items()} + if isinstance(value, list | tuple): + return [_jsonable(v) for v in value] + if isinstance(value, datetime): + return value.strftime(DATEFORMAT) + if isinstance(value, date): + return value.isoformat() + return value + + +def _edition_order(entry: dict) -> tuple[int, str]: + """Sort key that puts the newest edition last.""" + return _as_year(entry.get("year")) or 0, str(entry.get("start") or "") + + +def _new_item( + skill: str, + reason: str, + conference: str | None = None, + year: int | None = None, + *, + entry: dict | None = None, + latest_edition: dict | None = None, + queue_refs: list[dict] | None = None, + context: dict | None = None, + ambiguous_matches: list[dict] | None = None, +) -> dict: + """Create one worklist item in the output key order.""" + item = { + "id": None, + "skill": skill, + "reasons": [reason], + "conference": conference, + "year": year, + "entry": _jsonable(entry) if entry is not None else None, + "latest_edition": _jsonable(latest_edition) if latest_edition is not None else None, + "queue_refs": list(queue_refs or []), + "context": dict(context or {}), + } + if ambiguous_matches: + item["ambiguous_matches"] = ambiguous_matches + return item + + +def _item_id(item: dict) -> str: + """Return a stable id for a merged item.""" + year = "" if item["year"] is None else item["year"] + text = f"{item['skill']}|{item['conference'] or ''}|{year}" + if item["reasons"] == ["scout"]: + text = "scout" + return hashlib.sha1(text.encode("utf-8"), usedforsecurity=False).hexdigest() + + +def _by_name(item: dict) -> tuple[str, int]: + """Sort key for items within one group.""" + return (item["conference"] or "").casefold(), item["year"] or 0 + + +def deadline_checks(conferences: Iterable[dict], today: date) -> list[dict]: + """Return an enricher item for each entry whose deadline passed 1, 3 or 7 days ago. + + Days are counted from the UTC date the deadline falls on. The desk runs early in the UTC day, and an + AoE deadline closes at noon UTC the day after its own date, so a count from that date could send a + check while the CfP is still open. + + Parameters + ---------- + conferences : Iterable[dict] + Entries from conferences.yml. + today : date + The UTC date the worklist is for. + + Returns + ------- + list[dict] + Items with reason "deadline-check", most recent deadline first. + """ + items = [] + for entry in conferences: + deadline = effective_deadline(entry) + if deadline is None: + continue + days_since = (today - deadline["moment"].astimezone(timezone.utc).date()).days + if days_since not in CHECK_DAYS: + continue + context = { + "effective_deadline": deadline["moment"].isoformat(), + "deadline_field": deadline["field"], + "timezone": deadline["timezone"], + "days_since": days_since, + } + if not deadline["timezone_valid"]: + context["unknown_timezone"] = str(entry.get("timezone")) + items.append( + _new_item( + "enricher", + "deadline-check", + entry.get("conference"), + _as_year(entry.get("year")), + entry=entry, + context=context, + ), + ) + return sorted(items, key=lambda item: (item["context"]["days_since"], *_by_name(item))) + + +def tba_rotation(conferences: Iterable[dict], today: date) -> list[dict]: + """Return an enricher item for each TBA or TBD entry whose rotation slot is today. + + An entry whose cfp is None (no public CfP) or Cancelled has no deadline to find, so it is not rotated. + + Parameters + ---------- + conferences : Iterable[dict] + Entries from conferences.yml. + today : date + The UTC date the worklist is for. + + Returns + ------- + list[dict] + Items with reason "tba-rotation" for entries that start after today. + """ + items = [] + for entry in conferences: + if str(entry.get("cfp", "")).lower().strip() not in ROTATION_CFP: + continue + start = _as_date(entry.get("start")) + if start is None or start <= today: + continue + if rotation_slot(f"{entry.get('conference')}|{entry.get('year')}") != today.weekday(): + continue + items.append( + _new_item("enricher", "tba-rotation", entry.get("conference"), _as_year(entry.get("year")), entry=entry), + ) + return sorted(items, key=_by_name) + + +def stale_series(data: dict[str, list[dict]], today: date, known_mappings: dict) -> list[dict]: + """Return a new-year-checker item for each stale series whose rotation slot is today. + + Parameters + ---------- + data : dict[str, list[dict]] + Entries keyed "conferences", "archive" and "legacy". + today : date + The UTC date the worklist is for; its year is the lookback reference. + known_mappings : dict + Reverse titles.yml mapping from load_title_mappings(reverse=True). + + Returns + ------- + list[dict] + Items with reason "stale-series", each with the latest archived edition. + """ + archive = data.get("archive") or [] + editions: dict[tuple, dict] = {} + for entry in archive: + editions.setdefault((entry.get("conference"), entry.get("year")), entry) + candidates = find_candidates( + data.get("conferences") or [], + archive, + current_year=today.year, + known_mappings=known_mappings, + ) + items = [ + _new_item( + "new-year-checker", + "stale-series", + candidate["conference"], + latest_edition=editions.get((candidate["conference"], candidate["latest_year"])), + context={"latest_year": candidate["latest_year"]}, + ) + for candidate in candidates + if rotation_slot(candidate["conference"]) == today.weekday() + ] + return sorted(items, key=_by_name) + + +def _index(data: dict[str, list[dict]], known_mappings: dict) -> tuple[dict, dict]: + """Index every edition by series. + + Returns ``editions`` (series key to a list of (file, entry)) and ``lookup`` (the key of + every conference name and alt_name to the set of series keys it names). + """ + editions: dict[str, list[tuple[str, dict]]] = {} + lookup: dict[str, set[str]] = {} + for name, path in DATA_PATHS.items(): + for entry in data.get(name) or []: + conference = entry.get("conference") + if not isinstance(conference, str) or not conference.strip(): + continue + key = series_key(conference, known_mappings) + editions.setdefault(key, []).append((path, entry)) + for name_key in entry_keys(entry, known_mappings): + lookup.setdefault(name_key, set()).add(key) + return editions, lookup + + +def _queue_names(item: dict, payload: dict) -> list[str]: + """Return the names a queue item can be matched by, most specific first.""" + raw = [] + matched = payload.get("matched") + if item.get("kind") == "candidate" and payload.get("classification") == "MISSING_EDITION": + raw.append(matched.get("conference") if isinstance(matched, dict) else None) + raw.append(payload.get("conference")) + fields = payload.get("fields") + if isinstance(fields, dict): + raw.append(fields.get("conference")) + if isinstance(payload.get("names"), list): + raw.extend(payload["names"]) + raw.extend((item.get("title"), item.get("series"))) + names = [] + for value in raw: + if not isinstance(value, str): + continue + name = clean_watch_title(value)[0] + if name and name not in names: + names.append(name) + if not names and isinstance(item.get("file"), str): + names.append(Path(item["file"]).stem) + return names + + +def _queue_year(item: dict, payload: dict) -> int | None: + """Return the edition year a queue item is about, when known.""" + fields = payload.get("fields") if isinstance(payload.get("fields"), dict) else {} + title = item.get("title") + title_year = clean_watch_title(title)[1] if isinstance(title, str) else None + for value in (item.get("year"), payload.get("year"), fields.get("year"), title_year): + year = _as_year(value) + if year is not None: + return year + return None + + +def _match(names: list[str], lookup: dict, known_mappings: dict) -> tuple[str | None, list[str]]: + """Match names to series: the first unique match wins, else the first ambiguous one.""" + ambiguous = None + for name in names: + found = sorted(lookup.get(series_key(name, known_mappings), ())) + if len(found) == 1: + return name, found + if found and ambiguous is None: + ambiguous = (name, found) + if ambiguous is not None: + return ambiguous + return (names[0] if names else None), [] + + +def _pick_upcoming(upcoming: list[dict], year: int | None, today: date) -> dict: + """Return the upcoming entry for the year, else the next edition that has not ended. + + When every entry has ended (conferences.yml not yet sorted), the newest one is returned. + """ + for entry in upcoming: + if year is not None and _as_year(entry.get("year")) == year: + return entry + running = [entry for entry in upcoming if (_as_date(entry.get("end")) or date.max) >= today] + if running: + return min(running, key=_edition_order) + return max(upcoming, key=_edition_order) + + +def _edition_for(editions: list[tuple[str, dict]], year: int | None) -> dict | None: + """Return a series' edition for the year, from conferences.yml first, or None.""" + if year is None: + return None + for _, entry in editions: + if _as_year(entry.get("year")) == year: + return entry + return None + + +def _route_queue_item(item: dict, editions: dict, lookup: dict, known_mappings: dict, today: date) -> dict: + """Turn one queue item into a worklist item. + + A matched item takes its year from the entry it is bound to. Watch titles keep the year + of the edition they were created for, so a queue year that names no upcoming entry, or + an edition the new-year-checker already has, is moved to context["queue_year"]. + + A question, redirect or drift always goes to the enricher. When its series has no + upcoming entry, the item is bound to the edition for its year in any data file, and + otherwise context["no_entry"] says why entry is empty. + """ + kind = item.get("kind") + payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} + reason = kind if kind in QUEUE_KINDS else "signal" + year = _queue_year(item, payload) + name, matches = _match(_queue_names(item, payload), lookup, known_mappings) + upcoming = {key: [e for path, e in editions[key] if path == DATA_PATHS["conferences"]] for key in matches} + + if not matches: + skill = "scouter" + elif any(upcoming.values()): + skill = "enricher" + else: + skill = "new-year-checker" + if kind in ENRICHER_KINDS: + skill = "enricher" + elif kind == "registry": + skill = "scouter" + elif kind == "candidate": + skill = CANDIDATE_SKILLS.get(payload.get("classification"), skill) + + refs = [{"file": item.get("file"), "id": item.get("id")}] + if len(matches) != 1: + conference = normalize_conference_name(name, known_mappings) if name else None + representatives = [] + for key in matches: + if upcoming[key]: + path, entry = DATA_PATHS["conferences"], _pick_upcoming(upcoming[key], year, today) + else: + path, entry = max(editions[key], key=lambda edition: _edition_order(edition[1])) + representatives.append( + {"conference": entry.get("conference"), "year": _as_year(entry.get("year")), "file": path}, + ) + representatives.sort(key=lambda match: (match["conference"].casefold(), match["year"] or 0)) + context = {"no_entry": "no data file holds this series"} if skill == "enricher" and not matches else {} + return _new_item( + skill, + reason, + conference or name, + year, + queue_refs=refs, + context=context, + ambiguous_matches=representatives, + ) + + key = matches[0] + latest = max(editions[key], key=lambda edition: _edition_order(edition[1]))[1] + chosen = _pick_upcoming(upcoming[key], year, today) if upcoming[key] else None + conference = (chosen or latest).get("conference") + if skill == "enricher" and chosen is not None: + entry_year = _as_year(chosen.get("year")) + context = {"queue_year": year} if year is not None and year != entry_year else {} + return _new_item(skill, reason, conference, entry_year, entry=chosen, queue_refs=refs, context=context) + if skill == "new-year-checker": + context = {} + if year is not None and year <= (_as_year(latest.get("year")) or 0): + context, year = {"queue_year": year}, None + return _new_item(skill, reason, conference, year, latest_edition=latest, queue_refs=refs, context=context) + if skill == "enricher": + edition = _edition_for(editions[key], year) + if edition is not None: + return _new_item(skill, reason, edition.get("conference"), year, entry=edition, queue_refs=refs) + context = {"no_entry": "no data file holds an edition of this series for this year"} + return _new_item(skill, reason, conference, year, queue_refs=refs, context=context) + return _new_item(skill, reason, conference, year, queue_refs=refs) + + +def _merge(items: list[dict]) -> list[dict]: + """Merge items for the same (skill, conference, year), keeping first-seen order.""" + merged: dict[tuple, dict] = {} + for item in items: + key = (item["skill"], item["conference"], item["year"]) + target = merged.get(key) + if target is None: + merged[key] = item + continue + target["reasons"] = sorted(set(target["reasons"]) | set(item["reasons"]), key=REASONS.index) + target["queue_refs"].extend(ref for ref in item["queue_refs"] if ref not in target["queue_refs"]) + for field in ("entry", "latest_edition"): + if target[field] is None: + target[field] = item[field] + for name, value in item["context"].items(): + target["context"].setdefault(name, value) + if "ambiguous_matches" in item: + matches = target.setdefault("ambiguous_matches", []) + matches.extend(match for match in item["ambiguous_matches"] if match not in matches) + return list(merged.values()) + + +def build_worklist( + data: dict[str, list[dict]], + queue_items: Iterable[dict], + today: date, + *, + scout: bool = True, + generated: str | None = None, + known_mappings: dict | None = None, +) -> dict: + """Build the worklist document. + + Parameters + ---------- + data : dict[str, list[dict]] + Entries keyed "conferences", "archive" and "legacy", as from cfp_candidates.load_data. + queue_items : Iterable[dict] + Queue items as from load_queue: the queue schema fields plus "file" + ("queue/.json") and "series" (the file's series name). + today : date + The UTC date the worklist is for. + scout : bool + Add the open-ended scouting item. + generated : str, optional + Timestamp for the "generated" field. Defaults to the current UTC time. + known_mappings : dict, optional + Reverse titles.yml mapping from load_title_mappings(reverse=True). Loaded when omitted. + + Returns + ------- + dict + {"generated", "today", "items"}; see the module docstring for the order. + """ + if known_mappings is None: + _, known_mappings = load_title_mappings(reverse=True) + conferences = data.get("conferences") or [] + editions, lookup = _index(data, known_mappings) + + items = deadline_checks(conferences, today) + queue_order = sorted(queue_items, key=lambda q: (str(q.get("file")), str(q.get("received")), str(q.get("id")))) + items.extend(_route_queue_item(queue_item, editions, lookup, known_mappings, today) for queue_item in queue_order) + items.extend(tba_rotation(conferences, today)) + items.extend(stale_series(data, today, known_mappings)) + items = _merge(items) + if scout: + items.append(_new_item("scouter", "scout")) + for item in items: + item["id"] = _item_id(item) + + if generated is None: + generated = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return {"generated": generated, "today": today.isoformat(), "items": items} + + +def load_queue(queue_dir: Path) -> list[dict]: + """Read every queue/.json file of a desk-queue checkout. + + Parameters + ---------- + queue_dir : Path + The checkout, or its queue/ directory. + + Returns + ------- + list[dict] + Each item with "file" ("queue/.json") and "series" (from the file) added. + Unreadable files are reported on stderr and skipped. + """ + root = queue_dir / "queue" if (queue_dir / "queue").is_dir() else queue_dir + items = [] + for path in sorted(root.glob("*.json")): + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + print(f"desk_worklist: skipping {path}: {error}", file=sys.stderr) + continue + if not isinstance(document, dict) or not isinstance(document.get("items"), list): + print(f"desk_worklist: skipping {path}: no items list", file=sys.stderr) + continue + series = document.get("series") + items.extend( + {**raw, "file": f"queue/{path.name}", "series": series} + for raw in document["items"] + if isinstance(raw, dict) + ) + return items + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--queue-dir", type=Path, help="Checkout of the desk-queue branch, or its queue/ directory") + parser.add_argument("--base", type=Path, default=Path(), help="Repository root (default: current directory)") + parser.add_argument( + "--today", + type=date.fromisoformat, + help="Date to build the worklist for, YYYY-MM-DD (default: today in UTC)", + ) + parser.add_argument("--no-scout", action="store_true", help="Leave out the open-ended scouting item") + args = parser.parse_args(argv) + + if args.queue_dir is not None and not args.queue_dir.is_dir(): + parser.error(f"--queue-dir {args.queue_dir} is not a directory") + today = args.today or datetime.now(tz=timezone.utc).date() + queue_items = load_queue(args.queue_dir) if args.queue_dir is not None else [] + worklist = build_worklist(load_data(args.base), queue_items, today, scout=not args.no_scout) + print(json.dumps(worklist, indent=2)) + + +if __name__ == "__main__": + main()