From 05384ce1eef9b58077d886666e63f3d4adf75521 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Fri, 4 Sep 2026 20:06:20 +0000 Subject: [PATCH 1/6] ci: build only the Zephyr boards that have the changed module ci_set_matrix.py cannot ask make which Zephyr boards compile a module, so a change to any module built all 29 of them, and a frozen library update too although the port has no frozen modules. The module table of a Zephyr board is only known to its build, which writes it to autogen_board_info.toml. Keep it: build_release_files.py copies the file the push build generated to bin/zephyr-modules// .toml, from where the existing bin/ upload puts it on S3 next to mpy-cross and the stubs. The scheduler fetches it for the PR's base ref and builds a board when the table says the module is enabled, or when the table is missing, unreadable or does not know the module. frozen/ changes build no Zephyr boards; supervisor/ changes still build all. Until the first push build after this lands there is no table and every Zephyr board is built, as today. ZEPHYR_MODULES_URL overrides the location for testing. --- tools/build_release_files.py | 21 ++++++++++++++ tools/ci_set_matrix.py | 55 +++++++++++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/tools/build_release_files.py b/tools/build_release_files.py index 25ae0facd1b..7118c09a57b 100755 --- a/tools/build_release_files.py +++ b/tools/build_release_files.py @@ -32,6 +32,22 @@ build_all = os.environ.get("GITHUB_EVENT_NAME") != "pull_request" + +def save_zephyr_board_info(board, board_dir): + """Keep the module table a Zephyr build generated, under bin/ so it is uploaded. + + tools/ci_set_matrix.py reads it from S3 to decide which Zephyr boards a change + to a module concerns. + """ + ref = os.environ.get("GITHUB_REF_NAME") + board_info = board_dir / "autogen_board_info.toml" + if not ref or not board_info.exists(): + return + destination = pathlib.Path("../bin/zephyr-modules") / ref + destination.mkdir(parents=True, exist_ok=True) + shutil.copyfile(board_info, destination / f"{board}.toml") + + LANGUAGE_FIRST = "en_US" LANGUAGE_THRESHOLD = 10 * 1024 @@ -61,7 +77,9 @@ board_settings = {"CLEAN_REBUILD_LANGUAGES": []} with cp_toml.open("rb") as f: board_settings.update(tomllib.load(f)) + zephyr_board_dir = cp_toml.parent else: + zephyr_board_dir = None board_settings = get_settings_from_makefile("../ports/" + board_info["port"], board) board_settings["CIRCUITPY_BUILD_EXTENSIONS"] = [ extension.strip() @@ -141,6 +159,9 @@ if exit_status == 0: exit_status = 1 + if zephyr_board_dir and language == LANGUAGE_FIRST and make_result.returncode == 0: + save_zephyr_board_info(board, zephyr_board_dir) + print( "Build {board} for {language}{clean_build} took {build_duration:.2f}s and {success}".format( board=board, diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 17dace96969..60d4032b849 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -28,6 +28,8 @@ import json import pathlib import subprocess +import tomllib +import urllib.request from concurrent.futures import ThreadPoolExecutor tools_dir = pathlib.Path(__file__).resolve().parent @@ -71,6 +73,51 @@ GITHUB_MATRIX_LIMIT = 256 +# Zephyr boards don't use make, so their module tables can't be computed here. Each push +# build uploads the board's autogen_board_info.toml with the firmware +# (tools/build_release_files.py); a board whose table is missing, unreadable or does not +# know a module is built. +ZEPHYR_MODULES_URL = os.environ.get( + "ZEPHYR_MODULES_URL", + "https://adafruit-circuit-python.s3.amazonaws.com/bin/zephyr-modules/{ref}/{board}.toml", +) +zephyr_modules = {} + + +def fetch_zephyr_modules(boards): + ref = os.environ.get("GITHUB_BASE_REF") or os.environ.get("GITHUB_REF_NAME") or "main" + + def fetch(board): + url = ZEPHYR_MODULES_URL.format(board=board, ref=ref) + try: + with urllib.request.urlopen(url, timeout=10) as response: + return board, tomllib.loads(response.read().decode("utf-8"))["modules"] + except Exception as e: # noqa: BLE001 -- whatever went wrong, the board gets built + print(f" {board}: no module table ({e})") + return board, None + + need = [board for board in boards if board not in zephyr_modules] + with ThreadPoolExecutor(max_workers=os.cpu_count()) as ex: + zephyr_modules.update(ex.map(fetch, need)) + + +def zephyr_boards_for(file, module, boards): + """The Zephyr boards a change to `file` concerns, out of `boards`.""" + if file.startswith("frozen"): + # The port has no frozen modules. + return [] + if module is None: + return boards + fetch_zephyr_modules(boards) + selected = [ + board + for board in boards + if zephyr_modules.get(board) is None or zephyr_modules[board].get(module, True) + ] + print(f"Zephyr boards with {module}: {len(selected)} of {len(boards)}") + return selected + + PATTERN_WINDOWS = { ".github/", "extmod/", @@ -205,10 +252,10 @@ def get_settings(board): # the logic to build all boards breaks. boards = set(port_to_board[port] if port else all_board_ids) - # Zephyr boards don't use make, so build them and don't compute their settings. - for board in port_to_board["zephyr-cp"]: - if board in boards: - boards_to_build.add(board) + zephyr_boards = sorted(boards & port_to_board["zephyr-cp"]) + module = module_matches.group(2) if module_matches else None + boards_to_build.update(zephyr_boards_for(file, module, zephyr_boards)) + boards -= port_to_board["zephyr-cp"] for board in boards_to_build: if board in boards: From facbbb46b1bfd4e851347474c8b18672d1fd5fc5 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Thu, 10 Sep 2026 13:56:26 +0000 Subject: [PATCH 2/6] ci: read the committed module table, and annotate a stale one The board's build already commits autogen_board_info.toml and its header says other scripts use it, so read that instead of putting a fresh copy on S3. build_release_files.py is untouched again. The table now decides whether a board is built, so a stale one costs coverage instead of nothing. The build's warning about that sits in one board job's log among 29; as an annotation it shows on the run and next to the file in the pull request. The board builds once per language, so only its first build annotates. The scheduler also names the boards it left out. --- .../zephyr-cp/cptools/build_circuitpython.py | 11 +++++ tools/build_release_files.py | 21 -------- tools/ci_set_matrix.py | 49 +++++++++---------- 3 files changed, 35 insertions(+), 46 deletions(-) diff --git a/ports/zephyr-cp/cptools/build_circuitpython.py b/ports/zephyr-cp/cptools/build_circuitpython.py index 8594c245c81..c2a36eac8b3 100644 --- a/ports/zephyr-cp/cptools/build_circuitpython.py +++ b/ports/zephyr-cp/cptools/build_circuitpython.py @@ -632,6 +632,17 @@ async def build_circuitpython(): # noqa: C901 logger.warning( f"autogen_board_info.toml is missing or out of date. Please run `make BOARD={board}` locally and commit {autogen_board_info_fn}." ) + # Also as an annotation, so it shows on the run and next to the file in the + # pull request rather than in one board log out of 29. The board builds once + # per language, so only the first build of it says anything. + reported = builddir / "autogen_board_info.reported" + if not reported.exists(): + reported.touch() + print( + f"::warning file={autogen_board_info_fn.relative_to(srcdir)}::" + f"out of date, run `make BOARD={board}` and commit it", + flush=True, + ) autogen_modules.add(tomlkit.comment("extmod modules shared with MicroPython")) for extmod_module in EXTMOD_MODULES: enabled = extmod_module in enabled_modules diff --git a/tools/build_release_files.py b/tools/build_release_files.py index 7118c09a57b..25ae0facd1b 100755 --- a/tools/build_release_files.py +++ b/tools/build_release_files.py @@ -32,22 +32,6 @@ build_all = os.environ.get("GITHUB_EVENT_NAME") != "pull_request" - -def save_zephyr_board_info(board, board_dir): - """Keep the module table a Zephyr build generated, under bin/ so it is uploaded. - - tools/ci_set_matrix.py reads it from S3 to decide which Zephyr boards a change - to a module concerns. - """ - ref = os.environ.get("GITHUB_REF_NAME") - board_info = board_dir / "autogen_board_info.toml" - if not ref or not board_info.exists(): - return - destination = pathlib.Path("../bin/zephyr-modules") / ref - destination.mkdir(parents=True, exist_ok=True) - shutil.copyfile(board_info, destination / f"{board}.toml") - - LANGUAGE_FIRST = "en_US" LANGUAGE_THRESHOLD = 10 * 1024 @@ -77,9 +61,7 @@ def save_zephyr_board_info(board, board_dir): board_settings = {"CLEAN_REBUILD_LANGUAGES": []} with cp_toml.open("rb") as f: board_settings.update(tomllib.load(f)) - zephyr_board_dir = cp_toml.parent else: - zephyr_board_dir = None board_settings = get_settings_from_makefile("../ports/" + board_info["port"], board) board_settings["CIRCUITPY_BUILD_EXTENSIONS"] = [ extension.strip() @@ -159,9 +141,6 @@ def save_zephyr_board_info(board, board_dir): if exit_status == 0: exit_status = 1 - if zephyr_board_dir and language == LANGUAGE_FIRST and make_result.returncode == 0: - save_zephyr_board_info(board, zephyr_board_dir) - print( "Build {board} for {language}{clean_build} took {build_duration:.2f}s and {success}".format( board=board, diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 60d4032b849..912bae729c2 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -29,7 +29,6 @@ import pathlib import subprocess import tomllib -import urllib.request from concurrent.futures import ThreadPoolExecutor tools_dir = pathlib.Path(__file__).resolve().parent @@ -73,48 +72,48 @@ GITHUB_MATRIX_LIMIT = 256 -# Zephyr boards don't use make, so their module tables can't be computed here. Each push -# build uploads the board's autogen_board_info.toml with the firmware -# (tools/build_release_files.py); a board whose table is missing, unreadable or does not -# know a module is built. -ZEPHYR_MODULES_URL = os.environ.get( - "ZEPHYR_MODULES_URL", - "https://adafruit-circuit-python.s3.amazonaws.com/bin/zephyr-modules/{ref}/{board}.toml", -) -zephyr_modules = {} - +# Zephyr boards don't use make, so their module tables can't be computed here. Each +# board's build writes autogen_board_info.toml next to its circuitpython.toml and that +# file is committed; a board whose table is missing, unreadable or doesn't name a +# module is built. +ZEPHYR_BOARDS = tools_dir.parent / "ports" / "zephyr-cp" / "boards" +zephyr_modules = None -def fetch_zephyr_modules(boards): - ref = os.environ.get("GITHUB_BASE_REF") or os.environ.get("GITHUB_REF_NAME") or "main" - def fetch(board): - url = ZEPHYR_MODULES_URL.format(board=board, ref=ref) +def load_zephyr_modules(): + modules = {} + for board_info in ZEPHYR_BOARDS.glob("*/*/autogen_board_info.toml"): + board = f"{board_info.parent.parent.name}_{board_info.parent.name}" try: - with urllib.request.urlopen(url, timeout=10) as response: - return board, tomllib.loads(response.read().decode("utf-8"))["modules"] + with board_info.open("rb") as f: + modules[board] = tomllib.load(f)["modules"] except Exception as e: # noqa: BLE001 -- whatever went wrong, the board gets built - print(f" {board}: no module table ({e})") - return board, None - - need = [board for board in boards if board not in zephyr_modules] - with ThreadPoolExecutor(max_workers=os.cpu_count()) as ex: - zephyr_modules.update(ex.map(fetch, need)) + print(f" {board}: unusable module table ({e})") + return modules def zephyr_boards_for(file, module, boards): """The Zephyr boards a change to `file` concerns, out of `boards`.""" + global zephyr_modules if file.startswith("frozen"): # The port has no frozen modules. return [] if module is None: return boards - fetch_zephyr_modules(boards) + if zephyr_modules is None: + zephyr_modules = load_zephyr_modules() selected = [ board for board in boards - if zephyr_modules.get(board) is None or zephyr_modules[board].get(module, True) + if board not in zephyr_modules or zephyr_modules[board].get(module, True) ] + # Name the boards left out. Their tables say they don't have the module, and a table + # is only as fresh as the last build that committed it, so this is the record of what + # a stale one cost. + skipped = [board for board in boards if board not in selected] print(f"Zephyr boards with {module}: {len(selected)} of {len(boards)}") + if skipped: + print(f" no {module} according to their table: {', '.join(skipped)}") return selected From 341055d97e5598c83e17ca6c8616297119da2ba5 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Thu, 10 Sep 2026 14:09:09 +0000 Subject: [PATCH 3/6] TEMPORARY: make native_sim's table stale to show the annotation Flips synthio to false although the board has audiobusio. To be reverted once the run shows what a stale table looks like in a pull request. --- .../zephyr-cp/boards/native/native_sim/autogen_board_info.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/zephyr-cp/boards/native/native_sim/autogen_board_info.toml b/ports/zephyr-cp/boards/native/native_sim/autogen_board_info.toml index e738a839210..2b549cdcc5b 100644 --- a/ports/zephyr-cp/boards/native/native_sim/autogen_board_info.toml +++ b/ports/zephyr-cp/boards/native/native_sim/autogen_board_info.toml @@ -101,7 +101,7 @@ ssl = false storage = true struct = true supervisor = true -synthio = true # Zephyr board has audiobusio +synthio = false # Zephyr board has audiobusio terminalio = true # Zephyr board has busio tilepalettemapper = true # Zephyr board has busio time = true From ee6c95fb8861991629f51188e847ccbb870911b5 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Thu, 10 Sep 2026 15:27:05 +0000 Subject: [PATCH 4/6] Revert "TEMPORARY: make native_sim's table stale to show the annotation" This reverts commit 341055d97e5598c83e17ca6c8616297119da2ba5. --- .../zephyr-cp/boards/native/native_sim/autogen_board_info.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/zephyr-cp/boards/native/native_sim/autogen_board_info.toml b/ports/zephyr-cp/boards/native/native_sim/autogen_board_info.toml index 2b549cdcc5b..e738a839210 100644 --- a/ports/zephyr-cp/boards/native/native_sim/autogen_board_info.toml +++ b/ports/zephyr-cp/boards/native/native_sim/autogen_board_info.toml @@ -101,7 +101,7 @@ ssl = false storage = true struct = true supervisor = true -synthio = false # Zephyr board has audiobusio +synthio = true # Zephyr board has audiobusio terminalio = true # Zephyr board has busio tilepalettemapper = true # Zephyr board has busio time = true From 4884d20510e7872cfe14f61fc16d2cabb5e9d68f Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Thu, 10 Sep 2026 18:32:37 +0000 Subject: [PATCH 5/6] ci_set_matrix: decide the Zephyr boards in a plain loop One predicate per board, zephyr_board_has_module(), and a loop at the call site instead of set algebra and a helper returning a list. Same boards selected for a module, frozen and supervisor change. --- tools/ci_set_matrix.py | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index a38bf6c7158..4bf3d40806d 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -97,29 +97,13 @@ def load_zephyr_modules(): return modules -def zephyr_boards_for(file, module, boards): - """The Zephyr boards a change to `file` concerns, out of `boards`.""" +def zephyr_board_has_module(board, module): + """What the board's committed module table says. An unknown board or module counts + as yes, so the board gets built.""" global zephyr_modules - if file.startswith("frozen"): - # The port has no frozen modules. - return [] - if module is None: - return boards if zephyr_modules is None: zephyr_modules = load_zephyr_modules() - selected = [ - board - for board in boards - if board not in zephyr_modules or zephyr_modules[board].get(module, True) - ] - # Name the boards left out. Their tables say they don't have the module, and a table - # is only as fresh as the last build that committed it, so this is the record of what - # a stale one cost. - skipped = [board for board in boards if board not in selected] - print(f"Zephyr boards with {module}: {len(selected)} of {len(boards)}") - if skipped: - print(f" no {module} according to their table: {', '.join(skipped)}") - return selected + return board not in zephyr_modules or zephyr_modules[board].get(module, True) PATTERN_WINDOWS = { @@ -256,10 +240,22 @@ def get_settings(board): # the logic to build all boards breaks. boards = set(port_to_board[port] if port else all_board_ids) - zephyr_boards = sorted(boards & port_to_board["zephyr-cp"]) + # Zephyr boards don't use make, so decide them here from their committed + # module table and leave them out of the settings computation below. module = module_matches.group(2) if module_matches else None - boards_to_build.update(zephyr_boards_for(file, module, zephyr_boards)) - boards -= port_to_board["zephyr-cp"] + skipped = [] + for board in sorted(boards): + if board not in port_to_board["zephyr-cp"]: + continue + boards.remove(board) + if file.startswith("frozen"): + continue # the port has no frozen modules + if module is None or zephyr_board_has_module(board, module): + boards_to_build.add(board) + else: + skipped.append(board) + if skipped: + print(f"Zephyr boards without {module}, not built: {', '.join(skipped)}") for board in boards_to_build: if board in boards: From 3a45f2283047bd28a183f73b0aabf04a9915cdd2 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Thu, 10 Sep 2026 19:42:22 +0000 Subject: [PATCH 6/6] ci_set_matrix: simplify the Zephyr loop per review No sorted(), board_to_port[] for the port check, and no log of the boards left out. --- tools/ci_set_matrix.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 4bf3d40806d..4eef0060af5 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -243,19 +243,14 @@ def get_settings(board): # Zephyr boards don't use make, so decide them here from their committed # module table and leave them out of the settings computation below. module = module_matches.group(2) if module_matches else None - skipped = [] - for board in sorted(boards): - if board not in port_to_board["zephyr-cp"]: + for board in list(boards): # a copy, boards shrinks below + if board_to_port[board] != "zephyr-cp": continue boards.remove(board) if file.startswith("frozen"): continue # the port has no frozen modules if module is None or zephyr_board_has_module(board, module): boards_to_build.add(board) - else: - skipped.append(board) - if skipped: - print(f"Zephyr boards without {module}, not built: {', '.join(skipped)}") for board in boards_to_build: if board in boards: