diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5f8b79ee..fa0c90a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -203,17 +203,15 @@ jobs: # CI uses one per matrix job, dev machines use as many as built. B=build/esp32-${{ matrix.firmware }} PREFIX="firmware-${{ matrix.firmware }}-v$V" - # App + bootloader are unique per firmware, so staged with the prefix. - cp "$B/MoonLight.bin" "dist/${PREFIX}.bin" - cp "$B/bootloader/bootloader.bin" "dist/${PREFIX}-bootloader.bin" - # ota-data is byte-identical for ALL firmwares, and partition-table is - # identical within a flash-size group, so both are staged under SHARED names and the - # release uploads each once instead of one-per-firmware. Every matrix job - # writes the same shared filename(s); the flatten step keeps one copy - # (byte-identical, so last-writer-wins is correct). The generated - # manifests point at these same names (generate_manifest.py). - SIZE=$(jq -r .flash_settings.flash_size "$B/flasher_args.json" | tr 'A-Z' 'a-z') - cp "$B/partition_table/partition-table.bin" "dist/partition-table-$SIZE.bin" + # App, bootloader and partition table are unique per firmware (one flash size + # holds several layouts), so staged with the prefix. + cp "$B/MoonLight.bin" "dist/${PREFIX}.bin" + cp "$B/bootloader/bootloader.bin" "dist/${PREFIX}-bootloader.bin" + cp "$B/partition_table/partition-table.bin" "dist/${PREFIX}-partition-table.bin" + # ota-data is byte-identical for ALL firmwares, so it is staged under a SHARED + # name and the release uploads it once. Every matrix job writes the same file; + # the flatten step keeps one copy (byte-identical, so last-writer-wins is + # correct). The generated manifests point at this name (generate_manifest.py). cp "$B/ota_data_initial.bin" "dist/shared-ota-data.bin" # Per-firmware flasher_args.json: the release job feeds it to # generate_manifest.py so offsets come from the real build. @@ -743,7 +741,6 @@ jobs: dist/firmware-*.bin dist/firmware-*.hex dist/shared-*.bin - dist/partition-table-*.bin dist/manifest-*.json dist/MoonLight-*.tar.gz dist/MoonLight-*.zip @@ -979,7 +976,8 @@ jobs: # Use each release's OWN published manifest, not a freshly generated one: # the asset-naming scheme has changed across releases (v1.0.0 used # per-firmware `firmware--v-partition-table.bin` / `-ota-data.bin`; - # v2.0.0 uses shared `partition-table-.bin` / `shared-ota-data.bin`), + # v2.0.0 to v7.0.0 used shared `partition-table-.bin` / `shared-ota-data.bin`; + # later ones a per-firmware `-partition-table.bin` and `shared-ota-data.bin`), # so regenerating an old tag's manifest with today's generator points it at # files that release never published → a fetch-firmware 404. Each release's # own manifest already names its own files correctly; we just rewrite its diff --git a/docs/reference/testing.md b/docs/reference/testing.md index 9095971a..36cf5e74 100644 --- a/docs/reference/testing.md +++ b/docs/reference/testing.md @@ -162,7 +162,7 @@ buildImprovFrame(type=0x03, payload=[0x01]) == 49 4d 50 52 4f 56 01 03 01 01 e The JS suite proves the installer *chunks* a document correctly; the **device side that reassembles those chunks** is pinned by the C++ `unit_ImprovOpReassembler` suite (`src/core/ImprovOpReassembler.h`, the pure state machine behind the device's `APPLY_OP` handler, extracted from `platform_esp32_improv.cpp` so it's desktop-testable). It covers the full receive contract: in-order multi-chunk reassembly + NUL-termination, **duplicate-chunk rejection** and **out-of-order/skipped-seq rejection** (the guard against an installer retry corrupting the buffer), **overflow** rejection at the buffer-minus-NUL boundary, mid-stream `seq 0` abandoning a partial op, and clean recovery after every error. Encode (JS) + reassemble (C++) together prove APPLY_OP end to end without hardware. -**`test/python/test_installer_manifests.py`** (pytest), pins the web installer's per-release file contract. For every `ships: true` firmware in `mooninstaller/firmwares.json` it runs `moondeck/build/generate_manifest.py` (with a synthetic `flasher_args.json`, so no firmware build is needed) and asserts the manifest is valid (a `chipFamily` + non-empty `parts[]`) AND that **every part filename matches one of the globs the release workflow stages onto Pages** (`firmware-*.bin` / `shared-*.bin` / `partition-table-*.bin`, `shared-*.bin` covers the plain ota-data plus a MoonBase manifest's `shared-ota-data-slot0.bin` and `shared-moonbase-.bin`). A manifest that names a file outside those globs points at something the deploy never stages → the installer 404s at fetch-firmware (the failure that shipped a broken v2.0.0 installer). The test guards the manifest-generation ↔ staged-files contract; the *deploy mechanics* that stage them (per-tag, in `release.yml`) are workflow shell logic a unit test can't reach, so the two are complementary. +**`test/python/test_installer_manifests.py`** (pytest), pins the web installer's per-release file contract. For every `ships: true` firmware in `mooninstaller/firmwares.json` it runs `moondeck/build/generate_manifest.py` (with a synthetic `flasher_args.json`, so no firmware build is needed) and asserts the manifest is valid (a `chipFamily` + non-empty `parts[]`) AND that **every part filename matches one of the globs the release workflow stages onto Pages** (`firmware-*.bin` / `shared-*.bin`, where `firmware-*.bin` covers each firmware's own partition table and `shared-*.bin` covers the plain ota-data plus a MoonBase manifest's `shared-ota-data-slot0.bin` and `shared-moonbase-.bin`). A manifest that names a file outside those globs points at something the deploy never stages → the installer 404s at fetch-firmware (the failure that shipped a broken v2.0.0 installer). The test guards the manifest-generation ↔ staged-files contract; the *deploy mechanics* that stage them (per-tag, in `release.yml`) are workflow shell logic a unit test can't reach, so the two are complementary. MoonDeck's pure logic (catalog reverse-lookup, state migration) and the installer's op-walk / storage are the next host-side candidates as they accrete regression risk. diff --git a/moondeck/build/generate_manifest.py b/moondeck/build/generate_manifest.py index 009c82c8..bb2b71d1 100644 --- a/moondeck/build/generate_manifest.py +++ b/moondeck/build/generate_manifest.py @@ -46,17 +46,14 @@ # filenames the release publishes (the same names release.yml stages from # esp32/build/). # -# Two of the four parts are byte-identical across firmwares and so are shared -# (uploaded once, every firmware's manifest points at the shared name): -# * ota_data_initial.bin: a fixed 0xFF region, identical for ALL firmwares. -# * partition-table.bin: identical within a flash-size GROUP (4/8/16 MB), so -# it is named per group (`partition-table-.bin`). The {size} field is -# filled from flasher_args' flash_settings.flash_size (see resolve below). -# App + bootloader stay per-firmware ({prefix}), as both differ per variant. +# App, bootloader and partition table are per-firmware ({prefix}): firmwares of one flash +# size still differ in layout (a MoonBase table puts the app at 0xF0000, a dual-OTA table at +# 0x10000). Only ota_data_initial.bin, a fixed 0xFF region identical for ALL firmwares, is +# shared (uploaded once, every firmware's manifest points at the shared name). PART_NAME_MAP = { APP_BIN: "{prefix}.bin", "bootloader.bin": "{prefix}-bootloader.bin", - "partition-table.bin": "partition-table-{size}.bin", + "partition-table.bin": "{prefix}-partition-table.bin", "ota_data_initial.bin": "shared-ota-data.bin", } @@ -65,12 +62,10 @@ CHIP_FAMILIES = {f: chip_family(f) for f in FIRMWARES} -def parts_from_flasher_args(flasher_args: dict, prefix: str, size: str) -> list[dict]: +def parts_from_flasher_args(flasher_args: dict, prefix: str) -> list[dict]: """Convert IDF's flasher_args.json `flash_files` map into Web Tools parts. - `prefix` names the per-firmware parts (app + bootloader); `size` (e.g. "4mb") - names the shared partition-table group. `{prefix}` / `{size}` in the - PART_NAME_MAP templates are filled here; the shared parts ignore `prefix`. + `prefix` names the per-firmware parts; the shared part ignores it. """ flash_files = flasher_args.get("flash_files", {}) parts: list[dict] = [] @@ -84,7 +79,7 @@ def parts_from_flasher_args(flasher_args: dict, prefix: str, size: str) -> list[ continue offset = int(offset_hex, 16) if isinstance(offset_hex, str) else int(offset_hex) parts.append({ - "path": template.format(prefix=prefix, size=size), + "path": template.format(prefix=prefix), "offset": offset, }) # Web Tools doesn't require a specific order, but sorted-by-offset is the @@ -160,14 +155,7 @@ def esp32_parts(args, prefix: str) -> list[dict] | None: print(f"generate_manifest: {args.flasher_args} not valid JSON: {e}") return None - # Flash-size group for the shared partition-table name (partition-table-.bin). - # IDF records it in flasher_args; lowercased so "16MB" → "16mb". - size = str(flasher_args.get("flash_settings", {}).get("flash_size", "")).lower() - if not size: - print(f"generate_manifest: no flash_settings.flash_size in {args.flasher_args}") - return None - - parts = parts_from_flasher_args(flasher_args, prefix, size) + parts = parts_from_flasher_args(flasher_args, prefix) # MoonBase firmwares: flasher_args stages the app at the first app partition: the factory # slot, MoonBase's home: because IDF knows nothing about the two-image scheme. Remap the app diff --git a/moondeck/run/preview_installer.py b/moondeck/run/preview_installer.py index 67524561..44218018 100644 --- a/moondeck/run/preview_installer.py +++ b/moondeck/run/preview_installer.py @@ -139,16 +139,14 @@ def stage_local_builds(builds: list[Path]) -> list[str]: firmware = build_dir.name[len("esp32-"):] prefix = f"firmware-{firmware}-v{LOCAL_VERSION}" - # The four .bin files that go alongside a firmware in the release. Mirrors release.yml's "Stage release artifacts" step: app + bootloader are per-firmware; ota-data + partition-table are SHARED (ota-data once globally, partition-table once per flash-size group), matching the names generate_manifest.py emits so the preview's manifests resolve them. + # The four .bin files that go alongside a firmware in the release. Mirrors release.yml's "Stage release artifacts" step: app, bootloader and partition table are per-firmware; ota-data is SHARED, matching the names generate_manifest.py emits so the preview's manifests resolve them. try: - flasher = json.loads((build_dir / "flasher_args.json").read_text(encoding="utf-8")) - size = str(flasher.get("flash_settings", {}).get("flash_size", "")).lower() shutil.copy(build_dir / APP_BIN, releases_dir / f"{prefix}.bin") shutil.copy(build_dir / "bootloader" / "bootloader.bin", releases_dir / f"{prefix}-bootloader.bin") shutil.copy(build_dir / "partition_table" / "partition-table.bin", - releases_dir / f"partition-table-{size}.bin") + releases_dir / f"{prefix}-partition-table.bin") shutil.copy(build_dir / "ota_data_initial.bin", releases_dir / "shared-ota-data.bin") # MoonBase firmwares also ship the shared maintenance image + the slot-0 otadata their manifests reference (same names release.yml stages). diff --git a/mooninstaller/install.js b/mooninstaller/install.js index 48ce98ac..1b1a2c85 100644 --- a/mooninstaller/install.js +++ b/mooninstaller/install.js @@ -751,11 +751,11 @@ document.addEventListener('DOMContentLoaded', () => { + `(esptool-js) has no chip definition for it. Flash with esptool.py instead ` + `(which does support ${chip}); no dev environment needed:\n\n` + `1. Install esptool: pipx install esptool (or: pip install esptool)\n` - + `2. Download the 4 firmware-esp32s31-*.bin files from the release below.\n` + + `2. Download the 4 .bin files named below from the release.\n` + `3. Flash (replace , e.g. /dev/ttyUSB0 or COM3):\n\n` + ` esptool --chip esp32s31 --port --baud 460800 write-flash \\\n` + ` 0x2000 firmware-esp32s31--bootloader.bin \\\n` - + ` 0x8000 partition-table-16mb.bin \\\n` + + ` 0x8000 firmware-esp32s31--partition-table.bin \\\n` + ` 0xe000 shared-ota-data.bin \\\n` + ` 0x10000 firmware-esp32s31-.bin\n\n` + `Then re-open this page to provision WiFi, or use the device's own UI. ` diff --git a/test/js/installer-firmware-merge.test.mjs b/test/js/installer-firmware-merge.test.mjs index 1fd71d95..7a23d4a0 100644 --- a/test/js/installer-firmware-merge.test.mjs +++ b/test/js/installer-firmware-merge.test.mjs @@ -29,7 +29,7 @@ test("parseFirmwaresFromAssets: a firmware needs BOTH a manifest and an app .bin asset("firmware-esp32-v2.0.0.bin"), // bootloader / partition-table / ota-data are install fragments, not the app image asset("firmware-esp32-v2.0.0-bootloader.bin"), - asset("partition-table-4mb.bin"), + asset("firmware-esp32-v2.0.0-partition-table.bin"), // a manifest with NO matching app .bin (partial upload) — must be dropped asset("manifest-esp32s3-n16r8.json"), ]; diff --git a/test/python/test_installer_manifests.py b/test/python/test_installer_manifests.py index 7d51e93c..a9057ce1 100644 --- a/test/python/test_installer_manifests.py +++ b/test/python/test_installer_manifests.py @@ -5,7 +5,7 @@ parts to flash. Those parts MUST be files the release workflow actually stages onto Pages — the staging step (`.github/workflows/release.yml`) downloads them with these globs: - firmware-*.bin shared-ota-data.bin partition-table-*.bin + firmware-*.bin shared-*.bin A manifest that references a filename outside those globs points at a file that never reaches Pages → the installer 404s at fetch-firmware. That exact mismatch (a manifest referencing a @@ -33,8 +33,9 @@ # The exact globs the deploy's `gh release download` stages onto Pages (release.yml, # "Stage cumulative release content"). A manifest part must match one of these, or the -# installer fetches a file that isn't there. Keep in lockstep with that step's --pattern list. -STAGED_GLOBS = ["firmware-*.bin", "shared-*.bin", "partition-table-*.bin"] +# installer fetches a file that isn't there. Keep in lockstep with that step's --pattern list, +# less its `partition-table-*.bin`, which only the releases up to v7.0.0 publish. +STAGED_GLOBS = ["firmware-*.bin", "shared-*.bin"] # shared-*.bin covers shared-ota-data.bin plus the MoonBase pair (shared-moonbase-.bin, # shared-ota-data-slot0.bin); the same glob appears in release.yml's upload files and the # Pages self-host download patterns, which is exactly the sync this test guards. @@ -42,7 +43,6 @@ # A minimal but realistic IDF flasher_args.json — the four files every MoonLight build emits. # generate_manifest.py maps these to the bundle names (firmware--v.bin, etc.). FLASHER_ARGS = { - "flash_settings": {"flash_size": "4MB"}, "flash_files": { "0x1000": "bootloader/bootloader.bin", "0x8000": "partition_table/partition-table.bin", @@ -130,6 +130,16 @@ def test_app_and_bootloader_carry_the_version(tmp_path): assert f"firmware-{fw}-v{VERSION}-bootloader.bin" in names, f"{fw}: bootloader part misnamed: {names}" +def test_every_firmware_names_its_own_partition_table(tmp_path): + """Firmwares of one flash size still differ in layout (esp32-pico's MoonBase table puts the app at 0xF0000, esp32s3-n8r8's dual-OTA table at 0x10000), so each manifest names its own partition table: a name shared by flash size lets one firmware's table replace another's in the release, and the installer writes the wrong layout.""" + tables = {} + for fw in (f for f in _shipping_firmwares() if _platform(f) == "esp32"): + parts = _generate(fw, tmp_path)["builds"][0]["parts"] + tables[fw] = next(p["path"].rsplit("/", 1)[-1] for p in parts if p["offset"] == 0x8000) + assert len(set(tables.values())) == len(tables), f"firmwares share a partition-table file: {tables}" + assert all(name == f"firmware-{fw}-v{VERSION}-partition-table.bin" for fw, name in tables.items()), tables + + def test_a_teensy_manifest_is_its_flat_image_at_offset_zero(tmp_path): """The installer's Teensy writer takes one image starting at the flash base, so the manifest names the release's `.bin` at 0 under the Teensy family.""" teensy = [f for f in _shipping_firmwares() if _platform(f) == "teensy"]