diff --git a/CLAUDE.md b/CLAUDE.md index fc076d5..950ed66 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -165,13 +165,18 @@ Backends: `spi_flash.c` (fmc100), `spi_flash_hisfc350.c` (V1-era parts), U-Boot device discovery. - **Firmware** (`src/defib/firmware.py`) — downloads OpenIPC releases from GitHub, caches under `XDG_CACHE_HOME`. Most chips use one - `u-boot--universal.bin`; the u-boot-xmedia SoCs in - `PER_FLASH_TYPE_UBOOT` (hi3516ev200/ev300, hi3518ev300, hi3516dv200, - gk7205v500/v510/v530) publish `u-boot--nor.bin` and `-nand.bin` - instead, so callers pass `flash_type` (NOR when unknown). On NAND those SoCs - install the UBI-only layout (`NAND_UBI_LAYOUT` in `install/layout.py`: 768k - boot, 256k env, rest ubi), whose mtdparts/bootcmd/bootargs come from the - U-Boot default env; `NAND_LAYOUT` is the legacy split layout for other chips. + `u-boot--universal.bin`; the SoCs in `PER_FLASH_TYPE_UBOOT` (the + u-boot-xmedia ones: hi3516ev200/ev300, hi3518ev300, hi3516dv200, + gk7205v500/v510/v530; and the OpenIPC/u-boot gk7202v300, gk7205v200/v300, + gk7605v100) publish `u-boot--nor.bin` and `-nand.bin` instead, so + callers pass `flash_type` (NOR when unknown). On NAND those SoCs install the + UBI-only layout (`NAND_UBI_LAYOUT` in `install/layout.py`: 768k boot, 256k + env, rest ubi), whose mtdparts/bootcmd/bootargs come from the U-Boot default + env; `NAND_LAYOUT` is the legacy split layout for other chips. The + `NOR_SPLIT_SOCS` (gk7205v200 family) also install the split NOR package: + one `firmware.bin.` at 0x50000, the rest of the chip erased, which + the kernel divides at boot; their NOR U-Boot scans the layout, so the env + stage loads its defaults (`env default -a`) on either NOR layout. - **Capture** (`src/defib/capture/`) — record/replay UART sessions in `.dcap`. - Loose modules worth knowing: `flashdump.py` (dump flash through a U-Boot console), `ubi.py` (extract UBIFS volumes from raw UBI), `uboot_env.py`, diff --git a/README.md b/README.md index 07d7843..a5594df 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,33 @@ gk7205v510/v530 NAND packages are published under board `gk7205v500`. `install --nand` on any other chip still uses the older split layout (raw kernel partition, `mtdparts` and `bootcmd` set by the installer). +gk7202v300, gk7205v200, gk7205v300 and gk7605v100 take their U-Boot from +OpenIPC/u-boot (`openipc-hisilicon`), also published per flash type. That NOR +U-Boot works out the layout at every boot, so `install` loads its default +environment (`env default -a`, keeping the camera's `ethaddr`) instead of +writing `mtdparts` and `bootcmd`. Besides the fixed-slot package, their firmware +ships a split NOR package, `openipc.-nor--split.tgz`, and +`install` recognises it by its contents: + +| Offset | Size | Contents | +|--------|------|----------| +| `0x000000` | 256K | boot: `u-boot--nor.bin` | +| `0x040000` | 64K | env | +| `0x050000` | rest of chip | firmware: `firmware.bin.` (a hashed FIT, then the squashfs at the next 64 KiB); the kernel divides it into `kernel`, `rootfs` and `rootfs_data` at boot | + +```bash +defib install -c gk7205v300 \ + --firmware openipc.gk7205v300-nor-ultimate-split.tgz \ + -p /dev/ttyUSB0 --power-cycle +``` + +The `rootfs` stage writes `firmware.bin` at `0x50000` and erases the rest of the +chip, so the first boot starts with an empty overlay; the `kernel` and +`rootfs-data` stages have nothing of their own to do. Only that U-Boot boots the +image: a plan without the `uboot` stage warns that the one already on the flash +must be it. See the [OpenIPC wiki](https://github.com/OpenIPC/wiki/blob/master/en/nor-split-layout.md) +for the layout itself. + Targets that must bootstrap through a stock U-Boot use an explicit U-Boot variant. For example, HiWatch DS-I203 uses: @@ -135,7 +162,8 @@ cleanup behavior. The release U-Boot owns boot-critical hardware initialization such as DDR cold-init and RAM probing limits. Defib owns the layout it actually flashes: it detects NOR capacity, selects the standard OpenIPC 8/16/32 MiB layout, and -persists the matching `mtdparts` before the first Linux boot. Device runtime +persists the matching `mtdparts` before the first Linux boot (except on the +gk7205v200 family above, whose U-Boot sets `mtdparts` itself). Device runtime policy such as `osmem`, Linux PHY parameters and sensor selection belongs to the firmware/device profile. An explicit local U-Boot can still be supplied with `--uboot`. diff --git a/src/defib/firmware.py b/src/defib/firmware.py index f7d9902..06880da 100644 --- a/src/defib/firmware.py +++ b/src/defib/firmware.py @@ -32,7 +32,6 @@ # Chips that have pre-built u-boot-{chip}-universal.bin binaries on OpenIPC. # CV6xx chips are deliberately absent — see CV6XX_BOOT_VARIANTS below. AVAILABLE_FIRMWARE: set[str] = { - "gk7202v300", "gk7205v200", "gk7205v300", "gk7605v100", "hi3516av100", "hi3516av200", "hi3516av300", "hi3516cv100", "hi3516cv200", "hi3516cv300", "hi3516cv500", "hi3516dv100", "hi3516dv300", @@ -42,16 +41,28 @@ "t40a", "t40n", "t40xp", } -# SoCs whose OpenIPC U-Boot is built from OpenIPC/u-boot-xmedia and published -# once per flash type as u-boot-{chip}-nor.bin / u-boot-{chip}-nand.bin. The +# SoCs whose OpenIPC U-Boot is published once per flash type as +# u-boot-{chip}-nor.bin / u-boot-{chip}-nand.bin: the u-boot-xmedia SoCs, and +# the gk7205v200 family built from OpenIPC/u-boot (openipc-hisilicon). The # NAND build owns the UBI-only layout (768k boot, 256k env, rest ubi) in its # default environment, so a NOR image must never be installed on NAND or vice -# versa. The old u-boot-{chip}-universal.bin of the HiSilicon members was the -# retired u-boot-hi3516ev200 build with the split NAND layout; it is never -# downloaded for these SoCs again. +# versa. The old u-boot-{chip}-universal.bin of these SoCs came from retired +# per-SoC repositories (u-boot-hi3516ev200, u-boot-gk7205v200); it is never +# downloaded for them again. PER_FLASH_TYPE_UBOOT: frozenset[str] = frozenset({ "hi3516ev200", "hi3516ev300", "hi3518ev300", "hi3516dv200", "gk7205v500", "gk7205v510", "gk7205v530", + "gk7202v300", "gk7205v200", "gk7205v300", "gk7605v100", +}) + +# SoCs whose OpenIPC firmware also ships the split NOR layout, +# openipc.-nor--split.tgz: one firmware.bin (a hashed FIT, then +# the squashfs at the next 64 KiB) written at 0x50000, which the kernel divides +# into kernel, rootfs and rootfs_data at boot. Their OpenIPC/u-boot NOR build +# boots it, and also scans the fixed slots, so it owns mtdparts and bootcmd on +# either layout. +NOR_SPLIT_SOCS: frozenset[str] = frozenset({ + "gk7202v300", "gk7205v200", "gk7205v300", "gk7605v100", }) FLASH_TYPES: tuple[str, ...] = ("nor", "nand") @@ -145,6 +156,12 @@ def uses_per_flash_type_uboot(chip: str) -> bool: return CHIP_TO_FIRMWARE.get(base, base) in PER_FLASH_TYPE_UBOOT +def supports_nor_split(chip: str) -> bool: + """True for SoCs whose firmware and NOR U-Boot use the split NOR layout.""" + base = _strip_variant(chip).lower() + return CHIP_TO_FIRMWARE.get(base, base) in NOR_SPLIT_SOCS + + def asset_name(chip: str, flash_type: str | None = None) -> str | None: """Published release filename for a chip, or None if there isn't one. diff --git a/src/defib/install/firmware.py b/src/defib/install/firmware.py index b99a2e4..45975fb 100644 --- a/src/defib/install/firmware.py +++ b/src/defib/install/firmware.py @@ -17,12 +17,51 @@ class FirmwareBundle: For a UBI-only NAND package the kernel lives inside the rootfs image (``/boot/fitImage`` in the UBIFS volume): ``kernel_name`` is empty and ``kernel`` is ``b""``. + + For a split NOR package (``openipc.-nor--split.tgz``) the + one image is ``firmware.bin.``: it is carried as the rootfs, the + kernel is empty, and ``split`` is set. """ kernel_name: str kernel: bytes rootfs_name: str rootfs: bytes + split: bool = False + + +ERASE_BLOCK = 0x10000 +FDT_MAGIC = b"\xd0\x0d\xfe\xed" +SQUASHFS_MAGIC = b"hsqs" + + +def split_image_end(image: bytes) -> int: + """Where rootfs_data starts, relative to the image, for a split firmware.bin. + + The FIT's ``totalsize`` (header bytes 4..7, big-endian) rounded up to a + 64 KiB block is where the squashfs starts; its ``bytes_used`` (superblock + offset 40, little-endian) rounded up the same way is where it ends. The + kernel's mtdsplit, U-Boot and sysupgrade all draw the same lines. + Raises ValueError for anything else. + """ + if image[:4] != FDT_MAGIC: + raise ValueError("firmware.bin does not start with a FIT image") + fit = int.from_bytes(image[4:8], "big") + squash = (fit + ERASE_BLOCK - 1) // ERASE_BLOCK * ERASE_BLOCK + if not fit or squash + 48 > len(image) or image[squash:squash + 4] != SQUASHFS_MAGIC: + raise ValueError("firmware.bin has no squashfs at the 64 KiB block after its FIT") + used = int.from_bytes(image[squash + 40:squash + 48], "little") + end = squash + (used + ERASE_BLOCK - 1) // ERASE_BLOCK * ERASE_BLOCK + if not used or len(image) < squash + used: + raise ValueError( + "firmware.bin ends inside its squashfs: the image is truncated" + ) + if end < len(image): + raise ValueError( + "firmware.bin runs past the end of its squashfs: not an image as the " + "OpenIPC build packs it" + ) + return end def _is_ubi_member(name: str) -> bool: @@ -45,6 +84,8 @@ def load_firmware_bundle( rootfs_name = "" rootfs: bytes | None = None ubi_members: list[str] = [] + split_name = "" + split_image: bytes | None = None expected_md5: dict[str, str] = {} with tarfile.open(path, "r:gz") as archive: @@ -58,6 +99,9 @@ def load_firmware_bundle( line = stream.read().decode().strip() if line: expected_md5[name.removesuffix(".md5sum")] = line.split()[0] + elif name.rsplit("/", 1)[-1].startswith("firmware.bin"): + split_name = name + split_image = stream.read() elif ubi_only: if _is_ubi_member(name): ubi_members.append(name) @@ -70,7 +114,37 @@ def load_firmware_bundle( rootfs_name = name rootfs = stream.read() + if split_image is not None and not ubi_only: + if kernel or rootfs: + raise ValueError( + "tarball holds both firmware.bin and a separate kernel/rootfs; " + "expected one OpenIPC package or the other" + ) + expected = expected_md5.get(split_name) + if expected is not None and hashlib.md5(split_image).hexdigest() != expected: + raise ValueError(f"MD5 mismatch for {split_name}") + split_image_end(split_image) + board = split_name.rsplit("/", 1)[-1].removeprefix("firmware.bin").lstrip(".") + if boards is not None and board not in boards: + raise ValueError( + f"{split_name} is built for {board or 'an unnamed board'}, not " + + " or ".join(sorted(boards)) + + "; refusing to write another board's firmware" + ) + return FirmwareBundle( + kernel_name="", + kernel=b"", + rootfs_name=split_name, + rootfs=split_image, + split=True, + ) + if ubi_only: + if not rootfs and split_image is not None: + raise ValueError( + f"{split_name} is a split NOR package (openipc.-nor-" + "-split.tgz); it cannot be installed on NAND" + ) if not rootfs: raise ValueError( "tarball has no rootfs.ubi. image; this NAND layout needs " diff --git a/src/defib/install/layout.py b/src/defib/install/layout.py index 3a9bdd4..6a7764f 100644 --- a/src/defib/install/layout.py +++ b/src/defib/install/layout.py @@ -63,9 +63,28 @@ def ubi_nand_boards(chip: str) -> set[str]: - """The ``rootfs.ubi.`` suffixes a NAND package for ``chip`` may carry.""" + """The ``rootfs.ubi.`` suffixes a NAND package for ``chip`` may carry. + + An aliased chip (gk7201v200, gk7205v210, ...) installs the U-Boot of the SoC + it is published under, and that SoC's package with it. + """ + from defib.firmware import CHIP_TO_FIRMWARE + base = chip.partition(":")[0] - return {base, _UBI_NAND_BOARD.get(base, base)} + published = CHIP_TO_FIRMWARE.get(base.lower(), base) + return {base, published, _UBI_NAND_BOARD.get(base, base), _UBI_NAND_BOARD.get(published, published)} + +def split_boards(chip: str) -> set[str]: + """The ``firmware.bin.`` suffixes a split NOR package for ``chip`` may carry. + + Aliased chips (gk7205v210, gk7201v200, ...) install the image of the SoC + they are published under. + """ + from defib.firmware import CHIP_TO_FIRMWARE + + base = chip.partition(":")[0].lower() + return {base, CHIP_TO_FIRMWARE.get(base, base)} + _MTDPART_RE = re.compile( r"^(?P-|(?:0x[0-9a-f]+|\d+)[kmg]?)" @@ -207,9 +226,14 @@ def uboot_sf_lock_unsupported(response: str) -> bool: # subcommand. That is compatibility information only when the table itself # contains no sf lock entry. If sf lock is listed, a Usage response means our # invocation was rejected and must remain a hard failure. - usage_seen = any(line.startswith("usage:") for line in lines) + # + # Only the table counts. A console that echoes the command puts "sf lock 0" + # above "Usage:", and that echo is not an entry: OpenIPC/u-boot's sf has no + # lock subcommand at all and answers exactly this way. + usage_at = next((i for i, line in enumerate(lines) if line.startswith("usage:")), None) + usage_seen = usage_at is not None sf_usage: list[str] = [] - for line in lines: + for line in lines[usage_at or 0:] if usage_seen else []: candidate = line.removeprefix("usage:").strip() if line.startswith("usage:") else line if re.match(r"^sf(?:\s|$)", candidate): sf_usage.append(candidate) @@ -371,6 +395,28 @@ def detect_nor_size_mb(response: str) -> int | None: return None +NOR_SIZE_CANDIDATES_MB = (32, 16, 8) + + +def nor_size_probe_read(ram_addr: int, size_mb: int) -> str: + """A 16-byte ``sf read`` that only a chip of at least ``size_mb`` accepts. + + OpenIPC/u-boot (openipc-hisilicon) probes the flash at boot and its + ``sf probe`` prints nothing at the prompt, so the size it found is not on + offer; but ``sf read`` refuses an offset past the chip ("Offset exceeds + device limit"). The largest candidate whose last 16 bytes read is the size. + """ + return f"sf read 0x{ram_addr:x} 0x{size_mb * 1024 * 1024 - 16:x} 0x10" + + +def sf_read_succeeded(response: str) -> bool: + """True when ``sf read`` reported the read done and nothing refused it.""" + text = response.lower() + if "exceeds" in text or "past flash size" in text or "error" in text: + return False + return "read: ok" in text + + def select_nor_size_mb( override: int, detected: int | None, diff --git a/src/defib/install/orchestrator.py b/src/defib/install/orchestrator.py index 5a7c493..37a00af 100644 --- a/src/defib/install/orchestrator.py +++ b/src/defib/install/orchestrator.py @@ -13,7 +13,7 @@ import typer -from defib.install.firmware import load_firmware_bundle +from defib.install.firmware import load_firmware_bundle, split_image_end from defib.install.layout import ( NAND_LAYOUT, NAND_UBI_LAYOUT, @@ -24,10 +24,14 @@ erased_region_crc, mtdparts_partition_offset, nand_bootargs, + NOR_SIZE_CANDIDATES_MB, nor_layout, + nor_size_probe_read, + sf_read_succeeded, nor_mtdparts, parse_nand_erase_range, parse_uboot_crc32, + split_boards, ubi_nand_boards, select_nor_size_mb, set_uboot_env_verified, @@ -58,6 +62,7 @@ async def run_install(request: InstallRequest) -> None: get_cached_path, has_firmware, pad_to_size, + supports_nor_split, uses_per_flash_type_uboot, ) from defib.flashdump import get_ram_staging_addr, send_command @@ -224,15 +229,51 @@ def warn(message: str) -> None: if request.stages or request.skip_stages: console.print(f" Stages: [cyan]{', '.join(stages)}[/cyan]") + # The gk7205v200 family's OpenIPC/u-boot works out the NOR layout at every + # boot -- fixed slots or the split layout -- and owns mtdparts and bootcmd + # on both: its environment has no bootcmdnor to point at. + scanning_nor_uboot = not nand and supports_nor_split(chip) try: firmware = load_firmware_bundle( firmware_path, ubi_only=ubi_layout, - boards=ubi_nand_boards(chip) if ubi_layout else None, + boards=( + ubi_nand_boards(chip) + if ubi_layout + else split_boards(chip) if scanning_nor_uboot else None + ), ) except ValueError as exc: fail(str(exc)) + # openipc.-nor--split.tgz: one firmware.bin at 0x50000, + # which the kernel divides into kernel, rootfs and rootfs_data at boot. + split = firmware.split + if split: + if nand: + fail("A split NOR package (firmware.bin) cannot be installed on NAND", exit_code=2) + if not supports_nor_split(chip): + fail(f"{chip} does not use the split NOR layout (firmware.bin)", exit_code=2) + if has_stock_uboot: + fail( + "A stock U-Boot migration installs the fixed-slot package " + "(uImage and rootfs.squashfs), not firmware.bin", + exit_code=2, + ) + if wipe_rootfs_data and "rootfs" not in stage_set: + fail( + "--wipe-rootfs-data on the split layout needs the rootfs stage: " + "where rootfs_data starts depends on the image written there", + exit_code=2, + ) + if "rootfs" in stage_set and "uboot" not in stage_set: + warn( + "firmware.bin boots only on an OpenIPC U-Boot that knows the split " + "layout; the uboot stage is not in this plan, so the U-Boot " + "already on the flash must be one." + ) + needs_tftp = bool(stage_set & {"uboot", "rootfs"}) + kernel_name = firmware.kernel_name kernel_data = firmware.kernel rootfs_name = firmware.rootfs_name @@ -253,6 +294,11 @@ def warn(message: str) -> None: if ubi_layout: console.print(" Kernel: [cyan]inside the UBI image (/boot/fitImage)[/cyan]") console.print(f" UBI: [cyan]{rootfs_name}[/cyan] ({len(rootfs_data)} bytes)") + elif split: + console.print( + f" Image: [cyan]{rootfs_name}[/cyan] ({len(rootfs_data)} bytes; " + "split NOR layout, FIT + squashfs at 0x50000)" + ) else: console.print(f" Kernel: [cyan]{kernel_name}[/cyan] ({len(kernel_data)} bytes)") console.print(f" Rootfs: [cyan]{rootfs_name}[/cyan] ({len(rootfs_data)} bytes)") @@ -817,6 +863,21 @@ async def _set_env_verified_or_fail(key: str, value: str) -> None: nor_erase_block = int(block_match.group(1)) * 1024 detected_nor_size = detect_nor_size_mb(resp) + if detected_nor_size is None: + # A U-Boot that probed the flash at boot may print nothing here; + # ask the chip instead, with reads that only fit a big enough one. + # A refused read is the expected answer on a smaller chip, so it + # must not end the install; the download-mode client reports the + # command's status itself, the shell its text. + for candidate in NOR_SIZE_CANDIDATES_MB: + read_ok, read_resp = await _cmd_result( + nor_size_probe_read(ram_addr, candidate), + timeout=5.0, + allow_failure=True, + ) + if read_ok and (download_mode or sf_read_succeeded(read_resp)): + detected_nor_size = candidate + break if nor_size and detected_nor_size and nor_size != detected_nor_size: warn( f"--nor-size {nor_size} overrides detected " @@ -834,9 +895,18 @@ async def _set_env_verified_or_fail(key: str, value: str) -> None: k_off, k_sz = layout["kernel"] r_off, r_sz = layout["rootfs"] - if len(kernel_data) > k_sz: + if split: + # One image from the kernel offset; rootfs_data takes what is left, + # and needs at least one erase block of it. + chip_end = nor_size * 1024 * 1024 + if k_off + split_image_end(rootfs_data) + nor_erase_block > chip_end: + await close_and_fail( + f"{rootfs_name} does not fit {nor_size} MiB NOR with room " + f"for settings: it ends at 0x{k_off + split_image_end(rootfs_data):X}" + ) + elif len(kernel_data) > k_sz: await close_and_fail(f"Kernel too large: {len(kernel_data)} > {k_sz}") - if len(rootfs_data) > r_sz: + elif len(rootfs_data) > r_sz: await close_and_fail(f"Rootfs too large: {len(rootfs_data)} > {r_sz}") persistent_nor_stages = {"uboot", "kernel", "rootfs", "rootfs-data", "env"} @@ -858,6 +928,11 @@ async def _set_env_verified_or_fail(key: str, value: str) -> None: assert layout is not None k_off, k_sz = layout["kernel"] r_off, r_sz = layout["rootfs"] + if split: + # firmware.bin goes where the kernel slot starts, and the erase + # runs to the end of the chip: nothing of the old layout survives + # to be found by U-Boot's scan or mistaken for an overlay. + r_off, r_sz = k_off, nor_size * 1024 * 1024 - k_off # --- Step 5: Pick a TFTP backend, stage / start, then drive U-Boot --- # @@ -890,7 +965,7 @@ async def _set_env_verified_or_fail(key: str, value: str) -> None: tftp_files: dict[str, bytes] = {} if "uboot" in stage_set: tftp_files[tftp_alias["uboot"]] = uboot_data - if "kernel" in stage_set and not ubi_layout: + if "kernel" in stage_set and not ubi_layout and not split: tftp_files[tftp_alias["kernel"]] = kernel_data if "rootfs" in stage_set: tftp_files[tftp_alias["rootfs"]] = rootfs_data @@ -1199,10 +1274,13 @@ async def tftp_and_flash( f" Erase: 0x{flash_off:X}+0x{erase_len:X} " f"(payload 0x{len(orig_data):X})" ) - erase_resp = await _cmd( - erase_cmd, - timeout=180.0 if not nand else 120.0, + # About a second per 64 KiB block at worst: the split layout + # erases the whole chip after the environment here, 0x1FB0000 + # on 32 MiB, not one fixed-size partition. + erase_timeout = ( + max(180.0, erase_len / 0x10000 * 1.0) if not nand else 120.0 ) + erase_resp = await _cmd(erase_cmd, timeout=erase_timeout) erase_error = uboot_flash_command_error(erase_resp) if erase_error: console.print( @@ -1282,6 +1360,12 @@ async def tftp_and_flash( "layout; the kernel ships inside the UBI image as " "/boot/fitImage (written by the rootfs stage). Nothing to do." ) + elif "kernel" in stage_set and split: + if output == "human": + console.print( + "\n [bold]kernel[/bold]: the FIT at the start of " + "firmware.bin (written by the rootfs stage). Nothing to do." + ) elif "kernel" in stage_set: await tftp_and_flash( "kernel", tftp_alias["kernel"], kernel_data, k_off, k_sz @@ -1487,9 +1571,20 @@ async def flash_ubi_image() -> None: console.print(" [green]rootfs (UBI) OK[/green]") elif "rootfs" in stage_set: await tftp_and_flash( - "rootfs", tftp_alias["rootfs"], rootfs_data, r_off, r_sz + "firmware.bin" if split else "rootfs", + tftp_alias["rootfs"], rootfs_data, r_off, r_sz, ) + if "rootfs-data" in stage_set and split and output == "human": + if "rootfs" in stage_set: + note = ( + "the rootfs stage erased everything after firmware.bin; " + "the first boot makes an empty overlay there" + ) + else: + note = "the existing overlay is left untouched" + console.print(f"\n [bold]rootfs-data[/bold]: {note}. Nothing to do.") + if "rootfs-data" in stage_set and ubi_layout and output == "human": if "rootfs" in stage_set: note = "the UBI image just written carries an empty rootfs_data volume" @@ -1499,6 +1594,7 @@ async def flash_ubi_image() -> None: erase_rootfs_data = ( not nand + and not split and ( wipe_rootfs_data or ("rootfs-data" in stage_set and has_stock_uboot) @@ -1656,6 +1752,38 @@ async def flash_ubi_image() -> None: ) if output == "human": console.print(" U-Boot default environment loaded") + elif "env" in stage_set and scanning_nor_uboot and not has_stock_uboot: + # This U-Boot works out the NOR layout itself at every boot and + # sets mtdparts and the root device to match, so its defaults + # are the environment; only the camera's MAC is carried across. + if output == "human": + console.print( + "\n [bold]Setting boot environment[/bold] " + "(U-Boot scans the NOR layout)" + ) + if "uboot" in stage_set: + pre_default_resp = await _optional_printenv("ethaddr", timeout=5.0) + ubi_layout_eth = parse_printenv_value(pre_default_resp, "ethaddr") + # totalmem is the DRAM this U-Boot measured at boot, set in + # RAM only. Linux sizes its media memory from the saved + # value (set_allocator) and without one assumes 64 MB -- + # half of a gk7205v300. + mem_resp = await _optional_printenv("totalmem", timeout=5.0) + totalmem = parse_printenv_value(mem_resp, "totalmem") + default_resp = await _cmd("env default -a", timeout=5.0) + default_error = uboot_flash_command_error(default_resp) + if default_error: + raise RuntimeError( + f"env default -a failed ({default_error}): " + f"{default_resp.strip()[-200:]}" + ) + if totalmem: + await _cmd(f"setenv totalmem {totalmem}", timeout=3.0) + if output == "human": + console.print( + " U-Boot default environment loaded" + + (f", totalmem {totalmem}" if totalmem else "") + ) elif "env" in stage_set and nand: if output == "human": console.print("\n [bold]Setting boot environment[/bold] (NAND)") diff --git a/tests/test_firmware.py b/tests/test_firmware.py index 91116be..b9feb7f 100644 --- a/tests/test_firmware.py +++ b/tests/test_firmware.py @@ -35,8 +35,8 @@ def test_aliased_chip(self): assert "hi3518ev200" in url def test_url_format(self): - url = firmware_url("gk7205v200") - assert url == "https://github.com/OpenIPC/firmware/releases/download/latest/u-boot-gk7205v200-universal.bin" + url = firmware_url("hi3516cv300") + assert url == "https://github.com/OpenIPC/firmware/releases/download/latest/u-boot-hi3516cv300-universal.bin" class TestHasFirmware: @@ -58,7 +58,7 @@ def test_at_least_20_chips(self): assert len(AVAILABLE_FIRMWARE) >= 20 def test_common_chips_included(self): - for chip in ["gk7205v200", "hi3518ev200"]: + for chip in ["hi3516cv300", "hi3518ev200"]: assert chip in AVAILABLE_FIRMWARE def test_per_flash_type_socs_are_not_universal(self): @@ -265,11 +265,13 @@ def fake_urlopen(req, timeout=None): class TestPerFlashTypeUBoot: - """u-boot-xmedia SoCs publish u-boot--{nor,nand}.bin, not -universal.""" + """u-boot-xmedia SoCs and the gk7205v200 family (OpenIPC/u-boot) publish + u-boot--{nor,nand}.bin, not -universal.""" SOCS = ( "hi3516ev200", "hi3516ev300", "hi3518ev300", "hi3516dv200", "gk7205v500", "gk7205v510", "gk7205v530", + "gk7202v300", "gk7205v200", "gk7205v300", "gk7605v100", ) def test_set_matches_the_published_socs(self): diff --git a/tests/test_install_flash_helpers.py b/tests/test_install_flash_helpers.py index 117ee7a..1968f70 100644 --- a/tests/test_install_flash_helpers.py +++ b/tests/test_install_flash_helpers.py @@ -18,6 +18,7 @@ uboot_sf_lock_unsupported, verify_spi_environment_crc, ) +from defib.install.layout import nor_size_probe_read, sf_read_succeeded def test_generic_tftp_keeps_explicit_ram_address(): @@ -182,6 +183,22 @@ def test_sf_lock_usage_without_lock_entry_is_unsupported(): assert uboot_sf_lock_unsupported(response) is True +def test_sf_lock_echoed_above_usage_without_lock_entry_is_unsupported(): + """OpenIPC/u-boot (openipc-hisilicon): the console echoes the command, then + prints the sf table, which has `sf protect` but no `sf lock`.""" + response = ( + "sf lock 0\n" + "sf - SPI flash sub-system\n" + "\n" + "Usage:\n" + "sf probe [[bus:]cs] [hz] [mode] - init flash device on given SPI bus\n" + "sf read addr offset|partition len - read `len' bytes\n" + "sf protect lock/unlock sector len - protect/unprotect 'len' bytes\n" + "OpenIPC # " + ) + assert uboot_sf_lock_unsupported(response) is True + + def test_sf_lock_usage_with_lock_entry_is_not_unsupported(): response = "Usage:\nsf lock [offset] [len]\nOpenIPC # " assert uboot_sf_lock_unsupported(response) is False @@ -319,3 +336,17 @@ async def cmd(command: str, timeout: float = 0.0) -> str: ram_addr=0x82000000, ) ) + + +def test_nor_size_probe_read_targets_the_last_16_bytes(): + assert nor_size_probe_read(0x42000000, 8) == "sf read 0x42000000 0x7ffff0 0x10" + assert nor_size_probe_read(0x42000000, 16) == "sf read 0x42000000 0xfffff0 0x10" + + +def test_sf_read_succeeded_on_openipc_hisilicon_output(): + ok = "device 0 offset 0x7ffff0, size 0x10\nSF: 16 bytes @ 0x7ffff0 Read: OK\nOpenIPC # " + past = "Offset exceeds device limit\nsf - SPI flash sub-system\n\nUsage:\nOpenIPC # " + mainline = "ERROR: attempting read past flash size (0x800000)\nOpenIPC # " + assert sf_read_succeeded(ok) + assert not sf_read_succeeded(past) + assert not sf_read_succeeded(mainline) diff --git a/tests/test_install_nor_split.py b/tests/test_install_nor_split.py new file mode 100644 index 0000000..c59e85d --- /dev/null +++ b/tests/test_install_nor_split.py @@ -0,0 +1,287 @@ +"""Split NOR install for the gk7205v200 family (OpenIPC/u-boot openipc-hisilicon). + +openipc.-nor--split.tgz carries one firmware.bin: a hashed FIT, +then the squashfs at the next 64 KiB block. It is written at 0x50000 and the +rest of the chip is erased; the kernel divides it into kernel, rootfs and +rootfs_data at boot. That U-Boot works out the NOR layout itself, so the env +stage loads its defaults rather than writing fixed-slot mtdparts. +""" + +from __future__ import annotations + +import hashlib +import zlib +from pathlib import Path + +import pytest +import typer + +from defib.install import InstallRequest +from defib.install.firmware import load_firmware_bundle, split_image_end +from tests.test_nand_ubi_install import FACTORY_MAC, _patch_install, _write_tar + +BLOCK = 0x10000 +CHIP_8M = 8 * 1024 * 1024 + + +def _split_image(fit_size: int = 0x18000, squashfs_used: int = 0x2F000) -> bytes: + """A FIT whose totalsize rounds up to 0x20000, then a squashfs there.""" + fit = b"\xd0\x0d\xfe\xed" + fit_size.to_bytes(4, "big") + fit = fit.ljust((fit_size + BLOCK - 1) // BLOCK * BLOCK, b"\xab") + squashfs = b"hsqs" + b"\x00" * 36 + squashfs_used.to_bytes(8, "little") + squashfs = squashfs.ljust((squashfs_used + BLOCK - 1) // BLOCK * BLOCK, b"\xcd") + return fit + squashfs + + +def _split_package(path: Path, board: str = "gk7205v200", image: bytes | None = None, + md5: str | None = None) -> Path: + image = _split_image() if image is None else image + digest = md5 or hashlib.md5(image).hexdigest() + _write_tar(path, { + f"firmware.bin.{board}.md5sum": f"{digest} firmware.bin.{board}\n".encode(), + f"firmware.bin.{board}": image, + }) + return path + + +class FakeNorUBoot: + """An OpenIPC/u-boot NOR console over a byte-accurate 8 MiB chip.""" + + def __init__(self) -> None: + self.commands: list[str] = [] + self.tftp_files: dict[str, bytes] = {} + self.ram = b"" + self.flash = bytearray(b"\x5a" * CHIP_8M) # an old layout's leftovers + # totalmem as this U-Boot sets it at boot, from the DRAM it measured. + self.env: dict[str, str] = { + "ethaddr": FACTORY_MAC, "mtdparts": "saved", "totalmem": "128M", + } + + def reply(self, command: str) -> str: + return "".join(self._one(part.strip()) for part in command.split(";")) or "\nOpenIPC # " + + def _one(self, command: str) -> str: + self.commands.append(command) + prompt = "\nOpenIPC # " + words = command.split() + if not words: + return "" + if command == "sf lock 0": + # This U-Boot's sf has no lock subcommand: echo, then the table. + return ("sf lock 0\nsf - SPI flash sub-system\n\nUsage:\n" + "sf probe [[bus:]cs] [hz] [mode] - init flash device\n" + "sf protect lock/unlock sector len - protect/unprotect" + prompt) + if command == "sf probe 0": + # Probed at boot already: says nothing, as on the cameras. + return prompt + if words[0] in ("tftpboot", "tftp"): + self.ram = self.tftp_files[words[-1]] + return f"Bytes transferred = {len(self.ram)} (hex)" + prompt + if words[0] == "crc32": + size = int(words[2], 16) + return f"crc32 for {words[1]} ... ==> {zlib.crc32(self.ram[:size]):08x}" + prompt + if words[:2] == ["sf", "erase"]: + off, size = int(words[2], 16), int(words[3], 16) + assert off + size <= CHIP_8M + self.flash[off:off + size] = b"\xff" * size + return f"SF: {size} bytes @ {off:#x} Erased: OK" + prompt + if words[:2] == ["sf", "write"]: + off, size = int(words[3], 16), int(words[4], 16) + self.flash[off:off + size] = self.ram[:size] + return f"SF: {size} bytes @ {off:#x} Written: OK" + prompt + if words[:2] == ["sf", "read"]: + off, size = int(words[3], 16), int(words[4], 16) + if off + size > CHIP_8M: + return "Offset exceeds device limit\nsf - SPI flash sub-system\n\nUsage:" + prompt + self.ram = bytes(self.flash[off:off + size]) + return f"SF: {size} bytes @ {off:#x} Read: OK" + prompt + if words[0] == "printenv": + key = words[1] + if key in self.env: + return f"{key}={self.env[key]}" + prompt + return f'## Error: "{key}" not defined' + prompt + if words[0] == "setenv": + self.env[words[1]] = " ".join(words[2:]) + return prompt + if command == "env default -a": + self.env = {"bootcmd": "run bootflash"} + return "## Resetting to default environment" + prompt + if command == "saveenv": + return "Saving Environment to SPI Flash... OK" + prompt + if command == "reset": + return "resetting ..." + return prompt + + +def _seed_cache(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.setattr("sys.platform", "linux") + from defib.firmware import get_cache_dir + + cache = get_cache_dir() + for soc in ("gk7205v200", "hi3516ev300"): + (cache / f"u-boot-{soc}-nor.bin").write_bytes(b"N" * 0x36000) + (cache / f"u-boot-{soc}-universal.bin").write_bytes(b"X" * 0x30000) + + +async def _install(monkeypatch, tmp_path, firmware: Path, *, chip: str = "gk7205v200", + **kwargs) -> tuple[FakeNorUBoot, dict[str, object]]: + from defib.install.orchestrator import run_install + + _seed_cache(monkeypatch, tmp_path) + device = FakeNorUBoot() + seen = _patch_install(monkeypatch, device) + await run_install(InstallRequest( + chip=chip, firmware_path=str(firmware), port="COM15", nic="eth0", + output="json", **kwargs, + )) + return device, seen + + +# --- the package -------------------------------------------------------------- + +def test_split_package_is_recognised(tmp_path): + bundle = load_firmware_bundle(_split_package(tmp_path / "fw.tgz"), boards={"gk7205v200"}) + assert bundle.split + assert bundle.rootfs_name == "firmware.bin.gk7205v200" + assert bundle.rootfs == _split_image() + assert bundle.kernel == b"" + assert split_image_end(bundle.rootfs) == 0x50000 + + +def test_split_package_md5_mismatch(tmp_path): + with pytest.raises(ValueError, match="MD5 mismatch"): + load_firmware_bundle(_split_package(tmp_path / "fw.tgz", md5="0" * 32)) + + +def test_split_package_for_another_board(tmp_path): + with pytest.raises(ValueError, match="built for gk7205v300"): + load_firmware_bundle( + _split_package(tmp_path / "fw.tgz", board="gk7205v300"), boards={"gk7205v200"} + ) + + +@pytest.mark.parametrize("image, reason", [ + (b"\x27\x05\x19\x56" + b"\x00" * 0x1FFFC, "does not start with a FIT"), + (_split_image()[:0x20000] + b"\x00" * 0x30000, "no squashfs"), + (_split_image() + b"\x00" * BLOCK, "runs past the end of its squashfs"), + (_split_image()[:0x20000 + 0x2E000], "truncated"), +], ids=["uimage", "no-squashfs", "trailing-bytes", "truncated"]) +def test_split_package_shape_is_checked(tmp_path, image, reason): + with pytest.raises(ValueError, match=reason): + load_firmware_bundle(_split_package(tmp_path / "fw.tgz", image=image)) + + +# --- the install -------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_split_install_writes_firmware_bin_over_the_whole_chip(monkeypatch, tmp_path): + firmware = _split_package(tmp_path / "openipc.gk7205v200-nor-lite-split.tgz") + device, seen = await _install(monkeypatch, tmp_path, firmware) + + assert Path(str(seen["burned"])).name == "u-boot-gk7205v200-nor.bin" + assert sorted(device.tftp_files) == ["r", "u"] + assert seen["done_count"] == 2 + image = _split_image() + assert bytes(device.flash[0x50000:0x50000 + len(image)]) == image + # Nothing of the old layout is left for U-Boot's scan or the overlay. + assert bytes(device.flash[0x50000 + len(image):]) == b"\xff" * (CHIP_8M - 0x50000 - len(image)) + # The chip size came from reads, since sf probe says nothing. + assert "sf read 0x42000000 0x7ffff0 0x10" in device.commands + assert "sf erase 0x50000 0x7b0000" in device.commands + # U-Boot's own defaults, the MAC carried across; no fixed-slot env. + assert "env default -a" in device.commands + assert device.env.get("ethaddr") == FACTORY_MAC + assert device.env.get("totalmem") == "128M" + assert not any(c.startswith("setenv mtdparts") for c in device.commands) + assert not any("bootcmdnor" in c for c in device.commands) + + +@pytest.mark.asyncio +async def test_fixed_slot_install_on_this_uboot_keeps_its_bootcmd(monkeypatch, tmp_path): + """The openipc-hisilicon environment has no bootcmdnor to point bootcmd at.""" + firmware = tmp_path / "openipc.gk7205v200-nor-lite.tgz" + _write_tar(firmware, {"uImage.gk7205v200": b"K" * 1024, + "rootfs.squashfs.gk7205v200": b"R" * 2048}) + device, _ = await _install(monkeypatch, tmp_path, firmware) + + assert bytes(device.flash[0x50000:0x50400]) == b"K" * 1024 + assert bytes(device.flash[0x250000:0x250800]) == b"R" * 2048 + assert "env default -a" in device.commands + assert device.env.get("bootcmd") == "run bootflash" + assert not any("bootcmdnor" in c for c in device.commands) + + +@pytest.mark.asyncio +async def test_split_kernel_and_rootfs_data_stages_write_nothing(monkeypatch, tmp_path): + firmware = _split_package(tmp_path / "fw.tgz") + device, seen = await _install(monkeypatch, tmp_path, firmware, + stages=("kernel", "rootfs-data")) + assert not any(c.startswith(("sf erase", "sf write")) for c in device.commands) + assert "done_count" not in seen + + +@pytest.mark.asyncio +async def test_split_image_too_big_for_the_chip(monkeypatch, tmp_path): + image = _split_image(fit_size=0x18000, squashfs_used=0x7B0000 - 0x20000) # ends at the chip end + firmware = _split_package(tmp_path / "fw.tgz", image=image) + with pytest.raises(typer.Exit): + await _install(monkeypatch, tmp_path, firmware) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kwargs, code", [ + ({"chip": "hi3516ev300"}, 2), + # NAND here is the UBI layout, whose package loader names the mistake. + ({"nand": True}, 1), + ({"wipe_rootfs_data": True, "stages": ("rootfs-data",)}, 2), +]) +async def test_split_install_refusals(monkeypatch, tmp_path, capsys, kwargs, code): + board = "hi3516ev300" if kwargs.get("chip") == "hi3516ev300" else "gk7205v200" + firmware = _split_package(tmp_path / "fw.tgz", board=board) + with pytest.raises(typer.Exit) as exc: + await _install(monkeypatch, tmp_path, firmware, **kwargs) + assert exc.value.exit_code == code + if kwargs.get("nand"): + assert "cannot be installed on NAND" in capsys.readouterr().out + + +def test_aliased_chip_accepts_the_nand_package_it_is_published_under(): + from defib.install.layout import ubi_nand_boards + + assert "gk7205v200" in ubi_nand_boards("gk7201v200") + assert "gk7205v500" in ubi_nand_boards("gk7205v510") + + +@pytest.mark.asyncio +async def test_split_erase_timeout_scales_with_the_chip(monkeypatch, tmp_path): + """0x50000 to the end of a 32 MiB chip is 507 blocks, not one partition.""" + import defib.flashdump + + timeouts: dict[str, float] = {} + firmware = _split_package(tmp_path / "fw.tgz") + device = FakeNorUBoot() + device.flash = bytearray(b"\x5a" * 32 * 1024 * 1024) + _seed_cache(monkeypatch, tmp_path) + _patch_install(monkeypatch, device) + + async def send(transport_obj, command: str, timeout: float = 0.0, **kw): + timeouts[command] = timeout + if command.startswith("sf read") and "0x10" == command.split()[-1]: + device.commands.append(command) + return "SF: 16 bytes Read: OK\nOpenIPC # " + return device.reply(command) + + monkeypatch.setattr(defib.flashdump, "send_command", send) + from defib.install.orchestrator import run_install + + global CHIP_8M + saved, CHIP_8M = CHIP_8M, 32 * 1024 * 1024 + try: + await run_install(InstallRequest( + chip="gk7205v200", firmware_path=str(firmware), port="COM15", nic="eth0", + output="json", stages=("rootfs",), + )) + finally: + CHIP_8M = saved + assert timeouts["sf erase 0x50000 0x1fb0000"] >= 0x1FB0000 / 0x10000