diff --git a/.github/workflows/build-base-images.yml b/.github/workflows/build-base-images.yml index 40c3e4807..d370a0d1a 100644 --- a/.github/workflows/build-base-images.yml +++ b/.github/workflows/build-base-images.yml @@ -28,6 +28,14 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 + # QEMU + buildx let the amd64 runner build the arm64 image variant so + # --push-image can publish a multi-arch manifest. + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to ghcr.io uses: docker/login-action@v4 with: diff --git a/build-in-container.md b/build-in-container.md index 5aa8e0e2e..778be7813 100644 --- a/build-in-container.md +++ b/build-in-container.md @@ -55,6 +55,7 @@ None of the above arguments are required for `--update`. | `--shell` | | Drop into a bash shell inside the container for debugging | | `--list-platforms` | | List available platforms and exit | | `--source-dir` | parent of `buildscripts/` | Root directory containing repos | +| `--arch` | host architecture | Override the container architecture (see [Architecture](#architecture)) | ## Supported platforms @@ -92,6 +93,10 @@ The new entry in `platforms.json` needs: Don't copy this by hand — run `./build-in-container.py --update-sha --platform ` and it will fetch the current digest from Docker Hub and write it into `platforms.json`. +- `architectures` (optional): the list of docker platforms to publish, e.g. + `["linux/amd64", "linux/arm64"]`. Omit it to get the multi-arch default; set + it only to restrict a platform to specific architectures (see + [Architecture](#architecture)). Adding another RHEL-family platform (a new Rocky/RHEL major version) works the same way: add a `platforms.json` entry with `"dockerfile": "Dockerfile.rhel"` @@ -105,6 +110,41 @@ Docker Hub library images. Adding an entirely different, non-RHEL/non-Debian platform family (e.g. SUSE) would require a new `container/Dockerfile.` plus platform entries. +## Architecture + +By default the build runs on the host machine's architecture, and Docker picks +the matching image variant automatically. Use `--arch` to override this and +build for another architecture - the value is passed straight to Docker's +`--platform` flag: + +```bash +# Build an arm64 community agent .deb for Ubuntu 24 on an amd64 host +./build-in-container.py --platform ubuntu-24 --project community --role agent \ + --build-type DEBUG --arch linux/arm64 +``` + +The registry images are published as multi-arch manifests (`linux/amd64` and +`linux/arm64`), so `--arch` normally just pulls the matching variant. If the +registry does not provide the requested architecture (for example an older, +single-arch image that predates multi-arch support), the script falls back to +building the image locally for that architecture. + +Building a non-host architecture - whether locally or in CI - relies on +QEMU/binfmt emulation being registered on the build host. If it isn't set up, +register it once with: + +```bash +docker run --privileged --rm tonistiigi/binfmt --install all +``` + +Emulated builds are considerably slower than native ones. + +The set of architectures published for each platform defaults to `linux/amd64` +and `linux/arm64`. A platform can override this with an `"architectures"` list +in `platforms.json`. The `ubuntu-24-mingw` platform, for instance, +cross-compiles to Windows x64 regardless of the container's architecture, so it +is pinned to `["linux/amd64"]`. + ## How it works The system has three components: @@ -161,11 +201,10 @@ Images are hosted at `ghcr.io/cfengine` and versioned per-platform via ./build-in-container.py --platform ubuntu-22 --push-image ``` -`--push-image` always builds with `--no-cache` to pick up the latest upstream -packages, then pushes to the registry. However, you must be logged in to -`ghcr.io` first. You can log in with a personal access token (classic) that has -the write:packages scope. Alternatively, trigger the GitHub Actions workflow -which handles authentication automatically. +`--push-image` uses `docker buildx build --push` to build every architecture the +platform targets (`linux/amd64` and `linux/arm64` by default; see +[Architecture](#architecture)) and publish them under a single multi-arch +manifest. It always builds fresh to pick up the latest upstream packages. #### GitHub Actions workflow diff --git a/build-in-container.py b/build-in-container.py index 642070772..372b19385 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -21,6 +21,16 @@ IMAGE_REGISTRY = "ghcr.io/cfengine" CONFIG_PATH = Path(__file__).resolve().parent / "platforms.json" +# Architectures registry images are published for, unless a platform overrides +# it with an "architectures" list in platforms.json (e.g. the mingw cross-build, +# which always targets Windows x64 and only makes sense on amd64). +DEFAULT_ARCHITECTURES = ["linux/amd64", "linux/arm64"] + + +def platform_architectures(platform_config): + """Return the docker platforms a registry image is published for.""" + return platform_config.get("architectures", DEFAULT_ARCHITECTURES) + @functools.cache def get_config(): @@ -63,14 +73,25 @@ def image_needs_rebuild(image_tag, current_hash): return stored_hash != current_hash -def build_image(platform_name, platform_config, script_dir, rebuild=False): +def build_image(platform_name, platform_config, script_dir, rebuild=False, arch=None): """Build the Docker image for the given platform.""" image_tag = f"{platform_config['image_name']}:{platform_config['image_version']}" dockerfile_name = platform_config["dockerfile"] dockerfile_path = script_dir / "container" / dockerfile_name current_hash = dockerfile_hash(dockerfile_path) - if not rebuild and not image_needs_rebuild(image_tag, current_hash): + # A cached local image is only reusable when BOTH its Dockerfile hash and + # its architecture match. The hash check (has the Dockerfile text changed?) + # is arch-blind, and we only ever build single-arch images locally under one + # shared tag. Without the arch check a leftover image from an --arch run + # could be silently reused for a different target — or a host-arch build + # could reuse an arm64 image left behind by an earlier --arch build. + want_arch = arch if arch else host_docker_arch() + if ( + not rebuild + and not image_needs_rebuild(image_tag, current_hash) + and image_provides_arch(image_tag, want_arch) + ): log.info(f"Docker image {image_tag} is up to date.") return image_tag @@ -88,6 +109,9 @@ def build_image(platform_name, platform_config, script_dir, rebuild=False): image_tag, ] + if arch: + cmd.extend(["--platform", arch]) + for key, value in platform_config.get("extra_build_args", {}).items(): cmd.extend(["--build-arg", f"{key}={value}"]) @@ -117,39 +141,111 @@ def registry_image_ref(platform_name): return f"{IMAGE_REGISTRY}/{platform['image_name']}:{platform['image_version']}" -def pull_image(platform_name): +def host_docker_arch(): + """Return the Docker daemon's native architecture (e.g. "amd64", "arm64").""" + result = subprocess.run( + ["docker", "version", "--format", "{{.Server.Arch}}"], + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def image_provides_arch(ref, arch): + """Check whether a locally-present image matches the requested arch. + + `arch` may be a full docker platform string ("linux/arm64") or a bare + architecture ("arm64"); we compare its architecture component against the + image's own reported architecture. + """ + want = arch.rsplit("/", 1)[-1] + result = subprocess.run( + ["docker", "image", "inspect", "--format", "{{.Architecture}}", ref], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return False + return result.stdout.strip() == want + + +def pull_image(platform_name, arch=None): """Pull a pre-built image from the registry. - Returns the image reference on success or None on failure. + Returns the image reference on success or None on failure. When an arch is + requested, returns None if the registry image does not actually provide it + (e.g. a legacy single-arch image), so the caller can fall back to a local + build for the requested architecture. """ ref = registry_image_ref(platform_name) log.info(f"Pulling image {ref}...") + cmd = ["docker", "pull"] + if arch: + cmd.extend(["--platform", arch]) + cmd.append(ref) result = subprocess.run( - ["docker", "pull", ref], + cmd, capture_output=True, text=True, ) if result.returncode != 0: return None + if arch and not image_provides_arch(ref, arch): + log.warning(f"Registry image {ref} does not provide {arch}.") + return None return ref -def push_image(platform_name, local_tag): - """Tag a local image with a timestamped version and push it.""" - image_name = get_config()[platform_name]["image_name"] +def build_and_push_image(platform_name, platform_config, script_dir): + """Build a multi-arch image with buildx and push it to the registry. + + Multi-arch manifests cannot be produced by `docker build` + `docker tag` + (the local image store holds a single architecture), so this uses + `docker buildx build --platform ... --push` to build every target + architecture and push them under one manifest tag. Building a non-host + architecture relies on QEMU/binfmt being registered on the build host. + """ + image_name = platform_config["image_name"] version = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") ref = f"{IMAGE_REGISTRY}/{image_name}:{version}" - log.info(f"Tagging {local_tag} as {ref}...") - result = subprocess.run(["docker", "tag", local_tag, ref]) - if result.returncode != 0: - log.error("Docker tag failed.") - sys.exit(1) + dockerfile_path = script_dir / "container" / platform_config["dockerfile"] + current_hash = dockerfile_hash(dockerfile_path) + architectures = ",".join(platform_architectures(platform_config)) - log.info(f"Pushing {ref}...") - result = subprocess.run(["docker", "push", ref]) + log.info(f"Building and pushing multi-arch image {ref} ({architectures})...") + cmd = [ + "docker", + "buildx", + "build", + "--platform", + architectures, + "-f", + str(dockerfile_path), + "--build-arg", + f"BASE_IMAGE={platform_config['base_image']}@{platform_config['base_image_sha']}", + "--label", + f"dockerfile-hash={current_hash}", + "-t", + ref, + ] + + for key, value in platform_config.get("extra_build_args", {}).items(): + cmd.extend(["--build-arg", f"{key}={value}"]) + + # Expose ci/ as a named build context so the Dockerfile can COPY --from=ci + # the shared toolchain installers without widening the main build context. + cmd.extend(["--build-context", f"ci={script_dir / 'ci'}"]) + + # Build every architecture and push the resulting manifest in one step. + cmd.append("--push") + + # Build context is the container/ directory + cmd.append(str(script_dir / "container")) + + result = subprocess.run(cmd) if result.returncode != 0: - log.error("Docker push failed.") + log.error("Docker buildx build/push failed.") sys.exit(1) log.info(f"Update image_version to \"{version}\" in platforms.json.") @@ -259,6 +355,9 @@ def run_container(args, image_tag, source_dir, script_dir): cmd = ["docker", "run", "--rm", "--network", "host"] + if args.arch: + cmd.extend(["--platform", args.arch]) + if args.shell: cmd.extend(["-it"]) @@ -347,6 +446,11 @@ def parse_args(): action="store_true", help="List available platforms and exit", ) + parser.add_argument( + "--arch", + help="Override the container architecture, passed to docker's --platform " + "(e.g. linux/amd64, linux/arm64). Default: host architecture.", + ) parser.add_argument( "--source-dir", help="Root directory containing repos (default: parent of buildscripts/)", @@ -461,22 +565,24 @@ def main(): platform_config = get_config()[args.platform] if args.push_image: - image_tag = build_image( - args.platform, platform_config, script_dir, rebuild=True - ) - push_image(args.platform, image_tag) + build_and_push_image(args.platform, platform_config, script_dir) return - # Resolve image: pull from registry, fall back to local build + # Resolve image: pull from registry, fall back to local build. The registry + # holds multi-arch manifests, so a pull with --platform selects the right + # variant; if it isn't available (e.g. a legacy single-arch image) we build + # the requested architecture locally. if args.rebuild_image: image_tag = build_image( - args.platform, platform_config, script_dir, rebuild=True + args.platform, platform_config, script_dir, rebuild=True, arch=args.arch ) else: - image_tag = pull_image(args.platform) + image_tag = pull_image(args.platform, arch=args.arch) if image_tag is None: - log.warning("Registry pull failed, building image locally...") - image_tag = build_image(args.platform, platform_config, script_dir) + log.warning("Registry image unavailable, building image locally...") + image_tag = build_image( + args.platform, platform_config, script_dir, arch=args.arch + ) if not args.shell: log.info( diff --git a/platforms.json b/platforms.json index 536ddc1f2..4e1a44e4c 100644 --- a/platforms.json +++ b/platforms.json @@ -7,7 +7,8 @@ "dockerfile": "Dockerfile.debian", "extra_build_args": { "NCURSES_PKGS": "libncurses5 libncurses5-dev" - } + }, + "architectures": ["linux/amd64"] }, "ubuntu-22": { "image_name": "cfengine-builder-ubuntu-22", @@ -54,7 +55,8 @@ "CRB_REPO": "powertools", "PHP_MODULE_STREAM": "remi-8.3", "EXTRA_PKGS": "python3-rpm-macros platform-python-devel" - } + }, + "architectures": ["linux/amd64"] }, "rhel-9": { "image_name": "cfengine-builder-rhel-9", @@ -64,7 +66,8 @@ "dockerfile": "Dockerfile.rhel", "extra_build_args": { "PHP_MODULE_STREAM": "remi-8.3" - } + }, + "architectures": ["linux/amd64"] }, "rhel-10": { "image_name": "cfengine-builder-rhel-10", @@ -82,6 +85,7 @@ "base_image": "ubuntu:24.04", "base_image_sha": "sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90", "dockerfile": "Dockerfile.mingw", - "cross_target": "x64-mingw" + "cross_target": "x64-mingw", + "architectures": ["linux/amd64"] } }