Build images natively per architecture

.github/workflows/build.yaml

@@ -5,14 +5,28 @@ on:
     branches: [main]
   schedule:
     - cron: "0 00,12 * * *"  # Twice a day
+  workflow_dispatch:
+    inputs:
+      force:
+        description: Force rebuild and republish all image tags
+        default: true
+        type: boolean
+      image-name:
+        description: Override image name for this manual run
+        default: ""
+        type: string
+
+env:
+  IMAGE_NAME: ${{ inputs.image-name || vars.IMAGE_NAME || 'nikolaik/python-nodejs' }}
 
 jobs:
   generate-matrix:
     name: Generate build matrix
     runs-on: ubuntu-latest
     needs: [test]
     outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      version_matrix: ${{ steps.set-matrix.outputs.matrix }}
+      arch_matrix: ${{ steps.set-matrix.outputs.arch_matrix }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:
@@ -23,66 +37,107 @@ jobs:
       - name: Generate build matrix
         id: set-matrix
         run: |
-          FORCE=$(if git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then echo "--force"; else echo ""; fi)
+          FORCE=
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force }}" == "true" ]]; then
+            FORCE="--force"
+          elif git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then
+            FORCE="--force"
+          fi
           uv run dpn $FORCE build-matrix --event ${{ github.event_name }}
 
 
-  deploy:
-    name: ${{ matrix.key }}
-    runs-on: ubuntu-latest
-    if: needs.generate-matrix.outputs.matrix != ''
+  build-arch:
+    name: ${{ matrix.key }} (${{ matrix.arch }})
+    runs-on: ${{ matrix.runner }}
+    if: needs.generate-matrix.outputs.arch_matrix != ''
     needs: [generate-matrix]
     strategy:
-      matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.generate-matrix.outputs.arch_matrix) }}
     steps:
       # Setup
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
       - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
         with:
           enable-cache: true
       - name: Generate Dockerfile from config
-        run: uv run dpn dockerfile --context '${{ toJSON(matrix) }}'
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
+        run: |
+          context="$(echo '${{ toJSON(matrix) }}' | jq -c '{key, python, python_canonical, python_image, nodejs, nodejs_canonical, distro, platforms, digest}')"
+          uv run dpn dockerfile --context "${context}"
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
-      - name: Login to Docker Hub
-        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       # Build
       - name: Build image
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7
         with:
           context: .
           file: dockerfiles/${{ matrix.key }}.Dockerfile
+          platforms: ${{ matrix.platform }}
           load: true
-          tags: nikolaik/python-nodejs:${{ matrix.key }}
+          tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }}
 
       # Test
       - name: Run smoke tests
         run: |
-          docker run --rm nikolaik/python-nodejs:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version"
+          docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} sh -c \
+            "node --version && npm --version && yarn --version && \
+            python --version && pip --version && pipenv --version && \
+            poetry --version && uv --version"
 
-      # Push image
+      # Push
+      - name: Login to Docker Hub
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Push image
-        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7
-        id: build-and-push
+        run: docker push "${IMAGE_NAME}:${{ matrix.key }}-${{ matrix.arch }}"
+
+  deploy:
+    name: Publish ${{ matrix.key }}
+    runs-on: ubuntu-latest
+    if: needs.generate-matrix.outputs.version_matrix != ''
+    needs: [generate-matrix, build-arch]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.generate-matrix.outputs.version_matrix) }}
+    steps:
+      - name: Create local multi-arch manifest
+        run: |
+          refs=("${IMAGE_NAME}:${{ matrix.key }}-amd64")
+          if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then
+            refs+=("${IMAGE_NAME}:${{ matrix.key }}-arm64")
+          fi
+          docker manifest create "${IMAGE_NAME}:${{ matrix.key }}" "${refs[@]}"
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4
         with:
-          context: .
-          file: dockerfiles/${{ matrix.key }}.Dockerfile
-          platforms: ${{ join(matrix.platforms) }}
-          push: true
-          tags: nikolaik/python-nodejs:${{ matrix.key }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Install regctl
+        uses: regclient/actions/regctl-installer@1b705e32d40851370799ea5814e83d0a5f6a70dc # v0.1.0
+
+      - name: Push multi-arch manifest
+        id: push-manifest
+        run: |
+          docker manifest push "${IMAGE_NAME}:${{ matrix.key }}"
+          digest="$(regctl image digest "${IMAGE_NAME}:${{ matrix.key }}")"
+          echo "digest=${digest}" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
 
-      # Store build context
       - name: Add digest to build context
         run: |
           mkdir builds/
-          digest="${{ steps.build-and-push.outputs.digest }}"
-          echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" '. +={"digest": $digest}' >> "builds/${{ matrix.key }}.json"
+          digest="${{ steps.push-manifest.outputs.digest }}"
+          echo '${{ toJSON(matrix) }}' |
+            jq --arg digest "$digest" \
+              '. +={"digest": $digest}' \
+              >> "builds/${{ matrix.key }}.json"
 
       - name: Upload build context
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

src/docker_python_nodejs/build_matrix.py

@@ -25,15 +25,42 @@ def _github_action_set_output(key: str, value: str) -> None:
         sys.exit(1)
 
     with Path(GITHUB_OUTPUT).open("a") as fp:
-        fp.write(f"{key}={value}")
+        fp.write(f"{key}={value}\n")
+
+
+def _build_matrix_json(new_or_updated: list[BuildVersion]) -> str:
+    return json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else ""
+
+
+def _build_arch_matrix_json(new_or_updated: list[BuildVersion]) -> str:
+    if not new_or_updated:
+        return ""
+
+    include: list[dict[str, object]] = []
+    for version in new_or_updated:
+        include.extend(
+            (
+                dataclasses.asdict(version)
+                | {
+                    "platform": platform,
+                    "arch": platform.split("/")[1],
+                    "runner": "ubuntu-24.04-arm" if platform == "linux/arm64" else "ubuntu-latest",
+                }
+            )
+            for platform in version.platforms
+        )
+
+    return json.dumps({"include": include})
 
 
 def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None:
     if not new_or_updated and ci_event == CI_EVENT_SCHEDULED:
         logger.info("\n# Scheduled run with no new or updated versions. Doing nothing.")
         return
 
-    matrix = json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else ""
-    _github_action_set_output("MATRIX", matrix)
+    matrix = _build_matrix_json(new_or_updated)
+    arch_matrix = _build_arch_matrix_json(new_or_updated)
+    _github_action_set_output("matrix", matrix)
+    _github_action_set_output("arch_matrix", arch_matrix)
     logger.info("\n# New or updated versions:")
     logger.info("Nothing" if not new_or_updated else "\n".join(version.key for version in new_or_updated))

tests/test_all.py

@@ -8,6 +8,7 @@
 import pytest
 import responses
 
+from docker_python_nodejs.build_matrix import _build_arch_matrix_json
 from docker_python_nodejs.dockerfiles import render_dockerfile_with_context
 from docker_python_nodejs.readme import update_dynamic_readme
 from docker_python_nodejs.settings import BASE_PATH, DOCKERFILES_PATH
@@ -289,3 +290,40 @@ def test_find_new_or_updated_with_digest() -> None:
     res = find_new_or_updated([new], {existing.key: existing})
 
     assert len(res) == 0
+
+
+def test_build_arch_matrix_json(build_version: BuildVersion) -> None:
+    matrix = json.loads(_build_arch_matrix_json([build_version]))
+
+    assert matrix == {
+        "include": [
+            {
+                "key": "python3.11-nodejs20",
+                "python": "3.11",
+                "python_canonical": "3.11.3",
+                "python_image": "3.11.3-trixie",
+                "nodejs": "20",
+                "nodejs_canonical": "20.2.0",
+                "distro": "trixie",
+                "platforms": ["linux/amd64", "linux/arm64"],
+                "digest": "",
+                "platform": "linux/amd64",
+                "arch": "amd64",
+                "runner": "ubuntu-latest",
+            },
+            {
+                "key": "python3.11-nodejs20",
+                "python": "3.11",
+                "python_canonical": "3.11.3",
+                "python_image": "3.11.3-trixie",
+                "nodejs": "20",
+                "nodejs_canonical": "20.2.0",
+                "distro": "trixie",
+                "platforms": ["linux/amd64", "linux/arm64"],
+                "digest": "",
+                "platform": "linux/arm64",
+                "arch": "arm64",
+                "runner": "ubuntu-24.04-arm",
+            },
+        ],
+    }