Add UBI9 Server and Keeper images for Red Hat certification

.github/actionlint.yml

@@ -1,6 +1,7 @@
 self-hosted-runner:
   labels:
     - altinity-builder
+    - altinity-on-demand
     - altinity-func-tester
     - altinity-func-tester-aarch64
     - fuzzer-unit-tester

.github/workflows/docker_publish.yml

@@ -50,14 +50,17 @@ jobs:
     runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
     outputs:
       image_archives_path: ${{ steps.set_path.outputs.image_archives_path }}
+      component: ${{ steps.image_info.outputs.component }}
+      published_image: ${{ steps.image_info.outputs.published_image }}
     steps:
       - name: Docker Hub Login
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
       - name: Set clickhouse-server version as new tag
+        id: image_info
         run: |
           # Determine "clickhouse-server" or "clickhouse-keeper"
           echo "Input IMAGE: $IMAGE"
@@ -86,10 +89,15 @@ jobs:
           if [[ "$IMAGE" == *-alpine* ]]; then
             NEW_TAG="${NEW_TAG}-alpine"
           fi
+          if [[ "$IMAGE" == *-ubi9* ]]; then
+            NEW_TAG="${NEW_TAG}-ubi9"
+          fi
           echo "New tag: $NEW_TAG"
 
           # Export the new tag
           echo "NEW_TAG=$NEW_TAG" >> $GITHUB_ENV
+          echo "component=$COMPONENT" >> $GITHUB_OUTPUT
+          echo "published_image=altinity/$COMPONENT:$NEW_TAG" >> $GITHUB_OUTPUT
 
       - name: Process multiarch manifest
         run: |
@@ -148,3 +156,56 @@ jobs:
         run: |
           aws s3 sync image_archives/ "${{ inputs.s3_upload_path }}"
 
+  redhat-certification:
+    name: Submit Red Hat container certification
+    needs: republish
+    if: ${{ vars.RED_HAT_CERTIFICATION_ENABLED == 'true' && (github.event.inputs.release_environment || inputs.release_environment) == 'production' && contains(github.event.inputs.docker_image || inputs.docker_image, '-ubi9') }}
+    runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
+    env:
+      PREFLIGHT_VERSION: 1.21.0
+      PREFLIGHT_SHA256: 5e653135503c72f8702bbe31d7643197d12937c68086879133dd6b9650a9a449
+      PYXIS_API_TOKEN: ${{ secrets.RED_HAT_PYXIS_API_TOKEN }}
+      SERVER_COMPONENT_ID: ${{ secrets.RED_HAT_CLICKHOUSE_SERVER_CERTIFICATION_COMPONENT_ID }}
+      KEEPER_COMPONENT_ID: ${{ secrets.RED_HAT_CLICKHOUSE_KEEPER_CERTIFICATION_COMPONENT_ID }}
+      PUBLISHED_IMAGE: ${{ needs.republish.outputs.published_image }}
+      COMPONENT: ${{ needs.republish.outputs.component }}
+    steps:
+      - name: Install Preflight
+        run: |
+          set -euo pipefail
+          curl -fsSLo /tmp/preflight \
+            "https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/${PREFLIGHT_VERSION}/preflight-linux-amd64"
+          echo "${PREFLIGHT_SHA256}  /tmp/preflight" | sha256sum --check
+          sudo install -m 0755 /tmp/preflight /usr/local/bin/preflight
+
+      - name: Submit immutable multiarch image
+        run: |
+          set -euo pipefail
+          : "${PYXIS_API_TOKEN:?Set the RED_HAT_PYXIS_API_TOKEN secret}"
+          case "${COMPONENT}" in
+            clickhouse-server) component_id="${SERVER_COMPONENT_ID}" ;;
+            clickhouse-keeper) component_id="${KEEPER_COMPONENT_ID}" ;;
+            *) echo "Unknown component: ${COMPONENT}" >&2; exit 1 ;;
+          esac
+          : "${component_id:?Set the matching Red Hat certification component secret}"
+          digest=$(docker buildx imagetools inspect \
+            "${PUBLISHED_IMAGE}" --format '{{.Manifest.Digest}}')
+          [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]
+          pinned_image="${PUBLISHED_IMAGE%:*}@${digest}"
+          mkdir -p preflight-artifacts
+          PFLT_ARTIFACTS=preflight-artifacts \
+          PFLT_LOGFILE=preflight.log \
+          PFLT_PYXIS_API_TOKEN="${PYXIS_API_TOKEN}" \
+          PFLT_CERTIFICATION_COMPONENT_ID="${component_id}" \
+            preflight check container "${pinned_image}" --submit
+
+      - name: Upload certification evidence
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: redhat-certification-${{ needs.republish.outputs.component }}-${{ github.run_id }}
+          path: |
+            preflight-artifacts/
+            preflight.log
+          if-no-files-found: warn
+          retention-days: 365

.github/workflows/master.yml

@@ -6733,7 +6733,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suffix: ['', '-alpine']
+        suffix: ['', '-alpine', '-ubi9']
     uses: ./.github/workflows/grype_scan.yml
     secrets: inherit
     with:
@@ -6743,11 +6743,16 @@ jobs:
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
       if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'RG9ja2VyIGtlZXBlciBpbWFnZQ==') }}
+      strategy:
+        fail-fast: false
+        matrix:
+          suffix: ['', '-ubi9']
       uses: ./.github/workflows/grype_scan.yml
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
         version: ${{ fromJson(needs.config_workflow.outputs.data).workflow_config.custom_data.version.string }}
+        tag-suffix: ${{ matrix.suffix }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_binary]

.github/workflows/pull_request.yml

@@ -6253,7 +6253,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suffix: ['', '-alpine']
+        suffix: ['', '-alpine', '-ubi9']
     uses: ./.github/workflows/grype_scan.yml
     secrets: inherit
     with:
@@ -6263,11 +6263,16 @@ jobs:
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
       if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'RG9ja2VyIGtlZXBlciBpbWFnZQ==') }}
+      strategy:
+        fail-fast: false
+        matrix:
+          suffix: ['', '-ubi9']
       uses: ./.github/workflows/grype_scan.yml
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
         version: ${{ fromJson(needs.config_workflow.outputs.data).workflow_config.custom_data.version.string }}
+        tag-suffix: ${{ matrix.suffix }}
 
   RegressionTestsRelease:
     needs: [config_workflow, build_amd_binary, stateless_tests_amd_debug_parallel]

.github/workflows/release_builds.yml

@@ -1541,7 +1541,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suffix: ['', '-alpine']
+        suffix: ['', '-alpine', '-ubi9']
     uses: ./.github/workflows/grype_scan.yml
     secrets: inherit
     with:
@@ -1551,11 +1551,16 @@ jobs:
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
       if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'RG9ja2VyIGtlZXBlciBpbWFnZQ==') }}
+      strategy:
+        fail-fast: false
+        matrix:
+          suffix: ['', '-ubi9']
       uses: ./.github/workflows/grype_scan.yml
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
         version: ${{ fromJson(needs.config_workflow.outputs.data).workflow_config.custom_data.version.string }}
+        tag-suffix: ${{ matrix.suffix }}
 
   FinishCIReport:
     if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }}

.github/workflows/sign_and_release.yml

@@ -276,7 +276,7 @@ jobs:
     strategy:
       matrix:
         image_type: [server, keeper]
-        variant: ['', '-alpine']
+        variant: ['', '-alpine', '-ubi9']
     uses: ./.github/workflows/docker_publish.yml
     with:
       docker_image: altinityinfra/clickhouse-${{ matrix.image_type }}:${{ needs.extract-package-info.outputs.docker_version }}${{ matrix.variant }}

ci/jobs/docker_server.py

@@ -70,8 +70,8 @@ def docker_login(relogin: bool = True) -> None:
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        description="A program to build clickhouse-server image, both alpine and "
-        "ubuntu versions",
+        description="Build the supported clickhouse-server and clickhouse-keeper "
+        "container image variants",
     )
     parser.add_argument(
         "--tag-type",
@@ -101,7 +101,11 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument("--reports", default=True, help=argparse.SUPPRESS)
     parser.add_argument("--push", action="store_true", help=argparse.SUPPRESS)
-    parser.add_argument("--os", default=["ubuntu", "alpine", "distroless"], help=argparse.SUPPRESS)
+    parser.add_argument(
+        "--os",
+        default=["ubuntu", "alpine", "distroless", "ubi9"],
+        help=argparse.SUPPRESS,
+    )
     parser.add_argument(
         "--no-ubuntu",
         action=DelOS,
@@ -123,6 +127,13 @@ def parse_args() -> argparse.Namespace:
         default=argparse.SUPPRESS,
         help="don't build distroless image",
     )
+    parser.add_argument(
+        "--no-ubi9",
+        action=DelOS,
+        nargs=0,
+        default=argparse.SUPPRESS,
+        help="don't build UBI 9 image",
+    )
     parser.add_argument(
         "--allow-build-reuse",
         action="store_true",
@@ -219,11 +230,12 @@ def build_and_push_image(
         arch_tag = f"{tag}-{arch}"
         metadata_path = temp_path / arch_tag
         dockerfile = f"{image.path}/Dockerfile.{os}"
+        build_context = "." if os == "ubi9" else image.path
         cmd_args = list(init_args)
         urls = []
         if direct_urls:
             # distroless and ubuntu-server use an Ubuntu builder with dpkg, so they
-            # need .deb packages. alpine and ubuntu-keeper use .tgz packages.
+            # need .deb packages. Alpine, UBI, and ubuntu-keeper use .tgz packages.
             uses_deb = os == "distroless" or (
                 os == "ubuntu" and "clickhouse-server" in image.name
             )
@@ -259,7 +271,7 @@ def build_and_push_image(
                 f"--build-arg=VERSION='{version}'",
                 "--progress=plain",
                 f"--file={dockerfile}",
-                Path(image.path).as_posix(),
+                Path(build_context).as_posix(),
             ]
         )
         cmd = " ".join(cmd_args)

ci/praktika/yaml_additional_templates.py

@@ -44,7 +44,7 @@ class AltinityWorkflowTemplates:
     strategy:
       fail-fast: false
       matrix:
-        suffix: ['', '-alpine']
+        suffix: ['', '-alpine', '-ubi9']
     uses: ./.github/workflows/grype_scan.yml
     secrets: inherit
     with:
@@ -54,11 +54,16 @@ class AltinityWorkflowTemplates:
   GrypeScanKeeper:
       needs: [config_workflow, docker_keeper_image]
       if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'RG9ja2VyIGtlZXBlciBpbWFnZQ==') }}
+      strategy:
+        fail-fast: false
+        matrix:
+          suffix: ['', '-ubi9']
       uses: ./.github/workflows/grype_scan.yml
       secrets: inherit
       with:
         docker_image: altinityinfra/clickhouse-keeper
         version: ${{ fromJson(needs.config_workflow.outputs.data).workflow_config.custom_data.version.string }}
+        tag-suffix: ${{ matrix.suffix }}
 """,
         "RegressionPR": r"""
   RegressionTestsRelease:

docker/README.ubi9.md

@@ -0,0 +1,33 @@
+# UBI 9 release images
+
+Release builds produce Red Hat UBI 9 variants of the existing multi-architecture
+ClickHouse Server and ClickHouse Keeper images. They use the same release
+artifacts as the other image variants and are published with an `-ubi9` suffix:
+
+- `altinity/clickhouse-server:<version>-ubi9`
+- `altinity/clickhouse-keeper:<version>-ubi9`
+
+The Dockerfiles download the release `.tgz` artifacts and their `.sha512`
+files, verify every checksum, and install the files over a fresh UBI base. The
+final images run as the existing ClickHouse UID and GID, `101:101`.
+
+## Release and certification automation
+
+The normal release build includes `ubi9` in the Server and Keeper OS matrix.
+The normal publish workflow preserves the `-ubi9` suffix when it copies the
+multi-architecture manifest from the staging registry to Docker Hub. Grype also
+scans both UBI variants.
+
+For production releases, the publish workflow can submit the immutable UBI
+manifest digest to Red Hat Preflight. Configure these GitHub Actions settings:
+
+- Repository variable `RED_HAT_CERTIFICATION_ENABLED` set to `true`.
+- Repository secret `RED_HAT_PYXIS_API_TOKEN` containing the Red Hat API token.
+- Repository secret `RED_HAT_CLICKHOUSE_SERVER_CERTIFICATION_COMPONENT_ID`
+  containing the Server project component ID.
+- Repository secret `RED_HAT_CLICKHOUSE_KEEPER_CERTIFICATION_COMPONENT_ID`
+  containing the Keeper project component ID.
+
+Certification is intentionally skipped for staging releases and non-UBI image
+variants. Each submission also uploads the Preflight result as a workflow
+artifact for release auditing.

docker/keeper/Dockerfile.ubi9

@@ -0,0 +1,77 @@
+ARG UBI_IMAGE=registry.access.redhat.com/ubi9/ubi:9.8
+FROM --platform=${BUILDPLATFORM} ${UBI_IMAGE} AS artifacts
+
+ARG DIRECT_DOWNLOAD_URLS=""
+
+WORKDIR /tmp/clickhouse-install
+
+RUN if [ -z "${DIRECT_DOWNLOAD_URLS}" ]; then \
+        echo "DIRECT_DOWNLOAD_URLS is required" >&2; exit 1; \
+    fi \
+    && command -v curl gzip tar sha512sum >/dev/null \
+    && for url in ${DIRECT_DOWNLOAD_URLS}; do \
+        echo "Downloading ${url}" \
+        && curl --fail --location --retry 5 --retry-delay 1 --remote-name "${url}"; \
+    done \
+    && sed 's:/output/:/tmp/clickhouse-install/:' ./*.tgz.sha512 | sha512sum --check \
+    && mkdir -p /opt/clickhouse-root/lib \
+    && for archive in ./*.tgz; do \
+        tar xzf "${archive}" --strip-components=1 -C /opt/clickhouse-root; \
+    done
+
+FROM ${UBI_IMAGE}
+
+# Pull in security patches released against this UBI9 stream since the base
+# image tag was last published - a pinned tag does not update on its own.
+RUN dnf update -y && dnf clean all
+
+ARG VERSION
+ARG RELEASE=1
+
+LABEL name="Altinity ClickHouse Keeper" \
+    vendor="Altinity" \
+    maintainer="Altinity <support@altinity.com>" \
+    version="${VERSION}" \
+    release="${RELEASE}" \
+    summary="ClickHouse Keeper built and supported by Altinity" \
+    description="A Red Hat UBI-based ClickHouse Keeper container image." \
+    io.k8s.display-name="Altinity ClickHouse Keeper" \
+    io.openshift.tags="clickhouse,keeper,coordination"
+
+ARG DEFAULT_UID=101
+ARG DEFAULT_GID=101
+
+RUN command -v curl gzip tar sha512sum >/dev/null \
+    && groupadd --system --gid "${DEFAULT_GID}" clickhouse \
+    && useradd --system --uid "${DEFAULT_UID}" --gid clickhouse \
+        --home-dir /var/lib/clickhouse --shell /sbin/nologin clickhouse
+
+COPY --from=artifacts /opt/clickhouse-root/etc/ /etc/
+COPY --from=artifacts /opt/clickhouse-root/lib/ /usr/lib/
+COPY --from=artifacts /opt/clickhouse-root/usr/ /usr/
+
+ARG DEFAULT_CONFIG_DIR=/etc/clickhouse-keeper
+ARG DEFAULT_DATA_DIR=/var/lib/clickhouse
+ARG DEFAULT_LOG_DIR=/var/log/clickhouse-keeper
+
+RUN clickhouse-keeper --version \
+    && mkdir -p "${DEFAULT_CONFIG_DIR}" "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" \
+    && chown -R clickhouse:clickhouse \
+        "${DEFAULT_CONFIG_DIR}" "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" \
+    && chmod -R ugo+Xrw \
+        "${DEFAULT_CONFIG_DIR}" "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}"
+
+COPY --chmod=755 docker/keeper/entrypoint.sh /entrypoint.sh
+COPY LICENSE /licenses/LICENSE
+
+ENV CLICKHOUSE_WATCHDOG_ENABLE=0 \
+    LANG=C.UTF-8 \
+    TZ=UTC
+
+EXPOSE 9181 9234
+VOLUME ["/var/lib/clickhouse", "/var/log/clickhouse-keeper"]
+
+USER 101:101
+WORKDIR /var/lib/clickhouse
+
+ENTRYPOINT ["/entrypoint.sh"]