[release-4.22] USHIFT-7322: Run ostree and bootc cache builds in parallel

test/bin/ci_phase_iso_build.sh

@@ -51,27 +51,52 @@ download_build_cache() {
     return 1
 }
 
-# Run image build for the 'base' layers and update the cache:
+# Run image build for the 'base' and 'release' layers and update the cache:
 # - Upload build artifacts
 # - Update 'last' to point to the current build tag
 # - Clean up older images, preserving the 'last' and the previous build tag
 # Note that the build and upload are skipped if valid cached data already exists.
 update_build_cache() {
+    local pid_ostree pid_bootc9 pid_bootc10
+    local failed=false
+
     if ./bin/manage_build_cache.sh verify -b "${SCENARIO_BUILD_BRANCH}" -t "${SCENARIO_BUILD_TAG}" ; then
         echo "Valid build cache already exists for the '${SCENARIO_BUILD_BRANCH}' branch and '${SCENARIO_BUILD_TAG}' tag"
         echo "WARNING: Skipping cache build, update and cleanup procedures"
         return
     fi
 
-    # Build the composer-cli base layer and brew RPMs to be cached
-    $(dry_run) bash -x ./bin/build_images.sh -l ./image-blueprints/layer1-base
-    $(dry_run) bash -x ./bin/build_images.sh -l ./image-blueprints/layer4-release
+    # Re-build from source before updating the cache because some of the
+    # build artifacts may be cached
+    $(dry_run) bash -x ./bin/build_rpms.sh
 
-    # Build templates
+    # Build templates and run registry mirroring to be used by bootc image builds
     $(dry_run) bash -x ./bin/build_bootc_images.sh -g ./image-blueprints-bootc/templates
-    # Build the bootc base layer and brew RPMs to be cached
-    $(dry_run) bash -x ./bin/build_bootc_images.sh -l ./image-blueprints-bootc/el9/layer1-base -l ./image-blueprints-bootc/el10/layer1-base
-    $(dry_run) bash -x ./bin/build_bootc_images.sh -l ./image-blueprints-bootc/el9/layer4-release -l ./image-blueprints-bootc/el10/layer4-release
+
+    # Build the ostree layers in parallel
+    $(dry_run) bash -xc './bin/build_images.sh -l ./image-blueprints/layer1-base && ./bin/build_images.sh -l ./image-blueprints/layer4-release' &
+    pid_ostree=$!
+
+    # Build the bootc layers in parallel, skipping extraction and mirroring (-E)
+    # as they were already done by the templates build above
+    $(dry_run) bash -xc './bin/build_bootc_images.sh -E -l ./image-blueprints-bootc/el9/layer1-base  -l ./image-blueprints-bootc/el9/layer4-release' &
+    pid_bootc9=$!
+    $(dry_run) bash -xc './bin/build_bootc_images.sh -E -l ./image-blueprints-bootc/el10/layer1-base -l ./image-blueprints-bootc/el10/layer4-release' &
+    pid_bootc10=$!
+
+    # Wait for all the build processes to complete, exit early and clean up if any command fails
+    for pid in "${pid_ostree}" "${pid_bootc9}" "${pid_bootc10}"; do
+        if ! wait "${pid}"; then
+            failed=true
+            break
+        fi
+    done
+
+    if ${failed}; then
+        echo "ERROR: One of the image build commands failed"
+        kill "${pid_ostree}" "${pid_bootc9}" "${pid_bootc10}" 2>/dev/null || true
+        exit 1
+    fi
 
     # Prepare for the cache upload by stopping composer services and cleaning
     # temporary artifacts
@@ -199,10 +224,6 @@ fi
 # Check the build mode: "try using cache" (default) or "update cache"
 if [ $# -gt 0 ] && [ "$1" = "-update_cache" ] ; then
     if ${HAS_CACHE_ACCESS} ; then
-        # Re-build from source before updating the cache because some
-        # build artifacts may be cached
-        $(dry_run) bash -x ./bin/build_rpms.sh
-
         update_build_cache
     else
         echo "ERROR: Access to the build cache is not available"

test/bin/pyutils/build_bootc_images.py

@@ -31,9 +31,8 @@
 BREW_REPO = common.get_env_var('BREW_REPO')
 HOME_DIR = common.get_env_var("HOME")
 PULL_SECRET = common.get_env_var('PULL_SECRET', f"{HOME_DIR}/.pull-secret.json")
-# Switch to quay.io/centos-bootc/bootc-image-builder:latest if any new upstream
-# features are required
-BIB_IMAGE = "registry.redhat.io/rhel9/bootc-image-builder:latest"
+BIB_IMAGE_RHEL9 = "registry.redhat.io/rhel9/bootc-image-builder:latest"
+BIB_IMAGE = "registry.redhat.io/rhel10/bootc-image-builder:latest"
 IBC_IMAGE = "ghcr.io/osbuild/image-builder-cli:latest"
 GOMPLATE = common.get_env_var('GOMPLATE')
 MIRROR_REGISTRY = common.get_env_var('MIRROR_REGISTRY_URL')
@@ -48,7 +47,7 @@ def cleanup_atexit(dry_run):
         common.terminate_process(pid)
 
     # Terminate running image builder containers
-    for builder_image in [BIB_IMAGE, IBC_IMAGE]:
+    for builder_image in [BIB_IMAGE_RHEL9, BIB_IMAGE, IBC_IMAGE]:
         podman_args = [
             "sudo", "podman", "ps",
             "--filter", f"ancestor={builder_image}",
@@ -204,7 +203,7 @@ def extract_container_images(version, repo_spec, outfile, dry_run=False):
             f.write('\n')
 
         # Cleanup RPM files
-        rpm_list = list(map(str, image_path.glob("microshift-release-info-*.rpm")))
+        rpm_list = glob.glob(f"{image_path}/microshift-release-info-{version}*.rpm")
         common.run_command(["rm", "-f"] + rpm_list, dry_run)
     # Restore the current directory
     common.popd()
@@ -334,6 +333,12 @@ def should_skip(image, cached):
         common.run_command(["sed", f"s/^/{cf_outname}: /", cf_logfile], dry_run)
 
 
+def get_bib_image(bootc_imgref):
+    if "rhel9" in bootc_imgref or "rhel-9" in bootc_imgref:
+        return BIB_IMAGE_RHEL9
+    return BIB_IMAGE
+
+
 def process_image_bootc(groupdir, bootcfile, dry_run):
     bf_path, bf_outname, bf_outdir, bf_logfile = get_process_file_names(
         groupdir, bootcfile, BOOTC_ISO_DIR)
@@ -371,19 +376,20 @@ def should_skip(file):
     try:
         # Redirect the output to the log file
         with open(bf_logfile, 'w') as logfile:
+            # Read the image reference and select the matching BIB
+            bf_imgref = common.read_file_valid_lines(bf_outfile).strip()
+            bib_image = get_bib_image(bf_imgref)
+
             # Download the bootc image builder itself in case
             # it requires authorization for accessing the image
             pull_args = [
                 "sudo", "podman", "pull",
-                "--authfile", PULL_SECRET, BIB_IMAGE
+                "--authfile", PULL_SECRET, bib_image
             ]
             start = time.time()
             common.retry_on_exception(3, common.run_command_in_shell, pull_args, dry_run, logfile, logfile)
             common.record_junit(bf_path, "pull-bootc-bib", "OK", start)
 
-            # Read the image reference
-            bf_imgref = common.read_file_valid_lines(bf_outfile).strip()
-
             # Download the image to be used by bootc image builder.
             # Locally built images should also be downloaded in case they were
             # cached but not fetched from the mirror registry.
@@ -409,7 +415,7 @@ def should_skip(file):
             ]
             # Add the bootc image builder command line using local images
             build_args += [
-                BIB_IMAGE,
+                bib_image,
                 "--type", "anaconda-iso",
                 bf_imgref
             ]
@@ -654,7 +660,7 @@ def main():
     parser = argparse.ArgumentParser(description="Build image layers using Bootc Image Builder and Podman.")
     parser.add_argument("-d", "--dry-run", action="store_true", help="Dry run: skip executing build commands.")
     parser.add_argument("-f", "--force-rebuild", action="store_true", help="Force rebuilding images that already exist.")
-    parser.add_argument("-E", "--no-extract-images", action="store_true", help="Skip container image extraction.")
+    parser.add_argument("-E", "--no-extract-images", action="store_true", help="Skip container image extraction, template processing, and registry mirroring.")
     parser.add_argument("-X", "--skip-all-builds", action="store_true", help="Skip all image builds.")
     parser.add_argument("-b", "--build-type",
                         choices=["image-bootc", "image-installer", "containerfile", "container-encapsulate"],
@@ -709,11 +715,11 @@ def main():
 
         # Determine versions of RPM packages
         set_rpm_version_info_vars()
-        # Prepare container images list for mirroring registries
-        common.delete_file(CONTAINER_LIST)
         if args.no_extract_images:
-            common.print_msg("Skipping container image extraction")
+            common.print_msg("Skipping container image extraction and mirroring")
         else:
+            # Prepare container images list for mirroring registries
+            common.delete_file(CONTAINER_LIST)
             extract_container_images(SOURCE_VERSION, LOCAL_REPO, CONTAINER_LIST, args.dry_run)
             # The following images are specific to layers that use fake rpms built from source
             extract_container_images(f"4.{FAKE_NEXT_MINOR_VERSION}.*", NEXT_REPO, CONTAINER_LIST, args.dry_run)
@@ -732,20 +738,20 @@ def main():
                 extract_container_images(BREW_EC_RELEASE_VERSION, BREW_REPO, CONTAINER_LIST, args.dry_run)
             if BREW_NIGHTLY_RELEASE_VERSION:
                 extract_container_images(BREW_NIGHTLY_RELEASE_VERSION, BREW_REPO, CONTAINER_LIST, args.dry_run)
-        # Sort the images list, only leaving unique entries
-        common.sort_uniq_file(CONTAINER_LIST)
-        # Process package source templates
-        ipkgdir = f"{SCRIPTDIR}/../package-sources-bootc"
-        for ifile in os.listdir(ipkgdir):
-            # Create full path for output and input file names
-            ofile = os.path.join(BOOTC_IMAGE_DIR, ifile)
-            ifile = os.path.join(ipkgdir, ifile)
-            run_template_cmd(ifile, ofile, args.dry_run)
-
-        tpldir = os.path.join(SCRIPTDIR, "..", "image-blueprints-bootc", "templates")
-        process_template_files(tpldir, args.dry_run)
-        # Run the mirror registry
-        common.run_command([f"{SCRIPTDIR}/mirror_registry.sh"], args.dry_run)
+            # Sort the images list, only leaving unique entries
+            common.sort_uniq_file(CONTAINER_LIST)
+            # Process package source templates
+            ipkgdir = f"{SCRIPTDIR}/../package-sources-bootc"
+            for ifile in os.listdir(ipkgdir):
+                # Create full path for output and input file names
+                ofile = os.path.join(BOOTC_IMAGE_DIR, ifile)
+                ifile = os.path.join(ipkgdir, ifile)
+                run_template_cmd(ifile, ofile, args.dry_run)
+
+            tpldir = os.path.join(SCRIPTDIR, "..", "image-blueprints-bootc", "templates")
+            process_template_files(tpldir, args.dry_run)
+            # Run the mirror registry
+            common.run_command([f"{SCRIPTDIR}/mirror_registry.sh"], args.dry_run)
         # Skip all image builds
         if args.skip_all_builds:
             common.print_msg("Skipping all image builds")

test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile

@@ -5,15 +5,8 @@ ARG USHIFT_RPM_REPO_NAME=microshift-local
 ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME
 
 # Configure the RPM repositories
-# - No EUS repositories in staging environment
-# - Disable default repositories to avoid pre-release repo access errors
 COPY --chmod=755 ./bootc-images/rpm-repo-config.sh /tmp/rpm-repo-config.sh
-RUN /tmp/rpm-repo-config.sh --disable-all && rm -f /tmp/rpm-repo-config.sh
-
-# Configure the RHEL mirror RPM repositories (for use in the staging environment)
-ARG RHEL_MIRROR_REPO_NAME=rhel102-mirror.repo
-ARG RHEL_MIRROR_REPO_PATH=/etc/yum.repos.d/$RHEL_MIRROR_REPO_NAME
-COPY --chmod=644 ./bootc-images/$RHEL_MIRROR_REPO_NAME $RHEL_MIRROR_REPO_PATH
+RUN /tmp/rpm-repo-config.sh --enable-eus && rm -f /tmp/rpm-repo-config.sh
 
 # Copy the MicroShift repository contents
 COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH

test/image-blueprints-bootc/el10/layer1-base/group2/rhel102-bootc.image-bootc

@@ -1,7 +1 @@
-
-# TODO: Replace this by a RHEL 10.2 image when its RPM repositories are released.
-# Bootc Image Builder has a limitation does not allowing it to build ISO images
-# from containers that use repositories protected by credentials. Thus, we build
-# from the test agent container image which uses proxy for accessing repositories.
-# registry.redhat.io/rhel10/rhel-bootc:10.2
-localhost/rhel102-test-agent:latest
+registry.redhat.io/rhel10/rhel-bootc:10.2

test/image-blueprints-bootc/el10/layer1-base/group2/rhel102-installer.image-installer

@@ -1,2 +1 @@
-
-rhel-10.2
+rhel-10.2