ci: Add experimental RHEL 10 runner coverage

.github/actionlint.yaml

@@ -0,0 +1,5 @@
+self-hosted-runner:
+  labels:
+    - rhel10-x86_64-4c-16g
+    - rhel10-x86_64-16c-64g
+    - ubuntu-26.04

.github/actions/install-tmt/action.yml

@@ -4,9 +4,11 @@ runs:
   using: 'composite'
   steps:
     - name: Set up Python
+      id: python
       uses: actions/setup-python@v6
       with:
         python-version: '3.12.14'
+        update-environment: false
     - name: Install tmt
       shell: bash
       run: |
@@ -16,4 +18,5 @@ runs:
         # https://github.com/teemtee/tmt/issues/5068); bump deliberately.
         # renovate: datasource=pypi depName=tmt
         export VERSION=1.78.0
-        pip install --user "tmt[provision-virtual]==${VERSION}"
+        "${{ steps.python.outputs.python-path }}" -m pip install --user "tmt[provision-virtual]==${VERSION}"
+        echo "$HOME/.local/bin" >> "$GITHUB_PATH"

.github/workflows/ci.yml

@@ -140,11 +140,11 @@ jobs:
 
   # Run basic validation checks (linting, formatting, etc)
   validate:
-    runs-on: ubuntu-26.04
+    runs-on: rhel10-x86_64-16c-64g
     steps:
+      - name: Bootc host setup
+        uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10
       - uses: actions/checkout@v7
-      - name: Bootc Ubuntu Setup
-        uses: bootc-dev/actions/bootc-ubuntu-setup@main
       - name: Validate (default)
         run: just validate
   # Check for security vulnerabilities and license compliance
@@ -171,8 +171,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v7
-      - name: Bootc Ubuntu Setup
-        uses: bootc-dev/actions/bootc-ubuntu-setup@main
+      - name: Bootc host setup
+        uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10
       - name: Use native overlay instead of fuse-overlayfs
         run: |
           sudo sed -i 's|mount_program.*|mount_program = ""|' /etc/containers/storage.conf
@@ -236,11 +236,11 @@ jobs:
           done
   # Test that we can build documentation
   docs:
-    runs-on: ubuntu-26.04
+    runs-on: rhel10-x86_64-16c-64g
     steps:
+      - name: Bootc host setup
+        uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10
       - uses: actions/checkout@v7
-      - name: Bootc Ubuntu Setup
-        uses: bootc-dev/actions/bootc-ubuntu-setup@main
       - name: Build mdbook
         run: just build-mdbook
         env:
@@ -254,14 +254,14 @@ jobs:
       matrix:
         test_os: ${{ fromJson(needs.compute-ci-level.outputs.package_os_matrix) }}
 
-    runs-on: ubuntu-26.04
+    runs-on: rhel10-x86_64-16c-64g
     # Rawhide is best-effort; don't let it block merges
     continue-on-error: ${{ matrix.test_os == 'fedora-46' }}
 
     steps:
+      - name: Bootc host setup
+        uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10
       - uses: actions/checkout@v7
-      - name: Bootc Ubuntu Setup
-        uses: bootc-dev/actions/bootc-ubuntu-setup@main
 
       - name: Setup env
         run: |
@@ -285,6 +285,44 @@ jobs:
       - name: Unit tests
         run: just unit-tests
 
+  # Keep a minimal host-OS compatibility lane alongside the primary RHEL 10
+  # coverage: build the CentOS 10 package and boot its readonly TMT plan from
+  # an Ubuntu host.
+  test-ubuntu:
+    if: needs.compute-ci-level.outputs.run_heavy == 'true'
+    needs: compute-ci-level
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Bootc host setup
+        uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10
+        with:
+          libvirt: true
+      - uses: actions/checkout@v7
+      - name: Install tmt
+        uses: ./.github/actions/install-tmt
+
+      - name: Setup env
+        run: |
+          BASE=$(just pullspec-for-os base centos-10)
+          {
+            echo "BOOTC_base=${BASE}"
+            echo "RUST_BACKTRACE=full"
+          } >> "$GITHUB_ENV"
+
+      - name: Build CentOS 10 packages (and verify build system)
+        run: just check-buildsys
+
+      - name: Run TMT readonly tests
+        run: BOOTC_SKIP_PACKAGE=1 just test-tmt readonly
+
+      - name: Archive TMT logs
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tmt-log-centos-10-ubuntu-${{ env.ARCH }}
+          path: /var/tmp/tmt
+
   # Build bootc from source into a container image FROM each specified base `test_os`
   # running unit and integration tests (using TMT, leveraging the support for nested virtualization
   # in the GHA runners)
@@ -339,14 +377,14 @@ jobs:
           - bootloader: grub-cc
             seal_state: sealed
 
-    runs-on: ubuntu-26.04
+    runs-on: rhel10-x86_64-4c-16g
 
     steps:
-      - uses: actions/checkout@v7
-      - name: Bootc Ubuntu Setup
-        uses: bootc-dev/actions/bootc-ubuntu-setup@main
+      - name: Bootc host setup
+        uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10
         with:
           libvirt: true
+      - uses: actions/checkout@v7
       - name: Install tmt
         uses: ./.github/actions/install-tmt
 
@@ -438,14 +476,14 @@ jobs:
           - test_os: fedora-44
             variant: composefs
 
-    runs-on: ubuntu-26.04
+    runs-on: rhel10-x86_64-16c-64g
 
     steps:
-      - uses: actions/checkout@v7
-      - name: Bootc Ubuntu Setup
-        uses: bootc-dev/actions/bootc-ubuntu-setup@main
+      - name: Bootc host setup
+        uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10
         with:
           libvirt: true
+      - uses: actions/checkout@v7
       - name: Install tmt
         uses: ./.github/actions/install-tmt
 
@@ -500,14 +538,14 @@ jobs:
           # centos-9 ships an older dracut that lacks the auto-install of setup-root-conf.toml
           - test_os: centos-9
 
-    runs-on: ubuntu-26.04
+    runs-on: rhel10-x86_64-16c-64g
 
     steps:
-      - uses: actions/checkout@v7
-      - name: Bootc Ubuntu Setup
-        uses: bootc-dev/actions/bootc-ubuntu-setup@main
+      - name: Bootc host setup
+        uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10
         with:
           libvirt: true
+      - uses: actions/checkout@v7
       - name: Install tmt
         uses: ./.github/actions/install-tmt
 
@@ -569,11 +607,11 @@ jobs:
     runs-on: ubuntu-26.04
 
     steps:
-      - uses: actions/checkout@v7
-      - name: Bootc Ubuntu Setup
-        uses: bootc-dev/actions/bootc-ubuntu-setup@main
+      - name: Bootc host setup
+        uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10
         with:
           libvirt: true
+      - uses: actions/checkout@v7
       - name: Install tmt
         uses: ./.github/actions/install-tmt
 
@@ -609,14 +647,14 @@ jobs:
   test-container-export:
     if: needs.compute-ci-level.outputs.run_heavy == 'true'
     needs: [compute-ci-level, package]
-    runs-on: ubuntu-26.04
+    runs-on: rhel10-x86_64-16c-64g
 
     steps:
-      - uses: actions/checkout@v7
-      - name: Bootc Ubuntu Setup
-        uses: bootc-dev/actions/bootc-ubuntu-setup@main
+      - name: Bootc host setup
+        uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10
         with:
           libvirt: true
+      - uses: actions/checkout@v7
 
       - name: Setup env
         run: |
@@ -650,7 +688,7 @@ jobs:
   # rather than being skipped along with it.
   required-checks-heavy:
     if: ${{ always() && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.compute-ci-level.outputs.merge_queue_enabled != 'true')) }}
-    needs: [compute-ci-level, cargo-deny, validate, install-tests, docs, package, test-integration, test-upgrade, test-baseconfigs, test-container-export]
+    needs: [compute-ci-level, cargo-deny, validate, install-tests, docs, package, test-ubuntu, test-integration, test-upgrade, test-baseconfigs, test-container-export]
     # Cheap aggregation (jq only) — no need for a full VM.
     runs-on: ubuntu-slim
     steps:

crates/xtask/src/bcvk.rs

@@ -7,9 +7,19 @@
 use anyhow::Result;
 use camino::Utf8Path;
 use fn_error_context::context;
+use xshell::Cmd;
 
 use crate::{Bootloader, SealState};
 
+/// Remove inherited library search paths before launching a VM.
+///
+/// Build tooling can augment `LD_LIBRARY_PATH`, and libvirt may propagate it
+/// to the host QEMU process, where it can cause incompatible libraries to be
+/// loaded. Keep that environment out of `bcvk libvirt run` consistently.
+pub(crate) fn sanitize_bcvk_libvirt_run(cmd: Cmd<'_>) -> Cmd<'_> {
+    cmd.env_remove("LD_LIBRARY_PATH")
+}
+
 /// Default directory for secure boot test keys.
 const DEFAULT_SB_KEYS_DIR: &str = "target/test-secureboot";
 

crates/xtask/src/sysext.rs

@@ -27,7 +27,7 @@ use camino::Utf8Path;
 use fn_error_context::context;
 use xshell::{Shell, cmd};
 
-use crate::bcvk::BcvkInstallOpts;
+use crate::bcvk::{BcvkInstallOpts, sanitize_bcvk_libvirt_run};
 
 const SYSEXT_DIR: &str = "target/sysext";
 const DEV_VM_NAME: &str = "bootc-dev";
@@ -293,7 +293,9 @@ fn create_vm(sh: &Shell) -> Result<()> {
     bcvk_cmd = bcvk_cmd.args(&firmware_args);
 
     bcvk_cmd = bcvk_cmd.args(["--ssh-wait", &base_img]);
-    bcvk_cmd.run().context("Failed to create VM")?;
+    sanitize_bcvk_libvirt_run(bcvk_cmd)
+        .run()
+        .context("Failed to create VM")?;
 
     // Set up the sysext: create a symlink from /run/extensions/bootc
     // into the virtiofs-mounted versioned directory.

crates/xtask/src/tmt.rs

@@ -37,7 +37,7 @@ const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image";
 const DISTRO_CENTOS_9: &str = "centos-9";
 
 // Import the argument types from xtask.rs
-use crate::bcvk::BcvkInstallOpts;
+use crate::bcvk::{BcvkInstallOpts, sanitize_bcvk_libvirt_run};
 use crate::{RunTmtArgs, SealState, TmtProvisionArgs, out_of_sync_error};
 
 /// Generate a random alphanumeric suffix for VM names
@@ -228,21 +228,23 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result
     use std::time::Duration;
 
     let port_str = port.to_string();
+    let mut last_error = None;
     for attempt in 1..=SSH_CONNECTIVITY_MAX_ATTEMPTS {
         // Test with a complex command like TMT uses (exports + whoami)
         // Use IdentitiesOnly=yes to prevent ssh-agent from offering other keys
         let result = cmd!(
             sh,
             "ssh -i {key_path} -p {port_str} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o IdentitiesOnly=yes root@localhost 'export TEST=value; whoami'"
         )
-        .ignore_stderr()
-        .read();
+        .ignore_status()
+        .output();
 
-        match &result {
-            Ok(output) if output.trim() == "root" => {
+        match result {
+            Ok(output) if output.status.success() && output.stdout.trim_ascii() == b"root" => {
                 return Ok(());
             }
-            _ => {}
+            Ok(output) => last_error = ssh_stderr_diagnostic(&output.stderr),
+            Err(err) => last_error = Some(format!("failed to execute ssh: {err}")),
         }
 
         if attempt % 10 == 0 {
@@ -257,12 +259,20 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result
         }
     }
 
+    let diagnostic = last_error
+        .filter(|error| !error.is_empty())
+        .unwrap_or_else(|| "no SSH diagnostic was captured".to_owned());
     anyhow::bail!(
-        "SSH connectivity check failed after {} attempts",
+        "SSH connectivity check failed after {} attempts: {diagnostic}",
         SSH_CONNECTIVITY_MAX_ATTEMPTS
     )
 }
 
+fn ssh_stderr_diagnostic(stderr: &[u8]) -> Option<String> {
+    let diagnostic = String::from_utf8_lossy(stderr).trim().to_owned();
+    (!diagnostic.is_empty()).then_some(diagnostic)
+}
+
 #[derive(Debug, Default)]
 struct PlanMetadata {
     try_bind_storage: bool,
@@ -597,10 +607,10 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
 
         // Launch VM with bcvk
         let firmware_args_slice = firmware_args.as_slice();
-        let launch_result = cmd!(
+        let launch_result = sanitize_bcvk_libvirt_run(cmd!(
             sh,
             "bcvk libvirt run --name {vm_name} --detach {firmware_args_slice...} {COMMON_INST_ARGS...} {plan_bcvk_opts...} {log_dir_args...} {image}"
-        )
+        ))
         .run()
         .context("Launching VM with bcvk");
 
@@ -878,10 +888,10 @@ pub(crate) fn tmt_provision(sh: &Shell, args: &TmtProvisionArgs) -> Result<()> {
     // Launch VM with bcvk
     // Use ds=iid-datasource-none to disable cloud-init for faster boot
     let firmware_args_slice = firmware_args.as_slice();
-    cmd!(
+    sanitize_bcvk_libvirt_run(cmd!(
         sh,
         "bcvk libvirt run --name {vm_name} --detach {firmware_args_slice...} {COMMON_INST_ARGS...} {image}"
-    )
+    ))
     .run()
     .context("Launching VM with bcvk")?;
 
@@ -1414,6 +1424,21 @@ fn generate_integration() -> Result<(String, String)> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_ssh_stderr_diagnostic() {
+        let cases = [
+            (
+                b"connection refused\n".as_slice(),
+                Some("connection refused"),
+            ),
+            (b" \t\n".as_slice(), None),
+        ];
+
+        for (stderr, expected) in cases {
+            assert_eq!(ssh_stderr_diagnostic(stderr).as_deref(), expected);
+        }
+    }
+
     #[test]
     fn test_boot_context_values() {
         assert_eq!(