From a3dc4f17ceadd4a5764b59b7cf9cddaee0abc212 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 17 Sep 2026 16:51:10 -0400 Subject: [PATCH 1/3] tmt: Report SSH readiness failures Keep the final SSH client diagnostic from the readiness retry loop so transport, authentication, and execution failures are actionable without flooding CI logs. Assisted-by: AI --- crates/xtask/src/tmt.rs | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index b129b213f..796926b2d 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -228,6 +228,7 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result use std::time::Duration; let port_str = port.to_string(); + let mut last_error = None; for attempt in 1..=SSH_CONNECTIVITY_MAX_ATTEMPTS { // Test with a complex command like TMT uses (exports + whoami) // Use IdentitiesOnly=yes to prevent ssh-agent from offering other keys @@ -235,14 +236,15 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result sh, "ssh -i {key_path} -p {port_str} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o IdentitiesOnly=yes root@localhost 'export TEST=value; whoami'" ) - .ignore_stderr() - .read(); + .ignore_status() + .output(); - match &result { - Ok(output) if output.trim() == "root" => { + match result { + Ok(output) if output.status.success() && output.stdout.trim_ascii() == b"root" => { return Ok(()); } - _ => {} + Ok(output) => last_error = ssh_stderr_diagnostic(&output.stderr), + Err(err) => last_error = Some(format!("failed to execute ssh: {err}")), } if attempt % 10 == 0 { @@ -257,12 +259,20 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result } } + let diagnostic = last_error + .filter(|error| !error.is_empty()) + .unwrap_or_else(|| "no SSH diagnostic was captured".to_owned()); anyhow::bail!( - "SSH connectivity check failed after {} attempts", + "SSH connectivity check failed after {} attempts: {diagnostic}", SSH_CONNECTIVITY_MAX_ATTEMPTS ) } +fn ssh_stderr_diagnostic(stderr: &[u8]) -> Option { + let diagnostic = String::from_utf8_lossy(stderr).trim().to_owned(); + (!diagnostic.is_empty()).then_some(diagnostic) +} + #[derive(Debug, Default)] struct PlanMetadata { try_bind_storage: bool, @@ -1414,6 +1424,21 @@ fn generate_integration() -> Result<(String, String)> { mod tests { use super::*; + #[test] + fn test_ssh_stderr_diagnostic() { + let cases = [ + ( + b"connection refused\n".as_slice(), + Some("connection refused"), + ), + (b" \t\n".as_slice(), None), + ]; + + for (stderr, expected) in cases { + assert_eq!(ssh_stderr_diagnostic(stderr).as_deref(), expected); + } + } + #[test] fn test_boot_context_values() { assert_eq!( From 1695ff444c17099a926603b1b61c3c3f25df308a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 17 Sep 2026 16:51:10 -0400 Subject: [PATCH 2/3] ci: Move workloads to RHEL 10 runner Move the existing CI workloads to the stock RHEL 10 partner runner rather than maintaining a duplicate experimental workflow. The temporary shared host action keeps Ubuntu-specific setup out of this repository and provides the RHEL-packaged Rust and Cargo toolchain. Run host setup before checkout because the stock RHEL image does not provide git. Assisted-by: AI --- .github/actionlint.yaml | 5 ++ .github/actions/install-tmt/action.yml | 5 +- .github/workflows/ci.yml | 100 +++++++++++++++++-------- 3 files changed, 78 insertions(+), 32 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..7e76d8752 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,5 @@ +self-hosted-runner: + labels: + - rhel10-x86_64-4c-16g + - rhel10-x86_64-16c-64g + - ubuntu-26.04 diff --git a/.github/actions/install-tmt/action.yml b/.github/actions/install-tmt/action.yml index 97e336bf9..de6ef38b0 100644 --- a/.github/actions/install-tmt/action.yml +++ b/.github/actions/install-tmt/action.yml @@ -4,9 +4,11 @@ runs: using: 'composite' steps: - name: Set up Python + id: python uses: actions/setup-python@v6 with: python-version: '3.12.14' + update-environment: false - name: Install tmt shell: bash run: | @@ -16,4 +18,5 @@ runs: # https://github.com/teemtee/tmt/issues/5068); bump deliberately. # renovate: datasource=pypi depName=tmt export VERSION=1.78.0 - pip install --user "tmt[provision-virtual]==${VERSION}" + "${{ steps.python.outputs.python-path }}" -m pip install --user "tmt[provision-virtual]==${VERSION}" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58e32e385..860a8e326 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,11 +140,11 @@ jobs: # Run basic validation checks (linting, formatting, etc) validate: - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g steps: + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main - name: Validate (default) run: just validate # Check for security vulnerabilities and license compliance @@ -171,8 +171,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 - name: Use native overlay instead of fuse-overlayfs run: | sudo sed -i 's|mount_program.*|mount_program = ""|' /etc/containers/storage.conf @@ -236,11 +236,11 @@ jobs: done # Test that we can build documentation docs: - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g steps: + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main - name: Build mdbook run: just build-mdbook env: @@ -254,14 +254,14 @@ jobs: matrix: test_os: ${{ fromJson(needs.compute-ci-level.outputs.package_os_matrix) }} - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g # Rawhide is best-effort; don't let it block merges continue-on-error: ${{ matrix.test_os == 'fedora-46' }} steps: + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main - name: Setup env run: | @@ -285,6 +285,44 @@ jobs: - name: Unit tests run: just unit-tests + # Keep a minimal host-OS compatibility lane alongside the primary RHEL 10 + # coverage: build the CentOS 10 package and boot its readonly TMT plan from + # an Ubuntu host. + test-ubuntu: + if: needs.compute-ci-level.outputs.run_heavy == 'true' + needs: compute-ci-level + runs-on: ubuntu-26.04 + + steps: + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 + with: + libvirt: true + - uses: actions/checkout@v7 + - name: Install tmt + uses: ./.github/actions/install-tmt + + - name: Setup env + run: | + BASE=$(just pullspec-for-os base centos-10) + { + echo "BOOTC_base=${BASE}" + echo "RUST_BACKTRACE=full" + } >> "$GITHUB_ENV" + + - name: Build CentOS 10 packages (and verify build system) + run: just check-buildsys + + - name: Run TMT readonly tests + run: BOOTC_SKIP_PACKAGE=1 just test-tmt readonly + + - name: Archive TMT logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: tmt-log-centos-10-ubuntu-${{ env.ARCH }} + path: /var/tmp/tmt + # Build bootc from source into a container image FROM each specified base `test_os` # running unit and integration tests (using TMT, leveraging the support for nested virtualization # in the GHA runners) @@ -339,14 +377,14 @@ jobs: - bootloader: grub-cc seal_state: sealed - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-4c-16g steps: - - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 with: libvirt: true + - uses: actions/checkout@v7 - name: Install tmt uses: ./.github/actions/install-tmt @@ -438,14 +476,14 @@ jobs: - test_os: fedora-44 variant: composefs - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g steps: - - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 with: libvirt: true + - uses: actions/checkout@v7 - name: Install tmt uses: ./.github/actions/install-tmt @@ -500,14 +538,14 @@ jobs: # centos-9 ships an older dracut that lacks the auto-install of setup-root-conf.toml - test_os: centos-9 - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g steps: - - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 with: libvirt: true + - uses: actions/checkout@v7 - name: Install tmt uses: ./.github/actions/install-tmt @@ -569,11 +607,11 @@ jobs: runs-on: ubuntu-26.04 steps: - - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 with: libvirt: true + - uses: actions/checkout@v7 - name: Install tmt uses: ./.github/actions/install-tmt @@ -609,14 +647,14 @@ jobs: test-container-export: if: needs.compute-ci-level.outputs.run_heavy == 'true' needs: [compute-ci-level, package] - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g steps: - - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 with: libvirt: true + - uses: actions/checkout@v7 - name: Setup env run: | @@ -650,7 +688,7 @@ jobs: # rather than being skipped along with it. required-checks-heavy: if: ${{ always() && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.compute-ci-level.outputs.merge_queue_enabled != 'true')) }} - needs: [compute-ci-level, cargo-deny, validate, install-tests, docs, package, test-integration, test-upgrade, test-baseconfigs, test-container-export] + needs: [compute-ci-level, cargo-deny, validate, install-tests, docs, package, test-ubuntu, test-integration, test-upgrade, test-baseconfigs, test-container-export] # Cheap aggregation (jq only) — no need for a full VM. runs-on: ubuntu-slim steps: From ae1840af5f776f564b2191bb68f527387767dfd1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 17 Sep 2026 23:01:09 -0400 Subject: [PATCH 3/3] xtask: Sanitize QEMU library path Build tooling augments LD_LIBRARY_PATH while running xtask. Libvirt can propagate that environment into host QEMU processes, causing incompatible libraries to make QEMU exit before its monitor is available on RHEL 10. Keep inherited library search paths out of every bcvk VM launch. Assisted-by: AI --- crates/xtask/src/bcvk.rs | 10 ++++++++++ crates/xtask/src/sysext.rs | 6 ++++-- crates/xtask/src/tmt.rs | 10 +++++----- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/xtask/src/bcvk.rs b/crates/xtask/src/bcvk.rs index 520640ebf..6fcccaea7 100644 --- a/crates/xtask/src/bcvk.rs +++ b/crates/xtask/src/bcvk.rs @@ -7,9 +7,19 @@ use anyhow::Result; use camino::Utf8Path; use fn_error_context::context; +use xshell::Cmd; use crate::{Bootloader, SealState}; +/// Remove inherited library search paths before launching a VM. +/// +/// Build tooling can augment `LD_LIBRARY_PATH`, and libvirt may propagate it +/// to the host QEMU process, where it can cause incompatible libraries to be +/// loaded. Keep that environment out of `bcvk libvirt run` consistently. +pub(crate) fn sanitize_bcvk_libvirt_run(cmd: Cmd<'_>) -> Cmd<'_> { + cmd.env_remove("LD_LIBRARY_PATH") +} + /// Default directory for secure boot test keys. const DEFAULT_SB_KEYS_DIR: &str = "target/test-secureboot"; diff --git a/crates/xtask/src/sysext.rs b/crates/xtask/src/sysext.rs index 0c373b08d..0b36b6b68 100644 --- a/crates/xtask/src/sysext.rs +++ b/crates/xtask/src/sysext.rs @@ -27,7 +27,7 @@ use camino::Utf8Path; use fn_error_context::context; use xshell::{Shell, cmd}; -use crate::bcvk::BcvkInstallOpts; +use crate::bcvk::{BcvkInstallOpts, sanitize_bcvk_libvirt_run}; const SYSEXT_DIR: &str = "target/sysext"; const DEV_VM_NAME: &str = "bootc-dev"; @@ -293,7 +293,9 @@ fn create_vm(sh: &Shell) -> Result<()> { bcvk_cmd = bcvk_cmd.args(&firmware_args); bcvk_cmd = bcvk_cmd.args(["--ssh-wait", &base_img]); - bcvk_cmd.run().context("Failed to create VM")?; + sanitize_bcvk_libvirt_run(bcvk_cmd) + .run() + .context("Failed to create VM")?; // Set up the sysext: create a symlink from /run/extensions/bootc // into the virtiofs-mounted versioned directory. diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index 796926b2d..1c88f1057 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -37,7 +37,7 @@ const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image"; const DISTRO_CENTOS_9: &str = "centos-9"; // Import the argument types from xtask.rs -use crate::bcvk::BcvkInstallOpts; +use crate::bcvk::{BcvkInstallOpts, sanitize_bcvk_libvirt_run}; use crate::{RunTmtArgs, SealState, TmtProvisionArgs, out_of_sync_error}; /// Generate a random alphanumeric suffix for VM names @@ -607,10 +607,10 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { // Launch VM with bcvk let firmware_args_slice = firmware_args.as_slice(); - let launch_result = cmd!( + let launch_result = sanitize_bcvk_libvirt_run(cmd!( sh, "bcvk libvirt run --name {vm_name} --detach {firmware_args_slice...} {COMMON_INST_ARGS...} {plan_bcvk_opts...} {log_dir_args...} {image}" - ) + )) .run() .context("Launching VM with bcvk"); @@ -888,10 +888,10 @@ pub(crate) fn tmt_provision(sh: &Shell, args: &TmtProvisionArgs) -> Result<()> { // Launch VM with bcvk // Use ds=iid-datasource-none to disable cloud-init for faster boot let firmware_args_slice = firmware_args.as_slice(); - cmd!( + sanitize_bcvk_libvirt_run(cmd!( sh, "bcvk libvirt run --name {vm_name} --detach {firmware_args_slice...} {COMMON_INST_ARGS...} {image}" - ) + )) .run() .context("Launching VM with bcvk")?;