diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..7e76d8752 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,5 @@ +self-hosted-runner: + labels: + - rhel10-x86_64-4c-16g + - rhel10-x86_64-16c-64g + - ubuntu-26.04 diff --git a/.github/actions/install-tmt/action.yml b/.github/actions/install-tmt/action.yml index 97e336bf9..de6ef38b0 100644 --- a/.github/actions/install-tmt/action.yml +++ b/.github/actions/install-tmt/action.yml @@ -4,9 +4,11 @@ runs: using: 'composite' steps: - name: Set up Python + id: python uses: actions/setup-python@v6 with: python-version: '3.12.14' + update-environment: false - name: Install tmt shell: bash run: | @@ -16,4 +18,5 @@ runs: # https://github.com/teemtee/tmt/issues/5068); bump deliberately. # renovate: datasource=pypi depName=tmt export VERSION=1.78.0 - pip install --user "tmt[provision-virtual]==${VERSION}" + "${{ steps.python.outputs.python-path }}" -m pip install --user "tmt[provision-virtual]==${VERSION}" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58e32e385..860a8e326 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,11 +140,11 @@ jobs: # Run basic validation checks (linting, formatting, etc) validate: - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g steps: + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main - name: Validate (default) run: just validate # Check for security vulnerabilities and license compliance @@ -171,8 +171,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 - name: Use native overlay instead of fuse-overlayfs run: | sudo sed -i 's|mount_program.*|mount_program = ""|' /etc/containers/storage.conf @@ -236,11 +236,11 @@ jobs: done # Test that we can build documentation docs: - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g steps: + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main - name: Build mdbook run: just build-mdbook env: @@ -254,14 +254,14 @@ jobs: matrix: test_os: ${{ fromJson(needs.compute-ci-level.outputs.package_os_matrix) }} - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g # Rawhide is best-effort; don't let it block merges continue-on-error: ${{ matrix.test_os == 'fedora-46' }} steps: + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main - name: Setup env run: | @@ -285,6 +285,44 @@ jobs: - name: Unit tests run: just unit-tests + # Keep a minimal host-OS compatibility lane alongside the primary RHEL 10 + # coverage: build the CentOS 10 package and boot its readonly TMT plan from + # an Ubuntu host. + test-ubuntu: + if: needs.compute-ci-level.outputs.run_heavy == 'true' + needs: compute-ci-level + runs-on: ubuntu-26.04 + + steps: + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 + with: + libvirt: true + - uses: actions/checkout@v7 + - name: Install tmt + uses: ./.github/actions/install-tmt + + - name: Setup env + run: | + BASE=$(just pullspec-for-os base centos-10) + { + echo "BOOTC_base=${BASE}" + echo "RUST_BACKTRACE=full" + } >> "$GITHUB_ENV" + + - name: Build CentOS 10 packages (and verify build system) + run: just check-buildsys + + - name: Run TMT readonly tests + run: BOOTC_SKIP_PACKAGE=1 just test-tmt readonly + + - name: Archive TMT logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: tmt-log-centos-10-ubuntu-${{ env.ARCH }} + path: /var/tmp/tmt + # Build bootc from source into a container image FROM each specified base `test_os` # running unit and integration tests (using TMT, leveraging the support for nested virtualization # in the GHA runners) @@ -339,14 +377,14 @@ jobs: - bootloader: grub-cc seal_state: sealed - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-4c-16g steps: - - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 with: libvirt: true + - uses: actions/checkout@v7 - name: Install tmt uses: ./.github/actions/install-tmt @@ -438,14 +476,14 @@ jobs: - test_os: fedora-44 variant: composefs - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g steps: - - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 with: libvirt: true + - uses: actions/checkout@v7 - name: Install tmt uses: ./.github/actions/install-tmt @@ -500,14 +538,14 @@ jobs: # centos-9 ships an older dracut that lacks the auto-install of setup-root-conf.toml - test_os: centos-9 - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g steps: - - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 with: libvirt: true + - uses: actions/checkout@v7 - name: Install tmt uses: ./.github/actions/install-tmt @@ -569,11 +607,11 @@ jobs: runs-on: ubuntu-26.04 steps: - - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 with: libvirt: true + - uses: actions/checkout@v7 - name: Install tmt uses: ./.github/actions/install-tmt @@ -609,14 +647,14 @@ jobs: test-container-export: if: needs.compute-ci-level.outputs.run_heavy == 'true' needs: [compute-ci-level, package] - runs-on: ubuntu-26.04 + runs-on: rhel10-x86_64-16c-64g steps: - - uses: actions/checkout@v7 - - name: Bootc Ubuntu Setup - uses: bootc-dev/actions/bootc-ubuntu-setup@main + - name: Bootc host setup + uses: cgwalters-bot/actions/bootc-host-setup@bootc-host-setup-rhel10 with: libvirt: true + - uses: actions/checkout@v7 - name: Setup env run: | @@ -650,7 +688,7 @@ jobs: # rather than being skipped along with it. required-checks-heavy: if: ${{ always() && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && needs.compute-ci-level.outputs.merge_queue_enabled != 'true')) }} - needs: [compute-ci-level, cargo-deny, validate, install-tests, docs, package, test-integration, test-upgrade, test-baseconfigs, test-container-export] + needs: [compute-ci-level, cargo-deny, validate, install-tests, docs, package, test-ubuntu, test-integration, test-upgrade, test-baseconfigs, test-container-export] # Cheap aggregation (jq only) — no need for a full VM. runs-on: ubuntu-slim steps: diff --git a/crates/xtask/src/bcvk.rs b/crates/xtask/src/bcvk.rs index 520640ebf..6fcccaea7 100644 --- a/crates/xtask/src/bcvk.rs +++ b/crates/xtask/src/bcvk.rs @@ -7,9 +7,19 @@ use anyhow::Result; use camino::Utf8Path; use fn_error_context::context; +use xshell::Cmd; use crate::{Bootloader, SealState}; +/// Remove inherited library search paths before launching a VM. +/// +/// Build tooling can augment `LD_LIBRARY_PATH`, and libvirt may propagate it +/// to the host QEMU process, where it can cause incompatible libraries to be +/// loaded. Keep that environment out of `bcvk libvirt run` consistently. +pub(crate) fn sanitize_bcvk_libvirt_run(cmd: Cmd<'_>) -> Cmd<'_> { + cmd.env_remove("LD_LIBRARY_PATH") +} + /// Default directory for secure boot test keys. const DEFAULT_SB_KEYS_DIR: &str = "target/test-secureboot"; diff --git a/crates/xtask/src/sysext.rs b/crates/xtask/src/sysext.rs index 0c373b08d..0b36b6b68 100644 --- a/crates/xtask/src/sysext.rs +++ b/crates/xtask/src/sysext.rs @@ -27,7 +27,7 @@ use camino::Utf8Path; use fn_error_context::context; use xshell::{Shell, cmd}; -use crate::bcvk::BcvkInstallOpts; +use crate::bcvk::{BcvkInstallOpts, sanitize_bcvk_libvirt_run}; const SYSEXT_DIR: &str = "target/sysext"; const DEV_VM_NAME: &str = "bootc-dev"; @@ -293,7 +293,9 @@ fn create_vm(sh: &Shell) -> Result<()> { bcvk_cmd = bcvk_cmd.args(&firmware_args); bcvk_cmd = bcvk_cmd.args(["--ssh-wait", &base_img]); - bcvk_cmd.run().context("Failed to create VM")?; + sanitize_bcvk_libvirt_run(bcvk_cmd) + .run() + .context("Failed to create VM")?; // Set up the sysext: create a symlink from /run/extensions/bootc // into the virtiofs-mounted versioned directory. diff --git a/crates/xtask/src/tmt.rs b/crates/xtask/src/tmt.rs index b129b213f..1c88f1057 100644 --- a/crates/xtask/src/tmt.rs +++ b/crates/xtask/src/tmt.rs @@ -37,7 +37,7 @@ const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image"; const DISTRO_CENTOS_9: &str = "centos-9"; // Import the argument types from xtask.rs -use crate::bcvk::BcvkInstallOpts; +use crate::bcvk::{BcvkInstallOpts, sanitize_bcvk_libvirt_run}; use crate::{RunTmtArgs, SealState, TmtProvisionArgs, out_of_sync_error}; /// Generate a random alphanumeric suffix for VM names @@ -228,6 +228,7 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result use std::time::Duration; let port_str = port.to_string(); + let mut last_error = None; for attempt in 1..=SSH_CONNECTIVITY_MAX_ATTEMPTS { // Test with a complex command like TMT uses (exports + whoami) // Use IdentitiesOnly=yes to prevent ssh-agent from offering other keys @@ -235,14 +236,15 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result sh, "ssh -i {key_path} -p {port_str} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o IdentitiesOnly=yes root@localhost 'export TEST=value; whoami'" ) - .ignore_stderr() - .read(); + .ignore_status() + .output(); - match &result { - Ok(output) if output.trim() == "root" => { + match result { + Ok(output) if output.status.success() && output.stdout.trim_ascii() == b"root" => { return Ok(()); } - _ => {} + Ok(output) => last_error = ssh_stderr_diagnostic(&output.stderr), + Err(err) => last_error = Some(format!("failed to execute ssh: {err}")), } if attempt % 10 == 0 { @@ -257,12 +259,20 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result } } + let diagnostic = last_error + .filter(|error| !error.is_empty()) + .unwrap_or_else(|| "no SSH diagnostic was captured".to_owned()); anyhow::bail!( - "SSH connectivity check failed after {} attempts", + "SSH connectivity check failed after {} attempts: {diagnostic}", SSH_CONNECTIVITY_MAX_ATTEMPTS ) } +fn ssh_stderr_diagnostic(stderr: &[u8]) -> Option { + let diagnostic = String::from_utf8_lossy(stderr).trim().to_owned(); + (!diagnostic.is_empty()).then_some(diagnostic) +} + #[derive(Debug, Default)] struct PlanMetadata { try_bind_storage: bool, @@ -597,10 +607,10 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> { // Launch VM with bcvk let firmware_args_slice = firmware_args.as_slice(); - let launch_result = cmd!( + let launch_result = sanitize_bcvk_libvirt_run(cmd!( sh, "bcvk libvirt run --name {vm_name} --detach {firmware_args_slice...} {COMMON_INST_ARGS...} {plan_bcvk_opts...} {log_dir_args...} {image}" - ) + )) .run() .context("Launching VM with bcvk"); @@ -878,10 +888,10 @@ pub(crate) fn tmt_provision(sh: &Shell, args: &TmtProvisionArgs) -> Result<()> { // Launch VM with bcvk // Use ds=iid-datasource-none to disable cloud-init for faster boot let firmware_args_slice = firmware_args.as_slice(); - cmd!( + sanitize_bcvk_libvirt_run(cmd!( sh, "bcvk libvirt run --name {vm_name} --detach {firmware_args_slice...} {COMMON_INST_ARGS...} {image}" - ) + )) .run() .context("Launching VM with bcvk")?; @@ -1414,6 +1424,21 @@ fn generate_integration() -> Result<(String, String)> { mod tests { use super::*; + #[test] + fn test_ssh_stderr_diagnostic() { + let cases = [ + ( + b"connection refused\n".as_slice(), + Some("connection refused"), + ), + (b" \t\n".as_slice(), None), + ]; + + for (stderr, expected) in cases { + assert_eq!(ssh_stderr_diagnostic(stderr).as_deref(), expected); + } + } + #[test] fn test_boot_context_values() { assert_eq!(