From 176e2d895b7b345cd7f1bbf0d7f23acfa94d46d3 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 4 Oct 2026 17:36:50 +0200 Subject: [PATCH 1/2] No kernel asm names rbx beside an operand LLVM chooses rustc refuses `rbx` as a named asm operand on x86-64 because LLVM may reserve it as a base pointer (rustc_target/src/asm/x86.rs, `rbx_reserved`), but in a function that does not need a base pointer LLVM still allocates `rbx` to a `reg`-class operand. Two kernel sites saved `rbx` by hand around such an operand: - `cpu::cpuid`: `push rbx; cpuid; mov {ebx:e}, ebx; pop rbx` with `ebx = out(reg)`. When the output landed in rbx, the `pop` overwrote CPUID's EBX with the caller's rbx. The base kernel built on this host already does so at six sites (counters::bring_up x4, which holds hardware_id's leaf 0x1F/0xB/1 reads, record_panic, panic_reboot::arm): `mov ebx, ebx; pop rbx`. It now calls core's `__cpuid_count`, whose `mov {0:r}, rbx; cpuid; xchg {0:r}, rbx` is right under any allocation of its operand. - `vtd::table::Table::replace_pair`: `xchg {lo}, rbx; lock cmpxchg16b [{entry}]; mov rbx, {lo}` with both in `reg`. When `entry` landed in rbx, the xchg swapped the address out and cmpxchg16b took the entry's new low half as its target: #GP in domain::attach, the five red rows of #720's merge-queue run. Every operand now names its register (`r8`, `rsi`), the shape portable-atomic's x86_64 cmpxchg16b uses, so no allocation can alias rbx. No other non-naked asm! in the kernel names rbx; the naked and global asm that does (entries, switch, _start) has no compiler-chosen operand. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WcU2Dsw6mDYtwYfzVHPzM8 --- kernel/src/arch/x86_64/cpu.rs | 29 +++++++++-------------------- kernel/src/arch/x86_64/vtd/table.rs | 12 +++++++----- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index 0f514f69b12..abaa90ff5f8 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -152,27 +152,16 @@ pub fn leaf_7() -> (u32, u32, u32, u32) { if cpuid(0, 0).0 >= 7 { cpuid(7, 0) } else { (0, 0, 0, 0) } } -/// CPUID with both index registers; `rbx` is saved by hand since Rust reserves it as an operand. +/// CPUID with both index registers, as `(eax, ebx, ecx, edx)`; every caller +/// queries leaf 0 first, so an unsupported leaf isn't misread as data. +/// +/// `core`'s intrinsic and not an `asm!` of our own: `rbx` is CPUID's output +/// and LLVM's reserved register, yet LLVM may still allocate it to a `reg` +/// operand, so a hand-written save of `rbx` around a compiler-chosen +/// register can lose the output. pub fn cpuid(leaf: u32, subleaf: u32) -> (u32, u32, u32, u32) { - let eax: u32; - let ebx: u32; - let ecx: u32; - let edx: u32; - // SAFETY: cpuid can't fault; push/pop rbx is balanced, and every caller queries leaf 0 first, so an unsupported leaf isn't misread as data. - unsafe { - asm!( - "push rbx", - "cpuid", - "mov {ebx:e}, ebx", - "pop rbx", - ebx = out(reg) ebx, - inout("eax") leaf => eax, - inout("ecx") subleaf => ecx, - out("edx") edx, - options(nomem), - ); - } - (eax, ebx, ecx, edx) + let r = core::arch::x86_64::__cpuid_count(leaf, subleaf); + (r.eax, r.ebx, r.ecx, r.edx) } #[inline] diff --git a/kernel/src/arch/x86_64/vtd/table.rs b/kernel/src/arch/x86_64/vtd/table.rs index a8e923db8d5..0867e4b40fb 100644 --- a/kernel/src/arch/x86_64/vtd/table.rs +++ b/kernel/src/arch/x86_64/vtd/table.rs @@ -126,13 +126,15 @@ impl Table { // SAFETY: `entry` is 16 bytes of a table `Tables::alloc` never frees, // 16-byte aligned since a table is 4 KiB-aligned; `rbx`, which LLVM // reserves, holds `new`'s low half only between the two moves. + // Every operand names its register: LLVM may give `rbx` to a `reg` + // operand, and the `xchg` would then swap the address out of it. unsafe { core::arch::asm!( - "xchg {lo}, rbx", - "lock cmpxchg16b xmmword ptr [{entry}]", - "mov rbx, {lo}", - entry = in(reg) entry.addr(), - lo = inout(reg) new.0 => _, + "xchg r8, rbx", + "lock cmpxchg16b xmmword ptr [rsi]", + "mov rbx, r8", + in("rsi") entry.addr(), + inout("r8") new.0 => _, in("rcx") new.1, inout("rax") old.0 => found_lo, inout("rdx") old.1 => found_hi, From 179c852e7d67dde41f41c0ce1d3a036985f1fdf9 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 4 Oct 2026 18:20:08 +0200 Subject: [PATCH 2/2] run_on_stack names its registers; cpuid's doc drops a false claim AArch64's `run_on_stack` zeroed `x30` in its template beside a `func = in(reg)` operand. `x30` is in the `reg` class, so wherever LLVM gave `func` to `x30` the template zeroed it before `br`, and every CPU's idle entry branched to 0. The operands now name `x0` and `x16`, as `replace_pair` names its own on x86-64. `cpuid`'s doc said every caller queries leaf 0 first; `has_rdrand`, `counters::bring_up` and `control_regs` call leaf 1 without it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WcU2Dsw6mDYtwYfzVHPzM8 --- kernel/src/arch/aarch64/cpu.rs | 10 ++++++---- kernel/src/arch/x86_64/cpu.rs | 3 +-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/kernel/src/arch/aarch64/cpu.rs b/kernel/src/arch/aarch64/cpu.rs index b87c5044638..37738467f76 100644 --- a/kernel/src/arch/aarch64/cpu.rs +++ b/kernel/src/arch/aarch64/cpu.rs @@ -113,14 +113,16 @@ pub fn halt() -> ! { /// a 16-byte-aligned stack this CPU owns. pub unsafe fn run_on_stack(top: u64, func: extern "C" fn() -> !) -> ! { // SAFETY: the caller's contract. + // Every operand names its register: LLVM may give `x30` to a `reg` + // operand, and zeroing it would then branch to 0. unsafe { asm!( - "mov sp, {sp}", + "mov sp, x0", "mov x29, xzr", "mov x30, xzr", - "br {func}", - sp = in(reg) top, - func = in(reg) func as *const () as usize, + "br x16", + in("x0") top, + in("x16") func as *const () as usize, options(noreturn), ); } diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index a2fb5c2182b..699d8c5e7d7 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -152,8 +152,7 @@ pub fn leaf_7() -> (u32, u32, u32, u32) { if cpuid(0, 0).0 >= 7 { cpuid(7, 0) } else { (0, 0, 0, 0) } } -/// CPUID with both index registers, as `(eax, ebx, ecx, edx)`; every caller -/// queries leaf 0 first, so an unsupported leaf isn't misread as data. +/// CPUID with both index registers, as `(eax, ebx, ecx, edx)`. /// /// `core`'s intrinsic and not an `asm!` of our own: `rbx` is CPUID's output /// and LLVM's reserved register, yet LLVM may still allocate it to a `reg`