From cd3c0a571fa069de1d319bf68400b47f6ffc641c Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 02:56:44 +0200 Subject: [PATCH 1/9] toyos-amdvi: the interrupt half, the 128-bit remapping entry and its invalidation Stage 6 of issues/an-amd-vi-machine-isolates-no-device.md encodes, against AMD 48882 rev 3.11's own field positions: - a device table entry's interrupt half (Table 7, bits 191:128): IV, a 256-entry table (IntTabLen 1000b) at its root, IntCtl 00b (aborted) or 10b (remapped), and the pass bits where Table 7 puts Table 106's; `Setting::CLAIMED`, which passes nothing and gives no SysMgt (review B2 of the stage 5 design); - the 128-bit remapping entry, GuestMode 0 (Figure 17, Table 22; Figure 19 with XTEn): RemapEn, fixed, physical, the vector at 71:64, an 8-bit or a 32-bit destination; - INVALIDATE_INTERRUPT_TABLE (Figure 48), CONTROL's GAEn and XTEn, and EFR's GASup and XTSup. Host tests hold each against the specification's bit positions, and the bench's EFR as having GASup and XTSup. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- kernel/amdvi/src/dte.rs | 45 ++++++++++++++++-- kernel/amdvi/src/irte.rs | 46 ++++++++++++++++++ kernel/amdvi/src/lib.rs | 10 ++-- kernel/amdvi/src/queue.rs | 4 ++ kernel/amdvi/src/reg.rs | 14 ++++++ kernel/amdvi/tests/encodings.rs | 83 ++++++++++++++++++++++++++++++++- 6 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 kernel/amdvi/src/irte.rs diff --git a/kernel/amdvi/src/dte.rs b/kernel/amdvi/src/dte.rs index 611024a858..e50a7b9eee 100644 --- a/kernel/amdvi/src/dte.rs +++ b/kernel/amdvi/src/dte.rs @@ -21,21 +21,58 @@ const SYS_MGT_SHIFT: u32 = 40; pub const MODE: u8 = 4; /// What an IVHD device entry's DTE-setting byte declares for its requesters -/// (§5.2.2, Table 106). Of it, a translation entry carries `SysMgt` alone: -/// the pass bits are the interrupt half's, which nothing here sets valid. +/// (§5.2.2, Table 106): the `SysMgt` its translation half carries, and the +/// pass bits its interrupt half does. #[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] pub struct Setting(pub u8); impl Setting { + /// What a function a process drives is given whatever firmware declared: + /// no `SysMgt`, and no NMI, INIT, ExtInt or LINT forwarded unremapped + /// (Table 10), so its device reaches no CPU but by its own vectors. + pub const CLAIMED: Self = Self(0); + /// Bits 5:4, the `SysMgt` firmware declares the requester needs. pub const fn sys_mgt(self) -> u64 { (self.0 as u64 >> 4) & 0b11 } + + /// The pass bits, at their places in the interrupt half's doubleword + /// (Table 106 bits 0, 1, 2, 6, 7 to Table 7 bits 184, 185, 186, 190, 191). + const fn passes(self) -> u64 { + let byte = self.0 as u64; + (byte & 0b111) << 56 | (byte >> 6 & 0b11) << 62 + } +} + +/// `IV`, bit 128: the interrupt half is valid. +const IV: u64 = 1 << 0; +const INT_TAB_LEN_SHIFT: u32 = 1; +/// `IntCtl`, bits 189:188. +const INT_CTL_SHIFT: u32 = 60; +const INT_CTL_REMAPPED: u64 = 0b10; + +/// Entries in a requester's remapping table: 256, `IntTabLen` 1000b, one +/// per vector, 4 KiB of 128-bit entries. +pub const INTERRUPT_ENTRIES: usize = 256; + +/// One entry's interrupt half, bits 191:128, which an interrupt change +/// replaces in one 64-bit write (§2.2.2.2). +pub const fn interrupts_blocked(setting: Setting) -> u64 { + // `IntCtl` 00b: every fixed and arbitrated message target-aborted (Table 9). + IV | setting.passes() +} + +/// The interrupt half remapping every fixed and arbitrated message through +/// the 256-entry table at `table` (Table 7: root pointer 179:134, `IntTabLen` +/// 132:129). +pub const fn interrupts_remapped(table: Phys<12>, setting: Setting) -> u64 { + IV | 0b1000 << INT_TAB_LEN_SHIFT | table.get() | INT_CTL_REMAPPED << INT_CTL_SHIFT | setting.passes() } /// One entry's translation half, the 128 bits a translation change replaces -/// in one write (§2.2.2.2); the interrupt half above it stays zero, `IV` -/// clear, so the unit forwards a requester's interrupts unremapped (Table 9). +/// in one write (§2.2.2.2); the interrupt half above it is replaced on its +/// own. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Dte([u64; 2]); diff --git a/kernel/amdvi/src/irte.rs b/kernel/amdvi/src/irte.rs new file mode 100644 index 0000000000..575ed966f2 --- /dev/null +++ b/kernel/amdvi/src/irte.rs @@ -0,0 +1,46 @@ +//! A requester's interrupt remapping table entry in its 128-bit format +//! (§2.2.5.2, Figure 17 and Table 22, `GuestMode` 0; with `XTEn`, Figure 19): +//! selected by `CONTROL.GAEn` with `GAMEn` 000b (Table 21). A fixed message's +//! data bits 10:0 index the table (Figure 14), so with delivery mode 000b the +//! entry is the message's own vector. + +const REMAP_EN: u64 = 1 << 0; +const DESTINATION_SHIFT: u32 = 8; +/// `Vector`, bits 71:64: bits 7:0 of the second doubleword. +const VECTOR_SHIFT: u32 = 0; + +/// Where an entry delivers: the whole x2APIC id under `XTEn`, and without it +/// an 8-bit APIC id, bits 31:16 then reserved. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Destination { + Extended(u32), + Narrow(u8), +} + +/// An entry delivering `vector` to `destination`, fixed (`IntType` 000b), +/// physical (`DM` 0), no EOI requested; or [`ABSENT`], whose message is +/// target-aborted and recorded (`RemapEn` 0). +pub const fn entry(vector: u8, destination: Destination) -> [u64; 2] { + let (low, high) = match destination { + Destination::Extended(id) => ((id as u64 & 0xFF_FFFF) << DESTINATION_SHIFT, (id as u64 >> 24) << 56), + Destination::Narrow(id) => ((id as u64) << DESTINATION_SHIFT, 0), + }; + [REMAP_EN | low, (vector as u64) << VECTOR_SHIFT | high] +} + +pub const ABSENT: [u64; 2] = [0, 0]; + +/// Whether an entry read back delivers anything. +pub const fn present(words: [u64; 2]) -> bool { + words[0] & REMAP_EN != 0 +} + +/// The vector an entry read back delivers. +pub const fn vector(words: [u64; 2]) -> u8 { + (words[1] >> VECTOR_SHIFT) as u8 +} + +/// The destination an entry read back delivers to, all 32 bits. +pub const fn destination(words: [u64; 2]) -> u32 { + ((words[0] >> DESTINATION_SHIFT) & 0xFF_FFFF) as u32 | ((words[1] >> 56) as u32) << 24 +} diff --git a/kernel/amdvi/src/lib.rs b/kernel/amdvi/src/lib.rs index 9b22602ca6..23c5b6134a 100644 --- a/kernel/amdvi/src/lib.rs +++ b/kernel/amdvi/src/lib.rs @@ -1,11 +1,14 @@ //! Every word an AMD-Vi unit and its driver exchange, as AMD 48882 rev 3.11 //! lays it out: the device table entry ([`dte`]), the host page tables a -//! translating entry names ([`table`]), the commands and event records -//! ([`queue`]), and the register fields this driver writes or reads ([`reg`]). +//! translating entry names ([`table`]), the interrupt remapping table entry +//! ([`irte`]), the commands and event records ([`queue`]), and the register +//! fields this driver writes or reads ([`reg`]). //! //! The crate encodes one configuration: a requester is blocked, or translated //! through a four-level host table that maps 2 MiB pages, read and write, -//! forced coherent. No entry passes DMA untranslated. +//! forced coherent; and its fixed and arbitrated messages are aborted, or +//! remapped through a table of its own indexed by vector. No entry passes DMA +//! untranslated or a message unremapped. //! //! `no_std`, no allocation, no `unsafe`. @@ -13,6 +16,7 @@ #![forbid(unsafe_code)] pub mod dte; +pub mod irte; pub mod queue; pub mod reg; pub mod table; diff --git a/kernel/amdvi/src/queue.rs b/kernel/amdvi/src/queue.rs index a85789543d..90f1918105 100644 --- a/kernel/amdvi/src/queue.rs +++ b/kernel/amdvi/src/queue.rs @@ -20,6 +20,9 @@ pub enum Command { /// `INVALIDATE_IOMMU_PAGES` (Figure 46) over every address of one domain, /// directories with them: `S` and `PDE` set, address 7_FFFF_FFFF_FFFFh. InvalidateDomain(u16), + /// `INVALIDATE_INTERRUPT_TABLE` (Figure 48): everything the unit cached + /// of this id's remapping table. + InvalidateInterrupts(u16), /// `INVALIDATE_IOMMU_ALL` (Figure 51): every cache the unit holds. InvalidateAll, } @@ -30,6 +33,7 @@ impl Command { Self::CompletionWait { store, data } => [1 << CODE_SHIFT | store.get() | 1, data], Self::InvalidateDevice(id) => [2 << CODE_SHIFT | id as u64, 0], Self::InvalidateDomain(domain) => [3 << CODE_SHIFT | (domain as u64) << 32, 0x7_FFFF_FFFF_FFFF << 12 | 0b11], + Self::InvalidateInterrupts(id) => [5 << CODE_SHIFT | id as u64, 0], Self::InvalidateAll => [8 << CODE_SHIFT, 0], } } diff --git a/kernel/amdvi/src/reg.rs b/kernel/amdvi/src/reg.rs index 34d141cbb4..8bbe26596e 100644 --- a/kernel/amdvi/src/reg.rs +++ b/kernel/amdvi/src/reg.rs @@ -28,7 +28,11 @@ pub mod control { pub const CMD_BUF_EN: u64 = 1 << 12; pub const PPR_LOG_EN: u64 = 1 << 13; pub const PPR_INT_EN: u64 = 1 << 14; + /// The 128-bit remapping entry (Table 21), with `GAMEn` 000b. + pub const GA_EN: u64 = 1 << 17; pub const GA_LOG_EN: u64 = 1 << 28; + /// A remapping entry's 32-bit destination (§2.2.5.3); needs `GA_EN`. + pub const XT_EN: u64 = 1 << 50; pub const GA_INT_EN: u64 = 1 << 29; } @@ -54,6 +58,16 @@ impl Features { self.0 & 1 << 6 != 0 } + /// `XTSup`: a remapping entry can name an x2APIC destination. + pub const fn x2apic(self) -> bool { + self.0 & 1 << 2 != 0 + } + + /// `GASup`: the 128-bit remapping entry can be selected. + pub const fn guest_apic(self) -> bool { + self.0 & 1 << 7 != 0 + } + /// `HATS`, the host table levels the unit walks: 4, 5 or 6, or `None` /// for the reserved encoding. pub const fn host_levels(self) -> Option { diff --git a/kernel/amdvi/tests/encodings.rs b/kernel/amdvi/tests/encodings.rs index 6641a07fbc..be7808bac2 100644 --- a/kernel/amdvi/tests/encodings.rs +++ b/kernel/amdvi/tests/encodings.rs @@ -3,9 +3,10 @@ //! specification's figures and tables give them, and compared whole, so a //! field in the wrong place and a stray bit are both a difference. -use toyos_amdvi::dte::{Dte, Setting, DEVICE_IDS, ENTRY_BYTES}; +use toyos_amdvi::dte::{interrupts_blocked, interrupts_remapped, Dte, Setting, DEVICE_IDS, ENTRY_BYTES, INTERRUPT_ENTRIES}; +use toyos_amdvi::irte::{self, Destination}; use toyos_amdvi::queue::{event, name, Command, Event, ENTRIES}; -use toyos_amdvi::reg::{device_table, offset, ring, slot, Features}; +use toyos_amdvi::reg::{control, device_table, offset, ring, slot, Features}; use toyos_amdvi::table::{directory, entry, index, leaf, Entry, LEAF_LEVEL, ROOT_LEVEL}; use toyos_phys::Phys; @@ -181,3 +182,81 @@ fn the_feature_register_says_invalidate_all_and_host_levels() { assert_eq!(Features(0).host_levels(), Some(4)); assert_eq!(Features(word(&[(11, 10, 0b11)])).host_levels(), None); } + +/// Table 7, the interrupt half as bits 191:128 numbered from 128: `IV` [128], +/// `IntCtl` [189:188] 00b, and Table 106's pass bits where Table 7 puts them +/// (`INITPass` 184, `EIntPass` 185, `NMIPass` 186, `Lint0Pass` 190, +/// `Lint1Pass` 191). `SysMgt` and `ATCPermit` reach no interrupt half. +#[test] +fn a_blocked_interrupt_half_aborts_and_carries_firmwares_passes() { + let half = |fields: &[(u32, u32, u64)]| word(&fields.iter().map(|&(m, l, v)| (m - 128, l - 128, v)).collect::>()); + assert_eq!(interrupts_blocked(Setting(0)), half(&[(128, 128, 1)])); + let ioapic = Setting(0xD7); + assert_eq!( + interrupts_blocked(ioapic), + half(&[(128, 128, 1), (184, 184, 1), (185, 185, 1), (186, 186, 1), (190, 190, 1), (191, 191, 1)]) + ); +} + +/// Table 7: the table root at [179:134], 128-byte aligned, `IntTabLen` +/// [132:129] 1000b for 256 entries, `IntCtl` 10b; a claim's setting clears +/// every pass bit and `SysMgt` (Table 10, Table 7 bits 105:104). +#[test] +fn a_remapped_interrupt_half_names_its_table_and_a_claim_passes_nothing() { + const TABLE: u64 = 0x7_6543_2000; + let half = |fields: &[(u32, u32, u64)]| word(&fields.iter().map(|&(m, l, v)| (m - 128, l - 128, v)).collect::>()); + let table = Phys::new(TABLE).expect("aligned"); + assert_eq!(INTERRUPT_ENTRIES * 16, 4096); + assert_eq!( + interrupts_remapped(table, Setting::CLAIMED), + half(&[(128, 128, 1), (132, 129, 0b1000), (179, 134, TABLE >> 6), (189, 188, 0b10)]) + ); + assert_eq!( + interrupts_remapped(table, Setting(0xFF)), + half(&[ + (128, 128, 1), + (132, 129, 0b1000), + (179, 134, TABLE >> 6), + (184, 184, 1), + (185, 185, 1), + (186, 186, 1), + (189, 188, 0b10), + (190, 190, 1), + (191, 191, 1) + ]) + ); + let translated = Dte::translated(Phys::new(ROOT).expect("aligned"), 9, Setting::CLAIMED).words(); + assert_eq!(translated[1], word(&[(15, 0, 9)])); +} + +/// Figure 17 / Table 22 and Figure 19, `GuestMode` 0: `RemapEn` [0], `IntType` +/// [4:2] 000b, `DM` [6] 0, the vector at [71:64]; the destination at [15:8] +/// alone without `XTEn`, and at [31:8] and [127:120] with it. +#[test] +fn a_remapping_entry_delivers_its_vector_to_its_destination() { + let narrow = layout::<2>(&[(0, 0, 1), (15, 8, 0x07), (71, 64, 0x41)]); + assert_eq!(irte::entry(0x41, Destination::Narrow(7)), narrow); + let wide = layout::<2>(&[(0, 0, 1), (31, 8, 0x34_5678), (71, 64, 0xFB), (127, 120, 0x12)]); + let words = irte::entry(0xFB, Destination::Extended(0x1234_5678)); + assert_eq!(words, wide); + assert!(irte::present(words)); + assert_eq!((irte::vector(words), irte::destination(words)), (0xFB, 0x1234_5678)); + assert!(!irte::present(irte::ABSENT)); +} + +/// Figure 48: `INVALIDATE_INTERRUPT_TABLE`, code 05h, the id at [15:0]. +#[test] +fn invalidating_an_interrupt_table_names_its_requester() { + assert_eq!(Command::InvalidateInterrupts(0x00A2).words(), [word(&[(15, 0, 0x00A2), (63, 60, 5)]), 0]); +} + +/// MMIO 0018h `GAEn` [17] and `XTEn` [50]; MMIO 0030h `XTSup` [2], `GASup` +/// [7]. The AMD bench's unit reads 0x206d73ef22254ade. +#[test] +fn the_remapping_fields_sit_where_48882_puts_them() { + assert_eq!((control::GA_EN, control::XT_EN), (word(&[(17, 17, 1)]), word(&[(50, 50, 1)]))); + let bench = Features(0x206d_73ef_2225_4ade); + assert!(bench.guest_apic() && bench.x2apic()); + assert!(!Features(!word(&[(7, 7, 1)])).guest_apic()); + assert!(!Features(!word(&[(2, 2, 1)])).x2apic()); +} From b85bbb0ca78b84cc09823d87c1d9355ea5cf7e09 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 03:22:04 +0200 Subject: [PATCH 2/9] AMD-Vi remaps interrupts: a table per requester indexed by vector, 128-bit entries, claims through their own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 6 of issues/an-amd-vi-machine-isolates-no-device.md, the second PR of the design accepted on #909 (AMD 48882 rev 3.11). - Where every programmed unit has GASup (and XTSup where the CPUs run in x2APIC) and the IVRS names every I/O APIC the MADT has, the kernel remaps: CONTROL gains GAEn (and XTEn on x2APIC), every device table entry's interrupt half is IV with IntCtl 00b (each requester's fixed messages aborted) before IommuEn, and a requester's first entry makes it a 256-entry table of its own (IntTabLen 1000b), named by one 64-bit write and an INVALIDATE_DEVTAB_ENTRY (§2.2.2.2). Otherwise every interrupt half stays invalid and messages pass as written, and the remapping decision says which requirement is missing. - The table is indexed by MSI data bits 10:0 (Figure 14), so with fixed delivery the entry is the message's own vector: a function keeps the compatibility message it would carry with no unit, and reaches only the vectors its own table holds present. Source verification is the table's. - Kernel drivers' messages, claims' and I/O APIC pins' each write the entry at their vector; a pin keeps its redirection entry's ordinary format, so a level pin's EOI names the vector the CPU took. A claim takes its vector from #918's allocator; its release writes the entry absent and invalidates before the vector is given back. - A claim marks its function claimed: from then on its entry passes nothing firmware declared (no NMI, INIT, ExtInt, LINT) and gives it no SysMgt, whichever domain it names (stage 5 review B2). - A claim on a function no programmed unit serves, or one an IVMD names, is refused at the claim (`Refusal::NotIsolated`), before a domain or a vector is spent (stage 5 review S2). - `iommu_unit::interrupt` dispatches to the backend that remaps; the message, pin and refusal types it shares move there from VT-d, whose module keeps only its entries. - Harness: `Unit::AmdVi { intremap }` (`intremap=on,xtsup=on` and the split irqchip), `Profile::HeadlessUsbSpareAmdVi`, and `usbd_drives_the_spare_amdvi`, which also reads the claim's entry present at its vector. The AMD bench's row `amdvi_remaps_the_stick_controller` reads GAEn, every I/O APIC named, and each kernel-driven xHCI armed through its own entry. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- .../an-amd-vi-machine-isolates-no-device.md | 23 ++- kernel/src/arch/x86_64/amdvi/domain.rs | 16 +- kernel/src/arch/x86_64/amdvi/interrupt.rs | 188 ++++++++++++++++++ kernel/src/arch/x86_64/amdvi/mod.rs | 69 ++++++- kernel/src/arch/x86_64/ioapic.rs | 4 +- kernel/src/arch/x86_64/iommu_unit.rs | 133 ++++++++++++- kernel/src/arch/x86_64/vtd/interrupt.rs | 96 ++------- kernel/src/iommu/mod.rs | 10 + kernel/src/pcidev/mod.rs | 6 + tests/common/qemu.rs | 32 ++- tests/toyos.rs | 80 +++++++- 11 files changed, 530 insertions(+), 127 deletions(-) create mode 100644 kernel/src/arch/x86_64/amdvi/interrupt.rs diff --git a/issues/an-amd-vi-machine-isolates-no-device.md b/issues/an-amd-vi-machine-isolates-no-device.md index 58c8657787..36a8c713ad 100644 --- a/issues/an-amd-vi-machine-isolates-no-device.md +++ b/issues/an-amd-vi-machine-isolates-no-device.md @@ -10,18 +10,16 @@ Held by the orchestrator's AMD laptop track: its stage 5, amdvi-translate, build translation, and its stage 6, amdvi-intremap, interrupt remapping. On a machine whose IVRS describes AMD-Vi units, `kernel/src/arch/x86_64/amdvi/` -translates (stage 5): every requester id's entry blocks unless it names a -domain, enumerated functions sit on an identity domain until a kernel driver -moves one to its own, and the event log feeds `iommu::fault`. No interrupt -is remapped: every entry's interrupt half is invalid, so every -message-signalled interrupt is delivered as written, and a function a -process would drive is refused its claim (`iommu::remapping` is -`NotRemapped`), so no userland driver runs there. +translates (stage 5) and remaps (stage 6): every requester id's entry blocks +unless it names a domain, and where every programmed unit has `GASup` (and +`XTSup` on an x2APIC machine) and names every I/O APIC, every requester's +fixed and arbitrated messages go through a table of its own indexed by +vector, a claim's entry passing nothing firmware declared. -What remains is interrupt remapping (stage 6): a remapping table per -requester indexed by vector, the I/O APICs' requester ids from the IVRS's -special entries, `claim_msi`, a claim clearing its entry's pass bits and -`SysMgt`, and a claim of a function an IVMD names refused. +What remains is a claim on the AMD bench: both its xHCI controllers keep +their MSI-X table in BAR 0, their only BAR, so `pcidev` refuses to hand one +over (`issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md`), +and `xhci-leave=` names a controller by an id the two share. Constraints a reader would otherwise pay to re-derive: @@ -32,6 +30,9 @@ Constraints a reader would otherwise pay to re-derive: where the specification blocks it, so the blocked default is unobservable there; and its `IO_PAGE_FAULT` record carries no address (`issues/qemus-amd-iommu-page-fault-record-carries-no-address.md`). +- With `intremap=on` its IVRS names the I/O APIC (special entry, variety 1, + handle 0, as `00:14.0`), and with `xtsup=on` its EFR sets `XTSup`; this + suite's guests run in x2APIC, so a remapping guest needs both. - The AMD bench is in the metal suite; the orchestrator runs every boot. **Exit:** a guest boot on `Profile::MetalAmdVi` and a run on the AMD bench diff --git a/kernel/src/arch/x86_64/amdvi/domain.rs b/kernel/src/arch/x86_64/amdvi/domain.rs index 1012393b43..ad37aa9ce6 100644 --- a/kernel/src/arch/x86_64/amdvi/domain.rs +++ b/kernel/src/arch/x86_64/amdvi/domain.rs @@ -42,7 +42,8 @@ pub struct Space { struct Amd { domains: Domains, /// Firmware's DTE setting for each requester id, which an entry keeps - /// whichever domain it names. + /// whichever domain it names; [`Setting::CLAIMED`] from a function's first + /// claim on, for the boot. settings: Vec, /// The requesters an IVMD names. kept: Vec, @@ -61,6 +62,19 @@ pub(super) fn start(reserved: Vec<(u64, u64)>, settings: Vec, kept: Vec *DOMAINS.lock() = Some(Amd { domains, settings, kept, translated }); } +/// What requester `id`'s entry carries of firmware's declaration. +pub(super) fn setting(id: u16) -> Setting { + DOMAINS.lock().as_ref().expect("amdvi: an entry is written only once a unit translates").settings[usize::from(id)] +} + +/// Requester `id` is driven by a process: from now on its entry passes +/// nothing firmware declared and gives no `SysMgt`, whichever domain it names. +pub(super) fn claimed(id: u16) { + let mut held = DOMAINS.lock(); + let amd = held.as_mut().expect("amdvi: a claim exists only once a unit translates"); + amd.settings[usize::from(id)] = Setting::CLAIMED; +} + /// Why `stream` keeps the machine's own addresses rather than a domain of its /// own, if it does: no programmed unit serves it, so nothing would translate /// one; or an IVMD names it, so firmware reserves memory it must keep diff --git a/kernel/src/arch/x86_64/amdvi/interrupt.rs b/kernel/src/arch/x86_64/amdvi/interrupt.rs new file mode 100644 index 0000000000..78a5f33cdb --- /dev/null +++ b/kernel/src/arch/x86_64/amdvi/interrupt.rs @@ -0,0 +1,188 @@ +//! Interrupt remapping (AMD 48882 rev 3.11 §2.2.5): a table of 256 entries +//! per requester, indexed by vector, which the unit picks by that requester's +//! own device table entry. +//! +//! A fixed message's data bits 10:0 index its requester's table (Figure 14), +//! so a function is armed with the message it would carry with no unit at all, +//! and the entry at its vector says where that vector goes. Source +//! verification is the table's own: a requester reaches only the vectors its +//! table holds present, and a message on any other is target-aborted and +//! recorded (`RemapEn` 0, §2.2.5.2 step 7). +//! +//! Every entry is the 128-bit format, `CONTROL.GAEn` with `GAMEn` 000b and +//! `GuestMode` 0 (Table 21, Figure 17); its destination is 32 bits wide under +//! `XTEn` (§2.2.5.3, Figure 19), which an x2APIC machine needs. +//! +//! This module's lock is taken before `DOMAINS`, `UNITS` and `TABLES`, never +//! after any of them. + +use alloc::collections::BTreeMap; +use alloc::vec::Vec; + +use toyos_amdvi::dte::{interrupts_remapped, ENTRY_BYTES}; +use toyos_amdvi::irte::{self, Destination}; +use toyos_amdvi::queue::Command; +use toyos_phys::Phys as Address; + +use super::super::apic::{msi_message, narrow_destination, MSG_DEST, MSG_DEST_NARROW}; +use super::super::iommu_unit::interrupt::{Msi, Pin, Refused}; +use super::{domain, memory, submit, DEVICE_TABLE, TABLES}; +use crate::iommu::StreamId; +use crate::log; +use crate::sync::Lock; +use core::sync::atomic::Ordering; + +struct Remap { + /// Whether every unit remaps, decided once at boot: `None` leaves every + /// entry's interrupt half invalid and every message as written. + extended: Option, + /// Each requester's table, made on its first entry and kept. + tables: BTreeMap>, + /// The requester id firmware's IVRS gives each I/O APIC (§5.2.2, Table 109). + apics: Vec<(u8, u16)>, +} + +static REMAP: Lock = Lock::new(Remap { extended: None, tables: BTreeMap::new(), apics: Vec::new() }); + +/// Every unit remaps, with an entry's destination 32 bits wide where +/// `extended`; the I/O APICs speak as `apics`. +pub(super) fn start(extended: bool, apics: Vec<(u8, u16)>) { + let mut remap = REMAP.lock(); + remap.extended = Some(extended); + remap.apics = apics; +} + +pub fn is_armed() -> bool { + REMAP.lock().extended.is_some() +} + +/// The interrupt half of requester `id`'s entry, bits 191:128. +fn interrupt_half(id: u16) -> crate::mm::Mmio { + let table = DEVICE_TABLE.load(Ordering::Acquire); + memory(table + u64::from(id) * ENTRY_BYTES as u64 + 16, 8) +} + +impl Remap { + /// Entry `vector` of `id`'s table delivering it to `destination`, or + /// absent; the table made and named by `id`'s entry first where it has + /// none. Published to every unit before this returns. + fn write(&mut self, id: u16, vector: u8, entry: [u64; 2]) -> [u64; 2] { + let table = match self.tables.get(&id) { + Some(table) => *table, + None => { + let table = TABLES.lock().alloc(); + self.tables.insert(id, table); + // One 64-bit write of a half whose `IV` is already set, then + // the entry invalidated (§2.2.2.2). + interrupt_half(id).write_u64(0, interrupts_remapped(table, domain::setting(id))); + submit(&[Command::InvalidateDevice(id)]); + table + } + }; + let slot = memory(table.get() + u64::from(vector) * 16, 16); + // `RemapEn` is in the low doubleword: it goes last when an entry + // becomes present, and first when it stops being. + if irte::present(entry) { + slot.write_u64(8, entry[1]); + slot.write_u64(0, entry[0]); + } else { + slot.write_u64(0, entry[0]); + slot.write_u64(8, entry[1]); + } + submit(&[Command::InvalidateInterrupts(id)]); + [slot.read_u64(0), slot.read_u64(8)] + } + + fn destination(&self, apic: u32) -> Result { + match self.extended { + Some(true) => Ok(Destination::Extended(apic)), + Some(false) => narrow_destination(apic).map(Destination::Narrow).map_err(|_| Refused::DestinationTooWide(apic)), + None => unreachable!("amdvi: an entry written on a machine that does not remap"), + } + } +} + +/// What an entry holds, read back out of the table. +fn report(source: StreamId, vector: u8, read: [u64; 2]) { + log!( + "iommu: amdvi irte source={source} vector={vector:#04x} p={} dst={:#x} delivers={:#04x}", + u8::from(irte::present(read)), + irte::destination(read), + irte::vector(read), + ); +} + +/// `source`'s entry at `vector`, delivering it to this kernel's CPU. +fn arm(source: StreamId, vector: u8) -> Result<(), Refused> { + let mut remap = REMAP.lock(); + let destination = match remap.extended { + Some(true) => Destination::Extended(MSG_DEST), + _ => Destination::Narrow(MSG_DEST_NARROW), + }; + let read = remap.write(source.requester(), vector, irte::entry(vector, destination)); + drop(remap); + report(source, vector, read); + Ok(()) +} + +/// The message a kernel driver's `source` raises `vector` with: its own +/// compatibility message, an entry for it written first where every unit +/// remaps. +pub(in crate::arch::x86_64) fn msi(source: StreamId, vector: u8) -> Result { + if is_armed() { + arm(source, vector)?; + } + let (address, data) = msi_message(vector); + Ok(Msi { address, data }) +} + +/// A claim's entry at `vector` for `source`, whose entry from now on passes +/// nothing firmware declared for it and gives it no `SysMgt`. +pub(in crate::arch::x86_64) fn claim(vector: u8, source: StreamId) -> Result { + domain::claimed(source.requester()); + msi(source, vector) +} + +/// A claim's entry absent again and gone from every unit's cache. +pub(in crate::arch::x86_64) fn release(vector: u8, source: StreamId) { + let read = REMAP.lock().write(source.requester(), vector, irte::ABSENT); + log!("iommu: amdvi irte source={source} vector={vector:#04x} p={} released", u8::from(irte::present(read))); +} + +/// An I/O APIC pin's entry at its own vector, so its redirection entry stays +/// in the format it has with no unit: a level-triggered pin's EOI then names +/// the vector the CPU took. +pub(in crate::arch::x86_64) fn pin(apic_id: u8, vector: u8, dest: u32) -> Result { + let mut remap = REMAP.lock(); + let id = remap.apics.iter().find(|(apic, _)| *apic == apic_id).map(|(_, id)| *id); + let id = id.ok_or(Refused::ControllerUnnamed(apic_id))?; + let entry = irte::entry(vector, remap.destination(dest)?); + let read = remap.write(id, vector, entry); + drop(remap); + let [devfn, bus] = id.to_le_bytes(); + report(StreamId::pci(bus, devfn >> 3, devfn & 7), vector, read); + // The entry's destination is what the unit delivers to; the redirection + // entry's is never read. + Ok(Pin { low: 0, high: 0 }) +} + +/// The rule [`super::init`] decides remapping by, over what the units and +/// firmware said: `None`, said, where any requirement is missing. +pub(super) fn remappable(guest_apic: bool, x2apic_units: bool, x2apic_cpus: bool, named: &[(u8, u16)], apics: &[u8]) -> Option { + if !guest_apic { + log!("iommu: an AMD-Vi unit has GASup clear, so no message is remapped on this machine"); + return None; + } + if x2apic_cpus && !x2apic_units { + log!("iommu: the CPUs are in x2APIC and an AMD-Vi unit has XTSup clear, so no message is remapped"); + return None; + } + if let Some(apic) = apics.iter().find(|apic| !named.iter().any(|(named, _)| named == *apic)) { + log!( + "iommu: no programmed AMD-Vi unit names I/O APIC {apic} in the IVRS, so its pins could not be remapped \ + and no message is" + ); + return None; + } + Some(x2apic_cpus) +} diff --git a/kernel/src/arch/x86_64/amdvi/mod.rs b/kernel/src/arch/x86_64/amdvi/mod.rs index 5302d931ee..eeb5db77aa 100644 --- a/kernel/src/arch/x86_64/amdvi/mod.rs +++ b/kernel/src/arch/x86_64/amdvi/mod.rs @@ -8,12 +8,14 @@ //! blocked unless it names a domain, every enumerated function on the //! identity domain until a driver moves it to one of its own ([`domain`]), //! and the event log feeding `iommu::fault` ([`fault`]). A unit it cannot use -//! is logged by register value and left off. No interrupt is remapped: -//! every entry's interrupt half is invalid, so the unit forwards messages -//! as written (Table 9). +//! is logged by register value and left off. Where every unit programmed +//! can remap, every message is remapped through its requester's own table +//! ([`interrupt`]); where one cannot, every entry's interrupt half stays +//! invalid and messages pass as written (Table 9). pub mod domain; pub mod fault; +pub mod interrupt; #[cfg(feature = "boot-actuators")] pub(super) mod selftest; @@ -30,7 +32,7 @@ use crate::mm::{DirectMap, Mmio, PAGE_2M, PAGE_SIZE}; use crate::sync::Lock; use crate::time::{Duration, Tripwire}; use toyos_abi::boot::RootBridgeWindow; -use toyos_amdvi::dte::{Dte, Setting, DEVICE_IDS, ENTRY_BYTES}; +use toyos_amdvi::dte::{interrupts_blocked, Dte, Setting, DEVICE_IDS, ENTRY_BYTES}; use toyos_amdvi::queue::{Command, ENTRIES}; use toyos_amdvi::reg::{self, control, status, Features}; use toyos_phys::Phys as Address; @@ -129,6 +131,7 @@ struct Ready { base: u64, regs: Mmio, function: PciDevice, + features: Features, } /// Every unit the IVRS describes, switched off, and each this kernel can use @@ -163,6 +166,8 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) // The ids a unit that will be programmed serves: a function outside them // is behind a unit left off, and no domain of its own would translate. let mut translated: Vec<(u16, u16)> = Vec::new(); + // The I/O APICs a unit that will be programmed names, by MADT id. + let mut apics: Vec<(u8, u16)> = Vec::new(); let mut units = 0usize; for block in ivrs.blocks() { match block { @@ -175,10 +180,15 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) let function = describe_function(units, &unit, devices); let mut usable = Ok(()); let mut serves: Vec<(u16, u16)> = Vec::new(); + let mut specials: Vec<(u8, u16)> = Vec::new(); for entry in ivrs.devices(&unit) { match entry { Ok(entry) => { log!("iommu: amdvi unit{units} serves {}", served(&ivrs, &entry)); + // Variety 01h: the handle is an I/O APIC's MADT id (Table 109). + if let DeviceEntry::Special { handle, used, variety: 1, .. } = entry { + specials.push((handle, used)); + } match declare(&entry, &mut settings, enumerated) { Ok(ids) => serves.extend(ids), Err(why) => usable = Err(why), @@ -196,7 +206,9 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) match usable.and_then(|()| usable_unit(&unit, regs, function)) { Ok(function) => { translated.extend(serves); - ready.push(Ready { index: units, base: unit.base, regs, function }); + apics.extend(specials); + let features = Features(regs.read_u64(reg::EXTENDED_FEATURES)); + ready.push(Ready { index: units, base: unit.base, regs, function, features }); } Err(why) => log!("iommu: amdvi unit{units} is left off: {why}"), } @@ -258,12 +270,31 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) if end <= top { "inside" } else { "OUTSIDE" } ); } - let identity = build(devices, &settings); + let x2apic = super::control_regs::apic_mode() == super::control_regs::ApicMode::X2apic; + let remap = interrupt::remappable( + ready.iter().all(|unit| unit.features.guest_apic()), + ready.iter().all(|unit| unit.features.x2apic()), + x2apic, + &apics, + &super::ioapic::ids(), + ); + #[cfg(feature = "boot-actuators")] + let remap = if crate::actuator::iommu_no_remap() { None } else { remap }; + let identity = build(devices, &settings, remap.is_some()); let reserved = windows.iter().map(|w| (w.base, w.end())).chain(ivmds.iter().map(|m| (m.start, m.start + m.length))); domain::start(reserved.collect(), settings, ivmds.iter().map(|m| m.requesters).collect(), translated); log!("iommu: amdvi identity domain covers 0x0..{top:#x}, root={:#x}", identity.get()); + if let Some(extended) = remap { + log!( + "iommu: amdvi remaps every message through its requester's table: 128-bit entries, x2apic \ + destinations={}, {} I/O APIC(s) named", + yn(extended), + apics.len() + ); + interrupt::start(extended, apics); + } for unit in ready { - program(unit, devices); + program(unit, devices, remap); } true } @@ -330,7 +361,7 @@ fn usable_unit(unit: &Unit, regs: Mmio, function: Option) -> Result

Address<12> { +fn build(devices: &[PciDevice], settings: &[Setting], remap: bool) -> Address<12> { let mut tables = TABLES.lock(); let page = pmm::alloc_page().expect("amdvi: no physical memory for the device table"); let at = page.direct_map().phys(); @@ -342,6 +373,12 @@ fn build(devices: &[PciDevice], settings: &[Setting]) -> Address<12> { let slot = entry(id); slot.write_u64(8, words[1]); slot.write_u64(0, words[0]); + // Where every unit remaps, every requester's fixed and arbitrated + // messages are aborted until an entry of its own is written. + if remap { + memory(DEVICE_TABLE.load(Ordering::Acquire) + u64::from(id) * ENTRY_BYTES as u64 + 16, 8) + .write_u64(0, interrupts_blocked(*setting)); + } } let root = tables.alloc(); let mut phys = 0; @@ -418,8 +455,8 @@ fn submit(commands: &[Command]) { /// interrupt armed and the exclusion range cleared before the unit is on; /// then empties every cache it holds and names each enumerated /// function's entry. -fn program(unit: Ready, devices: &[PciDevice]) { - let Ready { index, base, regs, function } = unit; +fn program(unit: Ready, devices: &[PciDevice], remap: Option) { + let Ready { index, base, regs, function, .. } = unit; let (commands, events, done) = { let mut tables = TABLES.lock(); (tables.alloc(), tables.alloc(), tables.alloc()) @@ -441,7 +478,17 @@ fn program(unit: Ready, devices: &[PciDevice]) { // Written whole: a field firmware left, `DevTblSegEn` among them, would // otherwise change how the unit reads this table. `switch_off` logged // what it was handed over with. - let on = control::COHERENT | control::EVENT_LOG_EN | control::EVENT_INT_EN | control::CMD_BUF_EN | control::IOMMU_EN; + let remapping = match remap { + None => 0, + Some(false) => control::GA_EN, + Some(true) => control::GA_EN | control::XT_EN, + }; + let on = control::COHERENT + | control::EVENT_LOG_EN + | control::EVENT_INT_EN + | control::CMD_BUF_EN + | control::IOMMU_EN + | remapping; regs.write_u64(reg::CONTROL, on); let running = status::CMD_BUF_RUN | status::EVENT_LOG_RUN; spin_until( diff --git a/kernel/src/arch/x86_64/ioapic.rs b/kernel/src/arch/x86_64/ioapic.rs index b5ccc497b6..8c28819e5a 100644 --- a/kernel/src/arch/x86_64/ioapic.rs +++ b/kernel/src/arch/x86_64/ioapic.rs @@ -257,12 +257,12 @@ pub fn route( // Under no unit that remaps, the entry names its CPU itself; under one, a // pin with no entry of its own is refused rather than written in a form // the unit blocks. - let (index, high) = if !super::vtd::interrupt::is_armed() { + let (index, high) = if !super::iommu_unit::interrupt::is_armed() { let dest = super::apic::narrow_destination(dest_apic_id) .map_err(|_| RouteError::DestTooWide(dest_apic_id))?; (0, u32::from(dest) << 24) } else { - match super::vtd::interrupt::pin(unit.id, vector, dest_apic_id, level) { + match super::iommu_unit::interrupt::pin(unit.id, vector, dest_apic_id, level) { Ok(pin) => (pin.low, pin.high), Err(why) => return Err(RouteError::NotRemappable(gsi, why)), } diff --git a/kernel/src/arch/x86_64/iommu_unit.rs b/kernel/src/arch/x86_64/iommu_unit.rs index 951afe6f5f..9d391124b3 100644 --- a/kernel/src/arch/x86_64/iommu_unit.rs +++ b/kernel/src/arch/x86_64/iommu_unit.rs @@ -1,7 +1,6 @@ //! The x86-64 IOMMU: VT-d where firmware publishes a DMAR, AMD-Vi where it -//! publishes an IVRS, chosen once at [`init`] and asked through [`domain`] -//! and [`fault`]. Only VT-d remaps an interrupt -//! (`issues/an-amd-vi-machine-isolates-no-device.md`, stage 6). +//! publishes an IVRS, chosen once at [`init`] and asked through [`domain`], +//! [`fault`] and [`interrupt`]. use core::sync::atomic::{AtomicU8, Ordering}; @@ -11,8 +10,6 @@ use crate::mm::policy::MmioPolicy; use crate::mm::Mmio; use toyos_abi::boot::RootBridgeWindow; -pub use super::vtd::interrupt; - /// Which backend [`init`] found: none until it has, and then fixed. #[derive(Clone, Copy, PartialEq, Eq)] #[repr(u8)] @@ -204,3 +201,129 @@ pub mod fault { } } } + +/// Interrupt remapping, on whichever backend remaps: what a message is armed +/// with, and the claim's vector it delivers. +pub mod interrupt { + use super::{backend, Backend}; + use crate::arch::x86_64::idt::{user_dev, ClaimVector, DriverIrq}; + use crate::arch::x86_64::{amdvi, vtd}; + use crate::iommu::StreamId; + + pub struct Msi { + pub address: u32, + pub data: u32, + } + + /// The words an I/O APIC redirection entry carries for a remapped pin. + pub struct Pin { + pub low: u32, + pub high: u32, + } + + /// Why a source could not be given an entry. Carried rather than + /// collapsed: one message for all of them sends whoever reads it looking + /// in the wrong place. + #[derive(Clone, Copy)] + pub enum Refused { + /// Wider than the destination an entry holds without extended + /// interrupt mode. + DestinationTooWide(u32), + /// Every entry in the table is already spoken for. + TableFull, + /// Every vector a claim's message may carry is held by another claim. + NoVector, + /// Firmware named no requester id for this interrupt controller. + ControllerUnnamed(u8), + } + + impl Refused { + /// Whether this machine has run out of something a claim spends, + /// rather than this source being one no entry can be written for. + pub fn exhausted(self) -> bool { + matches!(self, Self::TableFull | Self::NoVector) + } + } + + impl core::fmt::Display for Refused { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::DestinationTooWide(id) => { + write!(f, "apic id {id:#x} does not fit a remapping entry's destination") + } + Self::TableFull => write!(f, "the interrupt remapping table is full"), + Self::NoVector => write!(f, "every vector a claimed function's message may carry is held"), + Self::ControllerUnnamed(id) => { + write!(f, "firmware named no requester id for interrupt controller {id}") + } + } + } + } + + /// A claim's message and the vector it delivers, held until [`release`]. + pub struct Claimed { + pub msi: Msi, + vector: ClaimVector, + /// VT-d's entry index; AMD-Vi's entry is the vector's own. + entry: Option, + } + + impl Claimed { + /// What the message raises, for the record. + pub fn interrupt(&self) -> impl core::fmt::Display { + self.vector + } + } + + pub fn is_armed() -> bool { + match backend() { + Backend::Vtd => vtd::interrupt::is_armed(), + Backend::AmdVi => amdvi::interrupt::is_armed(), + Backend::None => false, + } + } + + /// The message a kernel driver's `source` raises `irq` with. + pub fn msi(source: StreamId, irq: DriverIrq) -> Result { + match backend() { + Backend::AmdVi => amdvi::interrupt::msi(source, irq.vector()), + Backend::Vtd | Backend::None => vtd::interrupt::msi(source, irq), + } + } + + /// A vector for `pcidev` function `function` and an entry delivering it, + /// written for `source` and published to every unit before this returns. + pub fn claim(function: usize, source: StreamId) -> Result { + let vector = user_dev::take(function).ok_or(Refused::NoVector)?; + let made = match backend() { + Backend::AmdVi => amdvi::interrupt::claim(vector.vector(), source).map(|msi| (msi, None)), + Backend::Vtd => vtd::interrupt::claim(vector.vector(), source) + .map(|entry| (vtd::interrupt::message(entry), Some(entry))), + Backend::None => unreachable!("iommu: a claim's message made where nothing remaps"), + }; + match made { + Ok((msi, entry)) => Ok(Claimed { msi, vector, entry }), + Err(why) => { + user_dev::give_back(vector); + Err(why) + } + } + } + + /// A claim's entry gone from every unit, and then its vector free. + pub fn release(claimed: &Claimed, source: StreamId) { + match claimed.entry { + Some(entry) => vtd::interrupt::release(entry, source), + None => amdvi::interrupt::release(claimed.vector.vector(), source), + } + user_dev::give_back(claimed.vector); + } + + /// An I/O APIC pin's redirection words, its entry written first. + pub fn pin(apic_id: u8, vector: u8, dest: u32, level: bool) -> Result { + match backend() { + Backend::AmdVi => amdvi::interrupt::pin(apic_id, vector, dest), + Backend::Vtd | Backend::None => vtd::interrupt::pin(apic_id, vector, dest, level), + } + } +} diff --git a/kernel/src/arch/x86_64/vtd/interrupt.rs b/kernel/src/arch/x86_64/vtd/interrupt.rs index 784f429f61..cfc2f03b28 100644 --- a/kernel/src/arch/x86_64/vtd/interrupt.rs +++ b/kernel/src/arch/x86_64/vtd/interrupt.rs @@ -38,7 +38,7 @@ use alloc::vec::Vec; use kernel::owners::Owners; use super::super::apic::{narrow_destination, MSG_DEST}; -use super::super::idt::{self, ClaimVector, DriverIrq}; +use super::super::idt::DriverIrq; use crate::iommu::StreamId; use crate::sync::Lock; @@ -70,52 +70,7 @@ const MESSAGE_REMAPPABLE: u32 = 1 << 4; const MESSAGE_SUBHANDLE_VALID: u32 = 1 << 3; const PIN_REMAPPABLE: u32 = 1 << 16; -pub struct Msi { - pub address: u32, - pub data: u32, -} - -/// Why a source could not be given an entry. Carried rather than collapsed: -/// one message for all three sends whoever reads it looking in the wrong place. -#[derive(Clone, Copy)] -pub enum Refused { - /// Wider than the destination an entry holds without extended interrupt mode. - DestinationTooWide(u32), - /// Every entry in the table is already spoken for. - TableFull, - /// Every vector a claim's message may carry is held by another claim. - NoVector, - /// Firmware's device scopes named no requester id for this interrupt controller. - ControllerUnnamed(u8), -} - -impl Refused { - /// Whether this machine has run out of something a claim spends, rather - /// than this source being one no entry can be written for. - pub fn exhausted(self) -> bool { - matches!(self, Self::TableFull | Self::NoVector) - } -} - -impl core::fmt::Display for Refused { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::DestinationTooWide(id) => { - write!(f, "apic id {id:#x} does not fit a remapping entry's destination") - } - Self::TableFull => write!(f, "the interrupt remapping table is full"), - Self::NoVector => write!(f, "every vector a claimed function's message may carry is held"), - Self::ControllerUnnamed(id) => { - write!(f, "firmware named no requester id for interrupt controller {id}") - } - } - } -} - -pub struct Pin { - pub low: u32, - pub high: u32, -} +use super::super::iommu_unit::interrupt::{Msi, Pin, Refused}; struct Remap { /// `None` until a unit is armed, which is what "no source may use the remappable format" means. @@ -246,7 +201,7 @@ impl Remap { } /// The message that indexes entry `index` by its handle alone (Section 5.1.3). -fn message(index: usize) -> Msi { +pub(in crate::arch::x86_64) fn message(index: usize) -> Msi { let index = index as u32; Msi { address: crate::arch::MSI_DOORBELL @@ -269,49 +224,26 @@ pub fn msi(source: StreamId, irq: DriverIrq) -> Result { Ok(message(allocate(source, irq.vector(), MSG_DEST, false)?)) } -/// A claimed function's entry and the vector it delivers, held until -/// [`release`]. -pub struct Claimed { - pub msi: Msi, - entry: usize, - vector: ClaimVector, -} - -impl Claimed { - /// What the message raises, for the record. - pub fn interrupt(&self) -> impl core::fmt::Display { - self.vector - } -} - -/// A vector for `pcidev` function `function` and an entry delivering it, written -/// for `source` and published to every unit before this returns. -pub fn claim(function: usize, source: StreamId) -> Result { - let vector = idt::user_dev::take(function).ok_or(Refused::NoVector)?; - match allocate(source, vector.vector(), MSG_DEST, false) { - Ok(entry) => Ok(Claimed { msi: message(entry), entry, vector }), - Err(why) => { - idt::user_dev::give_back(vector); - Err(why) - } - } +/// An entry delivering a claim's `vector`, written for `source` and published +/// to every unit before this returns; its index, held until [`release`]. +pub(in crate::arch::x86_64) fn claim(vector: u8, source: StreamId) -> Result { + allocate(source, vector, MSG_DEST, false) } /// A claim's entry not present again and gone from every unit's cache, and -/// then its entry and vector free. -pub fn release(claimed: &Claimed, source: StreamId) { +/// then free. +pub(in crate::arch::x86_64) fn release(entry: usize, source: StreamId) { let lo = { let remap = REMAP.lock(); let Some(table) = remap.table else { - panic!("iommu: irte{} released with no table armed", claimed.entry); + panic!("iommu: irte{entry} released with no table armed"); }; - table.clear_pair(claimed.entry); + table.clear_pair(entry); super::invalidate_interrupt_entries(); - remap.held.give_back(claimed.entry); - table.read_pair(claimed.entry).0 + remap.held.give_back(entry); + table.read_pair(entry).0 }; - idt::user_dev::give_back(claimed.vector); - log!("iommu: irte{} source={source} p={} released", claimed.entry, lo & PRESENT); + log!("iommu: irte{entry} source={source} p={} released", lo & PRESENT); } pub fn pin(apic_id: u8, vector: u8, dest: u32, level: bool) -> Result { diff --git a/kernel/src/iommu/mod.rs b/kernel/src/iommu/mod.rs index 0ac953fe92..d097afaea6 100644 --- a/kernel/src/iommu/mod.rs +++ b/kernel/src/iommu/mod.rs @@ -313,6 +313,16 @@ pub fn remapping() -> Result { } } +/// Whether `function` can be given a domain of its own and a remapping entry +/// no other requester reaches, or why not: a function no programmed unit +/// serves, or one firmware reserves memory for, keeps the machine's own. +pub fn isolates(function: &crate::drivers::pci::PciDevice) -> Result<(), &'static str> { + match unit::domain::untranslated(StreamId::pci(function.bus, function.dev, function.func)) { + None => Ok(()), + Some(why) => Err(why), + } +} + /// A claimed function's message, raising an interrupt of its own and /// translated for [`Self::function`] alone. /// diff --git a/kernel/src/pcidev/mod.rs b/kernel/src/pcidev/mod.rs index 5ebec01b18..c80921a899 100644 --- a/kernel/src/pcidev/mod.rs +++ b/kernel/src/pcidev/mod.rs @@ -647,6 +647,8 @@ fn account_for(firmware: &[RootBridgeWindow], decoded: &[(u16, u64, u64)]) { #[derive(Clone, Copy)] enum Refusal { NotRemapped, + /// This machine remaps and translates, and not for this function. + NotIsolated(&'static str), /// The machine translates messages, and none could be made for this function. NoMessage(Refused), NoInterrupt, @@ -686,6 +688,9 @@ impl core::fmt::Display for Refusal { "its interrupts would not be remapped on this machine, and a message that is not \ remapped can raise any vector on any CPU" ), + Self::NotIsolated(why) => { + write!(f, "no address space or message of its own can be made for it: {why}") + } Self::NoMessage(why) => write!(f, "no message of its own could be made for it — {why}"), Self::NoInterrupt => write!( f, @@ -856,6 +861,7 @@ pub fn claim(id: PciId) -> Result<(PciFunctionInfo, Claim), ClaimError> { /// whatever addresses its registers still held. fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result { let remapping = crate::iommu::remapping().map_err(|_| Refusal::NotRemapped)?; + crate::iommu::isolates(&pci).map_err(Refusal::NotIsolated)?; // What the function's previous holder left mapped goes before anything // attaches to its domain, and whatever is left of a reset [`release`] started on this diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 4c020f44dd..9afb5378ab 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -711,6 +711,9 @@ pub enum Profile { /// unit: the only machine on which a translating AMD-Vi unit's refusal /// can be made and read. MetalAmdViTestdev, + /// [`Profile::HeadlessUsbSpare`] behind QEMU's AMD unit remapping + /// interrupts: a claim's message through its own requester's table. + HeadlessUsbSpareAmdVi, /// QEMU `virt` on AArch64 (GICv3, AAVMF): a GOP from `ramfb`, the boot /// stick on an xHCI, the PL011, a virtio-rng for firmware's /// `EFI_RNG_PROTOCOL`, and nothing else — no NIC, NVMe or IOMMU. The @@ -773,7 +776,8 @@ impl Profile { | Self::Metal | Self::MetalNoX2apic | Self::MetalAmdVi - | Self::MetalAmdViTestdev => Arch::X86_64, + | Self::MetalAmdViTestdev + | Self::HeadlessUsbSpareAmdVi => Arch::X86_64, } } @@ -835,11 +839,13 @@ pub const IOMMU_DEFAULT: Iommu = Iommu { aw_bits: 48, intremap: true, eim: false pub enum Unit { /// `intel-iommu`, with the capabilities it is staged with. Vtd(Iommu), - /// `amd-iommu`, interrupt remapping off, which needs no split irqchip, - /// and `dma-remap` on: without it QEMU 11.1.1 passes every function's DMA - /// untranslated whatever the unit is told (`hw/i386/amd_iommu.c:1165` - /// and `:1310` at v11.1.1), and a unit left blocking would block nothing. - AmdVi, + /// `amd-iommu` with `dma-remap` on: without it QEMU 11.1.1 passes every + /// function's DMA untranslated whatever the unit is told + /// (`hw/i386/amd_iommu.c:1165` and `:1310` at v11.1.1), and a unit left + /// blocking would block nothing. With interrupt remapping, which needs + /// the split irqchip, `xtsup` too: this suite's guests run in x2APIC, + /// and a unit without it remaps nothing for them. + AmdVi { intremap: bool }, } /// The controller every profile but [`Profile::MetalUsb`] gets. `nec-usb-xhci` @@ -1044,7 +1050,10 @@ impl Profile { virtio_gpu: false, }, Self::MetalNoX2apic => Self::Metal.shape(), - Self::MetalAmdVi => Shape { iommu: Some(Unit::AmdVi), ..Self::Metal.shape() }, + Self::MetalAmdVi => Shape { iommu: Some(Unit::AmdVi { intremap: false }), ..Self::Metal.shape() }, + Self::HeadlessUsbSpareAmdVi => { + Shape { iommu: Some(Unit::AmdVi { intremap: true }), ..Self::HeadlessUsbSpare.shape() } + } Self::MetalAmdViTestdev => Shape { probes: Probes::Testdevs, ..Self::MetalAmdVi.shape() }, Self::HeadlessNoIommu => Shape { iommu: None, ..Self::Headless.shape() }, Self::HeadlessNoUsb => Shape { xhci: &[], usb: &[], ..Self::Headless.shape() }, @@ -2136,10 +2145,10 @@ fn qemu_command( // is the option that does it, and it leaves i8042/ps2-kbd/ps2-mouse alone. qemu.arg("-nodefaults"); - // AMD-Vi is staged with no interrupt remapping, and `virt_smmu`'s unit - // needs no message of a function's. + // `virt_smmu`'s unit needs no message of a function's. let remapped = match (&shape.iommu, shape.probes) { (Some(Unit::Vtd(unit)), _) => unit.intremap, + (Some(Unit::AmdVi { intremap }), _) => *intremap, (_, Probes::Its) => true, _ => false, }; @@ -2205,9 +2214,12 @@ fn qemu_command( if unit.eim { "on" } else { "off" } )); } - Some(Unit::AmdVi) => { + Some(Unit::AmdVi { intremap: false }) => { qemu.arg("-device").arg("amd-iommu,intremap=off,dma-remap=on"); } + Some(Unit::AmdVi { intremap: true }) => { + qemu.arg("-device").arg("amd-iommu,intremap=on,xtsup=on,dma-remap=on"); + } None => {} } // A virtio function reaches memory through `vdev->dma_as`, the machine's own diff --git a/tests/toyos.rs b/tests/toyos.rs index 09403d635f..473f53a6d6 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -399,6 +399,11 @@ const MACHINE_TESTS: &[&str] = &[ // to fault. What usbd names on the bus is `toyos-xhci`'s and its // simulator's to hold. "usbd_drives_the_spare", + // The same behind QEMU's AMD unit remapping interrupts: the claim's + // message through an entry of its function's own table, which no host + // build has and which the AMD bench's controllers, their MSI-X tables in + // their register BARs, cannot yet be claimed to show. + "usbd_drives_the_spare_amdvi", // A boot keyboard reporting more keys than its six slots name: the // kernel's driver reading a report a device delivered over xHCI has no host // build, and the T14 binds no USB keyboard. @@ -993,6 +998,13 @@ const METAL: &[(&str, metal::Metal)] = &[ "amdvi_translates_the_stick_controller", metal::Metal { arms: TESTCASES, judge: |b| amdvi_translates(&b[0].kernel()) }, ), + ( + // An AMD-Vi unit remapping every message: the boot stick's + // controller's interrupt delivered through an entry of its own + // table, on real silicon, and every I/O APIC named. + "amdvi_remaps_the_stick_controller", + metal::Metal { arms: TESTCASES, judge: |b| amdvi_remaps(&b[0].kernel()) }, + ), ( // A machine whose units do not remap: the claim is refused before // anything on the function changes, and the boot goes on. @@ -3657,7 +3669,8 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "desktop_without_x2apic" => desktop_without_x2apic(), "nvme_disk_boots" => nvme_disk_boots(test_config), "hosted_clang_hello" => common::hostedclang::hosted_clang_hello(), - "usbd_drives_the_spare" => usbd_drives_the_spare(test_config), + "usbd_drives_the_spare" => usbd_drives_the_spare(test_config, qemu::Profile::HeadlessUsbSpare), + "usbd_drives_the_spare_amdvi" => usbd_drives_the_spare(test_config, qemu::Profile::HeadlessUsbSpareAmdVi), "usb_keyboard_rollover" => usb_keyboard_rollover(test_config), "vector_state_survives_a_switch" => vector_state_survives_a_switch(test_config), "the_opening_exchange_with_qemus_agent_completes" => the_opening_exchange_with_qemus_agent_completes(), @@ -3738,14 +3751,14 @@ const USBD_NOOP: &str = "usbd: a No-Op command completed, announced by an interr /// claim is refused because the kernel drives the function. Between them, a /// holder of the function is killed and the claim asked for again at once, /// each round, which only a release finished before the end grants. -fn usbd_drives_the_spare(test_config: &Path) -> Result<(), String> { +fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<(), String> { const JOB: &str = "usbd_spare"; const SPARE: &str = "1b36:000d"; let tests = compile::repo_root().join("tests/guest/rust-tests"); let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, JOB); let ends = qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, END_RETURNS_CLAIMS); let options = BootOptions { - profile: qemu::Profile::HeadlessUsbSpare, + profile, kernel_params: &["xhci-leave=1b36:000d"], ..Default::default() }; @@ -3774,6 +3787,16 @@ fn usbd_drives_the_spare(test_config: &Path) -> Result<(), String> { return Err(format!("{at} was not handed over once, to the one usbd:\n{said}")); } run.must_say(&format!("PCI {at}: msi address="))?; + let amdvi = matches!(profile.iommu(), Some(qemu::Unit::AmdVi { .. })); + if amdvi { + // The claim's entry in its own table, present at the vector the + // hand-over names, and absent again at the release. + let vector = said + .split_once(&handed) + .and_then(|(_, rest)| rest.split_whitespace().next()) + .ok_or_else(|| format!("the hand-over names no vector:\n{said}"))?; + run.must_say(&format!("iommu: amdvi irte source={at} vector={vector} p=1 "))?; + } let faulted = job_window(&mut qemu, "test_rs_usbd_spare fault")?; let fault = serial::Serial::named("usbd_spare fault", faulted.clone()); @@ -3787,7 +3810,10 @@ fn usbd_drives_the_spare(test_config: &Path) -> Result<(), String> { .find(|l| l.contains("iommu: DMA FAULT owner=process") && l.contains(&format!("stream={at} "))) .ok_or_else(|| format!("no DMA FAULT record names {at}:\n{faulted}"))?; let addr = record.split_once(" addr=").and_then(|(_, a)| a.split_whitespace().next()).unwrap_or_default(); - if u64::from_str_radix(addr.trim_start_matches("0x"), 16).ok() != u64::from_str_radix(aimed.trim_start_matches("0x"), 16).ok() { + // QEMU 11.1.1's AMD unit records no address + // (`issues/qemus-amd-iommu-page-fault-record-carries-no-address.md`). + let hex = |text: &str| u64::from_str_radix(text.trim_start_matches("0x"), 16).ok(); + if !amdvi && hex(addr) != hex(&aimed) { return Err(format!("the record faults {addr}, and the job aimed at {aimed}: {record}")); } eprintln!(" [usbd] {at}: handed over, its No-Op answered; aimed at {aimed}: {}", record.trim()); @@ -3801,7 +3827,7 @@ fn usbd_drives_the_spare(test_config: &Path) -> Result<(), String> { test_config, &[], &[(JOB.to_string(), bin)], - BootOptions { profile: qemu::Profile::HeadlessUsbSpare, ..Default::default() }, + BootOptions { profile, ..Default::default() }, ); serial::Serial::boot(&qemu).must_not_say("(xhci-leave)")?; let said = job_window(&mut qemu, "test_rs_usbd_spare refused")?; @@ -3841,6 +3867,50 @@ fn amdvi_translates(kernel: &serial::Serial) -> Result<(), String> { Ok(()) } +/// Every AMD-Vi unit remapping (`GAEn`, MMIO 0018h bit 17), every I/O APIC +/// the MADT has named by the IVRS, and each xHCI controller the kernel drives +/// armed through a present entry of its own table at the vector its message +/// carries, the stick bound by those interrupts. +fn amdvi_remaps(kernel: &serial::Serial) -> Result<(), String> { + let text = kernel.text(); + let remaps = kernel.must_say("iommu: amdvi remaps every message through its requester's table")?; + let units: Vec<&str> = + text.lines().filter(|l| l.contains("iommu: amdvi unit") && l.contains(" translating control=")).collect(); + for unit in &units { + let control = unit + .split_once(" control=") + .and_then(|(_, rest)| u64::from_str_radix(rest.split_whitespace().next()?.trim_start_matches("0x"), 16).ok()) + .ok_or_else(|| format!("{unit:?} names no control"))?; + if control & 1 << 17 == 0 { + return Err(format!("{unit:?} leaves GAEn clear")); + } + } + if units.is_empty() { + return Err(format!("no AMD-Vi unit is translating:\n{text}")); + } + let apics = text.lines().filter(|l| l.contains("ioapic: id=") && l.contains(" at 0x")).count(); + if !remaps.contains(&format!(" {apics} I/O APIC(s) named")) { + return Err(format!("{remaps:?} does not name the {apics} I/O APIC(s) the MADT has")); + } + kernel.must_not_say("iommu: DMA FAULT")?; + let controllers: Vec<&str> = text + .lines() + .filter_map(|l| l.split_once("xHCI: found at PCI ")?.1.split_whitespace().next()) + .collect(); + for at in &controllers { + let data = text + .lines() + .find_map(|l| l.split_once(&format!("PCI {at}: msix address="))?.1.split_once(" data=").map(|(_, d)| d.trim())) + .ok_or_else(|| format!("{at} was not armed on MSI-X:\n{text}"))?; + let vector = u32::from_str_radix(data.trim_start_matches("0x"), 16).map_err(|e| format!("{data}: {e}"))? & 0xFF; + kernel.must_say(&format!("iommu: amdvi irte source={at} vector={vector:#04x} p=1 "))?; + } + kernel.must_say("usb-storage: 1 device(s)")?; + eprintln!(" [amdvi] {}", remaps.trim()); + eprintln!(" [amdvi] {} controller(s) armed through their own entries, the stick bound", controllers.len()); + Ok(()) +} + /// The T14's Type-C controller left to a claim and driven by usbd: the /// kernel left it, the claim was handed over with its MSI in remappable /// format and its remapping entry written, the message reached its claim, and From f0462d2110e11c8d70035981e16db7241395e948 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 08:15:12 +0200 Subject: [PATCH 3/9] pcidev hands over an MSI-X table's BAR where the machine remaps; xhci-leave takes an address; the AMD bench's claim rows The orchestrator's ruling on #921: on a machine where every unit remaps, the BAR holding a function's MSI-X table is handed over, because the unit's per-requester table is the guard: a message the holder writes there reaches only the claim's own interrupt, and anything else is refused and recorded. A claim exists only where the machine remaps (`Refusal::NotRemapped` first), so `place_bars` takes the `Remapping` witness and no longer keeps the table's BAR back; `msix_bar` goes. The release masks the entry where the table is now, since its BAR may have moved since it was armed. Takes up issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md, its exit rewritten to that condition. - `xhci-leave=` names a function by `bus:device.function`: the AMD bench's two xHCI controllers share one id. The T14 row names `00:0d.0`; QEMU's spare is pinned at `00:10.0`. - `usbd_spare`'s arms move to `tests/guest/rust-tests/src/usbd_spare.rs`, with a binary each for the fault, foreign-vector and unremapped-refusal arms, because a metal job takes no argument. The new `vector` arm rewrites the claim's own MSI-X entry, through the BAR it now holds, to a vector not the claim's, and reads that usbd is never told of its No-Op. - QEMU: `Profile::HeadlessUsbSpareAmdVi`'s spare is armed on MSI-X with its table in its register BAR; `usbd_drives_the_spare_amdvi` adds the foreign-vector arm; `usbd_refused_without_amdvi_remapping` boots the AMD unit with `intremap=off` and reads the claim refused before anything on the function changes. QEMU records no fault for the refused message, filed as issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md. - AMD bench rows: `amdvi_usbd_drives_its_own_entry` (`03:00.4` handed over with its table's BAR, its No-Op answered through its own entry, aimed past its grant and recorded at that address, its holder's foreign vector refused and recorded and absent from the stop's unclaimed census), and `amdvi_claim_refused_without_remapping` (`iommu-no-remap`); the AMD unit now says when that actuator stands in for a unit that cannot remap. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...in-bar-0-cannot-be-driven-from-userland.md | 41 ++- ...o-fault-for-a-message-its-table-refuses.md | 26 ++ kernel/src/actuator.rs | 35 ++- kernel/src/arch/x86_64/amdvi/mod.rs | 7 +- kernel/src/drivers/pci.rs | 15 +- kernel/src/drivers/xhci/wait/boot.rs | 3 +- kernel/src/pcidev/mod.rs | 57 ++-- sdk/abi/src/pci.rs | 3 +- tests/common/claims.rs | 12 +- tests/common/qemu.rs | 24 +- .../rust-tests/src/bin/end_returns_claims.rs | 6 +- tests/guest/rust-tests/src/bin/usbd_spare.rs | 231 +------------- .../rust-tests/src/bin/usbd_spare_fault.rs | 10 + .../src/bin/usbd_spare_refused_unremapped.rs | 10 + .../rust-tests/src/bin/usbd_spare_vector.rs | 10 + tests/guest/rust-tests/src/usbd_spare.rs | 286 ++++++++++++++++++ tests/toyos.rs | 168 +++++++++- 17 files changed, 639 insertions(+), 305 deletions(-) create mode 100644 issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md create mode 100644 tests/guest/rust-tests/src/bin/usbd_spare_fault.rs create mode 100644 tests/guest/rust-tests/src/bin/usbd_spare_refused_unremapped.rs create mode 100644 tests/guest/rust-tests/src/bin/usbd_spare_vector.rs create mode 100644 tests/guest/rust-tests/src/usbd_spare.rs diff --git a/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md b/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md index 0d1693da25..70d93f1e2f 100644 --- a/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md +++ b/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md @@ -1,31 +1,28 @@ --- -status: open +status: assigned kind: defect opened: 2026-09-26 --- # A controller whose MSI-X table is in BAR 0 cannot be driven from userland -`pcidev` never maps the BAR holding a function's MSI-X table or PBA -(`kernel/src/pcidev/mod.rs`, `msix_bar`): a holder that could write the table -could aim the device's message at any address the LAPIC decodes. A window is a -whole 2 MiB page, so the refusal takes the whole BAR. NVMe keeps its registers -and doorbells in BAR 0, and a controller that puts its MSI-X table there too -has no BAR left to hand over: the claim is refused. +Held by PR #921 (AMD-Vi stage 6), under the orchestrator's ruling there: +on a machine where every unit remaps, `pcidev` hands over the BAR holding a +function's MSI-X table, because the unit's per-requester table is the guard +(a message the holder writes reaches only its own claim's interrupt, and +anything else is refused and recorded); where a unit does not remap, the +claim is refused anyway (`Refusal::NotRemapped`). -Measured on QEMU 11.1's NVMe, whose default puts the table in BAR 0: blockd's -claim answers `NotSupported` and the kernel says `pcidev: PCI 00:03.0 NOT -HANDED OVER — it publishes no memory BAR this claim may map, so its holder -would have no registers to drive it through`. The blockd tests boot the -controller with `msix-exclusive-bar=on`, which moves the table to a BAR of its -own (`tests/common/qemu.rs`, `BootOptions::userland_nvme`). Where the T14's -NVMe keeps its table is not measured; it matters from the small-kernel track's -step 9 (`issues/the-kernel-is-small-interrupts-post-and-threads-wait.md`), -when blockd is the only NVMe driver. +A claim exists only on a machine that remaps, so `kernel/src/pcidev/mod.rs` +no longer keeps the table's BAR back. QEMU's `qemu-xhci` with its table in +its register BAR is claimed and driven behind the AMD unit +(`usbd_drives_the_spare_amdvi`), and its holder's rewrite of its own entry +to a vector not its own reaches nobody; with the BAR withheld as before, the +claim is refused `NoMappableBar`. -**Exit condition.** A claim hands over the pages of a BAR that hold neither -the table nor the PBA, at the granularity those structures need (4 KiB, which -the user mappings do not have yet: -`issues/process-memory-is-2-mib-pages-and-that-caps-the-process-count.md`), -and a test drives QEMU's NVMe with its table in BAR 0 through blockd while a -write to the table's page is refused. +**Exit:** on the AMD bench, `03:00.4` (MSI-X table in BAR 0, its only BAR) +is claimed and driven by usbd through its own remapping entry, and its +holder's rewrite of its entry to a vector not its own is refused and +recorded against the claim (`amdvi_usbd_drives_its_own_entry`); and a +machine that does not remap still refuses the claim +(`amdvi_claim_refused_without_remapping`). diff --git a/issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md b/issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md new file mode 100644 index 0000000000..4333d90acd --- /dev/null +++ b/issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md @@ -0,0 +1,26 @@ +--- +status: open +kind: tooling +opened: 2026-10-14 +--- + +# QEMU's `amd-iommu` records no fault for a message its table refuses + +AMD 48882 rev 3.11 §2.2.5.2, step 7: a message whose remapping entry has +`RemapEn` clear is target-aborted and reported as an `IO_PAGE_FAULT` with `I` +set (Table 57). QEMU 11.1.1's `amd-iommu` (`intremap=on,xtsup=on`, 128-bit +entries) drops such a message and writes no event record: in +`usbd_drives_the_spare_amdvi`, a claim's holder rewrote its MSI-X entry from +the claim's vector 0x42 to 0x43, usbd's No-Op went unannounced ("the +bring-up's No-Op command: Silent"), and no `DMA FAULT` line followed (PR +#921's evidence, `g3` logs). + +So that test holds only that the message reached nobody; the AMD bench's +row `amdvi_usbd_drives_its_own_entry` holds the record, and that the vector +reached no CPU by the stop's census. + +Owner: the orchestrator's AMD laptop track +(`issues/an-amd-vi-machine-isolates-no-device.md`). + +**Exit:** under the QEMU `build/ci/qemu-version` names, that rewrite is +recorded against the claim, and the test asserts the record. diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index db5fbfecb0..c2171e66b9 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -211,7 +211,7 @@ actuators! { /// latch: a fatal path meeting a painter that will never let go. panel_painter_stalls = "panel-painter-stalls"; - /// Leave the xHCI controller `xhci-leave=:` names to a + /// Leave the xHCI controller at `xhci-leave=:.` to a /// claim: `drivers::xhci` never touches it. Read through [`xhci_left`]; /// judged by `usbd_drives_the_spare`. xhci_leave = "xhci-leave="; @@ -271,31 +271,40 @@ pub fn init(cmdline: &str) { fn take_value(name: &str, value: &str) { match name { "xhci-leave=" => { - let id = toyos_abi::syscall::PciId::parse(value).unwrap_or_else(|| { - panic!("boot parameter {name}{value}: not : in hex") + let at = function_address(value).unwrap_or_else(|| { + panic!("boot parameter {name}{value}: not :. in hex") }); - XHCI_LEFT.store(id.wire(), Ordering::Relaxed); + XHCI_LEFT.store(u64::from(at), Ordering::Relaxed); } _ => panic!("boot parameter {name}{value}: this kernel declares no such valued actuator"), } } -/// [`PciId::wire`](toyos_abi::syscall::PciId::wire) of the controller -/// `xhci-leave=` names, read only once that actuator is armed. +/// `bb:dd.f` in hex as a requester id, bus over device over function: a +/// machine with two controllers of one id names one by where it is. +#[cfg(feature = "boot-actuators")] +fn function_address(value: &str) -> Option { + let (bus, rest) = value.split_once(':')?; + let (dev, func) = rest.split_once('.')?; + let (bus, dev, func) = + (u8::from_str_radix(bus, 16).ok()?, u8::from_str_radix(dev, 16).ok()?, u8::from_str_radix(func, 16).ok()?); + (dev < 32 && func < 8).then_some(u16::from(bus) << 8 | u16::from(dev) << 3 | u16::from(func)) +} + +/// The requester id of the controller `xhci-leave=` names, read only once +/// that actuator is armed. #[cfg(feature = "boot-actuators")] static XHCI_LEFT: AtomicU64 = AtomicU64::new(0); -/// The xHCI controller `xhci-leave=` leaves to a claim, if it is armed. +/// The requester id of the xHCI controller `xhci-leave=` leaves to a claim, +/// if it is armed. #[cfg(feature = "boot-actuators")] -pub fn xhci_left() -> Option { - xhci_leave().then(|| { - toyos_abi::syscall::PciId::from_wire(XHCI_LEFT.load(Ordering::Relaxed)) - .expect("stored from an id's own word") - }) +pub fn xhci_left() -> Option { + xhci_leave().then(|| XHCI_LEFT.load(Ordering::Relaxed) as u16) } #[cfg(not(feature = "boot-actuators"))] -pub const fn xhci_left() -> Option { +pub const fn xhci_left() -> Option { None } diff --git a/kernel/src/arch/x86_64/amdvi/mod.rs b/kernel/src/arch/x86_64/amdvi/mod.rs index eeb5db77aa..b1cff9f8cf 100644 --- a/kernel/src/arch/x86_64/amdvi/mod.rs +++ b/kernel/src/arch/x86_64/amdvi/mod.rs @@ -279,7 +279,12 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) &super::ioapic::ids(), ); #[cfg(feature = "boot-actuators")] - let remap = if crate::actuator::iommu_no_remap() { None } else { remap }; + let remap = if crate::actuator::iommu_no_remap() { + log!("iommu: iommu-no-remap stands in for a unit that cannot remap, so no message is remapped"); + None + } else { + remap + }; let identity = build(devices, &settings, remap.is_some()); let reserved = windows.iter().map(|w| (w.base, w.end())).chain(ivmds.iter().map(|m| (m.start, m.start + m.length))); domain::start(reserved.collect(), settings, ivmds.iter().map(|m| m.requesters).collect(), translated); diff --git a/kernel/src/drivers/pci.rs b/kernel/src/drivers/pci.rs index eb9ac0881c..11e7b036ef 100644 --- a/kernel/src/drivers/pci.rs +++ b/kernel/src/drivers/pci.rs @@ -491,10 +491,21 @@ impl PciDevice { /// Point a claimed function's [`MSIX_ENTRY`] at its slot's own message and /// enable it. Takes the message and nothing else: the function is /// the one the entry was written for, and no other message reaches here. -pub fn arm_claimed_msix(message: &crate::iommu::Remapped) -> Result { +pub fn arm_claimed_msix(message: &crate::iommu::Remapped) -> Result<(), NoEntry> { let pci = message.function(); let entry = pci.msix_entry()?; - Ok(pci.write_msix(entry, message.address(), message.data())) + pci.write_msix(entry, message.address(), message.data()); + Ok(()) +} + +/// Mask a claimed function's [`MSIX_ENTRY`] where its table is now: the BAR +/// holding it may have moved since it was armed. +pub fn mask_claimed_msix(pci: &PciDevice) -> Result<(), NoEntry> { + let entry = pci.msix_entry()?; + let at = entry.address + MSIX_ENTRY as u64 * msix::ENTRY_BYTES; + crate::mm::paging::map_mmio(at, 0x1000, MmioPolicy::Uncacheable) + .write_u32(msix::ENTRY_VECTOR_CONTROL, msix::ENTRY_MASKED); + Ok(()) } /// [`arm_claimed_msix`] for a function with MSI and no MSI-X; `false` where it diff --git a/kernel/src/drivers/xhci/wait/boot.rs b/kernel/src/drivers/xhci/wait/boot.rs index 9c24061703..080bed0178 100644 --- a/kernel/src/drivers/xhci/wait/boot.rs +++ b/kernel/src/drivers/xhci/wait/boot.rs @@ -117,7 +117,8 @@ pub fn init(devices: &[PciDevice]) { for pci_dev in devices.iter().filter(|d| d.matches_class(0x0C, 0x03, Some(0x30))) { // Before anything that touches the function, `enable_bus_master`'s // record of it as this kernel's above all: that is what refuses a claim. - if crate::actuator::xhci_left().is_some_and(|id| pci_dev.is_id(id.vendor, id.device)) { + let at = u16::from(pci_dev.bus) << 8 | u16::from(pci_dev.dev) << 3 | u16::from(pci_dev.func); + if crate::actuator::xhci_left() == Some(at) { log!("xHCI: leaving PCI {:02x}:{:02x}.{} {:04x}:{:04x} to a claim (xhci-leave)", pci_dev.bus, pci_dev.dev, pci_dev.func, pci_dev.vendor_id(), pci_dev.device_id()); continue; diff --git a/kernel/src/pcidev/mod.rs b/kernel/src/pcidev/mod.rs index c80921a899..6fd531f515 100644 --- a/kernel/src/pcidev/mod.rs +++ b/kernel/src/pcidev/mod.rs @@ -49,12 +49,12 @@ //! that BAR rather than nothing at all, and a function this kernel could settle //! no BAR of ends at [`Refusal::NoMappableBar`]. //! -//! **The BAR holding the MSI-X table or PBA is never mapped**: a holder that -//! could rewrite the table could point the device's message at any address the -//! LAPIC decodes. **So a function is armed on MSI only where a walk that -//! reached its capability list's terminator found no MSI-X**: its message is -//! then a word of config space, which has no write path from userland, and it -//! has no table in a BAR for [`msix_bar`] to keep back. +//! **The BAR holding the MSI-X table or PBA is handed over like any other**, +//! because a claim exists only on a machine that remaps: whatever message its +//! holder writes into the table, the unit delivers only the claim's own +//! interrupt and refuses and records anything else. **A function is armed on +//! MSI only where a walk that reached its capability list's terminator found +//! no MSI-X.** //! //! **A function with no address space of its own is not handed over**, because //! every grant would answer with a physical address and a descriptor holding @@ -128,13 +128,13 @@ use toyos_abi::pci::{DeviceIrqRecord, PciFunctionInfo, BARS}; use toyos_abi::syscall::{PciId, RegWidth, SyscallError}; use toyos_dma::Register; use toyos_pci::bridge::Window; -use toyos_pci::{af, aperture, bar, express, msix, placement, pm, probe}; +use toyos_pci::{af, aperture, bar, express, placement, pm, probe}; use crate::device::{Claim, ClaimError}; -use crate::drivers::pci::{arm_claimed_msi, arm_claimed_msix, NoCapability, NoEntry, PciDevice}; +use crate::drivers::pci::{arm_claimed_msi, arm_claimed_msix, mask_claimed_msix, NoCapability, NoEntry, PciDevice}; use crate::iommu::{IommuError, OwnSpace, Refused, Remapped}; use crate::mm::policy::{CachePolicy, MmioPolicy}; -use crate::mm::{align_2m, DirectMap, Mmio, PAGE_2M}; +use crate::mm::{align_2m, DirectMap, PAGE_2M}; use crate::object::shm::{Region, SharedMemObject}; use crate::sync::Lock; use crate::inbox::PollEntry; @@ -266,15 +266,15 @@ struct Aimed { /// Both deliver the function's own interrupt into the same [`Interrupt`] and /// the claim answers the same handle either way. enum Armed { - /// This function's one MSI-X table entry, mapped for the kernel alone. - Msix(Mmio, Remapped), + /// This function's one MSI-X table entry. + Msix(Remapped), Msi(Remapped), } impl Armed { fn message(&self) -> &Remapped { match self { - Self::Msix(_, message) | Self::Msi(message) => message, + Self::Msix(message) | Self::Msi(message) => message, } } } @@ -878,9 +878,6 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result // leave the machine changed by a hand-over that did not happen. let Space { space, lend } = function_space(number).map_err(Refusal::Untranslated)?; - // The table's own BAR, so it can be left where it is and kept out of what - // the holder maps. - let table_bar = msix_bar(&pci); // Decode must be on for a BAR to answer, and off across each move. pci.enable_memory_space(); @@ -890,7 +887,7 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result // translation and its interrupt back. let message = crate::iommu::claim_msi(remapping, number, &pci).map_err(Refusal::NoMessage)?; let armed = match arm_claimed_msix(&message) { - Ok(entry) => Armed::Msix(entry, message), + Ok(()) => Armed::Msix(message), Err(NoEntry::Unusable) => return Err(Refusal::MsixUnusable), Err(NoEntry::NoTable(NoCapability::Truncated)) => return Err(Refusal::CapsTruncated), Err(NoEntry::NoTable(NoCapability::Absent)) if arm_claimed_msi(&message) => { @@ -901,7 +898,7 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result // From here a refusal has to undo: a vector is armed, and the arms below // move the function's BARs. - match place_bars(&pci, id, table_bar) { + match place_bars(&pci, id, remapping) { Ok((bar_at, bar_bytes)) => { space.attach(pci.bus, pci.dev, pci.func); Ok(Bound { @@ -920,7 +917,7 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result Err(why) => { // The entry goes not present once the function stops speaking. let message = match armed { - Armed::Msix(_, message) => { + Armed::Msix(message) => { pci.disable_msix(); message } @@ -945,7 +942,10 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result fn place_bars( pci: &PciDevice, id: PciId, - table_bar: Option, + // A BAR holding the MSI-X table is handed over too: on a machine that + // remaps, a message its holder writes there reaches only the claim's own + // interrupt, and anything else is refused and recorded by the unit. + _: crate::iommu::Remapping, ) -> Result<([u64; BARS], [u64; BARS]), Refusal> { let mut bar_at = [0u64; BARS]; let mut bar_bytes = [0u64; BARS]; @@ -959,7 +959,7 @@ fn place_bars( // whatever its register holds now: one a reset returned to zero reads // as unassigned, and it is the window that moved, not a BAR that is // not there. - if Some(index) == table_bar || (pci.memory_bar(index).is_err() && !was_cut(pci, index)) { + if pci.memory_bar(index).is_err() && !was_cut(pci, index) { index += step; continue; } @@ -1264,18 +1264,6 @@ fn take_residue(number: usize) -> Vec { residue } -/// Which BAR holds this function's MSI-X table or PBA, if any. -/// -/// Both, because both are the unit's structures and neither is the holder's: -/// the two live in one BAR on every device in reach, and a device that split -/// them costs the second BAR too rather than publishing one of them. -fn msix_bar(pci: &PciDevice) -> Option { - let cap = pci.capability(msix::CAP_ID).ok()?; - let control = cap.read_u16(msix::MESSAGE_CONTROL); - let table = msix::Msix::decode(control, cap.read_u32(msix::TABLE)).ok()?; - Some(table.bir()) -} - /// What this machine answers at `at + offset`. /// /// **One dword and no more.** This is a read of a register file this kernel @@ -1517,8 +1505,9 @@ fn release(number: usize) { fn tear_down(function: &Function, mut bound: Bound) { bound.pci.disable_bus_master(); let message = match bound.armed { - Armed::Msix(entry, message) => { - entry.write_u32(msix::ENTRY_VECTOR_CONTROL, msix::ENTRY_MASKED); + Armed::Msix(message) => { + // Armed through this table, so it is still there to mask. + assert!(mask_claimed_msix(&bound.pci).is_ok(), "pcidev: the MSI-X table a claim was armed through is gone"); message } Armed::Msi(message) => { diff --git a/sdk/abi/src/pci.rs b/sdk/abi/src/pci.rs index 2e22bb2965..e89d247c9e 100644 --- a/sdk/abi/src/pci.rs +++ b/sdk/abi/src/pci.rs @@ -31,8 +31,7 @@ crate::user_safe! { #[derive(Clone, Copy)] pub struct PciFunctionInfo { /// Byte size of each memory BAR; 0 where the function has none in that - /// slot, or where it is one the kernel will not map — the BAR holding this - /// function's MSI-X table or PBA reads 0 here. + /// slot, or where it is one the kernel will not map. pub bar_bytes: [u64; BARS], /// Where firmware put it. Identity for a log line, never a name a claim is /// looked up by: nothing in this ABI takes a bus/device/function. diff --git a/tests/common/claims.rs b/tests/common/claims.rs index add25e87ba..b27e108c67 100644 --- a/tests/common/claims.rs +++ b/tests/common/claims.rs @@ -137,12 +137,20 @@ pub fn clear_of_host_bridges(log: &Serial) -> Result<(), String> { pub fn refused_unremapped(log: &Serial) -> Result<(), String> { log.must_say(INTR_REMAP_CLEAR)?; let at = function(log)?; - log.must_say(&format!("pcidev: PCI {at} NOT HANDED OVER — {NOT_REMAPPED}"))?; log.must_not_say(&format!("[{I219}] handed over"))?; + refused_unremapped_at(log, &at)?; + log.must_say(bootlog::COMPLETE)?; + Ok(()) +} + +/// The claim of the function at `at` refused because nothing remaps its +/// messages, before anything on it changed: no message armed, no entry +/// written, no BAR moved. +pub fn refused_unremapped_at(log: &Serial, at: &str) -> Result<(), String> { + log.must_say(&format!("pcidev: PCI {at} NOT HANDED OVER — {NOT_REMAPPED}"))?; log.must_not_say(&format!("PCI {at}: msi address="))?; log.must_not_say(&format!("PCI {at}: msix address="))?; log.must_not_say(&format!(" source={at} "))?; log.must_not_say(&format!("pcidev: PCI {at} BAR"))?; - log.must_say(bootlog::COMPLETE)?; Ok(()) } diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 9afb5378ab..aaf45d4a32 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -712,8 +712,11 @@ pub enum Profile { /// can be made and read. MetalAmdViTestdev, /// [`Profile::HeadlessUsbSpare`] behind QEMU's AMD unit remapping - /// interrupts: a claim's message through its own requester's table. + /// interrupts, its spare armed on MSI-X with the table in the register + /// BAR: a claim's message through its own requester's table. HeadlessUsbSpareAmdVi, + /// [`Profile::HeadlessUsbSpareAmdVi`] with the unit remapping nothing. + HeadlessUsbSpareAmdViNoRemap, /// QEMU `virt` on AArch64 (GICv3, AAVMF): a GOP from `ramfb`, the boot /// stick on an xHCI, the PL011, a virtio-rng for firmware's /// `EFI_RNG_PROTOCOL`, and nothing else — no NIC, NVMe or IOMMU. The @@ -777,7 +780,8 @@ impl Profile { | Self::MetalNoX2apic | Self::MetalAmdVi | Self::MetalAmdViTestdev - | Self::HeadlessUsbSpareAmdVi => Arch::X86_64, + | Self::HeadlessUsbSpareAmdVi + | Self::HeadlessUsbSpareAmdViNoRemap => Arch::X86_64, } } @@ -860,7 +864,12 @@ const XHCI_DEFAULT: &str = "nec-usb-xhci,id=xhci"; /// question this machine asks. `msi=on` stated, because with MSI-X off and /// MSI left `auto` this QEMU's `qemu-xhci` offers neither, and the kernel /// refuses the claim for it. -const XHCI_SPARE: &str = "qemu-xhci,id=spare,msix=off,msi=on"; +const XHCI_SPARE: &str = "qemu-xhci,id=spare,addr=0x10,msix=off,msi=on"; +/// [`XHCI_SPARE`] armed on MSI-X, whose table QEMU puts in the controller's +/// register BAR, as the AMD bench's controllers have theirs. +const XHCI_SPARE_MSIX: &str = "qemu-xhci,id=spare,addr=0x10"; +/// Where both put the spare, which `xhci-leave=` names. +pub const SPARE_AT: &str = "00:10.0"; const SPARE_STICK: &str = "driver=null-co,node-name=spare-stick,size=67108864,read-zeroes=on"; /// Whether a machine has the virtio console and sound block. Which NIC it has @@ -1051,8 +1060,13 @@ impl Profile { }, Self::MetalNoX2apic => Self::Metal.shape(), Self::MetalAmdVi => Shape { iommu: Some(Unit::AmdVi { intremap: false }), ..Self::Metal.shape() }, - Self::HeadlessUsbSpareAmdVi => { - Shape { iommu: Some(Unit::AmdVi { intremap: true }), ..Self::HeadlessUsbSpare.shape() } + Self::HeadlessUsbSpareAmdVi => Shape { + iommu: Some(Unit::AmdVi { intremap: true }), + xhci: &[XHCI_DEFAULT, XHCI_SPARE_MSIX], + ..Self::HeadlessUsbSpare.shape() + }, + Self::HeadlessUsbSpareAmdViNoRemap => { + Shape { iommu: Some(Unit::AmdVi { intremap: false }), ..Self::HeadlessUsbSpareAmdVi.shape() } } Self::MetalAmdViTestdev => Shape { probes: Probes::Testdevs, ..Self::MetalAmdVi.shape() }, Self::HeadlessNoIommu => Shape { iommu: None, ..Self::Headless.shape() }, diff --git a/tests/guest/rust-tests/src/bin/end_returns_claims.rs b/tests/guest/rust-tests/src/bin/end_returns_claims.rs index 09c23f1e13..39099175ad 100644 --- a/tests/guest/rust-tests/src/bin/end_returns_claims.rs +++ b/tests/guest/rust-tests/src/bin/end_returns_claims.rs @@ -21,7 +21,11 @@ const SELF_PATH: &str = "/system/bin/test_rs_end_returns_claims"; /// The controllers a boot of this job leaves to a claim: QEMU's `qemu-xhci` /// and the T14's Type-C controller, as `usbd_spare`'s. -const SPARES: [PciId; 2] = [PciId { vendor: 0x1b36, device: 0x000d }, PciId { vendor: 0x8086, device: 0x9a13 }]; +const SPARES: [PciId; 3] = [ + PciId { vendor: 0x1b36, device: 0x000d }, + PciId { vendor: 0x8086, device: 0x9a13 }, + PciId { vendor: 0x1022, device: 0x1639 }, +]; const ROUNDS: u32 = 100; diff --git a/tests/guest/rust-tests/src/bin/usbd_spare.rs b/tests/guest/rust-tests/src/bin/usbd_spare.rs index d43b273946..b17bfe949e 100644 --- a/tests/guest/rust-tests/src/bin/usbd_spare.rs +++ b/tests/guest/rust-tests/src/bin/usbd_spare.rs @@ -1,226 +1,19 @@ -//! The xHCI controller the kernel was told to leave alone (`xhci-leave=`), -//! claimed by this job and driven by `/system/bin/usbd`, which it starts -//! itself so it can end it: this job holds the claim, the port usbd answers -//! `inspect` on, and usbd's process. -//! -//! - no argument: usbd started, asked once it says it has settled whether its -//! No-Op was answered by an interrupt through the claim, asked again across -//! a span in which nothing may wake it, and killed. Every answer is printed -//! for the harness to judge. -//! - `fault`: the claim's function aimed, with no usbd, at an address outside -//! every grant: the kernel's `DMA FAULT` record is the harness's to read, -//! and the claim refusing its interrupt read from then on is this job's. -//! - `refused`: the claim, on a kernel that drives the function, refused. +//! `usbd_spare` with its arm named by its argument, for a QEMU boot that +//! runs several in one: see `src/usbd_spare.rs`. A metal job takes no +//! argument, so each arm a metal row needs is a binary of its own beside +//! this one. -use std::collections::BTreeMap; -use std::io::{BufRead, BufReader}; -use std::os::toyos::process::CommandExt; -use std::process::{Child, Command, Stdio}; -use std::sync::mpsc; -use std::time::{Duration, Instant}; - -use toyos::endow::{Endowments, DEV_PREFIX, SERVE_PREFIX, SYSCAP_LABEL}; -use toyos::ipc::Connection; -use toyos::poller::{Poller, READABLE}; -use toyos::syscap::SysCap; -use toyos::volatile::Window; -use toyos::PciDev; -use toyos_abi::syscall::{DeviceRequest, PciId, SyscallError}; -use toyos_inspect::{Value, MSG_INSPECT, MSG_SNAPSHOT, USB}; - -const USBD: &str = "/system/bin/usbd"; - -/// The controllers a boot of this job leaves to a claim: QEMU's `qemu-xhci` -/// on the guest profile, and the T14's Type-C controller. A machine has one -/// of them; the other is absent. -const SPARES: [PciId; 2] = [PciId { vendor: 0x1b36, device: 0x000d }, PciId { vendor: 0x8086, device: 0x9a13 }]; - -/// How long usbd has to say it has settled. Policy, and loose: enumeration -/// is a few control transfers per device, and the guest may be slow. -const SETTLE_BOUND: Duration = Duration::from_secs(30); - -/// The span across which a parked usbd may wake for nothing but an event: -/// the absence is what is measured, so the span is the instrument. -const PARKED_SPAN: Duration = Duration::from_millis(500); - -/// What the fault arm aims the command ring at: a gigabyte past its one -/// grant, which nothing in the claim's domain maps. -const PAST_THE_GRANT: u64 = 1 << 30; +#[path = "../usbd_spare.rs"] +mod usbd_spare; fn main() { - let cap: SysCap = Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows a device-minting capability"); + let cap = usbd_spare::cap(); match std::env::args().nth(1).as_deref() { - None => drive(&cap), - Some("fault") => fault(&cap), - Some("refused") => refused(&cap), + None => usbd_spare::drive(&cap), + Some("fault") => usbd_spare::fault(&cap), + Some("refused") => usbd_spare::refused(&cap), + Some("vector") => usbd_spare::vector(&cap), + Some("refused_unremapped") => usbd_spare::refused_unremapped(&cap), Some(other) => panic!("usbd_spare: no arm {other:?}"), } } - -/// The one spare this machine has, claimed, or every claim's refusal. -fn claim(cap: &SysCap) -> (PciId, Result) { - let mut found = None; - for id in SPARES { - match cap.claim_pci::(id) { - Err(SyscallError::NotFound) => {} - answered => { - assert!(found.is_none(), "usbd_spare: two spares on one machine"); - found = Some((id, answered)); - } - } - } - found.expect("usbd_spare: neither spare controller is on this machine") -} - -fn spelled(id: PciId) -> String { - let mut buf = [0u8; DeviceRequest::MAX_NAME]; - DeviceRequest::Pci(id).write_name(&mut buf).to_string() -} - -/// usbd, started on `dev` and serving its port to this job, and the lines it -/// says. -struct Usbd { - child: Child, - said: mpsc::Receiver, - conn: toyos::port::Connector, -} - -impl Usbd { - fn start(id: PciId, dev: PciDev) -> Self { - let (acceptor, conn) = toyos::port::create().expect("usbd_spare: a port"); - let mut child = Command::new(USBD) - .endow(&format!("{DEV_PREFIX}{}", spelled(id)), toyos::AsHandle::as_handle(&dev).0) - .endow(&format!("{SERVE_PREFIX}{}", USB.port), acceptor.into_raw().0) - .stdout(Stdio::piped()) - .spawn() - .expect("usbd_spare: spawn usbd"); - // Moved: the spawn holds them now. - std::mem::forget(dev); - let out = BufReader::new(child.stdout.take().expect("usbd's piped stdout")); - let (tx, said) = mpsc::channel(); - std::thread::spawn(move || { - for line in out.lines() { - let Ok(line) = line else { return }; - println!("usbd_spare: usbd said: {line}"); - if tx.send(line).is_err() { - return; - } - } - }); - Self { child, said, conn } - } - - /// Wait for usbd's line starting `prefix`, bounded. - fn until(&self, prefix: &str) -> String { - let deadline = Instant::now() + SETTLE_BOUND; - loop { - let left = deadline.saturating_duration_since(Instant::now()); - match self.said.recv_timeout(left) { - Ok(line) if line.starts_with(prefix) => return line, - Ok(_) => {} - Err(why) => panic!("usbd_spare: usbd never said {prefix:?} within {SETTLE_BOUND:?}: {why:?}"), - } - } - } - - /// One `inspect` answer, printed whole. - fn ask(&self, when: &str) -> BTreeMap { - let ns = toyos::namespace::build().add(USB.port, &self.conn).finish().expect("usbd_spare: a namespace"); - let conn: Connection = ns.open(USB.port).expect("usbd_spare: a connection to usbd"); - conn.signal(MSG_INSPECT).expect("usbd_spare: the request"); - let header = conn.recv_header().expect("usbd_spare: usbd's answer"); - assert_eq!(header.msg_type, MSG_SNAPSHOT, "usbd_spare: usbd answered something that is not a snapshot"); - let mut buf = vec![0u8; header.len() as usize]; - let n = conn.recv_bytes(&header, &mut buf).expect("usbd_spare: the snapshot's bytes"); - let snapshot = toyos_inspect::decode(&buf[..n], USB).expect("usbd_spare: a snapshot for `usb`"); - for (path, value) in &snapshot { - println!("usbd_spare: {when}: {}", toyos_inspect::line(path, value)); - } - snapshot - } - - fn kill(mut self) { - self.child.kill().expect("usbd_spare: kill usbd"); - let status = self.child.wait().expect("usbd_spare: wait for usbd"); - assert!(!status.success(), "usbd_spare: a killed usbd ended {status:?}"); - } -} - -fn number(snapshot: &BTreeMap, path: &str) -> u64 { - match snapshot.get(path) { - Some(Value::U64(n)) => *n, - other => panic!("usbd_spare: `usb.{path}` is {other:?}, not a count"), - } -} - -fn drive(cap: &SysCap) { - let (id, claimed) = claim(cap); - let dev = claimed.unwrap_or_else(|why| panic!("usbd_spare: the claim on {} was refused: {why:?}", spelled(id))); - let usbd = Usbd::start(id, dev); - usbd.until("usbd: settled"); - let first = usbd.ask("settled"); - assert_eq!(number(&first, "usb.events.unannounced"), 0, "usbd_spare: an event no interrupt announced"); - assert_eq!(number(&first, "usb.operations.silent"), 0, "usbd_spare: an operation nothing answered"); - assert!(number(&first, "usb.interrupts.records") > 0, "usbd_spare: usbd read no interrupt record"); - assert_eq!(first.get("usb.noop").map(Value::to_string).as_deref(), Some("completed")); - - // Parked: across the span nothing wakes it that is not an event — no - // timeout, and no claim read ready without a record behind it. - let before = usbd.ask("parked, before"); - std::thread::sleep(PARKED_SPAN); - let after = usbd.ask("parked, after"); - for path in ["usb.wakes.timed", "usb.wakes.empty"] { - assert_eq!(number(&before, path), number(&after, path), "usbd_spare: {path} moved while usbd had nothing to do"); - } - assert_eq!(after.get("usb.deadline_ns").map(Value::to_string).as_deref(), Some("none")); - usbd.kill(); - println!("usbd_spare: usbd killed"); -} - -fn fault(cap: &SysCap) { - let (id, claimed) = claim(cap); - let dev = claimed.unwrap_or_else(|why| panic!("usbd_spare: the claim on {} was refused: {why:?}", spelled(id))); - let info = dev.describe().expect("usbd_spare: the claim's description"); - let bar = dev.map_bar(0, info.bar_bytes[0]).expect("usbd_spare: BAR 0"); - // SAFETY: the mapping is the BAR's length and lives as long as `bar`. - let regs = unsafe { Window::new(bar.as_ptr(), info.bar_bytes[0] as usize) }; - // One page granted, which is what starts the function mastering the bus. - let grant = dev.dma_alloc(4096).expect("usbd_spare: a grant"); - let op = usize::from(regs.read::(0)); - let db = (regs.read::(0x14) & !3) as usize; - let settle = |what: &str, done: &dyn Fn() -> bool| { - let began = Instant::now(); - while !done() { - assert!(began.elapsed() < Duration::from_secs(2), "usbd_spare: the controller {what}"); - } - }; - regs.write::(op, 0); - settle("never halted", &|| regs.read::(op + 4) & 1 != 0); - regs.write::(op, 1 << 1); - settle("held its reset", &|| regs.read::(op) & (1 << 1) == 0 && regs.read::(op + 4) & (1 << 11) == 0); - let aimed = grant.device_addr + PAST_THE_GRANT; - // The command ring there, the controller running, and its doorbell rung: - // its first fetch is a read of an address no grant maps. - regs.write::(op + 0x18, aimed | 1); - regs.write::(op, 1); - regs.write::(db, 0); - println!("usbd_spare: the command ring aimed at {aimed:#x}, past the grant at {:#x}", grant.device_addr); - - let poller = Poller::new(1); - poller.watch(&dev, READABLE, 0); - poller.wait(1, Duration::from_secs(10).as_nanos() as u64, |_| {}); - match dev.irq() { - Err(SyscallError::Io) => println!("usbd_spare: the claim refuses its interrupt read: Io"), - Ok(record) => panic!("usbd_spare: after the aimed fetch the claim answered a record of {}, not the unit's refusal", record.count), - Err(why) => panic!("usbd_spare: after the aimed fetch the claim answered {why:?}, not the unit's refusal"), - } -} - -fn refused(cap: &SysCap) { - let (id, claimed) = claim(cap); - match claimed { - Err(SyscallError::PermissionDenied) => println!("usbd_spare: the claim on {} was refused: PermissionDenied", spelled(id)), - Err(why) => panic!("usbd_spare: the claim was refused, by {why:?} and not because the kernel drives it"), - Ok(_) => panic!("usbd_spare: the claim on {} was granted, on a kernel that drives it", spelled(id)), - } -} diff --git a/tests/guest/rust-tests/src/bin/usbd_spare_fault.rs b/tests/guest/rust-tests/src/bin/usbd_spare_fault.rs new file mode 100644 index 0000000000..6c2c82fe3d --- /dev/null +++ b/tests/guest/rust-tests/src/bin/usbd_spare_fault.rs @@ -0,0 +1,10 @@ +//! `usbd_spare`'s `fault` arm as a binary of its own, for a metal job, which +//! takes no argument: see `src/usbd_spare.rs`. + +#[path = "../usbd_spare.rs"] +#[allow(dead_code, reason = "this binary runs one of the module's arms")] +mod usbd_spare; + +fn main() { + usbd_spare::fault(&usbd_spare::cap()); +} diff --git a/tests/guest/rust-tests/src/bin/usbd_spare_refused_unremapped.rs b/tests/guest/rust-tests/src/bin/usbd_spare_refused_unremapped.rs new file mode 100644 index 0000000000..7491890f4d --- /dev/null +++ b/tests/guest/rust-tests/src/bin/usbd_spare_refused_unremapped.rs @@ -0,0 +1,10 @@ +//! `usbd_spare`'s `refused_unremapped` arm as a binary of its own, for a metal job, which +//! takes no argument: see `src/usbd_spare.rs`. + +#[path = "../usbd_spare.rs"] +#[allow(dead_code, reason = "this binary runs one of the module's arms")] +mod usbd_spare; + +fn main() { + usbd_spare::refused_unremapped(&usbd_spare::cap()); +} diff --git a/tests/guest/rust-tests/src/bin/usbd_spare_vector.rs b/tests/guest/rust-tests/src/bin/usbd_spare_vector.rs new file mode 100644 index 0000000000..bc3d178dcc --- /dev/null +++ b/tests/guest/rust-tests/src/bin/usbd_spare_vector.rs @@ -0,0 +1,10 @@ +//! `usbd_spare`'s `vector` arm as a binary of its own, for a metal job, which +//! takes no argument: see `src/usbd_spare.rs`. + +#[path = "../usbd_spare.rs"] +#[allow(dead_code, reason = "this binary runs one of the module's arms")] +mod usbd_spare; + +fn main() { + usbd_spare::vector(&usbd_spare::cap()); +} diff --git a/tests/guest/rust-tests/src/usbd_spare.rs b/tests/guest/rust-tests/src/usbd_spare.rs new file mode 100644 index 0000000000..1625ba6b91 --- /dev/null +++ b/tests/guest/rust-tests/src/usbd_spare.rs @@ -0,0 +1,286 @@ +//! `usbd_spare`'s arms, one binary each for a metal job and all of them +//! behind one for QEMU. The xHCI controller the kernel was told to leave +//! alone (`xhci-leave=`), claimed by this job and driven by `/system/bin/usbd`, which it starts +//! itself so it can end it: this job holds the claim, the port usbd answers +//! `inspect` on, and usbd's process. +//! +//! - no argument: usbd started, asked once it says it has settled whether its +//! No-Op was answered by an interrupt through the claim, asked again across +//! a span in which nothing may wake it, and killed. Every answer is printed +//! for the harness to judge. +//! - `fault`: the claim's function aimed, with no usbd, at an address outside +//! every grant: the kernel's `DMA FAULT` record is the harness's to read, +//! and the claim refusing its interrupt read from then on is this job's. +//! - `refused`: the claim, on a kernel that drives the function, refused. +//! - `refused_unremapped`: the claim, on a machine that remaps nothing, +//! refused; the kernel's line says why. +//! - `vector`: the claim's MSI-X table entry rewritten, by this job through +//! the BAR that holds it, to a vector that is not the claim's, and usbd +//! started: its No-Op's message is the unit's to refuse and record, and +//! usbd settles never told of it. + +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader}; +use std::os::toyos::process::CommandExt; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +use toyos::endow::{Endowments, DEV_PREFIX, SERVE_PREFIX, SYSCAP_LABEL}; + +/// The device-minting capability the test runner endows every job with. +pub fn cap() -> SysCap { + Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows a device-minting capability") +} +use toyos::ipc::Connection; +use toyos::poller::{Poller, READABLE}; +use toyos::syscap::SysCap; +use toyos::volatile::Window; +use toyos::PciDev; +use toyos_abi::syscall::{DeviceRequest, PciId, SyscallError}; +use toyos_inspect::{Value, MSG_INSPECT, MSG_SNAPSHOT, USB}; + +const USBD: &str = "/system/bin/usbd"; + +/// The controllers a boot of this job leaves to a claim: QEMU's `qemu-xhci` +/// on the guest profile, and the T14's Type-C controller. A machine has one +/// of them; the other is absent. +const SPARES: [PciId; 3] = [ + PciId { vendor: 0x1b36, device: 0x000d }, + PciId { vendor: 0x8086, device: 0x9a13 }, + PciId { vendor: 0x1022, device: 0x1639 }, +]; + +/// How long usbd has to say it has settled. Policy, and loose: enumeration +/// is a few control transfers per device, and the guest may be slow. +const SETTLE_BOUND: Duration = Duration::from_secs(30); + +/// The span across which a parked usbd may wake for nothing but an event: +/// the absence is what is measured, so the span is the instrument. +const PARKED_SPAN: Duration = Duration::from_millis(500); + +/// What the fault arm aims the command ring at: a gigabyte past its one +/// grant, which nothing in the claim's domain maps. +const PAST_THE_GRANT: u64 = 1 << 30; + + +/// The one spare this machine has, claimed, or every claim's refusal. +fn claim(cap: &SysCap) -> (PciId, Result) { + let mut found = None; + for id in SPARES { + match cap.claim_pci::(id) { + Err(SyscallError::NotFound) => {} + answered => { + assert!(found.is_none(), "usbd_spare: two spares on one machine"); + found = Some((id, answered)); + } + } + } + found.expect("usbd_spare: neither spare controller is on this machine") +} + +fn spelled(id: PciId) -> String { + let mut buf = [0u8; DeviceRequest::MAX_NAME]; + DeviceRequest::Pci(id).write_name(&mut buf).to_string() +} + +/// usbd, started on `dev` and serving its port to this job, and the lines it +/// says. +struct Usbd { + child: Child, + said: mpsc::Receiver, + conn: toyos::port::Connector, +} + +impl Usbd { + fn start(id: PciId, dev: PciDev) -> Self { + let (acceptor, conn) = toyos::port::create().expect("usbd_spare: a port"); + let mut child = Command::new(USBD) + .endow(&format!("{DEV_PREFIX}{}", spelled(id)), toyos::AsHandle::as_handle(&dev).0) + .endow(&format!("{SERVE_PREFIX}{}", USB.port), acceptor.into_raw().0) + .stdout(Stdio::piped()) + .spawn() + .expect("usbd_spare: spawn usbd"); + // Moved: the spawn holds them now. + std::mem::forget(dev); + let out = BufReader::new(child.stdout.take().expect("usbd's piped stdout")); + let (tx, said) = mpsc::channel(); + std::thread::spawn(move || { + for line in out.lines() { + let Ok(line) = line else { return }; + println!("usbd_spare: usbd said: {line}"); + if tx.send(line).is_err() { + return; + } + } + }); + Self { child, said, conn } + } + + /// Wait for usbd's line starting `prefix`, bounded. + fn until(&self, prefix: &str) -> String { + let deadline = Instant::now() + SETTLE_BOUND; + loop { + let left = deadline.saturating_duration_since(Instant::now()); + match self.said.recv_timeout(left) { + Ok(line) if line.starts_with(prefix) => return line, + Ok(_) => {} + Err(why) => panic!("usbd_spare: usbd never said {prefix:?} within {SETTLE_BOUND:?}: {why:?}"), + } + } + } + + /// One `inspect` answer, printed whole. + fn ask(&self, when: &str) -> BTreeMap { + let ns = toyos::namespace::build().add(USB.port, &self.conn).finish().expect("usbd_spare: a namespace"); + let conn: Connection = ns.open(USB.port).expect("usbd_spare: a connection to usbd"); + conn.signal(MSG_INSPECT).expect("usbd_spare: the request"); + let header = conn.recv_header().expect("usbd_spare: usbd's answer"); + assert_eq!(header.msg_type, MSG_SNAPSHOT, "usbd_spare: usbd answered something that is not a snapshot"); + let mut buf = vec![0u8; header.len() as usize]; + let n = conn.recv_bytes(&header, &mut buf).expect("usbd_spare: the snapshot's bytes"); + let snapshot = toyos_inspect::decode(&buf[..n], USB).expect("usbd_spare: a snapshot for `usb`"); + for (path, value) in &snapshot { + println!("usbd_spare: {when}: {}", toyos_inspect::line(path, value)); + } + snapshot + } + + fn kill(mut self) { + self.child.kill().expect("usbd_spare: kill usbd"); + let status = self.child.wait().expect("usbd_spare: wait for usbd"); + assert!(!status.success(), "usbd_spare: a killed usbd ended {status:?}"); + } +} + +fn number(snapshot: &BTreeMap, path: &str) -> u64 { + match snapshot.get(path) { + Some(Value::U64(n)) => *n, + other => panic!("usbd_spare: `usb.{path}` is {other:?}, not a count"), + } +} + +pub fn drive(cap: &SysCap) { + let (id, claimed) = claim(cap); + let dev = claimed.unwrap_or_else(|why| panic!("usbd_spare: the claim on {} was refused: {why:?}", spelled(id))); + let usbd = Usbd::start(id, dev); + usbd.until("usbd: settled"); + let first = usbd.ask("settled"); + assert_eq!(number(&first, "usb.events.unannounced"), 0, "usbd_spare: an event no interrupt announced"); + assert_eq!(number(&first, "usb.operations.silent"), 0, "usbd_spare: an operation nothing answered"); + assert!(number(&first, "usb.interrupts.records") > 0, "usbd_spare: usbd read no interrupt record"); + assert_eq!(first.get("usb.noop").map(Value::to_string).as_deref(), Some("completed")); + + // Parked: across the span nothing wakes it that is not an event — no + // timeout, and no claim read ready without a record behind it. + let before = usbd.ask("parked, before"); + std::thread::sleep(PARKED_SPAN); + let after = usbd.ask("parked, after"); + for path in ["usb.wakes.timed", "usb.wakes.empty"] { + assert_eq!(number(&before, path), number(&after, path), "usbd_spare: {path} moved while usbd had nothing to do"); + } + assert_eq!(after.get("usb.deadline_ns").map(Value::to_string).as_deref(), Some("none")); + usbd.kill(); + println!("usbd_spare: usbd killed"); +} + +pub fn fault(cap: &SysCap) { + let (id, claimed) = claim(cap); + let dev = claimed.unwrap_or_else(|why| panic!("usbd_spare: the claim on {} was refused: {why:?}", spelled(id))); + let info = dev.describe().expect("usbd_spare: the claim's description"); + let bar = dev.map_bar(0, info.bar_bytes[0]).expect("usbd_spare: BAR 0"); + // SAFETY: the mapping is the BAR's length and lives as long as `bar`. + let regs = unsafe { Window::new(bar.as_ptr(), info.bar_bytes[0] as usize) }; + // One page granted, which is what starts the function mastering the bus. + let grant = dev.dma_alloc(4096).expect("usbd_spare: a grant"); + let op = usize::from(regs.read::(0)); + let db = (regs.read::(0x14) & !3) as usize; + let settle = |what: &str, done: &dyn Fn() -> bool| { + let began = Instant::now(); + while !done() { + assert!(began.elapsed() < Duration::from_secs(2), "usbd_spare: the controller {what}"); + } + }; + regs.write::(op, 0); + settle("never halted", &|| regs.read::(op + 4) & 1 != 0); + regs.write::(op, 1 << 1); + settle("held its reset", &|| regs.read::(op) & (1 << 1) == 0 && regs.read::(op + 4) & (1 << 11) == 0); + let aimed = grant.device_addr + PAST_THE_GRANT; + // The command ring there, the controller running, and its doorbell rung: + // its first fetch is a read of an address no grant maps. + regs.write::(op + 0x18, aimed | 1); + regs.write::(op, 1); + regs.write::(db, 0); + println!("usbd_spare: the command ring aimed at {aimed:#x}, past the grant at {:#x}", grant.device_addr); + + let poller = Poller::new(1); + poller.watch(&dev, READABLE, 0); + poller.wait(1, Duration::from_secs(10).as_nanos() as u64, |_| {}); + match dev.irq() { + Err(SyscallError::Io) => println!("usbd_spare: the claim refuses its interrupt read: Io"), + Ok(record) => panic!("usbd_spare: after the aimed fetch the claim answered a record of {}, not the unit's refusal", record.count), + Err(why) => panic!("usbd_spare: after the aimed fetch the claim answered {why:?}, not the unit's refusal"), + } +} + +pub fn refused(cap: &SysCap) { + let (id, claimed) = claim(cap); + match claimed { + Err(SyscallError::PermissionDenied) => println!("usbd_spare: the claim on {} was refused: PermissionDenied", spelled(id)), + Err(why) => panic!("usbd_spare: the claim was refused, by {why:?} and not because the kernel drives it"), + Ok(_) => panic!("usbd_spare: the claim on {} was granted, on a kernel that drives it", spelled(id)), + } +} + +/// The claim's MSI-X table entry 0, mapped through the BAR that holds it, and +/// its offset there, off the function's own capability chain (PCI 3.0 +/// §6.8.2: id 0x11, the table's BIR and offset in the dword at +4). Mapped +/// as far as that entry and no further, without the claim's description, +/// which is read once and is usbd's. +fn msix_table(dev: &PciDev) -> (toyos::shm::SharedMemory, usize) { + use toyos_abi::syscall::RegWidth; + let mut at = dev.config_read(0x34, RegWidth::U8).expect("usbd_spare: the capability pointer") & 0xFC; + for _ in 0..48 { + assert!(at != 0, "usbd_spare: the function has no MSI-X capability"); + let header = dev.config_read(at, RegWidth::U32).expect("usbd_spare: a capability header"); + if header & 0xFF == 0x11 { + let table = dev.config_read(at + 4, RegWidth::U32).expect("usbd_spare: the MSI-X table word"); + let offset = (table & !7) as usize; + let bar = dev + .map_bar(table & 7, offset as u64 + 16) + .unwrap_or_else(|why| panic!("usbd_spare: BAR {}, which holds the MSI-X table, was not handed over: {why:?}", table & 7)); + return (bar, offset); + } + at = (header >> 8) & 0xFC; + } + panic!("usbd_spare: the capability chain does not end"); +} + +pub fn vector(cap: &SysCap) { + let (id, claimed) = claim(cap); + let dev = claimed.unwrap_or_else(|why| panic!("usbd_spare: the claim on {} was refused: {why:?}", spelled(id))); + let (bar, table) = msix_table(&dev); + // SAFETY: the mapping is the BAR's length and lives as long as `bar`. + let regs = unsafe { Window::new(bar.as_ptr(), bar.len()) }; + // Entry 0's data word, at +8 of the entry (§6.8.2.6). + let data = regs.read::(table + 8); + let foreign = data ^ 1; + regs.write::(table + 8, foreign); + println!("usbd_spare: MSI-X entry 0's data {data:#x} rewritten to {foreign:#x}, a vector not the claim's"); + let usbd = Usbd::start(id, dev); + usbd.until("usbd: settled"); + let settled = usbd.ask("settled"); + let noop = settled.get("usb.noop").map(Value::to_string); + assert_ne!(noop.as_deref(), Some("completed"), "usbd_spare: usbd was told of its No-Op through a vector not its own"); + assert_eq!(number(&settled, "usb.interrupts.records"), 0, "usbd_spare: usbd read an interrupt record"); + usbd.kill(); + println!("usbd_spare: usbd settled and was never told of its No-Op (noop = {noop:?})"); +} + +pub fn refused_unremapped(cap: &SysCap) { + let (id, claimed) = claim(cap); + match claimed { + Err(why) => println!("usbd_spare: the claim on {} was refused: {why:?}", spelled(id)), + Ok(_) => panic!("usbd_spare: the claim on {} was granted, on a machine that remaps nothing", spelled(id)), + } +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 473f53a6d6..c48a95da94 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -197,6 +197,13 @@ const RUST_SKIP: &[&str] = &[ // boot's kernel does: `usbd_drives_the_spare` and the // `usbd_drives_the_type_c_controller` metal row run it. "usbd_spare", + // `usbd_spare`'s fault, foreign-vector and refusal arms, one binary each + // because a metal job takes no argument: the AMD bench's + // `amdvi_usbd_drives_its_own_entry` and + // `amdvi_claim_refused_without_remapping` rows run them. + "usbd_spare_fault", + "usbd_spare_vector", + "usbd_spare_refused_unremapped", // The same controller, handed to a holder that is killed and claimed // again at once, in the same boots as `usbd_spare`. "end_returns_claims", @@ -404,6 +411,10 @@ const MACHINE_TESTS: &[&str] = &[ // build has and which the AMD bench's controllers, their MSI-X tables in // their register BARs, cannot yet be claimed to show. "usbd_drives_the_spare_amdvi", + // The same machine with its AMD unit remapping nothing: the claim is + // refused before anything on the function changes. The AMD bench's arm is + // `claim_refused_without_remapping`'s. + "usbd_refused_without_amdvi_remapping", // A boot keyboard reporting more keys than its six slots name: the // kernel's driver reading a report a device delivered over xHCI has no host // build, and the T14 binds no USB keyboard. @@ -1005,6 +1016,51 @@ const METAL: &[(&str, metal::Metal)] = &[ "amdvi_remaps_the_stick_controller", metal::Metal { arms: TESTCASES, judge: |b| amdvi_remaps(&b[0].kernel()) }, ), + ( + // The AMD bench's Bluetooth controller's xHCI, `03:00.4`, left to a + // claim and driven by usbd through an entry of its own table, its + // MSI-X table in the register BAR it hands over; then aimed past its + // grant, recorded against the claim; then its holder rewriting its + // own table entry to a vector not its own, refused and recorded. + "amdvi_usbd_drives_its_own_entry", + metal::Metal { + arms: &[metal::once( + "testcases-amd-xhci-leave", + "tests/images/testcases.toml", + &["xhci-leave=03:00.4"], + &[ + "test_rs_usbd_spare", + "test_rs_usbd_spare_fault", + "test_rs_end_returns_claims", + "test_rs_usbd_spare_vector", + ], + )], + judge: |b| { + for job in ["test_rs_usbd_spare", "test_rs_usbd_spare_fault", "test_rs_end_returns_claims", "test_rs_usbd_spare_vector"] { + b[0].job_passed(job)?; + } + amdvi_usbd(&b[0].kernel(), &b[0].log()) + }, + }, + ), + ( + // The AMD bench with its unit remapping nothing (`iommu-no-remap`): + // the claim on `03:00.4` is refused before anything on it changes. + "amdvi_claim_refused_without_remapping", + metal::Metal { + arms: &[metal::once( + "testcases-amd-no-remap", + "tests/images/testcases.toml", + &["iommu-no-remap", "xhci-leave=03:00.4"], + &["test_rs_usbd_spare_refused_unremapped"], + )], + judge: |b| { + let kernel = b[0].kernel(); + kernel.must_say("no message is remapped")?; + claims::refused_unremapped_at(&kernel, AMD_XHCI_SPARE) + }, + }, + ), ( // A machine whose units do not remap: the claim is refused before // anything on the function changes, and the boot goes on. @@ -1030,7 +1086,7 @@ const METAL: &[(&str, metal::Metal)] = &[ arms: &[metal::once( "testcases-xhci-leave", "tests/images/testcases.toml", - &["xhci-leave=8086:9a13"], + &["xhci-leave=00:0d.0"], &["test_rs_usbd_spare", "test_rs_end_returns_claims"], )], judge: |b| { @@ -3671,6 +3727,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "hosted_clang_hello" => common::hostedclang::hosted_clang_hello(), "usbd_drives_the_spare" => usbd_drives_the_spare(test_config, qemu::Profile::HeadlessUsbSpare), "usbd_drives_the_spare_amdvi" => usbd_drives_the_spare(test_config, qemu::Profile::HeadlessUsbSpareAmdVi), + "usbd_refused_without_amdvi_remapping" => usbd_refused_without_remapping(test_config), "usb_keyboard_rollover" => usb_keyboard_rollover(test_config), "vector_state_survives_a_switch" => vector_state_survives_a_switch(test_config), "the_opening_exchange_with_qemus_agent_completes" => the_opening_exchange_with_qemus_agent_completes(), @@ -3725,6 +3782,13 @@ fn job_window(qemu: &mut QemuInstance, command: &str) -> Result Ok(result.serial) } +/// The AMD bench's Bluetooth controller's xHCI, the one `xhci-leave=` leaves +/// to a claim there: the stick is on the other, `03:00.3`. +const AMD_XHCI_SPARE: &str = "03:00.4"; + +/// QEMU's spare controller left to a claim, by the address the profile pins. +const XHCI_LEAVE_SPARE: &str = "xhci-leave=00:10.0"; + /// The job that kills a holder of the `xhci-leave=` controller and claims it /// again at once, in `usbd_spare`'s boots. const END_RETURNS_CLAIMS: &str = "end_returns_claims"; @@ -3759,7 +3823,7 @@ fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<( let ends = qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, END_RETURNS_CLAIMS); let options = BootOptions { profile, - kernel_params: &["xhci-leave=1b36:000d"], + kernel_params: &[XHCI_LEAVE_SPARE], ..Default::default() }; let mut qemu = QemuInstance::boot_with_options( @@ -3786,7 +3850,8 @@ fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<( if said.matches(&handed).count() != 1 { return Err(format!("{at} was not handed over once, to the one usbd:\n{said}")); } - run.must_say(&format!("PCI {at}: msi address="))?; + // MSI on the VT-d machine's spare, MSI-X on the AMD one's. + run.must_say(&format!("PCI {at}: msi"))?; let amdvi = matches!(profile.iommu(), Some(qemu::Unit::AmdVi { .. })); if amdvi { // The claim's entry in its own table, present at the vector the @@ -3820,6 +3885,9 @@ fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<( let ended = job_window(&mut qemu, "test_rs_end_returns_claims")?; let line = ends_returned_claims(&ended)?; eprintln!(" [usbd] {line}"); + if amdvi { + foreign_vector(&mut qemu, &at)?; + } drop(qemu); // The control: the same machine and no switch, so the kernel drives it. @@ -3837,6 +3905,100 @@ fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<( Ok(()) } +/// A machine whose AMD unit remaps nothing refuses the spare's claim by that +/// reason, before a vector is armed or a BAR moved. +fn usbd_refused_without_remapping(test_config: &Path) -> Result<(), String> { + const JOB: &str = "usbd_spare"; + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); + let options = BootOptions { + profile: qemu::Profile::HeadlessUsbSpareAmdViNoRemap, + kernel_params: &[XHCI_LEAVE_SPARE], + ..Default::default() + }; + let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[(JOB.to_string(), bin)], options); + let boot = serial::Serial::boot(&qemu); + boot.must_say("no message is remapped")?; + let said = job_window(&mut qemu, "test_rs_usbd_spare refused_unremapped")?; + claims::refused_unremapped_at(&serial::Serial::named("usbd_spare", said), qemu::SPARE_AT) +} + +/// The AMD bench's `03:00.4` left to a claim and driven by usbd: handed over +/// with its MSI-X table's BAR, armed through a present entry of its own table +/// at its claim's vector, its No-Op answered by that interrupt; then aimed +/// past its grant and recorded against the claim at the aimed address; then +/// its holder's rewrite of its own table entry to a vector not its own +/// refused and recorded against the claim, usbd never told of its No-Op. +fn amdvi_usbd(kernel: &serial::Serial, log: &serial::Serial) -> Result<(), String> { + const ID: &str = "1022:1639"; + let at = AMD_XHCI_SPARE; + kernel.must_say(&format!("xHCI: leaving PCI {at} {ID} to a claim (xhci-leave)"))?; + let handed = format!("pcidev: PCI {at} [{ID}] handed over, vector "); + let vector = kernel + .text() + .lines() + .find_map(|l| l.split_once(&handed)?.1.split_whitespace().next().map(str::to_string)) + .ok_or_else(|| format!("{at} was never handed over:\n{}", kernel.text()))?; + kernel.must_say(&format!("PCI {at}: msix address="))?; + kernel.must_say(&format!("iommu: amdvi irte source={at} vector={vector} p=1 "))?; + log.must_say(USBD_NOOP)?; + + let aimed = log + .text() + .lines() + .find_map(|l| l.split_once("usbd_spare: the command ring aimed at ")?.1.split_once(',').map(|(a, _)| a.to_string())) + .ok_or_else(|| format!("the fault arm never said where it aimed:\n{}", log.text()))?; + let hex = |text: &str| u64::from_str_radix(text.trim_start_matches("0x"), 16).ok(); + let records: Vec<&str> = kernel + .text() + .lines() + .filter(|l| l.contains("iommu: DMA FAULT owner=process") && l.contains(&format!("stream={at} "))) + .collect(); + let aimed_at = records + .iter() + .find(|l| l.split_once(" addr=").and_then(|(_, a)| hex(a.split_whitespace().next()?)) == hex(&aimed)) + .ok_or_else(|| format!("no record of {at} names the aimed {aimed}: {records:?}"))?; + log.must_say("usbd_spare: the claim refuses its interrupt read: Io")?; + + let rewritten = log.must_say("usbd_spare: MSI-X entry 0's data ")?; + log.must_say("usbd_spare: usbd settled and was never told of its No-Op")?; + // Refused by the unit, not delivered to a vector nobody holds: the stop's + // census names every vector the unclaimed gate took. + let foreign_vector = rewritten + .split_once(" rewritten to ") + .and_then(|(_, rest)| rest.split(',').next()) + .ok_or_else(|| format!("{rewritten:?} names no vector"))?; + let census = kernel.must_say("irq: unclaimed vectors")?; + if census.split_whitespace().any(|word| word == foreign_vector) { + return Err(format!("{foreign_vector} reached a CPU: {census}")); + } + // An interrupt's record states no read or write (48882 Table 57, `I`). + let foreign = records + .iter() + .find(|l| l.contains(" access=none ") && l.ends_with("IO_PAGE_FAULT")) + .ok_or_else(|| format!("no record of {at}'s message on a vector not its own: {records:?}"))?; + eprintln!(" [amdvi] {at}: handed over on {vector}, its No-Op answered through its own entry"); + eprintln!(" [amdvi] aimed at {aimed}: {}", aimed_at.trim()); + eprintln!(" [amdvi] a vector not its own: {}", foreign.trim()); + Ok(()) +} + +/// The claim's own MSI-X table entry rewritten by its holder to a vector not +/// the claim's: the unit refuses that message and records it against the +/// claim, and usbd is never told of its No-Op. The table is reachable only +/// because a machine that remaps hands over the BAR holding it. +fn foreign_vector(qemu: &mut QemuInstance, at: &str) -> Result<(), String> { + let said = job_window(qemu, "test_rs_usbd_spare vector")?; + let run = serial::Serial::named("usbd_spare vector", said); + let rewritten = run.must_say("usbd_spare: MSI-X entry 0's data ")?; + run.must_say("usbd_spare: usbd settled and was never told of its No-Op")?; + // QEMU 11.1.1's AMD unit drops the message and records nothing + // (`issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md`); + // the AMD bench's row holds the record. + run.must_not_say(&format!("DMA FAULT owner=process unit0 stream={at} "))?; + eprintln!(" [usbd] {}; usbd never told of its No-Op", rewritten.trim()); + Ok(()) +} + /// Every AMD-Vi unit translating, every xHCI controller the kernel drives /// moved to a domain of its own, the boot stick bound through one, and no /// DMA refused. From 4f1d6fc3aee42ec664713954b3f0d60f2491a56c Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 11:51:25 +0200 Subject: [PATCH 4/9] amdvi: SMI filter on every programmed unit, the table's BAR only where messages are fenced, and the build review's fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build review of #921 (6163050204) and the orchestrator's rulings (6163086612). - B1, ruled: every AMD-Vi unit with an SMI filter (EFR `SmiFSup` 01b) has it on and logging (CONTROL `SmiFEn`, `SmiFLogEn`; 48882 §2.1.5), its registers left as firmware set them and each said at boot. An SMI is no remapped message (Table 10, Table 19), so a holder's own MSI-X table could otherwise send one to every CPU. `Remapping::fences_tables` says whether a message written into a claim's table is fenced: VT-d and the ITS always, AMD-Vi only where every programmed unit filters. Where not, `pcidev` keeps the table's BAR back and says so by name. A refused SMI from a requester no claim holds is said and the machine goes on; a claimed function's is its holder's fault. - B2: a claim on a requester id an IVHD special entry uses (I/O APIC, HPET), or one the SMI filter passes, is refused at the claim (`iommu::isolates`, `unclaimable`); a claim asserts no other source wrote its requester's remapping table first. - B3: the release masks the MSI-X entry at an address from the BIR and offset captured at arming and the BAR address the kernel placed, never the device's answer; nothing there can panic on the device. - B4: the bench row reads the stop's census after the reset. - B5: QEMU's unit has no SMI filter, so under the ruling it hands no table over and the foreign-vector arm leaves QEMU; its spare is armed on MSI again. `usbd_table_kept_without_smi_filter` reads, on an MSI-X spare, the table's BAR kept back by name and the claim refused. The two issues' texts say what was measured. - B6: `device_bar_map`'s contract, the track issue's remaining work and exit, the guest-test registration, `iommu-no-remap`'s contract and the spare list's doc are true of the tree. - `FLASHABLE` rows for `xhci-leave=00:0d.0` and `xhci-leave=03:00.4`, the old id form gone; `iommu-no-remap`'s row covers AMD-Vi. - `usbd_spare`'s refusal arm is `unhanded`, for any refusal whose reason the kernel's line gives. One helper for an entry's interrupt half; the xHCI driver names the left controller by `pcidev::requester`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...in-bar-0-cannot-be-driven-from-userland.md | 32 ++++---- .../an-amd-vi-machine-isolates-no-device.md | 19 +++-- ...o-fault-for-a-message-its-table-refuses.md | 23 +++--- kernel/amdvi/src/queue.rs | 8 ++ kernel/amdvi/src/reg.rs | 41 +++++++++++ kernel/amdvi/tests/encodings.rs | 39 +++++++++- kernel/src/actuator.rs | 6 +- kernel/src/arch/aarch64/irqchip/its.rs | 7 ++ kernel/src/arch/aarch64/smmu/domain.rs | 5 ++ kernel/src/arch/x86_64/amdvi/domain.rs | 22 +++++- kernel/src/arch/x86_64/amdvi/fault.rs | 20 ++++- kernel/src/arch/x86_64/amdvi/interrupt.rs | 29 ++++++-- kernel/src/arch/x86_64/amdvi/mod.rs | 73 +++++++++++++++---- kernel/src/arch/x86_64/iommu_unit.rs | 22 ++++++ kernel/src/drivers/pci.rs | 30 +++++--- kernel/src/drivers/xhci/wait/boot.rs | 3 +- kernel/src/iommu/mod.rs | 20 ++++- kernel/src/pcidev/mod.rs | 62 ++++++++++------ sdk/abi/src/syscall.rs | 5 +- tests/common/qemu.rs | 21 ++++-- tests/guest/rust-tests/src/bin/usbd_spare.rs | 2 +- ...d_unremapped.rs => usbd_spare_unhanded.rs} | 4 +- tests/guest/rust-tests/src/usbd_spare.rs | 21 +++--- tests/metal/src/lib.rs | 27 ++++--- tests/toyos.rs | 72 ++++++++++-------- 25 files changed, 451 insertions(+), 162 deletions(-) rename tests/guest/rust-tests/src/bin/{usbd_spare_refused_unremapped.rs => usbd_spare_unhanded.rs} (55%) diff --git a/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md b/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md index 70d93f1e2f..b446088c64 100644 --- a/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md +++ b/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md @@ -6,23 +6,21 @@ opened: 2026-09-26 # A controller whose MSI-X table is in BAR 0 cannot be driven from userland -Held by PR #921 (AMD-Vi stage 6), under the orchestrator's ruling there: -on a machine where every unit remaps, `pcidev` hands over the BAR holding a -function's MSI-X table, because the unit's per-requester table is the guard -(a message the holder writes reaches only its own claim's interrupt, and -anything else is refused and recorded); where a unit does not remap, the -claim is refused anyway (`Refusal::NotRemapped`). +Held by PR #921 (AMD-Vi stage 6), under the orchestrator's rulings there: +where the machine fences a message written into a claim's MSI-X table, +`pcidev` hands over the BAR holding it; where it does not, that BAR is kept +back, said by name. VT-d fences by source-id verification and its block on +compatibility-format messages, the ITS by the device id the bridge stamps, +and AMD-Vi by the requester's own table together with every unit's SMI +filter, since an SMI is no remapped message (48882 Table 10). -A claim exists only on a machine that remaps, so `kernel/src/pcidev/mod.rs` -no longer keeps the table's BAR back. QEMU's `qemu-xhci` with its table in -its register BAR is claimed and driven behind the AMD unit -(`usbd_drives_the_spare_amdvi`), and its holder's rewrite of its own entry -to a vector not its own reaches nobody; with the BAR withheld as before, the -claim is refused `NoMappableBar`. +QEMU's AMD unit has no SMI filter, so `usbd_table_kept_without_smi_filter` +reads its spare's table BAR kept back and the claim refused +`NoMappableBar`. **Exit:** on the AMD bench, `03:00.4` (MSI-X table in BAR 0, its only BAR) -is claimed and driven by usbd through its own remapping entry, and its -holder's rewrite of its entry to a vector not its own is refused and -recorded against the claim (`amdvi_usbd_drives_its_own_entry`); and a -machine that does not remap still refuses the claim -(`amdvi_claim_refused_without_remapping`). +is claimed and driven by usbd through its own remapping entry with every +unit filtering SMIs, and its holder's rewrite of its entry to a vector not +its own is refused, recorded against the claim, and reaches no CPU +(`amdvi_usbd_drives_its_own_entry`); and a machine that does not remap still +refuses the claim (`amdvi_claim_refused_without_remapping`). diff --git a/issues/an-amd-vi-machine-isolates-no-device.md b/issues/an-amd-vi-machine-isolates-no-device.md index 36a8c713ad..8e527f9b29 100644 --- a/issues/an-amd-vi-machine-isolates-no-device.md +++ b/issues/an-amd-vi-machine-isolates-no-device.md @@ -16,10 +16,10 @@ unless it names a domain, and where every programmed unit has `GASup` (and fixed and arbitrated messages go through a table of its own indexed by vector, a claim's entry passing nothing firmware declared. -What remains is a claim on the AMD bench: both its xHCI controllers keep -their MSI-X table in BAR 0, their only BAR, so `pcidev` refuses to hand one -over (`issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md`), -and `xhci-leave=` names a controller by an id the two share. +What remains is the AMD bench's reading of a claim (PR #921 stages it): a +claimed `03:00.4` driven through its own entry, its MSI-X table's BAR handed +over because every unit filters SMIs, its holder's foreign vector refused +and recorded, and a machine that remaps nothing refusing the claim. Constraints a reader would otherwise pay to re-derive: @@ -35,7 +35,10 @@ Constraints a reader would otherwise pay to re-derive: suite's guests run in x2APIC, so a remapping guest needs both. - The AMD bench is in the metal suite; the orchestrator runs every boot. -**Exit:** a guest boot on `Profile::MetalAmdVi` and a run on the AMD bench -in which a kernel driver's DMA goes through a domain of its own, a DMA -outside it is recorded as a fault, and a claimed function's message is -delivered through its own remapping entry. +**Exit:** on the AMD bench, `amdvi_usbd_drives_its_own_entry` and +`amdvi_claim_refused_without_remapping` pass beside +`amdvi_translates_the_stick_controller` and +`amdvi_remaps_the_stick_controller`: a kernel driver's DMA through a domain +of its own, a DMA outside a claim's domain recorded, a claimed function's +message delivered through its own remapping entry and a foreign one refused +and recorded. diff --git a/issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md b/issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md index 4333d90acd..2bbc03ebdf 100644 --- a/issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md +++ b/issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md @@ -8,19 +8,20 @@ opened: 2026-10-14 AMD 48882 rev 3.11 §2.2.5.2, step 7: a message whose remapping entry has `RemapEn` clear is target-aborted and reported as an `IO_PAGE_FAULT` with `I` -set (Table 57). QEMU 11.1.1's `amd-iommu` (`intremap=on,xtsup=on`, 128-bit -entries) drops such a message and writes no event record: in -`usbd_drives_the_spare_amdvi`, a claim's holder rewrote its MSI-X entry from -the claim's vector 0x42 to 0x43, usbd's No-Op went unannounced ("the -bring-up's No-Op command: Silent"), and no `DMA FAULT` line followed (PR -#921's evidence, `g3` logs). +set (Table 57). Under QEMU 11.1.1's `amd-iommu` (`intremap=on,xtsup=on`, +128-bit entries), a claim's holder rewrote its MSI-X entry from the claim's +vector 0x42 to 0x43; usbd's No-Op went unannounced ("the bring-up's No-Op +command: Silent") and no event record was written (PR #921's evidence, `g3` +logs). Whether QEMU dropped the message or delivered it unremapped to 0x43, +which no claim held, was not measured. -So that test holds only that the message reached nobody; the AMD bench's -row `amdvi_usbd_drives_its_own_entry` holds the record, and that the vector -reached no CPU by the stop's census. +QEMU's unit has no SMI filter (`SmiFSup` 0), so under the ruling on #921 it +hands no table's BAR over and the arm no longer runs there; the AMD bench's +`amdvi_usbd_drives_its_own_entry` holds the record and the census. Owner: the orchestrator's AMD laptop track (`issues/an-amd-vi-machine-isolates-no-device.md`). -**Exit:** under the QEMU `build/ci/qemu-version` names, that rewrite is -recorded against the claim, and the test asserts the record. +**Exit:** under the QEMU `build/ci/qemu-version` names, a message on an +absent remapping entry is recorded against its requester, and a guest test +asserts the record. diff --git a/kernel/amdvi/src/queue.rs b/kernel/amdvi/src/queue.rs index 90f1918105..197700a2e8 100644 --- a/kernel/amdvi/src/queue.rs +++ b/kernel/amdvi/src/queue.rs @@ -70,6 +70,14 @@ pub const fn event(words: [u64; 2]) -> Event { Event::Requester { code, device, address: words[1], write } } +/// Whether a record's address is an SMI's (§2.1.5.2, Figure 3 and Table 4): +/// bits 47:40 zero, `FDF8h` at 39:24, `MT` 0010b at 7 and 4:2, `DM` and +/// bits 1:0 zero. The SMI filter records a refused SMI as an +/// `IO_PAGE_FAULT` at that address (§2.1.5.1). +pub const fn smi(address: u64) -> bool { + address & 0xFFFF_FFFF_FF00_00C3 == 0x0000_00FD_F800_0000 && (address >> 2) & 0b111 == 0b010 +} + /// The name 48882 gives an event code, for a record's line. pub const fn name(code: u8) -> &'static str { match code { diff --git a/kernel/amdvi/src/reg.rs b/kernel/amdvi/src/reg.rs index 8bbe26596e..0a7de20528 100644 --- a/kernel/amdvi/src/reg.rs +++ b/kernel/amdvi/src/reg.rs @@ -17,6 +17,28 @@ pub const COMMAND_TAIL: u64 = 0x2008; pub const EVENT_HEAD: u64 = 0x2010; pub const EVENT_TAIL: u64 = 0x2018; pub const STATUS: u64 = 0x2020; +/// The first SMI filter register (MMIO 0060h–00D8h, Table 80), 8 bytes each. +pub const SMI_FILTER: u64 = 0x0060; + +/// One SMI filter register (MMIO 00[60–D8]h): `SmiDID` 15:0, `SmiDV` 16, +/// `SmiFLock` 17. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct SmiFilter(pub u64); + +impl SmiFilter { + /// The requester id an SMI passes from, where the register is valid. + pub const fn passes(self) -> Option { + if self.0 & 1 << 16 != 0 { + Some(self.0 as u16) + } else { + None + } + } + + pub const fn locked(self) -> bool { + self.0 & 1 << 17 != 0 + } +} /// `CONTROL` (MMIO 0018h). pub mod control { @@ -28,6 +50,12 @@ pub mod control { pub const CMD_BUF_EN: u64 = 1 << 12; pub const PPR_LOG_EN: u64 = 1 << 13; pub const PPR_INT_EN: u64 = 1 << 14; + /// Upstream SMIs pass only from the ids the SMI filter registers name + /// (§2.1.5.1). + pub const SMIF_EN: u64 = 1 << 22; + /// An SMI the filter refuses is written to the event log as an + /// `IO_PAGE_FAULT` at its SMI address. + pub const SMIF_LOG_EN: u64 = 1 << 24; /// The 128-bit remapping entry (Table 21), with `GAMEn` 000b. pub const GA_EN: u64 = 1 << 17; pub const GA_LOG_EN: u64 = 1 << 28; @@ -68,6 +96,19 @@ impl Features { self.0 & 1 << 7 != 0 } + /// `SmiFSup` 01b: the SMI filter exists, with this many registers + /// (`SmiFRC`: 1, 2, 4, 8 or 16); `None` where SMIs always pass, or for a + /// reserved encoding of either field. + pub const fn smi_filters(self) -> Option { + if (self.0 >> 16) & 0b11 != 0b01 { + return None; + } + match (self.0 >> 18) & 0b111 { + count @ 0..=4 => Some(1 << count), + _ => None, + } + } + /// `HATS`, the host table levels the unit walks: 4, 5 or 6, or `None` /// for the reserved encoding. pub const fn host_levels(self) -> Option { diff --git a/kernel/amdvi/tests/encodings.rs b/kernel/amdvi/tests/encodings.rs index be7808bac2..dbd11a7376 100644 --- a/kernel/amdvi/tests/encodings.rs +++ b/kernel/amdvi/tests/encodings.rs @@ -5,8 +5,8 @@ use toyos_amdvi::dte::{interrupts_blocked, interrupts_remapped, Dte, Setting, DEVICE_IDS, ENTRY_BYTES, INTERRUPT_ENTRIES}; use toyos_amdvi::irte::{self, Destination}; -use toyos_amdvi::queue::{event, name, Command, Event, ENTRIES}; -use toyos_amdvi::reg::{control, device_table, offset, ring, slot, Features}; +use toyos_amdvi::queue::{event, name, smi, Command, Event, ENTRIES}; +use toyos_amdvi::reg::{control, device_table, offset, ring, slot, Features, SmiFilter, SMI_FILTER}; use toyos_amdvi::table::{directory, entry, index, leaf, Entry, LEAF_LEVEL, ROOT_LEVEL}; use toyos_phys::Phys; @@ -260,3 +260,38 @@ fn the_remapping_fields_sit_where_48882_puts_them() { assert!(!Features(!word(&[(7, 7, 1)])).guest_apic()); assert!(!Features(!word(&[(2, 2, 1)])).x2apic()); } + +/// MMIO 0018h `SmiFEn` [22], `SmiFLogEn` [24]; MMIO 0030h `SmiFSup` [17:16] +/// 01b and `SmiFRC` [20:18] counting 1, 2, 4, 8, 16; the filter registers at +/// 0060h, `SmiDID` [15:0], `SmiDV` [16], `SmiFLock` [17]. The bench's EFR +/// reads `SmiFSup` 01b and `SmiFRC` 001b: two registers. +#[test] +fn the_smi_filter_sits_where_48882_puts_it() { + assert_eq!((control::SMIF_EN, control::SMIF_LOG_EN), (word(&[(22, 22, 1)]), word(&[(24, 24, 1)]))); + assert_eq!(Features(0x206d_73ef_2225_4ade).smi_filters(), Some(2)); + assert_eq!(Features(word(&[(17, 16, 0b01), (20, 18, 0b100)])).smi_filters(), Some(16)); + assert_eq!(Features(word(&[(17, 16, 0b01), (20, 18, 0b101)])).smi_filters(), None); + assert_eq!(Features(word(&[(17, 16, 0b00), (20, 18, 0b001)])).smi_filters(), None); + assert_eq!(Features(word(&[(17, 16, 0b10)])).smi_filters(), None); + assert_eq!(SMI_FILTER, 0x60); + let valid = SmiFilter(word(&[(15, 0, 0x00A3), (16, 16, 1), (17, 17, 1)])); + assert_eq!((valid.passes(), valid.locked()), (Some(0x00A3), true)); + let unset = SmiFilter(word(&[(15, 0, 0x00A3)])); + assert_eq!((unset.passes(), unset.locked()), (None, false)); +} + +/// Figure 3 / Table 4: an SMI's address is `FDF8h` at [39:24], `MT[2:0]` +/// 010b at [4:2], `MT[3]` [7] and `DM` [6] zero, [1:0] zero, [47:40] zero; +/// the vector [23:16], destination [15:8] and `TM` [5] are ignored. +#[test] +fn an_smi_address_is_told_from_a_dma_address() { + let smi_at = word(&[(39, 24, 0xFDF8), (4, 2, 0b010)]); + assert!(smi(smi_at)); + assert!(smi(smi_at | word(&[(23, 16, 0xFF), (15, 8, 0xFF), (5, 5, 1)]))); + assert!(!smi(word(&[(39, 24, 0xFDF8), (4, 2, 0b000)]))); + assert!(!smi(smi_at | word(&[(7, 7, 1)]))); + assert!(!smi(smi_at | word(&[(6, 6, 1)]))); + assert!(!smi(smi_at | word(&[(40, 40, 1)]))); + assert!(!smi(0x4000_0000_0000)); + assert!(!smi(0xFEE0_0008)); +} diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index c2171e66b9..4e0f0a0cc6 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -137,8 +137,10 @@ actuators! { /// `amdvi_selftest`. amdvi_selftest = "amdvi-selftest"; - /// Read the DMAR's flags with `INTR_REMAP` clear, as on a platform whose - /// firmware says it does not remap. Judged by `claim_refused_without_remapping`. + /// Remap no message, as on a platform that cannot: VT-d reads the DMAR's + /// flags with `INTR_REMAP` clear, and AMD-Vi leaves every interrupt half + /// invalid. Judged by `claim_refused_without_remapping` and + /// `amdvi_claim_refused_without_remapping`. iommu_no_remap = "iommu-no-remap"; /// Leave every AP holding the CR0/CR4 that INIT left it. diff --git a/kernel/src/arch/aarch64/irqchip/its.rs b/kernel/src/arch/aarch64/irqchip/its.rs index 162fdba89b..ceee85a1d7 100644 --- a/kernel/src/arch/aarch64/irqchip/its.rs +++ b/kernel/src/arch/aarch64/irqchip/its.rs @@ -373,6 +373,13 @@ pub(in crate::arch::aarch64) fn doorbell_page() -> Option { } /// Whether an ITS is armed to give a claimed function its own LPI. +/// Whether a message a holder writes into its function's MSI-X table reaches +/// nothing but its claim's interrupt: always, since the ITS translates by the +/// device id the bridge stamps, through that device's own table. +pub fn fences_tables() -> bool { + true +} + pub fn is_armed() -> bool { ITS.lock().is_some() } diff --git a/kernel/src/arch/aarch64/smmu/domain.rs b/kernel/src/arch/aarch64/smmu/domain.rs index 6860760f02..710c7127b0 100644 --- a/kernel/src/arch/aarch64/smmu/domain.rs +++ b/kernel/src/arch/aarch64/smmu/domain.rs @@ -92,6 +92,11 @@ pub fn untranslated(_function: StreamId) -> Option<&'static str> { None } +/// Why no claim may have `function`: never, as [`untranslated`]. +pub fn unclaimable(_function: StreamId) -> Option<&'static str> { + None +} + /// A new domain, with `room` bytes of it handed out first: refused, with no id /// spent, where it would have less than that. pub fn create(room: u64) -> Result<(DomainId, Iova), IommuError> { diff --git a/kernel/src/arch/x86_64/amdvi/domain.rs b/kernel/src/arch/x86_64/amdvi/domain.rs index ad37aa9ce6..680d900639 100644 --- a/kernel/src/arch/x86_64/amdvi/domain.rs +++ b/kernel/src/arch/x86_64/amdvi/domain.rs @@ -49,6 +49,8 @@ struct Amd { kept: Vec, /// The ids a programmed unit serves, as inclusive ranges. translated: Vec<(u16, u16)>, + /// Requester ids the platform speaks as, and why no claim may have one. + platform: Vec<(u16, &'static str)>, } static DOMAINS: Lock> = Lock::new(None); @@ -56,10 +58,26 @@ static DOMAINS: Lock> = Lock::new(None); /// Before any domain is made, once a unit will translate: every one is built /// clear of `reserved`, `kept`'s requesters stay on the identity domain, and /// only a requester in `translated` is given one. -pub(super) fn start(reserved: Vec<(u64, u64)>, settings: Vec, kept: Vec, translated: Vec<(u16, u16)>) { +pub(super) fn start( + reserved: Vec<(u64, u64)>, + settings: Vec, + kept: Vec, + translated: Vec<(u16, u16)>, + platform: Vec<(u16, &'static str)>, +) { let mut domains = Domains::new(IDENTITY + 1); domains.avoid(reserved); - *DOMAINS.lock() = Some(Amd { domains, settings, kept, translated }); + *DOMAINS.lock() = Some(Amd { domains, settings, kept, translated, platform }); +} + +/// Why no claim may have `stream`, if none may: [`untranslated`]'s reasons, +/// and a requester id the platform speaks as, an I/O APIC's, an HPET's, or +/// one whose SMIs the unit's filter passes. +pub fn unclaimable(stream: StreamId) -> Option<&'static str> { + untranslated(stream).or_else(|| { + let id = stream.requester(); + DOMAINS.lock().as_ref()?.platform.iter().find(|(platform, _)| *platform == id).map(|(_, why)| *why) + }) } /// What requester `id`'s entry carries of firmware's declaration. diff --git a/kernel/src/arch/x86_64/amdvi/fault.rs b/kernel/src/arch/x86_64/amdvi/fault.rs index a1ccfd5f92..d9195de5c1 100644 --- a/kernel/src/arch/x86_64/amdvi/fault.rs +++ b/kernel/src/arch/x86_64/amdvi/fault.rs @@ -14,7 +14,7 @@ use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use crate::iommu::fault::{self as policy, Access, Fault, Who}; use crate::log; use crate::mm::{DirectMap, Mmio, PAGE_SIZE}; -use toyos_amdvi::queue::{event, name, Event, ENTRIES}; +use toyos_amdvi::queue::{event, name, smi, Event, ENTRIES}; use toyos_amdvi::reg::{self, control, status}; use super::{requester, spin_until, REGISTER_WINDOW}; @@ -27,10 +27,12 @@ struct FaultUnit { log: AtomicU64, /// Faults this unit has reported for the life of the boot. faults: AtomicU32, + /// SMIs its filter refused from a requester no claim holds. + smis: AtomicU32, } static UNITS: [FaultUnit; MAX_UNITS] = - [const { FaultUnit { regs: AtomicU64::new(0), log: AtomicU64::new(0), faults: AtomicU32::new(0) } }; MAX_UNITS]; + [const { FaultUnit { regs: AtomicU64::new(0), log: AtomicU64::new(0), faults: AtomicU32::new(0), smis: AtomicU32::new(0) } }; MAX_UNITS]; /// Publish unit `index`'s registers and event log to the handler, before its /// interrupt is armed. @@ -75,6 +77,20 @@ fn drain(index: usize, regs: Mmio, log: Mmio) -> usize { let at = reg::offset(head); let words = [log.read_u64(at), log.read_u64(at + 8)]; let fault = match event(words) { + // An SMI the filter refused (§2.1.5.1): a claimed function's is + // its holder's fault; any other's is said and the machine goes + // on, since no DMA of this kernel's went wrong. + Event::Requester { device, address, .. } if smi(address) && !crate::pcidev::note_fault(device) => { + let refused = UNITS[index].smis.fetch_add(1, Ordering::Relaxed) + 1; + if refused.is_power_of_two() { + log!( + "iommu: amdvi unit{index} SMI filter refused an SMI from {} at {address:#018x}, {refused} so far", + requester(device) + ); + } + head = (head + 1) % ENTRIES; + continue; + } Event::Requester { code, device, address, write } => Fault { who: Who::Function(requester(device)), address, diff --git a/kernel/src/arch/x86_64/amdvi/interrupt.rs b/kernel/src/arch/x86_64/amdvi/interrupt.rs index 78a5f33cdb..8c19f8577f 100644 --- a/kernel/src/arch/x86_64/amdvi/interrupt.rs +++ b/kernel/src/arch/x86_64/amdvi/interrupt.rs @@ -30,7 +30,7 @@ use super::{domain, memory, submit, DEVICE_TABLE, TABLES}; use crate::iommu::StreamId; use crate::log; use crate::sync::Lock; -use core::sync::atomic::Ordering; +use core::sync::atomic::{AtomicBool, Ordering}; struct Remap { /// Whether every unit remaps, decided once at boot: `None` leaves every @@ -44,6 +44,19 @@ struct Remap { static REMAP: Lock = Lock::new(Remap { extended: None, tables: BTreeMap::new(), apics: Vec::new() }); +/// Every programmed unit filters SMIs (§2.1.5): an SMI is no remapped message +/// (Table 10), so without the filter a holder's own MSI-X table could send +/// one to every CPU. +static SMI_FILTERED: AtomicBool = AtomicBool::new(false); + +pub(super) fn smi_filtered(everywhere: bool) { + SMI_FILTERED.store(everywhere, Ordering::Release); +} + +pub fn smi_filtered_everywhere() -> bool { + SMI_FILTERED.load(Ordering::Acquire) +} + /// Every unit remaps, with an entry's destination 32 bits wide where /// `extended`; the I/O APICs speak as `apics`. pub(super) fn start(extended: bool, apics: Vec<(u8, u16)>) { @@ -57,7 +70,7 @@ pub fn is_armed() -> bool { } /// The interrupt half of requester `id`'s entry, bits 191:128. -fn interrupt_half(id: u16) -> crate::mm::Mmio { +pub(super) fn interrupt_half(id: u16) -> crate::mm::Mmio { let table = DEVICE_TABLE.load(Ordering::Acquire); memory(table + u64::from(id) * ENTRY_BYTES as u64 + 16, 8) } @@ -113,7 +126,7 @@ fn report(source: StreamId, vector: u8, read: [u64; 2]) { } /// `source`'s entry at `vector`, delivering it to this kernel's CPU. -fn arm(source: StreamId, vector: u8) -> Result<(), Refused> { +fn arm(source: StreamId, vector: u8) { let mut remap = REMAP.lock(); let destination = match remap.extended { Some(true) => Destination::Extended(MSG_DEST), @@ -122,7 +135,6 @@ fn arm(source: StreamId, vector: u8) -> Result<(), Refused> { let read = remap.write(source.requester(), vector, irte::entry(vector, destination)); drop(remap); report(source, vector, read); - Ok(()) } /// The message a kernel driver's `source` raises `vector` with: its own @@ -130,7 +142,7 @@ fn arm(source: StreamId, vector: u8) -> Result<(), Refused> { /// remaps. pub(in crate::arch::x86_64) fn msi(source: StreamId, vector: u8) -> Result { if is_armed() { - arm(source, vector)?; + arm(source, vector); } let (address, data) = msi_message(vector); Ok(Msi { address, data }) @@ -139,6 +151,13 @@ pub(in crate::arch::x86_64) fn msi(source: StreamId, vector: u8) -> Result Result { + // A requester the platform or a kernel driver speaks as is refused + // before here (`iommu::isolates`); one that already has a table is one + // of those, and its entries keep firmware's pass bits. + assert!( + !REMAP.lock().tables.contains_key(&source.requester()) || domain::setting(source.requester()) == toyos_amdvi::dte::Setting::CLAIMED, + "amdvi: a claim on {source}, whose remapping table another source already wrote" + ); domain::claimed(source.requester()); msi(source, vector) } diff --git a/kernel/src/arch/x86_64/amdvi/mod.rs b/kernel/src/arch/x86_64/amdvi/mod.rs index b1cff9f8cf..3b9e293540 100644 --- a/kernel/src/arch/x86_64/amdvi/mod.rs +++ b/kernel/src/arch/x86_64/amdvi/mod.rs @@ -34,7 +34,7 @@ use crate::time::{Duration, Tripwire}; use toyos_abi::boot::RootBridgeWindow; use toyos_amdvi::dte::{interrupts_blocked, Dte, Setting, DEVICE_IDS, ENTRY_BYTES}; use toyos_amdvi::queue::{Command, ENTRIES}; -use toyos_amdvi::reg::{self, control, status, Features}; +use toyos_amdvi::reg::{self, control, status, Features, SmiFilter}; use toyos_phys::Phys as Address; use super::iommu_unit::{register_window, yn, MAX_UNITS}; @@ -62,6 +62,12 @@ const FIRMWARE_LEFT: [(u64, Option, &str); 8] = [ (control::IOMMU_EN, None, "IommuEn"), ]; +/// Why a requester an IVHD special entry names takes no claim: an I/O APIC's +/// or an HPET's messages are the platform's (Table 109). +const PLATFORM: [&str; 2] = ["the IVRS names it as an I/O APIC", "the IVRS names it as an HPET"]; +/// Why a requester the SMI filter passes takes no claim. +const SMI_SOURCE: &str = "the unit's SMI filter, as firmware left it, passes its SMIs"; + /// IVMD flags (Table 113). const IVMD_UNITY: u8 = 1 << 0; const IVMD_EXCLUSION: u8 = 1 << 3; @@ -132,6 +138,8 @@ struct Ready { regs: Mmio, function: PciDevice, features: Features, + /// The unit has an SMI filter, which [`program`] turns on. + filters_smi: bool, } /// Every unit the IVRS describes, switched off, and each this kernel can use @@ -168,6 +176,8 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) let mut translated: Vec<(u16, u16)> = Vec::new(); // The I/O APICs a unit that will be programmed names, by MADT id. let mut apics: Vec<(u8, u16)> = Vec::new(); + // Requester ids no claim may have: the platform speaks as them. + let mut platform: Vec<(u16, &'static str)> = Vec::new(); let mut units = 0usize; for block in ivrs.blocks() { match block { @@ -180,14 +190,15 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) let function = describe_function(units, &unit, devices); let mut usable = Ok(()); let mut serves: Vec<(u16, u16)> = Vec::new(); - let mut specials: Vec<(u8, u16)> = Vec::new(); + let mut specials: Vec<(u8, u8, u16)> = Vec::new(); for entry in ivrs.devices(&unit) { match entry { Ok(entry) => { log!("iommu: amdvi unit{units} serves {}", served(&ivrs, &entry)); - // Variety 01h: the handle is an I/O APIC's MADT id (Table 109). - if let DeviceEntry::Special { handle, used, variety: 1, .. } = entry { - specials.push((handle, used)); + // Variety 01h: the handle is an I/O APIC's MADT id; + // 02h, an HPET's number (Table 109). + if let DeviceEntry::Special { handle, used, variety, .. } = entry { + specials.push((variety, handle, used)); } match declare(&entry, &mut settings, enumerated) { Ok(ids) => serves.extend(ids), @@ -206,9 +217,13 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) match usable.and_then(|()| usable_unit(&unit, regs, function)) { Ok(function) => { translated.extend(serves); - apics.extend(specials); + apics.extend(specials.iter().filter(|(variety, ..)| *variety == 1).map(|&(_, handle, used)| (handle, used))); + platform.extend(specials.iter().map(|&(variety, _, used)| (used, PLATFORM[usize::from(variety == 2)]))); let features = Features(regs.read_u64(reg::EXTENDED_FEATURES)); - ready.push(Ready { index: units, base: unit.base, regs, function, features }); + let passes = smi_filter(units, regs, features); + let filters_smi = passes.is_some(); + platform.extend(passes.into_iter().flatten().map(|id| (id, SMI_SOURCE))); + ready.push(Ready { index: units, base: unit.base, regs, function, features, filters_smi }); } Err(why) => log!("iommu: amdvi unit{units} is left off: {why}"), } @@ -287,14 +302,17 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) }; let identity = build(devices, &settings, remap.is_some()); let reserved = windows.iter().map(|w| (w.base, w.end())).chain(ivmds.iter().map(|m| (m.start, m.start + m.length))); - domain::start(reserved.collect(), settings, ivmds.iter().map(|m| m.requesters).collect(), translated); + let filters_smi = ready.iter().all(|unit| unit.filters_smi); + domain::start(reserved.collect(), settings, ivmds.iter().map(|m| m.requesters).collect(), translated, platform); + interrupt::smi_filtered(filters_smi); log!("iommu: amdvi identity domain covers 0x0..{top:#x}, root={:#x}", identity.get()); if let Some(extended) = remap { log!( "iommu: amdvi remaps every message through its requester's table: 128-bit entries, x2apic \ - destinations={}, {} I/O APIC(s) named", + destinations={}, {} I/O APIC(s) named, SMIs filtered={}", yn(extended), - apics.len() + apics.len(), + yn(filters_smi) ); interrupt::start(extended, apics); } @@ -343,6 +361,30 @@ fn declare( Ok(Some((first, last))) } +/// The requester ids the unit's SMI filter passes, as firmware left it, each +/// register said; `None`, said, where the unit has no filter, so every SMI +/// passes (§2.1.5). +fn smi_filter(index: usize, regs: Mmio, features: Features) -> Option> { + let Some(count) = features.smi_filters() else { + log!("iommu: amdvi unit{index} has no SMI filter (EFR {:#018x}), so every SMI a device sends passes", features.0); + return None; + }; + let registers: Vec = + (0..count as u64).map(|i| SmiFilter(regs.read_u64(reg::SMI_FILTER + 8 * i))).collect(); + for (i, register) in registers.iter().enumerate() { + log!( + "iommu: amdvi unit{index} SMI filter {i} = {:#010x}: passes {}, {}", + register.0, + match register.passes() { + Some(id) => format!("{}", requester(id)), + None => String::from("nothing"), + }, + if register.locked() { "locked" } else { "unlocked" } + ); + } + Some(registers.iter().filter_map(|register| register.passes()).collect()) +} + /// The unit's function, where this driver can program the unit: segment 0, /// `INVALIDATE_IOMMU_ALL`, a host table depth it can name, and a function /// with an MSI capability to report faults on. @@ -381,8 +423,7 @@ fn build(devices: &[PciDevice], settings: &[Setting], remap: bool) -> Address<12 // Where every unit remaps, every requester's fixed and arbitrated // messages are aborted until an entry of its own is written. if remap { - memory(DEVICE_TABLE.load(Ordering::Acquire) + u64::from(id) * ENTRY_BYTES as u64 + 16, 8) - .write_u64(0, interrupts_blocked(*setting)); + interrupt::interrupt_half(id).write_u64(0, interrupts_blocked(*setting)); } } let root = tables.alloc(); @@ -461,7 +502,7 @@ fn submit(commands: &[Command]) { /// then empties every cache it holds and names each enumerated /// function's entry. fn program(unit: Ready, devices: &[PciDevice], remap: Option) { - let Ready { index, base, regs, function, .. } = unit; + let Ready { index, base, regs, function, filters_smi, .. } = unit; let (commands, events, done) = { let mut tables = TABLES.lock(); (tables.alloc(), tables.alloc(), tables.alloc()) @@ -488,12 +529,16 @@ fn program(unit: Ready, devices: &[PciDevice], remap: Option) { Some(false) => control::GA_EN, Some(true) => control::GA_EN | control::XT_EN, }; + // An SMI is no remapped message (Table 10, Table 19): only the filter + // keeps one a holder writes into its own MSI-X table from every CPU. + let smi = if filters_smi { control::SMIF_EN | control::SMIF_LOG_EN } else { 0 }; let on = control::COHERENT | control::EVENT_LOG_EN | control::EVENT_INT_EN | control::CMD_BUF_EN | control::IOMMU_EN - | remapping; + | remapping + | smi; regs.write_u64(reg::CONTROL, on); let running = status::CMD_BUF_RUN | status::EVENT_LOG_RUN; spin_until( diff --git a/kernel/src/arch/x86_64/iommu_unit.rs b/kernel/src/arch/x86_64/iommu_unit.rs index 9d391124b3..84a001e239 100644 --- a/kernel/src/arch/x86_64/iommu_unit.rs +++ b/kernel/src/arch/x86_64/iommu_unit.rs @@ -145,6 +145,16 @@ pub mod domain { } } + /// Why no claim may have `stream`, if none may: [`untranslated`]'s + /// reasons and the requester ids AMD-Vi's platform speaks as. VT-d + /// answers `None`. + pub fn unclaimable(stream: StreamId) -> Option<&'static str> { + match backend() { + Backend::AmdVi => amdvi::domain::unclaimable(stream), + Backend::Vtd | Backend::None => None, + } + } + /// The backend that minted a `DomainId`; there is one, or none was minted. fn minted() -> Backend { let found = backend(); @@ -275,6 +285,18 @@ pub mod interrupt { } } + /// Whether a message a holder writes into its function's MSI-X table + /// reaches nothing but its claim's interrupt: VT-d's source-id + /// verification and compatibility-format block say so; AMD-Vi's table + /// does only where every unit filters SMIs, which no table remaps. + pub fn fences_tables() -> bool { + match backend() { + Backend::Vtd => true, + Backend::AmdVi => amdvi::interrupt::smi_filtered_everywhere(), + Backend::None => false, + } + } + pub fn is_armed() -> bool { match backend() { Backend::Vtd => vtd::interrupt::is_armed(), diff --git a/kernel/src/drivers/pci.rs b/kernel/src/drivers/pci.rs index 2809bc3e68..9e2a1ba93e 100644 --- a/kernel/src/drivers/pci.rs +++ b/kernel/src/drivers/pci.rs @@ -49,6 +49,8 @@ pub(in crate::drivers) struct MsixEntry<'a> { cap: Capability<'a>, control: u16, address: u64, + bir: u8, + base: u64, } pub struct Capability<'a> { @@ -307,7 +309,7 @@ impl PciDevice { } }; match table.table_address(base) { - Ok(address) => Ok(MsixEntry { cap, control, address }), + Ok(address) => Ok(MsixEntry { cap, control, address, bir: table.bir(), base }), Err(why) => { log!("PCI {:02x}:{:02x}.{}: MSI-X not armed, {}", self.bus, self.dev, self.func, why); @@ -492,21 +494,29 @@ impl PciDevice { /// for it and /// enable it. Takes the message and nothing else: the function is /// the one the entry was written for, and no other message reaches here. -pub fn arm_claimed_msix(message: &crate::iommu::Remapped) -> Result<(), NoEntry> { +pub fn arm_claimed_msix(message: &crate::iommu::Remapped) -> Result { let pci = message.function(); let entry = pci.msix_entry()?; + let at = MsixAt { bar: entry.bir, offset: entry.address - entry.base, base: entry.base }; pci.write_msix(entry, message.address(), message.data()); - Ok(()) + Ok(at) } -/// Mask a claimed function's [`MSIX_ENTRY`] where its table is now: the BAR -/// holding it may have moved since it was armed. -pub fn mask_claimed_msix(pci: &PciDevice) -> Result<(), NoEntry> { - let entry = pci.msix_entry()?; - let at = entry.address + MSIX_ENTRY as u64 * msix::ENTRY_BYTES; - crate::mm::paging::map_mmio(at, 0x1000, MmioPolicy::Uncacheable) +/// Where a claimed function's armed [`MSIX_ENTRY`] is, read once at arming: +/// its table's BAR, the table's offset in it, and where that BAR was then. +#[derive(Clone, Copy)] +pub struct MsixAt { + pub bar: u8, + pub offset: u64, + pub base: u64, +} + +/// Mask the armed [`MSIX_ENTRY`] of a table at `table`, an address the +/// kernel placed or read at arming and never a device's later answer: a +/// device gone or answering ones takes the write as nothing. +pub fn mask_claimed_msix(table: u64) { + crate::mm::paging::map_mmio(table + MSIX_ENTRY as u64 * msix::ENTRY_BYTES, 0x1000, MmioPolicy::Uncacheable) .write_u32(msix::ENTRY_VECTOR_CONTROL, msix::ENTRY_MASKED); - Ok(()) } /// [`arm_claimed_msix`] for a function with MSI and no MSI-X; `false` where it diff --git a/kernel/src/drivers/xhci/wait/boot.rs b/kernel/src/drivers/xhci/wait/boot.rs index 080bed0178..423babdc85 100644 --- a/kernel/src/drivers/xhci/wait/boot.rs +++ b/kernel/src/drivers/xhci/wait/boot.rs @@ -117,8 +117,7 @@ pub fn init(devices: &[PciDevice]) { for pci_dev in devices.iter().filter(|d| d.matches_class(0x0C, 0x03, Some(0x30))) { // Before anything that touches the function, `enable_bus_master`'s // record of it as this kernel's above all: that is what refuses a claim. - let at = u16::from(pci_dev.bus) << 8 | u16::from(pci_dev.dev) << 3 | u16::from(pci_dev.func); - if crate::actuator::xhci_left() == Some(at) { + if crate::actuator::xhci_left() == Some(crate::pcidev::requester(pci_dev)) { log!("xHCI: leaving PCI {:02x}:{:02x}.{} {:04x}:{:04x} to a claim (xhci-leave)", pci_dev.bus, pci_dev.dev, pci_dev.func, pci_dev.vendor_id(), pci_dev.device_id()); continue; diff --git a/kernel/src/iommu/mod.rs b/kernel/src/iommu/mod.rs index d097afaea6..9304657955 100644 --- a/kernel/src/iommu/mod.rs +++ b/kernel/src/iommu/mod.rs @@ -299,12 +299,27 @@ pub struct MsiMessage { #[derive(Clone, Copy)] pub struct Remapping(()); +impl Remapping { + /// Whether a message a holder writes into its function's MSI-X table can + /// reach nothing but that claim's own interrupt, so the BAR holding the + /// table may be handed over: on VT-d by source-id verification, on the + /// ITS by the device id the bridge stamps, and on AMD-Vi by the + /// requester's table where every unit also filters SMIs, which no table + /// remaps. + pub fn fences_tables(self) -> bool { + messages::fences_tables() + } +} + /// Nothing on this machine translates a claimed function's message to its /// claim's own interrupt, so the message it would carry names any. pub struct NotRemapped; /// Whether a claimed function's message can be remapped: a fact of the /// machine, fixed before the first driver arms anything. +/// +/// Where it is, [`Remapping::fences_tables`] says whether whatever message a +/// holder writes into its function's own MSI-X table is fenced too. pub fn remapping() -> Result { if messages::is_armed() { Ok(Remapping(())) @@ -315,9 +330,10 @@ pub fn remapping() -> Result { /// Whether `function` can be given a domain of its own and a remapping entry /// no other requester reaches, or why not: a function no programmed unit -/// serves, or one firmware reserves memory for, keeps the machine's own. +/// serves, one firmware reserves memory for, or one whose requester id the +/// platform speaks as. pub fn isolates(function: &crate::drivers::pci::PciDevice) -> Result<(), &'static str> { - match unit::domain::untranslated(StreamId::pci(function.bus, function.dev, function.func)) { + match unit::domain::unclaimable(StreamId::pci(function.bus, function.dev, function.func)) { None => Ok(()), Some(why) => Err(why), } diff --git a/kernel/src/pcidev/mod.rs b/kernel/src/pcidev/mod.rs index ede87ad453..3e7e1e8957 100644 --- a/kernel/src/pcidev/mod.rs +++ b/kernel/src/pcidev/mod.rs @@ -49,12 +49,14 @@ //! that BAR rather than nothing at all, and a function this kernel could settle //! no BAR of ends at [`Refusal::NoMappableBar`]. //! -//! **The BAR holding the MSI-X table or PBA is handed over like any other**, -//! because a claim exists only on a machine that remaps: whatever message its -//! holder writes into the table, the unit delivers only the claim's own -//! interrupt and refuses and records anything else. **A function is armed on -//! MSI only where a walk that reached its capability list's terminator found -//! no MSI-X.** +//! **The BAR holding the MSI-X table is handed over only where the machine +//! fences a written message** ([`crate::iommu::Remapping::fences_tables`]): +//! there, whatever message its holder writes into the table reaches only the +//! claim's own interrupt, and anything else is refused and recorded. On +//! AMD-Vi that needs every unit's SMI filter on, since an SMI is no remapped +//! message; where it is not, the BAR is kept back, said by name. **A function +//! is armed on MSI only where a walk that reached its capability list's +//! terminator found no MSI-X.** //! //! **A function with no address space of its own is not handed over**, because //! every grant would answer with a physical address and a descriptor holding @@ -130,7 +132,7 @@ use toyos_pci::bridge::Window; use toyos_pci::{af, aperture, bar, express, placement, pm, probe}; use crate::device::{Claim, ClaimError}; -use crate::drivers::pci::{arm_claimed_msi, arm_claimed_msix, mask_claimed_msix, NoCapability, NoEntry, PciDevice}; +use crate::drivers::pci::{arm_claimed_msi, arm_claimed_msix, mask_claimed_msix, MsixAt, NoCapability, NoEntry, PciDevice}; use crate::iommu::{IommuError, OwnSpace, Refused, Remapped}; use crate::mm::policy::{CachePolicy, MmioPolicy}; use crate::mm::{align_2m, DirectMap, PAGE_2M}; @@ -265,15 +267,15 @@ struct Aimed { /// Both deliver the function's own interrupt into the same [`Interrupt`] and /// the claim answers the same handle either way. enum Armed { - /// This function's one MSI-X table entry. - Msix(Remapped), + /// This function's one MSI-X table entry, where it was found at arming. + Msix(MsixAt, Remapped), Msi(Remapped), } impl Armed { fn message(&self) -> &Remapped { match self { - Self::Msix(message) | Self::Msi(message) => message, + Self::Msix(_, message) | Self::Msi(message) => message, } } } @@ -886,7 +888,7 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result // translation and its interrupt back. let message = crate::iommu::claim_msi(remapping, number, &pci).map_err(Refusal::NoMessage)?; let armed = match arm_claimed_msix(&message) { - Ok(()) => Armed::Msix(message), + Ok(at) => Armed::Msix(at, message), Err(NoEntry::Unusable) => return Err(Refusal::MsixUnusable), Err(NoEntry::NoTable(NoCapability::Truncated)) => return Err(Refusal::CapsTruncated), Err(NoEntry::NoTable(NoCapability::Absent)) if arm_claimed_msi(&message) => { @@ -897,7 +899,23 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result // From here a refusal has to undo: a vector is armed, and the arms below // move the function's BARs. - match place_bars(&pci, id, remapping) { + // The table's BAR is the holder's only where a message written into it + // reaches nothing but this claim's interrupt. + let withheld = match &armed { + Armed::Msix(at, _) if !remapping.fences_tables() => { + log!( + "pcidev: PCI {:02x}:{:02x}.{} keeps BAR {}, which holds its MSI-X table, back: a message written \ + there would not be fenced on this machine", + pci.bus, + pci.dev, + pci.func, + at.bar + ); + Some(at.bar) + } + _ => None, + }; + match place_bars(&pci, id, withheld) { Ok((bar_at, bar_bytes)) => { space.attach(pci.bus, pci.dev, pci.func); Ok(Bound { @@ -916,7 +934,7 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result Err(why) => { // The entry goes not present once the function stops speaking. let message = match armed { - Armed::Msix(message) => { + Armed::Msix(_, message) => { pci.disable_msix(); message } @@ -941,10 +959,7 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result fn place_bars( pci: &PciDevice, id: PciId, - // A BAR holding the MSI-X table is handed over too: on a machine that - // remaps, a message its holder writes there reaches only the claim's own - // interrupt, and anything else is refused and recorded by the unit. - _: crate::iommu::Remapping, + withheld: Option, ) -> Result<([u64; BARS], [u64; BARS]), Refusal> { let mut bar_at = [0u64; BARS]; let mut bar_bytes = [0u64; BARS]; @@ -958,7 +973,7 @@ fn place_bars( // whatever its register holds now: one a reset returned to zero reads // as unassigned, and it is the window that moved, not a BAR that is // not there. - if pci.memory_bar(index).is_err() && !was_cut(pci, index) { + if Some(index) == withheld || (pci.memory_bar(index).is_err() && !was_cut(pci, index)) { index += step; continue; } @@ -1504,9 +1519,14 @@ fn release(number: usize) { fn tear_down(function: &Function, mut bound: Bound) { bound.pci.disable_bus_master(); let message = match bound.armed { - Armed::Msix(message) => { - // Armed through this table, so it is still there to mask. - assert!(mask_claimed_msix(&bound.pci).is_ok(), "pcidev: the MSI-X table a claim was armed through is gone"); + Armed::Msix(at, message) => { + // Where this kernel put the table's BAR, or where it was at + // arming if it stayed: never what the device answers now. + let base = match bound.bar_at[usize::from(at.bar)] { + 0 => at.base, + placed => placed, + }; + mask_claimed_msix(base + at.offset); message } Armed::Msi(message) => { diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 0514d57bd4..dec96701d1 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -2426,9 +2426,8 @@ pub fn pipe_map(handle: RawHandle) -> Result<*mut u8, SyscallError> { /// maps read/write and uncacheable. /// /// `InvalidArgument` for an index that names no memory BAR, and for the BAR -/// holding this function's MSI-X table or PBA: the vector in that table is the -/// kernel's, and a process that could write it could aim the device's interrupt -/// at any address the LAPIC decodes. +/// holding this function's MSI-X table where the machine would not fence a +/// message written there. /// /// Every call answers an object of its own over the same window, so a BAR /// whose handle was closed can be asked for again. diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index aaf45d4a32..15491fd1c9 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -712,9 +712,12 @@ pub enum Profile { /// can be made and read. MetalAmdViTestdev, /// [`Profile::HeadlessUsbSpare`] behind QEMU's AMD unit remapping - /// interrupts, its spare armed on MSI-X with the table in the register - /// BAR: a claim's message through its own requester's table. + /// interrupts: a claim's message through its own requester's table. HeadlessUsbSpareAmdVi, + /// [`Profile::HeadlessUsbSpareAmdVi`] with its spare armed on MSI-X, the + /// table in its register BAR, as the AMD bench's controllers have theirs; + /// QEMU's unit has no SMI filter, so that BAR is kept back. + HeadlessUsbSpareAmdViMsix, /// [`Profile::HeadlessUsbSpareAmdVi`] with the unit remapping nothing. HeadlessUsbSpareAmdViNoRemap, /// QEMU `virt` on AArch64 (GICv3, AAVMF): a GOP from `ramfb`, the boot @@ -781,7 +784,8 @@ impl Profile { | Self::MetalAmdVi | Self::MetalAmdViTestdev | Self::HeadlessUsbSpareAmdVi - | Self::HeadlessUsbSpareAmdViNoRemap => Arch::X86_64, + | Self::HeadlessUsbSpareAmdViNoRemap + | Self::HeadlessUsbSpareAmdViMsix => Arch::X86_64, } } @@ -1060,14 +1064,15 @@ impl Profile { }, Self::MetalNoX2apic => Self::Metal.shape(), Self::MetalAmdVi => Shape { iommu: Some(Unit::AmdVi { intremap: false }), ..Self::Metal.shape() }, - Self::HeadlessUsbSpareAmdVi => Shape { - iommu: Some(Unit::AmdVi { intremap: true }), - xhci: &[XHCI_DEFAULT, XHCI_SPARE_MSIX], - ..Self::HeadlessUsbSpare.shape() - }, + Self::HeadlessUsbSpareAmdVi => { + Shape { iommu: Some(Unit::AmdVi { intremap: true }), ..Self::HeadlessUsbSpare.shape() } + } Self::HeadlessUsbSpareAmdViNoRemap => { Shape { iommu: Some(Unit::AmdVi { intremap: false }), ..Self::HeadlessUsbSpareAmdVi.shape() } } + Self::HeadlessUsbSpareAmdViMsix => { + Shape { xhci: &[XHCI_DEFAULT, XHCI_SPARE_MSIX], ..Self::HeadlessUsbSpareAmdVi.shape() } + } Self::MetalAmdViTestdev => Shape { probes: Probes::Testdevs, ..Self::MetalAmdVi.shape() }, Self::HeadlessNoIommu => Shape { iommu: None, ..Self::Headless.shape() }, Self::HeadlessNoUsb => Shape { xhci: &[], usb: &[], ..Self::Headless.shape() }, diff --git a/tests/guest/rust-tests/src/bin/usbd_spare.rs b/tests/guest/rust-tests/src/bin/usbd_spare.rs index b17bfe949e..d2171a4ea8 100644 --- a/tests/guest/rust-tests/src/bin/usbd_spare.rs +++ b/tests/guest/rust-tests/src/bin/usbd_spare.rs @@ -13,7 +13,7 @@ fn main() { Some("fault") => usbd_spare::fault(&cap), Some("refused") => usbd_spare::refused(&cap), Some("vector") => usbd_spare::vector(&cap), - Some("refused_unremapped") => usbd_spare::refused_unremapped(&cap), + Some("unhanded") => usbd_spare::unhanded(&cap), Some(other) => panic!("usbd_spare: no arm {other:?}"), } } diff --git a/tests/guest/rust-tests/src/bin/usbd_spare_refused_unremapped.rs b/tests/guest/rust-tests/src/bin/usbd_spare_unhanded.rs similarity index 55% rename from tests/guest/rust-tests/src/bin/usbd_spare_refused_unremapped.rs rename to tests/guest/rust-tests/src/bin/usbd_spare_unhanded.rs index 7491890f4d..af4c40d087 100644 --- a/tests/guest/rust-tests/src/bin/usbd_spare_refused_unremapped.rs +++ b/tests/guest/rust-tests/src/bin/usbd_spare_unhanded.rs @@ -1,4 +1,4 @@ -//! `usbd_spare`'s `refused_unremapped` arm as a binary of its own, for a metal job, which +//! `usbd_spare`'s `unhanded` arm as a binary of its own, for a metal job, which //! takes no argument: see `src/usbd_spare.rs`. #[path = "../usbd_spare.rs"] @@ -6,5 +6,5 @@ mod usbd_spare; fn main() { - usbd_spare::refused_unremapped(&usbd_spare::cap()); + usbd_spare::unhanded(&usbd_spare::cap()); } diff --git a/tests/guest/rust-tests/src/usbd_spare.rs b/tests/guest/rust-tests/src/usbd_spare.rs index 1625ba6b91..4d8a520c43 100644 --- a/tests/guest/rust-tests/src/usbd_spare.rs +++ b/tests/guest/rust-tests/src/usbd_spare.rs @@ -12,7 +12,7 @@ //! every grant: the kernel's `DMA FAULT` record is the harness's to read, //! and the claim refusing its interrupt read from then on is this job's. //! - `refused`: the claim, on a kernel that drives the function, refused. -//! - `refused_unremapped`: the claim, on a machine that remaps nothing, +//! - `unhanded`: the claim, on a machine that will not hand it over, //! refused; the kernel's line says why. //! - `vector`: the claim's MSI-X table entry rewritten, by this job through //! the BAR that holds it, to a vector that is not the claim's, and usbd @@ -27,11 +27,6 @@ use std::sync::mpsc; use std::time::{Duration, Instant}; use toyos::endow::{Endowments, DEV_PREFIX, SERVE_PREFIX, SYSCAP_LABEL}; - -/// The device-minting capability the test runner endows every job with. -pub fn cap() -> SysCap { - Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows a device-minting capability") -} use toyos::ipc::Connection; use toyos::poller::{Poller, READABLE}; use toyos::syscap::SysCap; @@ -42,9 +37,15 @@ use toyos_inspect::{Value, MSG_INSPECT, MSG_SNAPSHOT, USB}; const USBD: &str = "/system/bin/usbd"; +/// The device-minting capability the test runner endows every job with. +pub fn cap() -> SysCap { + Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows a device-minting capability") +} + /// The controllers a boot of this job leaves to a claim: QEMU's `qemu-xhci` -/// on the guest profile, and the T14's Type-C controller. A machine has one -/// of them; the other is absent. +/// on the guest profiles, the T14's Type-C controller, and the AMD bench's +/// xHCI, whose id its other controller shares (`claim` takes the one the +/// kernel left). A machine has one of them. const SPARES: [PciId; 3] = [ PciId { vendor: 0x1b36, device: 0x000d }, PciId { vendor: 0x8086, device: 0x9a13 }, @@ -277,10 +278,10 @@ pub fn vector(cap: &SysCap) { println!("usbd_spare: usbd settled and was never told of its No-Op (noop = {noop:?})"); } -pub fn refused_unremapped(cap: &SysCap) { +pub fn unhanded(cap: &SysCap) { let (id, claimed) = claim(cap); match claimed { Err(why) => println!("usbd_spare: the claim on {} was refused: {why:?}", spelled(id)), - Ok(_) => panic!("usbd_spare: the claim on {} was granted, on a machine that remaps nothing", spelled(id)), + Ok(_) => panic!("usbd_spare: the claim on {} was granted, on a machine that would not hand it over", spelled(id)), } } diff --git a/tests/metal/src/lib.rs b/tests/metal/src/lib.rs index 0f52228d9e..b31a7c4650 100644 --- a/tests/metal/src/lib.rs +++ b/tests/metal/src/lib.rs @@ -768,9 +768,9 @@ pub const FLASHABLE: &[&str] = &[ // writes only the units' registers, which a reset returns to firmware, and // tables in memory this boot owns. "iommu-firmware-left", - // The DMAR's flags read with INTR_REMAP clear, so every IOMMU unit is left - // without interrupt remapping. It writes no register a reset does not - // return to firmware. + // No interrupt remapping: the DMAR's flags read with INTR_REMAP clear, or + // every AMD-Vi unit's interrupt halves left invalid. It writes no + // register a reset does not return to firmware. "iommu-no-remap", // The BSP is handed the T14's Intel update with its header's revision // raised by one and its checksum sealed again. Its payload is Intel's own, @@ -809,13 +809,20 @@ pub const FLASHABLE: &[&str] = &[ // hold, reaches no firmware state, and the worst // it leaves is a stick a replug clears — the defect the arm exists to stage. "usb-transport-break", - // The kernel leaves the Type-C xHCI controller to a claim, and `usbd` - // drives it through the unit: the handoff, a reset and its own rings, as - // this kernel's driver does every boot. Firmware owns neither of the - // T14's two (each reads `USBLEGSUP 0x01002201`, every SMI enable clear), - // and firmware takes the controller back at the next power-on. Only this - // id: the other controller carries the boot stick. - "xhci-leave=8086:9a13", + // The kernel leaves the T14's Type-C xHCI controller, `00:0d.0`, to a + // claim, and `usbd` drives it through the unit: the handoff, a reset and + // its own rings, as this kernel's driver does every boot. Firmware owns + // neither of the T14's two (each reads `USBLEGSUP 0x01002201`, every SMI + // enable clear), and firmware takes the controller back at the next + // power-on. Only this address: the other controller carries the boot + // stick. + "xhci-leave=00:0d.0", + // The AMD bench's `03:00.4`, the xHCI the Bluetooth module is on, left to + // a claim the same way: firmware owns it until the handoff `usbd` + // performs as this kernel's driver does every boot (`USBLEGSUP + // 0x00010401`), and takes it back at the next power-on. Only this + // address: `03:00.3`, which shares its id, carries the boot stick. + "xhci-leave=03:00.4", ]; /// The arm that stops the machine, named once: [`FLASHABLE`] rules on it and diff --git a/tests/toyos.rs b/tests/toyos.rs index c23a952af2..963b7f05a8 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -206,7 +206,7 @@ const RUST_SKIP: &[&str] = &[ // `amdvi_claim_refused_without_remapping` rows run them. "usbd_spare_fault", "usbd_spare_vector", - "usbd_spare_refused_unremapped", + "usbd_spare_unhanded", // The same controller, handed to a holder that is killed and claimed // again at once, in the same boots as `usbd_spare`. "end_returns_claims", @@ -440,13 +440,19 @@ const MACHINE_TESTS: &[&str] = &[ "usbd_drives_the_spare", // The same behind QEMU's AMD unit remapping interrupts: the claim's // message through an entry of its function's own table, which no host - // build has and which the AMD bench's controllers, their MSI-X tables in - // their register BARs, cannot yet be claimed to show. + // build has. The AMD bench's `amdvi_usbd_drives_its_own_entry` holds the + // same on silicon, with a foreign vector, which QEMU's unit cannot hand + // a table over for. "usbd_drives_the_spare_amdvi", // The same machine with its AMD unit remapping nothing: the claim is // refused before anything on the function changes. The AMD bench's arm is - // `claim_refused_without_remapping`'s. + // `amdvi_claim_refused_without_remapping`. "usbd_refused_without_amdvi_remapping", + // QEMU's AMD unit, which has no SMI filter, remapping a spare whose MSI-X + // table is in its register BAR: that BAR is kept back, said by name, and + // the claim is refused for want of another. No metal machine in reach has + // an AMD-Vi unit without the filter. + "usbd_table_kept_without_smi_filter", // A boot keyboard reporting more keys than its six slots name: the // kernel's driver reading a report a device delivered over xHCI has no host // build, and the T14 binds no USB keyboard. @@ -1100,7 +1106,7 @@ const METAL: &[(&str, metal::Metal)] = &[ for job in ["test_rs_usbd_spare", "test_rs_usbd_spare_fault", "test_rs_end_returns_claims", "test_rs_usbd_spare_vector"] { b[0].job_passed(job)?; } - amdvi_usbd(&b[0].kernel(), &b[0].log()) + amdvi_usbd(&b[0].kernel(), &b[0].log(), &b[0].after_the_reset()?) }, }, ), @@ -1113,7 +1119,7 @@ const METAL: &[(&str, metal::Metal)] = &[ "testcases-amd-no-remap", "tests/images/testcases.toml", &["iommu-no-remap", "xhci-leave=03:00.4"], - &["test_rs_usbd_spare_refused_unremapped"], + &["test_rs_usbd_spare_unhanded"], )], judge: |b| { let kernel = b[0].kernel(); @@ -3789,6 +3795,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "usbd_drives_the_spare" => usbd_drives_the_spare(test_config, qemu::Profile::HeadlessUsbSpare), "usbd_drives_the_spare_amdvi" => usbd_drives_the_spare(test_config, qemu::Profile::HeadlessUsbSpareAmdVi), "usbd_refused_without_amdvi_remapping" => usbd_refused_without_remapping(test_config), + "usbd_table_kept_without_smi_filter" => usbd_table_kept_without_smi_filter(test_config), "usb_keyboard_rollover" => usb_keyboard_rollover(test_config), "vector_state_survives_a_switch" => vector_state_survives_a_switch(test_config), "the_opening_exchange_with_qemus_agent_completes" => the_opening_exchange_with_qemus_agent_completes(), @@ -3911,9 +3918,8 @@ fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<( if said.matches(&handed).count() != 1 { return Err(format!("{at} was not handed over once, to the one usbd:\n{said}")); } - // MSI on the VT-d machine's spare, MSI-X on the AMD one's. - run.must_say(&format!("PCI {at}: msi"))?; let amdvi = matches!(profile.iommu(), Some(qemu::Unit::AmdVi { .. })); + run.must_say(&format!("PCI {at}: msi address="))?; if amdvi { // The claim's entry in its own table, present at the vector the // hand-over names, and absent again at the release. @@ -3946,9 +3952,6 @@ fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<( let ended = job_window(&mut qemu, "test_rs_end_returns_claims")?; let line = ends_returned_claims(&ended)?; eprintln!(" [usbd] {line}"); - if amdvi { - foreign_vector(&mut qemu, &at)?; - } drop(qemu); // The control: the same machine and no switch, so the kernel drives it. @@ -3966,6 +3969,30 @@ fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<( Ok(()) } +/// Where an AMD-Vi unit has no SMI filter, the BAR holding a claim's MSI-X +/// table is kept back, said by name: an SMI is no remapped message, so a +/// holder's own table could send one to every CPU (48882 Table 10). +fn usbd_table_kept_without_smi_filter(test_config: &Path) -> Result<(), String> { + const JOB: &str = "usbd_spare"; + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); + let options = BootOptions { + profile: qemu::Profile::HeadlessUsbSpareAmdViMsix, + kernel_params: &[XHCI_LEAVE_SPARE], + ..Default::default() + }; + let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[(JOB.to_string(), bin)], options); + let boot = serial::Serial::boot(&qemu); + boot.must_say("iommu: amdvi unit0 has no SMI filter")?; + boot.must_say("SMIs filtered=n")?; + let at = qemu::SPARE_AT; + let said = job_window(&mut qemu, "test_rs_usbd_spare unhanded")?; + let run = serial::Serial::named("usbd_spare", said); + let kept = run.must_say(&format!("pcidev: PCI {at} keeps BAR 0, which holds its MSI-X table, back"))?; + run.must_say(&format!("pcidev: PCI {at} NOT HANDED OVER — it publishes no memory BAR this claim may map"))?; + eprintln!(" [usbd] {}", kept.trim()); + Ok(()) +} + /// A machine whose AMD unit remaps nothing refuses the spare's claim by that /// reason, before a vector is armed or a BAR moved. fn usbd_refused_without_remapping(test_config: &Path) -> Result<(), String> { @@ -3979,7 +4006,7 @@ fn usbd_refused_without_remapping(test_config: &Path) -> Result<(), String> { let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[(JOB.to_string(), bin)], options); let boot = serial::Serial::boot(&qemu); boot.must_say("no message is remapped")?; - let said = job_window(&mut qemu, "test_rs_usbd_spare refused_unremapped")?; + let said = job_window(&mut qemu, "test_rs_usbd_spare unhanded")?; claims::refused_unremapped_at(&serial::Serial::named("usbd_spare", said), qemu::SPARE_AT) } @@ -3989,7 +4016,7 @@ fn usbd_refused_without_remapping(test_config: &Path) -> Result<(), String> { /// past its grant and recorded against the claim at the aimed address; then /// its holder's rewrite of its own table entry to a vector not its own /// refused and recorded against the claim, usbd never told of its No-Op. -fn amdvi_usbd(kernel: &serial::Serial, log: &serial::Serial) -> Result<(), String> { +fn amdvi_usbd(kernel: &serial::Serial, log: &serial::Serial, after: &serial::Serial) -> Result<(), String> { const ID: &str = "1022:1639"; let at = AMD_XHCI_SPARE; kernel.must_say(&format!("xHCI: leaving PCI {at} {ID} to a claim (xhci-leave)"))?; @@ -4028,7 +4055,8 @@ fn amdvi_usbd(kernel: &serial::Serial, log: &serial::Serial) -> Result<(), Strin .split_once(" rewritten to ") .and_then(|(_, rest)| rest.split(',').next()) .ok_or_else(|| format!("{rewritten:?} names no vector"))?; - let census = kernel.must_say("irq: unclaimed vectors")?; + // Logged inside the stop, past where the log file ends. + let census = after.must_say("irq: unclaimed vectors")?; if census.split_whitespace().any(|word| word == foreign_vector) { return Err(format!("{foreign_vector} reached a CPU: {census}")); } @@ -4043,22 +4071,6 @@ fn amdvi_usbd(kernel: &serial::Serial, log: &serial::Serial) -> Result<(), Strin Ok(()) } -/// The claim's own MSI-X table entry rewritten by its holder to a vector not -/// the claim's: the unit refuses that message and records it against the -/// claim, and usbd is never told of its No-Op. The table is reachable only -/// because a machine that remaps hands over the BAR holding it. -fn foreign_vector(qemu: &mut QemuInstance, at: &str) -> Result<(), String> { - let said = job_window(qemu, "test_rs_usbd_spare vector")?; - let run = serial::Serial::named("usbd_spare vector", said); - let rewritten = run.must_say("usbd_spare: MSI-X entry 0's data ")?; - run.must_say("usbd_spare: usbd settled and was never told of its No-Op")?; - // QEMU 11.1.1's AMD unit drops the message and records nothing - // (`issues/qemus-amd-iommu-records-no-fault-for-a-message-its-table-refuses.md`); - // the AMD bench's row holds the record. - run.must_not_say(&format!("DMA FAULT owner=process unit0 stream={at} "))?; - eprintln!(" [usbd] {}; usbd never told of its No-Op", rewritten.trim()); - Ok(()) -} /// Every AMD-Vi unit translating, every xHCI controller the kernel drives /// moved to a domain of its own, the boot stick bound through one, and no From 32067821980cc45e74007535a2b7c1528007fc43 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 12:54:17 +0200 Subject: [PATCH 5/9] amdvi: the SMI filter goes on once the kernel's drivers are settled, and every claim needs it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-3 review and the AMD bench's hang at 4f1d6fc3a. The hang: 4f1d6fc3a turned every unit's SMI filter on in `program`, before the kernel took its xHCI controllers from firmware, with no register valid on the bench (firmware names no source there). This bench resets through `SMI_CMD` (its FADT's reset register is port 0xB0), and an SMI the chipset raises upstream is what the filter refuses (48882 §2.1.5.1); with every source refused, neither the boot deadline's reset nor a panic's could end the boot. That is an inference from the earlier boot's log and the specification; the hung boot's own log is unread. - The filter is planned at `init` and turned on by `iommu::drivers_settled`, after every kernel driver took its functions and before userland runs. It passes what firmware left valid and every id the IVRS names the chipset's I/O APIC and HPET by, every register locked (§2.1.5.3); a unit with too few free registers filters nothing, said. The chipset raising the platform's SMIs under those ids is an inference the bench reads. - Every unit's filter is off before the reset register or `SLP_EN` is written, lock-free from the fault handler's unit table. - R1 (ruling a): a write to the interrupt range is an interrupt (§2.1.4.2), so withholding the table's BAR closed nothing. `iommu::remapping` is now the one gate: a claim exists only where every message reaches only its own interrupt, which on AMD-Vi needs every unit filtering SMIs; the refusal names which. The withheld-BAR path, `fences_tables`, `HeadlessUsbSpareAmdViMsix` and `usbd_table_kept_without_smi_filter` are gone; `usbd_drives_the_spare_amdvi` is `usbd_refused_without_smi_filter`. - R2: an SMI's address is read by Table 4's fields only, bits 63:48 not read; `toyos_amdvi::queue::event` decodes `RefusedSmi` itself. A claimed function's refused SMI is its holder's fault; any other's is logged, bounded, and the machine goes on. Whether a claim holds the function is read without a side effect (S1). - R3, R4, R5: `fences_claims` and `is_armed` documented apart; the spare's profile names 00:10.0; the FLASHABLE rows argue from the reset that ends every boot. - S2: the special entry's reason is a match on its variety. S3: the claim's comment names `pcidev::claim` for a kernel driver's function, and the SMI rationale lives at `fences_claims`. S4: the filter's registers are locked once written. - `amdvi_remaps` reads `SmiFEn` and `SmiFLogEn` on every unit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...in-bar-0-cannot-be-driven-from-userland.md | 17 ++- .../an-amd-vi-machine-isolates-no-device.md | 15 +- kernel/amdvi/src/queue.rs | 13 +- kernel/amdvi/src/reg.rs | 32 +++- kernel/amdvi/tests/encodings.rs | 56 +++++-- kernel/src/arch/aarch64/irqchip/its.rs | 15 +- kernel/src/arch/aarch64/smmu/mod.rs | 3 + kernel/src/arch/x86_64/amdvi/fault.rs | 37 ++++- kernel/src/arch/x86_64/amdvi/interrupt.rs | 11 +- kernel/src/arch/x86_64/amdvi/mod.rs | 110 ++++++++++---- kernel/src/arch/x86_64/iommu_unit.rs | 27 +++- kernel/src/arch/x86_64/power.rs | 7 +- kernel/src/iommu/mod.rs | 44 +++--- kernel/src/main.rs | 4 + kernel/src/pcidev/mod.rs | 60 ++++---- tests/common/claims.rs | 16 +- tests/common/qemu.rs | 26 +--- tests/metal/src/lib.rs | 17 ++- tests/toyos.rs | 139 ++++++++---------- 19 files changed, 392 insertions(+), 257 deletions(-) diff --git a/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md b/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md index b446088c64..5a1b19176c 100644 --- a/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md +++ b/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md @@ -7,16 +7,19 @@ opened: 2026-09-26 # A controller whose MSI-X table is in BAR 0 cannot be driven from userland Held by PR #921 (AMD-Vi stage 6), under the orchestrator's rulings there: -where the machine fences a message written into a claim's MSI-X table, -`pcidev` hands over the BAR holding it; where it does not, that BAR is kept -back, said by name. VT-d fences by source-id verification and its block on +a claim exists only where every message its function sends, whatever its +holder writes into its MSI-X table or aims a DMA at, reaches only its own +interrupt, and there `pcidev` hands over the table's BAR like any other. +VT-d holds that by source-id verification and its block on compatibility-format messages, the ITS by the device id the bridge stamps, and AMD-Vi by the requester's own table together with every unit's SMI -filter, since an SMI is no remapped message (48882 Table 10). +filter, since a write to the interrupt range is an interrupt (48882 +§2.1.4.2) and an SMI no remapped message (Table 10). Where an AMD-Vi unit +filters no SMI, every claim is refused by that reason. -QEMU's AMD unit has no SMI filter, so `usbd_table_kept_without_smi_filter` -reads its spare's table BAR kept back and the claim refused -`NoMappableBar`. +QEMU's AMD unit has no SMI filter, so QEMU drives no claim behind it: +`usbd_refused_without_smi_filter` reads the refusal, and only the AMD bench +reads a claim through an AMD-Vi entry. **Exit:** on the AMD bench, `03:00.4` (MSI-X table in BAR 0, its only BAR) is claimed and driven by usbd through its own remapping entry with every diff --git a/issues/an-amd-vi-machine-isolates-no-device.md b/issues/an-amd-vi-machine-isolates-no-device.md index 8e527f9b29..e8232a0ac6 100644 --- a/issues/an-amd-vi-machine-isolates-no-device.md +++ b/issues/an-amd-vi-machine-isolates-no-device.md @@ -17,9 +17,18 @@ fixed and arbitrated messages go through a table of its own indexed by vector, a claim's entry passing nothing firmware declared. What remains is the AMD bench's reading of a claim (PR #921 stages it): a -claimed `03:00.4` driven through its own entry, its MSI-X table's BAR handed -over because every unit filters SMIs, its holder's foreign vector refused -and recorded, and a machine that remaps nothing refusing the claim. +claimed `03:00.4` driven through its own entry with every unit filtering +SMIs, its holder's foreign vector refused and recorded, and a machine that +remaps nothing refusing the claim. + +The filter goes on once the kernel's drivers hold their functions, since a +controller firmware still owns raises SMIs to be handed over; it passes +what firmware left valid and the ids the IVRS names the chipset's I/O APIC +and HPET by, on the inference that the chipset raises the platform's SMIs +under the same ids, which §2.1.5.3 leaves to firmware to name. Every unit's +filter is off before the reset register or `SLP_EN` is written, since this +bench resets through `SMI_CMD`. The bench's boot reads which SMIs the +filter refused, by requester. Constraints a reader would otherwise pay to re-derive: diff --git a/kernel/amdvi/src/queue.rs b/kernel/amdvi/src/queue.rs index 197700a2e8..33ce1082b8 100644 --- a/kernel/amdvi/src/queue.rs +++ b/kernel/amdvi/src/queue.rs @@ -48,6 +48,9 @@ pub enum Event { /// `RX` clear (Figure 64). `write` is the access where the record states /// one; `address` is the record's, whatever it names. Requester { code: u8, device: u16, address: u64, write: Option }, + /// An `IO_PAGE_FAULT` at an SMI's address: an SMI from `device` the + /// filter refused (§2.1.5.1), no access of memory. + RefusedSmi { device: u16, address: u64 }, /// Every other record: an entry or command of this driver's the unit /// refused, or the unit's own hardware failing. Unit { code: u8 }, @@ -58,6 +61,7 @@ pub const fn event(words: [u64; 2]) -> Event { let code = (words[0] >> CODE_SHIFT) as u8; let write = match code { // `I` [51]: an interrupt carries no read or write; else `RW` [53]. + 0b0010 if smi(words[1]) => return Event::RefusedSmi { device, address: words[1] }, 0b0010 if words[0] & 1 << 51 != 0 => None, 0b0010 => Some(words[0] & 1 << 53 != 0), // `RX` [48] set is a `COMPLETE_PPR_REQUEST` of this driver's failing. @@ -72,10 +76,11 @@ pub const fn event(words: [u64; 2]) -> Event { /// Whether a record's address is an SMI's (§2.1.5.2, Figure 3 and Table 4): /// bits 47:40 zero, `FDF8h` at 39:24, `MT` 0010b at 7 and 4:2, `DM` and -/// bits 1:0 zero. The SMI filter records a refused SMI as an -/// `IO_PAGE_FAULT` at that address (§2.1.5.1). -pub const fn smi(address: u64) -> bool { - address & 0xFFFF_FFFF_FF00_00C3 == 0x0000_00FD_F800_0000 && (address >> 2) & 0b111 == 0b010 +/// bits 1:0 zero; bits 63:48 are not the figure's and are not read. The SMI +/// filter records a refused SMI as an `IO_PAGE_FAULT` at that address +/// (§2.1.5.1). +const fn smi(address: u64) -> bool { + address & 0x0000_FFFF_FF00_00C3 == 0x0000_00FD_F800_0000 && (address >> 2) & 0b111 == 0b010 } /// The name 48882 gives an event code, for a record's line. diff --git a/kernel/amdvi/src/reg.rs b/kernel/amdvi/src/reg.rs index 0a7de20528..12056d6ee6 100644 --- a/kernel/amdvi/src/reg.rs +++ b/kernel/amdvi/src/reg.rs @@ -25,10 +25,38 @@ pub const SMI_FILTER: u64 = 0x0060; #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct SmiFilter(pub u64); +/// `registers` as the filter is to pass SMIs from the ids firmware left +/// valid and from each of `platform`, every register locked (§2.1.5.3 steps +/// 3–6): an unlocked register firmware left invalid takes an id it left out, +/// and the rest are locked invalid. Refused with the first platform id no +/// such register is left for: a filter without it would refuse that +/// source's SMIs. +pub fn smi_filter(registers: &mut [SmiFilter], platform: &[u16]) -> Result<(), u16> { + for &id in platform { + if registers.iter().any(|register| register.passes() == Some(id)) { + continue; + } + let free = registers.iter_mut().find(|register| !register.locked() && register.passes().is_none()).ok_or(id)?; + *free = SmiFilter::passing(id); + } + for register in registers.iter_mut().filter(|register| !register.locked()) { + register.0 = register.passes().map_or(SmiFilter::LOCK, |id| SmiFilter::passing(id).0); + } + Ok(()) +} + impl SmiFilter { + const VALID: u64 = 1 << 16; + const LOCK: u64 = 1 << 17; + + /// Passes SMIs from `id`, locked. + pub const fn passing(id: u16) -> Self { + Self(id as u64 | Self::VALID | Self::LOCK) + } + /// The requester id an SMI passes from, where the register is valid. pub const fn passes(self) -> Option { - if self.0 & 1 << 16 != 0 { + if self.0 & Self::VALID != 0 { Some(self.0 as u16) } else { None @@ -36,7 +64,7 @@ impl SmiFilter { } pub const fn locked(self) -> bool { - self.0 & 1 << 17 != 0 + self.0 & Self::LOCK != 0 } } diff --git a/kernel/amdvi/tests/encodings.rs b/kernel/amdvi/tests/encodings.rs index dbd11a7376..7ad507591f 100644 --- a/kernel/amdvi/tests/encodings.rs +++ b/kernel/amdvi/tests/encodings.rs @@ -5,8 +5,8 @@ use toyos_amdvi::dte::{interrupts_blocked, interrupts_remapped, Dte, Setting, DEVICE_IDS, ENTRY_BYTES, INTERRUPT_ENTRIES}; use toyos_amdvi::irte::{self, Destination}; -use toyos_amdvi::queue::{event, name, smi, Command, Event, ENTRIES}; -use toyos_amdvi::reg::{control, device_table, offset, ring, slot, Features, SmiFilter, SMI_FILTER}; +use toyos_amdvi::queue::{event, name, Command, Event, ENTRIES}; +use toyos_amdvi::reg::{control, device_table, offset, ring, slot, smi_filter, Features, SmiFilter, SMI_FILTER}; use toyos_amdvi::table::{directory, entry, index, leaf, Entry, LEAF_LEVEL, ROOT_LEVEL}; use toyos_phys::Phys; @@ -280,18 +280,46 @@ fn the_smi_filter_sits_where_48882_puts_it() { assert_eq!((unset.passes(), unset.locked()), (None, false)); } -/// Figure 3 / Table 4: an SMI's address is `FDF8h` at [39:24], `MT[2:0]` -/// 010b at [4:2], `MT[3]` [7] and `DM` [6] zero, [1:0] zero, [47:40] zero; -/// the vector [23:16], destination [15:8] and `TM` [5] are ignored. +/// Figure 3 / Table 4, as the SMI filter records a refused SMI (§2.1.5.1): +/// an `IO_PAGE_FAULT` whose address is `FDF8h` at [39:24], `MT[2:0]` 010b at +/// [4:2], `MT[3]` [7] and `DM` [6] zero, [1:0] and [47:40] zero; the vector +/// [23:16], destination [15:8], `TM` [5] and [63:48] are not read. #[test] -fn an_smi_address_is_told_from_a_dma_address() { +fn an_smi_the_filter_refused_is_told_from_a_dma_fault() { + let fault = |address: u64| event([word(&[(15, 0, 0x00A3), (63, 60, 2)]), address]); let smi_at = word(&[(39, 24, 0xFDF8), (4, 2, 0b010)]); - assert!(smi(smi_at)); - assert!(smi(smi_at | word(&[(23, 16, 0xFF), (15, 8, 0xFF), (5, 5, 1)]))); - assert!(!smi(word(&[(39, 24, 0xFDF8), (4, 2, 0b000)]))); - assert!(!smi(smi_at | word(&[(7, 7, 1)]))); - assert!(!smi(smi_at | word(&[(6, 6, 1)]))); - assert!(!smi(smi_at | word(&[(40, 40, 1)]))); - assert!(!smi(0x4000_0000_0000)); - assert!(!smi(0xFEE0_0008)); + assert_eq!(fault(smi_at), Event::RefusedSmi { device: 0x00A3, address: smi_at }); + let ignored = smi_at | word(&[(23, 16, 0xFF), (15, 8, 0xFF), (5, 5, 1), (63, 48, 0xFFFF)]); + assert_eq!(fault(ignored), Event::RefusedSmi { device: 0x00A3, address: ignored }); + for not_smi in [ + word(&[(39, 24, 0xFDF8), (4, 2, 0b000)]), + smi_at | word(&[(7, 7, 1)]), + smi_at | word(&[(6, 6, 1)]), + smi_at | word(&[(40, 40, 1)]), + 0x4000_0000_0000, + 0xFEE0_0008, + ] { + assert!(matches!(fault(not_smi), Event::Requester { .. }), "{not_smi:#x}"); + } +} + +/// §2.1.5.3: the filter keeps every id firmware left valid, takes each +/// platform id it left out in an unlocked invalid register, and locks every +/// register; with too few such registers it is not one to turn on. +#[test] +fn the_smi_filter_passes_firmwares_and_the_platforms_sources_locked() { + let free = SmiFilter(0); + let held = SmiFilter(word(&[(15, 0, 0x00A3), (16, 16, 1)])); + let barred = SmiFilter(word(&[(15, 0, 0x0011), (17, 17, 1)])); + let passing = |id: u64| SmiFilter(word(&[(15, 0, id), (16, 16, 1), (17, 17, 1)])); + let locked_invalid = SmiFilter(word(&[(17, 17, 1)])); + let plan = |mut registers: Vec, platform: &[u16]| smi_filter(&mut registers, platform).map(|()| registers); + assert_eq!(plan(vec![free, free], &[0x00A0, 0x0001, 0x00A0]), Ok(vec![passing(0x00A0), passing(0x0001)])); + assert_eq!( + plan(vec![held, free, barred, free], &[0x00A3, 0x00A0]), + Ok(vec![passing(0x00A3), passing(0x00A0), barred, locked_invalid]) + ); + assert_eq!(plan(vec![free, barred], &[0x00A0, 0x0001]), Err(0x0001)); + assert_eq!(plan(vec![barred], &[0x0011]), Err(0x0011)); + assert_eq!(plan(vec![], &[]), Ok(vec![])); } diff --git a/kernel/src/arch/aarch64/irqchip/its.rs b/kernel/src/arch/aarch64/irqchip/its.rs index ceee85a1d7..a362684b3e 100644 --- a/kernel/src/arch/aarch64/irqchip/its.rs +++ b/kernel/src/arch/aarch64/irqchip/its.rs @@ -372,14 +372,17 @@ pub(in crate::arch::aarch64) fn doorbell_page() -> Option { ITS.lock().as_ref().map(|live| u64::from(live.doorbell) & !(crate::mm::PAGE_SIZE - 1)) } -/// Whether an ITS is armed to give a claimed function its own LPI. -/// Whether a message a holder writes into its function's MSI-X table reaches -/// nothing but its claim's interrupt: always, since the ITS translates by the -/// device id the bridge stamps, through that device's own table. -pub fn fences_tables() -> bool { - true +/// Whether a claimed function's every message reaches only its own LPI: where +/// an ITS is armed, which translates by the device id the bridge stamps. +pub fn fences_claims() -> Result<(), crate::iommu::NotRemapped> { + if is_armed() { + Ok(()) + } else { + Err(crate::iommu::NotRemapped::Unremapped) + } } +/// Whether an ITS is armed to give a claimed function its own LPI. pub fn is_armed() -> bool { ITS.lock().is_some() } diff --git a/kernel/src/arch/aarch64/smmu/mod.rs b/kernel/src/arch/aarch64/smmu/mod.rs index 17e379b803..3049cc2041 100644 --- a/kernel/src/arch/aarch64/smmu/mod.rs +++ b/kernel/src/arch/aarch64/smmu/mod.rs @@ -205,6 +205,9 @@ fn window(phys: u64, bytes: u64) -> Mmio { unsafe { Mmio::over_phys(DirectMap::from_phys(phys), bytes) } } +/// Nothing waits on the kernel's drivers: the SMMUv3 filters nothing by them. +pub fn drivers_settled() {} + /// The SMMUv3, armed; a refusal leaves every unit the IORT names aborting /// what it is given, or says why one could not be told to. pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[toyos_abi::boot::RootBridgeWindow]) { diff --git a/kernel/src/arch/x86_64/amdvi/fault.rs b/kernel/src/arch/x86_64/amdvi/fault.rs index d9195de5c1..cf56a56ec8 100644 --- a/kernel/src/arch/x86_64/amdvi/fault.rs +++ b/kernel/src/arch/x86_64/amdvi/fault.rs @@ -14,7 +14,7 @@ use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use crate::iommu::fault::{self as policy, Access, Fault, Who}; use crate::log; use crate::mm::{DirectMap, Mmio, PAGE_SIZE}; -use toyos_amdvi::queue::{event, name, smi, Event, ENTRIES}; +use toyos_amdvi::queue::{event, name, Event, ENTRIES}; use toyos_amdvi::reg::{self, control, status}; use super::{requester, spin_until, REGISTER_WINDOW}; @@ -76,11 +76,16 @@ fn drain(index: usize, regs: Mmio, log: Mmio) -> usize { while head != tail { let at = reg::offset(head); let words = [log.read_u64(at), log.read_u64(at + 8)]; - let fault = match event(words) { - // An SMI the filter refused (§2.1.5.1): a claimed function's is - // its holder's fault; any other's is said and the machine goes - // on, since no DMA of this kernel's went wrong. - Event::Requester { device, address, .. } if smi(address) && !crate::pcidev::note_fault(device) => { + let event = event(words); + let held = match event { + Event::RefusedSmi { device, .. } => crate::pcidev::is_held(device), + _ => false, + }; + let fault = match event { + // The filter kept it from every CPU (§2.1.5.1): a claimed + // function's is its holder's fault; any other's is said and the + // machine goes on, since nothing of this kernel's went wrong. + Event::RefusedSmi { device, address } if !held => { let refused = UNITS[index].smis.fetch_add(1, Ordering::Relaxed) + 1; if refused.is_power_of_two() { log!( @@ -91,6 +96,13 @@ fn drain(index: usize, regs: Mmio, log: Mmio) -> usize { head = (head + 1) % ENTRIES; continue; } + Event::RefusedSmi { device, address } => Fault { + who: Who::Function(requester(device)), + address, + access: Access::Write, + reason: 0b0010, + name: "an SMI the filter refused", + }, Event::Requester { code, device, address, write } => Fault { who: Who::Function(requester(device)), address, @@ -132,6 +144,19 @@ fn restart(index: usize, regs: Mmio) { regs.write_u64(reg::CONTROL, regs.read_u64(reg::CONTROL) | control::EVENT_LOG_EN); } +/// Every unit's SMI filter off, so the platform's SMIs pass whatever it +/// named: before the reset register is written, which on some machines is +/// `SMI_CMD` and resets in firmware's SMI handler. Takes no lock. +pub fn pass_every_smi() { + for unit in &UNITS { + let regs = unit.regs.load(Ordering::Acquire); + if regs != 0 { + let regs = window(regs, REGISTER_WINDOW); + regs.write_u64(reg::CONTROL, regs.read_u64(reg::CONTROL) & !control::SMIF_EN); + } + } +} + /// Records every unit has reported, for the selftest to wait on. #[cfg(feature = "boot-actuators")] pub(super) fn recorded() -> u32 { diff --git a/kernel/src/arch/x86_64/amdvi/interrupt.rs b/kernel/src/arch/x86_64/amdvi/interrupt.rs index 8c19f8577f..7d140758ea 100644 --- a/kernel/src/arch/x86_64/amdvi/interrupt.rs +++ b/kernel/src/arch/x86_64/amdvi/interrupt.rs @@ -44,9 +44,7 @@ struct Remap { static REMAP: Lock = Lock::new(Remap { extended: None, tables: BTreeMap::new(), apics: Vec::new() }); -/// Every programmed unit filters SMIs (§2.1.5): an SMI is no remapped message -/// (Table 10), so without the filter a holder's own MSI-X table could send -/// one to every CPU. +/// Every programmed unit filters SMIs, from [`super::settle`] on. static SMI_FILTERED: AtomicBool = AtomicBool::new(false); pub(super) fn smi_filtered(everywhere: bool) { @@ -151,9 +149,10 @@ pub(in crate::arch::x86_64) fn msi(source: StreamId, vector: u8) -> Result Result { - // A requester the platform or a kernel driver speaks as is refused - // before here (`iommu::isolates`); one that already has a table is one - // of those, and its entries keep firmware's pass bits. + // A requester the platform speaks as is refused before here + // (`iommu::isolates`), and one a kernel driver holds by `pcidev::claim`; + // one that already has a table is one of those, and its entries keep + // firmware's pass bits. assert!( !REMAP.lock().tables.contains_key(&source.requester()) || domain::setting(source.requester()) == toyos_amdvi::dte::Setting::CLAIMED, "amdvi: a claim on {source}, whose remapping table another source already wrote" diff --git a/kernel/src/arch/x86_64/amdvi/mod.rs b/kernel/src/arch/x86_64/amdvi/mod.rs index 3b9e293540..3e273a95f5 100644 --- a/kernel/src/arch/x86_64/amdvi/mod.rs +++ b/kernel/src/arch/x86_64/amdvi/mod.rs @@ -62,11 +62,9 @@ const FIRMWARE_LEFT: [(u64, Option, &str); 8] = [ (control::IOMMU_EN, None, "IommuEn"), ]; -/// Why a requester an IVHD special entry names takes no claim: an I/O APIC's -/// or an HPET's messages are the platform's (Table 109). -const PLATFORM: [&str; 2] = ["the IVRS names it as an I/O APIC", "the IVRS names it as an HPET"]; -/// Why a requester the SMI filter passes takes no claim. -const SMI_SOURCE: &str = "the unit's SMI filter, as firmware left it, passes its SMIs"; +/// Why a requester the SMI filter passes, which no special entry names, +/// takes no claim. +const SMI_SOURCE: &str = "firmware's SMI filter passes its SMIs"; /// IVMD flags (Table 113). const IVMD_UNITY: u8 = 1 << 0; @@ -138,8 +136,9 @@ struct Ready { regs: Mmio, function: PciDevice, features: Features, - /// The unit has an SMI filter, which [`program`] turns on. - filters_smi: bool, + /// What [`settle`] writes to the unit's SMI filter registers before it + /// turns the filter on; `None` where the unit filters no SMI. + smi: Option>, } /// Every unit the IVRS describes, switched off, and each this kernel can use @@ -218,12 +217,24 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) Ok(function) => { translated.extend(serves); apics.extend(specials.iter().filter(|(variety, ..)| *variety == 1).map(|&(_, handle, used)| (handle, used))); - platform.extend(specials.iter().map(|&(variety, _, used)| (used, PLATFORM[usize::from(variety == 2)]))); + // An I/O APIC's or an HPET's messages are the platform's (Table 109). + platform.extend(specials.iter().map(|&(variety, _, used)| { + let what = match variety { + 1 => "the IVRS names it as an I/O APIC", + 2 => "the IVRS names it as an HPET", + _ => "the IVRS names it as a platform source", + }; + (used, what) + })); let features = Features(regs.read_u64(reg::EXTENDED_FEATURES)); - let passes = smi_filter(units, regs, features); - let filters_smi = passes.is_some(); - platform.extend(passes.into_iter().flatten().map(|id| (id, SMI_SOURCE))); - ready.push(Ready { index: units, base: unit.base, regs, function, features, filters_smi }); + let chipset: Vec = specials.iter().map(|&(.., used)| used).collect(); + let smi = smi_filter(units, regs, features, &chipset); + for id in smi.iter().flatten().filter_map(|register| register.passes()) { + if !chipset.contains(&id) { + platform.push((id, SMI_SOURCE)); + } + } + ready.push(Ready { index: units, base: unit.base, regs, function, features, smi }); } Err(why) => log!("iommu: amdvi unit{units} is left off: {why}"), } @@ -302,17 +313,14 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) }; let identity = build(devices, &settings, remap.is_some()); let reserved = windows.iter().map(|w| (w.base, w.end())).chain(ivmds.iter().map(|m| (m.start, m.start + m.length))); - let filters_smi = ready.iter().all(|unit| unit.filters_smi); domain::start(reserved.collect(), settings, ivmds.iter().map(|m| m.requesters).collect(), translated, platform); - interrupt::smi_filtered(filters_smi); log!("iommu: amdvi identity domain covers 0x0..{top:#x}, root={:#x}", identity.get()); if let Some(extended) = remap { log!( "iommu: amdvi remaps every message through its requester's table: 128-bit entries, x2apic \ - destinations={}, {} I/O APIC(s) named, SMIs filtered={}", + destinations={}, {} I/O APIC(s) named", yn(extended), apics.len(), - yn(filters_smi) ); interrupt::start(extended, apics); } @@ -361,19 +369,21 @@ fn declare( Ok(Some((first, last))) } -/// The requester ids the unit's SMI filter passes, as firmware left it, each -/// register said; `None`, said, where the unit has no filter, so every SMI -/// passes (§2.1.5). -fn smi_filter(index: usize, regs: Mmio, features: Features) -> Option> { +/// The unit's SMI filter registers as [`settle`] is to write them: passing +/// what firmware left valid and every requester id the IVRS names the +/// chipset's own interrupt sources by, `chipset`, which are taken to raise +/// the platform's SMIs too, since §2.1.5.3 leaves naming those to firmware; +/// `None`, said, where the unit has no filter or too few registers left. +fn smi_filter(index: usize, regs: Mmio, features: Features, chipset: &[u16]) -> Option> { let Some(count) = features.smi_filters() else { log!("iommu: amdvi unit{index} has no SMI filter (EFR {:#018x}), so every SMI a device sends passes", features.0); return None; }; - let registers: Vec = + let mut registers: Vec = (0..count as u64).map(|i| SmiFilter(regs.read_u64(reg::SMI_FILTER + 8 * i))).collect(); for (i, register) in registers.iter().enumerate() { log!( - "iommu: amdvi unit{index} SMI filter {i} = {:#010x}: passes {}, {}", + "iommu: amdvi unit{index} SMI filter {i} = {:#010x} as firmware left it: passes {}, {}", register.0, match register.passes() { Some(id) => format!("{}", requester(id)), @@ -382,7 +392,17 @@ fn smi_filter(index: usize, regs: Mmio, features: Features) -> Option> if register.locked() { "locked" } else { "unlocked" } ); } - Some(registers.iter().filter_map(|register| register.passes()).collect()) + match reg::smi_filter(&mut registers, chipset) { + Ok(()) => Some(registers), + Err(id) => { + log!( + "iommu: amdvi unit{index} SMI filter has no register left to pass {}, a chipset source, so it \ + stays off and every SMI a device sends passes", + requester(id) + ); + None + } + } } /// The unit's function, where this driver can program the unit: segment 0, @@ -451,6 +471,8 @@ struct Live { tail: usize, done: Address<12>, waited: u64, + /// [`Ready::smi`], until [`settle`] writes it. + smi: Option>, } impl Live { @@ -502,7 +524,7 @@ fn submit(commands: &[Command]) { /// then empties every cache it holds and names each enumerated /// function's entry. fn program(unit: Ready, devices: &[PciDevice], remap: Option) { - let Ready { index, base, regs, function, filters_smi, .. } = unit; + let Ready { index, base, regs, function, smi, .. } = unit; let (commands, events, done) = { let mut tables = TABLES.lock(); (tables.alloc(), tables.alloc(), tables.alloc()) @@ -529,23 +551,19 @@ fn program(unit: Ready, devices: &[PciDevice], remap: Option) { Some(false) => control::GA_EN, Some(true) => control::GA_EN | control::XT_EN, }; - // An SMI is no remapped message (Table 10, Table 19): only the filter - // keeps one a holder writes into its own MSI-X table from every CPU. - let smi = if filters_smi { control::SMIF_EN | control::SMIF_LOG_EN } else { 0 }; let on = control::COHERENT | control::EVENT_LOG_EN | control::EVENT_INT_EN | control::CMD_BUF_EN | control::IOMMU_EN - | remapping - | smi; + | remapping; regs.write_u64(reg::CONTROL, on); let running = status::CMD_BUF_RUN | status::EVENT_LOG_RUN; spin_until( || regs.read_u32(reg::STATUS) & running == running, || format!("iommu: amdvi unit{index} never ran its command buffer and event log: STATUS={:#010x}", regs.read_u32(reg::STATUS)), ); - let mut live = Live { index, regs, commands: memory(commands.get(), PAGE_SIZE), tail: 0, done, waited: 0 }; + let mut live = Live { index, regs, commands: memory(commands.get(), PAGE_SIZE), tail: 0, done, waited: 0, smi }; // §2.4.8 needs `INVALIDATE_IOMMU_ALL` alone. QEMU 11.1.1 moves a function // off pass-through only on an `INVALIDATE_DEVTAB_ENTRY` that names it // (`issues/qemus-invalidate-iommu-all-leaves-every-function-untranslated.md`), @@ -565,6 +583,38 @@ fn program(unit: Ready, devices: &[PciDevice], remap: Option) { UNITS.lock().push(live); } +/// Every unit's SMI filter on, once every function this kernel drives is out +/// of firmware's hands: until then, a controller firmware still owns may +/// raise SMIs to be handed over (xHCI §4.22.1). A refused SMI is logged as an +/// `IO_PAGE_FAULT` (§2.1.5.1) for [`fault`]. +pub fn settle() { + let mut units = UNITS.lock(); + let mut everywhere = !units.is_empty(); + for unit in units.iter_mut() { + let Some(filter) = unit.smi.take() else { + everywhere = false; + continue; + }; + for (i, register) in (0u64..).zip(&filter) { + if !SmiFilter(unit.regs.read_u64(reg::SMI_FILTER + 8 * i)).locked() { + unit.regs.write_u64(reg::SMI_FILTER + 8 * i, register.0); + } + } + unit.regs.write_u64(reg::CONTROL, unit.regs.read_u64(reg::CONTROL) | control::SMIF_EN | control::SMIF_LOG_EN); + let passes: Vec = (0..filter.len() as u64) + .filter_map(|i| SmiFilter(unit.regs.read_u64(reg::SMI_FILTER + 8 * i)).passes()) + .map(|id| format!("{}", requester(id))) + .collect(); + log!( + "iommu: amdvi unit{} SMI filter on control={:#018x}, passing {}", + unit.index, + unit.regs.read_u64(reg::CONTROL), + passes.join(" ") + ); + } + interrupt::smi_filtered(everywhere); +} + /// A requester id, printed as `pci::enumerate` prints a function. fn requester(id: u16) -> StreamId { let [devfn, bus] = id.to_le_bytes(); diff --git a/kernel/src/arch/x86_64/iommu_unit.rs b/kernel/src/arch/x86_64/iommu_unit.rs index 84a001e239..59f471b28d 100644 --- a/kernel/src/arch/x86_64/iommu_unit.rs +++ b/kernel/src/arch/x86_64/iommu_unit.rs @@ -60,6 +60,14 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: &[RootBridgeWindow]) ); } +/// Every function the kernel drives is out of firmware's hands: AMD-Vi turns +/// its SMI filters on. +pub fn drivers_settled() { + if backend() == Backend::AmdVi { + super::amdvi::settle(); + } +} + /// Ceiling on units inventoried; a machine past it is told, not silently truncated. pub(super) const MAX_UNITS: usize = 16; @@ -285,15 +293,18 @@ pub mod interrupt { } } - /// Whether a message a holder writes into its function's MSI-X table - /// reaches nothing but its claim's interrupt: VT-d's source-id - /// verification and compatibility-format block say so; AMD-Vi's table - /// does only where every unit filters SMIs, which no table remaps. - pub fn fences_tables() -> bool { + /// Whether a claimed function's every message reaches only its own + /// interrupt: VT-d's source-id verification and its block on + /// compatibility format where it remaps; AMD-Vi's requester table where + /// it remaps and every unit filters SMIs, since a holder's write to the + /// interrupt range with an SMI's type is no remapped message (48882 + /// §2.1.4.2, Table 10, Table 19). + pub fn fences_claims() -> Result<(), crate::iommu::NotRemapped> { + use crate::iommu::NotRemapped; match backend() { - Backend::Vtd => true, - Backend::AmdVi => amdvi::interrupt::smi_filtered_everywhere(), - Backend::None => false, + _ if !is_armed() => Err(NotRemapped::Unremapped), + Backend::AmdVi if !amdvi::interrupt::smi_filtered_everywhere() => Err(NotRemapped::SmiUnfiltered), + _ => Ok(()), } } diff --git a/kernel/src/arch/x86_64/power.rs b/kernel/src/arch/x86_64/power.rs index ccf1c79f90..941d215c41 100644 --- a/kernel/src/arch/x86_64/power.rs +++ b/kernel/src/arch/x86_64/power.rs @@ -154,10 +154,13 @@ pub fn pm1a_control() -> Option { /// the FADT named. A machine with no reset register halts. Where the register /// is `SMI_CMD` this `out` is no write of `smi_cmd`'s: Table 5.9 asks the boot /// processor of a command to `SMI_CMD` and nothing of the reset, which is -/// made from whichever CPU ends the machine. +/// made from whichever CPU ends the machine. Every IOMMU's SMI filter is off +/// first: firmware may reset in its SMI handler, from an SMI its chipset +/// raises through the filter. // No fallback: 0xCF9, the keyboard controller and anything else are written only where a table named them. pub fn reset() -> ! { if let Some(reset) = RESET.get() { + super::amdvi::fault::pass_every_smi(); // SAFETY: the port `init_reset` decoded as an 8-bit System I/O register and declared, and the value is that register's. unsafe { cpu::outb(reset.port(0), RESET_VALUE.load(Ordering::Relaxed)) }; } @@ -211,6 +214,8 @@ pub fn off(Settled(taken): Settled) -> ! { } let typed = slp_typ.in_control(held & !SLP_EN); let smis_before = super::counters::read().smi; + // As for `reset`: firmware may enter S5 in its SMI handler. + super::amdvi::fault::pass_every_smi(); // SAFETY: the block `init_control` declared, and a `SLP_TYPa` the field holds: what it does is the firmware's, as its `\_S5` names it. unsafe { cpu::outw(control, typed); diff --git a/kernel/src/iommu/mod.rs b/kernel/src/iommu/mod.rs index 9304657955..b60994602a 100644 --- a/kernel/src/iommu/mod.rs +++ b/kernel/src/iommu/mod.rs @@ -286,6 +286,13 @@ pub fn init( unit::init(rsdp_addr, devices, windows); } +/// Every function the kernel drives has been taken from firmware, and no +/// claim has been granted yet: what the units guard against a holder may +/// now be guarded against firmware's handover too. +pub fn drivers_settled() { + unit::drivers_settled(); +} + /// Why a source could not be given a message: the architecture's own reasons. pub use messages::Refused; @@ -299,33 +306,22 @@ pub struct MsiMessage { #[derive(Clone, Copy)] pub struct Remapping(()); -impl Remapping { - /// Whether a message a holder writes into its function's MSI-X table can - /// reach nothing but that claim's own interrupt, so the BAR holding the - /// table may be handed over: on VT-d by source-id verification, on the - /// ITS by the device id the bridge stamps, and on AMD-Vi by the - /// requester's table where every unit also filters SMIs, which no table - /// remaps. - pub fn fences_tables(self) -> bool { - messages::fences_tables() - } +/// Why no claimed function's message would reach only its claim's interrupt. +#[derive(Clone, Copy)] +pub enum NotRemapped { + /// Nothing translates its message, so the message names any interrupt. + Unremapped, + /// Remapped, and a write its holder aims at the interrupt range could + /// still be an SMI no unit filters. + SmiUnfiltered, } -/// Nothing on this machine translates a claimed function's message to its -/// claim's own interrupt, so the message it would carry names any. -pub struct NotRemapped; - -/// Whether a claimed function's message can be remapped: a fact of the -/// machine, fixed before the first driver arms anything. -/// -/// Where it is, [`Remapping::fences_tables`] says whether whatever message a -/// holder writes into its function's own MSI-X table is fenced too. +/// Whether every message a claimed function sends, whatever its holder +/// writes into its MSI-X table or aims a DMA at, reaches only its claim's +/// interrupt: a fact of the machine, fixed before the first claim. Where it +/// is, the table's BAR is the holder's like any other. pub fn remapping() -> Result { - if messages::is_armed() { - Ok(Remapping(())) - } else { - Err(NotRemapped) - } + messages::fences_claims().map(|()| Remapping(())) } /// Whether `function` can be given a domain of its own and a remapping entry diff --git a/kernel/src/main.rs b/kernel/src/main.rs index a2a7abcf66..f1dd09760c 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -557,6 +557,10 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { boot_phase!("devices ready", t_devices); + // After every kernel driver took its functions from firmware, before + // userland runs and asks for a claim. + iommu::drivers_settled(); + report_log_destination(); let complete = cpu::counter(); boot_phase!("complete", 0); diff --git a/kernel/src/pcidev/mod.rs b/kernel/src/pcidev/mod.rs index 3e7e1e8957..32a694227f 100644 --- a/kernel/src/pcidev/mod.rs +++ b/kernel/src/pcidev/mod.rs @@ -49,14 +49,11 @@ //! that BAR rather than nothing at all, and a function this kernel could settle //! no BAR of ends at [`Refusal::NoMappableBar`]. //! -//! **The BAR holding the MSI-X table is handed over only where the machine -//! fences a written message** ([`crate::iommu::Remapping::fences_tables`]): -//! there, whatever message its holder writes into the table reaches only the -//! claim's own interrupt, and anything else is refused and recorded. On -//! AMD-Vi that needs every unit's SMI filter on, since an SMI is no remapped -//! message; where it is not, the BAR is kept back, said by name. **A function -//! is armed on MSI only where a walk that reached its capability list's -//! terminator found no MSI-X.** +//! **The BAR holding the MSI-X table is handed over like any other**: a claim +//! exists only where every message its function sends, whatever its holder +//! writes there or aims a DMA at, reaches only its own interrupt +//! ([`crate::iommu::remapping`]). **A function is armed on MSI only where a +//! walk that reached its capability list's terminator found no MSI-X.** //! //! **A function with no address space of its own is not handed over**, because //! every grant would answer with a physical address and a descriptor holding @@ -647,7 +644,7 @@ fn account_for(firmware: &[RootBridgeWindow], decoded: &[(u16, u64, u64)]) { /// place. #[derive(Clone, Copy)] enum Refusal { - NotRemapped, + NotRemapped(crate::iommu::NotRemapped), /// This machine remaps and translates, and not for this function. NotIsolated(&'static str), /// The machine translates messages, and none could be made for this function. @@ -684,11 +681,16 @@ enum Refusal { impl core::fmt::Display for Refusal { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { - Self::NotRemapped => write!( + Self::NotRemapped(crate::iommu::NotRemapped::Unremapped) => write!( f, "its interrupts would not be remapped on this machine, and a message that is not \ remapped can raise any vector on any CPU" ), + Self::NotRemapped(crate::iommu::NotRemapped::SmiUnfiltered) => write!( + f, + "an IOMMU unit on this machine filters no SMI, so a write its holder aimed at the \ + interrupt range could send one to every CPU" + ), Self::NotIsolated(why) => { write!(f, "no address space or message of its own can be made for it: {why}") } @@ -861,7 +863,7 @@ pub fn claim(id: PciId) -> Result<(PciFunctionInfo, Claim), ClaimError> { /// bus before its domain existed would be reaching physical memory with /// whatever addresses its registers still held. fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result { - let remapping = crate::iommu::remapping().map_err(|_| Refusal::NotRemapped)?; + let remapping = crate::iommu::remapping().map_err(Refusal::NotRemapped)?; crate::iommu::isolates(&pci).map_err(Refusal::NotIsolated)?; // What the function's previous holder left mapped goes before anything @@ -899,23 +901,7 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result // From here a refusal has to undo: a vector is armed, and the arms below // move the function's BARs. - // The table's BAR is the holder's only where a message written into it - // reaches nothing but this claim's interrupt. - let withheld = match &armed { - Armed::Msix(at, _) if !remapping.fences_tables() => { - log!( - "pcidev: PCI {:02x}:{:02x}.{} keeps BAR {}, which holds its MSI-X table, back: a message written \ - there would not be fenced on this machine", - pci.bus, - pci.dev, - pci.func, - at.bar - ); - Some(at.bar) - } - _ => None, - }; - match place_bars(&pci, id, withheld) { + match place_bars(&pci, id) { Ok((bar_at, bar_bytes)) => { space.attach(pci.bus, pci.dev, pci.func); Ok(Bound { @@ -959,7 +945,6 @@ fn bring_up(pci: PciDevice, id: PciId, number: usize) -> Result fn place_bars( pci: &PciDevice, id: PciId, - withheld: Option, ) -> Result<([u64; BARS], [u64; BARS]), Refusal> { let mut bar_at = [0u64; BARS]; let mut bar_bytes = [0u64; BARS]; @@ -973,7 +958,7 @@ fn place_bars( // whatever its register holds now: one a reset returned to zero reads // as unassigned, and it is the window that moved, not a BAR that is // not there. - if Some(index) == withheld || (pci.memory_bar(index).is_err() && !was_cut(pci, index)) { + if pci.memory_bar(index).is_err() && !was_cut(pci, index) { index += step; continue; } @@ -1841,18 +1826,25 @@ pub fn isr(number: usize) { /// answers refuses from here on, its interrupt read included, and the post /// wakes whoever waits on the claim to read that refusal, as a message's does. pub fn note_fault(who: u16) -> bool { - let Some(function) = functions().iter().find(|function| function.who == who) else { + let Some(function) = held(who) else { return false; }; - if !function.held.load(Ordering::Acquire) { - return false; - } function.irq.fault(); function.watch.post_in_place(); crate::preempt::set_need_resched(); true } +/// Requester `who`'s function, where a claim holds it; takes no lock. +fn held(who: u16) -> Option<&'static Function> { + functions().iter().find(|function| function.who == who && function.held.load(Ordering::Acquire)) +} + +/// Whether a claim holds requester `who`; takes no lock. +pub fn is_held(who: u16) -> bool { + held(who).is_some() +} + /// Register a poll on the watch of the function a claim holds. pub(crate) fn add_poll(binding: &Binding, entry: PollEntry) { binding.function().watch.add_poll(entry); diff --git a/tests/common/claims.rs b/tests/common/claims.rs index 2179a2cb95..2968025c17 100644 --- a/tests/common/claims.rs +++ b/tests/common/claims.rs @@ -19,7 +19,10 @@ pub const MSI: &str = "test_rs_i219_msi"; const MSI_SAYS: &str = "i219_msi: "; /// The kernel's reason on a claim's refusal line where the units do not remap. -const NOT_REMAPPED: &str = "its interrupts would not be remapped on this machine"; +pub const NOT_REMAPPED: &str = "its interrupts would not be remapped on this machine"; + +/// The kernel's reason where the units remap and one filters no SMI. +pub const SMI_UNFILTERED: &str = "an IOMMU unit on this machine filters no SMI"; /// The kernel's line where firmware's DMAR flags leave interrupt remapping off, /// which `iommu-no-remap` stands in for: the premise of the refusal. @@ -166,16 +169,15 @@ pub fn refused_unremapped(log: &Serial) -> Result<(), String> { log.must_say(INTR_REMAP_CLEAR)?; let at = function(log)?; log.must_not_say(&format!("[{I219}] handed over"))?; - refused_unremapped_at(log, &at)?; + refused_at(log, &at, NOT_REMAPPED)?; log.must_say(bootlog::COMPLETE)?; Ok(()) } -/// The claim of the function at `at` refused because nothing remaps its -/// messages, before anything on it changed: no message armed, no entry -/// written, no BAR moved. -pub fn refused_unremapped_at(log: &Serial, at: &str) -> Result<(), String> { - log.must_say(&format!("pcidev: PCI {at} NOT HANDED OVER — {NOT_REMAPPED}"))?; +/// The claim of the function at `at` refused for `why`, before anything on +/// it changed: no message armed, no entry written, no BAR moved. +pub fn refused_at(log: &Serial, at: &str, why: &str) -> Result<(), String> { + log.must_say(&format!("pcidev: PCI {at} NOT HANDED OVER — {why}"))?; log.must_not_say(&format!("PCI {at}: msi address="))?; log.must_not_say(&format!("PCI {at}: msix address="))?; log.must_not_say(&format!(" source={at} "))?; diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 15491fd1c9..97d9d471e1 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -685,11 +685,10 @@ pub enum Profile { /// function the kernel still drives itself beside the console, which is /// what `virtio-no-access-platform` withholds the bit from. HeadlessVirtioGpu, - /// [`Profile::Headless`] with a second controller, QEMU's `qemu-xhci` - /// on MSI and with no MSI-X, carrying a stick and a keyboard: the - /// controller `xhci-leave=1b36:000d` leaves to `usbd`, armed on MSI as - /// the T14's two are, because a claim never maps the BAR that holds an - /// MSI-X table and QEMU puts this one's in its registers' BAR. + /// [`Profile::Headless`] with a second controller at [`SPARE_AT`], QEMU's + /// `qemu-xhci` on MSI and with no MSI-X, carrying a stick and a keyboard: + /// the controller `xhci-leave=00:10.0` leaves to `usbd`, armed on MSI as + /// the T14's two are. HeadlessUsbSpare, /// M1 metal-sim: GOP, NVMe, xHCI with the boot stick on it, i8042 from /// q35, and nothing else -- no virtio device and no USB HID. This is the @@ -712,12 +711,8 @@ pub enum Profile { /// can be made and read. MetalAmdViTestdev, /// [`Profile::HeadlessUsbSpare`] behind QEMU's AMD unit remapping - /// interrupts: a claim's message through its own requester's table. + /// interrupts, which has no SMI filter. HeadlessUsbSpareAmdVi, - /// [`Profile::HeadlessUsbSpareAmdVi`] with its spare armed on MSI-X, the - /// table in its register BAR, as the AMD bench's controllers have theirs; - /// QEMU's unit has no SMI filter, so that BAR is kept back. - HeadlessUsbSpareAmdViMsix, /// [`Profile::HeadlessUsbSpareAmdVi`] with the unit remapping nothing. HeadlessUsbSpareAmdViNoRemap, /// QEMU `virt` on AArch64 (GICv3, AAVMF): a GOP from `ramfb`, the boot @@ -784,8 +779,7 @@ impl Profile { | Self::MetalAmdVi | Self::MetalAmdViTestdev | Self::HeadlessUsbSpareAmdVi - | Self::HeadlessUsbSpareAmdViNoRemap - | Self::HeadlessUsbSpareAmdViMsix => Arch::X86_64, + | Self::HeadlessUsbSpareAmdViNoRemap => Arch::X86_64, } } @@ -869,10 +863,7 @@ const XHCI_DEFAULT: &str = "nec-usb-xhci,id=xhci"; /// MSI left `auto` this QEMU's `qemu-xhci` offers neither, and the kernel /// refuses the claim for it. const XHCI_SPARE: &str = "qemu-xhci,id=spare,addr=0x10,msix=off,msi=on"; -/// [`XHCI_SPARE`] armed on MSI-X, whose table QEMU puts in the controller's -/// register BAR, as the AMD bench's controllers have theirs. -const XHCI_SPARE_MSIX: &str = "qemu-xhci,id=spare,addr=0x10"; -/// Where both put the spare, which `xhci-leave=` names. +/// Where [`XHCI_SPARE`] puts the spare, which `xhci-leave=` names. pub const SPARE_AT: &str = "00:10.0"; const SPARE_STICK: &str = "driver=null-co,node-name=spare-stick,size=67108864,read-zeroes=on"; @@ -1070,9 +1061,6 @@ impl Profile { Self::HeadlessUsbSpareAmdViNoRemap => { Shape { iommu: Some(Unit::AmdVi { intremap: false }), ..Self::HeadlessUsbSpareAmdVi.shape() } } - Self::HeadlessUsbSpareAmdViMsix => { - Shape { xhci: &[XHCI_DEFAULT, XHCI_SPARE_MSIX], ..Self::HeadlessUsbSpareAmdVi.shape() } - } Self::MetalAmdViTestdev => Shape { probes: Probes::Testdevs, ..Self::MetalAmdVi.shape() }, Self::HeadlessNoIommu => Shape { iommu: None, ..Self::Headless.shape() }, Self::HeadlessNoUsb => Shape { xhci: &[], usb: &[], ..Self::Headless.shape() }, diff --git a/tests/metal/src/lib.rs b/tests/metal/src/lib.rs index b31a7c4650..f09b0e9b4d 100644 --- a/tests/metal/src/lib.rs +++ b/tests/metal/src/lib.rs @@ -813,15 +813,18 @@ pub const FLASHABLE: &[&str] = &[ // claim, and `usbd` drives it through the unit: the handoff, a reset and // its own rings, as this kernel's driver does every boot. Firmware owns // neither of the T14's two (each reads `USBLEGSUP 0x01002201`, every SMI - // enable clear), and firmware takes the controller back at the next - // power-on. Only this address: the other controller carries the boot - // stick. + // enable clear). Whatever state a boot leaves it in, the reset that ends + // every boot resets the function and runs POST again, which takes it + // back. Only this address: the other controller carries the boot stick. "xhci-leave=00:0d.0", // The AMD bench's `03:00.4`, the xHCI the Bluetooth module is on, left to - // a claim the same way: firmware owns it until the handoff `usbd` - // performs as this kernel's driver does every boot (`USBLEGSUP - // 0x00010401`), and takes it back at the next power-on. Only this - // address: `03:00.3`, which shares its id, carries the boot stick. + // a claim the same way. Firmware owns it (`USBLEGSUP 0x00010401`) until + // `usbd` takes it; a boot may leave it halted with bus mastering off + // after the fault arm, or, where the claim is refused, still firmware's + // with its SMI on an ownership change armed. Either way the reset that + // ends every boot resets the function and runs POST again, which takes + // the controller back. Only this address: `03:00.3`, which shares its id, + // carries the boot stick. "xhci-leave=03:00.4", ]; diff --git a/tests/toyos.rs b/tests/toyos.rs index 963b7f05a8..cb6711023e 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -438,21 +438,16 @@ const MACHINE_TESTS: &[&str] = &[ // to fault. What usbd names on the bus is `toyos-xhci`'s and its // simulator's to hold. "usbd_drives_the_spare", - // The same behind QEMU's AMD unit remapping interrupts: the claim's - // message through an entry of its function's own table, which no host - // build has. The AMD bench's `amdvi_usbd_drives_its_own_entry` holds the - // same on silicon, with a foreign vector, which QEMU's unit cannot hand - // a table over for. - "usbd_drives_the_spare_amdvi", + // The same behind QEMU's AMD unit, remapping and with no SMI filter: the + // claim is refused by that reason before anything on the function + // changes. No metal machine in reach has an AMD-Vi unit without the + // filter; the AMD bench drives its claim in + // `amdvi_usbd_drives_its_own_entry`. + "usbd_refused_without_smi_filter", // The same machine with its AMD unit remapping nothing: the claim is - // refused before anything on the function changes. The AMD bench's arm is + // refused by that reason. The AMD bench's arm is // `amdvi_claim_refused_without_remapping`. "usbd_refused_without_amdvi_remapping", - // QEMU's AMD unit, which has no SMI filter, remapping a spare whose MSI-X - // table is in its register BAR: that BAR is kept back, said by name, and - // the claim is refused for want of another. No metal machine in reach has - // an AMD-Vi unit without the filter. - "usbd_table_kept_without_smi_filter", // A boot keyboard reporting more keys than its six slots name: the // kernel's driver reading a report a device delivered over xHCI has no host // build, and the T14 binds no USB keyboard. @@ -1124,7 +1119,7 @@ const METAL: &[(&str, metal::Metal)] = &[ judge: |b| { let kernel = b[0].kernel(); kernel.must_say("no message is remapped")?; - claims::refused_unremapped_at(&kernel, AMD_XHCI_SPARE) + claims::refused_at(&kernel, AMD_XHCI_SPARE, claims::NOT_REMAPPED) }, }, ), @@ -3792,10 +3787,19 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "desktop_without_x2apic" => desktop_without_x2apic(), "nvme_disk_boots" => nvme_disk_boots(test_config), "hosted_clang_hello" => common::hostedclang::hosted_clang_hello(), - "usbd_drives_the_spare" => usbd_drives_the_spare(test_config, qemu::Profile::HeadlessUsbSpare), - "usbd_drives_the_spare_amdvi" => usbd_drives_the_spare(test_config, qemu::Profile::HeadlessUsbSpareAmdVi), - "usbd_refused_without_amdvi_remapping" => usbd_refused_without_remapping(test_config), - "usbd_table_kept_without_smi_filter" => usbd_table_kept_without_smi_filter(test_config), + "usbd_drives_the_spare" => usbd_drives_the_spare(test_config), + "usbd_refused_without_smi_filter" => usbd_refused_on_amdvi( + test_config, + qemu::Profile::HeadlessUsbSpareAmdVi, + "has no SMI filter", + claims::SMI_UNFILTERED, + ), + "usbd_refused_without_amdvi_remapping" => usbd_refused_on_amdvi( + test_config, + qemu::Profile::HeadlessUsbSpareAmdViNoRemap, + "no message is remapped", + claims::NOT_REMAPPED, + ), "usb_keyboard_rollover" => usb_keyboard_rollover(test_config), "vector_state_survives_a_switch" => vector_state_survives_a_switch(test_config), "the_opening_exchange_with_qemus_agent_completes" => the_opening_exchange_with_qemus_agent_completes(), @@ -3883,12 +3887,13 @@ const USBD_NOOP: &str = "usbd: a No-Op command completed, announced by an interr /// claim is refused because the kernel drives the function. Between them, a /// holder of the function is killed and the claim asked for again at once, /// each round, which only a release finished before the end grants. -fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<(), String> { +fn usbd_drives_the_spare(test_config: &Path) -> Result<(), String> { const JOB: &str = "usbd_spare"; const SPARE: &str = "1b36:000d"; let tests = compile::repo_root().join("tests/guest/rust-tests"); let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, JOB); let ends = qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, END_RETURNS_CLAIMS); + let profile = qemu::Profile::HeadlessUsbSpare; let options = BootOptions { profile, kernel_params: &[XHCI_LEAVE_SPARE], @@ -3918,17 +3923,7 @@ fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<( if said.matches(&handed).count() != 1 { return Err(format!("{at} was not handed over once, to the one usbd:\n{said}")); } - let amdvi = matches!(profile.iommu(), Some(qemu::Unit::AmdVi { .. })); run.must_say(&format!("PCI {at}: msi address="))?; - if amdvi { - // The claim's entry in its own table, present at the vector the - // hand-over names, and absent again at the release. - let vector = said - .split_once(&handed) - .and_then(|(_, rest)| rest.split_whitespace().next()) - .ok_or_else(|| format!("the hand-over names no vector:\n{said}"))?; - run.must_say(&format!("iommu: amdvi irte source={at} vector={vector} p=1 "))?; - } let faulted = job_window(&mut qemu, "test_rs_usbd_spare fault")?; let fault = serial::Serial::named("usbd_spare fault", faulted.clone()); @@ -3942,10 +3937,8 @@ fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<( .find(|l| l.contains("iommu: DMA FAULT owner=process") && l.contains(&format!("stream={at} "))) .ok_or_else(|| format!("no DMA FAULT record names {at}:\n{faulted}"))?; let addr = record.split_once(" addr=").and_then(|(_, a)| a.split_whitespace().next()).unwrap_or_default(); - // QEMU 11.1.1's AMD unit records no address - // (`issues/qemus-amd-iommu-page-fault-record-carries-no-address.md`). let hex = |text: &str| u64::from_str_radix(text.trim_start_matches("0x"), 16).ok(); - if !amdvi && hex(addr) != hex(&aimed) { + if hex(addr) != hex(&aimed) { return Err(format!("the record faults {addr}, and the job aimed at {aimed}: {record}")); } eprintln!(" [usbd] {at}: handed over, its No-Op answered; aimed at {aimed}: {}", record.trim()); @@ -3969,45 +3962,16 @@ fn usbd_drives_the_spare(test_config: &Path, profile: qemu::Profile) -> Result<( Ok(()) } -/// Where an AMD-Vi unit has no SMI filter, the BAR holding a claim's MSI-X -/// table is kept back, said by name: an SMI is no remapped message, so a -/// holder's own table could send one to every CPU (48882 Table 10). -fn usbd_table_kept_without_smi_filter(test_config: &Path) -> Result<(), String> { +/// A machine behind QEMU's AMD unit, which `said` describes, refuses the +/// spare's claim for `why`, before a vector is armed or a BAR moved. +fn usbd_refused_on_amdvi(test_config: &Path, profile: qemu::Profile, said: &str, why: &str) -> Result<(), String> { const JOB: &str = "usbd_spare"; let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); - let options = BootOptions { - profile: qemu::Profile::HeadlessUsbSpareAmdViMsix, - kernel_params: &[XHCI_LEAVE_SPARE], - ..Default::default() - }; + let options = BootOptions { profile, kernel_params: &[XHCI_LEAVE_SPARE], ..Default::default() }; let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[(JOB.to_string(), bin)], options); - let boot = serial::Serial::boot(&qemu); - boot.must_say("iommu: amdvi unit0 has no SMI filter")?; - boot.must_say("SMIs filtered=n")?; - let at = qemu::SPARE_AT; - let said = job_window(&mut qemu, "test_rs_usbd_spare unhanded")?; - let run = serial::Serial::named("usbd_spare", said); - let kept = run.must_say(&format!("pcidev: PCI {at} keeps BAR 0, which holds its MSI-X table, back"))?; - run.must_say(&format!("pcidev: PCI {at} NOT HANDED OVER — it publishes no memory BAR this claim may map"))?; - eprintln!(" [usbd] {}", kept.trim()); - Ok(()) -} - -/// A machine whose AMD unit remaps nothing refuses the spare's claim by that -/// reason, before a vector is armed or a BAR moved. -fn usbd_refused_without_remapping(test_config: &Path) -> Result<(), String> { - const JOB: &str = "usbd_spare"; - let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); - let options = BootOptions { - profile: qemu::Profile::HeadlessUsbSpareAmdViNoRemap, - kernel_params: &[XHCI_LEAVE_SPARE], - ..Default::default() - }; - let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[(JOB.to_string(), bin)], options); - let boot = serial::Serial::boot(&qemu); - boot.must_say("no message is remapped")?; - let said = job_window(&mut qemu, "test_rs_usbd_spare unhanded")?; - claims::refused_unremapped_at(&serial::Serial::named("usbd_spare", said), qemu::SPARE_AT) + serial::Serial::boot(&qemu).must_say(said)?; + let run = job_window(&mut qemu, "test_rs_usbd_spare unhanded")?; + claims::refused_at(&serial::Serial::named("usbd_spare", run), qemu::SPARE_AT, why) } /// The AMD bench's `03:00.4` left to a claim and driven by usbd: handed over @@ -4071,7 +4035,6 @@ fn amdvi_usbd(kernel: &serial::Serial, log: &serial::Serial, after: &serial::Ser Ok(()) } - /// Every AMD-Vi unit translating, every xHCI controller the kernel drives /// moved to a domain of its own, the boot stick bound through one, and no /// DMA refused. @@ -4102,26 +4065,38 @@ fn amdvi_translates(kernel: &serial::Serial) -> Result<(), String> { Ok(()) } -/// Every AMD-Vi unit remapping (`GAEn`, MMIO 0018h bit 17), every I/O APIC -/// the MADT has named by the IVRS, and each xHCI controller the kernel drives -/// armed through a present entry of its own table at the vector its message -/// carries, the stick bound by those interrupts. +/// Every AMD-Vi unit remapping (`GAEn`, MMIO 0018h bit 17) and, once the +/// kernel's drivers are settled, filtering SMIs and logging each it refuses +/// (`SmiFEn` bit 22, `SmiFLogEn` bit 24); every I/O APIC the MADT has named +/// by the IVRS; and each xHCI controller the kernel drives armed through a +/// present entry of its own table at the vector its message carries, the +/// stick bound by those interrupts. fn amdvi_remaps(kernel: &serial::Serial) -> Result<(), String> { let text = kernel.text(); let remaps = kernel.must_say("iommu: amdvi remaps every message through its requester's table")?; + let control = |line: &str| { + line.split_once(" control=") + .and_then(|(_, rest)| u64::from_str_radix(rest.split([' ', ',']).next()?.trim_start_matches("0x"), 16).ok()) + .ok_or_else(|| format!("{line:?} names no control")) + }; let units: Vec<&str> = text.lines().filter(|l| l.contains("iommu: amdvi unit") && l.contains(" translating control=")).collect(); + if units.is_empty() { + return Err(format!("no AMD-Vi unit is translating:\n{text}")); + } for unit in &units { - let control = unit - .split_once(" control=") - .and_then(|(_, rest)| u64::from_str_radix(rest.split_whitespace().next()?.trim_start_matches("0x"), 16).ok()) - .ok_or_else(|| format!("{unit:?} names no control"))?; - if control & 1 << 17 == 0 { + if control(unit)? & 1 << 17 == 0 { return Err(format!("{unit:?} leaves GAEn clear")); } } - if units.is_empty() { - return Err(format!("no AMD-Vi unit is translating:\n{text}")); + let filters: Vec<&str> = text.lines().filter(|l| l.contains(" SMI filter on control=")).collect(); + if filters.len() != units.len() { + return Err(format!("{} unit(s) translate and {} filter SMIs: {filters:?}", units.len(), filters.len())); + } + for filter in &filters { + if control(filter)? & (1 << 22 | 1 << 24) != 1 << 22 | 1 << 24 { + return Err(format!("{filter:?} leaves SmiFEn or SmiFLogEn clear")); + } } let apics = text.lines().filter(|l| l.contains("ioapic: id=") && l.contains(" at 0x")).count(); if !remaps.contains(&format!(" {apics} I/O APIC(s) named")) { @@ -4142,6 +4117,12 @@ fn amdvi_remaps(kernel: &serial::Serial) -> Result<(), String> { } kernel.must_say("usb-storage: 1 device(s)")?; eprintln!(" [amdvi] {}", remaps.trim()); + for filter in filters { + eprintln!(" [amdvi] {}", filter.trim()); + } + for refused in text.lines().filter(|l| l.contains(" SMI filter refused an SMI from ")) { + eprintln!(" [amdvi] {}", refused.trim()); + } eprintln!(" [amdvi] {} controller(s) armed through their own entries, the stick bound", controllers.len()); Ok(()) } From 857b653333942ac3127daaf93f6ceab19e5603b9 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 20:40:09 +0200 Subject: [PATCH 6/9] toyos-amdvi: an SMI's record may carry the interrupt range sign-extended MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The round-3 panic on the AMD bench, photographed on its panel, shows the unit's IO_PAGE_FAULT for a message it refused from 00:14.0 at 0xfffffffdf8000020: bits 63:40 all ones, the interrupt range as 48882 §2.2.5 writes its base (FFFF_FFFD_F800_0000h), where Table 4 gives an SMI's 47:40 as zero. `smi` now takes 63:40 all zero or all ones; any other value there is no SMI's. The host test holds both forms and a partial extension. That record's MT reads 000b, not Table 4's SMI 010b, so this decoder still reads it as a requester's fault; whether the filter or the remapping table refused it is not settled by one reading (the PR says so). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016UzbDnM9m6nQWhbjGi6Lv1 --- kernel/amdvi/src/queue.rs | 13 ++++++++----- kernel/amdvi/tests/encodings.rs | 10 +++++++--- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/kernel/amdvi/src/queue.rs b/kernel/amdvi/src/queue.rs index 33ce1082b8..8aabca1dec 100644 --- a/kernel/amdvi/src/queue.rs +++ b/kernel/amdvi/src/queue.rs @@ -75,12 +75,15 @@ pub const fn event(words: [u64; 2]) -> Event { } /// Whether a record's address is an SMI's (§2.1.5.2, Figure 3 and Table 4): -/// bits 47:40 zero, `FDF8h` at 39:24, `MT` 0010b at 7 and 4:2, `DM` and -/// bits 1:0 zero; bits 63:48 are not the figure's and are not read. The SMI -/// filter records a refused SMI as an `IO_PAGE_FAULT` at that address -/// (§2.1.5.1). +/// `FDF8h` at 39:24, `MT` 0010b at 7 and 4:2, `DM` and bits 1:0 zero. The +/// SMI filter records a refused SMI as an `IO_PAGE_FAULT` at that address +/// (§2.1.5.1). Bits 63:40 are Table 4's zero or, as the unit writes the +/// interrupt range (`FFFF_FFFD_F800_0000h`, §2.2.5), all ones. const fn smi(address: u64) -> bool { - address & 0x0000_FFFF_FF00_00C3 == 0x0000_00FD_F800_0000 && (address >> 2) & 0b111 == 0b010 + let above = address >> 40; + (above == 0 || above == 0xFF_FFFF) + && address & 0xFF_FF00_00C3 == 0xFD_F800_0000 + && (address >> 2) & 0b111 == 0b010 } /// The name 48882 gives an event code, for a record's line. diff --git a/kernel/amdvi/tests/encodings.rs b/kernel/amdvi/tests/encodings.rs index 7ad507591f..b9c6aa004e 100644 --- a/kernel/amdvi/tests/encodings.rs +++ b/kernel/amdvi/tests/encodings.rs @@ -282,20 +282,24 @@ fn the_smi_filter_sits_where_48882_puts_it() { /// Figure 3 / Table 4, as the SMI filter records a refused SMI (§2.1.5.1): /// an `IO_PAGE_FAULT` whose address is `FDF8h` at [39:24], `MT[2:0]` 010b at -/// [4:2], `MT[3]` [7] and `DM` [6] zero, [1:0] and [47:40] zero; the vector -/// [23:16], destination [15:8], `TM` [5] and [63:48] are not read. +/// [4:2], `MT[3]` [7] and `DM` [6] zero, [1:0] zero, and [63:40] all zero +/// or, as the unit writes the interrupt range (§2.2.5), all ones; the +/// vector [23:16], destination [15:8] and `TM` [5] are not read. #[test] fn an_smi_the_filter_refused_is_told_from_a_dma_fault() { let fault = |address: u64| event([word(&[(15, 0, 0x00A3), (63, 60, 2)]), address]); let smi_at = word(&[(39, 24, 0xFDF8), (4, 2, 0b010)]); assert_eq!(fault(smi_at), Event::RefusedSmi { device: 0x00A3, address: smi_at }); - let ignored = smi_at | word(&[(23, 16, 0xFF), (15, 8, 0xFF), (5, 5, 1), (63, 48, 0xFFFF)]); + let ignored = smi_at | word(&[(23, 16, 0xFF), (15, 8, 0xFF), (5, 5, 1)]); assert_eq!(fault(ignored), Event::RefusedSmi { device: 0x00A3, address: ignored }); + let extended = smi_at | word(&[(63, 40, 0xFF_FFFF)]); + assert_eq!(fault(extended), Event::RefusedSmi { device: 0x00A3, address: extended }); for not_smi in [ word(&[(39, 24, 0xFDF8), (4, 2, 0b000)]), smi_at | word(&[(7, 7, 1)]), smi_at | word(&[(6, 6, 1)]), smi_at | word(&[(40, 40, 1)]), + smi_at | word(&[(63, 48, 0xFFFF)]), 0x4000_0000_0000, 0xFEE0_0008, ] { From a8791b689df54f24efe92b7ae7991d249dc18118 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 20:52:20 +0200 Subject: [PATCH 7/9] usbd_spare: the spare left to a claim is taken by value The merge's claim by arrival bound one arm's `Pci` by reference twice and the other once; the guest job did not build. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016UzbDnM9m6nQWhbjGi6Lv1 --- tests/guest/rust-tests/src/usbd_spare.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/guest/rust-tests/src/usbd_spare.rs b/tests/guest/rust-tests/src/usbd_spare.rs index f3b4ec859e..ab840c86a8 100644 --- a/tests/guest/rust-tests/src/usbd_spare.rs +++ b/tests/guest/rust-tests/src/usbd_spare.rs @@ -78,7 +78,7 @@ fn claim(cap: &SysCap) -> (PciId, Result) { _ => None, }) .collect(); - let left: Vec<_> = spares.iter().filter(|pci| pci.driven != Driven::Kernel).collect(); + let left: Vec<_> = spares.iter().copied().filter(|pci| pci.driven != Driven::Kernel).collect(); let pci = match (left.as_slice(), spares.as_slice()) { ([one], _) | ([], [one]) => *one, _ => panic!("usbd_spare: no one spare controller on this machine: {spares:?}"), From 1ceab43bcd47f9464632668378210c0c2ff84bf4 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 14 Oct 2026 23:51:37 +0200 Subject: [PATCH 8/9] amdvi: a message refused from a platform requester id is the platform's, whatever its type The orchestrator's ruling on #921 (6179195333). The round-3 panic's record was an IO_PAGE_FAULT from 00:14.0, the id the IVRS gives the chipset's I/O APIC and HPET, at 0xfffffffdf8000020: the interrupt range, `MT` 000b. Read as a DMA fault on a kernel-owned stream, it panicked the kernel. No kernel driver drives a platform id and a claim on one is refused (`domain::unclaimable`), so nothing of this kernel's went wrong. `toyos_amdvi::queue::event` decodes any IO_PAGE_FAULT at an interrupt-range address (Table 3, either address form) as `Event::Interrupt`, and `disposition` says whose it is: a platform id's whatever its type, a held function's its holder's fault, an SMI from anyone else said, any other a fault. The drain reads the platform ids from a lock-free table `init` publishes before any unit is armed, and says a platform message or an SMI with its id, type and address, bounded to powers of two. Host tests: the decoding of both address forms, and the disposition of the round-3 record from 00:14.0 (platform) and from anyone else (fault). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016UzbDnM9m6nQWhbjGi6Lv1 --- kernel/amdvi/src/queue.rs | 62 ++++++++++++++---- kernel/amdvi/tests/encodings.rs | 56 +++++++++------- kernel/src/arch/x86_64/amdvi/fault.rs | 92 ++++++++++++++++++--------- kernel/src/arch/x86_64/amdvi/mod.rs | 1 + tests/toyos.rs | 2 +- 5 files changed, 148 insertions(+), 65 deletions(-) diff --git a/kernel/amdvi/src/queue.rs b/kernel/amdvi/src/queue.rs index 8aabca1dec..0024a90c12 100644 --- a/kernel/amdvi/src/queue.rs +++ b/kernel/amdvi/src/queue.rs @@ -48,9 +48,10 @@ pub enum Event { /// `RX` clear (Figure 64). `write` is the access where the record states /// one; `address` is the record's, whatever it names. Requester { code: u8, device: u16, address: u64, write: Option }, - /// An `IO_PAGE_FAULT` at an SMI's address: an SMI from `device` the - /// filter refused (§2.1.5.1), no access of memory. - RefusedSmi { device: u16, address: u64 }, + /// An `IO_PAGE_FAULT` at an address in the interrupt range: a message + /// from `device` the unit refused, an SMI the filter refused among them + /// (§2.1.5.1), no access of memory. [`disposition`] says whose it is. + Interrupt { device: u16, address: u64 }, /// Every other record: an entry or command of this driver's the unit /// refused, or the unit's own hardware failing. Unit { code: u8 }, @@ -61,7 +62,7 @@ pub const fn event(words: [u64; 2]) -> Event { let code = (words[0] >> CODE_SHIFT) as u8; let write = match code { // `I` [51]: an interrupt carries no read or write; else `RW` [53]. - 0b0010 if smi(words[1]) => return Event::RefusedSmi { device, address: words[1] }, + 0b0010 if interrupt_range(words[1]) => return Event::Interrupt { device, address: words[1] }, 0b0010 if words[0] & 1 << 51 != 0 => None, 0b0010 => Some(words[0] & 1 << 53 != 0), // `RX` [48] set is a `COMPLETE_PPR_REQUEST` of this driver's failing. @@ -74,16 +75,51 @@ pub const fn event(words: [u64; 2]) -> Event { Event::Requester { code, device, address: words[1], write } } -/// Whether a record's address is an SMI's (§2.1.5.2, Figure 3 and Table 4): -/// `FDF8h` at 39:24, `MT` 0010b at 7 and 4:2, `DM` and bits 1:0 zero. The -/// SMI filter records a refused SMI as an `IO_PAGE_FAULT` at that address -/// (§2.1.5.1). Bits 63:40 are Table 4's zero or, as the unit writes the -/// interrupt range (`FFFF_FFFD_F800_0000h`, §2.2.5), all ones. -const fn smi(address: u64) -> bool { +/// Whether a record's address is in the interrupt range, `FD_F8xx_xxxxh` +/// (Table 3): bits 63:40 Table 4's zero or, as the unit writes the range's +/// base (`FFFF_FFFD_F800_0000h`, §2.2.5), all ones. +const fn interrupt_range(address: u64) -> bool { let above = address >> 40; - (above == 0 || above == 0xFF_FFFF) - && address & 0xFF_FF00_00C3 == 0xFD_F800_0000 - && (address >> 2) & 0b111 == 0b010 + (above == 0 || above == 0xFF_FFFF) && address & 0xFF_FF00_0000 == 0xFD_F800_0000 +} + +/// A message's type, `MT[3]` at bit 7 over `MT[2:0]` at 4:2 (Figure 3). +pub const fn message_type(address: u64) -> u8 { + ((address >> 4) & 0b1000 | (address >> 2) & 0b111) as u8 +} + +/// Whether an interrupt-range address is an SMI's (§2.1.5.2, Table 4): `MT` +/// 0010b, `DM` and bits 1:0 zero. +const fn smi(address: u64) -> bool { + message_type(address) == 0b0010 && address & 0b100_0011 == 0 +} + +/// Whose an [`Event::Interrupt`] is. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Disposition { + /// From a requester id the IVRS gives the platform, whatever its type: + /// the platform's message, said, and nothing of the kernel's went wrong. + Platform, + /// From a function a claim holds: its holder's fault. + Holder, + /// An SMI the filter refused from any other requester: said. + RefusedSmi, + /// Any other message the unit refused: a fault on its requester. + Fault, +} + +/// What an interrupt-range record at `address` ends, from a requester the +/// platform speaks as (`platform`) or a claim holds (`held`). +pub const fn disposition(address: u64, platform: bool, held: bool) -> Disposition { + if platform { + Disposition::Platform + } else if held { + Disposition::Holder + } else if smi(address) { + Disposition::RefusedSmi + } else { + Disposition::Fault + } } /// The name 48882 gives an event code, for a record's line. diff --git a/kernel/amdvi/tests/encodings.rs b/kernel/amdvi/tests/encodings.rs index b9c6aa004e..22f20027a2 100644 --- a/kernel/amdvi/tests/encodings.rs +++ b/kernel/amdvi/tests/encodings.rs @@ -5,7 +5,7 @@ use toyos_amdvi::dte::{interrupts_blocked, interrupts_remapped, Dte, Setting, DEVICE_IDS, ENTRY_BYTES, INTERRUPT_ENTRIES}; use toyos_amdvi::irte::{self, Destination}; -use toyos_amdvi::queue::{event, name, Command, Event, ENTRIES}; +use toyos_amdvi::queue::{disposition, event, message_type, name, Command, Disposition, Event, ENTRIES}; use toyos_amdvi::reg::{control, device_table, offset, ring, slot, smi_filter, Features, SmiFilter, SMI_FILTER}; use toyos_amdvi::table::{directory, entry, index, leaf, Entry, LEAF_LEVEL, ROOT_LEVEL}; use toyos_phys::Phys; @@ -280,30 +280,42 @@ fn the_smi_filter_sits_where_48882_puts_it() { assert_eq!((unset.passes(), unset.locked()), (None, false)); } -/// Figure 3 / Table 4, as the SMI filter records a refused SMI (§2.1.5.1): -/// an `IO_PAGE_FAULT` whose address is `FDF8h` at [39:24], `MT[2:0]` 010b at -/// [4:2], `MT[3]` [7] and `DM` [6] zero, [1:0] zero, and [63:40] all zero -/// or, as the unit writes the interrupt range (§2.2.5), all ones; the -/// vector [23:16], destination [15:8] and `TM` [5] are not read. +/// Table 3: an `IO_PAGE_FAULT` at an address in the interrupt range, +/// `FDF8h` at [39:24] with [63:40] all zero (Table 4) or all ones (§2.2.5's +/// base), is a refused message, whatever its type; anything else is a +/// requester's access. #[test] -fn an_smi_the_filter_refused_is_told_from_a_dma_fault() { +fn a_refused_message_is_told_from_a_dma_fault() { let fault = |address: u64| event([word(&[(15, 0, 0x00A3), (63, 60, 2)]), address]); let smi_at = word(&[(39, 24, 0xFDF8), (4, 2, 0b010)]); - assert_eq!(fault(smi_at), Event::RefusedSmi { device: 0x00A3, address: smi_at }); - let ignored = smi_at | word(&[(23, 16, 0xFF), (15, 8, 0xFF), (5, 5, 1)]); - assert_eq!(fault(ignored), Event::RefusedSmi { device: 0x00A3, address: ignored }); - let extended = smi_at | word(&[(63, 40, 0xFF_FFFF)]); - assert_eq!(fault(extended), Event::RefusedSmi { device: 0x00A3, address: extended }); - for not_smi in [ - word(&[(39, 24, 0xFDF8), (4, 2, 0b000)]), - smi_at | word(&[(7, 7, 1)]), - smi_at | word(&[(6, 6, 1)]), - smi_at | word(&[(40, 40, 1)]), - smi_at | word(&[(63, 48, 0xFFFF)]), - 0x4000_0000_0000, - 0xFEE0_0008, - ] { - assert!(matches!(fault(not_smi), Event::Requester { .. }), "{not_smi:#x}"); + let fixed = word(&[(63, 40, 0xFF_FFFF), (39, 24, 0xFDF8), (5, 5, 1)]); + for message in [smi_at, smi_at | word(&[(23, 16, 0xFF), (15, 8, 0xFF), (5, 5, 1)]), smi_at | word(&[(63, 40, 0xFF_FFFF)]), fixed] { + assert_eq!(fault(message), Event::Interrupt { device: 0x00A3, address: message }, "{message:#x}"); + } + for access in [smi_at | word(&[(40, 40, 1)]), smi_at | word(&[(63, 48, 0xFFFF)]), word(&[(39, 24, 0xFDF9)]), 0x4000_0000_0000, 0xFEE0_0008] { + assert!(matches!(fault(access), Event::Requester { .. }), "{access:#x}"); + } + assert_eq!(message_type(smi_at | word(&[(7, 7, 1)])), 0b1010); +} + +/// The ruling on #921: an interrupt-range record from an id the IVRS gives +/// the platform is the platform's, whatever its type; the round-3 panic's +/// record (`00:14.0`, `0xfffffffdf8000020`, `MT` 000b) among them. From +/// anyone else, a claim's is its holder's, an SMI (Table 4: `MT` 0010b, `DM` +/// and [1:0] zero) is said, and any other is a fault. +#[test] +fn a_platform_message_is_the_platforms_whatever_its_type() { + let round3 = 0xFFFF_FFFD_F800_0020; + assert_eq!(message_type(round3), 0b0000); + assert_eq!(disposition(round3, true, false), Disposition::Platform); + assert_eq!(disposition(round3, false, false), Disposition::Fault); + assert_eq!(disposition(round3, false, true), Disposition::Holder); + let smi = word(&[(39, 24, 0xFDF8), (4, 2, 0b010)]); + assert_eq!(disposition(smi, true, true), Disposition::Platform); + assert_eq!(disposition(smi, false, true), Disposition::Holder); + assert_eq!(disposition(smi, false, false), Disposition::RefusedSmi); + for not_smi in [smi | word(&[(7, 7, 1)]), smi | word(&[(6, 6, 1)]), smi | 1] { + assert_eq!(disposition(not_smi, false, false), Disposition::Fault, "{not_smi:#x}"); } } diff --git a/kernel/src/arch/x86_64/amdvi/fault.rs b/kernel/src/arch/x86_64/amdvi/fault.rs index cf56a56ec8..16bd5f8991 100644 --- a/kernel/src/arch/x86_64/amdvi/fault.rs +++ b/kernel/src/arch/x86_64/amdvi/fault.rs @@ -14,7 +14,7 @@ use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use crate::iommu::fault::{self as policy, Access, Fault, Who}; use crate::log; use crate::mm::{DirectMap, Mmio, PAGE_SIZE}; -use toyos_amdvi::queue::{event, name, Event, ENTRIES}; +use toyos_amdvi::queue::{disposition, event, message_type, name, Disposition, Event, ENTRIES}; use toyos_amdvi::reg::{self, control, status}; use super::{requester, spin_until, REGISTER_WINDOW}; @@ -27,12 +27,38 @@ struct FaultUnit { log: AtomicU64, /// Faults this unit has reported for the life of the boot. faults: AtomicU32, - /// SMIs its filter refused from a requester no claim holds. - smis: AtomicU32, + /// Messages it refused that were said rather than faulted: the + /// platform's, and SMIs from a requester no claim holds. + said: AtomicU32, } static UNITS: [FaultUnit; MAX_UNITS] = - [const { FaultUnit { regs: AtomicU64::new(0), log: AtomicU64::new(0), faults: AtomicU32::new(0), smis: AtomicU32::new(0) } }; MAX_UNITS]; + [const { FaultUnit { regs: AtomicU64::new(0), log: AtomicU64::new(0), faults: AtomicU32::new(0), said: AtomicU32::new(0) } }; MAX_UNITS]; + +/// The requester ids the platform speaks as, each `id | PRESENT`, for the +/// handler to read without a lock. +static PLATFORM: [AtomicU32; PLATFORM_IDS] = [const { AtomicU32::new(0) }; PLATFORM_IDS]; +const PLATFORM_IDS: usize = 32; +const PRESENT: u32 = 1 << 16; + +/// Publish the requester ids the platform speaks as, before any unit is +/// armed; past [`PLATFORM_IDS`] the rest are said and read as anyone's. +pub(super) fn platform(ids: impl Iterator) { + let mut ids = ids.peekable(); + for slot in &PLATFORM { + match ids.next() { + Some(id) => slot.store(u32::from(id) | PRESENT, Ordering::Relaxed), + None => return, + } + } + if ids.peek().is_some() { + log!("iommu: amdvi has more than {PLATFORM_IDS} platform requester ids; a message from the rest is read as anyone's"); + } +} + +fn is_platform(id: u16) -> bool { + PLATFORM.iter().any(|slot| slot.load(Ordering::Relaxed) == u32::from(id) | PRESENT) +} /// Publish unit `index`'s registers and event log to the handler, before its /// interrupt is armed. @@ -76,33 +102,41 @@ fn drain(index: usize, regs: Mmio, log: Mmio) -> usize { while head != tail { let at = reg::offset(head); let words = [log.read_u64(at), log.read_u64(at + 8)]; - let event = event(words); - let held = match event { - Event::RefusedSmi { device, .. } => crate::pcidev::is_held(device), - _ => false, - }; - let fault = match event { - // The filter kept it from every CPU (§2.1.5.1): a claimed - // function's is its holder's fault; any other's is said and the - // machine goes on, since nothing of this kernel's went wrong. - Event::RefusedSmi { device, address } if !held => { - let refused = UNITS[index].smis.fetch_add(1, Ordering::Relaxed) + 1; - if refused.is_power_of_two() { - log!( - "iommu: amdvi unit{index} SMI filter refused an SMI from {} at {address:#018x}, {refused} so far", - requester(device) - ); + let fault = match event(words) { + Event::Interrupt { device, address } => { + let who = requester(device); + let said = |what: &str| { + let n = UNITS[index].said.fetch_add(1, Ordering::Relaxed) + 1; + if n.is_power_of_two() { + log!( + "iommu: amdvi unit{index} refused {what} from {who}, type {:#06b} at {address:#018x}, {n} said so far", + message_type(address) + ); + } + }; + match disposition(address, is_platform(device), crate::pcidev::is_held(device)) { + // No kernel driver drives a platform id and no claim + // may take one (`domain::unclaimable`), so nothing of + // this kernel's went wrong. + Disposition::Platform => { + said("a platform message"); + head = (head + 1) % ENTRIES; + continue; + } + Disposition::RefusedSmi => { + said("an SMI"); + head = (head + 1) % ENTRIES; + continue; + } + Disposition::Holder | Disposition::Fault => Fault { + who: Who::Function(who), + address, + access: Access::Unrecorded, + reason: 0b0010, + name: name(0b0010), + }, } - head = (head + 1) % ENTRIES; - continue; } - Event::RefusedSmi { device, address } => Fault { - who: Who::Function(requester(device)), - address, - access: Access::Write, - reason: 0b0010, - name: "an SMI the filter refused", - }, Event::Requester { code, device, address, write } => Fault { who: Who::Function(requester(device)), address, diff --git a/kernel/src/arch/x86_64/amdvi/mod.rs b/kernel/src/arch/x86_64/amdvi/mod.rs index 5a680e64b4..b4cb79fd62 100644 --- a/kernel/src/arch/x86_64/amdvi/mod.rs +++ b/kernel/src/arch/x86_64/amdvi/mod.rs @@ -316,6 +316,7 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: Option<&[RootBridgeW let reserved = windows.map(|w| { w.iter().map(|w| (w.base, w.end())).chain(ivmds.iter().map(|m| (m.start, m.start + m.length))).collect() }); + fault::platform(platform.iter().map(|&(id, _)| id)); domain::start(reserved, settings, ivmds.iter().map(|m| m.requesters).collect(), translated, platform); log!("iommu: amdvi identity domain covers 0x0..{top:#x}, root={:#x}", identity.get()); if let Some(extended) = remap { diff --git a/tests/toyos.rs b/tests/toyos.rs index 203b44c647..c1ed50bdf0 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -4150,7 +4150,7 @@ fn amdvi_remaps(kernel: &serial::Serial) -> Result<(), String> { for filter in filters { eprintln!(" [amdvi] {}", filter.trim()); } - for refused in text.lines().filter(|l| l.contains(" SMI filter refused an SMI from ")) { + for refused in text.lines().filter(|l| l.contains("iommu: amdvi unit") && l.contains(" said so far")) { eprintln!(" [amdvi] {}", refused.trim()); } eprintln!(" [amdvi] {} controller(s) armed through their own entries, the stick bound", controllers.len()); From 4b052d558b35bd3a7c3388c3133dbd400dcd36a8 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 15 Oct 2026 00:23:55 +0200 Subject: [PATCH 9/9] Landing re-review: records without the withheld table BAR, the MSI-X issue closed into P1, `I` read as an interrupt B1: `device_bar_map`'s contract names no withheld BAR; on a machine that would not fence a message the claim is refused whole. B2: `issues/a-claims-own-refusals-are-read-by-nothing.md` loses the withheld-table bullet and exit clause; its subject, a claim's refusals read by nothing, stands. `every-driver-is-still-in-the-kernel.md` and `the-kernel-is-small-interrupts-post-and-threads-wait.md` no longer cite `pcidev::msix_bar` or a claim refused for its table's BAR. B3, ruled: `issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md` closes into P1 of `issues/there-is-no-wifi.md`, whose `03:00.4` bullet carries the dependency and its reason; P1's bullet rewriting that issue's exit goes, and so does the hypervisor issue's citation of it. The rule it leaves: a claim exists only where every message its function sends reaches only its own interrupt, and there the table's BAR is placed like any other (`kernel/src/pcidev/mod.rs`'s header). S1: an `IO_PAGE_FAULT` with `I` [51] set is `Event::Interrupt` whatever address it names (Table 57), so a refused platform message in another address form is the platform's too. S2: the platform ids are published once each. S3: `drivers_settled`'s doc and the claim's re-claim case. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016UzbDnM9m6nQWhbjGi6Lv1 --- ...claims-own-refusals-are-read-by-nothing.md | 15 +------- ...in-bar-0-cannot-be-driven-from-userland.md | 34 ------------------- issues/every-driver-is-still-in-the-kernel.md | 11 +++--- ...-small-interrupts-post-and-threads-wait.md | 10 +++--- issues/there-is-no-wifi.md | 4 --- ...oyos-runs-in-any-open-source-hypervisor.md | 4 --- kernel/amdvi/src/queue.rs | 14 ++++---- kernel/amdvi/tests/encodings.rs | 5 ++- kernel/src/arch/x86_64/amdvi/interrupt.rs | 6 ++-- kernel/src/arch/x86_64/amdvi/mod.rs | 5 ++- kernel/src/iommu/mod.rs | 5 ++- sdk/abi/src/syscall.rs | 4 +-- 12 files changed, 32 insertions(+), 85 deletions(-) delete mode 100644 issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md diff --git a/issues/a-claims-own-refusals-are-read-by-nothing.md b/issues/a-claims-own-refusals-are-read-by-nothing.md index ae50f83338..9930d1f940 100644 --- a/issues/a-claims-own-refusals-are-read-by-nothing.md +++ b/issues/a-claims-own-refusals-are-read-by-nothing.md @@ -22,17 +22,6 @@ Refusals of `kernel/src/pcidev/mod.rs` that no test reaches: - every bound `SYS_DEVICE_BAR_MAP` and `SYS_DEVICE_DMA_ALLOC` check, which is `bar_object`'s index and zero-length bounds and `dma_alloc`'s `MAX_GRANT_BYTES` and `MAX_GRANT_TOTAL`; -- the withholding of the MSI-X table's own BAR on the *ordinary* path: - `place_bars`'s `Some(index) == table_bar` (`kernel/src/pcidev/mod.rs`) - mutated to `(Some(index) == table_bar && false)` — the spelling that builds, - where a bare `false` is `unused variable: table_bar` — hands that BAR to the - holder of every function that publishes a table, and `https_tls13`, - `https_tls13_e1000e`, `pci_function_is_exclusive` and `userdev_dma_fault` are - each green on it. A holder that can write the - table aims the device's message at any address the LAPIC decodes, and - `msix_bar` is named as the boundary in the module header while nothing reads - it back. A refused claim is not the arm that covers this: the refusal spends - no BAR at all, so what is unread is the hand-over that succeeds. A one-field mutation of any of them — an index bound compared `<=` rather than `<`, a window overlap accepted, a grant total never summed — leaves every arm @@ -42,6 +31,4 @@ Owned by whoever next adds a boot config whose own test binary holds a claimable function: netd holds this machine's only claim, and a test that makes it fail this way costs the machine its NIC for that boot. Exit condition: a guest arm in which a test binary's claim is refused each of these ways, each one red against -a kernel whose corresponding check answers `Ok` — and, for the withheld table -BAR, one that holds a function publishing a table and is told that BAR's size -is zero. +a kernel whose corresponding check answers `Ok`. diff --git a/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md b/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md deleted file mode 100644 index f667989a7f..0000000000 --- a/issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -status: assigned -kind: defect -opened: 2026-09-26 ---- - -# A controller whose MSI-X table is in BAR 0 cannot be driven from userland - -Held by the orchestrator, with P1 of `issues/there-is-no-wifi.md`. Since -PR #921 (AMD-Vi stage 6), under the orchestrator's rulings there, a claim exists only where every message its function sends, whatever its -holder writes into its MSI-X table or aims a DMA at, reaches only its own -interrupt, and there `pcidev` hands over the table's BAR like any other. -VT-d holds that by source-id verification and its block on -compatibility-format messages, the ITS by the device id the bridge stamps, -and AMD-Vi by the requester's own table together with every unit's SMI -filter, since a write to the interrupt range is an interrupt (48882 -§2.1.4.2) and an SMI no remapped message (Table 10). Where an AMD-Vi unit -filters no SMI, every claim is refused by that reason. - -QEMU's AMD unit has no SMI filter, so QEMU drives no claim behind it: -`usbd_refused_without_smi_filter` reads the refusal, and only the AMD bench -reads a claim through an AMD-Vi entry. - -On the AMD bench that claim waits on P1 of `issues/there-is-no-wifi.md` -(a BAR stays where firmware put it, in 4 KiB pages): `03:00.4`'s only BAR, -which holds its table, sits behind a port whose window bus 3 fills, so it -cannot move to a 2 MiB slot of its own, and its claim is refused by name. - -**Depends on:** P1 of `issues/there-is-no-wifi.md`. - -**Exit:** P1's AMD bench reading of `03:00.4`, claimed with its BAR in place -and driven through its own remapping entry, its holder's foreign vector -refused and recorded; and a machine that does not remap still refusing the -claim (`amdvi_claim_refused_without_remapping`). diff --git a/issues/every-driver-is-still-in-the-kernel.md b/issues/every-driver-is-still-in-the-kernel.md index a2d8deb733..857f75471e 100644 --- a/issues/every-driver-is-still-in-the-kernel.md +++ b/issues/every-driver-is-still-in-the-kernel.md @@ -69,12 +69,11 @@ What is left of the staged work: splitting the controller.** It shares the controller, the event ring and the lock with the boot disk. - **Where each xHCI keeps its MSI-X table**, which decides whether usbd's - claim is handed over (`pcidev::msix_bar`). QEMU 11.1's `nec-usb-xhci` arms - MSI-X and publishes one memory BAR, BAR 0, so its table is there and the - claim is refused as - `issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md` - says; the device's `msix=off` leaves it on MSI, as `msix-exclusive-bar=on` - moves NVMe's table for blockd. The T14's two xHCI functions each publish + claim arms MSI-X or MSI (`bring_up`, `kernel/src/pcidev/mod.rs`). QEMU + 11.1's `nec-usb-xhci` arms MSI-X and publishes one memory BAR, BAR 0, so + its table is there, and a claim hands that BAR over with the rest where + the machine remaps; the device's `msix=off` leaves it on MSI, as + `msix-exclusive-bar=on` moves NVMe's table for blockd. The T14's two xHCI functions each publish only BAR 0 and armed MSI in the kernel's own bring-up, with no `MSI-X not armed` line and one remapping entry each (a `testcases` readback): no MSI-X capability came off either walk, so a claim arms MSI and hands BAR 0 over — diff --git a/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md b/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md index 6585f3bcae..e4d94d5e4c 100644 --- a/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md +++ b/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md @@ -225,12 +225,10 @@ Beside that order, each waiting on something outside it: over `inspect`; no class is bound, and no shipped image runs it. **Left**, first P1 (order item 4), then the whole xHCI, one usbd per controller (the orchestrator's stage-0 ruling). pcidev today arms MSI - only on a function with no MSI-X, and maps no BAR that holds the - table (`kernel/src/pcidev/mod.rs`). QEMU's xHCI keeps its MSI-X table - in the BAR that holds the registers, which refuses usbd's claim as it - refuses diskserver's - (`issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md`, - whose exit P1 rewrites), so QEMU's spare today is + only on a function with no MSI-X, and hands the BAR holding the + table over like any other where the machine remaps + (`kernel/src/pcidev/mod.rs`). QEMU's xHCI keeps its MSI-X table in + the BAR that holds the registers; QEMU's spare today is `qemu-xhci,msix=off,msi=on` (`XHCI_SPARE` in `tests/common/qemu.rs`). P1 serves usbd when that spare is `qemu-xhci` with its MSI-X left on and `usbd_drives_the_spare` passes on it. The T14's xHCI has no MSI-X diff --git a/issues/there-is-no-wifi.md b/issues/there-is-no-wifi.md index d575b55734..6e83e03242 100644 --- a/issues/there-is-no-wifi.md +++ b/issues/there-is-no-wifi.md @@ -151,10 +151,6 @@ page included.** Owner: the orchestrator. - `issues/a-bar-behind-a-bridge-with-no-free-2-mib-of-its-windows-is-not-handed-over.md` closed, and each of the two placement issues it names closed or its exit rewritten to the placement P1 lands; - - the exit of - `issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md` - rewritten, from a refused write to the table's page to the NVMe driven - through a table in BAR 0; - on the AMD bench, `03:00.4` (`xhci-leave=03:00.4`, its MSI-X table in BAR 0, its only BAR) claimed with that BAR in place and driven by usbd through its own remapping entry, every unit filtering SMIs: the diff --git a/issues/toyos-runs-in-any-open-source-hypervisor.md b/issues/toyos-runs-in-any-open-source-hypervisor.md index 4db7436d4c..3fea22c7a8 100644 --- a/issues/toyos-runs-in-any-open-source-hypervisor.md +++ b/issues/toyos-runs-in-any-open-source-hypervisor.md @@ -41,10 +41,6 @@ and, filed before this track: - `issues/a-machine-without-an-iommu-refuses-every-claim.md` — bhyve; crosvm for every device it emulates; and VirtualBox, QEMU and Cloud Hypervisor by default (`issues/no-claim-gets-an-address-space-under-virtio-iommu.md`). -- `issues/a-controller-whose-msix-table-is-in-bar-0-cannot-be-driven-from-userland.md` - — QEMU's NVMe by default; and every virtio function of Cloud Hypervisor and - crosvm, which keep the MSI-X table and PBA in the BAR their virtio - structures are in. - `issues/a-hid-interface-outside-the-boot-subclass-is-read-as-qemus-tablet.md` — VirtualBox's USB tablet, a subclass-0 interface; bhyve's declares the boot mouse subclass and protocol. diff --git a/kernel/amdvi/src/queue.rs b/kernel/amdvi/src/queue.rs index 0024a90c12..534ae5991c 100644 --- a/kernel/amdvi/src/queue.rs +++ b/kernel/amdvi/src/queue.rs @@ -48,9 +48,10 @@ pub enum Event { /// `RX` clear (Figure 64). `write` is the access where the record states /// one; `address` is the record's, whatever it names. Requester { code: u8, device: u16, address: u64, write: Option }, - /// An `IO_PAGE_FAULT` at an address in the interrupt range: a message - /// from `device` the unit refused, an SMI the filter refused among them - /// (§2.1.5.1), no access of memory. [`disposition`] says whose it is. + /// An `IO_PAGE_FAULT` with `I` set (Table 57) or at an address in the + /// interrupt range: a message from `device` the unit refused, an SMI the + /// filter refused among them (§2.1.5.1), no access of memory. + /// [`disposition`] says whose it is. Interrupt { device: u16, address: u64 }, /// Every other record: an entry or command of this driver's the unit /// refused, or the unit's own hardware failing. @@ -61,9 +62,10 @@ pub const fn event(words: [u64; 2]) -> Event { let device = words[0] as u16; let code = (words[0] >> CODE_SHIFT) as u8; let write = match code { - // `I` [51]: an interrupt carries no read or write; else `RW` [53]. - 0b0010 if interrupt_range(words[1]) => return Event::Interrupt { device, address: words[1] }, - 0b0010 if words[0] & 1 << 51 != 0 => None, + // `I` [51]: an interrupt, which carries no read or write; else `RW` [53]. + 0b0010 if words[0] & 1 << 51 != 0 || interrupt_range(words[1]) => { + return Event::Interrupt { device, address: words[1] } + } 0b0010 => Some(words[0] & 1 << 53 != 0), // `RX` [48] set is a `COMPLETE_PPR_REQUEST` of this driver's failing. 0b1001 if words[0] & 1 << 48 != 0 => return Event::Unit { code }, diff --git a/kernel/amdvi/tests/encodings.rs b/kernel/amdvi/tests/encodings.rs index 22f20027a2..a0ad589567 100644 --- a/kernel/amdvi/tests/encodings.rs +++ b/kernel/amdvi/tests/encodings.rs @@ -140,7 +140,7 @@ fn a_page_fault_is_its_requesters_with_its_access() { let write = [read[0] | word(&[(53, 53, 1)]), read[1]]; assert!(matches!(event(write), Event::Requester { write: Some(true), .. })); let message = [word(&[(15, 0, 0x0303), (51, 51, 1), (53, 53, 1), (63, 60, 2)]), 0xFEE0_0000]; - assert!(matches!(event(message), Event::Requester { device: 0x0303, write: None, .. })); + assert_eq!(event(message), Event::Interrupt { device: 0x0303, address: 0xFEE0_0000 }); } /// Every record a requester can cause is its, never the unit's: Figure 62 @@ -296,6 +296,9 @@ fn a_refused_message_is_told_from_a_dma_fault() { assert!(matches!(fault(access), Event::Requester { .. }), "{access:#x}"); } assert_eq!(message_type(smi_at | word(&[(7, 7, 1)])), 0b1010); + // `I` [51] set: an interrupt request, whatever address it names. + let flagged = event([word(&[(15, 0, 0x00A3), (51, 51, 1), (63, 60, 2)]), 0x4000_0000_0000]); + assert_eq!(flagged, Event::Interrupt { device: 0x00A3, address: 0x4000_0000_0000 }); } /// The ruling on #921: an interrupt-range record from an id the IVRS gives diff --git a/kernel/src/arch/x86_64/amdvi/interrupt.rs b/kernel/src/arch/x86_64/amdvi/interrupt.rs index 7d140758ea..b8954f6867 100644 --- a/kernel/src/arch/x86_64/amdvi/interrupt.rs +++ b/kernel/src/arch/x86_64/amdvi/interrupt.rs @@ -150,9 +150,9 @@ pub(in crate::arch::x86_64) fn msi(source: StreamId, vector: u8) -> Result Result { // A requester the platform speaks as is refused before here - // (`iommu::isolates`), and one a kernel driver holds by `pcidev::claim`; - // one that already has a table is one of those, and its entries keep - // firmware's pass bits. + // (`iommu::isolates`), and one a kernel driver holds by `pcidev::claim`. + // A table already written is a re-claim's, whose setting is `CLAIMED`; + // any other is one of those, whose entries keep firmware's pass bits. assert!( !REMAP.lock().tables.contains_key(&source.requester()) || domain::setting(source.requester()) == toyos_amdvi::dte::Setting::CLAIMED, "amdvi: a claim on {source}, whose remapping table another source already wrote" diff --git a/kernel/src/arch/x86_64/amdvi/mod.rs b/kernel/src/arch/x86_64/amdvi/mod.rs index b4cb79fd62..a9a96b6db5 100644 --- a/kernel/src/arch/x86_64/amdvi/mod.rs +++ b/kernel/src/arch/x86_64/amdvi/mod.rs @@ -316,7 +316,10 @@ pub fn init(rsdp_addr: u64, devices: &[PciDevice], windows: Option<&[RootBridgeW let reserved = windows.map(|w| { w.iter().map(|w| (w.base, w.end())).chain(ivmds.iter().map(|m| (m.start, m.start + m.length))).collect() }); - fault::platform(platform.iter().map(|&(id, _)| id)); + let mut ids: Vec = platform.iter().map(|&(id, _)| id).collect(); + ids.sort_unstable(); + ids.dedup(); + fault::platform(ids.into_iter()); domain::start(reserved, settings, ivmds.iter().map(|m| m.requesters).collect(), translated, platform); log!("iommu: amdvi identity domain covers 0x0..{top:#x}, root={:#x}", identity.get()); if let Some(extended) = remap { diff --git a/kernel/src/iommu/mod.rs b/kernel/src/iommu/mod.rs index e8347c9e91..1a4993a276 100644 --- a/kernel/src/iommu/mod.rs +++ b/kernel/src/iommu/mod.rs @@ -298,9 +298,8 @@ pub fn init( unit::init(rsdp_addr, devices, windows.map(|declared| declared.windows())); } -/// Every function the kernel drives has been taken from firmware, and no -/// claim has been granted yet: what the units guard against a holder may -/// now be guarded against firmware's handover too. +/// Once, after every kernel driver has taken its functions from firmware and +/// before userland runs: AMD-Vi turns its SMI filters on. pub fn drivers_settled() { unit::drivers_settled(); } diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 2acc23984f..6af98d2644 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -2341,9 +2341,7 @@ pub fn pipe_map(handle: RawHandle) -> Result<*mut u8, SyscallError> { /// Map one memory BAR of a claimed PCI function, as an object `SYS_SHM_MAP` /// maps read/write and uncacheable. /// -/// `InvalidArgument` for an index that names no memory BAR, and for the BAR -/// holding this function's MSI-X table where the machine would not fence a -/// message written there. +/// `InvalidArgument` for an index that names no memory BAR. /// /// Every call answers an object of its own over the same window, so a BAR /// whose handle was closed can be asked for again.