From 286536fb8606852bce23dcf10b0120b60691797c Mon Sep 17 00:00:00 2001 From: japabu Date: Fri, 9 Oct 2026 12:36:52 +0200 Subject: [PATCH 1/3] toyos-acpi decodes the IORT and the MADT's MSI controllers; toyos-smmu and toyos-its hold the SMMUv3's and the ITS's encodings Stage G1 of the ARM desktop: host-only, no kernel file. The kernel's SMMUv3 unit (G2) and its ITS (G3) are built on these. toyos-acpi: `iort` answers an IORT every node of which it walked, with the SMMUv3 node's base and four GSIVs, the ITS node's id, and `route(segment, rid)`: through which SMMUv3 under which StreamID and on to which ITS under which DeviceID, straight to an ITS past every SMMU, or mapped nowhere. The refusals are by name: a table or node revision not read against DEN 0049, a node the table cannot hold, an ID array over its node's fields or past its end, a range past the ID space, an output reference to no node or to one its source may not reach, a root complex single mapping, an ITS group of other than one, and an ID two mappings claim. `msi_controllers` walks the MADT for its ITS and GIC MSI Frame structures; the frame is not a `MadtEntry` variant because the kernel matches that enum exhaustively in two files this stage does not touch. The fixtures are QEMU 11.1.1 `virt`'s RSDP, XSDT, MADT and IORT in both GIC modes, captured with the firmware alone in the guest (`SOURCE` beside them). toyos-smmu: the identification registers judged (`probe`), the stream table entry that aborts and the one that translates by stage 1 through one context descriptor, the context descriptor, the register values, the queue index arithmetic, the commands, the event records, and the stage 1 table plan with its 2 MiB memory block and its one 4 KiB doorbell page. No bypass is spelled. toyos-its: the type register judged, the table and command queue registers, the commands, and the LPI configuration and pending tables with the redistributor registers that name them. The device and translation table entries are IMPLEMENTATION DEFINED and the ITS's own, so only their sizes are here. Every encoding is tested against the specification's field positions (IHI 0070 H.a, IHI 0069 H.b, DDI 0487 M.d, ACPI 6.5), written in the tests as `(msb, lsb, value)` triples and compared whole. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RvnWQFcMuGqTHYhvSnTe8A --- Cargo.lock | 8 + Cargo.toml | 2 + src/licence.rs | 40 ++ toyos-acpi/fixtures/qemu-11.1.1-virt/SOURCE | 28 + .../fixtures/qemu-11.1.1-virt/hwgic/apic.bin | Bin 0 -> 428 bytes .../fixtures/qemu-11.1.1-virt/hwgic/iort.bin | Bin 0 -> 172 bytes .../fixtures/qemu-11.1.1-virt/its/apic.bin | Bin 0 -> 424 bytes .../fixtures/qemu-11.1.1-virt/its/iort.bin | Bin 0 -> 236 bytes toyos-acpi/fixtures/qemu-11.1.1-virt/rsdp.bin | Bin 0 -> 36 bytes toyos-acpi/fixtures/qemu-11.1.1-virt/xsdt.bin | Bin 0 -> 100 bytes toyos-acpi/src/iort.rs | 383 +++++++++++ toyos-acpi/src/lib.rs | 9 +- toyos-acpi/src/madt.rs | 93 ++- toyos-acpi/tests/iort.rs | 596 ++++++++++++++++++ toyos-its/Cargo.toml | 17 + toyos-its/src/command.rs | 56 ++ toyos-its/src/lib.rs | 264 ++++++++ toyos-its/src/lpi.rs | 111 ++++ toyos-its/tests/encodings.rs | 294 +++++++++ toyos-smmu/Cargo.toml | 17 + toyos-smmu/src/config.rs | 82 +++ toyos-smmu/src/lib.rs | 56 ++ toyos-smmu/src/queue.rs | 205 ++++++ toyos-smmu/src/table.rs | 129 ++++ toyos-smmu/src/unit.rs | 249 ++++++++ toyos-smmu/tests/encodings.rs | 504 +++++++++++++++ 26 files changed, 3133 insertions(+), 10 deletions(-) create mode 100644 toyos-acpi/fixtures/qemu-11.1.1-virt/SOURCE create mode 100644 toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/apic.bin create mode 100644 toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/iort.bin create mode 100644 toyos-acpi/fixtures/qemu-11.1.1-virt/its/apic.bin create mode 100644 toyos-acpi/fixtures/qemu-11.1.1-virt/its/iort.bin create mode 100644 toyos-acpi/fixtures/qemu-11.1.1-virt/rsdp.bin create mode 100644 toyos-acpi/fixtures/qemu-11.1.1-virt/xsdt.bin create mode 100644 toyos-acpi/src/iort.rs create mode 100644 toyos-acpi/tests/iort.rs create mode 100644 toyos-its/Cargo.toml create mode 100644 toyos-its/src/command.rs create mode 100644 toyos-its/src/lib.rs create mode 100644 toyos-its/src/lpi.rs create mode 100644 toyos-its/tests/encodings.rs create mode 100644 toyos-smmu/Cargo.toml create mode 100644 toyos-smmu/src/config.rs create mode 100644 toyos-smmu/src/lib.rs create mode 100644 toyos-smmu/src/queue.rs create mode 100644 toyos-smmu/src/table.rs create mode 100644 toyos-smmu/src/unit.rs create mode 100644 toyos-smmu/tests/encodings.rs diff --git a/Cargo.lock b/Cargo.lock index c8cce35baf7..2621d67fc81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5870,6 +5870,10 @@ dependencies = [ "toyos-abi", ] +[[package]] +name = "toyos-its" +version = "0.1.0" + [[package]] name = "toyos-keymap" version = "0.1.0" @@ -5999,6 +6003,10 @@ version = "0.1.0" name = "toyos-rootimage" version = "0.1.0" +[[package]] +name = "toyos-smmu" +version = "0.1.0" + [[package]] name = "toyos-swap" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 788315a9e7a..49a3e48ce8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ members = [ "toyos-hda", "toyos-i219", "toyos-inspect", + "toyos-its", "toyos-keymap", "toyos-logstream", "toyos-manifest", @@ -57,6 +58,7 @@ members = [ "toyos-osrelease", "toyos-quiesce", "toyos-rootimage", + "toyos-smmu", "toyos-swap", "toyos-symbols", "toyos-tco", diff --git a/src/licence.rs b/src/licence.rs index 1c106d08124..6bfe3e1e1fd 100644 --- a/src/licence.rs +++ b/src/licence.rs @@ -270,6 +270,46 @@ pub const COMMITTED_FILES: &[(&str, &str, &str, Terms)] = &[ "ours: QEMU's own XSDT, captured by the commit that added toyos-acpi", Terms::Spdx("MIT OR Apache-2.0"), ), + // The tables QEMU 11.1.1 published to its `virt` machine in both of its + // GIC modes, read out of guest physical memory over the monitor with no + // operating system in the guest. Firmware output, as the q35 tables above: + // `toyos-acpi/tests/iort.rs` decodes them. + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/apic.bin", + "5f65e29aa3c1d4b7afce9f22c144797f2d4234a58e7fbeaae48c1744d0713976", + "ours: QEMU `virt`'s own MADT under the hypervisor's GIC, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/iort.bin", + "6b0a25a264422e32097f761d5d82cd7d5142bbe20a6d6ef22b7750c9ff7cd8e2", + "ours: QEMU `virt`'s own IORT under the hypervisor's GIC, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/its/apic.bin", + "ca5ef1376c5e30ee314a9a22c225fe34e9721889a6884bc36239c9d145759452", + "ours: QEMU `virt`'s own MADT under its emulated GIC and ITS, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/its/iort.bin", + "57b1d81a2995c018d623a8d4a0ecb7318624f03065c3deb820773f3c7d5c7a93", + "ours: QEMU `virt`'s own IORT under its emulated GIC and ITS, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/rsdp.bin", + "88f4591b661d951ae67bca4b740dbebf8e0382b1e27dc915ae44ceddb3a09df7", + "ours: QEMU `virt`'s own RSDP, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/xsdt.bin", + "8129cb6793e7b6f375e60f6e9780feff0b11b1e99da8c0b1dc4135700123725d", + "ours: QEMU `virt`'s own XSDT, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), ( "toyos-acpi/fixtures/ovmf-pure-efi/root-bridge-0.bin", "eb00e68be746a09ac7f0ce1ca492ce8c858e3af1152112b49ffb4708883acbfb", diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/SOURCE b/toyos-acpi/fixtures/qemu-11.1.1-virt/SOURCE new file mode 100644 index 00000000000..df4becb2b96 --- /dev/null +++ b/toyos-acpi/fixtures/qemu-11.1.1-virt/SOURCE @@ -0,0 +1,28 @@ +The RSDP, the XSDT, the MADT and the IORT QEMU 11.1.1 published to its `virt` +machine under edk2-stable202408 (QEMU's own `edk2-aarch64-code.fd`), with no +operating system in the guest: the firmware was left to reach its shell, and +the guest's memory was then written out by the monitor's `pmemsave`. Every +address was walked host-side from the RSDP, found by its signature and both +checksums, so no byte here came through the decoder these fixtures judge. Each +table sums to zero. + + qemu-system-aarch64 -M -accel hvf -cpu host -smp 4 -m 256 \ + -nodefaults -display none \ + -drive if=pflash,format=raw,readonly=on,file=edk2-aarch64-code.fd + + hwgic/ -M virt,gic-version=3,iommu=smmuv3 + the hypervisor's own GICv3, which has no ITS: the MADT ends in a + GIC MSI Frame structure, and the IORT is an SMMUv3 and a root complex + its/ -M virt,gic-version=3,its=on,iommu=smmuv3,kernel-irqchip=off + QEMU's emulated GICv3 with its ITS: the MADT ends in a GIC ITS + structure, and the IORT is an ITS group, an SMMUv3 and a root complex + +Both boots put every table at the same address, so the RSDP and the XSDT are +one file each: + + rsdp.bin 0x4cb43018 xsdt.bin 0x4cb43f18 + apic.bin 0x4cb43c98 iort.bin 0x4cb43198 + +Those are `tests/iort.rs`'s regions, so the XSDT's own entries point at them +and the walk under test is the real one. The XSDT's other six entries (FACP, +PPTT, GTDT, MCFG, SPCR, DBG2) name tables that are not here. diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/apic.bin b/toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/apic.bin new file mode 100644 index 0000000000000000000000000000000000000000..5a4643dd9698b5dbe07591280cc12f7dd92f627d GIT binary patch literal 428 zcmZ<^@N{0o$iTpI#mV2Lrka-|;U>IZospi4l0n!UHj|oVFFif0O^FVrG0AwD>-ylq`c_6(oz$XA=aY4iM S8v`#qoSDF4AUhlw0vG^?8Vdmc literal 0 HcmV?d00001 diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/iort.bin b/toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/iort.bin new file mode 100644 index 0000000000000000000000000000000000000000..0cf52b52f671637bf4dbc9e0fc80c3c73d0b01d3 GIT binary patch literal 172 zcmebD4+>esz`(#d)yd!4BUr&HBEVSz2pEB4AU23*0%8Lo1_2fq1{MelMzV5(SRi%i nAPXpv4aB)XoP#9EWWm4;QV+r^P#Q=v12GslKg4b25v<@85#X!<1VAAM5F13p0FMNW#lQh$F#{Rg0Wcl|15CX* zLWpD*$UKlaFbpz)RP$i&0OF2fW$N3VGc&YkYPg%7N zbp{0J=&2D{fEKB+`rw0FMX_XV>~5U+Y~yd);CJu@k6%w~%v<<*VJA_Dip)Vbk6*GV Qow{qHUFJb^BS8II53pGf0RR91 literal 0 HcmV?d00001 diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/rsdp.bin b/toyos-acpi/fixtures/qemu-11.1.1-virt/rsdp.bin new file mode 100644 index 0000000000000000000000000000000000000000..1361613561706b7e9da8dece42b3615b3bfd242b GIT binary patch literal 36 kcmWFvc2Nij2~r4f@^|(KR$yWP0u>;Zu;1bXVz&Y@0EXBG0RR91 literal 0 HcmV?d00001 diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/xsdt.bin b/toyos-acpi/fixtures/qemu-11.1.1-virt/xsdt.bin new file mode 100644 index 0000000000000000000000000000000000000000..fc360682de777dbcfba918e16b1b938f1b808cf8 GIT binary patch literal 100 zcmazDb_q#gU|?YEcJg=j2v%^42yj*a0!E-1h!zG)Nmy_30dg5;*g$Cm2rXd& = 5..=7; + +/// Table 4: type (0), length (1..3), revision (3), identifier (4..8), number +/// of ID mappings (8..12), reference to the ID array (12..16). +const NODE_HEADER: usize = 16; +/// Table 5: input base, number of IDs minus one, output base, output +/// reference, flags — five words. +const MAPPING: usize = 20; +/// Table 6: bit 0, "apply the output base regardless of the input IDs". +const SINGLE: u32 = 1; + +const ITS: u8 = 0; +const ROOT_COMPLEX: u8 = 2; +const SMMUV3: u8 = 4; + +/// Table 16: one reserved word that counted the group's ITSs before issue +/// E.g and is 1 since, then the one identifier. +const ITS_LEN: usize = NODE_HEADER + 8; +/// Table 18: memory access properties (16..24), ATS attribute (24..28), PCI +/// segment number (28..32), then the address size limit and, from node +/// revision 4, the PASID capabilities and flags. +const RC_SEGMENT: usize = 28; +/// Table 13: base address (16..24), flags (24..28), a reserved word, VATOS +/// (32..40), model (40..44), the event, PRI, GERR and sync GSIVs (44..60), +/// proximity domain (60..64), DeviceID mapping index (64..68). +const SMMU_BASE: usize = 16; +const SMMU_FLAGS: usize = 24; +const SMMU_GSIVS: usize = 44; +const SMMU_LEN: usize = 68; +/// Table 14, bit 0: `COHACC override`. +const SMMU_COHACC: u32 = 1; + +/// Why an IORT cannot be used. `at` is a node's offset in the table. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum IortRefused { + Table(TableError), + /// A table revision outside the ones decoded. + Revision(u8), + /// The node array starts inside the table's own header fields. + NodeArray { offset: u32 }, + /// The node at `at` declares a length too short for its type, or one the + /// table cannot hold; a node count that overruns the table ends here. + Node { at: usize, declared: usize }, + /// A node revision whose layout was not read against the specification. + NodeRevision { at: usize, kind: u8, revision: u8 }, + /// The node's ID array lies over its own fields or runs past its end, or + /// an ITS node, which has none, declares one. + Mappings { at: usize, count: u32, offset: u32 }, + /// A mapping's input or output range runs past the 32-bit ID space. + Range { at: usize, base: u32, last: u32 }, + /// A mapping's output reference names no node, or a node its source may + /// not output to: a root complex outputs to an SMMUv3 or an ITS, an + /// SMMUv3 to an ITS. + Output { at: usize, reference: u32 }, + /// A root complex mapping with the single-mapping flag, which no table + /// read here carries. + SingleMapping { at: usize }, + /// An ITS group of other than one ITS: deprecated by issue E.g, which + /// fixes the count at 1. + ItsGroup { at: usize, count: u32 }, + /// Two mappings claim this ID, which the table must route one way. + Overlap { id: u32 }, +} + +impl From for IortRefused { + fn from(error: TableError) -> Self { + Self::Table(error) + } +} + +/// An SMMUv3 as its node names it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Smmuv3 { + pub base: u64, + /// Table 14's `COHACC override`: what `SMMU_IDR0.COHACC` is read as. + pub coherent: bool, + /// The four wired interrupts; `None` where the node gives no GSIV. + pub event: Option, + pub pri: Option, + pub gerror: Option, + pub sync: Option, +} + +/// One node, in the shape its reader acts on. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Node { + /// `id` is the MADT GIC ITS structure's. + Its { id: u32 }, + RootComplex { segment: u32 }, + Smmuv3(Smmuv3), + /// A type not decoded here: a named component (1), an SMMUv1 or v2 (3), a + /// PMCG (5), a reserved memory range (6), an IWB (7), or a reserved one. + Other(u8), +} + +/// The DeviceID a requester's messages carry, and the ITS they carry it to. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct ItsDevice { + pub its: u32, + pub device: u32, +} + +/// Where one requester's transactions go. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Route { + /// Through this SMMUv3 under `stream`; `its` is where the SMMU's own node + /// maps that stream on to, and `None` where it maps it nowhere. + Translated { smmu: Smmuv3, stream: u32, its: Option }, + /// Straight to an ITS, past every SMMU. + Untranslated(ItsDevice), + /// No root complex of that segment maps the requester. + Unmapped, +} + +/// A node's header, bounded by the table. +#[derive(Clone, Copy)] +struct Raw { + at: usize, + kind: u8, + len: usize, + revision: u8, + mappings: u32, + array: u32, +} + +#[derive(Clone, Copy)] +struct Mapping { + input: u32, + /// The number of IDs, minus one. + span: u32, + output: u32, + reference: u32, + single: bool, +} + +impl Mapping { + fn maps(&self, id: u32) -> Option { + // `output + span` was checked not to wrap when the table was opened. + (!self.single && id >= self.input && id - self.input <= self.span).then(|| self.output + (id - self.input)) + } +} + +/// An IORT whose every node [`iort`] has walked. +#[derive(Clone, Copy)] +pub struct Iort

{ + table: Table

, + nodes: u32, + array: usize, +} + +/// The IORT at `rsdp_addr`, every node of it checked. +pub fn iort(phys: P, rsdp_addr: u64) -> Result, IortRefused> { + let table = find_table(phys, rsdp_addr, b"IORT", IORT_NEEDED)?; + let short = TableError::Length { declared: table.len() as u32, needed: IORT_NEEDED }; + let revision = table.byte(SDT_REVISION).ok_or(short)?; + if !REVISIONS.contains(&revision) { + return Err(IortRefused::Revision(revision)); + } + let nodes = table.u32_at(NODE_COUNT).ok_or(short)?; + let offset = table.u32_at(NODE_ARRAY).ok_or(short)?; + if (offset as usize) < IORT_NEEDED { + return Err(IortRefused::NodeArray { offset }); + } + let iort = Iort { table, nodes, array: offset as usize }; + for node in iort.raw_nodes() { + iort.check(node?)?; + } + Ok(iort) +} + +impl Iort

{ + /// Every node, in the table's order. + pub fn nodes(&self) -> impl Iterator + '_ { + // `iort` walked the same headers and decoded the same nodes. + self.raw_nodes().filter_map(|raw| self.decode(raw.ok()?).ok()) + } + + /// Where the PCI function `rid` of `segment` is routed. + pub fn route(&self, segment: u32, rid: u16) -> Result { + let mut route = None; + for raw in self.raw_nodes() { + let raw = raw?; + if self.decode(raw)? != (Node::RootComplex { segment }) { + continue; + } + for (mapping, id) in self.hits(raw, u32::from(rid)) { + let hop = match self.output(raw, mapping)? { + (_, Node::Its { id: its }) => Route::Untranslated(ItsDevice { its, device: id }), + (smmu_raw, Node::Smmuv3(smmu)) => { + let mut its = None; + for (onward, device) in self.hits(smmu_raw, id) { + let (_, Node::Its { id: to }) = self.output(smmu_raw, onward)? else { + return Err(IortRefused::Output { at: smmu_raw.at, reference: onward.reference }); + }; + if its.replace(ItsDevice { its: to, device }).is_some() { + return Err(IortRefused::Overlap { id }); + } + } + Route::Translated { smmu, stream: id, its } + } + _ => return Err(IortRefused::Output { at: raw.at, reference: mapping.reference }), + }; + if route.replace(hop).is_some() { + return Err(IortRefused::Overlap { id: u32::from(rid) }); + } + } + } + Ok(route.unwrap_or(Route::Unmapped)) + } + + /// The mappings of `raw` that hold `id`, each with the ID it puts out. + fn hits(&self, raw: Raw, id: u32) -> impl Iterator + '_ { + (0..raw.mappings).filter_map(move |i| { + let mapping = self.mapping(raw, i)?; + Some((mapping, mapping.maps(id)?)) + }) + } + + /// Walks by each node's own length, which [`Iort::raw`] bounds below by + /// the header and above by the table: at most `len / 16` steps. + fn raw_nodes(&self) -> impl Iterator> + '_ { + let mut at = self.array; + let mut halted = false; + (0..self.nodes).map_while(move |_| { + if halted { + return None; + } + let raw = self.raw(at); + match raw { + Ok(raw) => at += raw.len, + Err(_) => halted = true, + } + Some(raw) + }) + } + + fn raw(&self, at: usize) -> Result { + let t = &self.table; + let header = || Some((t.byte(at)?, t.u16_at(at + 1)?, t.byte(at + 3)?, t.u32_at(at + 8)?, t.u32_at(at + 12)?)); + let Some((kind, declared, revision, mappings, array)) = header() else { + return Err(IortRefused::Node { at, declared: 0 }); + }; + let len = usize::from(declared); + let floor = match kind { + ITS => ITS_LEN, + ROOT_COMPLEX => RC_SEGMENT + 4, + SMMUV3 => SMMU_LEN, + _ => NODE_HEADER, + }; + if len < floor || at + len > t.len() { + return Err(IortRefused::Node { at, declared: len }); + } + Ok(Raw { at, kind, len, revision, mappings, array }) + } + + /// Mapping `index` of a node [`Iort::check`] accepted the array of. + fn mapping(&self, raw: Raw, index: u32) -> Option { + let at = raw.at + raw.array as usize + index as usize * MAPPING; + let t = &self.table; + Some(Mapping { + input: t.u32_at(at)?, + span: t.u32_at(at + 4)?, + output: t.u32_at(at + 8)?, + reference: t.u32_at(at + 12)?, + single: t.u32_at(at + 16)? & SINGLE != 0, + }) + } + + /// The node a mapping of `from` outputs to. + fn output(&self, from: Raw, mapping: Mapping) -> Result<(Raw, Node), IortRefused> { + let refused = IortRefused::Output { at: from.at, reference: mapping.reference }; + for raw in self.raw_nodes() { + let raw = raw?; + if raw.at == mapping.reference as usize { + return Ok((raw, self.decode(raw)?)); + } + } + Err(refused) + } + + fn decode(&self, raw: Raw) -> Result { + let t = &self.table; + let Raw { at, kind, revision, .. } = raw; + let short = IortRefused::Node { at, declared: raw.len }; + let revised = |known: core::ops::RangeInclusive| { + if known.contains(&revision) { + Ok(()) + } else { + Err(IortRefused::NodeRevision { at, kind, revision }) + } + }; + Ok(match kind { + ITS => { + revised(1..=1)?; + let count = t.u32_at(at + NODE_HEADER).ok_or(short)?; + if count != 1 { + return Err(IortRefused::ItsGroup { at, count }); + } + Node::Its { id: t.u32_at(at + NODE_HEADER + 4).ok_or(short)? } + } + ROOT_COMPLEX => { + revised(3..=4)?; + Node::RootComplex { segment: t.u32_at(at + RC_SEGMENT).ok_or(short)? } + } + SMMUV3 => { + revised(4..=5)?; + let gsiv = |i: usize| t.u32_at(at + SMMU_GSIVS + 4 * i).map(NonZeroU32::new).ok_or(short); + Node::Smmuv3(Smmuv3 { + base: t.u64_at(at + SMMU_BASE).ok_or(short)?, + coherent: t.u32_at(at + SMMU_FLAGS).ok_or(short)? & SMMU_COHACC != 0, + event: gsiv(0)?, + pri: gsiv(1)?, + gerror: gsiv(2)?, + sync: gsiv(3)?, + }) + } + other => Node::Other(other), + }) + } + + /// One node's fields, its ID array's place and every mapping in it. + fn check(&self, raw: Raw) -> Result<(), IortRefused> { + let node = self.decode(raw)?; + // Where the node's own fields end, which its ID array may not lie over. + let fields = match (node, raw.revision) { + (Node::Other(_), _) => return Ok(()), + (Node::Its { .. }, _) => ITS_LEN, + (Node::RootComplex { .. }, 3) => 36, + (Node::RootComplex { .. }, _) => 40, + (Node::Smmuv3(_), _) => SMMU_LEN, + }; + let Raw { at, mappings: count, array: offset, .. } = raw; + let end = u64::from(offset) + u64::from(count) * MAPPING as u64; + let misplaced = (offset as usize) < fields || end > raw.len as u64; + if count != 0 && (misplaced || matches!(node, Node::Its { .. })) { + return Err(IortRefused::Mappings { at, count, offset }); + } + for index in 0..count { + let mapping = self.mapping(raw, index).ok_or(IortRefused::Mappings { at, count, offset })?; + let from_root = matches!(node, Node::RootComplex { .. }); + if mapping.single && from_root { + return Err(IortRefused::SingleMapping { at }); + } + if !mapping.single { + for base in [mapping.input, mapping.output] { + if base.checked_add(mapping.span).is_none() { + return Err(IortRefused::Range { at, base, last: mapping.span }); + } + } + } + match self.output(raw, mapping)?.1 { + Node::Its { .. } => {} + Node::Smmuv3(_) if from_root => {} + _ => return Err(IortRefused::Output { at, reference: mapping.reference }), + } + } + Ok(()) + } +} diff --git a/toyos-acpi/src/lib.rs b/toyos-acpi/src/lib.rs index 120e58501bb..f9c0cfed746 100644 --- a/toyos-acpi/src/lib.rs +++ b/toyos-acpi/src/lib.rs @@ -7,7 +7,8 @@ //! //! Multi-byte fields are composed from bytes, little-endian, so no firmware //! byte is transmuted into a type. Field offsets cite ACPI 6.5, except MCFG's -//! (PCI Firmware Specification), HPET's (IA-PC HPET Specification) and SPCR's (Microsoft's SPCR specification). +//! (PCI Firmware Specification), HPET's (IA-PC HPET Specification), SPCR's (Microsoft's SPCR specification) +//! and the IORT's (Arm DEN 0049). //! //! `no_std`, no allocation, no `unsafe`. @@ -18,6 +19,7 @@ mod ecdt; mod facs; mod fadt; mod gtdt; +mod iort; mod madt; mod resource; mod spcr; @@ -33,9 +35,10 @@ pub use fadt::{ FADT_FOR_FIXED_HARDWARE, FADT_FOR_RESET, FADT_PM1A_CNT_BLK, }; use fadt::FADT_DSDT; +pub use iort::{iort, Iort, IortRefused, ItsDevice, Node, Route, Smmuv3, IORT_NEEDED}; pub use madt::{ - isa_line, madt_entries, sci_line, Gicc, IoApicEntry, Line, MadtEntries, MadtEntry, MadtHalt, - Polarity, SourceOverride, Trigger, MADT_ENTRIES, + isa_line, madt_entries, msi_controllers, sci_line, Gicc, IoApicEntry, Line, MadtEntries, MadtEntry, MadtHalt, + MsiController, MsiFrame, Polarity, SourceOverride, SpiRange, Trigger, MADT_ENTRIES, }; pub use gtdt::{gtdt, Gtdt, TimerInterrupt, GTDT_NEEDED}; pub use resource::{memory_windows, ResourceError, MAX_LIST_BYTES}; diff --git a/toyos-acpi/src/madt.rs b/toyos-acpi/src/madt.rs index 5c87c337133..9b9a39b39fe 100644 --- a/toyos-acpi/src/madt.rs +++ b/toyos-acpi/src/madt.rs @@ -88,10 +88,18 @@ pub fn madt_entries(madt: &Table

) -> MadtEntries

{ } } -impl Iterator for MadtEntries

{ - type Item = Result; +/// One structure the list holds whole: its type, its declared length and +/// where its first byte is. +#[derive(Clone, Copy)] +struct Structure

{ + phys: P, + base: u64, + entry_type: u8, + declared: usize, +} - fn next(&mut self) -> Option { +impl MadtEntries

{ + fn structure(&mut self) -> Option, MadtHalt>> { if self.halted || self.offset + ENTRY_HEADER_LEN > self.list_len { return None; } @@ -108,10 +116,81 @@ impl Iterator for MadtEntries

{ })); } self.offset += declared; + Some(Ok(Structure { phys: self.table.phys(), base: self.table.base() + at as u64, entry_type, declared })) + } +} - let phys = self.table.phys(); - let base = self.table.base() + at as u64; - Some(Ok(match (entry_type, declared) { +impl Iterator for MadtEntries

{ + type Item = Result; + + fn next(&mut self) -> Option { + Some(self.structure()?.map(Structure::decode)) + } +} + +/// The SPIs a GICv2m frame raises, where its structure names them itself. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct SpiRange { + pub base: u16, + pub count: u16, +} + +/// MADT type 0xD (ACPI 6.5 §5.2.12.16, "GIC MSI Frame Structure"): a GICv2m +/// frame, whose doorbell takes the SPI to raise from the data written to it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct MsiFrame { + pub id: u32, + pub base: u64, + /// `None` where the flags' `SPI Count/Base Select` is clear and both are + /// the frame's own `MSI_TYPER`'s to say. + pub spis: Option, +} + +/// What a machine's functions write their message-signalled interrupts to. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum MsiController { + /// MADT type 0xF: an ITS, which looks an event up by its writer's DeviceID. + Its { id: u32, base: u64 }, + Frame(MsiFrame), +} + +/// Every MSI controller a validated MADT names, in its order: the walk +/// [`madt_entries`] makes, ending on the same [`MadtHalt`]. +pub fn msi_controllers(madt: &Table

) -> impl Iterator> { + let mut entries = madt_entries(madt); + core::iter::from_fn(move || { + loop { + let structure = match entries.structure()? { + Ok(structure) => structure, + Err(halt) => return Some(Err(halt)), + }; + if let Some(frame) = structure.msi_frame() { + return Some(Ok(MsiController::Frame(frame))); + } + if let MadtEntry::Its { id, base } = structure.decode() { + return Some(Ok(MsiController::Its { id, base })); + } + } + }) +} + +impl Structure

{ + /// ACPI 6.5 §5.2.12.16: GIC MSI Frame ID (4..8), Physical Base Address + /// (8..16), Flags (16..20) whose bit 0 is `SPI Count/Base Select`, SPI + /// Count (20..22), SPI Base (22..24). + fn msi_frame(self) -> Option { + let Self { phys, base, entry_type, declared } = self; + (entry_type == 0xD && declared >= 24).then(|| MsiFrame { + id: u32le(phys, base + 4), + base: u64le(phys, base + 8), + spis: (u32le(phys, base + 16) & 1 != 0) + .then(|| SpiRange { count: u16le(phys, base + 20), base: u16le(phys, base + 22) }), + }) + } + + fn decode(self) -> MadtEntry { + let Self { phys, base, entry_type, declared } = self; + match (entry_type, declared) { // Table 5.21: ACPI Processor UID (2), APIC ID (3), Flags (4..8). (0, 8..) => MadtEntry::LocalApic { apic_id: u32::from(phys.byte(base + 3)), @@ -150,7 +229,7 @@ impl Iterator for MadtEntries

{ // Table 5.44: GIC ITS ID (4..8), Physical Base Address (8..16). (0xF, 20..) => MadtEntry::Its { id: u32le(phys, base + 4), base: u64le(phys, base + 8) }, _ => MadtEntry::Other(entry_type), - })) + } } } diff --git a/toyos-acpi/tests/iort.rs b/toyos-acpi/tests/iort.rs new file mode 100644 index 00000000000..da1d0a5cd94 --- /dev/null +++ b/toyos-acpi/tests/iort.rs @@ -0,0 +1,596 @@ +//! The IORT and the MADT's MSI controllers: QEMU `virt`'s own tables in both +//! of its GIC modes, decoded against what was read off their bytes by hand +//! with DEN 0049 E.g and ACPI 6.5 open, and every refusal over crafted ones. + +mod common; + +use core::num::NonZeroU32; + +use common::{declare_len, entry, madt, reseal, rsdp, sdt, xsdt, Machine}; +use toyos_acpi::{ + find_table, iort, madt_entries, msi_controllers, IortRefused, ItsDevice, MadtEntry, MadtHalt, MsiController, + MsiFrame, Node, Route, Smmuv3, SpiRange, TableError, MADT_ENTRIES, +}; + +const RSDP: u64 = 0x4cb4_3018; +const XSDT: u64 = 0x4cb4_3f18; +const APIC: u64 = 0x4cb4_3c98; +const IORT: u64 = 0x4cb4_3198; + +const VIRT_RSDP: &[u8] = include_bytes!("../fixtures/qemu-11.1.1-virt/rsdp.bin"); +const VIRT_XSDT: &[u8] = include_bytes!("../fixtures/qemu-11.1.1-virt/xsdt.bin"); +/// `-M virt,gic-version=3,iommu=smmuv3` under HVF: the hypervisor's GIC. +const HWGIC: &[(u64, &[u8])] = &[ + (RSDP, VIRT_RSDP), + (XSDT, VIRT_XSDT), + (APIC, include_bytes!("../fixtures/qemu-11.1.1-virt/hwgic/apic.bin")), + (IORT, include_bytes!("../fixtures/qemu-11.1.1-virt/hwgic/iort.bin")), +]; +/// `-M virt,gic-version=3,its=on,iommu=smmuv3,kernel-irqchip=off`. +const ITS: &[(u64, &[u8])] = &[ + (RSDP, VIRT_RSDP), + (XSDT, VIRT_XSDT), + (APIC, include_bytes!("../fixtures/qemu-11.1.1-virt/its/apic.bin")), + (IORT, include_bytes!("../fixtures/qemu-11.1.1-virt/its/iort.bin")), +]; + +/// The node both modes publish: `arm-smmuv3` at `0x09050000`, `COHACC +/// override` set, and the GSIVs in the node's order — event, PRI, GERR, sync — +/// of which GERR is the higher of the last two. +fn virt_smmu() -> Smmuv3 { + Smmuv3 { + base: 0x0905_0000, + coherent: true, + event: NonZeroU32::new(106), + pri: NonZeroU32::new(107), + gerror: NonZeroU32::new(109), + sync: NonZeroU32::new(108), + } +} + +#[test] +fn the_hardware_gic_boot_routes_bus_zero_through_the_smmu_and_nothing_to_an_its() { + let table = iort(Machine { regions: HWGIC }, RSDP).expect("the IORT QEMU published"); + assert_eq!(table.nodes().collect::>(), [Node::Smmuv3(virt_smmu()), Node::RootComplex { segment: 0 }]); + for rid in [0x00, 0x08, 0xff] { + assert_eq!( + table.route(0, rid), + Ok(Route::Translated { smmu: virt_smmu(), stream: u32::from(rid), its: None }), + "requester {rid:#x}" + ); + } + // The one mapping is `0x00..=0xff`: bus 1 is mapped nowhere, and neither is another segment. + assert_eq!(table.route(0, 0x100), Ok(Route::Unmapped)); + assert_eq!(table.route(0, 0xffff), Ok(Route::Unmapped)); + assert_eq!(table.route(1, 0x08), Ok(Route::Unmapped)); +} + +#[test] +fn the_its_boot_routes_bus_zero_through_the_smmu_to_the_its_and_the_rest_past_it() { + let table = iort(Machine { regions: ITS }, RSDP).expect("the IORT QEMU published"); + assert_eq!( + table.nodes().collect::>(), + [Node::Its { id: 0 }, Node::Smmuv3(virt_smmu()), Node::RootComplex { segment: 0 }] + ); + for rid in [0x00, 0x08, 0xff] { + let id = u32::from(rid); + assert_eq!( + table.route(0, rid), + Ok(Route::Translated { smmu: virt_smmu(), stream: id, its: Some(ItsDevice { its: 0, device: id }) }), + "requester {rid:#x}" + ); + } + for rid in [0x100, 0x1234, 0xffff] { + assert_eq!( + table.route(0, rid), + Ok(Route::Untranslated(ItsDevice { its: 0, device: u32::from(rid) })), + "requester {rid:#x}" + ); + } + assert_eq!(table.route(1, 0x08), Ok(Route::Unmapped)); +} + +fn controllers(regions: &[(u64, &[u8])]) -> Vec> { + let table = find_table(Machine { regions }, RSDP, b"APIC", MADT_ENTRIES).expect("MADT"); + msi_controllers(&table).collect() +} + +/// `arm-gicv2m` at `0x08020000`, 64 SPIs from 80, named by the structure. +#[test] +fn the_hardware_gic_boot_names_a_v2m_frame_and_no_its() { + assert_eq!( + controllers(HWGIC), + [Ok(MsiController::Frame(MsiFrame { + id: 0, + base: 0x0802_0000, + spis: Some(SpiRange { base: 80, count: 64 }), + }))] + ); + // The entry walk a kernel already makes still answers it by its type alone. + let table = find_table(Machine { regions: HWGIC }, RSDP, b"APIC", MADT_ENTRIES).expect("MADT"); + assert_eq!(madt_entries(&table).last(), Some(Ok(MadtEntry::Other(0xD)))); +} + +/// `arm-gicv3-its` at `0x08080000`, under the id the IORT's ITS node carries. +#[test] +fn the_its_boot_names_the_its_the_iort_routes_to() { + assert_eq!(controllers(ITS), [Ok(MsiController::Its { id: 0, base: 0x0808_0000 })]); +} + +// --- crafted tables ------------------------------------------------------- + +const TABLE_AT: u64 = 0x9000; + +/// One node: Table 4's header over `fields`, then `mappings`, the ID array +/// placed straight after the fields. +fn node(kind: u8, revision: u8, fields: &[u8], mappings: &[[u32; 5]]) -> Vec { + let mut n = vec![kind, 0, 0, revision, 0, 0, 0, 0]; + n.extend((mappings.len() as u32).to_le_bytes()); + n.extend(if mappings.is_empty() { 0u32 } else { 16 + fields.len() as u32 }.to_le_bytes()); + n.extend(fields); + for word in mappings.iter().flatten() { + n.extend(word.to_le_bytes()); + } + let len = n.len() as u16; + n[1..3].copy_from_slice(&len.to_le_bytes()); + n +} + +fn its(id: u32) -> Vec { + node(0, 1, &[1u32.to_le_bytes(), id.to_le_bytes()].concat(), &[]) +} + +/// A revision 3 root complex of `segment`: 20 bytes of fields. +fn root_complex(segment: u32, mappings: &[[u32; 5]]) -> Vec { + let mut fields = vec![0u8; 20]; + fields[12..16].copy_from_slice(&segment.to_le_bytes()); + node(2, 3, &fields, mappings) +} + +/// A revision 4 SMMUv3 at `base` with no flag and no GSIV: 52 bytes of fields. +fn smmu(base: u64, mappings: &[[u32; 5]]) -> Vec { + let mut fields = vec![0u8; 52]; + fields[..8].copy_from_slice(&base.to_le_bytes()); + node(4, 4, &fields, mappings) +} + +/// An IORT of `revision` over `nodes`, its array straight after the header. +fn table(revision: u8, nodes: &[Vec]) -> Vec { + let mut body = Vec::new(); + body.extend((nodes.len() as u32).to_le_bytes()); + body.extend(48u32.to_le_bytes()); + body.extend(0u32.to_le_bytes()); + body.extend(nodes.concat()); + sdt(b"IORT", revision, &body) +} + +/// Where each of three nodes laid out as `[its, smmu, root complex]` starts: +/// the root complex after an SMMUv3 of no mapping, and of one. +const ITS_AT: u32 = 48; +const SMMU_AT: u32 = ITS_AT + 24; +const ROOT_AT: u32 = SMMU_AT + 68; +const ROOT_AFTER_ONE: u32 = ROOT_AT + 20; + +/// The ITS boot's shape with every number chosen: requesters `0x100..=0x1ff` +/// to streams `0x1000..`, streams `0x1000..=0x10ff` to DeviceIDs `0x20000..`. +fn three(root: &[[u32; 5]], onward: &[[u32; 5]]) -> Vec { + table(5, &[its(7), smmu(0x0905_0000, onward), root_complex(0, root)]) +} + +fn judged(bytes: &[u8], ask: impl FnOnce(&toyos_acpi::Iort>) -> T) -> Result { + let head = rsdp(0x1000, 2, 36); + let root = xsdt(&[TABLE_AT]); + let regions: &[(u64, &[u8])] = &[(0x800, &head), (0x1000, &root), (TABLE_AT, bytes)]; + iort(Machine { regions }, 0x800).map(|table| ask(&table)) +} + +fn refusal(bytes: &[u8]) -> Option { + judged(bytes, |_| ()).err() +} + +fn route(bytes: &[u8], segment: u32, rid: u16) -> Result { + judged(bytes, |table| table.route(segment, rid))? +} + +fn plain_smmu() -> Smmuv3 { + Smmuv3 { base: 0x0905_0000, coherent: false, event: None, pri: None, gerror: None, sync: None } +} + +#[test] +fn a_mapping_carries_an_id_by_its_offset_from_the_input_base_and_ends_at_its_last_id() { + let t = three(&[[0x100, 0xff, 0x1000, SMMU_AT, 0]], &[[0x1000, 0xff, 0x2_0000, ITS_AT, 0]]); + let through = |stream, device| { + Ok(Route::Translated { smmu: plain_smmu(), stream, its: Some(ItsDevice { its: 7, device }) }) + }; + assert_eq!(route(&t, 0, 0x100), through(0x1000, 0x2_0000)); + assert_eq!(route(&t, 0, 0x180), through(0x1080, 0x2_0080)); + // `Number of IDs` is the count minus one: the last ID is in, the next is out. + assert_eq!(route(&t, 0, 0x1ff), through(0x10ff, 0x2_00ff)); + assert_eq!(route(&t, 0, 0x200), Ok(Route::Unmapped)); + assert_eq!(route(&t, 0, 0x0ff), Ok(Route::Unmapped)); +} + +#[test] +fn a_stream_the_smmu_maps_nowhere_has_no_its() { + let t = three(&[[0, 0x1ff, 0, SMMU_AT, 0]], &[[0, 0xff, 0, ITS_AT, 0]]); + assert_eq!(route(&t, 0, 0x100), Ok(Route::Translated { smmu: plain_smmu(), stream: 0x100, its: None })); +} + +#[test] +fn a_revision_whose_layout_was_not_read_is_refused() { + for revision in [0, 4, 8, 0xff] { + assert_eq!(refusal(&table(revision, &[its(0)])), Some(IortRefused::Revision(revision))); + } + for revision in [5, 6, 7] { + assert_eq!(refusal(&table(revision, &[its(0)])), None); + } +} + +#[test] +fn a_table_too_short_for_its_own_header_fields_is_refused() { + let mut t = table(5, &[]); + declare_len(&mut t, 44); + assert_eq!(refusal(&t), Some(IortRefused::Table(TableError::Length { declared: 44, needed: 48 }))); +} + +#[test] +fn a_node_array_inside_the_header_is_refused() { + let mut t = table(5, &[its(0)]); + t[40..44].copy_from_slice(&40u32.to_le_bytes()); + reseal(&mut t); + assert_eq!(refusal(&t), Some(IortRefused::NodeArray { offset: 40 })); +} + +#[test] +fn a_node_count_that_overruns_the_table_is_refused_where_the_bytes_end() { + let mut t = table(5, &[its(0), its(1)]); + t[36..40].copy_from_slice(&3u32.to_le_bytes()); + reseal(&mut t); + assert_eq!(refusal(&t), Some(IortRefused::Node { at: 96, declared: 0 })); + // A count nothing could hold costs one refusal, not a walk of its length. + t[36..40].copy_from_slice(&u32::MAX.to_le_bytes()); + reseal(&mut t); + assert_eq!(refusal(&t), Some(IortRefused::Node { at: 96, declared: 0 })); +} + +#[test] +fn a_node_array_past_the_table_is_refused_as_its_first_node() { + let mut t = table(5, &[its(0)]); + t[40..44].copy_from_slice(&u32::MAX.to_le_bytes()); + reseal(&mut t); + assert_eq!(refusal(&t), Some(IortRefused::Node { at: u32::MAX as usize, declared: 0 })); +} + +#[test] +fn a_node_longer_than_the_table_or_shorter_than_its_type_is_refused() { + let mut long = table(5, &[its(0)]); + long[49..51].copy_from_slice(&25u16.to_le_bytes()); + reseal(&mut long); + assert_eq!(refusal(&long), Some(IortRefused::Node { at: 48, declared: 25 })); + + // Zero would never advance the walk; eight is below Table 4's own header. + for declared in [0u16, 8, 23] { + let mut short = table(5, &[its(0)]); + short[49..51].copy_from_slice(&declared.to_le_bytes()); + reseal(&mut short); + assert_eq!(refusal(&short), Some(IortRefused::Node { at: 48, declared: usize::from(declared) })); + } + + // An SMMUv3 node cut before its DeviceID mapping index, with bytes after it to read. + let mut cut = table(5, &[smmu(0, &[]), its(0)]); + cut[49..51].copy_from_slice(&64u16.to_le_bytes()); + reseal(&mut cut); + assert_eq!(refusal(&cut), Some(IortRefused::Node { at: 48, declared: 64 })); +} + +#[test] +fn a_node_revision_whose_layout_was_not_read_is_refused() { + let revised = |mut node: Vec, revision: u8| { + node[3] = revision; + refusal(&table(5, &[node])) + }; + for (kind, node, known, unknown) in [ + (0u8, its(0), &[1u8][..], &[0u8, 2][..]), + (2, root_complex(0, &[]), &[3, 4], &[2, 5]), + (4, smmu(0, &[]), &[4, 5], &[3, 6]), + ] { + for &revision in known { + assert_eq!(revised(node.clone(), revision), None, "type {kind} revision {revision}"); + } + for &revision in unknown { + assert_eq!( + revised(node.clone(), revision), + Some(IortRefused::NodeRevision { at: 48, kind, revision }), + "type {kind} revision {revision}" + ); + } + } +} + +/// A root complex whose ID array is `offset` into it and which is `pad` bytes +/// longer than its revision 3 fields. +fn root_with_array_at(revision: u8, offset: u32, pad: usize) -> Vec { + let mut node = node(2, revision, &vec![0u8; 20 + pad], &[[0, 0, 0, 48, 0]]); + node[12..16].copy_from_slice(&offset.to_le_bytes()); + node +} + +#[test] +fn an_id_array_lying_over_its_nodes_own_fields_is_refused() { + let with = |root: Vec| refusal(&table(5, &[its(0), root])); + // Revision 3's fields end at 36 and revision 4's, with its PASID capabilities and flags, at 40. + assert_eq!(with(root_with_array_at(3, 36, 0)), None); + assert_eq!(with(root_with_array_at(3, 32, 0)), Some(IortRefused::Mappings { at: 72, count: 1, offset: 32 })); + assert_eq!(with(root_with_array_at(4, 40, 4)), None); + assert_eq!(with(root_with_array_at(4, 36, 4)), Some(IortRefused::Mappings { at: 72, count: 1, offset: 36 })); + // And inside the node's header. + assert_eq!(with(root_with_array_at(3, 0, 0)), Some(IortRefused::Mappings { at: 72, count: 1, offset: 0 })); +} + +#[test] +fn an_id_array_running_past_its_node_is_refused() { + let counted = |count: u32| { + let mut root = root_complex(0, &[[0, 0, 0, 48, 0], [1, 0, 1, 48, 0]]); + root[8..12].copy_from_slice(&count.to_le_bytes()); + refusal(&table(5, &[its(0), root])) + }; + assert_eq!(counted(2), None); + assert_eq!(counted(3), Some(IortRefused::Mappings { at: 72, count: 3, offset: 36 })); + // A count whose bytes no 32-bit length could hold. + assert_eq!(counted(u32::MAX), Some(IortRefused::Mappings { at: 72, count: u32::MAX, offset: 36 })); + + // Past its node and inside the table: the third mapping would be the next node's bytes. + let mut root = root_complex(0, &[[0, 0, 0, 48, 0], [1, 0, 1, 48, 0]]); + root[8..12].copy_from_slice(&3u32.to_le_bytes()); + assert_eq!( + refusal(&table(5, &[its(0), root, its(1)])), + Some(IortRefused::Mappings { at: 72, count: 3, offset: 36 }) + ); + + // The array's start alone past the node. + let mut root = root_complex(0, &[[0, 0, 0, 48, 0]]); + root[12..16].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!( + refusal(&table(5, &[its(0), root])), + Some(IortRefused::Mappings { at: 72, count: 1, offset: u32::MAX }) + ); +} + +#[test] +fn an_its_node_declaring_an_id_array_is_refused() { + let fields = [1u32.to_le_bytes(), 0u32.to_le_bytes()].concat(); + let node = node(0, 1, &fields, &[[0, 0, 0, 48, 0]]); + assert_eq!(refusal(&table(5, &[node])), Some(IortRefused::Mappings { at: 48, count: 1, offset: 24 })); +} + +#[test] +fn a_mapping_whose_range_leaves_the_id_space_is_refused() { + let input = three(&[[0xffff_ff00, 0x100, 0, SMMU_AT, 0]], &[]); + assert_eq!(refusal(&input), Some(IortRefused::Range { at: 140, base: 0xffff_ff00, last: 0x100 })); + let output = three(&[[0, 0x100, 0xffff_ff00, SMMU_AT, 0]], &[]); + assert_eq!(refusal(&output), Some(IortRefused::Range { at: 140, base: 0xffff_ff00, last: 0x100 })); + // The whole space, to its last ID, is a range. + assert_eq!(refusal(&three(&[[0, u32::MAX, 0, SMMU_AT, 0]], &[])), None); + let onward = three(&[], &[[1, u32::MAX, 0, ITS_AT, 0]]); + assert_eq!(refusal(&onward), Some(IortRefused::Range { at: 72, base: 1, last: u32::MAX })); +} + +#[test] +fn an_output_reference_that_names_no_node_is_refused() { + // Zero, the table's own header, the middle of a node, one past the last node. + for reference in [0, 36, SMMU_AT + 4, ROOT_AT + 56, u32::MAX] { + assert_eq!( + refusal(&three(&[[0, 0xff, 0, reference, 0]], &[])), + Some(IortRefused::Output { at: 140, reference }), + "reference {reference}" + ); + } +} + +#[test] +fn an_output_reference_to_a_node_its_source_may_not_reach_is_refused() { + // A root complex to a root complex — itself. + assert_eq!( + refusal(&three(&[[0, 0xff, 0, ROOT_AT, 0]], &[])), + Some(IortRefused::Output { at: 140, reference: ROOT_AT }) + ); + // An SMMUv3 to an SMMUv3 — itself — and to a root complex. + for reference in [SMMU_AT, ROOT_AFTER_ONE] { + assert_eq!( + refusal(&three(&[], &[[0, 0xff, 0, reference, 0]])), + Some(IortRefused::Output { at: 72, reference }) + ); + } + // A root complex to a node type not decoded: an SMMUv1 or v2. + let old = table(5, &[node(3, 3, &[0u8; 44], &[]), root_complex(0, &[[0, 0xff, 0, 48, 0]])]); + assert_eq!(refusal(&old), Some(IortRefused::Output { at: 108, reference: 48 })); +} + +#[test] +fn a_node_type_not_decoded_is_answered_by_its_type_and_refuses_nothing() { + // A named component, a PMCG, a reserved memory range, an IWB, and a reserved type. + let others = [1u8, 5, 6, 7, 0x80].map(|kind| node(kind, 9, &[0xa5; 12], &[])); + let mut nodes = vec![its(3)]; + nodes.extend(others); + nodes.push(root_complex(0, &[[0, 0xff, 0, 48, 0]])); + let t = table(5, &nodes); + assert_eq!( + judged(&t, |table| table.nodes().collect::>()), + Ok(vec![ + Node::Its { id: 3 }, + Node::Other(1), + Node::Other(5), + Node::Other(6), + Node::Other(7), + Node::Other(0x80), + Node::RootComplex { segment: 0 }, + ]) + ); + assert_eq!(route(&t, 0, 8), Ok(Route::Untranslated(ItsDevice { its: 3, device: 8 }))); +} + +#[test] +fn a_single_mapping_is_the_smmus_own_message_and_never_a_root_complexs() { + assert_eq!( + refusal(&three(&[[0, 0xff, 0, SMMU_AT, 1]], &[])), + Some(IortRefused::SingleMapping { at: 140 }) + ); + // On the SMMUv3 it names the DeviceID of the unit's own interrupts: its + // input fields are ignored, so it routes no stream and its range is not judged. + let own = three(&[[0, 0xff, 0, SMMU_AT, 0]], &[[0, 0, 0x30, ITS_AT, 1]]); + assert_eq!(route(&own, 0, 0), Ok(Route::Translated { smmu: plain_smmu(), stream: 0, its: None })); + let wide = three(&[[0, 0xff, 0, SMMU_AT, 0]], &[[0xffff_ffff, 0xffff_ffff, 0x30, ITS_AT, 1]]); + assert_eq!(refusal(&wide), None); + // It still has to name an ITS. + assert_eq!( + refusal(&three(&[], &[[0, 0, 0x30, ROOT_AFTER_ONE, 1]])), + Some(IortRefused::Output { at: 72, reference: ROOT_AFTER_ONE }) + ); +} + +#[test] +fn an_its_group_of_other_than_one_its_is_refused() { + for count in [0u32, 2] { + let group = node(0, 1, &[count.to_le_bytes(), 0u32.to_le_bytes(), 1u32.to_le_bytes()].concat(), &[]); + assert_eq!(refusal(&table(5, &[group])), Some(IortRefused::ItsGroup { at: 48, count })); + } +} + +#[test] +fn an_id_two_mappings_claim_is_refused_when_it_is_asked_for() { + // In one root complex: `0x00..=0xff` and `0x80..=0x17f`. + let t = three(&[[0, 0xff, 0, SMMU_AT, 0], [0x80, 0xff, 0x80, ITS_AT, 0]], &[]); + assert_eq!(route(&t, 0, 0x7f), Ok(Route::Translated { smmu: plain_smmu(), stream: 0x7f, its: None })); + assert_eq!(route(&t, 0, 0x80), Err(IortRefused::Overlap { id: 0x80 })); + assert_eq!(route(&t, 0, 0x100), Ok(Route::Untranslated(ItsDevice { its: 7, device: 0x100 }))); + + // Across two root complexes of one segment; a second segment's is its own. + let two = |segment| { + table(5, &[its(7), root_complex(0, &[[0, 0xff, 0, 48, 0]]), root_complex(segment, &[[0, 0xff, 0x100, 48, 0]])]) + }; + assert_eq!(route(&two(0), 0, 8), Err(IortRefused::Overlap { id: 8 })); + assert_eq!(route(&two(1), 0, 8), Ok(Route::Untranslated(ItsDevice { its: 7, device: 8 }))); + assert_eq!(route(&two(1), 1, 8), Ok(Route::Untranslated(ItsDevice { its: 7, device: 0x108 }))); + + // In the SMMUv3, by the stream. + let onward = three(&[[0, 0xff, 0x40, SMMU_AT, 0]], &[[0, 0xff, 0, ITS_AT, 0], [0x48, 0, 0x900, ITS_AT, 0]]); + assert_eq!(route(&onward, 0, 8), Err(IortRefused::Overlap { id: 0x48 })); + assert_eq!( + route(&onward, 0, 9), + Ok(Route::Translated { smmu: plain_smmu(), stream: 0x49, its: Some(ItsDevice { its: 7, device: 0x49 }) }) + ); +} + +/// Firmware's bytes, any of them wrong: every single-byte change to either +/// captured IORT, and every length it could declare, is decoded or refused and +/// every requester of two buses asked for, without a panic and to an end. +#[test] +fn no_corruption_of_a_published_iort_panics_or_hangs_the_decoder() { + for regions in [HWGIC, ITS] { + let original = regions[3].1; + let ask = |bytes: &[u8]| { + let patched = [regions[0], regions[1], regions[2], (IORT, bytes)]; + if let Ok(table) = iort(Machine { regions: &patched }, RSDP) { + assert!(table.nodes().count() <= bytes.len() / 16); + for rid in 0..0x200 { + let _ = table.route(0, rid); + } + } + }; + for at in 36..original.len() { + for value in [0x00, 0x01, 0x7f, 0x80, 0xff, original[at].wrapping_add(1), original[at] ^ 0x10] { + let mut bytes = original.to_vec(); + bytes[at] = value; + reseal(&mut bytes); + ask(&bytes); + } + } + for len in 36..original.len() as u32 { + let mut bytes = original.to_vec(); + declare_len(&mut bytes, len); + ask(&bytes); + } + } +} + +// --- the MADT's MSI controllers ------------------------------------------- + +fn crafted_controllers(list: &[u8]) -> Vec> { + let t = madt(list); + let head = rsdp(0x1000, 2, 36); + let root = xsdt(&[TABLE_AT]); + let regions: &[(u64, &[u8])] = &[(0x800, &head), (0x1000, &root), (TABLE_AT, &t)]; + let table = find_table(Machine { regions }, 0x800, b"APIC", MADT_ENTRIES).expect("MADT"); + msi_controllers(&table).take(8).collect() +} + +/// A GIC MSI Frame structure's 22 bytes after its type and length. +fn frame(id: u32, base: u64, flags: u32, count: u16, spi_base: u16) -> Vec { + let mut body = vec![0u8; 2]; + body.extend(id.to_le_bytes()); + body.extend(base.to_le_bytes()); + body.extend(flags.to_le_bytes()); + body.extend(count.to_le_bytes()); + body.extend(spi_base.to_le_bytes()); + entry(0xD, 24, &body) +} + +#[test] +fn a_frame_names_its_spis_only_where_its_flag_says_the_structure_does() { + let mut list = frame(3, 0x0802_0000, 0, 64, 80); + list.extend(frame(4, 0x0803_0000, 1, 32, 144)); + // Every flag but the select bit: reserved, and no word about the SPIs. + list.extend(frame(5, 0x0804_0000, !1, 8, 200)); + assert_eq!( + crafted_controllers(&list), + [ + Ok(MsiController::Frame(MsiFrame { id: 3, base: 0x0802_0000, spis: None })), + Ok(MsiController::Frame(MsiFrame { + id: 4, + base: 0x0803_0000, + spis: Some(SpiRange { base: 144, count: 32 }) + })), + Ok(MsiController::Frame(MsiFrame { id: 5, base: 0x0804_0000, spis: None })), + ] + ); +} + +#[test] +fn the_controllers_come_in_the_tables_order_past_every_other_structure() { + // A GICD, an ITS, a frame, a second ITS. + let mut list = entry(0xC, 24, &[0u8; 22]); + let its = |id: u32, base: u64| { + let mut body = vec![0u8; 2]; + body.extend(id.to_le_bytes()); + body.extend(base.to_le_bytes()); + body.extend([0u8; 4]); + entry(0xF, 20, &body) + }; + list.extend(its(1, 0x0808_0000)); + list.extend(frame(2, 0x0802_0000, 0, 0, 0)); + list.extend(its(9, 0x0809_0000)); + assert_eq!( + crafted_controllers(&list), + [ + Ok(MsiController::Its { id: 1, base: 0x0808_0000 }), + Ok(MsiController::Frame(MsiFrame { id: 2, base: 0x0802_0000, spis: None })), + Ok(MsiController::Its { id: 9, base: 0x0809_0000 }), + ] + ); +} + +#[test] +fn a_frame_too_short_for_its_fields_is_no_controller_and_a_halt_ends_the_walk() { + // Twenty-two bytes: the SPI base would be read from the next structure. + let mut list = entry(0xD, 22, &[0xff; 20]); + list.extend(frame(1, 0x0802_0000, 0, 0, 0)); + list.extend(entry(0xD, 0, &[])); + list.extend(frame(2, 0x0803_0000, 0, 0, 0)); + assert_eq!( + crafted_controllers(&list), + [ + Ok(MsiController::Frame(MsiFrame { id: 1, base: 0x0802_0000, spis: None })), + Err(MadtHalt { at: 46, declared: 0, list_len: 72 }), + ] + ); +} diff --git a/toyos-its/Cargo.toml b/toyos-its/Cargo.toml new file mode 100644 index 00000000000..f6c8d3ed685 --- /dev/null +++ b/toyos-its/Cargo.toml @@ -0,0 +1,17 @@ +# A member of the host workspace (root `Cargo.toml`), like toyos-acpi and +# toyos-smmu: it is pure, it has no dependencies, and its tests run on the +# host. +# +# What lives here is every word a GICv3 ITS and its driver exchange, and the +# two tables a redistributor reads an LPI's configuration and pending state +# from. The ITS is what makes a message a claimed function writes arrive as +# that claim's interrupt and no other's, so its commands are held on the host +# to the specification's field positions. + +[package] +name = "toyos-its" +description = "The GICv3 ITS's encodings: commands, table sizing, LPI configuration and pending tables and register fields, pure, forbid(unsafe_code)." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false diff --git a/toyos-its/src/command.rs b/toyos-its/src/command.rs new file mode 100644 index 00000000000..7dc7d67c2ae --- /dev/null +++ b/toyos-its/src/command.rs @@ -0,0 +1,56 @@ +//! The ITS's commands (§5.3): 32 bytes each, four little-endian +//! doublewords, the command number in bits [7:0] of the first. + +use crate::lpi::Lpi; +use crate::{EventBits, Phys, Target}; + +/// One command. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Command { + /// `MAPD` with `V` set (§5.3.10): `device`'s events are translated by + /// the zeroed table at `table`. + MapDevice { device: u32, table: Phys<8>, events: EventBits }, + /// `MAPD` with `V` clear: `device` has no table, and its messages are + /// ignored. + UnmapDevice { device: u32 }, + /// `MAPC` with `V` set (§5.3.9): `collection`'s interrupts go to `target`. + MapCollection { collection: u16, target: Target }, + /// `MAPTI` (§5.3.12): `device`'s `event` is `lpi`, in `collection`. + MapEvent { device: u32, event: u32, lpi: Lpi, collection: u16 }, + /// `INV` (§5.3.6): reread the configuration of `device`'s `event`'s LPI. + Reconfigure { device: u32, event: u32 }, + /// `DISCARD` (§5.3.4): `device`'s `event` is no interrupt, and any + /// pending one is dropped. + Discard { device: u32, event: u32 }, + /// `SYNC` (§5.3.15): every command before it has taken effect at + /// `target` before the next is read. + Sync(Target), +} + +/// `V`, bit [63] of doubleword 2. +const V: u64 = 1 << 63; + +impl Command { + pub const fn words(self) -> [u64; 4] { + // DeviceID is bits [63:32] of doubleword 0, and EventID bits [31:0] of doubleword 1. + const fn device(number: u64, device: u32) -> u64 { + number | (device as u64) << 32 + } + match self { + // Size [4:0] of doubleword 1 is the EventID bits minus one; ITT_addr [51:8] of doubleword 2. + Self::MapDevice { device: id, table, events } => { + [device(0x08, id), (events.0 - 1) as u64, V | table.get(), 0] + } + Self::UnmapDevice { device: id } => [device(0x08, id), 0, 0, 0], + // RDbase [51:16] and ICID [15:0] of doubleword 2. + Self::MapCollection { collection, target } => [0x09, 0, V | target.0 << 16 | collection as u64, 0], + // pINTID [63:32] of doubleword 1; ICID [15:0] of doubleword 2. + Self::MapEvent { device: id, event, lpi, collection } => { + [device(0x0A, id), (lpi.intid() as u64) << 32 | event as u64, collection as u64, 0] + } + Self::Reconfigure { device: id, event } => [device(0x0C, id), event as u64, 0, 0], + Self::Discard { device: id, event } => [device(0x0F, id), event as u64, 0, 0], + Self::Sync(target) => [0x05, 0, target.0 << 16, 0], + } + } +} diff --git a/toyos-its/src/lib.rs b/toyos-its/src/lib.rs new file mode 100644 index 00000000000..a1a88fd0d7e --- /dev/null +++ b/toyos-its/src/lib.rs @@ -0,0 +1,264 @@ +//! Every word a GICv3 Interrupt Translation Service and its driver exchange, +//! as Arm IHI 0069 issue H.b lays it out: the ITS's registers and what its +//! type register says of it (this module), its commands ([`command`]), and +//! the two tables a redistributor reads an LPI's configuration and pending +//! state from ([`lpi`]). +//! +//! The device table and each device's interrupt translation table are the +//! ITS's own: their entry formats are IMPLEMENTATION DEFINED (§5.2.3, +//! §5.2.4), software gives the ITS zeroed memory of the size its registers +//! ask for and writes an entry only by a command. So no entry is encoded +//! here; the sizes are. +//! +//! Only physical LPIs through flat tables are spelled: no vPE, no indirect +//! table. What an ITS reports about itself is a device's word and is refused +//! by name where the driver could not act on it; nothing here panics on a +//! register's value. +//! +//! `no_std`, no allocation, no `unsafe`. + +#![no_std] +#![forbid(unsafe_code)] + +pub mod command; +pub mod lpi; + +/// A physical address the ITS or a redistributor is given, aligned to +/// `1 << ALIGN` bytes and below 2^48: inside every address field written +/// here, whichever page size the table it names is read in. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Phys(u64); + +impl Phys { + pub const fn new(address: u64) -> Option { + if address >> 48 == 0 && address & ((1 << ALIGN) - 1) == 0 { + Some(Self(address)) + } else { + None + } + } + + pub const fn get(self) -> u64 { + self.0 + } +} + +/// Offsets in the ITS's control frame (§12.18). +pub const GITS_CTLR: usize = 0x0000; +pub const GITS_TYPER: usize = 0x0008; +pub const GITS_CBASER: usize = 0x0080; +pub const GITS_CWRITER: usize = 0x0088; +pub const GITS_CREADR: usize = 0x0090; + +/// `GITS_BASER`, n = 0 to 7: one per table the ITS may ask memory for. +pub const fn gits_baser(n: usize) -> Option { + if n < 8 { + Some(0x0100 + 8 * n) + } else { + None + } +} + +/// The translation frame is the second 64 KiB frame, and `GITS_TRANSLATER` +/// is at 0x0040 of it: a function's message is one 32-bit write of its +/// EventID there, and the bus says which device wrote it. +pub const TRANSLATION_FRAME: u64 = 0x1_0000; +pub const GITS_TRANSLATER: u64 = 0x0040; + +/// `GITS_CTLR` (§12.19.4): `Enabled` [0]; `Quiescent` [31] reads set once a +/// disabled ITS has nothing in progress. +pub const CTLR_ENABLED: u32 = 1 << 0; +pub const CTLR_QUIESCENT: u32 = 1 << 31; + +/// What an ITS lacks that the driver needs. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Lacks { + /// `GITS_TYPER.Physical` clear: no physical LPIs. + PhysicalLpis, +} + +/// An ITS as its type register describes it (§12.19.13). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Its { + /// `ID_bits` [12:8] plus one: the widest EventID. + pub event_bits: u8, + /// `Devbits` [17:13] plus one: the widest DeviceID. + pub device_bits: u8, + /// `ITT_entry_size` [7:4] plus one: bytes of one translation table entry. + pub itt_entry_bytes: u8, + /// `HCC` [31:24]: collections the ITS holds without a collection table. + pub collections_held: u8, + /// `PTA` [19]: a collection's target is its redistributor's address + /// rather than its processor number. + physical_targets: bool, +} + +pub const fn probe(typer: u64) -> Result { + if typer & 1 == 0 { + return Err(Lacks::PhysicalLpis); + } + Ok(Its { + event_bits: (typer >> 8 & 0x1F) as u8 + 1, + device_bits: (typer >> 13 & 0x1F) as u8 + 1, + itt_entry_bytes: (typer >> 4 & 0xF) as u8 + 1, + collections_held: (typer >> 24) as u8, + physical_targets: typer & 1 << 19 != 0, + }) +} + +/// How many EventIDs one device's translation table holds, as `MAPD` says +/// it: made by [`Its::events`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct EventBits(u8); + +/// The redistributor a collection's interrupts go to, in the form this ITS +/// takes it (`RDbase`, §5.3.1): made by [`Its::target`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Target(u64); + +impl Its { + /// A translation table of `1 << bits` events. `None` for no bit, or more + /// than the ITS's EventIDs have. + pub const fn events(&self, bits: u8) -> Option { + if bits >= 1 && bits <= self.event_bits { + Some(EventBits(bits)) + } else { + None + } + } + + /// The zeroed bytes `MAPD` is given for that table, at a 256-byte + /// aligned address. + pub const fn itt_bytes(&self, events: EventBits) -> u64 { + (self.itt_entry_bytes as u64) << events.0 + } + + /// The redistributor at `frame` whose `GICR_TYPER.Processor_Number` is + /// `number`, as this ITS names it. + pub const fn target(&self, number: u16, frame: Phys<16>) -> Target { + Target(if self.physical_targets { frame.get() >> 16 } else { number as u64 }) + } +} + +/// What a `GITS_BASER` asks memory for (`Type`, bits [58:56]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Table { + /// `0b000`: the register backs no table. + Unimplemented, + /// `0b001`: one entry for every DeviceID. + Devices, + /// `0b100`: one entry for every collection past those the ITS holds. + Collections, + /// vPEs (`0b010`), or a reserved type. + Other(u8), +} + +/// The page a table is read in (`Page_Size`, bits [9:8]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PageSize { + K4, + K16, + K64, +} + +impl PageSize { + pub const fn bytes(self) -> u64 { + match self { + Self::K4 => 4 << 10, + Self::K16 => 16 << 10, + Self::K64 => 64 << 10, + } + } +} + +/// `GITS_BASER` as read (§12.19.1): the table it backs, the bytes of one +/// entry (`Entry_Size` [52:48] plus one) and the page size it holds — which +/// an ITS may fix, so it is read back after [`baser`] is written. +pub const fn table(baser: u64) -> (Table, u8, PageSize) { + let kind = match baser >> 56 & 0b111 { + 0b000 => Table::Unimplemented, + 0b001 => Table::Devices, + 0b100 => Table::Collections, + other => Table::Other(other as u8), + }; + let page = match baser >> 8 & 0b11 { + 0b00 => PageSize::K4, + 0b01 => PageSize::K16, + // `0b11` is reserved and treated as `0b10`. + _ => PageSize::K64, + }; + (kind, (baser >> 48 & 0x1F) as u8 + 1, page) +} + +/// `InnerCache` `0b111`, read-allocate write-allocate write-back, with +/// `OuterCache` zero, as the inner; `Shareability` [11:10] `0b01`, inner +/// shareable. `GITS_BASER` and `GITS_CBASER` hold them at the same bits. +const CACHED: u64 = 0b111 << 59 | 0b01 << 10; +const VALID: u64 = 1 << 63; + +/// `GITS_BASER` for a flat table of `pages` pages of `page` at `at`: +/// `Valid` [63], `Indirect` [62] clear, `Physical_Address` [47:12], +/// `Page_Size` [9:8], `Size` [7:0] the pages minus one. `None` for no page or +/// more than the 256 the field counts. +pub const fn baser(at: Phys<16>, page: PageSize, pages: u16) -> Option { + if pages == 0 || pages > 256 { + return None; + } + let size = match page { + PageSize::K4 => 0b00, + PageSize::K16 => 0b01, + PageSize::K64 => 0b10, + }; + Some(VALID | CACHED | at.get() | size << 8 | (pages - 1) as u64) +} + +/// The ITS's command queue: `pages` 4 KiB pages of 32-byte commands +/// (§5.2.8). `GITS_CWRITER` and `GITS_CREADR` each hold a byte offset into +/// it, `Offset` [19:5]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct CommandQueue { + bytes: u32, +} + +/// `GITS_CREADR.Stalled` [0]: the ITS stopped on a command it refused. +pub const CREADR_STALLED: u64 = 1 << 0; + +impl CommandQueue { + /// `None` for no page or more than the 256 `GITS_CBASER.Size` counts. + pub const fn new(pages: u16) -> Option { + if pages == 0 || pages > 256 { + None + } else { + Some(Self { bytes: pages as u32 * 4096 }) + } + } + + /// `GITS_CBASER` (§12.19.2) for the queue at `at`: `Valid` [63], + /// `Physical_Address` [51:12] with its low four bits zero, `Size` [7:0] + /// the pages minus one. + pub const fn cbaser(self, at: Phys<16>) -> u64 { + VALID | CACHED | at.get() | (self.bytes / 4096 - 1) as u64 + } + + /// The byte offset an offset register names: its `Offset` field alone, + /// and inside the queue whatever the ITS wrote there. + pub const fn offset(self, register: u64) -> u32 { + (register & 0xF_FFE0) as u32 % self.bytes + } + + /// The offset after `offset`, wrapping at the queue's end. + pub const fn after(self, offset: u32) -> u32 { + (self.offset(offset as u64) + 32) % self.bytes + } + + /// The ITS has read every command: the two offsets are equal. + pub const fn is_empty(self, writer: u32, reader: u32) -> bool { + writer == reader + } + + /// One more command would make the writer equal the reader, which reads + /// as empty: the queue holds one command fewer than its bytes do. + pub const fn is_full(self, writer: u32, reader: u32) -> bool { + self.after(writer) == reader + } +} diff --git a/toyos-its/src/lpi.rs b/toyos-its/src/lpi.rs new file mode 100644 index 00000000000..e252f93535b --- /dev/null +++ b/toyos-its/src/lpi.rs @@ -0,0 +1,111 @@ +//! LPIs (§5.1): their INTIDs, the configuration table every redistributor +//! reads and the pending table each one owns, and the redistributor registers +//! that name them. + +use crate::Phys; + +/// The first LPI's INTID (§2.2): everything below is an SGI, a PPI, an SPI +/// or special. +pub const FIRST: u32 = 8192; + +/// Offsets in a redistributor's `RD_base` frame (§12.10, Table 12-27). +pub const GICR_CTLR: usize = 0x0000; +pub const GICR_TYPER: usize = 0x0008; +pub const GICR_PROPBASER: usize = 0x0070; +pub const GICR_PENDBASER: usize = 0x0078; + +/// `GICR_CTLR.EnableLPIs` [0] (§12.11): from here the redistributor reads +/// both tables, and neither base register may be written. +pub const CTLR_ENABLE_LPIS: u32 = 1 << 0; + +/// `GICR_TYPER.PLPIS` [0] (§12.11.37): the redistributor takes physical LPIs. +pub const TYPER_PLPIS: u64 = 1 << 0; + +/// `GICR_TYPER.Processor_Number` [23:8]: how an ITS whose `GITS_TYPER.PTA` +/// is clear names this redistributor. +pub const fn processor_number(typer: u64) -> u16 { + (typer >> 8) as u16 +} + +/// An LPI's INTID, inside the space it was made in: [`Space::lpi`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Lpi(u32); + +impl Lpi { + pub const fn intid(self) -> u32 { + self.0 + } + + /// Where its byte is in the configuration table: the table's first byte + /// is INTID 8192's (§5.1.1). + pub const fn configuration_index(self) -> usize { + (self.0 - FIRST) as usize + } + + /// Where its bit is in a pending table, which is indexed by the INTID + /// itself: the byte, and the bit in it (§5.1.2). + pub const fn pending_bit(self) -> (usize, u8) { + ((self.0 / 8) as usize, (self.0 % 8) as u8) + } +} + +/// One LPI's byte in the configuration table (Table 5-1): the priority's +/// upper six bits in [7:2], bit [1] RES1, and the enable in bit [0]. +pub const fn configuration(priority: u8, enabled: bool) -> u8 { + priority & 0xFC | 0b10 | enabled as u8 +} + +/// The INTIDs of `id_bits` bits the tables are laid out for. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Space { + id_bits: u8, +} + +/// `InnerCache` [9:7] `0b111`, read-allocate write-allocate write-back, with +/// `OuterCache` [58:56] zero, as the inner; `Shareability` [11:10] `0b01`, +/// inner shareable. Both base registers hold them at the same bits. +const CACHED: u64 = 0b111 << 7 | 0b01 << 10; + +impl Space { + /// `None` below 14 bits, which reach no LPI, and above the 24 that keep + /// the configuration table within 16 MiB. + pub const fn new(id_bits: u8) -> Option { + if id_bits >= 14 && id_bits <= 24 { + Some(Self { id_bits }) + } else { + None + } + } + + /// `intid` as an LPI of this space. + pub const fn lpi(self, intid: u32) -> Option { + if intid >= FIRST && intid >> self.id_bits == 0 { + Some(Lpi(intid)) + } else { + None + } + } + + /// One byte for every LPI. + pub const fn configuration_bytes(self) -> usize { + (1 << self.id_bits) - FIRST as usize + } + + /// One bit for every INTID, the first 8192 included. + pub const fn pending_bytes(self) -> usize { + (1 << self.id_bits) / 8 + } + + /// `GICR_PROPBASER` (§12.11.33) for the configuration table at `table`: + /// `Physical_Address` [51:12] and `IDbits` [4:0], the bits minus one. + pub const fn propbaser(self, table: Phys<12>) -> u64 { + CACHED | table.get() | (self.id_bits - 1) as u64 + } +} + +/// `GICR_PENDBASER` (§12.11.32) for the pending table at `table`, which is +/// all zeroes: `Physical_Address` [51:16], and `PTZ` [62], which says so and +/// lets the redistributor not read it. +pub const fn pendbaser(table: Phys<16>) -> u64 { + 1 << 62 | CACHED | table.get() +} diff --git a/toyos-its/tests/encodings.rs b/toyos-its/tests/encodings.rs new file mode 100644 index 00000000000..3f2e5a77985 --- /dev/null +++ b/toyos-its/tests/encodings.rs @@ -0,0 +1,294 @@ +//! Every encoding against IHI 0069 H.b's own field positions: each expected +//! value is built from `(msb, lsb, value)` triples as the specification's +//! figures and field headings give them, and compared whole, so a field in +//! the wrong place and a stray bit are both a difference. + +use toyos_its::command::Command; +use toyos_its::lpi::{self, Space}; +use toyos_its::{baser, gits_baser, probe, table, CommandQueue, Its, Lacks, PageSize, Phys, Table}; + +/// `N` doublewords holding `value` in bits `[msb:lsb]` for every triple, and +/// zero everywhere else. +fn layout(fields: &[(u32, u32, u64)]) -> [u64; N] { + let mut words = [0u64; N]; + for &(msb, lsb, value) in fields { + let width = msb - lsb + 1; + assert!(width == 64 || value >> width == 0, "{value:#x} does not fit [{msb}:{lsb}]"); + for bit in 0..width { + if value >> bit & 1 != 0 { + let at = lsb + bit; + assert_eq!(words[(at / 64) as usize] >> (at % 64) & 1, 0, "two fields claim bit {at}"); + words[(at / 64) as usize] |= 1 << (at % 64); + } + } + } + words +} + +fn word(fields: &[(u32, u32, u64)]) -> u64 { + layout::<1>(fields)[0] +} + +/// A command's fields, each given by its doubleword and its bits in it. +fn command(fields: &[(u32, u32, u32, u64)]) -> [u64; 4] { + let absolute: Vec<_> = fields.iter().map(|&(dw, msb, lsb, value)| (dw * 64 + msb, dw * 64 + lsb, value)).collect(); + layout(&absolute) +} + +/// `GITS_TYPER` of an ITS of 16-bit EventIDs and DeviceIDs and 8-byte +/// translation entries, holding 3 collections itself, which names a +/// redistributor by `pta`: Physical [0], ITT_entry_size [7:4], ID_bits +/// [12:8], Devbits [17:13], PTA [19], HCC [31:24]. +fn typer(pta: u64) -> u64 { + word(&[(0, 0, 1), (7, 4, 7), (12, 8, 15), (17, 13, 15), (19, 19, pta), (31, 24, 3)]) +} + +fn its(pta: u64) -> Its { + probe(typer(pta)).expect("an ITS of physical LPIs") +} + +// --- §12.18, §12.19: the registers ------------------------------------------ + +#[test] +fn the_type_register_says_how_wide_an_its_is() { + let its = its(0); + assert_eq!((its.event_bits, its.device_bits, its.itt_entry_bytes, its.collections_held), (16, 16, 8, 3)); + // Each field at its largest, alone: every one is its value plus one but HCC. + let wide = |field: (u32, u32, u64)| probe(word(&[(0, 0, 1), field])).unwrap(); + assert_eq!(wide((12, 8, 31)).event_bits, 32); + assert_eq!(wide((17, 13, 31)).device_bits, 32); + assert_eq!(wide((7, 4, 15)).itt_entry_bytes, 16); + assert_eq!(wide((31, 24, 255)).collections_held, 255); + let least = probe(1).unwrap(); + assert_eq!((least.event_bits, least.device_bits, least.itt_entry_bytes, least.collections_held), (1, 1, 1, 0)); +} + +#[test] +fn an_its_without_physical_lpis_is_refused() { + assert_eq!(probe(typer(0) & !1), Err(Lacks::PhysicalLpis)); + assert_eq!(probe(!1), Err(Lacks::PhysicalLpis)); + assert!(probe(u64::MAX).is_ok()); +} + +#[test] +fn the_registers_sit_where_the_its_map_has_them() { + assert_eq!( + (toyos_its::GITS_CTLR, toyos_its::GITS_TYPER, toyos_its::GITS_CBASER, toyos_its::GITS_CWRITER, toyos_its::GITS_CREADR), + (0x0000, 0x0008, 0x0080, 0x0088, 0x0090) + ); + // GITS_BASER is 0x0100 to 0x0138. + assert_eq!((gits_baser(0), gits_baser(7), gits_baser(8)), (Some(0x0100), Some(0x0138), None)); + // GITS_TRANSLATER is 0x0040 of the second 64 KiB frame. + assert_eq!(toyos_its::TRANSLATION_FRAME + toyos_its::GITS_TRANSLATER, 0x1_0040); + assert_eq!(u64::from(toyos_its::CTLR_ENABLED | toyos_its::CTLR_QUIESCENT), word(&[(0, 0, 1), (31, 31, 1)])); + assert_eq!(toyos_its::CREADR_STALLED, word(&[(0, 0, 1)])); +} + +#[test] +fn a_table_register_says_what_it_backs_and_how_it_is_read() { + // Type [58:56], Entry_Size [52:48] the bytes minus one, Page_Size [9:8]. + let read = |kind: u64, entry: u64, page: u64| table(word(&[(58, 56, kind), (52, 48, entry), (9, 8, page)])); + assert_eq!(read(0b000, 0, 0b00), (Table::Unimplemented, 1, PageSize::K4)); + assert_eq!(read(0b001, 7, 0b10), (Table::Devices, 8, PageSize::K64)); + assert_eq!(read(0b100, 15, 0b01), (Table::Collections, 16, PageSize::K16)); + assert_eq!(read(0b010, 31, 0b11), (Table::Other(0b010), 32, PageSize::K64)); + assert_eq!(read(0b111, 0, 0b00).0, Table::Other(0b111)); + assert_eq!((PageSize::K4.bytes(), PageSize::K16.bytes(), PageSize::K64.bytes()), (4096, 16384, 65536)); +} + +#[test] +fn a_table_is_given_flat_valid_and_counted_in_its_own_pages() { + let at = Phys::<16>::new(0xffff_ffff_0000).expect("64 KiB aligned, below 2^48"); + let written = |page: u64, pages: u64| { + Some(word(&[ + (63, 63, 1), // Valid + (62, 62, 0), // Indirect + (61, 59, 0b111), // InnerCache: RaWaWb + (55, 53, 0b000), // OuterCache: as the inner + (47, 12, 0xffff_ffff_0000 >> 12), // Physical_Address + (11, 10, 0b01), // Shareability: inner + (9, 8, page), // Page_Size + (7, 0, pages - 1), // Size + ])) + }; + assert_eq!(baser(at, PageSize::K4, 1), written(0b00, 1)); + assert_eq!(baser(at, PageSize::K16, 8), written(0b01, 8)); + assert_eq!(baser(at, PageSize::K64, 256), written(0b10, 256)); + assert_eq!(baser(at, PageSize::K64, 0), None); + assert_eq!(baser(at, PageSize::K64, 257), None); +} + +#[test] +fn an_address_off_its_alignment_or_past_48_bits_is_no_address() { + assert_eq!(Phys::<8>::new(0x100).map(Phys::get), Some(0x100)); + assert_eq!(Phys::<8>::new(0x80), None); + assert_eq!(Phys::<16>::new(0x8000), None); + assert_eq!(Phys::<16>::new(1 << 48), None); + assert_eq!(Phys::<16>::new((1 << 48) - 0x1_0000).map(Phys::get), Some((1 << 48) - 0x1_0000)); +} + +// --- §5.2.8: the command queue ---------------------------------------------- + +#[test] +fn the_command_queue_is_counted_in_4_kib_pages_and_holds_one_command_fewer() { + let at = Phys::<16>::new(0x4003_0000).unwrap(); + // Valid [63], InnerCache [61:59], OuterCache [55:53], Physical_Address [51:12], Shareability [11:10], Size [7:0]. + let written = |pages: u64| { + word(&[(63, 63, 1), (61, 59, 0b111), (55, 53, 0), (51, 12, 0x4003_0000 >> 12), (11, 10, 0b01), (7, 0, pages - 1)]) + }; + assert_eq!(CommandQueue::new(1).unwrap().cbaser(at), written(1)); + assert_eq!(CommandQueue::new(256).unwrap().cbaser(at), written(256)); + assert!(CommandQueue::new(0).is_none() && CommandQueue::new(257).is_none()); + + // One page is 128 commands. Against a count kept beside it, over three laps. + let queue = CommandQueue::new(1).unwrap(); + let (mut writer, mut reader, mut held) = (0u32, 0u32, 0u32); + for step in 0..3 * 128 * 2 { + assert_eq!(queue.is_empty(writer, reader), held == 0, "step {step}"); + assert_eq!(queue.is_full(writer, reader), held == 127, "step {step}"); + if held < 127 && (step / 150u32).is_multiple_of(2) { + writer = queue.after(writer); + held += 1; + } else if held > 0 { + reader = queue.after(reader); + held -= 1; + } + assert!(writer < 4096 && reader < 4096 && writer % 32 == 0 && reader % 32 == 0); + } + assert_eq!(queue.after(4096 - 32), 0); + // An offset is the ITS's word: past the queue, it is still an offset in it. + assert_eq!(queue.after(u32::MAX), 0); + assert_eq!(queue.after(4096 + 64), 96); +} + +#[test] +fn an_offset_register_names_a_command_by_its_offset_field_alone() { + let queue = CommandQueue::new(2).unwrap(); + // Offset [19:5]; Stalled or Retry [0]; the rest reserved. + assert_eq!(queue.offset(word(&[(19, 5, 0x40), (0, 0, 1), (4, 1, 0xf), (63, 20, 0xfff_ffff_ffff)])), 0x40 << 5); + // An offset past the queue is a device's word: it still names a command in the queue. + assert!(queue.offset(word(&[(19, 5, 0x7fff)])) < 8192); +} + +// --- §5.3: commands --------------------------------------------------------- + +#[test] +fn each_command_holds_its_number_and_parameters_where_its_figure_has_them() { + let its = its(0); + let events = its.events(5).expect("32 events of an ITS of 16-bit EventIDs"); + let itt = Phys::<8>::new(0xffff_ffff_ff00).expect("256-byte aligned"); + let target = its.target(0x1234, Phys::new(0x080a_0000).unwrap()); + let lpi = Space::new(16).unwrap().lpi(0xabcd).expect("an LPI"); + + // MAPD: DeviceID [63:32]; Size [4:0] of DW1, the bits minus one; V [63] and ITT_addr [51:8] of DW2. + assert_eq!( + Command::MapDevice { device: 0xdead_beef, table: itt, events }.words(), + command(&[(0, 7, 0, 0x08), (0, 63, 32, 0xdead_beef), (1, 4, 0, 4), (2, 63, 63, 1), (2, 51, 8, 0xffff_ffff_ff00 >> 8)]) + ); + assert_eq!( + Command::UnmapDevice { device: 0xdead_beef }.words(), + command(&[(0, 7, 0, 0x08), (0, 63, 32, 0xdead_beef), (2, 63, 63, 0)]) + ); + // MAPC: V [63], RDbase [51:16] and ICID [15:0] of DW2. + assert_eq!( + Command::MapCollection { collection: 0xc011, target }.words(), + command(&[(0, 7, 0, 0x09), (2, 63, 63, 1), (2, 51, 16, 0x1234), (2, 15, 0, 0xc011)]) + ); + // MAPTI: DeviceID [63:32]; pINTID [63:32] and EventID [31:0] of DW1; ICID [15:0] of DW2. + assert_eq!( + Command::MapEvent { device: 0xdead_beef, event: 0x1f, lpi, collection: 0xc011 }.words(), + command(&[(0, 7, 0, 0x0A), (0, 63, 32, 0xdead_beef), (1, 63, 32, 0xabcd), (1, 31, 0, 0x1f), (2, 15, 0, 0xc011)]) + ); + // INV and DISCARD: DeviceID [63:32]; EventID [31:0] of DW1. + assert_eq!( + Command::Reconfigure { device: 0xdead_beef, event: 0xffff_ffff }.words(), + command(&[(0, 7, 0, 0x0C), (0, 63, 32, 0xdead_beef), (1, 31, 0, 0xffff_ffff)]) + ); + assert_eq!( + Command::Discard { device: 0xdead_beef, event: 0xffff_ffff }.words(), + command(&[(0, 7, 0, 0x0F), (0, 63, 32, 0xdead_beef), (1, 31, 0, 0xffff_ffff)]) + ); + // SYNC: RDbase [51:16] of DW2. + assert_eq!(Command::Sync(target).words(), command(&[(0, 7, 0, 0x05), (2, 51, 16, 0x1234)])); +} + +#[test] +fn a_redistributor_is_named_as_the_its_asks_by_number_or_by_address() { + let frame = Phys::<16>::new(0xffff_fffe_0000).unwrap(); + let rdbase = |pta: u64| Command::Sync(its(pta).target(0x1234, frame)).words()[2]; + // PTA clear: GICR_TYPER.Processor_Number. PTA set: bits [51:16] of the frame's address. + assert_eq!(rdbase(0), word(&[(51, 16, 0x1234)])); + assert_eq!(rdbase(1), word(&[(51, 16, 0xffff_fffe_0000 >> 16)])); +} + +#[test] +fn a_devices_events_are_at_least_two_and_no_more_than_the_its_has_bits_for() { + let its = its(0); + assert_eq!(its.events(0), None); + assert_eq!(its.events(17), None); + // 8 bytes an entry: 2 events are 16 bytes and 2^16 are 512 KiB. + assert_eq!(its.events(1).map(|events| its.itt_bytes(events)), Some(16)); + assert_eq!(its.events(16).map(|events| its.itt_bytes(events)), Some(8 << 16)); + let size = |bits| Command::MapDevice { device: 0, table: Phys::new(0).unwrap(), events: its.events(bits).unwrap() }.words()[1]; + assert_eq!((size(1), size(16)), (0, 15)); +} + +// --- §5.1: LPIs -------------------------------------------------------------- + +#[test] +fn an_lpi_is_an_intid_from_8192_inside_its_space() { + let space = Space::new(14).expect("the fewest bits that reach an LPI"); + assert_eq!(space.lpi(8191), None); + assert_eq!(space.lpi(8192).map(|lpi| lpi.intid()), Some(8192)); + assert_eq!(space.lpi(16383).map(|lpi| lpi.intid()), Some(16383)); + assert_eq!(space.lpi(16384), None); + assert_eq!(space.lpi(u32::MAX), None); + assert!(Space::new(13).is_none() && Space::new(24).is_some() && Space::new(25).is_none()); +} + +#[test] +fn the_configuration_table_starts_at_intid_8192_and_the_pending_table_at_zero() { + let space = Space::new(16).unwrap(); + // "(base address + (N - 8192))", one byte an LPI. + assert_eq!(space.lpi(8192).unwrap().configuration_index(), 0); + assert_eq!(space.lpi(65535).unwrap().configuration_index(), 65535 - 8192); + assert_eq!(space.configuration_bytes(), 65536 - 8192); + // "(base address + (N / 8))", bit "(N MOD 8)": one bit an INTID, the first 1 KiB below every LPI. + assert_eq!(space.lpi(8192).unwrap().pending_bit(), (1024, 0)); + assert_eq!(space.lpi(8199).unwrap().pending_bit(), (1024, 7)); + assert_eq!(space.lpi(65535).unwrap().pending_bit(), (8191, 7)); + assert_eq!(space.pending_bytes(), 8192); + assert_eq!((Space::new(14).unwrap().configuration_bytes(), Space::new(14).unwrap().pending_bytes()), (8192, 2048)); +} + +#[test] +fn an_lpis_configuration_is_its_priority_a_set_bit_and_its_enable() { + // Priority [7:2], bit [1] RES1, Enable [0]. + let byte = |priority: u64, enable: u64| word(&[(7, 2, priority), (1, 1, 1), (0, 0, enable)]) as u8; + assert_eq!(lpi::configuration(0xa0, true), byte(0xa0 >> 2, 1)); + assert_eq!(lpi::configuration(0xa0, false), byte(0xa0 >> 2, 0)); + // The priority's low two bits are not the table's to hold. + assert_eq!(lpi::configuration(0xff, false), byte(0x3f, 0)); + assert_eq!(lpi::configuration(0x03, true), byte(0, 1)); +} + +#[test] +fn the_redistributors_base_registers_name_the_two_tables() { + let space = Space::new(16).unwrap(); + // OuterCache [58:56], Physical_Address [51:12], Shareability [11:10], InnerCache [9:7], IDbits [4:0] the bits minus one. + assert_eq!( + space.propbaser(Phys::new(0xffff_ffff_f000).unwrap()), + word(&[(58, 56, 0), (51, 12, 0xffff_ffff_f000 >> 12), (11, 10, 0b01), (9, 7, 0b111), (4, 0, 15)]) + ); + assert_eq!(Space::new(14).unwrap().propbaser(Phys::new(0).unwrap()) & 0x1f, 13); + // PTZ [62], OuterCache [58:56], Physical_Address [51:16], Shareability [11:10], InnerCache [9:7]. + assert_eq!( + lpi::pendbaser(Phys::new(0xffff_ffff_0000).unwrap()), + word(&[(62, 62, 1), (58, 56, 0), (51, 16, 0xffff_ffff_0000 >> 16), (11, 10, 0b01), (9, 7, 0b111)]) + ); + assert_eq!((lpi::GICR_CTLR, lpi::GICR_TYPER, lpi::GICR_PROPBASER, lpi::GICR_PENDBASER), (0x0000, 0x0008, 0x0070, 0x0078)); + assert_eq!(u64::from(lpi::CTLR_ENABLE_LPIS), word(&[(0, 0, 1)])); + // GICR_TYPER: PLPIS [0], Processor_Number [23:8] under an affinity and flags it is not. + assert_eq!(lpi::TYPER_PLPIS, word(&[(0, 0, 1)])); + assert_eq!(lpi::processor_number(word(&[(63, 32, 0xffff_ffff), (23, 8, 0xbeef), (7, 0, 0xff), (31, 24, 0xff)])), 0xbeef); +} diff --git a/toyos-smmu/Cargo.toml b/toyos-smmu/Cargo.toml new file mode 100644 index 00000000000..5e9d3fad997 --- /dev/null +++ b/toyos-smmu/Cargo.toml @@ -0,0 +1,17 @@ +# A member of the host workspace (root `Cargo.toml`), like toyos-acpi and +# toyos-dma: it is pure, it has no dependencies, and its tests run on the host. +# +# What lives here is every word an SMMUv3 and its driver exchange. The unit +# decides which memory a device reaches, so a field one bit off is an isolation +# hole no boot shows: a stream that should abort translates, or two domains +# share a tag. Each encoding is held on the host to the specification's own +# field positions, and the kernel's `arch::aarch64` unit is left the registers +# and the memory. + +[package] +name = "toyos-smmu" +description = "The SMMUv3's encodings: stream table entries, context descriptors, commands, event records, register fields and the stage 1 table plan, pure, forbid(unsafe_code)." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false diff --git a/toyos-smmu/src/config.rs b/toyos-smmu/src/config.rs new file mode 100644 index 00000000000..b0d4c2b5106 --- /dev/null +++ b/toyos-smmu/src/config.rs @@ -0,0 +1,82 @@ +//! The stream table entry (§5.2) and the context descriptor (§5.4): 64 bytes +//! each, eight little-endian doublewords, a field's bit `n` being bit +//! `n % 64` of doubleword `n / 64`. + +use crate::unit::Unit; +use crate::{Asid, Phys}; + +/// A stream table entry. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Ste(pub [u64; 8]); + +/// `V`, bit [0]. +const STE_V: u64 = 1 << 0; +/// `Config`, bits [3:1]: `0b101` is stage 1 translate, stage 2 bypass. +const STE_STAGE1: u64 = 0b101 << 1; +/// Doubleword 1: `S1CIR` [67:66] and `S1COR` [69:68] `0b01`, write-back +/// read-allocate, and `S1CSH` [71:70] `0b11`, inner shareable — how the unit +/// reads the context descriptor. +const STE_CD_CACHED: u64 = 0b01 << 2 | 0b01 << 4 | 0b11 << 6; +/// `S1STALLD`, bit [91]. +const STE_S1STALLD: u64 = 1 << 27; + +impl Ste { + /// `V` set and `Config` `0b000`: every transaction of the stream is + /// aborted, and no event is recorded for it. An entry of all zeroes + /// aborts too, and records `C_BAD_STE` each time. + pub const ABORT: Self = Self([STE_V, 0, 0, 0, 0, 0, 0, 0]); + + /// The stream translated by stage 1 through the one context descriptor + /// at `context`: `S1ContextPtr` [55:6], `S1CDMax` [63:59] zero so a + /// transaction carrying a SubstreamID is aborted, `STRW` [95:94] zero for + /// the Non-secure EL1 regime, `EATS` [93:92] zero so ATS is refused, and + /// no stage 2 field. + pub const fn stage1(context: Phys<6>, unit: &Unit) -> Self { + let stall = if unit.stall_disable() { STE_S1STALLD } else { 0 }; + Self([STE_V | STE_STAGE1 | context.get(), STE_CD_CACHED | stall, 0, 0, 0, 0, 0, 0]) + } +} + +/// A context descriptor. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Cd(pub [u64; 8]); + +/// `T0SZ`, bits [5:0]: a 48-bit input, walked from level 0. +const CD_T0SZ: u64 = 64 - crate::table::INPUT_BITS as u64; +/// `TG0` [7:6] zero is the 4 KiB granule; `IR0` [9:8] and `OR0` [11:10] +/// `0b01`, write-back write-allocate, and `SH0` [13:12] `0b11`, inner +/// shareable — how the unit walks the tables; `EPD0` [14] and `ENDI` [15] +/// clear, walk `TTB0` little-endian. +const CD_WALK: u64 = 0b01 << 8 | 0b01 << 10 | 0b11 << 12; +/// `EPD1`, bit [30]: no walk through `TTB1`, so the upper half faults. +const CD_EPD1: u64 = 1 << 30; +/// `V`, bit [31]. +const CD_V: u64 = 1 << 31; +/// `AA64`, bit [41]: VMSAv8-64 descriptors. +const CD_AA64: u64 = 1 << 41; +/// `R` [45] records a translation-related fault and `A` [46] aborts its +/// transaction rather than completing it as read-as-zero, write-ignored; +/// `S` [44] stays clear, so none stalls (§5.5). +const CD_FAULTS: u64 = 1 << 45 | 1 << 46; +/// `ASET`, bit [47]: the ASID is the unit's own and no CPU process's. +const CD_ASET: u64 = 1 << 47; + +impl Cd { + /// The context whose tables start at `root`, tagged `asid`: `IPS` + /// [34:32] from the unit's output size, `ASID` [63:48], `TTB0` [119:68] + /// holding the root's bits [55:4], and `MAIR0` [223:192] the attributes + /// [`crate::table`]'s leaves index. `AFFD` [35] stays clear, and every + /// leaf written sets its access flag. + pub const fn new(root: Phys<12>, asid: Asid, unit: &Unit) -> Self { + let word0 = CD_T0SZ + | CD_WALK + | CD_EPD1 + | CD_V + | unit.ips() << 32 + | CD_AA64 + | CD_FAULTS + | CD_ASET + | (asid.get() as u64) << 48; + Self([word0, root.get(), 0, crate::table::MAIR0 as u64, 0, 0, 0, 0]) + } +} diff --git a/toyos-smmu/src/lib.rs b/toyos-smmu/src/lib.rs new file mode 100644 index 00000000000..6437affb45e --- /dev/null +++ b/toyos-smmu/src/lib.rs @@ -0,0 +1,56 @@ +//! Every word an SMMUv3 and its driver exchange, as Arm IHI 0070 issue H.a +//! lays it out: what the identification registers say a unit can do +//! ([`unit`]), the stream table entry and context descriptor that put a +//! stream under stage 1 translation or abort it ([`config`]), the commands and +//! the event records ([`queue`]), and the stage 1 tables a context descriptor +//! names ([`table`]). +//! +//! The crate encodes one configuration and no other: a stream either aborts +//! or is translated by stage 1 alone through one context descriptor, under +//! the Non-secure EL1 regime, with 4 KiB-granule VMSAv8-64 tables over a +//! 48-bit input. No stream bypasses, and nothing here spells a bypass. +//! +//! What a unit reports about itself is a device's word and is refused by name +//! where the driver could not act on it; nothing here panics on a register's +//! or a record's value. +//! +//! `no_std`, no allocation, no `unsafe`. + +#![no_std] +#![forbid(unsafe_code)] + +pub mod config; +pub mod queue; +pub mod table; +pub mod unit; + +/// A physical address the unit is given, aligned to `1 << ALIGN` bytes and +/// below 2^48: the widest address a 4 KiB-granule VMSAv8-64 descriptor +/// carries without `DS`, and inside every address field written here. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Phys(u64); + +impl Phys { + pub const fn new(address: u64) -> Option { + if address >> 48 == 0 && address & ((1 << ALIGN) - 1) == 0 { + Some(Self(address)) + } else { + None + } + } + + pub const fn get(self) -> u64 { + self.0 + } +} + +/// A context's tag in the unit's TLB, no wider than the unit's ASIDs: made by +/// [`unit::Unit::asid`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Asid(u16); + +impl Asid { + pub const fn get(self) -> u16 { + self.0 + } +} diff --git a/toyos-smmu/src/queue.rs b/toyos-smmu/src/queue.rs new file mode 100644 index 00000000000..92b28f1c3b1 --- /dev/null +++ b/toyos-smmu/src/queue.rs @@ -0,0 +1,205 @@ +//! The two queues: their index arithmetic (§3.5.1), the commands the driver +//! writes (chapter 4) and the event records it reads (§7.3). + +use crate::Asid; + +/// A circular queue of `1 << log2size` entries. An index register holds the +/// entry's index in its low `log2size` bits and a wrap flag in the next one +/// up, which toggles each time the index passes the top. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Queue { + log2size: u8, +} + +impl Queue { + /// `None` past the 19 bits an index register's `WR` and `RD` fields with + /// their wrap flag can hold (`IDR1.CMDQS`, `EVENTQS`). + pub const fn new(log2size: u8) -> Option { + if log2size <= 19 { + Some(Self { log2size }) + } else { + None + } + } + + /// The index and its wrap flag, which is all of an index register that + /// names a position. + const fn position(self, index: u32) -> u32 { + index & ((2 << self.log2size) - 1) + } + + /// The entry an index register names. + pub const fn slot(self, index: u32) -> usize { + (index & ((1 << self.log2size) - 1)) as usize + } + + /// Nothing to consume: the indexes and their wrap flags are equal. + pub const fn is_empty(self, prod: u32, cons: u32) -> bool { + self.position(prod) == self.position(cons) + } + + /// Nothing can be produced: the indexes are equal and the wrap flags differ. + pub const fn is_full(self, prod: u32, cons: u32) -> bool { + self.position(prod) ^ self.position(cons) == 1 << self.log2size + } + + /// The position after `index`, wrap flag toggled where it passes the + /// top: what the owner of an index writes back, index and flag at once. + pub const fn after(self, index: u32) -> u32 { + self.position(self.position(index) + 1) + } +} + +/// How a `CMD_SYNC` says it is done (§4.7.3, `CS`). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Signal { + /// `SIG_NONE`: `SMMU_CMDQ_CONS` passing it is the only sign. + None, + /// `SIG_IRQ` with `MSIAddress` zero: the unit's wired sync interrupt. + Irq, + /// `SIG_SEV`: an event to the CPUs, where `IDR0.SEV` is set. + Sev, +} + +/// One 16-byte command. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Command { + /// `CMD_CFGI_STE` (§4.3.1), `Leaf` set: forget one stream's entry and + /// every context descriptor reached through it. + ForgetStream(u32), + /// `CMD_CFGI_CD` (§4.3.3), SubstreamID 0 and `Leaf` set: forget the one + /// context descriptor a stream's entry names. + ForgetContext(u32), + /// `CMD_CFGI_ALL` (§4.3.9): forget every stream's configuration. + ForgetAll, + /// `CMD_TLBI_NH_ASID` (§4.4.2.2), VMID 0: drop every non-global + /// translation cached under one ASID. + InvalidateAsid(Asid), + /// `CMD_TLBI_NSNH_ALL` (§4.4.4.1): drop every Non-secure translation. + InvalidateAll, + /// `CMD_SYNC` (§4.7.3): consumed only once every command before it has + /// taken effect. + Sync(Signal), +} + +impl Command { + /// The command's two doublewords: the opcode in bits [7:0] of the first. + pub const fn words(self) -> [u64; 2] { + match self { + // StreamID [63:32]; Leaf [64]. + Self::ForgetStream(stream) => [0x03 | (stream as u64) << 32, 1], + // StreamID [63:32], SubstreamID [31:12] zero; Leaf [64]. + Self::ForgetContext(stream) => [0x05 | (stream as u64) << 32, 1], + // `CMD_CFGI_STE_RANGE` with Range [68:64] of 31. + Self::ForgetAll => [0x04, 31], + // ASID [63:48], VMID [47:32]. + Self::InvalidateAsid(asid) => [0x11 | (asid.get() as u64) << 48, 0], + Self::InvalidateAll => [0x30, 0], + // CS [13:12]. + Self::Sync(signal) => { + let cs = match signal { + Signal::None => 0b00, + Signal::Irq => 0b01, + Signal::Sev => 0b10, + }; + [0x46 | cs << 12, 0] + } + } + } +} + +/// An event's number, bits [7:0] of its record (§7.3). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Code { + /// `F_UUT` (0x01). + UnsupportedTransaction, + /// `C_BAD_STREAMID` (0x02): past the stream table. + BadStream, + /// `F_STE_FETCH` (0x03). + EntryFetch, + /// `C_BAD_STE` (0x04): an invalid or ILLEGAL stream table entry. + BadEntry, + /// `F_BAD_ATS_TREQ` (0x05). + AtsRequest, + /// `F_STREAM_DISABLED` (0x06). + StreamDisabled, + /// `F_TRANSL_FORBIDDEN` (0x07). + TranslatedForbidden, + /// `C_BAD_SUBSTREAMID` (0x08). + BadSubstream, + /// `F_CD_FETCH` (0x09). + ContextFetch, + /// `C_BAD_CD` (0x0A): an invalid or ILLEGAL context descriptor. + BadContext, + /// `F_WALK_EABT` (0x0B). + WalkAbort, + /// `F_TRANSLATION` (0x10): no valid descriptor for the address. + Translation, + /// `F_ADDR_SIZE` (0x11). + AddressSize, + /// `F_ACCESS` (0x12). + AccessFlag, + /// `F_PERMISSION` (0x13). + Permission, + /// `F_TLB_CONFLICT` (0x20). + TlbConflict, + /// `F_CFG_CONFLICT` (0x21). + ConfigurationConflict, + /// `E_PAGE_REQUEST` (0x24). + PageRequest, + /// `F_VMS_FETCH` (0x25). + VmsFetch, + /// A number this issue gives no record or the unit defines for itself. + Other(u8), +} + +/// What a device tried, where the record says: the four translation-related +/// faults carry it (§7.3.13 to §7.3.16). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Attempt { + /// `InputAddr`, bits [191:128]: the device address. + pub address: u64, + /// `RnW`, bit [99], clear. + pub write: bool, +} + +/// One 32-byte event record, decoded. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Event { + /// `StreamID`, bits [63:32]. + pub stream: u32, + pub code: Code, + pub attempt: Option, +} + +pub const fn event(record: [u64; 4]) -> Event { + let code = match record[0] as u8 { + 0x01 => Code::UnsupportedTransaction, + 0x02 => Code::BadStream, + 0x03 => Code::EntryFetch, + 0x04 => Code::BadEntry, + 0x05 => Code::AtsRequest, + 0x06 => Code::StreamDisabled, + 0x07 => Code::TranslatedForbidden, + 0x08 => Code::BadSubstream, + 0x09 => Code::ContextFetch, + 0x0A => Code::BadContext, + 0x0B => Code::WalkAbort, + 0x10 => Code::Translation, + 0x11 => Code::AddressSize, + 0x12 => Code::AccessFlag, + 0x13 => Code::Permission, + 0x20 => Code::TlbConflict, + 0x21 => Code::ConfigurationConflict, + 0x24 => Code::PageRequest, + 0x25 => Code::VmsFetch, + other => Code::Other(other), + }; + let attempt = match code { + Code::Translation | Code::AddressSize | Code::AccessFlag | Code::Permission => { + Some(Attempt { address: record[2], write: record[1] & 1 << 35 == 0 }) + } + _ => None, + }; + Event { stream: (record[0] >> 32) as u32, code, attempt } +} diff --git a/toyos-smmu/src/table.rs b/toyos-smmu/src/table.rs new file mode 100644 index 00000000000..08ec48a2712 --- /dev/null +++ b/toyos-smmu/src/table.rs @@ -0,0 +1,129 @@ +//! The stage 1 tables a context descriptor names: VMSAv8-64 with the 4 KiB +//! granule over a 48-bit input, four levels of 512 descriptors (Arm ARM, +//! DDI 0487 M.d, §D8.3.1), and the plan of one mapping in them. +//! +//! A device's memory is mapped by 2 MiB blocks at level 2. The page an ITS +//! takes a message on is the one 4 KiB page at level 3: a block there would +//! hand the device every register that shares its 2 MiB. + +use crate::Phys; + +/// The width of a device address these tables translate. +pub const INPUT_BITS: u32 = 48; +/// Levels 0 to 3, each indexed by nine bits of the address. +pub const LEVELS: usize = 4; + +/// `CD.MAIR0`: attribute 0 is Device-nGnRE (`0x04`), which the message page +/// is mapped with, and attribute 1 Normal write-back memory (`0xFF`). +pub const MAIR0: u32 = 0x04 | 0xFF << 8; +const ATTR_DEVICE: u64 = 0 << 2; +const ATTR_MEMORY: u64 = 1 << 2; + +/// Table D8-50 and Table D8-52: bit [0] valid; bit [1] set for a table +/// descriptor and a level 3 page, clear for a block. +const VALID: u64 = 1 << 0; +const TABLE_OR_PAGE: u64 = 1 << 1; +/// `AP[1]`, bit [6]: an unprivileged access is permitted. A PCIe transaction +/// without a PASID prefix is unprivileged (IHI 0070 §13.7), so a leaf without +/// it is one no device reaches. +const AP_UNPRIVILEGED: u64 = 1 << 6; +/// `AP[2]`, bit [7]: read-only. +const AP_READ_ONLY: u64 = 1 << 7; +/// `SH`, bits [9:8]: inner shareable. +const INNER_SHAREABLE: u64 = 0b11 << 8; +/// The access flag, bit [10]: clear, the first access faults. +const AF: u64 = 1 << 10; +/// `nG`, bit [11]: the translation is its context's alone. A leaf without it +/// is global, cached for every ASID of the regime and so for every domain, +/// and no invalidation by ASID removes it. +const NOT_GLOBAL: u64 = 1 << 11; +/// `PXN` [53] and `UXN` [54]: never executable. +const NEVER_EXECUTED: u64 = 1 << 53 | 1 << 54; + +const LEAF: u64 = VALID | AP_UNPRIVILEGED | AF | NOT_GLOBAL | NEVER_EXECUTED; + +/// What a device may do to memory mapped for it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Access { + Read, + ReadWrite, +} + +/// What one mapping puts at a device address. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Leaf { + /// 2 MiB of memory. + Memory(Phys<21>, Access), + /// The 4 KiB page holding the register a message is written to. + Doorbell(Phys<12>), +} + +/// The descriptor naming the next table down, at levels 0 to 2. Its +/// hierarchical fields stay clear: the leaf alone decides the access. +pub const fn next(table: Phys<12>) -> u64 { + table.get() | TABLE_OR_PAGE | VALID +} + +/// One mapping, planned: the index to follow in each table from the root, +/// and the descriptor to store at the last of them. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Path { + indices: [usize; LEVELS], + tables: usize, + /// The leaf: a level 2 block, or a level 3 page. + pub descriptor: u64, +} + +impl Path { + /// The index in the table at each level walked, the root first; the + /// last is the leaf's own slot, and every one before it holds a + /// [`next`] descriptor. + pub fn indices(&self) -> &[usize] { + &self.indices[..self.tables] + } +} + +/// The path to `leaf` at the device address `at`. `None` where `at` is past +/// the input or not aligned to what the leaf maps. +pub const fn plan(at: u64, leaf: Leaf) -> Option { + let (tables, size, descriptor) = match leaf { + Leaf::Memory(block, access) => { + let access = match access { + Access::Read => AP_READ_ONLY, + Access::ReadWrite => 0, + }; + (3, 1u64 << 21, block.get() | LEAF | ATTR_MEMORY | INNER_SHAREABLE | access) + } + Leaf::Doorbell(page) => (4, 1u64 << 12, page.get() | LEAF | TABLE_OR_PAGE | ATTR_DEVICE), + }; + if at >> INPUT_BITS != 0 || at & (size - 1) != 0 { + return None; + } + const fn index(at: u64, level: u32) -> usize { + (at >> (39 - 9 * level) & 0x1FF) as usize + } + Some(Path { indices: [index(at, 0), index(at, 1), index(at, 2), index(at, 3)], tables, descriptor }) +} + +/// What a descriptor read back from a table is, by the level it was read at. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Entry { + /// Nothing is mapped under it. + Invalid, + /// The next table down. + Table(Phys<12>), + /// A block or a page: something is mapped here. + Mapped, +} + +/// `descriptor` as the walk reads it at `level`: bit [1] names a table above +/// level 3 and a page at it, and a table's address is bits [47:12]. +pub const fn entry(descriptor: u64, level: usize) -> Entry { + if descriptor & VALID == 0 { + Entry::Invalid + } else if level < LEVELS - 1 && descriptor & TABLE_OR_PAGE != 0 { + Entry::Table(Phys(descriptor & 0x0000_FFFF_FFFF_F000)) + } else { + Entry::Mapped + } +} diff --git a/toyos-smmu/src/unit.rs b/toyos-smmu/src/unit.rs new file mode 100644 index 00000000000..d93b0581b96 --- /dev/null +++ b/toyos-smmu/src/unit.rs @@ -0,0 +1,249 @@ +//! The register map (§6.2) and the fields of the registers the driver reads +//! and writes (§6.3), and [`probe`]: what a unit's identification registers +//! let this crate's one configuration be built on. + +use crate::{Asid, Phys}; + +/// Offsets in register page 0 (§6.2.1). +pub const IDR0: usize = 0x000; +pub const IDR1: usize = 0x004; +pub const IDR5: usize = 0x014; +pub const CR0: usize = 0x020; +pub const CR0ACK: usize = 0x024; +pub const CR1: usize = 0x028; +pub const CR2: usize = 0x02C; +pub const GBPA: usize = 0x044; +pub const IRQ_CTRL: usize = 0x050; +pub const IRQ_CTRLACK: usize = 0x054; +pub const GERROR: usize = 0x060; +pub const GERRORN: usize = 0x064; +pub const STRTAB_BASE: usize = 0x080; +pub const STRTAB_BASE_CFG: usize = 0x088; +pub const CMDQ_BASE: usize = 0x090; +pub const CMDQ_PROD: usize = 0x098; +pub const CMDQ_CONS: usize = 0x09C; +pub const EVENTQ_BASE: usize = 0x0A0; +/// The event queue's indexes are in register page 1, 64 KiB above page 0 +/// (§6.1, §6.2.2): `SMMU_EVENTQ_PROD` and `SMMU_EVENTQ_CONS` at 0xA8 and 0xAC of it. +pub const EVENTQ_PROD: usize = 0x1_00A8; +pub const EVENTQ_CONS: usize = 0x1_00AC; + +/// `SMMU_CR0` (§6.3.9), and `SMMU_CR0ACK`, which reads each bit back once +/// the unit has acted on it. +pub const CR0_SMMUEN: u32 = 1 << 0; +pub const CR0_EVENTQEN: u32 = 1 << 2; +pub const CR0_CMDQEN: u32 = 1 << 3; + +/// `SMMU_CR1` (§6.3.11): the unit reads its stream table and its queues as +/// inner-shareable write-back memory — `TABLE_SH` [11:10], `TABLE_OC` [9:8], +/// `TABLE_IC` [7:6], `QUEUE_SH` [5:4], `QUEUE_OC` [3:2], `QUEUE_IC` [1:0], +/// each shareability `0b11` and each cacheability `0b01`. +pub const CR1_WRITE_BACK: u32 = 0b11 << 10 | 0b01 << 8 | 0b01 << 6 | 0b11 << 4 | 0b01 << 2 | 0b01; + +/// `SMMU_CR2` (§6.3.12): `RECINVSID` [1], so a transaction under a StreamID +/// past the stream table is recorded as well as aborted, and `PTM` [2], so +/// no CPU's broadcast TLB invalidation reaches the unit's entries: its ASIDs +/// are its own. `E2H` [0] stays clear. +pub const CR2_RECORD_PRIVATE: u32 = 1 << 1 | 1 << 2; + +/// `SMMU_GBPA` (§6.3.15): what happens to every transaction while `SMMUEN` +/// is clear. `ABORT` [20] aborts them; a write takes effect by carrying +/// `Update` [31], which reads back set until the unit has taken it (§6.3.15.1). +pub const GBPA_ABORT: u32 = 1 << 20; +pub const GBPA_UPDATE: u32 = 1 << 31; + +/// `SMMU_IRQ_CTRL` (§6.3.17), and `SMMU_IRQ_CTRLACK` as `SMMU_CR0ACK`. +pub const IRQ_GERROR: u32 = 1 << 0; +pub const IRQ_EVENTQ: u32 = 1 << 2; + +/// `SMMU_GERROR` (§6.3.19): an error is active while its bit differs from +/// `SMMU_GERRORN`'s, and writing `GERRORN` equal acknowledges it (§7.5). +pub const fn active_errors(gerror: u32, gerrorn: u32) -> u32 { + gerror ^ gerrorn +} + +/// `SMMU_GERROR.CMDQ_ERR`: the command queue stopped on [`CommandError`]. +pub const GERROR_CMDQ: u32 = 1 << 0; +/// `SMMU_GERROR.EVENTQ_ABT_ERR`: a write to the event queue aborted, and +/// records were lost. +pub const GERROR_EVENTQ_ABORT: u32 = 1 << 2; +/// `SMMU_GERROR.SFM_ERR`: the unit entered Service Failure Mode. +pub const GERROR_SERVICE_FAILURE: u32 = 1 << 8; + +/// `SMMU_CMDQ_CONS.ERR` [30:24] (§6.3.28, §7.1): why the command at `RD` +/// was not consumed. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum CommandError { + None, + /// `CERROR_ILL`: a reserved opcode or an invalid parameter. + Illegal, + /// `CERROR_ABT`: reading the command aborted. + Abort, + /// `CERROR_ATC_INV_SYNC`. + AtcInvalidation, + Reserved(u8), +} + +pub const fn command_error(cons: u32) -> CommandError { + match (cons >> 24 & 0x7F) as u8 { + 0 => CommandError::None, + 1 => CommandError::Illegal, + 2 => CommandError::Abort, + 3 => CommandError::AtcInvalidation, + other => CommandError::Reserved(other), + } +} + +/// `SMMU_EVENTQ_PROD.OVFLG` and `SMMU_EVENTQ_CONS.OVACKFLG` [31] (§7.4): +/// records were dropped on a full queue while the two differ, and the unit +/// reports no further overflow until the consumer writes its flag equal. +pub const EVENTQ_OVERFLOW: u32 = 1 << 31; + +/// What a unit lacks that this crate's one configuration needs. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Lacks { + /// `IDR0.S1P` clear: no stage 1 translation. + Stage1, + /// `IDR0.TTF[1]` clear: no VMSAv8-64 tables. + Aarch64Tables, + /// `IDR0.TTENDIAN` is big-endian only. + LittleEndianTables, + /// `IDR0.STALL_MODEL` is `0b10`, under which every fault stalls its + /// transaction until software answers it; or a reserved value. + Termination(u8), + /// `IDR5.GRAN4K` clear. + Granule4K, + /// `IDR1.TABLES_PRESET` or `IDR1.QUEUES_PRESET`: the unit's table or + /// queues sit at an address of its own choosing. + FixedStructures, + /// `IDR1.SIDSIZE`, `CMDQS` or `EVENTQS` holds a reserved value. + Size(u8), +} + +/// A unit the one configuration can be built on, and its sizes. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Unit { + /// `IDR1.SIDSIZE`: how many bits of StreamID the unit takes, at most 32. + pub stream_bits: u8, + /// `IDR1.CMDQS` and `EVENTQS`: the largest each queue may be, as log2 of + /// its entries. + pub command_queue_log2: u8, + pub event_queue_log2: u8, + /// `IDR5.OAS` in bits: no address the unit is given may reach past it. + pub output_bits: u8, + /// `IDR0.COHACC`, which the IORT's SMMUv3 node may override. + pub coherent: bool, + asid16: bool, + /// `IDR0.STALL_MODEL` is `0b00`: stalling is there to be disabled. + stall: bool, +} + +/// §6.3.1, §6.3.2 and §6.3.6: the three identification registers, judged. +pub const fn probe(idr0: u32, idr1: u32, idr5: u32) -> Result { + if idr0 & 1 << 1 == 0 { + return Err(Lacks::Stage1); + } + if idr0 & 1 << 3 == 0 { + return Err(Lacks::Aarch64Tables); + } + // `TTENDIAN` [22:21]: `0b00` mixed and `0b10` little-endian can be set to little-endian. + if idr0 >> 21 & 0b11 == 0b11 { + return Err(Lacks::LittleEndianTables); + } + // `STALL_MODEL` [25:24]: `0b00` stall and terminate, `0b01` terminate only. + let stall = match idr0 >> 24 & 0b11 { + 0b00 => true, + 0b01 => false, + other => return Err(Lacks::Termination(other as u8)), + }; + if idr5 & 1 << 4 == 0 { + return Err(Lacks::Granule4K); + } + if idr1 & (1 << 30 | 1 << 29) != 0 { + return Err(Lacks::FixedStructures); + } + let stream_bits = (idr1 & 0x3F) as u8; + let command_queue_log2 = (idr1 >> 21 & 0x1F) as u8; + let event_queue_log2 = (idr1 >> 16 & 0x1F) as u8; + if stream_bits > 32 { + return Err(Lacks::Size(stream_bits)); + } + if command_queue_log2 > 19 { + return Err(Lacks::Size(command_queue_log2)); + } + if event_queue_log2 > 19 { + return Err(Lacks::Size(event_queue_log2)); + } + Ok(Unit { + stream_bits, + command_queue_log2, + event_queue_log2, + output_bits: [32, 36, 40, 42, 44, 48, 52, 56][(idr5 & 0b111) as usize], + coherent: idr0 & 1 << 4 != 0, + asid16: idr0 & 1 << 12 != 0, + stall, + }) +} + +impl Unit { + /// `tag` as an ASID of this unit: eight bits where `IDR0.ASID16` is + /// clear, under which a wider one makes its context descriptor ILLEGAL. + pub const fn asid(&self, tag: u16) -> Option { + if self.asid16 || tag <= 0xFF { + Some(Asid(tag)) + } else { + None + } + } + + /// Whether `STE.S1STALLD` is set: it must be where stalling is + /// configurable and is ILLEGAL where it is not (§5.5). + pub(crate) const fn stall_disable(&self) -> bool { + self.stall + } + + /// `CD.IPS`, the widest of `TCR_ELx.PS`'s sizes inside the unit's output + /// size and the 48 bits these tables' descriptors carry. + pub(crate) const fn ips(&self) -> u64 { + match self.output_bits { + 32 => 0b000, + 36 => 0b001, + 40 => 0b010, + 42 => 0b011, + 44 => 0b100, + _ => 0b101, + } + } + + /// `SMMU_STRTAB_BASE` and `SMMU_STRTAB_BASE_CFG` (§6.3.24, §6.3.25) for + /// a linear table of `1 << log2size` entries at `table`: `RA` [62] and + /// the address, then `FMT` [17:16] zero and `LOG2SIZE` [5:0]. `None` + /// where the unit takes fewer StreamID bits, or the table is not aligned + /// to its own size as the unit reads its address. + pub const fn stream_table(&self, table: Phys<6>, log2size: u8) -> Option<(u64, u32)> { + if log2size > self.stream_bits || table.get() & ((64 << log2size) - 1) != 0 { + return None; + } + Some((1 << 62 | table.get(), log2size as u32)) + } + + /// `SMMU_CMDQ_BASE` (§6.3.26) for a queue of `1 << log2size` 16-byte + /// commands at `queue`: `RA` [62], the address, `LOG2SIZE` [4:0]. `None` + /// past the unit's `CMDQS`, or where the queue is not aligned to the + /// larger of its size and 32 bytes. + pub const fn command_queue(&self, queue: Phys<5>, log2size: u8) -> Option { + if log2size > self.command_queue_log2 || queue.get() & ((16 << log2size) - 1) != 0 { + return None; + } + Some(1 << 62 | queue.get() | log2size as u64) + } + + /// `SMMU_EVENTQ_BASE` (§6.3.29) for a queue of `1 << log2size` 32-byte + /// records: `WA` [62], the address, `LOG2SIZE` [4:0]. + pub const fn event_queue(&self, queue: Phys<5>, log2size: u8) -> Option { + if log2size > self.event_queue_log2 || queue.get() & ((32 << log2size) - 1) != 0 { + return None; + } + Some(1 << 62 | queue.get() | log2size as u64) + } +} diff --git a/toyos-smmu/tests/encodings.rs b/toyos-smmu/tests/encodings.rs new file mode 100644 index 00000000000..2ede94eac0b --- /dev/null +++ b/toyos-smmu/tests/encodings.rs @@ -0,0 +1,504 @@ +//! Every encoding against IHI 0070 H.a's own field positions: each expected +//! value is built from `(msb, lsb, value)` triples as the specification's +//! diagrams and field headings give them, and compared whole, so a field in +//! the wrong place and a stray bit are both a difference. + +use toyos_smmu::config::{Cd, Ste}; +use toyos_smmu::queue::{event, Attempt, Code, Command, Event, Queue, Signal}; +use toyos_smmu::table::{entry, next, plan, Access, Entry, Leaf, INPUT_BITS, MAIR0}; +use toyos_smmu::unit::{self, probe, CommandError, Lacks, Unit}; +use toyos_smmu::Phys; + +/// `N` doublewords holding `value` in bits `[msb:lsb]` for every triple, and +/// zero everywhere else. A field may cross a doubleword. +fn layout(fields: &[(u32, u32, u64)]) -> [u64; N] { + let mut words = [0u64; N]; + for &(msb, lsb, value) in fields { + let width = msb - lsb + 1; + assert!(width == 64 || value >> width == 0, "{value:#x} does not fit [{msb}:{lsb}]"); + for bit in 0..width { + if value >> bit & 1 != 0 { + let at = lsb + bit; + assert_eq!(words[(at / 64) as usize] >> (at % 64) & 1, 0, "two fields claim bit {at}"); + words[(at / 64) as usize] |= 1 << (at % 64); + } + } + } + words +} + +fn word(fields: &[(u32, u32, u64)]) -> u64 { + layout::<1>(fields)[0] +} + +/// The identification registers of a unit that lacks nothing: `S1P` [1], +/// `TTF` [3:2] VMSAv8-64 alone, `COHACC` [4], `ASID16` [12], little-endian +/// tables, stall and terminate; 16 StreamID bits, queues of 2^19 and 2^8; +/// `GRAN4K` [4] and a 44-bit output. +const IDR0: &[(u32, u32, u64)] = &[(1, 1, 1), (3, 2, 0b10), (4, 4, 1), (12, 12, 1), (22, 21, 0b10)]; +const IDR1: &[(u32, u32, u64)] = &[(5, 0, 16), (20, 16, 8), (25, 21, 19)]; +const IDR5: &[(u32, u32, u64)] = &[(2, 0, 0b100), (4, 4, 1)]; + +/// The fields of `base` with those of `change` in place of the same bits. +fn with(base: &[(u32, u32, u64)], change: &[(u32, u32, u64)]) -> u32 { + let kept: Vec<_> = base.iter().filter(|b| !change.iter().any(|c| c.0 == b.0)).chain(change).copied().collect(); + word(&kept) as u32 +} + +fn idr(idr0: &[(u32, u32, u64)], idr1: &[(u32, u32, u64)], idr5: &[(u32, u32, u64)]) -> Result { + probe(with(IDR0, idr0), with(IDR1, idr1), with(IDR5, idr5)) +} + +fn whole() -> Unit { + idr(&[], &[], &[]).expect("a unit that lacks nothing") +} + +/// `STALL_MODEL` [25:24] `0b01`: every fault terminates, and none can stall. +fn terminating() -> Unit { + idr(&[(25, 24, 0b01)], &[], &[]).expect("a unit that terminates every fault") +} + +// --- §6.3.1, §6.3.2, §6.3.6: the identification registers ------------------ + +#[test] +fn a_units_sizes_are_read_from_their_own_fields() { + let unit = whole(); + assert_eq!( + (unit.stream_bits, unit.command_queue_log2, unit.event_queue_log2, unit.output_bits, unit.coherent), + (16, 19, 8, 44, true) + ); + // Each field moved alone moves its own answer and no other. + let other = idr(&[(4, 4, 0)], &[(5, 0, 32), (20, 16, 19), (25, 21, 0)], &[(2, 0, 0b000)]).expect("a unit"); + assert_eq!( + (other.stream_bits, other.command_queue_log2, other.event_queue_log2, other.output_bits, other.coherent), + (32, 0, 19, 32, false) + ); + for (oas, bits) in [32u8, 36, 40, 42, 44, 48, 52, 56].into_iter().enumerate() { + assert_eq!(idr(&[], &[], &[(2, 0, oas as u64)]).map(|unit| unit.output_bits), Ok(bits)); + } +} + +#[test] +fn a_unit_that_lacks_what_the_configuration_needs_is_refused_by_name() { + assert_eq!(idr(&[(1, 1, 0)], &[], &[]), Err(Lacks::Stage1)); + // `TTF` `0b01` is VMSAv8-32 LPAE alone; `0b11` has both. + assert_eq!(idr(&[(3, 2, 0b01)], &[], &[]), Err(Lacks::Aarch64Tables)); + assert!(idr(&[(3, 2, 0b11)], &[], &[]).is_ok()); + assert_eq!(idr(&[(22, 21, 0b11)], &[], &[]), Err(Lacks::LittleEndianTables)); + assert!(idr(&[(22, 21, 0b00)], &[], &[]).is_ok()); + assert_eq!(idr(&[(25, 24, 0b10)], &[], &[]), Err(Lacks::Termination(0b10))); + assert_eq!(idr(&[(25, 24, 0b11)], &[], &[]), Err(Lacks::Termination(0b11))); + assert_eq!(idr(&[], &[], &[(4, 4, 0)]), Err(Lacks::Granule4K)); + assert_eq!(idr(&[], &[(30, 30, 1)], &[]), Err(Lacks::FixedStructures)); + assert_eq!(idr(&[], &[(29, 29, 1)], &[]), Err(Lacks::FixedStructures)); +} + +#[test] +fn a_reserved_size_is_refused_and_the_largest_defined_one_is_not() { + assert_eq!(idr(&[], &[(5, 0, 33)], &[]), Err(Lacks::Size(33))); + assert_eq!(idr(&[], &[(25, 21, 20)], &[]), Err(Lacks::Size(20))); + assert_eq!(idr(&[], &[(20, 16, 20)], &[]), Err(Lacks::Size(20))); + assert!(idr(&[], &[(5, 0, 32), (25, 21, 19), (20, 16, 19)], &[]).is_ok()); + // Every value three registers can hold is an answer. + for bits in [0u32, u32::MAX, 0x5555_5555, 0xaaaa_aaaa] { + let _ = probe(bits, bits, bits); + let _ = probe(with(IDR0, &[]), bits, bits); + } +} + +#[test] +fn an_asid_is_no_wider_than_the_units() { + let narrow = idr(&[(12, 12, 0)], &[], &[]).expect("a unit of 8-bit ASIDs"); + assert_eq!(narrow.asid(0xff).map(|asid| asid.get()), Some(0xff)); + assert_eq!(narrow.asid(0x100), None); + assert_eq!(whole().asid(0xffff).map(|asid| asid.get()), Some(0xffff)); +} + +// --- §5.2, §5.4: the stream table entry and the context descriptor --------- + +#[test] +fn an_aborting_entry_is_valid_and_configured_to_pass_nothing() { + // V [0] set; Config [3:1] `0b000`, "report abort to device, no event recorded". + assert_eq!(Ste::ABORT.0, layout(&[(0, 0, 1), (3, 1, 0b000)])); +} + +#[test] +fn a_translating_entry_is_stage_1_alone_through_one_context_descriptor() { + let context = Phys::<6>::new(0xffff_ffff_ffc0).expect("64-byte aligned, below 2^48"); + let fields = [ + (0, 0, 1), // V + (3, 1, 0b101), // Config: stage 1 translate, stage 2 bypass + (5, 4, 0), // S1Fmt, ignored with S1CDMax zero + (55, 6, 0xffff_ffff_ffc0 >> 6), // S1ContextPtr + (63, 59, 0), // S1CDMax: one CD, substreams disabled + (67, 66, 0b01), // S1CIR + (69, 68, 0b01), // S1COR + (71, 70, 0b11), // S1CSH + (93, 92, 0), // EATS + (95, 94, 0), // STRW: NS-EL1 + ]; + let mut stalling = fields.to_vec(); + stalling.push((91, 91, 1)); // S1STALLD, which a unit that can stall must be told + assert_eq!(Ste::stage1(context, &whole()).0, layout(&stalling)); + // And is ILLEGAL to set on one that cannot. + assert_eq!(Ste::stage1(context, &terminating()).0, layout(&fields)); +} + +#[test] +fn a_context_descriptor_walks_ttb0_alone_under_its_own_asid() { + let root = Phys::<12>::new(0xffff_ffff_f000).expect("a page below 2^48"); + let unit = whole(); + let asid = unit.asid(0xbeef).expect("a 16-bit ASID"); + assert_eq!( + Cd::new(root, asid, &unit).0, + layout(&[ + (5, 0, 16), // T0SZ: 48 bits + (7, 6, 0b00), // TG0: 4KB + (9, 8, 0b01), // IR0 + (11, 10, 0b01), // OR0 + (13, 12, 0b11), // SH0 + (14, 14, 0), // EPD0 + (15, 15, 0), // ENDI + (30, 30, 1), // EPD1 + (31, 31, 1), // V + (34, 32, 0b100), // IPS: 44 bits, the unit's OAS + (35, 35, 0), // AFFD + (41, 41, 1), // AA64 + (44, 44, 0), // S + (45, 45, 1), // R + (46, 46, 1), // A + (47, 47, 1), // ASET + (63, 48, 0xbeef), // ASID + (119, 68, 0xffff_ffff_f000 >> 4), // TTB0: address bits [55:4] + (223, 192, 0xff04), // MAIR0 + ]) + ); +} + +#[test] +fn a_context_descriptors_ips_is_the_units_output_size_and_no_more_than_48_bits() { + let root = Phys::<12>::new(0x1000).unwrap(); + // `TCR_ELx.PS`'s encodings for 32, 36, 40, 42, 44 and 48 bits; 52 and 56 are past these tables. + for (oas, ips) in [0b000u64, 0b001, 0b010, 0b011, 0b100, 0b101, 0b101, 0b101].into_iter().enumerate() { + let unit = idr(&[], &[], &[(2, 0, oas as u64)]).expect("a unit"); + let cd = Cd::new(root, unit.asid(1).unwrap(), &unit); + assert_eq!(cd.0[0] >> 32 & 0b111, ips, "OAS {oas:#b}"); + } +} + +#[test] +fn an_address_off_its_alignment_or_past_48_bits_is_no_address() { + assert_eq!(Phys::<6>::new(0x40).map(Phys::get), Some(0x40)); + assert_eq!(Phys::<6>::new(0x20), None); + assert_eq!(Phys::<12>::new(0xfff), None); + assert_eq!(Phys::<12>::new(0x800), None); + assert_eq!(Phys::<21>::new(0x10_0000), None); + assert_eq!(Phys::<21>::new(0x20_0000).map(Phys::get), Some(0x20_0000)); + assert_eq!(Phys::<12>::new(1 << 48), None); + assert_eq!(Phys::<12>::new((1 << 48) - 0x1000).map(Phys::get), Some((1 << 48) - 0x1000)); + assert_eq!(Phys::<0>::new(u64::MAX), None); +} + +// --- §6.3: the registers the driver writes --------------------------------- + +#[test] +fn the_control_values_hold_each_field_where_its_register_has_it() { + assert_eq!(u64::from(unit::CR0_SMMUEN | unit::CR0_EVENTQEN | unit::CR0_CMDQEN), word(&[(0, 0, 1), (2, 2, 1), (3, 3, 1)])); + // TABLE_SH, TABLE_OC, TABLE_IC, QUEUE_SH, QUEUE_OC, QUEUE_IC. + assert_eq!( + u64::from(unit::CR1_WRITE_BACK), + word(&[(11, 10, 0b11), (9, 8, 0b01), (7, 6, 0b01), (5, 4, 0b11), (3, 2, 0b01), (1, 0, 0b01)]) + ); + // E2H clear, RECINVSID, PTM. + assert_eq!(u64::from(unit::CR2_RECORD_PRIVATE), word(&[(0, 0, 0), (1, 1, 1), (2, 2, 1)])); + assert_eq!(u64::from(unit::GBPA_ABORT | unit::GBPA_UPDATE), word(&[(20, 20, 1), (31, 31, 1)])); + assert_eq!(u64::from(unit::IRQ_GERROR | unit::IRQ_EVENTQ), word(&[(0, 0, 1), (2, 2, 1)])); + assert_eq!( + u64::from(unit::GERROR_CMDQ | unit::GERROR_EVENTQ_ABORT | unit::GERROR_SERVICE_FAILURE), + word(&[(0, 0, 1), (2, 2, 1), (8, 8, 1)]) + ); + assert_eq!(u64::from(unit::EVENTQ_OVERFLOW), word(&[(31, 31, 1)])); +} + +#[test] +fn an_error_is_active_while_the_two_registers_differ_in_its_bit() { + assert_eq!(unit::active_errors(0, 0), 0); + assert_eq!(unit::active_errors(0b101, 0b100), 0b001); + // Acknowledged by writing the same value, whichever way the bit last toggled. + assert_eq!(unit::active_errors(0b101, 0b101), 0); + assert_eq!(unit::active_errors(0b000, 0b001), 0b001); +} + +#[test] +fn a_command_error_is_the_cons_registers_err_field_alone() { + let cons = |err: u64| word(&[(30, 24, err), (19, 0, 0xf_ffff), (31, 31, 1)]) as u32; + assert_eq!(unit::command_error(cons(0)), CommandError::None); + assert_eq!(unit::command_error(cons(1)), CommandError::Illegal); + assert_eq!(unit::command_error(cons(2)), CommandError::Abort); + assert_eq!(unit::command_error(cons(3)), CommandError::AtcInvalidation); + assert_eq!(unit::command_error(cons(0x7f)), CommandError::Reserved(0x7f)); +} + +#[test] +fn a_stream_table_is_linear_inside_the_units_streams_and_aligned_to_its_size() { + let unit = whole(); + // 2^8 entries of 64 bytes: 16 KiB. + let table = Phys::<6>::new(0x4000_4000).unwrap(); + assert_eq!( + unit.stream_table(table, 8), + // RA [62], ADDR [55:6]; FMT [17:16] linear, SPLIT [10:6] unused, LOG2SIZE [5:0]. + Some((word(&[(62, 62, 1), (55, 6, 0x4000_4000 >> 6)]), word(&[(17, 16, 0b00), (5, 0, 8)]) as u32)) + ); + assert_eq!(unit.stream_table(Phys::new(0x4000_2000).unwrap(), 8), None, "8 KiB into a 16 KiB table's alignment"); + assert_eq!(unit.stream_table(Phys::new(0x4000_0040).unwrap(), 0), Some((1 << 62 | 0x4000_0040, 0))); + // The unit takes 16 StreamID bits and no more. + let big = Phys::<6>::new(0x8000_0000).unwrap(); + assert!(unit.stream_table(big, 16).is_some()); + assert_eq!(unit.stream_table(big, 17), None); +} + +#[test] +fn a_queue_is_inside_the_units_size_and_aligned_to_its_own() { + let unit = whole(); + let at = Phys::<5>::new(0x4001_0000).unwrap(); + // RA or WA [62], ADDR [55:5], LOG2SIZE [4:0]. + let base = |log2: u64| word(&[(62, 62, 1), (55, 5, 0x4001_0000 >> 5), (4, 0, log2)]); + assert_eq!(unit.command_queue(at, 12), Some(base(12))); + assert_eq!(unit.event_queue(at, 8), Some(base(8))); + // 2^13 commands of 16 bytes are 128 KiB, and 2^12 records of 32 as many: `at` is 64 KiB aligned. + assert_eq!(unit.command_queue(at, 13), None); + assert_eq!(unit.event_queue(at, 12), None, "past EVENTQS and misaligned both"); + let unit = idr(&[], &[(20, 16, 19)], &[]).unwrap(); + assert_eq!(unit.event_queue(at, 11), Some(base(11))); + assert_eq!(unit.event_queue(at, 12), None, "128 KiB of records at a 64 KiB alignment"); + // The unit's largest and one more. + let low = Phys::<5>::new(0).unwrap(); + assert!(unit.command_queue(low, 19).is_some() && unit.event_queue(low, 19).is_some()); + let small = idr(&[], &[(25, 21, 4), (20, 16, 4)], &[]).unwrap(); + assert_eq!((small.command_queue(low, 5), small.event_queue(low, 5)), (None, None)); +} + +// --- §3.5.1: the queues' indexes -------------------------------------------- + +/// Against a count kept beside it: empty exactly at zero entries, full +/// exactly at all of them, and the slot the index that many steps on. +#[test] +fn a_queues_indexes_say_empty_full_and_slot_as_a_count_would() { + for log2size in 0..=4u8 { + let queue = Queue::new(log2size).unwrap(); + let size = 1usize << log2size; + // Start anywhere, the wrap flags and the bits above them included. + for start in [0u32, 1 << log2size, (2 << log2size) - 1, 0xfff0_0000 | 1 << log2size] { + let (mut prod, mut cons) = (start, start); + let (mut produced, mut consumed) = (0usize, 0usize); + let first = queue.slot(start); + // Fill, drain halfway, fill again, drain: three laps of the buffer. + for step in 0..6 * size { + let held = produced - consumed; + assert_eq!(queue.is_empty(prod, cons), held == 0, "size {size} step {step}"); + assert_eq!(queue.is_full(prod, cons), held == size, "size {size} step {step}"); + assert_eq!(queue.slot(prod), (first + produced) % size); + assert_eq!(queue.slot(cons), (first + consumed) % size); + if held < size && (step / size).is_multiple_of(2) { + prod = queue.after(prod); + produced += 1; + } else if held > 0 { + cons = queue.after(cons); + consumed += 1; + } + } + } + } + // The wrap flag toggles as the index passes the top, and nothing above it is written back. + let queue = Queue::new(3).unwrap(); + assert_eq!(queue.after(0b0_111), 0b1_000); + assert_eq!(queue.after(0b1_111), 0b0_000); + assert_eq!(queue.after(0xffff_fff7), 0b1_000); + // An index register is the unit's word: every bit set, it is still an index. + assert_eq!(queue.after(u32::MAX), 0); + assert_eq!(Queue::new(19).unwrap().after(u32::MAX), 0); + assert!(Queue::new(19).is_some() && Queue::new(20).is_none()); +} + +// --- chapter 4: commands ---------------------------------------------------- + +#[test] +fn each_command_holds_its_opcode_and_parameters_where_its_diagram_has_them() { + let asid = whole().asid(0xa5c3).unwrap(); + // CMD_CFGI_STE: StreamID [63:32], Leaf [64], SSec [10] clear. + assert_eq!(Command::ForgetStream(0xdead_beef).words(), layout(&[(7, 0, 0x03), (63, 32, 0xdead_beef), (64, 64, 1)])); + // CMD_CFGI_CD: StreamID [63:32], SubstreamID [31:12] zero, Leaf [64]. + assert_eq!( + Command::ForgetContext(0xdead_beef).words(), + layout(&[(7, 0, 0x05), (31, 12, 0), (63, 32, 0xdead_beef), (64, 64, 1)]) + ); + // CMD_CFGI_ALL: CMD_CFGI_STE_RANGE with Range [68:64] of 31. + assert_eq!(Command::ForgetAll.words(), layout(&[(7, 0, 0x04), (68, 64, 31)])); + // CMD_TLBI_NH_ASID: ASID [63:48], VMID [47:32]. + assert_eq!(Command::InvalidateAsid(asid).words(), layout(&[(7, 0, 0x11), (47, 32, 0), (63, 48, 0xa5c3)])); + assert_eq!(Command::InvalidateAll.words(), layout(&[(7, 0, 0x30)])); + // CMD_SYNC: CS [13:12]; MSIAddress [119:66] zero, so SIG_IRQ is the wired interrupt. + assert_eq!(Command::Sync(Signal::None).words(), layout(&[(7, 0, 0x46), (13, 12, 0b00)])); + assert_eq!(Command::Sync(Signal::Irq).words(), layout(&[(7, 0, 0x46), (13, 12, 0b01), (119, 66, 0)])); + assert_eq!(Command::Sync(Signal::Sev).words(), layout(&[(7, 0, 0x46), (13, 12, 0b10)])); +} + +// --- §7.3: event records ---------------------------------------------------- + +/// A record of `number` from `stream`, every field the four fault records +/// carry filled: InputAddr [191:128], RnW [99], InD [98] clear as a write's +/// is, PnU [97], S2 [103], CLASS [105:104], STAG [79:64], an IPA [247:204]. +fn record(number: u8, stream: u32, address: u64, read: bool) -> [u64; 4] { + layout(&[ + (7, 0, u64::from(number)), + (11, 11, 1), + (31, 12, 0xf_ffff), + (63, 32, u64::from(stream)), + (79, 64, 0xffff), + (97, 97, 1), + (98, 98, 0), + (99, 99, u64::from(read)), + (103, 103, 1), + (105, 104, 0b10), + (191, 128, address), + (247, 204, 0xfff_ffff_ffff), + ]) +} + +#[test] +fn a_translation_fault_names_its_stream_its_address_and_whether_it_wrote() { + for (number, code) in [ + (0x10, Code::Translation), + (0x11, Code::AddressSize), + (0x12, Code::AccessFlag), + (0x13, Code::Permission), + ] { + for read in [false, true] { + assert_eq!( + event(record(number, 0x0001_0008, 0xffff_8000_dead_b000, read)), + Event { + stream: 0x0001_0008, + code, + attempt: Some(Attempt { address: 0xffff_8000_dead_b000, write: !read }) + } + ); + } + } +} + +#[test] +fn every_event_number_is_its_own_code_and_only_a_fault_carries_an_attempt() { + let named = [ + (0x01, Code::UnsupportedTransaction), + (0x02, Code::BadStream), + (0x03, Code::EntryFetch), + (0x04, Code::BadEntry), + (0x05, Code::AtsRequest), + (0x06, Code::StreamDisabled), + (0x07, Code::TranslatedForbidden), + (0x08, Code::BadSubstream), + (0x09, Code::ContextFetch), + (0x0A, Code::BadContext), + (0x0B, Code::WalkAbort), + (0x10, Code::Translation), + (0x11, Code::AddressSize), + (0x12, Code::AccessFlag), + (0x13, Code::Permission), + (0x20, Code::TlbConflict), + (0x21, Code::ConfigurationConflict), + (0x24, Code::PageRequest), + (0x25, Code::VmsFetch), + ]; + for number in 0..=u8::MAX { + let decoded = event(record(number, 7, 0x1000, false)); + let code = named.iter().find(|(n, _)| *n == number).map_or(Code::Other(number), |(_, code)| *code); + assert_eq!((decoded.stream, decoded.code), (7, code), "event {number:#x}"); + assert_eq!(decoded.attempt.is_some(), (0x10..=0x13).contains(&number), "event {number:#x}"); + } +} + +// --- the stage 1 tables (DDI 0487 M.d §D8.3.1) ----------------------------- + +/// A device address whose four table indexes are 0x1a5, 0x0f0, 0x00f and 0x155. +const AT: u64 = 0x1a5 << 39 | 0x0f0 << 30 | 0x00f << 21; + +#[test] +fn memory_is_a_level_2_block_only_its_own_context_and_an_unprivileged_device_reach() { + let block = Phys::<21>::new(0xffff_ffe0_0000).unwrap(); + let fields = |read_only: u64| { + word(&[ + (0, 0, 1), // valid + (1, 1, 0), // a block + (4, 2, 1), // AttrIndx: MAIR0's attribute 1 + (6, 6, 1), // AP[1]: unprivileged access + (7, 7, read_only), // AP[2] + (9, 8, 0b11), // SH: inner shareable + (10, 10, 1), // AF + (11, 11, 1), // nG + (47, 21, 0xffff_ffe0_0000 >> 21), // the output address + (53, 53, 1), // PXN + (54, 54, 1), // UXN + ]) + }; + let path = plan(AT, Leaf::Memory(block, Access::ReadWrite)).expect("a 2 MiB aligned address"); + assert_eq!((path.indices(), path.descriptor), (&[0x1a5, 0x0f0, 0x00f][..], fields(0))); + let path = plan(AT, Leaf::Memory(block, Access::Read)).expect("a 2 MiB aligned address"); + assert_eq!((path.indices(), path.descriptor), (&[0x1a5, 0x0f0, 0x00f][..], fields(1))); +} + +#[test] +fn the_doorbell_is_one_level_3_page_of_device_memory() { + let page = Phys::<12>::new(0x0809_0000).unwrap(); + let path = plan(AT | 0x155 << 12, Leaf::Doorbell(page)).expect("a 4 KiB aligned address"); + assert_eq!(path.indices(), [0x1a5, 0x0f0, 0x00f, 0x155]); + assert_eq!( + path.descriptor, + word(&[ + (0, 0, 1), // valid + (1, 1, 1), // a page + (4, 2, 0), // AttrIndx: MAIR0's attribute 0, Device-nGnRE + (6, 6, 1), // AP[1] + (7, 7, 0), // AP[2]: written + (10, 10, 1), // AF + (11, 11, 1), // nG + (47, 12, 0x0809_0000 >> 12), // the page alone + (53, 53, 1), + (54, 54, 1), + ]) + ); + // Attribute 0 is Device-nGnRE and attribute 1 Normal write-back. + assert_eq!(MAIR0.to_le_bytes(), [0x04, 0xff, 0, 0]); +} + +#[test] +fn an_address_past_the_input_or_off_the_leafs_alignment_has_no_plan() { + let block = Leaf::Memory(Phys::new(0x20_0000).unwrap(), Access::ReadWrite); + let page = Leaf::Doorbell(Phys::new(0x1000).unwrap()); + assert_eq!(plan(1 << INPUT_BITS, block), None); + assert_eq!(plan(1 << INPUT_BITS, page), None); + assert_eq!(plan(!0x1f_ffff, block), None); + assert!(plan((1 << INPUT_BITS) - (1 << 21), block).is_some()); + // 4 KiB aligned is enough for the page and not for the block. + assert_eq!(plan(AT | 0x1000, block), None); + assert!(plan(AT | 0x1000, page).is_some()); + assert_eq!(plan(AT | 0x800, page), None); +} + +#[test] +fn a_descriptor_read_back_is_a_table_only_above_level_3() { + let table = Phys::<12>::new(0xffff_ffff_f000).unwrap(); + // Valid [0], table [1], the next table's address [47:12], and no hierarchical restriction. + assert_eq!(next(table), word(&[(0, 0, 1), (1, 1, 1), (47, 12, 0xffff_ffff_f000 >> 12)])); + for level in 0..3 { + assert_eq!(entry(next(table), level), Entry::Table(table)); + assert_eq!(entry(0, level), Entry::Invalid); + // Bit [0] alone decides validity: a stale address under it is nothing. + assert_eq!(entry(next(table) & !1, level), Entry::Invalid); + } + let block = plan(AT, Leaf::Memory(Phys::new(0x4020_0000).unwrap(), Access::ReadWrite)).unwrap(); + assert_eq!(entry(block.descriptor, 2), Entry::Mapped); + // A level 3 page sets bit [1] as a table descriptor does: it is no table. + let page = plan(AT, Leaf::Doorbell(Phys::new(0x0809_0000).unwrap())).unwrap(); + assert_eq!(entry(page.descriptor, 3), Entry::Mapped); + assert_eq!(entry(0, 3), Entry::Invalid); +} From f8022b7e2dde50cbe705216c3e269002744af440 Mon Sep 17 00:00:00 2001 From: japabu Date: Fri, 9 Oct 2026 14:04:37 +0200 Subject: [PATCH 2/3] The IORT's own-DeviceID mapping is read by its index, the unit's output size bounds what it is given, and the MADT is main's again Round 2 of the IORT decode and the SMMUv3 and ITS encodings, answering the review of 286536fb8. toyos-acpi. An SMMUv3 node's own DeviceID mapping is the one its DeviceID mapping index (offset 64) names: where Table 14's bit 4 says the index is valid from node revision 5, and before it wherever one of the four control interrupts has no GSIV, as DEN 0049 E.d words it. That entry's input base and length are not read, and it routes no stream. A single-mapping flag on any other entry refuses the table by name. An output reference is followed by the route that needs it and no longer by the open, which walked the nodes once per mapping: opening reads each node and mapping once, a route walks the nodes three times at most. The MADT's MSI controllers are gone, and madt.rs is main's byte for byte. The node's PRI GSIV is not answered, and its flag is named for what it is, an override. toyos-phys, new: the one Phys toyos-smmu and toyos-its each declared. toyos-smmu. Every address the unit is given is held to IDR5.OAS: the stream table and both queues by their whole extent, S1ContextPtr, TTB0, a next table and a leaf's output. Ste and Cd hold their words privately. The descriptors, the MAIR and PXN are toyos-bootmap's. The queues are two types made by the unit: Commands, and Events, whose `after` writes OVACKFLG as OVFLG was read. probe takes the IORT's override and refuses a unit that is not coherent. Signal::Sev and CMD_CFGI_CD are gone. toyos-its. An event, a collection and an LPI are types: an event of its device's EventBits, a collection of what GITS_TYPER (HCC, CCT, CIL, CIDbits) and the collection table hold, an LPI of a layout GICD_TYPER bounds in a redistributor whose GICR_PROPBASER read it back. The command queue's emptiness and fullness take the registers and mask them. A table register is a Backing that sizes its own pages. GITS_CTLR.Quiescent, HCC as a public field, the pending bit and the two redistributor offsets the kernel already declares are gone. What still has a second declaration in kernel/ is recorded in issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RvnWQFcMuGqTHYhvSnTe8A --- Cargo.lock | 11 + Cargo.toml | 1 + ...e-what-the-aarch64-kernel-also-declares.md | 48 +++ toyos-acpi/src/iort.rs | 151 ++++---- toyos-acpi/src/lib.rs | 4 +- toyos-acpi/src/madt.rs | 93 +---- toyos-acpi/tests/iort.rs | 315 ++++++++++------- toyos-bootmap/src/aarch64.rs | 2 +- toyos-its/Cargo.toml | 6 +- toyos-its/src/command.rs | 20 +- toyos-its/src/lib.rs | 196 +++++++---- toyos-its/src/lpi.rs | 90 +++-- toyos-its/tests/encodings.rs | 333 +++++++++++++----- toyos-phys/Cargo.toml | 12 + toyos-phys/src/lib.rs | 33 ++ toyos-phys/tests/phys.rs | 27 ++ toyos-smmu/Cargo.toml | 10 +- toyos-smmu/src/config.rs | 42 ++- toyos-smmu/src/lib.rs | 27 +- toyos-smmu/src/queue.rs | 115 ++++-- toyos-smmu/src/table.rs | 58 +-- toyos-smmu/src/unit.rs | 72 ++-- toyos-smmu/tests/encodings.rs | 293 ++++++++++----- 23 files changed, 1270 insertions(+), 689 deletions(-) create mode 100644 issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md create mode 100644 toyos-phys/Cargo.toml create mode 100644 toyos-phys/src/lib.rs create mode 100644 toyos-phys/tests/phys.rs diff --git a/Cargo.lock b/Cargo.lock index 2621d67fc81..ca92ccf3f50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5873,6 +5873,9 @@ dependencies = [ [[package]] name = "toyos-its" version = "0.1.0" +dependencies = [ + "toyos-phys", +] [[package]] name = "toyos-keymap" @@ -5991,6 +5994,10 @@ dependencies = [ "toyos-abi", ] +[[package]] +name = "toyos-phys" +version = "0.1.0" + [[package]] name = "toyos-ps2" version = "0.1.0" @@ -6006,6 +6013,10 @@ version = "0.1.0" [[package]] name = "toyos-smmu" version = "0.1.0" +dependencies = [ + "toyos-bootmap", + "toyos-phys", +] [[package]] name = "toyos-swap" diff --git a/Cargo.toml b/Cargo.toml index 49a3e48ce8a..41dbe85d606 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ members = [ "toyos-net-udp", "toyos-net-wire", "toyos-osrelease", + "toyos-phys", "toyos-quiesce", "toyos-rootimage", "toyos-smmu", diff --git a/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md b/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md new file mode 100644 index 00000000000..6430618fbf3 --- /dev/null +++ b/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md @@ -0,0 +1,48 @@ +--- +status: open +kind: defect +opened: 2026-10-09 +--- + +# The SMMUv3 and ITS crates declare what the AArch64 kernel also declares + +`toyos-smmu` and `toyos-its` came in with no kernel file touched, so each +declaration below has a second home in `kernel/`, and nothing holds the two +equal. `toyos-smmu` takes the stage 1 descriptors, the `MAIR` and `PXN` from +`toyos-bootmap::aarch64` already; what is left is what `toyos-bootmap` does not +declare. + +**The stage 1 descriptor.** `toyos-smmu/src/table.rs` declares `VALID` (bit +0), `TABLE_OR_PAGE` (bit 1), `AP_UNPRIVILEGED` (bit 6), `AP_READ_ONLY` (bit +7), `NOT_GLOBAL` (bit 11) and the level index `at >> (39 - 9 * level) & 0x1FF`. +`kernel/src/arch/aarch64/paging.rs` declares the same bits as `VALID`, +`TABLE`, `AP_EL0`, `AP_READ_ONLY` and `NOT_GLOBAL`, and the same `index`. It +also declares `INNER_SHAREABLE`, `OUTER_SHAREABLE`, `AF`, `PXN` and `UXN`, +which `toyos-bootmap/src/aarch64.rs` holds: the kernel's own copy of that +crate's. + +**The redistributor.** `toyos-its/src/lpi.rs` declares `GICR_CTLR.EnableLPIs`, +`GICR_TYPER.PLPIS` and `GICR_TYPER.Processor_Number`. `GICR_TYPER`'s other +fields (`VLPIS`, `Last`, the affinity) are `kernel/gicv3/src/lib.rs`'s, and +the two registers' offsets are `kernel/src/arch/aarch64/irqchip.rs`'s: one +register's fields have two crates. `toyos_its::TRANSLATION_FRAME` and +`toyos_gicv3::FRAME` are both the GIC's 64 KiB frame. + +**Owner.** The ARM desktop gate's next two stages, on +`issues/toyos-runs-on-arm64.md`'s stage 6: the SMMUv3 unit, which writes +these tables from `kernel/src/arch/aarch64/` beside `paging.rs`, owns the +descriptor; the ITS, which edits `irqchip.rs` and reads `GICR_TYPER` for +every CPU, owns the redistributor. + +**Exit.** Each bit and the level index has one declaration that the kernel's +tables, the loader's and the unit's are all written from, and each +redistributor register's fields have one crate: + +``` +rg -n 'const (VALID|TABLE|TABLE_OR_PAGE|AP_\w+|NOT_GLOBAL|INNER_SHAREABLE|OUTER_SHAREABLE|AF|PXN|UXN): u64' \ + kernel/src/arch/aarch64/paging.rs toyos-smmu/src/table.rs +rg -n 'fn index' kernel/src/arch/aarch64/paging.rs toyos-smmu/src/table.rs +``` + +print nothing, and `rg -l 'GICR_TYPER' toyos-its kernel/gicv3` names one +directory. diff --git a/toyos-acpi/src/iort.rs b/toyos-acpi/src/iort.rs index 17a6badf0bc..0c63c2abc3c 100644 --- a/toyos-acpi/src/iort.rs +++ b/toyos-acpi/src/iort.rs @@ -2,9 +2,13 @@ //! — which requester reaches which SMMUv3 under which StreamID, and which //! DeviceID its messages carry to which ITS. //! -//! [`iort`] answers a table every node of which it has walked: a node, a -//! mapping array or an output reference the table cannot hold is its refusal, -//! and so is a revision whose layout was not read against the specification. +//! [`iort`] answers a table every node of which it has walked: a node or a +//! mapping array the table cannot hold is its refusal, and so is a revision +//! whose layout was not read against the specification. An output reference +//! is followed, and judged, by the [`Iort::route`] that needs it: opening the +//! table and routing one requester each read it a bounded number of times, +//! whatever its nodes and mappings say. +//! //! Only the three node types that route a PCI function are decoded — the ITS, //! the root complex and the SMMUv3; any other is answered by its type alone, //! for its caller to rule on, and is refused as the output of a mapping. @@ -19,8 +23,8 @@ const NODE_COUNT: usize = SDT_HEADER_LEN; const NODE_ARRAY: usize = SDT_HEADER_LEN + 4; pub const IORT_NEEDED: usize = SDT_HEADER_LEN + 12; -/// The table revisions decoded: QEMU's `virt` publishes 5 and issue E.g is 7. -/// Every issue from E.b carries a revision in each node, which is what a +/// The table revisions decoded: 5 is issue E.d's, 6 issues E.e's and E.f's and +/// 7 issue E.g's. Each carries a revision in every node, which is what a /// node's own layout is judged by. const REVISIONS: core::ops::RangeInclusive = 5..=7; @@ -31,7 +35,7 @@ const NODE_HEADER: usize = 16; /// reference, flags — five words. const MAPPING: usize = 20; /// Table 6: bit 0, "apply the output base regardless of the input IDs". -const SINGLE: u32 = 1; +const SINGLE: u32 = 1 << 0; const ITS: u8 = 0; const ROOT_COMPLEX: u8 = 2; @@ -50,9 +54,12 @@ const RC_SEGMENT: usize = 28; const SMMU_BASE: usize = 16; const SMMU_FLAGS: usize = 24; const SMMU_GSIVS: usize = 44; +const SMMU_INDEX: usize = 64; const SMMU_LEN: usize = 68; /// Table 14, bit 0: `COHACC override`. -const SMMU_COHACC: u32 = 1; +const SMMU_COHACC: u32 = 1 << 0; +/// Table 14, bit 4, from node revision 5: `DeviceID mapping index valid`. +const SMMU_INDEXED: u32 = 1 << 4; /// Why an IORT cannot be used. `at` is a node's offset in the table. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -72,15 +79,17 @@ pub enum IortRefused { Mappings { at: usize, count: u32, offset: u32 }, /// A mapping's input or output range runs past the 32-bit ID space. Range { at: usize, base: u32, last: u32 }, - /// A mapping's output reference names no node, or a node its source may - /// not output to: a root complex outputs to an SMMUv3 or an ITS, an - /// SMMUv3 to an ITS. + /// The mapping a requester is routed by names no node with its output + /// reference, or a node its source may not output to: a root complex + /// outputs to an SMMUv3 or an ITS, an SMMUv3 to an ITS. Output { at: usize, reference: u32 }, - /// A root complex mapping with the single-mapping flag, which no table - /// read here carries. + /// A mapping with the single-mapping flag that is not its SMMUv3's own. + /// Table 6 allows one in a root complex and in an SMMUv3, where it puts + /// every input ID out as one; nothing here routes that. SingleMapping { at: usize }, - /// An ITS group of other than one ITS: deprecated by issue E.g, which - /// fixes the count at 1. + /// An ITS group of other than one ITS, which a table of revision 5 or 6 + /// may hold: issue E.g fixes the count at 1, and nothing here routes to + /// a group. ItsGroup { at: usize, count: u32 }, /// Two mappings claim this ID, which the table must route one way. Overlap { id: u32 }, @@ -96,11 +105,11 @@ impl From for IortRefused { #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Smmuv3 { pub base: u64, - /// Table 14's `COHACC override`: what `SMMU_IDR0.COHACC` is read as. - pub coherent: bool, - /// The four wired interrupts; `None` where the node gives no GSIV. + /// Table 14's `COHACC override`: set, the unit reads its tables and + /// queues coherently whatever its `SMMU_IDR0.COHACC` says. + pub coherent_override: bool, + /// The wired interrupts; `None` where the node gives no GSIV. pub event: Option, - pub pri: Option, pub gerror: Option, pub sync: Option, } @@ -128,7 +137,9 @@ pub struct ItsDevice { #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Route { /// Through this SMMUv3 under `stream`; `its` is where the SMMU's own node - /// maps that stream on to, and `None` where it maps it nowhere. + /// maps that stream on to, and `None` where it maps it nowhere. The + /// unit's own DeviceID (Table 13, `DeviceID mapping index`) is no + /// stream's. Translated { smmu: Smmuv3, stream: u32, its: Option }, /// Straight to an ITS, past every SMMU. Untranslated(ItsDevice), @@ -160,7 +171,7 @@ struct Mapping { impl Mapping { fn maps(&self, id: u32) -> Option { // `output + span` was checked not to wrap when the table was opened. - (!self.single && id >= self.input && id - self.input <= self.span).then(|| self.output + (id - self.input)) + (id >= self.input && id - self.input <= self.span).then(|| self.output + (id - self.input)) } } @@ -201,41 +212,70 @@ impl Iort

{ /// Where the PCI function `rid` of `segment` is routed. pub fn route(&self, segment: u32, rid: u16) -> Result { - let mut route = None; + let rid = u32::from(rid); + let mut hit = None; for raw in self.raw_nodes() { let raw = raw?; if self.decode(raw)? != (Node::RootComplex { segment }) { continue; } - for (mapping, id) in self.hits(raw, u32::from(rid)) { - let hop = match self.output(raw, mapping)? { - (_, Node::Its { id: its }) => Route::Untranslated(ItsDevice { its, device: id }), - (smmu_raw, Node::Smmuv3(smmu)) => { - let mut its = None; - for (onward, device) in self.hits(smmu_raw, id) { - let (_, Node::Its { id: to }) = self.output(smmu_raw, onward)? else { - return Err(IortRefused::Output { at: smmu_raw.at, reference: onward.reference }); - }; - if its.replace(ItsDevice { its: to, device }).is_some() { - return Err(IortRefused::Overlap { id }); - } - } - Route::Translated { smmu, stream: id, its } + for found in self.hits(raw, rid) { + if hit.replace((raw, found)).is_some() { + return Err(IortRefused::Overlap { id: rid }); + } + } + } + let Some((root, (mapping, id))) = hit else { + return Ok(Route::Unmapped); + }; + match self.output(root, mapping)? { + (_, Node::Its { id: its }) => Ok(Route::Untranslated(ItsDevice { its, device: id })), + (unit, Node::Smmuv3(smmu)) => { + let mut onward = None; + for found in self.hits(unit, id) { + if onward.replace(found).is_some() { + return Err(IortRefused::Overlap { id }); } - _ => return Err(IortRefused::Output { at: raw.at, reference: mapping.reference }), - }; - if route.replace(hop).is_some() { - return Err(IortRefused::Overlap { id: u32::from(rid) }); } + let its = match onward { + None => None, + Some((mapping, device)) => match self.output(unit, mapping)? { + (_, Node::Its { id: its }) => Some(ItsDevice { its, device }), + _ => return Err(IortRefused::Output { at: unit.at, reference: mapping.reference }), + }, + }; + Ok(Route::Translated { smmu, stream: id, its }) } + _ => Err(IortRefused::Output { at: root.at, reference: mapping.reference }), + } + } + + /// The index of the mapping that is an SMMUv3's own DeviceID (Table 13, + /// offset 64), where its node says it has one: by the flag from node + /// revision 5, and before it wherever a control interrupt has no GSIV. + /// The entry's input base and length are not read. + fn own(&self, raw: Raw) -> Option { + let t = &self.table; + if raw.kind != SMMUV3 { + return None; + } + let indexed = if raw.revision >= 5 { + t.u32_at(raw.at + SMMU_FLAGS)? & SMMU_INDEXED != 0 + } else { + (0..4).any(|i| t.u32_at(raw.at + SMMU_GSIVS + 4 * i) == Some(0)) + }; + if indexed { + t.u32_at(raw.at + SMMU_INDEX) + } else { + None } - Ok(route.unwrap_or(Route::Unmapped)) } /// The mappings of `raw` that hold `id`, each with the ID it puts out. fn hits(&self, raw: Raw, id: u32) -> impl Iterator + '_ { - (0..raw.mappings).filter_map(move |i| { - let mapping = self.mapping(raw, i)?; + let own = self.own(raw); + (0..raw.mappings).filter(move |index| Some(*index) != own).filter_map(move |index| { + let mapping = self.mapping(raw, index)?; Some((mapping, mapping.maps(id)?)) }) } @@ -290,7 +330,7 @@ impl Iort

{ }) } - /// The node a mapping of `from` outputs to. + /// The node a mapping of `from` outputs to: one walk of the nodes. fn output(&self, from: Raw, mapping: Mapping) -> Result<(Raw, Node), IortRefused> { let refused = IortRefused::Output { at: from.at, reference: mapping.reference }; for raw in self.raw_nodes() { @@ -328,12 +368,12 @@ impl Iort

{ } SMMUV3 => { revised(4..=5)?; + // Event, PRI, GERR, sync: the PRI queue is not used, and its GSIV not answered. let gsiv = |i: usize| t.u32_at(at + SMMU_GSIVS + 4 * i).map(NonZeroU32::new).ok_or(short); Node::Smmuv3(Smmuv3 { base: t.u64_at(at + SMMU_BASE).ok_or(short)?, - coherent: t.u32_at(at + SMMU_FLAGS).ok_or(short)? & SMMU_COHACC != 0, + coherent_override: t.u32_at(at + SMMU_FLAGS).ok_or(short)? & SMMU_COHACC != 0, event: gsiv(0)?, - pri: gsiv(1)?, gerror: gsiv(2)?, sync: gsiv(3)?, }) @@ -342,7 +382,7 @@ impl Iort

{ }) } - /// One node's fields, its ID array's place and every mapping in it. + /// One node's fields, its ID array's place and every mapping's own words. fn check(&self, raw: Raw) -> Result<(), IortRefused> { let node = self.decode(raw)?; // Where the node's own fields end, which its ID array may not lie over. @@ -359,24 +399,17 @@ impl Iort

{ if count != 0 && (misplaced || matches!(node, Node::Its { .. })) { return Err(IortRefused::Mappings { at, count, offset }); } - for index in 0..count { + let own = self.own(raw); + for index in (0..count).filter(|index| Some(*index) != own) { let mapping = self.mapping(raw, index).ok_or(IortRefused::Mappings { at, count, offset })?; - let from_root = matches!(node, Node::RootComplex { .. }); - if mapping.single && from_root { + if mapping.single { return Err(IortRefused::SingleMapping { at }); } - if !mapping.single { - for base in [mapping.input, mapping.output] { - if base.checked_add(mapping.span).is_none() { - return Err(IortRefused::Range { at, base, last: mapping.span }); - } + for base in [mapping.input, mapping.output] { + if base.checked_add(mapping.span).is_none() { + return Err(IortRefused::Range { at, base, last: mapping.span }); } } - match self.output(raw, mapping)?.1 { - Node::Its { .. } => {} - Node::Smmuv3(_) if from_root => {} - _ => return Err(IortRefused::Output { at, reference: mapping.reference }), - } } Ok(()) } diff --git a/toyos-acpi/src/lib.rs b/toyos-acpi/src/lib.rs index f9c0cfed746..c16d58441e1 100644 --- a/toyos-acpi/src/lib.rs +++ b/toyos-acpi/src/lib.rs @@ -37,8 +37,8 @@ pub use fadt::{ use fadt::FADT_DSDT; pub use iort::{iort, Iort, IortRefused, ItsDevice, Node, Route, Smmuv3, IORT_NEEDED}; pub use madt::{ - isa_line, madt_entries, msi_controllers, sci_line, Gicc, IoApicEntry, Line, MadtEntries, MadtEntry, MadtHalt, - MsiController, MsiFrame, Polarity, SourceOverride, SpiRange, Trigger, MADT_ENTRIES, + isa_line, madt_entries, sci_line, Gicc, IoApicEntry, Line, MadtEntries, MadtEntry, MadtHalt, + Polarity, SourceOverride, Trigger, MADT_ENTRIES, }; pub use gtdt::{gtdt, Gtdt, TimerInterrupt, GTDT_NEEDED}; pub use resource::{memory_windows, ResourceError, MAX_LIST_BYTES}; diff --git a/toyos-acpi/src/madt.rs b/toyos-acpi/src/madt.rs index 9b9a39b39fe..5c87c337133 100644 --- a/toyos-acpi/src/madt.rs +++ b/toyos-acpi/src/madt.rs @@ -88,18 +88,10 @@ pub fn madt_entries(madt: &Table

) -> MadtEntries

{ } } -/// One structure the list holds whole: its type, its declared length and -/// where its first byte is. -#[derive(Clone, Copy)] -struct Structure

{ - phys: P, - base: u64, - entry_type: u8, - declared: usize, -} +impl Iterator for MadtEntries

{ + type Item = Result; -impl MadtEntries

{ - fn structure(&mut self) -> Option, MadtHalt>> { + fn next(&mut self) -> Option { if self.halted || self.offset + ENTRY_HEADER_LEN > self.list_len { return None; } @@ -116,81 +108,10 @@ impl MadtEntries

{ })); } self.offset += declared; - Some(Ok(Structure { phys: self.table.phys(), base: self.table.base() + at as u64, entry_type, declared })) - } -} -impl Iterator for MadtEntries

{ - type Item = Result; - - fn next(&mut self) -> Option { - Some(self.structure()?.map(Structure::decode)) - } -} - -/// The SPIs a GICv2m frame raises, where its structure names them itself. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct SpiRange { - pub base: u16, - pub count: u16, -} - -/// MADT type 0xD (ACPI 6.5 §5.2.12.16, "GIC MSI Frame Structure"): a GICv2m -/// frame, whose doorbell takes the SPI to raise from the data written to it. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct MsiFrame { - pub id: u32, - pub base: u64, - /// `None` where the flags' `SPI Count/Base Select` is clear and both are - /// the frame's own `MSI_TYPER`'s to say. - pub spis: Option, -} - -/// What a machine's functions write their message-signalled interrupts to. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum MsiController { - /// MADT type 0xF: an ITS, which looks an event up by its writer's DeviceID. - Its { id: u32, base: u64 }, - Frame(MsiFrame), -} - -/// Every MSI controller a validated MADT names, in its order: the walk -/// [`madt_entries`] makes, ending on the same [`MadtHalt`]. -pub fn msi_controllers(madt: &Table

) -> impl Iterator> { - let mut entries = madt_entries(madt); - core::iter::from_fn(move || { - loop { - let structure = match entries.structure()? { - Ok(structure) => structure, - Err(halt) => return Some(Err(halt)), - }; - if let Some(frame) = structure.msi_frame() { - return Some(Ok(MsiController::Frame(frame))); - } - if let MadtEntry::Its { id, base } = structure.decode() { - return Some(Ok(MsiController::Its { id, base })); - } - } - }) -} - -impl Structure

{ - /// ACPI 6.5 §5.2.12.16: GIC MSI Frame ID (4..8), Physical Base Address - /// (8..16), Flags (16..20) whose bit 0 is `SPI Count/Base Select`, SPI - /// Count (20..22), SPI Base (22..24). - fn msi_frame(self) -> Option { - let Self { phys, base, entry_type, declared } = self; - (entry_type == 0xD && declared >= 24).then(|| MsiFrame { - id: u32le(phys, base + 4), - base: u64le(phys, base + 8), - spis: (u32le(phys, base + 16) & 1 != 0) - .then(|| SpiRange { count: u16le(phys, base + 20), base: u16le(phys, base + 22) }), - }) - } - - fn decode(self) -> MadtEntry { - let Self { phys, base, entry_type, declared } = self; - match (entry_type, declared) { + let phys = self.table.phys(); + let base = self.table.base() + at as u64; + Some(Ok(match (entry_type, declared) { // Table 5.21: ACPI Processor UID (2), APIC ID (3), Flags (4..8). (0, 8..) => MadtEntry::LocalApic { apic_id: u32::from(phys.byte(base + 3)), @@ -229,7 +150,7 @@ impl Structure

{ // Table 5.44: GIC ITS ID (4..8), Physical Base Address (8..16). (0xF, 20..) => MadtEntry::Its { id: u32le(phys, base + 4), base: u64le(phys, base + 8) }, _ => MadtEntry::Other(entry_type), - } + })) } } diff --git a/toyos-acpi/tests/iort.rs b/toyos-acpi/tests/iort.rs index da1d0a5cd94..f13b7984d53 100644 --- a/toyos-acpi/tests/iort.rs +++ b/toyos-acpi/tests/iort.rs @@ -1,15 +1,16 @@ -//! The IORT and the MADT's MSI controllers: QEMU `virt`'s own tables in both -//! of its GIC modes, decoded against what was read off their bytes by hand -//! with DEN 0049 E.g and ACPI 6.5 open, and every refusal over crafted ones. +//! The IORT: QEMU `virt`'s own tables in both of its GIC modes, decoded +//! against what was read off their bytes by hand with DEN 0049 open, and +//! every refusal over crafted ones. mod common; +use core::cell::Cell; use core::num::NonZeroU32; -use common::{declare_len, entry, madt, reseal, rsdp, sdt, xsdt, Machine}; +use common::{declare_len, reseal, rsdp, sdt, xsdt, Machine}; use toyos_acpi::{ - find_table, iort, madt_entries, msi_controllers, IortRefused, ItsDevice, MadtEntry, MadtHalt, MsiController, - MsiFrame, Node, Route, Smmuv3, SpiRange, TableError, MADT_ENTRIES, + find_table, iort, madt_entries, IortRefused, ItsDevice, MadtEntry, Node, Phys, Route, Smmuv3, TableError, + MADT_ENTRIES, }; const RSDP: u64 = 0x4cb4_3018; @@ -35,14 +36,15 @@ const ITS: &[(u64, &[u8])] = &[ ]; /// The node both modes publish: `arm-smmuv3` at `0x09050000`, `COHACC -/// override` set, and the GSIVs in the node's order — event, PRI, GERR, sync — -/// of which GERR is the higher of the last two. +/// override` set, and the GSIVs in the node's order — event 106, PRI 107, +/// GERR 109, sync 108 — of which GERR is the higher of the last two. Every +/// one is wired, so the revision 4 node's DeviceID mapping index is ignored +/// and its one mapping in the ITS boot routes streams. fn virt_smmu() -> Smmuv3 { Smmuv3 { base: 0x0905_0000, - coherent: true, + coherent_override: true, event: NonZeroU32::new(106), - pri: NonZeroU32::new(107), gerror: NonZeroU32::new(109), sync: NonZeroU32::new(108), } @@ -90,31 +92,24 @@ fn the_its_boot_routes_bus_zero_through_the_smmu_to_the_its_and_the_rest_past_it assert_eq!(table.route(1, 0x08), Ok(Route::Unmapped)); } -fn controllers(regions: &[(u64, &[u8])]) -> Vec> { +fn madt_of(regions: &[(u64, &[u8])]) -> Vec { let table = find_table(Machine { regions }, RSDP, b"APIC", MADT_ENTRIES).expect("MADT"); - msi_controllers(&table).collect() + madt_entries(&table).map(|entry| entry.expect("a structure the list holds whole")).collect() } -/// `arm-gicv2m` at `0x08020000`, 64 SPIs from 80, named by the structure. +/// `arm-gicv3-its` at `0x08080000`, under the id the IORT's ITS node carries. #[test] -fn the_hardware_gic_boot_names_a_v2m_frame_and_no_its() { - assert_eq!( - controllers(HWGIC), - [Ok(MsiController::Frame(MsiFrame { - id: 0, - base: 0x0802_0000, - spis: Some(SpiRange { base: 80, count: 64 }), - }))] - ); - // The entry walk a kernel already makes still answers it by its type alone. - let table = find_table(Machine { regions: HWGIC }, RSDP, b"APIC", MADT_ENTRIES).expect("MADT"); - assert_eq!(madt_entries(&table).last(), Some(Ok(MadtEntry::Other(0xD)))); +fn the_its_boots_madt_names_the_its_the_iort_routes_to() { + let its: Vec<_> = madt_of(ITS).into_iter().filter(|entry| matches!(entry, MadtEntry::Its { .. })).collect(); + assert_eq!(its, [MadtEntry::Its { id: 0, base: 0x0808_0000 }]); } -/// `arm-gicv3-its` at `0x08080000`, under the id the IORT's ITS node carries. +/// It ends in a GIC MSI Frame structure, type 0xD, which is no ITS. #[test] -fn the_its_boot_names_the_its_the_iort_routes_to() { - assert_eq!(controllers(ITS), [Ok(MsiController::Its { id: 0, base: 0x0808_0000 })]); +fn the_hardware_gic_boots_madt_names_no_its() { + let entries = madt_of(HWGIC); + assert!(!entries.iter().any(|entry| matches!(entry, MadtEntry::Its { .. }))); + assert_eq!(entries.last(), Some(&MadtEntry::Other(0xD))); } // --- crafted tables ------------------------------------------------------- @@ -147,11 +142,23 @@ fn root_complex(segment: u32, mappings: &[[u32; 5]]) -> Vec { node(2, 3, &fields, mappings) } -/// A revision 4 SMMUv3 at `base` with no flag and no GSIV: 52 bytes of fields. -fn smmu(base: u64, mappings: &[[u32; 5]]) -> Vec { +/// An SMMUv3 node of `revision` at `0x0905_0000`, 52 bytes of fields: Table +/// 13's flags (24), its event, PRI, GERR and sync GSIVs (44..60) and its +/// DeviceID mapping index (64). +fn smmu_with(revision: u8, flags: u32, gsivs: [u32; 4], index: u32, mappings: &[[u32; 5]]) -> Vec { let mut fields = vec![0u8; 52]; - fields[..8].copy_from_slice(&base.to_le_bytes()); - node(4, 4, &fields, mappings) + fields[..8].copy_from_slice(&0x0905_0000u64.to_le_bytes()); + fields[8..12].copy_from_slice(&flags.to_le_bytes()); + for (i, gsiv) in gsivs.iter().enumerate() { + fields[28 + 4 * i..32 + 4 * i].copy_from_slice(&gsiv.to_le_bytes()); + } + fields[48..52].copy_from_slice(&index.to_le_bytes()); + node(4, revision, &fields, mappings) +} + +/// A revision 5 SMMUv3 with no flag and no GSIV: none of its mappings is its own. +fn smmu(mappings: &[[u32; 5]]) -> Vec { + smmu_with(5, 0, [0; 4], 0, mappings) } /// An IORT of `revision` over `nodes`, its array straight after the header. @@ -174,7 +181,7 @@ const ROOT_AFTER_ONE: u32 = ROOT_AT + 20; /// The ITS boot's shape with every number chosen: requesters `0x100..=0x1ff` /// to streams `0x1000..`, streams `0x1000..=0x10ff` to DeviceIDs `0x20000..`. fn three(root: &[[u32; 5]], onward: &[[u32; 5]]) -> Vec { - table(5, &[its(7), smmu(0x0905_0000, onward), root_complex(0, root)]) + table(5, &[its(7), smmu(onward), root_complex(0, root)]) } fn judged(bytes: &[u8], ask: impl FnOnce(&toyos_acpi::Iort>) -> T) -> Result { @@ -193,7 +200,7 @@ fn route(bytes: &[u8], segment: u32, rid: u16) -> Result { } fn plain_smmu() -> Smmuv3 { - Smmuv3 { base: 0x0905_0000, coherent: false, event: None, pri: None, gerror: None, sync: None } + Smmuv3 { base: 0x0905_0000, coherent_override: false, event: None, gerror: None, sync: None } } #[test] @@ -277,7 +284,7 @@ fn a_node_longer_than_the_table_or_shorter_than_its_type_is_refused() { } // An SMMUv3 node cut before its DeviceID mapping index, with bytes after it to read. - let mut cut = table(5, &[smmu(0, &[]), its(0)]); + let mut cut = table(5, &[smmu(&[]), its(0)]); cut[49..51].copy_from_slice(&64u16.to_le_bytes()); reseal(&mut cut); assert_eq!(refusal(&cut), Some(IortRefused::Node { at: 48, declared: 64 })); @@ -292,7 +299,7 @@ fn a_node_revision_whose_layout_was_not_read_is_refused() { for (kind, node, known, unknown) in [ (0u8, its(0), &[1u8][..], &[0u8, 2][..]), (2, root_complex(0, &[]), &[3, 4], &[2, 5]), - (4, smmu(0, &[]), &[4, 5], &[3, 6]), + (4, smmu(&[]), &[4, 5], &[3, 6]), ] { for &revision in known { assert_eq!(revised(node.clone(), revision), None, "type {kind} revision {revision}"); @@ -376,34 +383,37 @@ fn a_mapping_whose_range_leaves_the_id_space_is_refused() { } #[test] -fn an_output_reference_that_names_no_node_is_refused() { +fn a_route_through_an_output_reference_that_names_no_node_is_refused() { // Zero, the table's own header, the middle of a node, one past the last node. for reference in [0, 36, SMMU_AT + 4, ROOT_AT + 56, u32::MAX] { - assert_eq!( - refusal(&three(&[[0, 0xff, 0, reference, 0]], &[])), - Some(IortRefused::Output { at: 140, reference }), - "reference {reference}" - ); + let t = three(&[[0, 0xff, 0, reference, 0]], &[]); + assert_eq!(route(&t, 0, 8), Err(IortRefused::Output { at: 140, reference }), "reference {reference}"); + // A requester the mapping does not hold never follows it. + assert_eq!(route(&t, 0, 0x100), Ok(Route::Unmapped), "reference {reference}"); } + // The SMMUv3's onward mapping is followed by the stream it holds. + let t = three(&[[0, 0x1ff, 0, SMMU_AT, 0]], &[[0, 0xff, 0, ITS_AT + 4, 0]]); + assert_eq!(route(&t, 0, 8), Err(IortRefused::Output { at: 72, reference: ITS_AT + 4 })); + assert_eq!(route(&t, 0, 0x100), Ok(Route::Translated { smmu: plain_smmu(), stream: 0x100, its: None })); } #[test] -fn an_output_reference_to_a_node_its_source_may_not_reach_is_refused() { +fn a_route_through_an_output_reference_to_a_node_its_source_may_not_reach_is_refused() { // A root complex to a root complex — itself. assert_eq!( - refusal(&three(&[[0, 0xff, 0, ROOT_AT, 0]], &[])), - Some(IortRefused::Output { at: 140, reference: ROOT_AT }) + route(&three(&[[0, 0xff, 0, ROOT_AT, 0]], &[]), 0, 8), + Err(IortRefused::Output { at: 140, reference: ROOT_AT }) ); // An SMMUv3 to an SMMUv3 — itself — and to a root complex. for reference in [SMMU_AT, ROOT_AFTER_ONE] { assert_eq!( - refusal(&three(&[], &[[0, 0xff, 0, reference, 0]])), - Some(IortRefused::Output { at: 72, reference }) + route(&three(&[[0, 0xff, 0, SMMU_AT, 0]], &[[0, 0xff, 0, reference, 0]]), 0, 8), + Err(IortRefused::Output { at: 72, reference }) ); } // A root complex to a node type not decoded: an SMMUv1 or v2. let old = table(5, &[node(3, 3, &[0u8; 44], &[]), root_complex(0, &[[0, 0xff, 0, 48, 0]])]); - assert_eq!(refusal(&old), Some(IortRefused::Output { at: 108, reference: 48 })); + assert_eq!(route(&old, 0, 8), Err(IortRefused::Output { at: 108, reference: 48 })); } #[test] @@ -430,22 +440,82 @@ fn a_node_type_not_decoded_is_answered_by_its_type_and_refuses_nothing() { } #[test] -fn a_single_mapping_is_the_smmus_own_message_and_never_a_root_complexs() { - assert_eq!( - refusal(&three(&[[0, 0xff, 0, SMMU_AT, 1]], &[])), - Some(IortRefused::SingleMapping { at: 140 }) - ); - // On the SMMUv3 it names the DeviceID of the unit's own interrupts: its - // input fields are ignored, so it routes no stream and its range is not judged. - let own = three(&[[0, 0xff, 0, SMMU_AT, 0]], &[[0, 0, 0x30, ITS_AT, 1]]); - assert_eq!(route(&own, 0, 0), Ok(Route::Translated { smmu: plain_smmu(), stream: 0, its: None })); - let wide = three(&[[0, 0xff, 0, SMMU_AT, 0]], &[[0xffff_ffff, 0xffff_ffff, 0x30, ITS_AT, 1]]); +fn a_single_mapping_that_is_not_its_smmus_own_is_refused() { + // Table 6 has it put every input ID out as one: a root complex's, and an SMMUv3's that its index does not name. + assert_eq!(refusal(&three(&[[0, 0xff, 0, SMMU_AT, 1]], &[])), Some(IortRefused::SingleMapping { at: 140 })); + assert_eq!(refusal(&three(&[], &[[0, 0, 0x30, ITS_AT, 1]])), Some(IortRefused::SingleMapping { at: 72 })); + let beside = smmu_with(5, INDEX_VALID, WIRED, 1, &[[0, 0, 0x30, ITS_AT, 1], [0, 0, 0x31, ITS_AT, 1]]); + assert_eq!(refusal(&behind(beside)), Some(IortRefused::SingleMapping { at: 72 })); + // The one it names carries the flag as Table 13 asks, and is no refusal. + let own = smmu_with(5, INDEX_VALID, WIRED, 0, &[[0, 0, 0x30, ITS_AT, 1]]); + assert_eq!(refusal(&behind(own.clone())), None); + assert_eq!(onward(&behind(own)), None); + // An index past the array names no entry: every mapping there is routes. + let past = behind(smmu_with(5, INDEX_VALID, WIRED, 1, &[[0, 0xff, 0x100, ITS_AT, 0]])); + assert_eq!(onward(&past), Some(ItsDevice { its: 7, device: 0x108 })); +} + +/// `virt`'s four GSIVs: every control interrupt wired. +const WIRED: [u32; 4] = [106, 107, 109, 108]; +/// Table 14, bit 4: `DeviceID mapping index valid`. +const INDEX_VALID: u32 = 1 << 4; + +/// An ITS of id 7, `smmu`, and a root complex sending requesters +/// `0x00..=0xff` to it as the same streams. +fn behind(smmu: Vec) -> Vec { + table(5, &[its(7), smmu, root_complex(0, &[[0, 0xff, 0, SMMU_AT, 0]])]) +} + +/// Where the SMMUv3's own node sends stream 8. +fn onward(table: &[u8]) -> Option { + match route(table, 0, 8) { + Ok(Route::Translated { stream: 8, its, .. }) => its, + other => panic!("requester 8 is stream 8 of the SMMUv3: {other:?}"), + } +} + +#[test] +fn the_mapping_a_revision_5_smmu_indexes_is_the_units_own_and_routes_no_stream() { + // An input base and length no range holds: they are not the table's to judge. + let wide = behind(smmu_with(5, INDEX_VALID, WIRED, 0, &[[0xffff_ffff, 0xffff_ffff, 0x30, ITS_AT, 0]])); assert_eq!(refusal(&wide), None); - // It still has to name an ITS. - assert_eq!( - refusal(&three(&[], &[[0, 0, 0x30, ROOT_AFTER_ONE, 1]])), - Some(IortRefused::Output { at: 72, reference: ROOT_AFTER_ONE }) - ); + assert_eq!(onward(&wide), None); + // And ones that would hold the stream: the entry is the unit's DeviceID and no stream's. + let plausible = behind(smmu_with(5, INDEX_VALID, WIRED, 0, &[[0, 0xffff, 0x30, ITS_AT, 0]])); + assert_eq!(onward(&plausible), None); + // The index picks the entry, and the other one still routes. + let second = smmu_with(5, INDEX_VALID, WIRED, 1, &[[0, 0xff, 0x100, ITS_AT, 0], [0, 0xffff, 0x30, ITS_AT, 0]]); + assert_eq!(onward(&behind(second)), Some(ItsDevice { its: 7, device: 0x108 })); + // Flag clear, the index is ignored whatever the GSIVs say. + let ignored = behind(smmu_with(5, 0, [0; 4], 0, &[[0, 0xff, 0x100, ITS_AT, 0]])); + assert_eq!(onward(&ignored), Some(ItsDevice { its: 7, device: 0x108 })); +} + +#[test] +fn a_revision_4_smmu_indexes_its_own_mapping_whenever_a_control_interrupt_is_not_wired() { + for unwired in 0..4 { + let mut gsivs = WIRED; + gsivs[unwired] = 0; + let own = behind(smmu_with(4, 0, gsivs, 0, &[[0xffff_ffff, 0xffff_ffff, 0x30, ITS_AT, 0]])); + assert_eq!(refusal(&own), None, "GSIV {unwired} zero"); + assert_eq!(onward(&own), None, "GSIV {unwired} zero"); + let plausible = behind(smmu_with(4, 0, gsivs, 0, &[[0, 0xffff, 0x30, ITS_AT, 0]])); + assert_eq!(onward(&plausible), None, "GSIV {unwired} zero"); + } + // Every one wired, as on `virt`: the field is ignored, and bit 4 of the flags is reserved here. + for flags in [0, INDEX_VALID] { + let wired = behind(smmu_with(4, flags, WIRED, 0, &[[0, 0xff, 0x100, ITS_AT, 0]])); + assert_eq!(onward(&wired), Some(ItsDevice { its: 7, device: 0x108 })); + } +} + +/// Only an SMMUv3 has a DeviceID mapping index. Read as one, this root +/// complex would have no GSIV and, where the index is, its second mapping's +/// output base: zero, naming the first. +#[test] +fn a_root_complex_has_no_mapping_of_its_own() { + let t = table(5, &[its(7), root_complex(0, &[[0, 0xff, 0, ITS_AT, 0], [0x100, 0xff, 0, ITS_AT, 0]])]); + assert_eq!(route(&t, 0, 8), Ok(Route::Untranslated(ItsDevice { its: 7, device: 8 }))); } #[test] @@ -513,84 +583,57 @@ fn no_corruption_of_a_published_iort_panics_or_hangs_the_decoder() { } } -// --- the MADT's MSI controllers ------------------------------------------- - -fn crafted_controllers(list: &[u8]) -> Vec> { - let t = madt(list); - let head = rsdp(0x1000, 2, 36); - let root = xsdt(&[TABLE_AT]); - let regions: &[(u64, &[u8])] = &[(0x800, &head), (0x1000, &root), (TABLE_AT, &t)]; - let table = find_table(Machine { regions }, 0x800, b"APIC", MADT_ENTRIES).expect("MADT"); - msi_controllers(&table).take(8).collect() +/// Memory that counts every byte the decoder reads of it. +#[derive(Clone, Copy)] +struct Counted<'a> { + machine: Machine<'a>, + reads: &'a Cell, } -/// A GIC MSI Frame structure's 22 bytes after its type and length. -fn frame(id: u32, base: u64, flags: u32, count: u16, spi_base: u16) -> Vec { - let mut body = vec![0u8; 2]; - body.extend(id.to_le_bytes()); - body.extend(base.to_le_bytes()); - body.extend(flags.to_le_bytes()); - body.extend(count.to_le_bytes()); - body.extend(spi_base.to_le_bytes()); - entry(0xD, 24, &body) -} +impl Phys for Counted<'_> { + fn readable(self, phys: u64, len: usize) -> bool { + self.machine.readable(phys, len) + } -#[test] -fn a_frame_names_its_spis_only_where_its_flag_says_the_structure_does() { - let mut list = frame(3, 0x0802_0000, 0, 64, 80); - list.extend(frame(4, 0x0803_0000, 1, 32, 144)); - // Every flag but the select bit: reserved, and no word about the SPIs. - list.extend(frame(5, 0x0804_0000, !1, 8, 200)); - assert_eq!( - crafted_controllers(&list), - [ - Ok(MsiController::Frame(MsiFrame { id: 3, base: 0x0802_0000, spis: None })), - Ok(MsiController::Frame(MsiFrame { - id: 4, - base: 0x0803_0000, - spis: Some(SpiRange { base: 144, count: 32 }) - })), - Ok(MsiController::Frame(MsiFrame { id: 5, base: 0x0804_0000, spis: None })), - ] - ); + fn byte(self, phys: u64) -> u8 { + self.reads.set(self.reads.get() + 1); + Phys::byte(self.machine, phys) + } } +/// 499 root complexes of 40 mappings each, every one naming the table's last +/// node: following each reference by a walk of the nodes when the table is +/// opened would read ten million node headers. Opening reads the table for +/// its checksum, each node and each mapping once; a route walks the nodes +/// once to find its root complex and once for each of the two references it +/// may follow. #[test] -fn the_controllers_come_in_the_tables_order_past_every_other_structure() { - // A GICD, an ITS, a frame, a second ITS. - let mut list = entry(0xC, 24, &[0u8; 22]); - let its = |id: u32, base: u64| { - let mut body = vec![0u8; 2]; - body.extend(id.to_le_bytes()); - body.extend(base.to_le_bytes()); - body.extend([0u8; 4]); - entry(0xF, 20, &body) - }; - list.extend(its(1, 0x0808_0000)); - list.extend(frame(2, 0x0802_0000, 0, 0, 0)); - list.extend(its(9, 0x0809_0000)); - assert_eq!( - crafted_controllers(&list), - [ - Ok(MsiController::Its { id: 1, base: 0x0808_0000 }), - Ok(MsiController::Frame(MsiFrame { id: 2, base: 0x0802_0000, spis: None })), - Ok(MsiController::Its { id: 9, base: 0x0809_0000 }), - ] - ); -} +fn opening_a_table_and_routing_a_requester_each_read_it_a_bounded_number_of_times() { + const ROOTS: u32 = 499; + const ROOT_LEN: u32 = 36 + 40 * 20; + let its_at = 48 + ROOTS * ROOT_LEN; + let mut nodes: Vec> = (0..ROOTS) + .map(|segment| { + let mappings: Vec<[u32; 5]> = (0..40).map(|i| [i * 16, 15, 0x1000 + i * 16, its_at, 0]).collect(); + root_complex(segment, &mappings) + }) + .collect(); + nodes.push(its(7)); + let bytes = table(5, &nodes); + assert_eq!(bytes.len(), its_at as usize + 24); -#[test] -fn a_frame_too_short_for_its_fields_is_no_controller_and_a_halt_ends_the_walk() { - // Twenty-two bytes: the SPI base would be read from the next structure. - let mut list = entry(0xD, 22, &[0xff; 20]); - list.extend(frame(1, 0x0802_0000, 0, 0, 0)); - list.extend(entry(0xD, 0, &[])); - list.extend(frame(2, 0x0803_0000, 0, 0, 0)); - assert_eq!( - crafted_controllers(&list), - [ - Ok(MsiController::Frame(MsiFrame { id: 1, base: 0x0802_0000, spis: None })), - Err(MadtHalt { at: 46, declared: 0, list_len: 72 }), - ] - ); + let head = rsdp(0x1000, 2, 36); + let root = xsdt(&[TABLE_AT]); + let regions: &[(u64, &[u8])] = &[(0x800, &head), (0x1000, &root), (TABLE_AT, &bytes)]; + let reads = Cell::new(0); + let opened = iort(Counted { machine: Machine { regions }, reads: &reads }, 0x800).expect("a table of 500 nodes"); + let open = reads.replace(0); + assert!(open <= 3 * bytes.len(), "opening {} bytes read {open}", bytes.len()); + + // The last mapping of the last root complex, so nothing is found early. + let device = 0x1000 + 39 * 16 + 15; + assert_eq!(opened.route(ROOTS - 1, 39 * 16 + 15), Ok(Route::Untranslated(ItsDevice { its: 7, device }))); + let routed = reads.replace(0); + assert!(routed <= bytes.len() / 8, "routing through {} bytes read {routed}", bytes.len()); + assert_eq!(opened.route(ROOTS, 0), Ok(Route::Unmapped)); } diff --git a/toyos-bootmap/src/aarch64.rs b/toyos-bootmap/src/aarch64.rs index 08b33f7542f..7db15d8a091 100644 --- a/toyos-bootmap/src/aarch64.rs +++ b/toyos-bootmap/src/aarch64.rs @@ -33,7 +33,7 @@ const OUTER_SHAREABLE: u64 = 0b10 << 8; /// The access flag, bit 10: set, so the first access does not fault. const AF: u64 = 1 << 10; /// Privileged execute-never, bit 53. -const PXN: u64 = 1 << 53; +pub const PXN: u64 = 1 << 53; /// Unprivileged execute-never, bit 54. const UXN: u64 = 1 << 54; diff --git a/toyos-its/Cargo.toml b/toyos-its/Cargo.toml index f6c8d3ed685..283165de4c6 100644 --- a/toyos-its/Cargo.toml +++ b/toyos-its/Cargo.toml @@ -1,6 +1,5 @@ # A member of the host workspace (root `Cargo.toml`), like toyos-acpi and -# toyos-smmu: it is pure, it has no dependencies, and its tests run on the -# host. +# toyos-smmu: it is pure and its tests run on the host. # # What lives here is every word a GICv3 ITS and its driver exchange, and the # two tables a redistributor reads an LPI's configuration and pending state @@ -15,3 +14,6 @@ version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" publish = false + +[dependencies] +toyos-phys = { path = "../toyos-phys" } diff --git a/toyos-its/src/command.rs b/toyos-its/src/command.rs index 7dc7d67c2ae..fb16bb3be42 100644 --- a/toyos-its/src/command.rs +++ b/toyos-its/src/command.rs @@ -1,8 +1,10 @@ //! The ITS's commands (§5.3): 32 bytes each, four little-endian //! doublewords, the command number in bits [7:0] of the first. +use toyos_phys::Phys; + use crate::lpi::Lpi; -use crate::{EventBits, Phys, Target}; +use crate::{Collection, Event, EventBits, Target}; /// One command. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -14,14 +16,14 @@ pub enum Command { /// ignored. UnmapDevice { device: u32 }, /// `MAPC` with `V` set (§5.3.9): `collection`'s interrupts go to `target`. - MapCollection { collection: u16, target: Target }, + MapCollection { collection: Collection, target: Target }, /// `MAPTI` (§5.3.12): `device`'s `event` is `lpi`, in `collection`. - MapEvent { device: u32, event: u32, lpi: Lpi, collection: u16 }, + MapEvent { device: u32, event: Event, lpi: Lpi, collection: Collection }, /// `INV` (§5.3.6): reread the configuration of `device`'s `event`'s LPI. - Reconfigure { device: u32, event: u32 }, + Reconfigure { device: u32, event: Event }, /// `DISCARD` (§5.3.4): `device`'s `event` is no interrupt, and any /// pending one is dropped. - Discard { device: u32, event: u32 }, + Discard { device: u32, event: Event }, /// `SYNC` (§5.3.15): every command before it has taken effect at /// `target` before the next is read. Sync(Target), @@ -43,13 +45,13 @@ impl Command { } Self::UnmapDevice { device: id } => [device(0x08, id), 0, 0, 0], // RDbase [51:16] and ICID [15:0] of doubleword 2. - Self::MapCollection { collection, target } => [0x09, 0, V | target.0 << 16 | collection as u64, 0], + Self::MapCollection { collection, target } => [0x09, 0, V | target.0 << 16 | collection.0 as u64, 0], // pINTID [63:32] of doubleword 1; ICID [15:0] of doubleword 2. Self::MapEvent { device: id, event, lpi, collection } => { - [device(0x0A, id), (lpi.intid() as u64) << 32 | event as u64, collection as u64, 0] + [device(0x0A, id), (lpi.intid() as u64) << 32 | event.0 as u64, collection.0 as u64, 0] } - Self::Reconfigure { device: id, event } => [device(0x0C, id), event as u64, 0, 0], - Self::Discard { device: id, event } => [device(0x0F, id), event as u64, 0, 0], + Self::Reconfigure { device: id, event } => [device(0x0C, id), event.0 as u64, 0, 0], + Self::Discard { device: id, event } => [device(0x0F, id), event.0 as u64, 0, 0], Self::Sync(target) => [0x05, 0, target.0 << 16, 0], } } diff --git a/toyos-its/src/lib.rs b/toyos-its/src/lib.rs index a1a88fd0d7e..12607412537 100644 --- a/toyos-its/src/lib.rs +++ b/toyos-its/src/lib.rs @@ -13,7 +13,8 @@ //! Only physical LPIs through flat tables are spelled: no vPE, no indirect //! table. What an ITS reports about itself is a device's word and is refused //! by name where the driver could not act on it; nothing here panics on a -//! register's value. +//! register's value. An event, a collection and an LPI are each a type made +//! from what the ITS, its tables and the distributor say there is of them. //! //! `no_std`, no allocation, no `unsafe`. @@ -23,34 +24,17 @@ pub mod command; pub mod lpi; -/// A physical address the ITS or a redistributor is given, aligned to -/// `1 << ALIGN` bytes and below 2^48: inside every address field written -/// here, whichever page size the table it names is read in. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct Phys(u64); - -impl Phys { - pub const fn new(address: u64) -> Option { - if address >> 48 == 0 && address & ((1 << ALIGN) - 1) == 0 { - Some(Self(address)) - } else { - None - } - } +use toyos_phys::Phys; - pub const fn get(self) -> u64 { - self.0 - } -} - -/// Offsets in the ITS's control frame (§12.18). +/// Offsets in the ITS's control frame (§12.18, Table 12-33). pub const GITS_CTLR: usize = 0x0000; pub const GITS_TYPER: usize = 0x0008; pub const GITS_CBASER: usize = 0x0080; pub const GITS_CWRITER: usize = 0x0088; pub const GITS_CREADR: usize = 0x0090; -/// `GITS_BASER`, n = 0 to 7: one per table the ITS may ask memory for. +/// `GITS_BASER`, n = 0 to 7 (§12.19.1): one per table the ITS may ask +/// memory for. pub const fn gits_baser(n: usize) -> Option { if n < 8 { Some(0x0100 + 8 * n) @@ -59,16 +43,15 @@ pub const fn gits_baser(n: usize) -> Option { } } -/// The translation frame is the second 64 KiB frame, and `GITS_TRANSLATER` -/// is at 0x0040 of it: a function's message is one 32-bit write of its -/// EventID there, and the bus says which device wrote it. +/// The translation frame is the second 64 KiB frame (§12.18), and +/// `GITS_TRANSLATER` is at 0x0040 of it (Table 12-34): a function's message +/// is one 32-bit write of its EventID there, and the bus says which device +/// wrote it. pub const TRANSLATION_FRAME: u64 = 0x1_0000; pub const GITS_TRANSLATER: u64 = 0x0040; -/// `GITS_CTLR` (§12.19.4): `Enabled` [0]; `Quiescent` [31] reads set once a -/// disabled ITS has nothing in progress. +/// `GITS_CTLR.Enabled` [0] (§12.19.4). pub const CTLR_ENABLED: u32 = 1 << 0; -pub const CTLR_QUIESCENT: u32 = 1 << 31; /// What an ITS lacks that the driver needs. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -80,14 +63,20 @@ pub enum Lacks { /// An ITS as its type register describes it (§12.19.13). #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Its { - /// `ID_bits` [12:8] plus one: the widest EventID. - pub event_bits: u8, - /// `Devbits` [17:13] plus one: the widest DeviceID. + /// `Devbits` [17:13] plus one: the widest DeviceID, and so how many + /// entries a flat device table holds. pub device_bits: u8, + /// `ID_bits` [12:8] plus one: the widest EventID. + event_bits: u8, /// `ITT_entry_size` [7:4] plus one: bytes of one translation table entry. - pub itt_entry_bytes: u8, + itt_entry_bytes: u8, /// `HCC` [31:24]: collections the ITS holds without a collection table. - pub collections_held: u8, + held: u8, + /// `CCT` [2]: those count beside a collection table's. + cumulative: bool, + /// The width of an ICID: 16 bits where `CIL` [36] is clear, else + /// `CIDbits` [35:32] plus one. + collection_bits: u8, /// `PTA` [19]: a collection's target is its redistributor's address /// rather than its processor number. physical_targets: bool, @@ -98,10 +87,12 @@ pub const fn probe(typer: u64) -> Result { return Err(Lacks::PhysicalLpis); } Ok(Its { - event_bits: (typer >> 8 & 0x1F) as u8 + 1, device_bits: (typer >> 13 & 0x1F) as u8 + 1, + event_bits: (typer >> 8 & 0x1F) as u8 + 1, itt_entry_bytes: (typer >> 4 & 0xF) as u8 + 1, - collections_held: (typer >> 24) as u8, + held: (typer >> 24) as u8, + cumulative: typer & 1 << 2 != 0, + collection_bits: if typer & 1 << 36 == 0 { 16 } else { (typer >> 32 & 0xF) as u8 + 1 }, physical_targets: typer & 1 << 19 != 0, }) } @@ -111,6 +102,40 @@ pub const fn probe(typer: u64) -> Result { #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct EventBits(u8); +/// An EventID inside the translation table it was made for: +/// [`EventBits::event`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Event(u32); + +impl EventBits { + /// `id` as an event of a device whose table holds this many. + pub const fn event(self, id: u32) -> Option { + if (id as u64) >> self.0 == 0 { + Some(Event(id)) + } else { + None + } + } +} + +/// How many collections an ITS has: made by [`Its::collections`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Collections(u32); + +/// A collection the ITS has, by its ICID: [`Collections::collection`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Collection(u16); + +impl Collections { + pub const fn collection(self, id: u16) -> Option { + if (id as u32) < self.0 { + Some(Collection(id)) + } else { + None + } + } +} + /// The redistributor a collection's interrupts go to, in the form this ITS /// takes it (`RDbase`, §5.3.1): made by [`Its::target`]. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -133,6 +158,17 @@ impl Its { (self.itt_entry_bytes as u64) << events.0 } + /// The collections this ITS has once its collection table holds + /// `in_table` entries, none where no `GITS_BASER` backs one (§5.2.2, + /// §5.3.1): the table's, those the ITS holds itself where it has no + /// table or counts them beside it, and no more than an ICID numbers. + pub const fn collections(&self, in_table: u64) -> Collections { + let held = if in_table == 0 || self.cumulative { self.held as u64 } else { 0 }; + let numbered = 1u64 << self.collection_bits; + let total = held.saturating_add(in_table); + Collections(if total < numbered { total } else { numbered } as u32) + } + /// The redistributor at `frame` whose `GICR_TYPER.Processor_Number` is /// `number`, as this ITS names it. pub const fn target(&self, number: u16, frame: Phys<16>) -> Target { @@ -171,11 +207,19 @@ impl PageSize { } } -/// `GITS_BASER` as read (§12.19.1): the table it backs, the bytes of one -/// entry (`Entry_Size` [52:48] plus one) and the page size it holds — which -/// an ITS may fix, so it is read back after [`baser`] is written. -pub const fn table(baser: u64) -> (Table, u8, PageSize) { - let kind = match baser >> 56 & 0b111 { +/// `GITS_BASER` as read (§12.19.1): the table it backs and the page size +/// it holds — which an ITS may fix, so it is read back after +/// [`Backing::baser`] is written. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Backing { + pub table: Table, + pub page: PageSize, + /// `Entry_Size` [52:48] plus one: the bytes of one entry. + entry_bytes: u8, +} + +pub const fn table(baser: u64) -> Backing { + let table = match baser >> 56 & 0b111 { 0b000 => Table::Unimplemented, 0b001 => Table::Devices, 0b100 => Table::Collections, @@ -187,7 +231,7 @@ pub const fn table(baser: u64) -> (Table, u8, PageSize) { // `0b11` is reserved and treated as `0b10`. _ => PageSize::K64, }; - (kind, (baser >> 48 & 0x1F) as u8 + 1, page) + Backing { table, page, entry_bytes: (baser >> 48 & 0x1F) as u8 + 1 } } /// `InnerCache` `0b111`, read-allocate write-allocate write-back, with @@ -196,25 +240,46 @@ pub const fn table(baser: u64) -> (Table, u8, PageSize) { const CACHED: u64 = 0b111 << 59 | 0b01 << 10; const VALID: u64 = 1 << 63; -/// `GITS_BASER` for a flat table of `pages` pages of `page` at `at`: -/// `Valid` [63], `Indirect` [62] clear, `Physical_Address` [47:12], -/// `Page_Size` [9:8], `Size` [7:0] the pages minus one. `None` for no page or -/// more than the 256 the field counts. -pub const fn baser(at: Phys<16>, page: PageSize, pages: u16) -> Option { - if pages == 0 || pages > 256 { - return None; +impl Backing { + /// The entries a flat table of `pages` of its pages holds. + pub const fn entries(self, pages: u16) -> u64 { + pages as u64 * self.page.bytes() / self.entry_bytes as u64 + } + + /// The fewest pages that hold `entries`. `None` for none, or more than + /// the 256 `Size` counts. + pub const fn pages(self, entries: u64) -> Option { + let Some(bytes) = entries.checked_mul(self.entry_bytes as u64) else { + return None; + }; + let pages = bytes.div_ceil(self.page.bytes()); + if pages >= 1 && pages <= 256 { + Some(pages as u16) + } else { + None + } + } + + /// `GITS_BASER` for a flat table of `pages` of its pages at `at`: + /// `Valid` [63], `Indirect` [62] clear, `Physical_Address` [47:12], + /// `Page_Size` [9:8], `Size` [7:0] the pages minus one. `None` for no + /// page or more than the 256 the field counts. + pub const fn baser(self, at: Phys<16>, pages: u16) -> Option { + if pages == 0 || pages > 256 { + return None; + } + let size = match self.page { + PageSize::K4 => 0b00, + PageSize::K16 => 0b01, + PageSize::K64 => 0b10, + }; + Some(VALID | CACHED | at.get() | size << 8 | (pages - 1) as u64) } - let size = match page { - PageSize::K4 => 0b00, - PageSize::K16 => 0b01, - PageSize::K64 => 0b10, - }; - Some(VALID | CACHED | at.get() | size << 8 | (pages - 1) as u64) } /// The ITS's command queue: `pages` 4 KiB pages of 32-byte commands /// (§5.2.8). `GITS_CWRITER` and `GITS_CREADR` each hold a byte offset into -/// it, `Offset` [19:5]. +/// it, `Offset` [19:5], which is all of either that names a command. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct CommandQueue { bytes: u32, @@ -246,19 +311,20 @@ impl CommandQueue { (register & 0xF_FFE0) as u32 % self.bytes } - /// The offset after `offset`, wrapping at the queue's end. - pub const fn after(self, offset: u32) -> u32 { - (self.offset(offset as u64) + 32) % self.bytes + /// What `GITS_CWRITER` is written with once the command at its offset is + /// in memory: the offset after, wrapping at the queue's end. + pub const fn after(self, cwriter: u64) -> u64 { + ((self.offset(cwriter) + 32) % self.bytes) as u64 } - /// The ITS has read every command: the two offsets are equal. - pub const fn is_empty(self, writer: u32, reader: u32) -> bool { - writer == reader + /// The ITS has read every command: the two registers' offsets are equal. + pub const fn is_empty(self, cwriter: u64, creadr: u64) -> bool { + self.offset(cwriter) == self.offset(creadr) } - /// One more command would make the writer equal the reader, which reads - /// as empty: the queue holds one command fewer than its bytes do. - pub const fn is_full(self, writer: u32, reader: u32) -> bool { - self.after(writer) == reader + /// One more command would make the writer's offset the reader's, which + /// reads as empty: the queue holds one command fewer than its bytes do. + pub const fn is_full(self, cwriter: u64, creadr: u64) -> bool { + self.after(cwriter) == self.offset(creadr) as u64 } } diff --git a/toyos-its/src/lpi.rs b/toyos-its/src/lpi.rs index e252f93535b..074c2346308 100644 --- a/toyos-its/src/lpi.rs +++ b/toyos-its/src/lpi.rs @@ -2,19 +2,17 @@ //! reads and the pending table each one owns, and the redistributor registers //! that name them. -use crate::Phys; +use toyos_phys::Phys; /// The first LPI's INTID (§2.2): everything below is an SGI, a PPI, an SPI /// or special. pub const FIRST: u32 = 8192; /// Offsets in a redistributor's `RD_base` frame (§12.10, Table 12-27). -pub const GICR_CTLR: usize = 0x0000; -pub const GICR_TYPER: usize = 0x0008; pub const GICR_PROPBASER: usize = 0x0070; pub const GICR_PENDBASER: usize = 0x0078; -/// `GICR_CTLR.EnableLPIs` [0] (§12.11): from here the redistributor reads +/// `GICR_CTLR.EnableLPIs` [0] (§12.11.2): from here the redistributor reads /// both tables, and neither base register may be written. pub const CTLR_ENABLE_LPIS: u32 = 1 << 0; @@ -27,7 +25,8 @@ pub const fn processor_number(typer: u64) -> u16 { (typer >> 8) as u16 } -/// An LPI's INTID, inside the space it was made in: [`Space::lpi`]. +/// An LPI's INTID, one the distributor and its redistributor both take: +/// [`Space::lpi`]. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Lpi(u32); @@ -41,12 +40,6 @@ impl Lpi { pub const fn configuration_index(self) -> usize { (self.0 - FIRST) as usize } - - /// Where its bit is in a pending table, which is indexed by the INTID - /// itself: the byte, and the bit in it (§5.1.2). - pub const fn pending_bit(self) -> (usize, u8) { - ((self.0 / 8) as usize, (self.0 % 8) as u8) - } } /// One LPI's byte in the configuration table (Table 5-1): the priority's @@ -55,10 +48,12 @@ pub const fn configuration(priority: u8, enabled: bool) -> u8 { priority & 0xFC | 0b10 | enabled as u8 } -/// The INTIDs of `id_bits` bits the tables are laid out for. +/// The two tables laid out for INTIDs of `id_bits` bits, which the +/// distributor has. #[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct Space { +pub struct Layout { id_bits: u8, + lpis: u32, } /// `InnerCache` [9:7] `0b111`, read-allocate write-allocate write-back, with @@ -66,33 +61,31 @@ pub struct Space { /// inner shareable. Both base registers hold them at the same bits. const CACHED: u64 = 0b111 << 7 | 0b01 << 10; -impl Space { - /// `None` below 14 bits, which reach no LPI, and above the 24 that keep - /// the configuration table within 16 MiB. - pub const fn new(id_bits: u8) -> Option { - if id_bits >= 14 && id_bits <= 24 { - Some(Self { id_bits }) - } else { - None +impl Layout { + /// `GICD_TYPER` (§12.9.38) judged: `None` where the distributor takes no + /// LPI (`LPIS` [17] clear), and for `id_bits` below 14, which reach no + /// LPI, or above its INTIDs' (`IDbits` [23:19] plus one). Its `num_LPIs` + /// [15:11], where not zero, counts the LPIs there are from 8192 on. + pub const fn new(gicd_typer: u32, id_bits: u8) -> Option { + let widest = (gicd_typer >> 19 & 0x1F) as u8 + 1; + if gicd_typer & 1 << 17 == 0 || id_bits < 14 || id_bits > widest { + return None; } + let numbered = (1u64 << id_bits) - FIRST as u64; + let counted = match gicd_typer >> 11 & 0x1F { + 0 => numbered, + count => 2u64 << count, + }; + Some(Self { id_bits, lpis: if counted < numbered { counted } else { numbered } as u32 }) } - /// `intid` as an LPI of this space. - pub const fn lpi(self, intid: u32) -> Option { - if intid >= FIRST && intid >> self.id_bits == 0 { - Some(Lpi(intid)) - } else { - None - } - } - - /// One byte for every LPI. - pub const fn configuration_bytes(self) -> usize { - (1 << self.id_bits) - FIRST as usize + /// One byte for every INTID from 8192 that `id_bits` bits number. + pub const fn configuration_bytes(self) -> u64 { + (1 << self.id_bits) - FIRST as u64 } /// One bit for every INTID, the first 8192 included. - pub const fn pending_bytes(self) -> usize { + pub const fn pending_bytes(self) -> u64 { (1 << self.id_bits) / 8 } @@ -101,6 +94,35 @@ impl Space { pub const fn propbaser(self, table: Phys<12>) -> u64 { CACHED | table.get() | (self.id_bits - 1) as u64 } + + /// The LPIs of a redistributor that reads `propbaser` back: `None` where + /// its `IDbits` is not what [`Layout::propbaser`] wrote, which a + /// redistributor that holds the register as its own answers. + pub const fn space(self, propbaser: u64) -> Option { + if (propbaser & 0x1F) as u8 == self.id_bits - 1 { + Some(Space { lpis: self.lpis }) + } else { + None + } + } +} + +/// The LPIs there are: those a [`Layout`]'s tables hold, that the +/// distributor counts, in a redistributor that took the layout. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Space { + lpis: u32, +} + +impl Space { + /// `intid` as an LPI of this space. + pub const fn lpi(self, intid: u32) -> Option { + if intid >= FIRST && intid - FIRST < self.lpis { + Some(Lpi(intid)) + } else { + None + } + } } /// `GICR_PENDBASER` (§12.11.32) for the pending table at `table`, which is diff --git a/toyos-its/tests/encodings.rs b/toyos-its/tests/encodings.rs index 3f2e5a77985..cf913d0cdf4 100644 --- a/toyos-its/tests/encodings.rs +++ b/toyos-its/tests/encodings.rs @@ -4,8 +4,9 @@ //! the wrong place and a stray bit are both a difference. use toyos_its::command::Command; -use toyos_its::lpi::{self, Space}; -use toyos_its::{baser, gits_baser, probe, table, CommandQueue, Its, Lacks, PageSize, Phys, Table}; +use toyos_its::lpi::{self, Layout, Space}; +use toyos_its::{gits_baser, probe, table, CommandQueue, Its, Lacks, PageSize, Table}; +use toyos_phys::Phys; /// `N` doublewords holding `value` in bits `[msb:lsb]` for every triple, and /// zero everywhere else. @@ -39,12 +40,25 @@ fn command(fields: &[(u32, u32, u32, u64)]) -> [u64; 4] { /// translation entries, holding 3 collections itself, which names a /// redistributor by `pta`: Physical [0], ITT_entry_size [7:4], ID_bits /// [12:8], Devbits [17:13], PTA [19], HCC [31:24]. -fn typer(pta: u64) -> u64 { - word(&[(0, 0, 1), (7, 4, 7), (12, 8, 15), (17, 13, 15), (19, 19, pta), (31, 24, 3)]) +const TYPER: &[(u32, u32, u64)] = &[(0, 0, 1), (7, 4, 7), (12, 8, 15), (17, 13, 15), (31, 24, 3)]; + +/// `TYPER` with `change`'s fields beside it. +fn typer(change: &[(u32, u32, u64)]) -> u64 { + word(&[TYPER, change].concat()) } fn its(pta: u64) -> Its { - probe(typer(pta)).expect("an ITS of physical LPIs") + probe(typer(&[(19, 19, pta)])).expect("an ITS of physical LPIs") +} + +/// `GICD_TYPER` of a distributor that takes LPIs under 16-bit INTIDs: LPIS +/// [17], IDbits [23:19] the bits minus one, num_LPIs [15:11] zero. +const GICD_TYPER: &[(u32, u32, u64)] = &[(17, 17, 1), (23, 19, 15)]; + +/// The LPIs of a redistributor that took a layout of `bits` bits under `GICD_TYPER`. +fn space(bits: u8) -> Space { + let layout = Layout::new(word(GICD_TYPER) as u32, bits).expect("bits the distributor has"); + layout.space(layout.propbaser(Phys::new(0x4000_0000).unwrap())).expect("the register as it was written") } // --- §12.18, §12.19: the registers ------------------------------------------ @@ -52,50 +66,58 @@ fn its(pta: u64) -> Its { #[test] fn the_type_register_says_how_wide_an_its_is() { let its = its(0); - assert_eq!((its.event_bits, its.device_bits, its.itt_entry_bytes, its.collections_held), (16, 16, 8, 3)); - // Each field at its largest, alone: every one is its value plus one but HCC. + assert_eq!(its.device_bits, 16); + // ID_bits: 16 bits of EventID and no more; ITT_entry_size: 8 bytes an entry. + assert!(its.events(16).is_some() && its.events(17).is_none()); + assert_eq!(its.events(1).map(|events| its.itt_bytes(events)), Some(16)); + // Each field at its largest, alone: every one is its value plus one. let wide = |field: (u32, u32, u64)| probe(word(&[(0, 0, 1), field])).unwrap(); - assert_eq!(wide((12, 8, 31)).event_bits, 32); assert_eq!(wide((17, 13, 31)).device_bits, 32); - assert_eq!(wide((7, 4, 15)).itt_entry_bytes, 16); - assert_eq!(wide((31, 24, 255)).collections_held, 255); + assert!(wide((12, 8, 31)).events(32).is_some()); + let entry = wide((7, 4, 15)); + assert_eq!(entry.events(1).map(|events| entry.itt_bytes(events)), Some(32)); let least = probe(1).unwrap(); - assert_eq!((least.event_bits, least.device_bits, least.itt_entry_bytes, least.collections_held), (1, 1, 1, 0)); + assert_eq!(least.device_bits, 1); + assert!(least.events(1).is_some() && least.events(2).is_none()); + assert_eq!(least.events(1).map(|events| least.itt_bytes(events)), Some(2)); } #[test] fn an_its_without_physical_lpis_is_refused() { - assert_eq!(probe(typer(0) & !1), Err(Lacks::PhysicalLpis)); + assert_eq!(probe(typer(&[]) & !1), Err(Lacks::PhysicalLpis)); assert_eq!(probe(!1), Err(Lacks::PhysicalLpis)); assert!(probe(u64::MAX).is_ok()); } -#[test] -fn the_registers_sit_where_the_its_map_has_them() { - assert_eq!( - (toyos_its::GITS_CTLR, toyos_its::GITS_TYPER, toyos_its::GITS_CBASER, toyos_its::GITS_CWRITER, toyos_its::GITS_CREADR), - (0x0000, 0x0008, 0x0080, 0x0088, 0x0090) - ); - // GITS_BASER is 0x0100 to 0x0138. - assert_eq!((gits_baser(0), gits_baser(7), gits_baser(8)), (Some(0x0100), Some(0x0138), None)); - // GITS_TRANSLATER is 0x0040 of the second 64 KiB frame. - assert_eq!(toyos_its::TRANSLATION_FRAME + toyos_its::GITS_TRANSLATER, 0x1_0040); - assert_eq!(u64::from(toyos_its::CTLR_ENABLED | toyos_its::CTLR_QUIESCENT), word(&[(0, 0, 1), (31, 31, 1)])); - assert_eq!(toyos_its::CREADR_STALLED, word(&[(0, 0, 1)])); -} - #[test] fn a_table_register_says_what_it_backs_and_how_it_is_read() { // Type [58:56], Entry_Size [52:48] the bytes minus one, Page_Size [9:8]. - let read = |kind: u64, entry: u64, page: u64| table(word(&[(58, 56, kind), (52, 48, entry), (9, 8, page)])); - assert_eq!(read(0b000, 0, 0b00), (Table::Unimplemented, 1, PageSize::K4)); - assert_eq!(read(0b001, 7, 0b10), (Table::Devices, 8, PageSize::K64)); - assert_eq!(read(0b100, 15, 0b01), (Table::Collections, 16, PageSize::K16)); - assert_eq!(read(0b010, 31, 0b11), (Table::Other(0b010), 32, PageSize::K64)); + let read = |kind: u64, entry: u64, page: u64| { + let backing = table(word(&[(58, 56, kind), (52, 48, entry), (9, 8, page)])); + (backing.table, backing.page, backing.entries(1)) + }; + assert_eq!(read(0b000, 0, 0b00), (Table::Unimplemented, PageSize::K4, 4096)); + assert_eq!(read(0b001, 7, 0b10), (Table::Devices, PageSize::K64, 65536 / 8)); + assert_eq!(read(0b100, 15, 0b01), (Table::Collections, PageSize::K16, 16384 / 16)); + assert_eq!(read(0b010, 31, 0b11), (Table::Other(0b010), PageSize::K64, 65536 / 32)); assert_eq!(read(0b111, 0, 0b00).0, Table::Other(0b111)); assert_eq!((PageSize::K4.bytes(), PageSize::K16.bytes(), PageSize::K64.bytes()), (4096, 16384, 65536)); } +#[test] +fn a_table_is_sized_in_its_own_pages_for_the_entries_it_must_hold() { + // 8 bytes an entry in 64 KiB pages: 8192 to the page. + let devices = table(word(&[(58, 56, 0b001), (52, 48, 7), (9, 8, 0b10)])); + assert_eq!(devices.pages(0), None); + assert_eq!((devices.pages(1), devices.pages(8192), devices.pages(8193)), (Some(1), Some(1), Some(2))); + // A flat table for every DeviceID of 16 bits is eight pages, and of 21 bits the field's 256. + assert_eq!(devices.pages(1 << 16), Some(8)); + assert_eq!(devices.pages(1 << 21), Some(256)); + assert_eq!(devices.pages((1 << 21) + 1), None); + assert_eq!(devices.pages(u64::MAX), None); + assert_eq!((devices.entries(8), devices.entries(256)), (1 << 16, 1 << 21)); +} + #[test] fn a_table_is_given_flat_valid_and_counted_in_its_own_pages() { let at = Phys::<16>::new(0xffff_ffff_0000).expect("64 KiB aligned, below 2^48"); @@ -111,20 +133,12 @@ fn a_table_is_given_flat_valid_and_counted_in_its_own_pages() { (7, 0, pages - 1), // Size ])) }; - assert_eq!(baser(at, PageSize::K4, 1), written(0b00, 1)); - assert_eq!(baser(at, PageSize::K16, 8), written(0b01, 8)); - assert_eq!(baser(at, PageSize::K64, 256), written(0b10, 256)); - assert_eq!(baser(at, PageSize::K64, 0), None); - assert_eq!(baser(at, PageSize::K64, 257), None); -} - -#[test] -fn an_address_off_its_alignment_or_past_48_bits_is_no_address() { - assert_eq!(Phys::<8>::new(0x100).map(Phys::get), Some(0x100)); - assert_eq!(Phys::<8>::new(0x80), None); - assert_eq!(Phys::<16>::new(0x8000), None); - assert_eq!(Phys::<16>::new(1 << 48), None); - assert_eq!(Phys::<16>::new((1 << 48) - 0x1_0000).map(Phys::get), Some((1 << 48) - 0x1_0000)); + let paged = |page: u64| table(word(&[(9, 8, page)])); + assert_eq!(paged(0b00).baser(at, 1), written(0b00, 1)); + assert_eq!(paged(0b01).baser(at, 8), written(0b01, 8)); + assert_eq!(paged(0b10).baser(at, 256), written(0b10, 256)); + assert_eq!(paged(0b10).baser(at, 0), None); + assert_eq!(paged(0b10).baser(at, 257), None); } // --- §5.2.8: the command queue ---------------------------------------------- @@ -142,7 +156,7 @@ fn the_command_queue_is_counted_in_4_kib_pages_and_holds_one_command_fewer() { // One page is 128 commands. Against a count kept beside it, over three laps. let queue = CommandQueue::new(1).unwrap(); - let (mut writer, mut reader, mut held) = (0u32, 0u32, 0u32); + let (mut writer, mut reader, mut held) = (0u64, 0u64, 0u32); for step in 0..3 * 128 * 2 { assert_eq!(queue.is_empty(writer, reader), held == 0, "step {step}"); assert_eq!(queue.is_full(writer, reader), held == 127, "step {step}"); @@ -157,7 +171,7 @@ fn the_command_queue_is_counted_in_4_kib_pages_and_holds_one_command_fewer() { } assert_eq!(queue.after(4096 - 32), 0); // An offset is the ITS's word: past the queue, it is still an offset in it. - assert_eq!(queue.after(u32::MAX), 0); + assert_eq!(queue.after(u64::MAX), 0); assert_eq!(queue.after(4096 + 64), 96); } @@ -165,9 +179,18 @@ fn the_command_queue_is_counted_in_4_kib_pages_and_holds_one_command_fewer() { fn an_offset_register_names_a_command_by_its_offset_field_alone() { let queue = CommandQueue::new(2).unwrap(); // Offset [19:5]; Stalled or Retry [0]; the rest reserved. - assert_eq!(queue.offset(word(&[(19, 5, 0x40), (0, 0, 1), (4, 1, 0xf), (63, 20, 0xfff_ffff_ffff)])), 0x40 << 5); + let beside = |offset: u64| word(&[(19, 5, offset), (0, 0, 1), (4, 1, 0xf), (63, 20, 0xfff_ffff_ffff)]); + assert_eq!(queue.offset(beside(0x40)), 0x40 << 5); // An offset past the queue is a device's word: it still names a command in the queue. assert!(queue.offset(word(&[(19, 5, 0x7fff)])) < 8192); + // `GITS_CREADR` with `Stalled` set and every reserved bit beside its offset is the same offset. + let clean = |offset: u64| word(&[(19, 5, offset)]); + assert!(queue.is_empty(clean(0x40), beside(0x40))); + assert!(!queue.is_empty(clean(0x41), beside(0x40))); + assert!(queue.is_full(clean(0x3f), beside(0x40))); + assert!(!queue.is_full(clean(0x40), beside(0x40))); + assert_eq!(queue.after(beside(0x40)), clean(0x41)); + assert_eq!(toyos_its::CREADR_STALLED, word(&[(0, 0, 1)])); } // --- §5.3: commands --------------------------------------------------------- @@ -176,11 +199,13 @@ fn an_offset_register_names_a_command_by_its_offset_field_alone() { fn each_command_holds_its_number_and_parameters_where_its_figure_has_them() { let its = its(0); let events = its.events(5).expect("32 events of an ITS of 16-bit EventIDs"); + let event = events.event(0x1f).expect("the last of 32 events"); let itt = Phys::<8>::new(0xffff_ffff_ff00).expect("256-byte aligned"); let target = its.target(0x1234, Phys::new(0x080a_0000).unwrap()); - let lpi = Space::new(16).unwrap().lpi(0xabcd).expect("an LPI"); + let collection = its.collections(0xc012).collection(0xc011).expect("a collection the table holds"); + let lpi = space(16).lpi(0xabcd).expect("an LPI"); - // MAPD: DeviceID [63:32]; Size [4:0] of DW1, the bits minus one; V [63] and ITT_addr [51:8] of DW2. + // MAPD (Figure 5-13): DeviceID [63:32]; Size [4:0] of DW1, the bits minus one; V [63] and ITT_addr [51:8] of DW2. assert_eq!( Command::MapDevice { device: 0xdead_beef, table: itt, events }.words(), command(&[(0, 7, 0, 0x08), (0, 63, 32, 0xdead_beef), (1, 4, 0, 4), (2, 63, 63, 1), (2, 51, 8, 0xffff_ffff_ff00 >> 8)]) @@ -189,29 +214,35 @@ fn each_command_holds_its_number_and_parameters_where_its_figure_has_them() { Command::UnmapDevice { device: 0xdead_beef }.words(), command(&[(0, 7, 0, 0x08), (0, 63, 32, 0xdead_beef), (2, 63, 63, 0)]) ); - // MAPC: V [63], RDbase [51:16] and ICID [15:0] of DW2. + // MAPC (Figure 5-12): V [63], RDbase [51:16] and ICID [15:0] of DW2. assert_eq!( - Command::MapCollection { collection: 0xc011, target }.words(), + Command::MapCollection { collection, target }.words(), command(&[(0, 7, 0, 0x09), (2, 63, 63, 1), (2, 51, 16, 0x1234), (2, 15, 0, 0xc011)]) ); - // MAPTI: DeviceID [63:32]; pINTID [63:32] and EventID [31:0] of DW1; ICID [15:0] of DW2. + // MAPTI (Figure 5-15): DeviceID [63:32]; pINTID [63:32] and EventID [31:0] of DW1; ICID [15:0] of DW2. assert_eq!( - Command::MapEvent { device: 0xdead_beef, event: 0x1f, lpi, collection: 0xc011 }.words(), + Command::MapEvent { device: 0xdead_beef, event, lpi, collection }.words(), command(&[(0, 7, 0, 0x0A), (0, 63, 32, 0xdead_beef), (1, 63, 32, 0xabcd), (1, 31, 0, 0x1f), (2, 15, 0, 0xc011)]) ); - // INV and DISCARD: DeviceID [63:32]; EventID [31:0] of DW1. - assert_eq!( - Command::Reconfigure { device: 0xdead_beef, event: 0xffff_ffff }.words(), - command(&[(0, 7, 0, 0x0C), (0, 63, 32, 0xdead_beef), (1, 31, 0, 0xffff_ffff)]) - ); - assert_eq!( - Command::Discard { device: 0xdead_beef, event: 0xffff_ffff }.words(), - command(&[(0, 7, 0, 0x0F), (0, 63, 32, 0xdead_beef), (1, 31, 0, 0xffff_ffff)]) - ); - // SYNC: RDbase [51:16] of DW2. + // SYNC (Figure 5-18): RDbase [51:16] of DW2. assert_eq!(Command::Sync(target).words(), command(&[(0, 7, 0, 0x05), (2, 51, 16, 0x1234)])); } +/// `DISCARD` (Figure 5-7) and `INV` (Figure 5-9), one layout under two +/// numbers: doubleword 0 holds DeviceID [63:32], RES0 [31:8] and the command +/// number [7:0]; doubleword 1 holds RES0 [63:32] and EventID [31:0]; +/// doublewords 2 and 3 are RES0. +#[test] +fn inv_and_discard_hold_a_device_and_an_event_and_nothing_else() { + let event = its(0).events(16).unwrap().event(0xa5c3).unwrap(); + let figure = |number: u64| command(&[(0, 63, 32, 0xdead_beef), (0, 31, 8, 0), (0, 7, 0, number), (1, 63, 32, 0), (1, 31, 0, 0xa5c3)]); + assert_eq!(Command::Discard { device: 0xdead_beef, event }.words(), figure(0x0F)); + assert_eq!(Command::Reconfigure { device: 0xdead_beef, event }.words(), figure(0x0C)); + // The widest event there is fills its field and reaches no other. + let widest = probe(word(&[(0, 0, 1), (12, 8, 31)])).unwrap().events(32).unwrap().event(u32::MAX).unwrap(); + assert_eq!(Command::Discard { device: 0, event: widest }.words(), [0x0F, 0xffff_ffff, 0, 0]); + assert_eq!(Command::Reconfigure { device: u32::MAX, event: widest }.words(), [0xffff_ffff_0000_000c, 0xffff_ffff, 0, 0]); +} #[test] fn a_redistributor_is_named_as_the_its_asks_by_number_or_by_address() { let frame = Phys::<16>::new(0xffff_fffe_0000).unwrap(); @@ -233,32 +264,126 @@ fn a_devices_events_are_at_least_two_and_no_more_than_the_its_has_bits_for() { assert_eq!((size(1), size(16)), (0, 15)); } +#[test] +fn an_event_is_one_its_devices_table_holds() { + let its = its(0); + let events = its.events(5).unwrap(); + assert!(events.event(0).is_some() && events.event(31).is_some()); + assert_eq!(events.event(32), None); + assert_eq!(events.event(u32::MAX), None); + // The widest table there is holds every EventID. + let wide = probe(word(&[(0, 0, 1), (12, 8, 31)])).unwrap(); + assert!(wide.events(32).unwrap().event(u32::MAX).is_some()); +} + +/// §5.2.2 and §5.3.1: the collections are the table's, with those the ITS +/// holds itself (`HCC`) where it has no table or `CCT` [2] counts both, and +/// an ICID is 16 bits or, under `CIL` [36], `CIDbits` [35:32] plus one. +#[test] +fn a_collection_is_one_the_its_holds_or_its_table_does() { + let last = |its: Its, in_table: u64, id: u16| its.collections(in_table).collection(id).is_some(); + // No table: the three the ITS holds. + let held = probe(typer(&[])).unwrap(); + assert!(last(held, 0, 2) && !last(held, 0, 3)); + // A table without CCT: the table's alone. + assert!(last(held, 8, 7) && !last(held, 8, 8)); + // With CCT: both. + let cumulative = probe(typer(&[(2, 2, 1)])).unwrap(); + assert!(last(cumulative, 8, 10) && !last(cumulative, 8, 11)); + assert!(last(cumulative, 0, 2) && !last(cumulative, 0, 3)); + // No more than an ICID numbers: all 2^16 of 16 bits, and 2^4 under CIL with CIDbits 3. + assert!(last(held, 1 << 20, u16::MAX)); + let narrow = probe(typer(&[(36, 36, 1), (35, 32, 3)])).unwrap(); + assert!(last(narrow, 1 << 20, 15) && !last(narrow, 1 << 20, 16)); + // CIDbits says nothing where CIL is clear. + let unlimited = probe(typer(&[(35, 32, 3)])).unwrap(); + assert!(last(unlimited, 1 << 20, 16)); + // An ITS that holds none and was given no table has none. + let none = probe(1).unwrap(); + assert!(!last(none, 0, 0)); + // Every bit set and a table of every entry: still no more than a 16-bit ICID numbers. + assert!(last(probe(u64::MAX).unwrap(), u64::MAX, u16::MAX)); +} + // --- §5.1: LPIs -------------------------------------------------------------- +/// `GICD_TYPER` (§12.9.38): LPIS [17], IDbits [23:19] the INTID bits minus +/// one, num_LPIs [15:11]. +#[test] +fn an_lpi_layout_is_inside_the_distributors_intids() { + let typer = |fields: &[(u32, u32, u64)]| word(fields) as u32; + // No LPIs at all, whatever the bits. + assert_eq!(Layout::new(typer(&[(23, 19, 15)]), 14), None); + // 16 bits of INTID: 14, the fewest that reach an LPI, to 16. + let sixteen = typer(GICD_TYPER); + assert_eq!(Layout::new(sixteen, 13), None); + assert!(Layout::new(sixteen, 14).is_some() && Layout::new(sixteen, 16).is_some()); + assert_eq!(Layout::new(sixteen, 17), None); + // The field at its largest is 32 bits, and every other field says nothing of it. + let widest = typer(&[(17, 17, 1), (23, 19, 31), (31, 24, 0xff), (16, 16, 1), (10, 0, 0x7ff)]); + assert!(Layout::new(widest, 32).is_some()); + assert_eq!(Layout::new(widest, 33), None); + assert_eq!(Layout::new(u32::MAX, u8::MAX), None); +} + #[test] -fn an_lpi_is_an_intid_from_8192_inside_its_space() { - let space = Space::new(14).expect("the fewest bits that reach an LPI"); - assert_eq!(space.lpi(8191), None); - assert_eq!(space.lpi(8192).map(|lpi| lpi.intid()), Some(8192)); - assert_eq!(space.lpi(16383).map(|lpi| lpi.intid()), Some(16383)); +fn an_lpi_is_an_intid_from_8192_that_the_tables_and_the_distributor_both_count() { + let fourteen = space(14); + assert_eq!(fourteen.lpi(8191), None); + assert_eq!(fourteen.lpi(8192).map(|lpi| lpi.intid()), Some(8192)); + assert_eq!(fourteen.lpi(16383).map(|lpi| lpi.intid()), Some(16383)); + assert_eq!(fourteen.lpi(16384), None); + assert_eq!(fourteen.lpi(u32::MAX), None); + assert_eq!(fourteen.lpi(0), None); + + // num_LPIs [15:11] of 9: 2^10 LPIs, 8192 to 9215, under tables laid out for 16 bits. + let counted = word(&[GICD_TYPER, &[(15, 11, 9)]].concat()) as u32; + let layout = Layout::new(counted, 16).unwrap(); + let space = layout.space(layout.propbaser(Phys::new(0).unwrap())).unwrap(); + assert!(space.lpi(9215).is_some()); + assert_eq!(space.lpi(9216), None); + // And it never counts more than the tables' bits number: 2^16 of them under 14 bits is 8192. + let over = word(&[GICD_TYPER, &[(15, 11, 15)]].concat()) as u32; + let layout = Layout::new(over, 14).unwrap(); + let space = layout.space(layout.propbaser(Phys::new(0).unwrap())).unwrap(); + assert!(space.lpi(16383).is_some()); assert_eq!(space.lpi(16384), None); - assert_eq!(space.lpi(u32::MAX), None); - assert!(Space::new(13).is_none() && Space::new(24).is_some() && Space::new(25).is_none()); + + // 32 bits: every INTID from 8192 up. + let widest = word(&[(17, 17, 1), (23, 19, 31)]) as u32; + let layout = Layout::new(widest, 32).unwrap(); + let space = layout.space(layout.propbaser(Phys::new(0).unwrap())).unwrap(); + assert!(space.lpi(u32::MAX).is_some() && space.lpi(8191).is_none()); + assert_eq!((layout.configuration_bytes(), layout.pending_bytes()), ((1 << 32) - 8192, 1 << 29)); +} + +/// §12.11.33: a redistributor may hold `GICR_PROPBASER` as its own, and its +/// `IDbits` [4:0] bounds the LPIs whatever was written. +#[test] +fn a_redistributor_that_reads_other_bits_back_has_no_lpis_of_the_layout() { + let layout = Layout::new(word(GICD_TYPER) as u32, 16).unwrap(); + let written = layout.propbaser(Phys::new(0x4000_0000).unwrap()); + assert!(layout.space(written).is_some()); + // Its cacheability and address may read back otherwise; its IDbits may not. + assert!(layout.space(written & 0x1f).is_some()); + assert!(layout.space(written | !0x1f).is_some()); + for other in [0u64, 13, 14, 16, 31] { + assert_eq!(layout.space(written & !0x1f | other), None, "IDbits {other}"); + } } #[test] fn the_configuration_table_starts_at_intid_8192_and_the_pending_table_at_zero() { - let space = Space::new(16).unwrap(); + let layout = Layout::new(word(GICD_TYPER) as u32, 16).unwrap(); + let space = space(16); // "(base address + (N - 8192))", one byte an LPI. assert_eq!(space.lpi(8192).unwrap().configuration_index(), 0); assert_eq!(space.lpi(65535).unwrap().configuration_index(), 65535 - 8192); - assert_eq!(space.configuration_bytes(), 65536 - 8192); - // "(base address + (N / 8))", bit "(N MOD 8)": one bit an INTID, the first 1 KiB below every LPI. - assert_eq!(space.lpi(8192).unwrap().pending_bit(), (1024, 0)); - assert_eq!(space.lpi(8199).unwrap().pending_bit(), (1024, 7)); - assert_eq!(space.lpi(65535).unwrap().pending_bit(), (8191, 7)); - assert_eq!(space.pending_bytes(), 8192); - assert_eq!((Space::new(14).unwrap().configuration_bytes(), Space::new(14).unwrap().pending_bytes()), (8192, 2048)); + assert_eq!(layout.configuration_bytes(), 65536 - 8192); + // One bit an INTID, the first 1 KiB below every LPI. + assert_eq!(layout.pending_bytes(), 8192); + let fourteen = Layout::new(word(GICD_TYPER) as u32, 14).unwrap(); + assert_eq!((fourteen.configuration_bytes(), fourteen.pending_bytes()), (8192, 2048)); } #[test] @@ -274,21 +399,63 @@ fn an_lpis_configuration_is_its_priority_a_set_bit_and_its_enable() { #[test] fn the_redistributors_base_registers_name_the_two_tables() { - let space = Space::new(16).unwrap(); + let layout = Layout::new(word(GICD_TYPER) as u32, 16).unwrap(); // OuterCache [58:56], Physical_Address [51:12], Shareability [11:10], InnerCache [9:7], IDbits [4:0] the bits minus one. assert_eq!( - space.propbaser(Phys::new(0xffff_ffff_f000).unwrap()), + layout.propbaser(Phys::new(0xffff_ffff_f000).unwrap()), word(&[(58, 56, 0), (51, 12, 0xffff_ffff_f000 >> 12), (11, 10, 0b01), (9, 7, 0b111), (4, 0, 15)]) ); - assert_eq!(Space::new(14).unwrap().propbaser(Phys::new(0).unwrap()) & 0x1f, 13); + assert_eq!(Layout::new(word(GICD_TYPER) as u32, 14).unwrap().propbaser(Phys::new(0).unwrap()) & 0x1f, 13); // PTZ [62], OuterCache [58:56], Physical_Address [51:16], Shareability [11:10], InnerCache [9:7]. assert_eq!( lpi::pendbaser(Phys::new(0xffff_ffff_0000).unwrap()), word(&[(62, 62, 1), (58, 56, 0), (51, 16, 0xffff_ffff_0000 >> 16), (11, 10, 0b01), (9, 7, 0b111)]) ); - assert_eq!((lpi::GICR_CTLR, lpi::GICR_TYPER, lpi::GICR_PROPBASER, lpi::GICR_PENDBASER), (0x0000, 0x0008, 0x0070, 0x0078)); assert_eq!(u64::from(lpi::CTLR_ENABLE_LPIS), word(&[(0, 0, 1)])); // GICR_TYPER: PLPIS [0], Processor_Number [23:8] under an affinity and flags it is not. assert_eq!(lpi::TYPER_PLPIS, word(&[(0, 0, 1)])); assert_eq!(lpi::processor_number(word(&[(63, 32, 0xffff_ffff), (23, 8, 0xbeef), (7, 0, 0xff), (31, 24, 0xff)])), 0xbeef); } + +// --- the register maps ------------------------------------------------------- + +/// §12.18: "The control registers, which are located at ITS_base + 0x000000. +/// The interrupt translation space, which is located at ITS_base + +/// 0x010000." Table 12-33, the ITS control register map, and Table 12-34, +/// the ITS translation register map, row by row; each register's own +/// description (§12.19.4, .13, .2, .5, .3, .1, .12) states the same offset. +#[test] +fn the_its_registers_sit_where_tables_12_33_and_12_34_have_them() { + let control: [(usize, usize); 5] = [ + (0x0000, toyos_its::GITS_CTLR), + (0x0008, toyos_its::GITS_TYPER), + (0x0080, toyos_its::GITS_CBASER), + (0x0088, toyos_its::GITS_CWRITER), + (0x0090, toyos_its::GITS_CREADR), + ]; + for (specified, declared) in control { + assert_eq!(declared, specified); + } + // "0x0100-0x0138 GITS_BASER", and §12.19.1's "0x0100 + (8 * n)", n = 0 - 7. + assert_eq!((0..9).map(gits_baser).collect::>(), [ + Some(0x0100), + Some(0x0108), + Some(0x0110), + Some(0x0118), + Some(0x0120), + Some(0x0128), + Some(0x0130), + Some(0x0138), + None + ]); + // "0x0040 GITS_TRANSLATER", of the frame at ITS_base + 0x010000. + assert_eq!((toyos_its::TRANSLATION_FRAME, toyos_its::GITS_TRANSLATER), (0x01_0000, 0x0040)); + assert_eq!(u64::from(toyos_its::CTLR_ENABLED), word(&[(0, 0, 1)])); +} + +/// §12.10, Table 12-27, the GIC physical LPI Redistributor register map, by +/// its offsets from `RD_base`; §12.11.33 and §12.11.32 state the same. +#[test] +fn the_redistributors_lpi_registers_sit_where_table_12_27_has_them() { + assert_eq!((lpi::GICR_PROPBASER, lpi::GICR_PENDBASER), (0x0070, 0x0078)); +} diff --git a/toyos-phys/Cargo.toml b/toyos-phys/Cargo.toml new file mode 100644 index 00000000000..0453caeaf4c --- /dev/null +++ b/toyos-phys/Cargo.toml @@ -0,0 +1,12 @@ +# A member of the host workspace (root `Cargo.toml`): what toyos-smmu and +# toyos-its both take an address as, declared once. An SMMUv3's and a GIC's +# address fields are bits [47:n] of an address or bits [51:n] with the top +# four zero, so one type bounds what either is given. + +[package] +name = "toyos-phys" +description = "A physical address below 2^48 by its alignment, the type an SMMUv3's and a GIC ITS's address fields are filled from, pure, forbid(unsafe_code)." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false diff --git a/toyos-phys/src/lib.rs b/toyos-phys/src/lib.rs new file mode 100644 index 00000000000..415c52a3b8c --- /dev/null +++ b/toyos-phys/src/lib.rs @@ -0,0 +1,33 @@ +//! [`Phys`]: a physical address aligned to `1 << ALIGN` bytes and below 2^48, +//! so a register or descriptor field that takes an address's bits `[47:ALIGN]` +//! holds every bit of one. +//! +//! `no_std`, no allocation, no `unsafe`. + +#![no_std] +#![forbid(unsafe_code)] + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Phys(u64); + +impl Phys { + const FIELD: u64 = (1 << 48) - (1 << ALIGN); + + pub const fn new(address: u64) -> Option { + if address & !Self::FIELD == 0 { + Some(Self(address)) + } else { + None + } + } + + /// The address a word holds in its bits `[47:ALIGN]`, whatever its other + /// bits are. + pub const fn of(word: u64) -> Self { + Self(word & Self::FIELD) + } + + pub const fn get(self) -> u64 { + self.0 + } +} diff --git a/toyos-phys/tests/phys.rs b/toyos-phys/tests/phys.rs new file mode 100644 index 00000000000..674b72db9d9 --- /dev/null +++ b/toyos-phys/tests/phys.rs @@ -0,0 +1,27 @@ +use toyos_phys::Phys; + +#[test] +fn an_address_off_its_alignment_or_past_48_bits_is_no_address() { + assert_eq!(Phys::<6>::new(0x40).map(Phys::get), Some(0x40)); + assert_eq!(Phys::<6>::new(0x20), None); + assert_eq!(Phys::<8>::new(0x100).map(Phys::get), Some(0x100)); + assert_eq!(Phys::<8>::new(0x80), None); + assert_eq!(Phys::<12>::new(0xfff), None); + assert_eq!(Phys::<12>::new(0x800), None); + assert_eq!(Phys::<16>::new(0x8000), None); + assert_eq!(Phys::<21>::new(0x10_0000), None); + assert_eq!(Phys::<21>::new(0x20_0000).map(Phys::get), Some(0x20_0000)); + assert_eq!(Phys::<12>::new(1 << 48), None); + assert_eq!(Phys::<12>::new((1 << 48) - 0x1000).map(Phys::get), Some((1 << 48) - 0x1000)); + assert_eq!(Phys::<16>::new((1 << 48) - 0x1_0000).map(Phys::get), Some((1 << 48) - 0x1_0000)); + assert_eq!(Phys::<0>::new(u64::MAX), None); + assert_eq!(Phys::<0>::new((1 << 48) - 1).map(Phys::get), Some((1 << 48) - 1)); +} + +#[test] +fn a_words_address_field_is_its_bits_from_the_alignment_to_47() { + assert_eq!(Phys::<12>::of(u64::MAX).get(), 0x0000_ffff_ffff_f000); + assert_eq!(Phys::<12>::of(0x8060_0000_4020_3fff).get(), 0x4020_3000); + assert_eq!(Phys::<21>::of(u64::MAX).get(), 0x0000_ffff_ffe0_0000); + assert_eq!(Phys::<12>::of(0x1000), Phys::new(0x1000).unwrap()); +} diff --git a/toyos-smmu/Cargo.toml b/toyos-smmu/Cargo.toml index 5e9d3fad997..0c5953a87d8 100644 --- a/toyos-smmu/Cargo.toml +++ b/toyos-smmu/Cargo.toml @@ -1,5 +1,5 @@ -# A member of the host workspace (root `Cargo.toml`), like toyos-acpi and -# toyos-dma: it is pure, it has no dependencies, and its tests run on the host. +# A member of the host workspace (root `Cargo.toml`), like toyos-acpi: it is +# pure and its tests run on the host. # # What lives here is every word an SMMUv3 and its driver exchange. The unit # decides which memory a device reaches, so a field one bit off is an isolation @@ -15,3 +15,9 @@ version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" publish = false + +[dependencies] +# The VMSAv8-64 stage 1 descriptors and the `MAIR` they index: the unit walks +# the format the loader's tables are written in, read from its one declaration. +toyos-bootmap = { path = "../toyos-bootmap" } +toyos-phys = { path = "../toyos-phys" } diff --git a/toyos-smmu/src/config.rs b/toyos-smmu/src/config.rs index b0d4c2b5106..6c28c1b4c81 100644 --- a/toyos-smmu/src/config.rs +++ b/toyos-smmu/src/config.rs @@ -2,12 +2,14 @@ //! each, eight little-endian doublewords, a field's bit `n` being bit //! `n % 64` of doubleword `n / 64`. +use toyos_phys::Phys; + use crate::unit::Unit; -use crate::{Asid, Phys}; +use crate::Asid; -/// A stream table entry. +/// A stream table entry: [`Ste::ABORT`] or [`Ste::stage1`], and no other. #[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct Ste(pub [u64; 8]); +pub struct Ste([u64; 8]); /// `V`, bit [0]. const STE_V: u64 = 1 << 0; @@ -30,16 +32,25 @@ impl Ste { /// at `context`: `S1ContextPtr` [55:6], `S1CDMax` [63:59] zero so a /// transaction carrying a SubstreamID is aborted, `STRW` [95:94] zero for /// the Non-secure EL1 regime, `EATS` [93:92] zero so ATS is refused, and - /// no stage 2 field. - pub const fn stage1(context: Phys<6>, unit: &Unit) -> Self { + /// no stage 2 field. `None` where the descriptor is past the unit's + /// output size. + pub const fn stage1(context: Phys<6>, unit: &Unit) -> Option { + if !unit.reaches(context.get(), 64) { + return None; + } let stall = if unit.stall_disable() { STE_S1STALLD } else { 0 }; - Self([STE_V | STE_STAGE1 | context.get(), STE_CD_CACHED | stall, 0, 0, 0, 0, 0, 0]) + Some(Self([STE_V | STE_STAGE1 | context.get(), STE_CD_CACHED | stall, 0, 0, 0, 0, 0, 0])) + } + + /// The entry's eight doublewords, as the stream table holds them. + pub const fn words(&self) -> [u64; 8] { + self.0 } } -/// A context descriptor. +/// A context descriptor: [`Cd::new`]'s, and no other. #[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct Cd(pub [u64; 8]); +pub struct Cd([u64; 8]); /// `T0SZ`, bits [5:0]: a 48-bit input, walked from level 0. const CD_T0SZ: u64 = 64 - crate::table::INPUT_BITS as u64; @@ -66,8 +77,12 @@ impl Cd { /// [34:32] from the unit's output size, `ASID` [63:48], `TTB0` [119:68] /// holding the root's bits [55:4], and `MAIR0` [223:192] the attributes /// [`crate::table`]'s leaves index. `AFFD` [35] stays clear, and every - /// leaf written sets its access flag. - pub const fn new(root: Phys<12>, asid: Asid, unit: &Unit) -> Self { + /// leaf written sets its access flag. `None` where the root table is + /// past the unit's output size. + pub const fn new(root: Phys<12>, asid: Asid, unit: &Unit) -> Option { + if !unit.reaches(root.get(), 4096) { + return None; + } let word0 = CD_T0SZ | CD_WALK | CD_EPD1 @@ -77,6 +92,11 @@ impl Cd { | CD_FAULTS | CD_ASET | (asid.get() as u64) << 48; - Self([word0, root.get(), 0, crate::table::MAIR0 as u64, 0, 0, 0, 0]) + Some(Self([word0, root.get(), 0, crate::table::MAIR0 as u64, 0, 0, 0, 0])) + } + + /// The descriptor's eight doublewords. + pub const fn words(&self) -> [u64; 8] { + self.0 } } diff --git a/toyos-smmu/src/lib.rs b/toyos-smmu/src/lib.rs index 6437affb45e..0ed9a36b154 100644 --- a/toyos-smmu/src/lib.rs +++ b/toyos-smmu/src/lib.rs @@ -8,11 +8,14 @@ //! The crate encodes one configuration and no other: a stream either aborts //! or is translated by stage 1 alone through one context descriptor, under //! the Non-secure EL1 regime, with 4 KiB-granule VMSAv8-64 tables over a -//! 48-bit input. No stream bypasses, and nothing here spells a bypass. +//! 48-bit input. No stream bypasses: [`config::Ste`] is made only as an +//! abort or as that translation, and holds its words where no other crate +//! can write them. //! //! What a unit reports about itself is a device's word and is refused by name //! where the driver could not act on it; nothing here panics on a register's -//! or a record's value. +//! or a record's value. An address is a [`toyos_phys::Phys`], and whatever +//! takes one for the unit to read answers `None` past the unit's output size. //! //! `no_std`, no allocation, no `unsafe`. @@ -24,26 +27,6 @@ pub mod queue; pub mod table; pub mod unit; -/// A physical address the unit is given, aligned to `1 << ALIGN` bytes and -/// below 2^48: the widest address a 4 KiB-granule VMSAv8-64 descriptor -/// carries without `DS`, and inside every address field written here. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct Phys(u64); - -impl Phys { - pub const fn new(address: u64) -> Option { - if address >> 48 == 0 && address & ((1 << ALIGN) - 1) == 0 { - Some(Self(address)) - } else { - None - } - } - - pub const fn get(self) -> u64 { - self.0 - } -} - /// A context's tag in the unit's TLB, no wider than the unit's ASIDs: made by /// [`unit::Unit::asid`]. #[derive(Clone, Copy, PartialEq, Eq, Debug)] diff --git a/toyos-smmu/src/queue.rs b/toyos-smmu/src/queue.rs index 92b28f1c3b1..dbfad87ce37 100644 --- a/toyos-smmu/src/queue.rs +++ b/toyos-smmu/src/queue.rs @@ -3,50 +3,99 @@ use crate::Asid; -/// A circular queue of `1 << log2size` entries. An index register holds the -/// entry's index in its low `log2size` bits and a wrap flag in the next one -/// up, which toggles each time the index passes the top. +/// The indexes of a circular queue of `1 << log2size` entries. An index +/// register holds the entry's index in its low `log2size` bits and a wrap +/// flag in the next one up, which toggles each time the index passes the top. #[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct Queue { - log2size: u8, +pub(crate) struct Indexes { + pub(crate) log2size: u8, } -impl Queue { - /// `None` past the 19 bits an index register's `WR` and `RD` fields with - /// their wrap flag can hold (`IDR1.CMDQS`, `EVENTQS`). - pub const fn new(log2size: u8) -> Option { - if log2size <= 19 { - Some(Self { log2size }) - } else { - None - } - } - +impl Indexes { /// The index and its wrap flag, which is all of an index register that /// names a position. - const fn position(self, index: u32) -> u32 { - index & ((2 << self.log2size) - 1) + const fn position(self, register: u32) -> u32 { + register & ((2 << self.log2size) - 1) } - /// The entry an index register names. - pub const fn slot(self, index: u32) -> usize { - (index & ((1 << self.log2size) - 1)) as usize + const fn slot(self, register: u32) -> usize { + (register & ((1 << self.log2size) - 1)) as usize } - /// Nothing to consume: the indexes and their wrap flags are equal. - pub const fn is_empty(self, prod: u32, cons: u32) -> bool { + /// The indexes and their wrap flags are equal. + const fn is_empty(self, prod: u32, cons: u32) -> bool { self.position(prod) == self.position(cons) } - /// Nothing can be produced: the indexes are equal and the wrap flags differ. + /// The position after `register`'s, wrap flag toggled where it passes + /// the top. + const fn after(self, register: u32) -> u32 { + self.position(self.position(register) + 1) + } +} + +/// The command queue's indexes, as the driver that produces into it reads +/// `SMMU_CMDQ_PROD` and `SMMU_CMDQ_CONS`: made by +/// [`Unit::command_queue`](crate::unit::Unit::command_queue). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Commands(pub(crate) Indexes); + +impl Commands { + /// The entry the next command is written to. + pub const fn slot(self, prod: u32) -> usize { + self.0.slot(prod) + } + + /// No command can be written: the indexes are equal and the wrap flags + /// differ. pub const fn is_full(self, prod: u32, cons: u32) -> bool { - self.position(prod) ^ self.position(cons) == 1 << self.log2size + self.0.position(prod) ^ self.0.position(cons) == 1 << self.0.log2size + } + + /// The unit has consumed every command written. + pub const fn is_empty(self, prod: u32, cons: u32) -> bool { + self.0.is_empty(prod, cons) + } + + /// What `SMMU_CMDQ_PROD` is written with once the command at `prod` is + /// in memory: the position after it, index and wrap flag at once. + pub const fn after(self, prod: u32) -> u32 { + self.0.after(prod) + } +} + +/// `SMMU_EVENTQ_PROD.OVFLG` and `SMMU_EVENTQ_CONS.OVACKFLG`, bit [31] of each. +const OVERFLOW: u32 = 1 << 31; + +/// The event queue's indexes, as the driver that consumes from it reads +/// `SMMU_EVENTQ_PROD` and `SMMU_EVENTQ_CONS`: made by +/// [`Unit::event_queue`](crate::unit::Unit::event_queue). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Events(pub(crate) Indexes); + +impl Events { + /// The entry the next record is read from. + pub const fn slot(self, cons: u32) -> usize { + self.0.slot(cons) + } + + /// No record to read. + pub const fn is_empty(self, prod: u32, cons: u32) -> bool { + self.0.is_empty(prod, cons) + } + + /// Records were dropped on a full queue and that is not yet + /// acknowledged: `OVFLG` differs from `OVACKFLG` (§7.4). + pub const fn overflowed(self, prod: u32, cons: u32) -> bool { + (prod ^ cons) & OVERFLOW != 0 } - /// The position after `index`, wrap flag toggled where it passes the - /// top: what the owner of an index writes back, index and flag at once. - pub const fn after(self, index: u32) -> u32 { - self.position(self.position(index) + 1) + /// What `SMMU_EVENTQ_CONS` is written with once the record at `cons` is + /// read: the position after it, and `OVACKFLG` equal to the `OVFLG` + /// `prod` was read with. That acknowledges an overflow seen in `prod`, + /// and no value this answers raises one the unit did not report. + pub const fn after(self, cons: u32, prod: u32) -> u32 { + self.0.after(cons) | prod & OVERFLOW } } @@ -57,8 +106,6 @@ pub enum Signal { None, /// `SIG_IRQ` with `MSIAddress` zero: the unit's wired sync interrupt. Irq, - /// `SIG_SEV`: an event to the CPUs, where `IDR0.SEV` is set. - Sev, } /// One 16-byte command. @@ -67,9 +114,6 @@ pub enum Command { /// `CMD_CFGI_STE` (§4.3.1), `Leaf` set: forget one stream's entry and /// every context descriptor reached through it. ForgetStream(u32), - /// `CMD_CFGI_CD` (§4.3.3), SubstreamID 0 and `Leaf` set: forget the one - /// context descriptor a stream's entry names. - ForgetContext(u32), /// `CMD_CFGI_ALL` (§4.3.9): forget every stream's configuration. ForgetAll, /// `CMD_TLBI_NH_ASID` (§4.4.2.2), VMID 0: drop every non-global @@ -88,8 +132,6 @@ impl Command { match self { // StreamID [63:32]; Leaf [64]. Self::ForgetStream(stream) => [0x03 | (stream as u64) << 32, 1], - // StreamID [63:32], SubstreamID [31:12] zero; Leaf [64]. - Self::ForgetContext(stream) => [0x05 | (stream as u64) << 32, 1], // `CMD_CFGI_STE_RANGE` with Range [68:64] of 31. Self::ForgetAll => [0x04, 31], // ASID [63:48], VMID [47:32]. @@ -100,7 +142,6 @@ impl Command { let cs = match signal { Signal::None => 0b00, Signal::Irq => 0b01, - Signal::Sev => 0b10, }; [0x46 | cs << 12, 0] } diff --git a/toyos-smmu/src/table.rs b/toyos-smmu/src/table.rs index 08ec48a2712..f8f9a2cf669 100644 --- a/toyos-smmu/src/table.rs +++ b/toyos-smmu/src/table.rs @@ -1,23 +1,28 @@ //! The stage 1 tables a context descriptor names: VMSAv8-64 with the 4 KiB //! granule over a 48-bit input, four levels of 512 descriptors (Arm ARM, -//! DDI 0487 M.d, §D8.3.1), and the plan of one mapping in them. +//! DDI 0487 M.d, §D8.3.1), and the plan of one mapping in them. The +//! descriptors are [`toyos_bootmap::aarch64`]'s, which the loader's tables are +//! written with, under the bits that make a leaf a device's and no CPU's. //! //! A device's memory is mapped by 2 MiB blocks at level 2. The page an ITS //! takes a message on is the one 4 KiB page at level 3: a block there would //! hand the device every register that shares its 2 MiB. -use crate::Phys; +use toyos_bootmap::aarch64::{block, page, table, MAIR, PXN}; +use toyos_bootmap::Cache; +use toyos_phys::Phys; + +use crate::unit::Unit; /// The width of a device address these tables translate. pub const INPUT_BITS: u32 = 48; /// Levels 0 to 3, each indexed by nine bits of the address. pub const LEVELS: usize = 4; -/// `CD.MAIR0`: attribute 0 is Device-nGnRE (`0x04`), which the message page -/// is mapped with, and attribute 1 Normal write-back memory (`0xFF`). -pub const MAIR0: u32 = 0x04 | 0xFF << 8; -const ATTR_DEVICE: u64 = 0 << 2; -const ATTR_MEMORY: u64 = 1 << 2; +/// `CD.MAIR0`, attributes 0 to 3 of the `MAIR` the descriptors' `AttrIndx` +/// names: a [`Cache::Device`] leaf's is Device-nGnRE and a [`Cache::Memory`] +/// leaf's Normal write-back. +pub const MAIR0: u32 = MAIR as u32; /// Table D8-50 and Table D8-52: bit [0] valid; bit [1] set for a table /// descriptor and a level 3 page, clear for a block. @@ -29,18 +34,15 @@ const TABLE_OR_PAGE: u64 = 1 << 1; const AP_UNPRIVILEGED: u64 = 1 << 6; /// `AP[2]`, bit [7]: read-only. const AP_READ_ONLY: u64 = 1 << 7; -/// `SH`, bits [9:8]: inner shareable. -const INNER_SHAREABLE: u64 = 0b11 << 8; -/// The access flag, bit [10]: clear, the first access faults. -const AF: u64 = 1 << 10; /// `nG`, bit [11]: the translation is its context's alone. A leaf without it /// is global, cached for every ASID of the regime and so for every domain, /// and no invalidation by ASID removes it. const NOT_GLOBAL: u64 = 1 << 11; -/// `PXN` [53] and `UXN` [54]: never executable. -const NEVER_EXECUTED: u64 = 1 << 53 | 1 << 54; -const LEAF: u64 = VALID | AP_UNPRIVILEGED | AF | NOT_GLOBAL | NEVER_EXECUTED; +/// What a device's leaf adds to the loader's: unprivileged, its context's +/// alone, and executable at no level — the loader's memory is executable at +/// EL1, which no device's access is. +const DEVICES: u64 = AP_UNPRIVILEGED | NOT_GLOBAL | PXN; /// What a device may do to memory mapped for it. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -59,9 +61,14 @@ pub enum Leaf { } /// The descriptor naming the next table down, at levels 0 to 2. Its -/// hierarchical fields stay clear: the leaf alone decides the access. -pub const fn next(table: Phys<12>) -> u64 { - table.get() | TABLE_OR_PAGE | VALID +/// hierarchical fields stay clear: the leaf alone decides the access. `None` +/// where the table is past the unit's output size. +pub const fn next(at: Phys<12>, unit: &Unit) -> Option { + if unit.reaches(at.get(), 4096) { + Some(table(at.get())) + } else { + None + } } /// One mapping, planned: the index to follow in each table from the root, @@ -84,19 +91,20 @@ impl Path { } /// The path to `leaf` at the device address `at`. `None` where `at` is past -/// the input or not aligned to what the leaf maps. -pub const fn plan(at: u64, leaf: Leaf) -> Option { - let (tables, size, descriptor) = match leaf { - Leaf::Memory(block, access) => { +/// the input or not aligned to what the leaf maps, or what the leaf maps is +/// past the unit's output size. +pub const fn plan(at: u64, leaf: Leaf, unit: &Unit) -> Option { + let (tables, size, output, descriptor) = match leaf { + Leaf::Memory(memory, access) => { let access = match access { Access::Read => AP_READ_ONLY, Access::ReadWrite => 0, }; - (3, 1u64 << 21, block.get() | LEAF | ATTR_MEMORY | INNER_SHAREABLE | access) + (3, 1u64 << 21, memory.get(), block(memory.get(), Cache::Memory) | DEVICES | access) } - Leaf::Doorbell(page) => (4, 1u64 << 12, page.get() | LEAF | TABLE_OR_PAGE | ATTR_DEVICE), + Leaf::Doorbell(register) => (4, 1u64 << 12, register.get(), page(register.get(), Cache::Device) | DEVICES), }; - if at >> INPUT_BITS != 0 || at & (size - 1) != 0 { + if at >> INPUT_BITS != 0 || at & (size - 1) != 0 || !unit.reaches(output, size) { return None; } const fn index(at: u64, level: u32) -> usize { @@ -122,7 +130,7 @@ pub const fn entry(descriptor: u64, level: usize) -> Entry { if descriptor & VALID == 0 { Entry::Invalid } else if level < LEVELS - 1 && descriptor & TABLE_OR_PAGE != 0 { - Entry::Table(Phys(descriptor & 0x0000_FFFF_FFFF_F000)) + Entry::Table(Phys::of(descriptor)) } else { Entry::Mapped } diff --git a/toyos-smmu/src/unit.rs b/toyos-smmu/src/unit.rs index d93b0581b96..b706371d68d 100644 --- a/toyos-smmu/src/unit.rs +++ b/toyos-smmu/src/unit.rs @@ -2,7 +2,10 @@ //! and writes (§6.3), and [`probe`]: what a unit's identification registers //! let this crate's one configuration be built on. -use crate::{Asid, Phys}; +use toyos_phys::Phys; + +use crate::queue::{Commands, Events, Indexes}; +use crate::Asid; /// Offsets in register page 0 (§6.2.1). pub const IDR0: usize = 0x000; @@ -94,11 +97,6 @@ pub const fn command_error(cons: u32) -> CommandError { } } -/// `SMMU_EVENTQ_PROD.OVFLG` and `SMMU_EVENTQ_CONS.OVACKFLG` [31] (§7.4): -/// records were dropped on a full queue while the two differ, and the unit -/// reports no further overflow until the consumer writes its flag equal. -pub const EVENTQ_OVERFLOW: u32 = 1 << 31; - /// What a unit lacks that this crate's one configuration needs. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Lacks { @@ -118,6 +116,11 @@ pub enum Lacks { FixedStructures, /// `IDR1.SIDSIZE`, `CMDQS` or `EVENTQS` holds a reserved value. Size(u8), + /// `IDR0.COHACC` clear with no override from the IORT: the unit does not + /// read its tables and queues coherently with the CPUs' caches, which + /// every cacheability written here ([`CR1_WRITE_BACK`], the stream table + /// entry's and the context descriptor's) takes it to. + CoherentAccess, } /// A unit the one configuration can be built on, and its sizes. @@ -129,17 +132,21 @@ pub struct Unit { /// its entries. pub command_queue_log2: u8, pub event_queue_log2: u8, - /// `IDR5.OAS` in bits: no address the unit is given may reach past it. - pub output_bits: u8, - /// `IDR0.COHACC`, which the IORT's SMMUv3 node may override. - pub coherent: bool, + /// `IDR5.OAS` in bits: the bits of an address field above it are RES0 + /// (§6.3.24), so nothing the unit is given may reach past it. + output_bits: u8, asid16: bool, /// `IDR0.STALL_MODEL` is `0b00`: stalling is there to be disabled. stall: bool, } -/// §6.3.1, §6.3.2 and §6.3.6: the three identification registers, judged. -pub const fn probe(idr0: u32, idr1: u32, idr5: u32) -> Result { +/// §6.3.1, §6.3.2 and §6.3.6: the three identification registers, judged, +/// with the IORT's `COHACC override` (DEN 0049 Table 14), which stands for +/// `IDR0.COHACC` [4] where it is set. +pub const fn probe(idr0: u32, idr1: u32, idr5: u32, coherent_override: bool) -> Result { + if idr0 & 1 << 4 == 0 && !coherent_override { + return Err(Lacks::CoherentAccess); + } if idr0 & 1 << 1 == 0 { return Err(Lacks::Stage1); } @@ -179,7 +186,6 @@ pub const fn probe(idr0: u32, idr1: u32, idr5: u32) -> Result { command_queue_log2, event_queue_log2, output_bits: [32, 36, 40, 42, 44, 48, 52, 56][(idr5 & 0b111) as usize], - coherent: idr0 & 1 << 4 != 0, asid16: idr0 & 1 << 12 != 0, stall, }) @@ -215,35 +221,49 @@ impl Unit { } } + /// Whether the unit reads all of `bytes` at `address`: none of them at + /// or past `1 << IDR5.OAS`. + pub(crate) const fn reaches(&self, address: u64, bytes: u64) -> bool { + address + bytes <= 1 << self.output_bits + } + + /// `bytes` at `address`, aligned to their own size, where the unit reads + /// them. + const fn holds(&self, address: u64, bytes: u64) -> bool { + address & (bytes - 1) == 0 && self.reaches(address, bytes) + } + /// `SMMU_STRTAB_BASE` and `SMMU_STRTAB_BASE_CFG` (§6.3.24, §6.3.25) for /// a linear table of `1 << log2size` entries at `table`: `RA` [62] and /// the address, then `FMT` [17:16] zero and `LOG2SIZE` [5:0]. `None` - /// where the unit takes fewer StreamID bits, or the table is not aligned - /// to its own size as the unit reads its address. + /// where the unit takes fewer StreamID bits, the table is not aligned to + /// its own size as the unit reads its address, or it ends past the + /// unit's output size. pub const fn stream_table(&self, table: Phys<6>, log2size: u8) -> Option<(u64, u32)> { - if log2size > self.stream_bits || table.get() & ((64 << log2size) - 1) != 0 { + if log2size > self.stream_bits || !self.holds(table.get(), 64 << log2size) { return None; } Some((1 << 62 | table.get(), log2size as u32)) } /// `SMMU_CMDQ_BASE` (§6.3.26) for a queue of `1 << log2size` 16-byte - /// commands at `queue`: `RA` [62], the address, `LOG2SIZE` [4:0]. `None` - /// past the unit's `CMDQS`, or where the queue is not aligned to the - /// larger of its size and 32 bytes. - pub const fn command_queue(&self, queue: Phys<5>, log2size: u8) -> Option { - if log2size > self.command_queue_log2 || queue.get() & ((16 << log2size) - 1) != 0 { + /// commands at `queue`, and its indexes: `RA` [62], the address, + /// `LOG2SIZE` [4:0]. `None` past the unit's `CMDQS` or its output size, + /// or where the queue is not aligned to the larger of its size and 32 + /// bytes. + pub const fn command_queue(&self, queue: Phys<5>, log2size: u8) -> Option<(u64, Commands)> { + if log2size > self.command_queue_log2 || !self.holds(queue.get(), 16 << log2size) { return None; } - Some(1 << 62 | queue.get() | log2size as u64) + Some((1 << 62 | queue.get() | log2size as u64, Commands(Indexes { log2size }))) } /// `SMMU_EVENTQ_BASE` (§6.3.29) for a queue of `1 << log2size` 32-byte - /// records: `WA` [62], the address, `LOG2SIZE` [4:0]. - pub const fn event_queue(&self, queue: Phys<5>, log2size: u8) -> Option { - if log2size > self.event_queue_log2 || queue.get() & ((32 << log2size) - 1) != 0 { + /// records, and its indexes: `WA` [62], the address, `LOG2SIZE` [4:0]. + pub const fn event_queue(&self, queue: Phys<5>, log2size: u8) -> Option<(u64, Events)> { + if log2size > self.event_queue_log2 || !self.holds(queue.get(), 32 << log2size) { return None; } - Some(1 << 62 | queue.get() | log2size as u64) + Some((1 << 62 | queue.get() | log2size as u64, Events(Indexes { log2size }))) } } diff --git a/toyos-smmu/tests/encodings.rs b/toyos-smmu/tests/encodings.rs index 2ede94eac0b..f25d80ae04d 100644 --- a/toyos-smmu/tests/encodings.rs +++ b/toyos-smmu/tests/encodings.rs @@ -3,11 +3,11 @@ //! diagrams and field headings give them, and compared whole, so a field in //! the wrong place and a stray bit are both a difference. +use toyos_phys::Phys; use toyos_smmu::config::{Cd, Ste}; -use toyos_smmu::queue::{event, Attempt, Code, Command, Event, Queue, Signal}; +use toyos_smmu::queue::{event, Attempt, Code, Command, Event, Signal}; use toyos_smmu::table::{entry, next, plan, Access, Entry, Leaf, INPUT_BITS, MAIR0}; use toyos_smmu::unit::{self, probe, CommandError, Lacks, Unit}; -use toyos_smmu::Phys; /// `N` doublewords holding `value` in bits `[msb:lsb]` for every triple, and /// zero everywhere else. A field may cross a doubleword. @@ -46,7 +46,7 @@ fn with(base: &[(u32, u32, u64)], change: &[(u32, u32, u64)]) -> u32 { } fn idr(idr0: &[(u32, u32, u64)], idr1: &[(u32, u32, u64)], idr5: &[(u32, u32, u64)]) -> Result { - probe(with(IDR0, idr0), with(IDR1, idr1), with(IDR5, idr5)) + probe(with(IDR0, idr0), with(IDR1, idr1), with(IDR5, idr5), false) } fn whole() -> Unit { @@ -63,21 +63,38 @@ fn terminating() -> Unit { #[test] fn a_units_sizes_are_read_from_their_own_fields() { let unit = whole(); - assert_eq!( - (unit.stream_bits, unit.command_queue_log2, unit.event_queue_log2, unit.output_bits, unit.coherent), - (16, 19, 8, 44, true) - ); + assert_eq!((unit.stream_bits, unit.command_queue_log2, unit.event_queue_log2), (16, 19, 8)); // Each field moved alone moves its own answer and no other. - let other = idr(&[(4, 4, 0)], &[(5, 0, 32), (20, 16, 19), (25, 21, 0)], &[(2, 0, 0b000)]).expect("a unit"); - assert_eq!( - (other.stream_bits, other.command_queue_log2, other.event_queue_log2, other.output_bits, other.coherent), - (32, 0, 19, 32, false) - ); - for (oas, bits) in [32u8, 36, 40, 42, 44, 48, 52, 56].into_iter().enumerate() { - assert_eq!(idr(&[], &[], &[(2, 0, oas as u64)]).map(|unit| unit.output_bits), Ok(bits)); + let other = idr(&[], &[(5, 0, 32), (20, 16, 19), (25, 21, 0)], &[]).expect("a unit"); + assert_eq!((other.stream_bits, other.command_queue_log2, other.event_queue_log2), (32, 0, 19)); +} + +/// `IDR5.OAS` [2:0]: 32, 36, 40, 42, 44, 48, 52 and 56 bits. The unit is +/// given a stream table that ends at its last byte and none a byte further. +#[test] +fn a_units_output_size_is_idr5s_and_bounds_what_it_is_given() { + for (oas, bits) in [32u32, 36, 40, 42, 44, 48, 52, 56].into_iter().enumerate() { + let unit = idr(&[], &[], &[(2, 0, oas as u64)]).expect("a unit"); + let end = 1u64 << bits.min(48); + assert!(unit.stream_table(Phys::new(end - 64).unwrap(), 0).is_some(), "OAS {oas:#b}"); + // Past 48 bits there is no address to refuse. + if let Some(past) = Phys::new(end) { + assert_eq!(unit.stream_table(past, 0), None, "OAS {oas:#b}"); + } } } +/// DEN 0049 Table 14: the IORT's `COHACC override` stands for `IDR0.COHACC` +/// [4] where it is set, and a unit neither calls coherent is refused. +#[test] +fn a_unit_that_is_not_coherent_is_refused_unless_the_iort_says_it_is() { + assert_eq!(idr(&[(4, 4, 0)], &[], &[]), Err(Lacks::CoherentAccess)); + let (idr0, idr1, idr5) = (with(IDR0, &[(4, 4, 0)]), with(IDR1, &[]), with(IDR5, &[])); + assert_eq!(probe(idr0, idr1, idr5, true), Ok(whole())); + // The override adds nothing to a unit that says so itself. + assert_eq!(probe(with(IDR0, &[]), idr1, idr5, true), Ok(whole())); +} + #[test] fn a_unit_that_lacks_what_the_configuration_needs_is_refused_by_name() { assert_eq!(idr(&[(1, 1, 0)], &[], &[]), Err(Lacks::Stage1)); @@ -101,8 +118,10 @@ fn a_reserved_size_is_refused_and_the_largest_defined_one_is_not() { assert!(idr(&[], &[(5, 0, 32), (25, 21, 19), (20, 16, 19)], &[]).is_ok()); // Every value three registers can hold is an answer. for bits in [0u32, u32::MAX, 0x5555_5555, 0xaaaa_aaaa] { - let _ = probe(bits, bits, bits); - let _ = probe(with(IDR0, &[]), bits, bits); + for coherent in [false, true] { + let _ = probe(bits, bits, bits, coherent); + let _ = probe(with(IDR0, &[]), bits, bits, coherent); + } } } @@ -119,17 +138,17 @@ fn an_asid_is_no_wider_than_the_units() { #[test] fn an_aborting_entry_is_valid_and_configured_to_pass_nothing() { // V [0] set; Config [3:1] `0b000`, "report abort to device, no event recorded". - assert_eq!(Ste::ABORT.0, layout(&[(0, 0, 1), (3, 1, 0b000)])); + assert_eq!(Ste::ABORT.words(), layout(&[(0, 0, 1), (3, 1, 0b000)])); } #[test] fn a_translating_entry_is_stage_1_alone_through_one_context_descriptor() { - let context = Phys::<6>::new(0xffff_ffff_ffc0).expect("64-byte aligned, below 2^48"); + let context = Phys::<6>::new(0xfff_ffff_ffc0).expect("64-byte aligned, inside the unit's 44 bits"); let fields = [ (0, 0, 1), // V (3, 1, 0b101), // Config: stage 1 translate, stage 2 bypass (5, 4, 0), // S1Fmt, ignored with S1CDMax zero - (55, 6, 0xffff_ffff_ffc0 >> 6), // S1ContextPtr + (55, 6, 0xfff_ffff_ffc0 >> 6), // S1ContextPtr (63, 59, 0), // S1CDMax: one CD, substreams disabled (67, 66, 0b01), // S1CIR (69, 68, 0b01), // S1COR @@ -139,19 +158,19 @@ fn a_translating_entry_is_stage_1_alone_through_one_context_descriptor() { ]; let mut stalling = fields.to_vec(); stalling.push((91, 91, 1)); // S1STALLD, which a unit that can stall must be told - assert_eq!(Ste::stage1(context, &whole()).0, layout(&stalling)); + assert_eq!(Ste::stage1(context, &whole()).map(|ste| ste.words()), Some(layout(&stalling))); // And is ILLEGAL to set on one that cannot. - assert_eq!(Ste::stage1(context, &terminating()).0, layout(&fields)); + assert_eq!(Ste::stage1(context, &terminating()).map(|ste| ste.words()), Some(layout(&fields))); } #[test] fn a_context_descriptor_walks_ttb0_alone_under_its_own_asid() { - let root = Phys::<12>::new(0xffff_ffff_f000).expect("a page below 2^48"); + let root = Phys::<12>::new(0xfff_ffff_f000).expect("a page inside the unit's 44 bits"); let unit = whole(); let asid = unit.asid(0xbeef).expect("a 16-bit ASID"); assert_eq!( - Cd::new(root, asid, &unit).0, - layout(&[ + Cd::new(root, asid, &unit).map(|cd| cd.words()), + Some(layout(&[ (5, 0, 16), // T0SZ: 48 bits (7, 6, 0b00), // TG0: 4KB (9, 8, 0b01), // IR0 @@ -169,9 +188,9 @@ fn a_context_descriptor_walks_ttb0_alone_under_its_own_asid() { (46, 46, 1), // A (47, 47, 1), // ASET (63, 48, 0xbeef), // ASID - (119, 68, 0xffff_ffff_f000 >> 4), // TTB0: address bits [55:4] - (223, 192, 0xff04), // MAIR0 - ]) + (119, 68, 0xfff_ffff_f000 >> 4), // TTB0: address bits [55:4] + (223, 192, 0x0044_ff04), // MAIR0: the loader's MAIR, attributes 0 to 3 + ])) ); } @@ -181,26 +200,50 @@ fn a_context_descriptors_ips_is_the_units_output_size_and_no_more_than_48_bits() // `TCR_ELx.PS`'s encodings for 32, 36, 40, 42, 44 and 48 bits; 52 and 56 are past these tables. for (oas, ips) in [0b000u64, 0b001, 0b010, 0b011, 0b100, 0b101, 0b101, 0b101].into_iter().enumerate() { let unit = idr(&[], &[], &[(2, 0, oas as u64)]).expect("a unit"); - let cd = Cd::new(root, unit.asid(1).unwrap(), &unit); - assert_eq!(cd.0[0] >> 32 & 0b111, ips, "OAS {oas:#b}"); + let cd = Cd::new(root, unit.asid(1).unwrap(), &unit).expect("a root in the first 4 GiB"); + assert_eq!(cd.words()[0] >> 32 & 0b111, ips, "OAS {oas:#b}"); } } +// --- §6.3: the registers the driver writes --------------------------------- + +/// §6.2.1, "Registers in Page 0", and §6.2.2, "Registers in Page 1" — +/// "Offsets are relative to the base of Page 1, 0x10000" — row by row. Each +/// register's own description states the same offset in its "Accessible at +/// offset ... from SMMUv3_PAGE_0" (§6.3.1, .2, .6, .9, .10, .11, .12, .15, +/// .17, .18, .19, .20, .24, .25, .26, .27, .28, .29) or "from SMMUv3_PAGE_1" +/// (§6.3.130, §6.3.131). #[test] -fn an_address_off_its_alignment_or_past_48_bits_is_no_address() { - assert_eq!(Phys::<6>::new(0x40).map(Phys::get), Some(0x40)); - assert_eq!(Phys::<6>::new(0x20), None); - assert_eq!(Phys::<12>::new(0xfff), None); - assert_eq!(Phys::<12>::new(0x800), None); - assert_eq!(Phys::<21>::new(0x10_0000), None); - assert_eq!(Phys::<21>::new(0x20_0000).map(Phys::get), Some(0x20_0000)); - assert_eq!(Phys::<12>::new(1 << 48), None); - assert_eq!(Phys::<12>::new((1 << 48) - 0x1000).map(Phys::get), Some((1 << 48) - 0x1000)); - assert_eq!(Phys::<0>::new(u64::MAX), None); +fn the_registers_sit_where_the_register_overview_has_them() { + let page0: [(usize, usize); 18] = [ + (0x0000, unit::IDR0), + (0x0004, unit::IDR1), + (0x0014, unit::IDR5), + (0x0020, unit::CR0), + (0x0024, unit::CR0ACK), + (0x0028, unit::CR1), + (0x002C, unit::CR2), + (0x0044, unit::GBPA), + (0x0050, unit::IRQ_CTRL), + (0x0054, unit::IRQ_CTRLACK), + (0x0060, unit::GERROR), + (0x0064, unit::GERRORN), + (0x0080, unit::STRTAB_BASE), + (0x0088, unit::STRTAB_BASE_CFG), + (0x0090, unit::CMDQ_BASE), + (0x0098, unit::CMDQ_PROD), + (0x009C, unit::CMDQ_CONS), + (0x00A0, unit::EVENTQ_BASE), + ]; + for (specified, declared) in page0 { + assert_eq!(declared, specified); + } + let page1: [(usize, usize); 2] = [(0x00A8, unit::EVENTQ_PROD), (0x00AC, unit::EVENTQ_CONS)]; + for (specified, declared) in page1 { + assert_eq!(declared, 0x1_0000 + specified); + } } -// --- §6.3: the registers the driver writes --------------------------------- - #[test] fn the_control_values_hold_each_field_where_its_register_has_it() { assert_eq!(u64::from(unit::CR0_SMMUEN | unit::CR0_EVENTQEN | unit::CR0_CMDQEN), word(&[(0, 0, 1), (2, 2, 1), (3, 3, 1)])); @@ -217,7 +260,6 @@ fn the_control_values_hold_each_field_where_its_register_has_it() { u64::from(unit::GERROR_CMDQ | unit::GERROR_EVENTQ_ABORT | unit::GERROR_SERVICE_FAILURE), word(&[(0, 0, 1), (2, 2, 1), (8, 8, 1)]) ); - assert_eq!(u64::from(unit::EVENTQ_OVERFLOW), word(&[(31, 31, 1)])); } #[test] @@ -249,6 +291,11 @@ fn a_stream_table_is_linear_inside_the_units_streams_and_aligned_to_its_size() { // RA [62], ADDR [55:6]; FMT [17:16] linear, SPLIT [10:6] unused, LOG2SIZE [5:0]. Some((word(&[(62, 62, 1), (55, 6, 0x4000_4000 >> 6)]), word(&[(17, 16, 0b00), (5, 0, 8)]) as u32)) ); + // A table larger than everything the unit reads: 2^27 entries are 8 GiB, on a unit of 32 output bits. + let small = idr(&[], &[(5, 0, 32)], &[(2, 0, 0b000)]).unwrap(); + let zero = Phys::<6>::new(0).unwrap(); + assert!(small.stream_table(zero, 26).is_some()); + assert_eq!(small.stream_table(zero, 27), None); assert_eq!(unit.stream_table(Phys::new(0x4000_2000).unwrap(), 8), None, "8 KiB into a 16 KiB table's alignment"); assert_eq!(unit.stream_table(Phys::new(0x4000_0040).unwrap(), 0), Some((1 << 62 | 0x4000_0040, 0))); // The unit takes 16 StreamID bits and no more. @@ -263,13 +310,13 @@ fn a_queue_is_inside_the_units_size_and_aligned_to_its_own() { let at = Phys::<5>::new(0x4001_0000).unwrap(); // RA or WA [62], ADDR [55:5], LOG2SIZE [4:0]. let base = |log2: u64| word(&[(62, 62, 1), (55, 5, 0x4001_0000 >> 5), (4, 0, log2)]); - assert_eq!(unit.command_queue(at, 12), Some(base(12))); - assert_eq!(unit.event_queue(at, 8), Some(base(8))); + assert_eq!(unit.command_queue(at, 12).map(|(register, _)| register), Some(base(12))); + assert_eq!(unit.event_queue(at, 8).map(|(register, _)| register), Some(base(8))); // 2^13 commands of 16 bytes are 128 KiB, and 2^12 records of 32 as many: `at` is 64 KiB aligned. assert_eq!(unit.command_queue(at, 13), None); assert_eq!(unit.event_queue(at, 12), None, "past EVENTQS and misaligned both"); let unit = idr(&[], &[(20, 16, 19)], &[]).unwrap(); - assert_eq!(unit.event_queue(at, 11), Some(base(11))); + assert_eq!(unit.event_queue(at, 11).map(|(register, _)| register), Some(base(11))); assert_eq!(unit.event_queue(at, 12), None, "128 KiB of records at a 64 KiB alignment"); // The unit's largest and one more. let low = Phys::<5>::new(0).unwrap(); @@ -278,46 +325,116 @@ fn a_queue_is_inside_the_units_size_and_aligned_to_its_own() { assert_eq!((small.command_queue(low, 5), small.event_queue(low, 5)), (None, None)); } +/// §6.3.24: the bits of `ADDR` above `IDR5.OAS` are RES0, and a unit need not +/// store them, so it would read its table or queue at the address without +/// them. +#[test] +fn a_table_or_queue_past_the_units_output_size_is_refused() { + let unit = idr(&[], &[(20, 16, 19)], &[(2, 0, 0b000)]).expect("a unit of 32 output bits"); + let past = 1u64 << 32; + assert_eq!(unit.stream_table(Phys::new(past).unwrap(), 0), None); + assert_eq!(unit.command_queue(Phys::new(past).unwrap(), 0), None); + assert_eq!(unit.event_queue(Phys::new(past).unwrap(), 0), None); + // Each ends at the unit's last byte and is given. + assert!(unit.stream_table(Phys::new(past - (64 << 8)).unwrap(), 8).is_some()); + assert!(unit.command_queue(Phys::new(past - (16 << 8)).unwrap(), 8).is_some()); + assert!(unit.event_queue(Phys::new(past - (32 << 8)).unwrap(), 8).is_some()); +} + +/// The same bound on what an entry, a descriptor and a table name: an +/// `S1ContextPtr`, a `TTB0`, a next table and a leaf's output address. +#[test] +fn a_descriptor_or_a_mapping_past_the_units_output_size_is_refused() { + let unit = idr(&[], &[], &[(2, 0, 0b000)]).expect("a unit of 32 output bits"); + let asid = unit.asid(1).unwrap(); + let past = 1u64 << 32; + assert_eq!(Ste::stage1(Phys::new(past).unwrap(), &unit), None); + assert!(Ste::stage1(Phys::new(past - 64).unwrap(), &unit).is_some()); + assert_eq!(Cd::new(Phys::new(past).unwrap(), asid, &unit), None); + assert!(Cd::new(Phys::new(past - 4096).unwrap(), asid, &unit).is_some()); + assert_eq!(next(Phys::new(past).unwrap(), &unit), None); + assert!(next(Phys::new(past - 4096).unwrap(), &unit).is_some()); + let memory = |at: u64| plan(AT, Leaf::Memory(Phys::new(at).unwrap(), Access::ReadWrite), &unit); + assert_eq!(memory(past), None); + assert!(memory(past - (1 << 21)).is_some()); + let doorbell = |at: u64| plan(AT, Leaf::Doorbell(Phys::new(at).unwrap()), &unit); + assert_eq!(doorbell(past), None); + assert!(doorbell(past - 4096).is_some()); + // A unit of 44 bits takes what one of 32 does not. + assert!(Ste::stage1(Phys::new(past).unwrap(), &whole()).is_some()); +} + // --- §3.5.1: the queues' indexes -------------------------------------------- -/// Against a count kept beside it: empty exactly at zero entries, full -/// exactly at all of them, and the slot the index that many steps on. +/// Against a count kept beside it: the command queue is full exactly at all +/// of its entries and consumed exactly at none, the event queue empty +/// exactly at none, and the slot the index that many steps on. #[test] fn a_queues_indexes_say_empty_full_and_slot_as_a_count_would() { + let unit = idr(&[], &[(20, 16, 19)], &[]).unwrap(); + let low = Phys::<5>::new(0).unwrap(); for log2size in 0..=4u8 { - let queue = Queue::new(log2size).unwrap(); + let (_, commands) = unit.command_queue(low, log2size).unwrap(); + let (_, events) = unit.event_queue(low, log2size).unwrap(); let size = 1usize << log2size; // Start anywhere, the wrap flags and the bits above them included. - for start in [0u32, 1 << log2size, (2 << log2size) - 1, 0xfff0_0000 | 1 << log2size] { + for start in [0u32, 1 << log2size, (2 << log2size) - 1, 0x7ff0_0000 | 1 << log2size] { let (mut prod, mut cons) = (start, start); let (mut produced, mut consumed) = (0usize, 0usize); - let first = queue.slot(start); + let first = commands.slot(start); // Fill, drain halfway, fill again, drain: three laps of the buffer. for step in 0..6 * size { let held = produced - consumed; - assert_eq!(queue.is_empty(prod, cons), held == 0, "size {size} step {step}"); - assert_eq!(queue.is_full(prod, cons), held == size, "size {size} step {step}"); - assert_eq!(queue.slot(prod), (first + produced) % size); - assert_eq!(queue.slot(cons), (first + consumed) % size); + assert_eq!(commands.is_empty(prod, cons), held == 0, "size {size} step {step}"); + assert_eq!(events.is_empty(prod, cons), held == 0, "size {size} step {step}"); + assert_eq!(commands.is_full(prod, cons), held == size, "size {size} step {step}"); + assert_eq!(commands.slot(prod), (first + produced) % size); + assert_eq!(events.slot(cons), (first + consumed) % size); if held < size && (step / size).is_multiple_of(2) { - prod = queue.after(prod); + prod = commands.after(prod); produced += 1; } else if held > 0 { - cons = queue.after(cons); + cons = events.after(cons, prod); consumed += 1; } } } } // The wrap flag toggles as the index passes the top, and nothing above it is written back. - let queue = Queue::new(3).unwrap(); - assert_eq!(queue.after(0b0_111), 0b1_000); - assert_eq!(queue.after(0b1_111), 0b0_000); - assert_eq!(queue.after(0xffff_fff7), 0b1_000); + let (_, commands) = unit.command_queue(low, 3).unwrap(); + assert_eq!(commands.after(0b0_111), 0b1_000); + assert_eq!(commands.after(0b1_111), 0b0_000); + assert_eq!(commands.after(0xffff_fff7), 0b1_000); // An index register is the unit's word: every bit set, it is still an index. - assert_eq!(queue.after(u32::MAX), 0); - assert_eq!(Queue::new(19).unwrap().after(u32::MAX), 0); - assert!(Queue::new(19).is_some() && Queue::new(20).is_none()); + assert_eq!(commands.after(u32::MAX), 0); + assert_eq!(unit.command_queue(low, 19).unwrap().1.after(u32::MAX), 0); + assert_eq!(unit.event_queue(low, 19).unwrap().1.after(0x7fff_ffff, 0), 0); +} + +/// §7.4: records were lost while `SMMU_EVENTQ_PROD.OVFLG` [31] differs from +/// `SMMU_EVENTQ_CONS.OVACKFLG` [31], and the consumer acknowledges by +/// writing its flag to the value it read `OVFLG` with. +#[test] +fn consuming_an_event_acknowledges_an_overflow_and_never_raises_one() { + let unit = idr(&[], &[(20, 16, 19)], &[]).unwrap(); + let (_, events) = unit.event_queue(Phys::new(0).unwrap(), 3).unwrap(); + const OVFLG: u32 = 1 << 31; + for (prod, cons) in [(0b0_101, 0b0_011), (OVFLG | 0b0_101, OVFLG | 0b0_011)] { + assert!(!events.overflowed(prod, cons)); + // The flag is written back as it was read: no overflow is forged. + let written = events.after(cons, prod); + assert_eq!(written, prod & OVFLG | 0b0_100); + assert!(!events.overflowed(prod, written)); + } + for (prod, cons) in [(OVFLG | 0b0_101, 0b0_011), (0b0_101, OVFLG | 0b0_011)] { + assert!(events.overflowed(prod, cons)); + let written = events.after(cons, prod); + assert_eq!(written, prod & OVFLG | 0b0_100); + assert!(!events.overflowed(prod, written)); + } + // The flags are no part of a position: a queue whose flags differ is as empty as its indexes say. + assert!(events.is_empty(OVFLG | 0b1_010, 0b1_010)); + assert_eq!(events.slot(OVFLG | 0b1_010), 0b010); } // --- chapter 4: commands ---------------------------------------------------- @@ -327,11 +444,6 @@ fn each_command_holds_its_opcode_and_parameters_where_its_diagram_has_them() { let asid = whole().asid(0xa5c3).unwrap(); // CMD_CFGI_STE: StreamID [63:32], Leaf [64], SSec [10] clear. assert_eq!(Command::ForgetStream(0xdead_beef).words(), layout(&[(7, 0, 0x03), (63, 32, 0xdead_beef), (64, 64, 1)])); - // CMD_CFGI_CD: StreamID [63:32], SubstreamID [31:12] zero, Leaf [64]. - assert_eq!( - Command::ForgetContext(0xdead_beef).words(), - layout(&[(7, 0, 0x05), (31, 12, 0), (63, 32, 0xdead_beef), (64, 64, 1)]) - ); // CMD_CFGI_ALL: CMD_CFGI_STE_RANGE with Range [68:64] of 31. assert_eq!(Command::ForgetAll.words(), layout(&[(7, 0, 0x04), (68, 64, 31)])); // CMD_TLBI_NH_ASID: ASID [63:48], VMID [47:32]. @@ -340,7 +452,6 @@ fn each_command_holds_its_opcode_and_parameters_where_its_diagram_has_them() { // CMD_SYNC: CS [13:12]; MSIAddress [119:66] zero, so SIG_IRQ is the wired interrupt. assert_eq!(Command::Sync(Signal::None).words(), layout(&[(7, 0, 0x46), (13, 12, 0b00)])); assert_eq!(Command::Sync(Signal::Irq).words(), layout(&[(7, 0, 0x46), (13, 12, 0b01), (119, 66, 0)])); - assert_eq!(Command::Sync(Signal::Sev).words(), layout(&[(7, 0, 0x46), (13, 12, 0b10)])); } // --- §7.3: event records ---------------------------------------------------- @@ -424,7 +535,7 @@ const AT: u64 = 0x1a5 << 39 | 0x0f0 << 30 | 0x00f << 21; #[test] fn memory_is_a_level_2_block_only_its_own_context_and_an_unprivileged_device_reach() { - let block = Phys::<21>::new(0xffff_ffe0_0000).unwrap(); + let block = Phys::<21>::new(0xfff_ffe0_0000).unwrap(); let fields = |read_only: u64| { word(&[ (0, 0, 1), // valid @@ -435,21 +546,22 @@ fn memory_is_a_level_2_block_only_its_own_context_and_an_unprivileged_device_rea (9, 8, 0b11), // SH: inner shareable (10, 10, 1), // AF (11, 11, 1), // nG - (47, 21, 0xffff_ffe0_0000 >> 21), // the output address + (47, 21, 0xfff_ffe0_0000 >> 21), // the output address (53, 53, 1), // PXN (54, 54, 1), // UXN ]) }; - let path = plan(AT, Leaf::Memory(block, Access::ReadWrite)).expect("a 2 MiB aligned address"); + let unit = whole(); + let path = plan(AT, Leaf::Memory(block, Access::ReadWrite), &unit).expect("a 2 MiB aligned address"); assert_eq!((path.indices(), path.descriptor), (&[0x1a5, 0x0f0, 0x00f][..], fields(0))); - let path = plan(AT, Leaf::Memory(block, Access::Read)).expect("a 2 MiB aligned address"); + let path = plan(AT, Leaf::Memory(block, Access::Read), &unit).expect("a 2 MiB aligned address"); assert_eq!((path.indices(), path.descriptor), (&[0x1a5, 0x0f0, 0x00f][..], fields(1))); } #[test] fn the_doorbell_is_one_level_3_page_of_device_memory() { let page = Phys::<12>::new(0x0809_0000).unwrap(); - let path = plan(AT | 0x155 << 12, Leaf::Doorbell(page)).expect("a 4 KiB aligned address"); + let path = plan(AT | 0x155 << 12, Leaf::Doorbell(page), &whole()).expect("a 4 KiB aligned address"); assert_eq!(path.indices(), [0x1a5, 0x0f0, 0x00f, 0x155]); assert_eq!( path.descriptor, @@ -466,39 +578,42 @@ fn the_doorbell_is_one_level_3_page_of_device_memory() { (54, 54, 1), ]) ); - // Attribute 0 is Device-nGnRE and attribute 1 Normal write-back. - assert_eq!(MAIR0.to_le_bytes(), [0x04, 0xff, 0, 0]); + // The attributes the two leaves' AttrIndx name: 0 is Device-nGnRE and 1 Normal write-back. + assert_eq!(MAIR0.to_le_bytes()[..2], [0x04, 0xff]); } #[test] fn an_address_past_the_input_or_off_the_leafs_alignment_has_no_plan() { + let unit = whole(); let block = Leaf::Memory(Phys::new(0x20_0000).unwrap(), Access::ReadWrite); let page = Leaf::Doorbell(Phys::new(0x1000).unwrap()); - assert_eq!(plan(1 << INPUT_BITS, block), None); - assert_eq!(plan(1 << INPUT_BITS, page), None); - assert_eq!(plan(!0x1f_ffff, block), None); - assert!(plan((1 << INPUT_BITS) - (1 << 21), block).is_some()); + assert_eq!(plan(1 << INPUT_BITS, block, &unit), None); + assert_eq!(plan(1 << INPUT_BITS, page, &unit), None); + assert_eq!(plan(!0x1f_ffff, block, &unit), None); + assert!(plan((1 << INPUT_BITS) - (1 << 21), block, &unit).is_some()); // 4 KiB aligned is enough for the page and not for the block. - assert_eq!(plan(AT | 0x1000, block), None); - assert!(plan(AT | 0x1000, page).is_some()); - assert_eq!(plan(AT | 0x800, page), None); + assert_eq!(plan(AT | 0x1000, block, &unit), None); + assert!(plan(AT | 0x1000, page, &unit).is_some()); + assert_eq!(plan(AT | 0x800, page, &unit), None); } #[test] fn a_descriptor_read_back_is_a_table_only_above_level_3() { - let table = Phys::<12>::new(0xffff_ffff_f000).unwrap(); + let unit = whole(); + let table = Phys::<12>::new(0xfff_ffff_f000).unwrap(); // Valid [0], table [1], the next table's address [47:12], and no hierarchical restriction. - assert_eq!(next(table), word(&[(0, 0, 1), (1, 1, 1), (47, 12, 0xffff_ffff_f000 >> 12)])); + let descriptor = next(table, &unit).expect("a table inside the unit's 44 bits"); + assert_eq!(descriptor, word(&[(0, 0, 1), (1, 1, 1), (47, 12, 0xfff_ffff_f000 >> 12)])); for level in 0..3 { - assert_eq!(entry(next(table), level), Entry::Table(table)); + assert_eq!(entry(descriptor, level), Entry::Table(table)); assert_eq!(entry(0, level), Entry::Invalid); // Bit [0] alone decides validity: a stale address under it is nothing. - assert_eq!(entry(next(table) & !1, level), Entry::Invalid); + assert_eq!(entry(descriptor & !1, level), Entry::Invalid); } - let block = plan(AT, Leaf::Memory(Phys::new(0x4020_0000).unwrap(), Access::ReadWrite)).unwrap(); + let block = plan(AT, Leaf::Memory(Phys::new(0x4020_0000).unwrap(), Access::ReadWrite), &unit).unwrap(); assert_eq!(entry(block.descriptor, 2), Entry::Mapped); // A level 3 page sets bit [1] as a table descriptor does: it is no table. - let page = plan(AT, Leaf::Doorbell(Phys::new(0x0809_0000).unwrap())).unwrap(); + let page = plan(AT, Leaf::Doorbell(Phys::new(0x0809_0000).unwrap()), &unit).unwrap(); assert_eq!(entry(page.descriptor, 3), Entry::Mapped); assert_eq!(entry(0, 3), Entry::Invalid); } From bf56c79efbbb3e1734302befccb4f1e2c2d43ba9 Mon Sep 17 00:00:00 2001 From: japabu Date: Fri, 9 Oct 2026 14:46:30 +0200 Subject: [PATCH 3/3] An SMMUv3 stream another mapping also reaches is refused, and an own-mapping index past its array is refused by name DEN 0049 E.g Table 13 makes the DeviceID mapping index "an index into the array of ID mapping". Where the node says the field is valid (the flag from node revision 5, an unwired control interrupt before it) an index at or past the array's count named no entry and read as a node with no mapping of its own, which routed: malformed firmware failing open. `iort` now refuses it as `IortRefused::OwnMapping`. `Route::Translated` named a stream without saying whether anyone else's transactions arrive under it. Overlap was judged on input IDs within the asked segment, so two RID ranges of one root complex, two segments, or a named component mapped onto the same streams of one SMMUv3 each answered `Translated` with the same stream, and a claim attaching its domain there would hand another function its DMA. The IORT holds every mapping that reaches the unit, so `route` now walks them once more and refuses a stream any other mapping puts out to the same unit as `IortRefused::SharedStream`, whatever the other node's type. A node type `check` does not decode has its ID array held to Table 4's header and its own length before that walk reads it. `IORT_NEEDED` leaves the crate's exports; nothing outside `iort.rs` reads it. The duplicated-declarations issue names its owner as `issues/toyos-runs-on-arm64.md`'s stage 6. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- ...e-what-the-aarch64-kernel-also-declares.md | 7 +- toyos-acpi/src/iort.rs | 57 +++++++-- toyos-acpi/src/lib.rs | 2 +- toyos-acpi/tests/iort.rs | 116 +++++++++++++++++- 4 files changed, 164 insertions(+), 18 deletions(-) diff --git a/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md b/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md index 6430618fbf3..06244c6ad8e 100644 --- a/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md +++ b/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md @@ -28,10 +28,9 @@ the two registers' offsets are `kernel/src/arch/aarch64/irqchip.rs`'s: one register's fields have two crates. `toyos_its::TRANSLATION_FRAME` and `toyos_gicv3::FRAME` are both the GIC's 64 KiB frame. -**Owner.** The ARM desktop gate's next two stages, on -`issues/toyos-runs-on-arm64.md`'s stage 6: the SMMUv3 unit, which writes -these tables from `kernel/src/arch/aarch64/` beside `paging.rs`, owns the -descriptor; the ITS, which edits `irqchip.rs` and reads `GICR_TYPER` for +**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6: the SMMUv3 unit, which +writes these tables from `kernel/src/arch/aarch64/` beside `paging.rs`, owns +the descriptor; the ITS, which edits `irqchip.rs` and reads `GICR_TYPER` for every CPU, owns the redistributor. **Exit.** Each bit and the level index has one declaration that the kernel's diff --git a/toyos-acpi/src/iort.rs b/toyos-acpi/src/iort.rs index 0c63c2abc3c..d8d5854fda4 100644 --- a/toyos-acpi/src/iort.rs +++ b/toyos-acpi/src/iort.rs @@ -21,7 +21,7 @@ use crate::{find_table, Phys, Table, TableError, SDT_HEADER_LEN, SDT_REVISION}; /// 40, one reserved word, and the node array no earlier than that. const NODE_COUNT: usize = SDT_HEADER_LEN; const NODE_ARRAY: usize = SDT_HEADER_LEN + 4; -pub const IORT_NEEDED: usize = SDT_HEADER_LEN + 12; +const IORT_NEEDED: usize = SDT_HEADER_LEN + 12; /// The table revisions decoded: 5 is issue E.d's, 6 issues E.e's and E.f's and /// 7 issue E.g's. Each carries a revision in every node, which is what a @@ -91,8 +91,15 @@ pub enum IortRefused { /// may hold: issue E.g fixes the count at 1, and nothing here routes to /// a group. ItsGroup { at: usize, count: u32 }, + /// An SMMUv3 whose node says it has a DeviceID mapping of its own names + /// no entry of its ID array: Table 13's index is into that array. + OwnMapping { at: usize, index: u32, count: u32 }, /// Two mappings claim this ID, which the table must route one way. Overlap { id: u32 }, + /// The requester's StreamID is put out to its SMMUv3 by another mapping + /// too, of the node at `at`: another requester's transactions arrive + /// under it, and a domain attached to the stream would be theirs as well. + SharedStream { stream: u32, at: usize }, } impl From for IortRefused { @@ -136,10 +143,10 @@ pub struct ItsDevice { /// Where one requester's transactions go. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Route { - /// Through this SMMUv3 under `stream`; `its` is where the SMMU's own node - /// maps that stream on to, and `None` where it maps it nowhere. The - /// unit's own DeviceID (Table 13, `DeviceID mapping index`) is no - /// stream's. + /// Through this SMMUv3 under `stream`, which no other mapping of the table + /// puts out to that unit; `its` is where the SMMU's own node maps that + /// stream on to, and `None` where it maps it nowhere. The unit's own + /// DeviceID (Table 13, `DeviceID mapping index`) is no stream's. Translated { smmu: Smmuv3, stream: u32, its: Option }, /// Straight to an ITS, past every SMMU. Untranslated(ItsDevice), @@ -225,7 +232,7 @@ impl Iort

{ } } } - let Some((root, (mapping, id))) = hit else { + let Some((root, (index, mapping, id))) = hit else { return Ok(Route::Unmapped); }; match self.output(root, mapping)? { @@ -239,11 +246,12 @@ impl Iort

{ } let its = match onward { None => None, - Some((mapping, device)) => match self.output(unit, mapping)? { + Some((_, mapping, device)) => match self.output(unit, mapping)? { (_, Node::Its { id: its }) => Some(ItsDevice { its, device }), _ => return Err(IortRefused::Output { at: unit.at, reference: mapping.reference }), }, }; + self.alone(unit, (root.at, index), id)?; Ok(Route::Translated { smmu, stream: id, its }) } _ => Err(IortRefused::Output { at: root.at, reference: mapping.reference }), @@ -271,15 +279,41 @@ impl Iort

{ } } - /// The mappings of `raw` that hold `id`, each with the ID it puts out. - fn hits(&self, raw: Raw, id: u32) -> impl Iterator + '_ { + /// The mappings of `raw` that hold `id`, each with its index and the ID + /// it puts out. + fn hits(&self, raw: Raw, id: u32) -> impl Iterator + '_ { let own = self.own(raw); (0..raw.mappings).filter(move |index| Some(*index) != own).filter_map(move |index| { let mapping = self.mapping(raw, index)?; - Some((mapping, mapping.maps(id)?)) + Some((index, mapping, mapping.maps(id)?)) }) } + /// Refuses `stream` of the SMMUv3 `unit` where any mapping but `by` — + /// mapping `by.1` of the node at `by.0` — puts it out to that unit, + /// whatever its node's type: one walk of the nodes and their mappings. + fn alone(&self, unit: Raw, by: (usize, u32), stream: u32) -> Result<(), IortRefused> { + for raw in self.raw_nodes() { + let raw = raw?; + let Raw { at, mappings: count, array: offset, .. } = raw; + // [`Iort::check`] placed the arrays of the types it decodes; a + // type it does not is held to Table 4's header and its own length. + let end = u64::from(offset) + u64::from(count) * MAPPING as u64; + if count != 0 && ((offset as usize) < NODE_HEADER || end > raw.len as u64) { + return Err(IortRefused::Mappings { at, count, offset }); + } + for index in (0..count).filter(|index| (at, *index) != by) { + let mapping = self.mapping(raw, index).ok_or(IortRefused::Mappings { at, count, offset })?; + let last = if mapping.single { 0 } else { mapping.span }; + let reaches = stream >= mapping.output && stream - mapping.output <= last; + if reaches && mapping.reference as usize == unit.at { + return Err(IortRefused::SharedStream { stream, at }); + } + } + } + Ok(()) + } + /// Walks by each node's own length, which [`Iort::raw`] bounds below by /// the header and above by the table: at most `len / 16` steps. fn raw_nodes(&self) -> impl Iterator> + '_ { @@ -400,6 +434,9 @@ impl Iort

{ return Err(IortRefused::Mappings { at, count, offset }); } let own = self.own(raw); + if let Some(index) = own.filter(|index| *index >= count) { + return Err(IortRefused::OwnMapping { at, index, count }); + } for index in (0..count).filter(|index| Some(*index) != own) { let mapping = self.mapping(raw, index).ok_or(IortRefused::Mappings { at, count, offset })?; if mapping.single { diff --git a/toyos-acpi/src/lib.rs b/toyos-acpi/src/lib.rs index c16d58441e1..794c4089665 100644 --- a/toyos-acpi/src/lib.rs +++ b/toyos-acpi/src/lib.rs @@ -35,7 +35,7 @@ pub use fadt::{ FADT_FOR_FIXED_HARDWARE, FADT_FOR_RESET, FADT_PM1A_CNT_BLK, }; use fadt::FADT_DSDT; -pub use iort::{iort, Iort, IortRefused, ItsDevice, Node, Route, Smmuv3, IORT_NEEDED}; +pub use iort::{iort, Iort, IortRefused, ItsDevice, Node, Route, Smmuv3}; pub use madt::{ isa_line, madt_entries, sci_line, Gicc, IoApicEntry, Line, MadtEntries, MadtEntry, MadtHalt, Polarity, SourceOverride, Trigger, MADT_ENTRIES, diff --git a/toyos-acpi/tests/iort.rs b/toyos-acpi/tests/iort.rs index f13b7984d53..f1db24f6774 100644 --- a/toyos-acpi/tests/iort.rs +++ b/toyos-acpi/tests/iort.rs @@ -299,7 +299,7 @@ fn a_node_revision_whose_layout_was_not_read_is_refused() { for (kind, node, known, unknown) in [ (0u8, its(0), &[1u8][..], &[0u8, 2][..]), (2, root_complex(0, &[]), &[3, 4], &[2, 5]), - (4, smmu(&[]), &[4, 5], &[3, 6]), + (4, smmu_with(5, 0, WIRED, 0, &[]), &[4, 5], &[3, 6]), ] { for &revision in known { assert_eq!(revised(node.clone(), revision), None, "type {kind} revision {revision}"); @@ -450,9 +450,99 @@ fn a_single_mapping_that_is_not_its_smmus_own_is_refused() { let own = smmu_with(5, INDEX_VALID, WIRED, 0, &[[0, 0, 0x30, ITS_AT, 1]]); assert_eq!(refusal(&behind(own.clone())), None); assert_eq!(onward(&behind(own)), None); - // An index past the array names no entry: every mapping there is routes. + // An index past the array names no entry, and is no reading that routes. let past = behind(smmu_with(5, INDEX_VALID, WIRED, 1, &[[0, 0xff, 0x100, ITS_AT, 0]])); - assert_eq!(onward(&past), Some(ItsDevice { its: 7, device: 0x108 })); + assert_eq!(refusal(&past), Some(IortRefused::OwnMapping { at: 72, index: 1, count: 1 })); +} + +#[test] +fn an_smmus_own_mapping_index_at_or_past_its_id_array_is_refused() { + let one = [[0, 0xff, 0x100, ITS_AT, 0]]; + for index in [1, 2, u32::MAX] { + let past = behind(smmu_with(5, INDEX_VALID, WIRED, index, &one)); + assert_eq!(refusal(&past), Some(IortRefused::OwnMapping { at: 72, index, count: 1 }), "index {index}"); + } + assert_eq!( + refusal(&behind(smmu_with(5, INDEX_VALID, WIRED, 0, &[]))), + Some(IortRefused::OwnMapping { at: 72, index: 0, count: 0 }) + ); + // Revision 4 reads the index wherever a control interrupt has no GSIV. + let unwired = [106, 0, 109, 108]; + assert_eq!( + refusal(&behind(smmu_with(4, 0, unwired, 1, &one))), + Some(IortRefused::OwnMapping { at: 72, index: 1, count: 1 }) + ); + // Where the node says the field is ignored, no value of it is refused. + for (revision, flags, gsivs) in [(5, 0, WIRED), (5, 0, [0; 4]), (4, INDEX_VALID, WIRED)] { + let ignored = behind(smmu_with(revision, flags, gsivs, 1, &one)); + assert_eq!(onward(&ignored), Some(ItsDevice { its: 7, device: 0x108 }), "revision {revision}"); + } +} + +/// Requesters `0x00..=0x0f` to streams `0x10..=0x1f` of the SMMUv3 at `SMMU_AT`. +const FIRST: [u32; 5] = [0, 0xf, 0x10, SMMU_AT, 0]; + +fn alone(stream: u32) -> Result { + Ok(Route::Translated { smmu: plain_smmu(), stream, its: None }) +} + +fn shared(stream: u32, at: u32) -> Result { + Err(IortRefused::SharedStream { stream, at: at as usize }) +} + +#[test] +fn a_stream_another_mapping_puts_out_to_the_same_smmu_is_refused() { + // Two RID ranges of one root complex onto `0x18..=0x1f`, and a third onto the streams after. + let t = three(&[FIRST, [0x100, 0xf, 0x18, SMMU_AT, 0], [0x200, 0xf, 0x28, SMMU_AT, 0]], &[]); + assert_eq!(route(&t, 0, 0x7), alone(0x17)); + assert_eq!(route(&t, 0, 0x8), shared(0x18, ROOT_AT)); + assert_eq!(route(&t, 0, 0xf), shared(0x1f, ROOT_AT)); + assert_eq!(route(&t, 0, 0x100), shared(0x18, ROOT_AT)); + assert_eq!(route(&t, 0, 0x107), shared(0x1f, ROOT_AT)); + assert_eq!(route(&t, 0, 0x108), alone(0x20)); + assert_eq!(route(&t, 0, 0x200), alone(0x28)); + + // Two segments onto the same streams of one unit. + const SECOND: u32 = ROOT_AT + 56; + let segments = table(5, &[its(7), smmu(&[]), root_complex(0, &[FIRST]), root_complex(1, &[FIRST])]); + assert_eq!(route(&segments, 0, 8), shared(0x18, SECOND)); + assert_eq!(route(&segments, 1, 8), shared(0x18, ROOT_AT)); + + // The same stream numbers of another unit are another unit's streams. + let units = table( + 5, + &[its(7), smmu(&[]), root_complex(0, &[FIRST]), smmu(&[]), root_complex(1, &[[0, 0xf, 0x10, SECOND, 0]])], + ); + assert_eq!(route(&units, 0, 8), alone(0x18)); + assert_eq!(route(&units, 1, 8), alone(0x18)); + + // A node of a type not decoded is a requester too: a named component's single mapping + // puts out its base alone, whatever its span. + let named = |reference| node(1, 4, &[0; 12], &[[0, 0xf, 0x18, reference, 1]]); + let t = table(5, &[its(7), smmu(&[]), root_complex(0, &[FIRST]), named(SMMU_AT)]); + assert_eq!(route(&t, 0, 8), shared(0x18, SECOND)); + assert_eq!(route(&t, 0, 7), alone(0x17)); + assert_eq!(route(&t, 0, 9), alone(0x19)); + // To an ITS, `0x18` is a DeviceID and no stream. + assert_eq!(route(&table(5, &[its(7), smmu(&[]), root_complex(0, &[FIRST]), named(ITS_AT)]), 0, 8), alone(0x18)); +} + +/// [`Iort::check`] places only the ID arrays of the types it decodes; a stream +/// is judged alone only once every other node's array is read where its +/// header puts it. +#[test] +fn a_stream_is_not_judged_alone_past_an_id_array_out_of_its_node() { + const NAMED: usize = (ROOT_AT + 56) as usize; + let at = |offset: u32, count: u32| { + let mut named = node(1, 4, &[0; 12], &[[0, 0, 0x30, ITS_AT, 1]]); + named[8..12].copy_from_slice(&count.to_le_bytes()); + named[12..16].copy_from_slice(&offset.to_le_bytes()); + route(&table(5, &[its(7), smmu(&[]), root_complex(0, &[FIRST]), named, its(9)]), 0, 8) + }; + assert_eq!(at(28, 1), alone(0x18)); + assert_eq!(at(28, 2), Err(IortRefused::Mappings { at: NAMED, count: 2, offset: 28 })); + assert_eq!(at(0, 1), Err(IortRefused::Mappings { at: NAMED, count: 1, offset: 0 })); + assert_eq!(at(12, 1), Err(IortRefused::Mappings { at: NAMED, count: 1, offset: 12 })); } /// `virt`'s four GSIVs: every control interrupt wired. @@ -636,4 +726,24 @@ fn opening_a_table_and_routing_a_requester_each_read_it_a_bounded_number_of_time let routed = reads.replace(0); assert!(routed <= bytes.len() / 8, "routing through {} bytes read {routed}", bytes.len()); assert_eq!(opened.route(ROOTS, 0), Ok(Route::Unmapped)); + + // Every mapping to one SMMUv3, each onto streams of its own: a stream is + // judged alone by one more walk of the nodes and their mappings. + let mut nodes: Vec> = vec![smmu(&[])]; + nodes.extend((0..ROOTS).map(|segment| { + let mappings: Vec<[u32; 5]> = + (0..40).map(|i| [i * 16, 15, (segment * 40 + i) * 16, 48, 0]).collect(); + root_complex(segment, &mappings) + })); + let bytes = table(5, &nodes); + let regions: &[(u64, &[u8])] = &[(0x800, &head), (0x1000, &root), (TABLE_AT, &bytes)]; + let opened = iort(Counted { machine: Machine { regions }, reads: &reads }, 0x800).expect("a table of 500 nodes"); + reads.set(0); + let stream = ((ROOTS - 1) * 40 + 39) * 16 + 15; + assert_eq!( + opened.route(ROOTS - 1, 39 * 16 + 15), + Ok(Route::Translated { smmu: plain_smmu(), stream, its: None }) + ); + let routed = reads.get(); + assert!(routed <= 2 * bytes.len(), "routing through {} bytes read {routed}", bytes.len()); }