diff --git a/Cargo.lock b/Cargo.lock index 13dd732cf84..3be3dc36d98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5879,6 +5879,13 @@ dependencies = [ "toyos-abi", ] +[[package]] +name = "toyos-its" +version = "0.1.0" +dependencies = [ + "toyos-phys", +] + [[package]] name = "toyos-keymap" version = "0.1.0" @@ -5996,6 +6003,10 @@ dependencies = [ "toyos-abi", ] +[[package]] +name = "toyos-phys" +version = "0.1.0" + [[package]] name = "toyos-ps2" version = "0.1.0" @@ -6008,6 +6019,14 @@ version = "0.1.0" name = "toyos-rootimage" version = "0.1.0" +[[package]] +name = "toyos-smmu" +version = "0.1.0" +dependencies = [ + "toyos-bootmap", + "toyos-phys", +] + [[package]] name = "toyos-swap" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 8bd6a1e6d22..70e32e5f104 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ members = [ "toyos-hda", "toyos-i219", "toyos-inspect", + "toyos-its", "toyos-keymap", "toyos-logstream", "toyos-manifest", @@ -56,8 +57,10 @@ members = [ "toyos-net-udp", "toyos-net-wire", "toyos-osrelease", + "toyos-phys", "toyos-quiesce", "toyos-rootimage", + "toyos-smmu", "toyos-swap", "toyos-symbols", "toyos-tco", diff --git a/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md b/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md new file mode 100644 index 00000000000..06244c6ad8e --- /dev/null +++ b/issues/the-smmuv3-and-its-crates-declare-what-the-aarch64-kernel-also-declares.md @@ -0,0 +1,47 @@ +--- +status: open +kind: defect +opened: 2026-10-09 +--- + +# The SMMUv3 and ITS crates declare what the AArch64 kernel also declares + +`toyos-smmu` and `toyos-its` came in with no kernel file touched, so each +declaration below has a second home in `kernel/`, and nothing holds the two +equal. `toyos-smmu` takes the stage 1 descriptors, the `MAIR` and `PXN` from +`toyos-bootmap::aarch64` already; what is left is what `toyos-bootmap` does not +declare. + +**The stage 1 descriptor.** `toyos-smmu/src/table.rs` declares `VALID` (bit +0), `TABLE_OR_PAGE` (bit 1), `AP_UNPRIVILEGED` (bit 6), `AP_READ_ONLY` (bit +7), `NOT_GLOBAL` (bit 11) and the level index `at >> (39 - 9 * level) & 0x1FF`. +`kernel/src/arch/aarch64/paging.rs` declares the same bits as `VALID`, +`TABLE`, `AP_EL0`, `AP_READ_ONLY` and `NOT_GLOBAL`, and the same `index`. It +also declares `INNER_SHAREABLE`, `OUTER_SHAREABLE`, `AF`, `PXN` and `UXN`, +which `toyos-bootmap/src/aarch64.rs` holds: the kernel's own copy of that +crate's. + +**The redistributor.** `toyos-its/src/lpi.rs` declares `GICR_CTLR.EnableLPIs`, +`GICR_TYPER.PLPIS` and `GICR_TYPER.Processor_Number`. `GICR_TYPER`'s other +fields (`VLPIS`, `Last`, the affinity) are `kernel/gicv3/src/lib.rs`'s, and +the two registers' offsets are `kernel/src/arch/aarch64/irqchip.rs`'s: one +register's fields have two crates. `toyos_its::TRANSLATION_FRAME` and +`toyos_gicv3::FRAME` are both the GIC's 64 KiB frame. + +**Owner.** `issues/toyos-runs-on-arm64.md`'s stage 6: the SMMUv3 unit, which +writes these tables from `kernel/src/arch/aarch64/` beside `paging.rs`, owns +the descriptor; the ITS, which edits `irqchip.rs` and reads `GICR_TYPER` for +every CPU, owns the redistributor. + +**Exit.** Each bit and the level index has one declaration that the kernel's +tables, the loader's and the unit's are all written from, and each +redistributor register's fields have one crate: + +``` +rg -n 'const (VALID|TABLE|TABLE_OR_PAGE|AP_\w+|NOT_GLOBAL|INNER_SHAREABLE|OUTER_SHAREABLE|AF|PXN|UXN): u64' \ + kernel/src/arch/aarch64/paging.rs toyos-smmu/src/table.rs +rg -n 'fn index' kernel/src/arch/aarch64/paging.rs toyos-smmu/src/table.rs +``` + +print nothing, and `rg -l 'GICR_TYPER' toyos-its kernel/gicv3` names one +directory. diff --git a/src/licence.rs b/src/licence.rs index 1c106d08124..6bfe3e1e1fd 100644 --- a/src/licence.rs +++ b/src/licence.rs @@ -270,6 +270,46 @@ pub const COMMITTED_FILES: &[(&str, &str, &str, Terms)] = &[ "ours: QEMU's own XSDT, captured by the commit that added toyos-acpi", Terms::Spdx("MIT OR Apache-2.0"), ), + // The tables QEMU 11.1.1 published to its `virt` machine in both of its + // GIC modes, read out of guest physical memory over the monitor with no + // operating system in the guest. Firmware output, as the q35 tables above: + // `toyos-acpi/tests/iort.rs` decodes them. + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/apic.bin", + "5f65e29aa3c1d4b7afce9f22c144797f2d4234a58e7fbeaae48c1744d0713976", + "ours: QEMU `virt`'s own MADT under the hypervisor's GIC, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/iort.bin", + "6b0a25a264422e32097f761d5d82cd7d5142bbe20a6d6ef22b7750c9ff7cd8e2", + "ours: QEMU `virt`'s own IORT under the hypervisor's GIC, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/its/apic.bin", + "ca5ef1376c5e30ee314a9a22c225fe34e9721889a6884bc36239c9d145759452", + "ours: QEMU `virt`'s own MADT under its emulated GIC and ITS, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/its/iort.bin", + "57b1d81a2995c018d623a8d4a0ecb7318624f03065c3deb820773f3c7d5c7a93", + "ours: QEMU `virt`'s own IORT under its emulated GIC and ITS, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/rsdp.bin", + "88f4591b661d951ae67bca4b740dbebf8e0382b1e27dc915ae44ceddb3a09df7", + "ours: QEMU `virt`'s own RSDP, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), + ( + "toyos-acpi/fixtures/qemu-11.1.1-virt/xsdt.bin", + "8129cb6793e7b6f375e60f6e9780feff0b11b1e99da8c0b1dc4135700123725d", + "ours: QEMU `virt`'s own XSDT, captured by the commit that decodes the IORT", + Terms::Spdx("MIT OR Apache-2.0"), + ), ( "toyos-acpi/fixtures/ovmf-pure-efi/root-bridge-0.bin", "eb00e68be746a09ac7f0ce1ca492ce8c858e3af1152112b49ffb4708883acbfb", diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/SOURCE b/toyos-acpi/fixtures/qemu-11.1.1-virt/SOURCE new file mode 100644 index 00000000000..df4becb2b96 --- /dev/null +++ b/toyos-acpi/fixtures/qemu-11.1.1-virt/SOURCE @@ -0,0 +1,28 @@ +The RSDP, the XSDT, the MADT and the IORT QEMU 11.1.1 published to its `virt` +machine under edk2-stable202408 (QEMU's own `edk2-aarch64-code.fd`), with no +operating system in the guest: the firmware was left to reach its shell, and +the guest's memory was then written out by the monitor's `pmemsave`. Every +address was walked host-side from the RSDP, found by its signature and both +checksums, so no byte here came through the decoder these fixtures judge. Each +table sums to zero. + + qemu-system-aarch64 -M -accel hvf -cpu host -smp 4 -m 256 \ + -nodefaults -display none \ + -drive if=pflash,format=raw,readonly=on,file=edk2-aarch64-code.fd + + hwgic/ -M virt,gic-version=3,iommu=smmuv3 + the hypervisor's own GICv3, which has no ITS: the MADT ends in a + GIC MSI Frame structure, and the IORT is an SMMUv3 and a root complex + its/ -M virt,gic-version=3,its=on,iommu=smmuv3,kernel-irqchip=off + QEMU's emulated GICv3 with its ITS: the MADT ends in a GIC ITS + structure, and the IORT is an ITS group, an SMMUv3 and a root complex + +Both boots put every table at the same address, so the RSDP and the XSDT are +one file each: + + rsdp.bin 0x4cb43018 xsdt.bin 0x4cb43f18 + apic.bin 0x4cb43c98 iort.bin 0x4cb43198 + +Those are `tests/iort.rs`'s regions, so the XSDT's own entries point at them +and the walk under test is the real one. The XSDT's other six entries (FACP, +PPTT, GTDT, MCFG, SPCR, DBG2) name tables that are not here. diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/apic.bin b/toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/apic.bin new file mode 100644 index 00000000000..5a4643dd969 Binary files /dev/null and b/toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/apic.bin differ diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/iort.bin b/toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/iort.bin new file mode 100644 index 00000000000..0cf52b52f67 Binary files /dev/null and b/toyos-acpi/fixtures/qemu-11.1.1-virt/hwgic/iort.bin differ diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/its/apic.bin b/toyos-acpi/fixtures/qemu-11.1.1-virt/its/apic.bin new file mode 100644 index 00000000000..7df2b73b126 Binary files /dev/null and b/toyos-acpi/fixtures/qemu-11.1.1-virt/its/apic.bin differ diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/its/iort.bin b/toyos-acpi/fixtures/qemu-11.1.1-virt/its/iort.bin new file mode 100644 index 00000000000..79861d87b49 Binary files /dev/null and b/toyos-acpi/fixtures/qemu-11.1.1-virt/its/iort.bin differ diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/rsdp.bin b/toyos-acpi/fixtures/qemu-11.1.1-virt/rsdp.bin new file mode 100644 index 00000000000..13616135617 Binary files /dev/null and b/toyos-acpi/fixtures/qemu-11.1.1-virt/rsdp.bin differ diff --git a/toyos-acpi/fixtures/qemu-11.1.1-virt/xsdt.bin b/toyos-acpi/fixtures/qemu-11.1.1-virt/xsdt.bin new file mode 100644 index 00000000000..fc360682de7 Binary files /dev/null and b/toyos-acpi/fixtures/qemu-11.1.1-virt/xsdt.bin differ diff --git a/toyos-acpi/src/iort.rs b/toyos-acpi/src/iort.rs new file mode 100644 index 00000000000..d8d5854fda4 --- /dev/null +++ b/toyos-acpi/src/iort.rs @@ -0,0 +1,453 @@ +//! The IORT (signature `IORT`): Arm's IO Remapping Table, DEN 0049 issue E.g +//! — which requester reaches which SMMUv3 under which StreamID, and which +//! DeviceID its messages carry to which ITS. +//! +//! [`iort`] answers a table every node of which it has walked: a node or a +//! mapping array the table cannot hold is its refusal, and so is a revision +//! whose layout was not read against the specification. An output reference +//! is followed, and judged, by the [`Iort::route`] that needs it: opening the +//! table and routing one requester each read it a bounded number of times, +//! whatever its nodes and mappings say. +//! +//! Only the three node types that route a PCI function are decoded — the ITS, +//! the root complex and the SMMUv3; any other is answered by its type alone, +//! for its caller to rule on, and is refused as the output of a mapping. + +use core::num::NonZeroU32; + +use crate::{find_table, Phys, Table, TableError, SDT_HEADER_LEN, SDT_REVISION}; + +/// Table 3: `Number of IORT Nodes` at 36, `Offset to Array of IORT Nodes` at +/// 40, one reserved word, and the node array no earlier than that. +const NODE_COUNT: usize = SDT_HEADER_LEN; +const NODE_ARRAY: usize = SDT_HEADER_LEN + 4; +const IORT_NEEDED: usize = SDT_HEADER_LEN + 12; + +/// The table revisions decoded: 5 is issue E.d's, 6 issues E.e's and E.f's and +/// 7 issue E.g's. Each carries a revision in every node, which is what a +/// node's own layout is judged by. +const REVISIONS: core::ops::RangeInclusive = 5..=7; + +/// Table 4: type (0), length (1..3), revision (3), identifier (4..8), number +/// of ID mappings (8..12), reference to the ID array (12..16). +const NODE_HEADER: usize = 16; +/// Table 5: input base, number of IDs minus one, output base, output +/// reference, flags — five words. +const MAPPING: usize = 20; +/// Table 6: bit 0, "apply the output base regardless of the input IDs". +const SINGLE: u32 = 1 << 0; + +const ITS: u8 = 0; +const ROOT_COMPLEX: u8 = 2; +const SMMUV3: u8 = 4; + +/// Table 16: one reserved word that counted the group's ITSs before issue +/// E.g and is 1 since, then the one identifier. +const ITS_LEN: usize = NODE_HEADER + 8; +/// Table 18: memory access properties (16..24), ATS attribute (24..28), PCI +/// segment number (28..32), then the address size limit and, from node +/// revision 4, the PASID capabilities and flags. +const RC_SEGMENT: usize = 28; +/// Table 13: base address (16..24), flags (24..28), a reserved word, VATOS +/// (32..40), model (40..44), the event, PRI, GERR and sync GSIVs (44..60), +/// proximity domain (60..64), DeviceID mapping index (64..68). +const SMMU_BASE: usize = 16; +const SMMU_FLAGS: usize = 24; +const SMMU_GSIVS: usize = 44; +const SMMU_INDEX: usize = 64; +const SMMU_LEN: usize = 68; +/// Table 14, bit 0: `COHACC override`. +const SMMU_COHACC: u32 = 1 << 0; +/// Table 14, bit 4, from node revision 5: `DeviceID mapping index valid`. +const SMMU_INDEXED: u32 = 1 << 4; + +/// Why an IORT cannot be used. `at` is a node's offset in the table. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum IortRefused { + Table(TableError), + /// A table revision outside the ones decoded. + Revision(u8), + /// The node array starts inside the table's own header fields. + NodeArray { offset: u32 }, + /// The node at `at` declares a length too short for its type, or one the + /// table cannot hold; a node count that overruns the table ends here. + Node { at: usize, declared: usize }, + /// A node revision whose layout was not read against the specification. + NodeRevision { at: usize, kind: u8, revision: u8 }, + /// The node's ID array lies over its own fields or runs past its end, or + /// an ITS node, which has none, declares one. + Mappings { at: usize, count: u32, offset: u32 }, + /// A mapping's input or output range runs past the 32-bit ID space. + Range { at: usize, base: u32, last: u32 }, + /// The mapping a requester is routed by names no node with its output + /// reference, or a node its source may not output to: a root complex + /// outputs to an SMMUv3 or an ITS, an SMMUv3 to an ITS. + Output { at: usize, reference: u32 }, + /// A mapping with the single-mapping flag that is not its SMMUv3's own. + /// Table 6 allows one in a root complex and in an SMMUv3, where it puts + /// every input ID out as one; nothing here routes that. + SingleMapping { at: usize }, + /// An ITS group of other than one ITS, which a table of revision 5 or 6 + /// may hold: issue E.g fixes the count at 1, and nothing here routes to + /// a group. + ItsGroup { at: usize, count: u32 }, + /// An SMMUv3 whose node says it has a DeviceID mapping of its own names + /// no entry of its ID array: Table 13's index is into that array. + OwnMapping { at: usize, index: u32, count: u32 }, + /// Two mappings claim this ID, which the table must route one way. + Overlap { id: u32 }, + /// The requester's StreamID is put out to its SMMUv3 by another mapping + /// too, of the node at `at`: another requester's transactions arrive + /// under it, and a domain attached to the stream would be theirs as well. + SharedStream { stream: u32, at: usize }, +} + +impl From for IortRefused { + fn from(error: TableError) -> Self { + Self::Table(error) + } +} + +/// An SMMUv3 as its node names it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Smmuv3 { + pub base: u64, + /// Table 14's `COHACC override`: set, the unit reads its tables and + /// queues coherently whatever its `SMMU_IDR0.COHACC` says. + pub coherent_override: bool, + /// The wired interrupts; `None` where the node gives no GSIV. + pub event: Option, + pub gerror: Option, + pub sync: Option, +} + +/// One node, in the shape its reader acts on. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Node { + /// `id` is the MADT GIC ITS structure's. + Its { id: u32 }, + RootComplex { segment: u32 }, + Smmuv3(Smmuv3), + /// A type not decoded here: a named component (1), an SMMUv1 or v2 (3), a + /// PMCG (5), a reserved memory range (6), an IWB (7), or a reserved one. + Other(u8), +} + +/// The DeviceID a requester's messages carry, and the ITS they carry it to. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct ItsDevice { + pub its: u32, + pub device: u32, +} + +/// Where one requester's transactions go. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Route { + /// Through this SMMUv3 under `stream`, which no other mapping of the table + /// puts out to that unit; `its` is where the SMMU's own node maps that + /// stream on to, and `None` where it maps it nowhere. The unit's own + /// DeviceID (Table 13, `DeviceID mapping index`) is no stream's. + Translated { smmu: Smmuv3, stream: u32, its: Option }, + /// Straight to an ITS, past every SMMU. + Untranslated(ItsDevice), + /// No root complex of that segment maps the requester. + Unmapped, +} + +/// A node's header, bounded by the table. +#[derive(Clone, Copy)] +struct Raw { + at: usize, + kind: u8, + len: usize, + revision: u8, + mappings: u32, + array: u32, +} + +#[derive(Clone, Copy)] +struct Mapping { + input: u32, + /// The number of IDs, minus one. + span: u32, + output: u32, + reference: u32, + single: bool, +} + +impl Mapping { + fn maps(&self, id: u32) -> Option { + // `output + span` was checked not to wrap when the table was opened. + (id >= self.input && id - self.input <= self.span).then(|| self.output + (id - self.input)) + } +} + +/// An IORT whose every node [`iort`] has walked. +#[derive(Clone, Copy)] +pub struct Iort

{ + table: Table

, + nodes: u32, + array: usize, +} + +/// The IORT at `rsdp_addr`, every node of it checked. +pub fn iort(phys: P, rsdp_addr: u64) -> Result, IortRefused> { + let table = find_table(phys, rsdp_addr, b"IORT", IORT_NEEDED)?; + let short = TableError::Length { declared: table.len() as u32, needed: IORT_NEEDED }; + let revision = table.byte(SDT_REVISION).ok_or(short)?; + if !REVISIONS.contains(&revision) { + return Err(IortRefused::Revision(revision)); + } + let nodes = table.u32_at(NODE_COUNT).ok_or(short)?; + let offset = table.u32_at(NODE_ARRAY).ok_or(short)?; + if (offset as usize) < IORT_NEEDED { + return Err(IortRefused::NodeArray { offset }); + } + let iort = Iort { table, nodes, array: offset as usize }; + for node in iort.raw_nodes() { + iort.check(node?)?; + } + Ok(iort) +} + +impl Iort

{ + /// Every node, in the table's order. + pub fn nodes(&self) -> impl Iterator + '_ { + // `iort` walked the same headers and decoded the same nodes. + self.raw_nodes().filter_map(|raw| self.decode(raw.ok()?).ok()) + } + + /// Where the PCI function `rid` of `segment` is routed. + pub fn route(&self, segment: u32, rid: u16) -> Result { + let rid = u32::from(rid); + let mut hit = None; + for raw in self.raw_nodes() { + let raw = raw?; + if self.decode(raw)? != (Node::RootComplex { segment }) { + continue; + } + for found in self.hits(raw, rid) { + if hit.replace((raw, found)).is_some() { + return Err(IortRefused::Overlap { id: rid }); + } + } + } + let Some((root, (index, mapping, id))) = hit else { + return Ok(Route::Unmapped); + }; + match self.output(root, mapping)? { + (_, Node::Its { id: its }) => Ok(Route::Untranslated(ItsDevice { its, device: id })), + (unit, Node::Smmuv3(smmu)) => { + let mut onward = None; + for found in self.hits(unit, id) { + if onward.replace(found).is_some() { + return Err(IortRefused::Overlap { id }); + } + } + let its = match onward { + None => None, + Some((_, mapping, device)) => match self.output(unit, mapping)? { + (_, Node::Its { id: its }) => Some(ItsDevice { its, device }), + _ => return Err(IortRefused::Output { at: unit.at, reference: mapping.reference }), + }, + }; + self.alone(unit, (root.at, index), id)?; + Ok(Route::Translated { smmu, stream: id, its }) + } + _ => Err(IortRefused::Output { at: root.at, reference: mapping.reference }), + } + } + + /// The index of the mapping that is an SMMUv3's own DeviceID (Table 13, + /// offset 64), where its node says it has one: by the flag from node + /// revision 5, and before it wherever a control interrupt has no GSIV. + /// The entry's input base and length are not read. + fn own(&self, raw: Raw) -> Option { + let t = &self.table; + if raw.kind != SMMUV3 { + return None; + } + let indexed = if raw.revision >= 5 { + t.u32_at(raw.at + SMMU_FLAGS)? & SMMU_INDEXED != 0 + } else { + (0..4).any(|i| t.u32_at(raw.at + SMMU_GSIVS + 4 * i) == Some(0)) + }; + if indexed { + t.u32_at(raw.at + SMMU_INDEX) + } else { + None + } + } + + /// The mappings of `raw` that hold `id`, each with its index and the ID + /// it puts out. + fn hits(&self, raw: Raw, id: u32) -> impl Iterator + '_ { + let own = self.own(raw); + (0..raw.mappings).filter(move |index| Some(*index) != own).filter_map(move |index| { + let mapping = self.mapping(raw, index)?; + Some((index, mapping, mapping.maps(id)?)) + }) + } + + /// Refuses `stream` of the SMMUv3 `unit` where any mapping but `by` — + /// mapping `by.1` of the node at `by.0` — puts it out to that unit, + /// whatever its node's type: one walk of the nodes and their mappings. + fn alone(&self, unit: Raw, by: (usize, u32), stream: u32) -> Result<(), IortRefused> { + for raw in self.raw_nodes() { + let raw = raw?; + let Raw { at, mappings: count, array: offset, .. } = raw; + // [`Iort::check`] placed the arrays of the types it decodes; a + // type it does not is held to Table 4's header and its own length. + let end = u64::from(offset) + u64::from(count) * MAPPING as u64; + if count != 0 && ((offset as usize) < NODE_HEADER || end > raw.len as u64) { + return Err(IortRefused::Mappings { at, count, offset }); + } + for index in (0..count).filter(|index| (at, *index) != by) { + let mapping = self.mapping(raw, index).ok_or(IortRefused::Mappings { at, count, offset })?; + let last = if mapping.single { 0 } else { mapping.span }; + let reaches = stream >= mapping.output && stream - mapping.output <= last; + if reaches && mapping.reference as usize == unit.at { + return Err(IortRefused::SharedStream { stream, at }); + } + } + } + Ok(()) + } + + /// Walks by each node's own length, which [`Iort::raw`] bounds below by + /// the header and above by the table: at most `len / 16` steps. + fn raw_nodes(&self) -> impl Iterator> + '_ { + let mut at = self.array; + let mut halted = false; + (0..self.nodes).map_while(move |_| { + if halted { + return None; + } + let raw = self.raw(at); + match raw { + Ok(raw) => at += raw.len, + Err(_) => halted = true, + } + Some(raw) + }) + } + + fn raw(&self, at: usize) -> Result { + let t = &self.table; + let header = || Some((t.byte(at)?, t.u16_at(at + 1)?, t.byte(at + 3)?, t.u32_at(at + 8)?, t.u32_at(at + 12)?)); + let Some((kind, declared, revision, mappings, array)) = header() else { + return Err(IortRefused::Node { at, declared: 0 }); + }; + let len = usize::from(declared); + let floor = match kind { + ITS => ITS_LEN, + ROOT_COMPLEX => RC_SEGMENT + 4, + SMMUV3 => SMMU_LEN, + _ => NODE_HEADER, + }; + if len < floor || at + len > t.len() { + return Err(IortRefused::Node { at, declared: len }); + } + Ok(Raw { at, kind, len, revision, mappings, array }) + } + + /// Mapping `index` of a node [`Iort::check`] accepted the array of. + fn mapping(&self, raw: Raw, index: u32) -> Option { + let at = raw.at + raw.array as usize + index as usize * MAPPING; + let t = &self.table; + Some(Mapping { + input: t.u32_at(at)?, + span: t.u32_at(at + 4)?, + output: t.u32_at(at + 8)?, + reference: t.u32_at(at + 12)?, + single: t.u32_at(at + 16)? & SINGLE != 0, + }) + } + + /// The node a mapping of `from` outputs to: one walk of the nodes. + fn output(&self, from: Raw, mapping: Mapping) -> Result<(Raw, Node), IortRefused> { + let refused = IortRefused::Output { at: from.at, reference: mapping.reference }; + for raw in self.raw_nodes() { + let raw = raw?; + if raw.at == mapping.reference as usize { + return Ok((raw, self.decode(raw)?)); + } + } + Err(refused) + } + + fn decode(&self, raw: Raw) -> Result { + let t = &self.table; + let Raw { at, kind, revision, .. } = raw; + let short = IortRefused::Node { at, declared: raw.len }; + let revised = |known: core::ops::RangeInclusive| { + if known.contains(&revision) { + Ok(()) + } else { + Err(IortRefused::NodeRevision { at, kind, revision }) + } + }; + Ok(match kind { + ITS => { + revised(1..=1)?; + let count = t.u32_at(at + NODE_HEADER).ok_or(short)?; + if count != 1 { + return Err(IortRefused::ItsGroup { at, count }); + } + Node::Its { id: t.u32_at(at + NODE_HEADER + 4).ok_or(short)? } + } + ROOT_COMPLEX => { + revised(3..=4)?; + Node::RootComplex { segment: t.u32_at(at + RC_SEGMENT).ok_or(short)? } + } + SMMUV3 => { + revised(4..=5)?; + // Event, PRI, GERR, sync: the PRI queue is not used, and its GSIV not answered. + let gsiv = |i: usize| t.u32_at(at + SMMU_GSIVS + 4 * i).map(NonZeroU32::new).ok_or(short); + Node::Smmuv3(Smmuv3 { + base: t.u64_at(at + SMMU_BASE).ok_or(short)?, + coherent_override: t.u32_at(at + SMMU_FLAGS).ok_or(short)? & SMMU_COHACC != 0, + event: gsiv(0)?, + gerror: gsiv(2)?, + sync: gsiv(3)?, + }) + } + other => Node::Other(other), + }) + } + + /// One node's fields, its ID array's place and every mapping's own words. + fn check(&self, raw: Raw) -> Result<(), IortRefused> { + let node = self.decode(raw)?; + // Where the node's own fields end, which its ID array may not lie over. + let fields = match (node, raw.revision) { + (Node::Other(_), _) => return Ok(()), + (Node::Its { .. }, _) => ITS_LEN, + (Node::RootComplex { .. }, 3) => 36, + (Node::RootComplex { .. }, _) => 40, + (Node::Smmuv3(_), _) => SMMU_LEN, + }; + let Raw { at, mappings: count, array: offset, .. } = raw; + let end = u64::from(offset) + u64::from(count) * MAPPING as u64; + let misplaced = (offset as usize) < fields || end > raw.len as u64; + if count != 0 && (misplaced || matches!(node, Node::Its { .. })) { + return Err(IortRefused::Mappings { at, count, offset }); + } + let own = self.own(raw); + if let Some(index) = own.filter(|index| *index >= count) { + return Err(IortRefused::OwnMapping { at, index, count }); + } + for index in (0..count).filter(|index| Some(*index) != own) { + let mapping = self.mapping(raw, index).ok_or(IortRefused::Mappings { at, count, offset })?; + if mapping.single { + return Err(IortRefused::SingleMapping { at }); + } + for base in [mapping.input, mapping.output] { + if base.checked_add(mapping.span).is_none() { + return Err(IortRefused::Range { at, base, last: mapping.span }); + } + } + } + Ok(()) + } +} diff --git a/toyos-acpi/src/lib.rs b/toyos-acpi/src/lib.rs index 120e58501bb..794c4089665 100644 --- a/toyos-acpi/src/lib.rs +++ b/toyos-acpi/src/lib.rs @@ -7,7 +7,8 @@ //! //! Multi-byte fields are composed from bytes, little-endian, so no firmware //! byte is transmuted into a type. Field offsets cite ACPI 6.5, except MCFG's -//! (PCI Firmware Specification), HPET's (IA-PC HPET Specification) and SPCR's (Microsoft's SPCR specification). +//! (PCI Firmware Specification), HPET's (IA-PC HPET Specification), SPCR's (Microsoft's SPCR specification) +//! and the IORT's (Arm DEN 0049). //! //! `no_std`, no allocation, no `unsafe`. @@ -18,6 +19,7 @@ mod ecdt; mod facs; mod fadt; mod gtdt; +mod iort; mod madt; mod resource; mod spcr; @@ -33,6 +35,7 @@ pub use fadt::{ FADT_FOR_FIXED_HARDWARE, FADT_FOR_RESET, FADT_PM1A_CNT_BLK, }; use fadt::FADT_DSDT; +pub use iort::{iort, Iort, IortRefused, ItsDevice, Node, Route, Smmuv3}; pub use madt::{ isa_line, madt_entries, sci_line, Gicc, IoApicEntry, Line, MadtEntries, MadtEntry, MadtHalt, Polarity, SourceOverride, Trigger, MADT_ENTRIES, diff --git a/toyos-acpi/tests/iort.rs b/toyos-acpi/tests/iort.rs new file mode 100644 index 00000000000..f1db24f6774 --- /dev/null +++ b/toyos-acpi/tests/iort.rs @@ -0,0 +1,749 @@ +//! The IORT: QEMU `virt`'s own tables in both of its GIC modes, decoded +//! against what was read off their bytes by hand with DEN 0049 open, and +//! every refusal over crafted ones. + +mod common; + +use core::cell::Cell; +use core::num::NonZeroU32; + +use common::{declare_len, reseal, rsdp, sdt, xsdt, Machine}; +use toyos_acpi::{ + find_table, iort, madt_entries, IortRefused, ItsDevice, MadtEntry, Node, Phys, Route, Smmuv3, TableError, + MADT_ENTRIES, +}; + +const RSDP: u64 = 0x4cb4_3018; +const XSDT: u64 = 0x4cb4_3f18; +const APIC: u64 = 0x4cb4_3c98; +const IORT: u64 = 0x4cb4_3198; + +const VIRT_RSDP: &[u8] = include_bytes!("../fixtures/qemu-11.1.1-virt/rsdp.bin"); +const VIRT_XSDT: &[u8] = include_bytes!("../fixtures/qemu-11.1.1-virt/xsdt.bin"); +/// `-M virt,gic-version=3,iommu=smmuv3` under HVF: the hypervisor's GIC. +const HWGIC: &[(u64, &[u8])] = &[ + (RSDP, VIRT_RSDP), + (XSDT, VIRT_XSDT), + (APIC, include_bytes!("../fixtures/qemu-11.1.1-virt/hwgic/apic.bin")), + (IORT, include_bytes!("../fixtures/qemu-11.1.1-virt/hwgic/iort.bin")), +]; +/// `-M virt,gic-version=3,its=on,iommu=smmuv3,kernel-irqchip=off`. +const ITS: &[(u64, &[u8])] = &[ + (RSDP, VIRT_RSDP), + (XSDT, VIRT_XSDT), + (APIC, include_bytes!("../fixtures/qemu-11.1.1-virt/its/apic.bin")), + (IORT, include_bytes!("../fixtures/qemu-11.1.1-virt/its/iort.bin")), +]; + +/// The node both modes publish: `arm-smmuv3` at `0x09050000`, `COHACC +/// override` set, and the GSIVs in the node's order — event 106, PRI 107, +/// GERR 109, sync 108 — of which GERR is the higher of the last two. Every +/// one is wired, so the revision 4 node's DeviceID mapping index is ignored +/// and its one mapping in the ITS boot routes streams. +fn virt_smmu() -> Smmuv3 { + Smmuv3 { + base: 0x0905_0000, + coherent_override: true, + event: NonZeroU32::new(106), + gerror: NonZeroU32::new(109), + sync: NonZeroU32::new(108), + } +} + +#[test] +fn the_hardware_gic_boot_routes_bus_zero_through_the_smmu_and_nothing_to_an_its() { + let table = iort(Machine { regions: HWGIC }, RSDP).expect("the IORT QEMU published"); + assert_eq!(table.nodes().collect::>(), [Node::Smmuv3(virt_smmu()), Node::RootComplex { segment: 0 }]); + for rid in [0x00, 0x08, 0xff] { + assert_eq!( + table.route(0, rid), + Ok(Route::Translated { smmu: virt_smmu(), stream: u32::from(rid), its: None }), + "requester {rid:#x}" + ); + } + // The one mapping is `0x00..=0xff`: bus 1 is mapped nowhere, and neither is another segment. + assert_eq!(table.route(0, 0x100), Ok(Route::Unmapped)); + assert_eq!(table.route(0, 0xffff), Ok(Route::Unmapped)); + assert_eq!(table.route(1, 0x08), Ok(Route::Unmapped)); +} + +#[test] +fn the_its_boot_routes_bus_zero_through_the_smmu_to_the_its_and_the_rest_past_it() { + let table = iort(Machine { regions: ITS }, RSDP).expect("the IORT QEMU published"); + assert_eq!( + table.nodes().collect::>(), + [Node::Its { id: 0 }, Node::Smmuv3(virt_smmu()), Node::RootComplex { segment: 0 }] + ); + for rid in [0x00, 0x08, 0xff] { + let id = u32::from(rid); + assert_eq!( + table.route(0, rid), + Ok(Route::Translated { smmu: virt_smmu(), stream: id, its: Some(ItsDevice { its: 0, device: id }) }), + "requester {rid:#x}" + ); + } + for rid in [0x100, 0x1234, 0xffff] { + assert_eq!( + table.route(0, rid), + Ok(Route::Untranslated(ItsDevice { its: 0, device: u32::from(rid) })), + "requester {rid:#x}" + ); + } + assert_eq!(table.route(1, 0x08), Ok(Route::Unmapped)); +} + +fn madt_of(regions: &[(u64, &[u8])]) -> Vec { + let table = find_table(Machine { regions }, RSDP, b"APIC", MADT_ENTRIES).expect("MADT"); + madt_entries(&table).map(|entry| entry.expect("a structure the list holds whole")).collect() +} + +/// `arm-gicv3-its` at `0x08080000`, under the id the IORT's ITS node carries. +#[test] +fn the_its_boots_madt_names_the_its_the_iort_routes_to() { + let its: Vec<_> = madt_of(ITS).into_iter().filter(|entry| matches!(entry, MadtEntry::Its { .. })).collect(); + assert_eq!(its, [MadtEntry::Its { id: 0, base: 0x0808_0000 }]); +} + +/// It ends in a GIC MSI Frame structure, type 0xD, which is no ITS. +#[test] +fn the_hardware_gic_boots_madt_names_no_its() { + let entries = madt_of(HWGIC); + assert!(!entries.iter().any(|entry| matches!(entry, MadtEntry::Its { .. }))); + assert_eq!(entries.last(), Some(&MadtEntry::Other(0xD))); +} + +// --- crafted tables ------------------------------------------------------- + +const TABLE_AT: u64 = 0x9000; + +/// One node: Table 4's header over `fields`, then `mappings`, the ID array +/// placed straight after the fields. +fn node(kind: u8, revision: u8, fields: &[u8], mappings: &[[u32; 5]]) -> Vec { + let mut n = vec![kind, 0, 0, revision, 0, 0, 0, 0]; + n.extend((mappings.len() as u32).to_le_bytes()); + n.extend(if mappings.is_empty() { 0u32 } else { 16 + fields.len() as u32 }.to_le_bytes()); + n.extend(fields); + for word in mappings.iter().flatten() { + n.extend(word.to_le_bytes()); + } + let len = n.len() as u16; + n[1..3].copy_from_slice(&len.to_le_bytes()); + n +} + +fn its(id: u32) -> Vec { + node(0, 1, &[1u32.to_le_bytes(), id.to_le_bytes()].concat(), &[]) +} + +/// A revision 3 root complex of `segment`: 20 bytes of fields. +fn root_complex(segment: u32, mappings: &[[u32; 5]]) -> Vec { + let mut fields = vec![0u8; 20]; + fields[12..16].copy_from_slice(&segment.to_le_bytes()); + node(2, 3, &fields, mappings) +} + +/// An SMMUv3 node of `revision` at `0x0905_0000`, 52 bytes of fields: Table +/// 13's flags (24), its event, PRI, GERR and sync GSIVs (44..60) and its +/// DeviceID mapping index (64). +fn smmu_with(revision: u8, flags: u32, gsivs: [u32; 4], index: u32, mappings: &[[u32; 5]]) -> Vec { + let mut fields = vec![0u8; 52]; + fields[..8].copy_from_slice(&0x0905_0000u64.to_le_bytes()); + fields[8..12].copy_from_slice(&flags.to_le_bytes()); + for (i, gsiv) in gsivs.iter().enumerate() { + fields[28 + 4 * i..32 + 4 * i].copy_from_slice(&gsiv.to_le_bytes()); + } + fields[48..52].copy_from_slice(&index.to_le_bytes()); + node(4, revision, &fields, mappings) +} + +/// A revision 5 SMMUv3 with no flag and no GSIV: none of its mappings is its own. +fn smmu(mappings: &[[u32; 5]]) -> Vec { + smmu_with(5, 0, [0; 4], 0, mappings) +} + +/// An IORT of `revision` over `nodes`, its array straight after the header. +fn table(revision: u8, nodes: &[Vec]) -> Vec { + let mut body = Vec::new(); + body.extend((nodes.len() as u32).to_le_bytes()); + body.extend(48u32.to_le_bytes()); + body.extend(0u32.to_le_bytes()); + body.extend(nodes.concat()); + sdt(b"IORT", revision, &body) +} + +/// Where each of three nodes laid out as `[its, smmu, root complex]` starts: +/// the root complex after an SMMUv3 of no mapping, and of one. +const ITS_AT: u32 = 48; +const SMMU_AT: u32 = ITS_AT + 24; +const ROOT_AT: u32 = SMMU_AT + 68; +const ROOT_AFTER_ONE: u32 = ROOT_AT + 20; + +/// The ITS boot's shape with every number chosen: requesters `0x100..=0x1ff` +/// to streams `0x1000..`, streams `0x1000..=0x10ff` to DeviceIDs `0x20000..`. +fn three(root: &[[u32; 5]], onward: &[[u32; 5]]) -> Vec { + table(5, &[its(7), smmu(onward), root_complex(0, root)]) +} + +fn judged(bytes: &[u8], ask: impl FnOnce(&toyos_acpi::Iort>) -> T) -> Result { + let head = rsdp(0x1000, 2, 36); + let root = xsdt(&[TABLE_AT]); + let regions: &[(u64, &[u8])] = &[(0x800, &head), (0x1000, &root), (TABLE_AT, bytes)]; + iort(Machine { regions }, 0x800).map(|table| ask(&table)) +} + +fn refusal(bytes: &[u8]) -> Option { + judged(bytes, |_| ()).err() +} + +fn route(bytes: &[u8], segment: u32, rid: u16) -> Result { + judged(bytes, |table| table.route(segment, rid))? +} + +fn plain_smmu() -> Smmuv3 { + Smmuv3 { base: 0x0905_0000, coherent_override: false, event: None, gerror: None, sync: None } +} + +#[test] +fn a_mapping_carries_an_id_by_its_offset_from_the_input_base_and_ends_at_its_last_id() { + let t = three(&[[0x100, 0xff, 0x1000, SMMU_AT, 0]], &[[0x1000, 0xff, 0x2_0000, ITS_AT, 0]]); + let through = |stream, device| { + Ok(Route::Translated { smmu: plain_smmu(), stream, its: Some(ItsDevice { its: 7, device }) }) + }; + assert_eq!(route(&t, 0, 0x100), through(0x1000, 0x2_0000)); + assert_eq!(route(&t, 0, 0x180), through(0x1080, 0x2_0080)); + // `Number of IDs` is the count minus one: the last ID is in, the next is out. + assert_eq!(route(&t, 0, 0x1ff), through(0x10ff, 0x2_00ff)); + assert_eq!(route(&t, 0, 0x200), Ok(Route::Unmapped)); + assert_eq!(route(&t, 0, 0x0ff), Ok(Route::Unmapped)); +} + +#[test] +fn a_stream_the_smmu_maps_nowhere_has_no_its() { + let t = three(&[[0, 0x1ff, 0, SMMU_AT, 0]], &[[0, 0xff, 0, ITS_AT, 0]]); + assert_eq!(route(&t, 0, 0x100), Ok(Route::Translated { smmu: plain_smmu(), stream: 0x100, its: None })); +} + +#[test] +fn a_revision_whose_layout_was_not_read_is_refused() { + for revision in [0, 4, 8, 0xff] { + assert_eq!(refusal(&table(revision, &[its(0)])), Some(IortRefused::Revision(revision))); + } + for revision in [5, 6, 7] { + assert_eq!(refusal(&table(revision, &[its(0)])), None); + } +} + +#[test] +fn a_table_too_short_for_its_own_header_fields_is_refused() { + let mut t = table(5, &[]); + declare_len(&mut t, 44); + assert_eq!(refusal(&t), Some(IortRefused::Table(TableError::Length { declared: 44, needed: 48 }))); +} + +#[test] +fn a_node_array_inside_the_header_is_refused() { + let mut t = table(5, &[its(0)]); + t[40..44].copy_from_slice(&40u32.to_le_bytes()); + reseal(&mut t); + assert_eq!(refusal(&t), Some(IortRefused::NodeArray { offset: 40 })); +} + +#[test] +fn a_node_count_that_overruns_the_table_is_refused_where_the_bytes_end() { + let mut t = table(5, &[its(0), its(1)]); + t[36..40].copy_from_slice(&3u32.to_le_bytes()); + reseal(&mut t); + assert_eq!(refusal(&t), Some(IortRefused::Node { at: 96, declared: 0 })); + // A count nothing could hold costs one refusal, not a walk of its length. + t[36..40].copy_from_slice(&u32::MAX.to_le_bytes()); + reseal(&mut t); + assert_eq!(refusal(&t), Some(IortRefused::Node { at: 96, declared: 0 })); +} + +#[test] +fn a_node_array_past_the_table_is_refused_as_its_first_node() { + let mut t = table(5, &[its(0)]); + t[40..44].copy_from_slice(&u32::MAX.to_le_bytes()); + reseal(&mut t); + assert_eq!(refusal(&t), Some(IortRefused::Node { at: u32::MAX as usize, declared: 0 })); +} + +#[test] +fn a_node_longer_than_the_table_or_shorter_than_its_type_is_refused() { + let mut long = table(5, &[its(0)]); + long[49..51].copy_from_slice(&25u16.to_le_bytes()); + reseal(&mut long); + assert_eq!(refusal(&long), Some(IortRefused::Node { at: 48, declared: 25 })); + + // Zero would never advance the walk; eight is below Table 4's own header. + for declared in [0u16, 8, 23] { + let mut short = table(5, &[its(0)]); + short[49..51].copy_from_slice(&declared.to_le_bytes()); + reseal(&mut short); + assert_eq!(refusal(&short), Some(IortRefused::Node { at: 48, declared: usize::from(declared) })); + } + + // An SMMUv3 node cut before its DeviceID mapping index, with bytes after it to read. + let mut cut = table(5, &[smmu(&[]), its(0)]); + cut[49..51].copy_from_slice(&64u16.to_le_bytes()); + reseal(&mut cut); + assert_eq!(refusal(&cut), Some(IortRefused::Node { at: 48, declared: 64 })); +} + +#[test] +fn a_node_revision_whose_layout_was_not_read_is_refused() { + let revised = |mut node: Vec, revision: u8| { + node[3] = revision; + refusal(&table(5, &[node])) + }; + for (kind, node, known, unknown) in [ + (0u8, its(0), &[1u8][..], &[0u8, 2][..]), + (2, root_complex(0, &[]), &[3, 4], &[2, 5]), + (4, smmu_with(5, 0, WIRED, 0, &[]), &[4, 5], &[3, 6]), + ] { + for &revision in known { + assert_eq!(revised(node.clone(), revision), None, "type {kind} revision {revision}"); + } + for &revision in unknown { + assert_eq!( + revised(node.clone(), revision), + Some(IortRefused::NodeRevision { at: 48, kind, revision }), + "type {kind} revision {revision}" + ); + } + } +} + +/// A root complex whose ID array is `offset` into it and which is `pad` bytes +/// longer than its revision 3 fields. +fn root_with_array_at(revision: u8, offset: u32, pad: usize) -> Vec { + let mut node = node(2, revision, &vec![0u8; 20 + pad], &[[0, 0, 0, 48, 0]]); + node[12..16].copy_from_slice(&offset.to_le_bytes()); + node +} + +#[test] +fn an_id_array_lying_over_its_nodes_own_fields_is_refused() { + let with = |root: Vec| refusal(&table(5, &[its(0), root])); + // Revision 3's fields end at 36 and revision 4's, with its PASID capabilities and flags, at 40. + assert_eq!(with(root_with_array_at(3, 36, 0)), None); + assert_eq!(with(root_with_array_at(3, 32, 0)), Some(IortRefused::Mappings { at: 72, count: 1, offset: 32 })); + assert_eq!(with(root_with_array_at(4, 40, 4)), None); + assert_eq!(with(root_with_array_at(4, 36, 4)), Some(IortRefused::Mappings { at: 72, count: 1, offset: 36 })); + // And inside the node's header. + assert_eq!(with(root_with_array_at(3, 0, 0)), Some(IortRefused::Mappings { at: 72, count: 1, offset: 0 })); +} + +#[test] +fn an_id_array_running_past_its_node_is_refused() { + let counted = |count: u32| { + let mut root = root_complex(0, &[[0, 0, 0, 48, 0], [1, 0, 1, 48, 0]]); + root[8..12].copy_from_slice(&count.to_le_bytes()); + refusal(&table(5, &[its(0), root])) + }; + assert_eq!(counted(2), None); + assert_eq!(counted(3), Some(IortRefused::Mappings { at: 72, count: 3, offset: 36 })); + // A count whose bytes no 32-bit length could hold. + assert_eq!(counted(u32::MAX), Some(IortRefused::Mappings { at: 72, count: u32::MAX, offset: 36 })); + + // Past its node and inside the table: the third mapping would be the next node's bytes. + let mut root = root_complex(0, &[[0, 0, 0, 48, 0], [1, 0, 1, 48, 0]]); + root[8..12].copy_from_slice(&3u32.to_le_bytes()); + assert_eq!( + refusal(&table(5, &[its(0), root, its(1)])), + Some(IortRefused::Mappings { at: 72, count: 3, offset: 36 }) + ); + + // The array's start alone past the node. + let mut root = root_complex(0, &[[0, 0, 0, 48, 0]]); + root[12..16].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!( + refusal(&table(5, &[its(0), root])), + Some(IortRefused::Mappings { at: 72, count: 1, offset: u32::MAX }) + ); +} + +#[test] +fn an_its_node_declaring_an_id_array_is_refused() { + let fields = [1u32.to_le_bytes(), 0u32.to_le_bytes()].concat(); + let node = node(0, 1, &fields, &[[0, 0, 0, 48, 0]]); + assert_eq!(refusal(&table(5, &[node])), Some(IortRefused::Mappings { at: 48, count: 1, offset: 24 })); +} + +#[test] +fn a_mapping_whose_range_leaves_the_id_space_is_refused() { + let input = three(&[[0xffff_ff00, 0x100, 0, SMMU_AT, 0]], &[]); + assert_eq!(refusal(&input), Some(IortRefused::Range { at: 140, base: 0xffff_ff00, last: 0x100 })); + let output = three(&[[0, 0x100, 0xffff_ff00, SMMU_AT, 0]], &[]); + assert_eq!(refusal(&output), Some(IortRefused::Range { at: 140, base: 0xffff_ff00, last: 0x100 })); + // The whole space, to its last ID, is a range. + assert_eq!(refusal(&three(&[[0, u32::MAX, 0, SMMU_AT, 0]], &[])), None); + let onward = three(&[], &[[1, u32::MAX, 0, ITS_AT, 0]]); + assert_eq!(refusal(&onward), Some(IortRefused::Range { at: 72, base: 1, last: u32::MAX })); +} + +#[test] +fn a_route_through_an_output_reference_that_names_no_node_is_refused() { + // Zero, the table's own header, the middle of a node, one past the last node. + for reference in [0, 36, SMMU_AT + 4, ROOT_AT + 56, u32::MAX] { + let t = three(&[[0, 0xff, 0, reference, 0]], &[]); + assert_eq!(route(&t, 0, 8), Err(IortRefused::Output { at: 140, reference }), "reference {reference}"); + // A requester the mapping does not hold never follows it. + assert_eq!(route(&t, 0, 0x100), Ok(Route::Unmapped), "reference {reference}"); + } + // The SMMUv3's onward mapping is followed by the stream it holds. + let t = three(&[[0, 0x1ff, 0, SMMU_AT, 0]], &[[0, 0xff, 0, ITS_AT + 4, 0]]); + assert_eq!(route(&t, 0, 8), Err(IortRefused::Output { at: 72, reference: ITS_AT + 4 })); + assert_eq!(route(&t, 0, 0x100), Ok(Route::Translated { smmu: plain_smmu(), stream: 0x100, its: None })); +} + +#[test] +fn a_route_through_an_output_reference_to_a_node_its_source_may_not_reach_is_refused() { + // A root complex to a root complex — itself. + assert_eq!( + route(&three(&[[0, 0xff, 0, ROOT_AT, 0]], &[]), 0, 8), + Err(IortRefused::Output { at: 140, reference: ROOT_AT }) + ); + // An SMMUv3 to an SMMUv3 — itself — and to a root complex. + for reference in [SMMU_AT, ROOT_AFTER_ONE] { + assert_eq!( + route(&three(&[[0, 0xff, 0, SMMU_AT, 0]], &[[0, 0xff, 0, reference, 0]]), 0, 8), + Err(IortRefused::Output { at: 72, reference }) + ); + } + // A root complex to a node type not decoded: an SMMUv1 or v2. + let old = table(5, &[node(3, 3, &[0u8; 44], &[]), root_complex(0, &[[0, 0xff, 0, 48, 0]])]); + assert_eq!(route(&old, 0, 8), Err(IortRefused::Output { at: 108, reference: 48 })); +} + +#[test] +fn a_node_type_not_decoded_is_answered_by_its_type_and_refuses_nothing() { + // A named component, a PMCG, a reserved memory range, an IWB, and a reserved type. + let others = [1u8, 5, 6, 7, 0x80].map(|kind| node(kind, 9, &[0xa5; 12], &[])); + let mut nodes = vec![its(3)]; + nodes.extend(others); + nodes.push(root_complex(0, &[[0, 0xff, 0, 48, 0]])); + let t = table(5, &nodes); + assert_eq!( + judged(&t, |table| table.nodes().collect::>()), + Ok(vec![ + Node::Its { id: 3 }, + Node::Other(1), + Node::Other(5), + Node::Other(6), + Node::Other(7), + Node::Other(0x80), + Node::RootComplex { segment: 0 }, + ]) + ); + assert_eq!(route(&t, 0, 8), Ok(Route::Untranslated(ItsDevice { its: 3, device: 8 }))); +} + +#[test] +fn a_single_mapping_that_is_not_its_smmus_own_is_refused() { + // Table 6 has it put every input ID out as one: a root complex's, and an SMMUv3's that its index does not name. + assert_eq!(refusal(&three(&[[0, 0xff, 0, SMMU_AT, 1]], &[])), Some(IortRefused::SingleMapping { at: 140 })); + assert_eq!(refusal(&three(&[], &[[0, 0, 0x30, ITS_AT, 1]])), Some(IortRefused::SingleMapping { at: 72 })); + let beside = smmu_with(5, INDEX_VALID, WIRED, 1, &[[0, 0, 0x30, ITS_AT, 1], [0, 0, 0x31, ITS_AT, 1]]); + assert_eq!(refusal(&behind(beside)), Some(IortRefused::SingleMapping { at: 72 })); + // The one it names carries the flag as Table 13 asks, and is no refusal. + let own = smmu_with(5, INDEX_VALID, WIRED, 0, &[[0, 0, 0x30, ITS_AT, 1]]); + assert_eq!(refusal(&behind(own.clone())), None); + assert_eq!(onward(&behind(own)), None); + // An index past the array names no entry, and is no reading that routes. + let past = behind(smmu_with(5, INDEX_VALID, WIRED, 1, &[[0, 0xff, 0x100, ITS_AT, 0]])); + assert_eq!(refusal(&past), Some(IortRefused::OwnMapping { at: 72, index: 1, count: 1 })); +} + +#[test] +fn an_smmus_own_mapping_index_at_or_past_its_id_array_is_refused() { + let one = [[0, 0xff, 0x100, ITS_AT, 0]]; + for index in [1, 2, u32::MAX] { + let past = behind(smmu_with(5, INDEX_VALID, WIRED, index, &one)); + assert_eq!(refusal(&past), Some(IortRefused::OwnMapping { at: 72, index, count: 1 }), "index {index}"); + } + assert_eq!( + refusal(&behind(smmu_with(5, INDEX_VALID, WIRED, 0, &[]))), + Some(IortRefused::OwnMapping { at: 72, index: 0, count: 0 }) + ); + // Revision 4 reads the index wherever a control interrupt has no GSIV. + let unwired = [106, 0, 109, 108]; + assert_eq!( + refusal(&behind(smmu_with(4, 0, unwired, 1, &one))), + Some(IortRefused::OwnMapping { at: 72, index: 1, count: 1 }) + ); + // Where the node says the field is ignored, no value of it is refused. + for (revision, flags, gsivs) in [(5, 0, WIRED), (5, 0, [0; 4]), (4, INDEX_VALID, WIRED)] { + let ignored = behind(smmu_with(revision, flags, gsivs, 1, &one)); + assert_eq!(onward(&ignored), Some(ItsDevice { its: 7, device: 0x108 }), "revision {revision}"); + } +} + +/// Requesters `0x00..=0x0f` to streams `0x10..=0x1f` of the SMMUv3 at `SMMU_AT`. +const FIRST: [u32; 5] = [0, 0xf, 0x10, SMMU_AT, 0]; + +fn alone(stream: u32) -> Result { + Ok(Route::Translated { smmu: plain_smmu(), stream, its: None }) +} + +fn shared(stream: u32, at: u32) -> Result { + Err(IortRefused::SharedStream { stream, at: at as usize }) +} + +#[test] +fn a_stream_another_mapping_puts_out_to_the_same_smmu_is_refused() { + // Two RID ranges of one root complex onto `0x18..=0x1f`, and a third onto the streams after. + let t = three(&[FIRST, [0x100, 0xf, 0x18, SMMU_AT, 0], [0x200, 0xf, 0x28, SMMU_AT, 0]], &[]); + assert_eq!(route(&t, 0, 0x7), alone(0x17)); + assert_eq!(route(&t, 0, 0x8), shared(0x18, ROOT_AT)); + assert_eq!(route(&t, 0, 0xf), shared(0x1f, ROOT_AT)); + assert_eq!(route(&t, 0, 0x100), shared(0x18, ROOT_AT)); + assert_eq!(route(&t, 0, 0x107), shared(0x1f, ROOT_AT)); + assert_eq!(route(&t, 0, 0x108), alone(0x20)); + assert_eq!(route(&t, 0, 0x200), alone(0x28)); + + // Two segments onto the same streams of one unit. + const SECOND: u32 = ROOT_AT + 56; + let segments = table(5, &[its(7), smmu(&[]), root_complex(0, &[FIRST]), root_complex(1, &[FIRST])]); + assert_eq!(route(&segments, 0, 8), shared(0x18, SECOND)); + assert_eq!(route(&segments, 1, 8), shared(0x18, ROOT_AT)); + + // The same stream numbers of another unit are another unit's streams. + let units = table( + 5, + &[its(7), smmu(&[]), root_complex(0, &[FIRST]), smmu(&[]), root_complex(1, &[[0, 0xf, 0x10, SECOND, 0]])], + ); + assert_eq!(route(&units, 0, 8), alone(0x18)); + assert_eq!(route(&units, 1, 8), alone(0x18)); + + // A node of a type not decoded is a requester too: a named component's single mapping + // puts out its base alone, whatever its span. + let named = |reference| node(1, 4, &[0; 12], &[[0, 0xf, 0x18, reference, 1]]); + let t = table(5, &[its(7), smmu(&[]), root_complex(0, &[FIRST]), named(SMMU_AT)]); + assert_eq!(route(&t, 0, 8), shared(0x18, SECOND)); + assert_eq!(route(&t, 0, 7), alone(0x17)); + assert_eq!(route(&t, 0, 9), alone(0x19)); + // To an ITS, `0x18` is a DeviceID and no stream. + assert_eq!(route(&table(5, &[its(7), smmu(&[]), root_complex(0, &[FIRST]), named(ITS_AT)]), 0, 8), alone(0x18)); +} + +/// [`Iort::check`] places only the ID arrays of the types it decodes; a stream +/// is judged alone only once every other node's array is read where its +/// header puts it. +#[test] +fn a_stream_is_not_judged_alone_past_an_id_array_out_of_its_node() { + const NAMED: usize = (ROOT_AT + 56) as usize; + let at = |offset: u32, count: u32| { + let mut named = node(1, 4, &[0; 12], &[[0, 0, 0x30, ITS_AT, 1]]); + named[8..12].copy_from_slice(&count.to_le_bytes()); + named[12..16].copy_from_slice(&offset.to_le_bytes()); + route(&table(5, &[its(7), smmu(&[]), root_complex(0, &[FIRST]), named, its(9)]), 0, 8) + }; + assert_eq!(at(28, 1), alone(0x18)); + assert_eq!(at(28, 2), Err(IortRefused::Mappings { at: NAMED, count: 2, offset: 28 })); + assert_eq!(at(0, 1), Err(IortRefused::Mappings { at: NAMED, count: 1, offset: 0 })); + assert_eq!(at(12, 1), Err(IortRefused::Mappings { at: NAMED, count: 1, offset: 12 })); +} + +/// `virt`'s four GSIVs: every control interrupt wired. +const WIRED: [u32; 4] = [106, 107, 109, 108]; +/// Table 14, bit 4: `DeviceID mapping index valid`. +const INDEX_VALID: u32 = 1 << 4; + +/// An ITS of id 7, `smmu`, and a root complex sending requesters +/// `0x00..=0xff` to it as the same streams. +fn behind(smmu: Vec) -> Vec { + table(5, &[its(7), smmu, root_complex(0, &[[0, 0xff, 0, SMMU_AT, 0]])]) +} + +/// Where the SMMUv3's own node sends stream 8. +fn onward(table: &[u8]) -> Option { + match route(table, 0, 8) { + Ok(Route::Translated { stream: 8, its, .. }) => its, + other => panic!("requester 8 is stream 8 of the SMMUv3: {other:?}"), + } +} + +#[test] +fn the_mapping_a_revision_5_smmu_indexes_is_the_units_own_and_routes_no_stream() { + // An input base and length no range holds: they are not the table's to judge. + let wide = behind(smmu_with(5, INDEX_VALID, WIRED, 0, &[[0xffff_ffff, 0xffff_ffff, 0x30, ITS_AT, 0]])); + assert_eq!(refusal(&wide), None); + assert_eq!(onward(&wide), None); + // And ones that would hold the stream: the entry is the unit's DeviceID and no stream's. + let plausible = behind(smmu_with(5, INDEX_VALID, WIRED, 0, &[[0, 0xffff, 0x30, ITS_AT, 0]])); + assert_eq!(onward(&plausible), None); + // The index picks the entry, and the other one still routes. + let second = smmu_with(5, INDEX_VALID, WIRED, 1, &[[0, 0xff, 0x100, ITS_AT, 0], [0, 0xffff, 0x30, ITS_AT, 0]]); + assert_eq!(onward(&behind(second)), Some(ItsDevice { its: 7, device: 0x108 })); + // Flag clear, the index is ignored whatever the GSIVs say. + let ignored = behind(smmu_with(5, 0, [0; 4], 0, &[[0, 0xff, 0x100, ITS_AT, 0]])); + assert_eq!(onward(&ignored), Some(ItsDevice { its: 7, device: 0x108 })); +} + +#[test] +fn a_revision_4_smmu_indexes_its_own_mapping_whenever_a_control_interrupt_is_not_wired() { + for unwired in 0..4 { + let mut gsivs = WIRED; + gsivs[unwired] = 0; + let own = behind(smmu_with(4, 0, gsivs, 0, &[[0xffff_ffff, 0xffff_ffff, 0x30, ITS_AT, 0]])); + assert_eq!(refusal(&own), None, "GSIV {unwired} zero"); + assert_eq!(onward(&own), None, "GSIV {unwired} zero"); + let plausible = behind(smmu_with(4, 0, gsivs, 0, &[[0, 0xffff, 0x30, ITS_AT, 0]])); + assert_eq!(onward(&plausible), None, "GSIV {unwired} zero"); + } + // Every one wired, as on `virt`: the field is ignored, and bit 4 of the flags is reserved here. + for flags in [0, INDEX_VALID] { + let wired = behind(smmu_with(4, flags, WIRED, 0, &[[0, 0xff, 0x100, ITS_AT, 0]])); + assert_eq!(onward(&wired), Some(ItsDevice { its: 7, device: 0x108 })); + } +} + +/// Only an SMMUv3 has a DeviceID mapping index. Read as one, this root +/// complex would have no GSIV and, where the index is, its second mapping's +/// output base: zero, naming the first. +#[test] +fn a_root_complex_has_no_mapping_of_its_own() { + let t = table(5, &[its(7), root_complex(0, &[[0, 0xff, 0, ITS_AT, 0], [0x100, 0xff, 0, ITS_AT, 0]])]); + assert_eq!(route(&t, 0, 8), Ok(Route::Untranslated(ItsDevice { its: 7, device: 8 }))); +} + +#[test] +fn an_its_group_of_other_than_one_its_is_refused() { + for count in [0u32, 2] { + let group = node(0, 1, &[count.to_le_bytes(), 0u32.to_le_bytes(), 1u32.to_le_bytes()].concat(), &[]); + assert_eq!(refusal(&table(5, &[group])), Some(IortRefused::ItsGroup { at: 48, count })); + } +} + +#[test] +fn an_id_two_mappings_claim_is_refused_when_it_is_asked_for() { + // In one root complex: `0x00..=0xff` and `0x80..=0x17f`. + let t = three(&[[0, 0xff, 0, SMMU_AT, 0], [0x80, 0xff, 0x80, ITS_AT, 0]], &[]); + assert_eq!(route(&t, 0, 0x7f), Ok(Route::Translated { smmu: plain_smmu(), stream: 0x7f, its: None })); + assert_eq!(route(&t, 0, 0x80), Err(IortRefused::Overlap { id: 0x80 })); + assert_eq!(route(&t, 0, 0x100), Ok(Route::Untranslated(ItsDevice { its: 7, device: 0x100 }))); + + // Across two root complexes of one segment; a second segment's is its own. + let two = |segment| { + table(5, &[its(7), root_complex(0, &[[0, 0xff, 0, 48, 0]]), root_complex(segment, &[[0, 0xff, 0x100, 48, 0]])]) + }; + assert_eq!(route(&two(0), 0, 8), Err(IortRefused::Overlap { id: 8 })); + assert_eq!(route(&two(1), 0, 8), Ok(Route::Untranslated(ItsDevice { its: 7, device: 8 }))); + assert_eq!(route(&two(1), 1, 8), Ok(Route::Untranslated(ItsDevice { its: 7, device: 0x108 }))); + + // In the SMMUv3, by the stream. + let onward = three(&[[0, 0xff, 0x40, SMMU_AT, 0]], &[[0, 0xff, 0, ITS_AT, 0], [0x48, 0, 0x900, ITS_AT, 0]]); + assert_eq!(route(&onward, 0, 8), Err(IortRefused::Overlap { id: 0x48 })); + assert_eq!( + route(&onward, 0, 9), + Ok(Route::Translated { smmu: plain_smmu(), stream: 0x49, its: Some(ItsDevice { its: 7, device: 0x49 }) }) + ); +} + +/// Firmware's bytes, any of them wrong: every single-byte change to either +/// captured IORT, and every length it could declare, is decoded or refused and +/// every requester of two buses asked for, without a panic and to an end. +#[test] +fn no_corruption_of_a_published_iort_panics_or_hangs_the_decoder() { + for regions in [HWGIC, ITS] { + let original = regions[3].1; + let ask = |bytes: &[u8]| { + let patched = [regions[0], regions[1], regions[2], (IORT, bytes)]; + if let Ok(table) = iort(Machine { regions: &patched }, RSDP) { + assert!(table.nodes().count() <= bytes.len() / 16); + for rid in 0..0x200 { + let _ = table.route(0, rid); + } + } + }; + for at in 36..original.len() { + for value in [0x00, 0x01, 0x7f, 0x80, 0xff, original[at].wrapping_add(1), original[at] ^ 0x10] { + let mut bytes = original.to_vec(); + bytes[at] = value; + reseal(&mut bytes); + ask(&bytes); + } + } + for len in 36..original.len() as u32 { + let mut bytes = original.to_vec(); + declare_len(&mut bytes, len); + ask(&bytes); + } + } +} + +/// Memory that counts every byte the decoder reads of it. +#[derive(Clone, Copy)] +struct Counted<'a> { + machine: Machine<'a>, + reads: &'a Cell, +} + +impl Phys for Counted<'_> { + fn readable(self, phys: u64, len: usize) -> bool { + self.machine.readable(phys, len) + } + + fn byte(self, phys: u64) -> u8 { + self.reads.set(self.reads.get() + 1); + Phys::byte(self.machine, phys) + } +} + +/// 499 root complexes of 40 mappings each, every one naming the table's last +/// node: following each reference by a walk of the nodes when the table is +/// opened would read ten million node headers. Opening reads the table for +/// its checksum, each node and each mapping once; a route walks the nodes +/// once to find its root complex and once for each of the two references it +/// may follow. +#[test] +fn opening_a_table_and_routing_a_requester_each_read_it_a_bounded_number_of_times() { + const ROOTS: u32 = 499; + const ROOT_LEN: u32 = 36 + 40 * 20; + let its_at = 48 + ROOTS * ROOT_LEN; + let mut nodes: Vec> = (0..ROOTS) + .map(|segment| { + let mappings: Vec<[u32; 5]> = (0..40).map(|i| [i * 16, 15, 0x1000 + i * 16, its_at, 0]).collect(); + root_complex(segment, &mappings) + }) + .collect(); + nodes.push(its(7)); + let bytes = table(5, &nodes); + assert_eq!(bytes.len(), its_at as usize + 24); + + let head = rsdp(0x1000, 2, 36); + let root = xsdt(&[TABLE_AT]); + let regions: &[(u64, &[u8])] = &[(0x800, &head), (0x1000, &root), (TABLE_AT, &bytes)]; + let reads = Cell::new(0); + let opened = iort(Counted { machine: Machine { regions }, reads: &reads }, 0x800).expect("a table of 500 nodes"); + let open = reads.replace(0); + assert!(open <= 3 * bytes.len(), "opening {} bytes read {open}", bytes.len()); + + // The last mapping of the last root complex, so nothing is found early. + let device = 0x1000 + 39 * 16 + 15; + assert_eq!(opened.route(ROOTS - 1, 39 * 16 + 15), Ok(Route::Untranslated(ItsDevice { its: 7, device }))); + let routed = reads.replace(0); + assert!(routed <= bytes.len() / 8, "routing through {} bytes read {routed}", bytes.len()); + assert_eq!(opened.route(ROOTS, 0), Ok(Route::Unmapped)); + + // Every mapping to one SMMUv3, each onto streams of its own: a stream is + // judged alone by one more walk of the nodes and their mappings. + let mut nodes: Vec> = vec![smmu(&[])]; + nodes.extend((0..ROOTS).map(|segment| { + let mappings: Vec<[u32; 5]> = + (0..40).map(|i| [i * 16, 15, (segment * 40 + i) * 16, 48, 0]).collect(); + root_complex(segment, &mappings) + })); + let bytes = table(5, &nodes); + let regions: &[(u64, &[u8])] = &[(0x800, &head), (0x1000, &root), (TABLE_AT, &bytes)]; + let opened = iort(Counted { machine: Machine { regions }, reads: &reads }, 0x800).expect("a table of 500 nodes"); + reads.set(0); + let stream = ((ROOTS - 1) * 40 + 39) * 16 + 15; + assert_eq!( + opened.route(ROOTS - 1, 39 * 16 + 15), + Ok(Route::Translated { smmu: plain_smmu(), stream, its: None }) + ); + let routed = reads.get(); + assert!(routed <= 2 * bytes.len(), "routing through {} bytes read {routed}", bytes.len()); +} diff --git a/toyos-bootmap/src/aarch64.rs b/toyos-bootmap/src/aarch64.rs index 08b33f7542f..7db15d8a091 100644 --- a/toyos-bootmap/src/aarch64.rs +++ b/toyos-bootmap/src/aarch64.rs @@ -33,7 +33,7 @@ const OUTER_SHAREABLE: u64 = 0b10 << 8; /// The access flag, bit 10: set, so the first access does not fault. const AF: u64 = 1 << 10; /// Privileged execute-never, bit 53. -const PXN: u64 = 1 << 53; +pub const PXN: u64 = 1 << 53; /// Unprivileged execute-never, bit 54. const UXN: u64 = 1 << 54; diff --git a/toyos-its/Cargo.toml b/toyos-its/Cargo.toml new file mode 100644 index 00000000000..283165de4c6 --- /dev/null +++ b/toyos-its/Cargo.toml @@ -0,0 +1,19 @@ +# A member of the host workspace (root `Cargo.toml`), like toyos-acpi and +# toyos-smmu: it is pure and its tests run on the host. +# +# What lives here is every word a GICv3 ITS and its driver exchange, and the +# two tables a redistributor reads an LPI's configuration and pending state +# from. The ITS is what makes a message a claimed function writes arrive as +# that claim's interrupt and no other's, so its commands are held on the host +# to the specification's field positions. + +[package] +name = "toyos-its" +description = "The GICv3 ITS's encodings: commands, table sizing, LPI configuration and pending tables and register fields, pure, forbid(unsafe_code)." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false + +[dependencies] +toyos-phys = { path = "../toyos-phys" } diff --git a/toyos-its/src/command.rs b/toyos-its/src/command.rs new file mode 100644 index 00000000000..fb16bb3be42 --- /dev/null +++ b/toyos-its/src/command.rs @@ -0,0 +1,58 @@ +//! The ITS's commands (§5.3): 32 bytes each, four little-endian +//! doublewords, the command number in bits [7:0] of the first. + +use toyos_phys::Phys; + +use crate::lpi::Lpi; +use crate::{Collection, Event, EventBits, Target}; + +/// One command. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Command { + /// `MAPD` with `V` set (§5.3.10): `device`'s events are translated by + /// the zeroed table at `table`. + MapDevice { device: u32, table: Phys<8>, events: EventBits }, + /// `MAPD` with `V` clear: `device` has no table, and its messages are + /// ignored. + UnmapDevice { device: u32 }, + /// `MAPC` with `V` set (§5.3.9): `collection`'s interrupts go to `target`. + MapCollection { collection: Collection, target: Target }, + /// `MAPTI` (§5.3.12): `device`'s `event` is `lpi`, in `collection`. + MapEvent { device: u32, event: Event, lpi: Lpi, collection: Collection }, + /// `INV` (§5.3.6): reread the configuration of `device`'s `event`'s LPI. + Reconfigure { device: u32, event: Event }, + /// `DISCARD` (§5.3.4): `device`'s `event` is no interrupt, and any + /// pending one is dropped. + Discard { device: u32, event: Event }, + /// `SYNC` (§5.3.15): every command before it has taken effect at + /// `target` before the next is read. + Sync(Target), +} + +/// `V`, bit [63] of doubleword 2. +const V: u64 = 1 << 63; + +impl Command { + pub const fn words(self) -> [u64; 4] { + // DeviceID is bits [63:32] of doubleword 0, and EventID bits [31:0] of doubleword 1. + const fn device(number: u64, device: u32) -> u64 { + number | (device as u64) << 32 + } + match self { + // Size [4:0] of doubleword 1 is the EventID bits minus one; ITT_addr [51:8] of doubleword 2. + Self::MapDevice { device: id, table, events } => { + [device(0x08, id), (events.0 - 1) as u64, V | table.get(), 0] + } + Self::UnmapDevice { device: id } => [device(0x08, id), 0, 0, 0], + // RDbase [51:16] and ICID [15:0] of doubleword 2. + Self::MapCollection { collection, target } => [0x09, 0, V | target.0 << 16 | collection.0 as u64, 0], + // pINTID [63:32] of doubleword 1; ICID [15:0] of doubleword 2. + Self::MapEvent { device: id, event, lpi, collection } => { + [device(0x0A, id), (lpi.intid() as u64) << 32 | event.0 as u64, collection.0 as u64, 0] + } + Self::Reconfigure { device: id, event } => [device(0x0C, id), event.0 as u64, 0, 0], + Self::Discard { device: id, event } => [device(0x0F, id), event.0 as u64, 0, 0], + Self::Sync(target) => [0x05, 0, target.0 << 16, 0], + } + } +} diff --git a/toyos-its/src/lib.rs b/toyos-its/src/lib.rs new file mode 100644 index 00000000000..12607412537 --- /dev/null +++ b/toyos-its/src/lib.rs @@ -0,0 +1,330 @@ +//! Every word a GICv3 Interrupt Translation Service and its driver exchange, +//! as Arm IHI 0069 issue H.b lays it out: the ITS's registers and what its +//! type register says of it (this module), its commands ([`command`]), and +//! the two tables a redistributor reads an LPI's configuration and pending +//! state from ([`lpi`]). +//! +//! The device table and each device's interrupt translation table are the +//! ITS's own: their entry formats are IMPLEMENTATION DEFINED (§5.2.3, +//! §5.2.4), software gives the ITS zeroed memory of the size its registers +//! ask for and writes an entry only by a command. So no entry is encoded +//! here; the sizes are. +//! +//! Only physical LPIs through flat tables are spelled: no vPE, no indirect +//! table. What an ITS reports about itself is a device's word and is refused +//! by name where the driver could not act on it; nothing here panics on a +//! register's value. An event, a collection and an LPI are each a type made +//! from what the ITS, its tables and the distributor say there is of them. +//! +//! `no_std`, no allocation, no `unsafe`. + +#![no_std] +#![forbid(unsafe_code)] + +pub mod command; +pub mod lpi; + +use toyos_phys::Phys; + +/// Offsets in the ITS's control frame (§12.18, Table 12-33). +pub const GITS_CTLR: usize = 0x0000; +pub const GITS_TYPER: usize = 0x0008; +pub const GITS_CBASER: usize = 0x0080; +pub const GITS_CWRITER: usize = 0x0088; +pub const GITS_CREADR: usize = 0x0090; + +/// `GITS_BASER`, n = 0 to 7 (§12.19.1): one per table the ITS may ask +/// memory for. +pub const fn gits_baser(n: usize) -> Option { + if n < 8 { + Some(0x0100 + 8 * n) + } else { + None + } +} + +/// The translation frame is the second 64 KiB frame (§12.18), and +/// `GITS_TRANSLATER` is at 0x0040 of it (Table 12-34): a function's message +/// is one 32-bit write of its EventID there, and the bus says which device +/// wrote it. +pub const TRANSLATION_FRAME: u64 = 0x1_0000; +pub const GITS_TRANSLATER: u64 = 0x0040; + +/// `GITS_CTLR.Enabled` [0] (§12.19.4). +pub const CTLR_ENABLED: u32 = 1 << 0; + +/// What an ITS lacks that the driver needs. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Lacks { + /// `GITS_TYPER.Physical` clear: no physical LPIs. + PhysicalLpis, +} + +/// An ITS as its type register describes it (§12.19.13). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Its { + /// `Devbits` [17:13] plus one: the widest DeviceID, and so how many + /// entries a flat device table holds. + pub device_bits: u8, + /// `ID_bits` [12:8] plus one: the widest EventID. + event_bits: u8, + /// `ITT_entry_size` [7:4] plus one: bytes of one translation table entry. + itt_entry_bytes: u8, + /// `HCC` [31:24]: collections the ITS holds without a collection table. + held: u8, + /// `CCT` [2]: those count beside a collection table's. + cumulative: bool, + /// The width of an ICID: 16 bits where `CIL` [36] is clear, else + /// `CIDbits` [35:32] plus one. + collection_bits: u8, + /// `PTA` [19]: a collection's target is its redistributor's address + /// rather than its processor number. + physical_targets: bool, +} + +pub const fn probe(typer: u64) -> Result { + if typer & 1 == 0 { + return Err(Lacks::PhysicalLpis); + } + Ok(Its { + device_bits: (typer >> 13 & 0x1F) as u8 + 1, + event_bits: (typer >> 8 & 0x1F) as u8 + 1, + itt_entry_bytes: (typer >> 4 & 0xF) as u8 + 1, + held: (typer >> 24) as u8, + cumulative: typer & 1 << 2 != 0, + collection_bits: if typer & 1 << 36 == 0 { 16 } else { (typer >> 32 & 0xF) as u8 + 1 }, + physical_targets: typer & 1 << 19 != 0, + }) +} + +/// How many EventIDs one device's translation table holds, as `MAPD` says +/// it: made by [`Its::events`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct EventBits(u8); + +/// An EventID inside the translation table it was made for: +/// [`EventBits::event`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Event(u32); + +impl EventBits { + /// `id` as an event of a device whose table holds this many. + pub const fn event(self, id: u32) -> Option { + if (id as u64) >> self.0 == 0 { + Some(Event(id)) + } else { + None + } + } +} + +/// How many collections an ITS has: made by [`Its::collections`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Collections(u32); + +/// A collection the ITS has, by its ICID: [`Collections::collection`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Collection(u16); + +impl Collections { + pub const fn collection(self, id: u16) -> Option { + if (id as u32) < self.0 { + Some(Collection(id)) + } else { + None + } + } +} + +/// The redistributor a collection's interrupts go to, in the form this ITS +/// takes it (`RDbase`, §5.3.1): made by [`Its::target`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Target(u64); + +impl Its { + /// A translation table of `1 << bits` events. `None` for no bit, or more + /// than the ITS's EventIDs have. + pub const fn events(&self, bits: u8) -> Option { + if bits >= 1 && bits <= self.event_bits { + Some(EventBits(bits)) + } else { + None + } + } + + /// The zeroed bytes `MAPD` is given for that table, at a 256-byte + /// aligned address. + pub const fn itt_bytes(&self, events: EventBits) -> u64 { + (self.itt_entry_bytes as u64) << events.0 + } + + /// The collections this ITS has once its collection table holds + /// `in_table` entries, none where no `GITS_BASER` backs one (§5.2.2, + /// §5.3.1): the table's, those the ITS holds itself where it has no + /// table or counts them beside it, and no more than an ICID numbers. + pub const fn collections(&self, in_table: u64) -> Collections { + let held = if in_table == 0 || self.cumulative { self.held as u64 } else { 0 }; + let numbered = 1u64 << self.collection_bits; + let total = held.saturating_add(in_table); + Collections(if total < numbered { total } else { numbered } as u32) + } + + /// The redistributor at `frame` whose `GICR_TYPER.Processor_Number` is + /// `number`, as this ITS names it. + pub const fn target(&self, number: u16, frame: Phys<16>) -> Target { + Target(if self.physical_targets { frame.get() >> 16 } else { number as u64 }) + } +} + +/// What a `GITS_BASER` asks memory for (`Type`, bits [58:56]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Table { + /// `0b000`: the register backs no table. + Unimplemented, + /// `0b001`: one entry for every DeviceID. + Devices, + /// `0b100`: one entry for every collection past those the ITS holds. + Collections, + /// vPEs (`0b010`), or a reserved type. + Other(u8), +} + +/// The page a table is read in (`Page_Size`, bits [9:8]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PageSize { + K4, + K16, + K64, +} + +impl PageSize { + pub const fn bytes(self) -> u64 { + match self { + Self::K4 => 4 << 10, + Self::K16 => 16 << 10, + Self::K64 => 64 << 10, + } + } +} + +/// `GITS_BASER` as read (§12.19.1): the table it backs and the page size +/// it holds — which an ITS may fix, so it is read back after +/// [`Backing::baser`] is written. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Backing { + pub table: Table, + pub page: PageSize, + /// `Entry_Size` [52:48] plus one: the bytes of one entry. + entry_bytes: u8, +} + +pub const fn table(baser: u64) -> Backing { + let table = match baser >> 56 & 0b111 { + 0b000 => Table::Unimplemented, + 0b001 => Table::Devices, + 0b100 => Table::Collections, + other => Table::Other(other as u8), + }; + let page = match baser >> 8 & 0b11 { + 0b00 => PageSize::K4, + 0b01 => PageSize::K16, + // `0b11` is reserved and treated as `0b10`. + _ => PageSize::K64, + }; + Backing { table, page, entry_bytes: (baser >> 48 & 0x1F) as u8 + 1 } +} + +/// `InnerCache` `0b111`, read-allocate write-allocate write-back, with +/// `OuterCache` zero, as the inner; `Shareability` [11:10] `0b01`, inner +/// shareable. `GITS_BASER` and `GITS_CBASER` hold them at the same bits. +const CACHED: u64 = 0b111 << 59 | 0b01 << 10; +const VALID: u64 = 1 << 63; + +impl Backing { + /// The entries a flat table of `pages` of its pages holds. + pub const fn entries(self, pages: u16) -> u64 { + pages as u64 * self.page.bytes() / self.entry_bytes as u64 + } + + /// The fewest pages that hold `entries`. `None` for none, or more than + /// the 256 `Size` counts. + pub const fn pages(self, entries: u64) -> Option { + let Some(bytes) = entries.checked_mul(self.entry_bytes as u64) else { + return None; + }; + let pages = bytes.div_ceil(self.page.bytes()); + if pages >= 1 && pages <= 256 { + Some(pages as u16) + } else { + None + } + } + + /// `GITS_BASER` for a flat table of `pages` of its pages at `at`: + /// `Valid` [63], `Indirect` [62] clear, `Physical_Address` [47:12], + /// `Page_Size` [9:8], `Size` [7:0] the pages minus one. `None` for no + /// page or more than the 256 the field counts. + pub const fn baser(self, at: Phys<16>, pages: u16) -> Option { + if pages == 0 || pages > 256 { + return None; + } + let size = match self.page { + PageSize::K4 => 0b00, + PageSize::K16 => 0b01, + PageSize::K64 => 0b10, + }; + Some(VALID | CACHED | at.get() | size << 8 | (pages - 1) as u64) + } +} + +/// The ITS's command queue: `pages` 4 KiB pages of 32-byte commands +/// (§5.2.8). `GITS_CWRITER` and `GITS_CREADR` each hold a byte offset into +/// it, `Offset` [19:5], which is all of either that names a command. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct CommandQueue { + bytes: u32, +} + +/// `GITS_CREADR.Stalled` [0]: the ITS stopped on a command it refused. +pub const CREADR_STALLED: u64 = 1 << 0; + +impl CommandQueue { + /// `None` for no page or more than the 256 `GITS_CBASER.Size` counts. + pub const fn new(pages: u16) -> Option { + if pages == 0 || pages > 256 { + None + } else { + Some(Self { bytes: pages as u32 * 4096 }) + } + } + + /// `GITS_CBASER` (§12.19.2) for the queue at `at`: `Valid` [63], + /// `Physical_Address` [51:12] with its low four bits zero, `Size` [7:0] + /// the pages minus one. + pub const fn cbaser(self, at: Phys<16>) -> u64 { + VALID | CACHED | at.get() | (self.bytes / 4096 - 1) as u64 + } + + /// The byte offset an offset register names: its `Offset` field alone, + /// and inside the queue whatever the ITS wrote there. + pub const fn offset(self, register: u64) -> u32 { + (register & 0xF_FFE0) as u32 % self.bytes + } + + /// What `GITS_CWRITER` is written with once the command at its offset is + /// in memory: the offset after, wrapping at the queue's end. + pub const fn after(self, cwriter: u64) -> u64 { + ((self.offset(cwriter) + 32) % self.bytes) as u64 + } + + /// The ITS has read every command: the two registers' offsets are equal. + pub const fn is_empty(self, cwriter: u64, creadr: u64) -> bool { + self.offset(cwriter) == self.offset(creadr) + } + + /// One more command would make the writer's offset the reader's, which + /// reads as empty: the queue holds one command fewer than its bytes do. + pub const fn is_full(self, cwriter: u64, creadr: u64) -> bool { + self.after(cwriter) == self.offset(creadr) as u64 + } +} diff --git a/toyos-its/src/lpi.rs b/toyos-its/src/lpi.rs new file mode 100644 index 00000000000..074c2346308 --- /dev/null +++ b/toyos-its/src/lpi.rs @@ -0,0 +1,133 @@ +//! LPIs (§5.1): their INTIDs, the configuration table every redistributor +//! reads and the pending table each one owns, and the redistributor registers +//! that name them. + +use toyos_phys::Phys; + +/// The first LPI's INTID (§2.2): everything below is an SGI, a PPI, an SPI +/// or special. +pub const FIRST: u32 = 8192; + +/// Offsets in a redistributor's `RD_base` frame (§12.10, Table 12-27). +pub const GICR_PROPBASER: usize = 0x0070; +pub const GICR_PENDBASER: usize = 0x0078; + +/// `GICR_CTLR.EnableLPIs` [0] (§12.11.2): from here the redistributor reads +/// both tables, and neither base register may be written. +pub const CTLR_ENABLE_LPIS: u32 = 1 << 0; + +/// `GICR_TYPER.PLPIS` [0] (§12.11.37): the redistributor takes physical LPIs. +pub const TYPER_PLPIS: u64 = 1 << 0; + +/// `GICR_TYPER.Processor_Number` [23:8]: how an ITS whose `GITS_TYPER.PTA` +/// is clear names this redistributor. +pub const fn processor_number(typer: u64) -> u16 { + (typer >> 8) as u16 +} + +/// An LPI's INTID, one the distributor and its redistributor both take: +/// [`Space::lpi`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Lpi(u32); + +impl Lpi { + pub const fn intid(self) -> u32 { + self.0 + } + + /// Where its byte is in the configuration table: the table's first byte + /// is INTID 8192's (§5.1.1). + pub const fn configuration_index(self) -> usize { + (self.0 - FIRST) as usize + } +} + +/// One LPI's byte in the configuration table (Table 5-1): the priority's +/// upper six bits in [7:2], bit [1] RES1, and the enable in bit [0]. +pub const fn configuration(priority: u8, enabled: bool) -> u8 { + priority & 0xFC | 0b10 | enabled as u8 +} + +/// The two tables laid out for INTIDs of `id_bits` bits, which the +/// distributor has. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Layout { + id_bits: u8, + lpis: u32, +} + +/// `InnerCache` [9:7] `0b111`, read-allocate write-allocate write-back, with +/// `OuterCache` [58:56] zero, as the inner; `Shareability` [11:10] `0b01`, +/// inner shareable. Both base registers hold them at the same bits. +const CACHED: u64 = 0b111 << 7 | 0b01 << 10; + +impl Layout { + /// `GICD_TYPER` (§12.9.38) judged: `None` where the distributor takes no + /// LPI (`LPIS` [17] clear), and for `id_bits` below 14, which reach no + /// LPI, or above its INTIDs' (`IDbits` [23:19] plus one). Its `num_LPIs` + /// [15:11], where not zero, counts the LPIs there are from 8192 on. + pub const fn new(gicd_typer: u32, id_bits: u8) -> Option { + let widest = (gicd_typer >> 19 & 0x1F) as u8 + 1; + if gicd_typer & 1 << 17 == 0 || id_bits < 14 || id_bits > widest { + return None; + } + let numbered = (1u64 << id_bits) - FIRST as u64; + let counted = match gicd_typer >> 11 & 0x1F { + 0 => numbered, + count => 2u64 << count, + }; + Some(Self { id_bits, lpis: if counted < numbered { counted } else { numbered } as u32 }) + } + + /// One byte for every INTID from 8192 that `id_bits` bits number. + pub const fn configuration_bytes(self) -> u64 { + (1 << self.id_bits) - FIRST as u64 + } + + /// One bit for every INTID, the first 8192 included. + pub const fn pending_bytes(self) -> u64 { + (1 << self.id_bits) / 8 + } + + /// `GICR_PROPBASER` (§12.11.33) for the configuration table at `table`: + /// `Physical_Address` [51:12] and `IDbits` [4:0], the bits minus one. + pub const fn propbaser(self, table: Phys<12>) -> u64 { + CACHED | table.get() | (self.id_bits - 1) as u64 + } + + /// The LPIs of a redistributor that reads `propbaser` back: `None` where + /// its `IDbits` is not what [`Layout::propbaser`] wrote, which a + /// redistributor that holds the register as its own answers. + pub const fn space(self, propbaser: u64) -> Option { + if (propbaser & 0x1F) as u8 == self.id_bits - 1 { + Some(Space { lpis: self.lpis }) + } else { + None + } + } +} + +/// The LPIs there are: those a [`Layout`]'s tables hold, that the +/// distributor counts, in a redistributor that took the layout. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Space { + lpis: u32, +} + +impl Space { + /// `intid` as an LPI of this space. + pub const fn lpi(self, intid: u32) -> Option { + if intid >= FIRST && intid - FIRST < self.lpis { + Some(Lpi(intid)) + } else { + None + } + } +} + +/// `GICR_PENDBASER` (§12.11.32) for the pending table at `table`, which is +/// all zeroes: `Physical_Address` [51:16], and `PTZ` [62], which says so and +/// lets the redistributor not read it. +pub const fn pendbaser(table: Phys<16>) -> u64 { + 1 << 62 | CACHED | table.get() +} diff --git a/toyos-its/tests/encodings.rs b/toyos-its/tests/encodings.rs new file mode 100644 index 00000000000..cf913d0cdf4 --- /dev/null +++ b/toyos-its/tests/encodings.rs @@ -0,0 +1,461 @@ +//! Every encoding against IHI 0069 H.b's own field positions: each expected +//! value is built from `(msb, lsb, value)` triples as the specification's +//! figures and field headings give them, and compared whole, so a field in +//! the wrong place and a stray bit are both a difference. + +use toyos_its::command::Command; +use toyos_its::lpi::{self, Layout, Space}; +use toyos_its::{gits_baser, probe, table, CommandQueue, Its, Lacks, PageSize, Table}; +use toyos_phys::Phys; + +/// `N` doublewords holding `value` in bits `[msb:lsb]` for every triple, and +/// zero everywhere else. +fn layout(fields: &[(u32, u32, u64)]) -> [u64; N] { + let mut words = [0u64; N]; + for &(msb, lsb, value) in fields { + let width = msb - lsb + 1; + assert!(width == 64 || value >> width == 0, "{value:#x} does not fit [{msb}:{lsb}]"); + for bit in 0..width { + if value >> bit & 1 != 0 { + let at = lsb + bit; + assert_eq!(words[(at / 64) as usize] >> (at % 64) & 1, 0, "two fields claim bit {at}"); + words[(at / 64) as usize] |= 1 << (at % 64); + } + } + } + words +} + +fn word(fields: &[(u32, u32, u64)]) -> u64 { + layout::<1>(fields)[0] +} + +/// A command's fields, each given by its doubleword and its bits in it. +fn command(fields: &[(u32, u32, u32, u64)]) -> [u64; 4] { + let absolute: Vec<_> = fields.iter().map(|&(dw, msb, lsb, value)| (dw * 64 + msb, dw * 64 + lsb, value)).collect(); + layout(&absolute) +} + +/// `GITS_TYPER` of an ITS of 16-bit EventIDs and DeviceIDs and 8-byte +/// translation entries, holding 3 collections itself, which names a +/// redistributor by `pta`: Physical [0], ITT_entry_size [7:4], ID_bits +/// [12:8], Devbits [17:13], PTA [19], HCC [31:24]. +const TYPER: &[(u32, u32, u64)] = &[(0, 0, 1), (7, 4, 7), (12, 8, 15), (17, 13, 15), (31, 24, 3)]; + +/// `TYPER` with `change`'s fields beside it. +fn typer(change: &[(u32, u32, u64)]) -> u64 { + word(&[TYPER, change].concat()) +} + +fn its(pta: u64) -> Its { + probe(typer(&[(19, 19, pta)])).expect("an ITS of physical LPIs") +} + +/// `GICD_TYPER` of a distributor that takes LPIs under 16-bit INTIDs: LPIS +/// [17], IDbits [23:19] the bits minus one, num_LPIs [15:11] zero. +const GICD_TYPER: &[(u32, u32, u64)] = &[(17, 17, 1), (23, 19, 15)]; + +/// The LPIs of a redistributor that took a layout of `bits` bits under `GICD_TYPER`. +fn space(bits: u8) -> Space { + let layout = Layout::new(word(GICD_TYPER) as u32, bits).expect("bits the distributor has"); + layout.space(layout.propbaser(Phys::new(0x4000_0000).unwrap())).expect("the register as it was written") +} + +// --- §12.18, §12.19: the registers ------------------------------------------ + +#[test] +fn the_type_register_says_how_wide_an_its_is() { + let its = its(0); + assert_eq!(its.device_bits, 16); + // ID_bits: 16 bits of EventID and no more; ITT_entry_size: 8 bytes an entry. + assert!(its.events(16).is_some() && its.events(17).is_none()); + assert_eq!(its.events(1).map(|events| its.itt_bytes(events)), Some(16)); + // Each field at its largest, alone: every one is its value plus one. + let wide = |field: (u32, u32, u64)| probe(word(&[(0, 0, 1), field])).unwrap(); + assert_eq!(wide((17, 13, 31)).device_bits, 32); + assert!(wide((12, 8, 31)).events(32).is_some()); + let entry = wide((7, 4, 15)); + assert_eq!(entry.events(1).map(|events| entry.itt_bytes(events)), Some(32)); + let least = probe(1).unwrap(); + assert_eq!(least.device_bits, 1); + assert!(least.events(1).is_some() && least.events(2).is_none()); + assert_eq!(least.events(1).map(|events| least.itt_bytes(events)), Some(2)); +} + +#[test] +fn an_its_without_physical_lpis_is_refused() { + assert_eq!(probe(typer(&[]) & !1), Err(Lacks::PhysicalLpis)); + assert_eq!(probe(!1), Err(Lacks::PhysicalLpis)); + assert!(probe(u64::MAX).is_ok()); +} + +#[test] +fn a_table_register_says_what_it_backs_and_how_it_is_read() { + // Type [58:56], Entry_Size [52:48] the bytes minus one, Page_Size [9:8]. + let read = |kind: u64, entry: u64, page: u64| { + let backing = table(word(&[(58, 56, kind), (52, 48, entry), (9, 8, page)])); + (backing.table, backing.page, backing.entries(1)) + }; + assert_eq!(read(0b000, 0, 0b00), (Table::Unimplemented, PageSize::K4, 4096)); + assert_eq!(read(0b001, 7, 0b10), (Table::Devices, PageSize::K64, 65536 / 8)); + assert_eq!(read(0b100, 15, 0b01), (Table::Collections, PageSize::K16, 16384 / 16)); + assert_eq!(read(0b010, 31, 0b11), (Table::Other(0b010), PageSize::K64, 65536 / 32)); + assert_eq!(read(0b111, 0, 0b00).0, Table::Other(0b111)); + assert_eq!((PageSize::K4.bytes(), PageSize::K16.bytes(), PageSize::K64.bytes()), (4096, 16384, 65536)); +} + +#[test] +fn a_table_is_sized_in_its_own_pages_for_the_entries_it_must_hold() { + // 8 bytes an entry in 64 KiB pages: 8192 to the page. + let devices = table(word(&[(58, 56, 0b001), (52, 48, 7), (9, 8, 0b10)])); + assert_eq!(devices.pages(0), None); + assert_eq!((devices.pages(1), devices.pages(8192), devices.pages(8193)), (Some(1), Some(1), Some(2))); + // A flat table for every DeviceID of 16 bits is eight pages, and of 21 bits the field's 256. + assert_eq!(devices.pages(1 << 16), Some(8)); + assert_eq!(devices.pages(1 << 21), Some(256)); + assert_eq!(devices.pages((1 << 21) + 1), None); + assert_eq!(devices.pages(u64::MAX), None); + assert_eq!((devices.entries(8), devices.entries(256)), (1 << 16, 1 << 21)); +} + +#[test] +fn a_table_is_given_flat_valid_and_counted_in_its_own_pages() { + let at = Phys::<16>::new(0xffff_ffff_0000).expect("64 KiB aligned, below 2^48"); + let written = |page: u64, pages: u64| { + Some(word(&[ + (63, 63, 1), // Valid + (62, 62, 0), // Indirect + (61, 59, 0b111), // InnerCache: RaWaWb + (55, 53, 0b000), // OuterCache: as the inner + (47, 12, 0xffff_ffff_0000 >> 12), // Physical_Address + (11, 10, 0b01), // Shareability: inner + (9, 8, page), // Page_Size + (7, 0, pages - 1), // Size + ])) + }; + let paged = |page: u64| table(word(&[(9, 8, page)])); + assert_eq!(paged(0b00).baser(at, 1), written(0b00, 1)); + assert_eq!(paged(0b01).baser(at, 8), written(0b01, 8)); + assert_eq!(paged(0b10).baser(at, 256), written(0b10, 256)); + assert_eq!(paged(0b10).baser(at, 0), None); + assert_eq!(paged(0b10).baser(at, 257), None); +} + +// --- §5.2.8: the command queue ---------------------------------------------- + +#[test] +fn the_command_queue_is_counted_in_4_kib_pages_and_holds_one_command_fewer() { + let at = Phys::<16>::new(0x4003_0000).unwrap(); + // Valid [63], InnerCache [61:59], OuterCache [55:53], Physical_Address [51:12], Shareability [11:10], Size [7:0]. + let written = |pages: u64| { + word(&[(63, 63, 1), (61, 59, 0b111), (55, 53, 0), (51, 12, 0x4003_0000 >> 12), (11, 10, 0b01), (7, 0, pages - 1)]) + }; + assert_eq!(CommandQueue::new(1).unwrap().cbaser(at), written(1)); + assert_eq!(CommandQueue::new(256).unwrap().cbaser(at), written(256)); + assert!(CommandQueue::new(0).is_none() && CommandQueue::new(257).is_none()); + + // One page is 128 commands. Against a count kept beside it, over three laps. + let queue = CommandQueue::new(1).unwrap(); + let (mut writer, mut reader, mut held) = (0u64, 0u64, 0u32); + for step in 0..3 * 128 * 2 { + assert_eq!(queue.is_empty(writer, reader), held == 0, "step {step}"); + assert_eq!(queue.is_full(writer, reader), held == 127, "step {step}"); + if held < 127 && (step / 150u32).is_multiple_of(2) { + writer = queue.after(writer); + held += 1; + } else if held > 0 { + reader = queue.after(reader); + held -= 1; + } + assert!(writer < 4096 && reader < 4096 && writer % 32 == 0 && reader % 32 == 0); + } + assert_eq!(queue.after(4096 - 32), 0); + // An offset is the ITS's word: past the queue, it is still an offset in it. + assert_eq!(queue.after(u64::MAX), 0); + assert_eq!(queue.after(4096 + 64), 96); +} + +#[test] +fn an_offset_register_names_a_command_by_its_offset_field_alone() { + let queue = CommandQueue::new(2).unwrap(); + // Offset [19:5]; Stalled or Retry [0]; the rest reserved. + let beside = |offset: u64| word(&[(19, 5, offset), (0, 0, 1), (4, 1, 0xf), (63, 20, 0xfff_ffff_ffff)]); + assert_eq!(queue.offset(beside(0x40)), 0x40 << 5); + // An offset past the queue is a device's word: it still names a command in the queue. + assert!(queue.offset(word(&[(19, 5, 0x7fff)])) < 8192); + // `GITS_CREADR` with `Stalled` set and every reserved bit beside its offset is the same offset. + let clean = |offset: u64| word(&[(19, 5, offset)]); + assert!(queue.is_empty(clean(0x40), beside(0x40))); + assert!(!queue.is_empty(clean(0x41), beside(0x40))); + assert!(queue.is_full(clean(0x3f), beside(0x40))); + assert!(!queue.is_full(clean(0x40), beside(0x40))); + assert_eq!(queue.after(beside(0x40)), clean(0x41)); + assert_eq!(toyos_its::CREADR_STALLED, word(&[(0, 0, 1)])); +} + +// --- §5.3: commands --------------------------------------------------------- + +#[test] +fn each_command_holds_its_number_and_parameters_where_its_figure_has_them() { + let its = its(0); + let events = its.events(5).expect("32 events of an ITS of 16-bit EventIDs"); + let event = events.event(0x1f).expect("the last of 32 events"); + let itt = Phys::<8>::new(0xffff_ffff_ff00).expect("256-byte aligned"); + let target = its.target(0x1234, Phys::new(0x080a_0000).unwrap()); + let collection = its.collections(0xc012).collection(0xc011).expect("a collection the table holds"); + let lpi = space(16).lpi(0xabcd).expect("an LPI"); + + // MAPD (Figure 5-13): DeviceID [63:32]; Size [4:0] of DW1, the bits minus one; V [63] and ITT_addr [51:8] of DW2. + assert_eq!( + Command::MapDevice { device: 0xdead_beef, table: itt, events }.words(), + command(&[(0, 7, 0, 0x08), (0, 63, 32, 0xdead_beef), (1, 4, 0, 4), (2, 63, 63, 1), (2, 51, 8, 0xffff_ffff_ff00 >> 8)]) + ); + assert_eq!( + Command::UnmapDevice { device: 0xdead_beef }.words(), + command(&[(0, 7, 0, 0x08), (0, 63, 32, 0xdead_beef), (2, 63, 63, 0)]) + ); + // MAPC (Figure 5-12): V [63], RDbase [51:16] and ICID [15:0] of DW2. + assert_eq!( + Command::MapCollection { collection, target }.words(), + command(&[(0, 7, 0, 0x09), (2, 63, 63, 1), (2, 51, 16, 0x1234), (2, 15, 0, 0xc011)]) + ); + // MAPTI (Figure 5-15): DeviceID [63:32]; pINTID [63:32] and EventID [31:0] of DW1; ICID [15:0] of DW2. + assert_eq!( + Command::MapEvent { device: 0xdead_beef, event, lpi, collection }.words(), + command(&[(0, 7, 0, 0x0A), (0, 63, 32, 0xdead_beef), (1, 63, 32, 0xabcd), (1, 31, 0, 0x1f), (2, 15, 0, 0xc011)]) + ); + // SYNC (Figure 5-18): RDbase [51:16] of DW2. + assert_eq!(Command::Sync(target).words(), command(&[(0, 7, 0, 0x05), (2, 51, 16, 0x1234)])); +} + +/// `DISCARD` (Figure 5-7) and `INV` (Figure 5-9), one layout under two +/// numbers: doubleword 0 holds DeviceID [63:32], RES0 [31:8] and the command +/// number [7:0]; doubleword 1 holds RES0 [63:32] and EventID [31:0]; +/// doublewords 2 and 3 are RES0. +#[test] +fn inv_and_discard_hold_a_device_and_an_event_and_nothing_else() { + let event = its(0).events(16).unwrap().event(0xa5c3).unwrap(); + let figure = |number: u64| command(&[(0, 63, 32, 0xdead_beef), (0, 31, 8, 0), (0, 7, 0, number), (1, 63, 32, 0), (1, 31, 0, 0xa5c3)]); + assert_eq!(Command::Discard { device: 0xdead_beef, event }.words(), figure(0x0F)); + assert_eq!(Command::Reconfigure { device: 0xdead_beef, event }.words(), figure(0x0C)); + // The widest event there is fills its field and reaches no other. + let widest = probe(word(&[(0, 0, 1), (12, 8, 31)])).unwrap().events(32).unwrap().event(u32::MAX).unwrap(); + assert_eq!(Command::Discard { device: 0, event: widest }.words(), [0x0F, 0xffff_ffff, 0, 0]); + assert_eq!(Command::Reconfigure { device: u32::MAX, event: widest }.words(), [0xffff_ffff_0000_000c, 0xffff_ffff, 0, 0]); +} +#[test] +fn a_redistributor_is_named_as_the_its_asks_by_number_or_by_address() { + let frame = Phys::<16>::new(0xffff_fffe_0000).unwrap(); + let rdbase = |pta: u64| Command::Sync(its(pta).target(0x1234, frame)).words()[2]; + // PTA clear: GICR_TYPER.Processor_Number. PTA set: bits [51:16] of the frame's address. + assert_eq!(rdbase(0), word(&[(51, 16, 0x1234)])); + assert_eq!(rdbase(1), word(&[(51, 16, 0xffff_fffe_0000 >> 16)])); +} + +#[test] +fn a_devices_events_are_at_least_two_and_no_more_than_the_its_has_bits_for() { + let its = its(0); + assert_eq!(its.events(0), None); + assert_eq!(its.events(17), None); + // 8 bytes an entry: 2 events are 16 bytes and 2^16 are 512 KiB. + assert_eq!(its.events(1).map(|events| its.itt_bytes(events)), Some(16)); + assert_eq!(its.events(16).map(|events| its.itt_bytes(events)), Some(8 << 16)); + let size = |bits| Command::MapDevice { device: 0, table: Phys::new(0).unwrap(), events: its.events(bits).unwrap() }.words()[1]; + assert_eq!((size(1), size(16)), (0, 15)); +} + +#[test] +fn an_event_is_one_its_devices_table_holds() { + let its = its(0); + let events = its.events(5).unwrap(); + assert!(events.event(0).is_some() && events.event(31).is_some()); + assert_eq!(events.event(32), None); + assert_eq!(events.event(u32::MAX), None); + // The widest table there is holds every EventID. + let wide = probe(word(&[(0, 0, 1), (12, 8, 31)])).unwrap(); + assert!(wide.events(32).unwrap().event(u32::MAX).is_some()); +} + +/// §5.2.2 and §5.3.1: the collections are the table's, with those the ITS +/// holds itself (`HCC`) where it has no table or `CCT` [2] counts both, and +/// an ICID is 16 bits or, under `CIL` [36], `CIDbits` [35:32] plus one. +#[test] +fn a_collection_is_one_the_its_holds_or_its_table_does() { + let last = |its: Its, in_table: u64, id: u16| its.collections(in_table).collection(id).is_some(); + // No table: the three the ITS holds. + let held = probe(typer(&[])).unwrap(); + assert!(last(held, 0, 2) && !last(held, 0, 3)); + // A table without CCT: the table's alone. + assert!(last(held, 8, 7) && !last(held, 8, 8)); + // With CCT: both. + let cumulative = probe(typer(&[(2, 2, 1)])).unwrap(); + assert!(last(cumulative, 8, 10) && !last(cumulative, 8, 11)); + assert!(last(cumulative, 0, 2) && !last(cumulative, 0, 3)); + // No more than an ICID numbers: all 2^16 of 16 bits, and 2^4 under CIL with CIDbits 3. + assert!(last(held, 1 << 20, u16::MAX)); + let narrow = probe(typer(&[(36, 36, 1), (35, 32, 3)])).unwrap(); + assert!(last(narrow, 1 << 20, 15) && !last(narrow, 1 << 20, 16)); + // CIDbits says nothing where CIL is clear. + let unlimited = probe(typer(&[(35, 32, 3)])).unwrap(); + assert!(last(unlimited, 1 << 20, 16)); + // An ITS that holds none and was given no table has none. + let none = probe(1).unwrap(); + assert!(!last(none, 0, 0)); + // Every bit set and a table of every entry: still no more than a 16-bit ICID numbers. + assert!(last(probe(u64::MAX).unwrap(), u64::MAX, u16::MAX)); +} + +// --- §5.1: LPIs -------------------------------------------------------------- + +/// `GICD_TYPER` (§12.9.38): LPIS [17], IDbits [23:19] the INTID bits minus +/// one, num_LPIs [15:11]. +#[test] +fn an_lpi_layout_is_inside_the_distributors_intids() { + let typer = |fields: &[(u32, u32, u64)]| word(fields) as u32; + // No LPIs at all, whatever the bits. + assert_eq!(Layout::new(typer(&[(23, 19, 15)]), 14), None); + // 16 bits of INTID: 14, the fewest that reach an LPI, to 16. + let sixteen = typer(GICD_TYPER); + assert_eq!(Layout::new(sixteen, 13), None); + assert!(Layout::new(sixteen, 14).is_some() && Layout::new(sixteen, 16).is_some()); + assert_eq!(Layout::new(sixteen, 17), None); + // The field at its largest is 32 bits, and every other field says nothing of it. + let widest = typer(&[(17, 17, 1), (23, 19, 31), (31, 24, 0xff), (16, 16, 1), (10, 0, 0x7ff)]); + assert!(Layout::new(widest, 32).is_some()); + assert_eq!(Layout::new(widest, 33), None); + assert_eq!(Layout::new(u32::MAX, u8::MAX), None); +} + +#[test] +fn an_lpi_is_an_intid_from_8192_that_the_tables_and_the_distributor_both_count() { + let fourteen = space(14); + assert_eq!(fourteen.lpi(8191), None); + assert_eq!(fourteen.lpi(8192).map(|lpi| lpi.intid()), Some(8192)); + assert_eq!(fourteen.lpi(16383).map(|lpi| lpi.intid()), Some(16383)); + assert_eq!(fourteen.lpi(16384), None); + assert_eq!(fourteen.lpi(u32::MAX), None); + assert_eq!(fourteen.lpi(0), None); + + // num_LPIs [15:11] of 9: 2^10 LPIs, 8192 to 9215, under tables laid out for 16 bits. + let counted = word(&[GICD_TYPER, &[(15, 11, 9)]].concat()) as u32; + let layout = Layout::new(counted, 16).unwrap(); + let space = layout.space(layout.propbaser(Phys::new(0).unwrap())).unwrap(); + assert!(space.lpi(9215).is_some()); + assert_eq!(space.lpi(9216), None); + // And it never counts more than the tables' bits number: 2^16 of them under 14 bits is 8192. + let over = word(&[GICD_TYPER, &[(15, 11, 15)]].concat()) as u32; + let layout = Layout::new(over, 14).unwrap(); + let space = layout.space(layout.propbaser(Phys::new(0).unwrap())).unwrap(); + assert!(space.lpi(16383).is_some()); + assert_eq!(space.lpi(16384), None); + + // 32 bits: every INTID from 8192 up. + let widest = word(&[(17, 17, 1), (23, 19, 31)]) as u32; + let layout = Layout::new(widest, 32).unwrap(); + let space = layout.space(layout.propbaser(Phys::new(0).unwrap())).unwrap(); + assert!(space.lpi(u32::MAX).is_some() && space.lpi(8191).is_none()); + assert_eq!((layout.configuration_bytes(), layout.pending_bytes()), ((1 << 32) - 8192, 1 << 29)); +} + +/// §12.11.33: a redistributor may hold `GICR_PROPBASER` as its own, and its +/// `IDbits` [4:0] bounds the LPIs whatever was written. +#[test] +fn a_redistributor_that_reads_other_bits_back_has_no_lpis_of_the_layout() { + let layout = Layout::new(word(GICD_TYPER) as u32, 16).unwrap(); + let written = layout.propbaser(Phys::new(0x4000_0000).unwrap()); + assert!(layout.space(written).is_some()); + // Its cacheability and address may read back otherwise; its IDbits may not. + assert!(layout.space(written & 0x1f).is_some()); + assert!(layout.space(written | !0x1f).is_some()); + for other in [0u64, 13, 14, 16, 31] { + assert_eq!(layout.space(written & !0x1f | other), None, "IDbits {other}"); + } +} + +#[test] +fn the_configuration_table_starts_at_intid_8192_and_the_pending_table_at_zero() { + let layout = Layout::new(word(GICD_TYPER) as u32, 16).unwrap(); + let space = space(16); + // "(base address + (N - 8192))", one byte an LPI. + assert_eq!(space.lpi(8192).unwrap().configuration_index(), 0); + assert_eq!(space.lpi(65535).unwrap().configuration_index(), 65535 - 8192); + assert_eq!(layout.configuration_bytes(), 65536 - 8192); + // One bit an INTID, the first 1 KiB below every LPI. + assert_eq!(layout.pending_bytes(), 8192); + let fourteen = Layout::new(word(GICD_TYPER) as u32, 14).unwrap(); + assert_eq!((fourteen.configuration_bytes(), fourteen.pending_bytes()), (8192, 2048)); +} + +#[test] +fn an_lpis_configuration_is_its_priority_a_set_bit_and_its_enable() { + // Priority [7:2], bit [1] RES1, Enable [0]. + let byte = |priority: u64, enable: u64| word(&[(7, 2, priority), (1, 1, 1), (0, 0, enable)]) as u8; + assert_eq!(lpi::configuration(0xa0, true), byte(0xa0 >> 2, 1)); + assert_eq!(lpi::configuration(0xa0, false), byte(0xa0 >> 2, 0)); + // The priority's low two bits are not the table's to hold. + assert_eq!(lpi::configuration(0xff, false), byte(0x3f, 0)); + assert_eq!(lpi::configuration(0x03, true), byte(0, 1)); +} + +#[test] +fn the_redistributors_base_registers_name_the_two_tables() { + let layout = Layout::new(word(GICD_TYPER) as u32, 16).unwrap(); + // OuterCache [58:56], Physical_Address [51:12], Shareability [11:10], InnerCache [9:7], IDbits [4:0] the bits minus one. + assert_eq!( + layout.propbaser(Phys::new(0xffff_ffff_f000).unwrap()), + word(&[(58, 56, 0), (51, 12, 0xffff_ffff_f000 >> 12), (11, 10, 0b01), (9, 7, 0b111), (4, 0, 15)]) + ); + assert_eq!(Layout::new(word(GICD_TYPER) as u32, 14).unwrap().propbaser(Phys::new(0).unwrap()) & 0x1f, 13); + // PTZ [62], OuterCache [58:56], Physical_Address [51:16], Shareability [11:10], InnerCache [9:7]. + assert_eq!( + lpi::pendbaser(Phys::new(0xffff_ffff_0000).unwrap()), + word(&[(62, 62, 1), (58, 56, 0), (51, 16, 0xffff_ffff_0000 >> 16), (11, 10, 0b01), (9, 7, 0b111)]) + ); + assert_eq!(u64::from(lpi::CTLR_ENABLE_LPIS), word(&[(0, 0, 1)])); + // GICR_TYPER: PLPIS [0], Processor_Number [23:8] under an affinity and flags it is not. + assert_eq!(lpi::TYPER_PLPIS, word(&[(0, 0, 1)])); + assert_eq!(lpi::processor_number(word(&[(63, 32, 0xffff_ffff), (23, 8, 0xbeef), (7, 0, 0xff), (31, 24, 0xff)])), 0xbeef); +} + +// --- the register maps ------------------------------------------------------- + +/// §12.18: "The control registers, which are located at ITS_base + 0x000000. +/// The interrupt translation space, which is located at ITS_base + +/// 0x010000." Table 12-33, the ITS control register map, and Table 12-34, +/// the ITS translation register map, row by row; each register's own +/// description (§12.19.4, .13, .2, .5, .3, .1, .12) states the same offset. +#[test] +fn the_its_registers_sit_where_tables_12_33_and_12_34_have_them() { + let control: [(usize, usize); 5] = [ + (0x0000, toyos_its::GITS_CTLR), + (0x0008, toyos_its::GITS_TYPER), + (0x0080, toyos_its::GITS_CBASER), + (0x0088, toyos_its::GITS_CWRITER), + (0x0090, toyos_its::GITS_CREADR), + ]; + for (specified, declared) in control { + assert_eq!(declared, specified); + } + // "0x0100-0x0138 GITS_BASER", and §12.19.1's "0x0100 + (8 * n)", n = 0 - 7. + assert_eq!((0..9).map(gits_baser).collect::>(), [ + Some(0x0100), + Some(0x0108), + Some(0x0110), + Some(0x0118), + Some(0x0120), + Some(0x0128), + Some(0x0130), + Some(0x0138), + None + ]); + // "0x0040 GITS_TRANSLATER", of the frame at ITS_base + 0x010000. + assert_eq!((toyos_its::TRANSLATION_FRAME, toyos_its::GITS_TRANSLATER), (0x01_0000, 0x0040)); + assert_eq!(u64::from(toyos_its::CTLR_ENABLED), word(&[(0, 0, 1)])); +} + +/// §12.10, Table 12-27, the GIC physical LPI Redistributor register map, by +/// its offsets from `RD_base`; §12.11.33 and §12.11.32 state the same. +#[test] +fn the_redistributors_lpi_registers_sit_where_table_12_27_has_them() { + assert_eq!((lpi::GICR_PROPBASER, lpi::GICR_PENDBASER), (0x0070, 0x0078)); +} diff --git a/toyos-phys/Cargo.toml b/toyos-phys/Cargo.toml new file mode 100644 index 00000000000..0453caeaf4c --- /dev/null +++ b/toyos-phys/Cargo.toml @@ -0,0 +1,12 @@ +# A member of the host workspace (root `Cargo.toml`): what toyos-smmu and +# toyos-its both take an address as, declared once. An SMMUv3's and a GIC's +# address fields are bits [47:n] of an address or bits [51:n] with the top +# four zero, so one type bounds what either is given. + +[package] +name = "toyos-phys" +description = "A physical address below 2^48 by its alignment, the type an SMMUv3's and a GIC ITS's address fields are filled from, pure, forbid(unsafe_code)." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false diff --git a/toyos-phys/src/lib.rs b/toyos-phys/src/lib.rs new file mode 100644 index 00000000000..415c52a3b8c --- /dev/null +++ b/toyos-phys/src/lib.rs @@ -0,0 +1,33 @@ +//! [`Phys`]: a physical address aligned to `1 << ALIGN` bytes and below 2^48, +//! so a register or descriptor field that takes an address's bits `[47:ALIGN]` +//! holds every bit of one. +//! +//! `no_std`, no allocation, no `unsafe`. + +#![no_std] +#![forbid(unsafe_code)] + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Phys(u64); + +impl Phys { + const FIELD: u64 = (1 << 48) - (1 << ALIGN); + + pub const fn new(address: u64) -> Option { + if address & !Self::FIELD == 0 { + Some(Self(address)) + } else { + None + } + } + + /// The address a word holds in its bits `[47:ALIGN]`, whatever its other + /// bits are. + pub const fn of(word: u64) -> Self { + Self(word & Self::FIELD) + } + + pub const fn get(self) -> u64 { + self.0 + } +} diff --git a/toyos-phys/tests/phys.rs b/toyos-phys/tests/phys.rs new file mode 100644 index 00000000000..674b72db9d9 --- /dev/null +++ b/toyos-phys/tests/phys.rs @@ -0,0 +1,27 @@ +use toyos_phys::Phys; + +#[test] +fn an_address_off_its_alignment_or_past_48_bits_is_no_address() { + assert_eq!(Phys::<6>::new(0x40).map(Phys::get), Some(0x40)); + assert_eq!(Phys::<6>::new(0x20), None); + assert_eq!(Phys::<8>::new(0x100).map(Phys::get), Some(0x100)); + assert_eq!(Phys::<8>::new(0x80), None); + assert_eq!(Phys::<12>::new(0xfff), None); + assert_eq!(Phys::<12>::new(0x800), None); + assert_eq!(Phys::<16>::new(0x8000), None); + assert_eq!(Phys::<21>::new(0x10_0000), None); + assert_eq!(Phys::<21>::new(0x20_0000).map(Phys::get), Some(0x20_0000)); + assert_eq!(Phys::<12>::new(1 << 48), None); + assert_eq!(Phys::<12>::new((1 << 48) - 0x1000).map(Phys::get), Some((1 << 48) - 0x1000)); + assert_eq!(Phys::<16>::new((1 << 48) - 0x1_0000).map(Phys::get), Some((1 << 48) - 0x1_0000)); + assert_eq!(Phys::<0>::new(u64::MAX), None); + assert_eq!(Phys::<0>::new((1 << 48) - 1).map(Phys::get), Some((1 << 48) - 1)); +} + +#[test] +fn a_words_address_field_is_its_bits_from_the_alignment_to_47() { + assert_eq!(Phys::<12>::of(u64::MAX).get(), 0x0000_ffff_ffff_f000); + assert_eq!(Phys::<12>::of(0x8060_0000_4020_3fff).get(), 0x4020_3000); + assert_eq!(Phys::<21>::of(u64::MAX).get(), 0x0000_ffff_ffe0_0000); + assert_eq!(Phys::<12>::of(0x1000), Phys::new(0x1000).unwrap()); +} diff --git a/toyos-smmu/Cargo.toml b/toyos-smmu/Cargo.toml new file mode 100644 index 00000000000..0c5953a87d8 --- /dev/null +++ b/toyos-smmu/Cargo.toml @@ -0,0 +1,23 @@ +# A member of the host workspace (root `Cargo.toml`), like toyos-acpi: it is +# pure and its tests run on the host. +# +# What lives here is every word an SMMUv3 and its driver exchange. The unit +# decides which memory a device reaches, so a field one bit off is an isolation +# hole no boot shows: a stream that should abort translates, or two domains +# share a tag. Each encoding is held on the host to the specification's own +# field positions, and the kernel's `arch::aarch64` unit is left the registers +# and the memory. + +[package] +name = "toyos-smmu" +description = "The SMMUv3's encodings: stream table entries, context descriptors, commands, event records, register fields and the stage 1 table plan, pure, forbid(unsafe_code)." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false + +[dependencies] +# The VMSAv8-64 stage 1 descriptors and the `MAIR` they index: the unit walks +# the format the loader's tables are written in, read from its one declaration. +toyos-bootmap = { path = "../toyos-bootmap" } +toyos-phys = { path = "../toyos-phys" } diff --git a/toyos-smmu/src/config.rs b/toyos-smmu/src/config.rs new file mode 100644 index 00000000000..6c28c1b4c81 --- /dev/null +++ b/toyos-smmu/src/config.rs @@ -0,0 +1,102 @@ +//! The stream table entry (§5.2) and the context descriptor (§5.4): 64 bytes +//! each, eight little-endian doublewords, a field's bit `n` being bit +//! `n % 64` of doubleword `n / 64`. + +use toyos_phys::Phys; + +use crate::unit::Unit; +use crate::Asid; + +/// A stream table entry: [`Ste::ABORT`] or [`Ste::stage1`], and no other. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Ste([u64; 8]); + +/// `V`, bit [0]. +const STE_V: u64 = 1 << 0; +/// `Config`, bits [3:1]: `0b101` is stage 1 translate, stage 2 bypass. +const STE_STAGE1: u64 = 0b101 << 1; +/// Doubleword 1: `S1CIR` [67:66] and `S1COR` [69:68] `0b01`, write-back +/// read-allocate, and `S1CSH` [71:70] `0b11`, inner shareable — how the unit +/// reads the context descriptor. +const STE_CD_CACHED: u64 = 0b01 << 2 | 0b01 << 4 | 0b11 << 6; +/// `S1STALLD`, bit [91]. +const STE_S1STALLD: u64 = 1 << 27; + +impl Ste { + /// `V` set and `Config` `0b000`: every transaction of the stream is + /// aborted, and no event is recorded for it. An entry of all zeroes + /// aborts too, and records `C_BAD_STE` each time. + pub const ABORT: Self = Self([STE_V, 0, 0, 0, 0, 0, 0, 0]); + + /// The stream translated by stage 1 through the one context descriptor + /// at `context`: `S1ContextPtr` [55:6], `S1CDMax` [63:59] zero so a + /// transaction carrying a SubstreamID is aborted, `STRW` [95:94] zero for + /// the Non-secure EL1 regime, `EATS` [93:92] zero so ATS is refused, and + /// no stage 2 field. `None` where the descriptor is past the unit's + /// output size. + pub const fn stage1(context: Phys<6>, unit: &Unit) -> Option { + if !unit.reaches(context.get(), 64) { + return None; + } + let stall = if unit.stall_disable() { STE_S1STALLD } else { 0 }; + Some(Self([STE_V | STE_STAGE1 | context.get(), STE_CD_CACHED | stall, 0, 0, 0, 0, 0, 0])) + } + + /// The entry's eight doublewords, as the stream table holds them. + pub const fn words(&self) -> [u64; 8] { + self.0 + } +} + +/// A context descriptor: [`Cd::new`]'s, and no other. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Cd([u64; 8]); + +/// `T0SZ`, bits [5:0]: a 48-bit input, walked from level 0. +const CD_T0SZ: u64 = 64 - crate::table::INPUT_BITS as u64; +/// `TG0` [7:6] zero is the 4 KiB granule; `IR0` [9:8] and `OR0` [11:10] +/// `0b01`, write-back write-allocate, and `SH0` [13:12] `0b11`, inner +/// shareable — how the unit walks the tables; `EPD0` [14] and `ENDI` [15] +/// clear, walk `TTB0` little-endian. +const CD_WALK: u64 = 0b01 << 8 | 0b01 << 10 | 0b11 << 12; +/// `EPD1`, bit [30]: no walk through `TTB1`, so the upper half faults. +const CD_EPD1: u64 = 1 << 30; +/// `V`, bit [31]. +const CD_V: u64 = 1 << 31; +/// `AA64`, bit [41]: VMSAv8-64 descriptors. +const CD_AA64: u64 = 1 << 41; +/// `R` [45] records a translation-related fault and `A` [46] aborts its +/// transaction rather than completing it as read-as-zero, write-ignored; +/// `S` [44] stays clear, so none stalls (§5.5). +const CD_FAULTS: u64 = 1 << 45 | 1 << 46; +/// `ASET`, bit [47]: the ASID is the unit's own and no CPU process's. +const CD_ASET: u64 = 1 << 47; + +impl Cd { + /// The context whose tables start at `root`, tagged `asid`: `IPS` + /// [34:32] from the unit's output size, `ASID` [63:48], `TTB0` [119:68] + /// holding the root's bits [55:4], and `MAIR0` [223:192] the attributes + /// [`crate::table`]'s leaves index. `AFFD` [35] stays clear, and every + /// leaf written sets its access flag. `None` where the root table is + /// past the unit's output size. + pub const fn new(root: Phys<12>, asid: Asid, unit: &Unit) -> Option { + if !unit.reaches(root.get(), 4096) { + return None; + } + let word0 = CD_T0SZ + | CD_WALK + | CD_EPD1 + | CD_V + | unit.ips() << 32 + | CD_AA64 + | CD_FAULTS + | CD_ASET + | (asid.get() as u64) << 48; + Some(Self([word0, root.get(), 0, crate::table::MAIR0 as u64, 0, 0, 0, 0])) + } + + /// The descriptor's eight doublewords. + pub const fn words(&self) -> [u64; 8] { + self.0 + } +} diff --git a/toyos-smmu/src/lib.rs b/toyos-smmu/src/lib.rs new file mode 100644 index 00000000000..0ed9a36b154 --- /dev/null +++ b/toyos-smmu/src/lib.rs @@ -0,0 +1,39 @@ +//! Every word an SMMUv3 and its driver exchange, as Arm IHI 0070 issue H.a +//! lays it out: what the identification registers say a unit can do +//! ([`unit`]), the stream table entry and context descriptor that put a +//! stream under stage 1 translation or abort it ([`config`]), the commands and +//! the event records ([`queue`]), and the stage 1 tables a context descriptor +//! names ([`table`]). +//! +//! The crate encodes one configuration and no other: a stream either aborts +//! or is translated by stage 1 alone through one context descriptor, under +//! the Non-secure EL1 regime, with 4 KiB-granule VMSAv8-64 tables over a +//! 48-bit input. No stream bypasses: [`config::Ste`] is made only as an +//! abort or as that translation, and holds its words where no other crate +//! can write them. +//! +//! What a unit reports about itself is a device's word and is refused by name +//! where the driver could not act on it; nothing here panics on a register's +//! or a record's value. An address is a [`toyos_phys::Phys`], and whatever +//! takes one for the unit to read answers `None` past the unit's output size. +//! +//! `no_std`, no allocation, no `unsafe`. + +#![no_std] +#![forbid(unsafe_code)] + +pub mod config; +pub mod queue; +pub mod table; +pub mod unit; + +/// A context's tag in the unit's TLB, no wider than the unit's ASIDs: made by +/// [`unit::Unit::asid`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Asid(u16); + +impl Asid { + pub const fn get(self) -> u16 { + self.0 + } +} diff --git a/toyos-smmu/src/queue.rs b/toyos-smmu/src/queue.rs new file mode 100644 index 00000000000..dbfad87ce37 --- /dev/null +++ b/toyos-smmu/src/queue.rs @@ -0,0 +1,246 @@ +//! The two queues: their index arithmetic (§3.5.1), the commands the driver +//! writes (chapter 4) and the event records it reads (§7.3). + +use crate::Asid; + +/// The indexes of a circular queue of `1 << log2size` entries. An index +/// register holds the entry's index in its low `log2size` bits and a wrap +/// flag in the next one up, which toggles each time the index passes the top. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct Indexes { + pub(crate) log2size: u8, +} + +impl Indexes { + /// The index and its wrap flag, which is all of an index register that + /// names a position. + const fn position(self, register: u32) -> u32 { + register & ((2 << self.log2size) - 1) + } + + const fn slot(self, register: u32) -> usize { + (register & ((1 << self.log2size) - 1)) as usize + } + + /// The indexes and their wrap flags are equal. + const fn is_empty(self, prod: u32, cons: u32) -> bool { + self.position(prod) == self.position(cons) + } + + /// The position after `register`'s, wrap flag toggled where it passes + /// the top. + const fn after(self, register: u32) -> u32 { + self.position(self.position(register) + 1) + } +} + +/// The command queue's indexes, as the driver that produces into it reads +/// `SMMU_CMDQ_PROD` and `SMMU_CMDQ_CONS`: made by +/// [`Unit::command_queue`](crate::unit::Unit::command_queue). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Commands(pub(crate) Indexes); + +impl Commands { + /// The entry the next command is written to. + pub const fn slot(self, prod: u32) -> usize { + self.0.slot(prod) + } + + /// No command can be written: the indexes are equal and the wrap flags + /// differ. + pub const fn is_full(self, prod: u32, cons: u32) -> bool { + self.0.position(prod) ^ self.0.position(cons) == 1 << self.0.log2size + } + + /// The unit has consumed every command written. + pub const fn is_empty(self, prod: u32, cons: u32) -> bool { + self.0.is_empty(prod, cons) + } + + /// What `SMMU_CMDQ_PROD` is written with once the command at `prod` is + /// in memory: the position after it, index and wrap flag at once. + pub const fn after(self, prod: u32) -> u32 { + self.0.after(prod) + } +} + +/// `SMMU_EVENTQ_PROD.OVFLG` and `SMMU_EVENTQ_CONS.OVACKFLG`, bit [31] of each. +const OVERFLOW: u32 = 1 << 31; + +/// The event queue's indexes, as the driver that consumes from it reads +/// `SMMU_EVENTQ_PROD` and `SMMU_EVENTQ_CONS`: made by +/// [`Unit::event_queue`](crate::unit::Unit::event_queue). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Events(pub(crate) Indexes); + +impl Events { + /// The entry the next record is read from. + pub const fn slot(self, cons: u32) -> usize { + self.0.slot(cons) + } + + /// No record to read. + pub const fn is_empty(self, prod: u32, cons: u32) -> bool { + self.0.is_empty(prod, cons) + } + + /// Records were dropped on a full queue and that is not yet + /// acknowledged: `OVFLG` differs from `OVACKFLG` (§7.4). + pub const fn overflowed(self, prod: u32, cons: u32) -> bool { + (prod ^ cons) & OVERFLOW != 0 + } + + /// What `SMMU_EVENTQ_CONS` is written with once the record at `cons` is + /// read: the position after it, and `OVACKFLG` equal to the `OVFLG` + /// `prod` was read with. That acknowledges an overflow seen in `prod`, + /// and no value this answers raises one the unit did not report. + pub const fn after(self, cons: u32, prod: u32) -> u32 { + self.0.after(cons) | prod & OVERFLOW + } +} + +/// How a `CMD_SYNC` says it is done (§4.7.3, `CS`). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Signal { + /// `SIG_NONE`: `SMMU_CMDQ_CONS` passing it is the only sign. + None, + /// `SIG_IRQ` with `MSIAddress` zero: the unit's wired sync interrupt. + Irq, +} + +/// One 16-byte command. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Command { + /// `CMD_CFGI_STE` (§4.3.1), `Leaf` set: forget one stream's entry and + /// every context descriptor reached through it. + ForgetStream(u32), + /// `CMD_CFGI_ALL` (§4.3.9): forget every stream's configuration. + ForgetAll, + /// `CMD_TLBI_NH_ASID` (§4.4.2.2), VMID 0: drop every non-global + /// translation cached under one ASID. + InvalidateAsid(Asid), + /// `CMD_TLBI_NSNH_ALL` (§4.4.4.1): drop every Non-secure translation. + InvalidateAll, + /// `CMD_SYNC` (§4.7.3): consumed only once every command before it has + /// taken effect. + Sync(Signal), +} + +impl Command { + /// The command's two doublewords: the opcode in bits [7:0] of the first. + pub const fn words(self) -> [u64; 2] { + match self { + // StreamID [63:32]; Leaf [64]. + Self::ForgetStream(stream) => [0x03 | (stream as u64) << 32, 1], + // `CMD_CFGI_STE_RANGE` with Range [68:64] of 31. + Self::ForgetAll => [0x04, 31], + // ASID [63:48], VMID [47:32]. + Self::InvalidateAsid(asid) => [0x11 | (asid.get() as u64) << 48, 0], + Self::InvalidateAll => [0x30, 0], + // CS [13:12]. + Self::Sync(signal) => { + let cs = match signal { + Signal::None => 0b00, + Signal::Irq => 0b01, + }; + [0x46 | cs << 12, 0] + } + } + } +} + +/// An event's number, bits [7:0] of its record (§7.3). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Code { + /// `F_UUT` (0x01). + UnsupportedTransaction, + /// `C_BAD_STREAMID` (0x02): past the stream table. + BadStream, + /// `F_STE_FETCH` (0x03). + EntryFetch, + /// `C_BAD_STE` (0x04): an invalid or ILLEGAL stream table entry. + BadEntry, + /// `F_BAD_ATS_TREQ` (0x05). + AtsRequest, + /// `F_STREAM_DISABLED` (0x06). + StreamDisabled, + /// `F_TRANSL_FORBIDDEN` (0x07). + TranslatedForbidden, + /// `C_BAD_SUBSTREAMID` (0x08). + BadSubstream, + /// `F_CD_FETCH` (0x09). + ContextFetch, + /// `C_BAD_CD` (0x0A): an invalid or ILLEGAL context descriptor. + BadContext, + /// `F_WALK_EABT` (0x0B). + WalkAbort, + /// `F_TRANSLATION` (0x10): no valid descriptor for the address. + Translation, + /// `F_ADDR_SIZE` (0x11). + AddressSize, + /// `F_ACCESS` (0x12). + AccessFlag, + /// `F_PERMISSION` (0x13). + Permission, + /// `F_TLB_CONFLICT` (0x20). + TlbConflict, + /// `F_CFG_CONFLICT` (0x21). + ConfigurationConflict, + /// `E_PAGE_REQUEST` (0x24). + PageRequest, + /// `F_VMS_FETCH` (0x25). + VmsFetch, + /// A number this issue gives no record or the unit defines for itself. + Other(u8), +} + +/// What a device tried, where the record says: the four translation-related +/// faults carry it (§7.3.13 to §7.3.16). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Attempt { + /// `InputAddr`, bits [191:128]: the device address. + pub address: u64, + /// `RnW`, bit [99], clear. + pub write: bool, +} + +/// One 32-byte event record, decoded. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Event { + /// `StreamID`, bits [63:32]. + pub stream: u32, + pub code: Code, + pub attempt: Option, +} + +pub const fn event(record: [u64; 4]) -> Event { + let code = match record[0] as u8 { + 0x01 => Code::UnsupportedTransaction, + 0x02 => Code::BadStream, + 0x03 => Code::EntryFetch, + 0x04 => Code::BadEntry, + 0x05 => Code::AtsRequest, + 0x06 => Code::StreamDisabled, + 0x07 => Code::TranslatedForbidden, + 0x08 => Code::BadSubstream, + 0x09 => Code::ContextFetch, + 0x0A => Code::BadContext, + 0x0B => Code::WalkAbort, + 0x10 => Code::Translation, + 0x11 => Code::AddressSize, + 0x12 => Code::AccessFlag, + 0x13 => Code::Permission, + 0x20 => Code::TlbConflict, + 0x21 => Code::ConfigurationConflict, + 0x24 => Code::PageRequest, + 0x25 => Code::VmsFetch, + other => Code::Other(other), + }; + let attempt = match code { + Code::Translation | Code::AddressSize | Code::AccessFlag | Code::Permission => { + Some(Attempt { address: record[2], write: record[1] & 1 << 35 == 0 }) + } + _ => None, + }; + Event { stream: (record[0] >> 32) as u32, code, attempt } +} diff --git a/toyos-smmu/src/table.rs b/toyos-smmu/src/table.rs new file mode 100644 index 00000000000..f8f9a2cf669 --- /dev/null +++ b/toyos-smmu/src/table.rs @@ -0,0 +1,137 @@ +//! The stage 1 tables a context descriptor names: VMSAv8-64 with the 4 KiB +//! granule over a 48-bit input, four levels of 512 descriptors (Arm ARM, +//! DDI 0487 M.d, §D8.3.1), and the plan of one mapping in them. The +//! descriptors are [`toyos_bootmap::aarch64`]'s, which the loader's tables are +//! written with, under the bits that make a leaf a device's and no CPU's. +//! +//! A device's memory is mapped by 2 MiB blocks at level 2. The page an ITS +//! takes a message on is the one 4 KiB page at level 3: a block there would +//! hand the device every register that shares its 2 MiB. + +use toyos_bootmap::aarch64::{block, page, table, MAIR, PXN}; +use toyos_bootmap::Cache; +use toyos_phys::Phys; + +use crate::unit::Unit; + +/// The width of a device address these tables translate. +pub const INPUT_BITS: u32 = 48; +/// Levels 0 to 3, each indexed by nine bits of the address. +pub const LEVELS: usize = 4; + +/// `CD.MAIR0`, attributes 0 to 3 of the `MAIR` the descriptors' `AttrIndx` +/// names: a [`Cache::Device`] leaf's is Device-nGnRE and a [`Cache::Memory`] +/// leaf's Normal write-back. +pub const MAIR0: u32 = MAIR as u32; + +/// Table D8-50 and Table D8-52: bit [0] valid; bit [1] set for a table +/// descriptor and a level 3 page, clear for a block. +const VALID: u64 = 1 << 0; +const TABLE_OR_PAGE: u64 = 1 << 1; +/// `AP[1]`, bit [6]: an unprivileged access is permitted. A PCIe transaction +/// without a PASID prefix is unprivileged (IHI 0070 §13.7), so a leaf without +/// it is one no device reaches. +const AP_UNPRIVILEGED: u64 = 1 << 6; +/// `AP[2]`, bit [7]: read-only. +const AP_READ_ONLY: u64 = 1 << 7; +/// `nG`, bit [11]: the translation is its context's alone. A leaf without it +/// is global, cached for every ASID of the regime and so for every domain, +/// and no invalidation by ASID removes it. +const NOT_GLOBAL: u64 = 1 << 11; + +/// What a device's leaf adds to the loader's: unprivileged, its context's +/// alone, and executable at no level — the loader's memory is executable at +/// EL1, which no device's access is. +const DEVICES: u64 = AP_UNPRIVILEGED | NOT_GLOBAL | PXN; + +/// What a device may do to memory mapped for it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Access { + Read, + ReadWrite, +} + +/// What one mapping puts at a device address. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Leaf { + /// 2 MiB of memory. + Memory(Phys<21>, Access), + /// The 4 KiB page holding the register a message is written to. + Doorbell(Phys<12>), +} + +/// The descriptor naming the next table down, at levels 0 to 2. Its +/// hierarchical fields stay clear: the leaf alone decides the access. `None` +/// where the table is past the unit's output size. +pub const fn next(at: Phys<12>, unit: &Unit) -> Option { + if unit.reaches(at.get(), 4096) { + Some(table(at.get())) + } else { + None + } +} + +/// One mapping, planned: the index to follow in each table from the root, +/// and the descriptor to store at the last of them. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Path { + indices: [usize; LEVELS], + tables: usize, + /// The leaf: a level 2 block, or a level 3 page. + pub descriptor: u64, +} + +impl Path { + /// The index in the table at each level walked, the root first; the + /// last is the leaf's own slot, and every one before it holds a + /// [`next`] descriptor. + pub fn indices(&self) -> &[usize] { + &self.indices[..self.tables] + } +} + +/// The path to `leaf` at the device address `at`. `None` where `at` is past +/// the input or not aligned to what the leaf maps, or what the leaf maps is +/// past the unit's output size. +pub const fn plan(at: u64, leaf: Leaf, unit: &Unit) -> Option { + let (tables, size, output, descriptor) = match leaf { + Leaf::Memory(memory, access) => { + let access = match access { + Access::Read => AP_READ_ONLY, + Access::ReadWrite => 0, + }; + (3, 1u64 << 21, memory.get(), block(memory.get(), Cache::Memory) | DEVICES | access) + } + Leaf::Doorbell(register) => (4, 1u64 << 12, register.get(), page(register.get(), Cache::Device) | DEVICES), + }; + if at >> INPUT_BITS != 0 || at & (size - 1) != 0 || !unit.reaches(output, size) { + return None; + } + const fn index(at: u64, level: u32) -> usize { + (at >> (39 - 9 * level) & 0x1FF) as usize + } + Some(Path { indices: [index(at, 0), index(at, 1), index(at, 2), index(at, 3)], tables, descriptor }) +} + +/// What a descriptor read back from a table is, by the level it was read at. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Entry { + /// Nothing is mapped under it. + Invalid, + /// The next table down. + Table(Phys<12>), + /// A block or a page: something is mapped here. + Mapped, +} + +/// `descriptor` as the walk reads it at `level`: bit [1] names a table above +/// level 3 and a page at it, and a table's address is bits [47:12]. +pub const fn entry(descriptor: u64, level: usize) -> Entry { + if descriptor & VALID == 0 { + Entry::Invalid + } else if level < LEVELS - 1 && descriptor & TABLE_OR_PAGE != 0 { + Entry::Table(Phys::of(descriptor)) + } else { + Entry::Mapped + } +} diff --git a/toyos-smmu/src/unit.rs b/toyos-smmu/src/unit.rs new file mode 100644 index 00000000000..b706371d68d --- /dev/null +++ b/toyos-smmu/src/unit.rs @@ -0,0 +1,269 @@ +//! The register map (§6.2) and the fields of the registers the driver reads +//! and writes (§6.3), and [`probe`]: what a unit's identification registers +//! let this crate's one configuration be built on. + +use toyos_phys::Phys; + +use crate::queue::{Commands, Events, Indexes}; +use crate::Asid; + +/// Offsets in register page 0 (§6.2.1). +pub const IDR0: usize = 0x000; +pub const IDR1: usize = 0x004; +pub const IDR5: usize = 0x014; +pub const CR0: usize = 0x020; +pub const CR0ACK: usize = 0x024; +pub const CR1: usize = 0x028; +pub const CR2: usize = 0x02C; +pub const GBPA: usize = 0x044; +pub const IRQ_CTRL: usize = 0x050; +pub const IRQ_CTRLACK: usize = 0x054; +pub const GERROR: usize = 0x060; +pub const GERRORN: usize = 0x064; +pub const STRTAB_BASE: usize = 0x080; +pub const STRTAB_BASE_CFG: usize = 0x088; +pub const CMDQ_BASE: usize = 0x090; +pub const CMDQ_PROD: usize = 0x098; +pub const CMDQ_CONS: usize = 0x09C; +pub const EVENTQ_BASE: usize = 0x0A0; +/// The event queue's indexes are in register page 1, 64 KiB above page 0 +/// (§6.1, §6.2.2): `SMMU_EVENTQ_PROD` and `SMMU_EVENTQ_CONS` at 0xA8 and 0xAC of it. +pub const EVENTQ_PROD: usize = 0x1_00A8; +pub const EVENTQ_CONS: usize = 0x1_00AC; + +/// `SMMU_CR0` (§6.3.9), and `SMMU_CR0ACK`, which reads each bit back once +/// the unit has acted on it. +pub const CR0_SMMUEN: u32 = 1 << 0; +pub const CR0_EVENTQEN: u32 = 1 << 2; +pub const CR0_CMDQEN: u32 = 1 << 3; + +/// `SMMU_CR1` (§6.3.11): the unit reads its stream table and its queues as +/// inner-shareable write-back memory — `TABLE_SH` [11:10], `TABLE_OC` [9:8], +/// `TABLE_IC` [7:6], `QUEUE_SH` [5:4], `QUEUE_OC` [3:2], `QUEUE_IC` [1:0], +/// each shareability `0b11` and each cacheability `0b01`. +pub const CR1_WRITE_BACK: u32 = 0b11 << 10 | 0b01 << 8 | 0b01 << 6 | 0b11 << 4 | 0b01 << 2 | 0b01; + +/// `SMMU_CR2` (§6.3.12): `RECINVSID` [1], so a transaction under a StreamID +/// past the stream table is recorded as well as aborted, and `PTM` [2], so +/// no CPU's broadcast TLB invalidation reaches the unit's entries: its ASIDs +/// are its own. `E2H` [0] stays clear. +pub const CR2_RECORD_PRIVATE: u32 = 1 << 1 | 1 << 2; + +/// `SMMU_GBPA` (§6.3.15): what happens to every transaction while `SMMUEN` +/// is clear. `ABORT` [20] aborts them; a write takes effect by carrying +/// `Update` [31], which reads back set until the unit has taken it (§6.3.15.1). +pub const GBPA_ABORT: u32 = 1 << 20; +pub const GBPA_UPDATE: u32 = 1 << 31; + +/// `SMMU_IRQ_CTRL` (§6.3.17), and `SMMU_IRQ_CTRLACK` as `SMMU_CR0ACK`. +pub const IRQ_GERROR: u32 = 1 << 0; +pub const IRQ_EVENTQ: u32 = 1 << 2; + +/// `SMMU_GERROR` (§6.3.19): an error is active while its bit differs from +/// `SMMU_GERRORN`'s, and writing `GERRORN` equal acknowledges it (§7.5). +pub const fn active_errors(gerror: u32, gerrorn: u32) -> u32 { + gerror ^ gerrorn +} + +/// `SMMU_GERROR.CMDQ_ERR`: the command queue stopped on [`CommandError`]. +pub const GERROR_CMDQ: u32 = 1 << 0; +/// `SMMU_GERROR.EVENTQ_ABT_ERR`: a write to the event queue aborted, and +/// records were lost. +pub const GERROR_EVENTQ_ABORT: u32 = 1 << 2; +/// `SMMU_GERROR.SFM_ERR`: the unit entered Service Failure Mode. +pub const GERROR_SERVICE_FAILURE: u32 = 1 << 8; + +/// `SMMU_CMDQ_CONS.ERR` [30:24] (§6.3.28, §7.1): why the command at `RD` +/// was not consumed. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum CommandError { + None, + /// `CERROR_ILL`: a reserved opcode or an invalid parameter. + Illegal, + /// `CERROR_ABT`: reading the command aborted. + Abort, + /// `CERROR_ATC_INV_SYNC`. + AtcInvalidation, + Reserved(u8), +} + +pub const fn command_error(cons: u32) -> CommandError { + match (cons >> 24 & 0x7F) as u8 { + 0 => CommandError::None, + 1 => CommandError::Illegal, + 2 => CommandError::Abort, + 3 => CommandError::AtcInvalidation, + other => CommandError::Reserved(other), + } +} + +/// What a unit lacks that this crate's one configuration needs. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Lacks { + /// `IDR0.S1P` clear: no stage 1 translation. + Stage1, + /// `IDR0.TTF[1]` clear: no VMSAv8-64 tables. + Aarch64Tables, + /// `IDR0.TTENDIAN` is big-endian only. + LittleEndianTables, + /// `IDR0.STALL_MODEL` is `0b10`, under which every fault stalls its + /// transaction until software answers it; or a reserved value. + Termination(u8), + /// `IDR5.GRAN4K` clear. + Granule4K, + /// `IDR1.TABLES_PRESET` or `IDR1.QUEUES_PRESET`: the unit's table or + /// queues sit at an address of its own choosing. + FixedStructures, + /// `IDR1.SIDSIZE`, `CMDQS` or `EVENTQS` holds a reserved value. + Size(u8), + /// `IDR0.COHACC` clear with no override from the IORT: the unit does not + /// read its tables and queues coherently with the CPUs' caches, which + /// every cacheability written here ([`CR1_WRITE_BACK`], the stream table + /// entry's and the context descriptor's) takes it to. + CoherentAccess, +} + +/// A unit the one configuration can be built on, and its sizes. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Unit { + /// `IDR1.SIDSIZE`: how many bits of StreamID the unit takes, at most 32. + pub stream_bits: u8, + /// `IDR1.CMDQS` and `EVENTQS`: the largest each queue may be, as log2 of + /// its entries. + pub command_queue_log2: u8, + pub event_queue_log2: u8, + /// `IDR5.OAS` in bits: the bits of an address field above it are RES0 + /// (§6.3.24), so nothing the unit is given may reach past it. + output_bits: u8, + asid16: bool, + /// `IDR0.STALL_MODEL` is `0b00`: stalling is there to be disabled. + stall: bool, +} + +/// §6.3.1, §6.3.2 and §6.3.6: the three identification registers, judged, +/// with the IORT's `COHACC override` (DEN 0049 Table 14), which stands for +/// `IDR0.COHACC` [4] where it is set. +pub const fn probe(idr0: u32, idr1: u32, idr5: u32, coherent_override: bool) -> Result { + if idr0 & 1 << 4 == 0 && !coherent_override { + return Err(Lacks::CoherentAccess); + } + if idr0 & 1 << 1 == 0 { + return Err(Lacks::Stage1); + } + if idr0 & 1 << 3 == 0 { + return Err(Lacks::Aarch64Tables); + } + // `TTENDIAN` [22:21]: `0b00` mixed and `0b10` little-endian can be set to little-endian. + if idr0 >> 21 & 0b11 == 0b11 { + return Err(Lacks::LittleEndianTables); + } + // `STALL_MODEL` [25:24]: `0b00` stall and terminate, `0b01` terminate only. + let stall = match idr0 >> 24 & 0b11 { + 0b00 => true, + 0b01 => false, + other => return Err(Lacks::Termination(other as u8)), + }; + if idr5 & 1 << 4 == 0 { + return Err(Lacks::Granule4K); + } + if idr1 & (1 << 30 | 1 << 29) != 0 { + return Err(Lacks::FixedStructures); + } + let stream_bits = (idr1 & 0x3F) as u8; + let command_queue_log2 = (idr1 >> 21 & 0x1F) as u8; + let event_queue_log2 = (idr1 >> 16 & 0x1F) as u8; + if stream_bits > 32 { + return Err(Lacks::Size(stream_bits)); + } + if command_queue_log2 > 19 { + return Err(Lacks::Size(command_queue_log2)); + } + if event_queue_log2 > 19 { + return Err(Lacks::Size(event_queue_log2)); + } + Ok(Unit { + stream_bits, + command_queue_log2, + event_queue_log2, + output_bits: [32, 36, 40, 42, 44, 48, 52, 56][(idr5 & 0b111) as usize], + asid16: idr0 & 1 << 12 != 0, + stall, + }) +} + +impl Unit { + /// `tag` as an ASID of this unit: eight bits where `IDR0.ASID16` is + /// clear, under which a wider one makes its context descriptor ILLEGAL. + pub const fn asid(&self, tag: u16) -> Option { + if self.asid16 || tag <= 0xFF { + Some(Asid(tag)) + } else { + None + } + } + + /// Whether `STE.S1STALLD` is set: it must be where stalling is + /// configurable and is ILLEGAL where it is not (§5.5). + pub(crate) const fn stall_disable(&self) -> bool { + self.stall + } + + /// `CD.IPS`, the widest of `TCR_ELx.PS`'s sizes inside the unit's output + /// size and the 48 bits these tables' descriptors carry. + pub(crate) const fn ips(&self) -> u64 { + match self.output_bits { + 32 => 0b000, + 36 => 0b001, + 40 => 0b010, + 42 => 0b011, + 44 => 0b100, + _ => 0b101, + } + } + + /// Whether the unit reads all of `bytes` at `address`: none of them at + /// or past `1 << IDR5.OAS`. + pub(crate) const fn reaches(&self, address: u64, bytes: u64) -> bool { + address + bytes <= 1 << self.output_bits + } + + /// `bytes` at `address`, aligned to their own size, where the unit reads + /// them. + const fn holds(&self, address: u64, bytes: u64) -> bool { + address & (bytes - 1) == 0 && self.reaches(address, bytes) + } + + /// `SMMU_STRTAB_BASE` and `SMMU_STRTAB_BASE_CFG` (§6.3.24, §6.3.25) for + /// a linear table of `1 << log2size` entries at `table`: `RA` [62] and + /// the address, then `FMT` [17:16] zero and `LOG2SIZE` [5:0]. `None` + /// where the unit takes fewer StreamID bits, the table is not aligned to + /// its own size as the unit reads its address, or it ends past the + /// unit's output size. + pub const fn stream_table(&self, table: Phys<6>, log2size: u8) -> Option<(u64, u32)> { + if log2size > self.stream_bits || !self.holds(table.get(), 64 << log2size) { + return None; + } + Some((1 << 62 | table.get(), log2size as u32)) + } + + /// `SMMU_CMDQ_BASE` (§6.3.26) for a queue of `1 << log2size` 16-byte + /// commands at `queue`, and its indexes: `RA` [62], the address, + /// `LOG2SIZE` [4:0]. `None` past the unit's `CMDQS` or its output size, + /// or where the queue is not aligned to the larger of its size and 32 + /// bytes. + pub const fn command_queue(&self, queue: Phys<5>, log2size: u8) -> Option<(u64, Commands)> { + if log2size > self.command_queue_log2 || !self.holds(queue.get(), 16 << log2size) { + return None; + } + Some((1 << 62 | queue.get() | log2size as u64, Commands(Indexes { log2size }))) + } + + /// `SMMU_EVENTQ_BASE` (§6.3.29) for a queue of `1 << log2size` 32-byte + /// records, and its indexes: `WA` [62], the address, `LOG2SIZE` [4:0]. + pub const fn event_queue(&self, queue: Phys<5>, log2size: u8) -> Option<(u64, Events)> { + if log2size > self.event_queue_log2 || !self.holds(queue.get(), 32 << log2size) { + return None; + } + Some((1 << 62 | queue.get() | log2size as u64, Events(Indexes { log2size }))) + } +} diff --git a/toyos-smmu/tests/encodings.rs b/toyos-smmu/tests/encodings.rs new file mode 100644 index 00000000000..f25d80ae04d --- /dev/null +++ b/toyos-smmu/tests/encodings.rs @@ -0,0 +1,619 @@ +//! Every encoding against IHI 0070 H.a's own field positions: each expected +//! value is built from `(msb, lsb, value)` triples as the specification's +//! diagrams and field headings give them, and compared whole, so a field in +//! the wrong place and a stray bit are both a difference. + +use toyos_phys::Phys; +use toyos_smmu::config::{Cd, Ste}; +use toyos_smmu::queue::{event, Attempt, Code, Command, Event, Signal}; +use toyos_smmu::table::{entry, next, plan, Access, Entry, Leaf, INPUT_BITS, MAIR0}; +use toyos_smmu::unit::{self, probe, CommandError, Lacks, Unit}; + +/// `N` doublewords holding `value` in bits `[msb:lsb]` for every triple, and +/// zero everywhere else. A field may cross a doubleword. +fn layout(fields: &[(u32, u32, u64)]) -> [u64; N] { + let mut words = [0u64; N]; + for &(msb, lsb, value) in fields { + let width = msb - lsb + 1; + assert!(width == 64 || value >> width == 0, "{value:#x} does not fit [{msb}:{lsb}]"); + for bit in 0..width { + if value >> bit & 1 != 0 { + let at = lsb + bit; + assert_eq!(words[(at / 64) as usize] >> (at % 64) & 1, 0, "two fields claim bit {at}"); + words[(at / 64) as usize] |= 1 << (at % 64); + } + } + } + words +} + +fn word(fields: &[(u32, u32, u64)]) -> u64 { + layout::<1>(fields)[0] +} + +/// The identification registers of a unit that lacks nothing: `S1P` [1], +/// `TTF` [3:2] VMSAv8-64 alone, `COHACC` [4], `ASID16` [12], little-endian +/// tables, stall and terminate; 16 StreamID bits, queues of 2^19 and 2^8; +/// `GRAN4K` [4] and a 44-bit output. +const IDR0: &[(u32, u32, u64)] = &[(1, 1, 1), (3, 2, 0b10), (4, 4, 1), (12, 12, 1), (22, 21, 0b10)]; +const IDR1: &[(u32, u32, u64)] = &[(5, 0, 16), (20, 16, 8), (25, 21, 19)]; +const IDR5: &[(u32, u32, u64)] = &[(2, 0, 0b100), (4, 4, 1)]; + +/// The fields of `base` with those of `change` in place of the same bits. +fn with(base: &[(u32, u32, u64)], change: &[(u32, u32, u64)]) -> u32 { + let kept: Vec<_> = base.iter().filter(|b| !change.iter().any(|c| c.0 == b.0)).chain(change).copied().collect(); + word(&kept) as u32 +} + +fn idr(idr0: &[(u32, u32, u64)], idr1: &[(u32, u32, u64)], idr5: &[(u32, u32, u64)]) -> Result { + probe(with(IDR0, idr0), with(IDR1, idr1), with(IDR5, idr5), false) +} + +fn whole() -> Unit { + idr(&[], &[], &[]).expect("a unit that lacks nothing") +} + +/// `STALL_MODEL` [25:24] `0b01`: every fault terminates, and none can stall. +fn terminating() -> Unit { + idr(&[(25, 24, 0b01)], &[], &[]).expect("a unit that terminates every fault") +} + +// --- §6.3.1, §6.3.2, §6.3.6: the identification registers ------------------ + +#[test] +fn a_units_sizes_are_read_from_their_own_fields() { + let unit = whole(); + assert_eq!((unit.stream_bits, unit.command_queue_log2, unit.event_queue_log2), (16, 19, 8)); + // Each field moved alone moves its own answer and no other. + let other = idr(&[], &[(5, 0, 32), (20, 16, 19), (25, 21, 0)], &[]).expect("a unit"); + assert_eq!((other.stream_bits, other.command_queue_log2, other.event_queue_log2), (32, 0, 19)); +} + +/// `IDR5.OAS` [2:0]: 32, 36, 40, 42, 44, 48, 52 and 56 bits. The unit is +/// given a stream table that ends at its last byte and none a byte further. +#[test] +fn a_units_output_size_is_idr5s_and_bounds_what_it_is_given() { + for (oas, bits) in [32u32, 36, 40, 42, 44, 48, 52, 56].into_iter().enumerate() { + let unit = idr(&[], &[], &[(2, 0, oas as u64)]).expect("a unit"); + let end = 1u64 << bits.min(48); + assert!(unit.stream_table(Phys::new(end - 64).unwrap(), 0).is_some(), "OAS {oas:#b}"); + // Past 48 bits there is no address to refuse. + if let Some(past) = Phys::new(end) { + assert_eq!(unit.stream_table(past, 0), None, "OAS {oas:#b}"); + } + } +} + +/// DEN 0049 Table 14: the IORT's `COHACC override` stands for `IDR0.COHACC` +/// [4] where it is set, and a unit neither calls coherent is refused. +#[test] +fn a_unit_that_is_not_coherent_is_refused_unless_the_iort_says_it_is() { + assert_eq!(idr(&[(4, 4, 0)], &[], &[]), Err(Lacks::CoherentAccess)); + let (idr0, idr1, idr5) = (with(IDR0, &[(4, 4, 0)]), with(IDR1, &[]), with(IDR5, &[])); + assert_eq!(probe(idr0, idr1, idr5, true), Ok(whole())); + // The override adds nothing to a unit that says so itself. + assert_eq!(probe(with(IDR0, &[]), idr1, idr5, true), Ok(whole())); +} + +#[test] +fn a_unit_that_lacks_what_the_configuration_needs_is_refused_by_name() { + assert_eq!(idr(&[(1, 1, 0)], &[], &[]), Err(Lacks::Stage1)); + // `TTF` `0b01` is VMSAv8-32 LPAE alone; `0b11` has both. + assert_eq!(idr(&[(3, 2, 0b01)], &[], &[]), Err(Lacks::Aarch64Tables)); + assert!(idr(&[(3, 2, 0b11)], &[], &[]).is_ok()); + assert_eq!(idr(&[(22, 21, 0b11)], &[], &[]), Err(Lacks::LittleEndianTables)); + assert!(idr(&[(22, 21, 0b00)], &[], &[]).is_ok()); + assert_eq!(idr(&[(25, 24, 0b10)], &[], &[]), Err(Lacks::Termination(0b10))); + assert_eq!(idr(&[(25, 24, 0b11)], &[], &[]), Err(Lacks::Termination(0b11))); + assert_eq!(idr(&[], &[], &[(4, 4, 0)]), Err(Lacks::Granule4K)); + assert_eq!(idr(&[], &[(30, 30, 1)], &[]), Err(Lacks::FixedStructures)); + assert_eq!(idr(&[], &[(29, 29, 1)], &[]), Err(Lacks::FixedStructures)); +} + +#[test] +fn a_reserved_size_is_refused_and_the_largest_defined_one_is_not() { + assert_eq!(idr(&[], &[(5, 0, 33)], &[]), Err(Lacks::Size(33))); + assert_eq!(idr(&[], &[(25, 21, 20)], &[]), Err(Lacks::Size(20))); + assert_eq!(idr(&[], &[(20, 16, 20)], &[]), Err(Lacks::Size(20))); + assert!(idr(&[], &[(5, 0, 32), (25, 21, 19), (20, 16, 19)], &[]).is_ok()); + // Every value three registers can hold is an answer. + for bits in [0u32, u32::MAX, 0x5555_5555, 0xaaaa_aaaa] { + for coherent in [false, true] { + let _ = probe(bits, bits, bits, coherent); + let _ = probe(with(IDR0, &[]), bits, bits, coherent); + } + } +} + +#[test] +fn an_asid_is_no_wider_than_the_units() { + let narrow = idr(&[(12, 12, 0)], &[], &[]).expect("a unit of 8-bit ASIDs"); + assert_eq!(narrow.asid(0xff).map(|asid| asid.get()), Some(0xff)); + assert_eq!(narrow.asid(0x100), None); + assert_eq!(whole().asid(0xffff).map(|asid| asid.get()), Some(0xffff)); +} + +// --- §5.2, §5.4: the stream table entry and the context descriptor --------- + +#[test] +fn an_aborting_entry_is_valid_and_configured_to_pass_nothing() { + // V [0] set; Config [3:1] `0b000`, "report abort to device, no event recorded". + assert_eq!(Ste::ABORT.words(), layout(&[(0, 0, 1), (3, 1, 0b000)])); +} + +#[test] +fn a_translating_entry_is_stage_1_alone_through_one_context_descriptor() { + let context = Phys::<6>::new(0xfff_ffff_ffc0).expect("64-byte aligned, inside the unit's 44 bits"); + let fields = [ + (0, 0, 1), // V + (3, 1, 0b101), // Config: stage 1 translate, stage 2 bypass + (5, 4, 0), // S1Fmt, ignored with S1CDMax zero + (55, 6, 0xfff_ffff_ffc0 >> 6), // S1ContextPtr + (63, 59, 0), // S1CDMax: one CD, substreams disabled + (67, 66, 0b01), // S1CIR + (69, 68, 0b01), // S1COR + (71, 70, 0b11), // S1CSH + (93, 92, 0), // EATS + (95, 94, 0), // STRW: NS-EL1 + ]; + let mut stalling = fields.to_vec(); + stalling.push((91, 91, 1)); // S1STALLD, which a unit that can stall must be told + assert_eq!(Ste::stage1(context, &whole()).map(|ste| ste.words()), Some(layout(&stalling))); + // And is ILLEGAL to set on one that cannot. + assert_eq!(Ste::stage1(context, &terminating()).map(|ste| ste.words()), Some(layout(&fields))); +} + +#[test] +fn a_context_descriptor_walks_ttb0_alone_under_its_own_asid() { + let root = Phys::<12>::new(0xfff_ffff_f000).expect("a page inside the unit's 44 bits"); + let unit = whole(); + let asid = unit.asid(0xbeef).expect("a 16-bit ASID"); + assert_eq!( + Cd::new(root, asid, &unit).map(|cd| cd.words()), + Some(layout(&[ + (5, 0, 16), // T0SZ: 48 bits + (7, 6, 0b00), // TG0: 4KB + (9, 8, 0b01), // IR0 + (11, 10, 0b01), // OR0 + (13, 12, 0b11), // SH0 + (14, 14, 0), // EPD0 + (15, 15, 0), // ENDI + (30, 30, 1), // EPD1 + (31, 31, 1), // V + (34, 32, 0b100), // IPS: 44 bits, the unit's OAS + (35, 35, 0), // AFFD + (41, 41, 1), // AA64 + (44, 44, 0), // S + (45, 45, 1), // R + (46, 46, 1), // A + (47, 47, 1), // ASET + (63, 48, 0xbeef), // ASID + (119, 68, 0xfff_ffff_f000 >> 4), // TTB0: address bits [55:4] + (223, 192, 0x0044_ff04), // MAIR0: the loader's MAIR, attributes 0 to 3 + ])) + ); +} + +#[test] +fn a_context_descriptors_ips_is_the_units_output_size_and_no_more_than_48_bits() { + let root = Phys::<12>::new(0x1000).unwrap(); + // `TCR_ELx.PS`'s encodings for 32, 36, 40, 42, 44 and 48 bits; 52 and 56 are past these tables. + for (oas, ips) in [0b000u64, 0b001, 0b010, 0b011, 0b100, 0b101, 0b101, 0b101].into_iter().enumerate() { + let unit = idr(&[], &[], &[(2, 0, oas as u64)]).expect("a unit"); + let cd = Cd::new(root, unit.asid(1).unwrap(), &unit).expect("a root in the first 4 GiB"); + assert_eq!(cd.words()[0] >> 32 & 0b111, ips, "OAS {oas:#b}"); + } +} + +// --- §6.3: the registers the driver writes --------------------------------- + +/// §6.2.1, "Registers in Page 0", and §6.2.2, "Registers in Page 1" — +/// "Offsets are relative to the base of Page 1, 0x10000" — row by row. Each +/// register's own description states the same offset in its "Accessible at +/// offset ... from SMMUv3_PAGE_0" (§6.3.1, .2, .6, .9, .10, .11, .12, .15, +/// .17, .18, .19, .20, .24, .25, .26, .27, .28, .29) or "from SMMUv3_PAGE_1" +/// (§6.3.130, §6.3.131). +#[test] +fn the_registers_sit_where_the_register_overview_has_them() { + let page0: [(usize, usize); 18] = [ + (0x0000, unit::IDR0), + (0x0004, unit::IDR1), + (0x0014, unit::IDR5), + (0x0020, unit::CR0), + (0x0024, unit::CR0ACK), + (0x0028, unit::CR1), + (0x002C, unit::CR2), + (0x0044, unit::GBPA), + (0x0050, unit::IRQ_CTRL), + (0x0054, unit::IRQ_CTRLACK), + (0x0060, unit::GERROR), + (0x0064, unit::GERRORN), + (0x0080, unit::STRTAB_BASE), + (0x0088, unit::STRTAB_BASE_CFG), + (0x0090, unit::CMDQ_BASE), + (0x0098, unit::CMDQ_PROD), + (0x009C, unit::CMDQ_CONS), + (0x00A0, unit::EVENTQ_BASE), + ]; + for (specified, declared) in page0 { + assert_eq!(declared, specified); + } + let page1: [(usize, usize); 2] = [(0x00A8, unit::EVENTQ_PROD), (0x00AC, unit::EVENTQ_CONS)]; + for (specified, declared) in page1 { + assert_eq!(declared, 0x1_0000 + specified); + } +} + +#[test] +fn the_control_values_hold_each_field_where_its_register_has_it() { + assert_eq!(u64::from(unit::CR0_SMMUEN | unit::CR0_EVENTQEN | unit::CR0_CMDQEN), word(&[(0, 0, 1), (2, 2, 1), (3, 3, 1)])); + // TABLE_SH, TABLE_OC, TABLE_IC, QUEUE_SH, QUEUE_OC, QUEUE_IC. + assert_eq!( + u64::from(unit::CR1_WRITE_BACK), + word(&[(11, 10, 0b11), (9, 8, 0b01), (7, 6, 0b01), (5, 4, 0b11), (3, 2, 0b01), (1, 0, 0b01)]) + ); + // E2H clear, RECINVSID, PTM. + assert_eq!(u64::from(unit::CR2_RECORD_PRIVATE), word(&[(0, 0, 0), (1, 1, 1), (2, 2, 1)])); + assert_eq!(u64::from(unit::GBPA_ABORT | unit::GBPA_UPDATE), word(&[(20, 20, 1), (31, 31, 1)])); + assert_eq!(u64::from(unit::IRQ_GERROR | unit::IRQ_EVENTQ), word(&[(0, 0, 1), (2, 2, 1)])); + assert_eq!( + u64::from(unit::GERROR_CMDQ | unit::GERROR_EVENTQ_ABORT | unit::GERROR_SERVICE_FAILURE), + word(&[(0, 0, 1), (2, 2, 1), (8, 8, 1)]) + ); +} + +#[test] +fn an_error_is_active_while_the_two_registers_differ_in_its_bit() { + assert_eq!(unit::active_errors(0, 0), 0); + assert_eq!(unit::active_errors(0b101, 0b100), 0b001); + // Acknowledged by writing the same value, whichever way the bit last toggled. + assert_eq!(unit::active_errors(0b101, 0b101), 0); + assert_eq!(unit::active_errors(0b000, 0b001), 0b001); +} + +#[test] +fn a_command_error_is_the_cons_registers_err_field_alone() { + let cons = |err: u64| word(&[(30, 24, err), (19, 0, 0xf_ffff), (31, 31, 1)]) as u32; + assert_eq!(unit::command_error(cons(0)), CommandError::None); + assert_eq!(unit::command_error(cons(1)), CommandError::Illegal); + assert_eq!(unit::command_error(cons(2)), CommandError::Abort); + assert_eq!(unit::command_error(cons(3)), CommandError::AtcInvalidation); + assert_eq!(unit::command_error(cons(0x7f)), CommandError::Reserved(0x7f)); +} + +#[test] +fn a_stream_table_is_linear_inside_the_units_streams_and_aligned_to_its_size() { + let unit = whole(); + // 2^8 entries of 64 bytes: 16 KiB. + let table = Phys::<6>::new(0x4000_4000).unwrap(); + assert_eq!( + unit.stream_table(table, 8), + // RA [62], ADDR [55:6]; FMT [17:16] linear, SPLIT [10:6] unused, LOG2SIZE [5:0]. + Some((word(&[(62, 62, 1), (55, 6, 0x4000_4000 >> 6)]), word(&[(17, 16, 0b00), (5, 0, 8)]) as u32)) + ); + // A table larger than everything the unit reads: 2^27 entries are 8 GiB, on a unit of 32 output bits. + let small = idr(&[], &[(5, 0, 32)], &[(2, 0, 0b000)]).unwrap(); + let zero = Phys::<6>::new(0).unwrap(); + assert!(small.stream_table(zero, 26).is_some()); + assert_eq!(small.stream_table(zero, 27), None); + assert_eq!(unit.stream_table(Phys::new(0x4000_2000).unwrap(), 8), None, "8 KiB into a 16 KiB table's alignment"); + assert_eq!(unit.stream_table(Phys::new(0x4000_0040).unwrap(), 0), Some((1 << 62 | 0x4000_0040, 0))); + // The unit takes 16 StreamID bits and no more. + let big = Phys::<6>::new(0x8000_0000).unwrap(); + assert!(unit.stream_table(big, 16).is_some()); + assert_eq!(unit.stream_table(big, 17), None); +} + +#[test] +fn a_queue_is_inside_the_units_size_and_aligned_to_its_own() { + let unit = whole(); + let at = Phys::<5>::new(0x4001_0000).unwrap(); + // RA or WA [62], ADDR [55:5], LOG2SIZE [4:0]. + let base = |log2: u64| word(&[(62, 62, 1), (55, 5, 0x4001_0000 >> 5), (4, 0, log2)]); + assert_eq!(unit.command_queue(at, 12).map(|(register, _)| register), Some(base(12))); + assert_eq!(unit.event_queue(at, 8).map(|(register, _)| register), Some(base(8))); + // 2^13 commands of 16 bytes are 128 KiB, and 2^12 records of 32 as many: `at` is 64 KiB aligned. + assert_eq!(unit.command_queue(at, 13), None); + assert_eq!(unit.event_queue(at, 12), None, "past EVENTQS and misaligned both"); + let unit = idr(&[], &[(20, 16, 19)], &[]).unwrap(); + assert_eq!(unit.event_queue(at, 11).map(|(register, _)| register), Some(base(11))); + assert_eq!(unit.event_queue(at, 12), None, "128 KiB of records at a 64 KiB alignment"); + // The unit's largest and one more. + let low = Phys::<5>::new(0).unwrap(); + assert!(unit.command_queue(low, 19).is_some() && unit.event_queue(low, 19).is_some()); + let small = idr(&[], &[(25, 21, 4), (20, 16, 4)], &[]).unwrap(); + assert_eq!((small.command_queue(low, 5), small.event_queue(low, 5)), (None, None)); +} + +/// §6.3.24: the bits of `ADDR` above `IDR5.OAS` are RES0, and a unit need not +/// store them, so it would read its table or queue at the address without +/// them. +#[test] +fn a_table_or_queue_past_the_units_output_size_is_refused() { + let unit = idr(&[], &[(20, 16, 19)], &[(2, 0, 0b000)]).expect("a unit of 32 output bits"); + let past = 1u64 << 32; + assert_eq!(unit.stream_table(Phys::new(past).unwrap(), 0), None); + assert_eq!(unit.command_queue(Phys::new(past).unwrap(), 0), None); + assert_eq!(unit.event_queue(Phys::new(past).unwrap(), 0), None); + // Each ends at the unit's last byte and is given. + assert!(unit.stream_table(Phys::new(past - (64 << 8)).unwrap(), 8).is_some()); + assert!(unit.command_queue(Phys::new(past - (16 << 8)).unwrap(), 8).is_some()); + assert!(unit.event_queue(Phys::new(past - (32 << 8)).unwrap(), 8).is_some()); +} + +/// The same bound on what an entry, a descriptor and a table name: an +/// `S1ContextPtr`, a `TTB0`, a next table and a leaf's output address. +#[test] +fn a_descriptor_or_a_mapping_past_the_units_output_size_is_refused() { + let unit = idr(&[], &[], &[(2, 0, 0b000)]).expect("a unit of 32 output bits"); + let asid = unit.asid(1).unwrap(); + let past = 1u64 << 32; + assert_eq!(Ste::stage1(Phys::new(past).unwrap(), &unit), None); + assert!(Ste::stage1(Phys::new(past - 64).unwrap(), &unit).is_some()); + assert_eq!(Cd::new(Phys::new(past).unwrap(), asid, &unit), None); + assert!(Cd::new(Phys::new(past - 4096).unwrap(), asid, &unit).is_some()); + assert_eq!(next(Phys::new(past).unwrap(), &unit), None); + assert!(next(Phys::new(past - 4096).unwrap(), &unit).is_some()); + let memory = |at: u64| plan(AT, Leaf::Memory(Phys::new(at).unwrap(), Access::ReadWrite), &unit); + assert_eq!(memory(past), None); + assert!(memory(past - (1 << 21)).is_some()); + let doorbell = |at: u64| plan(AT, Leaf::Doorbell(Phys::new(at).unwrap()), &unit); + assert_eq!(doorbell(past), None); + assert!(doorbell(past - 4096).is_some()); + // A unit of 44 bits takes what one of 32 does not. + assert!(Ste::stage1(Phys::new(past).unwrap(), &whole()).is_some()); +} + +// --- §3.5.1: the queues' indexes -------------------------------------------- + +/// Against a count kept beside it: the command queue is full exactly at all +/// of its entries and consumed exactly at none, the event queue empty +/// exactly at none, and the slot the index that many steps on. +#[test] +fn a_queues_indexes_say_empty_full_and_slot_as_a_count_would() { + let unit = idr(&[], &[(20, 16, 19)], &[]).unwrap(); + let low = Phys::<5>::new(0).unwrap(); + for log2size in 0..=4u8 { + let (_, commands) = unit.command_queue(low, log2size).unwrap(); + let (_, events) = unit.event_queue(low, log2size).unwrap(); + let size = 1usize << log2size; + // Start anywhere, the wrap flags and the bits above them included. + for start in [0u32, 1 << log2size, (2 << log2size) - 1, 0x7ff0_0000 | 1 << log2size] { + let (mut prod, mut cons) = (start, start); + let (mut produced, mut consumed) = (0usize, 0usize); + let first = commands.slot(start); + // Fill, drain halfway, fill again, drain: three laps of the buffer. + for step in 0..6 * size { + let held = produced - consumed; + assert_eq!(commands.is_empty(prod, cons), held == 0, "size {size} step {step}"); + assert_eq!(events.is_empty(prod, cons), held == 0, "size {size} step {step}"); + assert_eq!(commands.is_full(prod, cons), held == size, "size {size} step {step}"); + assert_eq!(commands.slot(prod), (first + produced) % size); + assert_eq!(events.slot(cons), (first + consumed) % size); + if held < size && (step / size).is_multiple_of(2) { + prod = commands.after(prod); + produced += 1; + } else if held > 0 { + cons = events.after(cons, prod); + consumed += 1; + } + } + } + } + // The wrap flag toggles as the index passes the top, and nothing above it is written back. + let (_, commands) = unit.command_queue(low, 3).unwrap(); + assert_eq!(commands.after(0b0_111), 0b1_000); + assert_eq!(commands.after(0b1_111), 0b0_000); + assert_eq!(commands.after(0xffff_fff7), 0b1_000); + // An index register is the unit's word: every bit set, it is still an index. + assert_eq!(commands.after(u32::MAX), 0); + assert_eq!(unit.command_queue(low, 19).unwrap().1.after(u32::MAX), 0); + assert_eq!(unit.event_queue(low, 19).unwrap().1.after(0x7fff_ffff, 0), 0); +} + +/// §7.4: records were lost while `SMMU_EVENTQ_PROD.OVFLG` [31] differs from +/// `SMMU_EVENTQ_CONS.OVACKFLG` [31], and the consumer acknowledges by +/// writing its flag to the value it read `OVFLG` with. +#[test] +fn consuming_an_event_acknowledges_an_overflow_and_never_raises_one() { + let unit = idr(&[], &[(20, 16, 19)], &[]).unwrap(); + let (_, events) = unit.event_queue(Phys::new(0).unwrap(), 3).unwrap(); + const OVFLG: u32 = 1 << 31; + for (prod, cons) in [(0b0_101, 0b0_011), (OVFLG | 0b0_101, OVFLG | 0b0_011)] { + assert!(!events.overflowed(prod, cons)); + // The flag is written back as it was read: no overflow is forged. + let written = events.after(cons, prod); + assert_eq!(written, prod & OVFLG | 0b0_100); + assert!(!events.overflowed(prod, written)); + } + for (prod, cons) in [(OVFLG | 0b0_101, 0b0_011), (0b0_101, OVFLG | 0b0_011)] { + assert!(events.overflowed(prod, cons)); + let written = events.after(cons, prod); + assert_eq!(written, prod & OVFLG | 0b0_100); + assert!(!events.overflowed(prod, written)); + } + // The flags are no part of a position: a queue whose flags differ is as empty as its indexes say. + assert!(events.is_empty(OVFLG | 0b1_010, 0b1_010)); + assert_eq!(events.slot(OVFLG | 0b1_010), 0b010); +} + +// --- chapter 4: commands ---------------------------------------------------- + +#[test] +fn each_command_holds_its_opcode_and_parameters_where_its_diagram_has_them() { + let asid = whole().asid(0xa5c3).unwrap(); + // CMD_CFGI_STE: StreamID [63:32], Leaf [64], SSec [10] clear. + assert_eq!(Command::ForgetStream(0xdead_beef).words(), layout(&[(7, 0, 0x03), (63, 32, 0xdead_beef), (64, 64, 1)])); + // CMD_CFGI_ALL: CMD_CFGI_STE_RANGE with Range [68:64] of 31. + assert_eq!(Command::ForgetAll.words(), layout(&[(7, 0, 0x04), (68, 64, 31)])); + // CMD_TLBI_NH_ASID: ASID [63:48], VMID [47:32]. + assert_eq!(Command::InvalidateAsid(asid).words(), layout(&[(7, 0, 0x11), (47, 32, 0), (63, 48, 0xa5c3)])); + assert_eq!(Command::InvalidateAll.words(), layout(&[(7, 0, 0x30)])); + // CMD_SYNC: CS [13:12]; MSIAddress [119:66] zero, so SIG_IRQ is the wired interrupt. + assert_eq!(Command::Sync(Signal::None).words(), layout(&[(7, 0, 0x46), (13, 12, 0b00)])); + assert_eq!(Command::Sync(Signal::Irq).words(), layout(&[(7, 0, 0x46), (13, 12, 0b01), (119, 66, 0)])); +} + +// --- §7.3: event records ---------------------------------------------------- + +/// A record of `number` from `stream`, every field the four fault records +/// carry filled: InputAddr [191:128], RnW [99], InD [98] clear as a write's +/// is, PnU [97], S2 [103], CLASS [105:104], STAG [79:64], an IPA [247:204]. +fn record(number: u8, stream: u32, address: u64, read: bool) -> [u64; 4] { + layout(&[ + (7, 0, u64::from(number)), + (11, 11, 1), + (31, 12, 0xf_ffff), + (63, 32, u64::from(stream)), + (79, 64, 0xffff), + (97, 97, 1), + (98, 98, 0), + (99, 99, u64::from(read)), + (103, 103, 1), + (105, 104, 0b10), + (191, 128, address), + (247, 204, 0xfff_ffff_ffff), + ]) +} + +#[test] +fn a_translation_fault_names_its_stream_its_address_and_whether_it_wrote() { + for (number, code) in [ + (0x10, Code::Translation), + (0x11, Code::AddressSize), + (0x12, Code::AccessFlag), + (0x13, Code::Permission), + ] { + for read in [false, true] { + assert_eq!( + event(record(number, 0x0001_0008, 0xffff_8000_dead_b000, read)), + Event { + stream: 0x0001_0008, + code, + attempt: Some(Attempt { address: 0xffff_8000_dead_b000, write: !read }) + } + ); + } + } +} + +#[test] +fn every_event_number_is_its_own_code_and_only_a_fault_carries_an_attempt() { + let named = [ + (0x01, Code::UnsupportedTransaction), + (0x02, Code::BadStream), + (0x03, Code::EntryFetch), + (0x04, Code::BadEntry), + (0x05, Code::AtsRequest), + (0x06, Code::StreamDisabled), + (0x07, Code::TranslatedForbidden), + (0x08, Code::BadSubstream), + (0x09, Code::ContextFetch), + (0x0A, Code::BadContext), + (0x0B, Code::WalkAbort), + (0x10, Code::Translation), + (0x11, Code::AddressSize), + (0x12, Code::AccessFlag), + (0x13, Code::Permission), + (0x20, Code::TlbConflict), + (0x21, Code::ConfigurationConflict), + (0x24, Code::PageRequest), + (0x25, Code::VmsFetch), + ]; + for number in 0..=u8::MAX { + let decoded = event(record(number, 7, 0x1000, false)); + let code = named.iter().find(|(n, _)| *n == number).map_or(Code::Other(number), |(_, code)| *code); + assert_eq!((decoded.stream, decoded.code), (7, code), "event {number:#x}"); + assert_eq!(decoded.attempt.is_some(), (0x10..=0x13).contains(&number), "event {number:#x}"); + } +} + +// --- the stage 1 tables (DDI 0487 M.d §D8.3.1) ----------------------------- + +/// A device address whose four table indexes are 0x1a5, 0x0f0, 0x00f and 0x155. +const AT: u64 = 0x1a5 << 39 | 0x0f0 << 30 | 0x00f << 21; + +#[test] +fn memory_is_a_level_2_block_only_its_own_context_and_an_unprivileged_device_reach() { + let block = Phys::<21>::new(0xfff_ffe0_0000).unwrap(); + let fields = |read_only: u64| { + word(&[ + (0, 0, 1), // valid + (1, 1, 0), // a block + (4, 2, 1), // AttrIndx: MAIR0's attribute 1 + (6, 6, 1), // AP[1]: unprivileged access + (7, 7, read_only), // AP[2] + (9, 8, 0b11), // SH: inner shareable + (10, 10, 1), // AF + (11, 11, 1), // nG + (47, 21, 0xfff_ffe0_0000 >> 21), // the output address + (53, 53, 1), // PXN + (54, 54, 1), // UXN + ]) + }; + let unit = whole(); + let path = plan(AT, Leaf::Memory(block, Access::ReadWrite), &unit).expect("a 2 MiB aligned address"); + assert_eq!((path.indices(), path.descriptor), (&[0x1a5, 0x0f0, 0x00f][..], fields(0))); + let path = plan(AT, Leaf::Memory(block, Access::Read), &unit).expect("a 2 MiB aligned address"); + assert_eq!((path.indices(), path.descriptor), (&[0x1a5, 0x0f0, 0x00f][..], fields(1))); +} + +#[test] +fn the_doorbell_is_one_level_3_page_of_device_memory() { + let page = Phys::<12>::new(0x0809_0000).unwrap(); + let path = plan(AT | 0x155 << 12, Leaf::Doorbell(page), &whole()).expect("a 4 KiB aligned address"); + assert_eq!(path.indices(), [0x1a5, 0x0f0, 0x00f, 0x155]); + assert_eq!( + path.descriptor, + word(&[ + (0, 0, 1), // valid + (1, 1, 1), // a page + (4, 2, 0), // AttrIndx: MAIR0's attribute 0, Device-nGnRE + (6, 6, 1), // AP[1] + (7, 7, 0), // AP[2]: written + (10, 10, 1), // AF + (11, 11, 1), // nG + (47, 12, 0x0809_0000 >> 12), // the page alone + (53, 53, 1), + (54, 54, 1), + ]) + ); + // The attributes the two leaves' AttrIndx name: 0 is Device-nGnRE and 1 Normal write-back. + assert_eq!(MAIR0.to_le_bytes()[..2], [0x04, 0xff]); +} + +#[test] +fn an_address_past_the_input_or_off_the_leafs_alignment_has_no_plan() { + let unit = whole(); + let block = Leaf::Memory(Phys::new(0x20_0000).unwrap(), Access::ReadWrite); + let page = Leaf::Doorbell(Phys::new(0x1000).unwrap()); + assert_eq!(plan(1 << INPUT_BITS, block, &unit), None); + assert_eq!(plan(1 << INPUT_BITS, page, &unit), None); + assert_eq!(plan(!0x1f_ffff, block, &unit), None); + assert!(plan((1 << INPUT_BITS) - (1 << 21), block, &unit).is_some()); + // 4 KiB aligned is enough for the page and not for the block. + assert_eq!(plan(AT | 0x1000, block, &unit), None); + assert!(plan(AT | 0x1000, page, &unit).is_some()); + assert_eq!(plan(AT | 0x800, page, &unit), None); +} + +#[test] +fn a_descriptor_read_back_is_a_table_only_above_level_3() { + let unit = whole(); + let table = Phys::<12>::new(0xfff_ffff_f000).unwrap(); + // Valid [0], table [1], the next table's address [47:12], and no hierarchical restriction. + let descriptor = next(table, &unit).expect("a table inside the unit's 44 bits"); + assert_eq!(descriptor, word(&[(0, 0, 1), (1, 1, 1), (47, 12, 0xfff_ffff_f000 >> 12)])); + for level in 0..3 { + assert_eq!(entry(descriptor, level), Entry::Table(table)); + assert_eq!(entry(0, level), Entry::Invalid); + // Bit [0] alone decides validity: a stale address under it is nothing. + assert_eq!(entry(descriptor & !1, level), Entry::Invalid); + } + let block = plan(AT, Leaf::Memory(Phys::new(0x4020_0000).unwrap(), Access::ReadWrite), &unit).unwrap(); + assert_eq!(entry(block.descriptor, 2), Entry::Mapped); + // A level 3 page sets bit [1] as a table descriptor does: it is no table. + let page = plan(AT, Leaf::Doorbell(Phys::new(0x0809_0000).unwrap()), &unit).unwrap(); + assert_eq!(entry(page.descriptor, 3), Entry::Mapped); + assert_eq!(entry(0, 3), Entry::Invalid); +}