AArch64 stage 4: the boot-timing handoff names the CPU's counter, and the loader hands the count the kernel's clock reads

bootloader/src/arch/aarch64.rs

@@ -15,12 +15,29 @@ pub fn typing(write_back: &[(u64, u64)]) -> Typing<'_> {
     Typing::ByMap(write_back)
 }
 
-/// The generic timer's virtual count, `CNTVCT_EL0`.
+/// The exception level the loader runs at, from `CurrentEL`.
+fn current_el() -> u64 {
+    let current: u64;
+    // SAFETY: reads `CurrentEL`, which EL1 and above may read.
+    unsafe { core::arch::asm!("mrs {}, currentel", out(reg) current, options(nomem, nostack, preserves_flags)) };
+    (current >> 2) & 0b11
+}
+
+/// The count the kernel's clock reads, its virtual count. At EL2 that is the
+/// physical count, `CNTPCT_EL0`, because the kernel's entry writes
+/// `CNTVOFF_EL2` zero, which resets UNKNOWN; at EL1 the offset is the
+/// hypervisor's for good and `CNTVCT_EL0` is read.
 pub fn counter() -> u64 {
     let count: u64;
-    // SAFETY: reads a counter EL1 and EL2 may always read; the `ISB` keeps the
-    // read in program order.
-    unsafe { core::arch::asm!("isb", "mrs {}, cntvct_el0", out(reg) count, options(nomem, nostack, preserves_flags)) };
+    if current_el() == 2 {
+        // SAFETY: reads a counter EL2 may always read; the `ISB` keeps the
+        // read in program order.
+        unsafe { core::arch::asm!("isb", "mrs {}, cntpct_el0", out(reg) count, options(nomem, nostack, preserves_flags)) };
+    } else {
+        // SAFETY: reads a counter EL1 may always read; the `ISB` keeps the
+        // read in program order.
+        unsafe { core::arch::asm!("isb", "mrs {}, cntvct_el0", out(reg) count, options(nomem, nostack, preserves_flags)) };
+    }
     count
 }
 
@@ -31,10 +48,7 @@ pub fn counter() -> u64 {
 /// the console still prints; the entry's read-back of `HCR_EL2` stays as the
 /// last line of defence.
 pub fn cpu_as_entered() -> Result<Option<alloc::string::String>, alloc::string::String> {
-    let current: u64;
-    // SAFETY: reads `CurrentEL`, which EL1 and above may read.
-    unsafe { core::arch::asm!("mrs {}, currentel", out(reg) current, options(nomem, nostack, preserves_flags)) };
-    let el = (current >> 2) & 0b11;
+    let el = current_el();
     if el != 2 {
         return Ok(Some(alloc::format!("CPU: entered at EL{el}")));
     }

bootloader/src/main.rs

@@ -499,13 +499,8 @@ fn report_reach(what: &str, at: u64, len: u64) {
     );
 }
 
-/// The CPU's free-running counter, which counts from reset.
-fn tsc() -> u64 {
-    arch::counter()
-}
-
 #[allow(clippy::too_many_arguments)]
-fn start_kernel(kernel: LoadedKernel, kernel_elf_bytes: vec::Vec<u8>, cmdline: vec::Vec<u8>, rsdp_addr: u64, gop: Option<GopInfo>, boot_part: Option<BootPartition>, log_partition_guid: [u8; 16], root_image: rootimage::RootImage, entry_tsc: u64, system_table: SystemTable<Boot>) -> ! {
+fn start_kernel(kernel: LoadedKernel, kernel_elf_bytes: vec::Vec<u8>, cmdline: vec::Vec<u8>, rsdp_addr: u64, gop: Option<GopInfo>, boot_part: Option<BootPartition>, log_partition_guid: [u8; 16], root_image: rootimage::RootImage, entry_counter: u64, system_table: SystemTable<Boot>) -> ! {
     // Said before it is refused, for `report_reach`'s reason.
     match arch::cpu_as_entered() {
         Ok(None) => {}
@@ -597,7 +592,7 @@ fn start_kernel(kernel: LoadedKernel, kernel_elf_bytes: vec::Vec<u8>, cmdline: v
             None => ([0u8; 16], 0, 0, 0),
         };
 
-    let (root_image_addr, root_image_len, root_partition_guid, root_read_tsc) = root_image.handoff();
+    let (root_image_addr, root_image_len, root_partition_guid, root_read_ticks) = root_image.handoff();
 
     // Built before the exit so the address the kernel is handed is one this
     // loader can still print and refuse on.
@@ -633,20 +628,20 @@ fn start_kernel(kernel: LoadedKernel, kernel_elf_bytes: vec::Vec<u8>, cmdline: v
         root_image_addr,
         root_image_len,
         root_partition_guid,
-        loader_entry_tsc: entry_tsc,
-        loader_handoff_tsc: 0,
-        root_read_tsc,
+        loader_entry_counter: entry_counter,
+        loader_handoff_counter: 0,
+        root_read_ticks,
     };
     report_reach(
         "Kernel arguments",
         &kernel_args as *const KernelArgs as u64,
         mem::size_of::<KernelArgs>() as u64,
     );
 
-    kernel_args.loader_handoff_tsc = tsc();
+    kernel_args.loader_handoff_counter = arch::counter();
     println!(
-        "Loader TSC: {entry_tsc} at entry, {} at the handoff",
-        kernel_args.loader_handoff_tsc,
+        "Loader counter: {entry_counter} at entry, {} at the handoff",
+        kernel_args.loader_handoff_counter,
     );
 
     // Last, and after every line above: a console write, a FAT write and a
@@ -750,8 +745,8 @@ fn end_this_pass(system_table: &SystemTable<Boot>, exit_event: Option<Event>) ->
 
 #[entry]
 fn main(handle: Handle, mut system_table: SystemTable<Boot>) -> Status {
-    // First: the TSC counts from reset, so this is what firmware took.
-    let entry_tsc = tsc();
+    // First, so this reading is firmware's time and none of the loader's.
+    let entry_counter = arch::counter();
     let exit_event = uefi_services::init(&mut system_table).unwrap();
     // First, because it covers everything below it: firmware starts a
     // five-minute countdown when it loads an image and resets the machine if
@@ -961,5 +956,5 @@ fn main(handle: Handle, mut system_table: SystemTable<Boot>) -> Status {
     watchdog::arm(&system_table, rsdp_addr, params);
 
     println!("Starting kernel...");
-    start_kernel(loaded_kernel, kernel_bytes, cmdline, rsdp_addr, gop, boot_part, log_guid, chosen.root, entry_tsc, system_table);
+    start_kernel(loaded_kernel, kernel_bytes, cmdline, rsdp_addr, gop, boot_part, log_guid, chosen.root, entry_counter, system_table);
 }

bootloader/src/rootimage.rs

@@ -58,15 +58,15 @@ pub struct RootImage {
     len: u64,
     /// The partition it was read from, raw as in its GPT entry.
     partition: [u8; 16],
-    /// The TSC cycles the read took.
-    cycles: u64,
+    /// The counter ticks the read took.
+    ticks: u64,
 }
 
 impl RootImage {
     /// Where the kernel is told the image is, which partition it came from, and
-    /// the cycles reading it took.
+    /// the counter ticks reading it took.
     pub fn handoff(&self) -> (u64, u64, [u8; 16], u64) {
-        (self.at, self.len, self.partition, self.cycles)
+        (self.at, self.len, self.partition, self.ticks)
     }
 
     /// The image's bytes, for the hash its slot's header names.
@@ -222,10 +222,10 @@ impl<'a> Disk<'a> {
         // Chunks are whole `BLOCK`s from a page-aligned buffer, so each one
         // keeps the `IoAlign` `open` checked against `BLOCK`.
         let chunk = chunk::chunk_bytes(CHUNK_BOUND, BLOCK, self.lba_bytes, granularity.unwrap_or(0));
-        let began = crate::tsc();
+        let began = crate::arch::counter();
         let mut device = Firmware { io: &self.io, media_id: self.media_id };
         let read = chunk::read(&mut device, part.first_lba(), self.lba_bytes, chunk, into);
-        let image = RootImage { at, len, partition: part.unique_guid().0, cycles: crate::tsc().wrapping_sub(began) };
+        let image = RootImage { at, len, partition: part.unique_guid().0, ticks: crate::arch::counter().wrapping_sub(began) };
         if let Err(failed) = read {
             let why = alloc::format!(
                 "the read of {} blocks at LBA {} failed: {:?}, after {} of {len} bytes read",
@@ -238,14 +238,14 @@ impl<'a> Disk<'a> {
             return Err(why);
         }
         println!(
-            "{READ_AT} {at:#x}+{len:#x} from LBA {}+{}, {chunk} bytes a request (optimal granularity: {}), in {} TSC cycles",
+            "{READ_AT} {at:#x}+{len:#x} from LBA {}+{}, {chunk} bytes a request (optimal granularity: {}), in {} counter ticks",
             part.first_lba(),
             len / u64::from(self.lba_bytes),
             match granularity {
                 Some(lbas) => alloc::format!("{lbas} block(s)"),
                 None => String::from("not reported"),
             },
-            image.cycles
+            image.ticks
         );
         Ok(image)
     }

bootloader/src/slot.rs

@@ -145,9 +145,9 @@ fn verify(
         println!("{HEAD} {letter}: ROOT: {why}");
         Refusal::Unreadable("root")
     })?;
-    let began = crate::tsc();
+    let began = crate::arch::counter();
     let root_hash = toyos_update::sha256(root.bytes());
-    println!("{HEAD} {letter}: ROOT hashed in {} TSC cycles", crate::tsc().wrapping_sub(began));
+    println!("{HEAD} {letter}: ROOT hashed in {} counter ticks", crate::arch::counter().wrapping_sub(began));
     if root_hash != header.root().sha256 {
         root.free(bs);
         return Err(Refusal::Hash("root"));

issues/boot-media/the-loader-does-only-what-must-precede-the-handover.md

@@ -188,7 +188,7 @@ Each stage lands on its own, in this order.
    `kernel_args_last_layout_refused` boots with
    `loader-writes-the-last-layout` and finds the kernel's refusal naming both
    words before any `black box:` record. Moving `layout` after
-   `root_read_tsc` fails to build, and so does padding `KernelArgs` back to
+   `root_read_ticks` fails to build, and so does padding `KernelArgs` back to
    1272 bytes: `size_of::<KernelArgs>()` is then stage 1's size again, the
    derived `LAYOUT` collapses onto the literal `LAST_LAYOUT`
    (`0x5459_0000 | 1272`), and `assert!(LAYOUT != LAST_LAYOUT)` fails the

issues/kernel/portable-kernel-code-names-the-tsc.md

@@ -12,8 +12,6 @@ portable kernel still calls it the TSC: `kernel/src/clock.rs`'s
 `AT_TSC`, and the xHCI driver's, `hardlockup`'s and `panic_reboot`'s waits
 and comments read it by that name. `clock::counter_ticks`, which the AArch64
 timer reads, is renamed; the rest reads as x86-64's on both machines.
-`issues/kernel/the-boot-timing-handoff-is-named-for-the-tsc.md` is the ABI's
-half of the same name.
 
 **Exit condition**: no item or comment outside `kernel/src/arch/x86_64/`
 names the TSC for the counter `crate::arch::cpu::counter` reads.

issues/kernel/toyos-runs-on-arm64.md

@@ -201,7 +201,6 @@ before any aarch64 file exists, with x86 as its only user:
 Each is its own issue, owned by the stage that removes it:
 
 - `issues/kernel/msi-and-pin-routing-take-an-x86-vector-and-apic-id.md` (stage 6)
-- `issues/kernel/the-boot-timing-handoff-is-named-for-the-tsc.md` (stage 4)
 - `issues/kernel/the-crash-evidence-records-x86-fault-registers.md` (stage 5)
 - `issues/kernel/the-aarch64-kernel-builds-with-dead-code-allowed.md` (stage 7)
 
@@ -262,8 +261,7 @@ Each stage names its exit; "measured" means a number from a run.
    exposes no RNDR and the kernel's hash seed refuses there until stage 6's
    virtio-rng. Each judges an event, never a rate: no QEMU test measures time.
    Owed before the exit holds: the interrupts-off window against x86's, a
-   measurement only metal can make, with no instrument on either arch yet;
-   `issues/kernel/the-boot-timing-handoff-is-named-for-the-tsc.md`; the
+   measurement only metal can make, with no instrument on either arch yet; the
    instruction-cache maintenance before a mapping is executable
    (`cache::make_executable`), the break-before-make ordering of a live
    entry's replacement, and the TLB flush before a reclaimed ASID is issued

kernel/src/main.rs

@@ -167,15 +167,15 @@ fn register_gpu(driver: Box<dyn gpu::Gpu>, info: gpu::GpuInfo) {
     gpu::register(driver, info);
 }
 
-/// The boot from power-on, off the loader's TSC readings and `complete`'s, at
-/// the calibrated rate. The TSC counts from reset, so the first span is
-/// firmware's unless firmware wrote the counter.
+/// The boot from power-on, off the loader's [`cpu::counter`] readings and
+/// `complete`'s, at the clock's rate. The first span is firmware's time since
+/// the counter started.
 fn report_power_on(args: &KernelArgs, complete: u64) {
     arch::boot::report_counter_origin();
-    let (entry, handoff) = (args.loader_entry_tsc, args.loader_handoff_tsc);
+    let (entry, handoff) = (args.loader_entry_counter, args.loader_handoff_counter);
     if handoff < entry || complete < handoff {
         log!(
-            "boot: the TSC went backwards: {entry} at the loader's entry, {handoff} at its handoff, \
+            "boot: the counter went backwards: {entry} at the loader's entry, {handoff} at its handoff, \
              {complete} at Boot: complete"
         );
         return;
@@ -185,7 +185,7 @@ fn report_power_on(args: &KernelArgs, complete: u64) {
         "boot: power-on to loader {} ms, loader {} ms (ROOT read {} ms), kernel to Boot: complete {} ms",
         ms(entry),
         ms(handoff - entry),
-        ms(args.root_read_tsc),
+        ms(args.root_read_ticks),
         ms(complete - handoff),
     );
 }
@@ -536,9 +536,9 @@ pub(crate) unsafe extern "C" fn kernel_main(kernel_args: &KernelArgs) -> ! {
     }
 
     report_log_destination();
-    let complete_tsc = cpu::counter();
+    let complete = cpu::counter();
     boot_phase!("complete", 0);
-    report_power_on(kernel_args, complete_tsc);
+    report_power_on(kernel_args, complete);
 
     #[cfg(feature = "boot-actuators")]
     if actuator::test_late_panic() {

src/kernelconsole.rs

@@ -57,7 +57,7 @@ mod tests {
          \"UEFI QEMU QEMU USB HARDDRIVE TOYOS0BOOTSTICK1\" from PciRoot(0x0)/Pci(0x1,0x0)/USB(0x0,0x0)\n\
          ToyOS Bootloader 1.0\n\
          ROOT: read into memory at 0x7c894000+0x800000 from LBA 212992+16384, 1048576 bytes a request \
-         (optimal granularity: not reported), in 22519000 TSC cycles\n\
+         (optimal granularity: not reported), in 22519000 counter ticks\n\
          Loader log: the kernel handoff begins, so ";
 
     const KERNEL: &str = "[kernel 0.000 cpu0 boot] black box: 0x8000000 is this boot's, 16344 bytes \

toyos-abi/src/boot.rs

@@ -101,13 +101,13 @@ pub struct KernelArgs {
     /// GPT entry like [`Self::boot_partition_guid`]; zero with no image. The
     /// kernel holds that partition so no claim writes the slot it is running.
     pub root_partition_guid: [u8; 16],
-    /// The time-stamp counter at the loader's entry and at its handoff, and the
-    /// cycles its read of ROOT took (zero with no image). The TSC counts from
-    /// reset, so the first is firmware's time since power-on unless firmware
-    /// wrote the counter; the kernel converts all three at its calibrated rate.
-    pub loader_entry_tsc: u64,
-    pub loader_handoff_tsc: u64,
-    pub root_read_tsc: u64,
+    /// The CPU's counter, the one the kernel's clock reads, at the loader's
+    /// entry and at its handoff, and the ticks its read of ROOT took (zero with
+    /// no image). The first is firmware's time since the counter started; the
+    /// kernel converts all three at its clock's rate.
+    pub loader_entry_counter: u64,
+    pub loader_handoff_counter: u64,
+    pub root_read_ticks: u64,
 }
 
 /// [`KernelArgs::layout`] for the struct this file declares: the struct's own
@@ -222,9 +222,9 @@ const _: () = {
     assert!(offset_of!(KernelArgs, root_image_addr) == 1216);
     assert!(offset_of!(KernelArgs, root_image_len) == 1224);
     assert!(offset_of!(KernelArgs, root_partition_guid) == 1232);
-    assert!(offset_of!(KernelArgs, loader_entry_tsc) == 1248);
-    assert!(offset_of!(KernelArgs, loader_handoff_tsc) == 1256);
-    assert!(offset_of!(KernelArgs, root_read_tsc) == 1264);
+    assert!(offset_of!(KernelArgs, loader_entry_counter) == 1248);
+    assert!(offset_of!(KernelArgs, loader_handoff_counter) == 1256);
+    assert!(offset_of!(KernelArgs, root_read_ticks) == 1264);
     assert!(size_of::<KernelArgs>() == 1272);
     assert!(LAYOUT as i32 > 1440 || (LAYOUT as i32) < -1440);
     assert!(align_of::<KernelArgs>() == 8);
@@ -317,9 +317,9 @@ mod tests {
         root_image_addr: 0,
         root_image_len: 0,
         root_partition_guid: [0; 16],
-        loader_entry_tsc: 0,
-        loader_handoff_tsc: 0,
-        root_read_tsc: 0,
+        loader_entry_counter: 0,
+        loader_handoff_counter: 0,
+        root_read_ticks: 0,
     };
 
     #[test]