diff --git a/Cargo.lock b/Cargo.lock index b4e0cfe43fe..459db388e8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -561,6 +561,7 @@ dependencies = [ "toyos-blackbox", "toyos-blockhold", "toyos-bootmap", + "toyos-cpuvuln", "toyos-dma", "toyos-elf", "toyos-elide", diff --git a/issues/design-debt/toyos-cpuvuln-has-one-consumer-and-lives-outside-it.md b/issues/design-debt/toyos-cpuvuln-has-one-consumer-and-lives-outside-it.md new file mode 100644 index 00000000000..5d9c3e09fd1 --- /dev/null +++ b/issues/design-debt/toyos-cpuvuln-has-one-consumer-and-lives-outside-it.md @@ -0,0 +1,19 @@ +--- +status: open +kind: defect +opened: 2026-10-04 +--- + +# toyos-cpuvuln has one consumer and lives outside it + +Since pull request #705 the kernel depends on `toyos-cpuvuln` for which +counters a CPU has (`toyos_cpuvuln::counters`), and nothing else depends on +it: a crate with one consumer, which `.claude/agents/reviewer.md`'s Fit puts +under that consumer as #703 put every other, still at the repository root as +a host workspace member. #705 left it +there because the move was outside its brief. + +Owner: the orchestrator, who briefs the move. **Exit**: the crate lives under +`kernel/`, every path naming `toyos-cpuvuln/` moved with it (the root +`Cargo.toml`'s members, `kernel/Cargo.toml`, and the issues citing its +fixtures), and `src/hostws.rs` green; or a second consumer depends on it. diff --git a/issues/diagnostics/toyos-explains-itself.md b/issues/diagnostics/toyos-explains-itself.md index a1f9badb6f1..8033a58663e 100644 --- a/issues/diagnostics/toyos-explains-itself.md +++ b/issues/diagnostics/toyos-explains-itself.md @@ -9,8 +9,8 @@ opened: 2026-10-04 ToyOS answers what it is doing, what it did and what it is made of from inside itself, with programs it ships. Today the answers are scattered: the log carries numbers in prose (`irq:`, `tlb:`, `PMM:`, `sched:`, `syscalls:`), -the trace ring has LLDB as its only reader, nothing reads APERF, MPERF, -`MSR_SMI_COUNT`, RAPL or C-state residency, a process's memory is a byte sum +the trace ring has LLDB as its only reader, nothing reads RAPL or C-state +residency, a process's memory is a byte sum that reads 0 under contention, and a user symbol is resolved by the kernel. Each pillar's own track carries its steps and its exits; this track holds the counters and sizes pillars, which no other track does. diff --git a/issues/hardware/a-boots-readback-is-lost-to-one-ssh-that-times-out.md b/issues/hardware/a-boots-readback-is-lost-to-one-ssh-that-times-out.md new file mode 100644 index 00000000000..2400145f70c --- /dev/null +++ b/issues/hardware/a-boots-readback-is-lost-to-one-ssh-that-times-out.md @@ -0,0 +1,32 @@ +--- +status: open +kind: tooling +opened: 2026-10-04 +--- + +# A boot's readback is lost to one ssh that times out + +`src/metal.rs`'s `read_mounted` reads each log file off the T14's stick by +one `ssh … cat` (`Driver::cat`), and `Driver::ssh` runs it once: a dial that +times out after the boot came back fails the boot's readback whole. Nothing +retries it, and nothing keeps the stick's copy anywhere else, so the next +boot's flash (`wipefs --all`, then `dd`) destroys the only record of the boot +that ran. Every row judged on that boot then fails on the harness's word, not +the boot's. + +Evidence: the full metal profile at `ce1786ff0` (pull request #705, the +orchestrator's Drive log). The `jobcase` boot's machine answered ssh again +48 s after its reboot, the boot stick enumerated, the log partition mounted +and unmounted, and between the two the read failed with `reading a log file +on the machine exited exit status: 255: Connection timed out during banner +exchange`; the next boot's flash followed. Its four rows +(`usb_reset_hands_devices_back`, `blackbox_done_chain`, `machine_reboot`, +`machine_soft_off_decoded`) failed on that line, 275 of 279 green; the same +image (sha256 checked), booted again, was judged green on all four. The development +machine's network dropped in the same minutes. + +Owner: the orchestrator, which holds the T14. **Exit**: a dial lost while +the stick is being read costs a retry bounded by a timeout that fails loudly, +not the boot, or the stick's log partition is copied off whole before anything +else can flash it; and a mutation that makes the first `cat` of a run's first +boot fail once is read back green. diff --git a/issues/hardware/linuxs-readings-of-the-t14-and-the-tcg-model-lack-reads-owed-before-the-t14s-wipe.md b/issues/hardware/linuxs-readings-of-the-t14-and-the-tcg-model-lack-reads-owed-before-the-t14s-wipe.md index 68e258ba237..e8c17fb8fe8 100644 --- a/issues/hardware/linuxs-readings-of-the-t14-and-the-tcg-model-lack-reads-owed-before-the-t14s-wipe.md +++ b/issues/hardware/linuxs-readings-of-the-t14-and-the-tcg-model-lack-reads-owed-before-the-t14s-wipe.md @@ -31,193 +31,15 @@ reads it. **Oracle**: that Linux. ## Linux's counter readings, read 2026-10-03 The orchestrator takes this Linux as the one independent oracle for the T14's -hardware counters (`issues/diagnostics/toyos-explains-itself.md`). Run as root -under Ubuntu's `6.8.0-142-generic` from 16:22:37 UTC, one after the other; -each output below is whole but for trailing blanks. No test reads them yet: -the T14 row the umbrella track's exit names does. - -Idle, 60 s: `turbostat --quiet --interval 10 --num_iterations 6 --show -Core,CPU,Avg_MHz,Busy%,Bzy_MHz,TSC_MHz,IRQ,SMI,CPU%c1,CPU%c6,CPU%c7,CoreTmp,PkgTmp,Pkg%pc2,Pkg%pc3,Pkg%pc6,Pkg%pc7,Pkg%pc8,Pkg%pc9,Pk%pc10,PkgWatt,CorWatt`. -Machine-wide per 10 s: busy 0.11 to 0.36%, busy clock 980 to 1668 MHz, TSC -2419 MHz, SMI 0, core C7 98.78 to 100.08%, package PC8 75.47 to 82.10%, -package 1.14 to 1.88 W, cores 0.07 to 0.13 W. - -``` -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt -- - 1 0.11 994 2419 1160 0 0.47 0.00 99.98 29 31 17.46 2.01 0.03 0.29 77.22 0.00 0.00 1.30 0.07 -0 0 1 0.11 1012 2419 131 0 0.12 0.00 100.42 26 31 17.46 2.01 0.03 0.29 77.22 0.00 0.00 1.30 0.07 -0 4 0 0.05 910 2419 53 0 0.12 -1 1 1 0.08 986 2419 57 0 0.30 0.00 100.18 27 -1 5 1 0.12 999 2419 119 0 0.30 -2 2 1 0.13 933 2419 423 0 1.29 0.00 99.04 29 -2 6 2 0.19 1002 2419 157 0 1.29 -3 3 1 0.14 978 2419 148 0 0.16 0.00 100.27 28 -3 7 1 0.09 1115 2419 72 0 0.16 -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt -- - 1 0.11 980 2419 1009 0 0.38 0.00 100.08 29 30 19.01 1.80 0.05 0.22 76.10 0.00 0.00 1.81 0.07 -0 0 1 0.07 975 2419 46 0 0.14 0.00 100.37 26 30 19.01 1.80 0.05 0.22 76.10 0.00 0.00 1.81 0.07 -0 4 1 0.11 913 2419 72 0 0.14 -1 1 0 0.03 997 2419 23 0 0.19 0.00 100.34 25 -1 5 1 0.13 993 2419 130 0 0.19 -2 2 1 0.11 943 2419 386 0 1.14 0.00 99.27 29 -2 6 2 0.18 1035 2419 155 0 1.14 -3 3 1 0.14 967 2419 141 0 0.06 0.00 100.36 27 -3 7 1 0.09 997 2419 56 0 0.06 -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt -- - 1 0.11 1005 2419 1125 0 0.42 0.00 100.03 29 31 19.47 1.35 0.06 0.23 75.87 0.00 0.00 1.43 0.07 -0 0 1 0.12 1081 2419 42 0 0.08 0.00 100.39 26 31 19.47 1.35 0.06 0.23 75.87 0.00 0.00 1.43 0.07 -0 4 1 0.13 998 2419 56 0 0.08 -1 1 1 0.10 1006 2419 127 0 0.22 0.00 100.21 25 -1 5 1 0.14 1004 2419 132 0 0.22 -2 2 2 0.16 981 2419 513 0 1.33 0.00 99.12 29 -2 6 1 0.08 1028 2419 75 0 1.33 -3 3 1 0.13 953 2419 137 0 0.05 0.00 100.42 28 -3 7 1 0.06 1019 2419 43 0 0.05 -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt -- - 6 0.36 1668 2419 1966 0 1.14 0.00 98.78 29 32 18.49 1.12 0.04 0.33 75.47 0.00 0.00 1.14 0.13 -0 0 7 0.43 1587 2419 231 0 0.90 0.00 99.10 26 32 18.49 1.12 0.04 0.33 75.47 0.00 0.00 1.14 0.13 -0 4 2 0.15 994 2419 96 0 0.90 -1 1 11 0.48 2410 2419 185 0 1.08 0.00 98.67 27 -1 5 7 0.46 1450 2419 348 0 1.08 -2 2 8 0.48 1564 2419 635 0 1.97 0.00 98.11 29 -2 6 1 0.08 1040 2419 72 0 1.97 -3 3 12 0.69 1725 2419 326 0 0.60 0.00 99.24 28 -3 7 1 0.08 1053 2419 73 0 0.60 -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt -- - 1 0.12 984 2419 1166 0 0.39 0.00 100.01 29 31 13.62 2.79 0.04 0.39 80.55 0.00 0.00 1.30 0.07 -0 0 0 0.05 957 2419 34 0 0.09 0.00 100.41 26 31 13.62 2.79 0.04 0.39 80.55 0.00 0.00 1.30 0.07 -0 4 1 0.13 943 2419 78 0 0.09 -1 1 1 0.05 982 2419 47 0 0.36 0.00 100.00 27 -1 5 2 0.23 1023 2419 186 0 0.36 -2 2 1 0.05 1001 2419 45 0 0.09 0.00 100.38 29 -2 6 2 0.15 989 2419 135 0 0.09 -3 3 2 0.22 959 2419 535 0 1.04 0.00 99.26 28 -3 7 1 0.10 994 2419 106 0 1.04 -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt -- - 1 0.12 980 2419 1156 0 0.39 0.00 100.03 28 31 13.35 1.53 0.07 0.41 82.10 0.00 0.00 1.88 0.07 -0 0 1 0.05 958 2419 40 0 0.16 0.00 100.38 26 31 13.35 1.53 0.07 0.41 82.10 0.00 0.00 1.88 0.07 -0 4 1 0.11 947 2419 82 0 0.16 -1 1 1 0.09 989 2419 76 0 0.26 0.00 100.09 26 -1 5 2 0.19 990 2419 174 0 0.26 -2 2 1 0.09 965 2419 335 0 0.96 0.00 99.42 28 -2 6 2 0.19 995 2419 178 0 0.96 -3 3 2 0.19 976 2419 242 0 0.18 0.00 100.24 28 -3 7 0 0.04 1021 2419 29 0 0.18 -``` - -Loaded, 60 s, eight `yes > /dev/null` started 2 s before: `for i in 1 2 3 4 5 -6 7 8; do timeout 65 sh -c "yes > /dev/null" & done; sleep 2; turbostat ---quiet --interval 10 --num_iterations 6 --show -Core,CPU,Avg_MHz,Busy%,Bzy_MHz,TSC_MHz,IRQ,SMI,CPU%c1,CPU%c6,CPU%c7,CoreTmp,PkgTmp,Pkg%pc2,Pkg%pc6,Pk%pc10,PkgWatt,CorWatt; -wait`. Busy 99.77%; busy clock 3800 MHz at 30.24 and 30.73 W package for the -first two intervals, 3555 MHz at 27.20 W in the third, then 3075 to 3094 MHz -at 19.62 to 19.92 W; about 10,000 -interrupts per CPU per 10 s; SMI 0. - -``` -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt -- - 3791 99.77 3800 2419 80691 0 0.00 0.00 0.00 71 71 0.00 0.00 0.00 30.24 27.88 -0 0 3791 99.77 3800 2419 10023 0 0.00 0.00 0.00 61 71 0.00 0.00 0.00 30.24 27.88 -0 4 3791 99.77 3800 2419 10021 0 0.00 -1 1 3791 99.77 3800 2419 10066 0 0.00 0.00 0.00 71 -1 5 3791 99.77 3800 2419 10102 0 0.00 -2 2 3791 99.77 3800 2419 10031 0 0.00 0.00 0.00 62 -2 6 3791 99.77 3800 2419 10274 0 0.00 -3 3 3791 99.77 3800 2419 10125 0 0.00 0.00 0.00 71 -3 7 3791 99.77 3800 2419 10049 0 0.00 -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt -- - 3791 99.77 3800 2419 80659 0 0.00 0.00 0.00 79 79 0.00 0.00 0.00 30.73 28.29 -0 0 3791 99.77 3800 2419 10012 0 0.00 0.00 0.00 68 79 0.00 0.00 0.00 30.73 28.29 -0 4 3791 99.77 3800 2419 10014 0 0.00 -1 1 3791 99.77 3800 2419 10026 0 0.00 0.00 0.00 79 -1 5 3791 99.77 3800 2419 10095 0 0.00 -2 2 3791 99.77 3800 2419 10017 0 0.00 0.00 0.00 69 -2 6 3791 99.77 3800 2419 10364 0 0.00 -3 3 3791 99.77 3800 2419 10122 0 0.00 0.00 0.00 78 -3 7 3791 99.77 3800 2419 10009 0 0.00 -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt -- - 3547 99.77 3555 2419 80555 0 0.00 0.00 0.00 66 66 0.00 0.00 0.00 27.20 24.72 -0 0 3547 99.77 3555 2419 10021 0 0.00 0.00 0.00 58 66 0.00 0.00 0.00 27.20 24.72 -0 4 3547 99.77 3555 2419 10010 0 0.00 -1 1 3547 99.77 3555 2419 10028 0 0.00 0.00 0.00 65 -1 5 3547 99.77 3555 2419 10037 0 0.00 -2 2 3547 99.77 3555 2419 10028 0 0.00 0.00 0.00 60 -2 6 3547 99.77 3555 2419 10246 0 0.00 -3 3 3547 99.77 3555 2419 10173 0 0.00 0.00 0.00 66 -3 7 3547 99.77 3555 2419 10012 0 0.00 -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt -- - 3068 99.77 3075 2419 80909 0 0.00 0.00 0.00 65 65 0.00 0.00 0.00 19.62 17.18 -0 0 3068 99.77 3075 2419 10054 0 0.00 0.00 0.00 59 65 0.00 0.00 0.00 19.62 17.18 -0 4 3068 99.77 3075 2419 10029 0 0.00 -1 1 3068 99.77 3075 2419 10121 0 0.00 0.00 0.00 65 -1 5 3068 99.77 3075 2419 10039 0 0.00 -2 2 3068 99.77 3075 2419 10069 0 0.00 0.00 0.00 59 -2 6 3068 99.77 3075 2419 10276 0 0.00 -3 3 3068 99.77 3075 2419 10229 0 0.00 0.00 0.00 64 -3 7 3068 99.76 3075 2419 10092 0 0.00 -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt -- - 3083 99.77 3090 2419 80593 0 0.00 0.00 0.00 65 65 0.00 0.00 0.00 19.85 17.43 -0 0 3083 99.77 3090 2419 10008 0 0.00 0.00 0.00 58 65 0.00 0.00 0.00 19.85 17.43 -0 4 3083 99.77 3090 2419 10020 0 0.00 -1 1 3083 99.77 3090 2419 10006 0 0.00 0.00 0.00 64 -1 5 3083 99.77 3090 2419 10019 0 0.00 -2 2 3083 99.77 3090 2419 10028 0 0.00 0.00 0.00 60 -2 6 3083 99.77 3090 2419 10282 0 0.00 -3 3 3083 99.77 3090 2419 10221 0 0.00 0.00 0.00 65 -3 7 3083 99.77 3090 2419 10009 0 0.00 -Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt -- - 3087 99.77 3094 2419 80544 0 0.00 0.00 0.00 65 65 0.00 0.00 0.00 19.92 17.49 -0 0 3087 99.77 3094 2419 10009 0 0.00 0.00 0.00 58 65 0.00 0.00 0.00 19.92 17.49 -0 4 3087 99.77 3094 2419 10011 0 0.00 -1 1 3087 99.77 3094 2419 10012 0 0.00 0.00 0.00 65 -1 5 3087 99.77 3094 2419 10009 0 0.00 -2 2 3087 99.77 3094 2419 10017 0 0.00 0.00 0.00 59 -2 6 3087 99.77 3094 2419 10273 0 0.00 -3 3 3087 99.77 3094 2419 10205 0 0.00 0.00 0.00 65 -3 7 3087 99.77 3094 2419 10008 0 0.00 -``` - -Idle, 60 s: `perf stat -a -A -e msr/aperf/,msr/mperf/,msr/smi/,msr/tsc/ sleep -60`. `msr/smi/` 0 on all eight CPUs; `msr/tsc/` 145,157,230,321 to -145,157,642,420 per CPU over 60.002902881 s, 2419.2 MHz. - -``` - Performance counter stats for 'system wide': - -CPU0 98,079,648 msr/aperf/ -CPU1 181,130,722 msr/aperf/ -CPU2 121,578,728 msr/aperf/ -CPU3 140,715,079 msr/aperf/ -CPU4 61,297,064 msr/aperf/ -CPU5 191,874,303 msr/aperf/ -CPU6 96,508,493 msr/aperf/ -CPU7 74,201,638 msr/aperf/ -CPU0 150,186,840 msr/mperf/ -CPU1 192,582,720 msr/mperf/ -CPU2 234,799,920 msr/mperf/ -CPU3 313,507,656 msr/mperf/ -CPU4 156,309,048 msr/mperf/ -CPU5 326,606,712 msr/mperf/ -CPU6 227,450,760 msr/mperf/ -CPU7 156,326,856 msr/mperf/ -CPU0 0 msr/smi/ -CPU1 0 msr/smi/ -CPU2 0 msr/smi/ -CPU3 0 msr/smi/ -CPU4 0 msr/smi/ -CPU5 0 msr/smi/ -CPU6 0 msr/smi/ -CPU7 0 msr/smi/ -CPU0 145,157,386,466 msr/tsc/ -CPU1 145,157,434,015 msr/tsc/ -CPU2 145,157,376,550 msr/tsc/ -CPU3 145,157,230,321 msr/tsc/ -CPU4 145,157,567,260 msr/tsc/ -CPU5 145,157,642,420 msr/tsc/ -CPU6 145,157,528,396 msr/tsc/ -CPU7 145,157,356,567 msr/tsc/ - - 60.002902881 seconds time elapsed -``` +hardware counters (`issues/diagnostics/toyos-explains-itself.md`). Read as +root under Ubuntu's `6.8.0-142-generic` from 16:22:37 UTC, one after the +other, and committed whole with their commands in the `SOURCE` beside them: +turbostat idle and under one `yes` per CPU in `tests/t14-linux/`, which +`counters_on_metal` prints ToyOS's reading beside, and `perf stat` over the +msr PMU in `toyos-cpuvuln/fixtures/t14/perf-msr.txt`, which +`the_t14_has_what_linux_counted_on_it` holds the T14's counter verdict to. +Idle, per 10 s: busy 0.11 to 0.36%, busy clock 980 to 1668 MHz, TSC 2419 +MHz, SMI 0. Loaded: busy 99.77%, busy clock 3800 MHz for the first two +intervals, 3555 MHz in the third, then 3075 to 3094 MHz; SMI 0. Owner: the orchestrator, which holds the T14 the exit runs on. diff --git a/issues/kernel/an-ipi-can-overtake-the-stores-it-announces.md b/issues/kernel/an-ipi-can-overtake-the-stores-it-announces.md deleted file mode 100644 index 8a9703ba757..00000000000 --- a/issues/kernel/an-ipi-can-overtake-the-stores-it-announces.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-30 ---- - -# An IPI can overtake the stores it announces - -Neither architecture orders a CPU's earlier stores before the interrupt it -raises to announce them, so on hardware the target can take the interrupt and -read the word it was sent for before that word has landed. - -- **AArch64**: `kernel/src/arch/aarch64/irqchip.rs`'s `raise` writes - `ICC_SGI1R_EL1` with no barrier before it. Only a `DSB` orders a system - register write after stores to Normal memory; Linux's `gic_ipi_send_mask` - (`drivers/irqchip/irq-gic-v3.c`) issues `dsb(ishst)` before the write for - exactly this. -- **x86-64**: `kernel/src/arch/x86_64/apic.rs`'s `Reg::write` of the ICR is a - `wrmsr`, which in x2APIC mode is not serializing (SDM Vol. 3A, "MSR Access - in x2APIC Mode"); Linux fences every x2APIC IPI with `weak_wrmsr_fence` - (`mfence; lfence`). - -One reader that depends on the order: `kernel/src/sched/dump.rs` stores -`OWES` for each sibling, then kicks it, and a sibling that takes its kick -before its `OWES` is visible answers nothing, which the dump then reports as -a silent CPU. - -No guest here reds on it, and none is expected to: under TCG the write is a -call into QEMU's interrupt controller model and under HVF a trap, and neither -is the hardware's ordering. The evidence is the architecture and Linux's code, -and the exit is the barrier on both, one instruction each. diff --git a/issues/kernel/toyos-runs-the-t14-under-load-below-the-clock-linux-reaches.md b/issues/kernel/toyos-runs-the-t14-under-load-below-the-clock-linux-reaches.md new file mode 100644 index 00000000000..8a74e88ce38 --- /dev/null +++ b/issues/kernel/toyos-runs-the-t14-under-load-below-the-clock-linux-reaches.md @@ -0,0 +1,28 @@ +--- +status: open +kind: defect +opened: 2026-10-04 +--- + +# ToyOS runs the T14 under load below the clock Linux reaches + +The `counters` metal row, at `ce1786ff0` (pull request #705), spun one thread +per CPU for its `spin` phase and read every CPU's busy clock as 2318 MHz +(TSC 2419 MHz times APERF over MPERF, MPERF at 0.986 to 0.990 of the stamp, +so every CPU was in C0 the whole span). Linux's turbostat on the same machine +under one `yes` per CPU (`tests/t14-linux/turbostat-loaded.txt`) reads +3800 MHz for its first 20 s and 3075 to 3094 MHz once the package settles near +20 W. ToyOS runs a loaded T14 at 61 to 75% of Linux's busy clock, below the +TSC's own rate. + +The CPU enumerates HWP (`hwp`, `hwp_epp` in +`toyos-cpuvuln/fixtures/t14/cpuinfo.txt`), and the kernel writes neither +`IA32_PM_ENABLE` nor `IA32_PERF_CTL`: the clock is whatever the firmware left. +`issues/kernel/the-scheduler-and-the-clocks-never-talk.md` is the track that +would couple the scheduler to frequency; this is the floor beneath it, a +machine that never leaves the firmware's operating point. + +Owner: the orchestrator, which holds the T14. **Exit**: the `counters` row +holds every CPU's busy clock across its spin to +Linux's loaded range in `tests/t14-linux/`, and the reading that held it is in +the closing pull request. diff --git a/issues/kernel/two-t14-cpus-idle-busier-under-toyos-than-any-under-linux.md b/issues/kernel/two-t14-cpus-idle-busier-under-toyos-than-any-under-linux.md new file mode 100644 index 00000000000..d8d97d89a72 --- /dev/null +++ b/issues/kernel/two-t14-cpus-idle-busier-under-toyos-than-any-under-linux.md @@ -0,0 +1,26 @@ +--- +status: open +kind: defect +opened: 2026-10-04 +--- + +# Two T14 CPUs idle busier under ToyOS than any under Linux + +The `counters` metal row, at `ce1786ff0` (pull request #705), read each CPU's +MPERF over its stamp across the idle second between its `idle0` and `idle1` +reads: cpu0 1.37% and cpu7 1.45% busy, cpu3 0.11%, the other five 0.00 or +0.01%. Linux's turbostat on the same machine, idle +(`tests/t14-linux/turbostat-idle.txt`), reads 0.11 to 0.36% machine-wide per +10 s and no CPU above 0.69%. So six CPUs idle deeper than Linux's and two +about four times busier than Linux's machine-wide busiest interval. + +A second boot, at `a059144e2`, read the same two CPUs again: cpu0 1.35% and +cpu7 1.42%, cpu4 0.37%, cpu3 0.11%, the other four 0.00 to 0.03%. The two +busy CPUs reproduce across boots. + +Not yet attributed: the row's own reader runs between the two reads, prints +its `idle0` lines and parks, and which CPUs it and the log's path ran on that +second is not recorded. Owner: the orchestrator, which holds the T14. +**Exit**: a reading that names what ran on cpu0 and +cpu7 across that second; then the cause is fixed, or, if it is the row's own +work, folded to the row's doc. diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index d7c8738dd07..20a78b0a455 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -42,6 +42,7 @@ dependencies = [ "toyos-blackbox", "toyos-blockhold", "toyos-bootmap", + "toyos-cpuvuln", "toyos-dma", "toyos-elf", "toyos-elide", @@ -100,6 +101,10 @@ dependencies = [ "toyos-abi", ] +[[package]] +name = "toyos-cpuvuln" +version = "0.1.0" + [[package]] name = "toyos-dma" version = "0.1.0" diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 7dea204185d..b383b429fd9 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -21,7 +21,7 @@ loom = [] # obligation is a model. wake-fence-off = [] # The same for the panic console's published descriptor: `kernel-loom` turns it on -# to drop the publisher's `Release` fence from `drivers/panic_console/published.rs`, +# to drop the publisher's `Release` fence from `seqlock.rs`, # and `kernel/loom/tests/panic_console_publish.rs` must red — a reader keeping a # descriptor that is half one publication and half the next. seqlock-writer-fence-off = [] @@ -53,6 +53,8 @@ reap-raise-relaxed = [] # `src/shootdown.rs`'s `serve` reads what it owes `Relaxed`, and # `tlb_shootdown` reds. shootdown-serve-relaxed = [] +# `src/shootdown.rs`'s `served` reads `Relaxed`, and `counters_round` reds. +shootdown-served-relaxed = [] # `src/log/shard.rs`'s publishing store goes `Relaxed`, and `log_record` reds. log-commit-release-off = [] # `src/log/registry.rs`'s pointer store and load go `Relaxed`, and @@ -496,6 +498,7 @@ toyos-elide = { path = "../toyos-elide" } toyos-blackbox = { path = "../toyos-blackbox" } toyos-blockhold = { path = "../toyos-blockhold" } toyos-bootmap = { path = "../toyos-bootmap" } +toyos-cpuvuln = { path = "../toyos-cpuvuln" } toyos-fat32 = { path = "../toyos-fat32" } toyos-elf = { path = "../toyos-elf" } toyos-gicv3 = { path = "gicv3" } diff --git a/kernel/loom/Cargo.toml b/kernel/loom/Cargo.toml index 2cbf96cb562..15526da33bd 100644 --- a/kernel/loom/Cargo.toml +++ b/kernel/loom/Cargo.toml @@ -90,6 +90,16 @@ reap-raise-relaxed = [] # Never on by default and never reachable from a kernel build, which declares # the same name only so `cfg` checking knows it. shootdown-serve-relaxed = [] +# The negative control for a counters round's answer. It makes +# `shootdown.rs`'s `served` read `Relaxed`, so a reader told a CPU answered can +# still copy out the answer before it, and `counters_round.rs` must red: +# +# cargo test --manifest-path kernel/loom/Cargo.toml --features shootdown-served-relaxed \ +# --test counters_round +# +# Never on by default and never reachable from a kernel build, which declares +# the same name only so `cfg` checking knows it. +shootdown-served-relaxed = [] # The negative control for the CPU roster's publication. It makes # `smp_roster.rs`'s `commit` publish the count with a `Relaxed` store, so a # reader can see a count over a slot the commit has not filled, and diff --git a/kernel/loom/src/lib.rs b/kernel/loom/src/lib.rs index e4896839edb..2185a7ac7ee 100644 --- a/kernel/loom/src/lib.rs +++ b/kernel/loom/src/lib.rs @@ -408,10 +408,11 @@ pub mod capture_access; #[path = "../../../toyos/src/log/ring.rs"] pub mod log_ring; -/// The panic console's published framebuffer descriptor, driven by -/// `tests/panic_console_publish.rs`. -#[path = "../../src/drivers/panic_console/published.rs"] -pub mod panic_console_published; +/// The seqlock the panic console's descriptor and each CPU's counters are +/// published through, driven by `tests/panic_console_publish.rs` and +/// `tests/counters_round.rs`. +#[path = "../../src/seqlock.rs"] +pub mod seqlock; /// The console backend's lock, driven by `tests/serial_lock.rs`; under `loom` /// alone, as is the [`arch::cpu`] shim it names. diff --git a/kernel/loom/tests/counters_round.rs b/kernel/loom/tests/counters_round.rs new file mode 100644 index 00000000000..866407c82bb --- /dev/null +++ b/kernel/loom/tests/counters_round.rs @@ -0,0 +1,70 @@ +//! Loom: a counters read that finds a CPU's round answered copies out that +//! CPU's answer to it, and not the one before. +//! +//! A CPU answers a round by publishing its block (`seqlock::Published`) inside +//! `Shootdown::serve`'s closure, and the reader waits for `served` before it +//! snapshots the block. The seqlock keeps a snapshot whole; the edge from +//! `serve`'s release to `served`'s acquire is what makes it this round's. On +//! x86 every load is an acquire, so no guest test can fail here, and the +//! negative control is a feature rather than a comment: +//! +//! ```text +//! cargo test --manifest-path kernel/loom/Cargo.toml --features shootdown-served-relaxed \ +//! --test counters_round +//! ``` +//! +//! makes `served` read `Relaxed`, and this file must red: the reader finds the +//! round answered and copies out something other than the answer. +#![cfg(feature = "loom")] + +use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; + +use kernel_loom::seqlock::Published; +use kernel_loom::shootdown::Shootdown; +use loom::sync::Arc; + +/// What the CPU's block held before the round, and what it answers the round with. +const BEFORE: [u64; 2] = [1, 1]; +const ANSWER: [u64; 2] = [2, 2]; + +/// How many times the reader looks for the answer. +const POLLS: usize = 3; + +/// Set by any execution in which the reader found the round answered, outside +/// the model so it speaks for all of them: without it the bounded poll could +/// pass every execution by never reaching the assertion. +static READ: AtomicBool = AtomicBool::new(false); + +#[test] +fn a_cpu_read_as_answered_is_read_with_its_answer() { + READ.store(false, SeqCst); + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(2); + builder.check(|| { + let rounds = Arc::new(Shootdown::new()); + let block = Arc::new(Published::new()); + block.publish(BEFORE); + let generation = rounds.issue(); + + let cpu1 = { + let (rounds, block) = (rounds.clone(), block.clone()); + loom::thread::spawn(move || rounds.serve(1, || block.publish(ANSWER))) + }; + + for _ in 0..POLLS { + if rounds.served(1, generation) { + READ.store(true, SeqCst); + assert_eq!( + block.snapshot(), + Some(ANSWER), + "cpu 1 read as having answered the round, and its block did not copy \ + out that answer", + ); + break; + } + loom::thread::yield_now(); + } + cpu1.join().unwrap(); + }); + assert!(READ.load(SeqCst), "no interleaving found the round answered, so the assertion never ran"); +} diff --git a/kernel/loom/tests/panic_console_publish.rs b/kernel/loom/tests/panic_console_publish.rs index 2544e850c39..06b4e4fa467 100644 --- a/kernel/loom/tests/panic_console_publish.rs +++ b/kernel/loom/tests/panic_console_publish.rs @@ -7,10 +7,13 @@ //! `seqlock-writer-fence-off` feature, which must red this file. #![cfg(feature = "loom")] -use kernel_loom::panic_console_published::{Published, WORDS}; +use kernel_loom::seqlock::Published; use loom::sync::Arc; use loom::thread; +/// The descriptor's width in the kernel. +const WORDS: usize = 4; + #[test] fn a_snapshot_is_one_publication_whole() { loom::model(|| { diff --git a/kernel/src/arch/aarch64/counters.rs b/kernel/src/arch/aarch64/counters.rs new file mode 100644 index 00000000000..b808d6c166b --- /dev/null +++ b/kernel/src/arch/aarch64/counters.rs @@ -0,0 +1,10 @@ +//! No model-specific counter is read on this architecture: the activity +//! monitors' cycle counters (`FEAT_AMUv1`) are unread. + +use crate::counters::Hardware; + +pub fn bring_up() {} + +pub fn read() -> Hardware { + Hardware { smi: None, aperf: None, mperf: None } +} diff --git a/kernel/src/arch/aarch64/irqchip.rs b/kernel/src/arch/aarch64/irqchip.rs index 13cfe1de5f5..f2faf93cc3f 100644 --- a/kernel/src/arch/aarch64/irqchip.rs +++ b/kernel/src/arch/aarch64/irqchip.rs @@ -35,8 +35,7 @@ use crate::hw::MIN_ONE_SHOT; /// which `super::msi_message` refuses on this machine. #[repr(u8)] pub(super) enum Intid { - /// Asks a CPU for a scheduler pass: x86-64's kick, which rides the - /// timer's vector there. + /// Asks a CPU for a scheduler pass, or for its counters: x86-64's kick. Kick = 0, /// Stops a CPU for good: [`stop_other_cpus`]'s. Halt, @@ -278,11 +277,15 @@ pub(super) fn timer_intid() -> u32 { TIMER_INTID.load(Relaxed) } -/// Write `ICC_SGI1R_EL1` whole: the SGI it names is raised. +/// Write `ICC_SGI1R_EL1` whole: the SGI it names is raised, after every store +/// before it. Only a `DSB` orders a system register write after stores to +/// Normal memory, so without it a target could take the SGI before the store +/// it announces (Linux's `gic_ipi_send_mask`, `dsb(ishst)`). fn raise(value: u64) { - // SAFETY: writes `ICC_SGI1R_EL1`, which raises an SGI and touches no - // memory; the `ISB` sends it before whatever follows. - unsafe { core::arch::asm!("msr S3_0_C12_C11_5, {}", "isb", in(reg) value, options(nomem, nostack, preserves_flags)) }; + // SAFETY: a barrier and a write of `ICC_SGI1R_EL1`, which raises an SGI; + // the `ISB` sends it before whatever follows. No `nomem`: the barrier + // orders the compiler's stores too. + unsafe { core::arch::asm!("dsb ishst", "msr S3_0_C12_C11_5, {}", "isb", in(reg) value, options(nostack, preserves_flags)) }; } /// Raise SGI `intid` on the CPU whose packed affinity is `target`. diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index b07ea6171a1..0e56393ffb3 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -16,6 +16,7 @@ pub mod boot; pub mod cache; pub mod console_uart; pub mod control_regs; +pub mod counters; pub mod cpu; pub mod entropy; pub mod entry; diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 298d52a54ed..69c8f725f9c 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -181,7 +181,11 @@ fn irq(from_el0: bool) { irqchip::SGI_HALT => cpu::halt(), irqchip::SGI_OFF => super::power::cpu_off(), irqchip::SGI_KICK => { - percpu::irq_took(Source::Timer); + percpu::irq_took(Source::Kick); + // Raised as an interrupt entry raises it, so the answer's post runs no pass here. + percpu::preempt_count_up(); + crate::counters::serve_here(); + percpu::preempt_count_down(); irqchip::end(intid); if from_el0 { crate::scheduler::do_preempt(); diff --git a/kernel/src/arch/x86_64/apic.rs b/kernel/src/arch/x86_64/apic.rs index e8f54fd34c9..1d23f226088 100644 --- a/kernel/src/arch/x86_64/apic.rs +++ b/kernel/src/arch/x86_64/apic.rs @@ -85,15 +85,24 @@ pub fn id() -> u32 { Reg::Id.read() as u32 } +/// Raise an interrupt `icr` names whole, after every store before it. An +/// x2APIC ICR write is not serializing (SDM Vol. 3A, "MSR Access in x2APIC +/// Mode"), so without the fence a target could take the interrupt before the +/// store it announces. +fn send(icr: u64) { + cpu::wrmsr_fence(); + Reg::Icr.write(icr); +} + /// Send INIT IPI to the specified APIC ID. pub fn send_init(apic_id: u32) { // ICR write: destination in the high 32 bits, 0x4500 = delivery INIT, level assert. - Reg::Icr.write(((apic_id as u64) << 32) | 0x4500); + send(((apic_id as u64) << 32) | 0x4500); } /// Send Startup IPI (SIPI) with the given vector (trampoline page number). pub fn send_sipi(apic_id: u32, vector: u8) { - Reg::Icr.write(((apic_id as u64) << 32) | 0x4600 | vector as u64); + send(((apic_id as u64) << 32) | 0x4600 | vector as u64); } /// Send EOI. @@ -128,12 +137,12 @@ pub fn send_self(vector: u8) { return; } // Destination shorthand = self (0b01 << 18), fixed delivery, level assert. - Reg::Icr.write(0x0004_4000 | vector as u64); + send(0x0004_4000 | vector as u64); } fn ipi_all_excluding_self(vector: u8) { // destination shorthand = all-excluding-self (0b11 << 18), fixed delivery - Reg::Icr.write(0x000C_0000 | vector as u64); + send(0x000C_0000 | vector as u64); } /// Ask every other CPU to flush its TLB. @@ -143,12 +152,12 @@ pub(super) fn tlb_ipi() { } } -/// Send the timer-vector IPI to one CPU, waking it if halted. +/// Send the kick IPI to one CPU, waking it if halted. // Targeted, not broadcast: a broadcast kick would preempt every sibling per wake and cannot scale. pub fn kick_cpu(cpu_id: u32) { if !X2APIC_ENABLED.load(Ordering::Relaxed) { return; } let apic_id = crate::smp::hardware_id(cpu_id); - Reg::Icr.write(((apic_id as u64) << 32) | 0x4000 | TIMER_VECTOR as u64); + send(((apic_id as u64) << 32) | 0x4000 | u64::from(super::idt::KICK_VECTOR)); } // Kicked, and not left to arrive on their own: a CPU halted in the idle path has stopped its own timer, so nothing else brings it to the next scheduler pass. @@ -166,7 +175,7 @@ pub fn kick_all_but_self() { pub fn send_nmi(cpu_id: u32) { if !X2APIC_ENABLED.load(Ordering::Relaxed) { return; } let apic_id = crate::smp::hardware_id(cpu_id); - Reg::Icr.write(((apic_id as u64) << 32) | 0x4400); + send(((apic_id as u64) << 32) | 0x4400); } /// Point this CPU's performance-counter LVT entry at NMI delivery, unmasked — @@ -190,7 +199,7 @@ pub fn arm_perf_nmi() { /// for one rather than CPUs that need halting. pub fn stop_other_cpus() { if X2APIC_ENABLED.load(Ordering::Relaxed) && crate::smp::is_ready() { - Reg::Icr.write(0x000C_0000 | 0xFD); + send(0x000C_0000 | 0xFD); } } diff --git a/kernel/src/arch/x86_64/counters.rs b/kernel/src/arch/x86_64/counters.rs new file mode 100644 index 00000000000..efbc1eb891e --- /dev/null +++ b/kernel/src/arch/x86_64/counters.rs @@ -0,0 +1,48 @@ +//! Which of this CPU's model-specific counters it reads: decided on the CPU +//! itself at bring-up, by `toyos_cpuvuln::counters` over its own CPUID. + +use core::sync::atomic::{AtomicU8, Ordering::Relaxed}; + +use toyos_cpuvuln::{CounterFacts, Vendor}; + +use super::{cpu, percpu}; +use crate::counters::Hardware; +use crate::scheduler::MAX_CPUS; + +const MSR_SMI_COUNT: u32 = 0x34; +const IA32_MPERF: u32 = 0xE7; +const IA32_APERF: u32 = 0xE8; + +const SMI: u8 = 1 << 0; +const APERF_MPERF: u8 = 1 << 1; + +/// Each CPU's verdict, written by that CPU before it reads a counter. +static ADMITTED: [AtomicU8; MAX_CPUS] = [const { AtomicU8::new(0) }; MAX_CPUS]; + +pub fn bring_up() { + let (max_leaf, ebx, ecx, edx) = cpu::cpuid(0, 0); + let id: [u8; 12] = core::array::from_fn(|i| [ebx, edx, ecx][i / 4].to_le_bytes()[i % 4]); + let (signature, _, cpuid_1_ecx, _) = cpu::cpuid(1, 0); + // As `init_scattered_cpuid_features` reads it: a level past the range's own is no level. + let cpuid_6_ecx = if (6..=0xFFFF).contains(&max_leaf) { cpu::cpuid(6, 0).2 } else { 0 }; + let verdict = toyos_cpuvuln::counters(&CounterFacts { + vendor: Vendor::from_id(&id), + signature, + cpuid_1_ecx, + cpuid_6_ecx, + }); + let bits = if verdict.smi { SMI } else { 0 } | if verdict.aperf_mperf { APERF_MPERF } else { 0 }; + ADMITTED[percpu::cpu_id() as usize].store(bits, Relaxed); +} + +/// This CPU's admitted counters. +pub fn read() -> Hardware { + let admitted = ADMITTED[percpu::cpu_id() as usize].load(Relaxed); + let msr = |bit: u8, msr: u32| (admitted & bit != 0).then(|| cpu::rdmsr(msr)); + Hardware { + // Bits 63:32 are reserved (SDM Vol. 4, MSR 34H). + smi: msr(SMI, MSR_SMI_COUNT).map(|count| count & 0xFFFF_FFFF), + aperf: msr(APERF_MPERF, IA32_APERF), + mperf: msr(APERF_MPERF, IA32_MPERF), + } +} diff --git a/kernel/src/arch/x86_64/cpu.rs b/kernel/src/arch/x86_64/cpu.rs index 0c08e730458..6e67f58bbf2 100644 --- a/kernel/src/arch/x86_64/cpu.rs +++ b/kernel/src/arch/x86_64/cpu.rs @@ -28,6 +28,15 @@ pub unsafe fn wrmsr(msr: u32, value: u64) { asm!("wrmsr", in("ecx") msr, in("eax") low, in("edx") high, options(nomem, nostack)); } +/// Every earlier load and store done before what follows, a `wrmsr` among it: +/// Linux's `weak_wrmsr_fence`. No `nomem`, so the compiler moves no access +/// across it either. +#[inline] +pub fn wrmsr_fence() { + // SAFETY: two fences; no register, flag or memory changes. + unsafe { asm!("mfence", "lfence", options(nostack, preserves_flags)) }; +} + #[inline] pub fn rdtsc() -> u64 { let lo: u32; diff --git a/kernel/src/arch/x86_64/idt/kick.rs b/kernel/src/arch/x86_64/idt/kick.rs new file mode 100644 index 00000000000..7e957753a8e --- /dev/null +++ b/kernel/src/arch/x86_64/idt/kick.rs @@ -0,0 +1,13 @@ +use super::device_irq::device_irq_entry; + +extern "sysv64" fn kick_handler() { + crate::arch::percpu::irq_took!(Kick); + crate::counters::serve_here(); + crate::preempt::set_need_resched(); + crate::arch::apic::eoi(); +} + +device_irq_entry! { + /// `apic::kick_cpu`'s IPI entry (see `device_irq_entry` for the asm contract). + pub(super) fn kick_entry => kick_handler +} diff --git a/kernel/src/arch/x86_64/idt/mod.rs b/kernel/src/arch/x86_64/idt/mod.rs index c6ae1ff6081..c70e307935c 100644 --- a/kernel/src/arch/x86_64/idt/mod.rs +++ b/kernel/src/arch/x86_64/idt/mod.rs @@ -4,6 +4,7 @@ mod dma_fault; mod hda; mod i8042; mod isa; +mod kick; mod nmi; pub(crate) mod spurious; mod timer; @@ -31,6 +32,9 @@ const PIC2_DATA: u16 = 0xA1; /// The vector both PS/2 lines are routed to. pub const I8042_VECTOR: u8 = Vector::I8042 as u8; +/// The vector `apic::kick_cpu` raises. +pub const KICK_VECTOR: u8 = Vector::Kick as u8; + /// The vector each `pio::GRANTABLE` row's lines are routed to, by row: the /// vector is how the kernel knows whose record an interrupt belongs to. pub const ISA_VECTORS: [u8; 1] = [Vector::Isa0 as u8]; @@ -267,6 +271,7 @@ idt_vectors! { ring3 UserDev2 = 0x2A, user_dev::user_dev2_entry; ring3 UserDev3 = 0x2B, user_dev::user_dev3_entry; ring3 Isa0 = 0x2C, isa::isa0_entry; + ring3 Kick = 0xFC, kick::kick_entry; // Ring 0 because it never returns: `cli; hlt` forever. ring0 HaltAll = 0xFD, stub_halt_all; ring3 TlbFlush = 0xFE, tlb::tlb_flush_entry; diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index f9f59e888a8..2ac9058d047 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -14,6 +14,7 @@ pub mod barrier; pub mod boot; pub mod cache; pub mod control_regs; +pub mod counters; pub mod console_uart; pub mod cpu; pub mod entropy; diff --git a/kernel/src/counters.rs b/kernel/src/counters.rs new file mode 100644 index 00000000000..b0a43211113 --- /dev/null +++ b/kernel/src/counters.rs @@ -0,0 +1,205 @@ +//! The machine's counters: per CPU, what [`Counter`] names, read by that CPU +//! into its own block and copied out by `SYS_COUNTERS`. +//! +//! **A CPU's hardware counters are readable only on that CPU**, so a read is a +//! round ([`Shootdown`]'s protocol): the reader issues a generation, answers +//! for its own CPU, kicks every other, and parks until each has answered or +//! the round's [`ANSWER`] bound has passed. A CPU answers from its kick +//! handler ([`serve_here`]) wherever it was; one whose interrupts stay closed +//! past the bound is reported stale with the last block it published, never +//! waited for without end. A read within [`JOIN`] of the last round's issue +//! waits on that round, so no reader makes a CPU take more than one kick per +//! [`JOIN`]. +//! +//! A model-specific counter is read only where the CPU's own verdict at +//! [`bring_up`] admitted it, and that bring-up's sample reads it once: a wrong +//! verdict is `#GP` at boot, never inside a read. + +use kernel::sched::task::WaitClass; +use toyos_abi::counters::{Counter, Record, RECORD_BYTES}; +use toyos_abi::handle::Rights; +use toyos_abi::syscall::SyscallError; + +use crate::arch::{irqchip, percpu, IrqGuard}; +use crate::irq_census::Source; +use crate::scheduler::{Parkable, MAX_CPUS}; +use crate::seqlock::Published; +use crate::shootdown::{Generation, Shootdown}; +use crate::sync::Lock; +use crate::time::{Budget, Cadence, Deadline, Duration, Instant}; +use crate::user_ptr::UserBytesMut; +use crate::watch::{self, IrqWatch}; + +/// How long a round waits for its answers. +const ANSWER: Budget = Budget::of( + Duration::from_millis(100), + "every CPU that has not answered is read stale, with the block it last published", +); + +/// The most often a round is issued. +const JOIN: Cadence = Cadence::every( + Duration::from_millis(10), + "a reader within it of the last round's issue waits on that round, so a kick per CPU per 10 ms at most", +); + +/// The counters a CPU's model-specific registers hold, where its bring-up +/// admitted them. +pub struct Hardware { + pub smi: Option, + pub aperf: Option, + pub mperf: Option, +} + +/// A block's words: the hardware id, a bit per counter it holds, and a word +/// per counter. +const WORDS: usize = 2 + Counter::COUNT; + +static ROUNDS: Shootdown = Shootdown::new(); +static BLOCKS: [Published; MAX_CPUS] = [const { Published::new() }; MAX_CPUS]; +/// Posted by every answer; a reader parks on it. +static ANSWERED: IrqWatch = IrqWatch::new(); +/// The last round issued, and when. +static LAST: Lock> = Lock::new(None); + +/// This CPU's verdict and its first block. Every CPU, once, before it is +/// asked anything. +pub fn bring_up() { + crate::arch::counters::bring_up(); + let me = percpu::cpu_id() as usize; + let words = sample(me); + BLOCKS[me].publish(words); + let [smi, aperf, mperf] = [Counter::Smi, Counter::Aperf, Counter::Mperf].map(|c| held(&words, c)); + log!("counters: cpu{me} reads smi={smi} aperf={aperf} mperf={mperf}"); +} + +fn held(words: &[u64; WORDS], counter: Counter) -> bool { + words[1] & 1 << counter as usize != 0 +} + +/// This CPU's block, read on it. +fn sample(me: usize) -> [u64; WORDS] { + let stamp = crate::arch::cpu::counter(); + let hardware = crate::arch::counters::read(); + let mut words = [0; WORDS]; + words[0] = u64::from(crate::arch::cpu::hardware_id()); + for counter in Counter::ALL { + let value = match counter { + Counter::Stamp => Some(stamp), + Counter::Smi => hardware.smi, + Counter::Aperf => hardware.aperf, + Counter::Mperf => hardware.mperf, + Counter::Kicks => crate::irq_census::deliveries(me as u32, Source::Kick), + }; + if let Some(value) = value { + words[1] |= 1 << counter as usize; + words[2 + counter as usize] = value; + } + } + words +} + +/// This CPU's answer to the round it owes, if it owes one: the kick handler's +/// on every architecture. +pub fn serve_here() { + answer(false); +} + +/// This CPU's answer, after kicking every other CPU when `kick` says so. +fn answer(kick: bool) { + let served = { + // One writer per block, and one CPU both skipped by the kick and answered for. + let _closed = IrqGuard::close(); + if kick { + irqchip::kick_all_but_self(); + } + let me = percpu::cpu_id() as usize; + #[cfg(feature = "test-actuators")] + if deaf::DEAF.load(core::sync::atomic::Ordering::Relaxed) == me { + return; + } + ROUNDS.serve_if_owed(me, || BLOCKS[me].publish(sample(me))) + }; + if served { + ANSWERED.post_in_place(); + } +} + +/// One record per online CPU into `out`, or how many CPUs there are when `out` +/// is empty; `rights` is what the caller's capability holds, a counter answered +/// only where they contain what it needs. +pub fn read(rights: Rights, out: &mut UserBytesMut) -> Result { + let cpus = crate::smp::cpu_count() as usize; + if out.is_empty() { + return Ok(cpus); + } + if out.len() < cpus * RECORD_BYTES { + return Err(SyscallError::ResourceExhausted); + } + let now = crate::clock::now(); + let (generation, issued, fresh) = { + let mut last = LAST.lock(); + match *last { + Some((generation, issued)) if now < issued + JOIN.duration() => (generation, issued, false), + _ => { + let generation = ROUNDS.issue(); + *last = Some((generation, now)); + (generation, now, true) + } + } + }; + // This CPU's kick, if it has one, waits behind this syscall. + answer(fresh); + let answered = || (0..cpus).all(|cpu| ROUNDS.served(cpu, generation)); + let deadline = Deadline::at(issued + ANSWER.duration()); + if watch::wait_until(&Parkable::at_entry(), &ANSWERED, 0, WaitClass::Other, deadline, answered).is_err() { + return Err(SyscallError::Gone); + } + for cpu in 0..cpus { + out.write_at(cpu * RECORD_BYTES, &record(cpu, generation, rights).encode().0); + } + Ok(cpus) +} + +fn record(cpu: usize, generation: Generation, rights: Rights) -> Record { + // Before the snapshot: a CPU read as answered is one whose block holds that answer or a later one. + let fresh = ROUNDS.served(cpu, generation); + let words = BLOCKS[cpu].snapshot(); + let mut values = [None; Counter::COUNT]; + if let Some(words) = &words { + for counter in Counter::ALL.into_iter().filter(|&c| held(words, c) && rights.contains(c.needs())) { + values[counter as usize] = Some(words[2 + counter as usize]); + } + } + Record { + cpu: cpu as u32, + hardware_id: words.map_or(0, |w| w[0] as u32), + stale: !fresh || words.is_none(), + values, + } +} + +/// `SYS_DEBUG`'s staging of a CPU that answers no round. +#[cfg(feature = "test-actuators")] +pub mod deaf { + use core::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + + use toyos_abi::syscall::SyscallError; + + /// Outside every CPU index, so it names none. + const NOBODY: usize = usize::MAX; + + pub(super) static DEAF: AtomicUsize = AtomicUsize::new(NOBODY); + + pub fn stage(cpu: u64) -> u64 { + if cpu >= u64::from(crate::smp::cpu_count()) { + return SyscallError::InvalidArgument.to_u64(); + } + DEAF.store(cpu as usize, Relaxed); + 0 + } + + pub fn end() -> u64 { + DEAF.store(NOBODY, Relaxed); + 0 + } +} diff --git a/kernel/src/drivers/panic_console/mod.rs b/kernel/src/drivers/panic_console/mod.rs index 89a2cafdc21..720d92c7697 100644 --- a/kernel/src/drivers/panic_console/mod.rs +++ b/kernel/src/drivers/panic_console/mod.rs @@ -12,7 +12,6 @@ mod access; mod latch; -mod published; use core::cell::UnsafeCell; use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; @@ -65,6 +64,9 @@ struct Fb { format: u32, } +/// Words in one published descriptor. +const FB_WORDS: usize = 4; + impl Fb { const DETACHED: Fb = Fb { ptr: core::ptr::null_mut(), @@ -76,7 +78,7 @@ impl Fb { }; /// The descriptor as the seqlock's words. - fn words(self) -> [u64; published::WORDS] { + fn words(self) -> [u64; FB_WORDS] { [ self.ptr as u64, self.bytes, @@ -85,7 +87,7 @@ impl Fb { ] } - fn from_words([ptr, bytes, stride_width, height_format]: [u64; published::WORDS]) -> Self { + fn from_words([ptr, bytes, stride_width, height_format]: [u64; FB_WORDS]) -> Self { Self { ptr: ptr as *mut u8, bytes, @@ -219,10 +221,10 @@ struct RenderedCell(UnsafeCell); // or swap atomically. `PAINTING`, `CAPTURE`, and `CAPTURE_ACCESS` serialise the three cells. unsafe impl Sync for RenderedCell {} -/// The descriptor painters draw through, behind a seqlock (`published`). +/// The descriptor painters draw through, behind a seqlock (`crate::seqlock`). /// Not a `Lock`: its guard drop can dispatch the scheduler, forbidden here. /// A publisher that dies mid-update leaves it changing forever, costing the screen and nothing else. -static FB: published::Published = published::Published::new(); +static FB: crate::seqlock::Published = crate::seqlock::Published::new(); /// Exactly one painter at a time, taken by every painter without exception. /// [`render`] and [`seal_wedge`] never release it; every other painter does, @@ -354,7 +356,7 @@ pub fn disable() { } /// One publication whole, or nothing: a descriptor still changing is no -/// screen this instant (`published`, and `kernel-loom`'s `panic_console_publish`). +/// screen this instant. fn snapshot() -> Option { let fb = Fb::from_words(FB.snapshot()?); (!fb.ptr.is_null()).then_some(fb) diff --git a/kernel/src/irq_census.rs b/kernel/src/irq_census.rs index 4b8df43a05e..98f1093f809 100644 --- a/kernel/src/irq_census.rs +++ b/kernel/src/irq_census.rs @@ -15,8 +15,10 @@ use crate::scheduler::MAX_CPUS; #[derive(Clone, Copy, PartialEq, Eq, Debug)] #[repr(usize)] pub enum Source { - /// Vector 0x20: shared by this CPU's LAPIC one-shot and every `irqchip::kick_cpu` IPI. + /// Vector 0x20, this CPU's LAPIC one-shot: an expiry and nothing else. Timer, + /// Vector 0xFC, `irqchip::kick_cpu`'s IPI. + Kick, /// Vector 0x21, xHCI MSI-X (or MSI). Xhci, /// Vectors 0x28-0x2C: the MSI-X of a PCI function a process drives, and @@ -45,11 +47,11 @@ pub enum Source { } impl Source { - pub const COUNT: usize = 11; + pub const COUNT: usize = 12; /// Order `tests/toyos.rs`'s `irq_census_conservation` parses back; must match variant order. pub const NAMES: [&'static str; Self::COUNT] = [ - "timer", "xhci", "userdev", "sound", "i8042", "dmafault", "hda", "tlb", "nmi", + "timer", "kick", "xhci", "userdev", "sound", "i8042", "dmafault", "hda", "tlb", "nmi", "spurious", "unclaimed", ]; } @@ -101,7 +103,6 @@ impl fmt::Display for Fields<'_> { } /// One CPU's delivery count for `source`, or `None` if that CPU has never been built. -#[cfg(feature = "boot-actuators")] pub fn deliveries(cpu: u32, source: Source) -> Option { read(cpu).map(|counts| counts[1 + source as usize]) } diff --git a/kernel/src/loader/mod.rs b/kernel/src/loader/mod.rs index 2526058146b..1e0484ded5c 100644 --- a/kernel/src/loader/mod.rs +++ b/kernel/src/loader/mod.rs @@ -886,7 +886,9 @@ pub fn spawn_supervisor() -> Pid { .union(Rights::WAIT) .union(Rights::POWER) .union(Rights::ROSTER) - .union(Rights::INVENTORY); + .union(Rights::INVENTORY) + .union(Rights::COUNTERS) + .union(Rights::TRACE); let cap_handle = handles .install(crate::object::HandleEntry::new(cap, rights)) .expect("spawn_supervisor: an empty table refused the system capability"); diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 5b38375ff06..4a1ce8185ab 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -13,6 +13,7 @@ static DEBUG_WAIT: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBo pub use mm::{UserAddr, DirectMap, PHYS_OFFSET}; mod invalidation; +mod seqlock; mod shootdown; mod sleeplock; mod smp; @@ -70,6 +71,7 @@ mod sched; mod hw; mod iommu; mod preempt; +mod counters; mod irq_census; #[cfg(feature = "mask-windows")] mod windows; @@ -396,6 +398,7 @@ pub(crate) unsafe extern "C" fn kernel_main(kernel_args: &KernelArgs) -> ! { acpi::inventory(kernel_args.rsdp_addr); let platform = arch::boot::interrupts(kernel_args.rsdp_addr); + counters::bring_up(); symbols::set_kernel_base(kernel_args.kernel_memory_addr); if !kernel_elf.is_empty() { symbols::load_kernel(kernel_elf, mm::PHYS_OFFSET + kernel_args.kernel_memory_addr); diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 8788e7bfd86..f8a9ec93d6f 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -1801,6 +1801,8 @@ pub fn ap_idle() -> ! { // invalidation the wait could not take. crate::arch::tlb::join(); + crate::counters::bring_up(); + // Once this CPU is committed and about to run something: the counter is // per CPU, so a CPU nobody arms here is one the hard-lockup bound does not cover. crate::hardlockup::arm_this_cpu(); diff --git a/kernel/src/quiesce.rs b/kernel/src/quiesce.rs index 7a36071ff03..55a7cf83086 100644 --- a/kernel/src/quiesce.rs +++ b/kernel/src/quiesce.rs @@ -147,7 +147,6 @@ pub fn stop() -> Record { let parkable = crate::scheduler::Parkable::at_entry(); let armed = watch::arm(&PROGRESS, 0, WaitClass::Other) .expect("quiesce::stop: the caller holds no task to park"); - // The kick is the timer vector, whose return to Ring 3 is the gate. crate::arch::irqchip::kick_all_but_self(); let cpus = crate::smp::cpu_count(); diff --git a/kernel/src/drivers/panic_console/published.rs b/kernel/src/seqlock.rs similarity index 68% rename from kernel/src/drivers/panic_console/published.rs rename to kernel/src/seqlock.rs index 44678a9c1d2..93520255d00 100644 --- a/kernel/src/drivers/panic_console/published.rs +++ b/kernel/src/seqlock.rs @@ -1,7 +1,7 @@ -//! The published framebuffer descriptor: a seqlock whose payload is atomic -//! words, so a reader racing the one publisher reads torn words it throws -//! away rather than racing a plain write (a data race, and undefined, in the -//! Rust memory model whatever the check after it concludes). +//! A value one publisher replaces while others read it with no lock: a seqlock +//! whose payload is atomic words, so a reader racing the publisher reads torn +//! words it throws away rather than racing a plain write (a data race, and +//! undefined, in the Rust memory model whatever the check after it concludes). //! //! The writer marks the sequence odd, fences `Release`, stores the words //! `Relaxed` and marks it even with a `Release` store; the reader loads the @@ -16,22 +16,19 @@ use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; #[cfg(feature = "loom")] use loom::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; -/// Words in one descriptor. -pub const WORDS: usize = 4; - -/// Tries a snapshot makes before answering that the descriptor is changing. +/// Tries a snapshot makes before answering that the value is changing. const TRIES: usize = 4; -pub struct Published { +pub struct Published { seq: AtomicU32, - words: [AtomicU64; WORDS], + words: [AtomicU64; N], } -impl Published { - // Must stay `const`: the kernel's is a `static`, and loom's atomics have no const constructor. +impl Published { + // Must stay `const`: the kernel's are `static`s, and loom's atomics have no const constructor. #[cfg(not(feature = "loom"))] pub const fn new() -> Self { - Self { seq: AtomicU32::new(0), words: [const { AtomicU64::new(0) }; WORDS] } + Self { seq: AtomicU32::new(0), words: [const { AtomicU64::new(0) }; N] } } #[allow(clippy::new_without_default)] @@ -40,10 +37,11 @@ impl Published { Self { seq: AtomicU32::new(0), words: core::array::from_fn(|_| AtomicU64::new(0)) } } - /// Replace the descriptor. One publisher at a time: the caller's own - /// exclusion (the boot sequence, a mode set's single-CPU window) is what - /// makes the load-then-store of `seq` sound. - pub fn publish(&self, words: [u64; WORDS]) { + /// Replace the value. One publisher at a time: the caller's own exclusion + /// (the boot sequence, a mode set's single-CPU window, the one CPU a + /// block belongs to with its interrupts closed) is what makes the + /// load-then-store of `seq` sound. + pub fn publish(&self, words: [u64; N]) { let seq = self.seq.load(Ordering::Relaxed); self.seq.store(seq.wrapping_add(1), Ordering::Relaxed); // Orders the odd mark before every word below: a reader that sees a @@ -56,9 +54,9 @@ impl Published { self.seq.store(seq.wrapping_add(2), Ordering::Release); } - /// The descriptor, whole, or `None` when every try met a publication in + /// The value, whole, or `None` when every try met a publication in /// progress. - pub fn snapshot(&self) -> Option<[u64; WORDS]> { + pub fn snapshot(&self) -> Option<[u64; N]> { for _ in 0..TRIES { let before = self.seq.load(Ordering::Acquire); if before & 1 != 0 { diff --git a/kernel/src/shootdown.rs b/kernel/src/shootdown.rs index 8a69e30b65c..4443e820f28 100644 --- a/kernel/src/shootdown.rs +++ b/kernel/src/shootdown.rs @@ -1,6 +1,8 @@ -//! The acknowledgement half of a TLB shootdown, with no hardware in it. +//! A generation one CPU issues and every other acknowledges, with no hardware +//! in it: a TLB shootdown's, and a counters read's round. //! Compiled a second time into `kernel/loom/` against loom's atomics, so this file must hold no `crate::` references. -//! The read must happen before the flush, or a target could publish a generation its flush has not yet completed. +//! The read must happen before the answer, or a target could publish a generation its answer does not cover; +//! and an initiator that sees a generation served sees everything that answer wrote. #[cfg(not(feature = "loom"))] use core::sync::atomic::{AtomicU64, Ordering}; @@ -18,6 +20,13 @@ const OWED: Ordering = Ordering::Acquire; #[cfg(feature = "shootdown-serve-relaxed")] const OWED: Ordering = Ordering::Relaxed; +// Acquire: pairs with `serve`'s Release, so what its closure wrote is visible once a generation reads served. +#[cfg(not(feature = "shootdown-served-relaxed"))] +const SERVED: Ordering = Ordering::Acquire; +// Relaxed arm: kernel-loom's negative-control mutation, never selected by a kernel build. +#[cfg(feature = "shootdown-served-relaxed")] +const SERVED: Ordering = Ordering::Relaxed; + /// Which shootdown a flush answers for: monotonic and machine-wide, so one target's flush answers every initiator waiting on it. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] pub struct Generation(u64); @@ -70,15 +79,16 @@ impl Shootdown { /// Has `cpu` flushed since `generation` was issued? pub fn served(&self, cpu: usize, generation: Generation) -> bool { - // Acquire: nothing reads through this edge yet, but `Relaxed` here would be silently unsafe once something does. - self.flushed[cpu].load(Ordering::Acquire) >= generation.0 + self.flushed[cpu].load(SERVED) >= generation.0 } - /// [`serve`](Self::serve) for a CPU not taking the interrupt. - pub fn serve_if_owed(&self, cpu: usize, flush: impl FnOnce()) { - if self.owes(cpu) { + /// [`serve`](Self::serve) for a CPU not taking the interrupt; whether it served. + pub fn serve_if_owed(&self, cpu: usize, flush: impl FnOnce()) -> bool { + let owed = self.owes(cpu); + if owed { self.serve(cpu, flush); } + owed } /// One turn of an initiator's wait for `cpu`: answer first, then ask. diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 3764111a12c..d2f3ef0c66a 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -41,7 +41,7 @@ use super::ipc::{ sys_port_create, sys_shm_create, sys_shm_map, }; use super::machine::{ - sys_device_inventory, sys_log_read, sys_reboot, sys_sched_info, sys_shutdown, sys_sysinfo, + sys_counters, sys_device_inventory, sys_log_read, sys_reboot, sys_sched_info, sys_shutdown, sys_sysinfo, MAX_INVENTORY_RECORDS, }; #[cfg(feature = "test-actuators")] @@ -280,6 +280,16 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a2), len) else { return bad_addr }; sys_device_inventory(RawHandle(a1 as u32), &mut buf) } + // What every CPU counts, so it costs Rights::COUNTERS; a CPU per record, + // so the count is bounded by the most CPUs there can be. + SYS_COUNTERS => { + let Ok(count) = Untrusted::new(a3).at_most(crate::scheduler::MAX_CPUS as u64) else { + return SyscallError::InvalidArgument.to_u64(); + }; + let len = count * toyos_abi::counters::RECORD_BYTES as u64; + let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a2), len) else { return bad_addr }; + sys_counters(RawHandle(a1 as u32), &mut buf) + } SYS_NANOSLEEP => sys_nanosleep(a1), SYS_HANDLE_DUP => sys_handle_dup(RawHandle(a1 as u32), a2), SYS_HANDLE_DUP_AT => sys_dup2(RawHandle(a1 as u32), a2), @@ -592,6 +602,10 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> process::debug_mark_spawn_hold(); 0 } + // A CPU silent past a round's bound, which a guest cannot make on demand: the reader's + // bound and its stale answer are the shipped paths. + DA::COUNTERS_DEAF => crate::counters::deaf::stage(a2), + DA::COUNTERS_HEAR => crate::counters::deaf::end(), _ => SyscallError::InvalidArgument.to_u64(), }, SYS_SCHED_INFO => match ctx.copy_out(UserAddr::new(a1), &sys_sched_info()) { diff --git a/kernel/src/syscall/handles.rs b/kernel/src/syscall/handles.rs index eeb19cdae5d..2b9decbebca 100644 --- a/kernel/src/syscall/handles.rs +++ b/kernel/src/syscall/handles.rs @@ -17,12 +17,12 @@ pub(super) fn with_object_ref( process::with_process_data(|data| data.handles.get_ref(h, need).map(f)) } -/// Requires that `syscap` resolves to a `SysCap` holding exactly `need`. -pub(super) fn demand_syscap(syscap: RawHandle, need: Rights) -> Result<(), crate::object::HandleError> { +/// Requires that `syscap` resolves to a `SysCap` holding `need`, and answers +/// every right it holds, read under the same guard. +pub(super) fn demand_syscap(syscap: RawHandle, need: Rights) -> Result { process::with_process_data(|data| { - data.handles - .get::(syscap, need) - .map(|_| ()) + data.handles.get::(syscap, need)?; + data.handles.rights_of(syscap) }) } diff --git a/kernel/src/syscall/machine.rs b/kernel/src/syscall/machine.rs index 4c2009f7492..df469aa4150 100644 --- a/kernel/src/syscall/machine.rs +++ b/kernel/src/syscall/machine.rs @@ -275,6 +275,20 @@ pub(super) fn sys_sched_info() -> toyos_abi::syscall::SchedInfo { } } +/// Every online CPU's counters into `out`, or the CPU count when `out` is +/// empty: requires a `SysCap` carrying `Rights::COUNTERS`, and answers the +/// counters that time programs only where it carries `Rights::TRACE` too. +pub(super) fn sys_counters(syscap: RawHandle, out: &mut UserBytesMut) -> u64 { + let rights = match demand_syscap(syscap, Rights::COUNTERS) { + Ok(rights) => rights, + Err(e) => return e.refuse(), + }; + match crate::counters::read(rights, out) { + Ok(records) => records as u64, + Err(e) => e.to_u64(), + } +} + /// The most records one `SYS_DEVICE_INVENTORY` buffer may be declared to hold, /// 64 KiB of them: a declared length past it is refused before it becomes a /// window, and a machine with more records than this is refused by name rather diff --git a/kernel/src/watch.rs b/kernel/src/watch.rs index 4e8dc035580..2c9ffab73a1 100644 --- a/kernel/src/watch.rs +++ b/kernel/src/watch.rs @@ -144,7 +144,8 @@ impl IrqWatch { /// Something about the object changed: wake every thread waiting on it and /// fire every poll where it stands, freeing nothing. A handler makes it /// with its CPU's preempt count raised, as `device_irq_entry` holds it, so - /// this post's own never reaches zero, and a pass, inside the interrupt. + /// this post's own never reaches zero, and a pass, inside the interrupt; a + /// thread's may reach zero and pass, as any post's does. pub fn post_in_place(&self) { preempt_off(|p| { let env = Poster { cpus: cpus(), kicker: &HW, preempt: p }; diff --git a/src/ci.rs b/src/ci.rs index d25e1412c23..f69ee1a8440 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -264,6 +264,9 @@ pub(crate) const CONTROLS: &[Control] = &[ Fails("an_acknowledged_flush_postdates_the_page_table_write"), Fails("one_serve_answers_two_concurrent_shootdowns"), ]), + red(KERNEL_LOOM, "shootdown-served-relaxed", Some("counters_round"), &[ + Fails("a_cpu_read_as_answered_is_read_with_its_answer"), + ]), red(KERNEL_LOOM, "roster-commit-relaxed", Some("smp_bringup"), &[ Fails("a_committed_count_never_outruns_its_slot"), ]), diff --git a/src/metalimage.rs b/src/metalimage.rs index 31a0ce94e14..83e91f7b84b 100644 --- a/src/metalimage.rs +++ b/src/metalimage.rs @@ -176,9 +176,9 @@ mod tests { .iter() .map(|v| v.as_str().unwrap()) .collect(); - // `tests/testcases` declares all five, and the derivation takes none + // `tests/testcases` declares all seven, and the derivation takes none // away and spells none twice; nor the connector it needs, declared too. - for right in ["device", "dup", "logread", "power", "roster"] { + for right in ["device", "dup", "logread", "power", "roster", "counters", "trace"] { assert_eq!( syscap.iter().filter(|r| **r == right).count(), 1, diff --git a/system.toml b/system.toml index 7ad1913158f..ee8411d2bf3 100644 --- a/system.toml +++ b/system.toml @@ -99,11 +99,12 @@ slots = true receives = ["netstack"] # The reader of every owner's `inspect` answer: a connector to each owner's -# port, and `inventory` for the kernel's `dev.*`, are the whole of what it can -# observe. +# port, `inventory` for the kernel's `dev.*`, and `counters` and `trace` for its +# `kernel.*`, are the whole of what it can observe. `trace` makes it an admin +# tool: every CPU's frequency and wake-ups are what time other programs. [programs.inspect] receives = ["netstack", "soundserver", "log", "compositor"] -syscap = ["inventory"] +syscap = ["inventory", "counters", "trace"] [programs.filepicker] service = true diff --git a/tests/checks.rs b/tests/checks.rs index db34f7fc875..39aee3dbae9 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -410,7 +410,7 @@ mod checks { use common::irqcensus::{windows_under, Measured}; let census = |cpu: u32| { format!( - "[kernel 0.1 cpu0] irq: cpu{cpu} total=0 timer=0 xhci=0 userdev=0 sound=0 i8042=0 \ + "[kernel 0.1 cpu0] irq: cpu{cpu} total=0 timer=0 kick=0 xhci=0 userdev=0 sound=0 i8042=0 \ dmafault=0 hda=0 tlb=0 nmi=0 spurious=0 unclaimed=0\n" ) }; diff --git a/tests/common/irqcensus.rs b/tests/common/irqcensus.rs index d3401be45aa..656bc7d50a0 100644 --- a/tests/common/irqcensus.rs +++ b/tests/common/irqcensus.rs @@ -23,8 +23,8 @@ use std::sync::Mutex; /// and [`Census::parse`] refuses a line whose fields are not exactly these, so /// a source added on one side and not the other is a red rather than a silently /// dropped column. -pub const SOURCES: [&str; 11] = [ - "timer", "xhci", "userdev", "sound", "i8042", "dmafault", "hda", "tlb", "nmi", "spurious", +pub const SOURCES: [&str; 12] = [ + "timer", "kick", "xhci", "userdev", "sound", "i8042", "dmafault", "hda", "tlb", "nmi", "spurious", "unclaimed", ]; diff --git a/tests/t14-linux/SOURCE b/tests/t14-linux/SOURCE new file mode 100644 index 00000000000..d59763d1e66 --- /dev/null +++ b/tests/t14-linux/SOURCE @@ -0,0 +1,14 @@ +Linux's turbostat on the ThinkPad T14 the metal suite runs on, under that +machine's Ubuntu (`6.8.0-142-generic`, the `powersave` governor), taken as +root by the orchestrator on 2026-10-03, one after the other, each as the tool +printed it; `counters_on_metal` in `tests/toyos.rs` prints ToyOS's reading +beside them. + + turbostat-idle.txt 60 s of an idle machine: + `turbostat --quiet --interval 10 --num_iterations 6 --show + Core,CPU,Avg_MHz,Busy%,Bzy_MHz,TSC_MHz,IRQ,SMI,CPU%c1,CPU%c6,CPU%c7,CoreTmp,PkgTmp,Pkg%pc2,Pkg%pc3,Pkg%pc6,Pkg%pc7,Pkg%pc8,Pkg%pc9,Pk%pc10,PkgWatt,CorWatt` + turbostat-loaded.txt 60 s with one `yes` per CPU, started 2 s before: + `for i in 1 2 3 4 5 6 7 8; do timeout 65 sh -c "yes > /dev/null" & done; + sleep 2; turbostat --quiet --interval 10 --num_iterations 6 --show + Core,CPU,Avg_MHz,Busy%,Bzy_MHz,TSC_MHz,IRQ,SMI,CPU%c1,CPU%c6,CPU%c7,CoreTmp,PkgTmp,Pkg%pc2,Pkg%pc6,Pk%pc10,PkgWatt,CorWatt; + wait` diff --git a/tests/t14-linux/turbostat-idle.txt b/tests/t14-linux/turbostat-idle.txt new file mode 100644 index 00000000000..298df608572 --- /dev/null +++ b/tests/t14-linux/turbostat-idle.txt @@ -0,0 +1,60 @@ +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt +- - 1 0.11 994 2419 1160 0 0.47 0.00 99.98 29 31 17.46 2.01 0.03 0.29 77.22 0.00 0.00 1.30 0.07 +0 0 1 0.11 1012 2419 131 0 0.12 0.00 100.42 26 31 17.46 2.01 0.03 0.29 77.22 0.00 0.00 1.30 0.07 +0 4 0 0.05 910 2419 53 0 0.12 +1 1 1 0.08 986 2419 57 0 0.30 0.00 100.18 27 +1 5 1 0.12 999 2419 119 0 0.30 +2 2 1 0.13 933 2419 423 0 1.29 0.00 99.04 29 +2 6 2 0.19 1002 2419 157 0 1.29 +3 3 1 0.14 978 2419 148 0 0.16 0.00 100.27 28 +3 7 1 0.09 1115 2419 72 0 0.16 +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt +- - 1 0.11 980 2419 1009 0 0.38 0.00 100.08 29 30 19.01 1.80 0.05 0.22 76.10 0.00 0.00 1.81 0.07 +0 0 1 0.07 975 2419 46 0 0.14 0.00 100.37 26 30 19.01 1.80 0.05 0.22 76.10 0.00 0.00 1.81 0.07 +0 4 1 0.11 913 2419 72 0 0.14 +1 1 0 0.03 997 2419 23 0 0.19 0.00 100.34 25 +1 5 1 0.13 993 2419 130 0 0.19 +2 2 1 0.11 943 2419 386 0 1.14 0.00 99.27 29 +2 6 2 0.18 1035 2419 155 0 1.14 +3 3 1 0.14 967 2419 141 0 0.06 0.00 100.36 27 +3 7 1 0.09 997 2419 56 0 0.06 +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt +- - 1 0.11 1005 2419 1125 0 0.42 0.00 100.03 29 31 19.47 1.35 0.06 0.23 75.87 0.00 0.00 1.43 0.07 +0 0 1 0.12 1081 2419 42 0 0.08 0.00 100.39 26 31 19.47 1.35 0.06 0.23 75.87 0.00 0.00 1.43 0.07 +0 4 1 0.13 998 2419 56 0 0.08 +1 1 1 0.10 1006 2419 127 0 0.22 0.00 100.21 25 +1 5 1 0.14 1004 2419 132 0 0.22 +2 2 2 0.16 981 2419 513 0 1.33 0.00 99.12 29 +2 6 1 0.08 1028 2419 75 0 1.33 +3 3 1 0.13 953 2419 137 0 0.05 0.00 100.42 28 +3 7 1 0.06 1019 2419 43 0 0.05 +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt +- - 6 0.36 1668 2419 1966 0 1.14 0.00 98.78 29 32 18.49 1.12 0.04 0.33 75.47 0.00 0.00 1.14 0.13 +0 0 7 0.43 1587 2419 231 0 0.90 0.00 99.10 26 32 18.49 1.12 0.04 0.33 75.47 0.00 0.00 1.14 0.13 +0 4 2 0.15 994 2419 96 0 0.90 +1 1 11 0.48 2410 2419 185 0 1.08 0.00 98.67 27 +1 5 7 0.46 1450 2419 348 0 1.08 +2 2 8 0.48 1564 2419 635 0 1.97 0.00 98.11 29 +2 6 1 0.08 1040 2419 72 0 1.97 +3 3 12 0.69 1725 2419 326 0 0.60 0.00 99.24 28 +3 7 1 0.08 1053 2419 73 0 0.60 +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt +- - 1 0.12 984 2419 1166 0 0.39 0.00 100.01 29 31 13.62 2.79 0.04 0.39 80.55 0.00 0.00 1.30 0.07 +0 0 0 0.05 957 2419 34 0 0.09 0.00 100.41 26 31 13.62 2.79 0.04 0.39 80.55 0.00 0.00 1.30 0.07 +0 4 1 0.13 943 2419 78 0 0.09 +1 1 1 0.05 982 2419 47 0 0.36 0.00 100.00 27 +1 5 2 0.23 1023 2419 186 0 0.36 +2 2 1 0.05 1001 2419 45 0 0.09 0.00 100.38 29 +2 6 2 0.15 989 2419 135 0 0.09 +3 3 2 0.22 959 2419 535 0 1.04 0.00 99.26 28 +3 7 1 0.10 994 2419 106 0 1.04 +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc3 Pkg%pc6 Pkg%pc7 Pkg%pc8 Pkg%pc9 Pk%pc10 PkgWatt CorWatt +- - 1 0.12 980 2419 1156 0 0.39 0.00 100.03 28 31 13.35 1.53 0.07 0.41 82.10 0.00 0.00 1.88 0.07 +0 0 1 0.05 958 2419 40 0 0.16 0.00 100.38 26 31 13.35 1.53 0.07 0.41 82.10 0.00 0.00 1.88 0.07 +0 4 1 0.11 947 2419 82 0 0.16 +1 1 1 0.09 989 2419 76 0 0.26 0.00 100.09 26 +1 5 2 0.19 990 2419 174 0 0.26 +2 2 1 0.09 965 2419 335 0 0.96 0.00 99.42 28 +2 6 2 0.19 995 2419 178 0 0.96 +3 3 2 0.19 976 2419 242 0 0.18 0.00 100.24 28 +3 7 0 0.04 1021 2419 29 0 0.18 diff --git a/tests/t14-linux/turbostat-loaded.txt b/tests/t14-linux/turbostat-loaded.txt new file mode 100644 index 00000000000..df181dbeeb5 --- /dev/null +++ b/tests/t14-linux/turbostat-loaded.txt @@ -0,0 +1,60 @@ +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt +- - 3791 99.77 3800 2419 80691 0 0.00 0.00 0.00 71 71 0.00 0.00 0.00 30.24 27.88 +0 0 3791 99.77 3800 2419 10023 0 0.00 0.00 0.00 61 71 0.00 0.00 0.00 30.24 27.88 +0 4 3791 99.77 3800 2419 10021 0 0.00 +1 1 3791 99.77 3800 2419 10066 0 0.00 0.00 0.00 71 +1 5 3791 99.77 3800 2419 10102 0 0.00 +2 2 3791 99.77 3800 2419 10031 0 0.00 0.00 0.00 62 +2 6 3791 99.77 3800 2419 10274 0 0.00 +3 3 3791 99.77 3800 2419 10125 0 0.00 0.00 0.00 71 +3 7 3791 99.77 3800 2419 10049 0 0.00 +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt +- - 3791 99.77 3800 2419 80659 0 0.00 0.00 0.00 79 79 0.00 0.00 0.00 30.73 28.29 +0 0 3791 99.77 3800 2419 10012 0 0.00 0.00 0.00 68 79 0.00 0.00 0.00 30.73 28.29 +0 4 3791 99.77 3800 2419 10014 0 0.00 +1 1 3791 99.77 3800 2419 10026 0 0.00 0.00 0.00 79 +1 5 3791 99.77 3800 2419 10095 0 0.00 +2 2 3791 99.77 3800 2419 10017 0 0.00 0.00 0.00 69 +2 6 3791 99.77 3800 2419 10364 0 0.00 +3 3 3791 99.77 3800 2419 10122 0 0.00 0.00 0.00 78 +3 7 3791 99.77 3800 2419 10009 0 0.00 +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt +- - 3547 99.77 3555 2419 80555 0 0.00 0.00 0.00 66 66 0.00 0.00 0.00 27.20 24.72 +0 0 3547 99.77 3555 2419 10021 0 0.00 0.00 0.00 58 66 0.00 0.00 0.00 27.20 24.72 +0 4 3547 99.77 3555 2419 10010 0 0.00 +1 1 3547 99.77 3555 2419 10028 0 0.00 0.00 0.00 65 +1 5 3547 99.77 3555 2419 10037 0 0.00 +2 2 3547 99.77 3555 2419 10028 0 0.00 0.00 0.00 60 +2 6 3547 99.77 3555 2419 10246 0 0.00 +3 3 3547 99.77 3555 2419 10173 0 0.00 0.00 0.00 66 +3 7 3547 99.77 3555 2419 10012 0 0.00 +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt +- - 3068 99.77 3075 2419 80909 0 0.00 0.00 0.00 65 65 0.00 0.00 0.00 19.62 17.18 +0 0 3068 99.77 3075 2419 10054 0 0.00 0.00 0.00 59 65 0.00 0.00 0.00 19.62 17.18 +0 4 3068 99.77 3075 2419 10029 0 0.00 +1 1 3068 99.77 3075 2419 10121 0 0.00 0.00 0.00 65 +1 5 3068 99.77 3075 2419 10039 0 0.00 +2 2 3068 99.77 3075 2419 10069 0 0.00 0.00 0.00 59 +2 6 3068 99.77 3075 2419 10276 0 0.00 +3 3 3068 99.77 3075 2419 10229 0 0.00 0.00 0.00 64 +3 7 3068 99.76 3075 2419 10092 0 0.00 +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt +- - 3083 99.77 3090 2419 80593 0 0.00 0.00 0.00 65 65 0.00 0.00 0.00 19.85 17.43 +0 0 3083 99.77 3090 2419 10008 0 0.00 0.00 0.00 58 65 0.00 0.00 0.00 19.85 17.43 +0 4 3083 99.77 3090 2419 10020 0 0.00 +1 1 3083 99.77 3090 2419 10006 0 0.00 0.00 0.00 64 +1 5 3083 99.77 3090 2419 10019 0 0.00 +2 2 3083 99.77 3090 2419 10028 0 0.00 0.00 0.00 60 +2 6 3083 99.77 3090 2419 10282 0 0.00 +3 3 3083 99.77 3090 2419 10221 0 0.00 0.00 0.00 65 +3 7 3083 99.77 3090 2419 10009 0 0.00 +Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz IRQ SMI CPU%c1 CPU%c6 CPU%c7 CoreTmp PkgTmp Pkg%pc2 Pkg%pc6 Pk%pc10 PkgWatt CorWatt +- - 3087 99.77 3094 2419 80544 0 0.00 0.00 0.00 65 65 0.00 0.00 0.00 19.92 17.49 +0 0 3087 99.77 3094 2419 10009 0 0.00 0.00 0.00 58 65 0.00 0.00 0.00 19.92 17.49 +0 4 3087 99.77 3094 2419 10011 0 0.00 +1 1 3087 99.77 3094 2419 10012 0 0.00 0.00 0.00 65 +1 5 3087 99.77 3094 2419 10009 0 0.00 +2 2 3087 99.77 3094 2419 10017 0 0.00 0.00 0.00 59 +2 6 3087 99.77 3094 2419 10273 0 0.00 +3 3 3087 99.77 3094 2419 10205 0 0.00 0.00 0.00 65 +3 7 3087 99.77 3094 2419 10008 0 0.00 diff --git a/tests/testcases/system.toml b/tests/testcases/system.toml index c0b5e30efff..bd88580d82c 100644 --- a/tests/testcases/system.toml +++ b/tests/testcases/system.toml @@ -36,9 +36,11 @@ syscap = ["rt"] # the same machine-wide answer and have no narrower question in the ABI. It is # also what `endowment_denied` narrows *away* to prove the refusal, so an estate # without it would make that arm vacuous rather than red. +# `counters` and `trace` because `counters_read` reads every counter there is, +# and narrows each away to prove its refusal. [programs.test-runner] receives = ["soundserver", "power"] -syscap = ["device", "dup", "logread", "power", "roster"] +syscap = ["device", "dup", "logread", "power", "roster", "counters", "trace"] [programs.toybox] receives = ["soundserver"] diff --git a/tests/toyos-rust-tests/src/bin/counters_metal.rs b/tests/toyos-rust-tests/src/bin/counters_metal.rs new file mode 100644 index 00000000000..a60790f2e2f --- /dev/null +++ b/tests/toyos-rust-tests/src/bin/counters_metal.rs @@ -0,0 +1,56 @@ +//! Every CPU's counters across a span of an idle machine and then across every +//! CPU spinning, printed as `inspect kernel.*` renders them, for the `counters` +//! metal row to judge (`tests/toyos.rs`). It asserts nothing: frequency, busy +//! fraction and firmware interrupts are the hardware's to say. +//! +//! Three reads: `idle0` and `idle1` either side of [`IDLE`], and `spin` after +//! [`SPIN`] iterations on a thread per CPU begun at `idle1`. Each read's +//! records follow a line with the clock after it and what the read took, a +//! whole round each, none joining another's. + +use std::time::{Duration, Instant}; + +use toyos::endow::{Endowments, SYSCAP_LABEL}; +use toyos::syscap::SysCap; +use toyos_abi::counters::{RawRecord, Record}; +use toyos_abi::syscall; + +/// The idle span: long enough that a CPU's busy fraction is its idle one and +/// not the reads'. +const IDLE: Duration = Duration::from_secs(1); + +/// Iterations of a dependent multiply-add per thread, about four cycles each: +/// over four seconds of a CPU at the T14's highest frequency, so `idle0` and +/// `spin` span more than twice its firmware's 2.2 s interrupt period whatever +/// the frequency. +const SPIN: u64 = 4_000_000_000; + +fn read(cap: &SysCap, phase: &str) { + let mut raw = vec![RawRecord::EMPTY; syscall::cpu_count() as usize]; + let asked = Instant::now(); + let n = cap.counters(&mut raw).expect("the estate's capability reads the counters"); + let took = asked.elapsed(); + println!("counters_metal {phase}: at {} ns, the read took {} ns", toyos_abi::clock::nanos_since_boot(), took.as_nanos()); + let records: Vec = raw[..n].iter().map(|r| Record::decode(r).expect("a record that decodes")).collect(); + for (path, value) in toyos_inspect::kernel::render(&records).expect("one record per cpu") { + println!("counters_metal {phase}: {}", toyos_inspect::line(&path, &value)); + } +} + +fn main() { + let cap: SysCap = Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows a capability"); + read(&cap, "idle0"); + std::thread::sleep(IDLE); + read(&cap, "idle1"); + std::thread::scope(|s| { + for _ in 0..syscall::cpu_count() { + s.spawn(|| { + let mut x = 1u64; + for _ in 0..SPIN { + x = std::hint::black_box(x.wrapping_mul(6364136223846793005).wrapping_add(1)); + } + }); + } + }); + read(&cap, "spin"); +} diff --git a/tests/toyos-rust-tests/src/bin/counters_read.rs b/tests/toyos-rust-tests/src/bin/counters_read.rs new file mode 100644 index 00000000000..8bb10c0b8e8 --- /dev/null +++ b/tests/toyos-rust-tests/src/bin/counters_read.rs @@ -0,0 +1,168 @@ +//! `SYS_COUNTERS`: every online CPU answers a read for itself, and what it +//! answers is a right's. +//! +//! **A guest, because the subject is the kick and the syscall.** A read is a +//! round of IPIs, each answered in the kick handler of the CPU it reached, +//! and the refusals are the syscall boundary's; no host test reaches either. +//! Nothing here is timed: the only clock is the harness's hang ceiling, which +//! a read waiting without its bound reaches. +//! +//! - The count: an empty buffer is answered the CPU count, which +//! `SYS_CPU_COUNT` says too. +//! - A round every CPU answered, read until there is one: each CPU's record +//! is in its place, carries its stamp, and names a hardware id no other +//! does — a reader that filled every block itself would name its own on +//! each. The next round: every stamp moved on, and every CPU but at most +//! the reader's own took a kick in between, which a kick counted as anything +//! else would not show. +//! - The rights: on `COUNTERS` alone the counters that time programs are +//! absent and the rest as before; without it, and on a handle that is no +//! capability, `PermissionDenied`; a buffer short of a CPU +//! `ResourceExhausted`, one longer than any machine `InvalidArgument`; and +//! a handle nobody holds ends the caller. +//! - Many readers at once, more than there are CPUs: every read returns. + +use std::io::Write; +use std::process::{Command, Stdio}; +use std::sync::OnceLock; + +use toyos::endow::{Endowments, SYSCAP_LABEL}; +use toyos::syscap::SysCap; +use toyos_abi::counters::{Counter, RawRecord, Record}; +use toyos_abi::handle::{Rights, HANDLE_INVALID}; +use toyos_abi::syscall::{self, SyscallError}; +use toyos_abi::RawHandle; + +const SELF_PATH: &str = "/system/bin/test_rs_counters_read"; + +/// `process::HANDLE_FAULT_EXIT_CODE`. +const HANDLE_FAULT: i32 = 139; + +/// Reads each of the many readers makes. +const READS: usize = 50; + +/// What the harness waits for on a guest whose console is all it reads. +const SAID: &str = "counters_read: every cpu answered for itself, and each counter is a right's"; + +fn main() { + if std::env::args().nth(1).as_deref() == Some("unheld") { + println!("reached unheld"); + std::io::stdout().flush().expect("flush the marker"); + let refused = syscall::counters(HANDLE_INVALID, &mut [RawRecord::EMPTY]); + panic!("a handle nobody holds was answered {refused:?} instead of ending the caller"); + } + let cpus = syscall::cpu_count() as usize; + assert_eq!(cap().counters(&mut []), Ok(cpus), "the empty buffer's count"); + + let first = answered(cap()); + let second = loop { + let next = answered(cap()); + // Equal stamps are the same round, joined; a later one is the next. + if next.iter().zip(&first).any(|(n, f)| n.get(Counter::Stamp) != f.get(Counter::Stamp)) { + break next; + } + }; + for (cpu, (f, s)) in first.iter().zip(&second).enumerate() { + assert!(s.get(Counter::Stamp) > f.get(Counter::Stamp), "cpu{cpu}'s stamp did not move: {f:?} then {s:?}"); + assert_eq!(f.hardware_id, s.hardware_id, "cpu{cpu} named two hardware ids"); + } + let kicked = first.iter().zip(&second).filter(|(f, s)| s.get(Counter::Kicks) > f.get(Counter::Kicks)).count(); + assert!(kicked + 1 >= cpus, "{kicked} of {cpus} cpus took a kick between two rounds: {first:?} then {second:?}"); + + rights(&second); + many_readers(cpus); + unheld(); + println!("{SAID}"); +} + +/// The estate's capability, taken once: taking an endowment is a swap. +fn cap() -> &'static SysCap { + static CAP: OnceLock = OnceLock::new(); + CAP.get_or_init(|| Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows a capability")) +} + +fn read(cap: &SysCap) -> Result, SyscallError> { + let mut raw = vec![RawRecord::EMPTY; syscall::cpu_count() as usize]; + let n = cap.counters(&mut raw)?; + assert_eq!(n, raw.len(), "a read answered {n} of {} cpus", raw.len()); + Ok(raw.iter().map(|r| Record::decode(r).expect("the kernel wrote a record that decodes")).collect()) +} + +/// The first read in which every CPU answered, held to what one must say. +fn answered(cap: &SysCap) -> Vec { + loop { + let records = read(cap).expect("the estate's capability reads the counters"); + if records.iter().any(|r| r.stale) { + continue; + } + for (cpu, r) in records.iter().enumerate() { + assert_eq!(r.cpu as usize, cpu, "record {cpu} names cpu{}", r.cpu); + assert!(r.get(Counter::Stamp).is_some(), "cpu{cpu} answered with no stamp: {r:?}"); + assert!(r.get(Counter::Kicks).is_some(), "the estate holds TRACE and cpu{cpu}'s kicks are absent"); + let ids = records.iter().filter(|o| o.hardware_id == r.hardware_id).count(); + assert_eq!(ids, 1, "{ids} cpus named hardware id {}: {records:?}", r.hardware_id); + } + return records; + } +} + +fn rights(full: &[Record]) { + let counters_only = cap().narrowed(Rights::TRANSFER.union(Rights::COUNTERS)).expect("a cap carrying COUNTERS"); + let narrow = read(&counters_only).expect("COUNTERS alone reads the counters"); + for (r, f) in narrow.iter().zip(full) { + for counter in Counter::ALL { + let answered = r.get(counter).is_some(); + if counter.needs().contains(Rights::TRACE) { + assert!(!answered, "COUNTERS alone was answered cpu{}'s {}", r.cpu, counter.name()); + } else if !r.stale { + assert_eq!(answered, f.get(counter).is_some(), "cpu{}'s {} on COUNTERS alone", r.cpu, counter.name()); + } + } + } + + let none = cap().narrowed(Rights::TRANSFER).expect("a cap carrying less"); + assert_eq!(none.counters(&mut []), Err(SyscallError::PermissionDenied), "a cap without COUNTERS counted the cpus"); + assert_eq!(read(&none).err(), Some(SyscallError::PermissionDenied), "a cap without COUNTERS read the counters"); + let mut buf = vec![RawRecord::EMPTY; full.len()]; + assert_eq!( + syscall::counters(RawHandle(1), &mut buf), + Err(SyscallError::PermissionDenied), + "a console was taken as a capability", + ); + // The demand is above every write, so a refusal leaves the buffer as it was. + assert!(buf.iter().all(|r| *r == RawRecord::EMPTY), "a refused read wrote into the caller's buffer"); + + if full.len() > 1 { + let mut short = vec![RawRecord::EMPTY; full.len() - 1]; + assert_eq!(cap().counters(&mut short), Err(SyscallError::ResourceExhausted), "a buffer short of a cpu"); + } + let mut wide = vec![RawRecord::EMPTY; 4096]; + assert_eq!(cap().counters(&mut wide), Err(SyscallError::InvalidArgument), "a buffer longer than any machine"); +} + +/// Twice as many readers as CPUs, each reading over and over: rounds overlap +/// and join, and every read returns. +fn many_readers(cpus: usize) { + std::thread::scope(|s| { + for _ in 0..2 * cpus { + s.spawn(|| { + for _ in 0..READS { + assert_eq!(read(cap()).map(|r| r.len()), Ok(cpus), "a concurrent read"); + } + }); + } + }); +} + +/// In a child of its own, since the kernel's answer is to end the caller. +fn unheld() { + let out = Command::new(SELF_PATH) + .arg("unheld") + .stdout(Stdio::piped()) + .spawn() + .expect("spawn the unheld child") + .wait_with_output() + .expect("wait the unheld child"); + assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "reached unheld", "the child never reached its call"); + assert_eq!(out.status.code(), Some(HANDLE_FAULT), "SYS_COUNTERS took a handle nobody holds"); +} diff --git a/tests/toyos-rust-tests/src/bin/counters_silent.rs b/tests/toyos-rust-tests/src/bin/counters_silent.rs new file mode 100644 index 00000000000..0fb5a2217dc --- /dev/null +++ b/tests/toyos-rust-tests/src/bin/counters_silent.rs @@ -0,0 +1,50 @@ +//! A CPU that answers no round is read stale, and the read returns: the +//! round's bound ends the wait a silent CPU would otherwise hold. +//! +//! **A guest on the test kernel**, because a CPU silent past the bound is what +//! a span with its interrupts closed makes, or a host that has not run a vCPU, +//! and no guest can make either on demand. `SYS_DEBUG`'s `COUNTERS_DEAF` has +//! one CPU answer no round until `COUNTERS_HEAR`; the reader's wait, its bound +//! and its stale record are the shipped paths. Nothing is timed: a read that +//! waited without its bound reaches the harness's hang ceiling. + +use toyos::endow::{Endowments, SYSCAP_LABEL}; +use toyos::syscap::SysCap; +use toyos_abi::counters::{Counter, RawRecord, Record}; +use toyos_abi::syscall::{self, debug_action, SyscallError}; + +fn read(cap: &SysCap) -> Vec { + let mut raw = vec![RawRecord::EMPTY; syscall::cpu_count() as usize]; + assert_eq!(cap.counters(&mut raw), Ok(raw.len()), "a read of every cpu"); + raw.iter().map(|r| Record::decode(r).expect("the kernel wrote a record that decodes")).collect() +} + +fn main() { + let cap: SysCap = Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows a capability"); + let cpus = syscall::cpu_count() as u64; + assert!(cpus >= 2, "one cpu has no other to silence"); + assert_eq!( + syscall::debug_with(debug_action::COUNTERS_DEAF, cpus), + SyscallError::InvalidArgument.to_u64(), + "a cpu past the machine's was staged silent", + ); + + let silent = (cpus - 1) as usize; + assert_eq!(syscall::debug_with(debug_action::COUNTERS_DEAF, silent as u64), 0); + let held = read(&cap); + let record = held[silent]; + assert!(record.stale, "cpu{silent} answers no round and was read fresh: {record:?}"); + // Stale is the last block it published, its bring-up's at least, and not nothing. + assert!(record.get(Counter::Stamp).is_some(), "cpu{silent}'s last block was not read: {record:?}"); + + assert_eq!(syscall::debug(debug_action::COUNTERS_HEAR), 0); + let heard = loop { + let records = read(&cap); + if !records[silent].stale { + break records[silent]; + } + }; + assert!(heard.get(Counter::Stamp) > record.get(Counter::Stamp), "cpu{silent} answered with an old stamp"); + assert_eq!(heard.hardware_id, record.hardware_id, "cpu{silent}'s stale record named another cpu"); + println!("counters_silent: cpu{silent} was read stale while silent, and answered once heard"); +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 9e55a58846d..8851ffce51f 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -53,6 +53,10 @@ const ACTUATOR_TESTS: &[&str] = &[ // Action 23: the kernel holds a spawn, once its child has landed, until // the child has ended — a child's end no caller can order inside its spawn. "spawn_child_ends_first", + // Actions 24 and 25: one CPU answers no counters round until it is heard + // again, which is what a CPU silent past a read's bound is to the reader, + // and nothing in a guest makes one on demand. + "counters_silent", ]; /// What [`ACTUATOR_TESTS`] boots: the one kernel that carries `SYS_DEBUG`, with @@ -93,6 +97,10 @@ const RUST_SKIP: &[&str] = &[ // metal rows run them. "isa_grant", "isa_lines", + // Its product is the T14's counters across an idle span and a spin on + // every CPU, which the `counters` metal row judges; on a guest it would be + // seconds of four CPUs spinning, read by nothing. + "counters_metal", // It takes the machine down; `virt_fatal_halts_the_others_first` runs it. "panic_halts_first", // Needs a launcher and a declared `cat` and shell, which `tests/testcases` @@ -135,6 +143,9 @@ const RUST_SKIP: &[&str] = &[ /// other, so neither answer is silence — `suite_split` is the gate. #[allow(dead_code, reason = "`suite_split` reads it, in `toyos-checks` alone")] const DRIVEN_AND_SHARED: &[&str] = &[ + // Its shared run is the x86-64 verdict; `virt_smp` builds it for AArch64 + // and runs it on that architecture's SMP case. + "counters_read", // Its shared run is the x86-64 verdict; `virt_readonly_copyout` builds it // for AArch64 and runs it on that architecture's job case. "abuse_readonly_copyout", @@ -360,6 +371,16 @@ const METAL: &[(&str, metal::Metal)] = &[ }, }, ), + ( + // Every CPU's counters as the shipped syscall reads them, across an + // idle second and a spin on every CPU: `counters_on_metal` says what + // is held and what is read beside Linux's. + "counters", + metal::Metal { + arms: &[metal::once("testcases", "tests/testcases", &[], &["test_rs_counters_metal"])], + judge: |b| counters_on_metal(b[0]), + }, + ), ( // The two numbers `syscall_cost` measures, printed by the job and read // off the stick: that it ran, and what it said, are the verdict. @@ -1416,17 +1437,25 @@ fn check_colors( /// `test_rs_abuse_readonly_copyout`. const VIRT_COPYOUT: &str = "abuse_readonly_copyout"; -fn virt_copyout(arch: toyos_build::arch::Arch) -> &'static [u8] { - use toyos_build::arch::Arch; - static X86_64: std::sync::OnceLock> = std::sync::OnceLock::new(); - static AARCH64: std::sync::OnceLock> = std::sync::OnceLock::new(); - let built = match arch { - Arch::X86_64 => &X86_64, - Arch::Aarch64 => &AARCH64, +/// `tests/toyos-rust-tests`' binary that `tests/virtsmpcase` runs as its job +/// `test_rs_counters_read`. +const VIRT_COUNTERS_READ: &str = "counters_read"; + +/// `tests/toyos-rust-tests`' binary `name` built for `arch`, once a run, as a +/// file a case's job list names on ROOT. +fn suite_bin(arch: toyos_build::arch::Arch, name: &'static str) -> (String, Vec) { + type Built = Vec<(toyos_build::arch::Arch, &'static str, Vec)>; + static BUILT: std::sync::Mutex = std::sync::Mutex::new(Vec::new()); + let mut built = BUILT.lock().expect("no build panics holding this"); + let bytes = match built.iter().find(|(a, n, _)| *a == arch && *n == name) { + Some((.., bytes)) => bytes.clone(), + None => { + let bytes = qemu::build_toyos_bin(arch, &compile::repo_root().join("tests/toyos-rust-tests"), name); + built.push((arch, name, bytes.clone())); + bytes + } }; - built.get_or_init(|| { - qemu::build_toyos_bin(arch, &compile::repo_root().join("tests/toyos-rust-tests"), VIRT_COPYOUT) - }) + (format!("bin/test_rs_{name}"), bytes) } /// Boot `tests/virtjobcase` on one CPU and judge its job `job`: it ends with @@ -1445,7 +1474,7 @@ fn virt_job(profile: qemu::Profile, job: &str, said: &str) -> Result<(), String> smp: 1, kernel_features: toyos_build::build::TEST_KERNEL, ready_marker: "control registers: SCTLR_EL1=", - extra_root_files: vec![(format!("bin/test_rs_{VIRT_COPYOUT}"), virt_copyout(profile.arch()).to_vec())], + extra_root_files: vec![suite_bin(profile.arch(), VIRT_COPYOUT)], ..Default::default() }, ); @@ -1525,15 +1554,22 @@ fn boot_virt_smp(options: BootOptions) -> QemuInstance { case, &[], &[], - BootOptions { ready_marker: "control registers: SCTLR_EL1=", ..options }, + BootOptions { + ready_marker: "control registers: SCTLR_EL1=", + extra_root_files: vec![suite_bin(options.profile.arch(), VIRT_COUNTERS_READ)], + ..options + }, ) } +/// What `counters_read` says once every CPU answered for itself. +const COUNTERS_READ_SAID: &str = "counters_read: every cpu answered for itself, and each counter is a right's"; + /// Boot `tests/virtsmpcase` on [`VIRT_CPUS`] CPUs under `profile`, whose /// firmware enters every CPU at EL`el` and whose FADT names PSCI's `conduit`: /// each CPU is started by `CPU_ON`, holds the control-register declaration as -/// entered there and joins the scheduler, and the case's job `unmap_touch` -/// ends with exit 0. Then its job `shutdown` stops the machine and powers it +/// entered there and joins the scheduler, and the case's jobs `unmap_touch` +/// and `test_rs_counters_read` end with exit 0. Then its job `shutdown` stops the machine and powers it /// off, every other CPU turned off first ([`psci_powered_off`]). fn virt_smp(profile: qemu::Profile, conduit: &str, el: u32) -> Result<(), String> { let trace = common::lane::dir().join(format!("virt_smp-{conduit}.psci")); @@ -1569,8 +1605,9 @@ fn virt_smp(profile: qemu::Profile, conduit: &str, el: u32) -> Result<(), String } } eprintln!(" [virt] {VIRT_CPUS} CPUs entered at EL{el}, started through {conduit}, and scheduling"); + let counted = judge_virt_job(&mut qemu, "test_rs_counters_read", COUNTERS_READ_SAID)?; let (console, calls) = - ended_through_psci(&mut qemu, &mut stop, serial, power::SHUTTING_DOWN, "guest-shutdown", &trace, |_| Vec::new())?; + ended_through_psci(&mut qemu, &mut stop, counted, power::SHUTTING_DOWN, "guest-shutdown", &trace, |_| Vec::new())?; let record = console .lines() .find_map(toyos_quiesce::Record::parse) @@ -3190,6 +3227,135 @@ fn field_between<'a>(log: &'a str, head: &str, tail: &str) -> Result<&'a str, St .ok_or_else(|| format!("no record carrying {head:?} and then {tail:?}")) } +/// The shortest span `counters_on_metal` reads the SMI count across: twice the +/// longest period between the T14's firmware interrupts that has been read +/// (`issues/hardware/the-t14s-firmware-interrupts-every-cpu-every-2-2-s-under-toyos.md`). +const SMI_SPAN_NS: u64 = 4_444_000_000; + +/// `counters_metal`'s three reads on the T14, held to what the hardware and +/// the bring-up said. +/// +/// Held: a record per CPU the bring-up started, naming the local APIC id the +/// bring-up gave that CPU and carrying the counters its `counters: cpuN +/// reads` line names, and none stale. From `idle0` to `spin`, at least +/// [`SMI_SPAN_NS`] apart, every CPU's SMI count rose alike and by two or more: +/// the firmware's legacy mode, the positive control ACPI stage 1's flatness +/// is read against, and the row that stage changes. Across the spin every +/// CPU's MPERF ran nine tenths of its stamp or more [e], a CPU in C0 the whole +/// span: MPERF counts at the TSC's rate there (SDM Vol. 3B, "Hardware +/// Coordination Feedback"). +/// +/// Read and not held, beside Linux's turbostat on the same machine +/// (`tests/t14-linux/`): each CPU's idle busy fraction, its busy +/// frequency under the spin, and what one round cost its reader. +fn counters_on_metal(back: &metal::Readback) -> Result<(), String> { + type Read<'a> = BTreeMap>; + back.job_passed("test_rs_counters_metal")?; + let log = back.log(); + let kernel = back.kernel(); + let cpus = back.cpus()? as usize; + let mut reads: BTreeMap<&str, Read> = BTreeMap::new(); + let mut clock: BTreeMap<&str, (u64, u64)> = BTreeMap::new(); + for line in log.text().lines() { + let Some((phase, said)) = line.split("counters_metal ").nth(1).and_then(|r| r.split_once(": ")) else { + continue; + }; + let number = |s: &str| s.trim().parse::().map_err(|_| format!("{s:?} in {line:?} is not a number")); + if let Some(rest) = said.strip_prefix("at ") { + let (at, took) = rest.split_once(" ns, the read took ").ok_or_else(|| format!("unreadable: {line}"))?; + clock.insert(phase, (number(at)?, number(took.trim_end_matches(" ns"))?)); + } else if let Some(rest) = said.strip_prefix("kernel.cpu.") { + let (path, value) = rest.split_once(" = ").ok_or_else(|| format!("unreadable: {line}"))?; + let (cpu, name) = path.split_once('.').ok_or_else(|| format!("unreadable: {line}"))?; + let value = match value { + "true" => 1, + "false" => 0, + v => number(v)?, + }; + reads.entry(phase).or_default().entry(number(cpu)? as usize).or_default().insert(name, value); + } + } + let phase = |name: &str| -> Result<(&Read, u64), String> { + let read = reads.get(name).ok_or_else(|| format!("counters_metal printed no {name} read"))?; + Ok((read, clock.get(name).ok_or_else(|| format!("counters_metal printed no {name} clock"))?.0)) + }; + let (idle0, at0) = phase("idle0")?; + let (idle1, _) = phase("idle1")?; + let (spin, at2) = phase("spin")?; + let bsp = kernel.must_say("percpu: BSP cpu_id=0 lapic_id=")?; + let mut roster = vec![bsp.rsplit("lapic_id=").next().unwrap_or_default().trim().to_string()]; + for cpu in 1..cpus { + roster.push(field_between(kernel.text(), &format!("SMP: AP cpu{cpu} lapic="), " online")?.to_string()); + } + for (name, read) in [("idle0", idle0), ("idle1", idle1), ("spin", spin)] { + if read.keys().copied().ne(0..cpus) { + return Err(format!("{name} read cpus {:?}, and the bring-up started {cpus}", read.keys())); + } + for (cpu, counters) in read { + let reads = kernel.must_say(&format!("counters: cpu{cpu} reads "))?; + for counter in ["smi", "aperf", "mperf"] { + if !reads.contains(&format!("{counter}={}", counters.contains_key(counter))) { + return Err(format!("{name}: cpu{cpu} carries {counters:?} and its bring-up said {reads:?}")); + } + } + if counters.get("stale") != Some(&0) || counters.get("hardware_id").map(u64::to_string) != Some(roster[*cpu].clone()) { + return Err(format!("{name}: cpu{cpu} is stale or not lapic {}: {counters:?}", roster[*cpu])); + } + } + } + for cpu in 0..cpus { + for (name, &value) in &idle1[&cpu] { + if idle0[&cpu][name] > value || value > spin[&cpu][name] { + return Err(format!("cpu{cpu}'s {name} went backwards: {idle0:?} {idle1:?} {spin:?}")); + } + } + } + if at2 - at0 < SMI_SPAN_NS { + return Err(format!("idle0 and spin are {} ns apart, short of {SMI_SPAN_NS}: lengthen the spin", at2 - at0)); + } + let delta = |a: &Read, b: &Read, cpu: usize, name: &str| b[&cpu][name] - a[&cpu][name]; + let smis: Vec = (0..cpus).map(|cpu| delta(idle0, spin, cpu, "smi")).collect(); + if smis[0] < 2 || smis.iter().any(|&n| n != smis[0]) { + return Err(format!("the SMI count rose by {smis:?} over {} ns, not alike and by two or more", at2 - at0)); + } + let tsc_mhz = delta(idle0, spin, 0, "stamp") as f64 * 1e3 / (at2 - at0) as f64; + let ratio = |a: &Read, b: &Read, cpu: usize, top: &str, bottom: &str| { + delta(a, b, cpu, top) as f64 / delta(a, b, cpu, bottom) as f64 + }; + let busy: Vec = (0..cpus).map(|cpu| ratio(idle1, spin, cpu, "mperf", "stamp")).collect(); + if busy.iter().any(|&b| b < 0.9) { + return Err(format!("MPERF ran {busy:?} of each stamp across the spin, some cpu under 0.9")); + } + let linux = |file: &str, column: &str| -> Result<(f64, f64), String> { + let mut rows = file.lines().map(|l| l.split('\t').collect::>()); + let header = rows.next().ok_or("an empty turbostat reading")?; + let at = header.iter().position(|c| *c == column).ok_or_else(|| format!("turbostat read no {column}"))?; + let values: Vec = rows.filter(|r| r[0] == "-").filter_map(|r| r.get(at)?.parse().ok()).collect(); + Ok(values.iter().fold((f64::MAX, f64::MIN), |(lo, hi), &v| (lo.min(v), hi.max(v)))) + }; + let idle = linux(include_str!("t14-linux/turbostat-idle.txt"), "Busy%")?; + let loaded = linux(include_str!("t14-linux/turbostat-loaded.txt"), "Bzy_MHz")?; + eprintln!( + " [counters] {cpus} cpus, SMI +{} each over {} ms; TSC {tsc_mhz:.0} MHz", + smis[0], + (at2 - at0) / 1_000_000 + ); + for (cpu, busy) in busy.iter().enumerate() { + eprintln!( + " [counters] cpu{cpu}: idle busy {:.2}% (Linux {:.2}-{:.2}%), spinning {:.0} MHz (Linux {:.0}-{:.0}), \ + busy {busy:.3}", + ratio(idle0, idle1, cpu, "mperf", "stamp") * 100.0, + idle.0, + idle.1, + tsc_mhz * ratio(idle1, spin, cpu, "aperf", "mperf"), + loaded.0, + loaded.1, + ); + } + eprintln!(" [counters] the spin's read, a whole round, took its reader {} ns", clock["spin"].1); + Ok(()) +} + /// The same, parsed. fn number_between(log: &str, head: &str, tail: &str) -> Result { let word = field_between(log, head, tail)?; diff --git a/tests/virtsmpcase/system.toml b/tests/virtsmpcase/system.toml index c5fd5458641..9e2db041d43 100644 --- a/tests/virtsmpcase/system.toml +++ b/tests/virtsmpcase/system.toml @@ -1,6 +1,8 @@ -# Every CPU the machine names, and the one job whose premise is more than one -# CPU: a page unmapped beside a thread that is still reading it. Then the -# machine powers off, every CPU but one turned off first. +# Every CPU the machine names, and the two jobs whose premise is more than one +# CPU: a page unmapped beside a thread that is still reading it, and a counters +# read every CPU answers for itself, the suite's `test_rs_counters_read`, which +# the harness puts on ROOT. Then the machine powers off, every CPU but one +# turned off first. [boot] start = ["logkeeper", "test-runner"] @@ -12,7 +14,9 @@ syscap = ["logread"] receives = ["power"] # Four minutes from boot: every job here runs on emulated CPUs, and a job past # the bound reboots the guest with the job named. -args = ["--bound-ms=240000", "unmap_touch", "shutdown"] +args = ["--bound-ms=240000", "unmap_touch", "test_rs_counters_read", "shutdown"] +# What `test_rs_counters_read` reads, and narrows away to prove its refusals. +syscap = ["dup", "counters", "trace"] [programs.kernelprobe] diff --git a/toyos-abi/src/counters.rs b/toyos-abi/src/counters.rs new file mode 100644 index 00000000000..fd7109da628 --- /dev/null +++ b/toyos-abi/src/counters.rs @@ -0,0 +1,211 @@ +//! What [`SYS_COUNTERS`] answers: one [`Record`] per online CPU, each holding +//! that CPU's [`Counter`]s as it read them itself. +//! +//! **A CPU's hardware counters are readable only on that CPU**, so a read asks +//! every CPU to copy its counters out and waits for the answers under a bound. +//! A CPU that has not answered by then is [`Record::stale`]: its values are the +//! last it gave, and its [`Counter::Stamp`] says when it gave them. A counter +//! the CPU does not have, or that the caller's capability may not read, is +//! absent and never zero; a record with no stamp is a CPU whose answer could +//! not be read whole, and says nothing about it but its index. +//! +//! Every value counts up from boot, so the difference of two reads is exact. +//! +//! [`SYS_COUNTERS`]: crate::syscall::SYS_COUNTERS + +use crate::handle::Rights; + +/// One counter a CPU keeps, by its index in [`Record::values`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Counter { + /// The CPU's own free-running counter (TSC, `CNTVCT_EL0`) when it read the + /// rest, in its ticks. + Stamp, + /// Times firmware took the CPU over (`MSR_SMI_COUNT`), 32 bits wide. + Smi, + /// Cycles at the frequency the CPU ran at, counted while it ran + /// (`IA32_APERF`). + Aperf, + /// Cycles at a fixed reference frequency, counted while it ran + /// (`IA32_MPERF`). + Mperf, + /// Kicks the CPU took: the interrupt another CPU sends it to wake it. + Kicks, +} + +impl Counter { + pub const COUNT: usize = 5; + pub const ALL: [Counter; Self::COUNT] = + [Self::Stamp, Self::Smi, Self::Aperf, Self::Mperf, Self::Kicks]; + + pub const fn name(self) -> &'static str { + match self { + Self::Stamp => "stamp", + Self::Smi => "smi", + Self::Aperf => "aperf", + Self::Mperf => "mperf", + Self::Kicks => "kicks", + } + } + + /// The rights a `SysCap` carries for this counter to be answered. + pub const fn needs(self) -> Rights { + match self { + Self::Stamp | Self::Smi => Rights::COUNTERS, + Self::Aperf | Self::Mperf | Self::Kicks => Rights::COUNTERS.union(Rights::TRACE), + } + } +} + +/// The width of every record: the CPU's index and hardware id, the flags, and +/// one word per counter. +pub const RECORD_BYTES: usize = 16 + 8 * Counter::COUNT; + +/// One record as it crosses the boundary. +#[repr(C)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct RawRecord(pub [u8; RECORD_BYTES]); + +impl RawRecord { + pub const EMPTY: Self = Self([0; RECORD_BYTES]); +} + +const STALE: u64 = 1; + +const fn present(counter: Counter) -> u64 { + 1 << (8 + counter as u32) +} + +/// One CPU's counters. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Record { + pub cpu: u32, + /// The id the CPU read off itself (its local APIC id, its packed MPIDR + /// affinity), meaningful where the stamp is present. + pub hardware_id: u32, + /// The CPU did not answer this read within its bound. + pub stale: bool, + pub values: [Option; Counter::COUNT], +} + +/// A record whose bytes no kernel writes. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Undecodable { + /// A flag bit no counter names. + Flags(u64), + /// A counter absent and its word not zero. + Absent(Counter), +} + +impl Record { + pub fn get(&self, counter: Counter) -> Option { + self.values[counter as usize] + } + + pub fn encode(&self) -> RawRecord { + let mut raw = [0u8; RECORD_BYTES]; + let mut flags = if self.stale { STALE } else { 0 }; + for counter in Counter::ALL { + if let Some(value) = self.get(counter) { + flags |= present(counter); + let at = 16 + 8 * counter as usize; + raw[at..at + 8].copy_from_slice(&value.to_le_bytes()); + } + } + raw[0..4].copy_from_slice(&self.cpu.to_le_bytes()); + raw[4..8].copy_from_slice(&self.hardware_id.to_le_bytes()); + raw[8..16].copy_from_slice(&flags.to_le_bytes()); + RawRecord(raw) + } + + pub fn decode(raw: &RawRecord) -> Result { + let word = |at: usize| u64::from_le_bytes(raw.0[at..at + 8].try_into().unwrap()); + let flags = word(8); + let known = Counter::ALL.iter().fold(STALE, |known, &c| known | present(c)); + if flags & !known != 0 { + return Err(Undecodable::Flags(flags & !known)); + } + let mut values = [None; Counter::COUNT]; + for counter in Counter::ALL { + let value = word(16 + 8 * counter as usize); + if flags & present(counter) != 0 { + values[counter as usize] = Some(value); + } else if value != 0 { + return Err(Undecodable::Absent(counter)); + } + } + Ok(Self { + cpu: u32::from_le_bytes(raw.0[0..4].try_into().unwrap()), + hardware_id: u32::from_le_bytes(raw.0[4..8].try_into().unwrap()), + stale: flags & STALE != 0, + values, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(stale: bool, values: [Option; Counter::COUNT]) -> Record { + Record { cpu: 7, hardware_id: 0x0102_0304, stale, values } + } + + #[test] + fn every_shape_of_record_reads_back_as_written() { + for r in [ + record(false, [Some(1), Some(u64::from(u32::MAX)), Some(3), Some(u64::MAX), Some(0)]), + record(true, [Some(9), None, None, None, Some(4)]), + record(true, [None; Counter::COUNT]), + ] { + assert_eq!(Record::decode(&r.encode()), Ok(r)); + } + } + + /// Absent is not zero: a zero that is there reads back there. + #[test] + fn a_present_zero_is_not_an_absent_counter() { + let r = record(false, [Some(5), Some(0), None, None, None]); + assert_eq!(Record::decode(&r.encode()).unwrap().get(Counter::Smi), Some(0)); + assert_eq!(Record::decode(&r.encode()).unwrap().get(Counter::Aperf), None); + } + + #[test] + fn a_flag_no_counter_names_is_refused() { + let mut raw = record(false, [Some(1); Counter::COUNT]).encode(); + raw.0[8] |= 1 << 1; + assert_eq!(Record::decode(&raw), Err(Undecodable::Flags(1 << 1))); + let mut raw = RawRecord::EMPTY; + raw.0[15] = 0x80; + assert_eq!(Record::decode(&raw), Err(Undecodable::Flags(1 << 63))); + } + + #[test] + fn a_word_under_an_absent_counter_is_refused() { + let mut raw = record(false, [Some(1), None, None, None, None]).encode(); + raw.0[16 + 8 * Counter::Mperf as usize] = 1; + assert_eq!(Record::decode(&raw), Err(Undecodable::Absent(Counter::Mperf))); + } + + /// The counters that time programs are the trace right's, and the two that + /// do not are readable on `COUNTERS` alone. + #[test] + fn what_times_a_program_needs_the_trace_right() { + for counter in Counter::ALL { + assert!(counter.needs().contains(Rights::COUNTERS), "{counter:?}"); + assert_eq!( + counter.needs().contains(Rights::TRACE), + matches!(counter, Counter::Aperf | Counter::Mperf | Counter::Kicks), + "{counter:?}" + ); + } + } + + #[test] + fn counters_are_indexed_by_their_place_and_named_once() { + for (i, counter) in Counter::ALL.iter().enumerate() { + assert_eq!(*counter as usize, i); + assert!(Counter::ALL[..i].iter().all(|c| c.name() != counter.name())); + } + } +} diff --git a/toyos-abi/src/handle.rs b/toyos-abi/src/handle.rs index 4542e1694e6..5c503c112a7 100644 --- a/toyos-abi/src/handle.rs +++ b/toyos-abi/src/handle.rs @@ -137,10 +137,31 @@ impl Rights { /// /// [`SYS_DEVICE_INVENTORY`]: crate::syscall::SYS_DEVICE_INVENTORY pub const INVENTORY: Rights = Rights(1 << 12); + /// On a `SysCap`: read the machine's counters. + /// + /// [`SYS_COUNTERS`] answers, per CPU, its own clock reading and how many + /// times firmware took it over, `MSR_SMI_COUNT` where the CPU counts that: + /// what the machine is doing, not what a program on it does. The counters + /// that time programs need [`TRACE`](Self::TRACE) beside this. + /// `inspect` holds it, and `test-runner`. + /// + /// [`SYS_COUNTERS`]: crate::syscall::SYS_COUNTERS + pub const COUNTERS: Rights = Rights(1 << 13); + /// On a `SysCap`: read what times the work of every other program. + /// + /// Beside [`COUNTERS`](Self::COUNTERS), [`SYS_COUNTERS`] answers each + /// CPU's `APERF` and `MPERF` — its frequency and busy fraction, the + /// Hertzbleed power channel — and the kicks it took, the wake-ups that time + /// keystrokes. Admin tools hold it, `inspect` and `test-runner`; no + /// ordinary program and no toybox applet does while a manifest row grants + /// a whole binary. + /// + /// [`SYS_COUNTERS`]: crate::syscall::SYS_COUNTERS + pub const TRACE: Rights = Rights(1 << 14); /// Every bit that has a caller. A wider set than this is a bug in whoever /// composed it, not a right nobody uses. - pub const ALL: Rights = Rights(0x1fff); + pub const ALL: Rights = Rights(0x7fff); pub const fn from_bits(bits: u32) -> Option { if bits & !Self::ALL.0 == 0 { Some(Rights(bits)) } else { None } @@ -171,7 +192,7 @@ impl Rights { /// refusal saying which right was missing. impl core::fmt::Debug for Rights { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - const NAMES: [(Rights, &str); 13] = [ + const NAMES: [(Rights, &str); 15] = [ (Rights::DUP, "DUP"), (Rights::TRANSFER, "TRANSFER"), (Rights::READ, "READ"), @@ -185,6 +206,8 @@ impl core::fmt::Debug for Rights { (Rights::POWER, "POWER"), (Rights::ROSTER, "ROSTER"), (Rights::INVENTORY, "INVENTORY"), + (Rights::COUNTERS, "COUNTERS"), + (Rights::TRACE, "TRACE"), ]; if self.0 == 0 { return f.write_str("NONE"); diff --git a/toyos-abi/src/lib.rs b/toyos-abi/src/lib.rs index 5de747e19ab..15c26e12ad4 100644 --- a/toyos-abi/src/lib.rs +++ b/toyos-abi/src/lib.rs @@ -18,6 +18,7 @@ mod arch; pub mod audio; pub mod boot; pub mod clock; +pub mod counters; pub mod handle; pub mod hda; pub mod inbox; diff --git a/toyos-abi/src/syscall.rs b/toyos-abi/src/syscall.rs index 493b2833bde..6b2d73c1652 100644 --- a/toyos-abi/src/syscall.rs +++ b/toyos-abi/src/syscall.rs @@ -259,6 +259,14 @@ pub const SYS_DEVICE_DMA_MAP: u64 = 122; /// [`device_dma_unmap`]. pub const SYS_DEVICE_DMA_UNMAP: u64 = 123; +/// Every online CPU's counters, as [`crate::counters`] records. Gated by +/// [`Rights::COUNTERS`] on a `SysCap`, and the counters that time programs by +/// [`Rights::TRACE`] beside it. See [`counters`]. +/// +/// [`Rights::COUNTERS`]: crate::handle::Rights::COUNTERS +/// [`Rights::TRACE`]: crate::handle::Rights::TRACE +pub const SYS_COUNTERS: u64 = 124; + /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. /// @@ -271,7 +279,7 @@ pub const SYSCALL_PROFILE_BINS: usize = 128; /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_DEVICE_DMA_UNMAP < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_COUNTERS < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; @@ -807,6 +815,13 @@ pub mod debug_action { /// ending inside the spawn that started it is a race no caller can order; /// the landing and the exit either side of the wait are the shipped paths. pub const HOLD_SPAWN_UNTIL_CHILD_ENDS: u64 = 23; + /// The CPU the argument names answers no [`SYS_COUNTERS`](super::SYS_COUNTERS) + /// round until [`COUNTERS_HEAR`]: a reader finds it as it finds a CPU + /// with its interrupts closed past the round's bound. A CPU out of range + /// answers `InvalidArgument`. + pub const COUNTERS_DEAF: u64 = 24; + /// Every CPU answers rounds again. + pub const COUNTERS_HEAR: u64 = 25; } /// Every kind of kernel object, in the order the kernel's own `kobject!` @@ -1956,6 +1971,23 @@ pub fn device_inventory( .map(|n| n as usize) } +/// One [`crate::counters`] record per online CPU, into `buf`; answers how many +/// were written. An empty `buf` asks how many CPUs there are and asks no CPU +/// anything; one with room for fewer is refused with +/// [`SyscallError::ResourceExhausted`], and one declared longer than the +/// kernel's most CPUs with [`SyscallError::InvalidArgument`]. +/// +/// `syscap` must carry [`crate::handle::Rights::COUNTERS`]: a capability +/// without it is refused with a word, and a handle the caller does not hold +/// ends it. +pub fn counters( + syscap: RawHandle, + buf: &mut [crate::counters::RawRecord], +) -> Result { + check(syscall(SYS_COUNTERS, syscap.0 as u64, buf.as_mut_ptr() as u64, buf.len() as u64, 0)) + .map(|n| n as usize) +} + /// Sleep for the given number of nanoseconds. pub fn nanosleep(nanos: u64) { syscall(SYS_NANOSLEEP, nanos, 0, 0, 0); diff --git a/toyos-cpuvuln/Cargo.toml b/toyos-cpuvuln/Cargo.toml index 77f5b8f4f14..2bee37a101e 100644 --- a/toyos-cpuvuln/Cargo.toml +++ b/toyos-cpuvuln/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "toyos-cpuvuln" -description = "Linux's verdict on each speculative-execution vulnerability and its mitigation choice, from a CPU's CPUID and MSR facts, pure." +description = "Linux's verdicts from a CPU's CPUID and MSR facts, pure: each speculative-execution vulnerability and its mitigation choice, and which model-specific counters the CPU has." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/toyos-cpuvuln/fixtures/t14/SOURCE b/toyos-cpuvuln/fixtures/t14/SOURCE index 8e5f29ff538..4b9d9c0e487 100644 --- a/toyos-cpuvuln/fixtures/t14/SOURCE +++ b/toyos-cpuvuln/fixtures/t14/SOURCE @@ -11,3 +11,10 @@ CPUID and the MSRs through the running kernel's /dev/cpu/*/cpuid and msr.txt one per line: msr cpu value; 0x122 and 0x10F are absent because ARCH_CAPABILITIES bit 7 and CPUID.(7,0):EDX bits 11 and 13 do not enumerate them + +One more reading of the same Ubuntu on the same machine, `6.8.0-142-generic` +with the `powersave` governor, taken as root by the orchestrator on +2026-10-03, as the tool printed it: + + perf-msr.txt `perf stat -a -A -e msr/aperf/,msr/mperf/,msr/smi/,msr/tsc/ sleep 60`, + the msr PMU's events per CPU across 60 s of an idle machine diff --git a/toyos-cpuvuln/fixtures/t14/perf-msr.txt b/toyos-cpuvuln/fixtures/t14/perf-msr.txt new file mode 100644 index 00000000000..ba322133350 --- /dev/null +++ b/toyos-cpuvuln/fixtures/t14/perf-msr.txt @@ -0,0 +1,38 @@ + + Performance counter stats for 'system wide': + +CPU0 98,079,648 msr/aperf/ +CPU1 181,130,722 msr/aperf/ +CPU2 121,578,728 msr/aperf/ +CPU3 140,715,079 msr/aperf/ +CPU4 61,297,064 msr/aperf/ +CPU5 191,874,303 msr/aperf/ +CPU6 96,508,493 msr/aperf/ +CPU7 74,201,638 msr/aperf/ +CPU0 150,186,840 msr/mperf/ +CPU1 192,582,720 msr/mperf/ +CPU2 234,799,920 msr/mperf/ +CPU3 313,507,656 msr/mperf/ +CPU4 156,309,048 msr/mperf/ +CPU5 326,606,712 msr/mperf/ +CPU6 227,450,760 msr/mperf/ +CPU7 156,326,856 msr/mperf/ +CPU0 0 msr/smi/ +CPU1 0 msr/smi/ +CPU2 0 msr/smi/ +CPU3 0 msr/smi/ +CPU4 0 msr/smi/ +CPU5 0 msr/smi/ +CPU6 0 msr/smi/ +CPU7 0 msr/smi/ +CPU0 145,157,386,466 msr/tsc/ +CPU1 145,157,434,015 msr/tsc/ +CPU2 145,157,376,550 msr/tsc/ +CPU3 145,157,230,321 msr/tsc/ +CPU4 145,157,567,260 msr/tsc/ +CPU5 145,157,642,420 msr/tsc/ +CPU6 145,157,528,396 msr/tsc/ +CPU7 145,157,356,567 msr/tsc/ + + 60.002902881 seconds time elapsed + diff --git a/toyos-cpuvuln/src/counters.rs b/toyos-cpuvuln/src/counters.rs new file mode 100644 index 00000000000..4ad06afcb62 --- /dev/null +++ b/toyos-cpuvuln/src/counters.rs @@ -0,0 +1,131 @@ +//! Which of a CPU's model-specific counters exist, as Linux's `msr` PMU decides +//! (`arch/x86/events/msr.c`): a counter a CPU lacks is `#GP` to read, and the +//! kernel has no fault fixup, so it reads only what this admits. +//! +//! **One departure from Linux**: `MSR_SMI_COUNT` is not admitted under a +//! hypervisor (`CPUID.1:ECX[31]`). Linux admits it there and its probe read +//! decides (`probe.c:43-47`), but a hypervisor answers it with its own count +//! for the guest (QEMU's `env->msr_smi_count`), not the firmware's. `APERF` +//! and `MPERF` are admitted wherever leaf 6 says, a hypervisor's included, as +//! Linux admits them: one that enumerates them and faults the read ends the +//! boot that read them first. + +use crate::{table, Ident, Vendor, CPUID_1_ECX_HYPERVISOR}; + +/// What one CPU says about itself, read on that CPU. +#[derive(Clone, Copy, Debug)] +pub struct CounterFacts { + pub vendor: Vendor, + /// CPUID.1:EAX. + pub signature: u32, + pub cpuid_1_ecx: u32, + /// 0 unless CPUID.0:EAX reaches 6, as `init_scattered_cpuid_features` + /// reads it (`scattered.c:65-71`). + pub cpuid_6_ecx: u32, +} + +/// The counters a CPU has. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Counters { + /// `IA32_APERF` and `IA32_MPERF`. + pub aperf_mperf: bool, + /// `MSR_SMI_COUNT`. + pub smi: bool, +} + +/// `X86_FEATURE_APERFMPERF` (`scattered.c:27`). +const CPUID_6_ECX_APERFMPERF: u32 = 1 << 0; + +pub fn counters(facts: &CounterFacts) -> Counters { + Counters { + // `test_aperfmperf` (`msr.c:20-23,158-159`), whatever the vendor. + aperf_mperf: facts.cpuid_6_ecx & CPUID_6_ECX_APERFMPERF != 0, + // `test_intel` (`msr.c:161`), outside a hypervisor. + smi: table::counts_smis(&Ident::new(facts.vendor, facts.signature)) + && facts.cpuid_1_ecx & CPUID_1_ECX_HYPERVISOR == 0, + } +} + +#[cfg(test)] +mod tests { + extern crate std; + + use std::collections::BTreeSet; + use std::vec::Vec; + + use super::*; + + /// The T14's CPU 0, off `fixtures/t14/cpuid.txt`. + fn t14() -> CounterFacts { + let leaves: Vec> = include_str!("../fixtures/t14/cpuid.txt") + .lines() + .map(|l| l.split(' ').map(|w| u32::from_str_radix(w, 16).expect(w)).collect()) + .collect(); + let leaf = |n: u32| leaves.iter().find(|l| l[..2] == [n, 0]).expect("a leaf the capture read"); + let id: Vec = [leaf(0)[3], leaf(0)[5], leaf(0)[4]].iter().flat_map(|r| r.to_le_bytes()).collect(); + assert!(leaf(0)[2] >= 6, "the T14 reaches leaf 6"); + CounterFacts { + vendor: Vendor::from_id(id[..].try_into().unwrap()), + signature: leaf(1)[2], + cpuid_1_ecx: leaf(1)[4], + cpuid_6_ecx: leaf(6)[4], + } + } + + /// QEMU 11.1.1's `qemu64`, the TCG model: `AuthenticAMD` family 0xF model + /// 0x6B (`target/i386/cpu.c:3545-3550`), and leaf 6's ECX zero on every + /// model (`cpu.c:8777-8782`). + const TCG: CounterFacts = CounterFacts { + vendor: Vendor::Amd, + signature: 0x0006_0fb1, + cpuid_1_ecx: 0, + cpuid_6_ecx: 0, + }; + + /// Linux on the T14 listed exactly the events whose test and probe read + /// passed (`probe.c:28-59`), so its `perf stat` over them is its verdict on + /// that CPU, and this is held to it. + #[test] + fn the_t14_has_what_linux_counted_on_it() { + let counted: BTreeSet<&str> = include_str!("../fixtures/t14/perf-msr.txt") + .lines() + .filter_map(|l| l.split_whitespace().nth(2)?.strip_prefix("msr/")?.strip_suffix('/')) + .collect(); + assert_eq!(counted, BTreeSet::from(["aperf", "mperf", "smi", "tsc"])); + assert_eq!(counters(&t14()), Counters { aperf_mperf: true, smi: true }); + } + + #[test] + fn the_tcg_model_has_none() { + assert_eq!(counters(&TCG), Counters { aperf_mperf: false, smi: false }); + } + + /// Under a hypervisor the SMI count is the hypervisor's, and APERF and + /// MPERF stay whatever leaf 6 says. + #[test] + fn a_guest_counts_no_smis() { + let guest = CounterFacts { cpuid_1_ecx: t14().cpuid_1_ecx | CPUID_1_ECX_HYPERVISOR, ..t14() }; + assert_eq!(counters(&guest), Counters { aperf_mperf: true, smi: false }); + } + + #[test] + fn an_amd_with_leaf_6_has_aperf_and_mperf_and_no_smi_count() { + let zen = CounterFacts { cpuid_6_ecx: CPUID_6_ECX_APERFMPERF, ..TCG }; + assert_eq!(counters(&zen), Counters { aperf_mperf: true, smi: false }); + } + + /// The model list is the T14's model and Linux's other rows, and nothing + /// in family 6 that Linux leaves out. + #[test] + fn an_intel_model_off_the_list_counts_no_smis() { + // Family 6 model 0x8C stepping 1 is the T14's; 0x66 is CANNONLAKE_L, + // which `test_intel` does not name. + let model = |m: u32| CounterFacts { signature: 0x0000_0601 | (m >> 4) << 16 | (m & 0xf) << 4, ..t14() }; + assert!(counters(&model(0x8C)).smi); + assert!(!counters(&model(0x66)).smi); + // Family 0xF Intel is outside `test_intel`'s family, and the T14's + // family and model on another vendor outside its vendor. + assert!(!counters(&CounterFacts { signature: 0x0000_0f41, ..t14() }).smi); + assert!(!counters(&CounterFacts { vendor: Vendor::Centaur, ..t14() }).smi); + } +} diff --git a/toyos-cpuvuln/src/lib.rs b/toyos-cpuvuln/src/lib.rs index 51f544b41af..181e3bd1125 100644 --- a/toyos-cpuvuln/src/lib.rs +++ b/toyos-cpuvuln/src/lib.rs @@ -15,6 +15,9 @@ //! Parity is exact: a line that rests on something the facts do not carry is //! [`Unmodelled`], never approximated. //! +//! [`counters`] is the same tag's other verdict on a CPU: which of its +//! model-specific counters exist. +//! //! Pure: no I/O, no allocation, no `unsafe`. The caller reads the facts. #![no_std] @@ -22,7 +25,10 @@ use core::fmt; +mod counters; mod table; + +pub use counters::{counters, CounterFacts, Counters}; #[cfg(test)] mod tests; diff --git a/toyos-cpuvuln/src/table.rs b/toyos-cpuvuln/src/table.rs index ed284e7b698..8b748b5fe73 100644 --- a/toyos-cpuvuln/src/table.rs +++ b/toyos-cpuvuln/src/table.rs @@ -1,6 +1,7 @@ -//! Linux's four CPU tables, row for row: `cpu_vuln_whitelist` and +//! Linux's five CPU tables, row for row: `cpu_vuln_whitelist` and //! `cpu_vuln_blacklist` (`common.c:1182-1344`), `spectre_bad_microcodes` -//! (`intel.c:141-163`) and `amd_check_tsa_microcode`'s (`amd.c:472-515`). A +//! (`intel.c:141-163`), `amd_check_tsa_microcode`'s (`amd.c:472-515`) and +//! `test_intel`'s (`arch/x86/events/msr.c:40-122`). A //! lookup in the first two is `x86_match_cpu` (`match.c:36-57`): the //! first row whose vendor, family, model and stepping all match, `0` matching //! any. @@ -110,6 +111,13 @@ pub(crate) fn blacklisted(id: &Ident, which: Bl) -> bool { // `arch/x86/include/asm/intel-family.h`, family 6, the models the tables name. const CORE_YONAH: u32 = 0x0E; +const NEHALEM: u32 = 0x1E; +const NEHALEM_G: u32 = 0x1F; +const NEHALEM_EP: u32 = 0x1A; +const NEHALEM_EX: u32 = 0x2E; +const WESTMERE: u32 = 0x25; +const WESTMERE_EP: u32 = 0x2C; +const WESTMERE_EX: u32 = 0x2F; const SANDYBRIDGE: u32 = 0x2A; const SANDYBRIDGE_X: u32 = 0x2D; const IVYBRIDGE: u32 = 0x3A; @@ -133,12 +141,14 @@ const CANNONLAKE_L: u32 = 0x66; const ICELAKE_X: u32 = 0x6A; const ICELAKE_D: u32 = 0x6C; const ICELAKE_L: u32 = 0x7E; +const ICELAKE: u32 = 0x7D; const ROCKETLAKE: u32 = 0xA7; const TIGERLAKE_L: u32 = 0x8C; const TIGERLAKE: u32 = 0x8D; const SAPPHIRERAPIDS_X: u32 = 0x8F; const EMERALDRAPIDS_X: u32 = 0xCF; const GRANITERAPIDS_X: u32 = 0xAD; +const GRANITERAPIDS_D: u32 = 0xAE; const LAKEFIELD: u32 = 0x8A; const ALDERLAKE: u32 = 0x97; const ALDERLAKE_L: u32 = 0x9A; @@ -146,6 +156,7 @@ const RAPTORLAKE: u32 = 0xB7; const RAPTORLAKE_P: u32 = 0xBA; const RAPTORLAKE_S: u32 = 0xBF; const METEORLAKE_L: u32 = 0xAA; +const METEORLAKE: u32 = 0xAC; const ARROWLAKE_H: u32 = 0xC5; const ARROWLAKE: u32 = 0xC6; const ARROWLAKE_U: u32 = 0xB5; @@ -344,3 +355,26 @@ pub(crate) fn tsa_microcode(id: &Ident, microcode: u32) -> bool { let key = (id.family - 0xf) << 16 | (id.model >> 4) << 12 | (id.model & 0xf) << 4 | id.stepping; ROWS.iter().find(|&&(k, _)| k == key).is_some_and(|&(_, min)| microcode >= min) } + +/// `test_intel` for `PERF_MSR_SMI` (`arch/x86/events/msr.c:40-122`): an Intel +/// family 6 CPU of one of these models counts its SMIs in `MSR_SMI_COUNT`. +pub(crate) fn counts_smis(id: &Ident) -> bool { + const MODELS: &[u32] = &[ + NEHALEM, NEHALEM_G, NEHALEM_EP, NEHALEM_EX, + WESTMERE, WESTMERE_EP, WESTMERE_EX, + SANDYBRIDGE, SANDYBRIDGE_X, + IVYBRIDGE, IVYBRIDGE_X, + HASWELL, HASWELL_X, HASWELL_L, HASWELL_G, + BROADWELL, BROADWELL_D, BROADWELL_G, BROADWELL_X, + SAPPHIRERAPIDS_X, EMERALDRAPIDS_X, GRANITERAPIDS_X, GRANITERAPIDS_D, + ATOM_SILVERMONT, ATOM_SILVERMONT_D, ATOM_AIRMONT, ATOM_AIRMONT_NP, + ATOM_GOLDMONT, ATOM_GOLDMONT_D, ATOM_GOLDMONT_PLUS, + ATOM_TREMONT_D, ATOM_TREMONT, ATOM_TREMONT_L, + XEON_PHI_KNL, XEON_PHI_KNM, + SKYLAKE_L, SKYLAKE, SKYLAKE_X, KABYLAKE_L, KABYLAKE, COMETLAKE_L, COMETLAKE, + ICELAKE_L, ICELAKE, ICELAKE_X, ICELAKE_D, TIGERLAKE_L, TIGERLAKE, ROCKETLAKE, + ALDERLAKE, ALDERLAKE_L, ATOM_GRACEMONT, RAPTORLAKE, RAPTORLAKE_P, RAPTORLAKE_S, + METEORLAKE, METEORLAKE_L, + ]; + id.vendor == Vendor::Intel && id.family == 6 && MODELS.contains(&id.model) +} diff --git a/toyos-inspect/src/dev.rs b/toyos-inspect/src/dev.rs index b7a8e81675b..3f917e4b085 100644 --- a/toyos-inspect/src/dev.rs +++ b/toyos-inspect/src/dev.rs @@ -1,6 +1,6 @@ //! The kernel's device inventory, rendered under `dev.*`. //! -//! The one root no port answers for: the reader asks the kernel with a +//! A root no port answers for: the reader asks the kernel with a //! `SysCap` carrying `Rights::INVENTORY` and renders its typed records here, so //! the path layout is decided in the same pure crate as every owner's. //! diff --git a/toyos-inspect/src/kernel.rs b/toyos-inspect/src/kernel.rs new file mode 100644 index 00000000000..ca5ff1513ff --- /dev/null +++ b/toyos-inspect/src/kernel.rs @@ -0,0 +1,107 @@ +//! The kernel's counters, rendered under `kernel.*`. +//! +//! A root no port answers for, as `dev` is: the reader asks the kernel on a +//! `SysCap` carrying `Rights::COUNTERS`, and `Rights::TRACE` for the counters +//! that time programs, and renders its records here. +//! +//! ```text +//! kernel.cpu..stale whether the CPU missed this read's bound +//! kernel.cpu..hardware_id the id the CPU read off itself +//! kernel.cpu.. each counter the record carries, by its ABI name +//! ``` +//! +//! **A counter the record does not carry has no path**, never a zero: the CPU +//! lacks it, or the capability may not read it. A record with no stamp is a CPU +//! whose answer could not be read whole, and renders as its `stale` alone. + +use alloc::collections::BTreeMap; +use alloc::format; +use alloc::string::String; + +use toyos_abi::counters::{Counter, Record}; + +use crate::dev::Repeated; +use crate::wire::Value; + +/// The root every counter's path is under. +pub const ROOT: &str = "kernel"; + +/// Every record as `kernel.*` paths, sorted; a CPU two records name is +/// refused by its path. +pub fn render(records: &[Record]) -> Result, Repeated> { + let mut out = BTreeMap::new(); + for record in records { + let at = format!("{ROOT}.cpu.{}", record.cpu); + let stale = format!("{at}.stale"); + if out.insert(stale.clone(), record.stale.into()).is_some() { + return Err(Repeated(stale)); + } + if record.get(Counter::Stamp).is_none() { + continue; + } + out.insert(format!("{at}.hardware_id"), record.hardware_id.into()); + for counter in Counter::ALL { + if let Some(value) = record.get(counter) { + out.insert(format!("{at}.{}", counter.name()), value.into()); + } + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + extern crate std; + + use alloc::vec; + use alloc::vec::Vec; + + use super::*; + use crate::check_path; + + fn record(cpu: u32, stale: bool, values: [Option; Counter::COUNT]) -> Record { + Record { cpu, hardware_id: cpu * 2, stale, values } + } + + /// The T14's shape: every counter on every CPU. + fn whole(cpu: u32) -> Record { + record(cpu, false, [Some(10), Some(4817), Some(100), Some(200), Some(3)]) + } + + #[test] + fn every_counter_a_record_carries_is_a_path_the_grammar_accepts() { + let got = render(&(0..8).map(whole).collect::>()).unwrap(); + assert_eq!(got.len(), 8 * (2 + Counter::COUNT)); + for path in got.keys() { + check_path(path).unwrap_or_else(|e| panic!("{path}: {e:?}")); + } + assert_eq!(got["kernel.cpu.7.smi"], Value::U64(4817)); + assert_eq!(got["kernel.cpu.7.hardware_id"], Value::U64(14)); + assert_eq!(got["kernel.cpu.7.stale"], Value::Bool(false)); + } + + /// A machine whose CPU counts no SMIs, read on a capability without + /// `TRACE`: only what the record carries, and no zero in place of the rest. + #[test] + fn an_absent_counter_has_no_path() { + let got = render(&[record(0, false, [Some(10), None, None, None, None])]).unwrap(); + let paths: Vec<&str> = got.keys().map(String::as_str).collect(); + assert_eq!(paths, vec!["kernel.cpu.0.hardware_id", "kernel.cpu.0.stale", "kernel.cpu.0.stamp"]); + } + + /// A stale CPU keeps the values it last gave, and says it is stale; one + /// whose answer was not read says that alone. + #[test] + fn a_stale_cpu_says_so_and_an_unread_one_says_only_that() { + let got = render(&[record(1, true, whole(1).values), record(2, true, [None; Counter::COUNT])]).unwrap(); + assert_eq!(got["kernel.cpu.1.stale"], Value::Bool(true)); + assert_eq!(got["kernel.cpu.1.mperf"], Value::U64(200)); + assert_eq!(got.keys().filter(|p| p.starts_with("kernel.cpu.2.")).count(), 1); + assert_eq!(got["kernel.cpu.2.stale"], Value::Bool(true)); + } + + #[test] + fn a_cpu_two_records_name_is_refused_by_its_path() { + assert_eq!(render(&[whole(3), whole(3)]), Err(Repeated("kernel.cpu.3.stale".into()))); + } +} diff --git a/toyos-inspect/src/lib.rs b/toyos-inspect/src/lib.rs index 46fcb96b11e..6fd823dd5d2 100644 --- a/toyos-inspect/src/lib.rs +++ b/toyos-inspect/src/lib.rs @@ -20,9 +20,11 @@ //! - [`Selector`]: the reader's frozen selector grammar. //! - [`line`] and [`json`]: the two renderings. //! - [`Invocation`]: the reader's command line. -//! - [`dev`]: the kernel's device inventory under `dev.*`, the one root no -//! port answers for — the reader asks the kernel for it on a `SysCap` -//! carrying `Rights::INVENTORY`. +//! - [`dev`]: the kernel's device inventory under `dev.*`, a root no port +//! answers for — the reader asks the kernel for it on a `SysCap` carrying +//! `Rights::INVENTORY`. +//! - [`kernel`]: the kernel's counters under `kernel.*`, asked for the same +//! way on `Rights::COUNTERS`. //! //! # The path grammar //! @@ -57,6 +59,7 @@ extern crate alloc; pub mod dev; +pub mod kernel; mod invocation; mod path; mod render; diff --git a/toyos-manifest/src/lib.rs b/toyos-manifest/src/lib.rs index 0dff91193ca..c27a6241616 100644 --- a/toyos-manifest/src/lib.rs +++ b/toyos-manifest/src/lib.rs @@ -124,6 +124,12 @@ const SYSCAP_RIGHTS: &[(&str, Rights)] = &[ // `SYS_DEVICE_INVENTORY`'s records. A census of the hardware and of which // program drives what, and `/system/bin/inspect` is what it is for. ("inventory", Rights::INVENTORY), + // Read the machine's counters: `SYS_COUNTERS`, each CPU's clock reading + // and how often firmware took it over. + ("counters", Rights::COUNTERS), + // Read what times every other program: each CPU's frequency, its busy + // fraction and its wake-ups. An admin tool's, and no applet's. + ("trace", Rights::TRACE), ]; /// The whole right set a program's `syscap` list asks for. diff --git a/toyos/src/syscap.rs b/toyos/src/syscap.rs index 4333a8473e1..9e20f70f690 100644 --- a/toyos/src/syscap.rs +++ b/toyos/src/syscap.rs @@ -2,7 +2,7 @@ //! //! Some things are reachable no other way — minting a device claim, entering //! the real-time band, listing every process in the machine, reading what the -//! machine is made of, and taking its power away, off or back to firmware — +//! machine is made of and what its CPUs count, and taking its power away, off or back to firmware — //! and each is one bit on a handle to this. The kernel makes exactly one at boot, for the supervisor, so the set of //! processes that can ever do any of them is exactly what the supervisor endowed. @@ -130,6 +130,19 @@ impl SysCap { syscall::device_inventory(self.0.raw(), buf) } + /// One `toyos_abi::counters` record per online CPU, into `buf`; answers + /// how many. An empty `buf` asks how many CPUs there are — see + /// [`syscall::counters`]. + /// + /// Needs [`Rights::COUNTERS`]; the counters that time programs come back + /// only where the capability carries [`Rights::TRACE`] too. + pub fn counters( + &self, + buf: &mut [toyos_abi::counters::RawRecord], + ) -> Result { + syscall::counters(self.0.raw(), buf) + } + /// Every inventory record, read whole or refused whole, never a shorter /// list; `buffer(n)` is `n` records to read into. /// diff --git a/userland/inspect/Cargo.toml b/userland/inspect/Cargo.toml index 90cdbe67045..7e794328448 100644 --- a/userland/inspect/Cargo.toml +++ b/userland/inspect/Cargo.toml @@ -16,4 +16,4 @@ toyos-abi = { path = "../../toyos-abi" } toyos-inspect = { path = "../../toyos-inspect" } [package.metadata.toyos.host] -exempt.manages = "ToyOS's machine inventory: what every owner's port and the kernel's `inventory` SysCap report" +exempt.manages = "ToyOS's machine inventory and counters: what every owner's port and the kernel's `inventory` and `counters` SysCap rights report" diff --git a/userland/inspect/src/main.rs b/userland/inspect/src/main.rs index cb71ae4373e..05002f9cb23 100644 --- a/userland/inspect/src/main.rs +++ b/userland/inspect/src/main.rs @@ -6,7 +6,8 @@ //! owner's port, and prints the paths the selector matches as `path = value` //! lines sorted by path, or as one JSON object. The grammar, the wire form and //! the renderings are `toyos-inspect`'s, and the question put to one owner is -//! [`inspect::ask`]; this file is which owners are asked, and the inventory. +//! [`inspect::ask`]; this file is which owners are asked, and the kernel's two +//! roots, the inventory and the counters. //! //! **An owner this process holds no connector for is a refusal, not a gap**: //! it is named on stderr and the run exits 2, so a pipe never mistakes a partial @@ -21,7 +22,9 @@ use std::io::Write; use inspect::ask; use toyos::endow::{Endowments, SYSCAP_LABEL}; use toyos::syscap::SysCap; +use toyos_abi::counters; use toyos_abi::inventory::{RawRecord, Record}; +use toyos_abi::syscall::SyscallError; use toyos_inspect::{Invocation, Value}; const USAGE: &str = "usage: inspect [--json] [SELECTOR]"; @@ -50,13 +53,21 @@ fn main() { } } - if run.selector.reaches(toyos_inspect::dev::ROOT) { - match inventory() { + // Taken once: taking an endowment is a swap, and both kernel roots ask on it. + let cap: Option = Endowments::get().take(SYSCAP_LABEL); + let kernel_roots: [(&str, fn(&SysCap) -> Result, String>); 2] = + [(toyos_inspect::dev::ROOT, inventory), (toyos_inspect::kernel::ROOT, cpu_counters)]; + for (root, ask) in kernel_roots.into_iter().filter(|(root, _)| run.selector.reaches(root)) { + let answer = cap.as_ref().ok_or_else(|| { + "this program holds no system capability, so the kernel's answer is not its to read" + .to_string() + }); + match answer.and_then(ask) { Ok(paths) => { found.extend(paths.into_iter().filter(|(path, _)| run.selector.matches(path))) } Err(why) => { - eprintln!("inspect: {}.*: {why}", toyos_inspect::dev::ROOT); + eprintln!("inspect: {root}.*: {why}"); refused = true; } } @@ -82,8 +93,9 @@ fn main() { /// The kernel's inventory, asked with this process's `SysCap`, and the /// machine `SYS_SYSINFO`'s ambient header describes, as `dev.*` paths. -fn inventory() -> Result, String> { - let records = records()?; +fn inventory(cap: &SysCap) -> Result, String> { + let records: Vec = + cap.records(|n| vec![RawRecord::EMPTY; n]).map_err(|why| why.to_string())?; let mut header = [0u8; toyos::system::SYSINFO_HEADER_SIZE]; if toyos::system::sysinfo(&mut header) != header.len() { return Err("the kernel wrote no machine header".to_string()); @@ -92,11 +104,20 @@ fn inventory() -> Result, String> { toyos_inspect::dev::render(&machine, &records).map_err(|why| why.to_string()) } -/// Every inventory record, asked with this process's `SysCap`. -fn records() -> Result, String> { - let Some(cap) = Endowments::get().take::(SYSCAP_LABEL) else { - return Err("this program holds no system capability, so the inventory is not its to read" - .to_string()); +/// Every CPU's counters, asked with this process's `SysCap`, as `kernel.*` +/// paths. The CPU count does not change after boot, so one count sizes the read. +fn cpu_counters(cap: &SysCap) -> Result, String> { + let refused = |e: SyscallError| match e { + SyscallError::PermissionDenied => { + "the kernel refused: this program's capability does not carry `counters`".to_string() + } + e => format!("the counters would not read: {e:?}"), }; - cap.records(|n| vec![RawRecord::EMPTY; n]).map_err(|why| why.to_string()) + let mut raw = vec![counters::RawRecord::EMPTY; cap.counters(&mut []).map_err(refused)?]; + let n = cap.counters(&mut raw).map_err(refused)?; + let records = raw[..n] + .iter() + .map(|r| counters::Record::decode(r).map_err(|why| format!("a record does not decode: {why:?}"))) + .collect::, _>>()?; + toyos_inspect::kernel::render(&records).map_err(|why| why.to_string()) }