diff --git a/.claude/agents/reviewer.md b/.claude/agents/reviewer.md index 3e6b39e6e16..9973b91643f 100644 --- a/.claude/agents/reviewer.md +++ b/.claude/agents/reviewer.md @@ -45,8 +45,11 @@ if it meets the bar above; otherwise it is a NOTE. ## What to look for -- **Fit.** Does the tree already do this? Is each new thing where it belongs: a pure decision in a - pure crate, the user/kernel boundary in `toyos-userbound`, a device claim in a userland server? +- **Fit.** Does the tree already do this? Is each new thing where it belongs: what one program alone + uses in that program's package, a decision of the kernel's in its library, and a crate of its own + only for what two programs share, one per subject, or for an input boundary, whose own source + decodes a word from outside its trust or bounds it by its form, whoever uses it; the user/kernel + boundary in `toyos-userbound`, a device claim in a userland server? One declaration read by every reader, refusal by name, authority moved in by the parent. Zero legacy: no shim, no workaround, no silent default. A BLOCKER each: a kernel addition that userland could own; a design made worse to spare the ABI. A new dependency only where it is the diff --git a/Cargo.lock b/Cargo.lock index 0b57a5dae2d..a664d0ed2a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -248,6 +248,17 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dlmalloc" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f5b01c17f85ee988d832c40e549a64bd89ab2c9f8d8a613bdf5122ae507e294" +dependencies = [ + "cfg-if", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "ed25519" version = "2.2.3" @@ -537,6 +548,38 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kernel" +version = "0.1.0" +dependencies = [ + "bcachefs", + "dlmalloc", + "hashbrown 0.16.1", + "rustc-demangle", + "toyos-abi", + "toyos-acpi", + "toyos-blackbox", + "toyos-blockhold", + "toyos-bootmap", + "toyos-dma", + "toyos-elf", + "toyos-elide", + "toyos-fat32", + "toyos-gicv3", + "toyos-gpt", + "toyos-hda", + "toyos-pci", + "toyos-ps2", + "toyos-quiesce", + "toyos-rootimage", + "toyos-symbols", + "toyos-tco", + "toyos-untrusted", + "toyos-userbound", + "toyos-wallclock", + "toyos-xhci", +] + [[package]] name = "kernel-loom" version = "0.1.0" @@ -545,6 +588,14 @@ dependencies = [ "toyos-abi", ] +[[package]] +name = "kernel-sim" +version = "0.1.0" +dependencies = [ + "kernel", + "rand", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -790,6 +841,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + [[package]] name = "rustc-std-workspace-core" version = "1.0.1" @@ -1131,6 +1188,7 @@ dependencies = [ "getrandom 0.3.4", "gpt", "image", + "kernel", "libc", "serde", "serde_json", @@ -1148,7 +1206,6 @@ dependencies = [ "toyos-logstream", "toyos-manifest", "toyos-quiesce", - "toyos-sched", "toyos-swap", "toyos-symbols", "toyos-tco", @@ -1344,17 +1401,6 @@ dependencies = [ "toyos-abi", ] -[[package]] -name = "toyos-pcid" -version = "0.1.0" - -[[package]] -name = "toyos-proclife" -version = "0.1.0" -dependencies = [ - "toyos-abi", -] - [[package]] name = "toyos-ps2" version = "0.1.0" @@ -1367,26 +1413,6 @@ version = "0.1.0" name = "toyos-rootimage" version = "0.1.0" -[[package]] -name = "toyos-sched" -version = "0.1.0" - -[[package]] -name = "toyos-sched-loom" -version = "0.1.0" -dependencies = [ - "loom", - "toyos-abi", -] - -[[package]] -name = "toyos-sched-sim" -version = "0.1.0" -dependencies = [ - "rand", - "toyos-sched", -] - [[package]] name = "toyos-swap" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 36ab963d8b9..17dbb338ff2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,8 @@ [workspace] members = [ "bcachefs", - "kernel-loom", + "kernel/loom", + "kernel/sim", "tests/libc-arch", "toyos-abi", "toyos-acpi", @@ -52,14 +53,9 @@ members = [ "toyos-net-udp", "toyos-net-wire", "toyos-pci", - "toyos-pcid", - "toyos-proclife", "toyos-ps2", "toyos-quiesce", "toyos-rootimage", - "toyos-sched", - "toyos-sched/loom", - "toyos-sched/sim", "toyos-swap", "toyos-symbols", "toyos-tco", @@ -80,7 +76,9 @@ default-members = ["."] exclude = [ # Cross-compiled by a `.cargo/config.toml` below them, to targets no host # test can run. Cargo refuses to merge an inherited `build.target` away, so - # these cannot be members at any price. + # these cannot be members at any price. The kernel's library is tested on the + # host from its own manifest all the same, and `kernel/loom` and `kernel/sim` + # are members: `members` names them, which outranks this list. "bootloader", "kernel", "userland", @@ -94,9 +92,15 @@ exclude = [ # `cargo test` in it against the host triple. "toyos", # Guest programs `src/build.rs` builds for `x86_64-unknown-toyos`, each from - # its own directory. Named, and not `tests/` whole, so the gate finds a host - # test under `tests/` that joined neither list. + # its own directory but `tls-multi-crate/dep`, its parent's dependency. Named, + # and not `tests/` whole, so the gate finds a host test under `tests/` that + # joined neither list. "tests/toyos-rust-tests", + "tests/toyos-rust-tests/tls-cranelift", + "tests/toyos-rust-tests/tls-dlopen-lib", + "tests/toyos-rust-tests/tls-lib", + "tests/toyos-rust-tests/tls-multi-crate", + "tests/toyos-rust-tests/tls-multi-crate/dep", # A host judge: a second implementation, whose dependency graph is not the # harness's to resolve (`src/build.rs`, `build_host_judges`). "tests/ssh-client-host", @@ -185,7 +189,9 @@ toyos-userbound = { path = "toyos-userbound" } # declaration. Pure and `no_std`, so it compiles for the host as it does for # the guest. toyos-i219 = { path = "toyos-i219" } -toyos-sched = { path = "toyos-sched" } +# The scheduler's quantum and the windows' hold, so the harness judges a boot +# against the constants the kernel schedules by. +kernel = { path = "kernel" } # The Bulk-Only phases, so the harness judging a wedge reads the word # `toyos_xhci::bot::Phase` declares instead of spelling it a second time. toyos-xhci = { path = "toyos-xhci" } diff --git a/issues/build/a-member-under-an-excluded-directory-leaves-the-workspace-with-its-gate-green.md b/issues/build/a-member-under-an-excluded-directory-leaves-the-workspace-with-its-gate-green.md deleted file mode 100644 index 9d6c322521f..00000000000 --- a/issues/build/a-member-under-an-excluded-directory-leaves-the-workspace-with-its-gate-green.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -status: assigned -kind: tooling -opened: 2026-10-03 ---- - -# A member under an excluded directory leaves the host workspace with its gate green - -`src/hostws.rs`'s `crate_dirs` prunes each `exclude` entry's directory whole, -so `every_crate_in_the_tree_joined_the_workspace_or_was_excluded_by_name` -never sees a `Cargo.toml` under one. Cargo does not: a directory `members` -names is a member even under an excluded one. That line is then the only thing -that makes it one, and dropping it takes the package's tests out of -`--ci host` with the gate green. `crate_dirs`'s doc, "pruned exactly as cargo -prunes them", is false. - -**Evidence:** at `0d3ec4388`, `exclude` named `tests` whole and `members` -named `tests/libc-arch`. Its `--ci host` ran `toyos_libc_copies`'s 35 tests. -With that member line dropped (`mutation-member.patch`, posted on #692), -`cargo test -p toyos-build --lib hostws::` exited 0. #692 ends that instance -by excluding the two packages under `tests/` that keep their own resolution by -name; at its head `31d730b63`, no member sits under an `exclude` entry. - -**The instance:** step 2 of -`issues/build/code-used-by-one-program-lives-in-that-program.md` makes -`kernel/loom` and `kernel/sim` members, and `exclude` names `kernel` whole. - -**Exit:** dropping from `members` a line that names a directory under an -`exclude` entry reds a `--ci host` gate, shown by that mutation on -`kernel/loom` or `kernel/sim` once step 2 has moved them, and `crate_dirs`'s -doc is true of the walk. Owner: the orchestrator, who owns the track's step 2. diff --git a/issues/build/cargo-run-inside-kernel-loom-or-kernel-sim-builds-for-a-bare-target.md b/issues/build/cargo-run-inside-kernel-loom-or-kernel-sim-builds-for-a-bare-target.md new file mode 100644 index 00000000000..11e9e3c3780 --- /dev/null +++ b/issues/build/cargo-run-inside-kernel-loom-or-kernel-sim-builds-for-a-bare-target.md @@ -0,0 +1,25 @@ +--- +status: assigned +kind: tooling +opened: 2026-10-03 +--- + +# Cargo run inside `kernel/loom` or `kernel/sim` builds for a bare target + +Cargo reads `.cargo/config.toml` from the working directory up, so a `cargo +test` run inside `kernel/loom` or `kernel/sim` takes `kernel/.cargo/config.toml`'s +`build.target = "x86_64-unknown-none"` and its `rustflags`. Run from the +repository root, `-p kernel-loom` and `-p kernel-sim` build for the host, which +is how `--ci host` runs them. + +**Evidence:** inside each, `cargo test --no-run` exits 101 on `can't find crate +for std` and `the x86_64-unknown-none target may not support the standard +library`: in `kernel/loom` on loom's own dependencies (`scoped-tls`, `log`, +`once_cell`), in `kernel/sim` on `getrandom`. A `.cargo/config.toml` of the +package's own with `build.target = "host-tuple"` overrides it (measured on +cargo 1.98.1), but builds into `target//` with the kernel's +`rustflags`: a second build of every crate the root's run builds. + +**Exit:** `cargo test` run inside `kernel/loom` and inside `kernel/sim` builds +for the host and is green. Held by the orchestrator, who put both under +`kernel/`. diff --git a/issues/build/code-used-by-one-program-lives-in-that-program.md b/issues/build/code-used-by-one-program-lives-in-that-program.md index 07e9be4b303..03f37595ae3 100644 --- a/issues/build/code-used-by-one-program-lives-in-that-program.md +++ b/issues/build/code-used-by-one-program-lives-in-that-program.md @@ -9,8 +9,7 @@ opened: 2026-09-29 A crate exists because two programs share it. What only the kernel uses is the kernel package's, what only one userland program uses is that program's, and shared crates with one subject are one crate (owner, 2026-09-29). No gate holds -the layout: step 2 writes it into `.claude/agents/reviewer.md`'s Fit line, -which until then puts a pure decision in a pure crate. +the layout: `.claude/agents/reviewer.md`'s Fit line states it. An input boundary is a crate of its own, whoever uses it: the no-panic track (`issues/kernel/a-panic-is-never-an-accident.md`) forbids its tier 1 per @@ -22,26 +21,12 @@ program named is neither. No step here merges one; each stays a crate under the no-panic track. Every step lands green on `--ci host` and `--build-only`. Test counts are what -`cargo test -p -- --list` lists today. Step 2 lands right after #592 -and #631, of which #631 has landed, before the latency work -(`issues/kernel/toyos-beats-linuxs-latency-on-the-t14.md`; owner, 2026-10-03). +`cargo test -p -- --list` lists today. Step 4 lands before the +latency work (`issues/kernel/toyos-beats-linuxs-latency-on-the-t14.md`): asked +whether all five steps land in the window right after #592, the owner chose +"All five steps — The whole consolidation right after #592 lands, one step per +change, before the speed work starts" (2026-10-03). -2. **The kernel's library.** `kernel/pure/` is the `kernel` package's lib, and - its bin is `test = false`. `toyos-pcid`, `toyos-proclife` and `toyos-sched` - move in. `kernel-loom` and `toyos-sched/loom` become `kernel/loom/`, and - `toyos-sched/sim` `kernel/sim/`. The harness dev-depends on the kernel, and - the build system does not depend on it. The library has no `tests/`, since - an integration test builds the binary for the host. `--ci host` tests it - with `sched-check`, the feature scheduler tests need. The Fit line - states this track's rule. - Closes `issues/build/the-pcid-negative-control-runs-nowhere.md`. - Check: the library lists at least 135 tests, `kernel/loom` 79 and - `kernel/sim` 53, and `--clippy` lints the library's tests on the host. - Every moved control, and pcid's `counting-allocator`, reds with its verdict, - and `declared_model_controls` reads the kernel's manifest and every one in - the host workspace, not a list. An `unsafe {}` planted in a module that was - `forbid(unsafe_code)` does not compile. `cargo tree -e normal -p - toyos-build` names no `kernel`. 4. **A crate one package uses goes under it, a crate of its own.** A crate of this tree with exactly one consumer moves under it. Its consumers are the packages that name it as a dependency of any kind, under any `cfg`, as diff --git a/issues/build/the-guest-suite-runs-only-what-no-cheaper-tier-reaches.md b/issues/build/the-guest-suite-runs-only-what-no-cheaper-tier-reaches.md index 45e79cd0810..3d92716960f 100644 --- a/issues/build/the-guest-suite-runs-only-what-no-cheaper-tier-reaches.md +++ b/issues/build/the-guest-suite-runs-only-what-no-cheaper-tier-reaches.md @@ -81,7 +81,7 @@ console's renderer. Exit: as stage A. ## Stage C: CPUs, memory, scheduling and the kernel log -Metal: test kernels armed on the T14. Host: `toyos-sched` and its sim, `kernel-loom`, +Metal: test kernels armed on the T14. Host: `kernel::sched` and `kernel-sim`, `kernel-loom`, `toyos-wallclock`. Exit: as stage A; an unrepresentable name is a type the kernel builds against and a compile-fail case. diff --git a/issues/build/the-pcid-negative-control-runs-nowhere.md b/issues/build/the-pcid-negative-control-runs-nowhere.md deleted file mode 100644 index 0cd7cfc69c4..00000000000 --- a/issues/build/the-pcid-negative-control-runs-nowhere.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-09-29 ---- - -# `toyos-pcid`'s negative control runs nowhere - -`toyos-pcid/Cargo.toml` declares `counting-allocator`, the control that reverts -`PcidPool` to the counter that reissued a live tag. Nothing runs it: -`src/ci.rs`'s `CONTROLS` has no row for it, and `src/build.rs`'s -`declared_model_controls` does not read `toyos-pcid/Cargo.toml`, so -`every_model_control_is_run` cannot notice. The control still has teeth, run by -hand: `cargo test -p toyos-pcid --features counting-allocator` exits 101 with -`tests::two_live_address_spaces_never_share_a_pcid ... FAILED`. - -**Exit:** the control is a `CONTROLS` row demanding that `FAILED` line, and -`declared_model_controls` reads every manifest in the host workspace rather than -a list, so a control declared in a crate the list forgot reds. diff --git a/issues/build/toyos-pcids-counting-allocator-control-is-run-by-no-gate.md b/issues/build/toyos-pcids-counting-allocator-control-is-run-by-no-gate.md deleted file mode 100644 index f30215ac83d..00000000000 --- a/issues/build/toyos-pcids-counting-allocator-control-is-run-by-no-gate.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-09-29 ---- - -# `toyos-pcid`'s `counting-allocator` control is run by no gate - -`toyos-pcid/Cargo.toml` declares `counting-allocator` as the negative control -its isolation tests must red under, and `oracle.rs`'s -`the_counting_allocator_produces_a_cross_space_read` exists only with it on. -No row of `src/ci.rs`'s `CONTROLS` names it and nothing in `src/` or -`.github/` passes it: `src/build.rs`'s `declared_model_controls` reads six -model manifests, and `toyos-pcid`'s is not one of them. Clippy compiles it -(`src/clippy.rs`'s `UNCONTROLLED`); nothing runs it. - -Exit: a `CONTROLS` row with its verdicts, and `toyos-pcid/Cargo.toml` among the -manifests `declared_model_controls` reads. diff --git a/issues/kernel/a-bounded-post-spends-its-wake-on-a-waiter-that-then-dies.md b/issues/kernel/a-bounded-post-spends-its-wake-on-a-waiter-that-then-dies.md index fb601c00de6..ec77998b07b 100644 --- a/issues/kernel/a-bounded-post-spends-its-wake-on-a-waiter-that-then-dies.md +++ b/issues/kernel/a-bounded-post-spends-its-wake-on-a-waiter-that-then-dies.md @@ -20,6 +20,6 @@ claim through `scheduler::wake_sched`. ## Exit condition A bounded post never counts a waiter whose commit answers `Killed` — a model in -`toyos-sched/loom/tests/loom_watch.rs` with two waiters on one token, one of +`kernel/loom/tests/loom_watch.rs` with two waiters on one token, one of them killed while committing, and a `post_n(token, 1)` racing it, where the live one must be reached. diff --git a/issues/kernel/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md b/issues/kernel/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md index 9d51c4fa901..03404324522 100644 --- a/issues/kernel/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md +++ b/issues/kernel/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md @@ -133,7 +133,7 @@ directory is `issues/isolation/every-program-sees-only-the-files-it-was-given.md the notice once one is handled, ignored or blocked: an ignored reason is dropped and an unhandled one ends the process with exit code 128 plus the signal's number. `kill` with one of the three is a quit with that reason. - *Exit*: host — `toyos-proclife`'s interleavings of a notice's read racing a + *Exit*: host — `kernel::proclife`'s interleavings of a notice's read racing a second quit kill only where the read comes second; of a spawn racing a quit on its spawner leave each child holding the reason once or started after the read; of a quit on a process that never watched leave every notice below it diff --git a/issues/kernel/a-kill-on-a-spawns-handle-before-its-child-lands-claims-nothing.md b/issues/kernel/a-kill-on-a-spawns-handle-before-its-child-lands-claims-nothing.md index 3689e076bcc..d2618675d9f 100644 --- a/issues/kernel/a-kill-on-a-spawns-handle-before-its-child-lands-claims-nothing.md +++ b/issues/kernel/a-kill-on-a-spawns-handle-before-its-child-lands-claims-nothing.md @@ -24,6 +24,6 @@ the same window are sound. *Exit*: a kill on a handle whose process has not landed ends that process — the landing claims it, as it claims a child under a claimed place — or the -handle resolves only once the child has landed; `toyos-proclife` scripts the +handle resolves only once the child has landed; `kernel::proclife` scripts the kill as a step between the commit and the landing, and no schedule leaves the child running. diff --git a/issues/kernel/a-tlb-shootdown-reaches-only-the-cpus-that-ran-the-space.md b/issues/kernel/a-tlb-shootdown-reaches-only-the-cpus-that-ran-the-space.md index 8c19c4414d9..f559b50598c 100644 --- a/issues/kernel/a-tlb-shootdown-reaches-only-the-cpus-that-ran-the-space.md +++ b/issues/kernel/a-tlb-shootdown-reaches-only-the-cpus-that-ran-the-space.md @@ -20,7 +20,7 @@ CPU touches a page the initiator unmaps and then faults; on each proving machine with PCID, the sibling touches the page and blocks off its pinned CPU, the initiator unmaps it while the sibling is blocked and a thread of another space runs there, and the sibling faults on touching it again after it wakes -there; cases in `kernel-loom/tests/tlb_shootdown.rs` switch a CPU into the +there; cases in `kernel/loom/tests/tlb_shootdown.rs` switch a CPU into the space, and out of it, while the initiator reads the target set; the T14's munmap figure. **Mutation**, each red: a target set that omits a CPU the space ran on; a CPU that joins the set after loading CR3; a CPU cleared from the set diff --git a/issues/kernel/desktop-window-child-freeze.md b/issues/kernel/desktop-window-child-freeze.md index 40727892834..e6e58d22998 100644 --- a/issues/kernel/desktop-window-child-freeze.md +++ b/issues/kernel/desktop-window-child-freeze.md @@ -171,9 +171,9 @@ written about. NMI probe taken on a reproduction confirms no CPU stopped taking scheduler passes during the freeze — nothing short of that instrument distinguishes this signature from a green run, which this entry has already shown proves nothing -either way. Owner: `toyos-sched`, the placement track that closed the +either way. Owner: `kernel::sched`, the placement track that closed the CPU-selection half of this family (`CpuHandle::answering`, -`toyos-sched/src/cpu.rs`) and is nearest the remaining half; held by the +`kernel/pure/sched/cpu.rs`) and is nearest the remaining half; held by the orchestrator. **Its test is deleted**: `cd685b10a` and `b20d3fd40` took `desktop_window_child` diff --git a/issues/kernel/thread-exits-completion-post-is-the-second-one.md b/issues/kernel/thread-exits-completion-post-is-the-second-one.md index e7b9739b119..c16fbc16c55 100644 --- a/issues/kernel/thread-exits-completion-post-is-the-second-one.md +++ b/issues/kernel/thread-exits-completion-post-is-the-second-one.md @@ -60,5 +60,5 @@ outcome on the same watch at `kernel/src/sched/payload.rs:222`. Two posts to one subject where the second is load-bearing regardless is either an undocumented promptness guarantee or dead code on the exit path — "one of two sentences and the evidence for it" is owed work, not an observation. Owed by the lifecycle -work in `toyos-proclife`, which is the only machine that can enumerate the +work in `kernel::proclife`, which is the only machine that can enumerate the interleaving either answer rests on. diff --git a/kernel/CLAUDE.md b/kernel/CLAUDE.md index fd6888d1e3e..f5ef4aeb294 100644 --- a/kernel/CLAUDE.md +++ b/kernel/CLAUDE.md @@ -1,6 +1,6 @@ # Kernel -The module header at the site owns its subsystem — read it before changing a module. The scheduler core is `toyos-sched/`, driven from `kernel/src/sched/`; every user transition's machine state is `arch/*/fpu.rs`, with AArch64's `entry.rs` and `trap.rs`; every syscall is `kernel/src/syscall/`, and `dispatch.rs` decodes every user pointer the ABI takes. +The module header at the site owns its subsystem — read it before changing a module. The scheduler core is `kernel/pure/sched/`, driven from `kernel/src/sched/`; every user transition's machine state is `arch/*/fpu.rs`, with AArch64's `entry.rs` and `trap.rs`; every syscall is `kernel/src/syscall/`, and `dispatch.rs` decodes every user pointer the ABI takes. ## Caveats that bite every agent @@ -14,7 +14,7 @@ The module header at the site owns its subsystem — read it before changing a m - **`drain_irqs` is the drivers' engine and nothing on it may wait** — a blocking call there empties the audio pipeline on every plug. - **A syscall that can block resolves its handle and clones the object out before it blocks** — a `with_object`/`with_process_data` guard held across a park is a runtime panic no compile check catches; the `SYS_FSYNC` arm in `syscall/dispatch.rs` is the pattern. - **A block-layer `BudgetExpired` is not-durable-yet and never a loss** — it is retried on a fresh budget above every lock; a flush that discards its pages on one splits a FAT mirror. -- **A decision the process table makes lives in `toyos-proclife`, never in `process.rs`** — its defects are interleavings and that crate is the only machine that can enumerate one. +- **A decision the process table makes lives in `kernel/pure/proclife/`, never in `process.rs`** — its defects are interleavings and its host tests are the only machine that can enumerate one. - **A task holds at most one watch registration** — a standing registration across a loop must not call anything that registers again. A double registration panics only at attempt ≥ 2, so the contention depth is the coverage. - **A console is per holder, minted at spawn** — the object *is* the line buffer. - **`ops::close` cancels a poll only for a source its object really ends** — `Watch::cancel_polls` answers every ring's poll on that watch; `ops::close_ends_polls` is where a new object kind answers. diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index 90c5ca15e1a..d7c8738dd07 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -50,12 +50,9 @@ dependencies = [ "toyos-gpt", "toyos-hda", "toyos-pci", - "toyos-pcid", - "toyos-proclife", "toyos-ps2", "toyos-quiesce", "toyos-rootimage", - "toyos-sched", "toyos-symbols", "toyos-tco", "toyos-untrusted", @@ -141,17 +138,6 @@ dependencies = [ "toyos-abi", ] -[[package]] -name = "toyos-pcid" -version = "0.1.0" - -[[package]] -name = "toyos-proclife" -version = "0.1.0" -dependencies = [ - "toyos-abi", -] - [[package]] name = "toyos-ps2" version = "0.1.0" @@ -164,10 +150,6 @@ version = "0.1.0" name = "toyos-rootimage" version = "0.1.0" -[[package]] -name = "toyos-sched" -version = "0.1.0" - [[package]] name = "toyos-symbols" version = "0.1.0" diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index b3c602b7550..f3d04a81dcb 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -15,28 +15,28 @@ license = "MIT OR Apache-2.0" loom = [] # Declared, never enabled here, and never enabled by any *kernel* build: # `kernel-loom` turns it on to remove the two `SeqCst` fences, and -# `kernel-loom/tests/log_wake.rs` must red when it does. A model that has never +# `kernel/loom/tests/log_wake.rs` must red when it does. A model that has never # failed proves nothing, and no guest test can stand in for it — on x86 a build # with either fence removed behaves identically, which is the whole reason the # obligation is a model. wake-fence-off = [] # The same for the panic console's published descriptor: `kernel-loom` turns it on # to drop the publisher's `Release` fence from `drivers/panic_console/published.rs`, -# and `kernel-loom/tests/panic_console_publish.rs` must red — a reader keeping a +# and `kernel/loom/tests/panic_console_publish.rs` must red — a reader keeping a # descriptor that is half one publication and half the next. seqlock-writer-fence-off = [] # The same for the console backend's lock: `kernel-loom` turns it on to build # `drivers/serial_lock.rs`'s `try_lock` with `then_some`, whose lost exchange -# releases the holder's lock, and `kernel-loom/tests/serial_lock.rs` must red. +# releases the holder's lock, and `kernel/loom/tests/serial_lock.rs` must red. serial-try-lock-then-some = [] # The same arrangement for a poll ring's one-shot answer: `kernel-loom` turns it # on to split `inbox/once.rs`'s exchange into a load and a store, and -# `kernel-loom/tests/poll_once.rs` must red when it does — a poll two posts +# `kernel/loom/tests/poll_once.rs` must red when it does — a poll two posts # both answer. poll-fire-load-store = [] # The same for a poll ring's answer: `kernel-loom` turns it on so # `inbox/polls.rs` answers a fired poll without looking at its object again, -# and `kernel-loom/tests/inbox_answer.rs` must red — a post with nothing to +# and `kernel/loom/tests/inbox_answer.rs` must red — a post with nothing to # read, answered. post-is-an-answer = [] # The same arrangement, five more times, one per `kernel-loom` model that had @@ -70,19 +70,137 @@ dump-report-relaxed = [] # *contended*-acquire model this tree has: `src/sleeplock.rs`'s two loads of # `now` — the ones that decide whose turn it is — go `Relaxed`, so the previous # holder's writes are unordered against the next holder's reads, and -# `kernel-loom/tests/sleep_lock.rs` reds. `Lock::lock`'s spin is unreachable by +# `kernel/loom/tests/sleep_lock.rs` reds. `Lock::lock`'s spin is unreachable by # loom (`src/sync.rs`'s `ACQUIRED` says so at the site); a parking acquire has # no unbounded branch, which is what makes this one modellable at all. sleeplock-acquire-off = [] # The same arrangement for the CPU roster's publication edge: `src/smp_roster.rs`'s # `commit` publishes the count with a `Relaxed` store instead of the release the # slot store needs, so a reader can see a count over an unfilled slot, and -# `kernel-loom/tests/smp_bringup.rs` reds. +# `kernel/loom/tests/smp_bringup.rs` reds. roster-commit-relaxed = [] # The same for the release/answer word: `src/smp_roster.rs` grows the base's # second store back, so a shootdown can see the machine released while it is not -# yet answering, and `kernel-loom/tests/smp_bringup.rs` reds. +# yet answering, and `kernel/loom/tests/smp_bringup.rs` reds. smp-ready-split = [] +# The scheduler core's loom controls: `kernel-loom` compiles `pure/sched/` +# into its own crate and turns each on, one at a time; declared here only so +# `cfg` checking knows the names. +# +# The doorbell's kick-pending edge in `pure/sched/mailbox.rs`'s +# `Doorbell::ring` goes `Relaxed`, and `kernel/loom/tests/loom_sleep.rs` must +# red — the sleep handshake's own negative control. +doorbell-kick-relaxed = [] +# The `SeqCst` fence `pure/sched/cpu.rs`'s `balance_fence` puts between the push +# half's surplus store and its read of a target's SLEEPING bit is weakened, and +# `kernel/loom/tests/loom_push.rs` must red — `Balance::PushOnSurplus`'s own +# negative control. +push-fence-relaxed = [] +# `pure/sched/task.rs`'s `begin_commit` goes blind to the notified bit — a post +# landing between a waiter's registration on a watch and its commit is lost — +# and `kernel/loom/tests/loom_watch.rs` must red. +commit-ignores-notify = [] +# `pure/sched/task.rs`'s notify answers a post that finds its bits already set +# off a load rather than a read-modify-write, and +# `kernel/loom/tests/loom_watch.rs` must red. +notify-flag-load-only = [] +# The two `SeqCst` fences of `pure/sched/watch.rs`'s `Gate` go, and +# `kernel/loom/tests/loom_watch.rs` must red — the stop's store-buffering pair. +gate-fence-off = [] +# The staleness test's negative control: `pure/sched/cpu.rs`'s +# `CpuHandle::answering` answers `true` for every CPU under it, which is what +# every path that chooses a CPU believed before it existed. `kernel-sim`'s +# `a_stopped_cpu_stops_taking_work` must red when it is on. No kernel build +# selects it, and one that did would place tasks onto CPUs that never pick +# them up. +placement-ignores-staleness = [] +# Compiles the escape hatch `kernel-sim`'s `old_steal_port` needs to re-create +# the OLD scheduler's steal-in-transit algorithm. The simulator enables it; no +# kernel build does. +protocol-port = [] +# `pure/pcid/`'s negative control: it reverts `PcidPool` to the bare monotonic +# counter that wrapped at 4095 and reissued a live tag, and the isolation tests +# red under it. Never a default and reached by no kernel build. +counting-allocator = [] +# The process lifecycle's controls (`pure/proclife/`), declared for the +# library's own tests and never enabled by any kernel build: each reverts one +# decision to the form this kernel really had, so the host model that refuses +# it is shown able to red at all. `src/ci.rs`'s `CONTROLS` runs each and +# demands the named test's own FAILED line; `src/build.rs`'s +# `every_model_control_is_run` reds on a control declared with no row behind +# it. +# +# `teardown::claim_teardown` stops being exclusive: every arrival wins. An exit +# and a kill then both tear one process down, both retire its threads and both +# publish an exit — which `ProcessObject::publish_exit` asserts against at run +# time, in a guest, on whichever boot the two happen to meet. +# `interleave::tests::an_exit_and_a_kill_never_both_tear_a_process_down` must +# red under this, and it does so in every ordering rather than in one. +mutate-claim-teardown-always-wins = [] +# `spawn::admit_thread_insert` stops re-reading the teardown claim, so the +# question is asked once — at the top of `process::spawn_thread`, before the +# TLS block, the kernel stack and the scheduler record are built. Everything +# that makes a thread happens in that window, and teardown claims the process +# under the very lock the insert takes. `interleave::tests::a_published_exit_ +# leaves_no_unretired_thread` must red under this. +mutate-spawn-skips-the-insert-recheck = [] +# The model's kill waits for its victim's exit before it returns. +# `interleave::tests::two_processes_killing_each_other_both_end` must red under +# this: each killer waits on a process whose thread is inside the other's kill. +mutate-kill-waits-for-its-victims = [] +# `teardown::leave` answers `Last` to every thread that leaves a claimed +# process, so the first one out tears it down under its siblings. +# `interleave::tests::an_exit_and_a_kill_never_both_tear_a_process_down` must +# red under this. +mutate-first-out-tears-down = [] +# `teardown::leave` marks the last one out dead before its teardown runs, so +# the machine's stop finds nothing running while that teardown frees and logs. +# `interleave::tests::the_last_one_out_is_in_its_process_until_its_teardown_is_done` +# must red under this. +mutate-last-out-leaves-before-its-teardown = [] +# `join::collect_zombie` gives up a thread of a process being torn down, whose +# mappings its siblings may still run in. +# `interleave::tests::a_join_racing_the_kill_that_takes_its_target` must red +# under this. +mutate-join-collects-in-a-teardown = [] +# `tree::land_child` stops re-reading its place's claim, so a spawn admitted +# before a kill lands unclaimed under the place after the walk read its children. +# `interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it` +# must red under this. +mutate-place-skips-the-insert-recheck = [] +# `tree::refuse_child` keeps a failed spawn's hold on its place, which then +# waits for a child that will never be and is never published. The same test +# must red under this. +mutate-refused-spawn-keeps-the-count = [] +# `tree::land_child` claims a child landed under a place claimed since its +# admission and answers no thread to retire, so the child runs on claimed, is +# never torn down, and holds its place unpublished. +# `interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it` +# must red under this. +mutate-landed-child-retires-nothing = [] +# `tree::teardown_done` publishes an end with its own teardown, whatever is +# still unpublished below it. `interleave::tests::an_exit_publishes_after_every_end_below_it` +# must red under this. +mutate-publish-before-the-children = [] +# The model's walk claims everything it owes in one lock section, as a kernel +# that held `PROCESS_TABLE` across its walk would. +# `interleave::tests::a_spawn_under_an_unrelated_process_lands_between_two_claims_of_one_walk` +# must red under this. +mutate-walk-in-one-hold = [] +# The model's spawn mints its caller's handle to the child once the spawn has +# landed and its retires are posted, as a kernel that installed it after +# `loader::spawn` returned would: a child claimed as it lands closes its table, +# its own `self` with it, before that. +# `interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it` +# must red under this. +mutate-spawner-handle-after-the-landing = [] +# The model's spawn mints its caller's handle to the child under the caller's +# lock and the child's own once that lock is given up, as a kernel whose commit +# installed the one before it minted the other would: another thread of the +# caller closes the only handle the object has. +# `interleave::tests::a_sibling_closing_a_spawns_handle_before_the_spawn_returns` +# must red under this. +mutate-spawner-handle-before-the-childs-own = [] # **Every actuator, and which one is armed comes from the boot parameter.** # `kernel/src/actuator.rs` declares every one of them with the claim each rests on; this # name is the whole of what a build decides about them. A kernel without it has @@ -121,19 +239,16 @@ test-actuators = [] # `cargo run -- --debug` exists to debug is the kernel an image ships. debug-wait = [] # The on-target half of the scheduler's invariants. -# Turns on `toyos-sched/check`, which asserts container-vs-state-word agreement -# and invariant T at the end of every scheduler pass, and asserts the pass -# itself fits `toyos_sched::cpu::MAX_PASS_NS` — the term the simulator's -# invariant I4 leaves out, because a simulated pass costs nothing. +# Asserts container-vs-state-word agreement and invariant T at the end of every +# scheduler pass (`pure/sched/invariants.rs`), and asserts the pass itself fits +# `kernel::sched::cpu::MAX_PASS_NS` — the term the simulator's invariant I4 +# leaves out, because a simulated pass costs nothing. # # Off by default because it is not free: each pass gains a walk of this CPU's # ready queue and parked map plus a second clock read, and that lands on the # audio path. A bring-up, bisect or bare-metal build wants it; a build being # measured does not. -# -# Not a boot parameter, and the one that could not be: it forwards to a -# dependency's cargo feature, which no runtime value can reach. -sched-check = ["toyos-sched/check"] +sched-check = [] # The longest interrupts-off and preemption-off window each CPU closed since the # last report, printed beside the IRQ census (`src/windows.rs`). Its own build # because it reads the counter at every masking transition and every preempt @@ -144,7 +259,7 @@ mask-windows = [] # bytes, taken when the driver's exclusive region is left and compared when it is # next entered, so a word that changed while nothing was allowed to change it is # reported as an offset, a field, a before and an after. `sched::driver`'s -# `tripwire` module and `toyos_sched::cpu`'s `tripwire` impl block carry the +# `tripwire` module and `kernel::sched::cpu`'s `tripwire` impl block carry the # argument, and the four deaths it exists for are named there. # # **Not `sched-check`, and the distinction is the whole point.** That one asserts @@ -158,10 +273,7 @@ mask-windows = [] # bytes at every entry to that region, which is every pass and every # `running`-peek. It compiles readers and no decision, so a kernel carrying it # schedules exactly as one without it. -# -# Not a boot parameter, and the one that could not be, for `sched-check`'s -# reason: it forwards to a dependency's cargo feature. -sched-tripwire = ["toyos-sched/tripwire"] +sched-tripwire = [] # The tripwire one layer below `sched-tripwire`: a band of known bytes on each # side of **every** heap allocation, read back when it is freed and, for the # running task's kernel stack, at every scheduler pass. `kernel/src/mm/alloc.rs` @@ -363,9 +475,21 @@ strip = "debuginfo" debug-assertions = true overflow-checks = true +[lib] +path = "pure/lib.rs" + +# The library's tests run on the host; the binary builds for a bare target only. +[[bin]] +name = "kernel" +path = "src/main.rs" +test = false + [dependencies] -bcachefs = { path = "../bcachefs", default-features = false } toyos-abi = { path = "../toyos-abi" } + +# The binary's alone, so a host build of the library compiles none of them. +[target.'cfg(target_os = "none")'.dependencies] +bcachefs = { path = "../bcachefs", default-features = false } toyos-acpi = { path = "../toyos-acpi" } toyos-dma = { path = "../toyos-dma" } toyos-elide = { path = "../toyos-elide" } @@ -378,13 +502,10 @@ toyos-gicv3 = { path = "../toyos-gicv3" } toyos-gpt = { path = "../toyos-gpt" } toyos-hda = { path = "../toyos-hda" } toyos-pci = { path = "../toyos-pci" } -toyos-pcid = { path = "../toyos-pcid" } toyos-tco = { path = "../toyos-tco" } -toyos-proclife = { path = "../toyos-proclife" } toyos-ps2 = { path = "../toyos-ps2" } toyos-rootimage = { path = "../toyos-rootimage" } toyos-quiesce = { path = "../toyos-quiesce" } -toyos-sched = { path = "../toyos-sched" } toyos-symbols = { path = "../toyos-symbols" } toyos-untrusted = { path = "../toyos-untrusted" } toyos-userbound = { path = "../toyos-userbound" } diff --git a/kernel-loom/Cargo.toml b/kernel/loom/Cargo.toml similarity index 59% rename from kernel-loom/Cargo.toml rename to kernel/loom/Cargo.toml index 9111bcfc29f..2cbf96cb562 100644 --- a/kernel-loom/Cargo.toml +++ b/kernel/loom/Cargo.toml @@ -6,26 +6,22 @@ edition = "2021" license = "MIT OR Apache-2.0" publish = false -# This package exists only to run the loom model checker over `kernel/src/sync.rs`, -# the ticket spinlock the whole kernel rests on. It compiles that one file into -# its own crate with `feature = "loom"` on, so its atomics and its cell become -# loom-instrumented ones and the model explores the real primitive rather than a -# transliteration of it — a transliteration is exactly the divergence risk a -# model checker is meant to remove. +# This package exists only to run the loom model checker over the kernel's own +# sources: `kernel/src/sync.rs`, the ticket spinlock the whole kernel rests on, +# the other kernel files `src/lib.rs` names, and the scheduler core, +# `kernel/pure/sched/`. It compiles them into its own crate with +# `feature = "loom"` on, so their atomics and cells become loom-instrumented ones +# and the models explore the real primitives rather than transliterations of +# them — a transliteration is exactly the divergence risk a model checker is +# meant to remove. # # A separate package, so `loom` and its transitive host crates stay out of -# `kernel/Cargo.lock` — the same arrangement, and the same reason, as -# `toyos-sched/loom`. It sits beside `kernel/` rather than inside it because -# `kernel/.cargo/config.toml` cross-compiles everything below it to -# `x86_64-unknown-none`, and cargo refuses to merge an inherited `build.target` -# away. -# -# The lockfile that isolates loom is no longer this package's own: it is a -# member of the host workspace (root `Cargo.toml`) and resolves in -# `/Cargo.lock`. What matters is unchanged and is what that sentence always -# meant — `kernel/` is *excluded* from that workspace and resolves against -# `kernel/Cargo.lock`, and nothing the kernel builds depends on this package, -# so loom and its transitive crates appear in neither. +# `kernel/Cargo.lock`: this is a member of the host workspace (root +# `Cargo.toml`) and resolves in `/Cargo.lock`, `kernel/` is excluded from that +# workspace and resolves against `kernel/Cargo.lock`, and nothing the kernel +# builds depends on this package. Cargo runs it from the repository root: run +# below `kernel/`, `kernel/.cargo/config.toml` cross-compiles it to a bare +# target. [features] default = ["loom"] @@ -34,7 +30,7 @@ loom = [] # `shard.rs`'s wake pair, and `log_wake.rs` must red under it — the one command # that proves the model has teeth: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features wake-fence-off \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features wake-fence-off \ # --test log_wake # # Never on by default and never reachable from a kernel build, which declares @@ -44,16 +40,21 @@ wake-fence-off = [] # `inbox/once.rs`'s exchange into a load and a store, so a post and the # registrant's own recheck can both answer one poll, and `poll_once.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features poll-fire-load-store \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features poll-fire-load-store \ # --test poll_once # -# Never on by default, and no kernel build can reach it. +# Never on by default, and no kernel build can reach it. `loom_watch.rs`'s ring +# entry is the same `Once`, and its three `*_completes_exactly_once` models +# must red under it too: +# +# cargo test -p kernel-loom --features poll-fire-load-store --test loom_watch +# poll-fire-load-store = [] # The negative control for a poll ring's answer. It makes `inbox/polls.rs` # answer a fired poll with its interest and look at nothing, so a post with # nothing behind it is an answer and `inbox_answer.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features post-is-an-answer \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features post-is-an-answer \ # --test inbox_answer # # Never on by default, and no kernel build can reach it. @@ -63,7 +64,7 @@ post-is-an-answer = [] # previous owner's writes are unordered against the next owner's reads, and # `ticket_lock.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features lock-acquire-off \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features lock-acquire-off \ # --test ticket_lock # # Never on by default and never reachable from a kernel build, which declares @@ -73,7 +74,7 @@ lock-acquire-off = [] # `reap_gate.rs`'s `raise` store `Relaxed`, so a claimer is handed a gate # without the work it was raised for, and `reap_gate.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features reap-raise-relaxed \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features reap-raise-relaxed \ # --test reap_gate # # Never on by default and never reachable from a kernel build, which declares @@ -83,7 +84,7 @@ reap-raise-relaxed = [] # `shootdown.rs`'s `serve` read what it owes `Relaxed`, so the flush no longer # postdates the initiator's page-table write, and `tlb_shootdown.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features shootdown-serve-relaxed \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features shootdown-serve-relaxed \ # --test tlb_shootdown # # Never on by default and never reachable from a kernel build, which declares @@ -94,7 +95,7 @@ shootdown-serve-relaxed = [] # reader can see a count over a slot the commit has not filled, and # `smp_bringup.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features roster-commit-relaxed \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features roster-commit-relaxed \ # --test smp_bringup # # Never on by default and never reachable from a kernel build, which declares @@ -105,7 +106,7 @@ roster-commit-relaxed = [] # stores — so a shootdown can see the machine released while it is not yet # answering, and `smp_bringup.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features smp-ready-split \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features smp-ready-split \ # --test smp_bringup # # Never on by default and never reachable from a kernel build, which declares @@ -116,7 +117,7 @@ smp-ready-split = [] # `Relaxed`, so a reader can find the number with the body behind it stale or # half written, and `log_record.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features log-commit-release-off \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features log-commit-release-off \ # --test log_record # # Never on by default and never reachable from a kernel build, which declares @@ -129,7 +130,7 @@ log-commit-release-off = [] # mut accesses` — the reader reaching the shard's atomics while the publisher # is still building them: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features shard-publish-relaxed \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features shard-publish-relaxed \ # --test log_publish # # Never on by default and never reachable from a kernel build, which declares @@ -141,7 +142,7 @@ shard-publish-relaxed = [] # holder's reads, and `sleep_lock.rs` must red with loom's own words, # `Causality violation: Concurrent write accesses to UnsafeCell`: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features sleeplock-acquire-off \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features sleeplock-acquire-off \ # --test sleep_lock # # Never on by default and never reachable from a kernel build, which declares @@ -151,7 +152,7 @@ sleeplock-acquire-off = [] # reader's `swap` of the count becomes a load and a store, which is the whole of # what this record's design is, and `device_irq.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features device-irq-lossy \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features device-irq-lossy \ # --test device_irq # # Never on by default and never reachable from a kernel build, which declares @@ -161,7 +162,7 @@ device-irq-lossy = [] # `dump_request.rs`'s `take` and `end_report` `Relaxed`, so a report is # unordered against the one before it, and `dump_request.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features dump-report-relaxed \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features dump-report-relaxed \ # --test dump_request # # Never on by default and never reachable from a kernel build, which declares @@ -173,7 +174,7 @@ dump-report-relaxed = [] # behind it unwritten, and `log_ring.rs` must red with loom's own words, # `Causality violation`: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features log-ring-publish-relaxed \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features log-ring-publish-relaxed \ # --test log_ring # # Never on by default; the SDK declares the same name only so `cfg` checking @@ -191,7 +192,7 @@ log-ring-loads-swapped = [] # reader can keep words of a publication whose mark it never saw, and # `panic_console_publish.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features seqlock-writer-fence-off \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features seqlock-writer-fence-off \ # --test panic_console_publish # # Never on by default and never reachable from a kernel build, which declares @@ -203,18 +204,93 @@ seqlock-writer-fence-off = [] # lost exchange too and whose drop then releases the holder's lock, and # `serial_lock.rs` must red: # -# cargo test --manifest-path kernel-loom/Cargo.toml --features serial-try-lock-then-some \ +# cargo test --manifest-path kernel/loom/Cargo.toml --features serial-try-lock-then-some \ # --test serial_lock # # Never on by default and never reachable from a kernel build, which declares # the same name only so `cfg` checking knows it. serial-try-lock-then-some = [] +# The scheduler core's models, `tests/loom_*.rs`. +# +# Model the mandatory preempt-disabled push away. The +# `preempted_producer_strands_suffix` case must FAIL under this feature — +# `tests/loom_mailbox.rs` asserts exactly that. +no-preempt-guard = [] +# The negative control for the sleep handshake's kick edge. +# It makes `mailbox.rs`'s `Doorbell::ring` read the doorbell's bits `Relaxed` +# instead of `AcqRel`, so a producer can miss a consumer's freshly published +# `SLEEPING` bit and elide the IPI that would have woken it, and +# `loom_sleep.rs` must red under it — the one command that proves the model +# has teeth: +# +# cargo test -p kernel-loom --features doorbell-kick-relaxed \ +# --test loom_sleep +# +# Never on by default; the kernel declares the same name only so `cfg` +# checking knows it. +doorbell-kick-relaxed = [] +# The negative control for `Balance::PushOnSurplus`'s ordering. It weakens +# `cpu.rs`'s `balance_fence` from a `SeqCst` fence to a release one, which +# orders stores against stores and nothing against a later load — so the push's +# surplus store and the sleeper's SLEEPING store may each be missed by the +# other's read, and the CPU halts with a surplus published and no probe +# outstanding. `loom_push.rs` must red under it: +# +# cargo test -p kernel-loom --features push-fence-relaxed --test loom_push +# +push-fence-relaxed = [] +# The one feature here that models nothing away. It lets the mailbox model's +# victim stop taking passes — what a CPU that stops does — and the model must +# red, because `MailboxNode::in_flight` is the consumer's to clear and a +# retired consumer clears nothing. It reproduces an open defect rather than +# proving a lie is caught, so unlike its siblings it goes away with the fix: +# +# cargo test -p kernel-loom --features victim-retires-mid-probe \ +# --test loom_mailbox +# +# The kernel declares no twin, because this decides nothing in +# `kernel/pure/sched/`. +victim-retires-mid-probe = [] +# The negative control for the watch. It makes `task.rs`'s `begin_commit` +# blind to the notified bit, which is a waiter that registers, checks and parks +# without the post that landed in between being able to stop it — the lost wake +# the watch exists to make unrepresentable — and `loom_watch.rs` must red: +# +# cargo test -p kernel-loom --features commit-ignores-notify --test loom_watch +# +commit-ignores-notify = [] +# The negative control for the notify's flagged arm. A post that finds its +# bits already set answers off a load instead of a read-modify-write, so a +# second post can read the word from before the waiter consumed the first, and +# `loom_watch.rs` must red: +# +# cargo test -p kernel-loom --features notify-flag-load-only --test loom_watch +# +notify-flag-load-only = [] +# The negative control for the stop's gate. It removes the two `SeqCst` fences +# of `watch.rs`'s `Gate`, so a poster's transition and the stop's sweep may +# each miss the other, and `loom_watch.rs` must red: +# +# cargo test -p kernel-loom --features gate-fence-off --test loom_watch +# +gate-fence-off = [] +# The control for `poll_racing`'s lost-completion half: its producer posts, then +# stores the readiness, the reverse of the order `pcidev::note_fault` keeps, so +# both `*_racing_a_post*_completes_exactly_once` models must red with a poll +# completed by neither: +# +# cargo test -p kernel-loom --features fault-posted-before-it-is-set --test loom_watch +# +# The kernel declares no twin, because this decides nothing in +# `kernel/pure/sched/`. +fault-posted-before-it-is-set = [] + [dependencies] loom = "0.7" # The record layout the shard is a ring of. `no_std`, no atomics of its own, # so it compiles into the model unchanged. -toyos-abi = { path = "../toyos-abi" } +toyos-abi = { path = "../../toyos-abi" } # **The models are compiled with optimisation, and that is a stack decision.** # `generator`, which loom's coroutines are built on, gives each one 32 KiB — @@ -234,8 +310,9 @@ path = "src/lib.rs" # so this crate exposes no unit tests of its own. test = false -# The kernel's actuator build: a shared file compiles a method for it that no -# model's feature turns on. A `cfg` name and not a feature, since a feature -# here is a model control. +# The kernel's features its shared files name and no model turns on: the +# actuator build, the scheduler core's instruments, and the controls the +# library's tests and the simulator run. A `cfg` name and not a feature, since a +# feature here is a model control. [lints.rust] -unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("boot-actuators"))'] } +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("boot-actuators", "sched-check", "sched-tripwire", "protocol-port", "placement-ignores-staleness"))'] } diff --git a/kernel-loom/src/lib.rs b/kernel/loom/src/lib.rs similarity index 90% rename from kernel-loom/src/lib.rs rename to kernel/loom/src/lib.rs index fecb8fd876e..e4896839edb 100644 --- a/kernel-loom/src/lib.rs +++ b/kernel/loom/src/lib.rs @@ -162,19 +162,19 @@ macro_rules! log { ($($arg:tt)*) => {{ let _ = format_args!($($arg)*); }}; } -#[path = "../../kernel/src/sync.rs"] +#[path = "../../src/sync.rs"] pub mod sync; -#[path = "../../kernel/src/shootdown.rs"] +#[path = "../../src/shootdown.rs"] pub mod shootdown; /// The CPU roster and the release/answer word, driven by `tests/smp_bringup.rs`; /// under `loom` alone, as is the [`arch::cpu`] shim it names. #[cfg(feature = "loom")] -#[path = "../../kernel/src/smp_roster.rs"] +#[path = "../../src/smp_roster.rs"] pub mod smp_roster; -#[path = "../../kernel/src/log/shard.rs"] +#[path = "../../src/log/shard.rs"] pub mod log_shard; /// `registry.rs` names its shard as `super::shard`, which in the kernel is @@ -183,14 +183,14 @@ pub mod log_shard; /// feature is on, which the crate's other invocation depends on. pub use log_shard as shard; -#[path = "../../kernel/src/log/registry.rs"] +#[path = "../../src/log/registry.rs"] pub mod log_registry; /// The walks `SYS_LOG_READ` and the console drain take, driven by /// `tests/log_cursor.rs` over real shards; without `loom` alone, as is the /// [`shards`] shim it names. #[cfg(not(feature = "loom"))] -#[path = "../../kernel/src/log/read.rs"] +#[path = "../../src/log/read.rs"] pub mod log_read; // The shards `read.rs` walks: in the kernel cpu0's and every AP's published @@ -213,23 +213,23 @@ pub fn install_shards(shards: [Option<&'static shard::Shard>; toyos_abi::log::MA SHARDS.with(|cell| cell.set(shards)); } -#[path = "../../kernel/src/sched/reap_gate.rs"] +#[path = "../../src/sched/reap_gate.rs"] pub mod reap_gate; /// Ctrl+Alt+D's request word. Pure atomics, so it compiles here unshimmed. -#[path = "../../kernel/src/sched/dump_request.rs"] +#[path = "../../src/sched/dump_request.rs"] pub mod dump_request; /// The interrupt record a claimed PCI function's ISR writes and its holder /// reads. Pure `core` atomics and nothing else, so it compiles here unshimmed. -#[path = "../../kernel/src/pcidev/record.rs"] +#[path = "../../src/pcidev/record.rs"] pub mod device_irq; /// A poll ring's one-shot answer. It names atomics and nothing else, and that /// narrowness is load-bearing: a file that named a ring's page or a watch could /// not be compiled here at all, and the race would stop being checked by /// anything. -#[path = "../../kernel/src/inbox/once.rs"] +#[path = "../../src/inbox/once.rs"] pub mod poll_once; /// `polls.rs` names its one-shot as `super::once`, which in the kernel is @@ -241,7 +241,7 @@ extern crate alloc; /// A ring's polls and when one is answered, driven against a fake object by /// `tests/inbox_answer.rs`. It names the one-shot above, `toyos-abi` and /// `alloc`, and nothing of the kernel's. -#[path = "../../kernel/src/inbox/polls.rs"] +#[path = "../../src/inbox/polls.rs"] pub mod inbox_polls; /// What `sleeplock.rs` names of the kernel's watch, and nothing more. @@ -253,8 +253,8 @@ pub mod inbox_polls; /// therefore proves is the ticket arithmetic and the acquire/release edge — /// mutual exclusion, ordering, and the order contenders are served in. What it /// does **not** prove is the wake handshake: that a post reaches a waiter -/// registered on a watch is `toyos-sched`'s, modelled in -/// `toyos-sched/loom/tests/loom_watch.rs` against the real watch and park. +/// registered on a watch is the scheduler core's, modelled in +/// `tests/loom_watch.rs` against the real watch and park. /// /// `Watch` is a stand-in for the same reason: nothing registers here, so /// [`watch::Watch::post_n`] has nobody to tell and answers zero. @@ -308,7 +308,7 @@ pub mod watch { /// /// Four items, and the arithmetic in [`TaskId`] is the whole of what is /// restated here rather than compiled — `kernel/src/scheduler.rs` names the -/// process table, the `toyos-sched` driver and half the kernel besides, so it +/// process table, the scheduler core's driver and half the kernel besides, so it /// cannot be a `#[path]` module the way `sync.rs` and `inbox/once.rs` are. pub mod scheduler { /// The right to give the CPU back. In the kernel its constructor asserts @@ -382,22 +382,22 @@ pub mod scheduler { /// The sleep lock, compiled a second time against loom's atomics. Its /// dependency surface is the two shim modules above and nothing else — the same /// narrowness `inbox/once.rs` carries, and for the same reason. -#[path = "../../kernel/src/sleeplock.rs"] +#[path = "../../src/sleeplock.rs"] pub mod sleeplock; /// The i8042's interrupt tally. `tests/i8042_tally.rs` is the only model whose /// subject is a *driver*, and it is here for the reason the others are: the /// property is "no reader ever sees this pair disagree", which is a claim about /// instants that no guest test can express and that x86's TSO hides. -#[path = "../../kernel/src/arch/x86_64/i8042/tally.rs"] +#[path = "../../src/arch/x86_64/i8042/tally.rs"] pub mod i8042_tally; /// The panic snapshot's owner and access state, driven together by /// `tests/panic_capture.rs`. -#[path = "../../kernel/src/drivers/panic_console/latch.rs"] +#[path = "../../src/drivers/panic_console/latch.rs"] pub mod capture_latch; -#[path = "../../kernel/src/drivers/panic_console/access.rs"] +#[path = "../../src/drivers/panic_console/access.rs"] pub mod capture_access; /// A program's log ring: not the kernel's, and here because it is the one @@ -405,16 +405,32 @@ pub mod capture_access; /// needs the same loom this crate already carries. `tests/log_ring.rs` drives /// its two protocols against loom's atomics; the file names nothing outside /// itself, so it needs no shim. -#[path = "../../toyos/src/log/ring.rs"] +#[path = "../../../toyos/src/log/ring.rs"] pub mod log_ring; /// The panic console's published framebuffer descriptor, driven by /// `tests/panic_console_publish.rs`. -#[path = "../../kernel/src/drivers/panic_console/published.rs"] +#[path = "../../src/drivers/panic_console/published.rs"] pub mod panic_console_published; /// The console backend's lock, driven by `tests/serial_lock.rs`; under `loom` /// alone, as is the [`arch::cpu`] shim it names. #[cfg(feature = "loom")] -#[path = "../../kernel/src/drivers/serial_lock.rs"] +#[path = "../../src/drivers/serial_lock.rs"] pub mod serial_lock; + +/// The scheduler core, `kernel/pure/sched/`, driven by `tests/loom_*.rs` over +/// [`sched_model`]'s scaffolding. +/// +/// Loom owns the primitives the simulator's step granularity assumes correct — +/// mailbox push/drain, doorbell edges, the ticket CAS protocol, kill-bit vs wake +/// ordering, retire-node re-post, the sleep handshake — and the simulator +/// (`kernel/sim/`) owns the protocol above them. Loom does not scale to the +/// whole scheduler; the simulator does not model weak memory. Under `loom` +/// alone, as is every model that drives it. +#[cfg(feature = "loom")] +#[path = "../../pure/sched/mod.rs"] +pub mod sched; + +#[cfg(feature = "loom")] +pub mod sched_model; diff --git a/toyos-sched/loom/src/model.rs b/kernel/loom/src/sched_model.rs similarity index 94% rename from toyos-sched/loom/src/model.rs rename to kernel/loom/src/sched_model.rs index f58573bb9fd..74b40339434 100644 --- a/toyos-sched/loom/src/model.rs +++ b/kernel/loom/src/sched_model.rs @@ -1,15 +1,17 @@ //! Scaffolding shared by the loom models: the message type, the modelled //! preempt count, the cell lock and the kick recorder. +#![deny(unsafe_code)] + use loom::sync::atomic::{AtomicUsize, Ordering}; use loom::sync::{Mutex, MutexGuard}; -use crate::hw::{CpuId, Kicker}; -use crate::mailbox::{PreemptGuard, SchedMsg}; -use crate::sync::Arc; -use crate::task::{TaskKey, TaskShared, WakeCause, WakeReason}; -use crate::sync::CellLock; -use crate::watch::Waiters; +use crate::sched::hw::{CpuId, Kicker}; +use crate::sched::mailbox::{PreemptGuard, SchedMsg}; +use crate::sched::sync::Arc; +use crate::sched::task::{TaskKey, TaskShared, WakeCause, WakeReason}; +use crate::sched::sync::CellLock; +use crate::sched::watch::Waiters; pub const CPU0: CpuId = CpuId(0); pub const CPU1: CpuId = CpuId(1); diff --git a/kernel-loom/tests/device_irq.rs b/kernel/loom/tests/device_irq.rs similarity index 97% rename from kernel-loom/tests/device_irq.rs rename to kernel/loom/tests/device_irq.rs index d5370bf87fb..e2c42d979bf 100644 --- a/kernel-loom/tests/device_irq.rs +++ b/kernel/loom/tests/device_irq.rs @@ -15,7 +15,7 @@ //! — a cargo feature rather than a comment: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features device-irq-lossy \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features device-irq-lossy \ //! --test device_irq //! ``` //! diff --git a/kernel-loom/tests/dump_request.rs b/kernel/loom/tests/dump_request.rs similarity index 98% rename from kernel-loom/tests/dump_request.rs rename to kernel/loom/tests/dump_request.rs index dde1309fd04..538cf5ade57 100644 --- a/kernel-loom/tests/dump_request.rs +++ b/kernel/loom/tests/dump_request.rs @@ -8,7 +8,7 @@ //! The negative control relaxes the report's handoff in the real word, and this file must red: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features dump-report-relaxed \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features dump-report-relaxed \ //! --test dump_request //! ``` #![cfg(feature = "loom")] diff --git a/kernel-loom/tests/i8042_tally.rs b/kernel/loom/tests/i8042_tally.rs similarity index 100% rename from kernel-loom/tests/i8042_tally.rs rename to kernel/loom/tests/i8042_tally.rs diff --git a/kernel-loom/tests/inbox_answer.rs b/kernel/loom/tests/inbox_answer.rs similarity index 99% rename from kernel-loom/tests/inbox_answer.rs rename to kernel/loom/tests/inbox_answer.rs index d51108fb5d4..7266833180d 100644 --- a/kernel-loom/tests/inbox_answer.rs +++ b/kernel/loom/tests/inbox_answer.rs @@ -11,14 +11,14 @@ //! In `loom::model`, because the one-shot under each poll is built from //! loom's atomics in this crate; every model here is one thread, and a watch //! another thread submits during a look is staged inside the look. A fire -//! racing the submitter's park is `toyos-sched-loom`'s +//! racing the submitter's park is `loom_watch.rs`'s //! `a_fire_racing_a_submitters_park_is_never_lost`, and two submitters of one //! ring its `an_answer_wakes_the_submitter_its_look_hid_the_poll_from`. //! //! The negative case is a cargo feature: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features post-is-an-answer \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features post-is-an-answer \ //! --test inbox_answer //! ``` //! diff --git a/kernel-loom/tests/log_body_words.rs b/kernel/loom/tests/log_body_words.rs similarity index 100% rename from kernel-loom/tests/log_body_words.rs rename to kernel/loom/tests/log_body_words.rs diff --git a/kernel-loom/tests/log_cursor.rs b/kernel/loom/tests/log_cursor.rs similarity index 100% rename from kernel-loom/tests/log_cursor.rs rename to kernel/loom/tests/log_cursor.rs diff --git a/kernel-loom/tests/log_publish.rs b/kernel/loom/tests/log_publish.rs similarity index 97% rename from kernel-loom/tests/log_publish.rs rename to kernel/loom/tests/log_publish.rs index cdf8bf48139..2ed5de4312e 100644 --- a/kernel-loom/tests/log_publish.rs +++ b/kernel/loom/tests/log_publish.rs @@ -10,7 +10,7 @@ //! The negative case is a cargo feature rather than a comment: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features shard-publish-relaxed \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features shard-publish-relaxed \ //! --test log_publish //! ``` //! diff --git a/kernel-loom/tests/log_record.rs b/kernel/loom/tests/log_record.rs similarity index 99% rename from kernel-loom/tests/log_record.rs rename to kernel/loom/tests/log_record.rs index 80b8712e671..c1149dc1f03 100644 --- a/kernel-loom/tests/log_record.rs +++ b/kernel/loom/tests/log_record.rs @@ -25,7 +25,7 @@ //! The negative case is a cargo feature rather than a comment: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features log-commit-release-off \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features log-commit-release-off \ //! --test log_record //! ``` //! diff --git a/kernel-loom/tests/log_ring.rs b/kernel/loom/tests/log_ring.rs similarity index 99% rename from kernel-loom/tests/log_ring.rs rename to kernel/loom/tests/log_ring.rs index ee0db147a39..c358af67430 100644 --- a/kernel-loom/tests/log_ring.rs +++ b/kernel/loom/tests/log_ring.rs @@ -11,7 +11,7 @@ //! The negative control is a cargo feature: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features log-ring-publish-relaxed \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features log-ring-publish-relaxed \ //! --test log_ring //! ``` //! diff --git a/kernel-loom/tests/log_wake.rs b/kernel/loom/tests/log_wake.rs similarity index 97% rename from kernel-loom/tests/log_wake.rs rename to kernel/loom/tests/log_wake.rs index 4cc42094f1b..e8567160a37 100644 --- a/kernel-loom/tests/log_wake.rs +++ b/kernel/loom/tests/log_wake.rs @@ -14,7 +14,7 @@ //! targets**, which is why the obligation is a model and why the model ships in //! the same chunk as the code. //! -//! **What this model is not about.** The park itself is `toyos-sched`'s +//! **What this model is not about.** The park itself is the scheduler core's //! two-phase commit: a producer that wins the flag writes `klogd`'s word — //! claiming it if it is committing or parked, flagging it if it is running — //! and the task's own next commit refuses to park over either. That handshake @@ -26,7 +26,7 @@ //! The negative case is a cargo feature rather than a comment: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features wake-fence-off \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features wake-fence-off \ //! --test log_wake //! ``` //! diff --git a/kernel-loom/tests/log_zeroed_init.rs b/kernel/loom/tests/log_zeroed_init.rs similarity index 100% rename from kernel-loom/tests/log_zeroed_init.rs rename to kernel/loom/tests/log_zeroed_init.rs diff --git a/kernel-loom/tests/loom_exploration.rs b/kernel/loom/tests/loom_exploration.rs similarity index 100% rename from kernel-loom/tests/loom_exploration.rs rename to kernel/loom/tests/loom_exploration.rs diff --git a/toyos-sched/loom/tests/loom_mailbox.rs b/kernel/loom/tests/loom_mailbox.rs similarity index 98% rename from toyos-sched/loom/tests/loom_mailbox.rs rename to kernel/loom/tests/loom_mailbox.rs index b200a38c061..0db13dfc424 100644 --- a/toyos-sched/loom/tests/loom_mailbox.rs +++ b/kernel/loom/tests/loom_mailbox.rs @@ -33,12 +33,14 @@ //! the forbidden interleaving, which is what //! `preempted_producer_strands_suffix` asserts. +#![cfg(feature = "loom")] + use loom::sync::atomic::{AtomicBool, Ordering}; use loom::sync::Arc; -use toyos_sched_loom::mailbox::{ +use kernel_loom::sched::mailbox::{ mailbox, Doorbell, MailboxConsumer, MailboxNode, MailboxProducer, Urgency, }; -use toyos_sched_loom::model::{model, IrqGuard, Msg, PreemptModel, RemoteGuard}; +use kernel_loom::sched_model::{model, IrqGuard, Msg, PreemptModel, RemoteGuard}; const LOCAL: Msg = Msg::Probe(1); const IRQ: Msg = Msg::Probe(2); @@ -158,7 +160,7 @@ fn preempt_disabled_push_keeps_the_queue_drainable() { /// The mandated negative case: with the guard modelled away the very same /// model MUST fail. Run it with -/// `cargo test -p toyos-sched-loom --features no-preempt-guard`; the caught +/// `cargo test -p kernel-loom --features no-preempt-guard`; the caught /// panic (printed by loom together with the offending execution) is the /// evidence. #[cfg(feature = "no-preempt-guard")] diff --git a/toyos-sched/loom/tests/loom_park.rs b/kernel/loom/tests/loom_park.rs similarity index 94% rename from toyos-sched/loom/tests/loom_park.rs rename to kernel/loom/tests/loom_park.rs index 2599fbd0178..998ab40d389 100644 --- a/toyos-sched/loom/tests/loom_park.rs +++ b/kernel/loom/tests/loom_park.rs @@ -5,12 +5,14 @@ //! race a *registration* runs against a post is the watch's, and //! `loom_watch.rs` has it. +#![cfg(feature = "loom")] + use loom::sync::Arc; -use toyos_sched_loom::cpu::{CpuHandle, CpuHandles}; -use toyos_sched_loom::mailbox::{mailbox, MailboxConsumer}; -use toyos_sched_loom::model::{model, Kicks, Msg, PreemptModel, RemoteGuard, CPU0}; -use toyos_sched_loom::park::{notify, prepare, Cancel, Cancelled, Commit, CurrentTask}; -use toyos_sched_loom::task::{ +use kernel_loom::sched::cpu::{CpuHandle, CpuHandles}; +use kernel_loom::sched::mailbox::{mailbox, MailboxConsumer}; +use kernel_loom::sched_model::{model, Kicks, Msg, PreemptModel, RemoteGuard, CPU0}; +use kernel_loom::sched::park::{notify, prepare, Cancel, Cancelled, Commit, CurrentTask}; +use kernel_loom::sched::task::{ Notify, TaskKey, TaskShared, TaskState, WaitClass, WakeCause, WakeReason, }; diff --git a/toyos-sched/loom/tests/loom_push.rs b/kernel/loom/tests/loom_push.rs similarity index 95% rename from toyos-sched/loom/tests/loom_push.rs rename to kernel/loom/tests/loom_push.rs index b87f7970c9c..b2a5a30a27e 100644 --- a/toyos-sched/loom/tests/loom_push.rs +++ b/kernel/loom/tests/loom_push.rs @@ -35,7 +35,7 @@ //! The negative control is a cargo feature rather than a comment: //! //! ```text -//! cargo test -p toyos-sched-loom --features push-fence-relaxed --test loom_push +//! cargo test -p kernel-loom --features push-fence-relaxed --test loom_push //! ``` //! //! makes `cpu::balance_fence` a release fence — which orders stores against @@ -43,17 +43,19 @@ //! [`a_cpu_that_halts_without_seeing_the_surplus_was_pushed`]. Verified //! 2026-08-22, both ways round. +#![cfg(feature = "loom")] + use loom::sync::atomic::{AtomicUsize, Ordering}; use loom::sync::Arc; -use toyos_sched_loom::cpu::{balance_fence, CpuHandle}; -use toyos_sched_loom::hw::CpuId; -use toyos_sched_loom::mailbox::{mailbox, Kick, MailboxConsumer}; -use toyos_sched_loom::model::{model, Msg}; +use kernel_loom::sched::cpu::{balance_fence, CpuHandle}; +use kernel_loom::sched::hw::CpuId; +use kernel_loom::sched::mailbox::{mailbox, Kick, MailboxConsumer}; +use kernel_loom::sched_model::{model, Msg}; /// The surplus at which a pass pushes and at which an idle pass probes — one /// inequality, `SchedPass::best_victim`'s, taken from the core so the model /// stages the shipped threshold. -const THRESHOLD: u32 = toyos_sched_loom::cpu::PUSH_THRESHOLD; +const THRESHOLD: u32 = kernel_loom::sched::cpu::PUSH_THRESHOLD; /// Two CPUs' shared faces: the one that gains surplus, and the one going to /// sleep. The `CpuHandle` is the real one, so `publish_surplus`, `surplus`, diff --git a/toyos-sched/loom/tests/loom_retire.rs b/kernel/loom/tests/loom_retire.rs similarity index 97% rename from toyos-sched/loom/tests/loom_retire.rs rename to kernel/loom/tests/loom_retire.rs index 3b59e6382a9..aecaf9b6116 100644 --- a/toyos-sched/loom/tests/loom_retire.rs +++ b/kernel/loom/tests/loom_retire.rs @@ -8,22 +8,24 @@ //! plus a message, so the cases worth checking are exactly the orderings of //! that bit and that node. +#![cfg(feature = "loom")] + use std::sync::atomic::{AtomicBool, Ordering as StdOrdering}; use loom::sync::Arc; -use toyos_sched_loom::cpu::{Balance, CpuHandle, CpuHandles, CpuSched, Env, RunToken, SchedPass}; -use toyos_sched_loom::fair::{FairShare, Frontier, ShareState}; -use toyos_sched_loom::hw::{CpuId, Hw, Kicker, Machine, Nanos, TraceEvent}; -use toyos_sched_loom::mailbox::{mailbox, MailboxConsumer, Urgency}; -use toyos_sched_loom::model::{ +use kernel_loom::sched::cpu::{Balance, CpuHandle, CpuHandles, CpuSched, Env, RunToken, SchedPass}; +use kernel_loom::sched::fair::{FairShare, Frontier, ShareState}; +use kernel_loom::sched::hw::{CpuId, Hw, Kicker, Machine, Nanos, TraceEvent}; +use kernel_loom::sched::mailbox::{mailbox, MailboxConsumer, Urgency}; +use kernel_loom::sched_model::{ model, IrqGuard, Kicks, LoomLock, Msg, PreemptModel, RemoteGuard, CPU0, CPU1, }; -use toyos_sched_loom::retire; -use toyos_sched_loom::task::{ +use kernel_loom::sched::retire; +use kernel_loom::sched::task::{ RtState, SchedPayload, TaskAccounting, TaskBuilder, TaskKey, TaskShared, TaskState, WaitClass, WakeCause, WakeReason, }; -use toyos_sched_loom::park::{notify, prepare, Cancel, Commit, CurrentTask}; +use kernel_loom::sched::park::{notify, prepare, Cancel, Commit, CurrentTask}; struct World { cpus: CpuHandles, @@ -293,7 +295,7 @@ fn a_retire_and_a_wake_never_both_claim_a_parked_task() { // The retirer: the kill bit, the message, and then — on the CPU that // owns the task — the claim `handle_retire` now makes. retire::begin(&task).post(&world.cpus, &world.kicks, &RemoteGuard); - let retire_claimed = matches!(task.claim_wake(), toyos_sched_loom::task::Claim::Parked(_)); + let retire_claimed = matches!(task.claim_wake(), kernel_loom::sched::task::Claim::Parked(_)); let wake_claimed = waker.join().unwrap(); let msgs = drain(&mut rx[0], &world.preempt); @@ -341,7 +343,7 @@ impl SchedPayload for Payload { /// The full message set (`crate::msg::Msg`), as opposed to the reduced [`Msg`] /// the primitive models use: a pass consumes `Adopt` and `Retire` too. -type TaskMsg = toyos_sched_loom::msg::Msg; +type TaskMsg = kernel_loom::sched::msg::Msg; /// An `Hw` that records nothing. Every question this model asks is about which /// of `CpuSched`'s own containers the task is in, and `CpuSched` answers those diff --git a/toyos-sched/loom/tests/loom_sleep.rs b/kernel/loom/tests/loom_sleep.rs similarity index 96% rename from toyos-sched/loom/tests/loom_sleep.rs rename to kernel/loom/tests/loom_sleep.rs index 6a23f817265..c3f1fede98b 100644 --- a/toyos-sched/loom/tests/loom_sleep.rs +++ b/kernel/loom/tests/loom_sleep.rs @@ -19,7 +19,7 @@ //! comment: //! //! ```text -//! cargo test -p toyos-sched-loom --features doorbell-kick-relaxed \ +//! cargo test -p kernel-loom --features doorbell-kick-relaxed \ //! --test loom_sleep //! ``` //! @@ -27,12 +27,14 @@ //! [`a_halted_cpu_with_queued_work_was_kicked`], whose assertion names the //! failure exactly: halted with messages still queued and no IPI in flight. +#![cfg(feature = "loom")] + use loom::sync::atomic::{AtomicUsize, Ordering}; use loom::sync::Arc; -use toyos_sched_loom::mailbox::{ +use kernel_loom::sched::mailbox::{ mailbox, Doorbell, Kick, MailboxConsumer, MailboxNode, MailboxProducer, Urgency, }; -use toyos_sched_loom::model::{model, Msg, PreemptModel, RemoteGuard}; +use kernel_loom::sched_model::{model, Msg, PreemptModel, RemoteGuard}; struct World { tx: MailboxProducer, diff --git a/toyos-sched/loom/tests/loom_watch.rs b/kernel/loom/tests/loom_watch.rs similarity index 97% rename from toyos-sched/loom/tests/loom_watch.rs rename to kernel/loom/tests/loom_watch.rs index 042924459b2..7fa996713e4 100644 --- a/toyos-sched/loom/tests/loom_watch.rs +++ b/kernel/loom/tests/loom_watch.rs @@ -15,7 +15,7 @@ //! and `a_post_racing_a_registration_leaves_nobody_parked` must red: //! //! ```text -//! cargo test -p toyos-sched-loom --features commit-ignores-notify --test loom_watch +//! cargo test -p kernel-loom --features commit-ignores-notify --test loom_watch //! ``` //! //! Four more controls: `notify-flag-load-only` lets a post that finds its bits @@ -37,36 +37,32 @@ //! page, the objects its looks read — names half the kernel and cannot be //! compiled here, so the models count answers instead of writing them. //! -//! [`TaskShared::notify`]: toyos_sched_loom::task::TaskShared::notify -//! [`Gate`]: toyos_sched_loom::watch::Gate +//! [`TaskShared::notify`]: kernel_loom::sched::task::TaskShared::notify +//! [`Gate`]: kernel_loom::sched::watch::Gate //! [`Once`]: once::Once -//! [`prepare`]: toyos_sched_loom::park::prepare -//! [`Ring::fire`]: toyos_sched_loom::watch::Ring::fire +//! [`prepare`]: kernel_loom::sched::park::prepare +//! [`Ring::fire`]: kernel_loom::sched::watch::Ring::fire + +#![cfg(feature = "loom")] extern crate alloc; use loom::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use loom::sync::Arc; -use toyos_sched_loom::cpu::{CpuHandle, CpuHandles}; -use toyos_sched_loom::hw::CpuId; -use toyos_sched_loom::mailbox::{mailbox, MailboxConsumer}; -use toyos_sched_loom::model::{ +use kernel_loom::sched::cpu::{CpuHandle, CpuHandles}; +use kernel_loom::sched::hw::CpuId; +use kernel_loom::sched::mailbox::{mailbox, MailboxConsumer}; +use kernel_loom::sched_model::{ model, watch_list, Kicks, LoomLock, Msg, PreemptModel, RemoteGuard, CPU0, CPU1, }; -use toyos_sched_loom::park::{prepare, Cancel, Commit, CurrentTask}; -use toyos_sched_loom::task::{ +use kernel_loom::sched::park::{prepare, Cancel, Commit, CurrentTask}; +use kernel_loom::sched::task::{ Claim, Refused, TaskKey, TaskShared, TaskState, WaitClass, WakeCause, WakeReason, }; -use toyos_sched_loom::sync::CellLock; -use toyos_sched_loom::watch::{Fire, Gate, Poster, Ring, Waiters, Watch}; - -#[path = "../../../kernel/src/inbox/once.rs"] -mod once; +use kernel_loom::sched::sync::CellLock; +use kernel_loom::sched::watch::{Fire, Gate, Poster, Ring, Waiters, Watch}; -/// Names the one-shot above as `super::once`. -#[expect(dead_code, reason = "every look here finds its object ready: a renewal and a refusal are `kernel-loom`'s `inbox_answer`'s")] -#[path = "../../../kernel/src/inbox/polls.rs"] -mod polls; +use kernel_loom::{inbox_polls as polls, poll_once as once}; /// One poll, as the kernel's is: its answer taken once, by the kernel's own /// [`once::Once`], across everything that may fire it; it counts what it diff --git a/kernel-loom/tests/panic_capture.rs b/kernel/loom/tests/panic_capture.rs similarity index 100% rename from kernel-loom/tests/panic_capture.rs rename to kernel/loom/tests/panic_capture.rs diff --git a/kernel-loom/tests/panic_console_publish.rs b/kernel/loom/tests/panic_console_publish.rs similarity index 100% rename from kernel-loom/tests/panic_console_publish.rs rename to kernel/loom/tests/panic_console_publish.rs diff --git a/kernel-loom/tests/poll_once.rs b/kernel/loom/tests/poll_once.rs similarity index 97% rename from kernel-loom/tests/poll_once.rs rename to kernel/loom/tests/poll_once.rs index b1896147129..06a8745cf4c 100644 --- a/kernel-loom/tests/poll_once.rs +++ b/kernel/loom/tests/poll_once.rs @@ -11,7 +11,7 @@ //! The negative case is a cargo feature: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features poll-fire-load-store \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features poll-fire-load-store \ //! --test poll_once //! ``` //! diff --git a/kernel-loom/tests/reap_gate.rs b/kernel/loom/tests/reap_gate.rs similarity index 98% rename from kernel-loom/tests/reap_gate.rs rename to kernel/loom/tests/reap_gate.rs index 73e126512c5..f91aba2db4e 100644 --- a/kernel-loom/tests/reap_gate.rs +++ b/kernel/loom/tests/reap_gate.rs @@ -20,7 +20,7 @@ //! a cargo feature rather than a comment: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features reap-raise-relaxed \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features reap-raise-relaxed \ //! --test reap_gate //! ``` //! diff --git a/kernel-loom/tests/serial_lock.rs b/kernel/loom/tests/serial_lock.rs similarity index 100% rename from kernel-loom/tests/serial_lock.rs rename to kernel/loom/tests/serial_lock.rs diff --git a/kernel-loom/tests/sleep_lock.rs b/kernel/loom/tests/sleep_lock.rs similarity index 98% rename from kernel-loom/tests/sleep_lock.rs rename to kernel/loom/tests/sleep_lock.rs index ed9cb8f10b0..2f0f205eed5 100644 --- a/kernel-loom/tests/sleep_lock.rs +++ b/kernel/loom/tests/sleep_lock.rs @@ -16,7 +16,7 @@ //! and the release-to-acquire edge that hands the next holder the previous //! one's writes. What is **not** under test is the wake handshake — the //! registration and the notified word that make a post reach a *parked* -//! waiter, which is `toyos-sched/loom/tests/loom_watch.rs`'s model. +//! waiter, which is `loom_watch.rs`'s model. //! //! Service order is FIFO by the arithmetic rather than by anything this file //! observes: `now` is advanced exactly once per release and a contender waits @@ -29,7 +29,7 @@ //! is a cargo feature rather than a comment: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features sleeplock-acquire-off \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features sleeplock-acquire-off \ //! --test sleep_lock //! ``` //! diff --git a/kernel-loom/tests/smp_bringup.rs b/kernel/loom/tests/smp_bringup.rs similarity index 100% rename from kernel-loom/tests/smp_bringup.rs rename to kernel/loom/tests/smp_bringup.rs diff --git a/kernel-loom/tests/ticket_lock.rs b/kernel/loom/tests/ticket_lock.rs similarity index 98% rename from kernel-loom/tests/ticket_lock.rs rename to kernel/loom/tests/ticket_lock.rs index 9a8192b9aa6..c14edd52af7 100644 --- a/kernel-loom/tests/ticket_lock.rs +++ b/kernel/loom/tests/ticket_lock.rs @@ -10,7 +10,7 @@ //! a cargo feature rather than a comment: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features lock-acquire-off \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features lock-acquire-off \ //! --test ticket_lock //! ``` //! diff --git a/kernel-loom/tests/tlb_shootdown.rs b/kernel/loom/tests/tlb_shootdown.rs similarity index 99% rename from kernel-loom/tests/tlb_shootdown.rs rename to kernel/loom/tests/tlb_shootdown.rs index 8827622f38f..939d6948a96 100644 --- a/kernel-loom/tests/tlb_shootdown.rs +++ b/kernel/loom/tests/tlb_shootdown.rs @@ -18,7 +18,7 @@ //! fail here. The negative case is a cargo feature rather than a comment: //! //! ```text -//! cargo test --manifest-path kernel-loom/Cargo.toml --features shootdown-serve-relaxed \ +//! cargo test --manifest-path kernel/loom/Cargo.toml --features shootdown-serve-relaxed \ //! --test tlb_shootdown //! ``` //! diff --git a/kernel/pure/lib.rs b/kernel/pure/lib.rs new file mode 100644 index 00000000000..67a1c47742a --- /dev/null +++ b/kernel/pure/lib.rs @@ -0,0 +1,15 @@ +//! What the kernel decides without touching the machine: the scheduler core +//! ([`sched`]), the process and thread lifecycle ([`proclife`]) and which PCID an +//! address space is handed ([`pcid`]). The kernel binary links it; the host runs +//! its tests, because none of it reads a register, a clock or a kernel lock. + +#![no_std] + +extern crate alloc; + +#[cfg(test)] +extern crate std; + +pub mod pcid; +pub mod proclife; +pub mod sched; diff --git a/toyos-pcid/src/lib.rs b/kernel/pure/pcid/mod.rs similarity index 99% rename from toyos-pcid/src/lib.rs rename to kernel/pure/pcid/mod.rs index e14a848f936..393c447a3de 100644 --- a/toyos-pcid/src/lib.rs +++ b/kernel/pure/pcid/mod.rs @@ -11,7 +11,6 @@ //! was never issued, so no CPU has cached it. Pure — the kernel supplies the //! lock and the shootdown, and [`oracle`] judges the decision against the SDM. -#![no_std] #![forbid(unsafe_code)] pub mod oracle; diff --git a/toyos-pcid/src/oracle.rs b/kernel/pure/pcid/oracle.rs similarity index 97% rename from toyos-pcid/src/oracle.rs rename to kernel/pure/pcid/oracle.rs index 9df6dc2a881..d74134f3cc2 100644 --- a/toyos-pcid/src/oracle.rs +++ b/kernel/pure/pcid/oracle.rs @@ -8,11 +8,11 @@ //! type 2 clears every tag, which a shootdown performs on each CPU (§4.10.4.1). //! Each space maps an address to its own frame, so one space consuming another's //! translation is the cross-space read the boundary forbids. [`drive`] runs the -//! reachable attack through both the real [`PcidPool`](crate::PcidPool) and this +//! reachable attack through both the real [`PcidPool`](crate::pcid::PcidPool) and this //! model, sharing no line with it: the owned allocator admits no cross-space //! read, the reverted counting allocator does. -use crate::{Alloc, Pcid, PcidPool}; +use crate::pcid::{Alloc, Pcid, PcidPool}; const TLB_CAP: usize = 256; @@ -117,7 +117,7 @@ fn alloc_now(pool: &mut PcidPool, tlb: &mut Tlb) -> Option { mod tests { extern crate std; use super::*; - use crate::MAX_USER_PCID; + use crate::pcid::MAX_USER_PCID; /// Teeth: a tag reissued to a second live space with no flush between is read /// across the boundary. diff --git a/toyos-proclife/src/interleave.rs b/kernel/pure/proclife/interleave.rs similarity index 99% rename from toyos-proclife/src/interleave.rs rename to kernel/pure/proclife/interleave.rs index f6b418e1e36..098f99d2906 100644 --- a/toyos-proclife/src/interleave.rs +++ b/kernel/pure/proclife/interleave.rs @@ -1,6 +1,6 @@ //! Every ordering of two CPUs inside one process's lifecycle. //! -//! **This is the file the crate exists for.** `kernel/src/process.rs` gives the +//! **This is the file this module exists for.** `kernel/src/process.rs` gives the //! process table lock up between every phase of a teardown and between both //! halves of a spawn, and its comments say what each window costs — "a thread //! enqueued now would be invisible to its retire sweep", "once it is published @@ -22,10 +22,10 @@ use alloc::vec; use alloc::vec::Vec; use std::collections::HashSet; -use crate::model::{Climb, World}; -use crate::table::Processes; -use crate::tree::{self, Admit, Admitted}; -use crate::{join, reap, spawn, teardown, Pid, Tid, Watch}; +use crate::proclife::model::{Climb, World}; +use crate::proclife::table::Processes; +use crate::proclife::tree::{self, Admit, Admitted}; +use crate::proclife::{join, reap, spawn, teardown, Pid, Tid, Watch}; /// `process::KILLED_EXIT_CODE`, which a walk claims every process below an end /// with. diff --git a/toyos-proclife/src/join.rs b/kernel/pure/proclife/join.rs similarity index 95% rename from toyos-proclife/src/join.rs rename to kernel/pure/proclife/join.rs index bf4c2ef1d85..8dbfa64ad32 100644 --- a/toyos-proclife/src/join.rs +++ b/kernel/pure/proclife/join.rs @@ -15,8 +15,8 @@ use core::cell::Cell; -use crate::table::{Lifecycle, Processes}; -use crate::{Pid, Tid}; +use crate::proclife::table::{Lifecycle, Processes}; +use crate::proclife::{Pid, Tid}; /// Why a join may never be answered. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -87,8 +87,8 @@ mod tests { use super::*; use core::cell::RefCell; - use crate::model::World; - use crate::ThreadLocation; + use crate::proclife::model::World; + use crate::proclife::ThreadLocation; #[test] fn a_live_thread_answers_wait_and_stays_in_the_table() { @@ -115,8 +115,8 @@ mod tests { let mut world = World::new(); let pid = world.spawn_process(); let t1 = world.spawn_thread(pid); - assert!(crate::teardown::claim_teardown(&mut world, pid, 137)); - let _ = crate::teardown::leave(&mut world, pid, t1, None); + assert!(crate::proclife::teardown::claim_teardown(&mut world, pid, 137)); + let _ = crate::proclife::teardown::leave(&mut world, pid, t1, None); assert_eq!(collect_zombie(&mut world, pid, t1), Ok(None)); assert!(world.get(pid).unwrap().location(t1).is_some(), "the entry keeps the thread"); } diff --git a/toyos-proclife/src/lib.rs b/kernel/pure/proclife/mod.rs similarity index 89% rename from toyos-proclife/src/lib.rs rename to kernel/pure/proclife/mod.rs index 08c5f89c6ab..96e24c6da50 100644 --- a/toyos-proclife/src/lib.rs +++ b/kernel/pure/proclife/mod.rs @@ -1,7 +1,7 @@ //! Every decision a process or a thread's lifecycle makes. //! //! The states are two and the questions about them are the whole of this -//! crate: a thread is [`Scheduled`] or it is a [`Zombie`] with a code, and a +//! module: a thread is [`Scheduled`] or it is a [`Zombie`] with a code, and a //! process is either being torn down by somebody or it is not — and, in //! [`tree`], where a process stands under the others, which its end takes //! down with it and when that end is published, and in [`pids`], which pid it @@ -16,18 +16,17 @@ //! be retiring. //! //! So the decisions live here and the effects stay in `kernel/src/process.rs`. -//! Nothing in this crate locks, retires a task, frees a page, reads a clock or +//! Nothing in this module locks, retires a task, frees a page, reads a clock or //! allocates a stack: the kernel gathers what a decision needs under //! `PROCESS_TABLE`, calls one function, and performs the value it is handed. //! Both halves are then reachable from a host test — [`interleave`] enumerates //! every ordering of a scripted pair of operations and checks the laws at every -//! state, which before this crate existed took a booted guest and a race that -//! had to land the wrong way. +//! state. //! //! ## What is *not* here //! //! The scheduler's own states. A live thread is running, ready or blocked and -//! `toyos-sched` is authoritative about which; this crate knows only that it is +//! [`crate::sched`] is authoritative about which; this module knows only that it is //! alive. The same split the kernel already had between [`ThreadLocation`] and //! `scheduler::task_sched_state`. //! @@ -41,19 +40,13 @@ //! two words it is about. It never performs the consequence: a //! [`teardown::ThreadExit::Sibling`] carries the [`Watch`] to post on and posts //! nothing, because the post must happen with the table lock given up and this -//! crate cannot know that the caller is holding it. +//! module cannot know that the caller is holding it. //! //! [`Scheduled`]: ThreadLocation::Scheduled //! [`Zombie`]: ThreadLocation::Zombie -#![no_std] #![forbid(unsafe_code)] -extern crate alloc; - -#[cfg(test)] -extern crate std; - pub mod join; pub mod pids; pub mod reap; @@ -109,7 +102,7 @@ impl ThreadLocation { /// /// The kernel resolves each to the real watch a post is made on — /// a thread's `ThreadSched::handle.watch()`, a process's -/// `ProcessObject::watch()` — and this crate decides only *which*. That is the +/// `ProcessObject::watch()` — and this module decides only *which*. That is the /// whole of the distinction a real defect turned on: `process::thread_exit` /// posted one wake and it was always the process's main thread, so a non-main /// thread joining a sibling was owed a wake nobody sent. diff --git a/toyos-proclife/src/model.rs b/kernel/pure/proclife/model.rs similarity index 99% rename from toyos-proclife/src/model.rs rename to kernel/pure/proclife/model.rs index 9d514e6c92c..f2334e80399 100644 --- a/toyos-proclife/src/model.rs +++ b/kernel/pure/proclife/model.rs @@ -17,10 +17,10 @@ use alloc::collections::{BTreeMap, BTreeSet}; use alloc::string::String; use alloc::vec::Vec; -use crate::table::{Lifecycle, Processes}; -use crate::teardown::{self, Leave}; -use crate::tree::{self, Admit, Publish}; -use crate::{Node, Pid, Pids, ThreadLocation, Tid, Watch}; +use crate::proclife::table::{Lifecycle, Processes}; +use crate::proclife::teardown::{self, Leave}; +use crate::proclife::tree::{self, Admit, Publish}; +use crate::proclife::{Node, Pid, Pids, ThreadLocation, Tid, Watch}; /// One process, as its lifecycle sees it. #[derive(Clone, PartialEq, Eq, Hash)] diff --git a/toyos-proclife/src/pids.rs b/kernel/pure/proclife/pids.rs similarity index 99% rename from toyos-proclife/src/pids.rs rename to kernel/pure/proclife/pids.rs index ab1345ccbb0..89e6a5e825c 100644 --- a/toyos-proclife/src/pids.rs +++ b/kernel/pure/proclife/pids.rs @@ -12,7 +12,7 @@ use alloc::vec::Vec; -use crate::Pid; +use crate::proclife::Pid; #[derive(Clone, PartialEq, Eq, Hash, Debug)] pub struct Pids { diff --git a/toyos-proclife/src/reap.rs b/kernel/pure/proclife/reap.rs similarity index 91% rename from toyos-proclife/src/reap.rs rename to kernel/pure/proclife/reap.rs index b20fc3213a4..f4279e2d00e 100644 --- a/toyos-proclife/src/reap.rs +++ b/kernel/pure/proclife/reap.rs @@ -13,12 +13,12 @@ //! entry and freed everything the process held; the load that pairs with it is //! the reason a pass that sees `finished` sees all of that too. //! -//! [`Processes::published_exit`]: crate::table::Processes::published_exit +//! [`Processes::published_exit`]: crate::proclife::table::Processes::published_exit use alloc::vec::Vec; -use crate::table::Processes; -use crate::Pid; +use crate::proclife::table::Processes; +use crate::proclife::Pid; /// Every process whose exit is published, in pid order. /// @@ -38,7 +38,7 @@ pub fn finished_pids(table: &T) -> Vec { #[cfg(test)] mod tests { use super::*; - use crate::model::World; + use crate::proclife::model::World; #[test] fn nothing_is_taken_from_a_table_of_live_processes() { diff --git a/toyos-proclife/src/spawn.rs b/kernel/pure/proclife/spawn.rs similarity index 95% rename from toyos-proclife/src/spawn.rs rename to kernel/pure/proclife/spawn.rs index 6fbb29a042b..354731ff1fb 100644 --- a/toyos-proclife/src/spawn.rs +++ b/kernel/pure/proclife/spawn.rs @@ -9,15 +9,15 @@ //! Everything that makes a thread happens between the two. //! //! A teardown claims the process under that same lock -//! ([`crate::teardown::claim_teardown`]), collects the tids it is going to +//! ([`crate::proclife::teardown::claim_teardown`]), collects the tids it is going to //! retire, and gives the lock up to retire them. A thread inserted after that //! collection is a thread the retire sweep never names. //! `interleave::tests::a_published_exit_leaves_no_unretired_thread` //! is that whole sentence as a host test, and //! `mutate-spawn-skips-the-insert-recheck` is what it reds under. -use crate::table::{Lifecycle, Processes}; -use crate::Pid; +use crate::proclife::table::{Lifecycle, Processes}; +use crate::proclife::Pid; /// Whether a new thread may join a process. #[must_use = "a refused spawn must answer its caller, not fall through"] @@ -77,8 +77,8 @@ fn admit(proc: Option<&P>) -> Admit { #[cfg(test)] mod tests { use super::*; - use crate::model::World; - use crate::teardown; + use crate::proclife::model::World; + use crate::proclife::teardown; #[test] fn a_live_process_admits_a_thread_at_both_moments() { diff --git a/toyos-proclife/src/table.rs b/kernel/pure/proclife/table.rs similarity index 88% rename from toyos-proclife/src/table.rs rename to kernel/pure/proclife/table.rs index a225a21bcfe..6add75acaee 100644 --- a/toyos-proclife/src/table.rs +++ b/kernel/pure/proclife/table.rs @@ -3,18 +3,18 @@ //! Two traits, and they are deliberately narrow: everything a lifecycle //! decision may read or write, and nothing else. The kernel's `ProcessEntry` //! carries an address space, a handle table, a symbol table, a name and a -//! scheduler record per thread, and not one decision in this crate can name any +//! scheduler record per thread, and not one decision in this module can name any //! of them — which is what makes the host model in `model.rs` a `BTreeMap` and //! not a simulated kernel. //! //! `each_thread` and `each_pid` take a `&mut dyn FnMut` rather than answering an //! iterator, because the kernel's two containers are a `hashbrown::HashMap` and -//! this crate's model is a `BTreeMap`: an associated iterator type would put +//! this module's model is a `BTreeMap`: an associated iterator type would put //! both spellings in the trait for no decision's benefit. A caller that needs //! an order sorts what it collected, and the two that do -//! ([`crate::teardown::retire_set`] and [`crate::reap::finished_pids`]) say so. +//! ([`crate::proclife::teardown::retire_set`] and [`crate::proclife::reap::finished_pids`]) say so. -use crate::{Node, Pid, Pids, ThreadLocation, Tid}; +use crate::proclife::{Node, Pid, Pids, ThreadLocation, Tid}; /// One process's lifecycle state: the whole of what a spawn, an exit, a kill, a /// join or a reap reads or writes about it. @@ -31,7 +31,7 @@ pub trait Lifecycle { self.teardown_code().is_some() } - /// Raise the teardown claim, with the code the exit publishes. [`crate::teardown::claim_teardown`] is the one + /// Raise the teardown claim, with the code the exit publishes. [`crate::proclife::teardown::claim_teardown`] is the one /// caller — the flag exists to be claimed exactly once, so raising it /// anywhere else is what that function is written to prevent. fn begin_teardown(&mut self, code: i32); @@ -52,7 +52,7 @@ pub trait Lifecycle { fn each_thread(&self, f: &mut dyn FnMut(Tid, ThreadLocation)); /// Where this process stands in the tree. Read and written by - /// [`crate::tree`] alone. + /// [`crate::proclife::tree`] alone. fn node(&self) -> &Node; fn node_mut(&mut self) -> &mut Node; } diff --git a/toyos-proclife/src/teardown.rs b/kernel/pure/proclife/teardown.rs similarity index 98% rename from toyos-proclife/src/teardown.rs rename to kernel/pure/proclife/teardown.rs index 6c95f3e3375..e0b6f0dd363 100644 --- a/toyos-proclife/src/teardown.rs +++ b/kernel/pure/proclife/teardown.rs @@ -17,8 +17,8 @@ use alloc::vec::Vec; -use crate::table::{Lifecycle, Processes}; -use crate::{Pid, ThreadLocation, Tid, Watch, TORN_DOWN_THREAD_CODE}; +use crate::proclife::table::{Lifecycle, Processes}; +use crate::proclife::{Pid, ThreadLocation, Tid, Watch, TORN_DOWN_THREAD_CODE}; /// Claim exclusive teardown of a process, for `code`. /// @@ -151,7 +151,7 @@ pub fn route_thread_exit(table: &T, pid: Pid, tid: Tid) -> ThreadE #[cfg(test)] mod tests { use super::*; - use crate::model::World; + use crate::proclife::model::World; #[cfg(not(feature = "mutate-claim-teardown-always-wins"))] #[test] diff --git a/toyos-proclife/src/tree.rs b/kernel/pure/proclife/tree.rs similarity index 98% rename from toyos-proclife/src/tree.rs rename to kernel/pure/proclife/tree.rs index ab54f54b77d..26070b0d243 100644 --- a/toyos-proclife/src/tree.rs +++ b/kernel/pure/proclife/tree.rs @@ -10,7 +10,7 @@ //! - **May a child be placed here?** [`admit_child`] at the top of a spawn, //! before anything is built: a place being torn down takes nothing more, //! and a child deeper than [`MAX_DEPTH`] below the supervisor is refused. It takes -//! the child's pid ([`crate::pids`]). +//! the child's pid ([`crate::proclife::pids`]). //! - **Who does an end take?** [`claim`] one process at a time: its claim is //! what closes admission under it, and the same hold reads its children. //! A child admitted before the claim and landed after it is claimed in the @@ -24,8 +24,8 @@ use alloc::vec::Vec; -use crate::table::{Lifecycle, Processes}; -use crate::{teardown, Pid, Tid}; +use crate::proclife::table::{Lifecycle, Processes}; +use crate::proclife::{teardown, Pid, Tid}; /// How far below the supervisor a process may be. Bounds the climb one teardown can /// owe. @@ -226,8 +226,8 @@ fn lower(table: &mut T, pid: Pid) -> Option { #[cfg(test)] mod tests { use super::*; - use crate::model::World; - use crate::Pids; + use crate::proclife::model::World; + use crate::proclife::Pids; /// A chain from the supervisor down to `MAX_DEPTH`: the last is admitted, and one /// more below it is refused naming the depth it would have had. diff --git a/toyos-sched/src/cpu.rs b/kernel/pure/sched/cpu.rs similarity index 98% rename from toyos-sched/src/cpu.rs rename to kernel/pure/sched/cpu.rs index 4aaf3240e1a..0c9738cbca1 100644 --- a/toyos-sched/src/cpu.rs +++ b/kernel/pure/sched/cpu.rs @@ -23,21 +23,21 @@ use alloc::collections::{BTreeMap, VecDeque}; use alloc::vec::Vec; use core::marker::PhantomData; -use crate::fair::{Frontier, QUANTUM_NS}; -use crate::hw::{CpuId, Hw, Nanos, TraceEvent, TraceKind}; -use crate::mailbox::{ +use crate::sched::fair::{Frontier, QUANTUM_NS}; +use crate::sched::hw::{CpuId, Hw, Nanos, TraceEvent, TraceKind}; +use crate::sched::mailbox::{ Doorbell, Kick, MailboxConsumer, MailboxNode, MailboxProducer, PostSlot, PreemptGuard, Quiesced, SchedMsg, SleepArm, Urgency, }; -use crate::msg::Msg; -use crate::queue::RunQueue; -use crate::sync::{fence, Arc, AtomicU32, AtomicU64, Ordering}; -use crate::task::{ +use crate::sched::msg::Msg; +use crate::sched::queue::RunQueue; +use crate::sched::sync::{fence, Arc, AtomicU32, AtomicU64, Ordering}; +use crate::sched::task::{ BlockedTask, Claim, DeadTask, ReadyTask, RunningTask, SchedPayload, TaskKey, TaskShared, TaskState, TransitTask, WaitClass, WakeCause, WakeReason, }; -use crate::timer::{TimerApplied, TimerPlan}; -use crate::park::{CommittedTicket, CurrentTask}; +use crate::sched::timer::{TimerApplied, TimerPlan}; +use crate::sched::park::{CommittedTicket, CurrentTask}; /// Permission to switch. Holds pointers into the stable Box-backed task /// records; constructed only by safe code in @@ -484,8 +484,8 @@ impl CpuSched { /// The stray-write tripwire's *layout*, so a driver can hold this record's bytes /// to **not having changed while nothing was allowed to change it**. /// -/// The reading itself is the driver's: this crate writes `unsafe` in -/// [`crate::mailbox`] and nowhere else, and a byte-level copy of a +/// The reading itself is the driver's: the scheduler core writes `unsafe` in +/// [`crate::sched::mailbox`] and nowhere else, and a byte-level copy of a /// record is exactly the kind of thing that rule exists to keep out of the state /// machine. What is here is what only this module can compute — where the one /// remotely-written field sits, and which field a byte offset lands in. @@ -510,8 +510,8 @@ impl CpuSched { /// /// Everything else here is written by the owning CPU alone, inside the driver's /// exclusive region, so any difference across a window in which that region was -/// not entered is a write nothing in this crate made. -#[cfg(feature = "tripwire")] +/// not entered is a write nothing in the scheduler core made. +#[cfg(feature = "sched-tripwire")] impl CpuSched { /// How many `u64` words a whole-record shadow takes. pub fn tripwire_words() -> usize { @@ -599,7 +599,7 @@ impl CpuSched { /// Clear the borrowed window at a park only `if now >= until`, so a lend /// blocked on before it ran out survives the block — which with - /// [`crate::task::RtState::arm`] re-arming at the next dispatch is a task + /// [`crate::sched::task::RtState::arm`] re-arming at the next dispatch is a task /// holding inherited RT forever off one lend. Invariant I9 must catch it; /// `scenarios::old_park_kept_the_lend` is the gate that proves it does. pub fn set_park_keeps_lapsed_lend(&mut self, keep: bool) { @@ -633,7 +633,7 @@ impl CpuSched { /// Order the fair band by something other than its insertion sequence. /// Invariant I13 must catch what that does to a share's threads; /// `scenarios::sibling_storm`'s two gates are what prove it does. - pub fn set_fair_order(&mut self, order: crate::queue::FairOrder) { + pub fn set_fair_order(&mut self, order: crate::sched::queue::FairOrder) { self.rq.set_order(order); } } @@ -690,7 +690,7 @@ pub enum Balance { /// and then loads the surplus. Without a `SeqCst` fence between each side's /// store and its load, both may miss, and the CPU sleeps through a surplus /// exactly as it does under [`Balance::Pull`] — a narrower window on the - /// same defect. `toyos-sched/loom/tests/loom_push.rs` is the model. + /// same defect. `kernel/loom/tests/loom_push.rs` is the model. PushOnSurplus { threshold: u32 }, } @@ -767,10 +767,10 @@ impl Balance { /// outcome, and it is a real instruction on the push path — `mfence` on x86 — /// which is part of what the push costs. **A cargo feature rather than a /// comment, because a model that has never failed proves nothing**: -/// `toyos-sched-loom`'s `push-fence-relaxed` weakens it to a release fence, +/// `kernel-loom`'s `push-fence-relaxed` weakens it to a release fence, /// which orders stores against stores and nothing against a later load, and -/// `loom/tests/loom_push.rs` must red under it. No kernel build can turn the -/// name on; the crate declares it only so `cfg` checking knows it. +/// `kernel/loom/tests/loom_push.rs` must red under it. The kernel declares the +/// name only so `cfg` checking knows it. #[cfg(not(feature = "push-fence-relaxed"))] const PUSH_FENCE: Ordering = Ordering::SeqCst; #[cfg(feature = "push-fence-relaxed")] @@ -879,7 +879,7 @@ impl CpuSched { /// /// **The loaded task is never migrated, and the rule is asserted here /// because this is where every migration passes.** - /// [`crate::queue::RunQueue::pop_surplus`]'s `loaded` argument covers the + /// [`crate::sched::queue::RunQueue::pop_surplus`]'s `loaded` argument covers the /// steal path and only the steal path; the /// wake-forward in [`CpuSched::place`] reaches this function too, and what /// keeps *it* correct is an argument rather than a check — a task `place` @@ -1160,7 +1160,7 @@ impl CpuSched { TaskState::Dead => {} state if home_of(state) == Some(self.id) => {} _ => { - crate::retire::chase(shared, env.cpus, env.hw, env.preempt); + crate::sched::retire::chase(shared, env.cpus, env.hw, env.preempt); } } } @@ -1246,7 +1246,7 @@ fn home_of(state: TaskState) -> Option { /// How long one scheduler pass is modelled to take on the machine it runs on, /// which is also how long preemption is off for. **Measured by a -/// `feature = "check"` build; asserted by nothing.** +/// `feature = "sched-check"` build; asserted by nothing.** /// /// The number is the simulator's own modelling error made explicit. The sim /// charges a pass **zero** time — every step it takes is either a workload op @@ -1316,11 +1316,11 @@ pub const DYING_CHUNK_NS: u64 = QUANTUM_NS / 10; /// Power-of-two buckets a pass-cost histogram keeps. The top one saturates at /// 2^30 ns ≈ 1.07 s, which is longer than any pass a machine survives. -#[cfg(feature = "check")] +#[cfg(feature = "sched-check")] pub const PASS_COST_BUCKETS: usize = 32; /// Bucket 0 is exactly zero; bucket `b > 0` covers `[2^(b-1), 2^b)` ns. -#[cfg(feature = "check")] +#[cfg(feature = "sched-check")] pub fn pass_cost_bucket(ns: u64) -> usize { ((u64::BITS - ns.leading_zeros()) as usize).min(PASS_COST_BUCKETS - 1) } @@ -1329,7 +1329,7 @@ pub fn pass_cost_bucket(ns: u64) -> usize { /// top one. A quantile is reported as one of these: "this fraction of passes /// cost *less than* this many nanoseconds" is the strongest true statement a /// histogram supports. -#[cfg(feature = "check")] +#[cfg(feature = "sched-check")] pub fn pass_cost_bucket_end(bucket: usize) -> u64 { if bucket >= PASS_COST_BUCKETS - 1 { u64::MAX @@ -1343,7 +1343,7 @@ pub fn pass_cost_bucket_end(bucket: usize) -> u64 { /// `over` is exact and the histogram is not, which is deliberate: rounding it /// to a power of two would lose the one number a reader compares against /// [`MAX_PASS_NS`] by eye. -#[cfg(feature = "check")] +#[cfg(feature = "sched-check")] #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct PassCostReport { pub cpu: CpuId, @@ -1357,7 +1357,7 @@ pub struct PassCostReport { pub buckets: [u64; PASS_COST_BUCKETS], } -#[cfg(feature = "check")] +#[cfg(feature = "sched-check")] impl PassCostReport { pub fn empty(cpu: CpuId) -> Self { Self { @@ -1392,7 +1392,7 @@ impl PassCostReport { } } -#[cfg(feature = "check")] +#[cfg(feature = "sched-check")] impl core::fmt::Display for PassCostReport { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( @@ -1419,13 +1419,13 @@ impl core::fmt::Display for PassCostReport { } /// One CPU's live pass-cost recorder, written only by that CPU and read by -/// anyone. Exists only in a `feature = "check"` build. +/// anyone. Exists only in a `feature = "sched-check"` build. /// /// Plain relaxed load/store rather than read-modify-write: the writer is the /// owning CPU inside its own pass, so there is no contention to lose to, and an /// uncontended `lock xadd` on the pass path is the operation that costs most /// under emulation. -#[cfg(feature = "check")] +#[cfg(feature = "sched-check")] pub struct PassCosts { count: AtomicU64, max_ns: AtomicU64, @@ -1433,7 +1433,7 @@ pub struct PassCosts { buckets: [AtomicU64; PASS_COST_BUCKETS], } -#[cfg(feature = "check")] +#[cfg(feature = "sched-check")] impl PassCosts { fn new() -> Self { Self { @@ -1670,10 +1670,10 @@ impl SchedPass<'_, '_, H, P, Disposed> { // The handle is taken out here for the same reason `hw` is: by the time // the measurement lands, `finish_inner` has consumed the pass and with // it every borrow of the `CpuSched`. - #[cfg(feature = "check")] + #[cfg(feature = "sched-check")] let (hw, handle, entered) = (self.env.hw, self.env.cpus.get(self.cpu.id), self.now); let action = self.finish_inner(); - #[cfg(feature = "check")] + #[cfg(feature = "sched-check")] handle.pass_costs().record(hw.now().since(entered)); action } @@ -1936,7 +1936,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { /// /// It can only move the arming **earlier**, which is why it cannot break /// invariant T: the invariant says the armed instant is no later than the - /// earliest event this CPU owes ([`crate::invariants::check_timer`]), and an + /// earliest event this CPU owes ([`crate::sched::invariants::check_timer`]), and an /// extra wake before that instant is a spurious pass, not a missed /// deadline. fn apply_timer_no_later_than(&mut self, extra: Option) -> TimerApplied { @@ -1951,8 +1951,8 @@ impl SchedPass<'_, '_, H, P, Disposed> { // Invariant T is a statement about a CPU *outside* a pass, and the // arming that makes it true is the last thing a pass does — so the // check cannot move earlier. - #[cfg(feature = "check")] - crate::invariants::check_cpu(self.cpu); + #[cfg(feature = "sched-check")] + crate::sched::invariants::check_cpu(self.cpu); TimerApplied::new(plan.armed()) } @@ -2122,7 +2122,7 @@ impl SchedPass<'_, '_, H, P, Disposed> { /// tell one sleeping CPU that there is something to come and get. /// /// **One CPU per pass, and a doorbell ring rather than a message.** The ring - /// is what [`crate::mailbox::Doorbell`] already does for every producer, and + /// is what [`crate::sched::mailbox::Doorbell`] already does for every producer, and /// its edge-coalescing does the rest: only the 0→1 kick edge on a target /// that reads SLEEPING costs an IPI, so a target that already has one coming /// is not kicked twice. The woken CPU finds an empty mailbox, runs its idle @@ -2191,7 +2191,7 @@ pub struct CpuHandle { /// The on-target counterpart to the simulator's invariants: the sim asserts /// what a pass *does*, this measures what a pass *costs*. /// - /// Everything else in `feature = "check"` is a statement about state the + /// Everything else in `feature = "sched-check"` is a statement about state the /// core owns, which is checkable in either world. Cost is not: the /// simulator's clock does not advance inside a step, so on the sim side the /// recorder is fed a modelled pass cost (`scenarios::overlong_pass`) and on @@ -2200,7 +2200,7 @@ pub struct CpuHandle { /// It lives on the handle rather than in the `CpuSched` because the /// measurement lands *after* the pass has consumed every borrow of that, /// and because a report has to be readable from outside a pass. - #[cfg(feature = "check")] + #[cfg(feature = "sched-check")] pass_costs: PassCosts, } @@ -2213,7 +2213,7 @@ impl CpuHandle { load: AtomicU32::new(0), surplus: AtomicU32::new(0), last_pass: AtomicU64::new(0), - #[cfg(feature = "check")] + #[cfg(feature = "sched-check")] pass_costs: PassCosts::new(), } } @@ -2222,7 +2222,7 @@ impl CpuHandle { self.id } - #[cfg(feature = "check")] + #[cfg(feature = "sched-check")] pub fn pass_costs(&self) -> &PassCosts { &self.pass_costs } @@ -2398,11 +2398,11 @@ mod tests { extern crate std; use super::*; - use crate::fair::{FairShare, ShareState}; - use crate::hw::{Kicker, Machine}; - use crate::mailbox::{mailbox, NoPreempt}; - use crate::sync::CellLock; - use crate::task::{RtState, TaskAccounting, TaskBuilder}; + use crate::sched::fair::{FairShare, ShareState}; + use crate::sched::hw::{Kicker, Machine}; + use crate::sched::mailbox::{mailbox, NoPreempt}; + use crate::sched::sync::CellLock; + use crate::sched::task::{RtState, TaskAccounting, TaskBuilder}; use std::sync::Mutex; struct TestLock(Mutex); @@ -2589,10 +2589,10 @@ mod tests { sched.running().expect("a running task to park").shared(), cpu, ); - let ticket = crate::park::prepare(¤t, crate::park::Cancel::Answers, WaitClass::Other) + let ticket = crate::sched::park::prepare(¤t, crate::sched::park::Cancel::Answers, WaitClass::Other) .expect("nothing notified this task"); let committed = match ticket.commit() { - crate::park::Commit::Parked(c) => c, + crate::sched::park::Commit::Parked(c) => c, _ => panic!("the commit refused an uncontended park"), }; let pass = SchedPass::begin(sched, env, NOW); @@ -2670,7 +2670,7 @@ mod tests { w.park_running(C0); assert_eq!(shared.state(), TaskState::Blocked(C0)); - crate::retire::begin(&shared).post(&w.handles, &w.hw, &NoPreempt); + crate::sched::retire::begin(&shared).post(&w.handles, &w.hw, &NoPreempt); { let (cpus, env) = w.split(); cpus[0].drain(env, NOW); @@ -2712,7 +2712,7 @@ mod tests { // A waker wins the claim; its message has not been pushed yet. assert_eq!(shared.claim_wake(), Claim::Parked(C0), "the waker owns it"); - crate::retire::begin(&shared).post(&w.handles, &w.hw, &NoPreempt); + crate::sched::retire::begin(&shared).post(&w.handles, &w.hw, &NoPreempt); { let (cpus, env) = w.split(); cpus[0].drain(env, NOW); @@ -2750,7 +2750,7 @@ mod tests { let (key, shared) = w.spawn(C0); assert_eq!(shared.state(), TaskState::Ready(C0)); - crate::retire::begin(&shared).post(&w.handles, &w.hw, &NoPreempt); + crate::sched::retire::begin(&shared).post(&w.handles, &w.hw, &NoPreempt); { let (cpus, env) = w.split(); cpus[0].drain(env, NOW); @@ -2772,7 +2772,7 @@ mod tests { w.run_a_pass(C0); assert_eq!(shared.state(), TaskState::Running(C0)); - crate::retire::begin(&shared).post(&w.handles, &w.hw, &NoPreempt); + crate::sched::retire::begin(&shared).post(&w.handles, &w.hw, &NoPreempt); w.run_a_pass(C0); assert_eq!(shared.state(), TaskState::Running(C0), "it keeps its stack"); @@ -3238,7 +3238,7 @@ mod tests { /// `a_corpse_is_not_starved_for_ever_by_a_spinning_rt_task` is the other /// direction and stages the aged one. Neither absolute holds: "a ready /// real-time task always preempts the normal band" does admit an exception - /// here, and this crate's own gates are what state its shape. + /// here, and the scheduler core's own gates are what state its shape. #[test] fn a_killed_task_does_not_starve_a_ready_rt_task() { let mut w = World::new(1); @@ -3777,9 +3777,9 @@ mod tests { } } -/// The pass-cost recorder's own arms, which need the `check` build the +/// The pass-cost recorder's own arms, which need the `sched-check` build the /// recorder only exists in — a separate module for that reason and no other. -#[cfg(all(test, feature = "check"))] +#[cfg(all(test, feature = "sched-check"))] mod pass_cost_tests { use super::*; diff --git a/toyos-sched/src/fair.rs b/kernel/pure/sched/fair.rs similarity index 99% rename from toyos-sched/src/fair.rs rename to kernel/pure/sched/fair.rs index cbcd4603002..299474be21f 100644 --- a/toyos-sched/src/fair.rs +++ b/kernel/pure/sched/fair.rs @@ -5,7 +5,7 @@ use core::num::NonZeroU32; use core::sync::atomic::{AtomicU64, Ordering}; -use crate::sync::CellLock; +use crate::sched::sync::CellLock; /// Clamp for stored lag at the Runnable→NonRunnable transition: how far /// behind (entitled catch-up) or ahead (throttled on wake) of the frontier a diff --git a/toyos-sched/src/hw.rs b/kernel/pure/sched/hw.rs similarity index 96% rename from toyos-sched/src/hw.rs rename to kernel/pure/sched/hw.rs index 6006cb79154..8c7cd832f0c 100644 --- a/toyos-sched/src/hw.rs +++ b/kernel/pure/sched/hw.rs @@ -8,8 +8,8 @@ //! clock and vcpu bookkeeping. No scheduling decision, state transition or //! ordering-sensitive code may live behind them. -use crate::cpu::{RunToken, SleepToken}; -use crate::task::{SchedPayload, TaskAccounting, TaskKey}; +use crate::sched::cpu::{RunToken, SleepToken}; +use crate::sched::task::{SchedPayload, TaskAccounting, TaskKey}; /// CPU identity. Always a field or a parameter, never an ambient query — /// `Hw` deliberately has no `cpu_id()`, so a wrong-CPU lookup is @@ -74,7 +74,7 @@ pub enum TraceKind { } /// The one effect a wake path needs from the world: the targeted kick IPI a -/// [`crate::mailbox::Kick::Send`] obliges the poster to deliver. Split out of +/// [`crate::sched::mailbox::Kick::Send`] obliges the poster to deliver. Split out of /// [`Machine`] so a watch's post and the retire protocol — which run at any wake /// site, not inside a scheduler pass — depend on nothing else. pub trait Kicker: Sync { diff --git a/toyos-sched/src/invariants.rs b/kernel/pure/sched/invariants.rs similarity index 96% rename from toyos-sched/src/invariants.rs rename to kernel/pure/sched/invariants.rs index a310ca085bb..e2df886c2dd 100644 --- a/toyos-sched/src/invariants.rs +++ b/kernel/pure/sched/invariants.rs @@ -8,13 +8,13 @@ //! belong to the simulator, which is the only world that can hold all CPUs at //! once; [`residents`] is what it walks this CPU with. //! -//! `feature = "check"` gates the call in `SchedPass::finish`, not this module: +//! `feature = "sched-check"` gates the call in `SchedPass::finish`, not this module: //! the simulator wants the enumeration unconditionally, and a check that is //! compiled but never called costs nothing. -use crate::cpu::CpuSched; -use crate::hw::Nanos; -use crate::task::{SchedPayload, TaskKey, TaskState}; +use crate::sched::cpu::CpuSched; +use crate::sched::hw::Nanos; +use crate::sched::task::{SchedPayload, TaskKey, TaskState}; /// Where a task value lives. One key may appear in exactly one of these, /// system-wide. diff --git a/toyos-sched/src/mailbox.rs b/kernel/pure/sched/mailbox.rs similarity index 96% rename from toyos-sched/src/mailbox.rs rename to kernel/pure/sched/mailbox.rs index d2a0808558d..fd5199e4d71 100644 --- a/toyos-sched/src/mailbox.rs +++ b/kernel/pure/sched/mailbox.rs @@ -8,7 +8,7 @@ //! representation**, and an ownership-carrying message cannot be dropped //! because the message *is* the owner. //! -//! This is the only module in the crate allowed to write `unsafe`. Everything +//! This is the only module of the scheduler core allowed to write `unsafe`. Everything //! it does rests on three invariants, each restated at its use site: //! //! * **N1 (single claim).** A node carries at most one message at a time — @@ -32,12 +32,12 @@ use core::marker::PhantomData; use core::mem::ManuallyDrop; use core::ptr; -use crate::sync::{Arc, AtomicBool, AtomicPtr, AtomicU32, Ordering}; -use crate::task::{TaskKey, TaskShared, WakeCause}; +use crate::sched::sync::{Arc, AtomicBool, AtomicPtr, AtomicU32, Ordering}; +use crate::sched::task::{TaskKey, TaskShared, WakeCause}; /// The message vocabulary the primitives need to speak. Wait queues and the /// retire protocol construct only these two, which is what keeps them free of -/// the task payload type; [`crate::msg::Msg`] is the full set. +/// the task payload type; [`crate::sched::msg::Msg`] is the full set. pub trait SchedMsg: Send + Sized { fn wake(key: TaskKey, cause: WakeCause) -> Self; @@ -447,12 +447,11 @@ const SLEEPING: u32 = 1 << 1; /// the IPI the target's own store had just earned. /// /// **A cargo feature rather than a comment, because a model that has never -/// failed proves nothing.** `toyos-sched-loom`'s `doorbell-kick-relaxed` makes -/// it `Relaxed` and `loom/tests/loom_sleep.rs` must red under it: +/// failed proves nothing.** `kernel-loom`'s `doorbell-kick-relaxed` makes +/// it `Relaxed` and `kernel/loom/tests/loom_sleep.rs` must red under it: /// `a_halted_cpu_with_queued_work_was_kicked` finds a schedule where the /// target halts with both messages queued and no IPI in flight — a -/// sleep-through. No kernel build can turn the name on: the crate declares it -/// only so `cfg` checking knows it. +/// sleep-through. The kernel declares the name only so `cfg` checking knows it. #[cfg(not(feature = "doorbell-kick-relaxed"))] const KICK: Ordering = Ordering::AcqRel; #[cfg(feature = "doorbell-kick-relaxed")] @@ -525,7 +524,7 @@ impl Default for Doorbell { } /// SLEEPING is published; the final mailbox check has not happened yet. The -/// only way to obtain a [`crate::cpu::SleepToken`] runs through here, so "halt with work +/// only way to obtain a [`crate::sched::cpu::SleepToken`] runs through here, so "halt with work /// queued" has no expression. #[must_use = "an armed sleep must be confirmed or abandoned"] pub struct SleepArm<'d> { @@ -537,7 +536,7 @@ pub struct SleepArm<'d> { pub struct Awake; /// SLEEPING was published before a mailbox-empty check that came back empty. -/// Half of what [`crate::cpu::SleepToken`] requires. +/// Half of what [`crate::sched::cpu::SleepToken`] requires. #[must_use] pub struct Quiesced { _private: (), @@ -548,7 +547,7 @@ impl SleepArm<'_> { /// else's is impossible, since a `MailboxConsumer` is `!Send` and lives /// in the CPU's own `CpuSched`. /// - /// Success yields [`Quiesced`], one of the two halves [`crate::cpu::SleepToken`] + /// Success yields [`Quiesced`], one of the two halves [`crate::sched::cpu::SleepToken`] /// needs — the other being the applied timer plan. A /// CPU therefore cannot halt with work queued *or* with a deadline /// pending and the timer unarmed, and neither fact is asserted anywhere: @@ -566,7 +565,7 @@ impl SleepArm<'_> { } } -/// The guard the crate's own unit tests push under. It lives here, in the +/// The guard the scheduler core's unit tests push under. It lives here, in the /// one module allowed to write `unsafe`, so that no other module needs an /// exemption to implement [`PreemptGuard`]. #[cfg(test)] @@ -574,7 +573,7 @@ pub(crate) struct NoPreempt; // SAFETY: the unit tests are single-threaded and run no scheduler, so the // executing context cannot be descheduled. Concurrency is loom's job -// (`toyos-sched/loom/`), where the guard models the real preempt count. +// (`kernel/loom/`), where the guard models the real preempt count. #[cfg(test)] unsafe impl PreemptGuard for NoPreempt {} diff --git a/toyos-sched/src/lib.rs b/kernel/pure/sched/mod.rs similarity index 81% rename from toyos-sched/src/lib.rs rename to kernel/pure/sched/mod.rs index ce70a84c921..514482dc50f 100644 --- a/toyos-sched/src/lib.rs +++ b/kernel/pure/sched/mod.rs @@ -5,16 +5,13 @@ //! simulator and under loom. //! //! Two host-side harnesses compile against these same sources: the simulator -//! (`toyos-sched/sim/`), which explores the protocol against invariants -//! I1–I13, and the `toyos-sched-loom` package (`toyos-sched/loom/`), which +//! (`kernel/sim/`), which explores the protocol against invariants +//! I1–I13, and the `kernel-loom` package (`kernel/loom/`), which //! swaps in loom's instrumented atomics — see [`sync`] for why loom is a //! separate package rather than a `cfg(loom)` dependency. -#![no_std] #![deny(unsafe_code)] -extern crate alloc; - pub mod cpu; pub mod fair; pub mod hw; @@ -28,4 +25,6 @@ pub mod sync; pub mod task; pub mod timer; pub mod watch; +#[cfg(test)] +mod watch_list; pub mod windows; diff --git a/toyos-sched/src/msg.rs b/kernel/pure/sched/msg.rs similarity index 92% rename from toyos-sched/src/msg.rs rename to kernel/pure/sched/msg.rs index 6ace4462ae1..03dffa8cc44 100644 --- a/toyos-sched/src/msg.rs +++ b/kernel/pure/sched/msg.rs @@ -5,13 +5,13 @@ //! overflow unrepresentable: a queue that cannot drop a message cannot //! lose a task, because the message is the task. //! -//! Its own module rather than part of `mailbox.rs`, so that the crate's single +//! Its own module rather than part of `mailbox.rs`, so that the scheduler core's single //! `#[allow(unsafe_code)]` island stays exactly the intrusive-list code. -use crate::hw::CpuId; -use crate::mailbox::{MailboxNode, SchedMsg}; -use crate::sync::Arc; -use crate::task::{SchedPayload, TaskKey, TaskShared, TransitTask, WakeCause}; +use crate::sched::hw::CpuId; +use crate::sched::mailbox::{MailboxNode, SchedMsg}; +use crate::sched::sync::Arc; +use crate::sched::task::{SchedPayload, TaskKey, TaskShared, TransitTask, WakeCause}; pub enum Msg { /// The target CPU owns the parked task; this is a request, not a diff --git a/toyos-sched/src/park.rs b/kernel/pure/sched/park.rs similarity index 96% rename from toyos-sched/src/park.rs rename to kernel/pure/sched/park.rs index c8f28976609..2e4eadf846d 100644 --- a/toyos-sched/src/park.rs +++ b/kernel/pure/sched/park.rs @@ -1,7 +1,7 @@ //! The one park: a running task parks on its own state word, and the one wake //! primitive, [`notify`], is what ends it. //! -//! There is no queue here. What a task waits *for* is a [`crate::watch::Watch`] +//! There is no queue here. What a task waits *for* is a [`crate::sched::watch::Watch`] //! it registered on; where it parks is itself. Every wake in the system — a //! post, a direct notify, a remote waker — terminates in //! [`TaskShared::notify`]'s read-modify-write of that word (a revoke is the @@ -31,11 +31,11 @@ use core::marker::PhantomData; -use crate::cpu::CpuHandles; -use crate::hw::{CpuId, Kicker}; -use crate::mailbox::{Kick, PreemptGuard, SchedMsg}; -use crate::sync::Arc; -use crate::task::{Gen, Notify, ParkOutcome, Refused, TaskShared, WaitClass, WakeCause}; +use crate::sched::cpu::CpuHandles; +use crate::sched::hw::{CpuId, Kicker}; +use crate::sched::mailbox::{Kick, PreemptGuard, SchedMsg}; +use crate::sched::sync::Arc; +use crate::sched::task::{Gen, Notify, ParkOutcome, Refused, TaskShared, WaitClass, WakeCause}; /// The running task, as the wait path sees it. Only `CpuSched::current_task` /// hands one out, so [`prepare`] cannot be called for anybody else's task. @@ -253,9 +253,9 @@ mod tests { extern crate std; use super::*; - use crate::cpu::{CpuHandle, CpuHandles}; - use crate::mailbox::{mailbox, MailboxConsumer, MailboxNode, NoPreempt}; - use crate::task::{TaskKey, TaskState, WakeReason}; + use crate::sched::cpu::{CpuHandle, CpuHandles}; + use crate::sched::mailbox::{mailbox, MailboxConsumer, MailboxNode, NoPreempt}; + use crate::sched::task::{TaskKey, TaskState, WakeReason}; use alloc::vec; use alloc::vec::Vec; use std::sync::Mutex; @@ -419,7 +419,7 @@ mod tests { let t = task(1); let _ = park(&t); - let cause = WakeCause::boosted(WakeReason::Woken, crate::hw::Nanos(1_000)); + let cause = WakeCause::boosted(WakeReason::Woken, crate::sched::hw::Nanos(1_000)); assert!(notify(&t, cause, &handles, &kicks, &NoPreempt).woke()); assert_eq!(rx.pop(&NoPreempt), Some(Msg::Wake(TaskKey(1), WakeReason::Woken))); assert_eq!(kicks.count(), 1, "boost wakes always kick"); diff --git a/toyos-sched/src/queue.rs b/kernel/pure/sched/queue.rs similarity index 95% rename from toyos-sched/src/queue.rs rename to kernel/pure/sched/queue.rs index 580afdc4765..31c323d9f28 100644 --- a/toyos-sched/src/queue.rs +++ b/kernel/pure/sched/queue.rs @@ -9,14 +9,14 @@ use alloc::collections::{BTreeMap, VecDeque}; -use crate::task::{ReadyTask, SchedPayload, TaskKey}; +use crate::sched::task::{ReadyTask, SchedPayload, TaskKey}; /// How the fair band decides between two ready threads of the *same* share — /// the tie-break simulator invariant I13 exists to hold. /// /// The two broken orderings are reproduced for the simulator's negative gates, /// behind a feature the kernel does not enable, exactly as -/// [`crate::cpu::CpuSched::set_park_keeps_lapsed_lend`] is. A check that cannot +/// [`crate::sched::cpu::CpuSched::set_park_keeps_lapsed_lend`] is. A check that cannot /// tell these apart from the shipped one is not measuring the rule it names. #[cfg(feature = "protocol-port")] #[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] @@ -128,7 +128,7 @@ impl RunQueue { let chosen = *self .fair .iter() - .filter(|(_, task)| crate::sync::Arc::ptr_eq(task.share(), &share)) + .filter(|(_, task)| crate::sched::sync::Arc::ptr_eq(task.share(), &share)) .min_by_key(|(_, task)| task.key().0) .expect("the leader is its own share's member") .0; @@ -136,13 +136,13 @@ impl RunQueue { Some((chosen.0, task)) } - /// The task a [`crate::msg::Msg::StealRequest`] is answered with: the + /// The task a [`crate::sched::msg::Msg::StealRequest`] is answered with: the /// *last* fair task, i.e. the one whose turn is furthest away. Handing /// over the next-to-run task instead would trade a cache-warm local /// dispatch for a two-hop transfer. /// /// **`loaded` is skipped, and that is a correctness rule and not a policy - /// one** — [`crate::cpu::SchedPass::answer_steal_requests`] carries the + /// one** — [`crate::sched::cpu::SchedPass::answer_steal_requests`] carries the /// derivation. It is also the *most likely* candidate here rather than an /// unlikely one: a task `preempt_if_due` has just returned to the band was /// charged for the quantum it spent, so its vruntime is the band's highest diff --git a/toyos-sched/src/retire.rs b/kernel/pure/sched/retire.rs similarity index 95% rename from toyos-sched/src/retire.rs rename to kernel/pure/sched/retire.rs index 9f4c530f003..fe0b7db291e 100644 --- a/toyos-sched/src/retire.rs +++ b/kernel/pure/sched/retire.rs @@ -7,11 +7,11 @@ //! chase is bounded by the number of in-flight hops (≤1 in practice). Nothing //! scans; the home CPU in the state word is the proof. -use crate::cpu::CpuHandles; -use crate::hw::{CpuId, Kicker}; -use crate::mailbox::{Kick, PreemptGuard, SchedMsg, Urgency}; -use crate::sync::Arc; -use crate::task::{TaskShared, TaskState}; +use crate::sched::cpu::CpuHandles; +use crate::sched::hw::{CpuId, Kicker}; +use crate::sched::mailbox::{Kick, PreemptGuard, SchedMsg, Urgency}; +use crate::sched::sync::Arc; +use crate::sched::task::{TaskShared, TaskState}; /// Exactly one retirer exists per task (process teardown or thread kill), and /// it holds this. Consumed by [`RetireTicket::post`]. @@ -105,9 +105,9 @@ mod tests { extern crate std; use super::*; - use crate::cpu::{CpuHandle, CpuHandles}; - use crate::mailbox::{mailbox, MailboxConsumer, NoPreempt}; - use crate::task::{Claim, TaskKey, WakeCause, WakeReason}; + use crate::sched::cpu::{CpuHandle, CpuHandles}; + use crate::sched::mailbox::{mailbox, MailboxConsumer, NoPreempt}; + use crate::sched::task::{Claim, TaskKey, WakeCause, WakeReason}; use alloc::vec; use alloc::vec::Vec; use std::sync::Mutex; @@ -272,7 +272,7 @@ mod tests { // The waker wins the claim; the retirer sets the kill bit. Both // messages are in flight at once, which is well-formed by // construction: they ride two distinct embedded nodes. - assert!(crate::park::notify( + assert!(crate::sched::park::notify( &t, WakeCause::new(WakeReason::Woken), &cpus, diff --git a/toyos-sched/src/sync.rs b/kernel/pure/sched/sync.rs similarity index 77% rename from toyos-sched/src/sync.rs rename to kernel/pure/sched/sync.rs index 1a08ca17168..813c7efc3c9 100644 --- a/toyos-sched/src/sync.rs +++ b/kernel/pure/sched/sync.rs @@ -1,12 +1,12 @@ //! Atomics and `Arc`, resolved to whichever world compiles these sources: //! `core`/`alloc` for the kernel and the simulator, loom's instrumented -//! atomics for the `toyos-sched-loom` package. +//! atomics for the `kernel-loom` package. //! //! A feature and a second package rather than the usual `--cfg loom`, because //! a `[target.'cfg(loom)'.dependencies]` entry lands in every lockfile that -//! resolves this crate — including `kernel/Cargo.lock`, which would gain loom -//! and its 30 transitive host crates. `loom` is declared here purely so `cfg` -//! checking knows the name; this package never enables it. +//! resolves the kernel — including `kernel/Cargo.lock`, which would gain loom +//! and its 30 transitive host crates. The kernel declares `loom` purely so `cfg` +//! checking knows the name, and never enables it. #[cfg(not(feature = "loom"))] pub use alloc::sync::Arc; @@ -23,7 +23,7 @@ pub use loom::sync::Arc; /// implementor is never held across a pass or a switch; the simulator and the /// loom models supply their own. /// -/// It lives here rather than in one of its users because the core crate may +/// It lives here rather than in one of its users because the scheduler core may /// not implement a lock itself — that would need `unsafe`, which only /// `mailbox.rs` is allowed to write. pub trait CellLock: Sync { diff --git a/toyos-sched/src/task.rs b/kernel/pure/sched/task.rs similarity index 98% rename from toyos-sched/src/task.rs rename to kernel/pure/sched/task.rs index 1c490d24cde..05a1fa399a5 100644 --- a/toyos-sched/src/task.rs +++ b/kernel/pure/sched/task.rs @@ -9,12 +9,12 @@ use alloc::boxed::Box; use core::ptr::addr_of_mut; -use crate::fair::{FairShare, ShareState, QUANTUM_NS}; -use crate::hw::{CpuId, Nanos}; -use crate::mailbox::MailboxNode; -use crate::msg::Msg; -use crate::sync::{Arc, AtomicBool, AtomicU64, CellLock, Ordering}; -use crate::park::CommittedTicket; +use crate::sched::fair::{FairShare, ShareState, QUANTUM_NS}; +use crate::sched::hw::{CpuId, Nanos}; +use crate::sched::mailbox::MailboxNode; +use crate::sched::msg::Msg; +use crate::sched::sync::{Arc, AtomicBool, AtomicU64, CellLock, Ordering}; +use crate::sched::park::CommittedTicket; /// Monotonic, never reused. Stale messages keyed by `TaskKey` are provably /// about a dead task and are benign no-ops. @@ -30,7 +30,7 @@ pub trait SchedPayload: Sized + Send + 'static { type Ctx: Sized + Send; /// The cell the per-process [`FairShare`] lives in. Supplied by the - /// environment because the core crate may not implement a lock itself + /// environment because the scheduler core may not implement a lock itself /// (see [`CellLock`]). type ShareLock: CellLock + Send; } @@ -63,10 +63,10 @@ impl WakeCause { /// RT and boost wakes must preempt the target promptly; ordinary wakes /// ride the target's next safe point. - pub fn urgency(&self) -> crate::mailbox::Urgency { + pub fn urgency(&self) -> crate::sched::mailbox::Urgency { match self.boost { - Some(_) => crate::mailbox::Urgency::Preempt, - None => crate::mailbox::Urgency::Normal, + Some(_) => crate::sched::mailbox::Urgency::Preempt, + None => crate::sched::mailbox::Urgency::Normal, } } } @@ -836,7 +836,7 @@ impl Task { self.0.since } - /// The stable address of the saved context, for [`crate::cpu::RunToken`]: + /// The stable address of the saved context, for [`crate::sched::cpu::RunToken`]: /// the record is boxed, so it outlives every container move the task makes. pub(crate) fn ctx_ptr(&mut self) -> *mut X::Ctx { addr_of_mut!(self.0.ctx) @@ -915,7 +915,7 @@ macro_rules! linear_state { /// /// It is not a right revoked, either: the thread is dying, its /// unwind is not real-time work, and the bounded deferral - /// ([`crate::cpu::DYING_AGE_NS`]) is what keeps that from starving + /// ([`crate::sched::cpu::DYING_AGE_NS`]) is what keeps that from starving /// it. `a_killed_rt_thread_unwinds_in_the_normal_band` is the gate. pub fn serves_rt_band(&self) -> bool { self.0.rt().is_rt() && !self.0.shared().kill_pending() @@ -937,7 +937,7 @@ macro_rules! linear_state { } linear_state!( - /// Exists only inside a [`crate::queue::RunQueue`], or as the argument of + /// Exists only inside a [`crate::sched::queue::RunQueue`], or as the argument of /// the insert that puts it there. ReadyTask ); @@ -1164,7 +1164,7 @@ impl RunningTask { } /// The stable address of this task's saved context, for - /// [`crate::cpu::RunToken`]. + /// [`crate::sched::cpu::RunToken`]. pub(crate) fn ctx_ptr(&mut self) -> *mut X::Ctx { self.0.ctx_ptr() } diff --git a/toyos-sched/src/timer.rs b/kernel/pure/sched/timer.rs similarity index 94% rename from toyos-sched/src/timer.rs rename to kernel/pure/sched/timer.rs index c3b5dfea791..6d65e6e48e7 100644 --- a/toyos-sched/src/timer.rs +++ b/kernel/pure/sched/timer.rs @@ -19,7 +19,7 @@ //! an extra wake before that instant is a spurious pass and never a missed //! deadline. -use crate::hw::Nanos; +use crate::sched::hw::Nanos; /// What the one-shot timer must be programmed to at the end of a pass. /// Produced by `finish()` after every parking change, applied *last* — which @@ -46,7 +46,7 @@ impl TimerPlan { /// Pull the arming forward to `at`, if a caller has a reason to be woken /// that no parked deadline and no quantum accounts for — today - /// [`crate::cpu::Balance::PullWithRearm`]'s re-probe, and nothing else. + /// [`crate::sched::cpu::Balance::PullWithRearm`]'s re-probe, and nothing else. /// /// One direction only, and that is what keeps invariant T out of it: the /// invariant bounds the armed instant from *above* by the earliest event the @@ -69,7 +69,7 @@ impl TimerPlan { } } -/// Proof that a [`TimerPlan`] reached the hardware. [`crate::cpu::SleepToken`] +/// Proof that a [`TimerPlan`] reached the hardware. [`crate::sched::cpu::SleepToken`] /// cannot be built without one, so "halted with a deadline pending and the /// timer unarmed" is unrepresentable rather than asserted. pub struct TimerApplied { diff --git a/toyos-sched/src/watch.rs b/kernel/pure/sched/watch.rs similarity index 97% rename from toyos-sched/src/watch.rs rename to kernel/pure/sched/watch.rs index 8a584beaf36..7fc6fb4f8da 100644 --- a/toyos-sched/src/watch.rs +++ b/kernel/pure/sched/watch.rs @@ -2,7 +2,7 @@ //! exactly one [`Watch`], and making an object waitable is that one field. //! //! A waiter is one of two things. A **thread** is registered for the length of -//! one wait and is *woken*: a post runs [`crate::park::notify`] on it, which +//! one wait and is *woken*: a post runs [`crate::sched::park::notify`] on it, which //! claims its word if it is parked or committing and flags it otherwise, so its //! own next commit rechecks instead of parking. A **ring** entry is one poll a //! process submitted and is *posted*: a post hands it to [`Ring::fire`], once, @@ -35,12 +35,12 @@ use alloc::vec::Vec; use core::marker::PhantomData; -use crate::cpu::CpuHandles; -use crate::hw::Kicker; -use crate::mailbox::{PreemptGuard, SchedMsg}; -use crate::park::{notify, revoke}; -use crate::sync::{fence, Arc, AtomicU32, CellLock, Ordering}; -use crate::task::{TaskShared, WakeCause}; +use crate::sched::cpu::CpuHandles; +use crate::sched::hw::Kicker; +use crate::sched::mailbox::{PreemptGuard, SchedMsg}; +use crate::sched::park::{notify, revoke}; +use crate::sched::sync::{fence, Arc, AtomicU32, CellLock, Ordering}; +use crate::sched::task::{TaskShared, WakeCause}; /// Why a ring entry is posted. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -394,11 +394,11 @@ mod tests { extern crate std; use super::*; - use crate::cpu::{CpuHandle, CpuHandles}; - use crate::hw::CpuId; - use crate::mailbox::{mailbox, MailboxConsumer, NoPreempt}; - use crate::park::{prepare, Cancel, Commit, CurrentTask}; - use crate::task::{Notify, Refused, TaskKey, TaskState, WaitClass, WakeReason}; + use crate::sched::cpu::{CpuHandle, CpuHandles}; + use crate::sched::hw::CpuId; + use crate::sched::mailbox::{mailbox, MailboxConsumer, NoPreempt}; + use crate::sched::park::{prepare, Cancel, Commit, CurrentTask}; + use crate::sched::task::{Notify, Refused, TaskKey, TaskState, WaitClass, WakeReason}; use alloc::vec; use core::sync::atomic::{AtomicU32, Ordering}; use std::sync::Mutex; @@ -544,7 +544,7 @@ mod tests { w.register(&elsewhere, 8); parked(&elsewhere); // The first waiter's deadline fired on its home CPU. - assert!(matches!(timed_out.claim_wake(), crate::task::Claim::Parked(C0))); + assert!(matches!(timed_out.claim_wake(), crate::sched::task::Claim::Parked(C0))); assert_eq!(w.post_n(7, 1, woken(), &env), 1); assert_eq!(rx.pop(&NoPreempt), Some(Msg::Wake(TaskKey(2)))); assert_eq!(rx.pop(&NoPreempt), None); diff --git a/toyos-sched/tests/watch_list.rs b/kernel/pure/sched/watch_list.rs similarity index 94% rename from toyos-sched/tests/watch_list.rs rename to kernel/pure/sched/watch_list.rs index 63c60e7208e..4a49440ef28 100644 --- a/toyos-sched/tests/watch_list.rs +++ b/kernel/pure/sched/watch_list.rs @@ -6,18 +6,24 @@ //! while it holds a list lock, and every one it makes inside a post in place. //! The list lock counts its sections, and can run a stage between two of them. +// A `GlobalAlloc` is `unsafe` to implement. +#![allow(unsafe_code)] + use std::alloc::{GlobalAlloc, Layout, System}; +use std::boxed::Box; use std::cell::{Cell, RefCell}; use std::sync::atomic::{AtomicU32, Ordering::{AcqRel, Acquire, Relaxed}}; -use std::sync::{Arc, Mutex}; - -use toyos_sched::cpu::{CpuHandle, CpuHandles}; -use toyos_sched::hw::{CpuId, Kicker}; -use toyos_sched::mailbox::{mailbox, PreemptGuard, SchedMsg}; -use toyos_sched::park::{prepare, Cancel, Commit, CurrentTask}; -use toyos_sched::sync::CellLock; -use toyos_sched::task::{TaskKey, TaskShared, TaskState, WaitClass, WakeCause, WakeReason}; -use toyos_sched::watch::{Fire, Poster, Ring, Waiters, Watch, FEW}; +use std::sync::Mutex; +use std::vec::Vec; +use std::{thread_local, vec}; + +use crate::sched::cpu::{CpuHandle, CpuHandles}; +use crate::sched::hw::{CpuId, Kicker}; +use crate::sched::mailbox::{mailbox, PreemptGuard, SchedMsg}; +use crate::sched::park::{prepare, Cancel, Commit, CurrentTask}; +use crate::sched::sync::{Arc, CellLock}; +use crate::sched::task::{TaskKey, TaskShared, TaskState, WaitClass, WakeCause, WakeReason}; +use crate::sched::watch::{Fire, Poster, Ring, Waiters, Watch, FEW}; thread_local! { static HELD: Cell = const { Cell::new(false) }; diff --git a/toyos-sched/src/windows.rs b/kernel/pure/sched/windows.rs similarity index 99% rename from toyos-sched/src/windows.rs rename to kernel/pure/sched/windows.rs index 8c80caffb45..6f949cd51d9 100644 --- a/toyos-sched/src/windows.rs +++ b/kernel/pure/sched/windows.rs @@ -22,7 +22,7 @@ use core::fmt; -use crate::sync::{AtomicU64, Ordering}; +use crate::sched::sync::{AtomicU64, Ordering}; /// A CPU that has not joined the scheduler, or that stopped being tracked: /// every transition is accepted and records nothing. diff --git a/toyos-sched/sim/Cargo.toml b/kernel/sim/Cargo.toml similarity index 50% rename from toyos-sched/sim/Cargo.toml rename to kernel/sim/Cargo.toml index eed75e2558b..68b9054cb1f 100644 --- a/toyos-sched/sim/Cargo.toml +++ b/kernel/sim/Cargo.toml @@ -1,15 +1,15 @@ [package] -name = "toyos-sched-sim" -description = "The deterministic host simulator for toyos-sched: machine, explorer, shrinker and scenario corpus." +name = "kernel-sim" +description = "The deterministic host simulator for the kernel's scheduler core: machine, explorer, shrinker and scenario corpus." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" [dependencies] -# `check` turns on the core's own container/state-word cross-checks, and -# `protocol-port` compiles the escape hatch the negative gate needs. Neither -# is enabled by the kernel. -toyos-sched = { path = "..", features = ["std", "check", "protocol-port"] } +# `sched-check` turns on the core's own container/state-word cross-checks, and +# `protocol-port` compiles the escape hatch the negative gate needs, which no +# kernel build carries. +kernel = { path = "..", features = ["sched-check", "protocol-port"] } rand = { version = "0.9", features = ["small_rng"] } [features] @@ -17,13 +17,4 @@ rand = { version = "0.9", features = ["small_rng"] } # selected on the package the test runs in. Never on by default: with it every # path that chooses a CPU believes a stopped one's published load, which is what # `a_stopped_cpu_stops_taking_work` measures the cost of. -placement-ignores-staleness = ["toyos-sched/placement-ignores-staleness"] - -[lib] -name = "toyos_sched_sim" -path = "src/lib.rs" - -[[bin]] -name = "toyos-sched-sim" -path = "src/main.rs" - +placement-ignores-staleness = ["kernel/placement-ignores-staleness"] diff --git a/toyos-sched/sim/corpus/double_drop_exit_race_baseline.trace b/kernel/sim/corpus/double_drop_exit_race_baseline.trace similarity index 100% rename from toyos-sched/sim/corpus/double_drop_exit_race_baseline.trace rename to kernel/sim/corpus/double_drop_exit_race_baseline.trace diff --git a/toyos-sched/sim/corpus/old_migrate_kept_the_corpse_i14.trace b/kernel/sim/corpus/old_migrate_kept_the_corpse_i14.trace similarity index 100% rename from toyos-sched/sim/corpus/old_migrate_kept_the_corpse_i14.trace rename to kernel/sim/corpus/old_migrate_kept_the_corpse_i14.trace diff --git a/toyos-sched/sim/corpus/old_steal_port_i1.trace b/kernel/sim/corpus/old_steal_port_i1.trace similarity index 100% rename from toyos-sched/sim/corpus/old_steal_port_i1.trace rename to kernel/sim/corpus/old_steal_port_i1.trace diff --git a/toyos-sched/sim/corpus/old_steal_port_i8.trace b/kernel/sim/corpus/old_steal_port_i8.trace similarity index 100% rename from toyos-sched/sim/corpus/old_steal_port_i8.trace rename to kernel/sim/corpus/old_steal_port_i8.trace diff --git a/toyos-sched/sim/src/choice.rs b/kernel/sim/src/choice.rs similarity index 100% rename from toyos-sched/sim/src/choice.rs rename to kernel/sim/src/choice.rs diff --git a/toyos-sched/sim/src/explore.rs b/kernel/sim/src/explore.rs similarity index 99% rename from toyos-sched/sim/src/explore.rs rename to kernel/sim/src/explore.rs index bb942f3bb1b..19e0526e740 100644 --- a/toyos-sched/sim/src/explore.rs +++ b/kernel/sim/src/explore.rs @@ -5,8 +5,8 @@ //! its decisions, so a failure is a *value* — a list of integers — that can be //! replayed, shrunk and committed as a regression. -use toyos_sched::cpu::PassCostReport; -use toyos_sched::hw::CpuId; +use kernel::sched::cpu::PassCostReport; +use kernel::sched::hw::CpuId; use crate::choice::ChoiceStream; use crate::invariants; diff --git a/toyos-sched/sim/src/hw_impl.rs b/kernel/sim/src/hw_impl.rs similarity index 97% rename from toyos-sched/sim/src/hw_impl.rs rename to kernel/sim/src/hw_impl.rs index f3b66ff1546..3fc8beea8e6 100644 --- a/toyos-sched/sim/src/hw_impl.rs +++ b/kernel/sim/src/hw_impl.rs @@ -10,9 +10,9 @@ use std::collections::HashMap; use std::sync::Mutex; -use toyos_sched::cpu::RunToken; -use toyos_sched::hw::{CpuId, Hw, Kicker, Machine, Nanos, TraceEvent}; -use toyos_sched::task::{TaskAccounting, TaskKey}; +use kernel::sched::cpu::RunToken; +use kernel::sched::hw::{CpuId, Hw, Kicker, Machine, Nanos, TraceEvent}; +use kernel::sched::task::{TaskAccounting, TaskKey}; use crate::payload::{SimCtx, SimPayload}; diff --git a/toyos-sched/sim/src/invariants.rs b/kernel/sim/src/invariants.rs similarity index 98% rename from toyos-sched/sim/src/invariants.rs rename to kernel/sim/src/invariants.rs index f0ae80c4810..1576906f8bc 100644 --- a/toyos-sched/sim/src/invariants.rs +++ b/kernel/sim/src/invariants.rs @@ -14,11 +14,11 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc as StdArc; -use toyos_sched::fair::{MAX_VRUNTIME_LAG_NS, QUANTUM_NS}; -use toyos_sched::invariants::{residents, Container}; -use toyos_sched::task::{TaskKey, TaskState}; +use kernel::sched::fair::{MAX_VRUNTIME_LAG_NS, QUANTUM_NS}; +use kernel::sched::invariants::{residents, Container}; +use kernel::sched::task::{TaskKey, TaskState}; -use toyos_sched::cpu::{DYING_AGE_NS, DYING_CHUNK_NS}; +use kernel::sched::cpu::{DYING_AGE_NS, DYING_CHUNK_NS}; use crate::latency::ReadyCause; use crate::vm::{FairEpoch, Vm, IPI_LATENCY_NS, RUN_CHUNK_NS, UNWIND_NS}; @@ -39,8 +39,8 @@ use crate::vm::{FairEpoch, Vm, IPI_LATENCY_NS, RUN_CHUNK_NS, UNWIND_NS}; /// /// What ships is neither absolute. `CpuSched::pick` takes the dying list ahead /// of the RT band only once its head has waited -/// [`toyos_sched::cpu::DYING_AGE_NS`], and dispatches it for exactly -/// [`toyos_sched::cpu::DYING_CHUNK_NS`] with `CpuSched::aged_grant` holding the +/// [`kernel::sched::cpu::DYING_AGE_NS`], and dispatches it for exactly +/// [`kernel::sched::cpu::DYING_CHUNK_NS`] with `CpuSched::aged_grant` holding the /// preemption off for that long and no longer. So a ready RT task gives up one /// chunk, and gives it up **at most once per age window**: the window this /// bound measures closes the instant the RT task runs, and the corpse's stamp @@ -106,7 +106,7 @@ fn retire_latency_bound(max_kernel_section: u64, peers: usize) -> u64 { /// length, which is the factor term 4 of [`retire_latency_bound`] carries. /// /// `CpuSched::pick` delivers an aged corpse one -/// [`toyos_sched::cpu::DYING_CHUNK_NS`] per +/// [`kernel::sched::cpu::DYING_CHUNK_NS`] per /// `DYING_AGE_NS + DYING_CHUNK_NS`, so `UNWIND_NS` of the victim's own CPU time /// takes that multiple of wall clock to spend when the RT band never empties. /// It is charged unconditionally because a bound is a worst case, and it is a @@ -308,7 +308,7 @@ fn check_retires(vm: &mut Vm<'_>) { let fresh = s.trace[vm.trace_cursor..] .iter() .filter_map(|ev| match ev.kind { - toyos_sched::hw::TraceKind::Migrate { task, to } => Some((task, to.0)), + kernel::sched::hw::TraceKind::Migrate { task, to } => Some((task, to.0)), _ => None, }) .collect(); @@ -789,7 +789,7 @@ fn open_thread_window(vm: &mut Vm<'_>, balanced: bool, live_threads: &BTreeSet) { // a preempt-off section can overrun the quantum it started inside, and the // model advances the clock in chunks, so the quantum's expiry is observed // one chunk late and the pass that acts on it lands a chunk after that. - let bound = toyos_sched::fair::QUANTUM_NS + vm.max_kernel_section() + 2 * RUN_CHUNK_NS; + let bound = kernel::sched::fair::QUANTUM_NS + vm.max_kernel_section() + 2 * RUN_CHUNK_NS; let problems: Vec = vm .boosted_run .iter() diff --git a/toyos-sched/sim/src/latency.rs b/kernel/sim/src/latency.rs similarity index 99% rename from toyos-sched/sim/src/latency.rs rename to kernel/sim/src/latency.rs index eac7cf02b04..20f1ef4a8f0 100644 --- a/toyos-sched/sim/src/latency.rs +++ b/kernel/sim/src/latency.rs @@ -10,7 +10,7 @@ //! bounds. //! //! **Four exact numbers rather than a histogram, and the fourth is the reason.** -//! A power-of-two histogram — [`toyos_sched::cpu::PassCosts`]' scheme, the one +//! A power-of-two histogram — [`kernel::sched::cpu::PassCosts`]' scheme, the one //! thing here that could have been borrowed — answers a quantile to within a //! factor of two, and every bound this suite states is within a factor of two of //! the quantity it bounds: one quantum against one quantum plus 2.2 ms of diff --git a/toyos-sched/sim/src/lib.rs b/kernel/sim/src/lib.rs similarity index 81% rename from toyos-sched/sim/src/lib.rs rename to kernel/sim/src/lib.rs index ea8728877b8..c0d10a20e6b 100644 --- a/toyos-sched/sim/src/lib.rs +++ b/kernel/sim/src/lib.rs @@ -1,4 +1,4 @@ -//! Host-side deterministic simulator for `toyos-sched`. +//! Host-side deterministic simulator for the kernel's scheduler core, `kernel::sched`. //! //! The virtual machine, the explorer, the shrinker, the corpus and the //! scenario library are here, and the exit criterion they exist to serve is @@ -14,8 +14,8 @@ //! which keeps a budget it can afford: //! //! ```text -//! cargo run --release -p toyos-sched-sim -- gate 10000 # 10^4 seeds/scenario -//! cargo run --release -p toyos-sched-sim -- fuzz-sweep 10000000 # 10^7 steps/scenario +//! cargo run --release -p kernel-sim -- gate 10000 # 10^4 seeds/scenario +//! cargo run --release -p kernel-sim -- fuzz-sweep 10000000 # 10^7 steps/scenario //! ``` // Three exemptions, each a single item and each unavoidable: implementing the diff --git a/toyos-sched/sim/src/main.rs b/kernel/sim/src/main.rs similarity index 98% rename from toyos-sched/sim/src/main.rs rename to kernel/sim/src/main.rs index feae57a2f1f..7eebe451970 100644 --- a/toyos-sched/sim/src/main.rs +++ b/kernel/sim/src/main.rs @@ -3,14 +3,14 @@ use std::io::Read; use std::process::ExitCode; -use toyos_sched_sim::choice::ChoiceStream; -use toyos_sched_sim::explore::run; -use toyos_sched_sim::scenarios; -use toyos_sched_sim::shrink; -use toyos_sched_sim::sweep; +use kernel_sim::choice::ChoiceStream; +use kernel_sim::explore::run; +use kernel_sim::scenarios; +use kernel_sim::shrink; +use kernel_sim::sweep; const USAGE: &str = "\ -usage: toyos-sched-sim [args] +usage: kernel-sim [args] run [seed] one seeded exploration pct [seed] one PCT-driven exploration fuzz decisions driven by raw fuzz bytes on stdin diff --git a/toyos-sched/sim/src/msg.rs b/kernel/sim/src/msg.rs similarity index 78% rename from toyos-sched/sim/src/msg.rs rename to kernel/sim/src/msg.rs index 1c95b0103bb..f45c97bf5ab 100644 --- a/toyos-sched/sim/src/msg.rs +++ b/kernel/sim/src/msg.rs @@ -1,8 +1,8 @@ //! Type aliases that pin the core's generics to the simulator's payload. -use toyos_sched::cpu::{CpuHandles, CpuSched, Env, SchedPass}; -use toyos_sched::msg::Msg; -use toyos_sched::watch::Watch; +use kernel::sched::cpu::{CpuHandles, CpuSched, Env, SchedPass}; +use kernel::sched::msg::Msg; +use kernel::sched::watch::Watch; use crate::hw_impl::SimHw; use crate::payload::{NoRing, SimPayload, SimPreempt, SimWatchList}; diff --git a/toyos-sched/sim/src/payload.rs b/kernel/sim/src/payload.rs similarity index 92% rename from toyos-sched/sim/src/payload.rs rename to kernel/sim/src/payload.rs index c67b3871412..e34fb1a64fd 100644 --- a/toyos-sched/sim/src/payload.rs +++ b/kernel/sim/src/payload.rs @@ -9,11 +9,11 @@ use std::sync::{Arc, Mutex}; -use toyos_sched::fair::ShareState; -use toyos_sched::mailbox::PreemptGuard; -use toyos_sched::sync::CellLock; -use toyos_sched::task::{SchedPayload, TaskKey}; -use toyos_sched::watch::{Fire, Ring, Waiters}; +use kernel::sched::fair::ShareState; +use kernel::sched::mailbox::PreemptGuard; +use kernel::sched::sync::CellLock; +use kernel::sched::task::{SchedPayload, TaskKey}; +use kernel::sched::watch::{Fire, Ring, Waiters}; use crate::msg::SimMsg; diff --git a/toyos-sched/sim/src/scenarios.rs b/kernel/sim/src/scenarios.rs similarity index 99% rename from toyos-sched/sim/src/scenarios.rs rename to kernel/sim/src/scenarios.rs index 91af1cd5130..2e82c8ec6ba 100644 --- a/toyos-sched/sim/src/scenarios.rs +++ b/kernel/sim/src/scenarios.rs @@ -5,9 +5,9 @@ //! workload, and a scenario that takes ten thousand steps to quiesce buys one //! schedule per second instead of a thousand. -use toyos_sched::cpu::Balance; -use toyos_sched::queue::FairOrder; -use toyos_sched::task::WaitClass; +use kernel::sched::cpu::Balance; +use kernel::sched::queue::FairOrder; +use kernel::sched::task::WaitClass; use crate::vm::RUN_CHUNK_NS; use crate::workload::{ @@ -46,7 +46,7 @@ fn scenario( // The shipped policy (owner decision 2026-08-23), so every scenario // that does not name its own drives the balance path the kernel runs. balance: Balance::PushOnSurplus { - threshold: toyos_sched::cpu::PUSH_THRESHOLD, + threshold: kernel::sched::cpu::PUSH_THRESHOLD, }, order: FairOrder::InsertSequence, pass_cost_ns: 0, @@ -717,7 +717,7 @@ pub fn fork_storm() -> Scenario { /// widths at `ae974921`, on one host, with no other measurement running: /// /// ```text -/// cargo run --release -p toyos-sched-sim -- measure fairness_storm: 500 +/// cargo run --release -p kernel-sim -- measure fairness_storm: 500 /// ``` /// /// The last column is `Outcome::fair_over_bound`, and it is **not** the amount @@ -855,7 +855,7 @@ const WORK: u64 = 60 * MS; /// at `QUANTUM + 2 × RUN_CHUNK` each). Every number came from /// /// ```text -/// cargo run --release -p toyos-sched-sim -- measure fairness_storm: +/// cargo run --release -p kernel-sim -- measure fairness_storm: /// ``` /// /// on `be4b34a` with this change applied, on one host, with no other @@ -1026,7 +1026,7 @@ pub fn fair_double_charge() -> Scenario { /// compiled in, or it is reading a clock that never moves, and every check build /// that ever came back green certified nothing about how long a pass takes. pub fn overlong_pass() -> Scenario { - let mut scenario = lost_wake_pipe().with_pass_cost(5 * toyos_sched::cpu::MAX_PASS_NS); + let mut scenario = lost_wake_pipe().with_pass_cost(5 * kernel::sched::cpu::MAX_PASS_NS); scenario.name = "overlong_pass"; scenario } @@ -1472,7 +1472,7 @@ pub const STOPPED_CPU_PROGRAMS: usize = 24; /// so its numbers are believed for a whole `STALE_PASS_NS` from clock zero. /// /// `sim/tests/policy.rs` measures `Outcome::never_ran` against it, and -/// `toyos-sched`'s `placement-ignores-staleness` is the arm that says what the +/// the kernel's `placement-ignores-staleness` is the arm that says what the /// number is when a stopped CPU's published load is believed. pub fn stopped_cpu() -> Scenario { let mut scenario = scenario( diff --git a/toyos-sched/sim/src/shrink.rs b/kernel/sim/src/shrink.rs similarity index 100% rename from toyos-sched/sim/src/shrink.rs rename to kernel/sim/src/shrink.rs diff --git a/toyos-sched/sim/src/sweep.rs b/kernel/sim/src/sweep.rs similarity index 100% rename from toyos-sched/sim/src/sweep.rs rename to kernel/sim/src/sweep.rs diff --git a/toyos-sched/sim/src/vm.rs b/kernel/sim/src/vm.rs similarity index 98% rename from toyos-sched/sim/src/vm.rs rename to kernel/sim/src/vm.rs index f57f056b105..f7e85f7c042 100644 --- a/toyos-sched/sim/src/vm.rs +++ b/kernel/sim/src/vm.rs @@ -14,20 +14,20 @@ use std::cell::Cell; use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc as StdArc; -use toyos_sched::cpu::{Action, CpuHandle, CpuSched, Env, SchedPass}; -use toyos_sched::fair::{FairShare, Frontier, ShareState}; -use toyos_sched::hw::{CpuId, Hw, Kicker, Machine, Nanos}; -use toyos_sched::invariants::Container; -use toyos_sched::mailbox::{mailbox, Kick, Urgency}; -use toyos_sched::msg::Msg; -use toyos_sched::retire; -use toyos_sched::sync::Arc; -use toyos_sched::task::{ +use kernel::sched::cpu::{Action, CpuHandle, CpuSched, Env, SchedPass}; +use kernel::sched::fair::{FairShare, Frontier, ShareState}; +use kernel::sched::hw::{CpuId, Hw, Kicker, Machine, Nanos}; +use kernel::sched::invariants::Container; +use kernel::sched::mailbox::{mailbox, Kick, Urgency}; +use kernel::sched::msg::Msg; +use kernel::sched::retire; +use kernel::sched::sync::Arc; +use kernel::sched::task::{ ReadyTask, RtState, TaskAccounting, TaskBuilder, TaskKey, TaskShared, TaskState, WaitClass, WakeCause, WakeReason, }; -use toyos_sched::park::{prepare, Cancel, Cancelled, Commit, CommittedTicket, WaitTicket}; -use toyos_sched::watch::{Poster, Waiters, Watch}; +use kernel::sched::park::{prepare, Cancel, Cancelled, Commit, CommittedTicket, WaitTicket}; +use kernel::sched::watch::{Poster, Waiters, Watch}; use crate::choice::ChoiceStream; use crate::hw_impl::SimHw; @@ -154,7 +154,7 @@ pub struct ProcState { /// Every step of that was true and the conclusion was a blindfold. /// /// So the quantity is bounded at its source instead: `CpuSched::pick`'s -/// [`toyos_sched::cpu::DYING_AGE_NS`] makes the RT band's precedence over a +/// [`kernel::sched::cpu::DYING_AGE_NS`] makes the RT band's precedence over a /// corpse a bounded deferral. The RT service the victim's CPU owed is *in* the /// number, which is the only way the number means anything. /// @@ -363,7 +363,7 @@ pub struct Vm<'q> { /// Invariant I9's accumulator: per task, the lend counter it was last seen /// with and the running time charged to it since, while boosted. Reset when /// the counter moves, which is the only way to tell a fresh lend from - /// [`toyos_sched::task::RtState::arm`]'s re-arm from outside the core. + /// [`kernel::sched::task::RtState::arm`]'s re-arm from outside the core. pub boosted_run: BTreeMap, /// Invariant I5's measurement: CPU nanoseconds delivered to each process, /// summed at the same clock advances `busy_ns` is, so the two are exact @@ -525,7 +525,7 @@ pub struct Vm<'q> { /// The surplus at which a victim is worth probing — /// `SchedPass::post_steal_probe`'s own inequality, the core's constant because /// [`Vm::probe_gap_ns`] has to ask the same question from outside the core. -const PROBE_WORTH_IT: u32 = toyos_sched::cpu::PUSH_THRESHOLD; +const PROBE_WORTH_IT: u32 = kernel::sched::cpu::PUSH_THRESHOLD; /// One contention window: a maximal interval over which the same set of /// fair-band processes was continuously runnable. @@ -1725,7 +1725,7 @@ impl<'q> Vm<'q> { fn scan_containers(&self, key: TaskKey) -> bool { (0..self.scenario.cpus) - .any(|cpu| toyos_sched::invariants::residents(&self.cpus[cpu]).any(|(k, _)| k == key)) + .any(|cpu| kernel::sched::invariants::residents(&self.cpus[cpu]).any(|(k, _)| k == key)) } /// Release the process's own reference, asserting what the kernel's diff --git a/toyos-sched/sim/src/workload.rs b/kernel/sim/src/workload.rs similarity index 98% rename from toyos-sched/sim/src/workload.rs rename to kernel/sim/src/workload.rs index 3f4e39c57d4..91e49661a93 100644 --- a/toyos-sched/sim/src/workload.rs +++ b/kernel/sim/src/workload.rs @@ -11,9 +11,9 @@ //! second opcode would be modelling a second wake path, and there is exactly //! one. -use toyos_sched::cpu::Balance; -use toyos_sched::queue::FairOrder; -use toyos_sched::task::WaitClass; +use kernel::sched::cpu::Balance; +use kernel::sched::queue::FairOrder; +use kernel::sched::task::WaitClass; #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Op { @@ -202,7 +202,7 @@ pub enum AgeShape { /// apart, or it is not measuring the shipped policy at all. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum ShareShape { - /// The shipped policy: one [`toyos_sched::fair::FairShare`] per process, + /// The shipped policy: one [`kernel::sched::fair::FairShare`] per process, /// reached through every thread of it. PerProcess, /// One share per thread, so a process buys CPU by forking. See diff --git a/toyos-sched/sim/tests/deadline_claim_race.rs b/kernel/sim/tests/deadline_claim_race.rs similarity index 93% rename from toyos-sched/sim/tests/deadline_claim_race.rs rename to kernel/sim/tests/deadline_claim_race.rs index 0a18c1384c2..663d9763e8e 100644 --- a/toyos-sched/sim/tests/deadline_claim_race.rs +++ b/kernel/sim/tests/deadline_claim_race.rs @@ -20,20 +20,20 @@ use std::sync::Arc as StdArc; -use toyos_sched::cpu::{Action, Balance, CpuHandle, CpuHandles, CpuSched, Env, SchedPass}; -use toyos_sched::fair::{FairShare, Frontier, ShareState}; -use toyos_sched::hw::{CpuId, Kicker, Nanos}; -use toyos_sched::mailbox::{mailbox, Kick, Urgency}; -use toyos_sched::msg::Msg; -use toyos_sched::sync::Arc; -use toyos_sched::task::{ +use kernel::sched::cpu::{Action, Balance, CpuHandle, CpuHandles, CpuSched, Env, SchedPass}; +use kernel::sched::fair::{FairShare, Frontier, ShareState}; +use kernel::sched::hw::{CpuId, Kicker, Nanos}; +use kernel::sched::mailbox::{mailbox, Kick, Urgency}; +use kernel::sched::msg::Msg; +use kernel::sched::sync::Arc; +use kernel::sched::task::{ Claim, RtState, TaskBuilder, TaskKey, TaskShared, WaitClass, WakeCause, WakeReason, }; -use toyos_sched::park::{prepare, Cancel, Commit}; +use kernel::sched::park::{prepare, Cancel, Commit}; -use toyos_sched_sim::hw_impl::SimHw; -use toyos_sched_sim::msg::SimMsg; -use toyos_sched_sim::payload::{MockAddressSpace, SimCtx, SimPayload, SimPreempt, StdLock}; +use kernel_sim::hw_impl::SimHw; +use kernel_sim::msg::SimMsg; +use kernel_sim::payload::{MockAddressSpace, SimCtx, SimPayload, SimPreempt, StdLock}; const CPU0: CpuId = CpuId(0); const KEY: TaskKey = TaskKey(1); diff --git a/toyos-sched/sim/tests/policy.rs b/kernel/sim/tests/policy.rs similarity index 98% rename from toyos-sched/sim/tests/policy.rs rename to kernel/sim/tests/policy.rs index c6f5adb4483..d6b3d855ca6 100644 --- a/toyos-sched/sim/tests/policy.rs +++ b/kernel/sim/tests/policy.rs @@ -56,7 +56,7 @@ //! quantity the protocol owns end to end. //! //! Every number came from this file at 16–40 seeds per point: -//! `cargo test -p toyos-sched-sim --test policy -- --nocapture`. The tables in +//! `cargo test -p kernel-sim --test policy -- --nocapture`. The tables in //! each test carry the full sweeps. //! //! # What the numbers say, in one paragraph @@ -96,15 +96,15 @@ //! a reader is owed that. The multi-CPU cases do vary, and there the sweep is //! doing what a sweep is for. -use toyos_sched::cpu::Balance; -use toyos_sched::cpu::STALE_PASS_NS; -use toyos_sched::fair::QUANTUM_NS; -use toyos_sched_sim::choice::ChoiceStream; -use toyos_sched_sim::explore::{run, Outcome}; -use toyos_sched_sim::latency::{Latency, ReadyCause}; -use toyos_sched_sim::scenarios; -use toyos_sched_sim::vm::{IPI_LATENCY_NS, RUN_CHUNK_NS}; -use toyos_sched_sim::workload::{Scenario, ShareShape}; +use kernel::sched::cpu::Balance; +use kernel::sched::cpu::STALE_PASS_NS; +use kernel::sched::fair::QUANTUM_NS; +use kernel_sim::choice::ChoiceStream; +use kernel_sim::explore::{run, Outcome}; +use kernel_sim::latency::{Latency, ReadyCause}; +use kernel_sim::scenarios; +use kernel_sim::vm::{IPI_LATENCY_NS, RUN_CHUNK_NS}; +use kernel_sim::workload::{Scenario, ShareShape}; const MS: u64 = 1_000_000; @@ -158,7 +158,7 @@ fn sweep(scenario: &Scenario, seeds: u64, mut fold: impl FnMut(&Outcome)) { /// /// **What it found.** `solo` finishes exactly `N+1` quanta late at every width /// above one, and the seeds do not disagree by a nanosecond -/// (`cargo test -p toyos-sched-sim --test policy -- --nocapture`): +/// (`cargo test -p kernel-sim --test policy -- --nocapture`): /// /// | N | T_solo | deficit | in quanta | `solo`'s share over that window | /// |---|---|---|---|---| @@ -789,7 +789,7 @@ const REARM_TIMES: u32 = 4; /// The surplus a push fires at: `SchedPass::post_steal_probe`'s own inequality, /// so a push never wakes a CPU whose victim would refuse it. The core's /// constant, which is also the kernel's shipped `threshold`. -const PUSH_THRESHOLD: u32 = toyos_sched::cpu::PUSH_THRESHOLD; +const PUSH_THRESHOLD: u32 = kernel::sched::cpu::PUSH_THRESHOLD; /// The four policies the two cure cases and the cost table are read across. fn policies() -> [(&'static str, Balance); 4] { @@ -1351,7 +1351,7 @@ fn a_runnable_task_waits_at_most_one_dispatch_per_rival() { samples += outcome .run_wait .iter() - .map(toyos_sched_sim::latency::RunWait::samples) + .map(kernel_sim::latency::RunWait::samples) .sum::(); }); println!( @@ -1391,7 +1391,7 @@ fn a_runnable_task_waits_at_most_one_dispatch_per_rival() { /// built from maxima reads such a machine as a quiet one. /// /// **The bound is derived.** A stopped CPU is a legal target while its doorbell -/// edge is still down, or while [`toyos_sched::cpu::STALE_PASS_NS`] has not +/// edge is still down, or while [`kernel::sched::cpu::STALE_PASS_NS`] has not /// elapsed since the pass it never took; both windows close at `STALE_PASS_NS` /// from clock zero, so the losses are the launcher's spawns inside it. /// diff --git a/toyos-sched/sim/tests/scenarios.rs b/kernel/sim/tests/scenarios.rs similarity index 99% rename from toyos-sched/sim/tests/scenarios.rs rename to kernel/sim/tests/scenarios.rs index dec9455d753..dcac8327b5b 100644 --- a/toyos-sched/sim/tests/scenarios.rs +++ b/kernel/sim/tests/scenarios.rs @@ -56,17 +56,17 @@ //! `gate` carries all ten gates and both controls: //! //! ```text -//! cargo run --release -p toyos-sched-sim -- gate 10000 -//! cargo run --release -p toyos-sched-sim -- fuzz-sweep 10000000 +//! cargo run --release -p kernel-sim -- gate 10000 +//! cargo run --release -p kernel-sim -- fuzz-sweep 10000000 //! ``` use std::collections::BTreeMap; -use toyos_sched_sim::choice::ChoiceStream; -use toyos_sched_sim::explore::run; -use toyos_sched_sim::scenarios; -use toyos_sched_sim::shrink; -use toyos_sched_sim::sweep; +use kernel_sim::choice::ChoiceStream; +use kernel_sim::explore::run; +use kernel_sim::scenarios; +use kernel_sim::shrink; +use kernel_sim::sweep; /// Seeds per scenario in the in-test sweep. Every seed is a complete /// exploration with all of I1–I13 checked after every step. @@ -342,7 +342,7 @@ fn a_retire_completes_inside_its_derived_bound() { /// lasts one chunk plus what the model can prove it owes elsewhere, which is /// what its doc promises and what the code did not do. /// -/// [`toyos_sched_sim::vm::Vm::unwind_at`] closes `Vm::enabled`'s gate on every +/// [`kernel_sim::vm::Vm::unwind_at`] closes `Vm::enabled`'s gate on every /// *other* CPU's `Exec` while one CPU owes an unwind step. Stamped only on the /// false→true transition, and read with a `find` that named one owed CPU and /// denied the step to every other, that gate stayed shut for a CPU's whole @@ -370,7 +370,7 @@ fn a_retire_completes_inside_its_derived_bound() { #[test] fn the_unwind_gate_lasts_one_chunk_and_not_one_unwind() { let scenario = scenarios::retire_under_balance(); - let bound = (2 + scenario.cpus as u64) * toyos_sched_sim::vm::RUN_CHUNK_NS; + let bound = (2 + scenario.cpus as u64) * kernel_sim::vm::RUN_CHUNK_NS; let mut worst = 0; for seed in 0..SEEDS { let mut choices = if seed % 2 == 0 { @@ -867,7 +867,7 @@ fn a_pass_that_overruns_its_budget_is_recorded() { // answer is known: every sample at the modelled cost, so every quantile // is the bucket that cost falls in. assert!( - report.count == 0 || report.quantile_upper_ns(1, 2) > toyos_sched::cpu::MAX_PASS_NS, + report.count == 0 || report.quantile_upper_ns(1, 2) > kernel::sched::cpu::MAX_PASS_NS, "cpu{}'s median came back at {} ns with every pass modelled at {cost}", report.cpu.0, report.quantile_upper_ns(1, 2), diff --git a/kernel/src/arch/aarch64/hw.rs b/kernel/src/arch/aarch64/hw.rs index 0109c8d5230..556400418d7 100644 --- a/kernel/src/arch/aarch64/hw.rs +++ b/kernel/src/arch/aarch64/hw.rs @@ -1,8 +1,8 @@ //! AArch64's half of `crate::hw::KernelHw`: the halt and the context switch. -use toyos_sched::cpu::RunToken; -use toyos_sched::hw::Hw; -use toyos_sched::task::{TaskAccounting, TaskKey}; +use kernel::sched::cpu::RunToken; +use kernel::sched::hw::Hw; +use kernel::sched::task::{TaskAccounting, TaskKey}; use super::switch::{context_switch, RETURN_AT}; use super::{cpu, percpu}; diff --git a/kernel/src/arch/aarch64/irqchip.rs b/kernel/src/arch/aarch64/irqchip.rs index 09e28273443..13cfe1de5f5 100644 --- a/kernel/src/arch/aarch64/irqchip.rs +++ b/kernel/src/arch/aarch64/irqchip.rs @@ -430,7 +430,7 @@ pub fn floor_selftest() { ctl & TIMER_ISTATUS != 0 }; settles(100, "the timer armed for its floor", due); - let now = arm_within(toyos_sched::fair::QUANTUM_NS); + let now = arm_within(kernel::sched::fair::QUANTUM_NS); let cval: u64; // SAFETY: reads the EL1 virtual timer's comparator. unsafe { core::arch::asm!("mrs {}, cntv_cval_el0", out(reg) cval, options(nomem, nostack, preserves_flags)) }; diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 4d40e40dc3b..811580f35b7 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -25,7 +25,7 @@ use alloc::boxed::Box; use alloc::vec::Vec; use toyos_bootmap::aarch64::{coverage, direct_map_end, Coverage, ATTR_DEVICE, ATTR_NORMAL, ATTR_NORMAL_NC}; -use toyos_pcid::{Alloc, Pcid, PcidPool}; +use kernel::pcid::{Alloc, Pcid, PcidPool}; use super::tlb; use crate::mm::policy::{CachePolicy, MmioPolicy, Prot, WindowProt}; @@ -279,7 +279,7 @@ impl Root { } /// No invalidation: the ASID is this space's alone, and a returned one was - /// dropped from every CPU before it was issued again (`toyos_pcid`). + /// dropped from every CPU before it was issued again. /// # Safety /// The underlying page tables must be valid and live. pub unsafe fn activate(self) { @@ -288,7 +288,7 @@ impl Root { } } -/// The ASID allocator: `toyos_pcid`'s tags, which 16-bit ASIDs hold whole, +/// The ASID allocator: `kernel::pcid`'s tags, which 16-bit ASIDs hold whole, /// and its reclaim given the shootdown it asks for. static ASIDS: Lock = Lock::new(PcidPool::new()); @@ -310,7 +310,7 @@ enum Asid { impl Asid { fn value(&self) -> u16 { match self { - Self::Kernel => toyos_pcid::KERNEL_PCID, + Self::Kernel => kernel::pcid::KERNEL_PCID, Self::User(guard) => guard.0.get(), } } diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 04caa6b0f89..298d52a54ed 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -18,7 +18,7 @@ use core::fmt; use core::sync::atomic::{AtomicU32, AtomicU64, Ordering::Relaxed}; -use toyos_sched::hw::{CpuId, Machine, TraceEvent, TraceKind}; +use kernel::sched::hw::{CpuId, Machine, TraceEvent, TraceKind}; use super::percpu::{self, CpuFaultState}; use super::{cpu, irqchip}; diff --git a/kernel/src/arch/x86_64/hw.rs b/kernel/src/arch/x86_64/hw.rs index e1479a01660..5e17d59f321 100644 --- a/kernel/src/arch/x86_64/hw.rs +++ b/kernel/src/arch/x86_64/hw.rs @@ -2,9 +2,9 @@ use core::arch::asm; -use toyos_sched::cpu::RunToken; -use toyos_sched::hw::Hw; -use toyos_sched::task::{TaskAccounting, TaskKey}; +use kernel::sched::cpu::RunToken; +use kernel::sched::hw::Hw; +use kernel::sched::task::{TaskAccounting, TaskKey}; use crate::arch::{cpu, percpu}; use crate::hw::{report_contexts, KernelHw}; @@ -255,7 +255,7 @@ static PROBE_WATCH: crate::watch::Watch = crate::watch::Watch::new(); pub fn sysret_ss_probe(parkable: &crate::scheduler::Parkable) { use crate::watch; use crate::time::{Deadline, Duration}; - use toyos_sched::task::WaitClass; + use kernel::sched::task::WaitClass; let Some(armed) = watch::arm(&PROBE_WATCH, 0, WaitClass::Other) else { crate::log!("sysret-ss: probe could not arm"); diff --git a/kernel/src/arch/x86_64/idt/timer.rs b/kernel/src/arch/x86_64/idt/timer.rs index 1ce8cb24fba..a5faaa65650 100644 --- a/kernel/src/arch/x86_64/idt/timer.rs +++ b/kernel/src/arch/x86_64/idt/timer.rs @@ -1,4 +1,4 @@ -use toyos_sched::hw::{CpuId, Machine, TraceEvent, TraceKind}; +use kernel::sched::hw::{CpuId, Machine, TraceEvent, TraceKind}; use crate::arch::entry::{restore_user_state, ring3_naked_asm, save_user_state}; use crate::hw::HW; diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index b8d0d70e859..137c4452a1b 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -11,7 +11,7 @@ use alloc::boxed::Box; use alloc::vec::Vec; use crate::hasher::HashMap; -use toyos_pcid::{Alloc, Pcid, PcidPool}; +use kernel::pcid::{Alloc, Pcid, PcidPool}; use crate::mm::{UserAddr, PAGE_2M}; use crate::mm::policy::{CachePolicy, MmioPolicy, Prot, WindowProt}; @@ -292,7 +292,7 @@ impl Cr3 { /// Sets NOFLUSH when PCID is active — sound only because a user tag is owned: /// no other live space holds it, and a reused one was flushed from every CPU - /// before this space took it (`toyos_pcid`). + /// before this space took it. /// # Safety /// The underlying page tables must be valid and live. pub unsafe fn activate(self) { @@ -311,7 +311,7 @@ impl Cr3 { } } -/// The user PCID allocator: `toyos_pcid` owns the decision that no live tag is +/// The user PCID allocator: `kernel::pcid` owns the decision that no live tag is /// reissued; here it is given the shootdown its [`Alloc::NeedsFlush`] asks for. static PCID_POOL: Lock = Lock::new(PcidPool::new()); @@ -336,7 +336,7 @@ enum PcidHandle { impl PcidHandle { fn value(&self) -> u16 { match self { - Self::Kernel => toyos_pcid::KERNEL_PCID, + Self::Kernel => kernel::pcid::KERNEL_PCID, Self::User(g) => g.0.get(), } } diff --git a/kernel/src/block.rs b/kernel/src/block.rs index 72f8fe87e85..6e2f7607f21 100644 --- a/kernel/src/block.rs +++ b/kernel/src/block.rs @@ -89,7 +89,7 @@ pub(crate) fn between_attempts(attempt: u32) { &parkable, handle.watch(), 0, - toyos_sched::task::WaitClass::Other, + kernel::sched::task::WaitClass::Other, deadline, || false, ); diff --git a/kernel/src/deadline.rs b/kernel/src/deadline.rs index 8fdc1d07159..d85aca36474 100644 --- a/kernel/src/deadline.rs +++ b/kernel/src/deadline.rs @@ -278,7 +278,7 @@ fn this_cpu() -> ! { // claims. crate::preempt::disable(); let arrived_awake = crate::arch::cpu::interrupts_enabled(); - crate::arch::irqchip::arm_within(toyos_sched::fair::QUANTUM_NS); + crate::arch::irqchip::arm_within(kernel::sched::fair::QUANTUM_NS); crate::arch::cpu::enable_interrupts(); log!( "wedge: cpu{} {}", diff --git a/kernel/src/hw.rs b/kernel/src/hw.rs index c40c8af0242..75740cbd90c 100644 --- a/kernel/src/hw.rs +++ b/kernel/src/hw.rs @@ -9,9 +9,9 @@ use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; -use toyos_sched::cpu::SleepToken; -use toyos_sched::fair::QUANTUM_NS; -use toyos_sched::hw::{CpuId, Kicker, Machine, Nanos, TraceEvent}; +use kernel::sched::cpu::SleepToken; +use kernel::sched::fair::QUANTUM_NS; +use kernel::sched::hw::{CpuId, Kicker, Machine, Nanos, TraceEvent}; use crate::arch::{irqchip, percpu}; use crate::sched::payload::KernelCtx; @@ -90,9 +90,8 @@ impl Machine for KernelHw { // timer, which is stopped — and then held in the kernel takes no timer // interrupt at all. `crate::deadline`'s poll and `crate::hardlockup`'s // sample both rest on some CPU taking one. Arming earlier than the - // scheduler planned is a spurious pass and never a missed deadline - // (`toyos_sched::timer::TimerPlan`), and the next pass replaces it - // either way. + // scheduler planned is a spurious pass and never a missed deadline, + // and the next pass replaces it either way. // // Asked, because it is only those two that need it: a boot under no // bound pays a timer read and two writes per wake for nothing. diff --git a/kernel/src/inbox/mod.rs b/kernel/src/inbox/mod.rs index 68d56fb0a48..124f06812dc 100644 --- a/kernel/src/inbox/mod.rs +++ b/kernel/src/inbox/mod.rs @@ -34,8 +34,8 @@ use alloc::sync::Arc; use core::sync::atomic::Ordering; -use toyos_sched::task::WaitClass; -use toyos_sched::watch::{Fire, Ring}; +use kernel::sched::task::WaitClass; +use kernel::sched::watch::{Fire, Ring}; use crate::object::shm::SharedMemObject; use crate::object::{ops, HandleError, KObjectRef}; diff --git a/kernel/src/inbox/polls.rs b/kernel/src/inbox/polls.rs index 974a66829bd..7c7111686c7 100644 --- a/kernel/src/inbox/polls.rs +++ b/kernel/src/inbox/polls.rs @@ -21,7 +21,7 @@ //! park over it: what wakes that one is the answer ([`complete`]), which //! posts the same watch once the completion is written. //! -//! Compiled a second time by `kernel-loom` and by `toyos-sched-loom`, so it +//! Compiled a second time by `kernel-loom`, so it //! names nothing of the kernel's. use alloc::sync::Arc; diff --git a/kernel/src/log/console.rs b/kernel/src/log/console.rs index 4da55020424..d175c33aaa7 100644 --- a/kernel/src/log/console.rs +++ b/kernel/src/log/console.rs @@ -15,8 +15,8 @@ use core::sync::atomic::{AtomicPtr, AtomicU64, AtomicU8, Ordering}; use alloc::sync::Arc; use toyos_abi::log::LogRecord; -use toyos_sched::task::{WaitClass, WakeCause, WakeReason}; -use toyos_sched::park::notify; +use kernel::sched::task::{WaitClass, WakeCause, WakeReason}; +use kernel::sched::park::notify; use crate::drivers::serial::{self, BackendGuard, PanicUart, MAX_CONSOLE_LINE}; use crate::hw::HW; diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index de9d68e653e..264db66bac9 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -9,7 +9,7 @@ use alloc::vec::Vec; use toyos_abi::handle::{RawHandle, Rights}; use toyos_abi::syscall::{FileType, OpenFlags, SeekFrom, SyscallError}; -use toyos_sched::task::WaitClass; +use kernel::sched::task::WaitClass; use crate::drivers::serial; use crate::file_cache; diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 88c157fe60d..8788e7bfd86 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -5,7 +5,7 @@ //! symbols off the task's own record ([`resolve_user_symbol`]), never off //! this table. //! -//! `toyos-proclife` decides lifecycle transitions; this file only performs them. +//! `kernel::proclife` decides lifecycle transitions; this file only performs them. use alloc::alloc::{alloc_zeroed, dealloc, Layout}; use alloc::string::String; @@ -29,8 +29,8 @@ pub use crate::scheduler::TaskId; use toyos_abi::syscall::{EndowEntry, SyscallError}; /// The lifecycle's decisions; this file only performs them. -pub use toyos_proclife::{ThreadLocation, Watch}; -use toyos_proclife::{join, reap, spawn as proclife_spawn, teardown as proclife, tree, Lifecycle, Node, Processes}; +pub use kernel::proclife::{ThreadLocation, Watch}; +use kernel::proclife::{join, reap, spawn as proclife_spawn, teardown as proclife, tree, Lifecycle, Node, Processes}; /// One `EndowEntry` on the wire; `loader::start` and [`Endowments::encode`] both index by it. pub const ENDOW_ENTRY_LEN: usize = core::mem::size_of::(); @@ -333,7 +333,7 @@ pub struct ProcessEntry { threads: crate::id_map::IdMap, /// Set once, with the exit's code, by the exit or kill that claims teardown; checked by `spawn_thread` so no thread appears after the retire set. teardown_code: Option, - /// Its parent, children and depth, answered by no syscall; `toyos_proclife::tree` reads and writes it. + /// Its parent, children and depth, answered by no syscall; `kernel::proclife::tree` reads and writes it. node: Node, /// The exit its teardown made, kept until every end below it is published. exit: Option, @@ -380,7 +380,7 @@ impl ProcessEntry { pub fn tearing_down(&self) -> bool { self.teardown_code.is_some() } } -/// The lifecycle face of an entry: only the fields `toyos-proclife` decides against, and nothing else this type carries. +/// The lifecycle face of an entry: only the fields `kernel::proclife` decides against, and nothing else this type carries. impl Lifecycle for ProcessEntry { fn main_tid(&self) -> Tid { self.main_tid } fn teardown_code(&self) -> Option { self.teardown_code } @@ -423,7 +423,7 @@ impl Processes for ProcessTable { f(pid); } } - fn pids(&mut self) -> &mut toyos_proclife::Pids { + fn pids(&mut self) -> &mut kernel::proclife::Pids { &mut self.pids } } @@ -670,15 +670,15 @@ impl IdleProof { } -/// Every process by pid, and the pids not yet issued (`toyos_proclife::pids`). +/// Every process by pid, and the pids not yet issued. pub struct ProcessTable { entries: crate::hasher::HashMap, - pids: toyos_proclife::Pids, + pids: kernel::proclife::Pids, } impl ProcessTable { fn new() -> Self { - Self { entries: crate::hasher::HashMap::default(), pids: toyos_proclife::Pids::default() } + Self { entries: crate::hasher::HashMap::default(), pids: kernel::proclife::Pids::default() } } pub fn get(&self, pid: Pid) -> Option<&ProcessEntry> { @@ -873,7 +873,7 @@ impl Admission { tree::Admit::Gone => Err(SyscallError::Gone), tree::Admit::TooDeep { depth } => { log!("spawn: refused under pid {} at depth {depth}, more than {} below the supervisor", - place.expect("spawn: the supervisor is admitted at depth 0"), toyos_proclife::MAX_DEPTH); + place.expect("spawn: the supervisor is admitted at depth 0"), kernel::proclife::MAX_DEPTH); Err(SyscallError::ResourceExhausted) } tree::Admit::NoPid => { @@ -1083,7 +1083,7 @@ fn teardown_bookkeeping(table: &mut ProcessTable, process_pid: Pid, code: i32) - cpu_ns } -/// Publish `first`, then every parent each publication lets go, child before parent: at most `toyos_proclife::MAX_DEPTH` + 1 publications, each with the table lock given up. +/// Publish `first`, then every parent each publication lets go, child before parent: at most `kernel::proclife::MAX_DEPTH` + 1 publications, each with the table lock given up. /// Once published an entry is reapable, so only its parent's entry is read after. fn publish_climb(first: Option) { let mut ready = first; @@ -1756,7 +1756,7 @@ pub fn debug_hold_marked_spawn(parent: Parent, child: &crate::object::process::P &parkable, child.watch(), 0, - toyos_sched::task::WaitClass::Other, + kernel::sched::task::WaitClass::Other, crate::time::Deadline::never(), || child.finished(), ); diff --git a/kernel/src/quiesce.rs b/kernel/src/quiesce.rs index 3d89bb8b229..7a36071ff03 100644 --- a/kernel/src/quiesce.rs +++ b/kernel/src/quiesce.rs @@ -39,8 +39,8 @@ use core::sync::atomic::{AtomicBool, AtomicU32, Ordering::AcqRel, Ordering::Relaxed}; use toyos_quiesce::{must_stop, Record, Sweep, ThreadId}; -use toyos_sched::task::WaitClass; -use toyos_sched::watch::Gate; +use kernel::sched::task::WaitClass; +use kernel::sched::watch::Gate; use crate::arch::percpu; use crate::watch::{self, Watch}; @@ -56,7 +56,7 @@ use crate::time::{Budget, Deadline, Duration}; /// without parking. A thread can therefore outlast this, which is why its /// expiry is a clause in the record. pub(crate) const PARK: Budget = Budget::of( - Duration::from_nanos(toyos_sched::fair::QUANTUM_NS + crate::block::OPERATION.nanos()), + Duration::from_nanos(kernel::sched::fair::QUANTUM_NS + crate::block::OPERATION.nanos()), "the reset lands wherever the threads that never reached a safe point are, and \ the record names how many", ); diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 97b54c993a3..66b50714e4b 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -1,6 +1,6 @@ //! The kernel driver for the scheduler core: plumbing only — percpu, //! the asm switch, the idle loop, the trampoline. Every scheduling -//! decision lives in `toyos-sched`. +//! decision lives in `kernel::sched`. //! //! A pass is not complete when it returns: restoring a context before its last `switch` instruction runs puts two CPUs on the same stack. @@ -11,14 +11,14 @@ use core::cell::UnsafeCell; use core::ptr; use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU64, Ordering}; -use toyos_sched::cpu::{Action, Balance, CpuHandle, CpuHandles, CpuSched, Env, SchedPass}; -use toyos_sched::fair::Frontier; -use toyos_sched::hw::{CpuId, Hw, Kicker, Machine, Nanos}; -use toyos_sched::mailbox::{mailbox, Kick, PreemptGuard, Urgency}; -use toyos_sched::msg::Msg; -use toyos_sched::task::{RtState, TaskBuilder, TaskKey, WaitClass}; -use toyos_sched::park::{prepare, Cancel, Commit, CurrentTask}; -use toyos_sched::task::Refused; +use kernel::sched::cpu::{Action, Balance, CpuHandle, CpuHandles, CpuSched, Env, SchedPass}; +use kernel::sched::fair::Frontier; +use kernel::sched::hw::{CpuId, Hw, Kicker, Machine, Nanos}; +use kernel::sched::mailbox::{mailbox, Kick, PreemptGuard, Urgency}; +use kernel::sched::msg::Msg; +use kernel::sched::task::{RtState, TaskBuilder, TaskKey, WaitClass}; +use kernel::sched::park::{prepare, Cancel, Commit, CurrentTask}; +use kernel::sched::task::Refused; use crate::arch::percpu; use crate::hw::HW; @@ -446,7 +446,7 @@ pub fn spawn(new: NewTask) -> (ThreadSched, CpuId) { shared: task.shared().clone(), }; let dst = match task.shared().state() { - toyos_sched::task::TaskState::InTransit(cpu) => cpu, + kernel::sched::task::TaskState::InTransit(cpu) => cpu, state => panic!("a freshly built task is not in transit: {state:?}"), }; preempt_off(|p| { @@ -485,7 +485,7 @@ fn env(preempt: &PreemptOff) -> Env<'_, crate::hw::KernelHw, PreemptOff> { frontier: &FRONTIER, preempt, balance: Balance::PushOnSurplus { - threshold: toyos_sched::cpu::PUSH_THRESHOLD, + threshold: kernel::sched::cpu::PUSH_THRESHOLD, }, } } @@ -742,7 +742,7 @@ pub fn current_symbols() -> Option> { } /// What the running task's marks say it does instead of returning to Ring 3 — one load, no clone, since an `Arc` refcount here is too costly on this path. -pub fn current_safe_point(stopping: bool) -> Option { +pub fn current_safe_point(stopping: bool) -> Option { try_with_cpu(|cpu| cpu.running().and_then(|t| t.shared().at_safe_point(stopping))).flatten() } @@ -761,7 +761,7 @@ pub fn current_address_space() -> Option { } pub fn with_current_acct( - f: impl FnOnce(&toyos_sched::task::TaskAccounting) -> R, + f: impl FnOnce(&kernel::sched::task::TaskAccounting) -> R, ) -> Option { try_with_cpu(|cpu| cpu.running().map(|t| f(t.acct()))).flatten() } @@ -826,7 +826,7 @@ pub fn running_id() -> Option { /// One parked task, flattened because a `ParkedView` borrows the `CpuSched`, which nothing outside this file may hold. pub struct ParkedInfo { pub id: TaskId, - pub class: toyos_sched::task::WaitClass, + pub class: kernel::sched::task::WaitClass, pub deadline: Option, /// When the park began. pub since: u64, diff --git a/kernel/src/sched/kthread.rs b/kernel/src/sched/kthread.rs index d26ceac57ca..ac3cf7618d6 100644 --- a/kernel/src/sched/kthread.rs +++ b/kernel/src/sched/kthread.rs @@ -15,7 +15,7 @@ use crate::process::{ use crate::scheduler::{self, TaskId}; use crate::symbols::SymbolTable; use crate::sync::Lock; -use toyos_proclife::Processes; +use kernel::proclife::Processes; use super::payload::ThreadSched; @@ -104,7 +104,7 @@ pub fn spawn(name: &str, body: extern "C" fn(u64) -> !, arg: u64) -> ThreadSched Arc::new(Lock::new(kernel_process_data(name))), Arc::clone(&syms), ThreadEntry::new(Arc::new(Lock::new(kernel_thread_data()))), - toyos_proclife::Node::root(), + kernel::proclife::Node::root(), )); let tid = table.get(pid).expect("kthread: the entry just inserted is gone").main_tid(); claim.publish(TaskId(pid, tid)); diff --git a/kernel/src/sched/mod.rs b/kernel/src/sched/mod.rs index 71df61c919e..0d152037da8 100644 --- a/kernel/src/sched/mod.rs +++ b/kernel/src/sched/mod.rs @@ -1,4 +1,4 @@ -//! The kernel's half of the scheduler core; `toyos-sched` decides. +//! The kernel's half of the scheduler core; `kernel::sched` decides. #![warn(clippy::undocumented_unsafe_blocks)] diff --git a/kernel/src/sched/payload.rs b/kernel/src/sched/payload.rs index 34d737df6bb..c4be530acd2 100644 --- a/kernel/src/sched/payload.rs +++ b/kernel/src/sched/payload.rs @@ -5,12 +5,12 @@ use alloc::sync::Arc; use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; -use toyos_sched::fair::{FairShare, ShareState}; -use toyos_sched::hw::Nanos; -use toyos_sched::msg::Msg; -use toyos_sched::sync::CellLock; -use toyos_sched::task::{SchedPayload, TaskAccounting, TaskShared, WaitClass}; -use toyos_sched::park::WaitTicket; +use kernel::sched::fair::{FairShare, ShareState}; +use kernel::sched::hw::Nanos; +use kernel::sched::msg::Msg; +use kernel::sched::sync::CellLock; +use kernel::sched::task::{SchedPayload, TaskAccounting, TaskShared, WaitClass}; +use kernel::sched::park::WaitTicket; use crate::watch::Watch; use crate::mm::paging::Root; @@ -158,7 +158,7 @@ pub struct ThreadSched { impl ThreadSched { pub fn sched_state(&self) -> u8 { - use toyos_sched::task::TaskState; + use kernel::sched::task::TaskState; match self.shared.state() { TaskState::Running(_) => SCHED_RUNNING, TaskState::Ready(_) | TaskState::WakeQueued(_) | TaskState::InTransit(_) => SCHED_READY, diff --git a/kernel/src/scheduler.rs b/kernel/src/scheduler.rs index 07e9c4ae1a4..1d6f1a3d192 100644 --- a/kernel/src/scheduler.rs +++ b/kernel/src/scheduler.rs @@ -8,9 +8,9 @@ use alloc::sync::Arc; use core::sync::atomic::{AtomicU64, Ordering}; use crate::hasher::HashMap; -use toyos_sched::fair::{ShareState, QUANTUM_NS}; -use toyos_sched::hw::{CpuId, Machine, Nanos}; -use toyos_sched::task::{Refused, SafePoint, WaitClass}; +use kernel::sched::fair::{ShareState, QUANTUM_NS}; +use kernel::sched::hw::{CpuId, Machine, Nanos}; +use kernel::sched::task::{Refused, SafePoint, WaitClass}; use crate::arch::percpu; use crate::watch::{self, Cancel}; @@ -366,7 +366,7 @@ pub fn do_preempt() { /// `kernel/src/quiesce.rs`'s header says why that boundary is the safe point. /// /// **One call and one match, so the two marks have no order to disagree -/// about**: `toyos_sched::task::SafePoint` ranks them, here and in +/// about**: `kernel::sched::task::SafePoint` ranks them, here and in /// `CpuSched::place` alike. #[track_caller] pub fn leave_user_if_due() { @@ -535,7 +535,7 @@ pub fn futex_wake(phys_addr: DirectMap, count: usize) -> u64 { /// The thread leaves at that safe point: `process::leave`. pub fn post_retire(sched: &ThreadSched) { preempt_off(|p| { - toyos_sched::retire::begin(&sched.shared).post(cpus(), &HW, p); + kernel::sched::retire::begin(&sched.shared).post(cpus(), &HW, p); }); } diff --git a/kernel/src/shootdown.rs b/kernel/src/shootdown.rs index 6c453a99c17..8a69e30b65c 100644 --- a/kernel/src/shootdown.rs +++ b/kernel/src/shootdown.rs @@ -1,5 +1,5 @@ //! The acknowledgement half of a TLB shootdown, with no hardware in it. -//! Compiled a second time into `kernel-loom/` against loom's atomics, so this file must hold no `crate::` references. +//! Compiled a second time into `kernel/loom/` against loom's atomics, so this file must hold no `crate::` references. //! The read must happen before the flush, or a target could publish a generation its flush has not yet completed. #[cfg(not(feature = "loom"))] diff --git a/kernel/src/syscall/io.rs b/kernel/src/syscall/io.rs index ad5c6d25f0e..684d6f1233c 100644 --- a/kernel/src/syscall/io.rs +++ b/kernel/src/syscall/io.rs @@ -14,7 +14,7 @@ use crate::{device, pipe, process}; use toyos_abi::handle::{RawHandle, Rights}; use toyos_abi::syscall::*; -use toyos_sched::task::WaitClass; +use kernel::sched::task::WaitClass; use super::cancelled; use super::handles::with_object_ref; diff --git a/kernel/src/syscall/ipc.rs b/kernel/src/syscall/ipc.rs index 475d60a6e8c..bb7431320a1 100644 --- a/kernel/src/syscall/ipc.rs +++ b/kernel/src/syscall/ipc.rs @@ -17,7 +17,7 @@ use crate::{pipe, process}; use toyos_abi::handle::{RawHandle, Rights}; use toyos_abi::syscall::*; -use toyos_sched::task::WaitClass; +use kernel::sched::task::WaitClass; use super::{cancelled, HANDLE_LEN}; use super::handles::handle_result; diff --git a/kernel/src/syscall/proc.rs b/kernel/src/syscall/proc.rs index b366965170a..80c97aef4f7 100644 --- a/kernel/src/syscall/proc.rs +++ b/kernel/src/syscall/proc.rs @@ -14,7 +14,7 @@ use crate::process; use toyos_abi::handle::{RawHandle, Rights}; use toyos_abi::syscall::*; -use toyos_sched::task::WaitClass; +use kernel::sched::task::WaitClass; use super::cancelled; use super::handles::demand_syscap; @@ -144,7 +144,7 @@ pub(super) fn sys_thread_join(tid: u64) -> u64 { // None means never existed or already collected; the predicate below answers both. let target = process::thread_sched(caller, tid); let parkable = crate::scheduler::Parkable::at_entry(); - let join = toyos_proclife::join::Join::default(); + let join = kernel::proclife::join::Join::default(); let ask = || process::ask_join(&join, tid, caller); loop { // Both refusals are one answer here; `JoinRefused` keeps them apart because they aren't the same fact. diff --git a/kernel/src/trace.rs b/kernel/src/trace.rs index 7b5ca3b76bf..5038d1d2f32 100644 --- a/kernel/src/trace.rs +++ b/kernel/src/trace.rs @@ -4,13 +4,13 @@ //! the head index; payload writes are non-atomic, so a reader may see a torn //! record for the most recent entry. //! -//! [`Record`] is the `repr(C)` wire form of `toyos_sched::hw::TraceEvent`; +//! [`Record`] is the `repr(C)` wire form of `kernel::sched::hw::TraceEvent`; //! [`record`] maps one onto the other and is `Machine::trace`. use core::cell::UnsafeCell; use core::sync::atomic::{AtomicU64, Ordering}; -use toyos_sched::hw::{TraceEvent, TraceKind}; +use kernel::sched::hw::{TraceEvent, TraceKind}; use crate::arch::percpu; diff --git a/kernel/src/usb_gate.rs b/kernel/src/usb_gate.rs index c47beb8c3c4..10011be7f87 100644 --- a/kernel/src/usb_gate.rs +++ b/kernel/src/usb_gate.rs @@ -56,7 +56,7 @@ pub fn sweep_under_load() { // than about this function. let interrupts_were_on = crate::arch::cpu::interrupts_enabled(); crate::preempt::disable(); - crate::arch::irqchip::arm_within(toyos_sched::fair::QUANTUM_NS); + crate::arch::irqchip::arm_within(kernel::sched::fair::QUANTUM_NS); crate::arch::cpu::enable_interrupts(); let mut buf = vec![0u8; WEDGE_CHUNK as usize * BLOCK]; let mut at = first; diff --git a/kernel/src/watch.rs b/kernel/src/watch.rs index 5dc4a17cad4..4e8dc035580 100644 --- a/kernel/src/watch.rs +++ b/kernel/src/watch.rs @@ -2,7 +2,7 @@ //! a waiter on it is either a thread, which a post *wakes*, or a user poll //! ring's entry, which a post *fires*. //! -//! The protocol is `toyos_sched::watch`'s and `toyos_sched::park`'s, and its +//! The protocol is `kernel::sched::watch`'s and `kernel::sched::park`'s, and its //! lost-wake argument is theirs: a thread registers before it reads its //! condition, and a post either precedes the registration or writes the //! thread's state word, which its own next commit reads. Nothing here keeps a @@ -20,12 +20,12 @@ use alloc::sync::Arc; -use toyos_sched::hw::Nanos; -use toyos_sched::sync::CellLock; -use toyos_sched::task::{Refused, WaitClass, WakeCause, WakeReason}; -use toyos_sched::watch::{Poster, Waiters}; +use kernel::sched::hw::Nanos; +use kernel::sched::sync::CellLock; +use kernel::sched::task::{Refused, WaitClass, WakeCause, WakeReason}; +use kernel::sched::watch::{Poster, Waiters}; -pub use toyos_sched::park::Cancel; +pub use kernel::sched::park::Cancel; use crate::hw::HW; use crate::inbox::PollEntry; @@ -37,7 +37,7 @@ use crate::time::Deadline; type List = Waiters; /// What an object holds to be waitable, its list behind `L`. -pub struct Waitable>(toyos_sched::watch::Watch); +pub struct Waitable>(kernel::sched::watch::Watch); /// A watch no interrupt handler reaches. pub type Watch = Waitable>; @@ -91,7 +91,7 @@ mod masked { impl Watch { pub const fn new() -> Self { - Self(toyos_sched::watch::Watch::new(KernelLock::new(Waiters::new()))) + Self(kernel::sched::watch::Watch::new(KernelLock::new(Waiters::new()))) } /// Something about the object changed: wake every thread waiting on it and @@ -138,7 +138,7 @@ impl Watch { impl IrqWatch { pub const fn new() -> Self { - Self(toyos_sched::watch::Watch::new(IrqLock::new(Waiters::new()))) + Self(kernel::sched::watch::Watch::new(IrqLock::new(Waiters::new()))) } /// Something about the object changed: wake every thread waiting on it and diff --git a/kernel/src/windows.rs b/kernel/src/windows.rs index 2b7e5eef161..3ffea02f6af 100644 --- a/kernel/src/windows.rs +++ b/kernel/src/windows.rs @@ -1,4 +1,4 @@ -//! Each CPU's [`toyos_sched::windows::Windows`], fed where this CPU's +//! Each CPU's [`kernel::sched::windows::Windows`], fed where this CPU's //! interrupts and preempt count change, and printed beside the IRQ census as //! `windows: cpuN irqs_off_ns=… preempt_off_ns=…` (`mask-windows` builds only). //! @@ -29,7 +29,7 @@ use core::sync::atomic::AtomicBool; use core::sync::atomic::Ordering::Relaxed; -use toyos_sched::windows::{Unseen, Windows, HELD_NS}; +use kernel::sched::windows::{Unseen, Windows, HELD_NS}; use crate::arch::{cpu, percpu}; use crate::sched::MAX_CPUS; diff --git a/src/build.rs b/src/build.rs index 9a2c6d839b6..bae3cc48b6c 100644 --- a/src/build.rs +++ b/src/build.rs @@ -1154,7 +1154,8 @@ pub fn flashable_params(root: &Path, asked: &[String]) -> Result<(), String> { /// separated argument. /// /// **Every name the caller asked for is checked against `kernel/Cargo.toml`, -/// and an unknown one stops the build by name.** Read from the manifest rather +/// and an unknown one stops the build by name**, as does a control or a name in +/// [`KERNEL_CARRIES`]: neither is a kernel build. Read from the manifest rather /// than listed here, so the check cannot drift from what cargo would accept — /// and, more to the point, so that deleting a feature takes its own command /// lines down with it. That is what a temporary feature needs: once one is @@ -1192,6 +1193,12 @@ fn kernel_features( lists them.", declared.join(", ") ); + assert!( + !crate::ci::CONTROLS.iter().any(|c| c.feature == name) + && !KERNEL_CARRIES.contains(&name.as_str()), + "--kernel-feature {name}: a model's negative control or a name the kernel \ + declares for another package's build of its sources, and never a kernel build" + ); features.push(name); } } @@ -1341,6 +1348,11 @@ fn declared_kernel_features(root: &Path) -> Vec { manifest.features.into_keys().collect() } +/// The features the kernel declares for another package's build of its +/// sources: `loom`, which `kernel-loom` turns on, and `protocol-port`, which +/// `kernel-sim` does. +const KERNEL_CARRIES: &[&str] = &["loom", "protocol-port"]; + /// The kernel every test that needs an actuator boots: all of them compiled in /// and none of them armed, plus the `SYS_DEBUG` number. /// @@ -1357,7 +1369,7 @@ pub const TEST_SUITE_KERNEL_BUILDS: [&str; 3] = /// [`SCHED_CHECK_KERNEL`]'s reason: one spelling, so one build. pub const MASK_WINDOWS_KERNEL: &[&str] = &["mask-windows"]; -/// The scheduler core's own asserts, compiled in: `toyos-sched/check`. +/// The scheduler core's own asserts, compiled in. /// /// One name, for [`TEST_KERNEL`]'s reason — a second spelling is a second /// kernel and nothing would say so. @@ -1577,7 +1589,7 @@ fn assert_entry_window_matches_features(features: &str, kernel: &[u8]) { /// The scheduler core's `feature = "check"` instruments, by their own text, and /// the two kernels that must disagree about carrying them. /// -/// Every one of these is a `#[cfg(feature = "check")]` site in `toyos-sched`. +/// Every one of these is a `#[cfg(feature = "sched-check")]` site in `kernel::sched`. /// Two are asserts from `invariants::check_cpu` — invariant T's armed-timer /// bound and the container-versus-state-word agreement. The third is the /// pass-cost report, which is a *measurement* @@ -1609,8 +1621,8 @@ fn assert_sched_check_matches_features(features: &str, kernel: &[u8]) { SCHED_CHECK_KERNEL, &SCHED_CHECK_LITERALS, "scheduler check instruments", - "`sched-check` forwards to `toyos-sched/check`, so a build that carries the feature and \ - not the instruments is a check build in name only.", + "`sched-check` compiles the scheduler core's instruments in, so a build that carries the \ + feature and not the instruments is a check build in name only.", ); } @@ -2594,207 +2606,168 @@ mod tests { ); } + /// Every kernel feature that is a kernel build of its own, each beside what + /// earned it one. Every other feature the kernel declares is a control a row + /// of `crate::ci::CONTROLS` runs, or one of [`KERNEL_CARRIES`]. + const KERNEL_BUILDS: &[&str] = &[ + "boot-actuators", + "debug-wait", + // Does this kernel reach a pass, a trap or a syscall with the + // direction flag set. No gate clears `DF` and + // `compiler_builtins::mem::memmove` sets it across three `rep` + // string operations, so before `arch::entry`'s `cld` it could be + // — and every `rep movs`/`rep stos` after that writes backwards. + // Its own build: a `pushfq` and a test on three hot paths, and + // the negative control for that `cld` in both directions. + "df-witness", + // Its control: a `std` one instruction before the reader that + // must refuse it. The witness fired zero times on an unclean + // kernel, which is a fact about where the flag reaches and would + // be indistinguishable from a broken reader without this. + "df-witness-mutate", + // The kernel this tree had before `arch::entry`'s `cld`: the + // instruction gone and `DF` back out of the `SYSCALL` mask, so a + // build carrying it inherits a set direction flag from whatever + // it interrupted. The negative control for that fix, and its own + // build because the defect is in a `naked_asm!` body on every + // ring transition, where a boot parameter would have to be a + // branch. + "entry-df-unclean", + // The two band shapes that separate the two readings + // `heap-tripwire`'s own result left standing — the bands absorb + // a bounded overrun, or they displace every allocation and the + // victim moved. No band can be zero-width and keep its + // placement, because the padding *is* the displacement, so the + // separation is per side: `notail` leaves the slack past a + // payload byte-for-byte what an unbanded build has, `nohead` + // puts the payload at the bottom of its own chunk. They are one + // experiment in two arms and refuse to build together. + "heap-band-nohead", + "heap-band-notail", + // The sweep's lock hold without the sweep. `heap-sweep` and + // `sched-tripwire` both multiply this class and both spend time + // on the pass path; only the sweep also holds `dlmalloc`'s lock + // while it does. This arm and `pass-spin` below spend one + // `HOLD_NS` with and without that lock, which is the one + // variable nobody has varied. + "heap-lockspin", + // The sweep that reads every live band rather than only the + // ones a `dealloc` reaches. Its own build for `heap-tripwire`'s + // reason twice over: the walk takes `dlmalloc`'s lock on the + // pass path, which nothing shipping may do. + "heap-sweep", + // `sched-tripwire`'s twin one layer down: a band of known bytes + // on each side of every heap allocation, read back at `dealloc` + // and — for the running task's kernel stack — at every pass. It + // earns a build of its own because the bands change what + // `GlobalAlloc::alloc` returns, which no boot parameter can + // reach: an allocation minted under one arm and freed under the + // other is a miscomputed base address. No suite builds it, so a + // full run pays nothing and a boot storm asks for it by name. + "heap-tripwire", + "mask-windows", + // `heap-lockspin`'s other arm: the same visit to the pass path, + // for the same span, without the allocator's lock. + "pass-spin", + "sched-check", + // The stray-write tripwire on the per-CPU `CpuSched` record: a + // byte shadow taken and compared at both ends of the driver's + // exclusive region, plus a walk of its three containers. It + // earns a build of its own because what it watches cannot be + // reached from a boot parameter — the shadow's subject is a + // whole record and the walk's is a container, and both are + // decided at compile time by a cargo feature, the same wall + // `sched-check` is behind. No suite builds it: it is + // not in `TEST_SUITE_KERNEL_BUILDS`, so a full run pays nothing + // for it and a boot storm asks for it by name. + "sched-tripwire", + // The two comparisons that ask who else is standing on a task's + // kernel stack: the words a Ring 3 entry takes its stack from, + // against the running task's own top, at every pass; and the one + // driver field this class has been caught changing inside a + // single call. Its own build because both halves are readers on + // hot paths, so it has to be in both arms of any comparison. + "stack-witness", + // The one window this class has never measured: the eight words + // `context_switch` pops, copied at `check_switch_frame` and + // compared from inside the switch, one instruction before the + // first `pop`, against the stack pointer the machine is standing + // on. Its own build for `stack-witness`'s reason and one more — + // the compare is a `call`, so the frame has to have been proven + // to be inside a real stack, which is why it turns that feature + // on. Its two mutation controls sit beside it, each staging one + // arm of what it watches. + "switch-witness", + "switch-witness-mutate-frame", + "switch-witness-mutate-rsp", + "test-actuators", + ]; + + /// Features of a host workspace member that choose a build's world rather + /// than revert a decision: loom's atomics, a crate's `std` and its default + /// set, the toolchain's `core` and `rustc-dep-of-std`, libc's `std-runtime`, + /// the signer's `sign`, and `flaws`, which compiles what a simulator's + /// negative gates select at run time. + const NOT_A_CONTROL: &[&str] = &[ + "core", + "default", + "flaws", + "loom", + "rustc-dep-of-std", + "sign", + "std", + "std-runtime", + ]; + /// The features a kernel build may still carry, and the whole list. /// - /// **The gate on the count.** Each name here is a kernel `cargo test` may - /// build beside the two, so adding one is a decision to pay the ~6.9 s of - /// wall clock and ~29.6 s of CPU measured for one extra kernel build per - /// full run after any kernel edit — and `boot-actuators` exists so that - /// the answer is almost always a parameter instead. + /// **The gate on the count.** Each name in [`KERNEL_BUILDS`] is a kernel + /// `cargo test` may build beside the two, so adding one is a decision to pay + /// the ~6.9 s of wall clock and ~29.6 s of CPU measured for one extra kernel + /// build per full run after any kernel edit — and `boot-actuators` exists so + /// that the answer is almost always a parameter instead. #[test] fn the_kernel_declares_only_the_builds_that_earned_one() { let root = Path::new(env!("CARGO_MANIFEST_DIR")); - let mut declared = declared_kernel_features(root); - declared.sort(); + let controls: BTreeSet<&str> = crate::ci::CONTROLS.iter().map(|c| c.feature).collect(); + let mut builds: Vec = declared_kernel_features(root) + .into_iter() + .filter(|f| !controls.contains(f.as_str()) && !KERNEL_CARRIES.contains(&f.as_str())) + .collect(); + builds.sort(); assert_eq!( - declared, - [ - "boot-actuators", - "debug-wait", - // Costs no kernel build, for `wake-fence-off`'s reason: only - // `kernel-loom` turns it on, and `device_irq` must red under it. - "device-irq-lossy", - // Does this kernel reach a pass, a trap or a syscall with the - // direction flag set. No gate clears `DF` and - // `compiler_builtins::mem::memmove` sets it across three `rep` - // string operations, so before `arch::entry`'s `cld` it could be - // — and every `rep movs`/`rep stos` after that writes backwards. - // Its own build: a `pushfq` and a test on three hot paths, and - // the negative control for that `cld` in both directions. - "df-witness", - // Its control: a `std` one instruction before the reader that - // must refuse it. The witness fired zero times on an unclean - // kernel, which is a fact about where the flag reaches and would - // be indistinguishable from a broken reader without this. - "df-witness-mutate", - // Costs no kernel build, for `wake-fence-off`'s reason: only - // `kernel-loom` turns it on, and `dump_request` must red under it. - "dump-report-relaxed", - // The kernel this tree had before `arch::entry`'s `cld`: the - // instruction gone and `DF` back out of the `SYSCALL` mask, so a - // build carrying it inherits a set direction flag from whatever - // it interrupted. The negative control for that fix, and its own - // build because the defect is in a `naked_asm!` body on every - // ring transition, where a boot parameter would have to be a - // branch. - "entry-df-unclean", - // The two band shapes that separate the two readings - // `heap-tripwire`'s own result left standing — the bands absorb - // a bounded overrun, or they displace every allocation and the - // victim moved. No band can be zero-width and keep its - // placement, because the padding *is* the displacement, so the - // separation is per side: `notail` leaves the slack past a - // payload byte-for-byte what an unbanded build has, `nohead` - // puts the payload at the bottom of its own chunk. They are one - // experiment in two arms and refuse to build together. - "heap-band-nohead", - "heap-band-notail", - // The sweep's lock hold without the sweep. `heap-sweep` and - // `sched-tripwire` both multiply this class and both spend time - // on the pass path; only the sweep also holds `dlmalloc`'s lock - // while it does. This arm and `pass-spin` below spend one - // `HOLD_NS` with and without that lock, which is the one - // variable nobody has varied. - "heap-lockspin", - // The sweep that reads every live band rather than only the - // ones a `dealloc` reaches. Its own build for `heap-tripwire`'s - // reason twice over: the walk takes `dlmalloc`'s lock on the - // pass path, which nothing shipping may do. - "heap-sweep", - // `sched-tripwire`'s twin one layer down: a band of known bytes - // on each side of every heap allocation, read back at `dealloc` - // and — for the running task's kernel stack — at every pass. It - // earns a build of its own because the bands change what - // `GlobalAlloc::alloc` returns, which no boot parameter can - // reach: an allocation minted under one arm and freed under the - // other is a miscomputed base address. No suite builds it, so a - // full run pays nothing and a boot storm asks for it by name. - "heap-tripwire", - // The four below cost no kernel build at all, for - // `wake-fence-off`'s reason: each is declared only so `cfg` - // checking knows the name, and turned on only by - // `kernel-loom`, one at a time, to relax the single edge its - // named model rests on and prove that model reds without it. - "lock-acquire-off", - "log-commit-release-off", - "loom", - "mask-windows", - // `heap-lockspin`'s other arm: the same visit to the pass path, - // for the same span, without the allocator's lock. - "pass-spin", - // `wake-fence-off`'s twin, for a poll ring's one-shot answer: - // turned on only by `kernel-loom`, to split `inbox/once.rs`'s - // exchange and prove `poll_once` reds without it. - "poll-fire-load-store", - // Costs no kernel build: turned on only by `kernel-loom`, so - // `inbox/polls.rs` answers a fired poll without a look and - // `inbox_answer` reds. - "post-is-an-answer", - "reap-raise-relaxed", - // `smp_roster.rs`'s count relaxed; `smp_bringup.rs` reds. - "roster-commit-relaxed", - "sched-check", - // The stray-write tripwire on the per-CPU `CpuSched` record: a - // byte shadow taken and compared at both ends of the driver's - // exclusive region, plus a walk of its three containers. It - // earns a build of its own because what it watches cannot be - // reached from a boot parameter — the shadow's subject is a - // whole record and the walk's is a container, and both are - // decided at compile time by a dependency's cargo feature, the - // same wall `sched-check` is behind. No suite builds it: it is - // not in `TEST_SUITE_KERNEL_BUILDS`, so a full run pays nothing - // for it and a boot storm asks for it by name. - "sched-tripwire", - // Costs no kernel build, for `wake-fence-off`'s reason: turned on - // only by `kernel-loom`, to drop the panic console publisher's - // `Release` fence and prove `panic_console_publish` reds without it. - "seqlock-writer-fence-off", - // Costs no kernel build: turned on only by `kernel-loom`, to build - // the backend lock's `try_lock` with `then_some` and prove - // `serial_lock` reds. - "serial-try-lock-then-some", - "shard-publish-relaxed", - "shootdown-serve-relaxed", - // The eighth loom control, and the first over a *contended* - // acquire: `src/sleeplock.rs`'s two loads of `now` go - // `Relaxed` and `kernel-loom/tests/sleep_lock.rs` reds. Costs - // no kernel build, for the same reason as the six above it. - "sleeplock-acquire-off", - // `smp_roster.rs`'s second release store; `smp_bringup.rs` reds. - "smp-ready-split", - // The two comparisons that ask who else is standing on a task's - // kernel stack: the words a Ring 3 entry takes its stack from, - // against the running task's own top, at every pass; and the one - // driver field this class has been caught changing inside a - // single call. Its own build because both halves are readers on - // hot paths, so it has to be in both arms of any comparison. - "stack-witness", - // The one window this class has never measured: the eight words - // `context_switch` pops, copied at `check_switch_frame` and - // compared from inside the switch, one instruction before the - // first `pop`, against the stack pointer the machine is standing - // on. Its own build for `stack-witness`'s reason and one more — - // the compare is a `call`, so the frame has to have been proven - // to be inside a real stack, which is why it turns that feature - // on. Its two mutation controls sit beside it, each staging one - // arm of what it watches. - "switch-witness", - "switch-witness-mutate-frame", - "switch-witness-mutate-rsp", - "test-actuators", - // Costs no kernel build at all, for `loom`'s reason: declared - // so `cfg` checking knows the name, and turned on only by - // `kernel-loom` — to remove the log wake path's two `SeqCst` - // fences and prove `log_wake` reds without them. - "wake-fence-off", - ], - "the kernel declares a feature this list does not account for" + builds, KERNEL_BUILDS, + "the kernel declares a feature that is neither a build this list accounts for nor a \ + control `src/ci.rs` runs" ); } - /// Every negative control the model crates declare — every feature name - /// besides the structural ones they carry for other reasons. - /// - /// **The list is the crates that hold a model of the kernel, not the crates - /// that use loom.** `kernel-loom` and `toyos-sched-loom` swap in loom's - /// instrumented atomics because their subjects are memory orderings; - /// `toyos-proclife`'s subject is which CPU takes the process table lock - /// next, and every decision in it is made under that lock. Both kinds are - /// a model with controls, and a control with no step behind it is the same - /// hole either way. + /// Every negative control the tree declares: every feature of the kernel's + /// manifest but its builds and [`KERNEL_CARRIES`], and every feature of every + /// host workspace member's but [`NOT_A_CONTROL`]. /// - /// **And `toyos-sched-sim` is one of them**: what a whole simulated machine - /// can show wrong is a policy rather than an ordering, so a control over one - /// lives there and would otherwise be the only kind this gate could not see. - /// - /// `loom` selects loom's instrumented atomics; `check`, `protocol-port`, - /// `tripwire` and `std` mirror `toyos-sched`'s own features so the shared - /// sources compile identically and name nothing a model turns on. Everything - /// else declared in any of these files is, by construction, a - /// `--features ` command that must red a named model — each file's own - /// comment beside the name carries the argument for why. - fn declared_model_controls(root: &Path) -> Vec<(&'static str, String)> { - const NOT_A_CONTROL: &[&str] = - &["loom", "check", "protocol-port", "tripwire", "std", "default"]; - let mut out = Vec::new(); - for (crate_name, manifest) in [ - ("kernel-loom", "kernel-loom/Cargo.toml"), - ("toyos-sched-loom", "toyos-sched/loom/Cargo.toml"), - ("toyos-sched-sim", "toyos-sched/sim/Cargo.toml"), - ("toyos-proclife", "toyos-proclife/Cargo.toml"), - ("toyos-blockring", "toyos-blockring/Cargo.toml"), - ("toyos-transport", "toyos-transport/Cargo.toml"), - ] { - let path = root.join(manifest); + /// **Every manifest, and no list of the crates that hold a model.** A model + /// of memory orderings, of the process table's interleavings, of a simulated + /// machine's policy or of an allocator's isolation each declares its controls + /// beside its own decisions, and a list of those crates is one a new model + /// can be left off — its control then declared and run nowhere, silently. + /// Each file's own comment beside a name carries the argument for it. + fn declared_model_controls(root: &Path) -> BTreeSet { + let mut out = BTreeSet::new(); + for dir in std::iter::once(crate::ci::KERNEL.to_string()).chain(crate::hostws::members(root)) { + let path = root.join(&dir).join("Cargo.toml"); let text = fs::read_to_string(&path) .unwrap_or_else(|e| panic!("Failed to read {}: {e}", path.display())); let parsed: KernelManifest = toml::from_str(&text) .unwrap_or_else(|e| panic!("Failed to parse {}: {e}", path.display())); - for name in parsed.features.into_keys() { - if !NOT_A_CONTROL.contains(&name.as_str()) { - out.push((crate_name, name)); - } - } + let skip: &[&[&str]] = if dir == crate::ci::KERNEL { + &[KERNEL_BUILDS, KERNEL_CARRIES] + } else { + &[NOT_A_CONTROL] + }; + out.extend( + parsed.features.into_keys().filter(|name| !skip.iter().any(|s| s.contains(&name.as_str()))), + ); } out } @@ -2806,12 +2779,10 @@ mod tests { #[test] fn every_model_control_is_run() { let root = Path::new(env!("CARGO_MANIFEST_DIR")); - let declared: BTreeSet = - declared_model_controls(root).into_iter().map(|(_, name)| name).collect(); let run: BTreeSet = crate::ci::CONTROLS.iter().map(|c| c.feature.to_string()).collect(); assert_eq!( - declared, run, + declared_model_controls(root), run, "a model's negative control is declared and not in src/ci.rs's CONTROLS, or a row \ there names a feature no model crate declares" ); diff --git a/src/ci.rs b/src/ci.rs index c6e5aca2c9b..4d87bb822d0 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -174,7 +174,7 @@ fn cargo_logged(dir: &Path, args: &[&str]) -> Result<(bool, String), String> { /// that takes away the one edge the model's property rests on, and the verdict /// lines the model must then print. pub(crate) struct Control { - /// The model's package in the host workspace. + /// The model's package: a member of the host workspace, or [`KERNEL`]. pub(crate) krate: &'static str, pub(crate) feature: &'static str, /// The test target the verdicts' tests are in; `None` is the library's own. @@ -217,10 +217,18 @@ impl Verdict { use Verdict::{Fails, Passes, Says}; +/// The kernel package, whose library's tests are its models. +pub(crate) const KERNEL: &str = "kernel"; +/// Its manifest, resolved on its own. Run from the repository root, cargo +/// builds it for the host: `kernel/.cargo/config.toml` is read only below +/// `kernel/`. +pub(crate) const KERNEL_MANIFEST: &str = "kernel/Cargo.toml"; +/// Where a host build from it lands: the root's `target/`, since `kernel/target` +/// is cleaned whole under the worktree's build lock, which `--ci host` does not +/// take. +pub(crate) const KERNEL_HOST_TARGET: &str = "target"; const KERNEL_LOOM: &str = "kernel-loom"; -const SCHED_LOOM: &str = "toyos-sched-loom"; -const SCHED_SIM: &str = "toyos-sched-sim"; -const PROCLIFE: &str = "toyos-proclife"; +const KERNEL_SIM: &str = "kernel-sim"; const BLOCKRING: &str = "toyos-blockring"; const TRANSPORT: &str = "toyos-transport"; @@ -302,23 +310,23 @@ pub(crate) const CONTROLS: &[Control] = &[ Fails("a_request_filed_during_a_report_is_reported"), ]), Control { - krate: SCHED_LOOM, + krate: KERNEL_LOOM, feature: "no-preempt-guard", test: Some("loom_mailbox"), must_red: false, verdicts: &[Passes("preempted_producer_strands_suffix")], }, - red(SCHED_LOOM, "doorbell-kick-relaxed", Some("loom_sleep"), &[Says { + red(KERNEL_LOOM, "doorbell-kick-relaxed", Some("loom_sleep"), &[Says { test: "a_halted_cpu_with_queued_work_was_kicked", message: "halted with 2 of 2 messages queued and no IPI in flight", }]), - red(SCHED_LOOM, "push-fence-relaxed", Some("loom_push"), &[Says { + red(KERNEL_LOOM, "push-fence-relaxed", Some("loom_push"), &[Says { test: "a_cpu_that_halts_without_seeing_the_surplus_was_pushed", message: "published and no push behind it", }]), // The watch's lost wake, staged: the waiter parks over a post it was flagged // with. - red(SCHED_LOOM, "commit-ignores-notify", Some("loom_watch"), &[ + red(KERNEL_LOOM, "commit-ignores-notify", Some("loom_watch"), &[ Says { test: "a_post_racing_a_registration_leaves_nobody_parked", message: "parked with the condition true and no wake owed: the post was lost", @@ -330,25 +338,25 @@ pub(crate) const CONTROLS: &[Control] = &[ ]), // The notify's flagged arm answering off a load: a second post reads the // word from before the waiter consumed the first flag. - red(SCHED_LOOM, "notify-flag-load-only", Some("loom_watch"), &[Says { + red(KERNEL_LOOM, "notify-flag-load-only", Some("loom_watch"), &[Says { test: "a_second_post_is_not_lost_to_a_flag_the_waiter_consumed", message: "parked with both conditions true and no wake owed: a post answered off a load", }]), // The stop's store-buffering pair with the gate's fences gone. - red(SCHED_LOOM, "gate-fence-off", Some("loom_watch"), &[Says { + red(KERNEL_LOOM, "gate-fence-off", Some("loom_watch"), &[Says { test: "a_transition_racing_an_opening_gate_is_never_missed", message: "the stop parked over a thread that had parked, and nothing posted it", }]), // `kernel-loom`'s control, over the kernel's `Once` as the watch models' // ring entry. - red(SCHED_LOOM, "poll-fire-load-store", Some("loom_watch"), &[ + red(KERNEL_LOOM, "poll-fire-load-store", Some("loom_watch"), &[ Fails("a_poll_registered_racing_a_post_completes_exactly_once"), Fails("a_poll_registered_racing_a_post_in_place_completes_exactly_once"), Fails("a_poll_on_two_watches_racing_both_posts_completes_exactly_once"), ]), // The ring models' lost-completion half: the producer posts before it // stores the readiness its registrant rechecks. - red(SCHED_LOOM, "fault-posted-before-it-is-set", Some("loom_watch"), &[ + red(KERNEL_LOOM, "fault-posted-before-it-is-set", Some("loom_watch"), &[ Says { test: "a_poll_registered_racing_a_post_completes_exactly_once", message: "a poll over a ready object was completed by neither", @@ -360,7 +368,7 @@ pub(crate) const CONTROLS: &[Control] = &[ // (`issues/kernel/steal-probe-node-dies-with-its-victim.md`) rather than // proving a lie is caught, and goes with its fix. Control { - krate: SCHED_LOOM, + krate: KERNEL_LOOM, feature: "victim-retires-mid-probe", test: Some("loom_mailbox"), must_red: false, @@ -370,51 +378,57 @@ pub(crate) const CONTROLS: &[Control] = &[ queue", }], }, - red(PROCLIFE, "mutate-spawn-skips-the-insert-recheck", None, &[ - Fails("interleave::tests::a_published_exit_leaves_no_unretired_thread"), - Fails("interleave::tests::a_kill_racing_a_spawn_leaves_no_unretired_thread"), + // The isolation test reds, and the SDM's own judge of the allocator sees + // the cross-space read it lets through. + red(KERNEL, "counting-allocator", None, &[ + Fails("pcid::tests::two_live_address_spaces_never_share_a_pcid"), + Passes("pcid::oracle::tests::the_counting_allocator_produces_a_cross_space_read"), ]), - red(PROCLIFE, "mutate-claim-teardown-always-wins", None, &[ - Fails("interleave::tests::an_exit_and_a_kill_never_both_tear_a_process_down"), + red(KERNEL, "mutate-spawn-skips-the-insert-recheck", None, &[ + Fails("proclife::interleave::tests::a_published_exit_leaves_no_unretired_thread"), + Fails("proclife::interleave::tests::a_kill_racing_a_spawn_leaves_no_unretired_thread"), ]), - red(PROCLIFE, "mutate-kill-waits-for-its-victims", None, &[ - Fails("interleave::tests::two_processes_killing_each_other_both_end"), - Fails("interleave::tests::a_kill_chain_of_three_ends"), + red(KERNEL, "mutate-claim-teardown-always-wins", None, &[ + Fails("proclife::interleave::tests::an_exit_and_a_kill_never_both_tear_a_process_down"), ]), - red(PROCLIFE, "mutate-first-out-tears-down", None, &[ - Fails("interleave::tests::an_exit_and_a_kill_never_both_tear_a_process_down"), + red(KERNEL, "mutate-kill-waits-for-its-victims", None, &[ + Fails("proclife::interleave::tests::two_processes_killing_each_other_both_end"), + Fails("proclife::interleave::tests::a_kill_chain_of_three_ends"), ]), - red(PROCLIFE, "mutate-join-collects-in-a-teardown", None, &[ - Fails("interleave::tests::a_join_racing_the_kill_that_takes_its_target"), + red(KERNEL, "mutate-first-out-tears-down", None, &[ + Fails("proclife::interleave::tests::an_exit_and_a_kill_never_both_tear_a_process_down"), ]), - red(PROCLIFE, "mutate-last-out-leaves-before-its-teardown", None, &[ - Fails("interleave::tests::the_last_one_out_is_in_its_process_until_its_teardown_is_done"), - Fails("teardown::tests::only_the_thread_that_empties_a_claimed_process_tears_it_down"), + red(KERNEL, "mutate-join-collects-in-a-teardown", None, &[ + Fails("proclife::interleave::tests::a_join_racing_the_kill_that_takes_its_target"), ]), - red(PROCLIFE, "mutate-place-skips-the-insert-recheck", None, &[ - Fails("interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it"), + red(KERNEL, "mutate-last-out-leaves-before-its-teardown", None, &[ + Fails("proclife::interleave::tests::the_last_one_out_is_in_its_process_until_its_teardown_is_done"), + Fails("proclife::teardown::tests::only_the_thread_that_empties_a_claimed_process_tears_it_down"), ]), - red(PROCLIFE, "mutate-refused-spawn-keeps-the-count", None, &[ - Fails("interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it"), + red(KERNEL, "mutate-place-skips-the-insert-recheck", None, &[ + Fails("proclife::interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it"), ]), - red(PROCLIFE, "mutate-landed-child-retires-nothing", None, &[ - Fails("interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it"), - Fails("tree::tests::a_child_landed_under_a_place_claimed_since_its_admission_is_claimed_with_it"), + red(KERNEL, "mutate-refused-spawn-keeps-the-count", None, &[ + Fails("proclife::interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it"), ]), - red(PROCLIFE, "mutate-publish-before-the-children", None, &[ - Fails("interleave::tests::an_exit_publishes_after_every_end_below_it"), + red(KERNEL, "mutate-landed-child-retires-nothing", None, &[ + Fails("proclife::interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it"), + Fails("proclife::tree::tests::a_child_landed_under_a_place_claimed_since_its_admission_is_claimed_with_it"), ]), - red(PROCLIFE, "mutate-walk-in-one-hold", None, &[ - Fails("interleave::tests::a_spawn_under_an_unrelated_process_lands_between_two_claims_of_one_walk"), + red(KERNEL, "mutate-publish-before-the-children", None, &[ + Fails("proclife::interleave::tests::an_exit_publishes_after_every_end_below_it"), ]), - red(PROCLIFE, "mutate-spawner-handle-after-the-landing", None, &[ - Fails("interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it"), - Fails("interleave::tests::a_spawn_racing_the_kill_of_its_own_spawner"), + red(KERNEL, "mutate-walk-in-one-hold", None, &[ + Fails("proclife::interleave::tests::a_spawn_under_an_unrelated_process_lands_between_two_claims_of_one_walk"), ]), - red(PROCLIFE, "mutate-spawner-handle-before-the-childs-own", None, &[ - Fails("interleave::tests::a_sibling_closing_a_spawns_handle_before_the_spawn_returns"), + red(KERNEL, "mutate-spawner-handle-after-the-landing", None, &[ + Fails("proclife::interleave::tests::a_spawn_racing_its_places_kill_leaves_nothing_under_it_and_publishes_it"), + Fails("proclife::interleave::tests::a_spawn_racing_the_kill_of_its_own_spawner"), ]), - red(SCHED_SIM, "placement-ignores-staleness", Some("policy"), &[ + red(KERNEL, "mutate-spawner-handle-before-the-childs-own", None, &[ + Fails("proclife::interleave::tests::a_sibling_closing_a_spawns_handle_before_the_spawn_returns"), + ]), + red(KERNEL_SIM, "placement-ignores-staleness", Some("policy"), &[ Fails("a_stopped_cpu_stops_taking_work"), ]), // The block protocol's three: a completion lost to a session's end, a @@ -466,7 +480,13 @@ fn judge_control(control: &Control, exited_green: bool, log: &str) -> Result Result { - let mut args = vec!["test", "-p", control.krate, "--features", control.feature]; + let mut args = vec!["test"]; + // Cargo refuses `--features` for a package outside the workspace. + match control.krate { + KERNEL => args.extend(["--manifest-path", KERNEL_MANIFEST, "--target-dir", KERNEL_HOST_TARGET]), + member => args.extend(["-p", member]), + } + args.extend(["--features", control.feature]); match control.test { Some(test) => args.extend(["--test", test]), None => args.push("--lib"), @@ -481,10 +501,11 @@ fn run_control(root: &Path, control: &Control) -> Result { } /// Every test that runs on the host and boots no guest. The build system's own -/// tests, every member of the host workspace, clippy with warnings denied, the -/// concurrency models' negative controls, every userland crate with a host test -/// ([`crate::userlandhost`], which also reds on a userland test none of them -/// runs), every app the images ship for each host ([`apps_for`]), and the SDK. +/// tests, every member of the host workspace, the kernel's library, clippy with +/// warnings denied, the concurrency models' negative controls, every userland +/// crate with a host test ([`crate::userlandhost`], which also reds on a +/// userland test none of them runs), every app the images ship for each host +/// ([`apps_for`]), and the SDK. /// /// **Every step runs against a `$TMPDIR` of this job's own, and the last step /// reds on anything left in it** but the lock `toyos_tmpdir` keeps there: a test @@ -523,6 +544,18 @@ fn host(root: &Path) -> Vec { step("the host workspace", || { cargo(root, &["test", "--workspace", "--exclude", "toyos-build"]) }), + step("the kernel's library", || { + cargo(root, &[ + "test", + "--manifest-path", + KERNEL_MANIFEST, + "--target-dir", + KERNEL_HOST_TARGET, + "--lib", + "--features", + "sched-check", + ]) + }), step("the licences of what ships", || crate::licence::judge(root)), ]); steps.push(step("clippy and the bare targets", || { @@ -548,8 +581,8 @@ fn host(root: &Path) -> Vec { steps.push(step("kernel-loom without loom", || { cargo(root, &[ "test", - "--manifest-path", - "kernel-loom/Cargo.toml", + "-p", + KERNEL_LOOM, "--no-default-features", "--test", "log_zeroed_init", @@ -1113,7 +1146,7 @@ mod tests { let says = control("doorbell-kick-relaxed"); let panicked = "running 1 test\n\n\ thread 'a_halted_cpu_with_queued_work_was_kicked' (77012) panicked at \ - toyos-sched/loom/tests/loom_sleep.rs:102:13:\n\ + kernel/loom/tests/loom_sleep.rs:102:13:\n\ halted with 2 of 2 messages queued and no IPI in flight — a sleep-through\n"; assert!(judge_control(says, false, panicked).is_ok()); diff --git a/src/clippy.rs b/src/clippy.rs index 442847244b4..53621bcd273 100644 --- a/src/clippy.rs +++ b/src/clippy.rs @@ -12,6 +12,7 @@ use std::path::Path; use std::process::Command; use crate::arch::Arch; +use crate::ci::{KERNEL_HOST_TARGET, KERNEL_MANIFEST}; /// The pedantic/nursery lints adopted one at a time, each on a measured finding /// (`issues/build/clippy-stage-two-is-lints-one-at-a-time.md`). @@ -27,7 +28,8 @@ const ADOPTED: &[&str] = &[ /// One `cargo clippy`. `dir` is relative to the repository root and empty for /// the root itself — `.cargo/config.toml` is found from the working directory, /// so the kernel and bootloader run from their own; a `$ADOPTED` token splices -/// [`ADOPTED`] there, and a `$CONTROLS` token [`control_features`]. +/// [`ADOPTED`] there, a `$CONTROLS` token [`control_features`], and a +/// `$KERNEL_CONTROLS` token those of them whose package is `kernel`. struct Shape { dir: &'static str, before: &'static [&'static str], @@ -47,12 +49,12 @@ const AARCH64_INSTRUMENTS: &str = "debug-wait,sched-check,sched-tripwire,heap-tr switch-witness-mutate-frame,switch-witness-mutate-rsp,\ mask-windows"; -const UNCONTROLLED: &[&str] = &["toyos-pcid/counting-allocator", "toyos-sched/tripwire"]; +const UNCONTROLLED: &[&str] = &["kernel/sched-tripwire"]; -/// Every model's negative control and [`UNCONTROLLED`], as one `--features` list. -fn control_features() -> String { +/// Every model's negative control and [`UNCONTROLLED`], as `package/feature`. +fn control_features() -> Vec { let controls = crate::ci::CONTROLS.iter().map(|c| format!("{}/{}", c.krate, c.feature)); - controls.chain(UNCONTROLLED.iter().map(|f| (*f).to_string())).collect::>().join(",") + controls.chain(UNCONTROLLED.iter().map(|f| (*f).to_string())).collect() } /// `--all-targets` on the host workspace only: on the bootloader and kernel a @@ -63,8 +65,11 @@ fn control_features() -> String { /// `undocumented_unsafe_blocks` is adopted per area as each area's /// justifications land. `toyos-xhci` has a shape of its own because the /// workspace run builds it only with `toyos-xhci-sim`'s `flaws`, never as the -/// kernel does. `toyos-sched-loom` has one because `victim-retires-mid-probe`'s +/// kernel does. `kernel-loom` has one because `victim-retires-mid-probe`'s /// test arm excludes `no-preempt-guard`, which `$CONTROLS` turns on beside it. +/// The kernel's library, no member of the host workspace, is linted on the host +/// from its own manifest: its tests as `--ci host` runs them, and again with +/// `$KERNEL_CONTROLS`. const SHAPES: &[Shape] = &[ Shape { dir: "", @@ -76,6 +81,36 @@ const SHAPES: &[Shape] = &[ before: &["--workspace", "--all-targets", "--keep-going", "--features", "$CONTROLS"], after: &["$ADOPTED", "-D", "warnings"], }, + Shape { + dir: "", + before: &[ + "--manifest-path", + KERNEL_MANIFEST, + "--target-dir", + KERNEL_HOST_TARGET, + "--lib", + "--tests", + "--features", + "sched-check", + ], + after: &["$ADOPTED", "-D", "warnings"], + }, + Shape { + dir: "", + before: &[ + "--manifest-path", + KERNEL_MANIFEST, + "--target-dir", + KERNEL_HOST_TARGET, + "--lib", + "--tests", + "--features", + "sched-check,protocol-port", + "--features", + "$KERNEL_CONTROLS", + ], + after: &["$ADOPTED", "-D", "warnings"], + }, Shape { dir: "kernel", before: &["--target", Arch::X86_64.kernel()], @@ -145,7 +180,7 @@ const SHAPES: &[Shape] = &[ dir: "", before: &[ "-p", - "toyos-sched-loom", + "kernel-loom", "--test", "loom_mailbox", "--features", @@ -169,7 +204,7 @@ pub const BARE_TARGETS: [&str; 4] = [Arch::X86_64.kernel(), Arch::X86_64.loader(), Arch::Aarch64.kernel(), Arch::Aarch64.loader()]; impl Shape { - /// The command as a reader writes it, `$ADOPTED` and `$CONTROLS` unexpanded. + /// The command as a reader writes it, its tokens unexpanded. fn line(&self) -> String { let mut parts = vec!["cargo clippy".to_string()]; parts.extend(self.before.iter().map(|s| (*s).to_string())); @@ -178,13 +213,21 @@ impl Shape { parts.join(" ") } - /// The arguments to `cargo clippy`, both tokens spliced in — what actually + /// The arguments to `cargo clippy`, every token spliced in — what actually /// runs. fn args(&self) -> Vec { + let kernel = format!("{}/", crate::ci::KERNEL); let mut args: Vec = self .before .iter() - .map(|s| if *s == "$CONTROLS" { control_features() } else { (*s).to_string() }) + .map(|s| match *s { + "$CONTROLS" => control_features().join(","), + "$KERNEL_CONTROLS" => { + let ours = control_features().into_iter().filter(|f| f.starts_with(&kernel)); + ours.collect::>().join(",") + } + arg => arg.to_string(), + }) .collect(); args.push("--".to_string()); for token in self.after { diff --git a/src/hostws.rs b/src/hostws.rs index c7dbd21d97e..0771a513dc4 100644 --- a/src/hostws.rs +++ b/src/hostws.rs @@ -133,7 +133,12 @@ pub fn target_dir(root: &Path, crate_dir: &Path) -> PathBuf { } /// Every directory under `root` holding a `Cargo.toml`, as paths relative to -/// `root`, with the excluded subtrees pruned exactly as cargo prunes them. +/// `root`, but the excluded ones. +/// +/// An excluded workspace, or an excluded directory that is no package, is +/// pruned whole. An excluded package's subdirectories are walked: cargo +/// excludes a crate nested in one too unless `members` names it, which is an +/// exclusion argued for the parent and never for it. /// /// `target`, `.git` and every other dotted directory go too: what is in them is /// build output and history, not a crate somebody has to have declared. @@ -149,8 +154,17 @@ fn crate_dirs(root: &Path) -> BTreeSet { #[cfg(test)] fn walk(root: &Path, dir: &Path, prune: &BTreeSet, found: &mut BTreeSet) { - if dir.join("Cargo.toml").is_file() { - let relative = rel(root, dir); + let relative = rel(root, dir); + let manifest = dir.join("Cargo.toml"); + if prune.contains(&relative) { + let package = std::fs::read_to_string(&manifest) + .ok() + .and_then(|text| text.parse::().ok()) + .is_some_and(|doc| doc.get("package").is_some() && doc.get("workspace").is_none()); + if !package { + return; + } + } else if manifest.is_file() { found.insert(if relative.is_empty() { ".".to_string() } else { relative }); } let Ok(entries) = std::fs::read_dir(dir) else { return }; @@ -164,18 +178,15 @@ fn walk(root: &Path, dir: &Path, prune: &BTreeSet, found: &mut BTreeSet< if name.starts_with('.') || name == "target" { continue; } - if prune.contains(&rel(root, &path)) { - continue; - } walk(root, &path, prune, found); } } /// The crate directories that joined neither list. /// -/// Excluded subtrees never reach here — they are pruned during the walk, which -/// is what excluding one means — so this is the whole question: a `Cargo.toml` -/// the tree contains and the `[workspace]` table does not account for. +/// Excluded directories never reach here — [`crate_dirs`] leaves them out — so +/// this is the whole question: a `Cargo.toml` the tree contains and the +/// `[workspace]` table does not account for. #[cfg(test)] fn unclaimed(members: &BTreeSet, found: &BTreeSet) -> Vec { found.difference(members).cloned().collect() @@ -325,7 +336,7 @@ mod tests { #[test] fn the_gate_refuses_a_crate_that_joined_neither_list() { let members: BTreeSet = - [".", "toyos-elf", "toyos-sched", "toyos-sched/sim"].iter().map(|s| s.to_string()).collect(); + [".", "toyos-elf", "toyos-xhci", "toyos-xhci/sim"].iter().map(|s| s.to_string()).collect(); let all_declared: BTreeSet = members.clone(); assert!(unclaimed(&members, &all_declared).is_empty()); @@ -336,11 +347,10 @@ mod tests { members.union(&["toyos-newthing".to_string()].into()).cloned().collect(); assert_eq!(unclaimed(&members, &with_a_newcomer), ["toyos-newthing"]); - // And a nested one, which is how `toyos-sched/loom` could have been - // lost when its parent stopped being a workspace of its own. + // And a nested one. let with_a_nested_newcomer: BTreeSet = - members.union(&["toyos-sched/loom".to_string()].into()).cloned().collect(); - assert_eq!(unclaimed(&members, &with_a_nested_newcomer), ["toyos-sched/loom"]); + members.union(&["toyos-xhci/newthing".to_string()].into()).cloned().collect(); + assert_eq!(unclaimed(&members, &with_a_nested_newcomer), ["toyos-xhci/newthing"]); } /// The walk has teeth only if it can find anything: it must reach the real @@ -353,10 +363,15 @@ mod tests { assert!(found.contains("."), "the walk did not find the root package"); assert!(found.contains("toyos-elf"), "the walk did not find toyos-elf: {found:?}"); assert!( - found.contains("toyos-sched/loom"), + found.contains("toyos-xhci/sim"), "the walk did not descend past the first level, so a nested member could \ go missing without this gate noticing: {found:?}" ); + assert!( + found.contains("kernel/loom") && !found.contains("kernel"), + "the walk did not descend into an excluded package, so a crate nested in one \ + could lose its `members` line without this gate noticing: {found:?}" + ); // With nothing declared a member, every one of them is a complaint — // the gate above is silent because the table accounts for the tree, not // because the walk found nothing to account for. diff --git a/src/kernelkeys.rs b/src/kernelkeys.rs index 2da0203b72a..4d28b91a29e 100644 --- a/src/kernelkeys.rs +++ b/src/kernelkeys.rs @@ -61,7 +61,7 @@ pub const DECLARED: &[Declared] = &[ Declared { file: "kernel/src/process.rs", ty: "HashMap", - keys: "the pid of the entry's `ProcessObject`, which only `toyos_proclife::Pids::take` issues, \ + keys: "the pid of the entry's `ProcessObject`, which only `kernel::proclife::Pids::take` issues, \ at a spawn's admission or a kernel thread's start", }, ]; diff --git a/tests/CLAUDE.md b/tests/CLAUDE.md index 01bb2c654c3..829095721d0 100644 --- a/tests/CLAUDE.md +++ b/tests/CLAUDE.md @@ -18,4 +18,4 @@ The mechanics live where the work is: profiles and shapes in `tests/common/`, re - **A stimulus sent through a channel that can silently lose it is verified before its effect is asserted** — QEMU's PS/2 queue drops the seventeenth byte, so typed input paces against the guest's report; a guest's console reaches the host as whole lines only, so a partial line exists on no channel. - **A harness field that can be silently inert is this suite's worst defect class** — where two options can describe the same guest they refuse each other by name, and an image is asked what it is armed with. - **A test whose premise is arranged by a defect passes for the wrong reason** — a staging device's resource has to survive its own enumeration. -- **Host suites** are the root `Cargo.toml`'s `[workspace]` members, the SDK and every userland crate `src/userlandhost.rs` finds a test in; `src/ci.rs`'s `host` runs them all. `kernel-loom/` and `toyos-sched/loom` are the memory-ordering checks — x86 TSO hides a missing acquire edge from every guest test. +- **Host suites** are the root `Cargo.toml`'s `[workspace]` members, the kernel's library, the SDK and every userland crate `src/userlandhost.rs` finds a test in; `src/ci.rs`'s `host` runs them all. `kernel/loom/` is the memory-ordering check — x86 TSO hides a missing acquire edge from every guest test. diff --git a/tests/checks.rs b/tests/checks.rs index 88cd2bdc418..db34f7fc875 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -419,7 +419,7 @@ mod checks { }; let report = |cpu0: (u64, u64), cpu1: (u64, u64)| [census(0), windows(0, cpu0), census(1), windows(1, cpu1)].concat(); let exit = |name: &str| format!("[kernel 0.1 cpu0] exit: {name} pid=9 code=0 cpu=1ms\n"); - let held_ns = toyos_sched::windows::HELD_NS; + let held_ns = kernel::sched::windows::HELD_NS; let hold = format!("[kernel 0.1 cpu1] windows: held cpu1 ns={}\n", held_ns + 7); // Since each CPU joined: longer on cpu0 than anything under the load. let first = [report((6_500_000, 6_400_000), (900, 800)), exit("test_rs_idle_span")].concat(); diff --git a/tests/checks/metal.rs b/tests/checks/metal.rs index bcf499e6658..8ccfcf79fc9 100644 --- a/tests/checks/metal.rs +++ b/tests/checks/metal.rs @@ -114,7 +114,7 @@ pub fn the_stop_owes_its_record_and_leaves_no_operation_open() { /// Each bound's lateness against the period of what polls it, a millisecond /// either way and no further. pub fn a_bound_fires_within_one_period_of_itself() { - let quantum = i64::try_from(toyos_sched::fair::QUANTUM_NS / 1_000_000).expect("ms"); + let quantum = i64::try_from(kernel::sched::fair::QUANTUM_NS / 1_000_000).expect("ms"); let deadline = |late: i64| planted(&expired_at(120_060 + late), &armed()).deadline_on_time(); assert_eq!(deadline(4), Ok(())); assert_eq!(deadline(quantum + 1), Ok(())); diff --git a/tests/common/irqcensus.rs b/tests/common/irqcensus.rs index d317a468056..d3401be45aa 100644 --- a/tests/common/irqcensus.rs +++ b/tests/common/irqcensus.rs @@ -254,8 +254,8 @@ pub fn windows_under(capture: &str, cpus: u32, load_exited: &str) -> Result