From 4ed2bf27c24b02e5f66bdd3a10b2bec68e1a465a Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 1 Oct 2026 07:03:53 +0200 Subject: [PATCH 1/6] An ABI edit builds ToyOS's std alone, and cleans only what was built for ToyOS A toyos-abi edit moved the one sysroot key, and the key's build compiled std from nothing for all six guest targets: ToyOS userland's two, which compile toyos-abi and toyos through std, and the kernel's and the loader's four, which compile neither. Then every crate's target directory was `cargo clean`ed whole, because its stamp named the sysroot's directory and every target's rlibs. The kernel, the bootloader and the host half of every crate (proc-macros, build scripts) went with what was built for ToyOS. The kernel's and the loader's libraries are now a key of their own (`sysroot::freestanding_key`): the std recipe, the stage-0 cargo, the fork's `library/` and `src/bootstrap/`, the compiler, and the manifests std's lockfile resolves (`toyos-abi/Cargo.toml`, `toyos/Cargo.toml`), which can move a shared dependency's version under resolver 1. They are built once per that key into `rust/build/freestanding//` and cloned into every sysroot that names it. The sysroot key names the freestanding key, the witness and the rest of the recipe. Each std build refuses dep-info that contradicts the split. A freestanding target's dep-info naming any toyos-abi or toyos source is refused, as a userland target's naming another worktree's already was. Both builds still start from nothing per key, so no build relies on cargo's mtime freshness across keys. A crate's stamp is now the sysroot's identity (`Sysroot::identity`): the compiler, and per guest target the key of its libraries. A moved compiler cleans as before. Moved libraries take `target//` and nothing else, so an ABI edit leaves the kernel's and the loader's target directories and every crate's host half to cargo's own incremental build. An installed toolchain's identity is the TOOLCHAIN it was published with. The removal goes through `worktree::remove_tree`: a Finder `.DS_Store` written into `userland/target/x86_64-unknown-toyos` mid-removal failed a plain `remove_dir_all` here with `Directory not empty`. The stamp write now panics on failure, as the clean beside it does. That closes issues/build/cleans-deps-stamp-write-is-swallowed.md, which named whoever next touched `clean` as its owner. What is still built for every key is filed: issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md and issues/build/a-sysroot-builds-both-architectures-userland-for-a-build-of-one.md. Measured with `cargo run -- --build-only` after a one-item toyos-abi edit, base and branch back to back on a shared 14-core host: - at load ~41: 648 s became 507 s; - at load 57 and 63: 758 s became 471 s, and CPU 1470 s became 1065 s; - under heavy load: 1637 s became 722 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016t9wjdQkB8SH7bmfUoiy6L --- ...chitectures-userland-for-a-build-of-one.md | 18 + .../cleans-deps-stamp-write-is-swallowed.md | 21 -- ...untime-is-rebuilt-for-every-sysroot-key.md | 19 + src/build.rs | 194 ++++++---- src/buildlock.rs | 17 +- src/keystore.rs | 1 + src/sysroot.rs | 352 ++++++++++++++---- src/toolchain.rs | 46 ++- src/worktree.rs | 3 +- 9 files changed, 490 insertions(+), 181 deletions(-) create mode 100644 issues/build/a-sysroot-builds-both-architectures-userland-for-a-build-of-one.md delete mode 100644 issues/build/cleans-deps-stamp-write-is-swallowed.md create mode 100644 issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md diff --git a/issues/build/a-sysroot-builds-both-architectures-userland-for-a-build-of-one.md b/issues/build/a-sysroot-builds-both-architectures-userland-for-a-build-of-one.md new file mode 100644 index 00000000000..6e14341eb40 --- /dev/null +++ b/issues/build/a-sysroot-builds-both-architectures-userland-for-a-build-of-one.md @@ -0,0 +1,18 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# A sysroot builds both architectures' userland for a build of one + +A sysroot is a whole toolchain for every guest target (`toolchain::GUEST_TARGETS`), +so a new key builds the std set of `x86_64-unknown-toyos` and of +`aarch64-unknown-toyos`, libc for both, and the C++ runtime for both, even when +the build that asked for it is `cargo run -- --build-only`, which links only +x86_64. Measured on the `wt/toyos-rebuild` branch, one item added to +`toyos-abi/src/clock.rs`: the aarch64 std set took 2m18s of a 4m12s std +build at load 41. + +**Exit**: a build makes the libraries of only the architectures it builds for, +and the sysroot's key and identity still name exactly what is in it. diff --git a/issues/build/cleans-deps-stamp-write-is-swallowed.md b/issues/build/cleans-deps-stamp-write-is-swallowed.md deleted file mode 100644 index ca3bf064a2d..00000000000 --- a/issues/build/cleans-deps-stamp-write-is-swallowed.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-28 ---- - -# `clean`'s deps-stamp write is swallowed - -`src/build.rs`'s `clean` ends with `fs::create_dir_all(&target).ok();` and -`fs::write(target.join(".deps-stamp"), fingerprint).ok();` — the two arms -above them (`cargo clean` and the per-target `remove_dir_all`) both panic on -failure, so these are the only swallowed errors left in the function. A stamp -write that fails is not reported, and `stale` reads it back as absent or -mismatched, so every later build re-decides `clean` was owed and pays a full -clean it does not need. - -**Owner:** whoever next touches `clean` or `stale` in `src/build.rs`. - -**Exit condition:** both calls panic on error like their two siblings, or the -`.ok()` is replaced with a call that folds a write failure into the same -refusal `stale` would otherwise silently manufacture. diff --git a/issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md b/issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md new file mode 100644 index 00000000000..0b728aceed5 --- /dev/null +++ b/issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md @@ -0,0 +1,19 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# The C++ runtime is rebuilt for every sysroot key + +`sysroot::build` builds libc++, libc++abi and libunwind for both userland +targets into every new sysroot (`libcxx::build`), so every `toyos-abi` edit +pays for them. Their inputs are the compiler's LLVM, `libcxx::OPTIONS` and the +C sysroot they compile and link against: libc's headers and its staticlib. An +ABI edit moves only the staticlib, and an edit that leaves its bytes the same +moves none of them. Measured on the `wt/toyos-rebuild` branch, one item added +to `toyos-abi/src/clock.rs`: 27 s and 38 s at load 41, 47 s and 54 s at load 68. + +**Exit**: the C++ runtime is keyed by what it is built from, the C sysroot's +bytes among them, and an ABI edit that leaves those bytes unchanged builds no +C++ runtime. diff --git a/src/build.rs b/src/build.rs index 15d9f5386ea..42e5cc7b07c 100644 --- a/src/build.rs +++ b/src/build.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::rc::Rc; use std::sync::Arc; -use std::time::{Duration, Instant, UNIX_EPOCH}; +use std::time::{Duration, Instant}; use serde::Deserialize; @@ -173,46 +173,22 @@ fn parse_config(path: &Path) -> SystemConfig { // --- Freshness checking --- -/// Fingerprint all external build dependencies that cargo cannot track: the -/// sysroot `toolchain` is — by where it is, which names its key, and by its -/// libraries. The linker is the sysroot's own `rust-lld`, which that key names -/// with the compiler it came with. -fn external_fingerprint(toolchain: &Path) -> String { - let sysroot = toolchain.join("lib/rustlib"); - let mut entries = vec![format!("sysroot:{}", toolchain.display())]; - - for triple in toolchain::GUEST_TARGETS { - let lib_dir = sysroot.join(format!("{triple}/lib")); - let Ok(rd) = fs::read_dir(&lib_dir) else { - continue; - }; - for entry in rd.flatten() { - let path = entry.path(); - if !path.is_file() { - continue; - } - let ext = path.extension().and_then(|e| e.to_str()); - if !matches!(ext, Some("rlib" | "rmeta")) { - continue; - } - if let Ok(meta) = path.metadata() { - let name = path.file_name().unwrap().to_string_lossy(); - let mtime = meta - .modified() - .ok() - .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) - .map(|d| d.as_nanos()) - .unwrap_or(0); - entries.push(format!("{triple}/{name}:{}:{mtime}", meta.len())); - } - } - } - - entries.sort(); - entries.join("\n") +/// What of a crate's target directory a sysroot's identity +/// (`sysroot::Sysroot::identity`) leaves stale against the one its +/// `.deps-stamp` records. Cargo keys nothing it reuses on the sysroot — every +/// ToyOS compiler prints one `rustc -vV` — so this is where a moved compiler or +/// library reaches a crate. +#[derive(Debug, PartialEq)] +enum Stale { + /// The compiler moved, or nothing records which one built it: all the guest + /// build wrote, its host half too. + All, + /// Only these guest targets' libraries moved: what was made for them, + /// `target//`, and nothing the compiler made for the host. + Targets(Vec), } -/// How much of a crate's target directory goes when the external deps change. +/// How much of a crate's target directory goes when the compiler moves. #[derive(Clone, Copy)] enum Clean { All, @@ -226,17 +202,32 @@ enum Clean { ToyosOnly, } -fn stale(root: &Path, crate_dir: &Path, fingerprint: &str) -> bool { +fn stale(root: &Path, crate_dir: &Path, identity: &str) -> Option { let stamp = hostws::target_dir(root, crate_dir).join(".deps-stamp"); - fs::read_to_string(&stamp).map_or(true, |stored| stored != fingerprint) + let Ok(stored) = fs::read_to_string(&stamp) else { return Some(Stale::All) }; + let parts = |text: &str| -> BTreeMap { + text.lines().filter_map(|l| l.split_once(' ')).map(|(part, id)| (part.into(), id.into())).collect() + }; + let (was, now) = (parts(&stored), parts(identity)); + if was.get("compiler") != now.get("compiler") { + return Some(Stale::All); + } + let moved: Vec = now.iter().filter(|(part, id)| was.get(*part) != Some(*id)).map(|(part, _)| part.clone()).collect(); + (!moved.is_empty()).then_some(Stale::Targets(moved)) } -fn clean(root: &Path, crate_dir: &Path, kind: Clean, fingerprint: &str) { +fn clean(root: &Path, crate_dir: &Path, kind: Clean, stale: &Stale, identity: &str) { // Where cargo actually wrote it. `toyos-ld` is a member of the host // workspace, so its guest build lands in the root's `target/`. let target = hostws::target_dir(root, crate_dir); - match kind { - Clean::All => { + let remove = |dirs: &mut dyn Iterator| { + for dir in dirs.filter(|dir| dir.exists()) { + eprintln!("external deps changed: cleaning {}", dir.display()); + crate::worktree::remove_tree(&dir); + } + }; + match (stale, kind) { + (Stale::All, Clean::All) => { // `cargo clean` in a member's directory cleans the whole workspace, // this build system's own target directory included. Nothing asks // for that today; refusing it by name is cheaper than finding out. @@ -254,22 +245,19 @@ fn clean(root: &Path, crate_dir: &Path, kind: Clean, fingerprint: &str) { .unwrap_or_else(|e| panic!("run cargo clean in {}: {e}", crate_dir.display())); assert!(status.success(), "cargo clean in {} exited {status}", crate_dir.display()); } - Clean::ToyosOnly => { + (Stale::All, Clean::ToyosOnly) => { let guest = Arch::ALL.iter().map(|arch| target.join(arch.userland())); - for dir in guest.chain([target.join(PROFILE)]) { - if dir.exists() { - eprintln!("external deps changed: cleaning {}", dir.display()); - fs::remove_dir_all(&dir).unwrap_or_else(|e| panic!("remove {}: {e}", dir.display())); - } - } + remove(&mut guest.chain([target.join(PROFILE)])); } + (Stale::Targets(moved), _) => remove(&mut moved.iter().map(|t| target.join(t))), } - fs::create_dir_all(&target).ok(); - fs::write(target.join(".deps-stamp"), fingerprint).ok(); + fs::create_dir_all(&target).unwrap_or_else(|e| panic!("create {}: {e}", target.display())); + let stamp = target.join(".deps-stamp"); + fs::write(&stamp, identity).unwrap_or_else(|e| panic!("write {}: {e}", stamp.display())); } -/// Drop the target directories the changed external deps invalidated. +/// Drop what in the target directories the sysroot's moved parts invalidated. /// /// Deciding and acting under one exclusive section is the whole point. Each of /// these cleans removes a tree another builder may be compiling into, and @@ -280,24 +268,22 @@ fn clean(root: &Path, crate_dir: &Path, kind: Clean, fingerprint: &str) { fn invalidate_stale( root: &Path, lock: &mut buildlock::Held, - toolchain: &Path, + identity: &str, targets: &[(PathBuf, Clean)], ) { lock.act_if( buildlock::Scope::Worktree, "clean crate targets against changed external deps", || { - let fp = external_fingerprint(toolchain); - let work: Vec<(PathBuf, Clean)> = targets + let work: Vec<(PathBuf, Clean, Stale)> = targets .iter() - .filter(|(dir, _)| stale(root, dir, &fp)) - .cloned() + .filter_map(|(dir, kind)| stale(root, dir, identity).map(|s| (dir.clone(), *kind, s))) .collect(); - (!work.is_empty()).then_some((fp, work)) + (!work.is_empty()).then_some(work) }, - |(fp, work)| { - for (dir, kind) in work { - clean(root, &dir, kind, &fp); + |work| { + for (dir, kind, stale) in work { + clean(root, &dir, kind, &stale, identity); } }, ); @@ -420,6 +406,9 @@ struct GuestEnv { /// Whether that sysroot's compiler is the primary's, the one the hosted /// rustc is built from (`src/compiler.rs`). primary_compiler: bool, + /// What its crates compile against beside their sources + /// (`sysroot::Sysroot::identity`). + identity: String, /// The public key the loader and `/system/bin/update` embed /// (`signing::KEY_ENV`): every guest build carries it, so no crate that /// names it can be built without it. @@ -433,6 +422,7 @@ impl GuestEnv { Self { toolchain: sysroot.dir.clone(), primary_compiler: sysroot.primary_compiler, + identity: sysroot.identity(), image_key: crate::signing::key().public_hex(), floor_scope: crate::signing::key().floor_scope().word(), } @@ -1844,7 +1834,7 @@ fn shipped_parts(root: &Path, boot: &Boot, plan: &Plan) -> (Vec, Vec, Ve let env = GuestEnv::new(&sysroot); let config = parse_config(&boot.config); - invalidate_stale(root, &mut lock, &env.toolchain, &config_targets(root, &config)); + invalidate_stale(root, &mut lock, &env.identity, &config_targets(root, &config)); // Same lock-and-stage as `build_test_image`: `cargo run --build-only` and // `cargo test` share these paths, so this races the harness too. The kernel @@ -2077,7 +2067,7 @@ pub fn build_test_parts( let sysroot = crate::toolchain::ensure(root, &mut lock); let env = GuestEnv::new(&sysroot); - invalidate_stale(root, &mut lock, &env.toolchain, &config_targets(root, &config)); + invalidate_stale(root, &mut lock, &env.identity, &config_targets(root, &config)); // Build and stage under one lock, released before `build_and_assemble`. // Releasing it there is deliberate and required: that build takes its own @@ -2294,7 +2284,7 @@ impl TestBuild { let mut lock = buildlock::shared(root, what); let sysroot = crate::toolchain::ensure(root, &mut lock); let env = GuestEnv::new(&sysroot); - invalidate_stale(root, &mut lock, &env.toolchain, stale_targets); + invalidate_stale(root, &mut lock, &env.identity, stale_targets); let artifact = buildlock::artifact(root); TestBuild { target: arch.userland(), env, _lock: lock, _artifact: artifact } } @@ -2415,7 +2405,7 @@ mod tests { guest.push(file(&format!("target/{PROFILE}/deps/libsyn-1.rlib"))); let host = file("target/debug/toyos-build"); - clean(&root, &root.join("ld"), Clean::ToyosOnly, "fingerprint"); + clean(&root, &root.join("ld"), Clean::ToyosOnly, &Stale::All, "fingerprint"); for gone in &guest { assert!(!gone.exists(), "{} survived a clean of what the guest build wrote", gone.display()); } @@ -2423,6 +2413,74 @@ mod tests { assert_eq!(fs::read_to_string(root.join("target/.deps-stamp")).unwrap(), "fingerprint"); } + /// **An ABI edit takes only what was built for ToyOS**: its sysroot moves + /// the userland targets' libraries and nothing else, so the kernel's target + /// directory stays whole, and of userland's only `target/` + /// goes — the host half the same compiler built stays. A moved compiler, + /// or a stamp that names none, is all of it. + #[test] + fn moved_libraries_take_what_was_built_for_them_and_a_moved_compiler_all() { + use crate::sysroot::Sysroot; + let root = toyos_tmpdir::TempDir::new("moved-libraries"); + fs::write(root.join("Cargo.toml"), "[workspace]\nmembers = []\n").unwrap(); + let file = |path: PathBuf| { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, "built").unwrap(); + path + }; + let (kernel, userland) = (root.join("kernel"), root.join("userland")); + let kernels = Arch::ALL.map(|arch| file(kernel.join(format!("target/{}/{PROFILE}/kernel", arch.kernel())))); + let kernel_host = file(kernel.join(format!("target/{PROFILE}/deps/libproc-1.dylib"))); + let programs = Arch::ALL.map(|arch| file(userland.join(format!("target/{}/{PROFILE}/init", arch.userland())))); + let userland_host = file(userland.join(format!("target/{PROFILE}/deps/libsyn-1.rlib"))); + + let before = Sysroot::of_parts("compiler", "freestanding", "toyos").identity(); + for dir in [&kernel, &userland] { + fs::write(dir.join("target/.deps-stamp"), &before).unwrap(); + assert_eq!(stale(&root, dir, &before), None, "{} was stale against its own stamp", dir.display()); + } + + let abi_edit = Sysroot::of_parts("compiler", "freestanding", "toyos, edited").identity(); + let mut toyos: Vec = Arch::ALL.map(|arch| arch.userland().to_string()).into(); + toyos.sort(); + for dir in [&kernel, &userland] { + let found = stale(&root, dir, &abi_edit); + assert_eq!(found, Some(Stale::Targets(toyos.clone())), "{}", dir.display()); + clean(&root, dir, Clean::All, &found.unwrap(), &abi_edit); + assert_eq!(stale(&root, dir, &abi_edit), None, "{} was not stamped", dir.display()); + } + for kept in kernels.iter().chain([&kernel_host, &userland_host]) { + assert!(kept.is_file(), "{} went, and nothing it was built from moved", kept.display()); + } + for gone in &programs { + assert!(!gone.exists(), "{} survived its target's libraries moving", gone.display()); + } + + let compiler = Sysroot::of_parts("another compiler", "freestanding", "toyos, edited").identity(); + assert_eq!(stale(&root, &kernel, &compiler), Some(Stale::All), "a moved compiler kept the host half"); + fs::write(userland.join("target/.deps-stamp"), "sysroot:/a/stamp/naming/no/compiler").unwrap(); + assert_eq!(stale(&root, &userland, &abi_edit), Some(Stale::All), "a stamp naming no compiler was trusted"); + } + + /// **A stamp that cannot be written stops the build**: one left behind would + /// call what the clean took current, or owe a clean every build after. + #[test] + fn a_stamp_that_cannot_be_written_panics() { + use std::os::unix::fs::PermissionsExt; + let root = toyos_tmpdir::TempDir::new("unwritable-stamp"); + fs::write(root.join("Cargo.toml"), "[workspace]\nmembers = []\n").unwrap(); + let target = root.join("kernel/target"); + fs::create_dir_all(&target).unwrap(); + fs::set_permissions(&target, fs::Permissions::from_mode(0o555)).unwrap(); + let failed = std::panic::catch_unwind(|| { + clean(&root, &root.join("kernel"), Clean::All, &Stale::Targets(vec![]), "identity") + }); + fs::set_permissions(&target, fs::Permissions::from_mode(0o755)).unwrap(); + let refusal = failed.expect_err("a stamp that was not written was taken for written"); + let refusal = refusal.downcast_ref::().expect("a formatted panic"); + assert!(refusal.starts_with(&format!("write {}", target.join(".deps-stamp").display())), "{refusal}"); + } + /// **A `cargo clean` that fails stops the build and stamps nothing**: a /// stamp over a target it did not clean calls another compiler's build /// current. @@ -2434,7 +2492,7 @@ mod tests { fs::create_dir_all(kernel.join("target")).unwrap(); fs::write(kernel.join("Cargo.toml"), "[package\n").unwrap(); - let failed = std::panic::catch_unwind(|| clean(&root, &kernel, Clean::All, "fingerprint")); + let failed = std::panic::catch_unwind(|| clean(&root, &kernel, Clean::All, &Stale::All, "fingerprint")); let refusal = failed.expect_err("a cargo clean that failed was taken for one that ran"); let refusal = refusal.downcast_ref::().expect("a formatted panic"); assert!(refusal.starts_with(&format!("cargo clean in {} exited", kernel.display())), "{refusal}"); diff --git a/src/buildlock.rs b/src/buildlock.rs index db145dfb30c..323376a200f 100644 --- a/src/buildlock.rs +++ b/src/buildlock.rs @@ -32,9 +32,10 @@ //! primary's (`src/compiler.rs`) is locked the same way under its own key, and //! neither it nor a sysroot built from it takes the global lock. //! -//! key's lock → a sysroot key's lock → the worktree build lock → the global one -//! → an LLVM key's lock → artifact. A compiler's or a sysroot's key lock is -//! taken with the worktree lock put down ([`Held::without_shared`]), because the +//! A compiler key's lock → a sysroot key's lock → a freestanding key's lock → the +//! worktree build lock → the global one → an LLVM key's lock → artifact. A +//! compiler's, a sysroot's or a freestanding key lock is taken with the +//! worktree lock put down ([`Held::without_shared`]), because the //! key's builder takes the worktree lock exclusively; an LLVM key's is taken //! inside the worktree or global lock covering the fork build directory its //! builder writes. @@ -215,12 +216,14 @@ pub fn artifact(root: &Path) -> Guard { exclusive(&root.join(LOCK_DIR).join("artifact"), "artifact lock", "artifact staging") } -/// A content-addressed product of the host, locked per key: a sysroot, a -/// compiler a worktree's fork checkout names (`src/compiler.rs`), or the LLVM -/// a compiler links (`src/llvm.rs`). +/// A content-addressed product of the host, locked per key: a sysroot, the +/// freestanding targets' libraries it carries (`src/sysroot.rs`), a compiler +/// a worktree's fork checkout names (`src/compiler.rs`), or the LLVM a +/// compiler links (`src/llvm.rs`). #[derive(Clone, Copy)] pub enum Keyed { Sysroot, + Freestanding, Compiler, Llvm, } @@ -229,6 +232,7 @@ impl Keyed { fn dir(self) -> &'static str { match self { Keyed::Sysroot => "sysroots", + Keyed::Freestanding => "freestanding", Keyed::Compiler => "compilers", Keyed::Llvm => "llvm", } @@ -237,6 +241,7 @@ impl Keyed { pub(crate) fn name(self) -> &'static str { match self { Keyed::Sysroot => "sysroot", + Keyed::Freestanding => "freestanding libraries", Keyed::Compiler => "compiler", Keyed::Llvm => "LLVM", } diff --git a/src/keystore.rs b/src/keystore.rs index 8f9dfdd8db1..149d1ee5651 100644 --- a/src/keystore.rs +++ b/src/keystore.rs @@ -23,6 +23,7 @@ use crate::sysroot::git_out; fn record_path(root: &Path, kind: Keyed) -> PathBuf { root.join(match kind { Keyed::Sysroot => "target/toyos-sysroot-key", + Keyed::Freestanding => "target/toyos-freestanding-key", Keyed::Compiler => "target/toyos-compiler-key", Keyed::Llvm => "target/toyos-llvm-key", }) diff --git a/src/sysroot.rs b/src/sysroot.rs index f2f5044afb5..40ae49440b4 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -12,6 +12,14 @@ //! different ABIs or different compilers never refuse or wait for each other, //! and main and every branch matching it share one copy. //! +//! **The kernel's and the loader's libraries compile none of those trees**, so +//! they are a key of their own ([`freestanding_key`]): the fork, the compiler, +//! and the manifests std's lockfile resolves. They are built once per that key +//! into `rust/build/freestanding//` and cloned into every sysroot naming +//! it, so an edit to the ABI builds ToyOS's std and nothing else, and a crate +//! built against a sysroot learns which targets' libraries moved +//! ([`Sysroot::identity`]). Each build refuses dep-info that says otherwise. +//! //! **Each worktree builds std in its own fork checkout, and nothing but the //! primary's own sync moves the primary's.** The primary builds in its `rust/`; //! a linked worktree in its own `rust/`, made on first need as a git worktree of @@ -25,15 +33,16 @@ //! //! Locks, in the one order every acquirer takes them: the compiler key's, if the //! compiler is a worktree's own; the sysroot key's (`buildlock::keyed_*`), with -//! this worktree's build lock put down; then, to build, this worktree's -//! exclusively (its fork build directory is written); then, if the compiler is -//! the primary's, the global one shared, because it is read; then the key of the -//! compiler's LLVM, held in use while the C++ runtime is built from its sources. +//! this worktree's build lock put down; then the freestanding key's, while its +//! libraries are cloned in; then, to build, this worktree's exclusively (its +//! fork build directory is written); then, if the compiler is the primary's, the +//! global one shared, because it is read; then the key of the compiler's LLVM, +//! held in use while the C++ runtime is built from its sources. //! -//! A sysroot no worktree names any more is removed by `keystore::sweep`, which -//! `--worktree remove` runs: each build records the key it used in its -//! worktree's `target/`, and a key no registered worktree records, that nobody -//! is making or using, goes. +//! A sysroot or freestanding libraries no worktree names any more are removed +//! by `keystore::sweep`, which `--worktree remove` runs: each build records the +//! keys it used in its worktree's `target/`, and a key no registered worktree +//! records, that nobody is making or using, goes. use std::collections::BTreeSet; use std::fs; @@ -57,22 +66,48 @@ pub const SYSROOT_SOURCES: [&str; 4] = pub(crate) const SYSROOT_MANIFESTS: [&str; 3] = ["toyos-abi/Cargo.toml", "toyos/Cargo.toml", "userland/libc/Cargo.toml"]; -/// The file a finished sysroot carries last, naming what it was built from. -/// A directory without it is a build that did not finish. +/// Of [`SYSROOT_MANIFESTS`], the ones std's lockfile resolves with the fork's +/// own: what of a worktree can move a freestanding target's dependency versions. +const STD_MANIFESTS: [&str; 2] = ["toyos-abi/Cargo.toml", "toyos/Cargo.toml"]; + +/// The file a finished sysroot or freestanding store carries last, naming what +/// it was built from. A directory without it is a build that did not finish. const SOURCES: &str = "SOURCES"; -/// What changes how a key's sources become a sysroot and is none of them: the -/// std build's recipe below. Moving it moves every key. -const RECIPE: &str = "bootstrap stage-0 local rebuild, profile compiler, no LLVM or Ninja, \ - libtoyos_c merged, libraries from the stamp, linked by rust-lld, \ - a C sysroot of libc's staticlib and headers per target, and its C++ runtime \ - built under n2 from the runtimes' sources of the compiler's LLVM; 8"; +/// What changes how a guest target's libraries are built and is none of their +/// sources. Moving it moves every key. +const STD_RECIPE: &str = "bootstrap stage-0 local rebuild, profile compiler, no LLVM or Ninja, \ + libraries from the stamp, linked by rust-lld; 1"; + +/// What changes how a key's sources become a sysroot beyond [`STD_RECIPE`]. +/// Moving it moves every sysroot key. +const RECIPE: &str = "libtoyos_c merged, a C sysroot of libc's staticlib and headers per target, \ + and its C++ runtime built under n2 from the runtimes' sources of the \ + compiler's LLVM, the freestanding libraries cloned from their key's; 9"; /// Every sysroot on this host. pub fn sysroots_dir(rust_dir: &Path) -> PathBuf { rust_dir.join("build/sysroots") } +/// Every freestanding target's libraries on this host, one directory per key. +pub fn freestanding_dir(rust_dir: &Path) -> PathBuf { + rust_dir.join("build/freestanding") +} + +/// Whether `target`'s libraries compile any of [`SYSROOT_SOURCES`]: ToyOS +/// userland's do, through std; the kernel's and the loader's are the fork's +/// alone, which is what lets them be a key of their own. +fn reads_worktree(target: &str) -> bool { + Arch::ALL.iter().any(|arch| arch.userland() == target) +} + +/// Of [`GUEST_TARGETS`], the ones whose libraries compile [`SYSROOT_SOURCES`] +/// (`true`), or the freestanding ones (`false`). +fn targets(reading: bool) -> Vec<&'static str> { + GUEST_TARGETS.into_iter().filter(|t| reads_worktree(t) == reading).collect() +} + /// A sysroot a build compiles against, held in use for as long as this lives. pub struct Sysroot { /// A toolchain directory: `RUSTUP_TOOLCHAIN` names it. @@ -80,15 +115,46 @@ pub struct Sysroot { /// Whether its compiler is the primary's, which the ToyOS-hosted rustc is /// built from. pub primary_compiler: bool, + compiler: String, + freestanding: String, + key: String, _using: Option, } impl Sysroot { /// A checkout whose toolchain arrived as an artifact has one sysroot, the /// artifact's, which `toolchain::check_installed_toolchain` has matched to - /// these sources. - pub(crate) fn installed(stage2: PathBuf) -> Self { - Self { dir: stage2, primary_compiler: true, _using: None } + /// these sources; `release`, the `TOOLCHAIN` it was published with, is the + /// identity of all of it. + pub(crate) fn installed(stage2: PathBuf, release: &str) -> Self { + let id = short(release.as_bytes()); + Self { dir: stage2, primary_compiler: true, compiler: id.clone(), freestanding: id.clone(), key: id, _using: None } + } + + /// A sysroot of these identities and no directory, for [`Sysroot::identity`]'s readers' tests. + #[cfg(test)] + pub(crate) fn of_parts(compiler: &str, freestanding: &str, key: &str) -> Self { + Self { + dir: PathBuf::new(), + primary_compiler: true, + compiler: compiler.into(), + freestanding: freestanding.into(), + key: key.into(), + _using: None, + } + } + + /// What a crate compiled against this sysroot compiles against beside its + /// own sources, a line each: `compiler `, which everything a crate's + /// build makes reads, its host half too; and ` ` per guest + /// target, which only what is made for that target reads. + pub fn identity(&self) -> String { + let mut lines = vec![format!("compiler {}", self.compiler)]; + for target in GUEST_TARGETS { + let libraries = if reads_worktree(target) { &self.key } else { &self.freestanding }; + lines.push(format!("{target} {libraries}")); + } + lines.join("\n") } } @@ -139,14 +205,17 @@ pub fn witness(root: &Path) -> String { lines.push(format!("{}:{}", rel.display(), short(&identity::of(&path, &data)))); } } - for manifest in SYSROOT_MANIFESTS { - let path = root.join(manifest); - let data = fs::read(&path).unwrap_or_else(|e| panic!("witness {}: {e}", path.display())); - lines.push(format!("{manifest}:{}", short(&data))); - } + lines.extend(SYSROOT_MANIFESTS.map(|manifest| manifest_line(root, manifest))); lines.join("\n") } +/// `manifest`'s witness line: its path and the hash of its bytes. +fn manifest_line(root: &Path, manifest: &str) -> String { + let path = root.join(manifest); + let data = fs::read(&path).unwrap_or_else(|e| panic!("witness {}: {e}", path.display())); + format!("{manifest}:{}", short(&data)) +} + /// The identity of the source files under `paths` of the git checkout `base`, /// as one hash. /// @@ -187,22 +256,31 @@ fn source_files(checkout: &Path, paths: &[&str], out: &mut Vec) { } } -/// The key of the sysroot `root` builds against with its std fork at `fork`, -/// compiled by `compiler`. -pub fn key(root: &Path, compiler: &Compiler, fork: &Path) -> String { +/// The key of the freestanding targets' libraries `root` builds against with +/// its std fork at `fork`, compiled by `compiler`: none of +/// [`SYSROOT_SOURCES`], and of `root` only [`STD_MANIFESTS`]. +pub fn freestanding_key(root: &Path, compiler: &Compiler, fork: &Path) -> String { let parts = [ - format!( - "{RECIPE}; cargo {STAGE0_CARGO}; targets {}; C++ runtime {:?}", - GUEST_TARGETS.join(" "), - crate::libcxx::OPTIONS - ), - witness(root), + format!("{STD_RECIPE}; cargo {STAGE0_CARGO}; targets {}", targets(false).join(" ")), + STD_MANIFESTS.map(|manifest| manifest_line(root, manifest)).join("\n"), tree_identity(fork, &["library", "src/bootstrap"]), compiler.identity(), ]; short(parts.join("\n\0\n").as_bytes()) } +/// The key of the sysroot `root` builds against, whose freestanding libraries +/// are `freestanding`'s ([`freestanding_key`], which names the fork, the +/// compiler and the std recipe the rest is built with too). +pub fn key(root: &Path, freestanding: &str) -> String { + let parts = [ + format!("{RECIPE}; targets {}; C++ runtime {:?}", targets(true).join(" "), crate::libcxx::OPTIONS), + witness(root), + freestanding.to_string(), + ]; + short(parts.join("\n\0\n").as_bytes()) +} + /// The commit this checkout's tree pins the std fork at: the index's, so a /// staged gitlink counts as the tree's. fn pinned_fork(root: &Path) -> String { @@ -320,28 +398,52 @@ fn held(root: &Path, key: &str, dir: &Path, make: impl FnMut()) -> Guard { pub fn ensure(root: &Path, rust_dir: &Path, lock: &mut Held) -> Sysroot { let fork = fork_checkout(root); let compiler = compiler::resolve(root, rust_dir, &fork, lock); - let key = key(root, &compiler, &fork); + let freestanding = freestanding_key(root, &compiler, &fork); + let key = key(root, &freestanding); let dir = sysroots_dir(rust_dir).join(&key); crate::keystore::record(root, Keyed::Sysroot, &key); + crate::keystore::record(root, Keyed::Freestanding, &freestanding); - let using = lock.without_shared(|| held(root, &key, &dir, || build(root, rust_dir, &compiler, &fork, &key, &dir))); - Sysroot { dir, primary_compiler: compiler.primary, _using: Some(using) } + let using = lock.without_shared(|| { + held(root, &key, &dir, || build(root, rust_dir, &compiler, &fork, &key, &freestanding, &dir)) + }); + let compiler_id = short(compiler.identity().as_bytes()); + Sysroot { dir, primary_compiler: compiler.primary, compiler: compiler_id, freestanding, key, _using: Some(using) } } /// Make the sysroot `key` names at `dir`, from `root`'s sources and the std fork -/// at `fork`, with `compiler`. The caller holds the key's lock. -fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &str, dir: &Path) { +/// at `fork`, with `compiler`, and the libraries the freestanding key +/// `freestanding` names. The caller holds the sysroot key's lock. +fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &str, freestanding: &str, dir: &Path) { + let store = freestanding_dir(rust_dir).join(freestanding); + let _freestanding = buildlock::keyed_made( + root, + Keyed::Freestanding, + freestanding, + || (!store.join(SOURCES).is_file()).then(|| format!("{} carries no {SOURCES}", store.display())), + || build_freestanding(root, compiler, fork, freestanding, &store), + ); let what = format!("building sysroot {key}"); let _worktree = buildlock::worktree_exclusive(root, &what); // Only the primary's compiler is rebuilt in place; one of a worktree's own // is written once and held in use by `compiler`. let _compiler = compiler.primary.then(|| buildlock::compiler_shared(root, &what)); - eprintln!("Building sysroot {key}: std from {}, the compiler {}", fork.display(), compiler.stage2.display()); + eprintln!( + "Building sysroot {key}: ToyOS's std from {}, the compiler {}, the freestanding libraries {freestanding}", + fork.display(), + compiler.stage2.display() + ); publish(compiler, dir, |partial| { - let built = build_std(root, compiler, fork); + let built = build_std(root, compiler, fork, &targets(true)); for target in GUEST_TARGETS { - place_std(&stamp(&built, target), &partial.join("lib/rustlib").join(target).join("lib")); + let lib = partial.join("lib/rustlib").join(target).join("lib"); + if reads_worktree(target) { + place_std(&stamp(&built, target), &lib); + } else { + remove(&lib); + clone_tree(&store.join(target), &lib); + } } let libc_target = dir.with_extension("libc-target"); for arch in Arch::ALL { @@ -358,7 +460,7 @@ fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &s } // The sources the key named are the ones built, or this is not that key's. - let again = self::key(root, compiler, fork); + let again = self::key(root, &freestanding_key(root, compiler, fork)); assert!( again == key, "the sources moved while sysroot {key} was being built (they are now {again}); \ @@ -368,6 +470,37 @@ fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &s }); } +/// Make the freestanding targets' libraries `key` names at `store`, from the std +/// fork at `fork` with `compiler`. The caller holds the key's lock. +fn build_freestanding(root: &Path, compiler: &Compiler, fork: &Path, key: &str, store: &Path) { + let what = format!("building the freestanding libraries {key}"); + let _worktree = buildlock::worktree_exclusive(root, &what); + let _compiler = compiler.primary.then(|| buildlock::compiler_shared(root, &what)); + let freestanding = targets(false); + eprintln!( + "Building the freestanding libraries {key}: {} from {}, the compiler {}", + freestanding.join(", "), + fork.display(), + compiler.stage2.display() + ); + let partial = store.with_extension("partial"); + remove(&partial); + let built = build_std(root, compiler, fork, &freestanding); + for target in freestanding { + place_std(&stamp(&built, target), &partial.join(target)); + } + let again = freestanding_key(root, compiler, fork); + assert!( + again == key, + "the sources moved while the freestanding libraries {key} were being built (they are now \ + {again}); nothing was kept, and the next build makes the ones they name" + ); + fs::write(partial.join(SOURCES), format!("{key}\nfork {}\n", fork.display())) + .unwrap_or_else(|e| panic!("write {}: {e}", partial.join(SOURCES).display())); + remove(store); + fs::rename(&partial, store).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", partial.display(), store.display())); +} + /// Put at `dir` a whole toolchain: `compiler`'s files and what `fill` adds to /// them, then the [`SOURCES`] `fill` returns, last. A `dir` already there is one /// [`unfinished`] refused, and it is replaced. A compiler that is not whole is @@ -397,39 +530,45 @@ fn publish(compiler: &Compiler, dir: &Path, fill: impl FnOnce(&Path) -> String) .unwrap_or_else(|e| panic!("rename {} -> {}: {e}", partial.display(), dir.display())); } -/// Compile the guest targets' libraries from `fork`'s `library/` with -/// `compiler`, and return the directory each target's is under. -fn build_std(root: &Path, compiler: &Compiler, fork: &Path) -> PathBuf { +/// Compile `targets`' libraries from `fork`'s `library/` with `compiler`, and +/// return the directory each target's is under. A target's dep-info names this +/// worktree's [`SYSROOT_SOURCES`] if it [`reads_worktree`], and none if not. +fn build_std(root: &Path, compiler: &Compiler, fork: &Path, targets: &[&str]) -> PathBuf { crate::ensure_submodule(fork, "library/backtrace"); let host = host_triple(); let build_dir = fork.join("build/toyos-std"); - prepare_std_build(&build_dir, &host, &compiler.identity()); + prepare_std_build(&build_dir, &host, &compiler.identity(), targets); let config = build_dir.join("bootstrap.toml"); fs::write(&config, std_config(&compiler.stage2, &bootstrap_cargo(), &build_dir, &host)) .unwrap_or_else(|e| panic!("write {}: {e}", config.display())); - let targets = GUEST_TARGETS.join(","); + let listed = targets.join(","); let args = ["build", "library", "--stage", "0", "--config", path_str(&config), "--warnings", "warn", - "--target", &targets]; + "--target", &listed]; let (ok, log) = toolchain::x_build(fork, &args, "std"); toolchain::refuse_on_compile_error(&log, "std"); assert!(ok, "the std build failed, and nothing in its output was a compile error"); - for arch in Arch::ALL { - toolchain::assert_std_built_from(root, &build_dir.join(&host).join("stage0-std").join(arch.userland())); + let built = build_dir.join(&host).join("stage0-std"); + for target in targets { + if reads_worktree(target) { + toolchain::assert_std_built_from(root, &built.join(target)); + } else { + toolchain::assert_std_reads_no_worktree(&built.join(target)); + } } - build_dir.join(&host).join("stage0-std") + built } -/// Ready the std build directory `build_dir` for a build by the compiler -/// `identity` names: nothing another compiler built, no LLVM, and no guest -/// target's std. -fn prepare_std_build(build_dir: &Path, host: &str, identity: &str) { +/// Ready the std build directory `build_dir` for a build of `targets` by the +/// compiler `identity` names: nothing another compiler built, no LLVM, and no +/// std of those targets. +fn prepare_std_build(build_dir: &Path, host: &str, identity: &str, targets: &[&str]) { fs::create_dir_all(build_dir).unwrap_or_else(|e| panic!("create {}: {e}", build_dir.display())); forget_another_compiler(build_dir, host, identity); crate::llvm::retire_in_tree(build_dir); // Bootstrap reuses what it built before and does not see a path dependency // outside the fork move, so each target's std starts from nothing. - for target in GUEST_TARGETS { + for target in targets { remove(&build_dir.join(host).join("stage0-std").join(target)); } } @@ -709,55 +848,108 @@ mod tests { (root, rust_dir, fork) } - /// **The key is the identity, and only the identity**: a comment in any tree - /// it reads — the ABI or the std fork — is the same sysroot, and a signature, - /// a line of the fork's code or another compiler is another. + /// **Each key is the identity of what it is built from, and only that**: a + /// comment in any tree they read moves neither; an edit to a tree only + /// ToyOS's std or the C sysroot compiles moves the sysroot and keeps its + /// freestanding libraries; a line of the fork's code, a manifest std's + /// lockfile resolves or another compiler moves both. #[test] fn a_comment_is_the_same_sysroot_and_a_signature_is_another() { let base = TempDir::new("key"); let (root, rust_dir, fork) = keyed(&base); - let k = || key(&root, &Compiler::primary(&rust_dir), &fork); - let base = k(); - assert_eq!(base.len(), 16, "{base}"); + let k = || { + let freestanding = freestanding_key(&root, &Compiler::primary(&rust_dir), &fork); + (key(&root, &freestanding), freestanding) + }; + let (sysroot, freestanding) = k(); + assert_eq!((sysroot.len(), freestanding.len()), (16, 16), "{sysroot} {freestanding}"); + let same = |what: &str| assert_eq!(k(), (sysroot.clone(), freestanding.clone()), "{what} moved a key"); + let sysroot_only = |what: &str| { + let (s, f) = k(); + assert_ne!(s, sysroot, "{what} kept the old sysroot"); + assert_eq!(f, freestanding, "{what} moved the freestanding libraries"); + }; + let both = |what: &str| { + let (s, f) = k(); + assert_ne!(s, sysroot, "{what} kept the old sysroot"); + assert_ne!(f, freestanding, "{what} kept the old freestanding libraries"); + }; let abi = root.join("toyos-abi/src/lib.rs"); write(&abi, "//! The crate.\n/// A, said better.\n// and a plain comment\npub struct A;\n"); - assert_eq!(k(), base, "a comment in toyos-abi made a new sysroot"); + same("a comment in toyos-abi"); write(&abi, "/// A.\npub struct A(pub u64);\n"); - assert_ne!(k(), base, "a signature change kept the old sysroot"); + sysroot_only("a signature change in toyos-abi"); write(&abi, "/// A.\npub struct A;\n"); - assert_eq!(k(), base); + same("toyos-abi as it was"); + + for tree in ["toyos/src", "userland/libc/src"] { + write(&root.join(tree).join("lib.rs"), "/// A.\npub struct A(u8);\n"); + sysroot_only(tree); + write(&root.join(tree).join("lib.rs"), "/// A.\npub struct A;\n"); + same(tree); + } let header = root.join("userland/libc/include/stdio.h"); write(&header, "int puts(const char *);\n"); - assert_ne!(k(), base, "a header the C sysroot carries kept the old sysroot"); + sysroot_only("a header the C sysroot carries"); fs::remove_file(&header).unwrap(); - assert_eq!(k(), base); + same("the C sysroot's headers as they were"); + + write(&root.join("userland/libc/Cargo.toml"), "[package]\nversion = \"0.2.0\"\n"); + sysroot_only("libc's manifest, which std's lockfile does not resolve,"); + write(&root.join("userland/libc/Cargo.toml"), "[package]\nversion = \"0.1.0\"\n"); + same("libc's manifest as it was"); let std = fork.join("library/std/src/lib.rs"); write(&std, "//! std, documented\npub fn exit() {}\n"); - assert_eq!(k(), base, "a comment in the std fork made a new sysroot"); + same("a comment in the std fork"); write(&std, "//! std\npub fn exit() { loop {} }\n"); - assert_ne!(k(), base, "a change to the fork's code kept the old sysroot"); + both("a change to the fork's code"); write(&std, "//! std\npub fn exit() {}\n"); - assert_eq!(k(), base); + same("the fork as it was"); // What a build and the desktop leave in the checkout is not its source. write(&fork.join("src/bootstrap/__pycache__/bootstrap.cpython-313.pyc"), "bytecode"); write(&fork.join("library/.DS_Store"), "finder"); - assert_eq!(k(), base, "a file git ignores moved the key"); + same("a file git ignores"); write(&fork.join("library/std/src/new.rs"), "pub fn new() {}\n"); - assert_ne!(k(), base, "an untracked source file was not in the key"); + both("an untracked source file"); fs::remove_file(fork.join("library/std/src/new.rs")).unwrap(); - assert_eq!(k(), base); + same("the fork without it"); - write(&root.join("toyos-abi/Cargo.toml"), "[package]\nversion = \"0.2.0\"\n"); - assert_ne!(k(), base, "a manifest change kept the old sysroot"); - write(&root.join("toyos-abi/Cargo.toml"), "[package]\nversion = \"0.1.0\"\n"); - assert_eq!(k(), base); + for manifest in STD_MANIFESTS { + write(&root.join(manifest), "[package]\nversion = \"0.2.0\"\n"); + both(manifest); + write(&root.join(manifest), "[package]\nversion = \"0.1.0\"\n"); + same(manifest); + } write(&rust_dir.join("build/toyos-compiler"), "tree-2"); - assert_ne!(k(), base, "another compiler kept the old sysroot"); + both("another compiler"); + } + + /// **The kernel's and the loader's targets are the freestanding ones, and + /// ToyOS userland's are the rest**: each guest target is one or the other, + /// and a sysroot's identity names the freestanding key for the first and + /// the sysroot key for the second. + #[test] + fn the_freestanding_targets_are_the_kernels_and_the_loaders() { + let mut freestanding: Vec<&str> = Arch::ALL.iter().flat_map(|arch| [arch.kernel(), arch.loader()]).collect(); + freestanding.sort_unstable(); + let mut said = targets(false); + said.sort_unstable(); + assert_eq!(said, freestanding); + assert_eq!(targets(true), Arch::ALL.map(Arch::userland)); + + let identity = Sysroot::of_parts("c", "f", "k").identity(); + let lines: Vec<&str> = identity.lines().collect(); + assert_eq!(lines[0], "compiler c"); + assert_eq!(lines.len(), 1 + GUEST_TARGETS.len()); + for target in GUEST_TARGETS { + let want = if freestanding.contains(&target) { "f" } else { "k" }; + assert!(lines.contains(&format!("{target} {want}").as_str()), "{identity}"); + } } /// **What one compiler compiled in a std build directory is never another's**: @@ -817,7 +1009,7 @@ mod tests { let build = base.join("toyos-std"); let host = host_triple(); let identity = Compiler::primary(&rust_dir).identity(); - prepare_std_build(&build, &host, &identity); + prepare_std_build(&build, &host, &identity, &GUEST_TARGETS); let llvm = [ build.join(&host).join("ci-llvm/lib/libLLVM.dylib"), build.join(&host).join("llvm/bin/llvm-config"), @@ -827,7 +1019,7 @@ mod tests { for file in llvm.iter().chain(&kept) { write(file, "built"); } - prepare_std_build(&build, &host, &identity); + prepare_std_build(&build, &host, &identity, &GUEST_TARGETS); for file in &llvm { assert!(!file.exists(), "{} outlived a std build's preparation", file.display()); } diff --git a/src/toolchain.rs b/src/toolchain.rs index 6f028083e9d..05aaebe7b83 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -80,10 +80,10 @@ const STD_SOURCES: [&str; 2] = ["toyos-abi/src", "toyos/src"]; /// /// One home for the list, because it is read four ways that must agree — the /// `stage1-std` cleans in [`full_bootstrap`], [`write_config`]'s bootstrap -/// `target` set, the libraries `src/sysroot.rs` builds and places, and -/// `src/build.rs`'s external fingerprint. A fifth spelling would silently leave -/// one of them building or fingerprinting a different set of targets than the -/// others. +/// `target` set, the libraries `src/sysroot.rs` builds and places, and the +/// identity it gives a crate's target directory. A fifth spelling would +/// silently leave one of them building or fingerprinting a different set of +/// targets than the others. pub const GUEST_TARGETS: [&str; 6] = [ Arch::X86_64.userland(), Arch::X86_64.kernel(), @@ -173,6 +173,21 @@ pub(crate) fn assert_std_built_from(root: &Path, dep_info: &Path) { ); } +/// Refuse a freestanding target's libraries whose dep-info under `dep_info` +/// names any `toyos-abi` or `toyos` source: their key names neither +/// (`src/sysroot.rs`), so a sysroot of another ABI would carry them unchanged. +pub(crate) fn assert_std_reads_no_worktree(dep_info: &Path) { + let sources = std_toyos_sources(dep_info); + assert!( + sources.is_empty(), + "the freestanding libraries under {} compiled {} toyos-abi or toyos sources, which their \ + key does not name:\n {}", + dep_info.display(), + sources.len(), + sources.join("\n "), + ); +} + /// What an installed toolchain's sysroot was built from, as its publisher /// recorded it (`src/release.rs`). fn witness_path(rust_dir: &Path) -> PathBuf { @@ -442,7 +457,11 @@ pub fn ensure(root: &Path, lock: &mut buildlock::Held) -> Sysroot { } Owner::Installed => { check_installed_toolchain(root, &rust_dir); - return Sysroot::installed(stage2(&rust_dir)); + let release = rust_dir.join("build/TOOLCHAIN"); + let release = fs::read_to_string(&release).unwrap_or_else(|e| { + panic!("{}: {e}; an installed toolchain carries the TOOLCHAIN it was published with", release.display()) + }); + return Sysroot::installed(stage2(&rust_dir), &release); } Owner::Us => {} } @@ -1239,6 +1258,23 @@ mod tests { ); } + /// **Freestanding libraries whose dep-info names the ABI are refused**: their + /// key does not name it, so a sysroot of another ABI would carry them. + #[test] + fn freestanding_libraries_that_read_the_abi_are_refused() { + let built = TempDir::new("freestanding-dep-info"); + let d = built.join("dist/build/core/1/core.d"); + fs::create_dir_all(d.parent().unwrap()).unwrap(); + fs::write(&d, "/x/rust/library/core/src/lib.rs: /x/rust/library/core/src/lib.rs\n").unwrap(); + assert_std_reads_no_worktree(&built); + + fs::write(built.join("dist/build/core/1/std.d"), "/x/toyos-abi/src/lib.rs:\n").unwrap(); + let refused = std::panic::catch_unwind(|| assert_std_reads_no_worktree(&built)) + .expect_err("freestanding libraries that compiled toyos-abi were taken"); + let said = refused.downcast_ref::().expect("a formatted refusal"); + assert!(said.contains("/x/toyos-abi/src/lib.rs"), "{said}"); + } + /// Verbatim from run `31370078581`, the run this check exists because of: /// the four warnings are real and were what the error had to be told apart /// from, and the allowance took it and reported a missing file 83 seconds diff --git a/src/worktree.rs b/src/worktree.rs index 47236ec82dd..ebc6be4e915 100644 --- a/src/worktree.rs +++ b/src/worktree.rs @@ -395,6 +395,7 @@ pub(crate) fn remove(root: &Path, path: &str) { let rust_dir = crate::toolchain::rust_dir(root); for (kind, store, what) in [ (Keyed::Sysroot, crate::sysroot::sysroots_dir(&rust_dir), "sysroot"), + (Keyed::Freestanding, crate::sysroot::freestanding_dir(&rust_dir), "freestanding libraries"), (Keyed::Compiler, crate::compiler::compilers_dir(&rust_dir), "compiler"), (Keyed::Llvm, crate::llvm::store(&rust_dir), "LLVM"), ] { @@ -457,7 +458,7 @@ fn remove_fork_checkout(root: &Path, at: &Path) { /// stops halfway — the `Directory not empty` git itself dies on. The removal /// runs again over what is left, at most [`PASSES`] times; a tree still refusing /// after that has a writer this cannot outrun, and the panic says so. -fn remove_tree(dir: &Path) { +pub(crate) fn remove_tree(dir: &Path) { for pass in 1..=PASSES { match fs::remove_dir_all(dir) { Ok(()) => return, From fd76860ff44dc5aaa5d2e51cab059ad571e238ff Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 1 Oct 2026 07:31:24 +0200 Subject: [PATCH 2/6] issues: the C++ runtime's scratch removal dies on a Finder file Found by this branch's `--build-only` gate on the merged tree: Finder wrote a `.DS_Store` into `.libcxx-x86_64` while `libcxx::build` removed it. That code is not this branch's, so it is filed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016t9wjdQkB8SH7bmfUoiy6L --- ...s-scratch-removal-dies-on-a-finder-file.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md diff --git a/issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md b/issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md new file mode 100644 index 00000000000..1d7a2187fa8 --- /dev/null +++ b/issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md @@ -0,0 +1,20 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# The C++ runtime's scratch removal dies on a Finder file + +`libcxx::build` ends with a plain `fs::remove_dir_all(scratch)`. On this host +Finder writes a `.DS_Store` into a directory while it is being emptied. A +`cargo run -- --build-only` on `wt/toyos-rebuild` died with `remove +…/rust/build/sysroots/d9ce291c409918f4.libcxx-x86_64: Directory not empty (os +error 66)`, and afterwards the directory held only that `.DS_Store`. The +sysroot was left at `.partial`, so the next build made it again. Every +other plain `remove_dir_all` of a build product meets the same writer: +`sysroot::publish`, `sysroot::remove` and `keystore::remove`. +`worktree::remove_tree` is the removal that outlasts it. + +**Exit**: every removal of a build product outlasts a writer that adds a +file while it runs, with a test that adds one. From 12bb289fa0a5d4812763845b1e71cecec58d26a7 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 1 Oct 2026 08:03:49 +0200 Subject: [PATCH 3/6] sysroot: one publish, a typed identity, and a key that reads std's configuration Answers the review of fd76860ff. - A fork edit is tested: `moved_libraries_take_what_was_built_for_them_and_a_moved_compiler_all` now has a kernel, a loader and a userland crate, and after the ABI edit an identity whose freestanding key moved too must leave all six targets stale and take the kernel's and the loader's builds, keeping every host half. - `publish(dir, fill)` is the one partial-then-rename protocol: the sysroot's fill clones the compiler, the freestanding store's places its libraries, and `unpublished` is the one `SOURCES` check, which `unfinished` and the freestanding key's lock both use. The whole-compiler refusal moves to the top of `build`, before the freestanding libraries, so "no std was built for one" is true of them too. Its partial, the replaced directory and the cloned-over freestanding `lib/` go through `worktree::remove_tree`. - The stamp is `sysroot::Identity`: rendered by `Display`, parsed by itself, and asked `stale(stamp)`; `Stale` moves beside it. A stamp that does not parse as one key per guest target is all stale, as before. - The identity's compiler is derived by `Identity::of` and tested: a driver rewritten in place leaves all of a crate stale. - Each guest target carries its role (`toolchain::GuestTarget`); which key its libraries are under is one exhaustive match on it, so the test that rebuilt the split from triples is gone. - The freestanding key reads `std_config` whole, with placeholder paths, and the two recipes are one `RECIPE` that moves every key. - Freestanding dep-info naming any file of the worktree outside its fork is refused, not only `toyos-abi` and `toyos` sources. Measured on this worktree's last std build: the freestanding targets' dep-info names 718 (`x86_64-unknown-none`) and 1291 (`x86_64-unknown-uefi`) absolute paths, none of them such a file; `x86_64-unknown-toyos`'s names 42 (17 under `toyos-abi/src`, 25 under `toyos/src`). - The C++ runtime issue's exit is the probe it was measured with; the cost of `core` and `alloc` being compiled again for every key is filed; the stale census, the `external_fingerprint` sentence and `src/CLAUDE.md`'s "a new key costs one std build of the three guest targets" are deleted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016t9wjdQkB8SH7bmfUoiy6L --- ...compiles-userlands-core-and-alloc-again.md | 25 ++ ...untime-is-rebuilt-for-every-sysroot-key.md | 11 +- ...s-scratch-removal-dies-on-a-finder-file.md | 4 +- ...chain-releases-report-the-same-rustc-vv.md | 3 +- src/CLAUDE.md | 2 +- src/build.rs | 113 +++--- src/sysroot.rs | 349 +++++++++++------- src/toolchain.rs | 136 ++++--- 8 files changed, 388 insertions(+), 255 deletions(-) create mode 100644 issues/build/every-sysroot-key-compiles-userlands-core-and-alloc-again.md diff --git a/issues/build/every-sysroot-key-compiles-userlands-core-and-alloc-again.md b/issues/build/every-sysroot-key-compiles-userlands-core-and-alloc-again.md new file mode 100644 index 00000000000..08e3238928d --- /dev/null +++ b/issues/build/every-sysroot-key-compiles-userlands-core-and-alloc-again.md @@ -0,0 +1,25 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# Every sysroot key compiles userland's `core` and `alloc` again + +`sysroot::build_std` empties each userland target's `stage0-std/` before it +builds (`prepare_std_build`), so every new sysroot key compiles `core`, `alloc` +and `compiler_builtins` for both userland targets from nothing, although they +read nothing of the worktree: in this branch's build of +`x86_64-unknown-toyos`, cargo's dep-info for them (`dist/libcore.d`, +`liballoc.d`, `libcompiler_builtins.d`) names 411, 714 and 630 paths, none in +the worktree outside its fork, where `libstd.d` names 42. The emptying rests +on the claim, in a comment there, that bootstrap does not see a path +dependency outside the fork move; nobody has measured a build that keeps the +directory. Measured on the `wt/toyos-rebuild` branch, one item added to +`toyos-abi/src/clock.rs`, from the timestamped build log: `core` and `alloc` +were the first 37 s of the 91 s `x86_64-unknown-toyos` std build, and 91 s of +the 158 s `aarch64-unknown-toyos` one, at a load that peaked at 186. + +**Exit**: one item added to `toyos-abi/src/clock.rs` after a build, and the +next `cargo run -- --build-only` prints no `Compiling core` and no +`Compiling alloc`. diff --git a/issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md b/issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md index 0b728aceed5..1fed43541a5 100644 --- a/issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md +++ b/issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md @@ -10,10 +10,9 @@ opened: 2026-10-01 targets into every new sysroot (`libcxx::build`), so every `toyos-abi` edit pays for them. Their inputs are the compiler's LLVM, `libcxx::OPTIONS` and the C sysroot they compile and link against: libc's headers and its staticlib. An -ABI edit moves only the staticlib, and an edit that leaves its bytes the same -moves none of them. Measured on the `wt/toyos-rebuild` branch, one item added -to `toyos-abi/src/clock.rs`: 27 s and 38 s at load 41, 47 s and 54 s at load 68. +ABI edit moves only the staticlib. Measured on the `wt/toyos-rebuild` branch, +one item added to `toyos-abi/src/clock.rs`: 27 s and 38 s at load 41, 47 s and +54 s at load 68. -**Exit**: the C++ runtime is keyed by what it is built from, the C sysroot's -bytes among them, and an ABI edit that leaves those bytes unchanged builds no -C++ runtime. +**Exit**: one item added to `toyos-abi/src/clock.rs` after a build, and the +next `cargo run -- --build-only` prints no `Building the C++ runtime`. diff --git a/issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md b/issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md index 1d7a2187fa8..78f53bac131 100644 --- a/issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md +++ b/issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md @@ -11,9 +11,7 @@ Finder writes a `.DS_Store` into a directory while it is being emptied. A `cargo run -- --build-only` on `wt/toyos-rebuild` died with `remove …/rust/build/sysroots/d9ce291c409918f4.libcxx-x86_64: Directory not empty (os error 66)`, and afterwards the directory held only that `.DS_Store`. The -sysroot was left at `.partial`, so the next build made it again. Every -other plain `remove_dir_all` of a build product meets the same writer: -`sysroot::publish`, `sysroot::remove` and `keystore::remove`. +sysroot was left at `.partial`, so the next build made it again. `worktree::remove_tree` is the removal that outlasts it. **Exit**: every removal of a build product outlasts a writer that adds a diff --git a/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md b/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md index 07c04589bfd..7f9caaad8cc 100644 --- a/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md +++ b/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md @@ -20,8 +20,7 @@ the new rustc refuses their metadata: `error[E0463]: can't find crate` for `cursor_icon`, `smol_str`, `raw_window_handle`, `keyboard_types` in winit-core. Recorded at the first external use of the `sdk-0.2.0` alias: Japabu/gbae run 33960322229 restored a target directory built by `5009ee0d7ea1cfad` and built -with `3845b80c8ba72421`. Inside this tree `src/build.rs`'s -`external_fingerprint` is the belt that hides it; a consumer has no belt. +with `3845b80c8ba72421`. Reproduce with any consumer: build with one release, swap the linked toolchain to another, build again in the same target directory. diff --git a/src/CLAUDE.md b/src/CLAUDE.md index b20e2caddc6..94c51792107 100644 --- a/src/CLAUDE.md +++ b/src/CLAUDE.md @@ -18,7 +18,7 @@ Loads when you read a file under `src/` — the root cargo project, package name ## The host's locks -- **Sysroots are content-addressed** (`src/sysroot.rs`): one per key — the identity (`src/identity.rs`, so a comment is no change) of `toyos-abi/src`, `toyos/src`, `userland/libc/src` and their manifests, the std fork's `rust/library/` and `rust/src/bootstrap/`, and the compiler — at `rust/build/sysroots//`, made by whichever worktree first needs it. Every build compiles against its own key's, so two worktrees with different ABIs never refuse or wait for each other; the only shared step is the primary's compiler, which a sysroot build reads under the global lock in shared mode. A new key costs one std build of the three guest targets; `--worktree remove` sweeps the keys no worktree records. +- **Sysroots are content-addressed** (`src/sysroot.rs`): one per key — the identity (`src/identity.rs`, so a comment is no change) of `toyos-abi/src`, `toyos/src`, `userland/libc/src` and their manifests, the std fork's `rust/library/` and `rust/src/bootstrap/`, and the compiler — at `rust/build/sysroots//`, made by whichever worktree first needs it. Every build compiles against its own key's, so two worktrees with different ABIs never refuse or wait for each other; the only shared step is the primary's compiler, which a sysroot build reads under the global lock in shared mode. `--worktree remove` sweeps the keys no worktree records. - **The std fork is built per worktree, and nothing but the primary's own sync moves the primary's `rust/`.** A linked worktree's `rust/` becomes, on its first build, a git worktree of the primary's fork repository at the commit its tree pins — that is where the fork is edited, committed and pinned. A worktree whose fork `compiler/` differs from the one the primary built builds its own compiler, keyed by that source and placed beside the primary's without touching it. If that checkout later falls behind the commit its tree pins (a merge moved the pin), the build moves the checkout to it itself, fetching from the primary's repository first if it holds the commit, unless the checkout has local changes, which it refuses to move out from under. - `src/buildlock.rs` serialises the stateful phases in two scopes: `Global` (the primary's compiler and the rustup link — one directory in `.git/`, shared by every worktree) and `Worktree` (the crate-target cleans, and this worktree's std build). Only `./x.py` typed by hand in `rust/` escapes it. - **Never kill a build that has taken the global lock** — the kill removes the shell wrappers, not the bootstrap, which inherits the file descriptor and runs on regardless; a toolchain rebuild interrupted or unobserved this way can leave `stage2/bin` without a `cargo`. diff --git a/src/build.rs b/src/build.rs index 42e5cc7b07c..59e358dc2fc 100644 --- a/src/build.rs +++ b/src/build.rs @@ -16,6 +16,7 @@ use crate::buildlock; use crate::flags; use crate::hostws; use crate::image; +use crate::sysroot::{Identity, Stale}; use crate::toolchain; thread_local! { @@ -173,21 +174,6 @@ fn parse_config(path: &Path) -> SystemConfig { // --- Freshness checking --- -/// What of a crate's target directory a sysroot's identity -/// (`sysroot::Sysroot::identity`) leaves stale against the one its -/// `.deps-stamp` records. Cargo keys nothing it reuses on the sysroot — every -/// ToyOS compiler prints one `rustc -vV` — so this is where a moved compiler or -/// library reaches a crate. -#[derive(Debug, PartialEq)] -enum Stale { - /// The compiler moved, or nothing records which one built it: all the guest - /// build wrote, its host half too. - All, - /// Only these guest targets' libraries moved: what was made for them, - /// `target//`, and nothing the compiler made for the host. - Targets(Vec), -} - /// How much of a crate's target directory goes when the compiler moves. #[derive(Clone, Copy)] enum Clean { @@ -202,21 +188,12 @@ enum Clean { ToyosOnly, } -fn stale(root: &Path, crate_dir: &Path, identity: &str) -> Option { +fn stale(root: &Path, crate_dir: &Path, identity: &Identity) -> Option { let stamp = hostws::target_dir(root, crate_dir).join(".deps-stamp"); - let Ok(stored) = fs::read_to_string(&stamp) else { return Some(Stale::All) }; - let parts = |text: &str| -> BTreeMap { - text.lines().filter_map(|l| l.split_once(' ')).map(|(part, id)| (part.into(), id.into())).collect() - }; - let (was, now) = (parts(&stored), parts(identity)); - if was.get("compiler") != now.get("compiler") { - return Some(Stale::All); - } - let moved: Vec = now.iter().filter(|(part, id)| was.get(*part) != Some(*id)).map(|(part, _)| part.clone()).collect(); - (!moved.is_empty()).then_some(Stale::Targets(moved)) + identity.stale(fs::read_to_string(&stamp).ok().as_deref()) } -fn clean(root: &Path, crate_dir: &Path, kind: Clean, stale: &Stale, identity: &str) { +fn clean(root: &Path, crate_dir: &Path, kind: Clean, stale: &Stale, identity: &Identity) { // Where cargo actually wrote it. `toyos-ld` is a member of the host // workspace, so its guest build lands in the root's `target/`. let target = hostws::target_dir(root, crate_dir); @@ -254,7 +231,7 @@ fn clean(root: &Path, crate_dir: &Path, kind: Clean, stale: &Stale, identity: &s fs::create_dir_all(&target).unwrap_or_else(|e| panic!("create {}: {e}", target.display())); let stamp = target.join(".deps-stamp"); - fs::write(&stamp, identity).unwrap_or_else(|e| panic!("write {}: {e}", stamp.display())); + fs::write(&stamp, identity.to_string()).unwrap_or_else(|e| panic!("write {}: {e}", stamp.display())); } /// Drop what in the target directories the sysroot's moved parts invalidated. @@ -268,7 +245,7 @@ fn clean(root: &Path, crate_dir: &Path, kind: Clean, stale: &Stale, identity: &s fn invalidate_stale( root: &Path, lock: &mut buildlock::Held, - identity: &str, + identity: &Identity, targets: &[(PathBuf, Clean)], ) { lock.act_if( @@ -407,8 +384,8 @@ struct GuestEnv { /// rustc is built from (`src/compiler.rs`). primary_compiler: bool, /// What its crates compile against beside their sources - /// (`sysroot::Sysroot::identity`). - identity: String, + /// (`sysroot::Identity`). + identity: Identity, /// The public key the loader and `/system/bin/update` embed /// (`signing::KEY_ENV`): every guest build carries it, so no crate that /// names it can be built without it. @@ -422,7 +399,7 @@ impl GuestEnv { Self { toolchain: sysroot.dir.clone(), primary_compiler: sysroot.primary_compiler, - identity: sysroot.identity(), + identity: sysroot.identity.clone(), image_key: crate::signing::key().public_hex(), floor_scope: crate::signing::key().floor_scope().word(), } @@ -2404,23 +2381,25 @@ mod tests { Arch::ALL.iter().map(|arch| file(&format!("target/{}/{PROFILE}/ld", arch.userland()))).collect(); guest.push(file(&format!("target/{PROFILE}/deps/libsyn-1.rlib"))); let host = file("target/debug/toyos-build"); + let identity = Identity::of_parts("compiler", "freestanding", "toyos"); - clean(&root, &root.join("ld"), Clean::ToyosOnly, &Stale::All, "fingerprint"); + clean(&root, &root.join("ld"), Clean::ToyosOnly, &Stale::All, &identity); for gone in &guest { assert!(!gone.exists(), "{} survived a clean of what the guest build wrote", gone.display()); } assert!(host.is_file(), "the host workspace's own build went"); - assert_eq!(fs::read_to_string(root.join("target/.deps-stamp")).unwrap(), "fingerprint"); + assert_eq!(fs::read_to_string(root.join("target/.deps-stamp")).unwrap(), identity.to_string()); } - /// **An ABI edit takes only what was built for ToyOS**: its sysroot moves - /// the userland targets' libraries and nothing else, so the kernel's target - /// directory stays whole, and of userland's only `target/` - /// goes — the host half the same compiler built stays. A moved compiler, - /// or a stamp that names none, is all of it. + /// **Moved libraries take what was built for their targets, and nothing + /// else**: an ABI edit moves the userland targets' libraries, so the + /// kernel's and the loader's target directories stay whole and of + /// userland's only `target/` goes; a fork edit moves every + /// target's, so the kernel's and the loader's builds go too. The host half + /// the same compiler built stays either way. A moved compiler, or a stamp + /// that names none, is all of it. #[test] fn moved_libraries_take_what_was_built_for_them_and_a_moved_compiler_all() { - use crate::sysroot::Sysroot; let root = toyos_tmpdir::TempDir::new("moved-libraries"); fs::write(root.join("Cargo.toml"), "[workspace]\nmembers = []\n").unwrap(); let file = |path: PathBuf| { @@ -2428,38 +2407,52 @@ mod tests { fs::write(&path, "built").unwrap(); path }; - let (kernel, userland) = (root.join("kernel"), root.join("userland")); + let (kernel, loader, userland) = (root.join("kernel"), root.join("bootloader"), root.join("userland")); + let crates = [&kernel, &loader, &userland]; let kernels = Arch::ALL.map(|arch| file(kernel.join(format!("target/{}/{PROFILE}/kernel", arch.kernel())))); - let kernel_host = file(kernel.join(format!("target/{PROFILE}/deps/libproc-1.dylib"))); + let loaders = Arch::ALL.map(|arch| file(loader.join(format!("target/{}/{PROFILE}/loader.efi", arch.loader())))); let programs = Arch::ALL.map(|arch| file(userland.join(format!("target/{}/{PROFILE}/init", arch.userland())))); - let userland_host = file(userland.join(format!("target/{PROFILE}/deps/libsyn-1.rlib"))); + let hosts = crates.map(|dir| file(dir.join(format!("target/{PROFILE}/deps/libproc-1.dylib")))); - let before = Sysroot::of_parts("compiler", "freestanding", "toyos").identity(); - for dir in [&kernel, &userland] { - fs::write(dir.join("target/.deps-stamp"), &before).unwrap(); + let before = Identity::of_parts("compiler", "freestanding", "toyos"); + for dir in crates { + fs::write(dir.join("target/.deps-stamp"), before.to_string()).unwrap(); assert_eq!(stale(&root, dir, &before), None, "{} was stale against its own stamp", dir.display()); } + let moved = |identity: &Identity, want: Vec<&'static str>| { + for dir in crates { + let found = stale(&root, dir, identity); + assert_eq!(found, Some(Stale::Targets(want.clone())), "{}", dir.display()); + clean(&root, dir, Clean::All, &found.unwrap(), identity); + assert_eq!(stale(&root, dir, identity), None, "{} was not stamped", dir.display()); + } + }; - let abi_edit = Sysroot::of_parts("compiler", "freestanding", "toyos, edited").identity(); - let mut toyos: Vec = Arch::ALL.map(|arch| arch.userland().to_string()).into(); + let mut toyos: Vec<&'static str> = Arch::ALL.map(Arch::userland).into(); toyos.sort(); - for dir in [&kernel, &userland] { - let found = stale(&root, dir, &abi_edit); - assert_eq!(found, Some(Stale::Targets(toyos.clone())), "{}", dir.display()); - clean(&root, dir, Clean::All, &found.unwrap(), &abi_edit); - assert_eq!(stale(&root, dir, &abi_edit), None, "{} was not stamped", dir.display()); - } - for kept in kernels.iter().chain([&kernel_host, &userland_host]) { + moved(&Identity::of_parts("compiler", "freestanding", "toyos, edited"), toyos); + for kept in kernels.iter().chain(&loaders).chain(&hosts) { assert!(kept.is_file(), "{} went, and nothing it was built from moved", kept.display()); } for gone in &programs { assert!(!gone.exists(), "{} survived its target's libraries moving", gone.display()); } - let compiler = Sysroot::of_parts("another compiler", "freestanding", "toyos, edited").identity(); + let mut all: Vec<&'static str> = Arch::ALL.into_iter().flat_map(|arch| [arch.userland(), arch.kernel(), arch.loader()]).collect(); + all.sort(); + let fork_edit = Identity::of_parts("compiler", "freestanding, edited", "toyos on the edited fork"); + moved(&fork_edit, all); + for gone in kernels.iter().chain(&loaders) { + assert!(!gone.exists(), "{} survived its target's libraries moving", gone.display()); + } + for kept in &hosts { + assert!(kept.is_file(), "{} went, and the compiler that built it did not move", kept.display()); + } + + let compiler = Identity::of_parts("another compiler", "freestanding, edited", "toyos on the edited fork"); assert_eq!(stale(&root, &kernel, &compiler), Some(Stale::All), "a moved compiler kept the host half"); fs::write(userland.join("target/.deps-stamp"), "sysroot:/a/stamp/naming/no/compiler").unwrap(); - assert_eq!(stale(&root, &userland, &abi_edit), Some(Stale::All), "a stamp naming no compiler was trusted"); + assert_eq!(stale(&root, &userland, &fork_edit), Some(Stale::All), "a stamp naming no compiler was trusted"); } /// **A stamp that cannot be written stops the build**: one left behind would @@ -2472,8 +2465,9 @@ mod tests { let target = root.join("kernel/target"); fs::create_dir_all(&target).unwrap(); fs::set_permissions(&target, fs::Permissions::from_mode(0o555)).unwrap(); + let identity = Identity::of_parts("compiler", "freestanding", "toyos"); let failed = std::panic::catch_unwind(|| { - clean(&root, &root.join("kernel"), Clean::All, &Stale::Targets(vec![]), "identity") + clean(&root, &root.join("kernel"), Clean::All, &Stale::Targets(vec![]), &identity) }); fs::set_permissions(&target, fs::Permissions::from_mode(0o755)).unwrap(); let refusal = failed.expect_err("a stamp that was not written was taken for written"); @@ -2492,7 +2486,8 @@ mod tests { fs::create_dir_all(kernel.join("target")).unwrap(); fs::write(kernel.join("Cargo.toml"), "[package\n").unwrap(); - let failed = std::panic::catch_unwind(|| clean(&root, &kernel, Clean::All, &Stale::All, "fingerprint")); + let identity = Identity::of_parts("compiler", "freestanding", "toyos"); + let failed = std::panic::catch_unwind(|| clean(&root, &kernel, Clean::All, &Stale::All, &identity)); let refusal = failed.expect_err("a cargo clean that failed was taken for one that ran"); let refusal = refusal.downcast_ref::().expect("a formatted panic"); assert!(refusal.starts_with(&format!("cargo clean in {} exited", kernel.display())), "{refusal}"); diff --git a/src/sysroot.rs b/src/sysroot.rs index e6f88d23299..ae0281412d1 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -16,9 +16,8 @@ //! they are a key of their own ([`freestanding_key`]): the fork, the compiler, //! and the manifests std's lockfile resolves. They are built once per that key //! into `rust/build/freestanding//` and cloned into every sysroot naming -//! it, so an edit to the ABI builds ToyOS's std and nothing else, and a crate -//! built against a sysroot learns which targets' libraries moved -//! ([`Sysroot::identity`]). Each build refuses dep-info that says otherwise. +//! it, and a crate built against a sysroot learns which targets' libraries +//! moved ([`Identity`]). Each build refuses dep-info that says otherwise. //! //! **Each worktree builds std in its own fork checkout, and nothing but the //! primary's own sync moves the primary's.** The primary builds in its `rust/`; @@ -44,7 +43,8 @@ //! keys it used in its worktree's `target/`, and a key no registered worktree //! records, that nobody is making or using, goes. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; @@ -56,7 +56,8 @@ use crate::buildlock::{self, Guard, Held, Keyed}; use crate::compiler::{self, Compiler}; use crate::identity; use crate::keystore::Key; -use crate::toolchain::{self, host_triple, Owner, GUEST_TARGETS}; +use crate::toolchain::{self, host_triple, GuestTarget, Owner, Role, GUEST_TARGETS}; +use crate::worktree::remove_tree; /// The per-worktree sources that end up inside a sysroot: std links `toyos-abi` /// and `toyos`, and `libtoyos_c.a` is `userland/libc`. @@ -71,20 +72,17 @@ pub(crate) const SYSROOT_MANIFESTS: [&str; 3] = /// own: what of a worktree can move a freestanding target's dependency versions. const STD_MANIFESTS: [&str; 2] = ["toyos-abi/Cargo.toml", "toyos/Cargo.toml"]; -/// The file a finished sysroot or freestanding store carries last, naming what -/// it was built from. A directory without it is a build that did not finish. +/// The file a directory [`publish`] made carries last, naming what it was +/// built from. A directory without it is a build that did not finish. const SOURCES: &str = "SOURCES"; -/// What changes how a guest target's libraries are built and is none of their -/// sources. Moving it moves every key. -const STD_RECIPE: &str = "bootstrap stage-0 local rebuild, profile compiler, no LLVM or Ninja, \ - libraries from the stamp, linked by rust-lld; 1"; - -/// What changes how a key's sources become a sysroot beyond [`STD_RECIPE`]. -/// Moving it moves every sysroot key. -const RECIPE: &str = "libtoyos_c merged, a C sysroot of libc's staticlib and headers per target, \ - and its C++ runtime built under n2 from the runtimes' sources of the \ - compiler's LLVM, the freestanding libraries cloned from their key's; 9"; +/// What changes how a key's sources become its libraries and its sysroot and +/// is none of them, nor std's configuration ([`std_config`]), which the +/// freestanding key reads whole. Moving it moves every key. +const RECIPE: &str = "bootstrap stage-0 local rebuild, libraries from the stamp, libtoyos_c merged, \ + a C sysroot of libc's staticlib and headers per target, and its C++ runtime \ + built under n2 from the runtimes' sources of the compiler's LLVM, the \ + freestanding libraries cloned from their key's; 10"; /// Every sysroot on this host. pub fn sysroots_dir(rust_dir: &Path) -> PathBuf { @@ -96,17 +94,29 @@ pub fn freestanding_dir(rust_dir: &Path) -> PathBuf { rust_dir.join("build/freestanding") } -/// Whether `target`'s libraries compile any of [`SYSROOT_SOURCES`]: ToyOS -/// userland's do, through std; the kernel's and the loader's are the fork's -/// alone, which is what lets them be a key of their own. -fn reads_worktree(target: &str) -> bool { - Arch::ALL.iter().any(|arch| arch.userland() == target) +/// Whose sources a guest target's libraries compile. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Libraries { + /// The fork's and [`SYSROOT_SOURCES`]: ToyOS userland's std, which names + /// `toyos-abi` and `toyos`. + Worktree, + /// The fork's alone: the kernel's `core` and `alloc` and the loader's std, + /// which is what lets them be a key of their own. + Freestanding, } -/// Of [`GUEST_TARGETS`], the ones whose libraries compile [`SYSROOT_SOURCES`] -/// (`true`), or the freestanding ones (`false`). -fn targets(reading: bool) -> Vec<&'static str> { - GUEST_TARGETS.into_iter().filter(|t| reads_worktree(t) == reading).collect() +impl Libraries { + fn of(target: GuestTarget) -> Self { + match target.role { + Role::Userland => Self::Worktree, + Role::Kernel | Role::Loader => Self::Freestanding, + } + } + + /// The triples of the [`GUEST_TARGETS`] whose libraries these are. + fn targets(self) -> Vec<&'static str> { + GUEST_TARGETS.into_iter().filter(|t| Self::of(*t) == self).map(GuestTarget::triple).collect() + } } /// A sysroot a build compiles against, held in use for as long as this lives. @@ -116,9 +126,7 @@ pub struct Sysroot { /// Whether its compiler is the primary's, which the ToyOS-hosted rustc is /// built from. pub primary_compiler: bool, - compiler: Key, - freestanding: Key, - key: Key, + pub identity: Identity, _using: Option, } @@ -129,33 +137,93 @@ impl Sysroot { /// identity of all of it. pub(crate) fn installed(stage2: PathBuf, release: &str) -> Self { let id = Key::of(release.as_bytes()); - Self { dir: stage2, primary_compiler: true, compiler: id.clone(), freestanding: id.clone(), key: id, _using: None } + Self { dir: stage2, primary_compiler: true, identity: Identity::new(id.clone(), &id, &id), _using: None } + } +} + +/// What a crate compiled against a sysroot compiles against beside its own +/// sources, as its target directory's `.deps-stamp` records it: the compiler, +/// which everything a crate's build makes reads, its host half too; and per +/// guest target the key of its libraries, which only what is made for that +/// target reads. +#[derive(Clone, Debug, PartialEq)] +pub struct Identity { + compiler: Key, + libraries: BTreeMap<&'static str, Key>, +} + +/// What of a crate's target directory a sysroot's [`Identity`] leaves stale. +/// Cargo keys nothing it reuses on the sysroot — every ToyOS compiler prints +/// one `rustc -vV` — so this is where a moved compiler or library reaches a +/// crate. +#[derive(Debug, PartialEq)] +pub enum Stale { + /// The compiler moved, or nothing records which one built it: all the guest + /// build wrote, its host half too. + All, + /// Only these guest targets' libraries moved: what was made for them, + /// `target//`, and nothing the compiler made for the host. + Targets(Vec<&'static str>), +} + +impl Identity { + /// The identity of a sysroot whose compiler is `compiler`, whose + /// freestanding targets' libraries are `freestanding`'s, and whose others + /// are `key`'s. + fn of(compiler: &Compiler, freestanding: &Key, key: &Key) -> Self { + Self::new(Key::of(compiler.identity().as_bytes()), freestanding, key) } - /// A sysroot of these identities and no directory, for [`Sysroot::identity`]'s readers' tests. + fn new(compiler: Key, freestanding: &Key, key: &Key) -> Self { + let libraries = GUEST_TARGETS.map(|target| match Libraries::of(target) { + Libraries::Worktree => (target.triple(), key.clone()), + Libraries::Freestanding => (target.triple(), freestanding.clone()), + }); + Self { compiler, libraries: libraries.into() } + } + + /// The identity of these parts' keys, for its readers' tests. #[cfg(test)] pub(crate) fn of_parts(compiler: &str, freestanding: &str, key: &str) -> Self { - Self { - dir: PathBuf::new(), - primary_compiler: true, - compiler: Key::of(compiler.as_bytes()), - freestanding: Key::of(freestanding.as_bytes()), - key: Key::of(key.as_bytes()), - _using: None, + let of = |part: &str| Key::of(part.as_bytes()); + Self::new(of(compiler), &of(freestanding), &of(key)) + } + + /// What this leaves stale of a target directory whose `.deps-stamp` says + /// `stamp`, if it has one: nothing, if it is this. + pub fn stale(&self, stamp: Option<&str>) -> Option { + let Some(was) = stamp.and_then(Self::parse) else { return Some(Stale::All) }; + if was.compiler != self.compiler { + return Some(Stale::All); } + let moved: Vec<&'static str> = self + .libraries + .iter() + .filter(|(target, key)| was.libraries.get(*target) != Some(key)) + .map(|(target, _)| *target) + .collect(); + (!moved.is_empty()).then_some(Stale::Targets(moved)) } - /// What a crate compiled against this sysroot compiles against beside its - /// own sources, a line each: `compiler `, which everything a crate's - /// build makes reads, its host half too; and ` ` per guest - /// target, which only what is made for that target reads. - pub fn identity(&self) -> String { - let mut lines = vec![format!("compiler {}", self.compiler)]; - for target in GUEST_TARGETS { - let libraries = if reads_worktree(target) { &self.key } else { &self.freestanding }; - lines.push(format!("{target} {libraries}")); + /// The identity `text` spells, if it names a key for each of the + /// [`GUEST_TARGETS`] and for nothing else. + fn parse(text: &str) -> Option { + let mut lines = text.lines(); + let compiler = Key::parse(lines.next()?.strip_prefix("compiler ")?)?; + let mut libraries = BTreeMap::new(); + for line in lines { + let (triple, key) = line.split_once(' ')?; + let target = GUEST_TARGETS.into_iter().find(|t| t.triple() == triple)?; + libraries.insert(target.triple(), Key::parse(key)?); } - lines.join("\n") + (libraries.len() == GUEST_TARGETS.len()).then_some(Self { compiler, libraries }) + } +} + +impl fmt::Display for Identity { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "compiler {}", self.compiler)?; + self.libraries.iter().try_for_each(|(triple, key)| write!(f, "\n{triple} {key}")) } } @@ -261,8 +329,10 @@ fn source_files(checkout: &Path, paths: &[&str], out: &mut Vec) { /// its std fork at `fork`, compiled by `compiler`: none of /// [`SYSROOT_SOURCES`], and of `root` only [`STD_MANIFESTS`]. pub fn freestanding_key(root: &Path, compiler: &Compiler, fork: &Path) -> Key { + let placeholder = Path::new(""); let parts = [ - format!("{STD_RECIPE}; cargo {STAGE0_CARGO}; targets {}", targets(false).join(" ")), + format!("{RECIPE}; cargo {STAGE0_CARGO}; targets {}", Libraries::Freestanding.targets().join(" ")), + std_config(placeholder, placeholder, placeholder, ""), STD_MANIFESTS.map(|manifest| manifest_line(root, manifest)).join("\n"), tree_identity(fork, &["library", "src/bootstrap"]), compiler.identity(), @@ -271,11 +341,11 @@ pub fn freestanding_key(root: &Path, compiler: &Compiler, fork: &Path) -> Key { } /// The key of the sysroot `root` builds against, whose freestanding libraries -/// are `freestanding`'s ([`freestanding_key`], which names the fork, the -/// compiler and the std recipe the rest is built with too). +/// are `freestanding`'s ([`freestanding_key`], which names the recipe, std's +/// configuration, the fork and the compiler the rest is built with too). pub fn key(root: &Path, freestanding: &Key) -> Key { let parts = [ - format!("{RECIPE}; targets {}; C++ runtime {:?}", targets(true).join(" "), crate::libcxx::OPTIONS), + format!("targets {}; C++ runtime {:?}", Libraries::Worktree.targets().join(" "), crate::libcxx::OPTIONS), witness(root), freestanding.to_string(), ]; @@ -376,16 +446,19 @@ pub fn fork_checkout(root: &Path) -> PathBuf { fork } -/// Why `dir` is not a finished sysroot, if it is not: no [`SOURCES`], or not a +/// Why `dir` is not a directory [`publish`] finished, if it is not: it carries +/// no [`SOURCES`]. +fn unpublished(dir: &Path) -> Option { + (!dir.join(SOURCES).is_file()).then(|| format!("{} carries no {SOURCES}", dir.display())) +} + +/// Why `dir` is not a finished sysroot, if it is not: [`unpublished`], or not a /// whole toolchain (`toolchain::toolchain_defect`). One found with the first and /// not the second is made again rather than trusted — all of it even when only /// its `bin/cargo` link dangles, because that is rare and a sysroot has no /// repair path. fn unfinished(dir: &Path) -> Option { - if !dir.join(SOURCES).is_file() { - return Some(format!("{} carries no {SOURCES}", dir.display())); - } - toolchain::toolchain_defect(dir) + unpublished(dir).or_else(|| toolchain::toolchain_defect(dir)) } /// The sysroot `key` names at `dir`, made by `make` if nobody has made it, and @@ -408,20 +481,35 @@ pub fn ensure(root: &Path, rust_dir: &Path, lock: &mut Held) -> Sysroot { let using = lock.without_shared(|| { held(root, &key, &dir, || build(root, rust_dir, &compiler, &fork, &key, &freestanding, &dir)) }); - let compiler_id = Key::of(compiler.identity().as_bytes()); - Sysroot { dir, primary_compiler: compiler.primary, compiler: compiler_id, freestanding, key, _using: Some(using) } + let identity = Identity::of(&compiler, &freestanding, &key); + Sysroot { dir, primary_compiler: compiler.primary, identity, _using: Some(using) } +} + +/// Refuse `compiler` unless it is a whole toolchain: no sysroot is made from +/// one that is not, and no std is built for one. +fn whole(compiler: &Compiler) { + if let Some(defect) = toolchain::toolchain_defect(&compiler.stage2) { + let fix = if compiler.primary { + "\nA bootstrap in the primary checkout was stopped before it finished: \ + `cargo run -- --build-only` there completes it." + } else { + "" + }; + panic!("no sysroot is made from {}, and no std was built for one: {defect}{fix}", compiler.stage2.display()); + } } /// Make the sysroot `key` names at `dir`, from `root`'s sources and the std fork /// at `fork`, with `compiler`, and the libraries the freestanding key /// `freestanding` names. The caller holds the sysroot key's lock. fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &Key, freestanding: &Key, dir: &Path) { + whole(compiler); let store = freestanding_dir(rust_dir).join(freestanding); let _freestanding = buildlock::keyed_made( root, Keyed::Freestanding, freestanding, - || (!store.join(SOURCES).is_file()).then(|| format!("{} carries no {SOURCES}", store.display())), + || unpublished(&store), || build_freestanding(root, compiler, fork, freestanding, &store), ); let what = format!("building sysroot {key}"); @@ -435,15 +523,18 @@ fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &K compiler.stage2.display() ); - publish(compiler, dir, |partial| { - let built = build_std(root, compiler, fork, &targets(true)); + publish(dir, |partial| { + clone_tree(&compiler.stage2, partial); + let built = build_std(root, compiler, fork, Libraries::Worktree); for target in GUEST_TARGETS { - let lib = partial.join("lib/rustlib").join(target).join("lib"); - if reads_worktree(target) { - place_std(&stamp(&built, target), &lib); - } else { - remove(&lib); - clone_tree(&store.join(target), &lib); + let triple = target.triple(); + let lib = partial.join("lib/rustlib").join(triple).join("lib"); + match Libraries::of(target) { + Libraries::Worktree => place_std(&stamp(&built, triple), &lib), + Libraries::Freestanding => { + remove_tree(&lib); + clone_tree(&store.join(triple), &lib); + } } } let libc_target = dir.with_extension("libc-target"); @@ -477,68 +568,50 @@ fn build_freestanding(root: &Path, compiler: &Compiler, fork: &Path, key: &Key, let what = format!("building the freestanding libraries {key}"); let _worktree = buildlock::worktree_exclusive(root, &what); let _compiler = compiler.primary.then(|| buildlock::compiler_shared(root, &what)); - let freestanding = targets(false); eprintln!( "Building the freestanding libraries {key}: {} from {}, the compiler {}", - freestanding.join(", "), + Libraries::Freestanding.targets().join(", "), fork.display(), compiler.stage2.display() ); - let partial = store.with_extension("partial"); - remove(&partial); - let built = build_std(root, compiler, fork, &freestanding); - for target in freestanding { - place_std(&stamp(&built, target), &partial.join(target)); - } - let again = freestanding_key(root, compiler, fork); - assert!( - again == *key, - "the sources moved while the freestanding libraries {key} were being built (they are now \ - {again}); nothing was kept, and the next build makes the ones they name" - ); - fs::write(partial.join(SOURCES), format!("{key}\nfork {}\n", fork.display())) - .unwrap_or_else(|e| panic!("write {}: {e}", partial.join(SOURCES).display())); - remove(store); - fs::rename(&partial, store).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", partial.display(), store.display())); + publish(store, |partial| { + let built = build_std(root, compiler, fork, Libraries::Freestanding); + for target in Libraries::Freestanding.targets() { + place_std(&stamp(&built, target), &partial.join(target)); + } + let again = freestanding_key(root, compiler, fork); + assert!( + again == *key, + "the sources moved while the freestanding libraries {key} were being built (they are \ + now {again}); nothing was kept, and the next build makes the ones they name" + ); + format!("{key}\nfork {}\n", fork.display()) + }); } -/// Put at `dir` a whole toolchain: `compiler`'s files and what `fill` adds to -/// them, then the [`SOURCES`] `fill` returns, last. A `dir` already there is one -/// [`unfinished`] refused, and it is replaced. A compiler that is not whole is -/// refused before `fill` runs, and nothing is published. -fn publish(compiler: &Compiler, dir: &Path, fill: impl FnOnce(&Path) -> String) { - if let Some(defect) = toolchain::toolchain_defect(&compiler.stage2) { - let fix = if compiler.primary { - "\nA bootstrap in the primary checkout was stopped before it finished: \ - `cargo run -- --build-only` there completes it." - } else { - "" - }; - panic!("no sysroot is made from {}, and no std was built for one: {defect}{fix}", compiler.stage2.display()); - } +/// Put at `dir` what `fill` makes at the path it is handed, then the +/// [`SOURCES`] it returns, last. A `dir` already there is one [`unpublished`] +/// or worse refused, and it is replaced. +fn publish(dir: &Path, fill: impl FnOnce(&Path) -> String) { let partial = dir.with_extension("partial"); - if partial.exists() { - fs::remove_dir_all(&partial).unwrap_or_else(|e| panic!("remove {}: {e}", partial.display())); - } - clone_tree(&compiler.stage2, &partial); + remove_tree(&partial); let sources = fill(&partial); fs::write(partial.join(SOURCES), sources) .unwrap_or_else(|e| panic!("write {}: {e}", partial.join(SOURCES).display())); - if dir.exists() { - fs::remove_dir_all(dir).unwrap_or_else(|e| panic!("remove {}: {e}", dir.display())); - } + remove_tree(dir); fs::rename(&partial, dir) .unwrap_or_else(|e| panic!("rename {} -> {}: {e}", partial.display(), dir.display())); } -/// Compile `targets`' libraries from `fork`'s `library/` with `compiler`, and -/// return the directory each target's is under. A target's dep-info names this -/// worktree's [`SYSROOT_SOURCES`] if it [`reads_worktree`], and none if not. -fn build_std(root: &Path, compiler: &Compiler, fork: &Path, targets: &[&str]) -> PathBuf { +/// Compile the libraries of `libraries`' targets from `fork`'s `library/` with +/// `compiler`, and return the directory each target's is under, once each +/// target's dep-info shows it read of `root` what `libraries` says. +fn build_std(root: &Path, compiler: &Compiler, fork: &Path, libraries: Libraries) -> PathBuf { crate::ensure_submodule(fork, "library/backtrace"); let host = host_triple(); let build_dir = fork.join("build/toyos-std"); - prepare_std_build(&build_dir, &host, &compiler.identity(), targets); + let targets = libraries.targets(); + prepare_std_build(&build_dir, &host, &compiler.identity(), &targets); let config = build_dir.join("bootstrap.toml"); fs::write(&config, std_config(&compiler.stage2, &bootstrap_cargo(), &build_dir, &host)) .unwrap_or_else(|e| panic!("write {}: {e}", config.display())); @@ -551,10 +624,9 @@ fn build_std(root: &Path, compiler: &Compiler, fork: &Path, targets: &[&str]) -> assert!(ok, "the std build failed, and nothing in its output was a compile error"); let built = build_dir.join(&host).join("stage0-std"); for target in targets { - if reads_worktree(target) { - toolchain::assert_std_built_from(root, &built.join(target)); - } else { - toolchain::assert_std_reads_no_worktree(&built.join(target)); + match libraries { + Libraries::Worktree => toolchain::assert_std_built_from(root, &built.join(target)), + Libraries::Freestanding => toolchain::assert_std_reads_no_worktree(root, fork, &built.join(target)), } } built @@ -675,7 +747,7 @@ fn place_std(stamp: &Path, lib: &Path) { /// and no Ninja, which bootstrap otherwise demands on `PATH` for the LLVM it does /// not build. fn std_config(compiler: &Path, cargo: &Path, build_dir: &Path, host: &str) -> String { - let targets = GUEST_TARGETS.iter().map(|t| format!("\"{t}\"")).collect::>().join(", "); + let targets = GUEST_TARGETS.map(|t| format!("\"{}\"", t.triple())).join(", "); let linker = toolchain::rust_lld(compiler); let userland: String = Arch::ALL .iter() @@ -930,27 +1002,19 @@ mod tests { both("another compiler"); } - /// **The kernel's and the loader's targets are the freestanding ones, and - /// ToyOS userland's are the rest**: each guest target is one or the other, - /// and a sysroot's identity names the freestanding key for the first and - /// the sysroot key for the second. + /// **A crate's compiler is the one that built it, rebuilt in place or + /// not**: the primary's is rebuilt where it stands, so a driver its rebuild + /// left leaves all of a crate stale, and the same driver none of it. #[test] - fn the_freestanding_targets_are_the_kernels_and_the_loaders() { - let mut freestanding: Vec<&str> = Arch::ALL.iter().flat_map(|arch| [arch.kernel(), arch.loader()]).collect(); - freestanding.sort_unstable(); - let mut said = targets(false); - said.sort_unstable(); - assert_eq!(said, freestanding); - assert_eq!(targets(true), Arch::ALL.map(Arch::userland)); - - let identity = Sysroot::of_parts("c", "f", "k").identity(); - let lines: Vec<&str> = identity.lines().collect(); - assert_eq!(lines[0], format!("compiler {}", Key::of(b"c"))); - assert_eq!(lines.len(), 1 + GUEST_TARGETS.len()); - for target in GUEST_TARGETS { - let want = Key::of(if freestanding.contains(&target) { b"f" } else { b"k" }); - assert!(lines.contains(&format!("{target} {want}").as_str()), "{identity}"); - } + fn a_compiler_rebuilt_in_place_leaves_all_of_a_crate_stale() { + let base = TempDir::new("identity-compiler"); + let (_root, rust_dir, _fork) = keyed(&base); + let (freestanding, key) = (Key::of(b"freestanding"), Key::of(b"key")); + let identity = || Identity::of(&Compiler::primary(&rust_dir), &freestanding, &key); + let stamp = identity().to_string(); + assert_eq!(identity().stale(Some(&stamp)), None, "a crate was stale against the compiler that built it"); + write(&toolchain::stage2(&rust_dir).join("lib/librustc_driver-1.dylib"), "a driver, rebuilt in place"); + assert_eq!(identity().stale(Some(&stamp)), Some(Stale::All), "a crate kept what the compiler made before its rebuild"); } /// **What one compiler compiled in a std build directory is never another's**: @@ -1010,7 +1074,7 @@ mod tests { let build = base.join("toyos-std"); let host = host_triple(); let identity = Compiler::primary(&rust_dir).identity(); - prepare_std_build(&build, &host, &identity, &GUEST_TARGETS); + prepare_std_build(&build, &host, &identity, &GUEST_TARGETS.map(GuestTarget::triple)); let llvm = [ build.join(&host).join("ci-llvm/lib/libLLVM.dylib"), build.join(&host).join("llvm/bin/llvm-config"), @@ -1020,7 +1084,7 @@ mod tests { for file in llvm.iter().chain(&kept) { write(file, "built"); } - prepare_std_build(&build, &host, &identity, &GUEST_TARGETS); + prepare_std_build(&build, &host, &identity, &GUEST_TARGETS.map(GuestTarget::triple)); for file in &llvm { assert!(!file.exists(), "{} outlived a std build's preparation", file.display()); } @@ -1188,7 +1252,9 @@ mod tests { let make = |dir: &Path, most: usize| { made.set(made.get() + 1); assert!(made.get() <= most, "a sysroot that was not whole was made again: make {}", made.get()); - publish(&compiler, dir, |partial| { + whole(&compiler); + publish(dir, |partial| { + clone_tree(&compiler.stage2, partial); write(&partial.join("lib/rustlib/x86_64-unknown-toyos/lib/libstd.rlib"), "std"); "found\n".to_string() }) @@ -1237,7 +1303,8 @@ mod tests { let said = refusal(|| { drop(held(&base, &key, &dir, || { once(); - publish(&compiler, &dir, |_| { + whole(&compiler); + publish(&dir, |_| { filled.set(true); "cloned\n".to_string() }) diff --git a/src/toolchain.rs b/src/toolchain.rs index 05aaebe7b83..33f608e0696 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -84,15 +84,40 @@ const STD_SOURCES: [&str; 2] = ["toyos-abi/src", "toyos/src"]; /// identity it gives a crate's target directory. A fifth spelling would /// silently leave one of them building or fingerprinting a different set of /// targets than the others. -pub const GUEST_TARGETS: [&str; 6] = [ - Arch::X86_64.userland(), - Arch::X86_64.kernel(), - Arch::X86_64.loader(), - Arch::Aarch64.userland(), - Arch::Aarch64.kernel(), - Arch::Aarch64.loader(), +pub const GUEST_TARGETS: [GuestTarget; 6] = [ + GuestTarget { arch: Arch::X86_64, role: Role::Userland }, + GuestTarget { arch: Arch::X86_64, role: Role::Kernel }, + GuestTarget { arch: Arch::X86_64, role: Role::Loader }, + GuestTarget { arch: Arch::Aarch64, role: Role::Userland }, + GuestTarget { arch: Arch::Aarch64, role: Role::Kernel }, + GuestTarget { arch: Arch::Aarch64, role: Role::Loader }, ]; +/// What a guest target's artifacts are. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Role { + Userland, + Kernel, + Loader, +} + +/// One of the [`GUEST_TARGETS`]. +#[derive(Clone, Copy, Debug)] +pub struct GuestTarget { + pub arch: Arch, + pub role: Role, +} + +impl GuestTarget { + pub const fn triple(self) -> &'static str { + match self.role { + Role::Userland => self.arch.userland(), + Role::Kernel => self.arch.kernel(), + Role::Loader => self.arch.loader(), + } + } +} + /// The one ToyOS the hosted rustc (`system.toml`'s `hosted-rustc`) is built to /// run on. pub const HOSTED_ARCH: Arch = Arch::X86_64; @@ -130,21 +155,22 @@ fn collect_dep_info(dir: &Path, out: &mut Vec) { } } +/// Every path one dep-info file names: what was built, and what it read. +fn paths_in_dep_info(text: &str) -> impl Iterator { + text.lines() + .filter(|line| !line.starts_with('#')) + .flat_map(str::split_ascii_whitespace) + .map(|word| word.strip_suffix(':').unwrap_or(word)) +} + /// The paths in one dep-info file that name a `toyos-abi/src` or `toyos/src` /// source. Split out from the filesystem so the gate below has a negative /// control that is a string literal. fn toyos_sources_in_dep_info(text: &str) -> Vec { - let mut out = Vec::new(); - for word in text.split_ascii_whitespace() { - let word = word.strip_suffix(':').unwrap_or(word); - if !word.ends_with(".rs") { - continue; - } - if STD_SOURCES.iter().any(|src| word.contains(&format!("/{src}/"))) { - out.push(word.to_string()); - } - } - out + paths_in_dep_info(text) + .filter(|path| path.ends_with(".rs") && STD_SOURCES.iter().any(|src| path.contains(&format!("/{src}/")))) + .map(str::to_string) + .collect() } /// Refuse a std whose dep-info under `dep_info` names another checkout's ABI. @@ -174,17 +200,30 @@ pub(crate) fn assert_std_built_from(root: &Path, dep_info: &Path) { } /// Refuse a freestanding target's libraries whose dep-info under `dep_info` -/// names any `toyos-abi` or `toyos` source: their key names neither -/// (`src/sysroot.rs`), so a sysroot of another ABI would carry them unchanged. -pub(crate) fn assert_std_reads_no_worktree(dep_info: &Path) { - let sources = std_toyos_sources(dep_info); +/// names any file of the worktree `root` outside its std fork `fork`: their key +/// reads nothing else of it but the manifests std's lockfile resolves +/// (`src/sysroot.rs`), so a sysroot of another worktree would carry them unchanged. +pub(crate) fn assert_std_reads_no_worktree(root: &Path, fork: &Path, dep_info: &Path) { + let real = |path: &Path| fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let (root, fork) = (real(root), real(fork)); + let mut deps = Vec::new(); + collect_dep_info(dep_info, &mut deps); + let mut read: Vec<&str> = deps + .iter() + .flat_map(|text| paths_in_dep_info(text)) + .filter(|path| Path::new(path).starts_with(&root) && !Path::new(path).starts_with(&fork)) + .collect(); + read.sort_unstable(); + read.dedup(); assert!( - sources.is_empty(), - "the freestanding libraries under {} compiled {} toyos-abi or toyos sources, which their \ - key does not name:\n {}", + read.is_empty(), + "the freestanding libraries under {} read {} files of the worktree {} outside its fork {}, \ + and their key reads none of its sources:\n {}", dep_info.display(), - sources.len(), - sources.join("\n "), + read.len(), + root.display(), + fork.display(), + read.join("\n "), ); } @@ -745,7 +784,7 @@ fn full_bootstrap(root: &Path, rust_dir: &Path, llvm: &Path) { // Clean cached std for all ToyOS targets so bootstrap picks up compiler changes // (e.g. target spec changes like default_uwtable that affect codegen). for target in GUEST_TARGETS { - let stage1_std = rust_dir.join(format!("build/{host}/stage1-std/{target}")); + let stage1_std = rust_dir.join(format!("build/{host}/stage1-std/{}", target.triple())); if stage1_std.exists() { fs::remove_dir_all(&stage1_std).ok(); } @@ -847,7 +886,7 @@ fn write_config(rust_dir: &Path, host: &str, with_hosted_rustc: bool, llvm: &Pat format!("host = [\"{host}\"]") }; let targets = std::iter::once(host) - .chain(GUEST_TARGETS) + .chain(GUEST_TARGETS.map(GuestTarget::triple)) .map(|t| format!("\"{t}\"")) .collect::>() .join(", "); @@ -1258,21 +1297,32 @@ mod tests { ); } - /// **Freestanding libraries whose dep-info names the ABI are refused**: their - /// key does not name it, so a sysroot of another ABI would carry them. + /// **Freestanding libraries whose dep-info names the worktree outside its + /// fork are refused**: their key reads none of it, so a sysroot of another + /// worktree would carry them. The fork's files and what lies outside the + /// worktree are what they are built from. #[test] - fn freestanding_libraries_that_read_the_abi_are_refused() { - let built = TempDir::new("freestanding-dep-info"); - let d = built.join("dist/build/core/1/core.d"); - fs::create_dir_all(d.parent().unwrap()).unwrap(); - fs::write(&d, "/x/rust/library/core/src/lib.rs: /x/rust/library/core/src/lib.rs\n").unwrap(); - assert_std_reads_no_worktree(&built); - - fs::write(built.join("dist/build/core/1/std.d"), "/x/toyos-abi/src/lib.rs:\n").unwrap(); - let refused = std::panic::catch_unwind(|| assert_std_reads_no_worktree(&built)) - .expect_err("freestanding libraries that compiled toyos-abi were taken"); - let said = refused.downcast_ref::().expect("a formatted refusal"); - assert!(said.contains("/x/toyos-abi/src/lib.rs"), "{said}"); + fn freestanding_libraries_that_read_the_worktree_are_refused() { + let base = TempDir::new("freestanding-dep-info"); + let root = fs::canonicalize(&*base).unwrap(); + let fork = root.join("rust"); + let built = fork.join("build/toyos-std/host/stage0-std/x86_64-unknown-none"); + let deps = built.join("dist/build/core/1"); + fs::create_dir_all(&deps).unwrap(); + let core = fork.join("library/core/src/lib.rs").display().to_string(); + let rlib = deps.join("libcore.rlib").display().to_string(); + let text = format!("{rlib}: {core} /registry/compiler_builtins/src/lib.rs\n\n{core}:\n# env-dep:CARGO_PKG_NAME=core\n"); + fs::write(deps.join("core.d"), text).unwrap(); + assert_std_reads_no_worktree(&root, &fork, &built); + + for read in ["toyos-abi/src/lib.rs", "userland/libc/src/lib.rs", "system.toml"] { + let read = root.join(read).display().to_string(); + fs::write(deps.join("other.d"), format!("{rlib}: {read}\n")).unwrap(); + let refused = std::panic::catch_unwind(|| assert_std_reads_no_worktree(&root, &fork, &built)) + .expect_err("freestanding libraries that read the worktree were taken"); + let said = refused.downcast_ref::().expect("a formatted refusal"); + assert!(said.contains(&read), "{said}"); + } } /// Verbatim from run `31370078581`, the run this check exists because of: From 34be209ebfad45448c9863c45a8f3e1364694bcd Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 1 Oct 2026 09:03:07 +0200 Subject: [PATCH 4/6] sysroot: a dep-info path is decided resolved, and the keys are derived once Round 2's review of #663 sent back two blockers, three notes and a removal. The freestanding refusal compared dep-info paths as written. A path through `..` (`/library/core/src/../../../../system.toml`, which is `/system.toml`) starts with the fork and passed, and a relative one started with neither and passed too. `toolchain::assert_std_reads_no_worktree` now resolves every path with `fs::canonicalize` before deciding anything, a relative one against the fork, and refuses one that resolves to nothing. The fork is the base because that is where bootstrap's cargo runs rustc: in this worktree's last std build, the six targets' dep-info names 1013 distinct relative paths, every one beginning `library/`, and all 1013 exist under the fork. A wrong base therefore names a missing file and is refused, rather than being taken for a file outside the worktree. The same check now refuses dep-info that names no source under the fork's `library/`, as `assert_std_built_from` refuses one naming no ABI source. Before, a target directory with no dep-info at all passed, because there was nothing in it to refuse. `ensure` passed the two library keys to the identity positionally, and no test reached that call: swapping them compiled and passed every test. `Keys::of` is now the one derivation of the freestanding key, the sysroot key and the identity. `ensure` and `build` use it, and so does the key test, which now also asserts what a crate stamped with the old identity finds stale. An ABI signature edit leaves the two `-toyos` targets stale, a fork code edit all six, and another compiler all of it. `freestanding_key` and `key` are private, so `Keys::of` is their only caller outside the tests. `freestanding_key` read `RECIPE` and `std_config` inline, so dropping the configuration from the key passed every test. `freestanding_key_of` takes both as parameters, as `llvm::key_of` does, and the key test moves each. Deleting `build`'s `whole(compiler)` passed every test, because the tests composed `whole` and `publish` themselves. `whole` now returns a `Whole`, and `publish_toolchain`, which clones the compiler and publishes, takes one. `build` and the tests both call it, so a `build` without `whole` no longer compiles. `whole` stays first in `build`, which keeps a compiler that is not whole from having freestanding libraries built for it. `GUEST_TARGETS`' census of its readers is deleted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016t9wjdQkB8SH7bmfUoiy6L --- src/sysroot.rs | 171 +++++++++++++++++++++++++++++++---------------- src/toolchain.rs | 107 +++++++++++++++++++---------- 2 files changed, 182 insertions(+), 96 deletions(-) diff --git a/src/sysroot.rs b/src/sysroot.rs index ae0281412d1..5a1b1693baa 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -167,13 +167,9 @@ pub enum Stale { } impl Identity { - /// The identity of a sysroot whose compiler is `compiler`, whose + /// The identity of a sysroot whose compiler's key is `compiler`, whose /// freestanding targets' libraries are `freestanding`'s, and whose others /// are `key`'s. - fn of(compiler: &Compiler, freestanding: &Key, key: &Key) -> Self { - Self::new(Key::of(compiler.identity().as_bytes()), freestanding, key) - } - fn new(compiler: Key, freestanding: &Key, key: &Key) -> Self { let libraries = GUEST_TARGETS.map(|target| match Libraries::of(target) { Libraries::Worktree => (target.triple(), key.clone()), @@ -325,14 +321,46 @@ fn source_files(checkout: &Path, paths: &[&str], out: &mut Vec) { } } +/// What a build of a worktree compiles against, by key. +#[derive(Debug, PartialEq)] +struct Keys { + /// Its freestanding targets' libraries' ([`freestanding_key`]). + freestanding: Key, + /// Its sysroot's ([`key`]). + sysroot: Key, + /// What a crate compiled against that sysroot records. + identity: Identity, +} + +impl Keys { + /// The keys of what `root` builds against with its std fork at `fork`, + /// compiled by `compiler`. + fn of(root: &Path, compiler: &Compiler, fork: &Path) -> Self { + let freestanding = freestanding_key(root, compiler, fork); + let sysroot = key(root, &freestanding); + let identity = Identity::new(Key::of(compiler.identity().as_bytes()), &freestanding, &sysroot); + Self { freestanding, sysroot, identity } + } +} + /// The key of the freestanding targets' libraries `root` builds against with /// its std fork at `fork`, compiled by `compiler`: none of /// [`SYSROOT_SOURCES`], and of `root` only [`STD_MANIFESTS`]. -pub fn freestanding_key(root: &Path, compiler: &Compiler, fork: &Path) -> Key { +fn freestanding_key(root: &Path, compiler: &Compiler, fork: &Path) -> Key { + freestanding_key_of(root, compiler, fork, RECIPE, &keyed_std_config()) +} + +/// [`std_config`] with no path of this host in it, as a key reads it. +fn keyed_std_config() -> String { let placeholder = Path::new(""); + std_config(placeholder, placeholder, placeholder, "") +} + +/// [`freestanding_key`], with the recipe and std's configuration it reads. +fn freestanding_key_of(root: &Path, compiler: &Compiler, fork: &Path, recipe: &str, config: &str) -> Key { let parts = [ - format!("{RECIPE}; cargo {STAGE0_CARGO}; targets {}", Libraries::Freestanding.targets().join(" ")), - std_config(placeholder, placeholder, placeholder, ""), + format!("{recipe}; cargo {STAGE0_CARGO}; targets {}", Libraries::Freestanding.targets().join(" ")), + config.to_string(), STD_MANIFESTS.map(|manifest| manifest_line(root, manifest)).join("\n"), tree_identity(fork, &["library", "src/bootstrap"]), compiler.identity(), @@ -343,7 +371,7 @@ pub fn freestanding_key(root: &Path, compiler: &Compiler, fork: &Path) -> Key { /// The key of the sysroot `root` builds against, whose freestanding libraries /// are `freestanding`'s ([`freestanding_key`], which names the recipe, std's /// configuration, the fork and the compiler the rest is built with too). -pub fn key(root: &Path, freestanding: &Key) -> Key { +fn key(root: &Path, freestanding: &Key) -> Key { let parts = [ format!("targets {}; C++ runtime {:?}", Libraries::Worktree.targets().join(" "), crate::libcxx::OPTIONS), witness(root), @@ -472,22 +500,23 @@ fn held(root: &Path, key: &Key, dir: &Path, make: impl FnMut()) -> Guard { pub fn ensure(root: &Path, rust_dir: &Path, lock: &mut Held) -> Sysroot { let fork = fork_checkout(root); let compiler = compiler::resolve(root, rust_dir, &fork, lock); - let freestanding = freestanding_key(root, &compiler, &fork); - let key = key(root, &freestanding); - let dir = sysroots_dir(rust_dir).join(&key); - crate::keystore::record(root, Keyed::Sysroot, &key); - crate::keystore::record(root, Keyed::Freestanding, &freestanding); - - let using = lock.without_shared(|| { - held(root, &key, &dir, || build(root, rust_dir, &compiler, &fork, &key, &freestanding, &dir)) - }); - let identity = Identity::of(&compiler, &freestanding, &key); - Sysroot { dir, primary_compiler: compiler.primary, identity, _using: Some(using) } + let keys = Keys::of(root, &compiler, &fork); + let dir = sysroots_dir(rust_dir).join(&keys.sysroot); + crate::keystore::record(root, Keyed::Sysroot, &keys.sysroot); + crate::keystore::record(root, Keyed::Freestanding, &keys.freestanding); + + let using = + lock.without_shared(|| held(root, &keys.sysroot, &dir, || build(root, rust_dir, &compiler, &fork, &keys, &dir))); + Sysroot { dir, primary_compiler: compiler.primary, identity: keys.identity, _using: Some(using) } } -/// Refuse `compiler` unless it is a whole toolchain: no sysroot is made from +/// A compiler [`whole`] found to be a whole toolchain: what a sysroot is made +/// from. +struct Whole<'a>(&'a Compiler); + +/// `compiler`, refused unless it is a whole toolchain: no sysroot is made from /// one that is not, and no std is built for one. -fn whole(compiler: &Compiler) { +fn whole(compiler: &Compiler) -> Whole<'_> { if let Some(defect) = toolchain::toolchain_defect(&compiler.stage2) { let fix = if compiler.primary { "\nA bootstrap in the primary checkout was stopped before it finished: \ @@ -497,34 +526,36 @@ fn whole(compiler: &Compiler) { }; panic!("no sysroot is made from {}, and no std was built for one: {defect}{fix}", compiler.stage2.display()); } + Whole(compiler) } -/// Make the sysroot `key` names at `dir`, from `root`'s sources and the std fork -/// at `fork`, with `compiler`, and the libraries the freestanding key -/// `freestanding` names. The caller holds the sysroot key's lock. -fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &Key, freestanding: &Key, dir: &Path) { - whole(compiler); - let store = freestanding_dir(rust_dir).join(freestanding); +/// Make the sysroot `keys` names at `dir`, from `root`'s sources and the std +/// fork at `fork`, with `compiler`, and the freestanding libraries `keys` +/// names. The caller holds the sysroot key's lock. +fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, keys: &Keys, dir: &Path) { + let made_from = whole(compiler); + let store = freestanding_dir(rust_dir).join(&keys.freestanding); let _freestanding = buildlock::keyed_made( root, Keyed::Freestanding, - freestanding, + &keys.freestanding, || unpublished(&store), - || build_freestanding(root, compiler, fork, freestanding, &store), + || build_freestanding(root, compiler, fork, &keys.freestanding, &store), ); - let what = format!("building sysroot {key}"); + let what = format!("building sysroot {}", keys.sysroot); let _worktree = buildlock::worktree_exclusive(root, &what); // Only the primary's compiler is rebuilt in place; one of a worktree's own // is written once and held in use by `compiler`. let _compiler = compiler.primary.then(|| buildlock::compiler_shared(root, &what)); eprintln!( - "Building sysroot {key}: ToyOS's std from {}, the compiler {}, the freestanding libraries {freestanding}", + "Building sysroot {}: ToyOS's std from {}, the compiler {}, the freestanding libraries {}", + keys.sysroot, fork.display(), - compiler.stage2.display() + compiler.stage2.display(), + keys.freestanding ); - publish(dir, |partial| { - clone_tree(&compiler.stage2, partial); + publish_toolchain(made_from, dir, |partial| { let built = build_std(root, compiler, fork, Libraries::Worktree); for target in GUEST_TARGETS { let triple = target.triple(); @@ -552,7 +583,7 @@ fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &K } // The sources the key named are the ones built, or this is not that key's. - let again = self::key(root, &freestanding_key(root, compiler, fork)); + let (key, again) = (&keys.sysroot, Keys::of(root, compiler, fork).sysroot); assert!( again == *key, "the sources moved while sysroot {key} was being built (they are now {again}); \ @@ -589,6 +620,15 @@ fn build_freestanding(root: &Path, compiler: &Compiler, fork: &Path, key: &Key, }); } +/// [`publish`] at `dir` a toolchain: `compiler`'s files, then what `fill` +/// adds to them. +fn publish_toolchain(compiler: Whole, dir: &Path, fill: impl FnOnce(&Path) -> String) { + publish(dir, |partial| { + clone_tree(&compiler.0.stage2, partial); + fill(partial) + }); +} + /// Put at `dir` what `fill` makes at the path it is handed, then the /// [`SOURCES`] it returns, last. A `dir` already there is one [`unpublished`] /// or worse refused, and it is replaced. @@ -925,27 +965,32 @@ mod tests { /// comment in any tree they read moves neither; an edit to a tree only /// ToyOS's std or the C sysroot compiles moves the sysroot and keeps its /// freestanding libraries; a line of the fork's code, a manifest std's - /// lockfile resolves or another compiler moves both. + /// lockfile resolves, the recipe, std's configuration or another compiler + /// moves both. A crate built against the old ones is stale in the targets + /// whose libraries moved, and in all of it when the compiler did. #[test] fn a_comment_is_the_same_sysroot_and_a_signature_is_another() { let base = TempDir::new("key"); let (root, rust_dir, fork) = keyed(&base); - let k = || { - let freestanding = freestanding_key(&root, &Compiler::primary(&rust_dir), &fork); - (key(&root, &freestanding), freestanding) - }; - let (sysroot, freestanding) = k(); - assert_eq!((sysroot.as_str().len(), freestanding.as_str().len()), (16, 16), "{sysroot} {freestanding}"); - let same = |what: &str| assert_eq!(k(), (sysroot.clone(), freestanding.clone()), "{what} moved a key"); + let k = || Keys::of(&root, &Compiler::primary(&rust_dir), &fork); + let was = k(); + assert_eq!((was.sysroot.as_str().len(), was.freestanding.as_str().len()), (16, 16), "{was:?}"); + let stamp = was.identity.to_string(); + let toyos = vec!["aarch64-unknown-toyos", "x86_64-unknown-toyos"]; + let mut all = GUEST_TARGETS.map(GuestTarget::triple).to_vec(); + all.sort(); + let same = |what: &str| assert_eq!(k(), was, "{what} moved a key"); let sysroot_only = |what: &str| { - let (s, f) = k(); - assert_ne!(s, sysroot, "{what} kept the old sysroot"); - assert_eq!(f, freestanding, "{what} moved the freestanding libraries"); + let now = k(); + assert_ne!(now.sysroot, was.sysroot, "{what} kept the old sysroot"); + assert_eq!(now.freestanding, was.freestanding, "{what} moved the freestanding libraries"); + assert_eq!(now.identity.stale(Some(&stamp)), Some(Stale::Targets(toyos.clone())), "{what}"); }; let both = |what: &str| { - let (s, f) = k(); - assert_ne!(s, sysroot, "{what} kept the old sysroot"); - assert_ne!(f, freestanding, "{what} kept the old freestanding libraries"); + let now = k(); + assert_ne!(now.sysroot, was.sysroot, "{what} kept the old sysroot"); + assert_ne!(now.freestanding, was.freestanding, "{what} kept the old freestanding libraries"); + assert_eq!(now.identity.stale(Some(&stamp)), Some(Stale::Targets(all.clone())), "{what}"); }; let abi = root.join("toyos-abi/src/lib.rs"); @@ -998,8 +1043,20 @@ mod tests { same(manifest); } + let compiler = Compiler::primary(&rust_dir); + let config = keyed_std_config(); + assert_eq!(freestanding_key_of(&root, &compiler, &fork, RECIPE, &config), was.freestanding); + for (what, moved) in [ + ("the recipe", freestanding_key_of(&root, &compiler, &fork, "another recipe", &config)), + ("std's configuration", freestanding_key_of(&root, &compiler, &fork, RECIPE, &format!("{config}\n[rust]\n"))), + ] { + assert_ne!(moved, was.freestanding, "{what} kept the old freestanding libraries"); + } + write(&rust_dir.join("build/toyos-compiler"), "tree-2"); - both("another compiler"); + let now = k(); + assert!(now.sysroot != was.sysroot && now.freestanding != was.freestanding, "another compiler kept a key: {now:?}"); + assert_eq!(now.identity.stale(Some(&stamp)), Some(Stale::All), "another compiler kept a crate's host half"); } /// **A crate's compiler is the one that built it, rebuilt in place or @@ -1008,9 +1065,8 @@ mod tests { #[test] fn a_compiler_rebuilt_in_place_leaves_all_of_a_crate_stale() { let base = TempDir::new("identity-compiler"); - let (_root, rust_dir, _fork) = keyed(&base); - let (freestanding, key) = (Key::of(b"freestanding"), Key::of(b"key")); - let identity = || Identity::of(&Compiler::primary(&rust_dir), &freestanding, &key); + let (root, rust_dir, fork) = keyed(&base); + let identity = || Keys::of(&root, &Compiler::primary(&rust_dir), &fork).identity; let stamp = identity().to_string(); assert_eq!(identity().stale(Some(&stamp)), None, "a crate was stale against the compiler that built it"); write(&toolchain::stage2(&rust_dir).join("lib/librustc_driver-1.dylib"), "a driver, rebuilt in place"); @@ -1252,9 +1308,7 @@ mod tests { let make = |dir: &Path, most: usize| { made.set(made.get() + 1); assert!(made.get() <= most, "a sysroot that was not whole was made again: make {}", made.get()); - whole(&compiler); - publish(dir, |partial| { - clone_tree(&compiler.stage2, partial); + publish_toolchain(whole(&compiler), dir, |partial| { write(&partial.join("lib/rustlib/x86_64-unknown-toyos/lib/libstd.rlib"), "std"); "found\n".to_string() }) @@ -1303,8 +1357,7 @@ mod tests { let said = refusal(|| { drop(held(&base, &key, &dir, || { once(); - whole(&compiler); - publish(&dir, |_| { + publish_toolchain(whole(&compiler), &dir, |_| { filled.set(true); "cloned\n".to_string() }) diff --git a/src/toolchain.rs b/src/toolchain.rs index 33f608e0696..4f8df3a4e5a 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; @@ -77,13 +78,6 @@ const STD_SOURCES: [&str; 2] = ["toyos-abi/src", "toyos/src"]; /// Every target a guest artifact is built for: ToyOS userland, the kernel's /// bare-metal target, and the UEFI bootloader. -/// -/// One home for the list, because it is read four ways that must agree — the -/// `stage1-std` cleans in [`full_bootstrap`], [`write_config`]'s bootstrap -/// `target` set, the libraries `src/sysroot.rs` builds and places, and the -/// identity it gives a crate's target directory. A fifth spelling would -/// silently leave one of them building or fingerprinting a different set of -/// targets than the others. pub const GUEST_TARGETS: [GuestTarget; 6] = [ GuestTarget { arch: Arch::X86_64, role: Role::Userland }, GuestTarget { arch: Arch::X86_64, role: Role::Kernel }, @@ -200,30 +194,43 @@ pub(crate) fn assert_std_built_from(root: &Path, dep_info: &Path) { } /// Refuse a freestanding target's libraries whose dep-info under `dep_info` -/// names any file of the worktree `root` outside its std fork `fork`: their key -/// reads nothing else of it but the manifests std's lockfile resolves -/// (`src/sysroot.rs`), so a sysroot of another worktree would carry them unchanged. +/// names no source of the std fork `fork`, or any file of the worktree `root` +/// outside it: their key reads nothing else of it but the manifests std's +/// lockfile resolves (`src/sysroot.rs`), so a sysroot of another worktree would +/// carry them unchanged. Every path is decided as the filesystem resolves it, +/// `..` and symlinks followed, and one that resolves to nothing is refused. pub(crate) fn assert_std_reads_no_worktree(root: &Path, fork: &Path, dep_info: &Path) { - let real = |path: &Path| fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); - let (root, fork) = (real(root), real(fork)); + let resolve = |path: &Path| { + fs::canonicalize(path) + .unwrap_or_else(|e| panic!("resolve {}, checking the std build under {}: {e}", path.display(), dep_info.display())) + }; + let (root, fork) = (resolve(root), resolve(fork)); let mut deps = Vec::new(); collect_dep_info(dep_info, &mut deps); - let mut read: Vec<&str> = deps + // Bootstrap's cargo runs rustc in the fork, so a relative path is the fork's. + let read: BTreeSet = + deps.iter().flat_map(|text| paths_in_dep_info(text)).map(|path| resolve(&fork.join(path))).collect(); + assert!( + read.iter().any(|path| path.starts_with(fork.join("library"))), + "the freestanding libraries under {} name no source of the fork {} at all, so the check that \ + they read nothing else of the worktree cannot answer. Cargo's dep-info moved.", + dep_info.display(), + fork.display(), + ); + let worktree: Vec = read .iter() - .flat_map(|text| paths_in_dep_info(text)) - .filter(|path| Path::new(path).starts_with(&root) && !Path::new(path).starts_with(&fork)) + .filter(|path| path.starts_with(&root) && !path.starts_with(&fork)) + .map(|path| path.display().to_string()) .collect(); - read.sort_unstable(); - read.dedup(); assert!( - read.is_empty(), + worktree.is_empty(), "the freestanding libraries under {} read {} files of the worktree {} outside its fork {}, \ and their key reads none of its sources:\n {}", dep_info.display(), - read.len(), + worktree.len(), root.display(), fork.display(), - read.join("\n "), + worktree.join("\n "), ); } @@ -1298,31 +1305,57 @@ mod tests { } /// **Freestanding libraries whose dep-info names the worktree outside its - /// fork are refused**: their key reads none of it, so a sysroot of another - /// worktree would carry them. The fork's files and what lies outside the - /// worktree are what they are built from. + /// fork are refused, however the path is spelt**: their key reads none of + /// it, so a sysroot of another worktree would carry them. The fork's files + /// and what lies outside the worktree are what they are built from. + /// Dep-info that names no source of the fork, or a file that is not there, + /// cannot be checked and is refused too. #[test] fn freestanding_libraries_that_read_the_worktree_are_refused() { - let base = TempDir::new("freestanding-dep-info"); - let root = fs::canonicalize(&*base).unwrap(); + let temp = TempDir::new("freestanding-dep-info"); + let base = fs::canonicalize(&*temp).unwrap(); + let (root, registry) = (base.join("worktree"), base.join("registry/compiler_builtins/src/lib.rs")); let fork = root.join("rust"); let built = fork.join("build/toyos-std/host/stage0-std/x86_64-unknown-none"); - let deps = built.join("dist/build/core/1"); - fs::create_dir_all(&deps).unwrap(); - let core = fork.join("library/core/src/lib.rs").display().to_string(); - let rlib = deps.join("libcore.rlib").display().to_string(); - let text = format!("{rlib}: {core} /registry/compiler_builtins/src/lib.rs\n\n{core}:\n# env-dep:CARGO_PKG_NAME=core\n"); - fs::write(deps.join("core.d"), text).unwrap(); + let out = built.join("dist/build/core/1/out"); + let file = |path: &Path| { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "read").unwrap(); + path.display().to_string() + }; + file(&fork.join("library/core/src/lib.rs")); + file(&fork.join("library/stdarch/crates/core_arch/src/mod.rs")); + let (registry, rlib, raw) = (file(®istry), file(&built.join("dist/libcore.rlib")), file(&out.join("libcore-1.rlib"))); + // Cargo's dep-info names a file whole; rustc's, a fork source by its path in the fork. + let arch = "library/core/src/../../stdarch/crates/core_arch/src/mod.rs"; + fs::write(built.join("dist/libcore.d"), format!("{rlib}: {}/{arch} {registry}\n", fork.display())).unwrap(); + let rustc = format!("{raw}: library/core/src/lib.rs {arch}\n\nlibrary/core/src/lib.rs:\n# env-dep:CARGO_PKG_NAME=core\n"); + fs::write(out.join("core-1.d"), rustc).unwrap(); assert_std_reads_no_worktree(&root, &fork, &built); - for read in ["toyos-abi/src/lib.rs", "userland/libc/src/lib.rs", "system.toml"] { - let read = root.join(read).display().to_string(); - fs::write(deps.join("other.d"), format!("{rlib}: {read}\n")).unwrap(); - let refused = std::panic::catch_unwind(|| assert_std_reads_no_worktree(&root, &fork, &built)) - .expect_err("freestanding libraries that read the worktree were taken"); - let said = refused.downcast_ref::().expect("a formatted refusal"); + let refusal = |dep_info: &Path| { + let refused = std::panic::catch_unwind(|| assert_std_reads_no_worktree(&root, &fork, dep_info)); + *refused.expect_err("unchecked dep-info was taken").downcast::().expect("a formatted refusal") + }; + let naming = |named: &str| { + fs::write(built.join("dist/other.d"), format!("{rlib}: {named}\n")).unwrap(); + refusal(&built) + }; + let system = file(&root.join("system.toml")); + for read in [file(&root.join("toyos-abi/src/lib.rs")), file(&root.join("userland/libc/src/lib.rs")), system.clone()] { + let said = naming(&read); assert!(said.contains(&read), "{said}"); } + let dotted = "library/core/src/../../../../system.toml"; + for spelt in [format!("{}/{dotted}", fork.display()), dotted.to_string()] { + let said = naming(&spelt); + assert!(said.contains(&system), "{spelt}: {said}"); + } + let said = naming("library/core/src/gone.rs"); + assert!(said.starts_with(&format!("resolve {}/library/core/src/gone.rs", fork.display())), "{said}"); + + let said = refusal(&built.join("none")); + assert!(said.contains("name no source of the fork"), "{said}"); } /// Verbatim from run `31370078581`, the run this check exists because of: From c10a1a75e89bc5c166d5e0e266a6686511446f44 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 1 Oct 2026 09:06:01 +0200 Subject: [PATCH 5/6] toolchain: the dep-info refusal test names the dep-info it took A refusal that does not fire now fails the test naming the dep-info it took, not `unchecked dep-info was taken: ()`. Under the round-2 mutations, that message was all that said which spelling had passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016t9wjdQkB8SH7bmfUoiy6L --- src/toolchain.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/toolchain.rs b/src/toolchain.rs index 4f8df3a4e5a..eca2f4739fe 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -1333,13 +1333,13 @@ mod tests { fs::write(out.join("core-1.d"), rustc).unwrap(); assert_std_reads_no_worktree(&root, &fork, &built); - let refusal = |dep_info: &Path| { + let refusal = |dep_info: &Path, what: &str| { let refused = std::panic::catch_unwind(|| assert_std_reads_no_worktree(&root, &fork, dep_info)); - *refused.expect_err("unchecked dep-info was taken").downcast::().expect("a formatted refusal") + *refused.err().unwrap_or_else(|| panic!("{what} was taken")).downcast::().expect("a formatted refusal") }; let naming = |named: &str| { fs::write(built.join("dist/other.d"), format!("{rlib}: {named}\n")).unwrap(); - refusal(&built) + refusal(&built, &format!("dep-info naming {named}")) }; let system = file(&root.join("system.toml")); for read in [file(&root.join("toyos-abi/src/lib.rs")), file(&root.join("userland/libc/src/lib.rs")), system.clone()] { @@ -1354,7 +1354,7 @@ mod tests { let said = naming("library/core/src/gone.rs"); assert!(said.starts_with(&format!("resolve {}/library/core/src/gone.rs", fork.display())), "{said}"); - let said = refusal(&built.join("none")); + let said = refusal(&built.join("none"), "a target directory without dep-info"); assert!(said.contains("name no source of the fork"), "{said}"); } From b2a387f911b305293166606446f29d85334b7c81 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 1 Oct 2026 09:51:01 +0200 Subject: [PATCH 6/6] sysroot: a symbolic link in the fork is refused, unreadable dep-info is refused, and only `whole` makes a `Whole` The freestanding key read the fork's `library/` and `src/bootstrap/` through `tree_identity`, whose file listing kept only what `symlink_metadata` called a file, so a symbolic link there was in no key: retargeting one kept the old freestanding libraries. `tree_identity` now takes what to make of a link. The freestanding key refuses one by name. `git -C rust ls-files -s library src/bootstrap` lists no entry of mode 120000 at fork commit aca5f527f, and the backtrace submodule none either, so the key of today's tree does not move. The compiler key cannot refuse a link: `src/tools` holds 5 (two LICENSE pairs and rust-analyzer's AGENTS.md), so a refusal there refuses every build of a worktree's own compiler. Hashing a link's target text instead would move every such compiler's key once, a compiler rebuild for each of the three on this host. It keeps skipping links, now by name (`Links::Skipped`), and `issues/build/a-compiler-key-reads-no-symbolic-link.md` records it, to go with the next compiler RECIPE bump, which moves those keys anyway. The dep-info check resolves a path through a link with `fs::canonicalize`; its test now names a link in the fork that points at the worktree's `system.toml`, which a lexical `..` fold would take. `collect_dep_info` skipped a directory entry it could not read and a `.d` that did not read as UTF-8, so that file's paths were never decided. Both panic now; a missing top directory is still empty, which both callers refuse. `Whole` lives in a private module whose only constructor is `whole`, so `Whole(compiler)` anywhere else in `sysroot.rs` does not compile. The key test asserts the freestanding key moves when the configuration it is handed is empty, so a `keyed_std_config` that returns nothing fails it. The module header no longer lists the freestanding key's parts. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016t9wjdQkB8SH7bmfUoiy6L --- .../a-compiler-key-reads-no-symbolic-link.md | 22 +++++ src/compiler.rs | 4 +- src/sysroot.rs | 99 ++++++++++++++----- src/toolchain.rs | 56 +++++++++-- 4 files changed, 145 insertions(+), 36 deletions(-) create mode 100644 issues/build/a-compiler-key-reads-no-symbolic-link.md diff --git a/issues/build/a-compiler-key-reads-no-symbolic-link.md b/issues/build/a-compiler-key-reads-no-symbolic-link.md new file mode 100644 index 00000000000..79076367f20 --- /dev/null +++ b/issues/build/a-compiler-key-reads-no-symbolic-link.md @@ -0,0 +1,22 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# A compiler key reads no symbolic link + +`compiler::key` reads `compiler/`, `src/tools/`, `src/stage0` and +`Cargo.lock` through `sysroot::tree_identity` with `Links::Skipped`, so a +symbolic link there is in no key: retargeting one, or editing what it names +outside those trees, keeps the old compiler. Refusing a link there, as the +freestanding key does, refuses every compiler build: +`git -C rust ls-files -s compiler src/tools src/stage0 Cargo.lock` at fork +commit `aca5f527f` lists 5 entries of mode `120000`, all under `src/tools` +(clippy's and rust-analyzer's `LICENSE-APACHE` and `LICENSE-MIT`, and +rust-analyzer's `AGENTS.md`). Hashing a link's target text instead moves the +key of every compiler of a worktree's own once. + +**Exit**: `Links::Skipped` is deleted and the compiler key hashes a link's +target text, landed with the next change to `compiler.rs`'s `RECIPE`, which +moves every compiler key anyway. diff --git a/src/compiler.rs b/src/compiler.rs index 804d788479c..5f2e9ad511c 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -48,7 +48,7 @@ use std::path::{Path, PathBuf}; use crate::buildlock::{self, Guard, Held, Keyed}; use crate::keystore::{self, Key}; -use crate::sysroot::{clone_tree, git_bytes, git_out, short, tree_identity}; +use crate::sysroot::{clone_tree, git_bytes, git_out, short, tree_identity, Links}; use crate::toolchain::{self, host_triple}; /// What changes how a key's sources become a compiler and is none of them: the @@ -202,7 +202,7 @@ pub fn key(fork: &Path) -> Key { /// [`key`], with the key of the LLVM `fork` names. fn key_with(fork: &Path, llvm: &Key) -> Key { - let parts = [RECIPE, &tree_identity(fork, &KEYED), llvm.as_str()]; + let parts = [RECIPE, &tree_identity(fork, &KEYED, Links::Skipped), llvm.as_str()]; Key::of(parts.join("\n\0\n").as_bytes()) } diff --git a/src/sysroot.rs b/src/sysroot.rs index 5a1b1693baa..868929b31bd 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -13,8 +13,7 @@ //! and main and every branch matching it share one copy. //! //! **The kernel's and the loader's libraries compile none of those trees**, so -//! they are a key of their own ([`freestanding_key`]): the fork, the compiler, -//! and the manifests std's lockfile resolves. They are built once per that key +//! they are a key of their own ([`freestanding_key`]). They are built once per that key //! into `rust/build/freestanding//` and cloned into every sysroot naming //! it, and a crate built against a sysroot learns which targets' libraries //! moved ([`Identity`]). Each build refuses dep-info that says otherwise. @@ -58,6 +57,7 @@ use crate::identity; use crate::keystore::Key; use crate::toolchain::{self, host_triple, GuestTarget, Owner, Role, GUEST_TARGETS}; use crate::worktree::remove_tree; +use whole_toolchain::{whole, Whole}; /// The per-worktree sources that end up inside a sysroot: std links `toyos-abi` /// and `toyos`, and `libtoyos_c.a` is `userland/libc`. @@ -288,9 +288,9 @@ fn manifest_line(root: &Path, manifest: &str) -> String { /// covers, into every submodule checked out there — never what a build or the /// desktop leaves beside them (bootstrap's `__pycache__`, Finder's /// `.DS_Store`), which would make a key that moves while it is being built. -pub(crate) fn tree_identity(base: &Path, paths: &[&str]) -> String { +pub(crate) fn tree_identity(base: &Path, paths: &[&str], links: Links) -> String { let mut files = Vec::new(); - source_files(base, paths, &mut files); + source_files(base, paths, links, &mut files); files.sort(); let mut hasher = Sha256::new(); for path in files { @@ -303,7 +303,17 @@ pub(crate) fn tree_identity(base: &Path, paths: &[&str]) -> String { hex(&hasher.finalize())[..16].to_string() } -fn source_files(checkout: &Path, paths: &[&str], out: &mut Vec) { +/// What [`tree_identity`] makes of a symbolic link, which git keeps as the path +/// it names and a build reads through. +#[derive(Clone, Copy)] +pub(crate) enum Links { + /// Refused by name. + Refused, + /// Left out of the identity (`issues/build/a-compiler-key-reads-no-symbolic-link.md`). + Skipped, +} + +fn source_files(checkout: &Path, paths: &[&str], links: Links, out: &mut Vec) { let mut args = vec!["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--"]; args.extend(paths); let listed = git_bytes(checkout, &args); @@ -313,9 +323,20 @@ fn source_files(checkout: &Path, paths: &[&str], out: &mut Vec) { if !seen.insert(path.clone()) { continue; } - if path.join(".git").exists() { - source_files(&path, &["."], out); - } else if fs::symlink_metadata(&path).is_ok_and(|m| m.is_file()) { + let Ok(meta) = fs::symlink_metadata(&path) else { continue }; + if meta.is_symlink() { + match links { + Links::Refused => panic!( + "{} is a symbolic link, and a key of {} reads none: git keeps the path it names, \ + and a build reads what is there", + path.display(), + checkout.display() + ), + Links::Skipped => {} + } + } else if path.join(".git").exists() { + source_files(&path, &["."], links, out); + } else if meta.is_file() { out.push(path); } } @@ -362,7 +383,7 @@ fn freestanding_key_of(root: &Path, compiler: &Compiler, fork: &Path, recipe: &s format!("{recipe}; cargo {STAGE0_CARGO}; targets {}", Libraries::Freestanding.targets().join(" ")), config.to_string(), STD_MANIFESTS.map(|manifest| manifest_line(root, manifest)).join("\n"), - tree_identity(fork, &["library", "src/bootstrap"]), + tree_identity(fork, &["library", "src/bootstrap"], Links::Refused), compiler.identity(), ]; Key::of(parts.join("\n\0\n").as_bytes()) @@ -510,23 +531,37 @@ pub fn ensure(root: &Path, rust_dir: &Path, lock: &mut Held) -> Sysroot { Sysroot { dir, primary_compiler: compiler.primary, identity: keys.identity, _using: Some(using) } } -/// A compiler [`whole`] found to be a whole toolchain: what a sysroot is made -/// from. -struct Whole<'a>(&'a Compiler); - -/// `compiler`, refused unless it is a whole toolchain: no sysroot is made from -/// one that is not, and no std is built for one. -fn whole(compiler: &Compiler) -> Whole<'_> { - if let Some(defect) = toolchain::toolchain_defect(&compiler.stage2) { - let fix = if compiler.primary { - "\nA bootstrap in the primary checkout was stopped before it finished: \ - `cargo run -- --build-only` there completes it." - } else { - "" - }; - panic!("no sysroot is made from {}, and no std was built for one: {defect}{fix}", compiler.stage2.display()); +/// Only [`whole`] makes a [`Whole`]. +mod whole_toolchain { + use std::path::Path; + + use crate::compiler::Compiler; + use crate::toolchain; + + /// A compiler [`whole`] found to be a whole toolchain: what a sysroot is + /// made from. + pub(super) struct Whole<'a>(&'a Compiler); + + impl Whole<'_> { + pub(super) fn stage2(&self) -> &Path { + &self.0.stage2 + } + } + + /// `compiler`, refused unless it is a whole toolchain: no sysroot is made + /// from one that is not, and no std is built for one. + pub(super) fn whole(compiler: &Compiler) -> Whole<'_> { + if let Some(defect) = toolchain::toolchain_defect(&compiler.stage2) { + let fix = if compiler.primary { + "\nA bootstrap in the primary checkout was stopped before it finished: \ + `cargo run -- --build-only` there completes it." + } else { + "" + }; + panic!("no sysroot is made from {}, and no std was built for one: {defect}{fix}", compiler.stage2.display()); + } + Whole(compiler) } - Whole(compiler) } /// Make the sysroot `keys` names at `dir`, from `root`'s sources and the std @@ -624,7 +659,7 @@ fn build_freestanding(root: &Path, compiler: &Compiler, fork: &Path, key: &Key, /// adds to them. fn publish_toolchain(compiler: Whole, dir: &Path, fill: impl FnOnce(&Path) -> String) { publish(dir, |partial| { - clone_tree(&compiler.0.stage2, partial); + clone_tree(compiler.stage2(), partial); fill(partial) }); } @@ -967,7 +1002,8 @@ mod tests { /// freestanding libraries; a line of the fork's code, a manifest std's /// lockfile resolves, the recipe, std's configuration or another compiler /// moves both. A crate built against the old ones is stale in the targets - /// whose libraries moved, and in all of it when the compiler did. + /// whose libraries moved, and in all of it when the compiler did. A + /// symbolic link in the fork, whose target no key would read, is refused. #[test] fn a_comment_is_the_same_sysroot_and_a_signature_is_another() { let base = TempDir::new("key"); @@ -1036,6 +1072,13 @@ mod tests { fs::remove_file(fork.join("library/std/src/new.rs")).unwrap(); same("the fork without it"); + let link = fork.join("library/std/src/linked.rs"); + std::os::unix::fs::symlink("lib.rs", &link).unwrap(); + let said = refusal(|| drop(k())); + assert!(said.starts_with(&format!("{} is a symbolic link", link.display())), "{said}"); + fs::remove_file(&link).unwrap(); + same("the fork without the link"); + for manifest in STD_MANIFESTS { write(&root.join(manifest), "[package]\nversion = \"0.2.0\"\n"); both(manifest); @@ -1046,6 +1089,8 @@ mod tests { let compiler = Compiler::primary(&rust_dir); let config = keyed_std_config(); assert_eq!(freestanding_key_of(&root, &compiler, &fork, RECIPE, &config), was.freestanding); + assert_ne!(freestanding_key_of(&root, &compiler, &fork, RECIPE, ""), was.freestanding, + "the freestanding key reads no std configuration"); for (what, moved) in [ ("the recipe", freestanding_key_of(&root, &compiler, &fork, "another recipe", &config)), ("std's configuration", freestanding_key_of(&root, &compiler, &fork, RECIPE, &format!("{config}\n[rust]\n"))), diff --git a/src/toolchain.rs b/src/toolchain.rs index eca2f4739fe..0126f827d76 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -135,16 +135,23 @@ fn std_toyos_sources(dep_info: &Path) -> Vec { found } +/// The text of every dep-info file under `dir`, none if there is no `dir`; one +/// that cannot be read is refused, since its paths would go undecided. fn collect_dep_info(dir: &Path, out: &mut Vec) { - let Ok(entries) = fs::read_dir(dir) else { return }; - for entry in entries.filter_map(Result::ok) { - let path = entry.path(); + match fs::metadata(dir) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => panic!("read {}: {e}", dir.display()), + Ok(_) => dep_info_under(dir, out), + } +} + +fn dep_info_under(dir: &Path, out: &mut Vec) { + for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { + let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", dir.display())).path(); if path.is_dir() { - collect_dep_info(&path, out); + dep_info_under(&path, out); } else if path.extension().is_some_and(|e| e == "d") { - if let Ok(text) = fs::read_to_string(&path) { - out.push(text); - } + out.push(fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()))); } } } @@ -1351,6 +1358,9 @@ mod tests { let said = naming(&spelt); assert!(said.contains(&system), "{spelt}: {said}"); } + std::os::unix::fs::symlink(&system, fork.join("library/core/src/linked.rs")).unwrap(); + let said = naming("library/core/src/linked.rs"); + assert!(said.contains(&system), "{said}"); let said = naming("library/core/src/gone.rs"); assert!(said.starts_with(&format!("resolve {}/library/core/src/gone.rs", fork.display())), "{said}"); @@ -1358,6 +1368,38 @@ mod tests { assert!(said.contains("name no source of the fork"), "{said}"); } + /// **Dep-info that cannot be read is refused, never skipped**, since the + /// paths in it would go undecided; only a directory that is not there + /// holds none. + #[test] + fn dep_info_that_cannot_be_read_is_refused() { + use std::os::unix::fs::PermissionsExt; + let temp = TempDir::new("dep-info-unread"); + let mut none = Vec::new(); + collect_dep_info(&temp.join("none"), &mut none); + assert!(none.is_empty(), "{none:?}"); + + let built = temp.join("x86_64-unknown-none"); + let dist = built.join("dist"); + fs::create_dir_all(&dist).unwrap(); + let collect = || std::panic::catch_unwind(|| collect_dep_info(&built, &mut Vec::new())); + let said = |refused: std::thread::Result<()>, what: &str| { + *refused.err().unwrap_or_else(|| panic!("{what} was skipped")).downcast::().expect("a formatted refusal") + }; + let d = dist.join("core.d"); + fs::write(&d, b"libcore.rlib: library/core/src/lib.rs \xff\n").unwrap(); + let refused = said(collect(), "dep-info that is not UTF-8"); + assert!(refused.starts_with(&format!("read {}", d.display())), "{refused}"); + fs::remove_file(&d).unwrap(); + + let mode = |bits| fs::set_permissions(&dist, fs::Permissions::from_mode(bits)).unwrap(); + mode(0o000); + let unread = collect(); + mode(0o755); + let refused = said(unread, "a directory that cannot be read"); + assert!(refused.starts_with(&format!("read {}", dist.display())), "{refused}"); + } + /// Verbatim from run `31370078581`, the run this check exists because of: /// the four warnings are real and were what the error had to be told apart /// from, and the allowance took it and reported a missing file 83 seconds