diff --git a/issues/build/a-compiler-key-reads-no-symbolic-link.md b/issues/build/a-compiler-key-reads-no-symbolic-link.md new file mode 100644 index 00000000000..79076367f20 --- /dev/null +++ b/issues/build/a-compiler-key-reads-no-symbolic-link.md @@ -0,0 +1,22 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# A compiler key reads no symbolic link + +`compiler::key` reads `compiler/`, `src/tools/`, `src/stage0` and +`Cargo.lock` through `sysroot::tree_identity` with `Links::Skipped`, so a +symbolic link there is in no key: retargeting one, or editing what it names +outside those trees, keeps the old compiler. Refusing a link there, as the +freestanding key does, refuses every compiler build: +`git -C rust ls-files -s compiler src/tools src/stage0 Cargo.lock` at fork +commit `aca5f527f` lists 5 entries of mode `120000`, all under `src/tools` +(clippy's and rust-analyzer's `LICENSE-APACHE` and `LICENSE-MIT`, and +rust-analyzer's `AGENTS.md`). Hashing a link's target text instead moves the +key of every compiler of a worktree's own once. + +**Exit**: `Links::Skipped` is deleted and the compiler key hashes a link's +target text, landed with the next change to `compiler.rs`'s `RECIPE`, which +moves every compiler key anyway. diff --git a/issues/build/a-sysroot-builds-both-architectures-userland-for-a-build-of-one.md b/issues/build/a-sysroot-builds-both-architectures-userland-for-a-build-of-one.md new file mode 100644 index 00000000000..6e14341eb40 --- /dev/null +++ b/issues/build/a-sysroot-builds-both-architectures-userland-for-a-build-of-one.md @@ -0,0 +1,18 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# A sysroot builds both architectures' userland for a build of one + +A sysroot is a whole toolchain for every guest target (`toolchain::GUEST_TARGETS`), +so a new key builds the std set of `x86_64-unknown-toyos` and of +`aarch64-unknown-toyos`, libc for both, and the C++ runtime for both, even when +the build that asked for it is `cargo run -- --build-only`, which links only +x86_64. Measured on the `wt/toyos-rebuild` branch, one item added to +`toyos-abi/src/clock.rs`: the aarch64 std set took 2m18s of a 4m12s std +build at load 41. + +**Exit**: a build makes the libraries of only the architectures it builds for, +and the sysroot's key and identity still name exactly what is in it. diff --git a/issues/build/cleans-deps-stamp-write-is-swallowed.md b/issues/build/cleans-deps-stamp-write-is-swallowed.md deleted file mode 100644 index ca3bf064a2d..00000000000 --- a/issues/build/cleans-deps-stamp-write-is-swallowed.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-28 ---- - -# `clean`'s deps-stamp write is swallowed - -`src/build.rs`'s `clean` ends with `fs::create_dir_all(&target).ok();` and -`fs::write(target.join(".deps-stamp"), fingerprint).ok();` — the two arms -above them (`cargo clean` and the per-target `remove_dir_all`) both panic on -failure, so these are the only swallowed errors left in the function. A stamp -write that fails is not reported, and `stale` reads it back as absent or -mismatched, so every later build re-decides `clean` was owed and pays a full -clean it does not need. - -**Owner:** whoever next touches `clean` or `stale` in `src/build.rs`. - -**Exit condition:** both calls panic on error like their two siblings, or the -`.ok()` is replaced with a call that folds a write failure into the same -refusal `stale` would otherwise silently manufacture. diff --git a/issues/build/every-sysroot-key-compiles-userlands-core-and-alloc-again.md b/issues/build/every-sysroot-key-compiles-userlands-core-and-alloc-again.md new file mode 100644 index 00000000000..08e3238928d --- /dev/null +++ b/issues/build/every-sysroot-key-compiles-userlands-core-and-alloc-again.md @@ -0,0 +1,25 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# Every sysroot key compiles userland's `core` and `alloc` again + +`sysroot::build_std` empties each userland target's `stage0-std/` before it +builds (`prepare_std_build`), so every new sysroot key compiles `core`, `alloc` +and `compiler_builtins` for both userland targets from nothing, although they +read nothing of the worktree: in this branch's build of +`x86_64-unknown-toyos`, cargo's dep-info for them (`dist/libcore.d`, +`liballoc.d`, `libcompiler_builtins.d`) names 411, 714 and 630 paths, none in +the worktree outside its fork, where `libstd.d` names 42. The emptying rests +on the claim, in a comment there, that bootstrap does not see a path +dependency outside the fork move; nobody has measured a build that keeps the +directory. Measured on the `wt/toyos-rebuild` branch, one item added to +`toyos-abi/src/clock.rs`, from the timestamped build log: `core` and `alloc` +were the first 37 s of the 91 s `x86_64-unknown-toyos` std build, and 91 s of +the 158 s `aarch64-unknown-toyos` one, at a load that peaked at 186. + +**Exit**: one item added to `toyos-abi/src/clock.rs` after a build, and the +next `cargo run -- --build-only` prints no `Compiling core` and no +`Compiling alloc`. diff --git a/issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md b/issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md new file mode 100644 index 00000000000..1fed43541a5 --- /dev/null +++ b/issues/build/the-cxx-runtime-is-rebuilt-for-every-sysroot-key.md @@ -0,0 +1,18 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# The C++ runtime is rebuilt for every sysroot key + +`sysroot::build` builds libc++, libc++abi and libunwind for both userland +targets into every new sysroot (`libcxx::build`), so every `toyos-abi` edit +pays for them. Their inputs are the compiler's LLVM, `libcxx::OPTIONS` and the +C sysroot they compile and link against: libc's headers and its staticlib. An +ABI edit moves only the staticlib. Measured on the `wt/toyos-rebuild` branch, +one item added to `toyos-abi/src/clock.rs`: 27 s and 38 s at load 41, 47 s and +54 s at load 68. + +**Exit**: one item added to `toyos-abi/src/clock.rs` after a build, and the +next `cargo run -- --build-only` prints no `Building the C++ runtime`. diff --git a/issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md b/issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md new file mode 100644 index 00000000000..78f53bac131 --- /dev/null +++ b/issues/build/the-cxx-runtimes-scratch-removal-dies-on-a-finder-file.md @@ -0,0 +1,18 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# The C++ runtime's scratch removal dies on a Finder file + +`libcxx::build` ends with a plain `fs::remove_dir_all(scratch)`. On this host +Finder writes a `.DS_Store` into a directory while it is being emptied. A +`cargo run -- --build-only` on `wt/toyos-rebuild` died with `remove +…/rust/build/sysroots/d9ce291c409918f4.libcxx-x86_64: Directory not empty (os +error 66)`, and afterwards the directory held only that `.DS_Store`. The +sysroot was left at `.partial`, so the next build made it again. +`worktree::remove_tree` is the removal that outlasts it. + +**Exit**: every removal of a build product outlasts a writer that adds a +file while it runs, with a test that adds one. diff --git a/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md b/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md index 07c04589bfd..7f9caaad8cc 100644 --- a/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md +++ b/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md @@ -20,8 +20,7 @@ the new rustc refuses their metadata: `error[E0463]: can't find crate` for `cursor_icon`, `smol_str`, `raw_window_handle`, `keyboard_types` in winit-core. Recorded at the first external use of the `sdk-0.2.0` alias: Japabu/gbae run 33960322229 restored a target directory built by `5009ee0d7ea1cfad` and built -with `3845b80c8ba72421`. Inside this tree `src/build.rs`'s -`external_fingerprint` is the belt that hides it; a consumer has no belt. +with `3845b80c8ba72421`. Reproduce with any consumer: build with one release, swap the linked toolchain to another, build again in the same target directory. diff --git a/src/CLAUDE.md b/src/CLAUDE.md index b20e2caddc6..94c51792107 100644 --- a/src/CLAUDE.md +++ b/src/CLAUDE.md @@ -18,7 +18,7 @@ Loads when you read a file under `src/` — the root cargo project, package name ## The host's locks -- **Sysroots are content-addressed** (`src/sysroot.rs`): one per key — the identity (`src/identity.rs`, so a comment is no change) of `toyos-abi/src`, `toyos/src`, `userland/libc/src` and their manifests, the std fork's `rust/library/` and `rust/src/bootstrap/`, and the compiler — at `rust/build/sysroots//`, made by whichever worktree first needs it. Every build compiles against its own key's, so two worktrees with different ABIs never refuse or wait for each other; the only shared step is the primary's compiler, which a sysroot build reads under the global lock in shared mode. A new key costs one std build of the three guest targets; `--worktree remove` sweeps the keys no worktree records. +- **Sysroots are content-addressed** (`src/sysroot.rs`): one per key — the identity (`src/identity.rs`, so a comment is no change) of `toyos-abi/src`, `toyos/src`, `userland/libc/src` and their manifests, the std fork's `rust/library/` and `rust/src/bootstrap/`, and the compiler — at `rust/build/sysroots//`, made by whichever worktree first needs it. Every build compiles against its own key's, so two worktrees with different ABIs never refuse or wait for each other; the only shared step is the primary's compiler, which a sysroot build reads under the global lock in shared mode. `--worktree remove` sweeps the keys no worktree records. - **The std fork is built per worktree, and nothing but the primary's own sync moves the primary's `rust/`.** A linked worktree's `rust/` becomes, on its first build, a git worktree of the primary's fork repository at the commit its tree pins — that is where the fork is edited, committed and pinned. A worktree whose fork `compiler/` differs from the one the primary built builds its own compiler, keyed by that source and placed beside the primary's without touching it. If that checkout later falls behind the commit its tree pins (a merge moved the pin), the build moves the checkout to it itself, fetching from the primary's repository first if it holds the commit, unless the checkout has local changes, which it refuses to move out from under. - `src/buildlock.rs` serialises the stateful phases in two scopes: `Global` (the primary's compiler and the rustup link — one directory in `.git/`, shared by every worktree) and `Worktree` (the crate-target cleans, and this worktree's std build). Only `./x.py` typed by hand in `rust/` escapes it. - **Never kill a build that has taken the global lock** — the kill removes the shell wrappers, not the bootstrap, which inherits the file descriptor and runs on regardless; a toolchain rebuild interrupted or unobserved this way can leave `stage2/bin` without a `cargo`. diff --git a/src/build.rs b/src/build.rs index f8932e7a13d..37a33feb876 100644 --- a/src/build.rs +++ b/src/build.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::rc::Rc; use std::sync::Arc; -use std::time::{Duration, Instant, UNIX_EPOCH}; +use std::time::{Duration, Instant}; use serde::Deserialize; @@ -16,6 +16,7 @@ use crate::buildlock; use crate::flags; use crate::hostws; use crate::image; +use crate::sysroot::{Identity, Stale}; use crate::toolchain; thread_local! { @@ -173,46 +174,7 @@ fn parse_config(path: &Path) -> SystemConfig { // --- Freshness checking --- -/// Fingerprint all external build dependencies that cargo cannot track: the -/// sysroot `toolchain` is — by where it is, which names its key, and by its -/// libraries. The linker is the sysroot's own `rust-lld`, which that key names -/// with the compiler it came with. -fn external_fingerprint(toolchain: &Path) -> String { - let sysroot = toolchain.join("lib/rustlib"); - let mut entries = vec![format!("sysroot:{}", toolchain.display())]; - - for triple in toolchain::GUEST_TARGETS { - let lib_dir = sysroot.join(format!("{triple}/lib")); - let Ok(rd) = fs::read_dir(&lib_dir) else { - continue; - }; - for entry in rd.flatten() { - let path = entry.path(); - if !path.is_file() { - continue; - } - let ext = path.extension().and_then(|e| e.to_str()); - if !matches!(ext, Some("rlib" | "rmeta")) { - continue; - } - if let Ok(meta) = path.metadata() { - let name = path.file_name().unwrap().to_string_lossy(); - let mtime = meta - .modified() - .ok() - .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) - .map(|d| d.as_nanos()) - .unwrap_or(0); - entries.push(format!("{triple}/{name}:{}:{mtime}", meta.len())); - } - } - } - - entries.sort(); - entries.join("\n") -} - -/// How much of a crate's target directory goes when the external deps change. +/// How much of a crate's target directory goes when the compiler moves. #[derive(Clone, Copy)] enum Clean { All, @@ -226,17 +188,23 @@ enum Clean { ToyosOnly, } -fn stale(root: &Path, crate_dir: &Path, fingerprint: &str) -> bool { +fn stale(root: &Path, crate_dir: &Path, identity: &Identity) -> Option { let stamp = hostws::target_dir(root, crate_dir).join(".deps-stamp"); - fs::read_to_string(&stamp).map_or(true, |stored| stored != fingerprint) + identity.stale(fs::read_to_string(&stamp).ok().as_deref()) } -fn clean(root: &Path, crate_dir: &Path, kind: Clean, fingerprint: &str) { +fn clean(root: &Path, crate_dir: &Path, kind: Clean, stale: &Stale, identity: &Identity) { // Where cargo actually wrote it. `toyos-ld` is a member of the host // workspace, so its guest build lands in the root's `target/`. let target = hostws::target_dir(root, crate_dir); - match kind { - Clean::All => { + let remove = |dirs: &mut dyn Iterator| { + for dir in dirs.filter(|dir| dir.exists()) { + eprintln!("external deps changed: cleaning {}", dir.display()); + crate::worktree::remove_tree(&dir); + } + }; + match (stale, kind) { + (Stale::All, Clean::All) => { // `cargo clean` in a member's directory cleans the whole workspace, // this build system's own target directory included. Nothing asks // for that today; refusing it by name is cheaper than finding out. @@ -254,22 +222,19 @@ fn clean(root: &Path, crate_dir: &Path, kind: Clean, fingerprint: &str) { .unwrap_or_else(|e| panic!("run cargo clean in {}: {e}", crate_dir.display())); assert!(status.success(), "cargo clean in {} exited {status}", crate_dir.display()); } - Clean::ToyosOnly => { + (Stale::All, Clean::ToyosOnly) => { let guest = Arch::ALL.iter().map(|arch| target.join(arch.userland())); - for dir in guest.chain([target.join(PROFILE)]) { - if dir.exists() { - eprintln!("external deps changed: cleaning {}", dir.display()); - fs::remove_dir_all(&dir).unwrap_or_else(|e| panic!("remove {}: {e}", dir.display())); - } - } + remove(&mut guest.chain([target.join(PROFILE)])); } + (Stale::Targets(moved), _) => remove(&mut moved.iter().map(|t| target.join(t))), } - fs::create_dir_all(&target).ok(); - fs::write(target.join(".deps-stamp"), fingerprint).ok(); + fs::create_dir_all(&target).unwrap_or_else(|e| panic!("create {}: {e}", target.display())); + let stamp = target.join(".deps-stamp"); + fs::write(&stamp, identity.to_string()).unwrap_or_else(|e| panic!("write {}: {e}", stamp.display())); } -/// Drop the target directories the changed external deps invalidated. +/// Drop what in the target directories the sysroot's moved parts invalidated. /// /// Deciding and acting under one exclusive section is the whole point. Each of /// these cleans removes a tree another builder may be compiling into, and @@ -280,24 +245,22 @@ fn clean(root: &Path, crate_dir: &Path, kind: Clean, fingerprint: &str) { fn invalidate_stale( root: &Path, lock: &mut buildlock::Held, - toolchain: &Path, + identity: &Identity, targets: &[(PathBuf, Clean)], ) { lock.act_if( buildlock::Scope::Worktree, "clean crate targets against changed external deps", || { - let fp = external_fingerprint(toolchain); - let work: Vec<(PathBuf, Clean)> = targets + let work: Vec<(PathBuf, Clean, Stale)> = targets .iter() - .filter(|(dir, _)| stale(root, dir, &fp)) - .cloned() + .filter_map(|(dir, kind)| stale(root, dir, identity).map(|s| (dir.clone(), *kind, s))) .collect(); - (!work.is_empty()).then_some((fp, work)) + (!work.is_empty()).then_some(work) }, - |(fp, work)| { - for (dir, kind) in work { - clean(root, &dir, kind, &fp); + |work| { + for (dir, kind, stale) in work { + clean(root, &dir, kind, &stale, identity); } }, ); @@ -420,6 +383,9 @@ struct GuestEnv { /// Whether that sysroot's compiler is the primary's, the one the hosted /// rustc is built from (`src/compiler.rs`). primary_compiler: bool, + /// What its crates compile against beside their sources + /// (`sysroot::Identity`). + identity: Identity, /// The public key the loader and `/system/bin/update` embed /// (`signing::KEY_ENV`): every guest build carries it, so no crate that /// names it can be built without it. @@ -433,6 +399,7 @@ impl GuestEnv { Self { toolchain: sysroot.dir.clone(), primary_compiler: sysroot.primary_compiler, + identity: sysroot.identity.clone(), image_key: crate::signing::key().public_hex(), floor_scope: crate::signing::key().floor_scope().word(), } @@ -1802,7 +1769,7 @@ fn shipped_parts(root: &Path, boot: &Boot, plan: &Plan) -> (Vec, Vec, Ve let env = GuestEnv::new(&sysroot); let config = parse_config(&boot.config); - invalidate_stale(root, &mut lock, &env.toolchain, &config_targets(root, &config)); + invalidate_stale(root, &mut lock, &env.identity, &config_targets(root, &config)); // Same lock-and-stage as `build_test_image`: `cargo run --build-only` and // `cargo test` share these paths, so this races the harness too. The kernel @@ -2035,7 +2002,7 @@ pub fn build_test_parts( let sysroot = crate::toolchain::ensure(root, &mut lock); let env = GuestEnv::new(&sysroot); - invalidate_stale(root, &mut lock, &env.toolchain, &config_targets(root, &config)); + invalidate_stale(root, &mut lock, &env.identity, &config_targets(root, &config)); // Build and stage under one lock, released before `build_and_assemble`. // Releasing it there is deliberate and required: that build takes its own @@ -2252,7 +2219,7 @@ impl TestBuild { let mut lock = buildlock::shared(root, what); let sysroot = crate::toolchain::ensure(root, &mut lock); let env = GuestEnv::new(&sysroot); - invalidate_stale(root, &mut lock, &env.toolchain, stale_targets); + invalidate_stale(root, &mut lock, &env.identity, stale_targets); let artifact = buildlock::artifact(root); TestBuild { target: arch.userland(), env, _lock: lock, _artifact: artifact } } @@ -2372,13 +2339,98 @@ mod tests { Arch::ALL.iter().map(|arch| file(&format!("target/{}/{PROFILE}/ld", arch.userland()))).collect(); guest.push(file(&format!("target/{PROFILE}/deps/libsyn-1.rlib"))); let host = file("target/debug/toyos-build"); + let identity = Identity::of_parts("compiler", "freestanding", "toyos"); - clean(&root, &root.join("ld"), Clean::ToyosOnly, "fingerprint"); + clean(&root, &root.join("ld"), Clean::ToyosOnly, &Stale::All, &identity); for gone in &guest { assert!(!gone.exists(), "{} survived a clean of what the guest build wrote", gone.display()); } assert!(host.is_file(), "the host workspace's own build went"); - assert_eq!(fs::read_to_string(root.join("target/.deps-stamp")).unwrap(), "fingerprint"); + assert_eq!(fs::read_to_string(root.join("target/.deps-stamp")).unwrap(), identity.to_string()); + } + + /// **Moved libraries take what was built for their targets, and nothing + /// else**: an ABI edit moves the userland targets' libraries, so the + /// kernel's and the loader's target directories stay whole and of + /// userland's only `target/` goes; a fork edit moves every + /// target's, so the kernel's and the loader's builds go too. The host half + /// the same compiler built stays either way. A moved compiler, or a stamp + /// that names none, is all of it. + #[test] + fn moved_libraries_take_what_was_built_for_them_and_a_moved_compiler_all() { + let root = toyos_tmpdir::TempDir::new("moved-libraries"); + fs::write(root.join("Cargo.toml"), "[workspace]\nmembers = []\n").unwrap(); + let file = |path: PathBuf| { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, "built").unwrap(); + path + }; + let (kernel, loader, userland) = (root.join("kernel"), root.join("bootloader"), root.join("userland")); + let crates = [&kernel, &loader, &userland]; + let kernels = Arch::ALL.map(|arch| file(kernel.join(format!("target/{}/{PROFILE}/kernel", arch.kernel())))); + let loaders = Arch::ALL.map(|arch| file(loader.join(format!("target/{}/{PROFILE}/loader.efi", arch.loader())))); + let programs = Arch::ALL.map(|arch| file(userland.join(format!("target/{}/{PROFILE}/init", arch.userland())))); + let hosts = crates.map(|dir| file(dir.join(format!("target/{PROFILE}/deps/libproc-1.dylib")))); + + let before = Identity::of_parts("compiler", "freestanding", "toyos"); + for dir in crates { + fs::write(dir.join("target/.deps-stamp"), before.to_string()).unwrap(); + assert_eq!(stale(&root, dir, &before), None, "{} was stale against its own stamp", dir.display()); + } + let moved = |identity: &Identity, want: Vec<&'static str>| { + for dir in crates { + let found = stale(&root, dir, identity); + assert_eq!(found, Some(Stale::Targets(want.clone())), "{}", dir.display()); + clean(&root, dir, Clean::All, &found.unwrap(), identity); + assert_eq!(stale(&root, dir, identity), None, "{} was not stamped", dir.display()); + } + }; + + let mut toyos: Vec<&'static str> = Arch::ALL.map(Arch::userland).into(); + toyos.sort(); + moved(&Identity::of_parts("compiler", "freestanding", "toyos, edited"), toyos); + for kept in kernels.iter().chain(&loaders).chain(&hosts) { + assert!(kept.is_file(), "{} went, and nothing it was built from moved", kept.display()); + } + for gone in &programs { + assert!(!gone.exists(), "{} survived its target's libraries moving", gone.display()); + } + + let mut all: Vec<&'static str> = Arch::ALL.into_iter().flat_map(|arch| [arch.userland(), arch.kernel(), arch.loader()]).collect(); + all.sort(); + let fork_edit = Identity::of_parts("compiler", "freestanding, edited", "toyos on the edited fork"); + moved(&fork_edit, all); + for gone in kernels.iter().chain(&loaders) { + assert!(!gone.exists(), "{} survived its target's libraries moving", gone.display()); + } + for kept in &hosts { + assert!(kept.is_file(), "{} went, and the compiler that built it did not move", kept.display()); + } + + let compiler = Identity::of_parts("another compiler", "freestanding, edited", "toyos on the edited fork"); + assert_eq!(stale(&root, &kernel, &compiler), Some(Stale::All), "a moved compiler kept the host half"); + fs::write(userland.join("target/.deps-stamp"), "sysroot:/a/stamp/naming/no/compiler").unwrap(); + assert_eq!(stale(&root, &userland, &fork_edit), Some(Stale::All), "a stamp naming no compiler was trusted"); + } + + /// **A stamp that cannot be written stops the build**: one left behind would + /// call what the clean took current, or owe a clean every build after. + #[test] + fn a_stamp_that_cannot_be_written_panics() { + use std::os::unix::fs::PermissionsExt; + let root = toyos_tmpdir::TempDir::new("unwritable-stamp"); + fs::write(root.join("Cargo.toml"), "[workspace]\nmembers = []\n").unwrap(); + let target = root.join("kernel/target"); + fs::create_dir_all(&target).unwrap(); + fs::set_permissions(&target, fs::Permissions::from_mode(0o555)).unwrap(); + let identity = Identity::of_parts("compiler", "freestanding", "toyos"); + let failed = std::panic::catch_unwind(|| { + clean(&root, &root.join("kernel"), Clean::All, &Stale::Targets(vec![]), &identity) + }); + fs::set_permissions(&target, fs::Permissions::from_mode(0o755)).unwrap(); + let refusal = failed.expect_err("a stamp that was not written was taken for written"); + let refusal = refusal.downcast_ref::().expect("a formatted panic"); + assert!(refusal.starts_with(&format!("write {}", target.join(".deps-stamp").display())), "{refusal}"); } /// **A `cargo clean` that fails stops the build and stamps nothing**: a @@ -2392,7 +2444,8 @@ mod tests { fs::create_dir_all(kernel.join("target")).unwrap(); fs::write(kernel.join("Cargo.toml"), "[package\n").unwrap(); - let failed = std::panic::catch_unwind(|| clean(&root, &kernel, Clean::All, "fingerprint")); + let identity = Identity::of_parts("compiler", "freestanding", "toyos"); + let failed = std::panic::catch_unwind(|| clean(&root, &kernel, Clean::All, &Stale::All, &identity)); let refusal = failed.expect_err("a cargo clean that failed was taken for one that ran"); let refusal = refusal.downcast_ref::().expect("a formatted panic"); assert!(refusal.starts_with(&format!("cargo clean in {} exited", kernel.display())), "{refusal}"); diff --git a/src/buildlock.rs b/src/buildlock.rs index 534f20c65bb..cdf7ed00ec0 100644 --- a/src/buildlock.rs +++ b/src/buildlock.rs @@ -32,9 +32,10 @@ //! primary's (`src/compiler.rs`) is locked the same way under its own key, and //! neither it nor a sysroot built from it takes the global lock. //! -//! key's lock → a sysroot key's lock → the worktree build lock → the global one -//! → an LLVM key's lock → artifact. A compiler's or a sysroot's key lock is -//! taken with the worktree lock put down ([`Held::without_shared`]), because the +//! A compiler key's lock → a sysroot key's lock → a freestanding key's lock → the +//! worktree build lock → the global one → an LLVM key's lock → artifact. A +//! compiler's, a sysroot's or a freestanding key lock is taken with the +//! worktree lock put down ([`Held::without_shared`]), because the //! key's builder takes the worktree lock exclusively; an LLVM key's is taken //! inside the worktree or global lock covering the fork build directory its //! builder writes. @@ -217,12 +218,14 @@ pub fn artifact(root: &Path) -> Guard { exclusive(&root.join(LOCK_DIR).join("artifact"), "artifact lock", "artifact staging") } -/// A content-addressed product of the host, locked per key: a sysroot, a -/// compiler a worktree's fork checkout names (`src/compiler.rs`), or the LLVM -/// a compiler links (`src/llvm.rs`). +/// A content-addressed product of the host, locked per key: a sysroot, the +/// freestanding targets' libraries it carries (`src/sysroot.rs`), a compiler +/// a worktree's fork checkout names (`src/compiler.rs`), or the LLVM a +/// compiler links (`src/llvm.rs`). #[derive(Clone, Copy)] pub enum Keyed { Sysroot, + Freestanding, Compiler, Llvm, } @@ -231,6 +234,7 @@ impl Keyed { fn dir(self) -> &'static str { match self { Keyed::Sysroot => "sysroots", + Keyed::Freestanding => "freestanding", Keyed::Compiler => "compilers", Keyed::Llvm => "llvm", } @@ -239,6 +243,7 @@ impl Keyed { pub(crate) fn name(self) -> &'static str { match self { Keyed::Sysroot => "sysroot", + Keyed::Freestanding => "freestanding libraries", Keyed::Compiler => "compiler", Keyed::Llvm => "LLVM", } diff --git a/src/compiler.rs b/src/compiler.rs index 804d788479c..5f2e9ad511c 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -48,7 +48,7 @@ use std::path::{Path, PathBuf}; use crate::buildlock::{self, Guard, Held, Keyed}; use crate::keystore::{self, Key}; -use crate::sysroot::{clone_tree, git_bytes, git_out, short, tree_identity}; +use crate::sysroot::{clone_tree, git_bytes, git_out, short, tree_identity, Links}; use crate::toolchain::{self, host_triple}; /// What changes how a key's sources become a compiler and is none of them: the @@ -202,7 +202,7 @@ pub fn key(fork: &Path) -> Key { /// [`key`], with the key of the LLVM `fork` names. fn key_with(fork: &Path, llvm: &Key) -> Key { - let parts = [RECIPE, &tree_identity(fork, &KEYED), llvm.as_str()]; + let parts = [RECIPE, &tree_identity(fork, &KEYED, Links::Skipped), llvm.as_str()]; Key::of(parts.join("\n\0\n").as_bytes()) } diff --git a/src/keystore.rs b/src/keystore.rs index 9a1037c561e..c35e97f3dc1 100644 --- a/src/keystore.rs +++ b/src/keystore.rs @@ -62,6 +62,7 @@ impl AsRef for Key { fn record_path(root: &Path, kind: Keyed) -> PathBuf { root.join(match kind { Keyed::Sysroot => "target/toyos-sysroot-key", + Keyed::Freestanding => "target/toyos-freestanding-key", Keyed::Compiler => "target/toyos-compiler-key", Keyed::Llvm => "target/toyos-llvm-key", }) diff --git a/src/sysroot.rs b/src/sysroot.rs index 05a969d96ed..868929b31bd 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -12,6 +12,12 @@ //! different ABIs or different compilers never refuse or wait for each other, //! and main and every branch matching it share one copy. //! +//! **The kernel's and the loader's libraries compile none of those trees**, so +//! they are a key of their own ([`freestanding_key`]). They are built once per that key +//! into `rust/build/freestanding//` and cloned into every sysroot naming +//! it, and a crate built against a sysroot learns which targets' libraries +//! moved ([`Identity`]). Each build refuses dep-info that says otherwise. +//! //! **Each worktree builds std in its own fork checkout, and nothing but the //! primary's own sync moves the primary's.** The primary builds in its `rust/`; //! a linked worktree in its own `rust/`, made on first need as a git worktree of @@ -25,17 +31,19 @@ //! //! Locks, in the one order every acquirer takes them: the compiler key's, if the //! compiler is a worktree's own; the sysroot key's (`buildlock::keyed_*`), with -//! this worktree's build lock put down; then, to build, this worktree's -//! exclusively (its fork build directory is written); then, if the compiler is -//! the primary's, the global one shared, because it is read; then the key of the -//! compiler's LLVM, held in use while the C++ runtime is built from its sources. +//! this worktree's build lock put down; then the freestanding key's, while its +//! libraries are cloned in; then, to build, this worktree's exclusively (its +//! fork build directory is written); then, if the compiler is the primary's, the +//! global one shared, because it is read; then the key of the compiler's LLVM, +//! held in use while the C++ runtime is built from its sources. //! -//! A sysroot no worktree names any more is removed by `keystore::sweep`, which -//! `--worktree remove` runs: each build records the key it used in its -//! worktree's `target/`, and a key no registered worktree records, that nobody -//! is making or using, goes. +//! A sysroot or freestanding libraries no worktree names any more are removed +//! by `keystore::sweep`, which `--worktree remove` runs: each build records the +//! keys it used in its worktree's `target/`, and a key no registered worktree +//! records, that nobody is making or using, goes. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; @@ -47,7 +55,9 @@ use crate::buildlock::{self, Guard, Held, Keyed}; use crate::compiler::{self, Compiler}; use crate::identity; use crate::keystore::Key; -use crate::toolchain::{self, host_triple, Owner, GUEST_TARGETS}; +use crate::toolchain::{self, host_triple, GuestTarget, Owner, Role, GUEST_TARGETS}; +use crate::worktree::remove_tree; +use whole_toolchain::{whole, Whole}; /// The per-worktree sources that end up inside a sysroot: std links `toyos-abi` /// and `toyos`, and `libtoyos_c.a` is `userland/libc`. @@ -58,22 +68,57 @@ pub const SYSROOT_SOURCES: [&str; 4] = pub(crate) const SYSROOT_MANIFESTS: [&str; 3] = ["toyos-abi/Cargo.toml", "toyos/Cargo.toml", "userland/libc/Cargo.toml"]; -/// The file a finished sysroot carries last, naming what it was built from. -/// A directory without it is a build that did not finish. +/// Of [`SYSROOT_MANIFESTS`], the ones std's lockfile resolves with the fork's +/// own: what of a worktree can move a freestanding target's dependency versions. +const STD_MANIFESTS: [&str; 2] = ["toyos-abi/Cargo.toml", "toyos/Cargo.toml"]; + +/// The file a directory [`publish`] made carries last, naming what it was +/// built from. A directory without it is a build that did not finish. const SOURCES: &str = "SOURCES"; -/// What changes how a key's sources become a sysroot and is none of them: the -/// std build's recipe below. Moving it moves every key. -const RECIPE: &str = "bootstrap stage-0 local rebuild, profile compiler, no LLVM or Ninja, \ - libtoyos_c merged, libraries from the stamp, linked by rust-lld, \ +/// What changes how a key's sources become its libraries and its sysroot and +/// is none of them, nor std's configuration ([`std_config`]), which the +/// freestanding key reads whole. Moving it moves every key. +const RECIPE: &str = "bootstrap stage-0 local rebuild, libraries from the stamp, libtoyos_c merged, \ a C sysroot of libc's staticlib and headers per target, and its C++ runtime \ - built under n2 from the runtimes' sources of the compiler's LLVM; 8"; + built under n2 from the runtimes' sources of the compiler's LLVM, the \ + freestanding libraries cloned from their key's; 10"; /// Every sysroot on this host. pub fn sysroots_dir(rust_dir: &Path) -> PathBuf { rust_dir.join("build/sysroots") } +/// Every freestanding target's libraries on this host, one directory per key. +pub fn freestanding_dir(rust_dir: &Path) -> PathBuf { + rust_dir.join("build/freestanding") +} + +/// Whose sources a guest target's libraries compile. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Libraries { + /// The fork's and [`SYSROOT_SOURCES`]: ToyOS userland's std, which names + /// `toyos-abi` and `toyos`. + Worktree, + /// The fork's alone: the kernel's `core` and `alloc` and the loader's std, + /// which is what lets them be a key of their own. + Freestanding, +} + +impl Libraries { + fn of(target: GuestTarget) -> Self { + match target.role { + Role::Userland => Self::Worktree, + Role::Kernel | Role::Loader => Self::Freestanding, + } + } + + /// The triples of the [`GUEST_TARGETS`] whose libraries these are. + fn targets(self) -> Vec<&'static str> { + GUEST_TARGETS.into_iter().filter(|t| Self::of(*t) == self).map(GuestTarget::triple).collect() + } +} + /// A sysroot a build compiles against, held in use for as long as this lives. pub struct Sysroot { /// A toolchain directory: `RUSTUP_TOOLCHAIN` names it. @@ -81,15 +126,100 @@ pub struct Sysroot { /// Whether its compiler is the primary's, which the ToyOS-hosted rustc is /// built from. pub primary_compiler: bool, + pub identity: Identity, _using: Option, } impl Sysroot { /// A checkout whose toolchain arrived as an artifact has one sysroot, the /// artifact's, which `toolchain::check_installed_toolchain` has matched to - /// these sources. - pub(crate) fn installed(stage2: PathBuf) -> Self { - Self { dir: stage2, primary_compiler: true, _using: None } + /// these sources; `release`, the `TOOLCHAIN` it was published with, is the + /// identity of all of it. + pub(crate) fn installed(stage2: PathBuf, release: &str) -> Self { + let id = Key::of(release.as_bytes()); + Self { dir: stage2, primary_compiler: true, identity: Identity::new(id.clone(), &id, &id), _using: None } + } +} + +/// What a crate compiled against a sysroot compiles against beside its own +/// sources, as its target directory's `.deps-stamp` records it: the compiler, +/// which everything a crate's build makes reads, its host half too; and per +/// guest target the key of its libraries, which only what is made for that +/// target reads. +#[derive(Clone, Debug, PartialEq)] +pub struct Identity { + compiler: Key, + libraries: BTreeMap<&'static str, Key>, +} + +/// What of a crate's target directory a sysroot's [`Identity`] leaves stale. +/// Cargo keys nothing it reuses on the sysroot — every ToyOS compiler prints +/// one `rustc -vV` — so this is where a moved compiler or library reaches a +/// crate. +#[derive(Debug, PartialEq)] +pub enum Stale { + /// The compiler moved, or nothing records which one built it: all the guest + /// build wrote, its host half too. + All, + /// Only these guest targets' libraries moved: what was made for them, + /// `target//`, and nothing the compiler made for the host. + Targets(Vec<&'static str>), +} + +impl Identity { + /// The identity of a sysroot whose compiler's key is `compiler`, whose + /// freestanding targets' libraries are `freestanding`'s, and whose others + /// are `key`'s. + fn new(compiler: Key, freestanding: &Key, key: &Key) -> Self { + let libraries = GUEST_TARGETS.map(|target| match Libraries::of(target) { + Libraries::Worktree => (target.triple(), key.clone()), + Libraries::Freestanding => (target.triple(), freestanding.clone()), + }); + Self { compiler, libraries: libraries.into() } + } + + /// The identity of these parts' keys, for its readers' tests. + #[cfg(test)] + pub(crate) fn of_parts(compiler: &str, freestanding: &str, key: &str) -> Self { + let of = |part: &str| Key::of(part.as_bytes()); + Self::new(of(compiler), &of(freestanding), &of(key)) + } + + /// What this leaves stale of a target directory whose `.deps-stamp` says + /// `stamp`, if it has one: nothing, if it is this. + pub fn stale(&self, stamp: Option<&str>) -> Option { + let Some(was) = stamp.and_then(Self::parse) else { return Some(Stale::All) }; + if was.compiler != self.compiler { + return Some(Stale::All); + } + let moved: Vec<&'static str> = self + .libraries + .iter() + .filter(|(target, key)| was.libraries.get(*target) != Some(key)) + .map(|(target, _)| *target) + .collect(); + (!moved.is_empty()).then_some(Stale::Targets(moved)) + } + + /// The identity `text` spells, if it names a key for each of the + /// [`GUEST_TARGETS`] and for nothing else. + fn parse(text: &str) -> Option { + let mut lines = text.lines(); + let compiler = Key::parse(lines.next()?.strip_prefix("compiler ")?)?; + let mut libraries = BTreeMap::new(); + for line in lines { + let (triple, key) = line.split_once(' ')?; + let target = GUEST_TARGETS.into_iter().find(|t| t.triple() == triple)?; + libraries.insert(target.triple(), Key::parse(key)?); + } + (libraries.len() == GUEST_TARGETS.len()).then_some(Self { compiler, libraries }) + } +} + +impl fmt::Display for Identity { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "compiler {}", self.compiler)?; + self.libraries.iter().try_for_each(|(triple, key)| write!(f, "\n{triple} {key}")) } } @@ -140,14 +270,17 @@ pub fn witness(root: &Path) -> String { lines.push(format!("{}:{}", rel.display(), short(&identity::of(&path, &data)))); } } - for manifest in SYSROOT_MANIFESTS { - let path = root.join(manifest); - let data = fs::read(&path).unwrap_or_else(|e| panic!("witness {}: {e}", path.display())); - lines.push(format!("{manifest}:{}", short(&data))); - } + lines.extend(SYSROOT_MANIFESTS.map(|manifest| manifest_line(root, manifest))); lines.join("\n") } +/// `manifest`'s witness line: its path and the hash of its bytes. +fn manifest_line(root: &Path, manifest: &str) -> String { + let path = root.join(manifest); + let data = fs::read(&path).unwrap_or_else(|e| panic!("witness {}: {e}", path.display())); + format!("{manifest}:{}", short(&data)) +} + /// The identity of the source files under `paths` of the git checkout `base`, /// as one hash. /// @@ -155,9 +288,9 @@ pub fn witness(root: &Path) -> String { /// covers, into every submodule checked out there — never what a build or the /// desktop leaves beside them (bootstrap's `__pycache__`, Finder's /// `.DS_Store`), which would make a key that moves while it is being built. -pub(crate) fn tree_identity(base: &Path, paths: &[&str]) -> String { +pub(crate) fn tree_identity(base: &Path, paths: &[&str], links: Links) -> String { let mut files = Vec::new(); - source_files(base, paths, &mut files); + source_files(base, paths, links, &mut files); files.sort(); let mut hasher = Sha256::new(); for path in files { @@ -170,7 +303,17 @@ pub(crate) fn tree_identity(base: &Path, paths: &[&str]) -> String { hex(&hasher.finalize())[..16].to_string() } -fn source_files(checkout: &Path, paths: &[&str], out: &mut Vec) { +/// What [`tree_identity`] makes of a symbolic link, which git keeps as the path +/// it names and a build reads through. +#[derive(Clone, Copy)] +pub(crate) enum Links { + /// Refused by name. + Refused, + /// Left out of the identity (`issues/build/a-compiler-key-reads-no-symbolic-link.md`). + Skipped, +} + +fn source_files(checkout: &Path, paths: &[&str], links: Links, out: &mut Vec) { let mut args = vec!["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--"]; args.extend(paths); let listed = git_bytes(checkout, &args); @@ -180,30 +323,84 @@ fn source_files(checkout: &Path, paths: &[&str], out: &mut Vec) { if !seen.insert(path.clone()) { continue; } - if path.join(".git").exists() { - source_files(&path, &["."], out); - } else if fs::symlink_metadata(&path).is_ok_and(|m| m.is_file()) { + let Ok(meta) = fs::symlink_metadata(&path) else { continue }; + if meta.is_symlink() { + match links { + Links::Refused => panic!( + "{} is a symbolic link, and a key of {} reads none: git keeps the path it names, \ + and a build reads what is there", + path.display(), + checkout.display() + ), + Links::Skipped => {} + } + } else if path.join(".git").exists() { + source_files(&path, &["."], links, out); + } else if meta.is_file() { out.push(path); } } } -/// The key of the sysroot `root` builds against with its std fork at `fork`, -/// compiled by `compiler`. -pub fn key(root: &Path, compiler: &Compiler, fork: &Path) -> Key { +/// What a build of a worktree compiles against, by key. +#[derive(Debug, PartialEq)] +struct Keys { + /// Its freestanding targets' libraries' ([`freestanding_key`]). + freestanding: Key, + /// Its sysroot's ([`key`]). + sysroot: Key, + /// What a crate compiled against that sysroot records. + identity: Identity, +} + +impl Keys { + /// The keys of what `root` builds against with its std fork at `fork`, + /// compiled by `compiler`. + fn of(root: &Path, compiler: &Compiler, fork: &Path) -> Self { + let freestanding = freestanding_key(root, compiler, fork); + let sysroot = key(root, &freestanding); + let identity = Identity::new(Key::of(compiler.identity().as_bytes()), &freestanding, &sysroot); + Self { freestanding, sysroot, identity } + } +} + +/// The key of the freestanding targets' libraries `root` builds against with +/// its std fork at `fork`, compiled by `compiler`: none of +/// [`SYSROOT_SOURCES`], and of `root` only [`STD_MANIFESTS`]. +fn freestanding_key(root: &Path, compiler: &Compiler, fork: &Path) -> Key { + freestanding_key_of(root, compiler, fork, RECIPE, &keyed_std_config()) +} + +/// [`std_config`] with no path of this host in it, as a key reads it. +fn keyed_std_config() -> String { + let placeholder = Path::new(""); + std_config(placeholder, placeholder, placeholder, "") +} + +/// [`freestanding_key`], with the recipe and std's configuration it reads. +fn freestanding_key_of(root: &Path, compiler: &Compiler, fork: &Path, recipe: &str, config: &str) -> Key { let parts = [ - format!( - "{RECIPE}; cargo {STAGE0_CARGO}; targets {}; C++ runtime {:?}", - GUEST_TARGETS.join(" "), - crate::libcxx::OPTIONS - ), - witness(root), - tree_identity(fork, &["library", "src/bootstrap"]), + format!("{recipe}; cargo {STAGE0_CARGO}; targets {}", Libraries::Freestanding.targets().join(" ")), + config.to_string(), + STD_MANIFESTS.map(|manifest| manifest_line(root, manifest)).join("\n"), + tree_identity(fork, &["library", "src/bootstrap"], Links::Refused), compiler.identity(), ]; Key::of(parts.join("\n\0\n").as_bytes()) } +/// The key of the sysroot `root` builds against, whose freestanding libraries +/// are `freestanding`'s ([`freestanding_key`], which names the recipe, std's +/// configuration, the fork and the compiler the rest is built with too). +fn key(root: &Path, freestanding: &Key) -> Key { + let parts = [ + format!("targets {}; C++ runtime {:?}", Libraries::Worktree.targets().join(" "), crate::libcxx::OPTIONS), + witness(root), + freestanding.to_string(), + ]; + Key::of(parts.join("\n\0\n").as_bytes()) +} + /// The commit this checkout's tree pins the std fork at: the index's, so a /// staged gitlink counts as the tree's. fn pinned_fork(root: &Path) -> String { @@ -298,16 +495,19 @@ pub fn fork_checkout(root: &Path) -> PathBuf { fork } -/// Why `dir` is not a finished sysroot, if it is not: no [`SOURCES`], or not a +/// Why `dir` is not a directory [`publish`] finished, if it is not: it carries +/// no [`SOURCES`]. +fn unpublished(dir: &Path) -> Option { + (!dir.join(SOURCES).is_file()).then(|| format!("{} carries no {SOURCES}", dir.display())) +} + +/// Why `dir` is not a finished sysroot, if it is not: [`unpublished`], or not a /// whole toolchain (`toolchain::toolchain_defect`). One found with the first and /// not the second is made again rather than trusted — all of it even when only /// its `bin/cargo` link dangles, because that is rare and a sysroot has no /// repair path. fn unfinished(dir: &Path) -> Option { - if !dir.join(SOURCES).is_file() { - return Some(format!("{} carries no {SOURCES}", dir.display())); - } - toolchain::toolchain_defect(dir) + unpublished(dir).or_else(|| toolchain::toolchain_defect(dir)) } /// The sysroot `key` names at `dir`, made by `make` if nobody has made it, and @@ -321,28 +521,87 @@ fn held(root: &Path, key: &Key, dir: &Path, make: impl FnMut()) -> Guard { pub fn ensure(root: &Path, rust_dir: &Path, lock: &mut Held) -> Sysroot { let fork = fork_checkout(root); let compiler = compiler::resolve(root, rust_dir, &fork, lock); - let key = key(root, &compiler, &fork); - let dir = sysroots_dir(rust_dir).join(&key); - crate::keystore::record(root, Keyed::Sysroot, &key); + let keys = Keys::of(root, &compiler, &fork); + let dir = sysroots_dir(rust_dir).join(&keys.sysroot); + crate::keystore::record(root, Keyed::Sysroot, &keys.sysroot); + crate::keystore::record(root, Keyed::Freestanding, &keys.freestanding); + + let using = + lock.without_shared(|| held(root, &keys.sysroot, &dir, || build(root, rust_dir, &compiler, &fork, &keys, &dir))); + Sysroot { dir, primary_compiler: compiler.primary, identity: keys.identity, _using: Some(using) } +} + +/// Only [`whole`] makes a [`Whole`]. +mod whole_toolchain { + use std::path::Path; - let using = lock.without_shared(|| held(root, &key, &dir, || build(root, rust_dir, &compiler, &fork, &key, &dir))); - Sysroot { dir, primary_compiler: compiler.primary, _using: Some(using) } + use crate::compiler::Compiler; + use crate::toolchain; + + /// A compiler [`whole`] found to be a whole toolchain: what a sysroot is + /// made from. + pub(super) struct Whole<'a>(&'a Compiler); + + impl Whole<'_> { + pub(super) fn stage2(&self) -> &Path { + &self.0.stage2 + } + } + + /// `compiler`, refused unless it is a whole toolchain: no sysroot is made + /// from one that is not, and no std is built for one. + pub(super) fn whole(compiler: &Compiler) -> Whole<'_> { + if let Some(defect) = toolchain::toolchain_defect(&compiler.stage2) { + let fix = if compiler.primary { + "\nA bootstrap in the primary checkout was stopped before it finished: \ + `cargo run -- --build-only` there completes it." + } else { + "" + }; + panic!("no sysroot is made from {}, and no std was built for one: {defect}{fix}", compiler.stage2.display()); + } + Whole(compiler) + } } -/// Make the sysroot `key` names at `dir`, from `root`'s sources and the std fork -/// at `fork`, with `compiler`. The caller holds the key's lock. -fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &Key, dir: &Path) { - let what = format!("building sysroot {key}"); +/// Make the sysroot `keys` names at `dir`, from `root`'s sources and the std +/// fork at `fork`, with `compiler`, and the freestanding libraries `keys` +/// names. The caller holds the sysroot key's lock. +fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, keys: &Keys, dir: &Path) { + let made_from = whole(compiler); + let store = freestanding_dir(rust_dir).join(&keys.freestanding); + let _freestanding = buildlock::keyed_made( + root, + Keyed::Freestanding, + &keys.freestanding, + || unpublished(&store), + || build_freestanding(root, compiler, fork, &keys.freestanding, &store), + ); + let what = format!("building sysroot {}", keys.sysroot); let _worktree = buildlock::worktree_exclusive(root, &what); // Only the primary's compiler is rebuilt in place; one of a worktree's own // is written once and held in use by `compiler`. let _compiler = compiler.primary.then(|| buildlock::compiler_shared(root, &what)); - eprintln!("Building sysroot {key}: std from {}, the compiler {}", fork.display(), compiler.stage2.display()); + eprintln!( + "Building sysroot {}: ToyOS's std from {}, the compiler {}, the freestanding libraries {}", + keys.sysroot, + fork.display(), + compiler.stage2.display(), + keys.freestanding + ); - publish(compiler, dir, |partial| { - let built = build_std(root, compiler, fork); + publish_toolchain(made_from, dir, |partial| { + let built = build_std(root, compiler, fork, Libraries::Worktree); for target in GUEST_TARGETS { - place_std(&stamp(&built, target), &partial.join("lib/rustlib").join(target).join("lib")); + let triple = target.triple(); + let lib = partial.join("lib/rustlib").join(triple).join("lib"); + match Libraries::of(target) { + Libraries::Worktree => place_std(&stamp(&built, triple), &lib), + Libraries::Freestanding => { + remove_tree(&lib); + clone_tree(&store.join(triple), &lib); + } + } } let libc_target = dir.with_extension("libc-target"); for arch in Arch::ALL { @@ -359,7 +618,7 @@ fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &K } // The sources the key named are the ones built, or this is not that key's. - let again = self::key(root, compiler, fork); + let (key, again) = (&keys.sysroot, Keys::of(root, compiler, fork).sysroot); assert!( again == *key, "the sources moved while sysroot {key} was being built (they are now {again}); \ @@ -369,68 +628,95 @@ fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &K }); } -/// Put at `dir` a whole toolchain: `compiler`'s files and what `fill` adds to -/// them, then the [`SOURCES`] `fill` returns, last. A `dir` already there is one -/// [`unfinished`] refused, and it is replaced. A compiler that is not whole is -/// refused before `fill` runs, and nothing is published. -fn publish(compiler: &Compiler, dir: &Path, fill: impl FnOnce(&Path) -> String) { - if let Some(defect) = toolchain::toolchain_defect(&compiler.stage2) { - let fix = if compiler.primary { - "\nA bootstrap in the primary checkout was stopped before it finished: \ - `cargo run -- --build-only` there completes it." - } else { - "" - }; - panic!("no sysroot is made from {}, and no std was built for one: {defect}{fix}", compiler.stage2.display()); - } +/// Make the freestanding targets' libraries `key` names at `store`, from the std +/// fork at `fork` with `compiler`. The caller holds the key's lock. +fn build_freestanding(root: &Path, compiler: &Compiler, fork: &Path, key: &Key, store: &Path) { + let what = format!("building the freestanding libraries {key}"); + let _worktree = buildlock::worktree_exclusive(root, &what); + let _compiler = compiler.primary.then(|| buildlock::compiler_shared(root, &what)); + eprintln!( + "Building the freestanding libraries {key}: {} from {}, the compiler {}", + Libraries::Freestanding.targets().join(", "), + fork.display(), + compiler.stage2.display() + ); + publish(store, |partial| { + let built = build_std(root, compiler, fork, Libraries::Freestanding); + for target in Libraries::Freestanding.targets() { + place_std(&stamp(&built, target), &partial.join(target)); + } + let again = freestanding_key(root, compiler, fork); + assert!( + again == *key, + "the sources moved while the freestanding libraries {key} were being built (they are \ + now {again}); nothing was kept, and the next build makes the ones they name" + ); + format!("{key}\nfork {}\n", fork.display()) + }); +} + +/// [`publish`] at `dir` a toolchain: `compiler`'s files, then what `fill` +/// adds to them. +fn publish_toolchain(compiler: Whole, dir: &Path, fill: impl FnOnce(&Path) -> String) { + publish(dir, |partial| { + clone_tree(compiler.stage2(), partial); + fill(partial) + }); +} + +/// Put at `dir` what `fill` makes at the path it is handed, then the +/// [`SOURCES`] it returns, last. A `dir` already there is one [`unpublished`] +/// or worse refused, and it is replaced. +fn publish(dir: &Path, fill: impl FnOnce(&Path) -> String) { let partial = dir.with_extension("partial"); - if partial.exists() { - fs::remove_dir_all(&partial).unwrap_or_else(|e| panic!("remove {}: {e}", partial.display())); - } - clone_tree(&compiler.stage2, &partial); + remove_tree(&partial); let sources = fill(&partial); fs::write(partial.join(SOURCES), sources) .unwrap_or_else(|e| panic!("write {}: {e}", partial.join(SOURCES).display())); - if dir.exists() { - fs::remove_dir_all(dir).unwrap_or_else(|e| panic!("remove {}: {e}", dir.display())); - } + remove_tree(dir); fs::rename(&partial, dir) .unwrap_or_else(|e| panic!("rename {} -> {}: {e}", partial.display(), dir.display())); } -/// Compile the guest targets' libraries from `fork`'s `library/` with -/// `compiler`, and return the directory each target's is under. -fn build_std(root: &Path, compiler: &Compiler, fork: &Path) -> PathBuf { +/// Compile the libraries of `libraries`' targets from `fork`'s `library/` with +/// `compiler`, and return the directory each target's is under, once each +/// target's dep-info shows it read of `root` what `libraries` says. +fn build_std(root: &Path, compiler: &Compiler, fork: &Path, libraries: Libraries) -> PathBuf { crate::ensure_submodule(fork, "library/backtrace"); let host = host_triple(); let build_dir = fork.join("build/toyos-std"); - prepare_std_build(&build_dir, &host, &compiler.identity()); + let targets = libraries.targets(); + prepare_std_build(&build_dir, &host, &compiler.identity(), &targets); let config = build_dir.join("bootstrap.toml"); fs::write(&config, std_config(&compiler.stage2, &bootstrap_cargo(), &build_dir, &host)) .unwrap_or_else(|e| panic!("write {}: {e}", config.display())); - let targets = GUEST_TARGETS.join(","); + let listed = targets.join(","); let args = ["build", "library", "--stage", "0", "--config", path_str(&config), "--warnings", "warn", - "--target", &targets]; + "--target", &listed]; let (ok, log) = toolchain::x_build(fork, &args, "std"); toolchain::refuse_on_compile_error(&log, "std"); assert!(ok, "the std build failed, and nothing in its output was a compile error"); - for arch in Arch::ALL { - toolchain::assert_std_built_from(root, &build_dir.join(&host).join("stage0-std").join(arch.userland())); + let built = build_dir.join(&host).join("stage0-std"); + for target in targets { + match libraries { + Libraries::Worktree => toolchain::assert_std_built_from(root, &built.join(target)), + Libraries::Freestanding => toolchain::assert_std_reads_no_worktree(root, fork, &built.join(target)), + } } - build_dir.join(&host).join("stage0-std") + built } -/// Ready the std build directory `build_dir` for a build by the compiler -/// `identity` names: nothing another compiler built, no LLVM, and no guest -/// target's std. -fn prepare_std_build(build_dir: &Path, host: &str, identity: &str) { +/// Ready the std build directory `build_dir` for a build of `targets` by the +/// compiler `identity` names: nothing another compiler built, no LLVM, and no +/// std of those targets. +fn prepare_std_build(build_dir: &Path, host: &str, identity: &str, targets: &[&str]) { fs::create_dir_all(build_dir).unwrap_or_else(|e| panic!("create {}: {e}", build_dir.display())); forget_another_compiler(build_dir, host, identity); crate::llvm::retire_in_tree(build_dir); // Bootstrap reuses what it built before and does not see a path dependency // outside the fork move, so each target's std starts from nothing. - for target in GUEST_TARGETS { + for target in targets { remove(&build_dir.join(host).join("stage0-std").join(target)); } } @@ -536,7 +822,7 @@ fn place_std(stamp: &Path, lib: &Path) { /// and no Ninja, which bootstrap otherwise demands on `PATH` for the LLVM it does /// not build. fn std_config(compiler: &Path, cargo: &Path, build_dir: &Path, host: &str) -> String { - let targets = GUEST_TARGETS.iter().map(|t| format!("\"{t}\"")).collect::>().join(", "); + let targets = GUEST_TARGETS.map(|t| format!("\"{}\"", t.triple())).join(", "); let linker = toolchain::rust_lld(compiler); let userland: String = Arch::ALL .iter() @@ -710,55 +996,126 @@ mod tests { (root, rust_dir, fork) } - /// **The key is the identity, and only the identity**: a comment in any tree - /// it reads — the ABI or the std fork — is the same sysroot, and a signature, - /// a line of the fork's code or another compiler is another. + /// **Each key is the identity of what it is built from, and only that**: a + /// comment in any tree they read moves neither; an edit to a tree only + /// ToyOS's std or the C sysroot compiles moves the sysroot and keeps its + /// freestanding libraries; a line of the fork's code, a manifest std's + /// lockfile resolves, the recipe, std's configuration or another compiler + /// moves both. A crate built against the old ones is stale in the targets + /// whose libraries moved, and in all of it when the compiler did. A + /// symbolic link in the fork, whose target no key would read, is refused. #[test] fn a_comment_is_the_same_sysroot_and_a_signature_is_another() { let base = TempDir::new("key"); let (root, rust_dir, fork) = keyed(&base); - let k = || key(&root, &Compiler::primary(&rust_dir), &fork); - let base = k(); - assert_eq!(base.as_str().len(), 16, "{base}"); + let k = || Keys::of(&root, &Compiler::primary(&rust_dir), &fork); + let was = k(); + assert_eq!((was.sysroot.as_str().len(), was.freestanding.as_str().len()), (16, 16), "{was:?}"); + let stamp = was.identity.to_string(); + let toyos = vec!["aarch64-unknown-toyos", "x86_64-unknown-toyos"]; + let mut all = GUEST_TARGETS.map(GuestTarget::triple).to_vec(); + all.sort(); + let same = |what: &str| assert_eq!(k(), was, "{what} moved a key"); + let sysroot_only = |what: &str| { + let now = k(); + assert_ne!(now.sysroot, was.sysroot, "{what} kept the old sysroot"); + assert_eq!(now.freestanding, was.freestanding, "{what} moved the freestanding libraries"); + assert_eq!(now.identity.stale(Some(&stamp)), Some(Stale::Targets(toyos.clone())), "{what}"); + }; + let both = |what: &str| { + let now = k(); + assert_ne!(now.sysroot, was.sysroot, "{what} kept the old sysroot"); + assert_ne!(now.freestanding, was.freestanding, "{what} kept the old freestanding libraries"); + assert_eq!(now.identity.stale(Some(&stamp)), Some(Stale::Targets(all.clone())), "{what}"); + }; let abi = root.join("toyos-abi/src/lib.rs"); write(&abi, "//! The crate.\n/// A, said better.\n// and a plain comment\npub struct A;\n"); - assert_eq!(k(), base, "a comment in toyos-abi made a new sysroot"); + same("a comment in toyos-abi"); write(&abi, "/// A.\npub struct A(pub u64);\n"); - assert_ne!(k(), base, "a signature change kept the old sysroot"); + sysroot_only("a signature change in toyos-abi"); write(&abi, "/// A.\npub struct A;\n"); - assert_eq!(k(), base); + same("toyos-abi as it was"); + + for tree in ["toyos/src", "userland/libc/src"] { + write(&root.join(tree).join("lib.rs"), "/// A.\npub struct A(u8);\n"); + sysroot_only(tree); + write(&root.join(tree).join("lib.rs"), "/// A.\npub struct A;\n"); + same(tree); + } let header = root.join("userland/libc/include/stdio.h"); write(&header, "int puts(const char *);\n"); - assert_ne!(k(), base, "a header the C sysroot carries kept the old sysroot"); + sysroot_only("a header the C sysroot carries"); fs::remove_file(&header).unwrap(); - assert_eq!(k(), base); + same("the C sysroot's headers as they were"); + + write(&root.join("userland/libc/Cargo.toml"), "[package]\nversion = \"0.2.0\"\n"); + sysroot_only("libc's manifest, which std's lockfile does not resolve,"); + write(&root.join("userland/libc/Cargo.toml"), "[package]\nversion = \"0.1.0\"\n"); + same("libc's manifest as it was"); let std = fork.join("library/std/src/lib.rs"); write(&std, "//! std, documented\npub fn exit() {}\n"); - assert_eq!(k(), base, "a comment in the std fork made a new sysroot"); + same("a comment in the std fork"); write(&std, "//! std\npub fn exit() { loop {} }\n"); - assert_ne!(k(), base, "a change to the fork's code kept the old sysroot"); + both("a change to the fork's code"); write(&std, "//! std\npub fn exit() {}\n"); - assert_eq!(k(), base); + same("the fork as it was"); // What a build and the desktop leave in the checkout is not its source. write(&fork.join("src/bootstrap/__pycache__/bootstrap.cpython-313.pyc"), "bytecode"); write(&fork.join("library/.DS_Store"), "finder"); - assert_eq!(k(), base, "a file git ignores moved the key"); + same("a file git ignores"); write(&fork.join("library/std/src/new.rs"), "pub fn new() {}\n"); - assert_ne!(k(), base, "an untracked source file was not in the key"); + both("an untracked source file"); fs::remove_file(fork.join("library/std/src/new.rs")).unwrap(); - assert_eq!(k(), base); + same("the fork without it"); + + let link = fork.join("library/std/src/linked.rs"); + std::os::unix::fs::symlink("lib.rs", &link).unwrap(); + let said = refusal(|| drop(k())); + assert!(said.starts_with(&format!("{} is a symbolic link", link.display())), "{said}"); + fs::remove_file(&link).unwrap(); + same("the fork without the link"); + + for manifest in STD_MANIFESTS { + write(&root.join(manifest), "[package]\nversion = \"0.2.0\"\n"); + both(manifest); + write(&root.join(manifest), "[package]\nversion = \"0.1.0\"\n"); + same(manifest); + } - write(&root.join("toyos-abi/Cargo.toml"), "[package]\nversion = \"0.2.0\"\n"); - assert_ne!(k(), base, "a manifest change kept the old sysroot"); - write(&root.join("toyos-abi/Cargo.toml"), "[package]\nversion = \"0.1.0\"\n"); - assert_eq!(k(), base); + let compiler = Compiler::primary(&rust_dir); + let config = keyed_std_config(); + assert_eq!(freestanding_key_of(&root, &compiler, &fork, RECIPE, &config), was.freestanding); + assert_ne!(freestanding_key_of(&root, &compiler, &fork, RECIPE, ""), was.freestanding, + "the freestanding key reads no std configuration"); + for (what, moved) in [ + ("the recipe", freestanding_key_of(&root, &compiler, &fork, "another recipe", &config)), + ("std's configuration", freestanding_key_of(&root, &compiler, &fork, RECIPE, &format!("{config}\n[rust]\n"))), + ] { + assert_ne!(moved, was.freestanding, "{what} kept the old freestanding libraries"); + } write(&rust_dir.join("build/toyos-compiler"), "tree-2"); - assert_ne!(k(), base, "another compiler kept the old sysroot"); + let now = k(); + assert!(now.sysroot != was.sysroot && now.freestanding != was.freestanding, "another compiler kept a key: {now:?}"); + assert_eq!(now.identity.stale(Some(&stamp)), Some(Stale::All), "another compiler kept a crate's host half"); + } + + /// **A crate's compiler is the one that built it, rebuilt in place or + /// not**: the primary's is rebuilt where it stands, so a driver its rebuild + /// left leaves all of a crate stale, and the same driver none of it. + #[test] + fn a_compiler_rebuilt_in_place_leaves_all_of_a_crate_stale() { + let base = TempDir::new("identity-compiler"); + let (root, rust_dir, fork) = keyed(&base); + let identity = || Keys::of(&root, &Compiler::primary(&rust_dir), &fork).identity; + let stamp = identity().to_string(); + assert_eq!(identity().stale(Some(&stamp)), None, "a crate was stale against the compiler that built it"); + write(&toolchain::stage2(&rust_dir).join("lib/librustc_driver-1.dylib"), "a driver, rebuilt in place"); + assert_eq!(identity().stale(Some(&stamp)), Some(Stale::All), "a crate kept what the compiler made before its rebuild"); } /// **What one compiler compiled in a std build directory is never another's**: @@ -818,7 +1175,7 @@ mod tests { let build = base.join("toyos-std"); let host = host_triple(); let identity = Compiler::primary(&rust_dir).identity(); - prepare_std_build(&build, &host, &identity); + prepare_std_build(&build, &host, &identity, &GUEST_TARGETS.map(GuestTarget::triple)); let llvm = [ build.join(&host).join("ci-llvm/lib/libLLVM.dylib"), build.join(&host).join("llvm/bin/llvm-config"), @@ -828,7 +1185,7 @@ mod tests { for file in llvm.iter().chain(&kept) { write(file, "built"); } - prepare_std_build(&build, &host, &identity); + prepare_std_build(&build, &host, &identity, &GUEST_TARGETS.map(GuestTarget::triple)); for file in &llvm { assert!(!file.exists(), "{} outlived a std build's preparation", file.display()); } @@ -996,7 +1353,7 @@ mod tests { let make = |dir: &Path, most: usize| { made.set(made.get() + 1); assert!(made.get() <= most, "a sysroot that was not whole was made again: make {}", made.get()); - publish(&compiler, dir, |partial| { + publish_toolchain(whole(&compiler), dir, |partial| { write(&partial.join("lib/rustlib/x86_64-unknown-toyos/lib/libstd.rlib"), "std"); "found\n".to_string() }) @@ -1045,7 +1402,7 @@ mod tests { let said = refusal(|| { drop(held(&base, &key, &dir, || { once(); - publish(&compiler, &dir, |_| { + publish_toolchain(whole(&compiler), &dir, |_| { filled.set(true); "cloned\n".to_string() }) diff --git a/src/toolchain.rs b/src/toolchain.rs index 6f028083e9d..0126f827d76 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; @@ -77,22 +78,40 @@ const STD_SOURCES: [&str; 2] = ["toyos-abi/src", "toyos/src"]; /// Every target a guest artifact is built for: ToyOS userland, the kernel's /// bare-metal target, and the UEFI bootloader. -/// -/// One home for the list, because it is read four ways that must agree — the -/// `stage1-std` cleans in [`full_bootstrap`], [`write_config`]'s bootstrap -/// `target` set, the libraries `src/sysroot.rs` builds and places, and -/// `src/build.rs`'s external fingerprint. A fifth spelling would silently leave -/// one of them building or fingerprinting a different set of targets than the -/// others. -pub const GUEST_TARGETS: [&str; 6] = [ - Arch::X86_64.userland(), - Arch::X86_64.kernel(), - Arch::X86_64.loader(), - Arch::Aarch64.userland(), - Arch::Aarch64.kernel(), - Arch::Aarch64.loader(), +pub const GUEST_TARGETS: [GuestTarget; 6] = [ + GuestTarget { arch: Arch::X86_64, role: Role::Userland }, + GuestTarget { arch: Arch::X86_64, role: Role::Kernel }, + GuestTarget { arch: Arch::X86_64, role: Role::Loader }, + GuestTarget { arch: Arch::Aarch64, role: Role::Userland }, + GuestTarget { arch: Arch::Aarch64, role: Role::Kernel }, + GuestTarget { arch: Arch::Aarch64, role: Role::Loader }, ]; +/// What a guest target's artifacts are. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Role { + Userland, + Kernel, + Loader, +} + +/// One of the [`GUEST_TARGETS`]. +#[derive(Clone, Copy, Debug)] +pub struct GuestTarget { + pub arch: Arch, + pub role: Role, +} + +impl GuestTarget { + pub const fn triple(self) -> &'static str { + match self.role { + Role::Userland => self.arch.userland(), + Role::Kernel => self.arch.kernel(), + Role::Loader => self.arch.loader(), + } + } +} + /// The one ToyOS the hosted rustc (`system.toml`'s `hosted-rustc`) is built to /// run on. pub const HOSTED_ARCH: Arch = Arch::X86_64; @@ -116,35 +135,43 @@ fn std_toyos_sources(dep_info: &Path) -> Vec { found } +/// The text of every dep-info file under `dir`, none if there is no `dir`; one +/// that cannot be read is refused, since its paths would go undecided. fn collect_dep_info(dir: &Path, out: &mut Vec) { - let Ok(entries) = fs::read_dir(dir) else { return }; - for entry in entries.filter_map(Result::ok) { - let path = entry.path(); + match fs::metadata(dir) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => panic!("read {}: {e}", dir.display()), + Ok(_) => dep_info_under(dir, out), + } +} + +fn dep_info_under(dir: &Path, out: &mut Vec) { + for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { + let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", dir.display())).path(); if path.is_dir() { - collect_dep_info(&path, out); + dep_info_under(&path, out); } else if path.extension().is_some_and(|e| e == "d") { - if let Ok(text) = fs::read_to_string(&path) { - out.push(text); - } + out.push(fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()))); } } } +/// Every path one dep-info file names: what was built, and what it read. +fn paths_in_dep_info(text: &str) -> impl Iterator { + text.lines() + .filter(|line| !line.starts_with('#')) + .flat_map(str::split_ascii_whitespace) + .map(|word| word.strip_suffix(':').unwrap_or(word)) +} + /// The paths in one dep-info file that name a `toyos-abi/src` or `toyos/src` /// source. Split out from the filesystem so the gate below has a negative /// control that is a string literal. fn toyos_sources_in_dep_info(text: &str) -> Vec { - let mut out = Vec::new(); - for word in text.split_ascii_whitespace() { - let word = word.strip_suffix(':').unwrap_or(word); - if !word.ends_with(".rs") { - continue; - } - if STD_SOURCES.iter().any(|src| word.contains(&format!("/{src}/"))) { - out.push(word.to_string()); - } - } - out + paths_in_dep_info(text) + .filter(|path| path.ends_with(".rs") && STD_SOURCES.iter().any(|src| path.contains(&format!("/{src}/")))) + .map(str::to_string) + .collect() } /// Refuse a std whose dep-info under `dep_info` names another checkout's ABI. @@ -173,6 +200,47 @@ pub(crate) fn assert_std_built_from(root: &Path, dep_info: &Path) { ); } +/// Refuse a freestanding target's libraries whose dep-info under `dep_info` +/// names no source of the std fork `fork`, or any file of the worktree `root` +/// outside it: their key reads nothing else of it but the manifests std's +/// lockfile resolves (`src/sysroot.rs`), so a sysroot of another worktree would +/// carry them unchanged. Every path is decided as the filesystem resolves it, +/// `..` and symlinks followed, and one that resolves to nothing is refused. +pub(crate) fn assert_std_reads_no_worktree(root: &Path, fork: &Path, dep_info: &Path) { + let resolve = |path: &Path| { + fs::canonicalize(path) + .unwrap_or_else(|e| panic!("resolve {}, checking the std build under {}: {e}", path.display(), dep_info.display())) + }; + let (root, fork) = (resolve(root), resolve(fork)); + let mut deps = Vec::new(); + collect_dep_info(dep_info, &mut deps); + // Bootstrap's cargo runs rustc in the fork, so a relative path is the fork's. + let read: BTreeSet = + deps.iter().flat_map(|text| paths_in_dep_info(text)).map(|path| resolve(&fork.join(path))).collect(); + assert!( + read.iter().any(|path| path.starts_with(fork.join("library"))), + "the freestanding libraries under {} name no source of the fork {} at all, so the check that \ + they read nothing else of the worktree cannot answer. Cargo's dep-info moved.", + dep_info.display(), + fork.display(), + ); + let worktree: Vec = read + .iter() + .filter(|path| path.starts_with(&root) && !path.starts_with(&fork)) + .map(|path| path.display().to_string()) + .collect(); + assert!( + worktree.is_empty(), + "the freestanding libraries under {} read {} files of the worktree {} outside its fork {}, \ + and their key reads none of its sources:\n {}", + dep_info.display(), + worktree.len(), + root.display(), + fork.display(), + worktree.join("\n "), + ); +} + /// What an installed toolchain's sysroot was built from, as its publisher /// recorded it (`src/release.rs`). fn witness_path(rust_dir: &Path) -> PathBuf { @@ -442,7 +510,11 @@ pub fn ensure(root: &Path, lock: &mut buildlock::Held) -> Sysroot { } Owner::Installed => { check_installed_toolchain(root, &rust_dir); - return Sysroot::installed(stage2(&rust_dir)); + let release = rust_dir.join("build/TOOLCHAIN"); + let release = fs::read_to_string(&release).unwrap_or_else(|e| { + panic!("{}: {e}; an installed toolchain carries the TOOLCHAIN it was published with", release.display()) + }); + return Sysroot::installed(stage2(&rust_dir), &release); } Owner::Us => {} } @@ -726,7 +798,7 @@ fn full_bootstrap(root: &Path, rust_dir: &Path, llvm: &Path) { // Clean cached std for all ToyOS targets so bootstrap picks up compiler changes // (e.g. target spec changes like default_uwtable that affect codegen). for target in GUEST_TARGETS { - let stage1_std = rust_dir.join(format!("build/{host}/stage1-std/{target}")); + let stage1_std = rust_dir.join(format!("build/{host}/stage1-std/{}", target.triple())); if stage1_std.exists() { fs::remove_dir_all(&stage1_std).ok(); } @@ -828,7 +900,7 @@ fn write_config(rust_dir: &Path, host: &str, with_hosted_rustc: bool, llvm: &Pat format!("host = [\"{host}\"]") }; let targets = std::iter::once(host) - .chain(GUEST_TARGETS) + .chain(GUEST_TARGETS.map(GuestTarget::triple)) .map(|t| format!("\"{t}\"")) .collect::>() .join(", "); @@ -1239,6 +1311,95 @@ mod tests { ); } + /// **Freestanding libraries whose dep-info names the worktree outside its + /// fork are refused, however the path is spelt**: their key reads none of + /// it, so a sysroot of another worktree would carry them. The fork's files + /// and what lies outside the worktree are what they are built from. + /// Dep-info that names no source of the fork, or a file that is not there, + /// cannot be checked and is refused too. + #[test] + fn freestanding_libraries_that_read_the_worktree_are_refused() { + let temp = TempDir::new("freestanding-dep-info"); + let base = fs::canonicalize(&*temp).unwrap(); + let (root, registry) = (base.join("worktree"), base.join("registry/compiler_builtins/src/lib.rs")); + let fork = root.join("rust"); + let built = fork.join("build/toyos-std/host/stage0-std/x86_64-unknown-none"); + let out = built.join("dist/build/core/1/out"); + let file = |path: &Path| { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "read").unwrap(); + path.display().to_string() + }; + file(&fork.join("library/core/src/lib.rs")); + file(&fork.join("library/stdarch/crates/core_arch/src/mod.rs")); + let (registry, rlib, raw) = (file(®istry), file(&built.join("dist/libcore.rlib")), file(&out.join("libcore-1.rlib"))); + // Cargo's dep-info names a file whole; rustc's, a fork source by its path in the fork. + let arch = "library/core/src/../../stdarch/crates/core_arch/src/mod.rs"; + fs::write(built.join("dist/libcore.d"), format!("{rlib}: {}/{arch} {registry}\n", fork.display())).unwrap(); + let rustc = format!("{raw}: library/core/src/lib.rs {arch}\n\nlibrary/core/src/lib.rs:\n# env-dep:CARGO_PKG_NAME=core\n"); + fs::write(out.join("core-1.d"), rustc).unwrap(); + assert_std_reads_no_worktree(&root, &fork, &built); + + let refusal = |dep_info: &Path, what: &str| { + let refused = std::panic::catch_unwind(|| assert_std_reads_no_worktree(&root, &fork, dep_info)); + *refused.err().unwrap_or_else(|| panic!("{what} was taken")).downcast::().expect("a formatted refusal") + }; + let naming = |named: &str| { + fs::write(built.join("dist/other.d"), format!("{rlib}: {named}\n")).unwrap(); + refusal(&built, &format!("dep-info naming {named}")) + }; + let system = file(&root.join("system.toml")); + for read in [file(&root.join("toyos-abi/src/lib.rs")), file(&root.join("userland/libc/src/lib.rs")), system.clone()] { + let said = naming(&read); + assert!(said.contains(&read), "{said}"); + } + let dotted = "library/core/src/../../../../system.toml"; + for spelt in [format!("{}/{dotted}", fork.display()), dotted.to_string()] { + let said = naming(&spelt); + assert!(said.contains(&system), "{spelt}: {said}"); + } + std::os::unix::fs::symlink(&system, fork.join("library/core/src/linked.rs")).unwrap(); + let said = naming("library/core/src/linked.rs"); + assert!(said.contains(&system), "{said}"); + let said = naming("library/core/src/gone.rs"); + assert!(said.starts_with(&format!("resolve {}/library/core/src/gone.rs", fork.display())), "{said}"); + + let said = refusal(&built.join("none"), "a target directory without dep-info"); + assert!(said.contains("name no source of the fork"), "{said}"); + } + + /// **Dep-info that cannot be read is refused, never skipped**, since the + /// paths in it would go undecided; only a directory that is not there + /// holds none. + #[test] + fn dep_info_that_cannot_be_read_is_refused() { + use std::os::unix::fs::PermissionsExt; + let temp = TempDir::new("dep-info-unread"); + let mut none = Vec::new(); + collect_dep_info(&temp.join("none"), &mut none); + assert!(none.is_empty(), "{none:?}"); + + let built = temp.join("x86_64-unknown-none"); + let dist = built.join("dist"); + fs::create_dir_all(&dist).unwrap(); + let collect = || std::panic::catch_unwind(|| collect_dep_info(&built, &mut Vec::new())); + let said = |refused: std::thread::Result<()>, what: &str| { + *refused.err().unwrap_or_else(|| panic!("{what} was skipped")).downcast::().expect("a formatted refusal") + }; + let d = dist.join("core.d"); + fs::write(&d, b"libcore.rlib: library/core/src/lib.rs \xff\n").unwrap(); + let refused = said(collect(), "dep-info that is not UTF-8"); + assert!(refused.starts_with(&format!("read {}", d.display())), "{refused}"); + fs::remove_file(&d).unwrap(); + + let mode = |bits| fs::set_permissions(&dist, fs::Permissions::from_mode(bits)).unwrap(); + mode(0o000); + let unread = collect(); + mode(0o755); + let refused = said(unread, "a directory that cannot be read"); + assert!(refused.starts_with(&format!("read {}", dist.display())), "{refused}"); + } + /// Verbatim from run `31370078581`, the run this check exists because of: /// the four warnings are real and were what the error had to be told apart /// from, and the allowance took it and reported a missing file 83 seconds diff --git a/src/worktree.rs b/src/worktree.rs index 47236ec82dd..ebc6be4e915 100644 --- a/src/worktree.rs +++ b/src/worktree.rs @@ -395,6 +395,7 @@ pub(crate) fn remove(root: &Path, path: &str) { let rust_dir = crate::toolchain::rust_dir(root); for (kind, store, what) in [ (Keyed::Sysroot, crate::sysroot::sysroots_dir(&rust_dir), "sysroot"), + (Keyed::Freestanding, crate::sysroot::freestanding_dir(&rust_dir), "freestanding libraries"), (Keyed::Compiler, crate::compiler::compilers_dir(&rust_dir), "compiler"), (Keyed::Llvm, crate::llvm::store(&rust_dir), "LLVM"), ] { @@ -457,7 +458,7 @@ fn remove_fork_checkout(root: &Path, at: &Path) { /// stops halfway — the `Directory not empty` git itself dies on. The removal /// runs again over what is left, at most [`PASSES`] times; a tree still refusing /// after that has a writer this cannot outrun, and the panic says so. -fn remove_tree(dir: &Path) { +pub(crate) fn remove_tree(dir: &Path) { for pass in 1..=PASSES { match fs::remove_dir_all(dir) { Ok(()) => return,