diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 558bcb2bfe1..e53578aa31d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -62,14 +62,15 @@ jobs: fetch-depth: 0 # CMake and Ninja are what bootstrap builds LLVM and clang with, pinned to - # the versions Ubuntu 24.04 released (src/main.rs's `ALSO_USED`). - - name: disk, QEMU, CMake and Ninja + # the versions Ubuntu 24.04 released; Perl and make build the OpenSSL of + # the toolchain's cargo (src/main.rs's `ALSO_USED`). + - name: disk, QEMU, CMake, Ninja, Perl and make run: | sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ /usr/local/share/boost /usr/local/.ghcup sudo apt-get update -qq sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq qemu-system-x86 zstd \ - cmake=3.28.3-1build7 ninja-build=1.11.1-2 + cmake=3.28.3-1build7 ninja-build=1.11.1-2 perl make - env: GH_TOKEN: ${{ github.token }} diff --git a/CLAUDE.md b/CLAUDE.md index 285817eedf0..026aab0244f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ An operating system built from scratch in Rust, held to a production-grade engin | `kernel/CLAUDE.md` | the caveats that bite kernel work | | `userland/CLAUDE.md` | the server doctrine, and the caveats that bite userland work | | `tests/CLAUDE.md` | the caveats that bite the harness | -| `src/CLAUDE.md` | boot modes, the locks, worktrees — the operational file | +| `src/CLAUDE.md` | boot modes, the toolchain store, worktrees — the operational file | | `issues/README.md` | the issue tracker: one file per issue, typed by kind; `ls` is the index | | `.claude/agents/reviewer.md` | the review prompt the orchestrator spawns a reviewer with | @@ -82,7 +82,7 @@ The root `Cargo.toml`'s `[workspace]` `members` and `exclude` lists account for ## Workflow -**One agent, one worktree, one branch.** `cargo run -- --worktree add ` makes one; never `git worktree add` by hand — the naive path clones the rust fork's history and takes the machine-global toolchain name from every other checkout. The primary checkout is not a workspace: it owns `rust/`, the rustup link and `main`; `cargo run -- --sync` moves it onto whatever GitHub merged. +**One agent, one worktree, one branch.** `git worktree add --no-track -b wt/ origin/main` makes one and `git worktree remove ` takes it; never `git submodule update` in one — that clones the rust fork's history again. The primary checkout is not a workspace: it owns `rust/`, the rustup link and `main`; `cargo run -- --sync` moves it onto whatever GitHub merged. - Stay on the current task. File what you find in `issues/` and do not go fix it; one file per issue, its README has the shape. - If something blocks, stop and report it. Don't work around it. diff --git a/Cargo.toml b/Cargo.toml index 57ebd52d1eb..01f8f07f95c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,7 +109,7 @@ default-run = "toyos-build" fatfs = "0.3.6" fontdue = "0.9" gpt = "3.1.0" -# `statvfs` for `worktree::free_bytes`, and the unprivileged ICMP datagram +# `flock` for `src/dirlock.rs`, and the unprivileged ICMP datagram # socket `icmp::echo` asks a metal boot over, which is the platform call that # replaces a `ping` binary. libc = "0.2" diff --git a/README.md b/README.md index 670bfac6b13..5918498e447 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ One command, and a complete OS boots. No Make, no Docker, no LLVM to install, no cross-toolchain to assemble. Everything that boots is built by a toolchain in this repository. -Rust and QEMU, plus the four things `rustc`'s own bootstrap needs on every +Rust and QEMU, plus what `rustc`'s own bootstrap needs on every platform — [Prerequisites](#prerequisites) says exactly what they are and why nothing that boots goes near them. @@ -224,20 +224,24 @@ same font the kernel blits. - QEMU - A C compiler on `PATH` as `cc`, and a Python 3 - CMake and Ninja +- Perl and `make` `rustc` links every **host** binary through `cc`, which rustup does not install. `rust/x`, the entry point to rustc's own bootstrap, is a shell script whose whole job is to find a Python to run `bootstrap.py` with — so a clean clone needs one, and so does every toolchain change. And that bootstrap builds LLVM and clang from source with CMake and Ninja, whenever the LLVM commit -`rust/` names has not been built on the machine before. +`rust/` names has not been built on the machine before. The toolchain's cargo +is the fork's own, built with its compiler, and it carries its own OpenSSL, +which `openssl-src` configures with Perl and builds with `make`. Nothing in the OS goes near any of them. `bootloader/`, `kernel/` and `userland/` all link with the toolchain's `rust-lld`, and no image contains a C -toolchain or a Python. On macOS `cc` and Python arrive with the Xcode Command -Line Tools, and CMake and Ninja come from Homebrew (`brew install cmake -ninja`); on Debian and Ubuntu they are `build-essential`, `python3`, `cmake` -and `ninja-build`. +toolchain or a Python. On macOS `cc`, Python and `make` arrive with the Xcode +Command Line Tools and Perl with macOS itself, and CMake and Ninja come from +Homebrew (`brew install cmake ninja`); on Debian and Ubuntu they are +`build-essential`, which brings `make` and Perl, `python3`, `cmake` and +`ninja-build`. `cargo run` names anything it needs and cannot find, before it does anything else — including the Python that only the toolchain bootstrap runs, which diff --git a/examples/imgstat.rs b/examples/imgstat.rs index a3bda833daf..903a8d05dcc 100644 --- a/examples/imgstat.rs +++ b/examples/imgstat.rs @@ -94,15 +94,9 @@ fn main() { } } -/// Which of the four things on ROOT an entry is. -/// -/// The order matters: `bin/rustc` is the toolchain's, not userland's. +/// Which of the things on ROOT an entry is. fn group_of(name: &str) -> &'static str { - if name.starts_with("lib/") { - "hosted rustc lib/" - } else if name.starts_with("bin/rustc") { - "hosted rustc bin/" - } else if name.starts_with("bin/") { + if name.starts_with("bin/") { "userland bin/" } else if name.starts_with("share/") { "assets share/" diff --git a/issues/README.md b/issues/README.md index c617c43bc91..8577ff3715f 100644 --- a/issues/README.md +++ b/issues/README.md @@ -135,9 +135,7 @@ with it. slug as well as the path.** The slug is the identity, and a pointer written as a bare name is invisible to a path search. Search the *tree* rather than the checkout (`git grep `): `rg` skips dotfile directories without `--hidden`, -and `.github/` holds citations too. Then read where the hits are. One in a comment -under `toyos-abi/src`, `toyos/src` or a published crate changes no identity -(`src/identity.rs`), so it owes no version and builds no sysroot. One in +and `.github/` holds citations too. Then read where the hits are. One in `src/redlist.rs` is a disabled test's `issue`: the row goes with the file. ## Two area notes, carried over from the file this replaced diff --git a/issues/build/a-killed-keystore-writer-leaves-an-orphan-temp-in-target.md b/issues/build/a-killed-store-record-writer-leaves-an-orphan-temp-in-target.md similarity index 68% rename from issues/build/a-killed-keystore-writer-leaves-an-orphan-temp-in-target.md rename to issues/build/a-killed-store-record-writer-leaves-an-orphan-temp-in-target.md index 1d16051be45..795c17b3362 100644 --- a/issues/build/a-killed-keystore-writer-leaves-an-orphan-temp-in-target.md +++ b/issues/build/a-killed-store-record-writer-leaves-an-orphan-temp-in-target.md @@ -4,9 +4,9 @@ kind: tooling opened: 2026-09-29 --- -# A killed keystore writer leaves an orphan temp in `target/` +# A killed store record writer leaves an orphan temp in `target/` -`record_by` in `src/keystore.rs` writes a uniquely named temp beside the record +`record` in `src/store.rs` writes a uniquely named temp beside the record and renames it over. A writer killed between the write and the rename leaves that temp behind, and because its name is unique nothing later overwrites it: one orphan per kill. diff --git a/issues/build/a-lock-wait-in-the-build-has-no-ceiling.md b/issues/build/a-lock-wait-in-the-build-has-no-ceiling.md new file mode 100644 index 00000000000..f30050eae18 --- /dev/null +++ b/issues/build/a-lock-wait-in-the-build-has-no-ceiling.md @@ -0,0 +1,16 @@ +--- +status: open +kind: tooling +opened: 2026-09-29 +--- + +# A lock wait in the build has no ceiling + +Every blocking lock in `src/dirlock.rs` says what it waits for every 30 s and +waits for as long as its holder lives. A killed holder releases it, but a live +one that hangs — a bootstrap stuck in a fetch, a maker blocked on a terminal — +holds every build waiting for that key, that checkout or that store forever, +each of them saying so every 30 s and none of them failing. + +Exit: a wait whose holder has made no progress past a bound its kind of hold +declares fails loudly, naming the holder's pid and what it holds. diff --git a/issues/build/cargo-package-fails-in-a-linked-worktree.md b/issues/build/cargo-package-fails-in-a-linked-worktree.md index 69b8987674f..c49999d064e 100644 --- a/issues/build/cargo-package-fails-in-a-linked-worktree.md +++ b/issues/build/cargo-package-fails-in-a-linked-worktree.md @@ -7,20 +7,19 @@ opened: 2026-09-26 # `cargo package`/`cargo publish --dry-run` cannot run inside a linked worktree `cargo package -p ` (tried `toyos-abi` and `toyos-ld`) -reds with a bare `error: No such file or directory (os error 2)` in a worktree -made by `cargo run -- --worktree add`, right after cargo's own trace logs +reds with a bare `error: No such file or directory (os error 2)` in a worktree, +right after cargo's own trace logs `found a git repo` and `found (git) Cargo.toml`, inside `cargo::ops::cargo_package::vcs::check_repo_state` — before it prints anything about a dirty tree, and regardless of `--allow-dirty` or a fully clean working tree (confirmed with `git stash`). The identical command against the identical crate exits 0 in the **primary** checkout. Every worktree carries dozens of submodule entries (`userland/*`, `rust`) registered in the shared `.git/config` -but not checked out on disk (`rust`'s own empty stub included, per -`src/worktree.rs`) — a repo shape only a linked worktree has, and the likely -reason cargo's git-repo-state walk (`git2`) chokes there and not in the -primary. +but not checked out on disk (`rust`'s own empty stub included) — a repo shape +only a linked worktree has, and the likely reason cargo's git-repo-state walk +(`git2`) chokes there and not in the primary. -Reproduce: from any `--worktree add` checkout, `cargo package -p toyos-abi +Reproduce: from any linked worktree, `cargo package -p toyos-abi --no-verify --allow-dirty` exits 101 with that message; the same command in the primary checkout exits 0. diff --git a/issues/build/every-build-still-removes-an-llvm-no-build-makes-in-tree-any-more.md b/issues/build/every-build-still-removes-an-llvm-no-build-makes-in-tree-any-more.md deleted file mode 100644 index 8fc7f04ffff..00000000000 --- a/issues/build/every-build-still-removes-an-llvm-no-build-makes-in-tree-any-more.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-09-29 ---- - -# Every build still removes an in-tree LLVM that no build makes any more - -`llvm::retire_in_tree` removes a bootstrap build directory's own `/llvm`, -`/lld`, `/ci-llvm` and `cache/llvm-*`. It runs at four sites on -every build: the primary's `reassemble`, `compiler::place`, `compiler::choose` -on the way back to the primary's compiler, and the std build -(`sysroot::prepare_std_build`). - -Since the LLVM store, no build makes any of these. Every compiler build links -the store's `llvm-config`, and the std build sets `download-ci-llvm = false`. -What the four sites remove is what a build directory kept from before the -store. After a checkout's first build at that code, they remove nothing, but -they keep asking, on every build, a `read_dir` of `cache/` and a `stat` of -each name. - -Exit: delete `retire_in_tree`, `in_tree` and the four call sites once no -checkout that builds holds a build directory made before the store. A host -cannot know that for any other host, so the owner sets the date. diff --git a/issues/build/no-test-covers-the-pid-in-a-keystore-records-temp-name.md b/issues/build/no-test-covers-the-pid-in-a-store-records-temp-name.md similarity index 58% rename from issues/build/no-test-covers-the-pid-in-a-keystore-records-temp-name.md rename to issues/build/no-test-covers-the-pid-in-a-store-records-temp-name.md index 91a406b4732..f30b50ceb56 100644 --- a/issues/build/no-test-covers-the-pid-in-a-keystore-records-temp-name.md +++ b/issues/build/no-test-covers-the-pid-in-a-store-records-temp-name.md @@ -4,16 +4,16 @@ kind: tooling opened: 2026-09-29 --- -# No test covers the pid in a keystore record's temp name +# No test covers the pid in a store record's temp name -`record_by` in `src/keystore.rs` names its temp file -`...new` so that concurrent writers of one record never +`record` in `src/store.rs` names its temp file +`.-.new` so that concurrent writers of one record never share one. The counter distinguishes threads of one process; the pid is what distinguishes two processes in one worktree, and nothing tests it: dropping -`std::process::id()` from the format leaves `cargo test --lib keystore` green +`std::process::id()` from the format leaves `cargo test --lib store::` green (exit 0, measured by that mutation and that command) while two processes writing the same record share a temp name again. Exit condition: a test arm that races a child process against the parent on one -record, using the re-exec pattern of `rerun` in `src/buildlock.rs`, and goes red +record, using the re-exec pattern of `rerun` in `src/dirlock.rs`, and goes red when the pid is dropped from the format. diff --git a/issues/build/record-reads-compiler-source-after-the-bootstrap-it-records.md b/issues/build/record-reads-compiler-source-after-the-bootstrap-it-records.md deleted file mode 100644 index 9300b15baea..00000000000 --- a/issues/build/record-reads-compiler-source-after-the-bootstrap-it-records.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-09-28 ---- - -# `record` reads `compiler/` as it stands after the bootstrap it records - -`compiler::record` (`src/compiler.rs`) is called once `reassemble` has finished -building `stage2`, and it computes `source(rust_dir)` at that point — the -`compiler/` tree, working diff and untracked files as they read *then*, not as -they read when the bootstrap it is recording began. A `compiler/` edit made -while that bootstrap was running (`x.py build` takes minutes) is folded into -the record even though `stage2` was built without it, so the next build sees -`primary_is_current` return true for a `stage2` that does not contain the edit, -and skips a bootstrap that is actually owed. - -Exit condition: `record` (or whoever calls it) captures `source(rust_dir)` -before the bootstrap starts, not after, and a test edits `compiler/` mid-build -(a `bootstrap` closure that writes a file before returning) and asserts the -next `primary_is_current` is false. - -Owner: whoever next touches `compiler::record` or `rebuild_compiler`. diff --git a/issues/build/stub-worktrees-build-their-toolchains-one-at-a-time.md b/issues/build/stub-worktrees-build-their-toolchains-one-at-a-time.md new file mode 100644 index 00000000000..49699355d8f --- /dev/null +++ b/issues/build/stub-worktrees-build-their-toolchains-one-at-a-time.md @@ -0,0 +1,21 @@ +--- +status: open +kind: tooling +opened: 2026-09-30 +--- + +# Stub worktrees build their toolchains one at a time + +A linked worktree whose `rust/` is the stub builds every toolchain its key +lacks in the host's one shared checkout, `/rust/build/fork/`, held +exclusively for the whole build (`sysroot::Fork::checkout`). A second stub +worktree needing a key of its own waits behind the first: behind a compiler +build, which took 22:05 there (`Build completed successfully in 0:22:05`), or a +sysroot's std build, which took 6:29 (`0:06:29`), both from one +`cargo run -- --build-only` of this store's branch on this host. The old +layout built std in each worktree's own `rust/`, side by side. What the queue +costs with several stub worktrees building at once has not been measured. + +Exit: the wait of several stub worktrees whose keys differ is measured on the +host and the owner accepts that number, or their builds no longer share one +checkout. diff --git a/issues/build/the-ack-delay-abi-doc-names-one-cpu.md b/issues/build/the-ack-delay-abi-doc-names-one-cpu.md index aa9d5253f7e..3cf20b00d4c 100644 --- a/issues/build/the-ack-delay-abi-doc-names-one-cpu.md +++ b/issues/build/the-ack-delay-abi-doc-names-one-cpu.md @@ -21,9 +21,6 @@ sentence userland reads to learn what action 12 does describes a selection the kernel does not make, omits the answer it returns, and says nothing about how long what it leaves behind lasts. -`toyos-abi/src` is one of `toolchain::SYSROOT_SOURCES`, so the correction is a -single-commit branch of its own. - **Exit condition.** The doc names what the kernel does: each other CPU in turn, the smallest of those waits returned, and the arming left standing against every other CPU until a disarm or the end of the two-second window, whichever comes diff --git a/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md b/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md index 77ff0585707..ee88f3c4e2a 100644 --- a/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md +++ b/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md @@ -17,9 +17,20 @@ arrives and is not one. M4 and M5 are stages of `issues/build/toyos-builds-itsel | Python | LLVM's CMake, whenever this host builds an LLVM (`src/llvm.rs`) | admitted: no Rust tool does the job, LLVM's CMake requires one (`find_package(Python3 … REQUIRED)` in `rust/src/llvm-project/llvm/CMakeLists.txt`) | M5 runs it in the guest | | CMake | rustc's bootstrap, for LLVM and clang; `src/llvm.rs`, for the LLVM's key | admitted: no Rust tool does the job, LLVM, clang and LLD are described in CMake, and upstream's only other descriptions are a GN overlay it does not support and a Bazel one | M5 runs it in the guest | | Ninja | runs the build CMake generates for LLVM | refused: a Rust tool does it, n2 (`github.com/evmar/n2` at `b1fead5`), named `ninja` as its README directs for CMake: CMake's Ninja generator configured `rust/src/llvm-project/llvm` for it and it built `llvm-tblgen`, exit 0 each; named `n2`, CMake refuses its version, exit 1 | rustc's bootstrap builds LLVM under n2 | -| `git` for worktrees, submodules, checkouts, fixtures and rustc's bootstrap | adds, removes and prunes worktrees (`src/worktree.rs`, `src/sysroot.rs`); updates submodules (`src/lib.rs`, `src/sysroot.rs`, `src/licence.rs`, `src/release.rs`); fetches the fork from the primary's and checks it out (`src/sysroot.rs`); fast-forwards the primary (`src/sync.rs`); makes the tests' fixture repositories; runs inside rustc's bootstrap | admitted: no Rust tool does the job, gitoxide 0.85 adds, removes and prunes no worktree, updates no submodule, stages, resets and pushes nothing, checks out only a fresh clone and fetches a local path by spawning `git`; a fixture must be what `git` makes, and bootstrap runs `git` itself | M4 runs it in the guest | +| Perl | every compiler build (`src/compiler.rs`) builds the toolchain's cargo, whose `vendored-openssl` builds OpenSSL through `openssl-src`, and its `./Configure` and `util/dofile.pl` are Perl: on any host that builds a compiler, and on the nightly's `build` runner | admitted: no Rust tool does the job, upstream cargo hard-wires git2's `https` and `ssh` features, and `libssh2-sys` depends on `openssl-sys` unconditionally on Unix; strykelang 0.17.58 (`crates.io/crates/strykelang`), a Perl 5 in Rust, stops at `Configure` line 2141, exit 255 | the toolchain's cargo links no OpenSSL | +| `make` on a Linux host, GNU make | the same build: `openssl-src` runs `make depend`, `make build_libs` and `make install_dev`, which recurse through `$(MAKE)` | refused: a Rust tool does it, omake 0.2.0 (`crates.io/crates/omake`), measured below | Perl's | +| `make` on a macOS host, `/Library/Developer/CommandLineTools/usr/bin/make` | the same three targets, run by that path (the build script's record, `openssl-sys/012194ed4534a869/run/stdout:1461`) | refused: one host OS alone, it arrives only with Apple's Command Line Tools, which the macOS row refuses; and a Rust tool does it, omake, measured below | Perl's | +| `sh` under `make` and Perl | `make` runs every recipe line of those targets with `/bin/sh -c`, bash 3.2 on macOS, and their `echo`, `[`, `set` and `exit` are its builtins; Perl's `Configure` asks `cc` for its predefined macros through `/bin/sh` (`cc -dM -E -x c /dev/null 2>&1 \|`) | refused: a Rust tool does it, brush 0.4.0 (`crates.io/crates/brush-shell`), as `make`'s `SHELL`, measured below; Perl's `/bin/sh` was not replaced | Perl's | +| `rm` under `make` | `build_libs` empties each archive with `$(RM)`, `rm -f`, before it fills it, and each C object's recipe removes its new dependency file when `cmp` finds it unchanged (openssl's `Configurations/unix-Makefile.tmpl`) | refused: a Rust tool does it, uutils coreutils 0.12.0 (`crates.io/crates/coreutils`), measured below | Perl's | +| `touch` under `make` | each C object's recipe touches its new dependency file, silently, so no record shows it | refused: a Rust tool does it, uutils coreutils 0.12.0, measured below | Perl's | +| `cmp` under `make` | each C object's recipe compares its new dependency file with the old, silently | refused: a Rust tool does it, uutils diffutils 0.5.0 (`crates.io/crates/diffutils`), measured below | Perl's | +| `mv` under `make` | each C object's recipe moves a changed dependency file into place, and `install_dev` moves each library it installs into place | refused: a Rust tool does it, uutils coreutils 0.12.0, measured below | Perl's | +| `basename` under `make install_dev` | names each header, library, `.pc` and CMake file `install_dev` installs, silently | refused: a Rust tool does it, uutils coreutils 0.12.0, measured below | Perl's | +| `cp` under `make install_dev` | copies each of them, silently | refused: a Rust tool does it, uutils coreutils 0.12.0, measured below | Perl's | +| `chmod` under `make install_dev` | sets each one's mode, silently | refused: a Rust tool does it, uutils coreutils 0.12.0, measured below | Perl's | +| `git` for worktrees, submodules, checkouts, fixtures and rustc's bootstrap | adds and prunes worktrees (`src/sysroot.rs`); updates submodules (`src/lib.rs`, `src/licence.rs`, `src/release.rs`); checks the fork out and resets its submodules (`src/sysroot.rs`); fast-forwards the primary (`src/sync.rs`); makes the tests' fixture repositories; runs inside rustc's bootstrap | admitted: no Rust tool does the job, gitoxide 0.85 adds, removes and prunes no worktree, updates no submodule, stages, resets and pushes nothing, checks out only a fresh clone and fetches a local path by spawning `git`; a fixture must be what `git` makes, and bootstrap runs `git` itself | M4 runs it in the guest | | `git` for reads, a config write, and clones and fetches over HTTPS | `rev-parse`, `show-ref`, `for-each-ref`, `rev-list`, `log`, `branch --contains`, `merge-base`, `ls-tree`, `ls-files`, `cat-file`, `config --get-regexp`, `worktree list`, `status`, `diff`, `ls-remote` and `grep`, in the build system and its tests; `config --global --add safe.directory` in the nightly's containers; `src/sync.rs`'s fetch of `origin`; every workflow's checkout | refused: a Rust tool does it, gitoxide 0.85, which reads refs, objects, the index, config, worktrees and status, adds a value to a config file and writes it (gix-config 0.58's `File::section_mut_or_create_new`, `SectionMut::push`, `File::write_to`), walks history, diffs, and lists, fetches and clones a remote over HTTPS; `grep` is a search of the files its index names | those are gitoxide's | -| `cc`, `c++` and `ar` on a Linux host, `build-essential` on the nightly's runners | rustc links every host binary through `cc`; `cc` and `c++` compile LLVM, clang, LLD and `rustc_llvm` (`src/llvm.rs` names both to bootstrap) and `ring`'s C for `tests/https-server-host` and `tests/https-fetch-host`; `ar` archives what `cc::Build` compiles | admitted: no Rust tool compiles C or C++, or takes rustc's host link | M5: no host in the loop | +| `cc`, `c++` and `ar` on a Linux host, `build-essential` on the nightly's runners | rustc links every host binary through `cc`; `cc` and `c++` compile LLVM, clang, LLD and `rustc_llvm` (`src/llvm.rs` names both to bootstrap) and `ring`'s C for `tests/https-server-host` and `tests/https-fetch-host`; `ar` archives what `cc::Build` compiles; under `make`, `cc` compiles the toolchain cargo's OpenSSL and `ar` archives it and indexes it as `ar s`, the `RANLIB` bootstrap sets (`rust/src/bootstrap/src/core/builder/cargo.rs`) | admitted: no Rust tool compiles C or C++, or takes rustc's host link | M5: no host in the loop | | the toolchain's own `clang`, `llvm-ar`, `rust-lld` and `llvm-config`, built from `ToyOSOrg/llvm-project` | rustc links every guest binary with `rust-lld`; `clang` compiles the C corpus and `hello.c` (`tests/common/compile.rs`, `tests/common/clang.rs`) and, with `llvm-ar`, doomgeneric through `cc::Build` (`src/clang.rs`); rustc's bootstrap asks `llvm-config` how to link LLVM | admitted: our fork's C++, which ToyOS can one day build and run; no Rust tool compiles C, `cc::Build` archives with an `ar`, bootstrap reads LLVM through `llvm-config`, and `CLAUDE.md` links everything with `rust-lld` | M5: no host in the loop | | `ovmf-generic` | the UEFI firmware of the nightly's guest containers (`src/firmware.rs`), packaged by Debian apart from QEMU | admitted: QEMU's own firmware, and no Rust firmware does its job | the instrument's QEMU carries its own firmware | | `ca-certificates` | the trust store `git` and `curl` verify against in the nightly's containers | admitted: data both of them need | goes when neither runs there | @@ -42,8 +53,20 @@ arrives and is not one. M4 and M5 are stages of `issues/build/toyos-builds-itsel | `sudo` on macOS | `diag/flash.sh` and the README's macOS flashing steps run `dd` under it | admitted: no Rust tool raises a process to root on macOS; sudo-rs "is targeted for FreeBSD and Linux-based operating systems only" (its README at `89bae8a`) | goes with both flashes by hand | | `sudo` on Linux | the README's Linux flashing steps run `dd` under it | refused: a Rust tool does it, sudo-rs | the README's Linux steps run sudo-rs | | `sh` running rustup's `rustup-init.sh` | the nightly's three rustup installs, fetched with `curl` | refused: a Rust tool does it, rustup's own `rustup-init` binary | a job installs rustup without it | -| `nightly.yml`, job `build`, step "disk, QEMU, CMake and Ninja" | `sudo rm -rf` of five preinstalled SDK directories, then `sudo apt-get update` and `install` | refused: shell of our own | each step is one command | +| `nightly.yml`, job `build`, step "disk, QEMU, CMake, Ninja, Perl and make" | `sudo rm -rf` of five preinstalled SDK directories, then `sudo apt-get update` and `install` | refused: shell of our own | each step is one command | | `nightly.yml`, step `deps`, which jobs `guest` and `tcg` share | the snapshot archive read out of `debian.sources` with `sed`, checked with `test` and written back with `echo` and `rm`, a three-attempt `apt-get` loop with `sleep` and `cat`, `git config`, and rustup through `curl`, `sh` and `echo` | refused: shell of our own | each step is one command | | `nightly.yml`, job `portability-linux`, step "deps" | the same loop, `git config`, and rustup the same way | refused: shell of our own | each step is one command | | `nightly.yml`, job `portability-macos`, the rustup step | `curl`, `sh rustup-init.sh`, and `echo` into `$GITHUB_PATH` | refused: shell of our own | each step is one command | | `umask 077 && cat > ` | `src/metal.rs` stages the sudoers rule on the T14 with it | refused: shell of our own | Ubuntu leaves the metal loop | + +The rows under `make` name every tool `make -n depend`, `build_libs` and +`install_dev` print beside `cc`, `ar` and `perl`, and a Rust tool did each one's +work in its place. On macOS, OpenSSL 3.6.3 configured as `openssl-src` 300.6.1 +configures it for `darwin64-arm64-cc`, omake ran its Makefile with brush as +`SHELL` and uutils coreutils and diffutils first on `PATH`: the three targets +exited 0 each, the headers installed are identical to the compiler build's +record (`diff -r`, exit 0), and each library has the record's members (`ar t`, +`cmp`, exit 0). omake with the host's shell and tools, and GNU make with brush +and uutils, did the same, exit 0 each. The controls: omake with +`SHELL=/usr/bin/false` exits 2, and GNU make with a `touch` that is +`/usr/bin/false` first on `PATH` fails the object whose recipe runs it, exit 2. diff --git a/issues/build/the-build-system-does-not-compile-on-windows.md b/issues/build/the-build-system-does-not-compile-on-windows.md index 00af36ae92f..9a442895b78 100644 --- a/issues/build/the-build-system-does-not-compile-on-windows.md +++ b/issues/build/the-build-system-does-not-compile-on-windows.md @@ -4,28 +4,12 @@ kind: tooling opened: 2026-08-19 --- -# The build system does not compile on Windows, and it is three subsystems rather than one call +# The build system does not compile on Windows -Seven errors, in `buildlock`, `toolchain` and `worktree`. Measured 2026-09-01: +`libc` itself builds for `x86_64-pc-windows-msvc`. Every other crate in the +graph, first-party and third-party, checked clean. -``` -error[E0433]: cannot find `unix` in `os` src/buildlock.rs:59:14 (AsRawFd) -error[E0433]: cannot find `unix` in `os` src/toolchain.rs:927:14 (symlink) -error[E0433]: cannot find `unix` in `os` src/toolchain.rs:1813:14 (symlink) -error[E0425]: cannot find type `statvfs` in crate `libc` src/worktree.rs:305:24 -error[E0425]: cannot find function `statvfs` in crate `libc` src/worktree.rs:309:29 -error[E0599]: no method named `as_raw_fd` found for reference `&std::fs::File` - src/buildlock.rs:666:32 -error[E0599]: no method named `as_raw_fd` found for reference `&std::fs::File` - src/buildlock.rs:680:32 -``` - -`libc` itself builds for `x86_64-pc-windows-msvc`; it is `statvfs` that is not -there. Every other crate in the graph, first-party and third-party, checked -clean. The `#[cfg(unix)]` at `src/ci.rs:489` is still the only conditional -compilation in the build system. - -`src/tether.rs` is a fourth: `std::os::unix` and a pseudo-terminal per child, behind a Linux and macOS `cfg` pair with no Windows arm. +`src/tether.rs`: `std::os::unix` and a pseudo-terminal per child, behind a Linux and macOS `cfg` pair with no Windows arm. ## The judge, and it needs no Windows host and no download @@ -35,11 +19,13 @@ __CARGO_TESTS_ONLY_SRC_ROOT= CARGO_TARGET_DIR= \ --target x86_64-pc-windows-msvc --offline -p toyos-build --all-targets ``` -`` is `src/CLAUDE.md`'s std-src-root recipe with one addition: a +`` is `src/CLAUDE.md`'s std-src-root recipe with two additions: a workspace `Cargo.toml` whose members are `library/std`, `library/sysroot`, `library/proc_macro`, `library/panic_abort` and `library/test`, and whose `[patch.crates-io]` is `library/Cargo.toml`'s four entries with `library/` -prepended to each path. It works because the fork vendors `library/windows-sys` +prepended to each path; and `library/Cargo.lock` copied beside it, without +which `--offline` resolution refuses the yanked `moto-rt` 0.16.4. It works +because the fork vendors `library/windows-sys` and `library/windows_link`, so a Windows `std` builds from the tree — a plain `cargo check --target x86_64-pc-windows-msvc` instead says *"the `x86_64-pc-windows-msvc` target may not be installed"* and asks for @@ -47,26 +33,20 @@ and `library/windows_link`, so a Windows `std` builds from the tree — a plain ## Compiling is not working, and that is why the cheap half is refused -Each of the three wants a Windows call whose semantics differ in kind from the -Unix one it replaces, and none of the three can be run by anybody here: +Each wants a Windows call whose semantics differ in kind from the +Unix one it replaces, and none can be run by anybody here: - `std::os::windows::fs::symlink_dir` needs the privilege or developer mode - Windows does not grant by default, so `link_host_target` and - `provision_toolchain_cargo` would compile and fail at run time — the quieter - kind of broken. + Windows does not grant by default. - `flock` is advisory and whole-file; `LockFileEx` is mandatory and byte-range. - `buildlock` is what serialises the shared sysroot across every worktree. -- `statvfs` against `GetDiskFreeSpaceExW`. -So a green Windows compile would say nothing about a working Windows build, -and it would say it in the one subsystem whose failure mode is two checkouts -silently sharing a sysroot. +So a green Windows compile would say nothing about a working Windows build. ## The self-hosting question underneath The north star is that nothing rests on a host binary and that everything can eventually run inside ToyOS. `symlink` is the question in miniature: either -ToyOS grows symbolic links, or the two `toolchain.rs` sites need a shape that +ToyOS grows symbolic links, or the sites need a shape that does not need one — a copy, a directory junction, or a sysroot layout that does not require aliasing a directory at all. Deciding that is worth more than a -`#[cfg]` pair, and it decides two of the seven errors. +`#[cfg]` pair. diff --git a/issues/build/the-fork-checkout-runs-git-submodule-in-a-linked-worktree.md b/issues/build/the-fork-checkout-runs-git-submodule-in-a-linked-worktree.md index 96b22d6b0f2..2d091c1e29e 100644 --- a/issues/build/the-fork-checkout-runs-git-submodule-in-a-linked-worktree.md +++ b/issues/build/the-fork-checkout-runs-git-submodule-in-a-linked-worktree.md @@ -6,22 +6,15 @@ opened: 2026-09-30 # The fork checkout runs `git submodule` in a linked worktree -`sysroot::fork_checkout` makes a worktree's `rust/` a linked worktree of the -primary's `rust` (`git worktree add --detach`). When the primary's -`library/backtrace` lacks the commit the fork pins, it runs -`git submodule update --init library/backtrace` inside that linked worktree, -which `.claude/agents/implementer.md` forbids: `git submodule` in a linked -worktree writes `core.worktree` into shared config and breaks git in the -primary checkout's `rust/`. The orchestrator measured that for -`git submodule update rust` in a linked worktree of the monorepo, which set the -primary's `.git/modules/rust/config` `core.worktree` to a path that does not -exist; this arm is the same command one level down and is unmeasured. - -`ensure_submodule` (`src/lib.rs`) runs `git submodule update --init -library/backtrace` in the same fork checkout, from `sysroot::build_std` and -`compiler::build_in_fork`, whenever that checkout's `library/backtrace` is -empty or gone: what a first `fork_checkout` leaves when it stops after adding -the fork's worktree and before adding `library/backtrace`. +`sysroot::shared` makes the host's shared fork checkout a linked worktree of +the primary's `rust` (`git worktree add --detach`), and a linked worktree's own +`rust/` is one too. `ensure_submodule` (`src/lib.rs`) runs +`git submodule update --init library/backtrace` in either, from +`sysroot::build_std` and `compiler::build_in_fork`, whenever its +`library/backtrace` is empty or gone, which `.claude/agents/implementer.md` +forbids. The orchestrator measured that for `git submodule update rust` in a +linked worktree of the monorepo, which set the primary's +`.git/modules/rust/config` `core.worktree` to a path that does not exist. **Exit**: the build system runs no `git submodule` in a linked worktree's fork checkout. diff --git a/issues/build/the-hosted-rustc-is-not-built.md b/issues/build/the-hosted-rustc-is-not-built.md new file mode 100644 index 00000000000..cf69b57caa5 --- /dev/null +++ b/issues/build/the-hosted-rustc-is-not-built.md @@ -0,0 +1,16 @@ +--- +status: open +kind: track +opened: 2026-09-29 +--- + +# The hosted rustc is not built + +Nothing builds the ToyOS-hosted rustc (`x86_64-unknown-toyos`, Cranelift) any +more, and no image or release carries one. + +Exit: a store product keyed on a compiler and the ABI trees builds the hosted +rustc, the licences of the compiler it ships are read, an image carries it, and +a guest test compiles and runs a program with it +(`issues/build/toyos-builds-itself.md`, M3; +`issues/build/the-hosted-rustc-names-a-linker-toyos-does-not-have.md`). diff --git a/issues/build/the-hosted-rustc-names-a-linker-toyos-does-not-have.md b/issues/build/the-hosted-rustc-names-a-linker-toyos-does-not-have.md index 192cc12fed5..00baa0b6ce9 100644 --- a/issues/build/the-hosted-rustc-names-a-linker-toyos-does-not-have.md +++ b/issues/build/the-hosted-rustc-names-a-linker-toyos-does-not-have.md @@ -6,13 +6,12 @@ opened: 2026-09-26 # The ToyOS-hosted rustc names a linker ToyOS does not have -`x86_64-unknown-toyos` names `rust-lld`, and the rustc `src/toolchain.rs` -builds for a ToyOS host carries that target spec into the guest, where no +`x86_64-unknown-toyos` names `rust-lld`, and the rustc for a ToyOS host +carries that target spec into the guest, where no `rust-lld` exists: LLD runs on ToyOS only once clang and libc++ do. The linker that does run there is the frozen `/system/bin/toyos-ld`, which that rustc reaches only when told `-C linker=toyos-ld`. No image ships the hosted rustc -today — `src/build.rs` refuses `hosted-rustc = true` until its licences are -read — so nothing links through it yet. +today, so nothing links through it yet. Exit: the hosted rustc links a program inside ToyOS through a linker the image carries, and a guest test compiles and runs one. diff --git a/issues/build/the-primarys-compiler-record-is-written-by-truncate-then-write.md b/issues/build/the-primarys-compiler-record-is-written-by-truncate-then-write.md deleted file mode 100644 index 80ef1a72c99..00000000000 --- a/issues/build/the-primarys-compiler-record-is-written-by-truncate-then-write.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-09-29 ---- - -# The primary's compiler record is written by truncate-then-write - -`record` in `src/compiler.rs` writes `build/toyos-compiler` with `fs::write`, -which truncates and then writes. A linked worktree reads that file in -`primary_is` without the primary's lock, so a read between the two sees an empty -or partial record. It compares unequal to what the worktree's source names, and -the worktree builds a compiler of its own that the primary already has. - -Exit condition: the record is written whole or not at all (a temp beside it, -renamed over), as `keystore::record` does for its records. diff --git a/issues/build/the-shipping-kernel-actuator-gate-reads-the-checkouts-own-path.md b/issues/build/the-shipping-kernel-actuator-gate-reads-the-checkouts-own-path.md index 1d6a5be3da3..b9c201def8b 100644 --- a/issues/build/the-shipping-kernel-actuator-gate-reads-the-checkouts-own-path.md +++ b/issues/build/the-shipping-kernel-actuator-gate-reads-the-checkouts-own-path.md @@ -11,9 +11,7 @@ carries an actuator by searching the kernel image for each declared name as a byte string. The kernel image embeds absolute source paths, so the search also matches the directory the checkout sits in. -Measured on a worktree made by the documented command, -`cargo run -- --worktree add /Users/jan/Dev/jan/toyos-heartbeat`, at -`dc38a054`: +Measured on a worktree at `dc38a054`: ``` $ cargo run -- --build-only diff --git a/issues/build/the-toolchain-store-shares-its-host-with-the-layout-it-replaced.md b/issues/build/the-toolchain-store-shares-its-host-with-the-layout-it-replaced.md new file mode 100644 index 00000000000..53449e2279e --- /dev/null +++ b/issues/build/the-toolchain-store-shares-its-host-with-the-layout-it-replaced.md @@ -0,0 +1,45 @@ +--- +status: open +kind: tooling +opened: 2026-09-29 +--- + +# The toolchain store shares its host with the layout it replaced + +Until every worktree on a host has merged the store (`src/store.rs`), two +layouts build into one `rust/build/`, and neither sees the other's holds: + +- A build on the old layout runs `keystore::sweep`, which removes every key no + worktree records and every `.*` name in a store directory. A store key + that a build holds by `flock` and has not yet recorded is in reach of it. +- `store::collect` sees neither the old layout's `buildlock` holds nor its + `.making` claims, so a key an old-layout build is using and no worktree + records is in reach of it. +- Every worktree `rust/` the old layout made carries `library/backtrace` as a + git worktree of the primary's clone (20 on this host, `git worktree list` + there). An old-layout build in one whose backtrace is at any commit but its + gitlink — after a gitlink bump, or after the checkout moved, which leaves its + submodules where they were — has bootstrap run `git submodule update` over it + (`update_submodule` in the fork's `src/bootstrap/src/core/config/config.rs`), + which rewrites that clone's `core.worktree`, as `git submodule update rust` + did to the primary's fork repository once. A store build refuses such a + checkout by name before bootstrap runs in it (`sysroot::Fork::checkout`). +- An old-layout sweep takes a name up to its first dot for a key, so Finder's + `.DS_Store` in a store directory (`rust/build/sysroots/` and + `rust/build/llvm/` on this host) is the key `""`, whose lock is + `.git/toyos-build-locks/sysroots/` itself: the sweep panics "build lock: open + …/toyos-build-locks/sysroots/: Is a directory", as main's `--worktree remove` + did, and so does every old-layout build that places a key, after placing it. + +And the old layout leaves on disk what nothing on the store reads: +`rust/build//stage2` and the rest of `rust/build/` in the primary +(54G on this host, `du -sh`), `rust/build/x86_64-unknown-toyos`, +`rust/build/toyos-compiler`, `rust/build/toyos-sysroot-claimant`, +`.git/toyos-build-locks`, each worktree's `.build-locks/` (12 on this host) and +the `.build-locks/` line in `.gitignore`. + +Exit: no registered worktree on the host builds with a tree older than the +store's landing, and no fork checkout's submodule is a git worktree of another +clone (`git worktree list` in the primary's backtrace clone names that clone +alone); then what is listed above is removed and the `.gitignore` line goes, +in one pull request that closes this. diff --git a/issues/build/the-tooling-is-a-review-prompt-and-three-workflows.md b/issues/build/the-tooling-is-a-review-prompt-and-three-workflows.md index 81c48178ae0..f26342ecc2b 100644 --- a/issues/build/the-tooling-is-a-review-prompt-and-three-workflows.md +++ b/issues/build/the-tooling-is-a-review-prompt-and-three-workflows.md @@ -12,22 +12,17 @@ and the gates that held them go. - A test is green and fast or it is deleted in the same pull request and filed; then `src/redlist.rs`, `35383398^:src/durations.rs` and `tests/test-durations` have no subject and go. -- The toolchain is content-addressed by the four trees that produce it, one - directory per hash, never mutated; then the sysroot claim, `src/buildlock.rs` - and `src/worktree.rs` go. - The toolchain builds cargo from the Rust fork's submodule and ships it: one - cargo matching rustc. - - A shared cargo `target-dir` is safe only under `-Z checksum-freshness`; - stable cargo ignores it silently. - - The build system and the worktree config invoke the shipped cargo and no - other. - - `kernel/`, `bootloader/` and `userland/` inherit a redirected - `build.target-dir` unless their `.cargo/config.toml` sets - `target-dir = "target"`. - - `stage_artifact` in `src/build.rs` builds its path outside - `hostws::target_dir`. - - `cargo clean` follows a shared target dir. - - Artifact size, bootstrap delta and CI cache keys are unmeasured. +- A shared cargo `target-dir` is safe only under `-Z checksum-freshness`; + stable cargo ignores it silently. +- The build system and the worktree config invoke the shipped cargo and no + other. +- `kernel/`, `bootloader/` and `userland/` inherit a redirected + `build.target-dir` unless their `.cargo/config.toml` sets + `target-dir = "target"`. +- `stage_artifact` in `src/build.rs` builds its path outside + `hostws::target_dir`. +- `cargo clean` follows a shared target dir. +- Artifact size and CI cache keys are unmeasured. - The nine workflows become three — `pr`, `nightly`, `publish`; then `a5b25a75^:src/mergehealth.rs` goes, and the ABI-lands-alone rule moves into the review prompt. diff --git a/issues/build/worktree-remove-leaves-wt-branch-behind-so-the-name-is-refused-later.md b/issues/build/worktree-remove-leaves-wt-branch-behind-so-the-name-is-refused-later.md index f5fecf0dbdc..edc50672d8e 100644 --- a/issues/build/worktree-remove-leaves-wt-branch-behind-so-the-name-is-refused-later.md +++ b/issues/build/worktree-remove-leaves-wt-branch-behind-so-the-name-is-refused-later.md @@ -4,16 +4,13 @@ kind: tooling opened: 2026-09-29 --- -# `--worktree remove` leaves the local `wt/` branch behind, so the name is refused later +# `git worktree remove` leaves the local `wt/` branch behind, so the name is refused later -`remove` unregisters and deletes the worktree directory but never deletes the -local branch (`src/worktree.rs:391`, the `eprintln!` that says so). If nothing -was ever committed on it, the branch is an ancestor of `origin/main` and -`add`'s `refuse_if_no_commit_beyond_main` (`src/worktree.rs:162`) later refuses -the same name — correctly, since it truly carries no commit beyond -`origin/main`, but the caller still has to notice and run `git branch -d +`git worktree remove ` unregisters and deletes the worktree and keeps +its branch, and `git worktree add --no-track -b wt/ origin/main`, +the command `CLAUDE.md` gives, then refuses the name: `fatal: a branch named +'wt/' already exists`, exit 255. Somebody has to run `git branch -d wt/` by hand before the name is usable again. -**Exit**: `remove` deletes the local branch itself when it carries no commit -beyond `origin/main` (the same check `add` uses), so a name that was never -used becomes free again without a manual step. +**Exit**: a name whose branch carries no commit beyond `origin/main` is usable +again after `git worktree remove` without a manual step. diff --git a/issues/diagnostics/a-record-cannot-name-thread-zero.md b/issues/diagnostics/a-record-cannot-name-thread-zero.md index 3003c5330b0..0a788a7d3a3 100644 --- a/issues/diagnostics/a-record-cannot-name-thread-zero.md +++ b/issues/diagnostics/a-record-cannot-name-thread-zero.md @@ -45,9 +45,8 @@ one that drops `tid=0`. ## Why it was not fixed there -`toyos-abi/src/log.rs` is a sysroot source (`src/toolchain.rs`'s -`SYSROOT_SOURCES`), so an ABI change lands on its own pull request and the -kernel-side commit could not carry one. +`toyos-abi/src/log.rs` is a sysroot source, so an ABI change lands on its own +pull request and the kernel-side commit could not carry one. ## The options diff --git a/issues/filesystem/log-flush-retry-hung-arm-reds-with-no-offline-line.md b/issues/filesystem/log-flush-retry-hung-arm-reds-with-no-offline-line.md new file mode 100644 index 00000000000..426f8d2ac65 --- /dev/null +++ b/issues/filesystem/log-flush-retry-hung-arm-reds-with-no-offline-line.md @@ -0,0 +1,53 @@ +--- +status: open +kind: defect +opened: 2026-09-30 +--- + +# `log_flush_retry`'s hung arm reds with no " is offline: " line + +Nightly run 36709239346, guest shard 10 of 12, at `886cee668` (a head of +`wt/toyos-castore`, which touches nothing under `kernel/`, `userland/logd/` or +`tests/common/volumes.rs`: `git diff --stat 84471bc58...886cee668` over those +paths is empty), reds `log_flush_retry`. Main's nightly 36696295750, at +`ace064f9`, has guest (10) green. The run's log, from its first failure line +to the verdict: + +``` +2026-09-30T14:14:48.7585381Z FAIL log_flush_retry: no " is offline: " in the log, so the staged hung device never met its recovery: +2026-09-30T14:14:48.7585953Z what it said: +2026-09-30T14:14:48.7586444Z [kernel 0.195 cpu0] gpt: the boot volume names 74F67C14-767C-42F3-AF92-65F64C089D73 as the log partition +2026-09-30T14:14:48.7587214Z [kernel 0.196 cpu0] gpt: firmware booted us from partition D432BA2E-E295-40D8-9E6F-CB4B0A98DF78 at LBA 2048+69632 +2026-09-30T14:14:48.7589989Z [kernel 0.257 cpu0] gpt: device 1 carries the DATA candidate AD636D9A-EFA6-49BD-A166-3780FFE2E253 at LBA 2048+258048 +2026-09-30T14:14:48.7591038Z [kernel 0.259 cpu0] gpt: device 1 has 1 partitions and none of them is ours +2026-09-30T14:14:48.7591946Z [kernel 0.397 cpu0] usb-storage: slot 1 vendor "QEMU " product "QEMU HARDDISK " +2026-09-30T14:14:48.7592809Z [kernel 0.399 cpu0] usb-storage: slot 1 serial number "TOYOS0BOOTSTICK1" +2026-09-30T14:14:48.7593777Z [kernel 0.401 cpu0] usb-storage: disk 0 ready on slot 1, 28672 blocks of 512 B (112 MiB), msc_block +0x10000 +2026-09-30T14:14:48.7594959Z [kernel 0.405 cpu0] usb-storage: 1 device(s) +2026-09-30T14:14:48.7595921Z [kernel 0.412 cpu0] gpt: device 16 carries the log partition 74F67C14-767C-42F3-AF92-65F64C089D73 at LBA 73728+69632, entry 2 of 5 +2026-09-30T14:14:48.7597468Z [kernel 0.414 cpu0] gpt: device 16 carries the boot partition at LBA 2048+69632 (512-byte blocks), entry 0 of 5 on disk BD204DF6-758B-42D6-B62B-052F4FED7BD6 +2026-09-30T14:14:48.7598993Z [kernel 0.427 cpu0] boot-volume: partition mounted from device 16, 35651584 bytes of a 35651584-byte partition at device offset 1048576, 512-byte sectors, 512-byte clusters, 68552 clusters +2026-09-30T14:14:48.7600219Z [kernel 0.431 cpu0] log-volume: partition mounted from device 16, 35651584 bytes of a 35651584-byte partition at device offset 37748736, 512-byte sectors, 512-byte clusters, 68552 clusters +2026-09-30T14:14:48.7601302Z [kernel 0.499 cpu1] usb-storage: 00:02.0 slot 1 transport broke on SCSI 0x2a: a staged break skipped the data phase wait; break 1 of 3 running +2026-09-30T14:14:48.7602354Z [kernel 0.499 cpu1] usb-storage: 00:02.0 slot 1 is owed the data of the command that broke, so nothing can be asked of it on the Bulk-Out: its port is reset with no class reset before it +2026-09-30T14:14:48.7603384Z [kernel 0.550 cpu1] usb-storage: 00:02.0 slot 1 would not take SET_CONFIGURATION(1) after its port reset: a staged break skipped the status stage wait +2026-09-30T14:14:48.7604196Z [kernel 0.550 cpu1] usb-storage: 00:02.0 slot 1 the port reset was not answered; break 2 of 3 running +2026-09-30T14:14:48.7605208Z [kernel 0.550 cpu1] usb-storage: 00:02.0 slot 1 broke 2 times running; its port reset did not bring the transport back +2026-09-30T14:14:48.7605816Z {0.556 logd} logd: cannot create /log/2026-09-30-141447.log: other error +2026-09-30T14:14:48.7606412Z {0.556 logd} logd: no /log on this machine - this boot's kernel log is on the console only (2026-09-30 14:14:47 UTC) +2026-09-30T14:14:48.7606917Z FAIL log_flush_retry (8s) +``` + +The staged break (`usb-transport-break`, `usb-reset-break`) ran its first two +breaks, and the kernel said the port reset did not bring the transport back; +logd's give-up line follows 6 ms of guest time later, and no line holding +" is offline: " is in the log. The hung arm (`tests/common/volumes.rs`) reads +the console only until the first line holding "on the console only", and then +requires all three of "transport broke on SCSI", "the port reset was not +answered; break 2 of 3 running" and " is offline: ". + +`issues/filesystem/log-flush-retry-deadman-arm.md` records the test's other +ways to red; this shape is not among them. + +**Exit**: the hung arm reads " is offline: " whenever the staged break runs, +or the test waits for the line the kernel's recovery actually ends on. diff --git a/issues/kernel/the-kernel-still-parses-what-userland-writes.md b/issues/kernel/the-kernel-still-parses-what-userland-writes.md index cc6206ffb9d..9ab795122da 100644 --- a/issues/kernel/the-kernel-still-parses-what-userland-writes.md +++ b/issues/kernel/the-kernel-still-parses-what-userland-writes.md @@ -29,11 +29,8 @@ seven of the loader's twelve bounds are over quantities a workload sets and two have no bound at all; and two of those ceilings are *already* exceeded by artifacts this tree builds. -**It has a deadline.** Nothing shipped is dynamically linked today, so the move -is pure deletion. The day `hosted-rustc` turns on, a very large shared object is -dlopened into a kernel whose cache never evicts, and every one of those bounds -becomes load-bearing at once. Do it after the completion architecture lands and -before that day. Independent of everything else; may run as soon as a slot frees. +Nothing shipped is dynamically linked today, so the move +is pure deletion. Do it after the completion architecture lands. Independent of everything else; may run as soon as a slot frees. **Move 2 — filesystem daemons**, sequenced after the completion architecture. A crafted image attacks the kernel rather than a sandboxed daemon. The FS daemon diff --git a/src/CLAUDE.md b/src/CLAUDE.md index b20e2caddc6..57cc5acd1d8 100644 --- a/src/CLAUDE.md +++ b/src/CLAUDE.md @@ -16,17 +16,18 @@ Loads when you read a file under `src/` — the root cargo project, package name - `system.toml` defines which programs to build and the init sequence. - **A workflow step runs `cargo run -- --ci ` and nothing else; logic in YAML is a defect.** -## The host's locks +## The toolchain store -- **Sysroots are content-addressed** (`src/sysroot.rs`): one per key — the identity (`src/identity.rs`, so a comment is no change) of `toyos-abi/src`, `toyos/src`, `userland/libc/src` and their manifests, the std fork's `rust/library/` and `rust/src/bootstrap/`, and the compiler — at `rust/build/sysroots//`, made by whichever worktree first needs it. Every build compiles against its own key's, so two worktrees with different ABIs never refuse or wait for each other; the only shared step is the primary's compiler, which a sysroot build reads under the global lock in shared mode. A new key costs one std build of the three guest targets; `--worktree remove` sweeps the keys no worktree records. -- **The std fork is built per worktree, and nothing but the primary's own sync moves the primary's `rust/`.** A linked worktree's `rust/` becomes, on its first build, a git worktree of the primary's fork repository at the commit its tree pins — that is where the fork is edited, committed and pinned. A worktree whose fork `compiler/` differs from the one the primary built builds its own compiler, keyed by that source and placed beside the primary's without touching it. If that checkout later falls behind the commit its tree pins (a merge moved the pin), the build moves the checkout to it itself, fetching from the primary's repository first if it holds the commit, unless the checkout has local changes, which it refuses to move out from under. -- `src/buildlock.rs` serialises the stateful phases in two scopes: `Global` (the primary's compiler and the rustup link — one directory in `.git/`, shared by every worktree) and `Worktree` (the crate-target cleans, and this worktree's std build). Only `./x.py` typed by hand in `rust/` escapes it. -- **Never kill a build that has taken the global lock** — the kill removes the shell wrappers, not the bootstrap, which inherits the file descriptor and runs on regardless; a toolchain rebuild interrupted or unobserved this way can leave `stage2/bin` without a `cargo`. -- **The lock order is a constraint at every acquirer** — `src/buildlock.rs`'s header states it. Every blocking lock repeats itself every 30 s — a queue is never silence. +- **Every LLVM, compiler and sysroot is a directory of the store** (`src/store.rs`): `rust/build///` in the primary, one per key, read-only, placed whole by a rename or not at all, and made by whichever checkout first needs it — the primary is no different. A key is one function of a recipe and the git hashes of the four trees the toolchain is built from: the rust fork, `toyos-abi`, `toyos` and `userland/libc`, edits included. A build compiles against its own sysroot's key, so two worktrees with different ABIs never refuse each other. +- **The rustup `toyos` toolchain names `rust/build/toyos`**, a link the primary's build moves to its sysroot by a rename. +- **A toolchain is built in a fork checkout beside the building worktree's ABI trees** (`src/sysroot.rs`): the primary's `rust/`; a linked worktree's own `rust/` while it holds fork work its pin does not (`git -C /rust worktree add --detach /rust ` makes one); otherwise the host's one shared checkout, `/rust/build/fork/`, which such builds make toolchains in one at a time. A worktree whose `rust/` is the stub holds no fork state. +- **A killed build places nothing**; the bootstrap the kill orphans runs on in its fork checkout's build directory. +- A lock is `flock` on a directory (`src/dirlock.rs`), and every blocking one repeats itself every 30 s — a queue is never silence. ## Worktrees -- Everything under a worktree — targets, images, `.build-locks/`, its fork checkout — is its own; the object stores, the compiler and the rustup link are the primary checkout's, and ownership is derived from `git rev-parse --git-common-dir`, never recorded. +- `git worktree add --no-track -b wt/ origin/main` makes one and `git worktree remove ` takes it. A worktree holding a fork checkout of its own in `rust/` is refused by git until that checkout goes first. Never `git submodule update` in a linked worktree: that is a second 913 MiB clone of the fork. +- Everything under a worktree — targets, images, its fork checkout — is its own; the object stores, the store and the rustup link are the primary checkout's, and ownership is derived from `git rev-parse --git-common-dir`, never recorded. - **A linked worktree's `main` ref is only as current as the primary's last `--sync`: anything asking "does this branch differ from main" diffs against `origin/main`.** - **Type-checking a std edit without building a sysroot**: point `__CARGO_TESTS_ONLY_SRC_ROOT` at a tree holding an APFS clone of `rust/library` (`cp -Rc`), a workspace `Cargo.toml` naming `library/std`, and symlinks to `toyos-abi`/`toyos`; then `CARGO_TARGET_DIR= cargo +toyos build -Z build-std=std,panic_abort --target x86_64-unknown-toyos --offline`. Delete `/**/.fingerprint/std-*` between runs — cargo does not re-fingerprint std under `-Zbuild-std`. @@ -34,4 +35,3 @@ Loads when you read a file under `src/` — the root cargo project, package name - **Every CI lane is GitHub-hosted and no workflow may name a self-hosted label** — a `runs-on:` naming one queues until it times out rather than failing, so `src/ci.rs`'s `workflows_run_against_main_on_hosted_runners` refuses it; a measurement owed on hardware goes to the metal loop, not to a runner. - **A workflow job that runs in a container adds `safe.directory` itself** — `actions/checkout` sets it into a temporary global config it discards when its step ends, so the first git command a container step runs after checkout dies on a dubiously-owned repository. -- **A `stage1-std//dist/deps` temp-dir error means a concurrent build**, never a broken checkout; never repair or force-rebuild the toolchain. diff --git a/src/build.rs b/src/build.rs index 7e43ae2bd8d..95a28084ba9 100644 --- a/src/build.rs +++ b/src/build.rs @@ -12,7 +12,7 @@ use serde::Deserialize; use crate::arch::Arch; use crate::assets; -use crate::buildlock; +use crate::dirlock::Lock; use crate::flags; use crate::hostws; use crate::image; @@ -72,15 +72,13 @@ impl Drop for ArtifactBuildTimer { // --- Config --- #[derive(Deserialize)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] struct SystemConfig { #[serde(default)] programs: BTreeMap, #[serde(default)] symlinks: BTreeMap, #[serde(default)] - hosted_rustc: bool, - #[serde(default)] assets: Vec, /// What `/system/bin/init` starts at boot. Program *keys*, never paths — a path /// here is a second spelling of a `[programs]` key and is what let a boot @@ -277,30 +275,22 @@ fn clean(root: &Path, crate_dir: &Path, kind: Clean, fingerprint: &str) { /// `target//.cargo-lock`, inside what the clean deletes. Two processes /// that each decided before either acted would still both clean, which is the /// pair of `cargo clean`s that died with ENOENT on each other's files. -fn invalidate_stale( - root: &Path, - lock: &mut buildlock::Held, - toolchain: &Path, - targets: &[(PathBuf, Clean)], -) { - lock.act_if( - buildlock::Scope::Worktree, - "clean crate targets against changed external deps", - || { - let fp = external_fingerprint(toolchain); - let work: Vec<(PathBuf, Clean)> = targets - .iter() - .filter(|(dir, _)| stale(root, dir, &fp)) - .cloned() - .collect(); - (!work.is_empty()).then_some((fp, work)) - }, - |(fp, work)| { +fn invalidate_stale(root: &Path, lock: &mut Lock, toolchain: &Path, targets: &[(PathBuf, Clean)]) { + let work = || { + let fp = external_fingerprint(toolchain); + let work: Vec<(PathBuf, Clean)> = targets.iter().filter(|(dir, _)| stale(root, dir, &fp)).cloned().collect(); + (!work.is_empty()).then_some((fp, work)) + }; + if work().is_none() { + return; + } + lock.exclusively("cleaning crate targets against changed external deps, behind the other builds in this worktree", || { + if let Some((fp, work)) = work() { for (dir, kind) in work { clean(root, &dir, kind, &fp); } - }, - ); + } + }); } /// Every target directory a config builds into, and how much of each goes when @@ -417,9 +407,6 @@ pub const PROFILE: &str = "toyos"; #[derive(Clone)] struct GuestEnv { toolchain: PathBuf, - /// Whether that sysroot's compiler is the primary's, the one the hosted - /// rustc is built from (`src/compiler.rs`). - primary_compiler: bool, /// The public key the loader and `/system/bin/update` embed /// (`signing::KEY_ENV`): every guest build carries it, so no crate that /// names it can be built without it. @@ -432,7 +419,6 @@ impl GuestEnv { fn new(sysroot: &crate::sysroot::Sysroot) -> Self { Self { toolchain: sysroot.dir.clone(), - primary_compiler: sysroot.primary_compiler, image_key: crate::signing::key().public_hex(), floor_scope: crate::signing::key().floor_scope().word(), } @@ -499,7 +485,7 @@ fn cargo_build( // userland build and root-image assembly, and only then reads the artifact back. // Seconds to minutes, during which another config's build overwrites it. // -// So: hold [`buildlock::artifact`] across each build→stage pair, and copy the +// So: hold [`artifact`] across each build→stage pair, and copy the // artifact to a name carrying what it is actually keyed by. Readers use the // staged name, which no other config can overwrite. // @@ -539,8 +525,23 @@ fn key_hash(parts: &[&str]) -> u64 { h.finish() } +/// This worktree's build lock, shared, held for a build's whole length so no +/// clean of its crate targets ([`invalidate_stale`]) lands inside it: the +/// worktree's own directory, which no build removes. +fn worktree_lock(root: &Path, what: &str) -> Lock { + Lock::shared(root, &format!("{what}, behind a clean of this worktree's crate targets")) +} + +/// The lock over the shared cargo artifact paths, held across each +/// build→stage pair: this worktree's `target/`, which no build removes. +fn artifact(root: &Path) -> Lock { + let target = root.join("target"); + fs::create_dir_all(&target).unwrap_or_else(|e| panic!("create {}: {e}", target.display())); + Lock::exclusive(&target, "staging artifacts, behind another build in this worktree") +} + /// Copy a just-built artifact to a path carrying its build key, and return that -/// path. Must be called with [`buildlock::artifact`] held, before anything else +/// path. Must be called with [`artifact`] held, before anything else /// can rebuild the same crate. fn stage_artifact(root: &Path, built: &Path, stem: &str, key: u64) -> PathBuf { let staged = root.join(format!("target/{stem}-{key:016x}")); @@ -742,22 +743,6 @@ fn build_and_assemble( build_programs(root, config, env, quiet, arch, &mut root_files); root_files.push((toyos_manifest::PATH.to_string(), render_manifest(config))); - if config.hosted_rustc { - assert!( - arch == toolchain::HOSTED_ARCH, - "hosted-rustc is built to run on {}, and this image is for {}", - toolchain::HOSTED_ARCH.name(), - arch.name() - ); - assert!( - env.primary_compiler, - "hosted-rustc ships the primary checkout's hosted compiler, and this worktree builds with \ - a compiler of its own (src/compiler.rs): the image would carry a rustc that is not the \ - one its programs were built with" - ); - collect_hosted_rustc(root, &env.toolchain, &mut root_files); - } - if !config.assets.is_empty() { let programs: BTreeSet<&str> = config.programs.keys().map(String::as_str).collect(); root_files.extend(assets::collect(&config.assets, &programs)); @@ -772,11 +757,7 @@ fn build_and_assemble( .map(|(k, v)| (k.clone(), v.clone())) .collect(); - let mut programs: BTreeSet<&str> = config.programs.keys().map(String::as_str).collect(); - if config.hosted_rustc { - // `collect_hosted_rustc` puts it there and no row can. - programs.insert("rustc"); - } + let programs: BTreeSet<&str> = config.programs.keys().map(String::as_str).collect(); // Targets are inventoried beside the files: `bin/ls -> /system/bin/ghost` reaches a // program as surely as a file would, and the files alone walk past it. let targets: Vec = @@ -862,7 +843,7 @@ fn build_programs( // says nothing about a read between them — `ioapic_topology` died on // `Failed to read binary for toybox` while another worker's config was // relinking it, and was green the moment it was re-run alone. - let _artifact = buildlock::artifact(root); + let _artifact = artifact(root); if !workspace_packages.is_empty() { let mut extra: Vec<&str> = Vec::new(); for pkg in &workspace_packages { @@ -1126,20 +1107,11 @@ pub struct Shipped { } /// [`Shipped`], read out of the modes' configs the way [`build`] reads them. -/// -/// **A config that ships the hosted compiler is refused**: its dependencies are -/// the rust fork's `compiler/` workspace, which no reader of this answer walks. pub fn shipped(root: &Path) -> Result { let mut crates = BTreeSet::new(); let mut assets = BTreeSet::new(); for boot in [Boot::shipped(root), Boot::diag(root), Boot::console(root)] { let config = parse_config(&boot.config); - if config.hosted_rustc { - return Err(format!( - "{} sets hosted-rustc, and nothing reads the licences of the compiler it ships", - boot.config.display() - )); - } crates.extend(config_crates(root, &config).into_iter().map(|c| (c.dir, c.features))); assets.extend(config.assets.iter().map(|dir| root.join(dir))); } @@ -1725,9 +1697,9 @@ fn assert_sched_check_matches_features(features: &str, kernel: &[u8]) { /// shipped uncertified. There is now one place to add an assertion, and it is /// the kernel of every image this build system produces that gets it. The /// caller has already run `cargo_build` on the kernel crate and must hold -/// [`buildlock::artifact`], since the stage below copies the shared cargo path. +/// [`artifact`], since the stage below copies the shared cargo path. /// Stage the loader `arch`'s build just wrote, under the key that names it. -/// The caller holds [`buildlock::artifact`], as [`stage_and_certify_kernel`]'s does. +/// The caller holds [`artifact`], as [`stage_and_certify_kernel`]'s does. fn stage_loader(root: &Path, arch: Arch, env: &GuestEnv) -> PathBuf { stage_artifact( root, @@ -1838,8 +1810,8 @@ fn shipped_parts(root: &Path, boot: &Boot, plan: &Plan) -> (Vec, Vec, Ve // Held until the last staged artifact has been read back, so no clean of // this worktree's crate targets can land inside this build. - let mut lock = buildlock::shared(root, "build"); - let sysroot = toolchain::ensure(root, &mut lock); + let mut lock = worktree_lock(root, "a build"); + let sysroot = toolchain::ensure(root); let env = GuestEnv::new(&sysroot); let config = parse_config(&boot.config); @@ -1851,7 +1823,7 @@ fn shipped_parts(root: &Path, boot: &Boot, plan: &Plan) -> (Vec, Vec, Ve // is staged and certified through the same [`stage_and_certify_kernel`] that // path uses, so neither can grow an assertion the other lacks. let (kernel_bytes, bl_art) = { - let _artifact = buildlock::artifact(root); + let _artifact = artifact(root); let kernel_handle = { let root = root.to_path_buf(); let env = env.clone(); @@ -2073,20 +2045,20 @@ pub fn build_test_parts( // Held to the end of the function: the staged artifacts below are read // back after the userland build, and a clean landing in between is the // same defect as one landing mid-compile. - let mut lock = buildlock::shared(root, "test image"); - let sysroot = crate::toolchain::ensure(root, &mut lock); + let mut lock = worktree_lock(root, "a test image"); + let sysroot = crate::toolchain::ensure(root); let env = GuestEnv::new(&sysroot); invalidate_stale(root, &mut lock, &env.toolchain, &config_targets(root, &config)); // Build and stage under one lock, released before `build_and_assemble`. // Releasing it there is deliberate and required: that build takes its own - // `buildlock::artifact` across its build→read window (it reads shared cargo + // `artifact` across its build→read window (it reads shared cargo // paths too), so holding this one across the long userland build would // deadlock the process against itself — and the staged copies below are // already immune to another config's rebuild. let (kernel_bytes, bl_bytes) = { - let _artifact = buildlock::artifact(root); + let _artifact = artifact(root); let kernel = KERNEL.get_or_build(kernel_key, || { let mut kernel_extra: Vec<&str> = Vec::new(); if !features.is_empty() { @@ -2162,7 +2134,7 @@ pub fn https_fetch_host(root: &Path) -> PathBuf { /// image carries. Read under the artifact lock, as every image build reads it. pub fn copy_guest_program(root: &Path, arch: Arch, name: &str, to: &Path) -> Result<(), String> { let from = root.join(format!("userland/target/{}/{PROFILE}/{name}", arch.userland())); - let _artifact = buildlock::artifact(root); + let _artifact = artifact(root); fs::copy(&from, to) .map(|_| ()) .map_err(|e| format!("{} to {}: {e}", from.display(), to.display())) @@ -2285,18 +2257,18 @@ pub fn build_toyos_bins(root: &Path, arch: Arch, crate_path: &Path, quiet: bool) struct TestBuild { target: &'static str, env: GuestEnv, - _lock: buildlock::Held, - _artifact: buildlock::Guard, + _sysroot: crate::sysroot::Sysroot, + _lock: Lock, + _artifact: Lock, } impl TestBuild { fn begin(root: &Path, arch: Arch, what: &str, stale_targets: &[(PathBuf, Clean)]) -> Self { - let mut lock = buildlock::shared(root, what); - let sysroot = crate::toolchain::ensure(root, &mut lock); + let mut lock = worktree_lock(root, what); + let sysroot = crate::toolchain::ensure(root); let env = GuestEnv::new(&sysroot); invalidate_stale(root, &mut lock, &env.toolchain, stale_targets); - let artifact = buildlock::artifact(root); - TestBuild { target: arch.userland(), env, _lock: lock, _artifact: artifact } + TestBuild { target: arch.userland(), env, _sysroot: sysroot, _lock: lock, _artifact: artifact(root) } } } @@ -2310,69 +2282,6 @@ pub fn build_toyos_bin(root: &Path, arch: Arch, crate_path: &Path, name: &str, q fs::read(&binary).unwrap_or_else(|e| panic!("read the test binary {}: {e}", binary.display())) } -// --- Internal helpers --- - -/// The ToyOS-hosted rustc, and the target libraries it compiles against: this -/// build's own sysroot's, so the compiler on the image links what the image's -/// programs link. The hosted compiler itself is the primary's, read under the -/// lock its rebuild takes. -fn collect_hosted_rustc(root: &Path, toolchain: &Path, root_files: &mut Vec<(String, Vec)>) { - let _compiler = buildlock::compiler_shared(root, "reading the hosted rustc"); - let target = toolchain::HOSTED_ARCH.userland(); - let sysroot = toolchain::rust_dir(root).join(format!("build/{target}/stage2")); - assert!( - sysroot.exists(), - "Hosted rustc sysroot missing: {}", - sysroot.display() - ); - - let rustc = sysroot.join("bin/rustc"); - assert!( - rustc.exists(), - "Hosted rustc binary missing: {}", - rustc.display() - ); - root_files.push(("bin/rustc".to_string(), fs::read(&rustc).unwrap())); - - if let Ok(entries) = fs::read_dir(sysroot.join("lib")) { - for entry in entries.flatten() { - let path = entry.path(); - if path.extension().is_some_and(|e| e == "so") { - let name = path.file_name().unwrap().to_str().unwrap().to_string(); - let data = fs::read(&path).unwrap(); - root_files.push((format!("lib/{name}"), data)); - } - } - } - - let backends = sysroot.join(format!("lib/rustlib/{target}/codegen-backends")); - if backends.exists() { - for entry in fs::read_dir(&backends).into_iter().flatten().flatten() { - let path = entry.path(); - if path.extension().is_some_and(|e| e == "so") { - let name = path.file_name().unwrap().to_str().unwrap().to_string(); - let data = fs::read(&path).unwrap(); - root_files.push(( - format!("lib/rustlib/{target}/codegen-backends/{name}"), - data, - )); - } - } - } - - let rlibs = toolchain.join(format!("lib/rustlib/{target}/lib")); - for entry in fs::read_dir(&rlibs).unwrap_or_else(|e| panic!("read {}: {e}", rlibs.display())) { - let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", rlibs.display())).path(); - if path.extension().is_some_and(|e| e == "rlib" || e == "rmeta") { - let name = path.file_name().unwrap().to_str().unwrap().to_string(); - root_files.push(( - format!("lib/rustlib/{target}/lib/{name}"), - fs::read(&path).unwrap(), - )); - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -3199,7 +3108,7 @@ mod tests { receives_have_providers(&load(cfg)).unwrap_or_else(|e| panic!("{cfg}: {e}")); } let bad: SystemConfig = - toml::from_str("init = []\n[programs.client]\nreceives = [\"ghost\"]\n").unwrap(); + toml::from_str("[programs.client]\nreceives = [\"ghost\"]\n").unwrap(); assert!(receives_have_providers(&bad).is_err()); } @@ -3248,13 +3157,13 @@ mod tests { apps_receive_a_served_name(&load(cfg)).unwrap_or_else(|e| panic!("{cfg}: {e}")); } let provided: SystemConfig = toml::from_str( - "init = []\n[apps]\nreceives = [\"surface\"]\n\ + "[apps]\nreceives = [\"surface\"]\n\ [programs.terminal]\nprovides = [\"surface\"]\n", ) .unwrap(); assert!(apps_receive_a_served_name(&provided).is_err()); let ghost: SystemConfig = - toml::from_str("init = []\n[apps]\nreceives = [\"ghost\"]\n").unwrap(); + toml::from_str("[apps]\nreceives = [\"ghost\"]\n").unwrap(); assert!(apps_receive_a_served_name(&ghost).is_err()); } @@ -3285,7 +3194,7 @@ mod tests { provides_disjoint_from_serves(&load(cfg)).unwrap_or_else(|e| panic!("{cfg}: {e}")); } let bad: SystemConfig = toml::from_str( - "init = []\n[programs.a]\nserves = [\"x\"]\n[programs.b]\nprovides = [\"x\"]\n", + "[programs.a]\nserves = [\"x\"]\n[programs.b]\nprovides = [\"x\"]\n", ) .unwrap(); assert!(provides_disjoint_from_serves(&bad).is_err()); @@ -3340,7 +3249,7 @@ mod tests { .expect_err("the excused entry no longer collides with anything"); assert!(staged.contains(STAGED_COLLISION.2), "{staged}"); let bad: SystemConfig = toml::from_str( - "init = []\n[programs.a]\ndevices = [\"framebuffer\"]\n\ + "[programs.a]\ndevices = [\"framebuffer\"]\n\ [programs.b]\ndevices = [\"framebuffer\"]\n", ) .unwrap(); @@ -3473,7 +3382,7 @@ mod tests { } let armed_on = |device: &str, args: &str| { let cfg: SystemConfig = toml::from_str(&format!( - "init = []\n[programs.netd]\ndevices = [\"{device}\"]\nargs = [{args}]\n" + "[programs.netd]\ndevices = [\"{device}\"]\nargs = [{args}]\n" )) .unwrap(); an_armed_intel_actuator_claims_a_card_the_driver_opens(&cfg, &cards) @@ -3551,7 +3460,7 @@ mod tests { claims_no_device(&load("diag/system.toml")) .unwrap_or_else(|e| panic!("diag/system.toml: {e}")); let bad: SystemConfig = - toml::from_str("init = []\n[programs.x]\ndevices = [\"framebuffer\"]\n").unwrap(); + toml::from_str("[programs.x]\ndevices = [\"framebuffer\"]\n").unwrap(); assert!(claims_no_device(&bad).is_err()); } @@ -3571,7 +3480,7 @@ mod tests { for cfg in ALL_CONFIGS { started_programs_are_declared(&load(cfg)).unwrap_or_else(|e| panic!("{cfg}: {e}")); } - let bad: SystemConfig = toml::from_str("init = []\n[boot]\nstart = [\"ghost\"]\n").unwrap(); + let bad: SystemConfig = toml::from_str("[boot]\nstart = [\"ghost\"]\n").unwrap(); assert!(started_programs_are_declared(&bad).is_err()); } diff --git a/src/buildlock.rs b/src/buildlock.rs deleted file mode 100644 index db145dfb30c..00000000000 --- a/src/buildlock.rs +++ /dev/null @@ -1,1056 +0,0 @@ -//! Serialising the build system's stateful phases across the builds running -//! against this repository. -//! -//! Cargo's own build lock cannot do this job. `src/build.rs`'s `invalidate_stale` -//! runs the `clean` that `cargo clean`s a crate's `target/`, and cargo's lock -//! lives inside it at `target//.cargo-lock` — the clean deletes the -//! file the other process's lock is on. So these files live outside every -//! directory the build system removes: a lock on an inode that can be unlinked -//! and recreated under a waiter is not a lock. -//! -//! Two modes, because two plain `cargo build`s of different packages are -//! cargo's business and serialising those would destroy the parallelism the -//! builds depend on: -//! -//! - **shared** — "I am building against the state as it stands". Any number -//! at once. -//! - **exclusive** — "I am replacing it": the rust bootstrap, this worktree's -//! std build, the `cargo clean`s. One at a time, and never while a build -//! holds the shared mode. -//! -//! And two [`Scope`]s: a crate target directory is shared by the builds in one -//! worktree, while the primary's `rust/build` — the compiler every sysroot is -//! cloned from and compiled by — is shared by every worktree at once. A build -//! holds its worktree's lock shared for its whole length and the global one not -//! at all: it compiles against its own content-addressed sysroot -//! (`src/sysroot.rs`), which nothing rewrites. Only a sysroot being *made* -//! reads the compiler, and it holds [`compiler_shared`] while it does. -//! -//! A sysroot's own lock ([`keyed_building`], [`keyed_using`]) is per key, -//! so two worktrees with different ABIs never meet in it, and two with the same -//! one build it once. A compiler a worktree's fork checkout names apart from the -//! primary's (`src/compiler.rs`) is locked the same way under its own key, and -//! neither it nor a sysroot built from it takes the global lock. -//! -//! key's lock → a sysroot key's lock → the worktree build lock → the global one -//! → an LLVM key's lock → artifact. A compiler's or a sysroot's key lock is -//! taken with the worktree lock put down ([`Held::without_shared`]), because the -//! key's builder takes the worktree lock exclusively; an LLVM key's is taken -//! inside the worktree or global lock covering the fork build directory its -//! builder writes. -//! -//! Holder death: `flock` is released by the kernel when the open file -//! description closes, so a builder that is SIGKILLed mid-phase — routine here -//! — strands nothing, which a lock file with a pid in it could not promise. -//! Established on this host (Darwin 25.5.0) rather than assumed, and -//! `killed_holder_releases_the_lock` keeps it that way. - -use std::fs; -use std::io::{self, Read, Seek, SeekFrom, Write}; -use std::os::unix::io::AsRawFd; -use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; - -unsafe extern "C" { - fn flock(fd: i32, operation: i32) -> i32; - fn kill(pid: i32, sig: i32) -> i32; -} - -const LOCK_SH: i32 = 1; -const LOCK_EX: i32 = 2; -const LOCK_NB: i32 = 4; - -/// How often a wait that is lasting repeats itself. -/// -/// Shortened under `cfg(test)` so the gate on the repetition costs a second -/// instead of a minute; every process in those gates is this same binary, so -/// both sides of a wait agree on it. -#[cfg(not(test))] -const HEARTBEAT: Duration = Duration::from_secs(30); -#[cfg(test)] -const HEARTBEAT: Duration = Duration::from_millis(300); - -const LOCK_DIR: &str = ".build-locks"; -/// Inside the git common directory: the one place every worktree of this -/// repository names identically, and one the build system never cleans. -const GLOBAL_LOCK_DIR: &str = "toyos-build-locks"; - -/// The one directory every worktree of this repository names identically. -fn git_lock_dir(root: &Path) -> PathBuf { - crate::git_common_dir(root).join(GLOBAL_LOCK_DIR) -} - -/// Which shared state a phase replaces, and so which lock has to serialise it. -/// -/// Stated at every call site rather than inferred, because the two are not -/// interchangeable in either direction: a toolchain phase taken in the worktree -/// scope serialises nothing across worktrees, and a target-directory clean -/// taken in the global scope stalls builds it has no business stalling. -#[derive(Clone, Copy, PartialEq)] -pub enum Scope { - /// State every worktree shares: the primary's `rust/` build tree — the - /// compiler every sysroot is made with — and the machine-global rustup link. - Global, - /// State this worktree alone owns — its crate target directories. Two - /// worktrees cleaning their own have nothing to say to each other. - Worktree, -} - -/// A held lock. Releasing it is closing the file. -#[must_use] -pub struct Guard { - file: fs::File, - /// Exclusive holders record who they are, and clear it on the way out so a - /// waiter never names a process that has already finished. - records_holder: bool, -} - -impl Drop for Guard { - fn drop(&mut self) { - if self.records_holder { - write_note(&mut self.file, ""); - } - } -} - -/// The worktree's build lock, held in shared mode for the length of one build so -/// no clean of its crate targets lands inside it. The global lock is not held: -/// what a build reads of the shared tree is its own sysroot, which nothing -/// rewrites once it is made. -pub struct Held { - worktree_dir: PathBuf, - global_dir: PathBuf, - what: String, - /// `None` only while [`Held::without_shared`] has it put down, which is the - /// whole reason this is an `Option`. - guard: Option, -} - -/// Take the build lock in shared mode for `what`, and hold it until the -/// returned value is dropped — which must be after the last artifact the build -/// reads back, not merely after the last thing it writes: a clean landing -/// between a `cargo build` and the read of what it built is the same defect. -pub fn shared(root: &Path, what: &str) -> Held { - let mut held = Held { - worktree_dir: root.join(LOCK_DIR), - global_dir: git_lock_dir(root), - what: what.to_string(), - guard: None, - }; - held.guard = Some(held.take_shared()); - held -} - -impl Held { - /// Ask `decide`, and if it reports work, do that work under `scope`'s - /// exclusive lock. - /// - /// `decide` runs first under the shared lock this value holds, so a phase - /// with nothing to do costs no serialisation at all. When it does report - /// work the shared lock is dropped, the exclusive one taken, and `decide` - /// asked **again**: whatever it saw a moment ago may have been done by the - /// process that held the lock in between, and only this second answer is - /// acted on. Serialising the action alone would still double-clean. - /// - /// The shared lock goes down whichever scope is escalated: holding it while - /// queueing for the global one is a deadlock with a process holding the - /// global one that wants this worktree's. - pub fn act_if( - &mut self, - scope: Scope, - phase: &str, - decide: impl Fn() -> Option, - act: impl FnOnce(W), - ) { - if decide().is_none() { - return; - } - let dir = match scope { - Scope::Global => self.global_dir.clone(), - Scope::Worktree => self.worktree_dir.clone(), - }; - self.without_shared(|| { - let _exclusive = acquire(&dir, LOCK_EX, phase, BUILD); - if let Some(work) = decide() { - act(work); - } - }); - } - - /// Run `f` with this worktree's shared lock put down and take it back after: - /// for a lock that orders before it, as a sysroot key's does. - pub fn without_shared(&mut self, f: impl FnOnce() -> R) -> R { - self.guard = None; - let out = f(); - self.guard = Some(self.take_shared()); - out - } - - fn take_shared(&self) -> Guard { - acquire(&self.worktree_dir, LOCK_SH, &self.what, BUILD) - } -} - -/// This worktree's build lock, exclusively: what a sysroot build holds while it -/// writes the worktree's fork build directory, with the key's lock already held. -pub fn worktree_exclusive(root: &Path, what: &str) -> Guard { - acquire(&root.join(LOCK_DIR), LOCK_EX, what, BUILD) -} - -/// The global lock in shared mode: "I am reading the primary's compiler". A -/// sysroot build holds it from its std compile to its clone of `stage2`, so a -/// toolchain rebuild does not land inside either. -pub fn compiler_shared(root: &Path, what: &str) -> Guard { - acquire(&git_lock_dir(root), LOCK_SH, what, BUILD) -} - -/// Exclusive lock over the shared cargo artifact paths. -/// -/// Cargo keys an artifact path on (crate, target, profile) and nothing else, so -/// every config writes and reads one path; this is held across each build→stage -/// pair so the staged copy is of what this build produced. Separate from the -/// build lock proper because every builder needs it and builders hold the build -/// lock in *shared* mode by design. -pub fn artifact(root: &Path) -> Guard { - exclusive(&root.join(LOCK_DIR).join("artifact"), "artifact lock", "artifact staging") -} - -/// A content-addressed product of the host, locked per key: a sysroot, a -/// compiler a worktree's fork checkout names (`src/compiler.rs`), or the LLVM -/// a compiler links (`src/llvm.rs`). -#[derive(Clone, Copy)] -pub enum Keyed { - Sysroot, - Compiler, - Llvm, -} - -impl Keyed { - fn dir(self) -> &'static str { - match self { - Keyed::Sysroot => "sysroots", - Keyed::Compiler => "compilers", - Keyed::Llvm => "llvm", - } - } - - pub(crate) fn name(self) -> &'static str { - match self { - Keyed::Sysroot => "sysroot", - Keyed::Compiler => "compiler", - Keyed::Llvm => "LLVM", - } - } -} - -/// Make what `key` names: exclusive, and waited for by every other process that -/// wants the same key, which then finds it made. -fn keyed_building(root: &Path, kind: Keyed, key: &str) -> Guard { - let lock = format!("{} lock", kind.name()); - exclusive(&keyed_lock_path(root, kind, key), &lock, &format!("building {} {key}", kind.name())) -} - -/// Use what `key` names: shared, so any number of builds use it at once, a -/// builder of it is waited for, and a sweep cannot remove it. -fn keyed_using(root: &Path, kind: Keyed, key: &str) -> Guard { - let path = keyed_lock_path(root, kind, key); - let file = open_lock_file(&path); - if !try_lock(&file, LOCK_SH) { - let lock = format!("{} lock", kind.name()); - let what = format!("using {} {key}", kind.name()); - let holder = describe_holder(&path) - .unwrap_or_else(|| "held, but the holder left no readable note".to_string()); - announce(&lock, &what, &holder); - take_lock_announcing(&file, LOCK_SH, &path, &lock, &what); - } - Guard { file, records_holder: false } -} - -/// What `key` names, held in use and whole: made by `make` under -/// [`keyed_building`] while `defect`, which says why it is not whole, says it is -/// not. A `make` that leaves it not whole is refused by that defect rather than -/// run again. -pub fn keyed_made( - root: &Path, - kind: Keyed, - key: &str, - defect: impl Fn() -> Option, - mut make: impl FnMut(), -) -> Guard { - loop { - let using = keyed_using(root, kind, key); - if defect().is_none() { - return using; - } - drop(using); - let _building = keyed_building(root, kind, key); - if defect().is_some() { - make(); - if let Some(defect) = defect() { - panic!("{} {key} was made, and is not whole: {defect}", kind.name()); - } - } - } -} - -/// What `key` names, exclusively and only if nobody is making or using it: what -/// a sweep holds while it removes one. -pub fn keyed_idle(root: &Path, kind: Keyed, key: &str) -> Option { - let file = open_lock_file(&keyed_lock_path(root, kind, key)); - try_lock(&file, LOCK_EX).then_some(Guard { file, records_holder: false }) -} - -fn keyed_lock_path(root: &Path, kind: Keyed, key: &str) -> PathBuf { - git_lock_dir(root).join(kind.dir()).join(key) -} - -/// One lock file, taken exclusively and held until the guard drops. -/// -/// For the locks with no shared mode: every holder writes the note, so a waiter -/// can always be told who it is waiting for. -fn exclusive(path: &Path, lock: &str, what: &str) -> Guard { - let file = open_lock_file(path); - let start = Instant::now(); - if !try_lock(&file, LOCK_EX) { - let holder = describe_holder(path) - .unwrap_or_else(|| "held, but the holder left no readable note".to_string()); - announce(lock, what, &holder); - take_lock_announcing(&file, LOCK_EX, path, lock, what); - eprintln!("[build-lock] {what} acquired after {:.1?}", start.elapsed()); - } - let mut guard = Guard { file, records_holder: true }; - write_note(&mut guard.file, ¬e_text(what)); - guard -} - -/// One lock with a shared and an exclusive mode, in the two words a waiting -/// agent needs. -/// -/// The second field exists because the shared mode cannot leave a note: one -/// `state` file carries one, and shared holders come several at a time. So what -/// to say about them is a property of the lock rather than something -/// [`describe_holder`] could work out. -#[derive(Clone, Copy)] -struct Lock { - name: &'static str, - shared_holders: &'static str, - queued_ahead: &'static str, -} - -const BUILD: Lock = Lock { - name: "build lock", - shared_holders: "held by other builds in this tree", - queued_ahead: "an exclusive phase is queued ahead of it", -}; - -/// Acquire one mode of a two-file lock. -/// -/// Two files, not one. `flock` has no writer preference — measured on this -/// host, four shared churners kept an exclusive waiter out for the whole 5.5 s -/// they ran — and the exclusive phases are exactly the long, silent ones an -/// agent kills and retries. So an exclusive acquirer holds `intent` while it -/// queues for `state`, which makes later shared acquirers line up behind it -/// instead of overtaking it. `intent` is always taken before `state` and -/// dropped as soon as `state` is held, so nothing ever waits on `intent` while -/// holding `state`. -fn acquire(dir: &Path, op: i32, what: &str, lock: Lock) -> Guard { - let intent_path = dir.join("intent"); - let state_path = dir.join("state"); - let intent = open_lock_file(&intent_path); - let state = open_lock_file(&state_path); - let label = format!("{}, {what}", if op == LOCK_EX { "exclusive" } else { "shared" }); - - let start = Instant::now(); - let mut waited = false; - - if !try_lock(&intent, op) { - announce(lock.name, &label, lock.queued_ahead); - waited = true; - take_lock_announcing(&intent, op, &intent_path, lock.name, &label); - } - if !try_lock(&state, op) { - if !waited { - let holder = describe_holder(&state_path) - .unwrap_or_else(|| lock.shared_holders.to_string()); - announce(lock.name, &label, &holder); - waited = true; - } - take_lock_announcing(&state, op, &state_path, lock.name, &label); - } - drop(intent); - - if waited { - eprintln!("[build-lock] acquired ({label}) after {:.1?}", start.elapsed()); - } - - let records_holder = op == LOCK_EX; - let mut guard = Guard { file: state, records_holder }; - if records_holder { - write_note(&mut guard.file, ¬e_text(what)); - } - guard -} - -/// An agent staring at silence kills and retries, which is the pathology this -/// module exists to remove — so a wait says what it is waiting for and, when -/// that can be established, who has it. -fn announce(lock: &str, label: &str, holder: &str) { - eprintln!("[build-lock] waiting for the {lock} ({label}) — {holder}"); -} - -/// [`take_lock`], saying every 30 s that it is still waiting and who for. -/// -/// The kernel keeps the queue and a thread does the talking: nothing here polls -/// a lock, so `flock`'s own ordering is given up nowhere. The holder is re-read -/// each time, so the message follows the queue forward rather than naming the -/// process that was in front when the wait began. -fn take_lock_announcing(file: &fs::File, op: i32, path: &Path, lock: &str, label: &str) { - use std::sync::mpsc::{channel, RecvTimeoutError}; - - // A channel and not a polled flag: this runs on every *contended* - // acquisition, the artifact lock among them, and a flag checked every few - // milliseconds would put that granularity on the front of each one. The - // sender dropping wakes the thread at once and it never sleeps past the - // acquisition. - let (tx, rx) = channel::<()>(); - let heartbeat = { - let path = path.to_path_buf(); - let lock = lock.to_string(); - let label = label.to_string(); - std::thread::spawn(move || { - let began = Instant::now(); - while rx.recv_timeout(HEARTBEAT) == Err(RecvTimeoutError::Timeout) { - let holder = describe_holder(&path) - .unwrap_or_else(|| "the holder left no readable note".to_string()); - eprintln!( - "[build-lock] still waiting for the {lock} ({label}), {:.0?} so far — {holder}", - began.elapsed() - ); - } - }) - }; - take_lock(file, op, path); - drop(tx); - heartbeat.join().expect("the lock heartbeat panicked"); -} - -/// What the last exclusive holder of `path` recorded, if it is still running. -/// -/// A killed holder leaves its note behind, so the pid is checked before it is -/// named: telling a waiting agent to go look at a dead pid is worse than -/// telling it nothing. `None` is that "nothing" — what to say instead is the -/// caller's, because a lock with a shared mode is usually held by holders who -/// never wrote a note at all, and a lock without one never is. -fn describe_holder(path: &Path) -> Option { - let mut file = fs::File::open(path).ok()?; - let mut text = String::new(); - file.read_to_string(&mut text).ok()?; - let mut parts = text.trim().splitn(3, ' '); - let (pid, since, what) = (parts.next()?, parts.next()?, parts.next()?); - let (pid, since) = (pid.parse::().ok()?, since.parse::().ok()?); - if !alive(pid) { - return None; - } - let secs = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs().saturating_sub(since)) - .unwrap_or(0); - Some(format!("held by pid {pid} ({what}), {secs}s so far")) -} - -fn alive(pid: i32) -> bool { - // SAFETY: signal 0 runs the existence and permission checks and delivers - // nothing. - if unsafe { kill(pid, 0) } == 0 { - return true; - } - io::Error::last_os_error().kind() == io::ErrorKind::PermissionDenied -} - -fn note_text(what: &str) -> String { - let since = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - format!("{} {since} {what}", std::process::id()) -} - -/// The note is advisory — it names a holder in a waiter's message and nothing -/// reads it to decide anything — so failing to write it must not fail a build. -fn write_note(file: &mut fs::File, text: &str) { - let _ = file - .seek(SeekFrom::Start(0)) - .and_then(|_| file.set_len(0)) - .and_then(|_| file.write_all(text.as_bytes())) - .and_then(|_| file.flush()); -} - -fn open_lock_file(path: &Path) -> fs::File { - let dir = path.parent().expect("lock path has a parent"); - fs::create_dir_all(dir).unwrap_or_else(|e| panic!("build lock: create {}: {e}", dir.display())); - // Never truncating: the file carries the holder note, and `File::create` - // would wipe a live holder's. - fs::OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(false) - .open(path) - .unwrap_or_else(|e| panic!("build lock: open {}: {e}", path.display())) -} - -fn take_lock(file: &fs::File, op: i32, path: &Path) { - loop { - // SAFETY: `file` owns the fd for the duration of the call and of the - // guard the caller builds from it. - if unsafe { flock(file.as_raw_fd(), op) } == 0 { - return; - } - let err = io::Error::last_os_error(); - if err.kind() == io::ErrorKind::Interrupted { - continue; - } - panic!("build lock: flock on {}: {err}", path.display()); - } -} - -fn try_lock(file: &fs::File, op: i32) -> bool { - loop { - // SAFETY: as in `take_lock`. - if unsafe { flock(file.as_raw_fd(), op | LOCK_NB) } == 0 { - return true; - } - let err = io::Error::last_os_error(); - match err.kind() { - io::ErrorKind::Interrupted => continue, - io::ErrorKind::WouldBlock => return false, - _ => panic!("build lock: flock: {err}"), - } - } -} - -#[cfg(test)] -pub(crate) mod tests { - use super::*; - use std::ffi::OsStr; - use std::process::{Child, Command}; - use std::time::Duration; - use toyos_tmpdir::TempDir; - - // Two processes are the point. `flock` is per open file description, so a - // single-process test would prove nothing about the thing that actually - // races in this tree. The child is this same test binary, re-run with one - // `#[ignore]`d test selected by name and its role in the environment, so an - // ordinary `cargo test` never runs the child half on its own. - const ROLE: &str = "TOYOS_BUILDLOCK_TEST_ROLE"; - const ROOT: &str = "TOYOS_BUILDLOCK_TEST_ROOT"; - const MARKS: &str = "TOYOS_BUILDLOCK_TEST_MARKS"; - const KEY: &str = "TOYOS_BUILDLOCK_TEST_KEY"; - - /// A lock held by a process of its own, which [`Elsewhere::release`] waits - /// to exit. - /// - /// A test never asserts free a lock this process has held: another test - /// thread's spawn copies every descriptor open at that moment into its child - /// until the child's exec, and the copy holds the lock past the drop. - pub(crate) struct Elsewhere { - child: Child, - marks: TempDir, - released: bool, - } - - impl Elsewhere { - /// Run the `#[ignore]`d test `role` names, with `env`, and return once - /// it has called [`hold_until_released`]. - pub(crate) fn hold(role: &str, env: &[(&str, &OsStr)]) -> Self { - let marks = TempDir::new("buildlock-elsewhere"); - let mut child = rerun(role) - .envs(env.iter().copied()) - .env(MARKS, &marks) - .spawn() - .expect("spawn the holder"); - let deadline = Instant::now() + Duration::from_secs(20); - while !marks.join("held").exists() { - if let Some(status) = child.try_wait().unwrap() { - panic!("{role} exited before it took its lock: {status}"); - } - if Instant::now() >= deadline { - child.kill().unwrap(); - panic!("{role} never took its lock in 20 s, killed: {}", child.wait().unwrap()); - } - std::thread::sleep(Duration::from_millis(5)); - } - Elsewhere { child, marks, released: false } - } - - pub(crate) fn id(&self) -> u32 { - self.child.id() - } - - /// Let go, and return once the holder has exited. - pub(crate) fn release(mut self) { - self.released = true; - touch(&self.marks.join("release")); - assert!(self.child.wait().unwrap().success(), "the holder failed"); - } - } - - impl Drop for Elsewhere { - /// An assertion between `hold` and `release` skips `release`; without - /// this, the holder it leaked keeps running and its lock held past - /// the test that dropped it. - fn drop(&mut self) { - if self.released { - return; - } - let _ = self.child.kill(); - let _ = self.child.wait(); - } - } - - /// The holder's half of [`Elsewhere`]. - pub(crate) fn hold_until_released() { - let marks = PathBuf::from(std::env::var(MARKS).expect("a holder runs under Elsewhere::hold")); - touch(&marks.join("held")); - assert!(appeared(&marks.join("release"), Duration::from_secs(20)), "the holder was never released"); - } - - /// Sysroot `key` of `root`, held in use by a process of its own. - pub(crate) fn sysroot_used_elsewhere(root: &Path, key: &str) -> Elsewhere { - let env = [(ROLE, OsStr::new("use-sysroot")), (ROOT, root.as_os_str()), (KEY, OsStr::new(key))]; - Elsewhere::hold("buildlock::tests::child_role", &env) - } - - /// What `role` of [`child_role`] takes in `root`, held by a process of its own. - fn held_elsewhere(root: &Path, role: &str) -> Elsewhere { - Elsewhere::hold("buildlock::tests::child_role", &[(ROLE, OsStr::new(role)), (ROOT, root.as_os_str())]) - } - - /// A git repository, because the global scope is keyed on the common - /// directory and a scratch tree that is not one would exercise a path the - /// build system never takes. - fn scratch(name: &str) -> TempDir { - let dir = TempDir::new(&format!("buildlock-{name}")); - let ok = Command::new("git") - .args(["init", "-q"]) - .current_dir(&dir) - .status() - .expect("git init") - .success(); - assert!(ok, "git init in {}", dir.display()); - dir - } - - fn worktree_lock_dir(root: &Path) -> PathBuf { - root.join(LOCK_DIR) - } - - /// This test binary, to run the one `#[ignore]`d test `test` names. - pub(crate) fn rerun(test: &str) -> Command { - let mut rerun = Command::new(std::env::current_exe().unwrap()); - rerun.args(["--exact", test, "--include-ignored", "--nocapture"]); - rerun - } - - fn child(root: &Path, role: &str) -> Child { - rerun("buildlock::tests::child_role") - .env(ROLE, role) - .env(ROOT, root) - .spawn() - .expect("spawn the competing process") - } - - fn appeared(path: &Path, within: Duration) -> bool { - let deadline = Instant::now() + within; - while !path.exists() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(5)); - } - path.exists() - } - - fn touch(path: &Path) { - fs::write(path, b"").unwrap(); - } - - /// Hold whatever this role took until the test that spawned it is gone. - /// - /// A flat `sleep(600)` is what these roles used to do, and it is wrong in - /// exactly the case they exist for: when the *parent* assertion fails, the - /// test never reaches its `kill`, and two children go on holding the - /// harness's stdout for ten minutes — so the negative control an agent runs - /// on purpose wedges the run it was checking. Costs one `getppid` every - /// 100 ms and needs no reaper. - fn until_orphaned() { - unsafe extern "C" { - fn getppid() -> i32; - } - let deadline = Instant::now() + Duration::from_secs(600); - // SAFETY: `getppid` takes nothing and cannot fail. - while Instant::now() < deadline && unsafe { getppid() } > 1 { - std::thread::sleep(Duration::from_millis(100)); - } - } - - /// A fresh fd per probe: a successful `try_lock` *holds* what it took, and - /// polling on one fd would itself be the thing keeping the writer out. - fn intent_is_taken(root: &Path) -> bool { - !try_lock(&open_lock_file(&worktree_lock_dir(root).join("intent")), LOCK_SH) - } - - fn note(root: &Path, line: &str) { - let mut f = fs::OpenOptions::new() - .create(true) - .append(true) - .open(root.join("order.log")) - .unwrap(); - writeln!(f, "{line}").unwrap(); - } - - #[test] - #[ignore = "the competing process for the tests below; never runs on its own"] - fn child_role() { - let role = std::env::var(ROLE) - .unwrap_or_else(|_| panic!("child_role ran without {ROLE}; it is not a test")); - let root = PathBuf::from(std::env::var(ROOT).unwrap()); - match role.as_str() { - "hold-exclusive" => { - let mut held = shared(&root, "child"); - held.act_if(Scope::Worktree, "child exclusive phase", || Some(()), |()| hold_until_released()); - } - "hold-exclusive-forever" => { - let mut held = shared(&root, "child"); - held.act_if( - Scope::Worktree, - "child exclusive phase", - || Some(()), - |()| { - touch(&root.join("held")); - until_orphaned(); - }, - ); - } - "hold-artifact-forever" => { - let _landing = artifact(&root); - touch(&root.join("held")); - until_orphaned(); - } - "want-exclusive" => { - let mut held = shared(&root, "child"); - held.act_if(Scope::Worktree, "queued exclusive phase", || Some(()), |()| note(&root, "ex")); - } - "want-shared" => { - let _held = shared(&root, "child"); - note(&root, "sh"); - } - "hold-sysroot-build" => { - let _building = keyed_building(&root, Keyed::Sysroot, "k1"); - hold_until_released(); - note(&root, "built"); - } - "want-artifact" => { - let _landing = artifact(&root); - note(&root, "landed"); - } - "want-sysroot" => { - let _using = keyed_using(&root, Keyed::Sysroot, "k1"); - note(&root, "used"); - } - "use-sysroot" => { - let _using = keyed_using(&root, Keyed::Sysroot, &std::env::var(KEY).unwrap()); - hold_until_released(); - } - "clean" | "clean-unlocked" => { - touch(&root.join("cleaner-ready")); - assert!(appeared(&root.join("builder-mid"), Duration::from_secs(20))); - let target = root.join("crate/target"); - if role == "clean" { - let mut held = shared(&root, "child"); - held.act_if( - Scope::Worktree, - "clean the crate target", - || target.exists().then_some(()), - |()| fs::remove_dir_all(&target).unwrap(), - ); - } else { - fs::remove_dir_all(&target).unwrap(); - } - touch(&root.join("cleaner-done")); - } - other => panic!("unknown child role {other}"), - } - } - - #[test] - fn exclusive_excludes_every_other_acquirer() { - let root = scratch("exclusive"); - let kid = held_elsewhere(&root, "hold-exclusive"); - - let state = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(!try_lock(&state, LOCK_SH), "a build got in while an exclusive phase ran"); - assert!(!try_lock(&state, LOCK_EX), "two exclusive phases at once"); - let holder = describe_holder(&worktree_lock_dir(&root).join("state")) - .expect("no holder note"); - assert!( - holder.starts_with(&format!("held by pid {} ", kid.id())), - "the waiting side cannot name the holder: {holder}" - ); - - kid.release(); - drop(state); - let _mine = shared(&root, "parent"); - } - - #[test] - fn killed_holder_releases_the_lock() { - let root = scratch("killed"); - let mut kid = child(&root, "hold-exclusive-forever"); - assert!(appeared(&root.join("held"), Duration::from_secs(20)), "child never acquired"); - - let state = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(!try_lock(&state, LOCK_EX), "the lock was not actually held"); - - kid.kill().unwrap(); - kid.wait().unwrap(); - - assert!(try_lock(&state, LOCK_EX), "a SIGKILLed holder stranded the lock"); - // And the note it left behind names a pid that is gone, so nobody is - // sent to wait on it. - assert_eq!(describe_holder(&worktree_lock_dir(&root).join("state")), None); - } - - #[test] - fn shared_admits_shared() { - let root = scratch("shared"); - let _mine = shared(&root, "parent"); - let second = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(try_lock(&second, LOCK_SH), "two builds cannot run at once"); - drop(second); - let third = open_lock_file(&worktree_lock_dir(&root).join("state")); - assert!(!try_lock(&third, LOCK_EX), "a clean got in while a build was running"); - } - - /// Two worktrees of one repository must name one global lock file and two - /// worktree ones. - /// - /// Getting either half backwards is silent — every build still runs. One - /// global file per worktree means the phases that replace the shared sysroot - /// stop excluding each other, which is the defect worktrees were introduced - /// without; one worktree file for all of them means a clean of a target - /// directory stalls builds that cannot see it. - #[test] - fn worktrees_share_the_global_lock_and_not_the_worktree_one() { - let root = scratch("worktrees"); - fs::write(root.join("f"), b"x").unwrap(); - git(&root, &["add", "f"]); - git(&root, &["commit", "-qm", "init"]); - let linked = root.join("wt"); - git(&root, &["worktree", "add", "-q", linked.to_str().unwrap(), "-b", "wt"]); - - let mine = shared(&root, "primary"); - let theirs = shared(&linked, "linked"); - assert_eq!( - mine.global_dir, theirs.global_dir, - "two worktrees disagree about where the global lock lives" - ); - assert_ne!( - mine.worktree_dir, theirs.worktree_dir, - "two worktrees share one target-directory lock" - ); - - // Naming one path is not yet excluding on it: `flock` conflicts between - // open file descriptions, so a second handle on the shared file is the - // question a second process would ask. A build compiles against its own - // sysroot and reads no compiler, so a toolchain rebuild waits for no - // build in either worktree; a sysroot being made reads the compiler, so - // the rebuild waits for that. - let global = open_lock_file(&git_common_lock_dir(&root).join("state")); - assert!(try_lock(&global, LOCK_EX), "a build kept the toolchain from being rebuilt"); - drop(global); - drop(theirs); - drop(mine); - let making = compiler_shared(&linked, "a sysroot build in the worktree"); - let global = open_lock_file(&git_common_lock_dir(&root).join("state")); - assert!( - !try_lock(&global, LOCK_EX), - "a toolchain rebuild could land inside a sysroot build in another worktree" - ); - drop(making); - } - - fn git_common_lock_dir(root: &Path) -> PathBuf { - crate::git_common_dir(root).join(GLOBAL_LOCK_DIR) - } - - fn git(dir: &Path, args: &[&str]) { - let ok = Command::new("git") - .args(["-c", "commit.gpgsign=false", "-c", "user.email=t@t", "-c", "user.name=t"]) - .args(crate::gitfixture::NO_AUTO_MAINTENANCE) - .args(args) - .current_dir(dir) - .status() - .expect("run git") - .success(); - assert!(ok, "git {args:?} in {}", dir.display()); - } - - /// `flock` alone would let a stream of builds starve the rebuild they are - /// all waiting for; the `intent` file is what stops that, and this is the - /// gate on it. - #[test] - fn a_queued_exclusive_phase_goes_first() { - let root = scratch("preference"); - let mine = shared(&root, "parent"); - - let mut writer = child(&root, "want-exclusive"); - let deadline = Instant::now() + Duration::from_secs(20); - while !intent_is_taken(&root) { - assert!(Instant::now() < deadline, "the exclusive child never queued"); - std::thread::sleep(Duration::from_millis(5)); - } - - let mut reader = child(&root, "want-shared"); - assert!( - !appeared(&root.join("order.log"), Duration::from_millis(300)), - "a build overtook a queued exclusive phase" - ); - - drop(mine); - assert!(writer.wait().unwrap().success()); - assert!(reader.wait().unwrap().success()); - assert_eq!(fs::read_to_string(root.join("order.log")).unwrap(), "ex\nsh\n"); - } - - /// [`Elsewhere::release`] returns only once the holder has exited and been - /// reaped: a zombie still answers `kill(pid, 0)`. - #[test] - fn a_released_holder_is_gone() { - let root = scratch("released"); - let user = sysroot_used_elsewhere(&root, "k"); - let pid = user.id() as i32; - user.release(); - assert!(!alive(pid), "release returned before the holder was reaped"); - } - - /// **One key is built once, and two keys never meet.** A second process - /// wanting the key being built waits on the builder's lock — not on a - /// timer — and gets it only once the build is done; a process making - /// another key is not held at all; and a sweep cannot take a key that - /// somebody is making or using. - #[test] - fn a_key_being_built_is_waited_for_and_another_key_is_not() { - let root = scratch("sysroot-keys"); - let builder = held_elsewhere(&root, "hold-sysroot-build"); - - assert!(keyed_idle(&root, Keyed::Sysroot, "k1").is_none(), "a sweep could remove a key being built"); - let other = keyed_building(&root, Keyed::Sysroot, "k2"); - drop(other); - - let mut user = child(&root, "want-sysroot"); - assert!( - !appeared(&root.join("order.log"), Duration::from_millis(300)), - "a build used a sysroot while it was still being made" - ); - builder.release(); - assert!(user.wait().unwrap().success()); - assert_eq!(fs::read_to_string(root.join("order.log")).unwrap(), "built\nused\n"); - - let user = sysroot_used_elsewhere(&root, "k1"); - assert!(keyed_idle(&root, Keyed::Sysroot, "k1").is_none(), "a sweep could remove a key in use"); - user.release(); - assert!(keyed_idle(&root, Keyed::Sysroot, "k1").is_some(), "a sweep could not remove a key nobody uses"); - } - - #[test] - fn a_lasting_wait_keeps_saying_so() { - let root = scratch("heartbeat"); - let mut holder = child(&root, "hold-artifact-forever"); - assert!(appeared(&root.join("held"), Duration::from_secs(20)), "child never acquired"); - - let mut queued = rerun("buildlock::tests::child_role") - .env(ROLE, "want-artifact") - .env(ROOT, &root) - .stderr(std::process::Stdio::piped()) - .spawn() - .expect("spawn the queued landing"); - - let (tx, rx) = std::sync::mpsc::channel::(); - let stderr = queued.stderr.take().unwrap(); - std::thread::spawn(move || { - use std::io::BufRead; - for line in std::io::BufReader::new(stderr).lines().map_while(Result::ok) { - if tx.send(line).is_err() { - return; - } - } - }); - - let mut repeats = Vec::new(); - let deadline = Instant::now() + Duration::from_secs(20); - while repeats.len() < 2 && Instant::now() < deadline { - let left = deadline.saturating_duration_since(Instant::now()); - match rx.recv_timeout(left) { - Ok(line) if line.contains("still waiting for the artifact lock") => { - repeats.push(line); - } - Ok(_) => {} - Err(_) => break, - } - } - queued.kill().unwrap(); - queued.wait().unwrap(); - holder.kill().unwrap(); - holder.wait().unwrap(); - - assert!( - repeats.len() >= 2, - "a queued landing said {} times that it was still waiting: {repeats:?}", - repeats.len() - ); - assert!( - repeats[0].contains(&format!("pid {}", holder.id())), - "the repeat does not name the holder: {}", - repeats[0] - ); - } - - /// The defect this module exists for, staged so that it is not itself a - /// race: a clean lands in the middle of a build in the same target - /// directory, and the build's next write finds the directory gone. Run once - /// without the lock to show the ENOENT, once with it to show the clean - /// waiting its turn. - #[test] - fn a_clean_cannot_land_inside_a_build() { - let unlocked = clean_racing_a_build(false); - assert_eq!( - unlocked.unwrap_err().kind(), - io::ErrorKind::NotFound, - "unlocked, the clean was expected to pull the target dir out from under the build" - ); - clean_racing_a_build(true) - .expect("locked, the build's write must not land in a cleaned directory"); - } - - fn clean_racing_a_build(locked: bool) -> io::Result<()> { - let root = scratch(if locked { "race-locked" } else { "race-unlocked" }); - let target = root.join("crate/target"); - fs::create_dir_all(&target).unwrap(); - - let mut kid = child(&root, if locked { "clean" } else { "clean-unlocked" }); - assert!(appeared(&root.join("cleaner-ready"), Duration::from_secs(20))); - let guard = locked.then(|| shared(&root, "parent build")); - - fs::write(target.join("a.o"), b"a").unwrap(); - touch(&root.join("builder-mid")); - let cleaned = appeared(&root.join("cleaner-done"), Duration::from_millis(700)); - assert_eq!(cleaned, !locked, "the clean's turn came at the wrong time"); - - let outcome = fs::write(target.join("b.o"), b"b"); - - drop(guard); - assert!(kid.wait().unwrap().success()); - // Delayed, never dropped: the clean still happens, after the build. - assert!(root.join("cleaner-done").exists()); - assert!(!target.exists()); - outcome - } -} diff --git a/src/compiler.rs b/src/compiler.rs index 706183645ff..c344794c2f8 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -1,349 +1,134 @@ -//! The compiler a worktree's sysroot is cloned from and compiled by: the -//! primary's, or one of its own, content-addressed. +//! The compiler a sysroot is cloned from and compiled by: a product of the +//! store (`src/store.rs`), shared by every checkout whose fork names it, the +//! primary's among them. //! -//! **Every worktree builds with the compiler its own fork checkout names.** The -//! primary's `stage2` is built from what the primary's `rust/compiler/` holds, -//! and [`record`] writes which that is. A linked worktree whose fork checkout -//! holds the same `compiler/` ([`source`]) compiles with that one. One whose -//! `compiler/` differs — a new target spec, a codegen change — gets its own: -//! built by bootstrap in its own fork checkout, under that checkout's -//! `build/toyos-compiler/`, and placed at `rust/build/compilers//`, where -//! the key ([`key`]) is the identity (`src/identity.rs`) of the checkout's -//! `compiler/`, `src/tools/`, `src/stage0` and `Cargo.lock`, the key of the -//! LLVM it links, which names `src/bootstrap`, and [`RECIPE`]. Nothing writes -//! that directory after its [`SOURCE`] file exists, and two worktrees naming -//! the same compiler share one copy. -//! -//! **LLVM is the host's, built from `src/llvm-project`** (`src/llvm.rs`), and -//! linked through its `llvm-config`. [`source`] names that LLVM's key, so -//! another LLVM is another compiler, the primary's among them, and an LLVM -//! checkout or a `src/bootstrap` holding what no commit does names none. A -//! worktree records the LLVM its compiler links for as long as it builds with -//! that compiler. -//! -//! **A compiler of a worktree's own never touches what the others build with**: -//! not the primary's `stage2`, not its record, not the machine-global rustup -//! `toyos` link — a sysroot is named by its directory, never by a toolchain -//! name, so no link is made. The global lock is not taken either; nothing of -//! the primary's is read but the LLVM store, under its key's own lock. -//! -//! Locks, in the one order every acquirer takes them: the key's -//! (`buildlock::keyed_*` with [`Keyed::Compiler`]), with this worktree's build -//! lock put down, held shared for as long as a sysroot is being made from it; -//! then, to build, this worktree's exclusively, because its fork build -//! directory is written; then the LLVM key's, held shared while it is linked. -//! -//! A compiler no worktree names any more is removed by `keystore::sweep`, which -//! `--worktree remove` and every placement run: each build records the key it used in its -//! worktree's `target/`, and a key no registered worktree records, that nobody -//! is making or using, goes. -//! -//! What such a worktree's image cannot carry is the ToyOS-hosted rustc: that is -//! the primary's, built from the primary's `compiler/`, so `hosted-rustc` in a -//! worktree building with its own compiler is refused by name -//! (`src/build.rs`). +//! **Its key** ([`key`]) is [`RECIPE`], the key of the LLVM it links, and the +//! fork's `compiler/`, `src/tools/`, `src/stage0` and `Cargo.lock`. Bootstrap +//! builds it in the fork checkout's [`BUILD_DIR`], for the host alone, against +//! that LLVM (`src/llvm.rs`) through its `llvm-config`; its `stage2` is placed +//! at `compilers//stage2/`. use std::fs; use std::path::{Path, PathBuf}; -use crate::buildlock::{self, Guard, Held, Keyed}; -use crate::keystore; -use crate::sysroot::{clone_tree, git_bytes, git_out, short, tree_identity}; +use crate::llvm; +use crate::store::{self, Kind, Sources}; +use crate::sysroot::{clone_tree, Fork}; use crate::toolchain::{self, host_triple}; /// What changes how a key's sources become a compiler and is none of them: the /// build below. Moving it moves every key. -const RECIPE: &str = "bootstrap stage 2 of compiler/rustc and library, profile compiler, host only, with rust-lld, host linker pinned, LLVM, clang and LLD from the host's LLVM; 5"; +const RECIPE: &str = "bootstrap stage 2 of compiler/rustc, library and src/tools/cargo linked statically, profile compiler, host only, with rust-lld, host linker pinned, LLVM, clang and LLD from the host's LLVM, without the links to its source; 9"; -/// What a compiler's key is the identity of, in its fork checkout. -const KEYED: [&str; 4] = ["compiler", "src/tools", "src/stage0", "Cargo.lock"]; +/// What bootstrap links into a `stage2` from the checkout that built it, which +/// a product of the store does not carry (`store::publish`). +const SOURCE_LINKS: [&str; 2] = ["lib/rustlib/src", "lib/rustlib/rustc-src"]; -/// The submodule a compiler is built against by commit: its LLVM, which -/// bootstrap builds from that commit, so its content is never read. -pub(crate) const LLVM: &str = "src/llvm-project"; +/// What bootstrap compiles for a compiler, as it names them. +const BUILT: [&str; 3] = ["compiler/rustc", "library", "src/tools/cargo"]; -/// Where a fork checkout builds a compiler of its own. -const BUILD_DIR: &str = "build/toyos-compiler"; +/// Where a fork checkout builds its compiler, kept between builds so the next +/// one is incremental. +const BUILD_DIR: &str = "build/toyos-rustc"; -/// The file a finished compiler carries last, naming what it was built from. A -/// directory without it is a build that did not finish. -const SOURCE: &str = "SOURCE"; +/// What the host rustc links its own binaries with, held to one answer: bootstrap +/// otherwise ties it to `lld` for `x86_64-unknown-linux-gnu`. +const HOST_LINKER_PIN: &str = "default-linker-linux-override = \"off\""; /// A compiler, held in use for as long as this lives. pub struct Compiler { /// Its toolchain directory: `bin/rustc`, `lib/`. pub stage2: PathBuf, - /// The file naming what it was built from. - record: PathBuf, - /// Whether it is the primary's, the one the hosted rustc and the rustup - /// link are built from. - pub primary: bool, - _using: Option, -} - -impl Compiler { - /// The primary's `stage2`. - pub fn primary(rust_dir: &Path) -> Self { - Self { stage2: toolchain::stage2(rust_dir), record: primary_record(rust_dir), primary: true, _using: None } - } - - /// The compiler as a sysroot's key sees it: the source it was built from, - /// and the driver that build left, so a rebuild of the same source is a new - /// compiler too. - pub fn identity(&self) -> String { - let source = fs::read_to_string(&self.record).unwrap_or_else(|_| { - panic!( - "{} is missing, so no sysroot can say which compiler it was built with.\n\ - The primary checkout writes the primary's: run `cargo run -- --build-only` there once.", - self.record.display(), - ) - }); - let lib = self.stage2.join("lib"); - let driver = fs::read_dir(&lib) - .unwrap_or_else(|e| panic!("read {}: {e}", lib.display())) - .flatten() - .find(|e| e.file_name().to_string_lossy().starts_with("librustc_driver")) - .unwrap_or_else(|| panic!("{} holds no librustc_driver", lib.display())); - let meta = driver.metadata().unwrap_or_else(|e| panic!("stat the driver: {e}")); - let mtime = meta - .modified() - .ok() - .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) - .map_or(0, |d| d.as_nanos()); - format!("{} {} {} {mtime}", source.trim(), driver.file_name().to_string_lossy(), meta.len()) - } -} - -/// The primary's record of which `compiler/` its `stage2` was built from. -pub(crate) fn primary_record(rust_dir: &Path) -> PathBuf { - rust_dir.join("build/toyos-compiler") -} - -/// Every compiler of a worktree's own on this host. -pub fn compilers_dir(rust_dir: &Path) -> PathBuf { - rust_dir.join("build/compilers") -} - -/// [`compiler_source`] and the key of the LLVM it links. -pub fn source(checkout: &Path) -> String { - source_with(checkout, &crate::llvm::key(checkout)) -} - -/// [`source`], with the key of the LLVM `checkout` names. -fn source_with(checkout: &Path, llvm: &str) -> String { - format!("{} llvm {llvm}", compiler_source(checkout)) -} - -/// What `checkout`'s `compiler/` is: its commit's tree, and whatever the working -/// tree changes in it — an edit, or a file git does not track yet, which is -/// what a new target spec is before its commit. -fn compiler_source(checkout: &Path) -> String { - let tree = git_out(checkout, &["rev-parse", "HEAD:compiler"]); - let mut local = git_bytes(checkout, &["diff", "HEAD", "--", "compiler"]); - let untracked = git_bytes(checkout, &["ls-files", "-z", "--others", "--exclude-standard", "--", "compiler"]); - for name in untracked.split(|b| *b == 0).filter(|n| !n.is_empty()) { - let path = checkout.join(String::from_utf8_lossy(name).as_ref()); - local.extend_from_slice(name); - local.push(0); - local.extend(fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()))); - local.push(0); - } - if local.is_empty() { - tree.trim().to_string() - } else { - format!("{} with local changes {}", tree.trim(), short(&local)) - } -} - -/// Record which compiler the primary's `stage2` is. The primary calls this -/// after a toolchain build. -pub fn record(rust_dir: &Path) { - let at = primary_record(rust_dir); - let want = source(rust_dir); - if fs::read_to_string(&at).ok().as_deref() != Some(want.as_str()) { - fs::write(&at, &want).unwrap_or_else(|e| panic!("write {}: {e}", at.display())); - } -} - -/// Remove the record of which compiler the primary's `stage2` is. The primary -/// calls this before a toolchain build. -pub fn forget(rust_dir: &Path) { - let at = primary_record(rust_dir); - match fs::remove_file(&at) { - Err(e) if e.kind() != std::io::ErrorKind::NotFound => panic!("remove {}: {e}", at.display()), - _ => {} - } -} - -/// Whether the primary's `stage2` is the compiler `checkout`'s `compiler/` -/// names — `Err` when nothing records which compiler that is. -/// -/// **The source's content, never its files' times**: a checkout that rewrites a -/// file with the bytes it had is no new compiler. -fn primary_is(rust_dir: &Path, checkout: &Path, llvm: &str) -> Result { - let names = source_with(checkout, llvm); - Ok(fs::read_to_string(primary_record(rust_dir))?.trim() == names) -} - -/// Whether the primary's `stage2` is built from what its own `rust/compiler/` -/// holds: false until a bootstrap has finished and [`record`]ed it. -pub fn primary_is_current(rust_dir: &Path) -> bool { - match primary_is(rust_dir, rust_dir, &crate::llvm::key(rust_dir)) { - Ok(current) => current, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, - Err(e) => panic!("read {}: {e}", primary_record(rust_dir).display()), - } -} - -/// The key of the compiler `fork`'s sources name: their content. -pub fn key(fork: &Path) -> String { - key_with(fork, &crate::llvm::key(fork)) -} - -/// [`key`], with the key of the LLVM `fork` names. -fn key_with(fork: &Path, llvm: &str) -> String { - let parts = [RECIPE, &tree_identity(fork, &KEYED), llvm]; - short(parts.join("\n\0\n").as_bytes()) + pub key: String, + _held: store::Held, } -/// The LLVM commit `fork` builds against: the one its `HEAD` records, refused -/// when its index stages another, because bootstrap checks out the index's. An -/// LLVM change is a commit there and a gitlink here, so a checkout holding -/// anything no commit does is refused rather than named by its commit. -pub(crate) fn llvm_commit(fork: &Path) -> String { - let checkout = fork.join(LLVM); - // Exactly what bootstrap's LLVM stamp hashes beyond the commit; the untracked - // cache spares each call a walk of the whole tree. - let status = ["-c", "core.untrackedCache=true", "status", "--porcelain", "--untracked-files=normal"]; - let edited = checkout.join(".git").exists() && !git_bytes(&checkout, &status).is_empty(); - assert!( - !edited, - "{} holds changes no commit does, and a compiler is keyed on the commit its gitlink \ - names: commit them there and record that commit in {}", - checkout.display(), - fork.display(), - ); - let recorded = git_out(fork, &["ls-tree", "HEAD", LLVM]); - let committed = match recorded.split_whitespace().collect::>().as_slice() { - ["160000", "commit", sha, _] => sha.to_string(), - _ => panic!("{} records no {LLVM} gitlink: `git ls-tree HEAD {LLVM}` said {recorded:?}", fork.display()), - }; - let indexed = git_out(fork, &["ls-files", "--stage", LLVM]); - let staged = match indexed.split_whitespace().collect::>().as_slice() { - ["160000", sha, "0", _] => sha.to_string(), - _ => panic!("{} indexes no {LLVM} gitlink: `git ls-files --stage {LLVM}` said {indexed:?}", fork.display()), - }; - assert!( - staged == committed, - "{} stages {LLVM} at {staged}, and its HEAD records {committed}: bootstrap builds the one \ - staged, and nothing is keyed on what no commit holds; commit the gitlink, or unstage it", - fork.display(), - ); - committed +/// The key of the compiler `sources` name. +pub fn key(sources: &Sources) -> String { + let parts = [&llvm::key(sources), sources.get("compiler"), sources.get("src/tools"), sources.get("src/stage0"), sources.get("Cargo.lock")]; + store::key(RECIPE, &parts) } -/// The compiler `root`'s fork checkout at `fork` names: the primary's where its -/// `compiler/` is the one the primary's was built from, and otherwise its own, -/// built if nobody has built it, held in use for as long as the returned value -/// lives. -pub fn resolve(root: &Path, rust_dir: &Path, fork: &Path, lock: &mut Held) -> Compiler { - lock.without_shared(|| choose(root, rust_dir, fork, |fork| build_in_fork(root, rust_dir, fork))) +/// The compiler `sources` name, built from `fork` if nobody has built it, and +/// held in use for as long as the returned value lives. `root` records its key +/// and its LLVM's, so both stay while it builds with them. +pub fn resolve(root: &Path, rust_dir: &Path, fork: &Fork, sources: &Sources) -> Compiler { + choose(root, rust_dir, fork, sources, |dir| build_in_fork(root, rust_dir, dir, sources)) } /// [`resolve`] with the build that makes a compiler's `stage2` passed in, so a /// test can stand in for bootstrap: `build` compiles the fork checkout it is /// given and returns the `stage2` it left there. -fn choose(root: &Path, rust_dir: &Path, fork: &Path, build: impl Fn(&Path) -> PathBuf) -> Compiler { - if fork == rust_dir { - return Compiler::primary(rust_dir); - } - let llvm = crate::llvm::key(fork); - let names_primary = primary_is(rust_dir, fork, &llvm).unwrap_or_else(|e| { - panic!( - "{} cannot be read ({e}), so nothing says which compiler the primary's stage2 is, \ - and no worktree can know whether it names that one.\n\ - The primary checkout writes it: run `cargo run -- --build-only` there once.", - primary_record(rust_dir).display(), - ) +fn choose(root: &Path, rust_dir: &Path, fork: &Fork, sources: &Sources, build: impl Fn(&Path) -> PathBuf) -> Compiler { + store::record(root, Kind::Llvm, &llvm::key(sources)); + let key = key(sources); + let held = store::get(root, rust_dir, Kind::Compiler, &key, |partial| { + let checkout = fork.checkout(root); + fill(root, &checkout.dir, &key, partial, &build); }); - // The LLVM record follows the compiler's: a compiler links the LLVM its key - // names, and the primary's is recorded by the primary. - if names_primary { - keystore::forget(root, Keyed::Compiler); - keystore::forget(root, Keyed::Llvm); - let build = fork.join(BUILD_DIR); - if !crate::llvm::in_tree(&build).is_empty() { - let _worktree = buildlock::worktree_exclusive(root, "removing the LLVM its compiler build built"); - crate::llvm::retire_in_tree(&build); - } - return Compiler::primary(rust_dir); - } - let key = key_with(fork, &llvm); - let dir = compilers_dir(rust_dir).join(&key); - keystore::record(root, Keyed::Llvm, &llvm); - let using = keystore::made( - root, - Keyed::Compiler, - &compilers_dir(rust_dir), - &key, - || (!dir.join(SOURCE).is_file()).then(|| format!("{} carries no {SOURCE}", dir.display())), - || place(root, fork, &key, &dir, &build), - ); - Compiler { stage2: dir.join("stage2"), record: dir.join(SOURCE), primary: false, _using: Some(using) } + Compiler { stage2: held.dir.join("stage2"), key, _held: held } } -/// Build the compiler `key` names from `fork` and put it at `dir`. The caller -/// holds the key's lock. -fn place(root: &Path, fork: &Path, key: &str, dir: &Path, build: &impl Fn(&Path) -> PathBuf) { - let what = format!("building compiler {key}"); - let _worktree = buildlock::worktree_exclusive(root, &what); - eprintln!("Building compiler {key} in {}: its compiler/ is not the one the primary's was built from", fork.display()); +/// Build the compiler `key` names from the fork checkout at `fork` into +/// `partial`. +fn fill(root: &Path, fork: &Path, key: &str, partial: &Path, build: &impl Fn(&Path) -> PathBuf) { + eprintln!("Building compiler {key} in {}: nobody on this host has", fork.display()); let stage2 = build(fork); - crate::llvm::retire_in_tree(&fork.join(BUILD_DIR)); - let partial = dir.with_extension("partial"); - if partial.exists() { - fs::remove_dir_all(&partial).unwrap_or_else(|e| panic!("remove {}: {e}", partial.display())); - } clone_tree(&stage2, &partial.join("stage2")); - // The sources the key named are the ones built, or this is not that key's. - let again = self::key(fork); + for link in SOURCE_LINKS { + let at = partial.join("stage2").join(link); + fs::remove_dir_all(&at).unwrap_or_else(|e| panic!("remove {}: {e}", at.display())); + } + let again = self::key(&Sources::of(root, fork)); assert!( again == key, "the fork's compiler sources moved while compiler {key} was being built (they are now \ {again}); nothing was kept, and the next build makes the one they name" ); - fs::write(partial.join(SOURCE), format!("{key}\n")) - .unwrap_or_else(|e| panic!("write {}: {e}", partial.join(SOURCE).display())); - fs::rename(&partial, dir).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", partial.display(), dir.display())); + store::assert_built_at_gitlinks(fork, &BUILT, &format!("compiler {key}")); + if let Some(defect) = toolchain::toolchain_defect(&partial.join("stage2")) { + panic!("compiler {key} was made, and is not whole: {defect}"); + } } -/// Bootstrap's build of the compiler in `fork`, into its own build directory, -/// against the LLVM `fork` names (`src/llvm.rs`), and the `stage2` it made, -/// with the cargo and the clang every toolchain directory carries. -fn build_in_fork(root: &Path, rust_dir: &Path, fork: &Path) -> PathBuf { +/// Bootstrap's build of the compiler in `fork`, into [`BUILD_DIR`], against the +/// LLVM `sources` name, and the `stage2` it made, with the fork's own cargo and +/// the clang every toolchain directory carries. +fn build_in_fork(root: &Path, rust_dir: &Path, fork: &Path, sources: &Sources) -> PathBuf { crate::ensure_submodule(fork, "library/backtrace"); - let llvm = crate::llvm::resolve(root, rust_dir, fork); + let llvm = llvm::resolve(root, rust_dir, fork, sources); let host = host_triple(); let build_dir = fork.join(BUILD_DIR); fs::create_dir_all(&build_dir).unwrap_or_else(|e| panic!("create {}: {e}", build_dir.display())); let config = build_dir.join("bootstrap.toml"); fs::write(&config, config_text(&build_dir, &host, &llvm.dir)).unwrap_or_else(|e| panic!("write {}: {e}", config.display())); let config = config.to_str().unwrap_or_else(|| panic!("{} is not UTF-8", config.display())); - let args = ["build", "--stage", "2", "--config", config, "--warnings", "warn", "compiler/rustc", "library"]; + let args: Vec<&str> = ["build", "--stage", "2", "--config", config, "--warnings", "warn"].into_iter().chain(BUILT).collect(); let (ok, log) = toolchain::x_build(fork, &args, "the compiler"); toolchain::refuse_on_compile_error(&log, "the compiler"); assert!(ok, "the compiler build in {} failed, and nothing in its output was a compile error", fork.display()); let stage2 = build_dir.join(&host).join("stage2"); assert!(stage2.join("bin/rustc").is_file(), "the compiler build left no {}", stage2.join("bin/rustc").display()); - toolchain::provision_toolchain_cargo(&stage2); + let tools: Vec = fs::read_dir(build_dir.join(&host)) + .unwrap_or_else(|e| panic!("read {}: {e}", build_dir.join(&host).display())) + .map(|e| e.unwrap_or_else(|e| panic!("read the build directory: {e}")).path().join("cargo")) + .filter(|cargo| cargo.parent().is_some_and(|d| d.to_string_lossy().ends_with("-tools-bin")) && cargo.is_file()) + .collect(); + let [cargo] = tools.as_slice() else { panic!("the compiler build left cargo at {tools:?}, not one place") }; + // Removed first: a link left there would be copied through, onto what it names. + match fs::remove_file(stage2.join("bin/cargo")) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => panic!("remove {}: {e}", stage2.join("bin/cargo").display()), + _ => {} + } + fs::copy(cargo, stage2.join("bin/cargo")).unwrap_or_else(|e| panic!("copy {} into {}: {e}", cargo.display(), stage2.display())); crate::clang::provision(&stage2, &llvm.dir); toolchain::assert_toolchain_is_honest(&stage2); stage2 } -/// Bootstrap's configuration for a compiler of a worktree's own: the primary's -/// `profile` and options, for the host alone, since every guest target's -/// libraries are the sysroot's to build, linking the LLVM at `llvm`. +/// Bootstrap's configuration for a compiler: for the host alone, since every +/// guest target's libraries are the sysroot's to build, linking the LLVM at +/// `llvm`. fn config_text(build_dir: &Path, host: &str, llvm: &Path) -> String { format!( r#"change-id = "ignore" @@ -353,6 +138,7 @@ profile = "compiler" build-dir = "{build_dir}" host = ["{host}"] target = ["{host}"] +cargo-native-static = true [llvm] {llvm} @@ -367,438 +153,137 @@ lld = true "#, build_dir = build_dir.display(), llvm = crate::clang::LLVM_CONFIG, - pin = toolchain::HOST_LINKER_PIN, - external = crate::llvm::host_lines(llvm), + pin = HOST_LINKER_PIN, + external = llvm::host_lines(llvm), ) } #[cfg(test)] -pub(crate) mod tests { +mod tests { use std::cell::Cell; - use toyos_tmpdir::TempDir; - use std::process::Command; - use super::*; + use crate::store::tests::{behind_a_staged_gitlink, estate, git, refusal, write, LLVM_B}; - pub(crate) fn git(dir: &Path, args: &[&str]) -> String { - let out = Command::new("git") - .args(["-c", "commit.gpgsign=false", "-c", "user.email=t@t", "-c", "user.name=t"]) - .args(["-c", "protocol.file.allow=always", "-c", "init.defaultBranch=main"]) - .args(crate::gitfixture::NO_AUTO_MAINTENANCE) - .args(args) - .current_dir(dir) - .output() - .expect("run git"); - assert!(out.status.success(), "git {args:?} in {}: {}", dir.display(), String::from_utf8_lossy(&out.stderr)); - String::from_utf8(out.stdout).unwrap().trim().to_string() - } - - pub(crate) fn write(path: &Path, text: &str) { - fs::create_dir_all(path.parent().unwrap()).unwrap(); - fs::write(path, text).unwrap(); - } - - /// Every file under `dir` with its bytes, for "nothing here changed". - fn snapshot(dir: &Path) -> Vec<(PathBuf, Vec)> { - let mut out = Vec::new(); - let mut stack = vec![dir.to_path_buf()]; - while let Some(at) = stack.pop() { - for entry in fs::read_dir(&at).unwrap().flatten() { - let path = entry.path(); - if path.is_dir() { - stack.push(path); - } else { - out.push((path.clone(), fs::read(&path).unwrap())); - } - } + /// Bootstrap's stand-in: a `stage2` that says which target spec it knows, + /// with what every toolchain directory carries. + fn fake_build(fork: &Path) -> PathBuf { + let stage2 = fork.join(BUILD_DIR).join("stage2"); + let spec = fs::read_to_string(fork.join("compiler/rustc_target/src/lib.rs")).unwrap(); + write(&stage2.join("bin/rustc"), &format!("a rustc knowing {spec}")); + write(&stage2.join("lib/librustc_driver-1.dylib"), &spec); + let lld = toolchain::rust_lld(&stage2); + for tool in ["rust-lld", "llvm-ar", "clang", "ld.lld"] { + write(&lld.with_file_name(tool), tool); } - out.sort(); - out - } - - /// The LLVM commits the fixtures' forks record. Nothing reads their content. - pub(crate) const LLVM_A: &str = "1111111111111111111111111111111111111111"; - pub(crate) const LLVM_B: &str = "2222222222222222222222222222222222222222"; - - /// A primary whose `rust` pins fork commit `C0` and has built a compiler - /// from it, and three linked worktrees: `same` pins `C0`, `a` and `b` each - /// pin a commit whose `compiler/` is its own. - pub(crate) fn estate(scratch: &Path) -> (PathBuf, PathBuf, [PathBuf; 3]) { - let base = fs::canonicalize(scratch).unwrap(); - - let fork = base.join("fork-src"); - fs::create_dir_all(&fork).unwrap(); - git(&fork, &["init", "-q"]); - write(&fork.join("compiler/rustc_target/src/lib.rs"), "pub fn targets() {}\n"); - write(&fork.join("src/bootstrap/src/lib.rs"), "fn main() {}\n"); - write(&fork.join("src/stage0"), "compiler_version=beta\n"); - write(&fork.join("Cargo.lock"), "# lock\n"); - write(&fork.join("library/std/src/lib.rs"), "pub fn a() {}\n"); - write(&fork.join(".gitignore"), "/build\n"); - git(&fork, &["add", "-A"]); - git(&fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{LLVM_A},{LLVM}")]); - // What an uninitialised submodule leaves, so `commit -a` keeps the gitlink. - fs::create_dir_all(fork.join(LLVM)).unwrap(); - git(&fork, &["commit", "-qm", "C0"]); - let c0 = git(&fork, &["rev-parse", "HEAD"]); - let mut pins = Vec::new(); - for spec in ["pub fn targets() { aarch64() }\n", "pub fn targets() { riscv() }\n"] { - git(&fork, &["checkout", "-q", &c0]); - write(&fork.join("compiler/rustc_target/src/lib.rs"), spec); - git(&fork, &["commit", "-qam", "a target"]); - pins.push(git(&fork, &["rev-parse", "HEAD"])); + write(&lld.parent().unwrap().parent().unwrap().join("lib/clang/22/include/stddef.h"), "stddef"); + write(&stage2.join("bin/cargo"), "cargo"); + for link in SOURCE_LINKS { + let at = stage2.join(link).join("rust"); + fs::create_dir_all(at.parent().unwrap()).unwrap(); + let _ = fs::remove_file(&at); + std::os::unix::fs::symlink(fork, &at).unwrap(); } - git(&fork, &["checkout", "-q", &c0]); - - let primary = base.join("primary"); - fs::create_dir_all(&primary).unwrap(); - git(&primary, &["init", "-q"]); - write(&primary.join("README"), "x\n"); - git(&primary, &["submodule", "add", "-q", fork.to_str().unwrap(), "rust"]); - git(&primary, &["add", "-A"]); - git(&primary, &["commit", "-qm", "pins C0"]); - let rust_dir = primary.join("rust"); - write(&toolchain::stage2(&rust_dir).join("bin/rustc"), "the primary's rustc"); - write(&toolchain::stage2(&rust_dir).join("lib/librustc_driver-0.dylib"), "the primary's driver"); - record(&rust_dir); + stage2 + } - let mut linked = Vec::new(); - for (name, pin) in [("same", c0.as_str()), ("a", pins[0].as_str()), ("b", pins[1].as_str())] { - let wt = base.join(name); - git(&primary, &["worktree", "add", "-q", "-b", name, wt.to_str().unwrap()]); - let _ = fs::remove_dir(wt.join("rust")); - git(&rust_dir, &["worktree", "add", "-q", "--detach", wt.join("rust").to_str().unwrap(), pin]); - linked.push(wt); - } - (primary, rust_dir, linked.try_into().unwrap()) + fn sources(root: &Path) -> Sources { + Sources::of(root, &root.join("rust")) } - /// **Two worktrees with different compilers build side by side, and the - /// primary's toolchain is untouched by either.** Each gets its own compiler - /// at its own key, built once and found again; one that names the primary's - /// `compiler/` builds nothing and gets the primary's; the primary's `stage2`, - /// its record and the rustup `toyos` link are byte-for-byte what they were; - /// a sweep takes a compiler only once no worktree names it. + /// **One compiler per key, whoever asks**: the primary and a worktree whose + /// fork names the same `compiler/` share one; two worktrees with different + /// compilers build side by side, each once, and find theirs again; an + /// untracked file in `compiler/` is a new compiler and committing it is not + /// another. #[test] - fn worktrees_with_different_compilers_coexist_and_the_primary_s_is_untouched() { - let scratch = TempDir::new("compiler"); - let (primary, rust_dir, [same, a, b]) = estate(&scratch); - let before = snapshot(&rust_dir.join("build")); - let link = toolchain::rustup_link(); + fn one_compiler_per_key_whoever_asks() { + let e = estate("compiler"); let builds = Cell::new(0); - let fake = |fork: &Path| { + let counted = |fork: &Path| { builds.set(builds.get() + 1); fake_build(fork) }; - - let mine = choose(&same, &rust_dir, &same.join("rust"), fake); - assert!(mine.primary && mine.stage2 == toolchain::stage2(&rust_dir)); - assert_eq!(builds.get(), 0, "a worktree naming the primary's compiler built one"); - - let ca = choose(&a, &rust_dir, &a.join("rust"), fake); - let cb = choose(&b, &rust_dir, &b.join("rust"), fake); - assert_eq!(builds.get(), 2); - assert!(!ca.primary && !cb.primary); + let primary = choose(&e.primary, &e.rust_dir, &Fork::Checkout(e.rust_dir.clone()), &sources(&e.primary), counted); + let same = choose(&e.same, &e.rust_dir, &Fork::Checkout(e.same.join("rust")), &sources(&e.same), counted); + assert_eq!((same.stage2, builds.get()), (primary.stage2.clone(), 1), "one compiler/ built two compilers"); + assert!(SOURCE_LINKS.iter().all(|l| !primary.stage2.join(l).exists()), "a compiler names the checkout that built it"); + + let ca = choose(&e.a, &e.rust_dir, &Fork::Checkout(e.a.join("rust")), &sources(&e.a), counted); + let cb = choose(&e.b, &e.rust_dir, &Fork::Checkout(e.b.join("rust")), &sources(&e.b), counted); + assert_eq!(builds.get(), 3); assert_ne!(ca.stage2, cb.stage2, "two compilers were given one directory"); - assert!(ca.stage2.starts_with(compilers_dir(&rust_dir)) && cb.stage2.starts_with(compilers_dir(&rust_dir))); assert!(fs::read_to_string(ca.stage2.join("bin/rustc")).unwrap().contains("aarch64")); assert!(fs::read_to_string(cb.stage2.join("bin/rustc")).unwrap().contains("riscv")); - assert_ne!(ca.identity(), cb.identity()); - assert_ne!(ca.identity(), Compiler::primary(&rust_dir).identity()); - - // Found again, not rebuilt; and still both there. - let again = choose(&a, &rust_dir, &a.join("rust"), fake); - assert_eq!((again.stage2, builds.get()), (ca.stage2.clone(), 2)); - assert!(ca.stage2.join("bin/rustc").is_file() && cb.stage2.join("bin/rustc").is_file()); - - // An uncommitted file in `compiler/` is a new compiler too, and - // committing it is not another one. - let pinned = git(&a.join("rust"), &["rev-parse", "HEAD"]); - write(&a.join("rust/compiler/rustc_target/src/new_target.rs"), "pub fn t() {}\n"); - let ca2 = choose(&a, &rust_dir, &a.join("rust"), fake); - assert_ne!(ca2.stage2, ca.stage2, "an untracked target spec kept the old compiler"); - git(&a.join("rust"), &["add", "-A"]); - git(&a.join("rust"), &["commit", "-qm", "the target, committed"]); - let committed = choose(&a, &rust_dir, &a.join("rust"), fake); - assert_eq!((committed.stage2, builds.get()), (ca2.stage2.clone(), 3), "a commit rebuilt the compiler"); - git(&a.join("rust"), &["checkout", "-q", &pinned]); - - // The primary's own: nothing under its `build/` but `compilers/` moved. - let after: Vec<_> = snapshot(&rust_dir.join("build")) - .into_iter() - .filter(|(p, _)| !p.starts_with(compilers_dir(&rust_dir))) - .collect(); - assert_eq!(after, before, "the primary's stage2 or its record was written"); - assert_eq!(git(&rust_dir, &["rev-parse", "HEAD"]), git(&primary, &["rev-parse", "HEAD:rust"])); - assert_eq!(toolchain::rustup_link(), link, "the machine-global toyos link moved"); - - // A sweep takes the compiler nobody names, and only that one — and - // not while it is still in use, though nobody names it any more. - let spec = a.join("rust/compiler/rustc_target/src/orphan.rs"); - write(&spec, "pub fn o() {}\n"); - let orphan = compilers_dir(&rust_dir).join(key(&a.join("rust"))); - let user = chosen_elsewhere(&a, &rust_dir); - fs::remove_file(&spec).unwrap(); - let kept = choose(&a, &rust_dir, &a.join("rust"), fake); - let sweep = |root: &Path, rust_dir: &Path| keystore::sweep(root, Keyed::Compiler, &compilers_dir(rust_dir)); - assert_eq!(sweep(&primary, &rust_dir), Vec::::new(), "the sweep took a compiler still in use"); - assert!(orphan.is_dir() && ca2.stage2.is_dir()); - user.release(); - drop(cb); - assert_eq!(sweep(&primary, &rust_dir), [orphan], "the sweep took a compiler a worktree names, or left one nobody does"); - assert!(kept.stage2.is_dir() && ca2.stage2.is_dir()); - - // A placement sweeps too: the compiler an edit replaces goes once nobody - // uses it, and the one another worktree names stays. - let spec = a.join("rust/compiler/rustc_target/src/another.rs"); - write(&spec, "pub fn u() {}\n"); - let replaced = compilers_dir(&rust_dir).join(key(&a.join("rust"))); - chosen_elsewhere(&a, &rust_dir).release(); - let named = choose(&b, &rust_dir, &b.join("rust"), fake).stage2; - write(&spec, "pub fn v() {}\n"); - let ca3 = choose(&a, &rust_dir, &a.join("rust"), fake); - assert!(!replaced.exists(), "placing a compiler left the one it replaced, which nobody names"); - assert!(ca3.stage2.is_dir() && named.is_dir()); - } - - /// **A worktree names the LLVM of the compiler it builds with, and only - /// that one**: one it placed or found placed, never one it has gone back to - /// the primary's from; and its build directory keeps no LLVM of its own - /// either way. - #[test] - fn the_llvm_record_follows_the_compiler_in_use() { - let scratch = TempDir::new("compiler-llvm-record"); - let (primary, rust_dir, [_same, a, _b]) = estate(&scratch); - let fork = a.join("rust"); - let store = crate::llvm::store(&rust_dir); - let llvm = store.join(crate::llvm::key(&fork)); - let sweep = || keystore::sweep(&primary, Keyed::Llvm, &store); - - let own = fork.join(BUILD_DIR).join(host_triple()).join("llvm"); - write(&own.join("bin/llvm-config"), "the build directory's own"); - drop(choose(&a, &rust_dir, &fork, fake_build)); - assert!(!own.exists(), "a compiler built against the store left the LLVM its build directory built"); - fs::create_dir_all(&llvm).unwrap(); - assert_eq!(sweep(), Vec::::new(), "the LLVM of a worktree's own compiler was swept"); - - keystore::forget(&a, Keyed::Llvm); - let never = |_: &Path| -> PathBuf { panic!("a placed compiler was built again") }; - drop(choose(&a, &rust_dir, &fork, never)); - assert_eq!(keystore::recorded(&a, Keyed::Llvm), Some(crate::llvm::key(&fork)), "a placed compiler's LLVM went unrecorded"); - - git(&fork, &["checkout", "-q", &git(&rust_dir, &["rev-parse", "HEAD"])]); - write(&own.join("bin/llvm-config"), "the build directory's own, from before the store"); - assert!(choose(&a, &rust_dir, &fork, never).primary); - assert_eq!(sweep(), [llvm], "the LLVM of a compiler the worktree no longer builds with stayed"); - assert!(!own.exists(), "a worktree back on the primary's compiler kept the LLVM its build directory built"); - } - - const WORKTREE: &str = "TOYOS_COMPILER_TEST_WORKTREE"; - const RUST_DIR: &str = "TOYOS_COMPILER_TEST_RUST_DIR"; + let again = choose(&e.a, &e.rust_dir, &Fork::Checkout(e.a.join("rust")), &sources(&e.a), counted); + assert_eq!((again.stage2, builds.get()), (ca.stage2.clone(), 3), "a placed compiler was built again"); - /// The competing process for the test above: the compiler the worktree in - /// [`WORKTREE`] names, chosen and held in use until released. - #[test] - #[ignore = "the competing process for the test above; never runs on its own"] - fn child_role() { - let worktree = std::env::var(WORKTREE).unwrap_or_else(|_| panic!("child_role ran without {WORKTREE}; it is not a test")); - let worktree = PathBuf::from(worktree); - let rust_dir = PathBuf::from(std::env::var(RUST_DIR).unwrap()); - let chosen = choose(&worktree, &rust_dir, &worktree.join("rust"), fake_build); - assert!(!chosen.primary, "the holder was given the primary's compiler, which holds no key"); - buildlock::tests::hold_until_released(); - } - - /// The compiler `worktree` names, made if nobody has and held in use by a - /// process of its own. - fn chosen_elsewhere(worktree: &Path, rust_dir: &Path) -> buildlock::tests::Elsewhere { - let env = [(WORKTREE, worktree.as_os_str()), (RUST_DIR, rust_dir.as_os_str())]; - buildlock::tests::Elsewhere::hold("compiler::tests::child_role", &env) - } - - /// Bootstrap's stand-in: a `stage2` that says which target spec it knows. - fn fake_build(fork: &Path) -> PathBuf { - let stage2 = fork.join("build/toyos-compiler/stage2"); - let spec = fs::read_to_string(fork.join("compiler/rustc_target/src/lib.rs")).unwrap(); - write(&stage2.join("bin/rustc"), &format!("a rustc knowing {spec}")); - write(&stage2.join("lib/librustc_driver-1.dylib"), &spec); - stage2 - } - - /// **A primary with no record of its compiler is refused by name**, never - /// read as "no compiler": that reading made every worktree build its own. - #[test] - fn a_missing_primary_record_is_refused_and_builds_nothing() { - let scratch = TempDir::new("compiler-record"); - let (_primary, rust_dir, [same, _, _]) = estate(&scratch); - fs::remove_file(primary_record(&rust_dir)).unwrap(); - let builds = Cell::new(0); - let fake = |fork: &Path| { - builds.set(builds.get() + 1); - fork.join("build/toyos-compiler/stage2") - }; - let refused = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - choose(&same, &rust_dir, &same.join("rust"), fake); - })); - let why = refused.expect_err("a worktree resolved a compiler with no primary record"); - let why = why.downcast_ref::().cloned().unwrap_or_default(); - assert!(why.contains("toyos-compiler cannot be read"), "{why}"); - assert_eq!(builds.get(), 0, "a missing record built a compiler"); + let fork = e.a.join("rust"); + write(&fork.join("compiler/rustc_target/src/new_target.rs"), "pub fn t() {}\n"); + let untracked = choose(&e.a, &e.rust_dir, &Fork::Checkout(fork.clone()), &sources(&e.a), counted); + assert_ne!(untracked.stage2, ca.stage2, "an untracked target spec kept the old compiler"); + git(&fork, &["add", "-A"]); + git(&fork, &["commit", "-qm", "the target, committed"]); + let committed = choose(&e.a, &e.rust_dir, &Fork::Checkout(fork), &sources(&e.a), counted); + assert_eq!((committed.stage2, builds.get()), (untracked.stage2, 4), "a commit rebuilt the compiler"); + assert_eq!(store::recorded(&e.a, Kind::Llvm), Some(llvm::key(&sources(&e.a))), "the LLVM a compiler links went unrecorded"); } - /// **The primary bootstraps when its `compiler/` holds other content, and - /// never because its files' times moved**: every file rewritten with its own - /// bytes is the compiler just recorded. + /// **The key is content, never files' times**, and every source a compiler + /// is built from moves it: its tools by content, its LLVM by commit. #[test] - fn only_the_compiler_s_content_makes_the_primary_bootstrap() { - let scratch = TempDir::new("compiler-current"); - let (_primary, rust_dir, _) = estate(&scratch); - assert!(primary_is_current(&rust_dir), "the compiler just recorded is not current"); - - let files = snapshot(&rust_dir.join("compiler")); + fn the_key_is_what_the_compiler_is_built_from() { + let e = estate("compiler-key"); + let fork = e.same.join("rust"); + let before = key(&sources(&e.same)); + let spec = fork.join("compiler/rustc_target/src/lib.rs"); let later = std::time::SystemTime::now() + std::time::Duration::from_secs(3600); - for (file, bytes) in &files { - fs::write(file, bytes).unwrap(); - fs::File::options().write(true).open(file).unwrap().set_modified(later).unwrap(); - assert_eq!(fs::metadata(file).unwrap().modified().unwrap(), later); - } - assert!(!files.is_empty()); - assert!( - primary_is_current(&rust_dir), - "every file of compiler/ was rewritten with its own bytes, and the primary would bootstrap" - ); - - let spec = rust_dir.join("compiler/rustc_target/src/lib.rs"); - let held = fs::read(&spec).unwrap(); - write(&spec, "pub fn targets() { riscv() }\n"); - assert!(!primary_is_current(&rust_dir), "a change to compiler/ kept the old stage2"); - fs::write(&spec, held).unwrap(); - assert!(primary_is_current(&rust_dir), "compiler/ put back is not the compiler recorded"); - - write(&rust_dir.join("compiler/rustc_target/src/new_target.rs"), "pub fn t() {}\n"); - assert!(!primary_is_current(&rust_dir), "an untracked file in compiler/ kept the old stage2"); - fs::remove_file(rust_dir.join("compiler/rustc_target/src/new_target.rs")).unwrap(); - - fs::remove_file(primary_record(&rust_dir)).unwrap(); - assert!(!primary_is_current(&rust_dir), "a stage2 nothing recorded was taken for current"); - } - - /// Write the primary's record as it was written before its compiler linked - /// the host's LLVM: its `compiler/` and its LLVM commit. - pub(crate) fn record_before_the_store(rust_dir: &Path) { - fs::write(primary_record(rust_dir), format!("{} llvm {}", compiler_source(rust_dir), llvm_commit(rust_dir))).unwrap(); - } - - /// `fork`'s LLVM checked out at a commit of its own. - pub(crate) fn llvm_checkout(fork: &Path) -> PathBuf { - let llvm = fork.join(LLVM); - git(&llvm, &["init", "-q"]); - write(&llvm.join("llvm/lib/IR/Core.cpp"), "int core;\n"); - git(&llvm, &["add", "-A"]); - git(&llvm, &["commit", "-qm", "LLVM"]); - llvm - } - - /// What `f` panicked with; `expect` if it returned. - fn refusal(expect: &str, f: impl FnOnce()) -> String { - let refused = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).expect_err(expect); - refused.downcast_ref::().cloned().unwrap_or_default() + fs::File::options().write(true).open(&spec).unwrap().set_modified(later).unwrap(); + assert_eq!(key(&sources(&e.same)), before, "a file's time moved the key"); + write(&fork.join("src/tools/lld-wrapper/src/main.rs"), "fn main() { 1; }\n"); + let tools = key(&sources(&e.same)); + assert_ne!(tools, before, "a tool's source did not move the key"); + git(&fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{LLVM_B},{}", llvm::LLVM)]); + assert_ne!(key(&sources(&e.same)), tools, "another LLVM gitlink did not move the key"); + write(&fork.join("library/std/src/lib.rs"), "pub fn z() {}\n"); + let std = key(&sources(&e.same)); + git(&fork, &["checkout", "-q", "--", "library"]); + assert_eq!(key(&sources(&e.same)), std, "a std edit moved the compiler's key"); } - /// An LLVM checkout holding what no commit does, an edit or a file git does - /// not track, names no compiler of a worktree's. + /// **A compiler whose sources moved while it was built is never placed.** #[test] - fn an_uncommitted_llvm_edit_is_refused() { - let scratch = TempDir::new("compiler-llvm-edit"); - let (_primary, rust_dir, [same, _, _]) = estate(&scratch); - let fork = same.join("rust"); - let llvm = llvm_checkout(&fork); - let committed = key(&fork); - let builds = Cell::new(0); - let fake = |fork: &Path| { - builds.set(builds.get() + 1); - fork.join("build/toyos-compiler/stage2") + fn a_compiler_whose_sources_moved_is_never_placed() { + let e = estate("compiler-moved"); + let fork = e.a.join("rust"); + let moving = |fork: &Path| { + write(&fork.join("compiler/rustc_target/src/lib.rs"), "pub fn targets() { moved() }\n"); + fake_build(fork) }; - - write(&llvm.join("llvm/lib/IR/Core.cpp"), "int core_edited;\n"); - let said = refusal("an uncommitted LLVM edit named a compiler", || { - choose(&same, &rust_dir, &fork, fake); + let said = refusal("a compiler whose sources moved was placed", || { + choose(&e.a, &e.rust_dir, &Fork::Checkout(fork.clone()), &sources(&e.a), moving); }); - assert!(said.contains("holds changes no commit does"), "{said}"); - git(&llvm, &["commit", "-qam", "the edit"]); - assert_eq!(key(&fork), committed, "the key read the submodule's commit rather than the gitlink"); - - write(&llvm.join("llvm/lib/IR/Untracked.cpp"), "int untracked;\n"); - let said = refusal("an untracked file in LLVM named a compiler", || { - choose(&same, &rust_dir, &fork, fake); - }); - assert!(said.contains("holds changes no commit does"), "{said}"); - assert_eq!(builds.get(), 0, "an LLVM checkout no commit holds built a compiler"); - } - - /// The primary records no LLVM edit as the commit it is an edit of. - #[test] - fn the_primary_records_no_uncommitted_llvm_edit() { - let scratch = TempDir::new("compiler-llvm-primary"); - let (_primary, rust_dir, _) = estate(&scratch); - let llvm = llvm_checkout(&rust_dir); - let before = fs::read_to_string(primary_record(&rust_dir)).unwrap(); - write(&llvm.join("llvm/lib/IR/Core.cpp"), "int core_edited;\n"); - let said = refusal("the primary recorded an uncommitted LLVM edit as its commit", || record(&rust_dir)); - assert!(said.contains("holds changes no commit does"), "{said}"); - assert_eq!(fs::read_to_string(primary_record(&rust_dir)).unwrap(), before); + assert!(said.contains("moved while compiler"), "{said}"); + let placed: Vec<_> = fs::read_dir(Kind::Compiler.dir(&e.rust_dir)).into_iter().flatten().flatten().map(|e| e.file_name()).collect(); + assert!(placed.is_empty(), "placed: {placed:?}"); } - /// Every source a compiler is built from moves its key: LLVM by commit, - /// the tools by content. + /// **A compiler built from a submodule at another commit than its gitlink + /// is never placed**, `library/backtrace` or the `src/tools/cargo` it + /// ships: bootstrap leaves either at `HEAD`'s gitlink under a staged one, + /// and builds it. #[test] - fn llvm_and_the_tools_move_the_key() { - let scratch = TempDir::new("compiler-key"); - let (_primary, _rust_dir, [same, _, _]) = estate(&scratch); - let fork = same.join("rust"); - let before = key(&fork); - write(&fork.join("src/tools/lld-wrapper/src/main.rs"), "fn main() { 1; }\n"); - let tools = key(&fork); - assert_ne!(tools, before, "a tool's source did not move the key"); - git(&fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{LLVM_B},{LLVM}")]); - let said = refusal("a gitlink staged and not committed named a compiler", || { - key(&fork); - }); - assert!(said.contains("stages"), "{said}"); - git(&fork, &["commit", "-qm", "another LLVM"]); - assert_ne!(key(&fork), tools, "another LLVM commit did not move the key"); - } - - /// A primary record naming another LLVM, or none, is another compiler. - #[test] - fn another_llvm_is_another_compiler() { - let scratch = TempDir::new("compiler-llvm"); - let (_primary, rust_dir, [same, _, _]) = estate(&scratch); - let builds = Cell::new(0); - let fake = |fork: &Path| { - builds.set(builds.get() + 1); - let stage2 = fork.join("build/toyos-compiler/stage2"); - write(&stage2.join("bin/rustc"), "a rustc"); - write(&stage2.join("lib/librustc_driver-2.dylib"), "a driver"); - stage2 - }; - let fork = same.join("rust"); - assert!(choose(&same, &rust_dir, &fork, fake).primary, "the primary's own compiler/ and LLVM built one"); - - let record = primary_record(&rust_dir); - let recorded = fs::read_to_string(&record).unwrap(); - let (compiler, llvm) = recorded.rsplit_once(" llvm ").expect("the record names its LLVM"); - assert_eq!(llvm, crate::llvm::key(&rust_dir)); - fs::write(&record, compiler).unwrap(); - assert!(!choose(&same, &rust_dir, &fork, fake).primary, "a record naming no LLVM was taken for this one"); - assert_eq!(builds.get(), 1); - fs::write(&record, &recorded).unwrap(); - - git(&fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{LLVM_B},{LLVM}")]); - git(&fork, &["commit", "-qm", "the ToyOS LLVM"]); - let mine = choose(&same, &rust_dir, &fork, fake); - assert!(!mine.primary, "a worktree pinning another LLVM took the primary's compiler"); - assert_eq!(builds.get(), 2); + fn a_compiler_built_off_a_submodule_s_gitlink_is_never_placed() { + for path in ["library/backtrace", "src/tools/cargo"] { + let e = estate("compiler-gitlink"); + let fork = e.a.join("rust"); + let (head, staged) = behind_a_staged_gitlink(&fork, path); + let said = refusal(&format!("a compiler built from a {path} its gitlink does not name was placed"), || { + choose(&e.a, &e.rust_dir, &Fork::Checkout(fork.clone()), &sources(&e.a), fake_build); + }); + assert!(said.contains(&format!("{} is at {head}, and its gitlink names {staged}", fork.join(path).display())), "{said}"); + let placed: Vec<_> = fs::read_dir(Kind::Compiler.dir(&e.rust_dir)).into_iter().flatten().flatten().map(|e| e.file_name()).collect(); + assert!(placed.is_empty(), "placed: {placed:?}"); + } } } diff --git a/src/dirlock.rs b/src/dirlock.rs new file mode 100644 index 00000000000..004b979fcb4 --- /dev/null +++ b/src/dirlock.rs @@ -0,0 +1,221 @@ +//! A lock is `flock(2)` on an open directory. The kernel releases it +//! when the holder's descriptor closes, so a killed holder strands nothing, and +//! the descriptor is close-on-exec, so no child a holder spawns keeps it held. + +use std::fs::File; +use std::io::ErrorKind; +use std::os::unix::io::AsRawFd; +use std::path::Path; +use std::sync::mpsc::{channel, RecvTimeoutError}; +use std::time::{Duration, Instant}; + +/// How often a lasting wait says again what it waits for: an agent that sees +/// silence kills the wait and retries. +const HEARTBEAT: Duration = Duration::from_secs(30); + +/// A held lock; dropping it releases it. +pub struct Lock { + file: File, +} + +impl Lock { + /// `dir` shared: any number of holders at once, and no exclusive one. + pub fn shared(dir: &Path, what: &str) -> Self { + Self::there(dir, libc::LOCK_SH, what).unwrap_or_else(|| panic!("lock {}: it is not there", dir.display())) + } + + /// `dir` exclusively. + pub fn exclusive(dir: &Path, what: &str) -> Self { + Self::there(dir, libc::LOCK_EX, what).unwrap_or_else(|| panic!("lock {}: it is not there", dir.display())) + } + + /// `dir` shared, or `None` if there is no `dir`. + pub(crate) fn shared_if_there(dir: &Path, what: &str) -> Option { + Self::there(dir, libc::LOCK_SH, what) + } + + /// `dir` exclusively if nobody holds it, without waiting; `None` if somebody + /// does or there is no `dir`. + pub(crate) fn try_exclusive(dir: &Path) -> Option { + let file = open(dir)?; + attempt(&file, libc::LOCK_EX | libc::LOCK_NB).then_some(Self { file }) + } + + /// Run `f` holding this lock exclusively, and hold it shared again after. + /// The conversion is not atomic, so `f` decides afresh what to do. + pub fn exclusively(&mut self, what: &str, f: impl FnOnce() -> R) -> R { + take(&self.file, libc::LOCK_EX, what); + let out = f(); + take(&self.file, libc::LOCK_SH, what); + out + } + + /// The directory this holds, open. + pub(crate) fn file(&self) -> &File { + &self.file + } + + fn there(dir: &Path, op: i32, what: &str) -> Option { + let file = open(dir)?; + take(&file, op, what); + Some(Self { file }) + } +} + +fn open(dir: &Path) -> Option { + match File::open(dir) { + Ok(file) => Some(file), + Err(e) if e.kind() == ErrorKind::NotFound => None, + Err(e) => panic!("open {}: {e}", dir.display()), + } +} + +/// Take `op` on `file`, saying so when it has to wait and every [`HEARTBEAT`] +/// while it does. +fn take(file: &File, op: i32, what: &str) { + if attempt(file, op | libc::LOCK_NB) { + return; + } + eprintln!("[lock] waiting: {what}"); + let (tx, rx) = channel::<()>(); + let heartbeat = { + let what = what.to_string(); + std::thread::spawn(move || { + let began = Instant::now(); + while rx.recv_timeout(HEARTBEAT) == Err(RecvTimeoutError::Timeout) { + eprintln!("[lock] still waiting, {:.0?} so far: {what}", began.elapsed()); + } + }) + }; + assert!(attempt(file, op), "a blocking flock returned without the lock"); + drop(tx); + heartbeat.join().expect("the lock heartbeat panicked"); +} + +/// Whether `flock(op)` took the lock; `false` only for a non-blocking `op` +/// somebody else's lock refused. +fn attempt(file: &File, op: i32) -> bool { + loop { + // SAFETY: `file` owns the descriptor for the length of the call. + if unsafe { libc::flock(file.as_raw_fd(), op) } == 0 { + return true; + } + let err = std::io::Error::last_os_error(); + match err.kind() { + ErrorKind::Interrupted => continue, + ErrorKind::WouldBlock => return false, + _ => panic!("flock: {err}"), + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::ffi::OsStr; + use std::path::PathBuf; + use std::process::{Child, Command}; + use toyos_tmpdir::TempDir; + + const MARKS: &str = "TOYOS_DIRLOCK_TEST_MARKS"; + + /// This test binary, to run the one `#[ignore]`d test `test` names. + pub(crate) fn rerun(test: &str) -> Command { + let mut rerun = Command::new(std::env::current_exe().unwrap()); + rerun.args(["--exact", test, "--include-ignored", "--nocapture"]); + rerun + } + + /// A process of its own running the `#[ignore]`d test `role` names, which + /// has reached [`held_until_killed`]. + /// + /// Another process because a lock is per open file description and a test + /// thread's spawn copies every descriptor open at that moment until its exec: + /// what a test asserts about a lock held elsewhere, it asserts of a process. + pub(crate) struct Elsewhere { + child: Child, + _marks: TempDir, + } + + impl Elsewhere { + pub(crate) fn hold(role: &str, env: &[(&str, &OsStr)]) -> Self { + let marks = TempDir::new("dirlock-elsewhere"); + let mut child = rerun(role).envs(env.iter().copied()).env(MARKS, &marks).spawn().expect("spawn the holder"); + let deadline = Instant::now() + Duration::from_secs(20); + while !marks.join("held").exists() { + if let Some(status) = child.try_wait().unwrap() { + panic!("{role} exited before it held anything: {status}"); + } + if Instant::now() >= deadline { + child.kill().unwrap(); + panic!("{role} held nothing in 20 s, killed: {}", child.wait().unwrap()); + } + std::thread::sleep(Duration::from_millis(5)); + } + Self { child, _marks: marks } + } + + /// SIGKILL it, and return once it is reaped. + pub(crate) fn kill(mut self) { + self.child.kill().unwrap(); + self.child.wait().unwrap(); + } + } + + impl Drop for Elsewhere { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } + } + + /// `file`, a directory opened earlier, exclusively if nobody holds it: a + /// lock granted after whatever named it may have moved on. + pub(crate) fn try_exclusive_opened(file: File) -> Option { + attempt(&file, libc::LOCK_EX | libc::LOCK_NB).then_some(Lock { file }) + } + + /// The holder's half of [`Elsewhere`]: say it holds, and hold until it is + /// killed, or fail loudly after a minute. + pub(crate) fn held_until_killed() { + let marks = PathBuf::from(std::env::var(MARKS).expect("a holder runs under Elsewhere::hold")); + std::fs::write(marks.join("held"), b"").unwrap(); + std::thread::sleep(Duration::from_secs(60)); + panic!("the holder was never killed"); + } + + const DIR: &str = "TOYOS_DIRLOCK_TEST_DIR"; + + #[test] + #[ignore = "the holder for the tests below; never runs on its own"] + fn hold_exclusive() { + let dir = PathBuf::from(std::env::var(DIR).unwrap_or_else(|_| panic!("hold_exclusive ran without {DIR}; it is not a test"))); + let _held = Lock::exclusive(&dir, "the test holder"); + held_until_killed(); + } + + /// **An exclusive holder excludes, and a killed one strands nothing.** + #[test] + fn an_exclusive_lock_excludes_until_its_holder_dies() { + let dir = TempDir::new("dirlock"); + let holder = Elsewhere::hold("dirlock::tests::hold_exclusive", &[(DIR, dir.as_os_str())]); + assert!(Lock::try_exclusive(&dir).is_none(), "two exclusive holders at once"); + holder.kill(); + assert!(Lock::try_exclusive(&dir).is_some(), "a SIGKILLed holder stranded the lock"); + } + + /// **Shared admits shared and refuses exclusive**, and a conversion + /// excludes the other shared holders while it lasts. + #[test] + fn shared_admits_shared_and_refuses_exclusive() { + let dir = TempDir::new("dirlock-shared"); + let shared_now = || attempt(&open(&dir).unwrap(), libc::LOCK_SH | libc::LOCK_NB); + let mut mine = Lock::shared(&dir, "one"); + let theirs = Lock::shared(&dir, "two"); + assert!(Lock::try_exclusive(&dir).is_none(), "an exclusive holder got in beside shared ones"); + drop(theirs); + mine.exclusively("clean", || assert!(!shared_now(), "a shared holder got in beside an exclusive one")); + assert!(shared_now(), "the conversion back left the lock exclusive"); + assert!(Lock::shared_if_there(&dir.join("absent"), "absent").is_none()); + } +} diff --git a/src/flags.rs b/src/flags.rs index 9200b194d03..9c3667eca20 100644 --- a/src/flags.rs +++ b/src/flags.rs @@ -29,7 +29,7 @@ pub(crate) enum Value { /// answers about every row, and `--known-red ` about one. Optional, /// Every word after it — the flag names a subcommand that owns the rest of - /// the line, as `--worktree add ` does. + /// the line. Rest, } @@ -73,7 +73,6 @@ declare_flags!(pub CARGO_RUN = { pub REGEN_FONT = "--regen-font", None; pub REGEN_WALLPAPER = "--regen-wallpaper", None; pub REGEN_SOUNDFONT = "--regen-soundfont", Next; - pub WORKTREE = "--worktree", Rest; /// Mint the owner's image-signing key where `signing::owner_key_path` /// says, refusing to replace one. pub SIGNING_KEY_NEW = "--signing-key-new", None; @@ -167,8 +166,8 @@ pub(crate) struct Walk<'a> { impl Walk<'_> { /// A flag whose value no reader of this line would get — every shape that /// reaches a reader as a silent default, and the whole of what either - /// command line asks. A flag with nothing after it — the end of the line, - /// an empty `--smp=`, or a `--worktree` owning no words — answers `None` to + /// command line asks. A flag with nothing after it — the end of the line + /// or an empty `--smp=` — answers `None` to /// [`Vocabulary::value`] and `&[]` to [`Vocabulary::rest`]; a flag written /// twice has every use but one dropped; and an inline value is dropped by /// exactly two readers, `Value::None` having none and `rest` taking only the @@ -339,8 +338,8 @@ mod tests { fn an_inline_value_is_refused_exactly_where_it_would_be_dropped() { let debug = refusal(&["--debug=1"]); assert!(debug.contains("--debug takes no value"), "{debug}"); - let worktree = refusal(&["--worktree=add"]); - assert!(worktree.contains("--worktree "), "{worktree}"); + let ci = refusal(&["--ci=host"]); + assert!(ci.contains("--ci "), "{ci}"); let line = argv(&["--smp=4", "--known-red=audio_tone", "--kernel-param=slow"]); assert!(matches!(check(&line), Outcome::Proceed)); assert_eq!(CARGO_RUN.value(&line, &SMP), Some("4")); @@ -396,7 +395,7 @@ mod tests { for words in [ vec!["--kernel-param", "--help"], vec!["--boot-config", "--help"], - vec!["--worktree", "add", "--help"], + vec!["--ci", "host", "--help"], vec!["--boot-config", "diag", "--build-only"], ] { assert!(matches!(checked(&words), Outcome::Proceed), "{words:?} must proceed"); @@ -404,8 +403,8 @@ mod tests { let line = argv(&["--kernel-param", "--help"]); assert_eq!(CARGO_RUN.values(&line, &KERNEL_PARAM), ["--help"]); assert!(!CARGO_RUN.present(&line, &HELP), "the actuator's name is not the flag"); - let worktree = argv(&["--worktree", "add", "/tmp/wt"]); - assert_eq!(CARGO_RUN.rest(&worktree, &WORKTREE), ["add", "/tmp/wt"]); + let ci = argv(&["--ci", "host", "--help"]); + assert_eq!(CARGO_RUN.rest(&ci, &CI), ["host", "--help"]); assert_eq!(CARGO_RUN.value(&argv(&["--boot-config", "diag"]), &BOOT_CONFIG), Some("diag")); assert_eq!(CARGO_RUN.value(&argv(&["--build-only"]), &BOOT_CONFIG), None); } diff --git a/src/hostws.rs b/src/hostws.rs index 3eff316f364..b956d285cb2 100644 --- a/src/hostws.rs +++ b/src/hostws.rs @@ -119,9 +119,7 @@ pub fn is_member(root: &Path, crate_dir: &Path) -> bool { /// the rule above is about enablement and not about the feature: the flag is /// honoured by a nightly-capable cargo and *silently ignored* by any other, and /// nothing a `.cargo/config.toml` can say travels with the shared `target-dir` -/// it would also carry. This host's rustup default is stable, and -/// `src/toolchain.rs`'s `host_cargo` lends the `toyos` toolchain whatever cargo -/// the machine has — stable's, on every CI runner. Sharing on with +/// it would also carry. This host's rustup default is stable. Sharing on with /// freshness off is the mis-link above, so this function joins one path and /// stays out of it. pub fn target_dir(root: &Path, crate_dir: &Path) -> PathBuf { diff --git a/src/identity.rs b/src/identity.rs deleted file mode 100644 index c366ad718eb..00000000000 --- a/src/identity.rs +++ /dev/null @@ -1,301 +0,0 @@ -//! What a source file is to a build: its token stream, not its text. -//! -//! **One definition, read by every question of the form "did this source -//! change what gets built"**: the key a sysroot is filed under. A comment -//! — a doc comment included — and the whitespace around tokens change no item, -//! no layout and no code, so a change made only of them is no new sysroot. -//! -//! A `.rs` file is lexed just far enough to find its comments: every string, -//! raw string and character literal is kept byte for byte, every comment and -//! every run of whitespace becomes one space, and nothing else moves. Two files -//! with equal identities therefore lex to the same tokens. The converse is not -//! claimed: space appearing where there was none (`A;` to `A ;`) is a change, -//! because between two punctuation characters it can be one (`&&` against -//! `& &`), and telling those apart is a lexer this does not need to be. Any -//! other file is its bytes. -//! -//! What this gives up, deliberately: rustdoc output, and the line numbers a -//! panic location or debuginfo carries — a comment that adds a line moves those -//! and is still not a new sysroot. A `#![deny(missing_docs)]` crate is the one -//! place a doc comment decides whether a build succeeds; none of the crates -//! this is asked about carries it, and the crate's own build is what would -//! refuse. - -use std::borrow::Cow; -use std::path::Path; - -/// `bytes`, as what a build of the file at `path` can see of them. -pub fn of<'a>(path: &Path, bytes: &'a [u8]) -> Cow<'a, [u8]> { - if path.extension().is_some_and(|e| e == "rs") { - Cow::Owned(rust_tokens(bytes)) - } else { - Cow::Borrowed(bytes) - } -} - -fn is_space(b: u8) -> bool { - matches!(b, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c) -} - -fn is_word(b: u8) -> bool { - b.is_ascii_alphanumeric() || b == b'_' || b >= 0x80 -} - -/// The length of the UTF-8 character whose first byte is `lead`. -fn char_len(lead: u8) -> usize { - match lead { - 0xf0..=0xff => 4, - 0xe0..=0xef => 3, - 0xc0..=0xdf => 2, - _ => 1, - } -} - -/// The source with every comment and every run of whitespace reduced to one -/// space between the tokens it separated. -fn rust_tokens(b: &[u8]) -> Vec { - let mut out = Vec::with_capacity(b.len()); - let mut gap = false; - let emit = |out: &mut Vec, gap: &mut bool, token: &[u8]| { - if *gap && !out.is_empty() { - out.push(b' '); - } - *gap = false; - out.extend_from_slice(token); - }; - let n = b.len(); - let mut i = 0; - while i < n { - let c = b[i]; - if is_space(c) { - gap = true; - i += 1; - } else if b[i..].starts_with(b"//") { - while i < n && b[i] != b'\n' { - i += 1; - } - gap = true; - } else if b[i..].starts_with(b"/*") { - // Nested, as Rust's block comments are. - let mut depth = 0usize; - while i < n { - if b[i..].starts_with(b"/*") { - depth += 1; - i += 2; - } else if b[i..].starts_with(b"*/") { - depth -= 1; - i += 2; - if depth == 0 { - break; - } - } else { - i += 1; - } - } - gap = true; - } else if c == b'"' { - let end = quoted_end(b, i); - emit(&mut out, &mut gap, &b[i..end]); - i = end; - } else if c == b'\'' { - let end = char_literal_end(b, i).unwrap_or(i + 1); - emit(&mut out, &mut gap, &b[i..end]); - i = end; - } else if is_word(c) { - let mut end = i; - while end < n && is_word(b[end]) { - end += 1; - } - let end = match &b[i..end] { - b"r" | b"br" | b"cr" => raw_string_end(b, end).unwrap_or(end), - _ => end, - }; - emit(&mut out, &mut gap, &b[i..end]); - i = end; - } else { - emit(&mut out, &mut gap, &b[i..=i]); - i += 1; - } - } - out -} - -/// One past the `"` closing the string opened at `open`, escapes honoured. -fn quoted_end(b: &[u8], open: usize) -> usize { - let mut j = open + 1; - while j < b.len() { - match b[j] { - b'\\' => j += 2, - b'"' => return j + 1, - _ => j += 1, - } - } - b.len() -} - -/// One past a character literal opening at `open`, or `None` where the `'` -/// begins a lifetime or a label. -fn char_literal_end(b: &[u8], open: usize) -> Option { - let first = *b.get(open + 1)?; - if first == b'\\' { - // The escaped character itself, then on to the close: `'\''`, `'\u{2764}'`. - let mut j = open + 3; - while j < b.len() && b[j] != b'\'' { - j += 1; - } - return Some((j + 1).min(b.len())); - } - let close = open + 1 + char_len(first); - (b.get(close) == Some(&b'\'')).then_some(close + 1) -} - -/// One past a raw string whose prefix ends at `after_prefix`, or `None` where -/// the prefix is an identifier (`r#match`) or a lone `r`. -fn raw_string_end(b: &[u8], after_prefix: usize) -> Option { - let mut j = after_prefix; - while b.get(j) == Some(&b'#') { - j += 1; - } - if b.get(j) != Some(&b'"') { - return None; - } - let hashes = j - after_prefix; - let mut k = j + 1; - while k < b.len() { - if b[k] == b'"' && b[k + 1..].iter().take(hashes).filter(|&&h| h == b'#').count() == hashes - { - return Some(k + 1 + hashes); - } - k += 1; - } - Some(b.len()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn rs(text: &str) -> String { - String::from_utf8(of(Path::new("x.rs"), text.as_bytes()).into_owned()).unwrap() - } - - fn same(a: &str, b: &str) -> bool { - rs(a) == rs(b) - } - - /// **A recorded real failure, verbatim**: the hunk of `toyos-abi/src/syscall.rs` - /// that `c00056b9` (#499) changed, which cost a minor bump of three crates - /// and a shared-sysroot rebuild in every worktree. Its identity did not move. - #[test] - fn the_doc_fix_that_cost_a_version_and_a_sysroot_is_neither() { - let before = " pub const CANARY_CHANGED: u64 = 11; - /// Make the last CPU a shootdown waits for answer `arg` nanoseconds late, - /// and take it away again. - pub const TLB_ACK_DELAY_ARM: u64 = 12; -"; - let after = " pub const CANARY_CHANGED: u64 = 11; - /// Hold each other CPU's acknowledgement back for `arg` nanoseconds in - /// turn, one at a time, and answer the smallest wait any of them cost the - /// initiator — `0` on a machine with no other CPU to hold back. The - /// arming is then left standing against every other CPU until - /// `TLB_ACK_DELAY_DISARM` or the end of a fresh two-second window, - /// whichever comes first. - pub const TLB_ACK_DELAY_ARM: u64 = 12; -"; - assert!(same(before, after)); - // `439244cf`'s, from `toyos-abi/src/boot.rs`, the same shape. - assert!(same( - "/// The most windows the loader will carry: four memory windows on each of\n\ - /// sixteen root bridges.\npub const MAX_ROOT_BRIDGE_WINDOWS: usize = 64;\n", - "/// The most windows the loader will carry.\npub const MAX_ROOT_BRIDGE_WINDOWS: usize = 64;\n", - )); - } - - /// And the other direction, which a function that ignored everything would - /// pass the test above with: a value, a type, a name, a field. - #[test] - fn a_signature_or_a_value_is_a_change() { - assert!(!same("pub const TLB_ACK_DELAY_ARM: u64 = 12;", "pub const TLB_ACK_DELAY_ARM: u64 = 13;")); - assert!(!same("pub struct A;", "pub struct A(pub u64);")); - assert!(!same("pub fn f(a: u32) {}", "pub fn f(a: u64) {}")); - assert!(!same("pub fn f() {}", "pub fn g() {}")); - assert!(!same("#[repr(C)] struct S { a: u8, b: u32 }", "#[repr(C)] struct S { b: u32, a: u8 }")); - // Whitespace separates tokens, so its presence is a change and only its amount is not. - assert!(!same("a b", "ab")); - assert!(same("a b", "a\n\tb")); - assert!(same("a/**/b", "a b")); - assert!(!same("a/**/b", "ab")); - } - - /// Everything that only looks like a comment is kept, byte for byte. - #[test] - fn a_literal_is_never_read_as_a_comment() { - assert!(!same(r#"const U: &str = "http://a";"#, r#"const U: &str = "http://b";"#)); - assert!(!same(r#"const U: &str = "a /* b */ c";"#, r#"const U: &str = "a c";"#)); - assert!(!same("const S: &str = \"a b\";", "const S: &str = \"a b\";")); - assert!(!same(r#"const Q: &str = "\" // x";"#, r#"const Q: &str = "\" // y";"#)); - assert!(!same(r###"const R: &str = r#"a "// x" b"#;"###, r###"const R: &str = r#"a "// y" b"#;"###)); - assert!(!same(r#"const B: &[u8] = br"// x";"#, r#"const B: &[u8] = br"// y";"#)); - assert!(!same("const C: char = '\"'; // \"\nconst D: u8 = 1;", "const C: char = '\"'; // \"\nconst D: u8 = 2;")); - assert!(!same("const C: char = '\\''; const D: &str = \"// x\";", "const C: char = '\\''; const D: &str = \"// y\";")); - assert!(!same("const C: char = 'é'; const D: &str = \"// x\";", "const C: char = 'é'; const D: &str = \"// y\";")); - // A lifetime is not a character literal, and a raw identifier is not a raw string. - assert!(same("fn f<'a>(x: &'a str) -> &'a str { x } // one", "fn f<'a>(x: &'a str) -> &'a str { x }")); - assert!(!same("fn f<'a>(x: &'a str) { \"// x\"; }", "fn f<'a>(x: &'a str) { \"// y\"; }")); - assert!(same("let r#match = 1; // one", "let r#match = 1;")); - // Block comments nest. - assert!(same("a /* x /* y */ z */ b", "a b")); - assert!(!same("a /* x /* y */ z */ b", "a z */ b")); - } - - /// Anything but Rust is its bytes. - #[test] - fn a_file_that_is_not_rust_is_its_bytes() { - let toml = b"[package] # a comment\n"; - assert_eq!(&*of(Path::new("Cargo.toml"), toml), toml); - assert_eq!(&*of(Path::new("abi.h"), b"/* c */ int a;"), b"/* c */ int a;"); - } - - /// **The tree itself as the corpus**: every `.rs` file of every crate this - /// is asked about lexes to a fixed point, and deleting every line of it that - /// is a doc comment changes nothing. - #[test] - fn every_doc_comment_in_the_sysroot_crates_is_invisible() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")); - let mut files = Vec::new(); - for dir in crate::sysroot::SYSROOT_SOURCES { - walk(&root.join(dir), &mut files); - } - assert!(files.len() > 50, "the corpus walk found {} files", files.len()); - let mut docs = 0; - for path in files { - let text = std::fs::read_to_string(&path).unwrap(); - let id = rs(&text); - assert_eq!(rs(&id), id, "{} does not lex to a fixed point", path.display()); - let undocumented: String = text - .lines() - .filter(|l| { - let doc = l.trim_start().starts_with("///") || l.trim_start().starts_with("//!"); - docs += usize::from(doc); - !doc - }) - .map(|l| format!("{l}\n")) - .collect(); - assert_eq!(rs(&undocumented), id, "{}'s doc comments reach its identity", path.display()); - } - assert!(docs > 1000, "the corpus holds {docs} doc lines, which is not the tree"); - } - - fn walk(dir: &Path, out: &mut Vec) { - for entry in std::fs::read_dir(dir).unwrap().flatten() { - let path = entry.path(); - if path.is_dir() { - if path.file_name().is_some_and(|n| n != "target") { - walk(&path, out); - } - } else if path.extension().is_some_and(|e| e == "rs") { - out.push(path); - } - } - } -} diff --git a/src/keystore.rs b/src/keystore.rs deleted file mode 100644 index 8f9dfdd8db1..00000000000 --- a/src/keystore.rs +++ /dev/null @@ -1,288 +0,0 @@ -//! What every content-addressed product of the host ([`Keyed`]) shares: the -//! record each worktree keeps of the key it uses, and the sweep that removes a -//! key no registered worktree records and nobody is making or using. -//! -//! A product lives at `//`, whole once its maker renamed it there; -//! any other name beginning `.` is one half-made or half-removed. A whole -//! one is renamed out of the way before anything in it is removed ([`retire`]), -//! so a sweep that is stopped leaves nothing at `/` but what was made. A -//! product may be read-only, directories and all: removing one gives its -//! directories back their write permission first ([`remove`]). - -use std::collections::{BTreeMap, BTreeSet}; -use std::fs; -use std::io::ErrorKind; -use std::os::unix::fs::PermissionsExt; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; - -use crate::buildlock::{self, Guard, Keyed}; -use crate::sysroot::git_out; - -/// Where `root`'s builds record the key of `kind` they use. -fn record_path(root: &Path, kind: Keyed) -> PathBuf { - root.join(match kind { - Keyed::Sysroot => "target/toyos-sysroot-key", - Keyed::Compiler => "target/toyos-compiler-key", - Keyed::Llvm => "target/toyos-llvm-key", - }) -} - -/// Record that `root` uses `kind`'s `key`: whole or not at all, so a sweep -/// never reads a record half-written. -pub fn record(root: &Path, kind: Keyed, key: &str) { - record_by(root, kind, key, |path, key| fs::write(path, key).unwrap_or_else(|e| panic!("write {}: {e}", path.display()))); -} - -static TEMPS: AtomicU64 = AtomicU64::new(0); - -/// [`record`], writing with `write`, so a test can stop it. -fn record_by(root: &Path, kind: Keyed, key: &str, write: impl FnOnce(&Path, &str)) { - let path = record_path(root, kind); - let dir = path.parent().expect("a file under target/"); - fs::create_dir_all(dir).unwrap_or_else(|e| panic!("create {}: {e}", dir.display())); - // Its own name: concurrent writers of one record never share a temp file. - let written = path.with_extension(format!("{}.{}.new", std::process::id(), TEMPS.fetch_add(1, Ordering::Relaxed))); - write(&written, key); - fs::rename(&written, &path).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", written.display(), path.display())); -} - -/// Record that `root` uses `kind`'s `key`, whose product is `store/`, and -/// hold it in use: made by `make` first when `defect` says it is not whole -/// (`buildlock::keyed_made`), and then `store` swept, so the product one -/// replaced goes once nobody names it. -pub fn made( - root: &Path, - kind: Keyed, - store: &Path, - key: &str, - defect: impl Fn() -> Option, - mut make: impl FnMut(), -) -> Guard { - record(root, kind, key); - let mut placed = false; - let using = buildlock::keyed_made(root, kind, key, defect, || { - make(); - placed = true; - }); - if placed { - for gone in sweep(root, kind, store) { - eprintln!("Removed {} {}: no worktree names it", kind.name(), gone.display()); - } - } - using -} - -/// Record that `root` uses no `kind` of its own. -pub fn forget(root: &Path, kind: Keyed) { - let path = record_path(root, kind); - match fs::remove_file(&path) { - Err(e) if e.kind() != ErrorKind::NotFound => panic!("remove {}: {e}", path.display()), - _ => {} - } -} - -/// The key of `kind` `root` records, if it records one. -pub fn recorded(root: &Path, kind: Keyed) -> Option { - let path = record_path(root, kind); - match fs::read_to_string(&path) { - Ok(key) => Some(key.trim().to_string()), - Err(e) if e.kind() == ErrorKind::NotFound => None, - Err(e) => panic!("read {}: {e}", path.display()), - } -} - -/// Remove from `store` every `kind` no registered worktree of `root` records -/// and nobody is making or using, and every half-made or half-removed one -/// nobody is making. Returns what went. -pub fn sweep(root: &Path, kind: Keyed, store: &Path) -> Vec { - sweep_by(root, kind, store, remove) -} - -/// [`sweep`], removing with `remove`, so a test can stop it. -pub(crate) fn sweep_by(root: &Path, kind: Keyed, store: &Path, remove: impl Fn(&Path) + Copy) -> Vec { - let entries = match fs::read_dir(store) { - Ok(entries) => entries, - Err(e) if e.kind() == ErrorKind::NotFound => return Vec::new(), - Err(e) => panic!("read {}: {e}", store.display()), - }; - let named: BTreeSet = git_out(root, &["worktree", "list", "--porcelain"]) - .lines() - .filter_map(|l| l.strip_prefix("worktree ")) - .filter_map(|w| recorded(Path::new(w), kind)) - .collect(); - let mut by_key: BTreeMap> = BTreeMap::new(); - for entry in entries { - let entry = entry.unwrap_or_else(|e| panic!("read {}: {e}", store.display())); - let name = entry - .file_name() - .into_string() - .unwrap_or_else(|n| panic!("{} holds {n:?}, which names no key", store.display())); - let key = name.split_once('.').map_or(name.as_str(), |(key, _)| key).to_string(); - by_key.entry(key).or_default().push(name); - } - let mut removed = Vec::new(); - for (key, mut names) in by_key { - names.retain(|name| *name != key || !named.contains(&key)); - if names.is_empty() { - continue; - } - let Some(_idle) = buildlock::keyed_idle(root, kind, &key) else { continue }; - // The whole one last: its `retire` goes where a stopped one's remains were. - names.sort_by_key(|name| *name == key); - for name in names { - let path = store.join(&name); - if name == key { - retire_by(&path, remove); - } else { - remove(&path); - } - removed.push(path); - } - } - removed -} - -/// Remove the directory `path` if it is there, renamed to `.swept` before -/// anything in it is removed; what a stopped one left there goes first. -pub fn retire(path: &Path) { - retire_by(path, remove); -} - -/// [`retire`], removing with `remove`, so a test can stop it. -fn retire_by(path: &Path, remove: impl Fn(&Path)) { - let away = path.with_extension("swept"); - if away.exists() { - remove(&away); - } - if path.exists() { - fs::rename(path, &away).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", path.display(), away.display())); - remove(&away); - } -} - -/// Remove the directory `path` and all it holds, read-only or not. -pub fn remove(path: &Path) { - writable(path); - fs::remove_dir_all(path).unwrap_or_else(|e| panic!("remove {}: {e}", path.display())); -} - -/// Give `dir` and every directory under it back its owner's write permission. -pub(crate) fn writable(dir: &Path) { - let meta = fs::metadata(dir).unwrap_or_else(|e| panic!("stat {}: {e}", dir.display())); - let mut permissions = meta.permissions(); - permissions.set_mode(permissions.mode() | 0o700); - fs::set_permissions(dir, permissions).unwrap_or_else(|e| panic!("chmod {}: {e}", dir.display())); - for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { - let entry = entry.unwrap_or_else(|e| panic!("read {}: {e}", dir.display())); - if entry.file_type().unwrap_or_else(|e| panic!("stat {}: {e}", entry.path().display())).is_dir() { - writable(&entry.path()); - } - } -} - -#[cfg(test)] -mod tests { - use toyos_tmpdir::TempDir; - - use super::*; - use crate::compiler::tests::{git, write}; - - /// A key no registered worktree records goes, read-only or not, and so does - /// a half-made or half-removed one; a key a worktree records stays, and so - /// does one somebody is using; an unreadable record is refused, never read - /// as "names nothing". - #[test] - fn a_sweep_removes_what_no_worktree_names_and_nobody_uses() { - let root = TempDir::new("sweep"); - git(&root, &["init", "-q"]); - write(&root.join("f"), "x\n"); - git(&root, &["add", "f"]); - git(&root, &["commit", "-qm", "init"]); - let linked = root.join("linked"); - git(&root, &["worktree", "add", "-q", "-b", "wt", linked.to_str().unwrap()]); - - let dir = root.join("store"); - for name in ["named", "linked-named", "in-use", "orphan", "named.partial", "gone.swept", "orphan.swept"] { - fs::create_dir_all(dir.join(name).join("sub")).unwrap(); - } - for read_only in ["orphan/sub", "orphan", "named.partial"] { - fs::set_permissions(dir.join(read_only), fs::Permissions::from_mode(0o555)).unwrap(); - } - record(&root, Keyed::Sysroot, "named"); - record(&linked, Keyed::Sysroot, "linked-named"); - let user = buildlock::tests::sysroot_used_elsewhere(&root, "in-use"); - - let mut removed = sweep(&root, Keyed::Sysroot, &dir); - removed.sort(); - let want = ["gone.swept", "named.partial", "orphan", "orphan.swept"].map(|n| dir.join(n)); - assert_eq!(removed, want); - for stays in ["named", "linked-named", "in-use"] { - assert!(dir.join(stays).is_dir(), "{stays} was swept"); - } - for gone in want { - assert!(!gone.exists(), "{} was reported and kept", gone.display()); - } - user.release(); - assert_eq!(sweep(&root, Keyed::Sysroot, &dir), [dir.join("in-use")]); - - fs::remove_file(record_path(&linked, Keyed::Sysroot)).unwrap(); - fs::create_dir(record_path(&linked, Keyed::Sysroot)).unwrap(); - let unreadable = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| sweep(&root, Keyed::Sysroot, &dir))); - assert!(unreadable.is_err(), "an unreadable record was read as naming nothing"); - assert!(dir.join("linked-named").is_dir(), "an unreadable record's key was swept"); - } - - /// **A record stopped mid-write leaves the one before it readable**: the - /// new key is written beside it and renamed over it whole. - #[test] - fn a_stopped_record_leaves_the_one_before_it() { - let root = TempDir::new("record"); - record(&root, Keyed::Llvm, "0123456789abcdef"); - let stopped = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - record_by(&root, Keyed::Llvm, "fedcba9876543210", |path, key| { - fs::write(path, &key[..8]).unwrap(); - panic!("stopped"); - }) - })); - assert!(stopped.is_err(), "the stand-in write was never asked"); - assert_eq!(recorded(&root, Keyed::Llvm).as_deref(), Some("0123456789abcdef"), "a record stopped mid-write was read"); - record(&root, Keyed::Llvm, "fedcba9876543210"); - assert_eq!(recorded(&root, Keyed::Llvm).as_deref(), Some("fedcba9876543210")); - } - - /// Concurrent records of one kind never share a temp file, so none fails - /// and the record holds one of the keys written whole. - #[test] - fn concurrent_records_of_one_kind_all_land() { - let root = TempDir::new("race"); - let keys: Vec = (0..8).map(|i| format!("{i:016x}")).collect(); - for _ in 0..50 { - std::thread::scope(|s| { - let handles: Vec<_> = keys.iter().map(|key| s.spawn(|| record(&root, Keyed::Llvm, key))).collect(); - for h in handles { - h.join().expect("a concurrent record panicked"); - } - }); - let got = recorded(&root, Keyed::Llvm).unwrap(); - assert!(keys.contains(&got), "the record holds {got:?}, none of the keys written"); - } - } - - /// **A retire stopped halfway leaves nothing at the name it removes**: what - /// it removes is out of the way before anything in it goes, so a stopped - /// sweep leaves nothing that passes for whole, and the next retire takes - /// what the stopped one left. - #[test] - fn a_stopped_retire_leaves_nothing_at_its_name() { - let store = TempDir::new("retire"); - let whole = store.join("key"); - write(&whole.join("SOURCE"), "key\n"); - write(&whole.join("lib/libLLVMCore.a"), "core"); - let stopped = std::panic::catch_unwind(|| retire_by(&whole, |_| panic!("stopped"))); - assert!(stopped.is_err(), "the stand-in removal was never asked"); - assert!(!whole.exists(), "a stopped retire left {}", whole.display()); - retire(&whole); - assert!(!whole.with_extension("swept").exists(), "the next retire kept what the stopped one left"); - } -} diff --git a/src/lib.rs b/src/lib.rs index e4a81196fb4..eb66523b96f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,11 +2,11 @@ pub mod arch; pub mod assets; pub mod bootlog; pub mod build; -pub mod buildlock; pub mod ci; pub mod clang; pub mod clippy; pub mod compiler; +pub mod dirlock; /// What the untouched-disk gate compares a device against, in `tests/`. pub mod fingerprint; pub mod firmware; @@ -15,7 +15,6 @@ pub mod gitfixture; pub mod flags; pub mod hostws; pub mod icmp; -pub mod identity; pub mod image; pub mod kernelconsole; pub mod signing; @@ -23,7 +22,6 @@ pub mod signing; /// but its own tests. #[cfg(test)] pub mod kernelkeys; -pub mod keystore; pub mod lan; pub mod libc; pub mod llvm; @@ -42,6 +40,7 @@ pub mod soundfont; /// build system at all. #[cfg(test)] pub mod sourcegate; +pub mod store; pub mod sync; pub mod sysroot; pub mod testargs; @@ -49,7 +48,6 @@ pub mod tether; pub mod toolchain; pub mod userlandhost; pub mod wallpaper; -pub mod worktree; use std::path::{Path, PathBuf}; use std::process::Command; diff --git a/src/libc.rs b/src/libc.rs index 758721fd2d6..f5004aa2fec 100644 --- a/src/libc.rs +++ b/src/libc.rs @@ -12,7 +12,7 @@ pub const FEATURES: &str = "std-runtime"; /// Build toyos-libc against the toolchain at `toolchain`, in `target_dir`, and /// install it there as `libtoyos_c.a`. Part of making a sysroot -/// (`src/sysroot.rs`), whose key `userland/libc/src` is one of. +/// (`src/sysroot.rs`). pub fn build(root: &Path, toolchain: &Path, target_dir: &Path, arch: Arch) { let dest = toolchain.join(format!("lib/rustlib/{}/lib/libtoyos_c.a", arch.userland())); @@ -26,10 +26,12 @@ pub fn build(root: &Path, toolchain: &Path, target_dir: &Path, arch: Arch) { // it, so the installed archive would not be rebuilt and the manifest would // then claim something the artifact does not have. // - // --message-format=json to discover the exact rlib artifacts. + // --message-format=json to discover the exact rlib artifacts; --locked here + // and in `build_c`, so the lockfile the sysroot's key names is the one built with. let output = Command::new("cargo") .args([ "build", + "--locked", "--release", "--target", arch.userland(), @@ -95,7 +97,7 @@ pub fn build(root: &Path, toolchain: &Path, target_dir: &Path, arch: Arch) { pub fn build_c(root: &Path, toolchain: &Path, target_dir: &Path, arch: Arch) { let target = arch.userland(); let output = Command::new("cargo") - .args(["rustc", "--release", "--target", target, "--crate-type", "staticlib", "--manifest-path"]) + .args(["rustc", "--locked", "--release", "--target", target, "--crate-type", "staticlib", "--manifest-path"]) .arg(root.join(CRATE).join("Cargo.toml")) .arg("--target-dir") .arg(target_dir) diff --git a/src/licence.rs b/src/licence.rs index 437c9ac1924..ae02905edb7 100644 --- a/src/licence.rs +++ b/src/licence.rs @@ -52,6 +52,7 @@ use std::process::Command; use serde_json::Value; use crate::build::Features; +use crate::toolchain::Owner; /// What a shipped crate or file may be under. `OR` passes if any branch does, /// `AND` only if every part does. @@ -1121,19 +1122,53 @@ fn metadata( serde_json::from_slice(&out).map_err(|e| format!("cargo metadata printed no JSON: {e}")) } -/// The fork's `library/`, checked out at the commit this tree pins. A checkout -/// whose `rust/` was never initialised — a CI runner's — fetches that commit -/// alone. +/// The fork's `library/` this tree builds std from, read with no checkout +/// held: a primary checkout's `rust/`, whose pinned commit alone is fetched +/// when it was never initialised, as a CI runner's is; a linked worktree's own +/// checkout while it holds fork work; otherwise the commit a linked worktree +/// pins, read out of the primary's fork repository beside links to its +/// `toyos-abi` and `toyos`, which `library/std` names as `../../../`. A linked +/// worktree never runs `git submodule`. fn std_library(root: &Path) -> Result { - let fork = crate::sysroot::fork_checkout(root); - if !fork.join("library/Cargo.toml").exists() { - run( - Command::new("git") - .args(["submodule", "update", "--init", "--depth", "1", "rust"]) - .current_dir(root), - "git submodule update --init --depth 1 rust", - )?; - } + let fork = match crate::toolchain::owner(root) { + Owner::Elsewhere(primary) => { + let own = root.join("rust"); + let pinned = crate::sysroot::pinned_fork(root); + if own.join(".git").exists() && crate::sysroot::holds_work(&own, &pinned) { + own + } else { + let base = root.join("target/licence/pinned"); + if base.exists() { + std::fs::remove_dir_all(&base).map_err(|e| format!("remove {}: {e}", base.display()))?; + } + std::fs::create_dir_all(&base).map_err(|e| format!("create {}: {e}", base.display()))?; + let scratch = toyos_tmpdir::TempDir::new("licence-index"); + let index = scratch.join("index"); + let fork = base.join("rust"); + let (tree, prefix) = (format!("{pinned}:library"), format!("--prefix={}/", fork.display())); + for args in [["read-tree", "--prefix=library/", tree.as_str()], ["checkout-index", "--all", prefix.as_str()]] { + crate::sysroot::git(&primary.join("rust"), &args, Some(&index))?; + } + for tree in ["toyos-abi", "toyos"] { + std::os::unix::fs::symlink(root.join(tree), base.join(tree)) + .map_err(|e| format!("link {}: {e}", base.join(tree).display()))?; + } + fork + } + } + Owner::Us | Owner::Installed => { + let fork = root.join("rust"); + if !fork.join("library/Cargo.toml").exists() { + run( + Command::new("git") + .args(["submodule", "update", "--init", "--depth", "1", "rust"]) + .current_dir(root), + "git submodule update --init --depth 1 rust", + )?; + } + fork + } + }; Ok(fork.join("library")) } @@ -1874,4 +1909,32 @@ prose. } } } + + /// **A linked worktree's std library is read out of the primary's fork + /// repository, never checked out**: a stub worktree gets the `library/` of + /// the commit it pins, beside its own `toyos-abi`, holds no fork checkout + /// for it, and runs no `git submodule`; an own checkout behind its pin is + /// read as the pin and not moved. + #[test] + fn a_linked_worktree_reads_its_pinned_library_and_checks_nothing_out() { + use crate::store::tests::{estate, git}; + let e = estate("licence-stub"); + let stub = e.same.parent().unwrap().join("stub"); + git(&e.primary, &["worktree", "add", "-q", "-b", "stub", stub.to_str().unwrap()]); + let library = std_library(&stub).expect("a stub worktree's std library"); + assert!(library.starts_with(stub.join("target")), "{}", library.display()); + assert_eq!(std::fs::read_to_string(library.join("std/src/lib.rs")).unwrap(), "pub fn a() {}\n"); + let beside = std::fs::canonicalize(library.join("std/../../../toyos-abi")).unwrap(); + assert_eq!(beside, std::fs::canonicalize(stub.join("toyos-abi")).unwrap()); + assert!(!e.primary.join(".git/worktrees/stub/modules").exists(), "git submodule ran in a linked worktree"); + assert!(!e.rust_dir.join(crate::sysroot::SHARED).exists(), "a read took the host's shared checkout"); + + let own = e.b.join("rust"); + let pin = git(&own, &["rev-parse", "HEAD"]); + git(&e.b, &["update-index", "--cacheinfo", &format!("160000,{pin},rust")]); + git(&own, &["checkout", "-q", "--detach", "HEAD~1"]); + let behind = git(&own, &["rev-parse", "HEAD"]); + assert!(std_library(&e.b).unwrap().starts_with(e.b.join("target")), "a checkout behind its pin was read as it stands"); + assert_eq!(git(&own, &["rev-parse", "HEAD"]), behind, "a read moved a checkout behind its pin"); + } } diff --git a/src/llvm.rs b/src/llvm.rs index 4c66bbfce78..c5c6e6b5a65 100644 --- a/src/llvm.rs +++ b/src/llvm.rs @@ -1,47 +1,27 @@ -//! The LLVM every host compiler links, with its clang and LLD, content-addressed: -//! one per key on this host, shared by every compiler build that names it. +//! The LLVM every host compiler links, with its clang and LLD: a product of the +//! store (`src/store.rs`), one per key on this host. //! -//! **An LLVM is a function of its key** ([`key`]): the `src/llvm-project` commit -//! the fork checkout's gitlink names (`compiler::llvm_commit`, which refuses a -//! checkout holding what no commit does and a gitlink staged and not committed), -//! the committed tree of its `src/bootstrap` (one holding what no commit does is -//! refused), the bootstrap configuration below, [`RECIPE`], and the tools the -//! host builds it with ([`host_tools`]). `rust/build/llvm//` in the primary -//! is bootstrap's install of that LLVM and its clang, with its LLD in `bin/` -//! beside `llvm-config`, made by whichever build first needs it ([`resolve`]), -//! and stored only when it was built from what the key names. Once its -//! [`SOURCE`] file exists it is read-only, its directories as well as its files. -//! Every compiler build, the primary's and a worktree's own, names it as the -//! host's `llvm-config` with `llvm-has-rust-patches`, so bootstrap builds no -//! LLVM and takes LLD from beside it as `rust-lld`; `clang::provision` copies its -//! clang. Once a build directory's compiler is built against it, the LLVM that -//! directory built itself goes ([`retire_in_tree`]). +//! **Its key** ([`key`]) is the `src/llvm-project` commit the fork's gitlink +//! names, the fork's `src/bootstrap`, the bootstrap configuration below, +//! [`RECIPE`], and the tools the host builds it with ([`host_tools`]). Its +//! directory is bootstrap's install of that LLVM and its clang, with its LLD in +//! `bin/` beside `llvm-config`. Every compiler build names it as the host's +//! `llvm-config` with `llvm-has-rust-patches`, so bootstrap builds no LLVM and +//! takes LLD from beside it as `rust-lld`; `clang::provision` copies its clang. //! //! **Nothing of the environment it is asked from reaches it but //! [`ENVIRONMENT`]**: the build and every tool its key asks run with the rest //! cleared ([`clear`]), the configuration names the C and C++ compilers by path, //! and every host library LLVM would otherwise find and link is turned off //! ([`NO_HOST_LIBRARIES`]). -//! -//! Its lock (`buildlock::keyed_*` with [`Keyed::Llvm`]) is taken inside the -//! worktree or global lock that covers the fork build directory its maker -//! writes, `build/toyos-llvm/`, which is removed once the LLVM is placed. -//! -//! An LLVM no worktree names any more is removed by `keystore::sweep`, which -//! `--worktree remove` and every placement run: each worktree records the key of -//! the LLVM its compiler links, and a key no registered worktree records, that -//! nobody is making or using, goes. -use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::OnceLock; -use crate::buildlock::{Guard, Keyed}; -use crate::compiler::{llvm_commit, LLVM}; -use crate::keystore; -use crate::sysroot::{clone_tree, git_bytes, git_out, short}; +use crate::store::{self, Kind, Sources}; +use crate::sysroot::clone_tree; use crate::toolchain::{self, host_triple}; /// What changes how a key's sources become an LLVM and is none of the other @@ -89,13 +69,9 @@ const KEPT: [&str; 3] = ["bin", "include", "lib"]; /// What a compiler build and `clang::provision` read of an LLVM. const TOOLS: [&str; 4] = ["bin/llvm-config", "bin/lld", "bin/clang", "bin/llvm-ar"]; -/// The file a finished LLVM carries last, naming its key. A directory without -/// it is a build that did not finish. -const SOURCE: &str = "SOURCE"; -/// The fork's bootstrap, whose `Llvm` step and `compiler` profile decide how -/// LLVM is configured. -const BOOTSTRAP: &str = "src/bootstrap"; +/// The fork's LLVM checkout, and the tree of [`Sources`] naming its commit. +pub(crate) const LLVM: &str = "src/llvm-project"; /// The build directory the key's configuration names. const KEYED_BUILD_DIR: &str = ""; @@ -104,7 +80,7 @@ const KEYED_BUILD_DIR: &str = ""; pub struct Llvm { /// Its install: `bin/`, `include/`, `lib/`. pub dir: PathBuf, - _using: Guard, + _held: store::Held, } /// The `[target.]` lines that make a `bootstrap.toml` link the LLVM at @@ -113,22 +89,14 @@ pub fn host_lines(dir: &Path) -> String { format!("llvm-config = \"{}\"\nllvm-has-rust-patches = true", dir.join("bin/llvm-config").display()) } -/// Every LLVM on this host. -pub fn store(rust_dir: &Path) -> PathBuf { - rust_dir.join("build/llvm") -} - -/// The key of the LLVM `fork` names; refused while its `src/bootstrap` holds -/// changes no commit does. -pub fn key(fork: &Path) -> String { +/// The key of the LLVM `sources` name. +pub fn key(sources: &Sources) -> String { let tools = host_tools(); - key_of(fork, RECIPE, &config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), tools), &tools.identity) + key_of(sources, RECIPE, &config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), tools), &tools.identity) } -fn key_of(fork: &Path, recipe: &str, config: &str, tools: &str) -> String { - refuse_uncommitted_bootstrap(fork); - let bootstrap = git_out(fork, &["rev-parse", &format!("HEAD:{BOOTSTRAP}")]); - short([recipe, config, &llvm_commit(fork), bootstrap.trim(), tools].join("\n\0\n").as_bytes()) +fn key_of(sources: &Sources, recipe: &str, config: &str, tools: &str) -> String { + store::key(recipe, &[config, sources.get(LLVM), sources.get("src/bootstrap"), tools]) } /// Give `command` nothing of this process's environment but [`ENVIRONMENT`]. @@ -192,138 +160,52 @@ fn on_path(name: &str) -> PathBuf { fs::canonicalize(&found).unwrap_or_else(|e| panic!("resolve {}: {e}", found.display())) } -/// The LLVM `fork` names, made if nobody on this host has made it, and held in -/// use for as long as the returned value lives. `root` records its key. -pub fn resolve(root: &Path, rust_dir: &Path, fork: &Path) -> Llvm { - choose(root, rust_dir, fork, build_in_fork) +/// The LLVM `sources` name, made from the fork checkout at `fork` if nobody on +/// this host has made it, and held in use for as long as the returned value +/// lives. `root` records its key. +pub fn resolve(root: &Path, rust_dir: &Path, fork: &Path, sources: &Sources) -> Llvm { + choose(root, rust_dir, fork, sources, build_in_fork) } /// [`resolve`] with the build that makes an LLVM passed in, so a test can stand /// in for bootstrap: `build` builds in the fork checkout it is given and returns /// the build directory, holding `/llvm` and `/lld`. -fn choose(root: &Path, rust_dir: &Path, fork: &Path, build: impl Fn(&Path) -> PathBuf) -> Llvm { - let key = key(fork); - let dir = store(rust_dir).join(&key); - let using = keystore::made(root, Keyed::Llvm, &store(rust_dir), &key, || defect(&dir), || place(fork, &key, &dir, &build)); - Llvm { dir, _using: using } +fn choose(root: &Path, rust_dir: &Path, fork: &Path, sources: &Sources, build: impl Fn(&Path) -> PathBuf) -> Llvm { + let key = key(sources); + let held = store::get(root, rust_dir, Kind::Llvm, &key, |partial| fill(root, fork, &key, partial, &build)); + Llvm { dir: held.dir.clone(), _held: held } } -/// Why `dir` is not a finished LLVM, if it is not. +/// Why `dir` is not a whole LLVM, if it is not. fn defect(dir: &Path) -> Option { - if !dir.join(SOURCE).is_file() { - return Some(format!("{} carries no {SOURCE}", dir.display())); - } let kept = KEPT.iter().map(|k| dir.join(k)).filter(|p| !p.is_dir()); let tools = TOOLS.iter().map(|t| dir.join(t)).filter(|p| !p.is_file()); let gone: Vec = kept.chain(tools).map(|p| p.display().to_string()).collect(); (!gone.is_empty()).then(|| format!("{} carries no {}", dir.display(), gone.join(", "))) } -/// Refuse what `fork`'s `src/bootstrap` holds that no commit does: an LLVM is -/// keyed on the tree its commit records. -fn refuse_uncommitted_bootstrap(fork: &Path) { - let status = git_bytes(fork, &["status", "--porcelain", "--untracked-files=normal", "--", BOOTSTRAP]); - assert!( - status.is_empty(), - "{} holds changes no commit does, and an LLVM is keyed on the tree its commit records: \ - commit them, and the build makes the LLVM they name\n{}", - fork.join(BOOTSTRAP).display(), - String::from_utf8_lossy(&status), - ); -} - -/// Build the LLVM `key` names from `fork` and put it at `dir`. The caller holds -/// the key's lock. -fn place(fork: &Path, key: &str, dir: &Path, build: &impl Fn(&Path) -> PathBuf) { +/// Build the LLVM `key` names from `fork` into `partial`. +fn fill(root: &Path, fork: &Path, key: &str, partial: &Path, build: &impl Fn(&Path) -> PathBuf) { eprintln!("Building LLVM {key} in {}: nobody on this host has", fork.display()); let built = build(fork); let host = host_triple(); - let partial = dir.with_extension("partial"); - if partial.exists() { - keystore::remove(&partial); - } for part in KEPT { clone_tree(&built.join(&host).join("llvm").join(part), &partial.join(part)); } let lld = built.join(&host).join("lld/bin/lld"); fs::copy(&lld, partial.join("bin/lld")) .unwrap_or_else(|e| panic!("copy {} -> {}: {e}", lld.display(), partial.join("bin/lld").display())); - // What was built is what the key names, or it is not that key's: the key - // refuses a bootstrap the build left holding what no commit does. - let again = self::key(fork); + let again = self::key(&Sources::of(root, fork)); assert!( again == key, "the fork's LLVM sources moved while LLVM {key} was being built (they now name {again}); \ nothing was kept, and the next build makes the one they name" ); - let checkout = fork.join(LLVM); - assert!(checkout.join(".git").exists(), "the LLVM build left no checkout at {}", checkout.display()); - let (built_from, commit) = (git_out(&checkout, &["rev-parse", "HEAD"]), llvm_commit(fork)); - // Bootstrap's `Llvm` step checks the gitlink's commit out before it builds, - // so a checkout behind it, the key never reads, is moved first. - assert!( - built_from.trim() == commit, - "{} is checked out at {}, and its gitlink names {commit}: bootstrap built the commit checked \ - out, which is not LLVM {key}'s; nothing was kept. `git -C {} submodule update {LLVM}` checks \ - the gitlink's commit out", - checkout.display(), - built_from.trim(), - fork.display(), - ); - fs::write(partial.join(SOURCE), format!("{key}\n")) - .unwrap_or_else(|e| panic!("write {}: {e}", partial.join(SOURCE).display())); - read_only(&partial); - keystore::retire(dir); - fs::rename(&partial, dir).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", partial.display(), dir.display())); - fs::remove_dir_all(&built).unwrap_or_else(|e| panic!("remove {}: {e}", built.display())); -} - -/// Take write permission from every file and directory under `dir`, and from -/// `dir`. -fn read_only(dir: &Path) { - for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { - let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", dir.display())).path(); - let meta = fs::symlink_metadata(&path).unwrap_or_else(|e| panic!("stat {}: {e}", path.display())); - if meta.is_dir() { - read_only(&path); - } else if meta.is_file() { - let mut permissions = meta.permissions(); - permissions.set_readonly(true); - fs::set_permissions(&path, permissions).unwrap_or_else(|e| panic!("chmod {}: {e}", path.display())); - } - } - let mut permissions = fs::metadata(dir).unwrap_or_else(|e| panic!("stat {}: {e}", dir.display())).permissions(); - permissions.set_readonly(true); - fs::set_permissions(dir, permissions).unwrap_or_else(|e| panic!("chmod {}: {e}", dir.display())); -} - -/// What the bootstrap build directory `build` holds of an LLVM of its own: -/// bootstrap's LLVM and LLD, and `download-ci-llvm`'s with its downloads, -/// whole or as a stopped removal left them. -pub fn in_tree(build: &Path) -> Vec { - let host = build.join(host_triple()); - let mut own: Vec = ["llvm", "lld", "ci-llvm"].iter().map(|d| host.join(d)).collect(); - let cache = build.join("cache"); - if cache.is_dir() { - let downloads: BTreeSet = fs::read_dir(&cache) - .unwrap_or_else(|e| panic!("read {}: {e}", cache.display())) - .map(|e| e.unwrap_or_else(|e| panic!("read {}: {e}", cache.display())).file_name()) - .map(|name| name.to_string_lossy().trim_end_matches(".swept").to_string()) - .filter(|name| name.starts_with("llvm-")) - .collect(); - own.extend(downloads.iter().map(|name| cache.join(name))); - } - own.retain(|dir| dir.exists() || dir.with_extension("swept").exists()); - own -} - -/// Remove [`in_tree`]. The caller holds the lock covering `build`, where -/// nothing builds an LLVM or downloads one any more. -pub fn retire_in_tree(build: &Path) { - for dir in in_tree(build) { - eprintln!("Removing {}: builds here link the host's LLVM or none", dir.display()); - keystore::retire(&dir); + store::assert_built_at_gitlinks(fork, &[LLVM], &format!("LLVM {key}")); + if let Some(defect) = defect(partial) { + panic!("LLVM {key} was made, and is not whole: {defect}"); } + fs::remove_dir_all(&built).unwrap_or_else(|e| panic!("remove {}: {e}", built.display())); } /// Bootstrap's build of LLVM, clang and LLD in `fork`, into its own build @@ -371,38 +253,13 @@ cxx = "{cxx}" ) } + #[cfg(test)] mod tests { use std::cell::Cell; - use toyos_tmpdir::TempDir; - use super::*; - use crate::buildlock; - use crate::compiler::tests::{estate, git, write, LLVM_B}; - - /// A scratch directory whose read-only LLVMs are made writable again before - /// it goes. - struct Scratch(TempDir); - - impl Scratch { - fn new(name: &str) -> Self { - Self(TempDir::new(name)) - } - } - - impl std::ops::Deref for Scratch { - type Target = Path; - fn deref(&self) -> &Path { - &self.0 - } - } - - impl Drop for Scratch { - fn drop(&mut self) { - keystore::writable(&self.0); - } - } + use crate::store::tests::{estate, git, refusal, write, Estate, LLVM_B}; /// Bootstrap's stand-in: what its LLVM and LLD builds leave in the build /// directory, CMake's tree among them. @@ -454,52 +311,41 @@ mod tests { /// `estate`, every fork's LLVM checked out at the one commit its gitlink /// records. - fn estate_built(scratch: &Path) -> (PathBuf, PathBuf, [PathBuf; 3]) { - let (primary, rust_dir, forks) = estate(scratch); - for worktree in &forks { + fn estate_built(name: &str) -> Estate { + let e = estate(name); + for worktree in [&e.same, &e.a, &e.b] { pin_llvm(&worktree.join("rust"), "A"); } - pin_llvm(&rust_dir, "A"); - (primary, rust_dir, forks) + pin_llvm(&e.rust_dir, "A"); + e } - /// What `f` panicked with; `expect` if it returned. - fn refusal(expect: &str, f: impl FnOnce()) -> String { - let refused = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).expect_err(expect); - refused.downcast_ref::().cloned().unwrap_or_default() + fn sources(root: &Path) -> Sources { + Sources::of(root, &root.join("rust")) } - /// **One LLVM per key, made once and found again**: two worktrees whose + /// **One LLVM per key, made once and found again**: worktrees whose /// compilers differ and whose LLVM commit is one share one LLVM, and the - /// second build makes nothing and writes nothing; what is kept is the - /// install and its LLD, never CMake's tree, and the build directory goes. + /// later ones make nothing; what is kept is the install and its LLD, never + /// CMake's tree, and the build directory goes. #[test] fn two_compilers_of_one_llvm_make_it_once() { - let scratch = Scratch::new("llvm"); - let (primary, rust_dir, [same, a, b]) = estate_built(&scratch); + let e = estate_built("llvm"); let makes = Cell::new(0); let once = |fork: &Path| { makes.set(makes.get() + 1); assert_eq!(makes.get(), 1, "an LLVM whose key was made was made again"); fake_build(fork) }; - - let la = choose(&a, &rust_dir, &a.join("rust"), once); - assert_eq!(makes.get(), 1); + let la = choose(&e.a, &e.rust_dir, &e.a.join("rust"), &sources(&e.a), once); assert_eq!(defect(&la.dir), None); assert_eq!(fs::read_to_string(la.dir.join("bin/lld")).unwrap(), "the lld"); assert_eq!(fs::read_link(la.dir.join("bin/clang")).unwrap(), Path::new("clang-22")); - assert!(la.dir.join("lib/clang/22/include/stddef.h").is_file()); assert!(!la.dir.join("build").exists(), "CMake's tree was kept"); - assert!(!a.join("rust/build/toyos-llvm").exists(), "the build directory outlived the placement"); - - let before = snapshot(&store(&rust_dir)); - let lb = choose(&b, &rust_dir, &b.join("rust"), once); - let primary_s = choose(&primary, &rust_dir, &rust_dir, once); - let same_s = choose(&same, &rust_dir, &same.join("rust"), once); - assert_eq!(makes.get(), 1); - assert!(lb.dir == la.dir && primary_s.dir == la.dir && same_s.dir == la.dir, "one LLVM commit named two LLVMs"); - assert_eq!(snapshot(&store(&rust_dir)), before, "an LLVM was written after it was whole"); + assert!(!e.a.join("rust/build/toyos-llvm").exists(), "the build directory outlived the placement"); + for root in [&e.b, &e.same, &e.primary] { + assert_eq!(choose(root, &e.rust_dir, &root.join("rust"), &sources(root), once).dir, la.dir, "one LLVM commit named two LLVMs"); + } } /// **A placed LLVM cannot be written**, through its own path or through a @@ -507,88 +353,67 @@ mod tests { /// removed, replaced or added. #[test] fn a_placed_llvm_is_never_written() { - let scratch = Scratch::new("llvm-read-only"); - let (_primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); - let dir = choose(&a, &rust_dir, &a.join("rust"), fake_build).dir; - let stage = scratch.join("stage1-rust-lld"); + let e = estate_built("llvm-read-only"); + let dir = choose(&e.a, &e.rust_dir, &e.a.join("rust"), &sources(&e.a), fake_build).dir; + let stage = e.a.join("stage1-rust-lld"); fs::hard_link(dir.join("bin/lld"), &stage).unwrap(); let denied = |what: &str, done: std::io::Result<()>| { assert_eq!(done.map_err(|e| e.kind()).err(), Some(std::io::ErrorKind::PermissionDenied), "{what}"); }; - for file in [dir.join("bin/lld"), stage, dir.join("lib/libLLVMCore.a"), dir.join(SOURCE)] { + for file in [dir.join("bin/lld"), stage, dir.join("lib/libLLVMCore.a")] { denied(&format!("{} could be written", file.display()), fs::OpenOptions::new().write(true).open(&file).map(drop)); } denied("a placed tool could be removed", fs::remove_file(dir.join("bin/lld"))); denied("a file could be added to a placed LLVM", fs::write(dir.join("bin/new"), "x")); - denied("a placed LLVM could be emptied", fs::remove_dir_all(dir.join("include"))); } - /// **An LLVM that is not whole is made again, all of it**: one whose - /// `SOURCE` says it finished and that lost a tool, or a directory it keeps, - /// is replaced. + /// **A make that leaves an LLVM not whole is refused, and nothing is + /// placed.** #[test] - fn an_llvm_that_is_not_whole_is_made_again() { - let scratch = Scratch::new("llvm-whole"); - let (_primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); - let makes = Cell::new(0); - let counted = |fork: &Path| { - makes.set(makes.get() + 1); - fake_build(fork) + fn an_llvm_made_not_whole_is_never_placed() { + let e = estate_built("llvm-whole"); + let without_lld = |fork: &Path| { + let built = fake_build(fork); + fs::write(built.join(host_triple()).join("lld/bin/lld"), "").unwrap(); + fs::remove_file(built.join(host_triple()).join("llvm/bin/llvm-ar")).unwrap(); + built }; - let dir = choose(&a, &rust_dir, &a.join("rust"), counted).dir; - for (lost, made) in [("bin/lld", 2), ("lib", 3)] { - keystore::writable(&dir); - let lost = dir.join(lost); - if lost.is_dir() { - fs::remove_dir_all(&lost).unwrap(); - } else { - fs::remove_file(&lost).unwrap(); - } - assert!(defect(&dir).is_some_and(|d| d.contains(&lost.display().to_string())), "{:?}", defect(&dir)); - let again = choose(&a, &rust_dir, &a.join("rust"), counted); - assert_eq!((makes.get(), defect(&again.dir)), (made, None)); - } + let said = refusal("an LLVM without llvm-ar was placed", || { + choose(&e.a, &e.rust_dir, &e.a.join("rust"), &sources(&e.a), without_lld); + }); + assert!(said.contains("is not whole") && said.contains("bin/llvm-ar"), "{said}"); + assert!(!Kind::Llvm.dir(&e.rust_dir).join(key(&sources(&e.a))).exists()); } - /// **The key is the LLVM and nothing else**: the same inputs give the same - /// key; each of the recipe, the configuration (its `[llvm]` and its host), - /// the host's tools, the committed LLVM gitlink and the committed - /// `src/bootstrap` moves it; a compiler edit does not. + /// **The key is the LLVM and nothing else**: each of the recipe, the + /// configuration (its `[llvm]` and its host), the host's tools, the LLVM + /// gitlink and `src/bootstrap` moves it; a compiler edit does not. #[test] fn the_key_moves_with_the_llvm_and_only_with_it() { - let scratch = Scratch::new("llvm-key"); - let (_primary, rust_dir, [same, a, _b]) = estate(&scratch); - let fork = same.join("rust"); + let e = estate("llvm-key"); + let fork = e.same.join("rust"); let tools = host_tools(); let config = config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), tools); - let base = key(&fork); - assert_eq!(key_of(&fork, RECIPE, &config, &tools.identity), base); + let s = sources(&e.same); + let base = key(&s); + assert_eq!(key_of(&s, RECIPE, &config, &tools.identity), base); let linux = config_text(Path::new(KEYED_BUILD_DIR), "x86_64-unknown-linux-gnu", tools); for (what, other) in [ - ("the recipe", key_of(&fork, "another recipe", &config, &tools.identity)), - ("the [llvm]", key_of(&fork, RECIPE, &config.replace("X86", "RISCV;X86"), &tools.identity)), - ("the host", key_of(&fork, RECIPE, &linux, &tools.identity)), - ("the host's tools", key_of(&fork, RECIPE, &config, "/usr/bin/gcc\ngcc 14\n")), + ("the recipe", key_of(&s, "another recipe", &config, &tools.identity)), + ("the [llvm]", key_of(&s, RECIPE, &config.replace("X86", "RISCV;X86"), &tools.identity)), + ("the host", key_of(&s, RECIPE, &linux, &tools.identity)), + ("the host's tools", key_of(&s, RECIPE, &config, "/usr/bin/gcc\ngcc 14\n")), ] { assert_ne!(other, base, "{what} did not move the key"); } - - assert_eq!(key(&fork), key(&rust_dir), "one LLVM commit named two keys"); - assert_eq!(key(&fork), key(&a.join("rust")), "a compiler/ edit moved the LLVM key"); - write(&fork.join("compiler/rustc_target/src/new_target.rs"), "pub fn t() {}\n"); - git(&fork, &["add", "-A"]); - git(&fork, &["commit", "-qm", "a target"]); - assert_eq!(key(&fork), base, "a compiler/ commit moved the LLVM key"); + assert_eq!(key(&sources(&e.a)), base, "a compiler/ edit moved the LLVM key"); git(&fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{LLVM_B},{LLVM}")]); git(&fork, &["commit", "-qm", "another LLVM"]); - let moved = key(&fork); + let moved = key(&sources(&e.same)); assert_ne!(moved, base, "another LLVM commit kept the key"); - write(&fork.join("src/bootstrap/src/core/build_steps/llvm.rs"), "cfg.define(\"LLVM_ENABLE_ZLIB\", \"OFF\");\n"); - git(&fork, &["add", "-A"]); - git(&fork, &["commit", "-qm", "another LLVM step"]); - assert_ne!(key(&fork), moved, "another LLVM step in bootstrap kept the key"); + assert_ne!(key(&sources(&e.same)), moved, "another LLVM step in bootstrap kept the key"); } /// **The tools the key names are the host's**: the C and C++ compilers the @@ -617,16 +442,15 @@ mod tests { #[test] #[cfg(target_os = "macos")] fn two_sdk_versions_are_two_keys() { - let scratch = Scratch::new("llvm-sdk"); - let (_primary, _rust_dir, [same, _a, _b]) = estate(&scratch); - let fork = same.join("rust"); + let e = estate("llvm-sdk"); + let s = sources(&e.same); let [older, newer] = ["26.0", "27.0"].map(|version| { let tools = tools_with(|question| match question { "--show-sdk-path" => "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk\n".to_string(), "--show-sdk-version" => format!("{version}\n"), other => panic!("xcrun was asked {other}"), }); - key_of(&fork, RECIPE, &config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), &tools), &tools.identity) + key_of(&s, RECIPE, &config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), &tools), &tools.identity) }); assert_ne!(older, newer, "two SDK versions at one SDK path named one LLVM"); } @@ -644,28 +468,7 @@ mod tests { assert!(config.contains(&format!("\n{off}")), "{config}"); } - /// **A checkout behind its gitlink names the gitlink's LLVM**: the key - /// reads the gitlink and never the checkout, and the build, which checks the - /// gitlink's commit out first as bootstrap's `Llvm` step does, stores it. - #[test] - fn a_checkout_behind_its_gitlink_is_built_at_the_gitlink() { - let scratch = Scratch::new("llvm-behind"); - let (_primary, rust_dir, [same, a, _b]) = estate_built(&scratch); - let (fork, at_gitlink) = (a.join("rust"), same.join("rust")); - for fork in [&fork, &at_gitlink] { - pin_llvm(fork, "B"); - } - check_out_llvm(&fork, "A"); - assert_eq!(key(&fork), key(&at_gitlink), "a checkout behind its gitlink moved the key"); - let updating = |fork: &Path| { - check_out_llvm(fork, "B"); - fake_build(fork) - }; - let llvm = choose(&a, &rust_dir, &fork, updating); - assert_eq!((llvm.dir.clone(), defect(&llvm.dir)), (store(&rust_dir).join(key(&at_gitlink)), None)); - } - - const FORK: &str = "TOYOS_LLVM_TEST_FORK"; + const ROOT: &str = "TOYOS_LLVM_TEST_ROOT"; /// What a caller's environment may hold that would reach an LLVM build: /// flags, compilers, tools, and the SDK and deployment target. @@ -691,17 +494,16 @@ mod tests { #[test] fn the_caller_s_environment_reaches_neither_the_build_nor_the_key() { use std::os::unix::fs::PermissionsExt; - let scratch = Scratch::new("llvm-environment"); - let (_primary, _rust_dir, [same, _a, _b]) = estate(&scratch); - let fork = same.join("rust"); + let e = estate("llvm-environment"); + let fork = e.same.join("rust"); write(&fork.join("library/Cargo.lock"), "# lock\n"); write(&fork.join("x"), "#!/bin/sh\nenv > build/toyos-llvm/environment\n"); fs::set_permissions(fork.join("x"), fs::Permissions::from_mode(0o755)).unwrap(); - let out = buildlock::tests::rerun("llvm::tests::keyed_and_built").env(FORK, &fork).envs(AMBIENT).output().unwrap(); + let out = crate::dirlock::tests::rerun("llvm::tests::keyed_and_built").env(ROOT, &e.same).envs(AMBIENT).output().unwrap(); assert!(out.status.success(), "{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)); let built = fork.join("build/toyos-llvm"); - assert_eq!(fs::read_to_string(built.join("key")).unwrap(), key(&fork), "the caller's environment moved the key"); + assert_eq!(fs::read_to_string(built.join("key")).unwrap(), key(&sources(&e.same)), "the caller's environment moved the key"); let seen = fs::read_to_string(built.join("environment")).unwrap(); let allowed = ["PATH", "TMPDIR", "BOOTSTRAP_SKIP_TARGET_SANITY", "PWD", "OLDPWD", "SHLVL", "_"]; for name in seen.lines().filter_map(|l| l.split_once('=')).map(|(name, _)| name) { @@ -710,253 +512,54 @@ mod tests { assert!(seen.lines().any(|l| l.starts_with("PATH=")), "the build saw no PATH: {seen}"); } - /// The process [`the_caller_s_environment_reaches_neither_the_build_nor_the_key`] - /// runs: the key of the fork in [`FORK`] and its build, the key written - /// beside what the build wrote. + /// The process the environment test runs: the key of the worktree in + /// [`ROOT`] and its fork's build, the key written beside what the build wrote. #[test] #[ignore = "the process the environment test runs; never runs on its own"] fn keyed_and_built() { - let fork = PathBuf::from(std::env::var(FORK).unwrap_or_else(|_| panic!("keyed_and_built ran without {FORK}; it is not a test"))); - let key = key(&fork); - let built = build_in_fork(&fork); + let root = PathBuf::from(std::env::var(ROOT).unwrap_or_else(|_| panic!("keyed_and_built ran without {ROOT}; it is not a test"))); + let key = key(&sources(&root)); + let built = build_in_fork(&root.join("rust")); fs::write(built.join("key"), key).unwrap(); } - /// **A gitlink staged and not committed names no LLVM**: bootstrap checks - /// out the index's, and the key would name HEAD's. + /// **Only an LLVM built from what its key names is placed**: not from an + /// LLVM checkout holding what no commit does, refused before anything is + /// built; not from a checkout other than the gitlink's; not when the + /// sources moved while it was being built. #[test] - fn a_staged_llvm_gitlink_is_refused() { - let scratch = Scratch::new("llvm-staged"); - let (_primary, _rust_dir, [same, _a, _b]) = estate(&scratch); - let fork = same.join("rust"); - git(&fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{LLVM_B},{LLVM}")]); - let said = refusal("a staged gitlink named an LLVM", || { - key(&fork); + fn what_the_key_does_not_name_is_never_placed() { + let e = estate_built("llvm-dirt"); + let fork = e.a.join("rust"); + let never = |_: &Path| -> PathBuf { panic!("an LLVM was built from a checkout no commit holds") }; + + write(&fork.join(LLVM).join("llvm/lib/IR/Core.cpp"), "int core_edited;\n"); + let said = refusal("an uncommitted LLVM edit was built", || { + choose(&e.a, &e.rust_dir, &fork, &sources(&e.a), never); }); - assert!(said.contains("stages") && said.contains(LLVM_B), "{said}"); - } - - /// **An uncommitted `src/bootstrap` names no LLVM**, not even one already - /// stored: the key refuses it, and nothing is built or resolved. - #[test] - fn an_uncommitted_bootstrap_names_no_llvm() { - let scratch = Scratch::new("llvm-dirty-key"); - let (_primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); - let fork = a.join("rust"); - drop(choose(&a, &rust_dir, &fork, fake_build)); - write(&fork.join("src/bootstrap/src/core/build_steps/llvm.rs"), "cfg.define(\"LLVM_ENABLE_ZLIB\", \"ON\");\n"); - let never = |_: &Path| -> PathBuf { panic!("an LLVM was built from a bootstrap no commit holds") }; - let said = refusal("an uncommitted bootstrap edit resolved to the stored LLVM", || { - choose(&a, &rust_dir, &fork, never); - }); - assert!(said.contains("src/bootstrap holds changes no commit does"), "{said}"); - let said = refusal("an uncommitted bootstrap edit named an LLVM", || { - key(&fork); - }); - assert!(said.contains("src/bootstrap holds changes no commit does"), "{said}"); - } - - /// **Only an LLVM built from what its key names is stored**: not from a - /// `src/bootstrap` holding what no commit does, which is refused before - /// anything is built, and after, when the build left it so; not from an - /// LLVM checkout other than the gitlink's; not when the sources moved while - /// it was being built. - #[test] - fn what_the_key_does_not_name_is_never_stored() { - let scratch = Scratch::new("llvm-dirt"); - let (_primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); - let fork = a.join("rust"); - let step = fork.join("src/bootstrap/src/core/build_steps/llvm.rs"); - let never = |_: &Path| -> PathBuf { panic!("an LLVM was built from a bootstrap no commit holds") }; - - write(&step, "cfg.define(\"LLVM_ENABLE_ZLIB\", \"OFF\");\n"); - let said = refusal("an uncommitted bootstrap edit was built", || { - choose(&a, &rust_dir, &fork, never); - }); - assert!(said.contains("src/bootstrap holds changes no commit does"), "{said}"); - git(&fork, &["add", "-A"]); - git(&fork, &["commit", "-qm", "the step"]); - - let dirtying = |fork: &Path| { - write(&fork.join("src/bootstrap/src/core/build_steps/llvm.rs"), "cfg.define(\"LLVM_ENABLE_ZSTD\", \"ON\");\n"); - fake_build(fork) - }; - let said = refusal("an LLVM built while its bootstrap was edited was stored", || { - choose(&a, &rust_dir, &fork, dirtying); - }); - assert!(said.contains("src/bootstrap holds changes no commit does"), "{said}"); - git(&fork, &["checkout", "-q", "--", "src/bootstrap"]); + assert!(said.contains("holds changes no commit does"), "{said}"); + fs::remove_dir_all(fork.join(LLVM).join("llvm/lib")).unwrap(); let lagging = |fork: &Path| { check_out_llvm(fork, "A"); fake_build(fork) }; pin_llvm(&fork, "B"); - let said = refusal("an LLVM checkout other than the gitlink's was stored", || { - choose(&a, &rust_dir, &fork, lagging); + let said = refusal("an LLVM checkout other than the gitlink's was placed", || { + choose(&e.a, &e.rust_dir, &fork, &sources(&e.a), lagging); }); - assert!(said.contains("is checked out at") && said.contains("submodule update src/llvm-project"), "{said}"); - check_out_llvm(&fork, "B"); + assert!(said.contains(&format!("{} is at", fork.join(LLVM).display())) && said.contains("its gitlink names"), "{said}"); + let step = fork.join("src/bootstrap/src/core/build_steps/llvm.rs"); let moving = |fork: &Path| { write(&step, "cfg.define(\"LLVM_ENABLE_ZSTD\", \"OFF\");\n"); - git(fork, &["commit", "-qam", "moved while built"]); fake_build(fork) }; - let said = refusal("an LLVM whose sources moved while it was built was stored", || { - choose(&a, &rust_dir, &fork, moving); + let said = refusal("an LLVM whose sources moved while it was built was placed", || { + choose(&e.a, &e.rust_dir, &fork, &sources(&e.a), moving); }); assert!(said.contains("moved while LLVM"), "{said}"); - - let stored: Vec<_> = fs::read_dir(store(&rust_dir)).unwrap().flatten().map(|e| e.file_name()).collect(); - assert!(stored.iter().all(|n| n.to_string_lossy().ends_with(".partial")), "stored: {stored:?}"); - } - - const WORKTREE: &str = "TOYOS_LLVM_TEST_WORKTREE"; - const RUST_DIR: &str = "TOYOS_LLVM_TEST_RUST_DIR"; - const ROLE: &str = "TOYOS_LLVM_TEST_ROLE"; - - /// The competing process for the tests below: the LLVM the worktree in - /// [`WORKTREE`] names, held in use until released — or, as `make`, held - /// while it is being made. - #[test] - #[ignore = "the competing process for the tests below; never runs on its own"] - fn child_role() { - let worktree = PathBuf::from(std::env::var(WORKTREE).unwrap_or_else(|_| panic!("child_role ran without {WORKTREE}; it is not a test"))); - let rust_dir = PathBuf::from(std::env::var(RUST_DIR).unwrap()); - match std::env::var(ROLE).unwrap().as_str() { - "use" => { - let _held = choose(&worktree, &rust_dir, &worktree.join("rust"), fake_build); - buildlock::tests::hold_until_released(); - } - "make" => { - let held = |fork: &Path| { - buildlock::tests::hold_until_released(); - fake_build(fork) - }; - choose(&worktree, &rust_dir, &worktree.join("rust"), held); - } - other => panic!("unknown child role {other}"), - } - } - - fn elsewhere(role: &str, worktree: &Path, rust_dir: &Path) -> buildlock::tests::Elsewhere { - let env = [(ROLE, std::ffi::OsStr::new(role)), (WORKTREE, worktree.as_os_str()), (RUST_DIR, rust_dir.as_os_str())]; - buildlock::tests::Elsewhere::hold("llvm::tests::child_role", &env) - } - - /// **An LLVM another process is making is waited for, not made again.** - #[test] - fn an_llvm_being_made_elsewhere_is_not_made_again() { - let scratch = Scratch::new("llvm-made-elsewhere"); - let (primary, rust_dir, [_same, _a, b]) = estate_built(&scratch); - let maker = elsewhere("make", &b, &rust_dir); - let made = key(&b.join("rust")); - assert!(buildlock::keyed_idle(&primary, Keyed::Llvm, &made).is_none(), "a sweep could take an LLVM being made"); - maker.release(); - let never = |_: &Path| -> PathBuf { panic!("an LLVM another process made was made here too") }; - assert_eq!(defect(&choose(&b, &rust_dir, &b.join("rust"), never).dir), None); - } - - /// **A sweep takes an LLVM only once no worktree names it and nobody uses - /// it**: `a` moves to another LLVM while a process of its own still uses - /// the first, and then `b` names the first until it moves too. - #[test] - fn an_llvm_is_swept_once_nobody_names_or_uses_it() { - let scratch = Scratch::new("llvm-sweep"); - let (primary, rust_dir, [_same, a, b]) = estate_built(&scratch); - let user = elsewhere("use", &a, &rust_dir); - let first = store(&rust_dir).join(key(&a.join("rust"))); - - let fork = a.join("rust"); - pin_llvm(&fork, "B"); - let second = choose(&a, &rust_dir, &fork, fake_build); - assert_ne!(second.dir, first); - assert!(first.is_dir(), "placing an LLVM swept one still in use"); - - keystore::record(&b, Keyed::Llvm, &key(&b.join("rust"))); - user.release(); - let swept = |root: &Path| keystore::sweep(root, Keyed::Llvm, &store(&rust_dir)); - assert_eq!(swept(&primary), Vec::::new(), "the sweep took an LLVM a worktree names"); - keystore::record(&b, Keyed::Llvm, &key(&fork)); - assert_eq!(swept(&primary), [first], "the sweep kept an LLVM nobody names, or took a named one"); - assert!(second.dir.is_dir()); - } - - /// **A sweep stopped halfway leaves no LLVM that passes for whole**: the - /// entry is renamed away from its key before anything in it is removed, and - /// the next sweep takes what the stopped one left. - #[test] - fn a_stopped_sweep_leaves_no_llvm_that_passes_for_whole() { - let scratch = Scratch::new("llvm-sweep-stopped"); - let (primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); - let dir = choose(&a, &rust_dir, &a.join("rust"), fake_build).dir; - keystore::forget(&a, Keyed::Llvm); - let halfway = |path: &Path| { - keystore::writable(path); - fs::remove_file(path.join("lib/libLLVMCore.a")).unwrap(); - panic!("stopped"); - }; - let stopped = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - keystore::sweep_by(&primary, Keyed::Llvm, &store(&rust_dir), halfway) - })); - assert!(stopped.is_err(), "the stand-in removal was never asked"); - assert!(defect(&dir).is_some(), "a stopped sweep left {} passing for whole", dir.display()); - assert_eq!(keystore::sweep(&primary, Keyed::Llvm, &store(&rust_dir)), [dir.with_extension("swept")]); - } - - /// **An LLVM a worktree resolved stays once nothing uses it**: its record, - /// not its use, is what names it. - #[test] - fn a_resolved_llvm_is_named_by_its_worktree() { - let scratch = Scratch::new("llvm-named"); - let (primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); - elsewhere("use", &a, &rust_dir).release(); - let dir = store(&rust_dir).join(key(&a.join("rust"))); - assert_eq!(keystore::sweep(&primary, Keyed::Llvm, &store(&rust_dir)), Vec::::new()); - assert_eq!(defect(&dir), None, "the sweep took an LLVM the worktree that resolved it names"); - } - - /// **A build directory whose compiler links the host's LLVM keeps none of - /// its own**: bootstrap's LLVM and LLD, `download-ci-llvm`'s and its - /// downloads go, and what a stopped removal left; the rest stays. - #[test] - fn a_build_directory_keeps_no_llvm_of_its_own() { - let build = TempDir::new("llvm-in-tree"); - let host = build.join(host_triple()); - for file in ["llvm/bin/llvm-config", "lld/bin/lld", "ci-llvm/lib/libLLVM.dylib", "llvm.swept/bin/clang", "stage2/bin/rustc"] { - write(&host.join(file), "x"); - } - for file in ["cache/llvm-1111-false/rust-dev.tar.xz", "cache/llvm-2222-false.swept/rust-dev.tar.xz", "cache/2026-07-13/rustc.tar.xz"] { - write(&build.join(file), "x"); - } - retire_in_tree(&build); - for gone in ["llvm", "lld", "ci-llvm", "llvm.swept"] { - assert!(!host.join(gone).exists(), "{gone} stayed"); - } - let cache: Vec<_> = fs::read_dir(build.join("cache")).unwrap().flatten().map(|e| e.file_name()).collect(); - assert_eq!(cache, ["2026-07-13"]); - assert!(host.join("stage2/bin/rustc").is_file()); - } - - /// Every file under `dir` with its bytes, and every link with its target. - fn snapshot(dir: &Path) -> Vec<(PathBuf, Vec)> { - let mut out = Vec::new(); - let mut stack = vec![dir.to_path_buf()]; - while let Some(at) = stack.pop() { - for entry in fs::read_dir(&at).unwrap().flatten() { - let path = entry.path(); - let meta = fs::symlink_metadata(&path).unwrap(); - if meta.file_type().is_symlink() { - out.push((path.clone(), fs::read_link(&path).unwrap().into_os_string().into_encoded_bytes())); - } else if meta.is_dir() { - stack.push(path); - } else { - out.push((path.clone(), fs::read(&path).unwrap())); - } - } - } - out.sort(); - out + let placed: Vec<_> = fs::read_dir(Kind::Llvm.dir(&e.rust_dir)).into_iter().flatten().flatten().map(|e| e.file_name()).collect(); + assert!(placed.is_empty(), "placed: {placed:?}"); } } diff --git a/src/main.rs b/src/main.rs index 3ed298f68d6..449db66cb9f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,6 +48,12 @@ const ALSO_USED: &[Tool] = &[ why: "rustc's bootstrap builds LLVM and clang with it, under CMake; `brew install \ ninja` on macOS", }, + Tool { + any: &["perl"], + why: "a compiler build builds the toolchain's cargo, whose OpenSSL `openssl-src` \ + configures with Perl", + }, + Tool { any: &["make"], why: "a compiler build builds the toolchain's cargo, whose OpenSSL `openssl-src` builds with make" }, ]; /// Where the OS would find `name`, if anywhere. @@ -231,11 +237,6 @@ fn main() { return; } - if asked(&flags::WORKTREE) { - toyos_build::worktree::dispatch(&root, &args); - return; - } - // Only where the submodules belong. In a linked worktree `rust/` is an empty // stub and initialising it clones the whole rust history again, into a git // directory of its own that shares no objects with the one beside it. @@ -243,8 +244,6 @@ fn main() { toyos_build::ensure_submodules(&root); } - // Toolchain included: `build` holds the build lock across both, so no other - // agent's clean or bootstrap can land between the two. let plan = toyos_build::build::plan_for(&root, &boot, debug, &args); if let Some(out) = update_image { toyos_build::build::build_update(&root, &boot, &plan, &out); diff --git a/src/release.rs b/src/release.rs index 0f9d910b4ca..2934a91628d 100644 --- a/src/release.rs +++ b/src/release.rs @@ -18,14 +18,11 @@ use std::process::{Command, Stdio}; use sha2::{Digest, Sha256}; use toyos_tmpdir::TempDir; -use crate::toolchain::HOSTED_ARCH; - /// What the tag hashes, as `git rev-parse HEAD:` names them. The last is /// this file. fn trees() -> Vec<&'static str> { std::iter::once("rust") - .chain(crate::sysroot::SYSROOT_SOURCES) - .chain(crate::sysroot::SYSROOT_MANIFESTS) + .chain(crate::store::ABI_TREES) .chain([crate::clang::SOURCE, file!()]) .collect() } @@ -183,6 +180,15 @@ fn run(cmd: &mut Command) -> Result<(), String> { status.success().then_some(()).ok_or_else(|| format!("{cmd:?} exited {status}")) } +/// Check `root`'s `rust/` out at the commit it pins, in a primary checkout +/// alone: a linked worktree's `git submodule` clones the fork a second time. +fn check_out_fork(root: &Path) -> Result<(), String> { + match crate::toolchain::owner(root) { + crate::toolchain::Owner::Us => run(Command::new("git").args(["submodule", "update", "--init", "rust"]).current_dir(root)), + _ => Err(format!("{} is no primary checkout, and a release is built in one", root.display())), + } +} + /// Whether `gh` says `tag` carries [`ASSET`]. fn published(root: &Path, tag: &str) -> bool { Command::new("gh") @@ -225,7 +231,7 @@ pub fn ensure_published(root: &Path) -> Result { /// Bootstrap, check the glibc floor, package, publish, and wait for the asset. fn build(root: &Path, tag: &str, tmp: &Path) -> Result<(), String> { - run(Command::new("git").args(["submodule", "update", "--init", "rust"]).current_dir(root))?; + check_out_fork(root)?; // Bootstrap takes `HEAD^1` as the upstream commit whose artifacts to fetch // when it sees GitHub Actions; in this fork that is our own merge, which // rust-lang's CI never built. @@ -239,7 +245,7 @@ fn build(root: &Path, tag: &str, tmp: &Path) -> Result<(), String> { // the compiler with the guest libraries and `libtoyos_c.a` this tree's // sources name, recorded beside the witness an installer checks it by. let build = root.join("rust/build"); - let key = crate::keystore::recorded(root, crate::buildlock::Keyed::Sysroot).ok_or("the build recorded no sysroot key")?; + let key = crate::store::recorded(root, crate::store::Kind::Sysroot).ok_or("the build recorded no sysroot key")?; let sysroot = format!("sysroots/{key}"); let stage2 = build.join(&sysroot); fs::write(build.join("toyos-sysroot-witness"), crate::sysroot::witness(root)) @@ -254,17 +260,13 @@ fn build(root: &Path, tag: &str, tmp: &Path) -> Result<(), String> { } fs::copy(tmp.join("TOOLCHAIN"), build.join("TOOLCHAIN")).map_err(|e| e.to_string())?; - // `lib/rustlib/` and the sysroot's `bin/cargo` are links into this - // runner's own toolchain; `Owner::Installed` recreates both. GNU tar's - // `--transform` renames the sysroot to the path an installer links. + // GNU tar's `--transform` renames the sysroot to the path an installer links. let tarball = tmp.join(ASSET); let mut tar = Command::new("tar") .arg("-C") .arg(&build) - .arg(format!("--exclude={}/stage2/lib/rustlib/{HOST}", HOSTED_ARCH.userland())) - .arg(format!("--exclude={sysroot}/bin/cargo")) .arg(format!("--transform=s,^{sysroot},{HOST}/stage2,")) - .args(["-c", &sysroot, &format!("{}/stage2", HOSTED_ARCH.userland())]) + .args(["-c", &sysroot]) .args(["toyos-sysroot-witness", "TOOLCHAIN"]) .stdout(Stdio::piped()) .spawn() @@ -380,10 +382,9 @@ fn notes(root: &Path, tag: &str, manifest: &str) -> Result { mkdir -p toyos-toolchain curl -sSL {url} | tar --zstd -x -C toyos-toolchain rustup toolchain link toyos toyos-toolchain/{HOST}/stage2 - ln -s \"$(rustup which cargo)\" toyos-toolchain/{HOST}/stage2/bin/cargo cargo +toyos build --target x86_64-unknown-toyos -rustc's ToyOS target names `rust-lld` as its linker, and the toolchain carries it where rustc looks for it, so nothing goes on `PATH`. The `cargo` symlink is not shipped because its path would be the publisher's. +rustc's ToyOS target names `rust-lld` as its linker, and the toolchain carries it where rustc looks for it, so nothing goes on `PATH`. Its `cargo` is the fork's own, built with this `rustc`. ## C @@ -439,6 +440,18 @@ fn alias(root: &Path, manifest: &str, tmp: &Path) -> Result { mod tests { use super::*; + /// **A release is built in a primary checkout alone**: a linked worktree's + /// is refused, and `git submodule` never runs there. + #[test] + fn a_linked_worktree_checks_out_no_fork() { + let e = crate::store::tests::estate("release-linked"); + let stub = e.same.parent().unwrap().join("stub"); + git(&e.primary, &["worktree", "add", "-q", "-b", "stub", stub.to_str().unwrap()]); + let said = check_out_fork(&stub).expect_err("a linked worktree checked the fork out"); + assert!(said.contains("is no primary checkout"), "{said}"); + assert!(!e.primary.join(".git/worktrees/stub/modules").exists(), "git submodule ran in a linked worktree"); + } + /// The packaging is one of the trees its own tag hashes. #[test] fn the_tag_hashes_this_file() { diff --git a/src/store.rs b/src/store.rs new file mode 100644 index 00000000000..f68eec479a3 --- /dev/null +++ b/src/store.rs @@ -0,0 +1,1210 @@ +//! The host's toolchain store: every LLVM, compiler and sysroot a build on this +//! host has made, at `/rust/build///`, one directory per +//! key, read-only and never written once it is there. +//! +//! **A key is [`key`] of a recipe and the git hashes of what the product is +//! built from, and nothing else.** Those hashes are [`Sources`]: the four trees +//! a toolchain is made of — the rust fork, `toyos-abi`, `toyos` and +//! `userland/libc` — as git hashes them where they stand, edits included. +//! +//! **A product is there whole or not at all, and the store holds nothing else.** +//! One maker at a time holds the claim `.making/` (`src/dirlock.rs`), +//! fills it and renames it into the store as ``; a build that finds it held +//! waits for its maker instead of making the key again, and one whose maker is +//! dead takes it away and makes the key afresh. A rename onto a key already +//! placed fails, and the loser removes its copy. A product in use is held +//! shared, and holds no link that leaves it. +//! +//! **A key stays while a registered worktree records it, [`CURRENT`] names it, +//! or somebody holds it; everything else goes** — every other key, and whatever +//! a dead maker or a stopped collection left. [`collect`] runs after every +//! placement, and decides under the kind's store held exclusively. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::ErrorKind; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use sha2::{Digest, Sha256}; +use toyos_tmpdir::TempDir; + +use crate::dirlock::Lock; +use crate::sysroot::git; + +/// A product the store holds. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub enum Kind { + Llvm, + Compiler, + Sysroot, +} + +impl Kind { + const ALL: [Kind; 3] = [Kind::Llvm, Kind::Compiler, Kind::Sysroot]; + + fn name(self) -> &'static str { + match self { + Kind::Llvm => "LLVM", + Kind::Compiler => "compiler", + Kind::Sysroot => "sysroot", + } + } + + /// Where this kind's keys are, in the primary's `rust/`. + pub fn dir(self, rust_dir: &Path) -> PathBuf { + rust_dir.join("build").join(match self { + Kind::Llvm => "llvm", + Kind::Compiler => "compilers", + Kind::Sysroot => "sysroots", + }) + } + + /// Where a worktree records the key of this kind its build uses. + fn record(self, root: &Path) -> PathBuf { + root.join(match self { + Kind::Llvm => "target/toyos-llvm-key", + Kind::Compiler => "target/toyos-compiler-key", + Kind::Sysroot => "target/toyos-sysroot-key", + }) + } +} + +/// The one stable path the rustup `toyos` toolchain names: a link, in the +/// primary's `rust/build/`, to the sysroot of the primary's last build. +pub fn current(rust_dir: &Path) -> PathBuf { + rust_dir.join(CURRENT) +} + +/// [`current`], relative to the primary's `rust/`. +pub const CURRENT: &str = "build/toyos"; + +/// The first 16 hex digits of the SHA-256 of `data`. +pub(crate) fn short(data: &[u8]) -> String { + Sha256::digest(data).iter().take(8).map(|b| format!("{b:02x}")).collect() +} + +/// The key of a product made by `recipe` from `parts`: the one key function. +pub fn key(recipe: &str, parts: &[&str]) -> String { + let all: Vec<&str> = std::iter::once(recipe).chain(parts.iter().copied()).collect(); + short(all.join("\n\0\n").as_bytes()) +} + +/// What of the rust fork a toolchain is built from: its LLVM by the commit +/// the fork names, bootstrap, the compiler and its tools, and std. +pub const FORK_TREES: [&str; 7] = + ["src/llvm-project", "src/bootstrap", "compiler", "src/tools", "src/stage0", "Cargo.lock", "library"]; + +/// The three trees of this repository std and `libtoyos_c.a` compile. +pub const ABI_TREES: [&str; 3] = ["toyos-abi", "toyos", "userland/libc"]; + +/// The four trees a toolchain is built from, as git hashes them. +#[derive(PartialEq, Debug)] +pub struct Sources(BTreeMap<&'static str, String>); + +impl Sources { + /// `root`'s ABI trees, and the fork checkout at `fork`, as they stand. + pub fn of(root: &Path, fork: &Path) -> Self { + Self::with(root, FORK_TREES.into_iter().zip(trees(fork, &FORK_TREES, Relocked::Yes))) + } + + /// `root`'s ABI trees as they stand, and the fork's as `commit` holds them + /// in the fork repository at `rust_dir`: what a clean checkout of `commit` + /// hashes to, asked of no checkout. + pub fn pinned(root: &Path, rust_dir: &Path, commit: &str) -> Self { + let spec: Vec = FORK_TREES.iter().map(|tree| format!("{commit}:{tree}")).collect(); + let args: Vec<&str> = std::iter::once("rev-parse").chain(spec.iter().map(String::as_str)).collect(); + let hashes = git(rust_dir, &args, None).unwrap_or_else(|e| { + panic!("{e}\nThe fork repository at {} holds no commit {commit}, which this tree pins: fetch it there", rust_dir.display()) + }); + let hashes: Vec = String::from_utf8_lossy(&hashes).lines().map(str::to_string).collect(); + Self::with(root, FORK_TREES.into_iter().zip(hashes)) + } + + fn with(root: &Path, fork: impl Iterator) -> Self { + Self(fork.chain(ABI_TREES.into_iter().zip(trees(root, &ABI_TREES, Relocked::No))).collect()) + } + + /// The hash of `tree`, one of [`FORK_TREES`] or [`ABI_TREES`]. + pub fn get(&self, tree: &str) -> &str { + self.0.get(tree).unwrap_or_else(|| panic!("{tree} is not one of the trees a toolchain is built from")) + } +} + +/// Whether a running bootstrap rewrites a checkout's `Cargo.lock`s. +#[derive(Clone, Copy, PartialEq)] +pub enum Relocked { + /// The fork's: bootstrap rewrites them while it runs and puts them back + /// after, and a key read meanwhile would name neither, so each is keyed as + /// the index holds it. + Yes, + /// Any other checkout's: a lockfile is keyed as it stands, like every file. + No, +} + +/// The git hash of each of `paths` in the checkout `repo` as it stands: +/// committed, staged or neither, untracked files included and ignored ones not. +/// A submodule is the commit its gitlink names, and never the one its checkout +/// happens to be at; a checkout of one holding changes no commit does is +/// refused, since bootstrap builds them and the gitlink does not name them. +/// Hashed through a copy of the checkout's index, so the checkout's own is +/// never written. +pub fn trees(repo: &Path, paths: &[&str], lockfiles: Relocked) -> Vec { + let scratch = TempDir::new("store-index"); + let index = scratch.join("index"); + let run = |dir: &Path, args: &[&str], index: Option<&Path>| { + let out = git(dir, args, index).unwrap_or_else(|e| panic!("{e}")); + String::from_utf8(out).unwrap_or_else(|e| panic!("git {args:?} printed no UTF-8: {e}")) + }; + let real = run(repo, &["rev-parse", "--path-format=absolute", "--git-path", "index"], None); + fs::copy(real.trim(), &index).unwrap_or_else(|e| panic!("copy {}: {e}", real.trim())); + let gitlinks: Vec = gitlinks(repo, paths, Some(&index)).into_iter().map(|(path, _)| path).collect(); + for checkout in gitlinks.iter().map(|path| repo.join(path)).filter(|checkout| checkout.join(".git").exists()) { + let changes = run(&checkout, &["--no-optional-locks", "status", "--porcelain"], None); + assert!( + changes.is_empty(), + "{} holds changes no commit does, and a submodule is keyed on the commit its gitlink names: \ + commit them there and record that commit in {}\n{changes}", + checkout.display(), + repo.display(), + ); + } + let mut excluded: Vec = gitlinks.iter().map(|path| format!(":(exclude){path}")).collect(); + if lockfiles == Relocked::Yes { + excluded.push(":(exclude,glob)**/Cargo.lock".to_string()); + } + let added = paths.iter().filter(|p| lockfiles == Relocked::No || !p.ends_with("Cargo.lock")).copied(); + let add: Vec<&str> = ["add", "-A", "--"].into_iter().chain(added).chain(excluded.iter().map(String::as_str)).collect(); + run(repo, &add, Some(&index)); + let tree = run(repo, &["write-tree"], Some(&index)); + let spec: Vec = paths.iter().map(|p| format!("{}:{p}", tree.trim())).collect(); + let spec: Vec<&str> = std::iter::once("rev-parse").chain(spec.iter().map(String::as_str)).collect(); + run(repo, &spec, None).lines().map(str::to_string).collect() +} + +/// Each submodule under `paths` in the checkout `repo`, every one when there +/// are none, with the commit its gitlink in the index names; in `index`, if +/// given, rather than the checkout's own. +pub(crate) fn gitlinks(repo: &Path, paths: &[&str], index: Option<&Path>) -> Vec<(String, String)> { + let args: Vec<&str> = ["ls-files", "--stage", "--"].into_iter().chain(paths.iter().copied()).collect(); + let listed = git(repo, &args, index).unwrap_or_else(|e| panic!("{e}")); + let listed = String::from_utf8(listed).unwrap_or_else(|e| panic!("git {args:?} printed no UTF-8: {e}")); + listed + .lines() + .filter_map(|line| { + let (entry, path) = line.split_once('\t')?; + let mut words = entry.split(' '); + (words.next() == Some("160000")).then(|| (path.to_string(), words.next().expect("a gitlink names a commit").to_string())) + }) + .collect() +} + +/// Refuse `what`, which bootstrap just built in the fork checkout `fork` from +/// `built`, the paths it compiles, unless every submodule under them is +/// checked out at the commit its gitlink in the index names. Bootstrap leaves +/// a submodule that is at `HEAD`'s gitlink where it is, under a staged one too, +/// and one it fails to move; it builds whatever is there, and from an empty one +/// nothing. +pub fn assert_built_at_gitlinks(fork: &Path, built: &[&str], what: &str) { + for (path, gitlink) in gitlinks(fork, built, None) { + let checkout = fork.join(path); + if !checkout.join(".git").exists() { + let empty = match fs::read_dir(&checkout) { + Ok(mut entries) => entries.next().is_none(), + Err(e) if e.kind() == ErrorKind::NotFound => true, + Err(e) => panic!("read {}: {e}", checkout.display()), + }; + assert!( + empty, + "{} holds files and is no checkout of its gitlink {gitlink}, and bootstrap built {what} from \ + them; nothing was kept", + checkout.display(), + ); + continue; + } + let at = git(&checkout, &["rev-parse", "HEAD"], None).unwrap_or_else(|e| panic!("{e}")); + let at = String::from_utf8_lossy(&at); + assert!( + at.trim() == gitlink, + "{} is at {}, and its gitlink names {gitlink}: bootstrap built {what} from the commit checked \ + out there, which its sources do not name; nothing was kept. `git -C {} checkout --detach \ + {gitlink}` checks the gitlink's commit out", + checkout.display(), + at.trim(), + checkout.display(), + ); + } +} + +/// A product in use: shared, so any number of builds use it at once and +/// [`collect`] cannot take it. +pub struct Held { + pub dir: PathBuf, + _lock: Lock, +} + +/// The product `key` names, held in use; made by `make` first when nobody has +/// made it. `make` fills the directory it is given with the whole product or +/// panics; `root` records the key before anything is looked at. +pub fn get(root: &Path, rust_dir: &Path, kind: Kind, key: &str, mut make: impl FnMut(&Path)) -> Held { + record(root, kind, key); + let store = kind.dir(rust_dir); + let dir = store.join(key); + loop { + if let Some(held) = in_use(&dir, kind, key) { + return held; + } + match claim(&store, key, &Lock::try_exclusive) { + Claim::Mine(making, lock) if dir.is_dir() => remove(&take_away(&lock, &making, &claims(&store), key)), + Claim::Mine(making, lock) => { + eprintln!("Making {} {key}", kind.name()); + make(&making); + let placed = publish(&making, &lock, &dir, remove); + // Its waiters take the key now, not behind the collection. + drop(lock); + if placed { + for gone in collect(root, rust_dir) { + eprintln!("Removed {}: nothing names it", gone.display()); + } + } + } + Claim::Theirs(making) => { + let pid = fs::read_to_string(making.join(MAKER)).unwrap_or_default(); + drop(Lock::shared_if_there(&making, &format!("{} {key} is being made by pid {}", kind.name(), pid.trim()))); + } + } + } +} + +/// `dir`, held in use, if it is there. +fn in_use(dir: &Path, kind: Kind, key: &str) -> Option { + let lock = named(dir, Lock::shared_if_there(dir, &format!("{} {key} is being removed", kind.name()))?)?; + Some(Held { dir: dir.to_path_buf(), _lock: lock }) +} + +/// `lock`, if it holds the directory `dir` still names: a key or a claim is +/// renamed away before it is removed, and a lock taken on what was opened +/// before that holds what is being removed. +fn named(dir: &Path, lock: Lock) -> Option { + let named = fs::metadata(dir).ok()?.ino(); + let held = lock.file().metadata().unwrap_or_else(|e| panic!("stat {}: {e}", dir.display())).ino(); + (named == held).then_some(lock) +} + +/// `path` exclusively, taken by `take` as [`Lock::try_exclusive`] takes it, if +/// nobody holds it and `path` still names what was taken. +fn unheld(path: &Path, take: &impl Fn(&Path) -> Option) -> Option { + named(path, take(path)?) +} + +/// Who makes a key: this process, holding its claim, or the process that does. +enum Claim { + Mine(PathBuf, Lock), + Theirs(PathBuf), +} + +/// The file a maker writes its pid in, so a waiter can say whom it waits for. +const MAKER: &str = ".maker"; + +static MADE: AtomicU64 = AtomicU64::new(0); + +/// Claim the making of `key` in `store`. The claim is a directory made and held +/// under a name of its own and renamed to `` among the claims, which a rename never +/// replaces once it holds anything: so exactly one process holds the name, and +/// it held it before anybody could see it. One whose holder is dead, which +/// `take` takes as [`Lock::try_exclusive`] does, is taken away, and claimed +/// afresh. +fn claim(store: &Path, key: &str, take: &impl Fn(&Path) -> Option) -> Claim { + let claims = claims(store); + fs::create_dir_all(&claims).unwrap_or_else(|e| panic!("create {}: {e}", claims.display())); + let making = claims.join(key); + loop { + let n = MADE.fetch_add(1, Ordering::Relaxed); + let mine = claims.join(format!("{key}.{}-{n}.partial", std::process::id())); + fs::create_dir(&mine).unwrap_or_else(|e| panic!("create {}: {e}", mine.display())); + let lock = Lock::exclusive(&mine, "a probe of a claim just made"); + fs::write(mine.join(MAKER), std::process::id().to_string()).unwrap_or_else(|e| panic!("write {}: {e}", mine.display())); + match fs::rename(&mine, &making) { + Ok(()) => return Claim::Mine(making, lock), + Err(e) if placed_before(&e, &making) => remove(&mine), + Err(e) => panic!("rename {} -> {}: {e}", mine.display(), making.display()), + } + drop(lock); + let Some(dead) = unheld(&making, take) else { return Claim::Theirs(making) }; + let away = take_away(&dead, &making, &claims, key); + drop(dead); + remove(&away); + } +} + +/// Place what was made at `made`, which `held` holds, as the key `dir`, +/// read-only; `false`, and `made` taken away and removed with `remove`, which a +/// test acts in the gap before, if another maker placed it first. One holding +/// a link that leaves it is refused: its bytes would name whoever made it. +pub(crate) fn publish(made: &Path, held: &Lock, dir: &Path, remove: impl Fn(&Path)) -> bool { + let _ = fs::remove_file(made.join(MAKER)); + let out = links_out(made, made); + assert!(out.is_empty(), "{} holds links that leave it, and a key is only what it names: {out:?}", made.display()); + read_only(made); + // Renaming a directory writes its `..`, so its own mode waits for the rename. + set_writable(made, true); + let store = dir.parent().expect("a key is in a store"); + fs::create_dir_all(store).unwrap_or_else(|e| panic!("create {}: {e}", store.display())); + match fs::rename(made, dir) { + Ok(()) => { + set_writable(dir, false); + true + } + Err(e) if placed_before(&e, dir) => { + let key = dir.file_name().and_then(|k| k.to_str()).expect("a key is a name"); + remove(&take_away(held, made, made.parent().expect("what was made is beside its store"), key)); + false + } + Err(e) => panic!("rename {} -> {}: {e}", made.display(), dir.display()), + } +} + +/// Whether `e`, from a rename of a directory onto `to`, says `to` was already +/// there holding something: not empty, or, when it is read-only, not writable. +fn placed_before(e: &std::io::Error, to: &Path) -> bool { + match e.kind() { + ErrorKind::DirectoryNotEmpty | ErrorKind::AlreadyExists => true, + ErrorKind::PermissionDenied => to.is_dir(), + _ => false, + } +} + +/// Every link under `dir` whose target is outside `product`. +fn links_out(product: &Path, dir: &Path) -> Vec { + let mut out = Vec::new(); + for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { + let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", dir.display())).path(); + let meta = fs::symlink_metadata(&path).unwrap_or_else(|e| panic!("stat {}: {e}", path.display())); + if meta.is_dir() { + out.extend(links_out(product, &path)); + } else if meta.file_type().is_symlink() { + let target = fs::read_link(&path).unwrap_or_else(|e| panic!("readlink {}: {e}", path.display())); + let mut depth = dir.strip_prefix(product).expect("walked from the product").components().count(); + let leaves = target.components().any(|part| match part { + Component::Normal(_) => { + depth += 1; + false + } + Component::CurDir => false, + Component::ParentDir => match depth.checked_sub(1) { + Some(up) => { + depth = up; + false + } + None => true, + }, + Component::RootDir | Component::Prefix(_) => true, + }); + if leaves { + out.push(path); + } + } + } + out +} + +/// Record that `root`'s builds use `kind`'s `key`: whole or not at all, so +/// [`collect`] never reads a record half-written. A name that is no key is +/// refused, so the store never locks or removes one. +pub fn record(root: &Path, kind: Kind, key: &str) { + assert!(is_key(key), "{key:?} is no key: a key is 16 hex digits"); + let path = kind.record(root); + let dir = path.parent().expect("a record is under target/"); + fs::create_dir_all(dir).unwrap_or_else(|e| panic!("create {}: {e}", dir.display())); + let written = path.with_extension(format!("{}-{}.new", std::process::id(), MADE.fetch_add(1, Ordering::Relaxed))); + fs::write(&written, key).unwrap_or_else(|e| panic!("write {}: {e}", written.display())); + fs::rename(&written, &path).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", written.display(), path.display())); +} + +/// `kind`'s store, made if it is not there. +fn store(rust_dir: &Path, kind: Kind) -> PathBuf { + let store = kind.dir(rust_dir); + fs::create_dir_all(&store).unwrap_or_else(|e| panic!("create {}: {e}", store.display())); + store +} + +/// The key of `kind` `root` records, if it records one. +pub fn recorded(root: &Path, kind: Kind) -> Option { + let path = kind.record(root); + match fs::read_to_string(&path) { + Ok(key) => Some(key.trim().to_string()), + Err(e) if e.kind() == ErrorKind::NotFound => None, + Err(e) => panic!("read {}: {e}", path.display()), + } +} + +/// Remove from the store everything no registered worktree of `root` records, +/// [`CURRENT`] does not name, and nobody holds. Returns what went. +pub fn collect(root: &Path, rust_dir: &Path) -> Vec { + collect_by(root, rust_dir, &Lock::try_exclusive, remove) +} + +/// [`collect`], taking what may go with `take` and removing with `remove`, so +/// a test can stop it or act in the gap before either. It acts on the store's +/// own names alone, a key or a name among the claims ([`claimed`]), and leaves +/// every other name where it is. +fn collect_by(root: &Path, rust_dir: &Path, take: &impl Fn(&Path) -> Option, remove: impl Fn(&Path)) -> Vec { + let listed = git(root, &["worktree", "list", "--porcelain"], None).unwrap_or_else(|e| panic!("{e}")); + let worktrees: Vec = + String::from_utf8_lossy(&listed).lines().filter_map(|l| l.strip_prefix("worktree ")).map(PathBuf::from).collect(); + let mut removed = Vec::new(); + for kind in Kind::ALL { + let store = store(rust_dir, kind); + let claims = claims(&store); + fs::create_dir_all(&claims).unwrap_or_else(|e| panic!("create {}: {e}", claims.display())); + let mut away = Vec::new(); + { + let _deciding = Lock::exclusive(&store, &format!("the {} store, behind another collection", kind.name())); + let mut kept: BTreeSet = worktrees.iter().filter_map(|w| recorded(w, kind)).collect(); + if kind == Kind::Sysroot { + if let Ok(link) = fs::read_link(current(rust_dir)) { + kept.extend(link.file_name().map(|k| k.to_string_lossy().into_owned())); + } + } + for key in entries(&store).into_iter().filter(|name| is_key(name) && !kept.contains(name)) { + let path = store.join(&key); + let Some(held) = unheld(&path, take) else { continue }; + set_writable(&path, true); + away.push((take_away(&held, &path, &claims, &key), path)); + } + } + for (gone, path) in away { + remove(&gone); + removed.push(path); + } + for name in entries(&claims) { + let Some((key, maker)) = claimed(&name) else { continue }; + // A claim is held from before its name exists; a partial or a + // removal, only once its maker holds it. + if maker.is_some_and(alive) { + continue; + } + let path = claims.join(&name); + let Some(held) = unheld(&path, take) else { continue }; + let gone = take_away(&held, &path, &claims, key); + drop(held); + remove(&gone); + removed.push(path); + } + } + removed +} + +/// Rename `path`, which `_held` holds exclusively, to a removal of this +/// process's among `claims`, and return that: what is removed is out of the +/// way first, so a collection that is stopped leaves nothing at its name that +/// passes for whole, and nobody takes the name back while it goes. +fn take_away(_held: &Lock, path: &Path, claims: &Path, key: &str) -> PathBuf { + let n = MADE.fetch_add(1, Ordering::Relaxed); + let gone = claims.join(format!("{key}.{}-{n}.gone", std::process::id())); + fs::rename(path, &gone).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", path.display(), gone.display())); + gone +} + +/// Where `store`'s keys are claimed, made and removed: beside it, so the store +/// itself holds whole keys and nothing else. +fn claims(store: &Path) -> PathBuf { + store.with_extension("making") +} + +/// The UTF-8 names in `store`, none when there is no `store`: no other name is +/// the store's. +fn entries(store: &Path) -> Vec { + let listing = match fs::read_dir(store) { + Ok(listing) => listing, + Err(e) if e.kind() == ErrorKind::NotFound => return Vec::new(), + Err(e) => panic!("read {}: {e}", store.display()), + }; + let mut names: Vec = listing + .map(|e| e.unwrap_or_else(|e| panic!("read {}: {e}", store.display())).file_name()) + .filter_map(|n| n.into_string().ok()) + .collect(); + names.sort(); + names +} + +/// Whether `name` is a key: what [`key`] makes, 16 lowercase hex digits. +fn is_key(name: &str) -> bool { + name.len() == 16 && name.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// The key a name among the claims is of, and the pid a partial's or a +/// removal's name carries: ``, `.-.partial` or +/// `.-.gone`, as [`claim`] and [`take_away`] name them. `None` +/// for every other name. +fn claimed(name: &str) -> Option<(&str, Option)> { + let mut parts = name.split('.'); + let key = parts.next().filter(|key| is_key(key))?; + let Some(made) = parts.next() else { return Some((key, None)) }; + let (pid, n) = made.split_once('-')?; + let pid = pid.parse::().ok().filter(|pid| *pid > 0)?; + n.parse::().ok()?; + (matches!(parts.next(), Some("partial" | "gone")) && parts.next().is_none()).then_some((key, Some(pid))) +} + +/// Whether the process `pid` is running: a maker between creating its partial +/// and holding it is not yet told apart from a dead one by the lock alone. +fn alive(pid: i32) -> bool { + // SAFETY: signal 0 runs the existence and permission checks and delivers nothing. + unsafe { libc::kill(pid, 0) == 0 || std::io::Error::last_os_error().kind() == ErrorKind::PermissionDenied } +} + +/// Take write permission from every file and directory under `dir`, and from `dir`. +fn read_only(dir: &Path) { + for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { + let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", dir.display())).path(); + let meta = fs::symlink_metadata(&path).unwrap_or_else(|e| panic!("stat {}: {e}", path.display())); + if meta.is_dir() { + read_only(&path); + } else if meta.is_file() { + let mut permissions = meta.permissions(); + permissions.set_readonly(true); + fs::set_permissions(&path, permissions).unwrap_or_else(|e| panic!("chmod {}: {e}", path.display())); + } + } + let mut permissions = fs::metadata(dir).unwrap_or_else(|e| panic!("stat {}: {e}", dir.display())).permissions(); + permissions.set_readonly(true); + fs::set_permissions(dir, permissions).unwrap_or_else(|e| panic!("chmod {}: {e}", dir.display())); +} + +/// Give `dir` alone its owner's write permission, or take it. +fn set_writable(dir: &Path, writable: bool) { + let mut permissions = fs::metadata(dir).unwrap_or_else(|e| panic!("stat {}: {e}", dir.display())).permissions(); + permissions.set_mode(if writable { permissions.mode() | 0o200 } else { permissions.mode() & !0o222 }); + fs::set_permissions(dir, permissions).unwrap_or_else(|e| panic!("chmod {}: {e}", dir.display())); +} + +/// Remove the directory `path` and all it holds, read-only or not. +pub(crate) fn remove(path: &Path) { + writable(path); + fs::remove_dir_all(path).unwrap_or_else(|e| panic!("remove {}: {e}", path.display())); +} + +/// Give `dir` and every directory under it back its owner's write permission. +pub(crate) fn writable(dir: &Path) { + let meta = fs::metadata(dir).unwrap_or_else(|e| panic!("stat {}: {e}", dir.display())); + let mut permissions = meta.permissions(); + permissions.set_mode(permissions.mode() | 0o700); + fs::set_permissions(dir, permissions).unwrap_or_else(|e| panic!("chmod {}: {e}", dir.display())); + for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { + let entry = entry.unwrap_or_else(|e| panic!("read {}: {e}", dir.display())); + if entry.file_type().unwrap_or_else(|e| panic!("stat {}: {e}", entry.path().display())).is_dir() { + writable(&entry.path()); + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use std::cell::{Cell, RefCell}; + use std::process::Command; + + use super::*; + use crate::dirlock::tests::{held_until_killed, Elsewhere}; + + pub(crate) fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(["-c", "commit.gpgsign=false", "-c", "user.email=t@t", "-c", "user.name=t"]) + .args(["-c", "protocol.file.allow=always", "-c", "init.defaultBranch=main"]) + .args(crate::gitfixture::NO_AUTO_MAINTENANCE) + .args(args) + .current_dir(dir) + .output() + .expect("run git"); + assert!(out.status.success(), "git {args:?} in {}: {}", dir.display(), String::from_utf8_lossy(&out.stderr)); + String::from_utf8(out.stdout).unwrap().trim().to_string() + } + + pub(crate) fn write(path: &Path, text: &str) { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + + /// The LLVM commits the fixtures' forks record. Nothing reads their content. + pub(crate) const LLVM_A: &str = "1111111111111111111111111111111111111111"; + pub(crate) const LLVM_B: &str = "2222222222222222222222222222222222222222"; + + /// A primary checkout whose `rust` pins fork commit `C0`, and three linked + /// worktrees whose `rust/` is a fork checkout of their own: `same` at `C0`, + /// `a` and `b` each at a commit whose `compiler/` is its own. + pub(crate) struct Estate { + pub primary: PathBuf, + pub rust_dir: PathBuf, + pub same: PathBuf, + pub a: PathBuf, + pub b: PathBuf, + scratch: TempDir, + } + + impl Drop for Estate { + /// The store is read-only, and the scratch goes whole. + fn drop(&mut self) { + writable(&self.scratch); + } + } + + pub(crate) fn estate(name: &str) -> Estate { + let scratch = TempDir::new(name); + let base = fs::canonicalize(&scratch).unwrap(); + + let backtrace = base.join("backtrace-src"); + fs::create_dir_all(&backtrace).unwrap(); + git(&backtrace, &["init", "-q"]); + write(&backtrace.join("lib.rs"), "pub fn trace() {}\n"); + git(&backtrace, &["add", "-A"]); + git(&backtrace, &["commit", "-qm", "backtrace"]); + + let cargo = base.join("cargo-src"); + fs::create_dir_all(&cargo).unwrap(); + git(&cargo, &["init", "-q"]); + write(&cargo.join("Cargo.toml"), "[package]\nname = \"cargo\"\n"); + git(&cargo, &["add", "-A"]); + git(&cargo, &["commit", "-qm", "cargo"]); + + let fork = base.join("fork-src"); + fs::create_dir_all(&fork).unwrap(); + git(&fork, &["init", "-q"]); + git(&fork, &["submodule", "add", "-q", backtrace.to_str().unwrap(), "library/backtrace"]); + git(&fork, &["submodule", "add", "-q", cargo.to_str().unwrap(), "src/tools/cargo"]); + write(&fork.join("compiler/rustc_target/src/lib.rs"), "pub fn targets() {}\n"); + write(&fork.join("src/bootstrap/src/lib.rs"), "fn main() {}\n"); + write(&fork.join("src/tools/lld-wrapper/src/main.rs"), "fn main() {}\n"); + write(&fork.join("src/stage0"), "compiler_version=beta\n"); + write(&fork.join("Cargo.lock"), "# lock\n"); + write(&fork.join("library/std/src/lib.rs"), "pub fn a() {}\n"); + write(&fork.join(".gitignore"), "/build\n"); + write(&fork.join("x.py"), "# bootstrap\n"); + git(&fork, &["add", "-A"]); + git(&fork, &["update-index", "--add", "--cacheinfo", &format!("160000,{LLVM_A},{}", crate::llvm::LLVM)]); + // What an uninitialised submodule leaves, so `commit -a` keeps the gitlink. + fs::create_dir_all(fork.join(crate::llvm::LLVM)).unwrap(); + git(&fork, &["commit", "-qm", "C0"]); + let c0 = git(&fork, &["rev-parse", "HEAD"]); + let mut pins = Vec::new(); + for spec in ["pub fn targets() { aarch64() }\n", "pub fn targets() { riscv() }\n"] { + git(&fork, &["checkout", "-q", &c0]); + write(&fork.join("compiler/rustc_target/src/lib.rs"), spec); + git(&fork, &["commit", "-qam", "a target"]); + pins.push(git(&fork, &["rev-parse", "HEAD"])); + } + git(&fork, &["checkout", "-q", &c0]); + + let primary = base.join("primary"); + fs::create_dir_all(&primary).unwrap(); + git(&primary, &["init", "-q"]); + for tree in ABI_TREES { + write(&primary.join(tree).join("src/lib.rs"), "pub struct A;\n"); + } + write(&primary.join("userland/libc/Cargo.lock"), "# libc's lock\n"); + write(&primary.join(".gitignore"), "target/\n"); + git(&primary, &["submodule", "add", "-q", fork.to_str().unwrap(), "rust"]); + git(&primary, &["add", "-A"]); + git(&primary, &["commit", "-qm", "pins C0"]); + let rust_dir = primary.join("rust"); + git(&rust_dir, &["submodule", "update", "-q", "--init", "library/backtrace"]); + + let mut linked = Vec::new(); + for (name, pin) in [("same", c0.as_str()), ("a", pins[0].as_str()), ("b", pins[1].as_str())] { + let wt = base.join(name); + git(&primary, &["worktree", "add", "-q", "-b", name, wt.to_str().unwrap()]); + let _ = fs::remove_dir(wt.join("rust")); + git(&rust_dir, &["worktree", "add", "-q", "--detach", wt.join("rust").to_str().unwrap(), pin]); + linked.push(wt); + } + let [same, a, b]: [PathBuf; 3] = linked.try_into().unwrap(); + Estate { primary, rust_dir, same, a, b, scratch } + } + + /// Check `fork`'s submodule `path` out at its gitlink, stage a newer commit + /// of it, and leave the checkout where it was: a staged bump, under which + /// bootstrap leaves the submodule at `HEAD`'s gitlink. Returns the commit + /// checked out and the one staged. + pub(crate) fn behind_a_staged_gitlink(fork: &Path, path: &str) -> (String, String) { + let submodule = fork.join(path); + git(fork, &["submodule", "update", "-q", "--init", path]); + let head = git(&submodule, &["rev-parse", "HEAD"]); + write(&submodule.join("newer.rs"), "pub fn newer() {}\n"); + git(&submodule, &["add", "newer.rs"]); + git(&submodule, &["commit", "-qm", "a newer commit"]); + let staged = git(&submodule, &["rev-parse", "HEAD"]); + git(fork, &["add", path]); + git(&submodule, &["checkout", "-q", "--detach", &head]); + (head, staged) + } + + /// What `f` panicked with; `expect` if it returned. + pub(crate) fn refusal(expect: &str, f: impl FnOnce()) -> String { + let refused = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).expect_err(expect); + refused.downcast_ref::().cloned().unwrap_or_default() + } + + /// The key most tests make, use or collect. + const K: &str = "0123456789abcdef"; + + /// A key of its own for `what`. + fn key_for(what: &str) -> String { + key(what, &[]) + } + + /// Make a stand-in product, one file saying who made it, and publish it as + /// `key` in `store`. + fn placed(store: &Path, key: &str, by: &str) -> bool { + let made = store.join(format!("{by}.made")); + write(&made.join("made-by"), by); + publish(&made, &Lock::exclusive(&made, "its maker"), &store.join(key), remove) + } + + /// **Concurrent makers of one key make it once**: every other one waits for + /// the maker and uses what it placed. + #[test] + fn concurrent_makers_of_one_key_make_it_once() { + let e = estate("store-concurrent"); + let makes = std::sync::atomic::AtomicUsize::new(0); + let dirs: Vec = std::thread::scope(|s| { + let handles: Vec<_> = (0..6) + .map(|_| { + s.spawn(|| { + let held = get(&e.same, &e.rust_dir, Kind::Sysroot, K, |dir| { + makes.fetch_add(1, Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(200)); + write(&dir.join("made-by"), "a maker"); + }); + held.dir + }) + }) + .collect(); + handles.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert_eq!(makes.load(Ordering::SeqCst), 1, "one key was made more than once"); + assert!(dirs.iter().all(|d| *d == Kind::Sysroot.dir(&e.rust_dir).join(K))); + assert_eq!(entries(&Kind::Sysroot.dir(&e.rust_dir)), [K]); + assert!(entries(&claims(&Kind::Sysroot.dir(&e.rust_dir))).is_empty(), "a claim outlived its placement"); + } + + /// **The loser of a race to place a key discards its copy**, and what the + /// winner placed is what stays. + #[test] + fn the_loser_of_a_placement_discards_its_copy() { + let e = estate("store-loser"); + let store = Kind::Sysroot.dir(&e.rust_dir); + assert!(placed(&store, K, "the first")); + assert!(!placed(&store, K, "the second"), "a second placement of one key won"); + assert_eq!(fs::read_to_string(store.join(K).join("made-by")).unwrap(), "the first"); + assert_eq!(entries(&store), [K], "the loser's copy stayed"); + let written = fs::OpenOptions::new().write(true).open(store.join(K).join("made-by")); + assert_eq!(written.map_err(|e| e.kind()).err(), Some(ErrorKind::PermissionDenied), "a placed product can be written"); + let added = fs::write(store.join(K).join("new"), "x"); + assert_eq!(added.map_err(|e| e.kind()).err(), Some(ErrorKind::PermissionDenied), "a placed key's own directory can be written"); + } + + /// **The loser of a placement takes its claim away before removing it**: a + /// build that claims the key in the gap gets the name, and never a claim + /// that is being removed under it. + #[test] + fn a_losing_claim_is_taken_away_before_it_is_removed() { + let e = estate("store-lost"); + let store = Kind::Sysroot.dir(&e.rust_dir); + assert!(placed(&store, K, "the first")); + let Claim::Mine(making, lock) = claim(&store, K, &Lock::try_exclusive) else { panic!("a free claim was not taken") }; + write(&making.join("made-by"), "the second"); + let taken = RefCell::new(None); + let won = publish(&making, &lock, &store.join(K), |gone| { + taken.replace(Some(claim(&store, K, &Lock::try_exclusive))); + remove(gone); + }); + assert!(!won, "a second placement of one key won"); + let taken = taken.into_inner().expect("the removal was never asked"); + assert!(matches!(taken, Claim::Mine(..)), "a build claiming the key while the loser's claim went did not get it"); + } + + /// **A product holding a link out of itself is never placed**, and one whose + /// links stay inside it is. + #[test] + fn a_product_linking_out_of_itself_is_refused() { + let e = estate("store-links"); + let store = Kind::Compiler.dir(&e.rust_dir); + let made = store.join("inside.made"); + write(&made.join("lib/rustlib/bin/rust-lld"), "lld"); + std::os::unix::fs::symlink("rust-lld", made.join("lib/rustlib/bin/ld.lld")).unwrap(); + std::os::unix::fs::symlink("../bin", made.join("lib/rustlib/up")).unwrap(); + assert!(publish(&made, &Lock::exclusive(&made, "its maker"), &store.join("inside"), remove)); + for (name, target) in [("absolute", e.primary.join("rust")), ("escaping", PathBuf::from("../../../elsewhere"))] { + let made = store.join(format!("{name}.made")); + write(&made.join("lib/rustlib/x"), "x"); + std::os::unix::fs::symlink(&target, made.join("lib/rustlib/src")).unwrap(); + let said = refusal("a product linking out of itself was placed", || { + publish(&made, &Lock::exclusive(&made, "its maker"), &store.join(name), remove); + }); + assert!(said.contains("links that leave it"), "{said}"); + assert!(!store.join(name).exists()); + } + } + + const ROOT: &str = "TOYOS_STORE_TEST_ROOT"; + const RUST_DIR: &str = "TOYOS_STORE_TEST_RUST_DIR"; + + #[test] + #[ignore = "the maker the test below kills; never runs on its own"] + fn a_maker_that_never_finishes() { + let root = PathBuf::from(std::env::var(ROOT).unwrap_or_else(|_| panic!("run without {ROOT}; it is not a test"))); + let rust_dir = PathBuf::from(std::env::var(RUST_DIR).unwrap()); + get(&root, &rust_dir, Kind::Compiler, K, |dir| { + write(&dir.join("half"), "half of it"); + held_until_killed(); + }); + } + + /// **A maker killed halfway leaves nothing under the key**: while it runs + /// its partial is held and nobody makes the key beside it; once it is dead + /// the next build makes the key whole, and a collection takes the partial. + #[test] + fn a_killed_maker_leaves_no_half_product() { + let e = estate("store-killed"); + let store = Kind::Compiler.dir(&e.rust_dir); + let env = [(ROOT, e.same.as_os_str()), (RUST_DIR, e.rust_dir.as_os_str())]; + let maker = Elsewhere::hold("store::tests::a_maker_that_never_finishes", &env); + assert!(!store.join(K).exists(), "a product was visible under its key before it was whole"); + assert!(Lock::try_exclusive(&claims(&store).join(K)).is_none(), "a key being made is not held"); + maker.kill(); + assert!(!store.join(K).exists(), "a killed maker left its half under the key"); + assert!(Lock::try_exclusive(&claims(&store).join(K)).is_some(), "a dead maker's claim is still held"); + + let made = Cell::new(0); + let held = get(&e.same, &e.rust_dir, Kind::Compiler, K, |dir| { + made.set(made.get() + 1); + write(&dir.join("whole"), "all of it"); + }); + assert_eq!(made.get(), 1); + assert!(held.dir.join("whole").is_file() && !held.dir.join("half").exists()); + assert_eq!(entries(&store), [K]); + assert!(entries(&claims(&store)).is_empty(), "a dead maker's claim outlived the key's making"); + } + + /// **A collection keeps what a registered worktree records, what `CURRENT` + /// names and what somebody holds**, and takes every other key, read-only + /// or not, and whatever a dead maker or a stopped collection left. + #[test] + fn a_collection_keeps_what_is_named_held_or_current() { + let e = estate("store-collect"); + let store = Kind::Sysroot.dir(&e.rust_dir); + let [named_primary, named_linked, held, pointed, orphan] = ["named-primary", "named-linked", "held", "current", "orphan"].map(key_for); + for key in [&named_primary, &named_linked, &held, &pointed, &orphan] { + placed(&store, key, key); + } + let claims = claims(&store); + let leftovers = [key_for("j"), format!("{K}.2000000000-0.partial"), format!("{orphan}.2000000000-1.gone")]; + for leftover in &leftovers { + write(&claims.join(leftover).join("x"), "left"); + } + // A maker that is running, between making its partial and holding it. + let making = format!("{K}.{}-9.partial", std::process::id()); + write(&claims.join(&making).join("x"), "being made"); + record(&e.primary, Kind::Sysroot, &named_primary); + record(&e.a, Kind::Sysroot, &named_linked); + std::os::unix::fs::symlink(format!("sysroots/{pointed}"), current(&e.rust_dir)).unwrap(); + let holding = Lock::shared(&store.join(&held), "a build using it"); + + let mut removed = collect(&e.primary, &e.rust_dir); + removed.sort(); + let mut gone = vec![store.join(&orphan)]; + gone.extend(leftovers.iter().map(|n| claims.join(n))); + gone.sort(); + assert_eq!(removed, gone); + let mut kept = vec![named_primary, named_linked.clone(), held.clone(), pointed]; + kept.sort(); + assert_eq!(entries(&store), kept); + assert_eq!(entries(&claims), [making], "a running maker's partial was taken"); + drop(holding); + assert_eq!(collect(&e.primary, &e.rust_dir), [store.join(&held)], "a key nobody names or holds stayed"); + + fs::remove_file(Kind::Sysroot.record(&e.a)).unwrap(); + fs::create_dir(Kind::Sysroot.record(&e.a)).unwrap(); + refusal("an unreadable record was read as naming nothing", || { + collect(&e.primary, &e.rust_dir); + }); + assert!(store.join(&named_linked).is_dir(), "an unreadable record's key was taken"); + } + + /// **A collection acts on the store's own names alone**: a file Finder + /// leaves, or any name that is no key, claim, partial or removal, stays + /// where it is, in the store or among its claims, and no collection stops + /// at it. + #[test] + fn a_collection_leaves_every_name_not_the_store_s() { + let e = estate("store-strays"); + let store = Kind::Sysroot.dir(&e.rust_dir); + placed(&store, K, "a key nobody names"); + let claims = claims(&store); + let strays = [ + store.join(".DS_Store"), + store.join("notes"), + store.join(format!("{K}.partial")), + claims.join(".DS_Store"), + claims.join("k"), + claims.join(format!("{K}.notes")), + claims.join(format!("{K}.2000000000-0.partial.old")), + ]; + for stray in &strays { + write(stray, "none of the store's"); + } + assert_eq!(collect(&e.primary, &e.rust_dir), [store.join(K)]); + assert_eq!(collect(&e.primary, &e.rust_dir), Vec::::new()); + let left: Vec<&PathBuf> = strays.iter().filter(|s| !s.is_file()).collect(); + assert!(left.is_empty(), "a collection took {left:?}"); + } + + /// **A name that is no key is refused before the store is touched**: an + /// empty one would name the store itself. + #[test] + fn a_name_that_is_no_key_is_refused() { + let e = estate("store-no-key"); + let store = Kind::Sysroot.dir(&e.rust_dir); + placed(&store, K, "a key"); + for name in ["", "../compilers", "0123456789ABCDEF", "0123456789abcdef0"] { + let said = refusal("a name that is no key was used as one", || { + get(&e.same, &e.rust_dir, Kind::Sysroot, name, |_| panic!("made {name:?}")); + }); + assert!(said.contains("is no key"), "{said}"); + } + assert_eq!(recorded(&e.same, Kind::Sysroot), None, "a name that is no key was recorded"); + assert!(Lock::try_exclusive(&store).is_some(), "the store is held"); + } + + /// **A lock granted on a key a collection renamed away holds nothing**: a + /// build that opened the key before the rename and was granted its lock + /// after it gets the key placed since, or nothing, and never what is being + /// removed. + #[test] + fn a_lock_on_a_key_renamed_away_is_not_the_key() { + let e = estate("store-renamed"); + let store = Kind::Sysroot.dir(&e.rust_dir); + placed(&store, K, "the first"); + let dir = store.join(K); + let opened_before = Lock::shared(&dir, "a build using it"); + set_writable(&dir, true); + fs::rename(&dir, store.join(format!("{K}.away"))).unwrap(); + placed(&store, K, "the second"); + assert!(named(&dir, opened_before).is_none(), "a lock on the key renamed away was taken for the key"); + named(&dir, Lock::shared(&dir, "a build using it")).expect("the key placed since"); + assert_eq!(fs::read_to_string(dir.join("made-by")).unwrap(), "the second"); + } + + /// **A collection that is stopped leaves nothing at a key's name**: what it + /// removes is out of the way before anything in it goes. + #[test] + fn a_stopped_collection_leaves_nothing_at_its_name() { + let e = estate("store-stopped"); + let store = Kind::Sysroot.dir(&e.rust_dir); + placed(&store, K, "a key nobody names"); + let stopped = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + collect_by(&e.primary, &e.rust_dir, &Lock::try_exclusive, |_| panic!("stopped")) + })); + assert!(stopped.is_err(), "the stand-in removal was never asked"); + assert!(!store.join(K).exists(), "a stopped collection left the key it was removing at its name"); + assert!(entries(&store).is_empty()); + } + + /// **A claim taken afresh in the gap before a collection removes a dead + /// one survives it**: what the collection removes is the dead claim, never + /// the name a maker has taken back. + #[test] + fn a_claim_taken_back_before_a_removal_survives_it() { + let e = estate("store-gap"); + let store = Kind::Compiler.dir(&e.rust_dir); + write(&claims(&store).join(K).join(MAKER), "a maker that died"); + let taken = RefCell::new(Vec::new()); + collect_by(&e.primary, &e.rust_dir, &Lock::try_exclusive, |gone| { + // Another build, in the gap: it takes the dead claim away and + // claims the key afresh. + if taken.borrow().is_empty() { + taken.borrow_mut().push(claim(&store, K, &Lock::try_exclusive)); + } + remove(gone); + }); + let claim = taken.into_inner().pop().expect("the removal was never asked"); + assert!(matches!(claim, Claim::Mine(..)), "the fresh maker did not get the claim"); + assert!(claims(&store).join(K).is_dir(), "the collection removed the claim a live maker took back"); + assert!(Lock::try_exclusive(&claims(&store).join(K)).is_none(), "the claim at the key's name is not the live maker's"); + } + + /// Take `path` as [`Lock::try_exclusive`] does, and let another build claim + /// `K` afresh between the open and the lock: it takes the dead claim + /// `path` names away first. What that build got is left in `taken`. + fn claimed_between(path: &Path, store: &Path, taken: &RefCell>) -> Option { + let opened = fs::File::open(path).unwrap(); + taken.replace(Some(claim(store, K, &Lock::try_exclusive))); + crate::dirlock::tests::try_exclusive_opened(opened) + } + + /// Whether the claim at `K`'s name in `store` is the one `taken` holds. + fn still_held(store: &Path, taken: RefCell>) -> bool { + let live = taken.into_inner().expect("the lock was never taken"); + matches!(live, Claim::Mine(..)) && claims(store).join(K).is_dir() && Lock::try_exclusive(&claims(store).join(K)).is_none() + } + + /// **A collection takes a claim only through the name it renames**: one + /// that opened a dead claim, and was granted its lock after another build + /// took that claim away and claimed the key afresh, holds nothing at the + /// name and leaves the live claim there. + #[test] + fn a_collection_locked_after_a_claim_was_taken_back_leaves_it() { + let e = estate("store-open-lock"); + let store = Kind::Compiler.dir(&e.rust_dir); + write(&claims(&store).join(K).join(MAKER), "a maker that died"); + let taken = RefCell::new(None); + collect_by(&e.primary, &e.rust_dir, &|path: &Path| claimed_between(path, &store, &taken), remove); + assert!(still_held(&store, taken), "the collection took the claim a live maker holds"); + } + + /// **A claim takes a dead claim only through the name it renames**, as a + /// collection does, and waits for the live one it finds there instead. + #[test] + fn a_claim_locked_after_a_claim_was_taken_back_leaves_it() { + let e = estate("store-open-claim"); + let store = Kind::Compiler.dir(&e.rust_dir); + write(&claims(&store).join(K).join(MAKER), "a maker that died"); + let taken = RefCell::new(None); + let mine = claim(&store, K, &|path: &Path| claimed_between(path, &store, &taken)); + assert!(matches!(mine, Claim::Theirs(_)), "a claim took the name another build holds"); + assert!(still_held(&store, taken), "a claim took the claim a live maker holds"); + } + + /// **A key recorded while a collection runs is kept by every decision made + /// after the record**: the collection asks for the records of each kind + /// when it decides that kind, not once before it starts. + #[test] + fn a_key_recorded_while_a_collection_runs_is_kept() { + let e = estate("store-late"); + let late = key_for("late"); + placed(&Kind::Llvm.dir(&e.rust_dir), &key_for("first"), "an LLVM nobody names"); + placed(&Kind::Compiler.dir(&e.rust_dir), &late, "a compiler recorded while the collection runs"); + collect_by(&e.primary, &e.rust_dir, &Lock::try_exclusive, |gone| { + record(&e.same, Kind::Compiler, &late); + remove(gone); + }); + assert!(Kind::Compiler.dir(&e.rust_dir).join(&late).is_dir(), "a key recorded before its kind was decided was taken"); + } + + /// **A key is the recipe and the trees, byte for byte, as they stand**: a + /// comment is another key, so is an untracked file, and an ignored one or + /// an edit put back is not; a submodule is its gitlink and not its + /// checkout; and asking writes nothing to the checkout's index. + #[test] + fn a_key_is_the_recipe_and_the_trees_as_they_stand() { + let e = estate("store-key"); + let fork = e.same.join("rust"); + let k = || key("recipe", &[Sources::of(&e.same, &fork).get("toyos-abi"), Sources::of(&e.same, &fork).get("library")]); + let index = fs::read(e.primary.join(".git/worktrees/same/index")).unwrap(); + let base = k(); + assert_eq!(base.len(), 16); + assert_ne!(key("another recipe", &[]), key("recipe", &[])); + assert_ne!(key("recipe", &["ab", "c"]), key("recipe", &["a", "bc"]), "parts that differ only in where they split are one key"); + + let abi = e.same.join("toyos-abi/src/lib.rs"); + write(&abi, "/// A comment.\npub struct A;\n"); + assert_ne!(k(), base, "a comment kept the key"); + write(&abi, "pub struct A;\n"); + assert_eq!(k(), base, "an edit put back is another key"); + write(&e.same.join("toyos-abi/src/new.rs"), "pub struct B;\n"); + assert_ne!(k(), base, "an untracked file kept the key"); + fs::remove_file(e.same.join("toyos-abi/src/new.rs")).unwrap(); + write(&fork.join("library/std/src/lib.rs"), "pub fn b() {}\n"); + assert_ne!(k(), base, "a fork edit kept the key"); + git(&fork, &["checkout", "-q", "--", "library"]); + write(&fork.join("build/out"), "ignored"); + assert_eq!(k(), base, "an ignored file moved the key"); + assert_eq!(fs::read(e.primary.join(".git/worktrees/same/index")).unwrap(), index, "asking wrote the index"); + let lock = || Sources::of(&e.same, &fork).get("Cargo.lock").to_string(); + let committed = lock(); + write(&fork.join("Cargo.lock"), "# re-locked by a bootstrap that is running\n"); + assert_eq!(lock(), committed, "a lockfile a running bootstrap rewrote moved the key"); + git(&fork, &["add", "Cargo.lock"]); + assert_ne!(lock(), committed, "a staged lockfile kept the key"); + git(&fork, &["reset", "-q", "--hard"]); + + let llvm = || Sources::of(&e.same, &fork).get(crate::llvm::LLVM).to_string(); + assert_eq!(llvm(), LLVM_A); + let checkout = fork.join(crate::llvm::LLVM); + git(&checkout, &["init", "-q"]); + write(&checkout.join("f"), "x"); + git(&checkout, &["add", "-A"]); + git(&checkout, &["commit", "-qm", "a checkout elsewhere"]); + assert_eq!(llvm(), LLVM_A, "a submodule was keyed by its checkout, not its gitlink"); + git(&fork, &["update-index", "--cacheinfo", &format!("160000,{LLVM_B},{}", crate::llvm::LLVM)]); + assert_eq!(llvm(), LLVM_B, "a staged gitlink is not what bootstrap checks out"); + } + + /// **A submodule checked out holding changes no commit does is refused**, + /// an edit or an untracked file: bootstrap builds them, and the gitlink the + /// key names does not name them. + #[test] + fn a_submodule_holding_uncommitted_changes_is_refused() { + let e = estate("store-submodule-edit"); + let library = || Sources::of(&e.primary, &e.rust_dir).get("library").to_string(); + let clean = library(); + let backtrace = e.rust_dir.join("library/backtrace"); + for (file, text) in [("lib.rs", "pub fn trace() { edited() }\n"), ("new.rs", "pub fn new() {}\n")] { + let before = fs::read_to_string(backtrace.join(file)).ok(); + write(&backtrace.join(file), text); + let said = refusal("a change in a checked-out submodule was keyed as its gitlink", || { + library(); + }); + let named = format!("{} holds changes no commit does", backtrace.display()); + assert!(said.contains(&named) && said.contains(file), "{said}"); + match before { + Some(text) => write(&backtrace.join(file), &text), + None => fs::remove_file(backtrace.join(file)).unwrap(), + } + } + assert_eq!(library(), clean); + } + + /// **A build from a submodule's directory holding files and no checkout is + /// refused**: no gitlink names what bootstrap built from them. + #[test] + fn a_build_from_a_submodule_holding_files_and_no_checkout_is_refused() { + let e = estate("store-no-checkout"); + let fork = e.a.join("rust"); + let backtrace = fork.join("library/backtrace"); + assert_built_at_gitlinks(&fork, &["library"], "a sysroot"); + write(&backtrace.join("lib.rs"), "pub fn trace() {}\n"); + let said = refusal("a build from files no gitlink names was kept", || { + assert_built_at_gitlinks(&fork, &["library"], "a sysroot"); + }); + assert!(said.contains(&format!("{} holds files and is no checkout of its gitlink", backtrace.display())), "{said}"); + } + + /// **A maker lets go of the key it placed before it collects**: a build + /// waiting for that key takes it while the collection that follows is held + /// back. + #[test] + fn a_placed_key_is_free_while_its_maker_collects() { + use std::time::{Duration, Instant}; + let e = estate("store-free"); + let key = Kind::Sysroot.dir(&e.rust_dir).join(K); + let deciding = Lock::shared(&store(&e.rust_dir, Kind::Llvm), "holding the collection back"); + let free = std::thread::scope(|s| { + let maker = s.spawn(|| get(&e.same, &e.rust_dir, Kind::Sysroot, K, |dir| write(&dir.join("made-by"), "the maker"))); + let deadline = Instant::now() + Duration::from_secs(20); + let free = loop { + if Lock::try_exclusive(&key).is_some() { + break true; + } + if Instant::now() >= deadline { + break false; + } + std::thread::sleep(Duration::from_millis(5)); + }; + drop(deciding); + maker.join().unwrap(); + free + }); + assert!(free, "the key its maker placed stayed held while it collected, 20 s"); + } +} diff --git a/src/sync.rs b/src/sync.rs index d1f26653f6d..c1de416c6b7 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -55,16 +55,13 @@ fn sync(root: &Path) -> Result { let behind = git(&primary, &["rev-list", "--count", "main..origin/main"])?; if behind.trim() == "0" { let at = git(&primary, &["rev-parse", "--short", "main"])?; - return Ok(format!( - "fetched origin; this host's main is current at {at}{}", - reclaimable(root) - )); + return Ok(format!("fetched origin; this host's main is current at {at}")); } let said = match fast_forward(&primary)? { Some((before, after, commits)) => format!("{before} -> {after} ({commits} commit(s))"), None => format!("already at {}", git(&primary, &["rev-parse", "--short", "main"])?), }; - Ok(format!("fetched origin; this host's main {said}{}", reclaimable(root))) + Ok(format!("fetched origin; this host's main {said}")) } /// The move this call made, or `None` when a concurrent `--sync` had already @@ -82,15 +79,6 @@ fn fast_forward(primary: &Path) -> Result, Stri Ok(Some((short(&before)?, short(&after)?, commits))) } -/// What this host could give back, said where it becomes true. -/// -/// A worktree whose branch has landed has no reason to hold its build caches, -/// and `--sync` runs at exactly the moment that becomes true of one. -fn reclaimable(root: &Path) -> String { - crate::worktree::reclaim_line(&crate::worktree::survey(root, false)) - .map_or_else(String::new, |line| format!("\n[sync] {line}")) -} - /// This host's `main` has commits GitHub does not, so it is not a cache of /// `origin/main` any more and nothing can fast-forward it. fn stranded(primary: &Path) -> String { diff --git a/src/sysroot.rs b/src/sysroot.rs index 684ae2e9cbd..a40cd13c69f 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -1,84 +1,46 @@ -//! Content-addressed sysroots: one per source identity, made by whichever -//! worktree first needs it, and shared by every worktree whose sources match. +//! Sysroots: a product of the store (`src/store.rs`), one per key, made by +//! whichever checkout first needs it and shared by every checkout whose +//! sources match. //! -//! **A sysroot is a function of its key.** The key ([`key`]) is the identity -//! (`src/identity.rs`, so a comment is no change) of everything a sysroot is -//! built from: the three trees std and `libtoyos_c.a` compile -//! ([`SYSROOT_SOURCES`]), the std fork's `library/` and `src/bootstrap/` in the -//! checkout that builds it, and the compiler that builds it. `rust/build/ -//! sysroots//` is a whole toolchain — the compiler's files cloned from its -//! `stage2`, the guest targets' libraries built from this key's sources. A build -//! compiles against the directory its own key names, so two worktrees with -//! different ABIs or different compilers never refuse or wait for each other, -//! and main and every branch matching it share one copy. +//! **A sysroot is a function of its key** ([`key`]): [`RECIPE`], the key of the +//! compiler that builds it, the std fork's `library/` and `src/bootstrap/`, and +//! the three trees std and `libtoyos_c.a` compile, `toyos-abi`, `toyos` and +//! `userland/libc`. `sysroots//` is a whole toolchain — the compiler's +//! files cloned from its `stage2`, the guest targets' libraries built from this +//! key's sources — and a build names it as `RUSTUP_TOOLCHAIN`. //! -//! **Each worktree builds std in its own fork checkout, and nothing but the -//! primary's own sync moves the primary's.** The primary builds in its `rust/`; -//! a linked worktree in its own `rust/`, made on first need as a git worktree of -//! the primary's fork repository at the commit this tree pins ([`fork_checkout`]). -//! `library/std` names `toyos-abi` and `toyos` as `../../../`, so each -//! checkout's std compiles against its own worktree's ABI with nothing -//! rewritten. The build is bootstrap's stage-0 local rebuild: the compiler the -//! checkout names (`src/compiler.rs` — the primary's `stage2`, or one of the -//! worktree's own where its `compiler/` differs) compiles the checkout's -//! `library/` for the guest targets into `/build/toyos-std/`. -//! -//! Locks, in the one order every acquirer takes them: the compiler key's, if the -//! compiler is a worktree's own; the sysroot key's (`buildlock::keyed_*`), with -//! this worktree's build lock put down; then, to build, this worktree's -//! exclusively (its fork build directory is written); then, if the compiler is -//! the primary's, the global one shared, because it is read. -//! -//! A sysroot no worktree names any more is removed by `keystore::sweep`, which -//! `--worktree remove` runs: each build records the key it used in its -//! worktree's `target/`, and a key no registered worktree records, that nobody -//! is making or using, goes. +//! **The fork a checkout's toolchain is built from is a [`Fork`]**: the +//! primary's `rust/`, or a linked worktree's own `rust/` while it holds work +//! the pin does not, keyed as it stands and built where it is; for every other +//! linked worktree, the commit its tree pins, keyed from the primary's objects +//! and built in the host's one shared checkout, [`SHARED`], which such builds +//! hold one at a time. Bootstrap's stage-0 local rebuild compiles a checkout's +//! `library/` for the guest targets into its `build/toyos-std/`; `library/std` +//! names `toyos-abi` and `toyos` as `../../../`, so the checkout sits beside the +//! building worktree's trees, or beside links to them. -use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use sha2::{Digest, Sha256}; - use crate::arch::Arch; -use crate::buildlock::{self, Guard, Held, Keyed}; use crate::compiler::{self, Compiler}; -use crate::identity; +use crate::dirlock::Lock; +use crate::store::{self, Kind, Relocked, Sources, ABI_TREES}; use crate::toolchain::{self, host_triple, Owner, GUEST_TARGETS}; -/// The per-worktree sources that end up inside a sysroot: std links `toyos-abi` -/// and `toyos`, and `libtoyos_c.a` is `userland/libc`. -pub const SYSROOT_SOURCES: [&str; 4] = - ["toyos-abi/src", "toyos/src", "userland/libc/src", "userland/libc/include"]; - -/// Their manifests, whose features and versions decide the same build. -pub(crate) const SYSROOT_MANIFESTS: [&str; 3] = - ["toyos-abi/Cargo.toml", "toyos/Cargo.toml", "userland/libc/Cargo.toml"]; - -/// The file a finished sysroot carries last, naming what it was built from. -/// A directory without it is a build that did not finish. -const SOURCES: &str = "SOURCES"; - /// What changes how a key's sources become a sysroot and is none of them: the /// std build's recipe below. Moving it moves every key. const RECIPE: &str = "bootstrap stage-0 local rebuild, profile compiler, no LLVM, \ libtoyos_c merged, libraries from the stamp, linked by rust-lld, \ - a C sysroot of libc's staticlib and headers per target; 5"; - -/// Every sysroot on this host. -pub fn sysroots_dir(rust_dir: &Path) -> PathBuf { - rust_dir.join("build/sysroots") -} + a C sysroot of libc's staticlib and headers per target, \ + run by the compiler's own cargo; 7"; /// A sysroot a build compiles against, held in use for as long as this lives. pub struct Sysroot { /// A toolchain directory: `RUSTUP_TOOLCHAIN` names it. pub dir: PathBuf, - /// Whether its compiler is the primary's, which the ToyOS-hosted rustc is - /// built from. - pub primary_compiler: bool, - _using: Option, + _held: Option, } impl Sysroot { @@ -86,120 +48,29 @@ impl Sysroot { /// artifact's, which `toolchain::check_installed_toolchain` has matched to /// these sources. pub(crate) fn installed(stage2: PathBuf) -> Self { - Self { dir: stage2, primary_compiler: true, _using: None } + Self { dir: stage2, _held: None } } } -fn hex(digest: &[u8]) -> String { - digest.iter().map(|b| format!("{b:02x}")).collect() -} - -/// The first 16 hex digits of the SHA-256 of `data`. -pub(crate) fn short(data: &[u8]) -> String { - hex(&Sha256::digest(data))[..16].to_string() -} - -/// Every file under `dir` a build reads, sorted: no `target/` and no dotted -/// directory, which is where a checkout keeps what it did not write. -fn files_under(dir: &Path, out: &mut Vec) { - let Ok(entries) = fs::read_dir(dir) else { return }; - for entry in entries.flatten() { - let path = entry.path(); - let name = entry.file_name(); - let name = name.to_string_lossy(); - let Ok(meta) = fs::symlink_metadata(&path) else { continue }; - if meta.is_dir() { - if !name.starts_with('.') && name != "target" { - files_under(&path, out); - } - } else if meta.is_file() && name != ".git" { - out.push(path); - } - } -} - -/// One line per `.rs`, `.toml` and `.h` file of [`SYSROOT_SOURCES`] under -/// `root`, and per [`SYSROOT_MANIFESTS`] entry: its repository-relative path and -/// the hash of its identity. -/// -/// Also what a published toolchain records, so an installed one is matched to a -/// checkout by the same function (`src/release.rs`). +/// What a published toolchain records of the trees its std and libc compiled, +/// so an installed one is matched to a checkout by the same hashes the store +/// keys on (`src/release.rs`). pub fn witness(root: &Path) -> String { - let mut lines = Vec::new(); - for tree in SYSROOT_SOURCES { - let mut files = Vec::new(); - files_under(&root.join(tree), &mut files); - files.retain(|p| p.extension().is_some_and(|e| e == "rs" || e == "toml" || e == "h")); - files.sort(); - for path in files { - let data = fs::read(&path).unwrap_or_else(|e| panic!("witness {}: {e}", path.display())); - let rel = path.strip_prefix(root).unwrap_or(&path); - lines.push(format!("{}:{}", rel.display(), short(&identity::of(&path, &data)))); - } - } - for manifest in SYSROOT_MANIFESTS { - let path = root.join(manifest); - let data = fs::read(&path).unwrap_or_else(|e| panic!("witness {}: {e}", path.display())); - lines.push(format!("{manifest}:{}", short(&data))); - } - lines.join("\n") -} - -/// The identity of the source files under `paths` of the git checkout `base`, -/// as one hash. -/// -/// **Source as git sees it**: tracked files and untracked ones no ignore rule -/// covers, into every submodule checked out there — never what a build or the -/// desktop leaves beside them (bootstrap's `__pycache__`, Finder's -/// `.DS_Store`), which would make a key that moves while it is being built. -pub(crate) fn tree_identity(base: &Path, paths: &[&str]) -> String { - let mut files = Vec::new(); - source_files(base, paths, &mut files); - files.sort(); - let mut hasher = Sha256::new(); - for path in files { - let data = fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); - hasher.update(path.strip_prefix(base).unwrap_or(&path).to_string_lossy().as_bytes()); - hasher.update([0]); - hasher.update(&*identity::of(&path, &data)); - hasher.update([0]); - } - hex(&hasher.finalize())[..16].to_string() -} - -fn source_files(checkout: &Path, paths: &[&str], out: &mut Vec) { - let mut args = vec!["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--"]; - args.extend(paths); - let listed = git_bytes(checkout, &args); - let mut seen = BTreeSet::new(); - for entry in listed.split(|b| *b == 0).filter(|e| !e.is_empty()) { - let path = checkout.join(String::from_utf8_lossy(entry).as_ref()); - if !seen.insert(path.clone()) { - continue; - } - if path.join(".git").exists() { - source_files(&path, &["."], out); - } else if fs::symlink_metadata(&path).is_ok_and(|m| m.is_file()) { - out.push(path); - } - } + let hashes = store::trees(root, &ABI_TREES, Relocked::No); + ABI_TREES.iter().zip(hashes).map(|(tree, hash)| format!("{tree}:{hash}\n")).collect() } -/// The key of the sysroot `root` builds against with its std fork at `fork`, -/// compiled by `compiler`. -pub fn key(root: &Path, compiler: &Compiler, fork: &Path) -> String { - let parts = [ - format!("{RECIPE}; cargo {STAGE0_CARGO}; targets {}", GUEST_TARGETS.join(" ")), - witness(root), - tree_identity(fork, &["library", "src/bootstrap"]), - compiler.identity(), - ]; - short(parts.join("\n\0\n").as_bytes()) +/// The key of the sysroot the compiler `compiler` builds from `sources`. +pub fn key(compiler: &str, sources: &Sources) -> String { + let recipe = format!("{RECIPE}; targets {}", GUEST_TARGETS.join(" ")); + let trees = ["library", "src/bootstrap"].into_iter().chain(ABI_TREES).map(|tree| sources.get(tree)); + let parts: Vec<&str> = std::iter::once(compiler).chain(trees).collect(); + store::key(&recipe, &parts) } /// The commit this checkout's tree pins the std fork at: the index's, so a /// staged gitlink counts as the tree's. -fn pinned_fork(root: &Path) -> String { +pub(crate) fn pinned_fork(root: &Path) -> String { let entry = git_out(root, &["ls-files", "-s", "--", "rust"]); let mut words = entry.split_whitespace(); match (words.next(), words.next()) { @@ -208,180 +79,231 @@ fn pinned_fork(root: &Path) -> String { } } -/// The fork checkout `root`'s std is built in. -/// -/// The primary's is its own `rust/`. A linked worktree's `rust/` starts as the -/// empty stub `git worktree add` leaves; it is made here, the first time it is -/// needed, as a git worktree of the primary's fork repository at the commit -/// this tree pins, sharing its objects — and `library/backtrace` the same way -/// from the primary's, or by git's own clone where the primary does not hold -/// that commit. -/// -/// A checkout that exists is used as it stands, which is where an agent edits -/// the fork; one whose `HEAD` is neither the pinned commit nor ahead of it is -/// moved there itself, fetching the commit from the primary's repository first -/// if the checkout does not already hold it, unless the checkout has local -/// changes, in which case it is refused by name rather than moved out from -/// under whoever made them. -pub fn fork_checkout(root: &Path) -> PathBuf { - let fork = root.join("rust"); - let primary = match toolchain::owner(root) { - Owner::Us => return fork, - Owner::Installed => panic!("an installed toolchain has no fork checkout to build std in"), - Owner::Elsewhere(primary) => primary, - }; - let pinned = pinned_fork(root); - if !fork.join(".git").exists() { - let stub = fs::read_dir(&fork).map_or(0, |d| d.count()); - assert!( - stub == 0, - "{} is neither a fork checkout nor the empty stub a worktree starts with", - fork.display() - ); - let _ = fs::remove_dir(&fork); - eprintln!("Making {} a fork checkout at {pinned} (a git worktree of the primary's)", fork.display()); - git_run(&primary.join("rust"), &["worktree", "add", "--detach", path_str(&fork), &pinned]); - let backtrace = git_out(&fork, &["ls-tree", "HEAD", "library/backtrace"]); - let commit = backtrace.split_whitespace().nth(2).unwrap_or_else(|| { - panic!("{} pins no library/backtrace: {backtrace:?}", fork.display()) - }); - let theirs = primary.join("rust/library/backtrace"); - let held = Command::new("git") - .args(["cat-file", "-e", &format!("{commit}^{{commit}}")]) - .current_dir(&theirs) - .status() - .is_ok_and(|s| s.success()); - let at = fork.join("library/backtrace"); - if held { - let _ = fs::remove_dir(&at); - git_run(&theirs, &["worktree", "add", "--detach", path_str(&at), commit]); +/// Where, in the primary's `rust/`, the host builds the toolchain of every +/// linked worktree whose `rust/` is the stub: one git worktree of the primary's +/// fork repository, sharing its objects, at `rust/`, and `toyos-abi` and +/// `toyos` beside it as links to the trees of the build that holds it. Its +/// submodules are clones in its own git directory and never checkouts of the +/// primary's: bootstrap moves a submodule with `git submodule update`, which +/// over a checkout of another clone rewrites that clone's `core.worktree`. +pub const SHARED: &str = "build/fork"; + +/// The rust fork a checkout's toolchain is built from. +pub enum Fork { + /// A fork checkout, keyed as it stands and built where it is: the + /// primary's `rust/`, or a linked worktree's own, which is where the fork + /// is edited. + Checkout(PathBuf), + /// The commit a linked worktree pins, in the fork repository at `rust_dir`, + /// the primary's: keyed from its objects and built in [`SHARED`]. + Pinned { rust_dir: PathBuf, commit: String }, +} + +impl Fork { + /// The fork `root`'s toolchain is built from. + /// + /// A linked worktree's own checkout is built where it is only while it + /// holds work its pin does not, uncommitted or committed ahead of it; at + /// the pin and clean it is the pin. One behind its pin is moved there, + /// unless it holds uncommitted work, which is refused rather than moved out + /// from under whoever made it. The primary's is built as it stands, and + /// refused behind its pin, since nothing but its owner moves it. + pub fn of(root: &Path) -> Fork { + let pinned = pinned_fork(root); + let own = root.join("rust"); + let primary = match toolchain::owner(root) { + Owner::Installed => panic!("an installed toolchain has no fork to build from"), + Owner::Us => { + let head = head(&own); + assert!( + at_or_ahead(&own, &pinned, &head), + "{} is at {head}, and this tree pins the fork at {pinned}, which that is not at or \ + ahead of: a build here would make a toolchain this tree does not name. Move it \ + there: `git -C {} checkout --detach {pinned}`", + own.display(), + own.display(), + ); + return Fork::Checkout(own); + } + Owner::Elsewhere(primary) => primary, + }; + let shared = Fork::Pinned { rust_dir: primary.join("rust"), commit: pinned.clone() }; + if !own.join(".git").exists() { + return shared; + } + if !at_or_ahead(&own, &pinned, &head(&own)) { + let _held = Lock::exclusive(&own, &format!("{}, behind a build in it", own.display())); + let was = head(&own); + if !at_or_ahead(&own, &pinned, &was) { + let edits = work(&own); + assert!( + edits.is_empty(), + "{} is at {was} with uncommitted work, and this tree pins the fork at {pinned}, which \ + that is not at or ahead of: a build here would make a toolchain this tree does not \ + name, and moving the checkout would lose that work.\n{edits}", + own.display(), + ); + git_out(&own, &["checkout", "--detach", "-q", &pinned]); + eprintln!("{} was at {was}, not at or ahead of this tree's pin {pinned}: checked it out", own.display()); + } + } + if holds_work(&own, &pinned) { + Fork::Checkout(own) } else { - git_run(&fork, &["submodule", "update", "--init", "library/backtrace"]); + shared } - return fork; } - let head = git_out(&fork, &["rev-parse", "HEAD"]); - let head = head.trim(); - let ahead = Command::new("git") - .args(["merge-base", "--is-ancestor", &pinned, head]) - .current_dir(&fork) - .status() - .is_ok_and(|s| s.success()); - if head == pinned || ahead { - return fork; + + /// `root`'s sources: its ABI trees as they stand, and this fork's trees. + pub fn sources(&self, root: &Path) -> Sources { + match self { + Fork::Checkout(dir) => Sources::of(root, dir), + Fork::Pinned { rust_dir, commit } => Sources::pinned(root, rust_dir, commit), + } } - let dirty = git_out(&fork, &["status", "--porcelain", "--ignore-submodules=none"]); - assert!( - dirty.is_empty(), - "{} is at {head} with uncommitted work, and this tree pins the fork at {pinned}, which \ - that is not ahead of: a build here would compile a std this tree does not name, and \ - moving the checkout would lose that work.\n{dirty}", - fork.display(), - ); - let held = Command::new("git") - .args(["cat-file", "-e", &format!("{pinned}^{{commit}}")]) - .current_dir(&fork) - .status() - .is_ok_and(|s| s.success()); - if !held { - git_run(&fork, &["fetch", path_str(&primary.join("rust")), &pinned]); + + /// A checkout to build `root`'s toolchain in, held for it alone for as + /// long as the returned value lives: each build there empties the build + /// directory the one before it built in. One a submodule of which is a git + /// worktree of another clone is refused: bootstrap moves every submodule + /// checked out to its gitlink with `git submodule update`, which over such + /// a worktree rewrites that clone's `core.worktree`. In the host's shared + /// checkout, each submodule checked out is reset to its commit, with + /// nothing untracked. + pub fn checkout(&self, root: &Path) -> Checkout { + let checkout = match self { + Fork::Checkout(dir) => { + Checkout { _held: Lock::exclusive(dir, &format!("a toolchain build in {}", dir.display())), dir: dir.clone() } + } + Fork::Pinned { rust_dir, commit } => shared(rust_dir, root, commit), + }; + let gitlinks = store::gitlinks(&checkout.dir, &[], None); + for submodule in gitlinks.into_iter().map(|(path, _)| checkout.dir.join(path)).filter(|s| s.join(".git").exists()) { + let dirs = git_out(&submodule, &["rev-parse", "--path-format=absolute", "--git-dir", "--git-common-dir"]); + let (own, common) = dirs.trim().split_once('\n').expect("git names two directories"); + assert!( + own == common, + "{} is a git worktree of {common}, another clone, and bootstrap moves a submodule with \ + `git submodule update`, which over it rewrites that clone's `core.worktree`; nothing was \ + built. `git -C {common} worktree remove --force {}` takes it, and the next build clones \ + the submodule into {}'s own git directory", + submodule.display(), + submodule.display(), + checkout.dir.display(), + ); + if let Fork::Pinned { .. } = self { + git_out(&submodule, &["reset", "-q", "--hard"]); + git_out(&submodule, &["clean", "-dffxq"]); + } + } + checkout } - git_run(&fork, &["checkout", "--detach", "-q", &pinned]); - eprintln!("{} was at {head}, behind this tree's pin {pinned}: checked it out", fork.display()); - fork } -/// Why `dir` is not a finished sysroot, if it is not: no [`SOURCES`], or not a -/// whole toolchain (`toolchain::toolchain_defect`). One found with the first and -/// not the second is made again rather than trusted — all of it even when only -/// its `bin/cargo` link dangles, because that is rare and a sysroot has no -/// repair path. -fn unfinished(dir: &Path) -> Option { - if !dir.join(SOURCES).is_file() { - return Some(format!("{} carries no {SOURCES}", dir.display())); - } - toolchain::toolchain_defect(dir) +/// A fork checkout held by one build. +pub struct Checkout { + pub dir: PathBuf, + _held: Lock, } -/// The sysroot `key` names at `dir`, made by `make` if nobody has made it, and -/// held in use for as long as the returned guard lives. -fn held(root: &Path, key: &str, dir: &Path, make: impl FnMut()) -> Guard { - buildlock::keyed_made(root, Keyed::Sysroot, key, || unfinished(dir), make) +/// The commit the fork checkout `dir` is at. +fn head(dir: &Path) -> String { + git_out(dir, &["rev-parse", "HEAD"]).trim().to_string() } -/// The sysroot this worktree's sources name, made if nobody has made it, and -/// held in use for as long as the returned value lives. -pub fn ensure(root: &Path, rust_dir: &Path, lock: &mut Held) -> Sysroot { - let fork = fork_checkout(root); - let compiler = compiler::resolve(root, rust_dir, &fork, lock); - let key = key(root, &compiler, &fork); - let dir = sysroots_dir(rust_dir).join(&key); - crate::keystore::record(root, Keyed::Sysroot, &key); +/// Whether `head`, in the fork checkout `dir`, is `commit` or ahead of it. +fn at_or_ahead(dir: &Path, commit: &str, head: &str) -> bool { + git(dir, &["merge-base", "--is-ancestor", commit, head], None).is_ok() +} - let using = lock.without_shared(|| held(root, &key, &dir, || build(root, &compiler, &fork, &key, &dir))); - Sysroot { dir, primary_compiler: compiler.primary, _using: Some(using) } +/// Whether the fork checkout `own` holds work the commit `pinned` does not: +/// commits ahead of it, or [`work`]. +pub(crate) fn holds_work(own: &Path, pinned: &str) -> bool { + let head = head(own); + at_or_ahead(own, pinned, &head) && (head != pinned || !work(own).is_empty()) } -/// Make the sysroot `key` names at `dir`, from `root`'s sources and the std fork -/// at `fork`, with `compiler`. The caller holds the key's lock. -fn build(root: &Path, compiler: &Compiler, fork: &Path, key: &str, dir: &Path) { - let what = format!("building sysroot {key}"); - let _worktree = buildlock::worktree_exclusive(root, &what); - // Only the primary's compiler is rebuilt in place; one of a worktree's own - // is written once and held in use by `compiler`. - let _compiler = compiler.primary.then(|| buildlock::compiler_shared(root, &what)); - eprintln!("Building sysroot {key}: std from {}, the compiler {}", fork.display(), compiler.stage2.display()); +/// What `git status` says the fork checkout `own` holds that its commit does +/// not, less a submodule checked out at another commit than its gitlink and +/// no more: bootstrap moves that one to its gitlink, so the checkout builds as +/// its commit. +fn work(own: &Path) -> String { + let status = git_out(own, &["--no-optional-locks", "status", "--porcelain=v2", "--ignore-submodules=none"]); + status.lines().filter(|l| !l.starts_with("1 .M SC.. ")).map(|l| format!("{l}\n")).collect() +} - publish(compiler, dir, |partial| { - let built = build_std(root, compiler, fork); - for target in GUEST_TARGETS { - place_std(&stamp(&built, target), &partial.join("lib/rustlib").join(target).join("lib")); - } - let libc_target = dir.with_extension("libc-target"); - for arch in Arch::ALL { - crate::libc::build(root, partial, &libc_target, arch); - crate::libc::build_c(root, partial, &libc_target, arch); - } - let _ = fs::remove_dir_all(&libc_target); +/// [`SHARED`] in the fork repository at `rust_dir`, held for `root`, at +/// `commit`, beside links to `root`'s ABI trees. +fn shared(rust_dir: &Path, root: &Path, commit: &str) -> Checkout { + let base = rust_dir.join(SHARED); + fs::create_dir_all(&base).unwrap_or_else(|e| panic!("create {}: {e}", base.display())); + let held = Lock::exclusive(&base, &format!("{}, behind another worktree's toolchain build", base.display())); + let dir = base.join("rust"); + if !dir.join(".git").exists() { + eprintln!("Making {} a fork checkout (a git worktree of {})", dir.display(), rust_dir.display()); + remove(&dir); + git_out(rust_dir, &["worktree", "prune"]); + git_out(rust_dir, &["worktree", "add", "--detach", path_str(&dir), commit]); + } + git_out(&dir, &["checkout", "--detach", "--force", "-q", commit]); + git_out(&dir, &["clean", "-d", "--force", "-q"]); + for tree in ["toyos-abi", "toyos"] { + toolchain::swap_link(&root.join(tree), &base.join(tree)); + } + Checkout { dir, _held: held } +} - // The sources the key named are the ones built, or this is not that key's. - let again = self::key(root, compiler, fork); +/// The sysroot this worktree's sources name, made if nobody has made it, and +/// held in use for as long as the returned value lives. +pub fn ensure(root: &Path, rust_dir: &Path) -> Sysroot { + let fork = Fork::of(root); + let sources = fork.sources(root); + let compiler = compiler::resolve(root, rust_dir, &fork, &sources); + let key = key(&compiler.key, &sources); + let held = store::get(root, rust_dir, Kind::Sysroot, &key, |partial| { + let checkout = fork.checkout(root); + assemble(&compiler.stage2, &checkout.dir, partial, |partial| build(root, &compiler, &checkout.dir, partial)); + let again = self::key(&compiler.key, &Sources::of(root, &checkout.dir)); assert!( again == key, "the sources moved while sysroot {key} was being built (they are now {again}); \ nothing was kept, and the next build makes the one they name" ); - format!("{key}\nfork {}\n{}\n", fork.display(), witness(root)) }); + if let Some(defect) = toolchain::toolchain_defect(&held.dir) { + panic!("sysroot {key} at {} is not whole: {defect}", held.dir.display()); + } + Sysroot { dir: held.dir.clone(), _held: Some(held) } } -/// Put at `dir` a whole toolchain: `compiler`'s files and what `fill` adds to -/// them, then the [`SOURCES`] `fill` returns, last. A `dir` already there is one -/// [`unfinished`] refused, and it is replaced. A compiler that is not whole is -/// refused before `fill` runs, and nothing is published. -fn publish(compiler: &Compiler, dir: &Path, fill: impl FnOnce(&Path) -> String) { - if let Some(defect) = toolchain::toolchain_defect(&compiler.stage2) { - let fix = if compiler.primary { - "\nA bootstrap in the primary checkout was stopped before it finished: \ - `cargo run -- --build-only` there completes it." - } else { - "" - }; - panic!("no sysroot is made from {}, and no std was built for one: {defect}{fix}", compiler.stage2.display()); +/// Put in `partial` a whole toolchain: the compiler's files at `stage2` and +/// what `fill` builds from the fork checkout `fork` and adds to them. One that +/// is not whole, or built from a submodule its gitlink does not name, is +/// refused. +fn assemble(stage2: &Path, fork: &Path, partial: &Path, fill: impl FnOnce(&Path)) { + clone_tree(stage2, partial); + fill(partial); + store::assert_built_at_gitlinks(fork, &["library"], "a sysroot"); + if let Some(defect) = toolchain::toolchain_defect(partial) { + panic!("a sysroot was made from {}, and is not whole: {defect}", stage2.display()); } - let partial = dir.with_extension("partial"); - if partial.exists() { - fs::remove_dir_all(&partial).unwrap_or_else(|e| panic!("remove {}: {e}", partial.display())); +} + +/// Build the guest targets' libraries from `fork`'s `library/` and `root`'s libc +/// with `compiler`, into `partial`. +fn build(root: &Path, compiler: &Compiler, fork: &Path, partial: &Path) { + eprintln!("Building a sysroot: std from {}, the compiler {}", fork.display(), compiler.key); + let built = build_std(root, compiler, fork); + for target in GUEST_TARGETS { + place_std(&stamp(&built, target), &partial.join("lib/rustlib").join(target).join("lib")); } - clone_tree(&compiler.stage2, &partial); - let sources = fill(&partial); - fs::write(partial.join(SOURCES), sources) - .unwrap_or_else(|e| panic!("write {}: {e}", partial.join(SOURCES).display())); - if dir.exists() { - fs::remove_dir_all(dir).unwrap_or_else(|e| panic!("remove {}: {e}", dir.display())); + // Inside the product being made, which nothing else writes or collects. + let libc_target = partial.join(".libc-target"); + for arch in Arch::ALL { + crate::libc::build(root, partial, &libc_target, arch); + crate::libc::build_c(root, partial, &libc_target, arch); } - fs::rename(&partial, dir) - .unwrap_or_else(|e| panic!("rename {} -> {}: {e}", partial.display(), dir.display())); + fs::remove_dir_all(&libc_target).unwrap_or_else(|e| panic!("remove {}: {e}", libc_target.display())); } /// Compile the guest targets' libraries from `fork`'s `library/` with @@ -390,9 +312,9 @@ fn build_std(root: &Path, compiler: &Compiler, fork: &Path) -> PathBuf { crate::ensure_submodule(fork, "library/backtrace"); let host = host_triple(); let build_dir = fork.join("build/toyos-std"); - prepare_std_build(&build_dir, &host, &compiler.identity()); + prepare_std_build(&build_dir, &host, &compiler.key); let config = build_dir.join("bootstrap.toml"); - fs::write(&config, std_config(&compiler.stage2, &bootstrap_cargo(), &build_dir, &host)) + fs::write(&config, std_config(&compiler.stage2, &compiler.stage2.join("bin/cargo"), &build_dir, &host)) .unwrap_or_else(|e| panic!("write {}: {e}", config.display())); let targets = GUEST_TARGETS.join(","); @@ -407,13 +329,11 @@ fn build_std(root: &Path, compiler: &Compiler, fork: &Path) -> PathBuf { build_dir.join(&host).join("stage0-std") } -/// Ready the std build directory `build_dir` for a build by the compiler -/// `identity` names: nothing another compiler built, no LLVM, and no guest -/// target's std. -fn prepare_std_build(build_dir: &Path, host: &str, identity: &str) { +/// Ready the std build directory `build_dir` for a build by the compiler `key` +/// names: nothing another compiler built, and no guest target's std. +fn prepare_std_build(build_dir: &Path, host: &str, key: &str) { fs::create_dir_all(build_dir).unwrap_or_else(|e| panic!("create {}: {e}", build_dir.display())); - forget_another_compiler(build_dir, host, identity); - crate::llvm::retire_in_tree(build_dir); + forget_another_compiler(build_dir, host, key); // Bootstrap reuses what it built before and does not see a path dependency // outside the fork move, so each target's std starts from nothing. for target in GUEST_TARGETS { @@ -422,7 +342,7 @@ fn prepare_std_build(build_dir: &Path, host: &str, identity: &str) { } /// Empty the std build directory `build_dir` of all but what bootstrap -/// downloaded unless `identity` ([`Compiler::identity`]) is the compiler its +/// downloaded unless `identity`, a compiler's key, is the compiler its /// `compiled-by` records as having compiled the rest, then record `identity` /// there. /// @@ -519,6 +439,8 @@ fn place_std(stamp: &Path, lib: &Path) { /// a stage-0 build searches for tools decides nothing; and no rpath, which bootstrap /// spells as a C driver's `-Wl,` arguments that a linker run directly refuses. /// No LLVM: std builds none, and the profile's `download-ci-llvm` fetches one. +/// The cargo is the compiler's own: a local rebuild passes it the flags of the +/// fork's own version, which any other cargo may refuse. fn std_config(compiler: &Path, cargo: &Path, build_dir: &Path, host: &str) -> String { let targets = GUEST_TARGETS.iter().map(|t| format!("\"{t}\"")).collect::>().join(", "); let linker = toolchain::rust_lld(compiler); @@ -550,34 +472,9 @@ lld = false ) } -/// The rustup toolchain whose cargo runs bootstrap's stage-0 std build. -/// -/// A local rebuild passes cargo the flags of the fork's own version — the -/// fork's bootstrap spells `-Zembed-metadata=no` for it, which the fork's -/// stage-0 beta cargo refuses — so this is a nightly of the fork's version, and -/// it moves when an upstream merge moves that version: bootstrap refuses any -/// other by name (`Unexpected cargo version`). -const STAGE0_CARGO: &str = "nightly-2026-07-22"; - -/// [`STAGE0_CARGO`]'s cargo, installed through rustup the first time a sysroot -/// is built without it. -fn bootstrap_cargo() -> PathBuf { - let name = format!("{STAGE0_CARGO}-{}", host_triple()); - let cargo = toolchain::rustup_home().expect("a rustup home").join("toolchains").join(&name).join("bin/cargo"); - if !cargo.exists() { - eprintln!("Installing {STAGE0_CARGO}, whose cargo builds std for a sysroot..."); - let ok = Command::new("rustup") - .args(["toolchain", "install", STAGE0_CARGO, "--profile", "minimal"]) - .status() - .is_ok_and(|s| s.success()); - assert!(ok && cargo.exists(), "rustup could not install {STAGE0_CARGO}, so {} is missing", cargo.display()); - } - cargo -} - -/// Copy `from` to `to`, a symbolic link as a link: `stage2`'s own point at -/// things that outlive it. `fs::copy` clones on APFS and reflinks where Linux -/// can, so a sysroot costs the bytes its own libraries differ by. +/// Copy `from` to `to`, a symbolic link as a link. `fs::copy` clones on APFS +/// and reflinks where Linux can, so a sysroot costs the bytes its own libraries +/// differ by. pub(crate) fn clone_tree(from: &Path, to: &Path) { fs::create_dir_all(to).unwrap_or_else(|e| panic!("create {}: {e}", to.display())); for entry in fs::read_dir(from).unwrap_or_else(|e| panic!("read {}: {e}", from.display())).flatten() { @@ -601,13 +498,16 @@ fn path_str(path: &Path) -> &str { path.to_str().unwrap_or_else(|| panic!("{} is not UTF-8", path.display())) } -/// `Err` names the command, the directory and what git said. -fn git_try(dir: &Path, args: &[&str]) -> Result, String> { - let out = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .map_err(|e| format!("run git in {}: {e}", dir.display()))?; +/// What `git args` printed in `dir`, with `index` as its index if given: the +/// build system's one git runner. `Err` names the command, the directory and +/// what git said. +pub(crate) fn git(dir: &Path, args: &[&str], index: Option<&Path>) -> Result, String> { + let mut command = Command::new("git"); + command.args(args).current_dir(dir); + if let Some(index) = index { + command.env("GIT_INDEX_FILE", index); + } + let out = command.output().map_err(|e| format!("run git in {}: {e}", dir.display()))?; if !out.status.success() { let stderr = String::from_utf8_lossy(&out.stderr); return Err(format!("git {args:?} in {}: {}", dir.display(), stderr.trim())); @@ -616,7 +516,7 @@ fn git_try(dir: &Path, args: &[&str]) -> Result, String> { } pub(crate) fn git_bytes(dir: &Path, args: &[&str]) -> Vec { - git_try(dir, args).unwrap_or_else(|e| panic!("{e}")) + git(dir, args, None).unwrap_or_else(|e| panic!("{e}")) } pub(crate) fn git_out(dir: &Path, args: &[&str]) -> String { @@ -628,7 +528,7 @@ pub(crate) fn git_out(dir: &Path, args: &[&str]) -> String { /// name rather than rewritten into one git does not track. pub(crate) fn tracked_files(dir: &Path, pathspecs: &[&str]) -> Result, String> { let args = [&["ls-files", "-z", "--"][..], pathspecs].concat(); - let listing = git_try(dir, &args)?; + let listing = git(dir, &args, None)?; let names = listing.split(|b| *b == 0).filter(|f| !f.is_empty()); names .map(|f| { @@ -639,109 +539,35 @@ pub(crate) fn tracked_files(dir: &Path, pathspecs: &[&str]) -> Result String { - let out = Command::new("git") - .args(["-c", "commit.gpgsign=false", "-c", "user.email=t@t", "-c", "user.name=t"]) - .args(["-c", "protocol.file.allow=always", "-c", "init.defaultBranch=main"]) - .args(crate::gitfixture::NO_AUTO_MAINTENANCE) - .args(args) - .current_dir(dir) - .output() - .expect("run git"); - assert!(out.status.success(), "git {args:?} in {}: {}", dir.display(), String::from_utf8_lossy(&out.stderr)); - String::from_utf8(out.stdout).unwrap().trim().to_string() - } - - fn write(path: &Path, text: &str) { - fs::create_dir_all(path.parent().unwrap()).unwrap(); - fs::write(path, text).unwrap(); - } - - /// A worktree's three trees, a fork checkout and a compiler, laid out the - /// way the key reads them. - fn keyed(base: &Path) -> (PathBuf, PathBuf, PathBuf) { - let root = base.join("root"); - for tree in SYSROOT_SOURCES { - write(&root.join(tree).join("lib.rs"), "/// A.\npub struct A;\n"); - } - for manifest in SYSROOT_MANIFESTS { - write(&root.join(manifest), "[package]\nversion = \"0.1.0\"\n"); - } - let fork = base.join("fork"); - write(&fork.join("library/std/src/lib.rs"), "//! std\npub fn exit() {}\n"); - write(&fork.join("src/bootstrap/src/lib.rs"), "fn main() {}\n"); - write(&fork.join(".gitignore"), "__pycache__\n.DS_Store\n"); - git(&fork, &["init", "-q"]); - let rust_dir = base.join("rust"); - write(&rust_dir.join("build/toyos-compiler"), "tree-1"); - write(&toolchain::stage2(&rust_dir).join("lib/librustc_driver-1.dylib"), "a driver"); - (root, rust_dir, fork) - } - - /// **The key is the identity, and only the identity**: a comment in any tree - /// it reads — the ABI or the std fork — is the same sysroot, and a signature, - /// a line of the fork's code or another compiler is another. + /// **The key is the compiler's and the trees std and libc are built + /// from**: an ABI edit, a std edit and another compiler are each another + /// sysroot, and a compiler edit alone reaches it only through the + /// compiler's key. #[test] - fn a_comment_is_the_same_sysroot_and_a_signature_is_another() { - let base = TempDir::new("key"); - let (root, rust_dir, fork) = keyed(&base); - let k = || key(&root, &Compiler::primary(&rust_dir), &fork); - let base = k(); - assert_eq!(base.len(), 16, "{base}"); - - let abi = root.join("toyos-abi/src/lib.rs"); - write(&abi, "//! The crate.\n/// A, said better.\n// and a plain comment\npub struct A;\n"); - assert_eq!(k(), base, "a comment in toyos-abi made a new sysroot"); - write(&abi, "/// A.\npub struct A(pub u64);\n"); - assert_ne!(k(), base, "a signature change kept the old sysroot"); - write(&abi, "/// A.\npub struct A;\n"); - assert_eq!(k(), base); - - let header = root.join("userland/libc/include/stdio.h"); - write(&header, "int puts(const char *);\n"); - assert_ne!(k(), base, "a header the C sysroot carries kept the old sysroot"); - fs::remove_file(&header).unwrap(); - assert_eq!(k(), base); - - let std = fork.join("library/std/src/lib.rs"); - write(&std, "//! std, documented\npub fn exit() {}\n"); - assert_eq!(k(), base, "a comment in the std fork made a new sysroot"); - write(&std, "//! std\npub fn exit() { loop {} }\n"); - assert_ne!(k(), base, "a change to the fork's code kept the old sysroot"); - write(&std, "//! std\npub fn exit() {}\n"); - assert_eq!(k(), base); - - // What a build and the desktop leave in the checkout is not its source. - write(&fork.join("src/bootstrap/__pycache__/bootstrap.cpython-313.pyc"), "bytecode"); - write(&fork.join("library/.DS_Store"), "finder"); - assert_eq!(k(), base, "a file git ignores moved the key"); - write(&fork.join("library/std/src/new.rs"), "pub fn new() {}\n"); - assert_ne!(k(), base, "an untracked source file was not in the key"); - fs::remove_file(fork.join("library/std/src/new.rs")).unwrap(); - assert_eq!(k(), base); - - write(&root.join("toyos-abi/Cargo.toml"), "[package]\nversion = \"0.2.0\"\n"); - assert_ne!(k(), base, "a manifest change kept the old sysroot"); - write(&root.join("toyos-abi/Cargo.toml"), "[package]\nversion = \"0.1.0\"\n"); - assert_eq!(k(), base); - - write(&rust_dir.join("build/toyos-compiler"), "tree-2"); - assert_ne!(k(), base, "another compiler kept the old sysroot"); + fn a_sysroot_key_is_its_compiler_and_its_trees() { + let e = estate("sysroot-key"); + let fork = e.same.join("rust"); + let k = |compiler: &str| key(compiler, &Sources::of(&e.same, &fork)); + let base = k("c1"); + assert_ne!(k("c2"), base, "another compiler kept the sysroot"); + write(&e.same.join("userland/libc/src/lib.rs"), "pub struct B;\n"); + assert_ne!(k("c1"), base, "a libc edit kept the sysroot"); + git(&e.same, &["checkout", "-q", "--", "userland"]); + write(&e.same.join("userland/libc/Cargo.lock"), "# re-locked, and neither staged nor committed\n"); + assert_ne!(k("c1"), base, "an edit to libc's own lockfile kept the sysroot"); + git(&e.same, &["checkout", "-q", "--", "userland"]); + assert_eq!(k("c1"), base); + write(&fork.join("library/std/src/lib.rs"), "pub fn b() {}\n"); + assert_ne!(k("c1"), base, "a std edit kept the sysroot"); + git(&fork, &["checkout", "-q", "--", "library"]); + write(&fork.join("compiler/rustc_target/src/lib.rs"), "pub fn t() { x() }\n"); + assert_eq!(k("c1"), base, "a compiler edit reached the sysroot but through the compiler's key"); } /// **What one compiler compiled in a std build directory is never another's**: @@ -750,74 +576,36 @@ mod tests { /// stay either way. #[test] fn another_compiler_s_std_build_goes_and_the_same_one_s_stays() { - let base = TempDir::new("compiled-by"); - let (_root, rust_dir, _fork) = keyed(&base); - let build = base.join("toyos-std"); + let build = TempDir::new("compiled-by"); let compiled = [ build.join("bootstrap/debug/deps/libserde-1.rlib"), build.join("host/stage0-std/dist/build/std/build-script-build"), build.join("host/a-directory-bootstrap-adds/lib.rlib"), build.join("tmp/cc-rs-out-dir/out.o"), build.join("host/a-stamp-bootstrap-writes"), - build.join("host/ci-llvm/lib/libLLVM.dylib"), - ]; - let downloaded = [ - build.join("cache/2026-07-13/rustc.tar.xz"), - build.join("host/rustfmt/bin/rustfmt"), ]; + let downloaded = [build.join("cache/2026-07-13/rustc.tar.xz"), build.join("host/rustfmt/bin/rustfmt")]; let lay = || { for file in compiled.iter().chain(&downloaded) { write(file, "built"); } }; - let identity = || Compiler::primary(&rust_dir).identity(); - lay(); - forget_another_compiler(&build, "host", &identity()); + forget_another_compiler(&build, "host", "c1"); for file in &compiled { assert!(!file.exists(), "{} was kept, and no record names a compiler for it", file.display()); } assert!(downloaded.iter().all(|f| f.is_file()), "a download went"); - lay(); - forget_another_compiler(&build, "host", &identity()); + forget_another_compiler(&build, "host", "c1"); assert!(compiled.iter().all(|f| f.is_file()), "the same compiler's build went"); - - write(&rust_dir.join("build/toyos-compiler"), "tree-2"); - forget_another_compiler(&build, "host", &identity()); + forget_another_compiler(&build, "host", "c2"); for file in &compiled { assert!(!file.exists(), "{} was kept for another compiler", file.display()); } assert!(downloaded.iter().all(|f| f.is_file()), "a download went"); } - /// **A std build directory keeps no LLVM, even under the compiler that - /// built the rest**: bootstrap's and `download-ci-llvm`'s go with their - /// download, and what that compiler built and the other downloads stay. - #[test] - fn a_std_build_under_the_same_compiler_keeps_no_llvm() { - let base = TempDir::new("std-llvm"); - let (_root, rust_dir, _fork) = keyed(&base); - let build = base.join("toyos-std"); - let host = host_triple(); - let identity = Compiler::primary(&rust_dir).identity(); - prepare_std_build(&build, &host, &identity); - let llvm = [ - build.join(&host).join("ci-llvm/lib/libLLVM.dylib"), - build.join(&host).join("llvm/bin/llvm-config"), - build.join("cache/llvm-ad3d0bc-false/rust-dev.tar.xz"), - ]; - let kept = [build.join("bootstrap/debug/deps/libserde-1.rlib"), build.join("cache/2026-07-13/rustc.tar.xz")]; - for file in llvm.iter().chain(&kept) { - write(file, "built"); - } - prepare_std_build(&build, &host, &identity); - for file in &llvm { - assert!(!file.exists(), "{} outlived a std build's preparation", file.display()); - } - assert!(kept.iter().all(|f| f.is_file()), "the same compiler's build went"); - } - /// **A std build fetches no LLVM**: it builds none, and the `compiler` /// profile would download one. #[test] @@ -831,222 +619,205 @@ mod tests { #[test] fn a_switch_that_cannot_remove_records_nothing_and_the_next_one_removes() { use std::os::unix::fs::PermissionsExt; - let base = TempDir::new("compiled-by-stuck"); - let (_root, rust_dir, _fork) = keyed(&base); - let build = base.join("toyos-std"); - let identity = || Compiler::primary(&rust_dir).identity(); - fs::create_dir_all(&build).unwrap(); - forget_another_compiler(&build, "host", &identity()); + let build = TempDir::new("compiled-by-stuck"); + forget_another_compiler(&build, "host", "c1"); let deps = build.join("bootstrap/debug/deps"); write(&deps.join("libserde-1.rlib"), "built"); - write(&rust_dir.join("build/toyos-compiler"), "tree-2"); let mode = |bits| fs::set_permissions(&deps, fs::Permissions::from_mode(bits)).unwrap(); - mode(0o555); - let stuck = std::panic::catch_unwind(|| forget_another_compiler(&build, "host", &identity())); + let stuck = std::panic::catch_unwind(|| forget_another_compiler(&build, "host", "c2")); mode(0o755); let refusal = stuck.expect_err("a build that could not be removed was taken for removed"); let refusal = refusal.downcast_ref::().expect("a formatted panic"); assert!(refusal.starts_with(&format!("remove {}", build.join("bootstrap").display())), "{refusal}"); - assert_ne!(fs::read_to_string(build.join("compiled-by")).unwrap(), identity(), - "the new compiler was recorded over a build it did not remove"); - - forget_another_compiler(&build, "host", &identity()); + assert_eq!(fs::read_to_string(build.join("compiled-by")).unwrap(), "c1", "the new compiler was recorded over a build it did not remove"); + forget_another_compiler(&build, "host", "c2"); assert!(!build.join("bootstrap").exists(), "the next call kept the build the stuck one could not remove"); - assert_eq!(fs::read_to_string(build.join("compiled-by")).unwrap(), identity()); - } - - /// A primary with the fork as its `rust` submodule at `C1`, the fork's `C2` - /// one library change later, and a linked worktree whose tree pins `C2`. - fn two_pins(base: &Path) -> (PathBuf, PathBuf, String, String) { - let bt = base.join("backtrace-src"); - fs::create_dir_all(&bt).unwrap(); - git(&bt, &["init", "-q"]); - write(&bt.join("lib.rs"), "pub fn trace() {}\n"); - git(&bt, &["add", "-A"]); - git(&bt, &["commit", "-qm", "backtrace"]); - - let fork = base.join("fork-src"); - fs::create_dir_all(&fork).unwrap(); - git(&fork, &["init", "-q"]); - write(&fork.join("library/std/src/lib.rs"), "pub fn a() {}\n"); - write(&fork.join("compiler/lib.rs"), "\n"); - write(&fork.join("x.py"), "\n"); - git(&fork, &["submodule", "add", "-q", bt.to_str().unwrap(), "library/backtrace"]); - git(&fork, &["add", "-A"]); - git(&fork, &["commit", "-qm", "C1"]); - let c1 = git(&fork, &["rev-parse", "HEAD"]); - write(&fork.join("library/std/src/lib.rs"), "pub fn b() {}\n"); - git(&fork, &["commit", "-qam", "C2"]); - let c2 = git(&fork, &["rev-parse", "HEAD"]); - - let primary = base.join("primary"); - fs::create_dir_all(&primary).unwrap(); - git(&primary, &["init", "-q"]); - write(&primary.join("toyos-abi/src/lib.rs"), "pub struct A;\n"); - git(&primary, &["submodule", "add", "-q", fork.to_str().unwrap(), "rust"]); - git(&primary.join("rust"), &["checkout", "-q", &c1]); - git(&primary, &["add", "-A"]); - git(&primary, &["submodule", "update", "-q", "--init", "--recursive"]); - git(&primary, &["commit", "-qm", "pins C1"]); - - let linked = base.join("linked"); - git(&primary, &["worktree", "add", "-q", "-b", "wt", linked.to_str().unwrap()]); - git(&linked, &["update-index", "--cacheinfo", &format!("160000,{c2},rust")]); - git(&linked, &["commit", "-qm", "pins C2"]); - (primary, linked, c1, c2) + assert_eq!(fs::read_to_string(build.join("compiled-by")).unwrap(), "c2"); } - /// **A worktree pinning another fork commit gets a checkout of its own at - /// that commit, and the primary's is not touched** — neither its `HEAD` nor - /// a file of its tree; the worktree's own `git status` is clean, because the - /// checkout is what its gitlink names. + /// **A worktree whose `rust/` is the stub holds no fork checkout of its + /// own**: its toolchain is keyed from the primary's objects at the commit its + /// tree pins, which is what a clean checkout of that commit hashes to, and + /// built in the host's one shared checkout — held by one build at a time, + /// moved to the pin of the build holding it with whatever a killed build left + /// gone, beside links to that build's ABI trees. The primary's fork is not + /// touched, and plain `git worktree remove` takes the worktree whole. #[test] - fn a_worktree_pinning_another_fork_commit_gets_its_own_checkout() { - let base = TempDir::new("fork-pins"); - let (primary, linked, c1, c2) = two_pins(&base); - let before = git(&primary.join("rust"), &["status", "--porcelain"]); - - let fork = fork_checkout(&linked); - - assert_eq!(fork, linked.join("rust")); - assert_eq!(git(&fork, &["rev-parse", "HEAD"]), c2); - assert_eq!(fs::read_to_string(fork.join("library/std/src/lib.rs")).unwrap(), "pub fn b() {}\n"); - assert!(fork.join("library/backtrace/lib.rs").is_file(), "the nested fork submodule is missing"); - assert_eq!(git(&primary.join("rust"), &["rev-parse", "HEAD"]), c1, "the primary's fork moved"); - assert_eq!(git(&primary.join("rust"), &["status", "--porcelain"]), before); - assert_eq!( - fs::read_to_string(primary.join("rust/library/std/src/lib.rs")).unwrap(), - "pub fn a() {}\n", - "the primary's fork tree was written" - ); - assert_eq!(git(&linked, &["status", "--porcelain"]), "", "the worktree is not clean"); - - // Work on the fork in the worktree's own checkout is what it builds. - write(&fork.join("library/std/src/lib.rs"), "pub fn c() {}\n"); - git(&fork, &["commit", "-qam", "C3, the agent's own"]); - assert_eq!(fork_checkout(&linked), fork); - - // A clean checkout behind what the tree pins is moved to the pin itself. - git(&fork, &["checkout", "-q", &c1]); - assert_eq!(fork_checkout(&linked), fork); - assert_eq!(git(&fork, &["rev-parse", "HEAD"]), c2, "a checkout behind its pin was not moved to it"); - - // One with local changes is never moved out from under whoever made them. - git(&fork, &["checkout", "-q", &c1]); + fn a_stub_worktree_builds_in_the_host_s_shared_checkout() { + let e = estate("fork-shared"); + let stub = |name: &str| { + let worktree = e.same.parent().unwrap().join(name); + git(&e.primary, &["worktree", "add", "-q", "-b", name, worktree.to_str().unwrap()]); + worktree + }; + let beside = |dir: &Path, tree: &str| fs::canonicalize(dir.join("library/std/../../..").join(tree)).unwrap(); + let linked = stub("stub"); + let pinned = git(&e.primary, &["rev-parse", "HEAD:rust"]); + let before = git(&e.rust_dir, &["rev-parse", "HEAD"]); + + let fork = Fork::of(&linked); + assert!(matches!(&fork, Fork::Pinned { commit, .. } if *commit == pinned)); + assert_eq!(fork.sources(&linked), Sources::of(&linked, &e.same.join("rust")), "the pin's trees are not a clean checkout's"); + assert!(!linked.join("rust/.git").exists() && !linked.join("target").exists(), "a stub worktree holds fork state"); + + let checkout = fork.checkout(&linked); + assert_eq!(checkout.dir, e.rust_dir.join(SHARED).join("rust")); + assert_eq!(git(&checkout.dir, &["rev-parse", "HEAD"]), pinned); + assert_eq!(beside(&checkout.dir, "toyos-abi"), fs::canonicalize(linked.join("toyos-abi")).unwrap()); + let backtrace = fs::read_dir(checkout.dir.join("library/backtrace")).unwrap().count(); + assert_eq!(backtrace, 0, "a submodule was checked out of the primary's clone, which its update would take over"); + assert!(Lock::try_exclusive(&e.rust_dir.join(SHARED)).is_none(), "the shared checkout is not held"); + assert_eq!(git(&e.rust_dir, &["rev-parse", "HEAD"]), before, "the primary's fork moved"); + write(&checkout.dir.join("library/std/src/lib.rs"), "left by a killed build"); + write(&checkout.dir.join("left.rs"), "left by a killed build"); + git(&checkout.dir, &["submodule", "update", "-q", "--init", "library/backtrace"]); + write(&checkout.dir.join("library/backtrace/lib.rs"), "left by a killed build"); + write(&checkout.dir.join("library/backtrace/left.rs"), "left by a killed build"); + drop(checkout); + + let other = stub("other"); + let moved = git(&e.a.join("rust"), &["rev-parse", "HEAD"]); + git(&other, &["update-index", "--cacheinfo", &format!("160000,{moved},rust")]); + let checkout = Fork::of(&other).checkout(&other); + assert_eq!(git(&checkout.dir, &["rev-parse", "HEAD"]), moved, "a moved pin kept the old checkout"); + let left = git(&checkout.dir, &["status", "--porcelain", "--ignore-submodules=none"]); + assert_eq!(left, "", "what a killed build left stayed"); + assert_eq!(beside(&checkout.dir, "toyos"), fs::canonicalize(other.join("toyos")).unwrap(), "the links name the last build's trees"); + drop(checkout); + + git(&e.primary, &["worktree", "remove", linked.to_str().unwrap()]); + assert!(!linked.exists(), "git worktree remove left {}", linked.display()); + } + + /// **A linked worktree's own fork checkout is built where it is only while + /// it holds work its pin does not**: commits ahead or uncommitted work are + /// built as they stand, where they are; at the pin and clean it is the pin, + /// built in the shared checkout; behind the pin it is moved there when clean + /// and refused, its work named, when not. The primary's behind its pin is + /// refused, since nothing but its owner moves it. + #[test] + fn a_fork_checkout_is_built_where_its_work_is() { + let e = estate("fork-own"); + let fork = e.a.join("rust"); + let ahead = git(&fork, &["rev-parse", "HEAD"]); + let where_it_is = |f: Fork| matches!(f, Fork::Checkout(dir) if dir == fork); + let as_the_pin = |f: Fork| matches!(f, Fork::Pinned { commit, .. } if commit == ahead); + assert!(where_it_is(Fork::of(&e.a)), "commits ahead of the pin were built as the pin"); + git(&e.a, &["add", "rust"]); + git(&e.a, &["commit", "-qm", "pins a"]); + assert!(as_the_pin(Fork::of(&e.a)), "a clean checkout at its pin was built where it is"); write(&fork.join("library/std/src/lib.rs"), "pub fn uncommitted() {}\n"); - let refused = std::panic::catch_unwind(|| fork_checkout(&linked)) - .expect_err("a fork checkout with local changes was moved out from under them"); - let message = refused.downcast::().expect("a formatted refusal"); - assert!(message.contains(&c1) && message.contains(&c2), "{message}"); - } - - /// The primary's compiler under `base`: `rustc` and `rust-lld`, and the C - /// toolchain `src/clang.rs` provisions beside them if `clang`; no cargo. - fn primary_compiler(base: &Path, clang: bool) -> Compiler { - let compiler = Compiler::primary(&base.join("rust")); - write(&compiler.stage2.join("bin/rustc"), "rustc"); - let lld = toolchain::rust_lld(&compiler.stage2); - write(&lld, "lld"); - write(&lld.with_file_name("llvm-ar"), "llvm-ar"); - if clang { - for tool in ["clang", "ld.lld"] { - write(&lld.with_file_name(tool), tool); - } - write(&lld.parent().unwrap().parent().unwrap().join("lib/clang/22/include/stddef.h"), "stddef"); - } - compiler - } - - /// What a panic in `f` said. - fn refusal(f: impl FnOnce()) -> String { - let refused = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).expect_err("nothing was refused"); - *refused.downcast::().expect("a formatted refusal") + assert!(where_it_is(Fork::of(&e.a)), "uncommitted work was built as the pin"); + git(&fork, &["checkout", "-q", "HEAD~1"]); + let said = refusal("a fork checkout behind its pin was moved over uncommitted work", || { + Fork::of(&e.a); + }); + assert!(said.contains("uncommitted work") && said.contains("library/std/src/lib.rs"), "{said}"); + git(&fork, &["checkout", "-q", "--", "library"]); + assert!(as_the_pin(Fork::of(&e.a)), "a clean checkout behind its pin was not built as the pin"); + assert_eq!(git(&fork, &["rev-parse", "HEAD"]), ahead, "a clean checkout behind its pin was not moved to it"); + + git(&e.primary, &["update-index", "--cacheinfo", &format!("160000,{ahead},rust")]); + let said = refusal("the primary's fork behind its pin was built", || { + Fork::of(&e.primary); + }); + assert!(said.contains("is not at or ahead of"), "{said}"); } - /// **A sysroot is whole, or it is made again**: a `stage2` without cargo — - /// what bootstrap leaves until the primary completes it — is refused by - /// name and nothing is published, and one found with its `SOURCES` and - /// without its cargo is rebuilt rather than trusted, once. + /// **A submodule holding an edit or an untracked file is work**: a linked + /// worktree's own checkout at its pin, clean but for either, is built where + /// it is, and never as the pin, which holds neither. #[test] - fn a_sysroot_is_whole_or_it_is_made_again() { - let base = TempDir::new("whole"); - git(&base, &["init", "-q"]); - let compiler = primary_compiler(&base, true); - let made = std::cell::Cell::new(0); - // `most` bounds the makes so far, so a make that loops fails rather than hangs. - let make = |dir: &Path, most: usize| { - made.set(made.get() + 1); - assert!(made.get() <= most, "a sysroot that was not whole was made again: make {}", made.get()); - publish(&compiler, dir, |partial| { - write(&partial.join("lib/rustlib/x86_64-unknown-toyos/lib/libstd.rlib"), "std"); - "found\n".to_string() - }) - }; - - let fresh = sysroots_dir(&base.join("rust")).join("fresh"); - let said = refusal(|| drop(held(&base, "fresh", &fresh, || make(&fresh, 1)))); - assert!(said.contains("is missing cargo") && said.contains("`cargo run -- --build-only`"), "{said}"); - assert!(!fresh.exists() && !fresh.with_extension("partial").exists(), "a sysroot was published from a stage2 without cargo"); - assert_eq!(made.get(), 1); - - let dir = sysroots_dir(&base.join("rust")).join("found"); - clone_tree(&compiler.stage2, &dir); - write(&dir.join(SOURCES), "found\n"); - toolchain::provision_toolchain_cargo(&compiler.stage2); - let using = held(&base, "found", &dir, || make(&dir, 2)); - assert_eq!(made.get(), 2, "a sysroot without its cargo was trusted because it has SOURCES"); - assert_eq!(toolchain::toolchain_defect(&dir), None); - assert!(dir.join("lib/rustlib/x86_64-unknown-toyos/lib/libstd.rlib").is_file()); - drop(using); - drop(held(&base, "found", &dir, || make(&dir, 2))); - assert_eq!(made.get(), 2, "a whole sysroot was made again"); + fn a_submodule_holding_changes_is_work() { + let e = estate("fork-submodule-work"); + let fork = e.a.join("rust"); + let backtrace = fork.join("library/backtrace"); + git(&e.a, &["add", "rust"]); + git(&e.a, &["commit", "-qm", "pins a"]); + git(&fork, &["submodule", "update", "-q", "--init", "library/backtrace"]); + assert!(matches!(Fork::of(&e.a), Fork::Pinned { .. }), "a clean checkout at its pin was built where it is"); + let where_it_is = |f: Fork| matches!(f, Fork::Checkout(dir) if dir == fork); + write(&backtrace.join("lib.rs"), "pub fn trace() { edited() }\n"); + assert!(where_it_is(Fork::of(&e.a)), "an edit in a submodule was built as the pin"); + git(&backtrace, &["checkout", "-q", "--", "lib.rs"]); + write(&backtrace.join("new.rs"), "pub fn new() {}\n"); + assert!(where_it_is(Fork::of(&e.a)), "an untracked file in a submodule was built as the pin"); + } + + /// **A clean checkout moved to a pin that moves a submodule's gitlink is + /// the pin**: `git checkout` leaves the submodule at the old gitlink, which + /// is no work of anybody's. + #[test] + fn a_submodule_left_at_the_old_gitlink_is_no_work() { + let e = estate("fork-old-gitlink"); + let fork = e.same.join("rust"); + let backtrace = fork.join("library/backtrace"); + git(&fork, &["submodule", "update", "-q", "--init", "library/backtrace"]); + let old = git(&backtrace, &["rev-parse", "HEAD"]); + write(&backtrace.join("lib.rs"), "pub fn trace() { moved() }\n"); + git(&backtrace, &["commit", "-qam", "a newer backtrace"]); + git(&fork, &["commit", "-qam", "moves backtrace's gitlink"]); + let pin = git(&fork, &["rev-parse", "HEAD"]); + git(&fork, &["checkout", "-q", "--detach", "HEAD~1"]); + git(&backtrace, &["checkout", "-q", &old]); + git(&e.same, &["update-index", "--cacheinfo", &format!("160000,{pin},rust")]); + + let moved = Fork::of(&e.same); + assert_eq!(git(&fork, &["rev-parse", "HEAD"]), pin, "a clean checkout behind its pin was not moved to it"); + assert!(matches!(moved, Fork::Pinned { commit, .. } if commit == pin), "a submodule at the old gitlink was built in place"); + } + + /// **A fork checkout whose submodule is a git worktree of another clone is + /// refused before anything is built in it**, as main made them, and one + /// whose submodules are its own is not. + #[test] + fn a_submodule_of_another_clone_is_refused_before_a_build() { + let e = estate("fork-borrowed"); + let primarys = e.rust_dir.join("library/backtrace"); + drop(Fork::of(&e.primary).checkout(&e.primary)); + let backtrace = e.a.join("rust/library/backtrace"); + fs::remove_dir(&backtrace).unwrap(); + git(&primarys, &["worktree", "add", "-q", "--detach", backtrace.to_str().unwrap(), "HEAD"]); + let said = refusal("a checkout whose submodule is another clone's worktree was built in", || { + Fork::of(&e.a).checkout(&e.a); + }); + assert!(said.contains(&format!("{} is a git worktree of", backtrace.display())) && said.contains("nothing was built"), "{said}"); } - /// **A sysroot that cannot be made whole is refused after one make, never - /// made again**: a `stage2` without clang — what a stopped bootstrap leaves — - /// is refused before any std is built for it, with nothing published, and a - /// make that leaves its sysroot not whole is refused by what it lacks. + /// **A sysroot that is not whole is refused**: a compiler without clang + /// makes one without it. #[test] - fn a_sysroot_that_cannot_be_made_whole_is_made_once_and_refused() { - let base = TempDir::new("no-clang"); - git(&base, &["init", "-q"]); - let compiler = primary_compiler(&base, false); - toolchain::provision_toolchain_cargo(&compiler.stage2); - let made = std::cell::Cell::new(0); - let once = || { - made.set(made.get() + 1); - assert_eq!(made.get(), 1, "a sysroot that was not whole was made again"); - }; - - let dir = sysroots_dir(&base.join("rust")).join("cloned"); - let filled = std::cell::Cell::new(false); - let said = refusal(|| { - drop(held(&base, "cloned", &dir, || { - once(); - publish(&compiler, &dir, |_| { - filled.set(true); - "cloned\n".to_string() - }) - })) + fn a_sysroot_that_is_not_whole_is_refused() { + let e = estate("sysroot-whole"); + let base = TempDir::new("sysroot-whole"); + let stage2 = base.join("stage2"); + let lld = toolchain::rust_lld(&stage2); + write(&stage2.join("bin/rustc"), "rustc"); + write(&lld, "lld"); + write(&lld.with_file_name("llvm-ar"), "llvm-ar"); + write(&stage2.join("bin/cargo"), "cargo"); + let said = refusal("a sysroot without clang was taken for whole", || { + assemble(&stage2, &e.same.join("rust"), &base.join("partial"), |partial| write(&partial.join("lib/rustlib/x/lib/libstd.rlib"), "std")); }); - assert!(said.contains("carries no") && said.contains("/clang"), "{said}"); - assert!(said.contains(&compiler.stage2.display().to_string()) && said.contains("`cargo run -- --build-only`"), "{said}"); - assert!(!filled.get(), "a std was built for a sysroot of a compiler without clang"); - assert!(!dir.exists() && !dir.with_extension("partial").exists(), "a sysroot was published from a stage2 without clang"); - assert_eq!(made.get(), 1); + assert!(said.contains("is not whole") && said.contains("clang"), "{said}"); + } - made.set(0); - let dir = sysroots_dir(&base.join("rust")).join("made"); - let said = refusal(|| { - drop(held(&base, "made", &dir, || { - once(); - clone_tree(&compiler.stage2, &dir); - write(&dir.join(SOURCES), "made\n"); - })) + /// **A sysroot built from a submodule at another commit than its gitlink + /// is refused**: bootstrap leaves `library/backtrace` at `HEAD`'s gitlink + /// under a staged one, and builds it. + #[test] + fn a_sysroot_built_off_a_submodule_s_gitlink_is_refused() { + let e = estate("sysroot-gitlink"); + let fork = e.a.join("rust"); + let (head, staged) = behind_a_staged_gitlink(&fork, "library/backtrace"); + let base = TempDir::new("sysroot-gitlink"); + write(&base.join("stage2/bin/rustc"), "rustc"); + let said = refusal("a sysroot built from a submodule its gitlink does not name was taken", || { + assemble(&base.join("stage2"), &fork, &base.join("partial"), |_| {}); }); - assert!(said.starts_with("sysroot made was made, and is not whole") && said.contains("/clang"), "{said}"); - assert_eq!(made.get(), 1); + let named = format!("{} is at {head}, and its gitlink names {staged}", fork.join("library/backtrace").display()); + assert!(said.contains(&named), "{said}"); } /// **What a stage-0 std build made is what its stamp names**: its @@ -1075,28 +846,4 @@ mod tests { let refused = std::panic::catch_unwind(|| place_std(&built.join(".libstd-stamp"), &lib)); assert!(refused.is_err(), "a host library was placed in a guest target"); } - - /// `--worktree remove` takes the worktree's fork checkout with it — git will - /// not remove a worktree around one — unless that checkout holds the only - /// copy of something. - #[test] - fn a_removed_worktree_takes_its_fork_checkout_and_refuses_to_lose_fork_work() { - let base = TempDir::new("fork-remove"); - let (primary, linked, _c1, _c2) = two_pins(&base); - let fork = fork_checkout(&linked); - write(&fork.join("library/std/src/lib.rs"), "pub fn unsaved() {}\n"); - let refused = std::panic::catch_unwind(|| crate::worktree::remove(&primary, linked.to_str().unwrap())); - assert!(refused.is_err(), "a fork checkout with uncommitted work was removed"); - assert!(fork.join("library/std/src/lib.rs").is_file()); - - git(&fork, &["commit", "-qam", "committed, and on no ref"]); - let refused = std::panic::catch_unwind(|| crate::worktree::remove(&primary, linked.to_str().unwrap())); - assert!(refused.is_err(), "a fork commit no ref reaches was thrown away"); - - git(&fork, &["branch", "kept"]); - crate::worktree::remove(&primary, linked.to_str().unwrap()); - assert!(!linked.exists(), "{} is still on disk", linked.display()); - let listed = git(&primary.join("rust"), &["worktree", "list", "--porcelain"]); - assert_eq!(listed.lines().filter(|l| l.starts_with("worktree ")).count(), 1, "{listed}"); - } } diff --git a/src/toolchain.rs b/src/toolchain.rs index 6f028083e9d..9865f7ecbef 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -4,18 +4,8 @@ use std::process::Command; use std::sync::OnceLock; use crate::arch::Arch; -use crate::buildlock; -use crate::buildlock::Scope; -use crate::sysroot::{self, Sysroot, SYSROOT_SOURCES}; - -/// Whether the primary's compiler needs a bootstrap. `invalidate_hosted` -/// separates "the compiler changed" from "the rustup link is missing": only the -/// first makes the ToyOS-hosted rustc stale, and rebuilding that one costs -/// minutes. -#[derive(Clone, Copy, PartialEq, Debug)] -struct Bootstrap { - invalidate_hosted: bool, -} +use crate::store::{self, ABI_TREES}; +use crate::sysroot::{self, Sysroot}; /// Which checkout holds the `rust/` submodule and the toolchain built from it. pub enum Owner { @@ -33,15 +23,10 @@ pub enum Owner { Installed, } -/// One `rust/` per repository, in the primary checkout, and every worktree -/// compiles against it. -/// -/// Not a policy — an affordance. A second checkout of that submodule is a -/// 913 MiB clone (git gives a linked worktree its own, sharing no objects), and -/// a second `build/` beside it is 47 GiB. `git worktree add` leaves `rust/` an -/// empty stub, and leaving it empty is what keeps `git status` clean: git -/// refuses a symlink where a gitlink belongs, and errors out of every command -/// rather than just that one. +/// One `rust/` per repository, in the primary checkout, holding the fork's +/// objects and the store every checkout builds into. A linked worktree never +/// initialises its own: git would clone the fork's history again, 913 MiB +/// sharing no objects. pub fn owner(root: &Path) -> Owner { let primary = crate::primary_checkout(root); let same = fs::canonicalize(root).map(|r| r == primary).unwrap_or(false); @@ -78,12 +63,9 @@ const STD_SOURCES: [&str; 2] = ["toyos-abi/src", "toyos/src"]; /// Every target a guest artifact is built for: ToyOS userland, the kernel's /// bare-metal target, and the UEFI bootloader. /// -/// One home for the list, because it is read four ways that must agree — the -/// `stage1-std` cleans in [`full_bootstrap`], [`write_config`]'s bootstrap -/// `target` set, the libraries `src/sysroot.rs` builds and places, and -/// `src/build.rs`'s external fingerprint. A fifth spelling would silently leave -/// one of them building or fingerprinting a different set of targets than the -/// others. +/// One home for the list, because it is read in ways that must agree: the +/// libraries `src/sysroot.rs` builds and places, and `src/build.rs`'s external +/// fingerprint. pub const GUEST_TARGETS: [&str; 6] = [ Arch::X86_64.userland(), Arch::X86_64.kernel(), @@ -93,11 +75,7 @@ pub const GUEST_TARGETS: [&str; 6] = [ Arch::Aarch64.loader(), ]; -/// The one ToyOS the hosted rustc (`system.toml`'s `hosted-rustc`) is built to -/// run on. -pub const HOSTED_ARCH: Arch = Arch::X86_64; - -/// The primary's compiler, which every sysroot is cloned from and compiled by. +/// Where an installed toolchain is, as `src/release.rs` unpacks it. pub(crate) fn stage2(rust_dir: &Path) -> PathBuf { rust_dir.join(format!("build/{}/stage2", host_triple())) } @@ -161,13 +139,12 @@ pub(crate) fn assert_std_built_from(root: &Path, dep_info: &Path) { dep_info.display(), ); let root = fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); - let foreign: Vec<&String> = - sources.iter().filter(|p| !Path::new(p).starts_with(&root)).collect(); + // Resolved: the shared checkout reaches a worktree's trees through links. + let resolved = |p: &String| fs::canonicalize(p).unwrap_or_else(|e| panic!("resolve {p}, which std compiled: {e}")); + let foreign: Vec<&String> = sources.iter().filter(|p| !resolved(p).starts_with(&root)).collect(); assert!( foreign.is_empty(), - "std was compiled against {} sources that are not this worktree's:\n {}\n\ - `library/std` names them as `../../../`, so the fork checkout that built it is not \ - this worktree's own.", + "std was compiled against {} sources that are not this worktree's:\n {}", foreign.len(), foreign.iter().map(|p| p.as_str()).collect::>().join("\n "), ); @@ -179,21 +156,20 @@ fn witness_path(rust_dir: &Path) -> PathBuf { rust_dir.join("build/toyos-sysroot-witness") } - /// The lines of a witness belonging to `trees`. fn witness_subset(text: &str, trees: &[&str]) -> String { text.lines() - .filter(|l| trees.iter().any(|t| l.starts_with(t))) + .filter(|l| trees.iter().any(|t| l.starts_with(&format!("{t}:")))) .collect::>() .join("\n") } -/// Which of [`SYSROOT_SOURCES`] this worktree disagrees with the sysroot about. +/// Which of [`ABI_TREES`] this worktree disagrees with the sysroot about. fn differing_trees(recorded: Option<&str>, current: &str) -> String { let Some(recorded) = recorded else { return "nothing recorded what the sysroot was built from".to_string(); }; - let names: Vec<&str> = SYSROOT_SOURCES + let names: Vec<&str> = ABI_TREES .iter() .copied() .filter(|t| witness_subset(recorded, &[t]) != witness_subset(current, &[t])) @@ -232,73 +208,6 @@ fn narrated_binaries(bin: &Path) -> Vec<&'static str> { TOOLCHAIN_BINARIES.into_iter().filter(|name| !bin.join(name).exists()).collect() } -/// The `cargo` this machine can lend the `toyos` toolchain. -/// -/// **A nightly one if rustup has it, and the host's otherwise — which is what -/// rustup itself falls back to, measured on both machines that matter.** The -/// dev host has a `nightly-` installed and rustup's narration names it -/// (`falling back to ".../nightly-aarch64-apple-darwin/bin/cargo"`, cargo -/// 1.96.0-nightly); every CI runner installs `--profile minimal -/// --default-toolchain stable` and nothing else, so rustup falls back to -/// stable's. Provisioning in that order changes the cargo behind no ToyOS build -/// anywhere: it removes the narration and nothing else. -/// -/// The order is not decoration. `-Z` is refused outside the nightly channel, so -/// stable's cargo (1.97.1) and bootstrap's stage0 cargo (1.98.0-beta.2) both -/// refuse the `-Zbuild-std` std type-check `src/CLAUDE.md` documents — measured, -/// both — while the nightly rustup already falls back to accepts it. Picking -/// "the host cargo" flatly would have taken that away from the dev host and -/// called it a cleanup. -/// -/// The host's is resolved through `rustc --print sysroot` for the same reason -/// [`link_host_target`] does: it is whatever stable toolchain this machine has, -/// and it is not a path any artifact can know. -fn host_cargo() -> &'static Path { - static CARGO: OnceLock = OnceLock::new(); - CARGO.get_or_init(|| { - if let Some(home) = rustup_home() { - let nightly = home.join(format!("toolchains/nightly-{}/bin/cargo", host_triple())); - if nightly.exists() { - return nightly; - } - } - let cargo = host_sysroot().join("bin/cargo"); - assert!( - cargo.exists(), - "there is no cargo at {}, so the toyos toolchain cannot be given one and every \ - cargo invocation under it will narrate a fallback.", - cargo.display(), - ); - cargo - }) -} - -/// Whether the toolchain's `cargo` is not the one this machine would lend it. -/// -/// The question is what the link *points at*, not whether a file is there: a -/// `bin/cargo` that arrived inside the published artifact names a path only the -/// publisher had, and a build that took it for provisioned would keep narrating -/// — or worse, run another platform's binary. -fn cargo_link_stale(stage2: &Path) -> bool { - fs::read_link(stage2.join("bin/cargo")).ok().as_deref() != Some(host_cargo()) -} - -/// Put a `cargo` beside the toolchain's `rustc`. -/// -/// **A symlink, and what survives the artifact round-trip is this step rather -/// than the link.** `src/release.rs` excludes it from the tarball for the reason -/// it excludes `lib/rustlib/`: it names a path only the publishing runner -/// has, and a copy would put a 32 MB host binary into a 401 MiB artifact to -/// stand in for a file the consumer can make in a microsecond. `Owner::Installed` -/// makes it, exactly as it makes the host target. -pub(crate) fn provision_toolchain_cargo(stage2: &Path) { - let at = stage2.join("bin/cargo"); - let _ = fs::remove_file(&at); - std::os::unix::fs::symlink(host_cargo(), &at).unwrap_or_else(|e| { - panic!("Failed to symlink {} -> {}: {e}", at.display(), host_cargo().display()) - }); -} - /// Why the toolchain at `stage2` is not whole, if it is not: a binary rustup /// would narrate a fallback for, no linker for the guest targets, or no C /// toolchain (`src/clang.rs`). @@ -312,7 +221,7 @@ pub(crate) fn toolchain_defect(stage2: &Path) -> Option { return Some(format!( "the toyos toolchain at {} is missing {}, so rustup answers for {} by falling back to \ another toolchain and narrating it on every invocation.\n\ - provision_toolchain_cargo is the step that puts them there, and it did not.", + The compiler build puts them there (`src/compiler.rs`), and it did not.", bin.display(), narrated.join(" and "), if narrated.len() == 1 { "it" } else { "them" }, @@ -324,7 +233,7 @@ pub(crate) fn toolchain_defect(stage2: &Path) -> Option { if !lld.is_file() { return Some(format!( "the toyos toolchain at {} carries no {}, the linker every guest target names: \ - bootstrap puts it there when `write_config` says `lld = true`, and it did not", + bootstrap puts it there when its configuration says `lld = true`, and it did not", stage2.display(), lld.display(), )); @@ -343,171 +252,51 @@ pub(crate) fn assert_toolchain_is_honest(stage2: &Path) { } } -/// Whether the primary's toolchain lacks what bootstrap does not put there: -/// `stage2`'s cargo and clang, or the host target in the hosted rustc's sysroot. -fn incomplete(rust_dir: &Path) -> bool { - let stage2 = stage2(rust_dir); - cargo_link_stale(&stage2) || crate::clang::defect(&stage2).is_some() || host_target_missing(rust_dir) -} - -/// Give the primary's toolchain what [`incomplete`] finds missing, its clang by -/// `provision_clang`. -fn complete(rust_dir: &Path, provision_clang: impl FnOnce(&Path)) { - let stage2 = stage2(rust_dir); - if cargo_link_stale(&stage2) { - provision_toolchain_cargo(&stage2); - } - if crate::clang::defect(&stage2).is_some() { - provision_clang(&stage2); - } - if host_target_missing(rust_dir) { - link_host_target(rust_dir); - } -} - -/// Run `bootstrap` in the primary's `rust/` against the LLVM at `llvm`, then -/// remove the LLVM its build directory built itself (`llvm::retire_in_tree`) -/// and [`complete`] what it reassembled. Called inside the act that holds the -/// global lock exclusively. -/// -/// **In the same hold, because bootstrap recreates `stage2` without its cargo -/// and clang**: a completion under a hold of its own queues behind every sysroot -/// build that takes the lock shared in between, and those last minutes. -fn reassemble(rust_dir: &Path, llvm: &Path, bootstrap: impl FnOnce()) { - bootstrap(); - crate::llvm::retire_in_tree(&rust_dir.join("build")); - complete(rust_dir, |stage2| crate::clang::provision(stage2, llvm)); -} - -/// [`reassemble`] the primary's compiler with `bootstrap`, with nothing recording -/// which compiler `stage2` is until it is whole: a bootstrap that is stopped is -/// run again by the primary, and refused by name in every linked worktree. -fn rebuild_compiler(rust_dir: &Path, llvm: &Path, bootstrap: impl FnOnce()) { - crate::compiler::forget(rust_dir); - reassemble(rust_dir, llvm, bootstrap); - crate::compiler::record(rust_dir); -} - -/// What the primary bootstraps: a new compiler when `stage2` is not the one its -/// `compiler/` names, and the same one again when rustup has no `toyos` -/// toolchain to run. -fn bootstrap(current: bool, toolchain_exists: bool) -> Option { - if !current { - Some(Bootstrap { invalidate_hosted: true }) - } else { - (!toolchain_exists).then_some(Bootstrap { invalidate_hosted: false }) - } -} - -/// Ensure the toolchain is up to date, and return the sysroot this checkout's -/// sources name — made if nobody has made it (`src/sysroot.rs`). -/// -/// Every step decides under the caller's shared lock and acts under the -/// exclusive one, so the common answer — nothing to do — costs no -/// serialisation, and two agents cannot both conclude the compiler is stale -/// and both start `x.py build` in the same directory. That pair is what left a -/// half-written `librustc_driver` for cargo to probe, and cargo memoises a -/// failed probe (`issues/build/`). +/// The sysroot this checkout's sources name, made if nobody has made it +/// (`src/sysroot.rs`), and held in use for as long as the returned value lives. /// -/// The steps are ordered, and each invalidates what it makes stale rather than -/// threading a `rebuilt` flag through: a step that decides for itself still -/// decides correctly when the process before it was killed halfway. -/// -/// Only the primary checkout builds the compiler. Every checkout builds the -/// sysroot its own sources name, in its own fork checkout. -/// -/// **The lock only covers builds routed through here.** A `./x.py build` typed -/// by hand in `rust/` takes no lock at all, and can still lose the race this -/// serialises: one builder's bootstrap removes and recreates -/// `stage1-std//dist/deps` while another's `rustc` creates a temp file -/// inside it, and the loser dies compiling `core` with `couldn't create a temp -/// dir: No such file or directory`. -pub fn ensure(root: &Path, lock: &mut buildlock::Held) -> Sysroot { +/// Every checkout makes what its own sources name, into the store; nothing is +/// rebuilt in place. The primary's build alone points the rustup `toyos` +/// toolchain at what it built ([`link`]). +pub fn ensure(root: &Path) -> Sysroot { let rust_dir = rust_dir(root); - let stamps_dir = root.join("target/stamps"); - fs::create_dir_all(&stamps_dir).ok(); - - let owner = owner(root); - - match owner { - Owner::Elsewhere(primary) => { - assert!( - stage2(&rust_dir).join("bin/rustc").exists(), - "there is no compiler to build with: {} does not exist.\n\ - The primary checkout builds it — run `cargo run -- --build-only` in {} first.", - stage2(&rust_dir).display(), - primary.display() - ); - return sysroot::ensure(root, &rust_dir, lock); - } + match owner(root) { Owner::Installed => { check_installed_toolchain(root, &rust_dir); - return Sysroot::installed(stage2(&rust_dir)); + Sysroot::installed(stage2(&rust_dir)) + } + Owner::Elsewhere(_) => sysroot::ensure(root, &rust_dir), + Owner::Us => { + let sysroot = sysroot::ensure(root, &rust_dir); + let home = rustup_home().expect("a rustup home"); + link(&home, &rust_dir, &sysroot.dir); + sysroot } - Owner::Us => {} } +} - let hosted_stamp = stamps_dir.join("hosted-rustc.stamp"); - lock.act_if( - Scope::Global, - "build the rust toolchain", - || { - let current = crate::compiler::primary_is_current(&rust_dir); - let toolchain_exists = Command::new("rustup") - .args(["run", "toyos", "rustc", "--version"]) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status() - .map(|s| s.success()) - .unwrap_or(false); - bootstrap(current, toolchain_exists) - }, - |kind| { - eprintln!("Building full toolchain (this takes a while on first run)..."); - let llvm = crate::llvm::resolve(root, &rust_dir, &rust_dir); - rebuild_compiler(&rust_dir, &llvm.dir, || full_bootstrap(root, &rust_dir, &llvm.dir)); - if kind.invalidate_hosted { - let _ = fs::remove_file(&hosted_stamp); - } - }, - ); - - let hosted_rustc = rust_dir.join(format!("build/{}/stage2/bin/rustc", HOSTED_ARCH.userland())); - lock.act_if( - Scope::Global, - "build the ToyOS-hosted rustc", - || (!hosted_stamp.exists() || !hosted_rustc.exists()).then_some(()), - |()| { - let llvm = crate::llvm::resolve(root, &rust_dir, &rust_dir); - reassemble(&rust_dir, &llvm.dir, || build_hosted_rustc(&rust_dir, &llvm.dir)); - assert!(hosted_rustc.exists(), "Failed to build hosted rustc"); - fs::write(&hosted_stamp, "").unwrap(); - }, - ); - - let stage2 = stage2(&rust_dir); - lock.act_if( - Scope::Global, - "link the toyos rustup toolchain", - || link_stale(&stage2).then_some(()), - |()| { - let status = Command::new("rustup") - .args(["toolchain", "link", "toyos", stage2.to_str().unwrap()]) - .status() - .unwrap_or_else(|e| panic!("Failed to run rustup: {e}")); - assert!(status.success(), "rustup toolchain link failed"); - }, - ); - - lock.act_if( - Scope::Global, - "complete the toyos toolchain", - || incomplete(&rust_dir).then_some(()), - |()| complete(&rust_dir, |stage2| crate::clang::provision(stage2, &crate::llvm::resolve(root, &rust_dir, &rust_dir).dir)), - ); - assert_toolchain_is_honest(&stage2); +/// Point the rustup `toyos` toolchain at `sysroot`, through the one stable path +/// it ever names, [`store::current`]: that link is replaced by a rename, so no +/// `rustc` run through rustup ever finds the name dangling, and rustup's own +/// is made once. +fn link(rustup_home: &Path, rust_dir: &Path, sysroot: &Path) { + let current = store::current(rust_dir); + let target = sysroot.strip_prefix(current.parent().expect("the link has a parent")).unwrap_or(sysroot); + swap_link(target, ¤t); + swap_link(¤t, &rustup_home.join("toolchains/toyos")); +} - sysroot::ensure(root, &rust_dir, lock) +/// Make `at` a link to `to`, by a rename over whatever `at` was. +pub(crate) fn swap_link(to: &Path, at: &Path) { + if fs::read_link(at).is_ok_and(|now| now == to) { + return; + } + let dir = at.parent().expect("a link has a parent"); + fs::create_dir_all(dir).unwrap_or_else(|e| panic!("create {}: {e}", dir.display())); + let fresh = at.with_extension(format!("{}.new", std::process::id())); + let _ = fs::remove_file(&fresh); + std::os::unix::fs::symlink(to, &fresh).unwrap_or_else(|e| panic!("link {} -> {}: {e}", fresh.display(), to.display())); + fs::rename(&fresh, at).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", fresh.display(), at.display())); } /// Everything a checkout may do with a toolchain it did not build: check that @@ -529,18 +318,6 @@ fn check_installed_toolchain(root: &Path, rust_dir: &Path) { stage2.display(), ); - // Recreated rather than shipped: both of these point into whatever stable - // toolchain this machine has, which is not a path any artifact can know. - // This is CI's whole share of the cargo provisioning — it links its - // toolchain fresh from the published artifact on every run, so nothing - // upstream of the download can have put one there. Its clang is the - // artifact's own, so this is not `complete`. - if host_target_missing(rust_dir) { - link_host_target(rust_dir); - } - if cargo_link_stale(&stage2) { - provision_toolchain_cargo(&stage2); - } assert_toolchain_is_honest(&stage2); let want = sysroot::witness(root); @@ -555,31 +332,6 @@ fn check_installed_toolchain(root: &Path, rust_dir: &Path) { ); } -/// Whether the `toyos` rustup toolchain points anywhere other than `stage2`. -/// -/// `rustup toolchain link` unlinks and recreates the symlink rather than -/// replacing it atomically, so every call opens a window in which -/// `~/.rustup/toolchains/toyos` does not resolve. Any concurrent `rustc` proxy -/// invocation landing in that window dies with `'rustc' is not installed for the -/// custom toolchain 'toyos'` — which reads as a broken toolchain rather than as -/// contention, because a probe run a moment later succeeds. -/// -/// This ran unconditionally on every `ensure`, i.e. every build. With five -/// agents building in one tree it cost one of them eleven consecutive -/// `cargo test` invocations over about fifteen minutes, while -/// `RUSTUP_TOOLCHAIN=toyos rustc --version` succeeded 20 out of 20 between the -/// attempts. -/// -/// A mismatched or absent link still re-links, so a moved tree or a fresh clone -/// behaves as before; only the no-op case is skipped. -/// -/// Reached only from the primary checkout, which is what makes the window above -/// a window of one: a linked worktree that re-linked would point the name at a -/// stage2 nobody else has. -fn link_stale(stage2: &Path) -> bool { - rustup_link().is_none_or(|current| current != stage2) -} - /// Run bootstrap in the fork checkout `rust_dir`, streaming its output where it /// was going anyway and keeping a copy, with both the checkout's lockfiles put /// back as they were when this returns, however the build that holds them @@ -698,195 +450,6 @@ pub(crate) fn refuse_on_compile_error(log: &[String], what: &str) { panic!("{what} did not compile:\n{}", log[at..end].join("\n")); } -/// What an `x build` failure the artifact check is willing to tolerate actually -/// said, so that "expected" is a claim the reader can check. -fn tolerated_failure(log: &[String], what: &str) { - let errors: Vec<&str> = - log.iter().map(String::as_str).filter(|l| l.trim_start().starts_with("error")).collect(); - eprintln!( - "Note: {what} exited non-zero and the artifacts it must produce are all there, so this \ - is the ToyOS rustdoc link failure. It reported:\n{}", - if errors.is_empty() { - " (no line beginning `error`)".to_string() - } else { - errors.join("\n") - } - ); -} - -fn full_bootstrap(root: &Path, rust_dir: &Path, llvm: &Path) { - // Ensure library/backtrace is checked out — std depends on it. - // Other rust submodules (llvm, docs, cargo) are handled by bootstrap on demand. - crate::ensure_submodule(rust_dir, "library/backtrace"); - - // Write bootstrap.toml — ToyOS as target only, not host (fast rebuilds) - let host = host_triple(); - write_config(rust_dir, &host, false, llvm); - - // Clean cached std for all ToyOS targets so bootstrap picks up compiler changes - // (e.g. target spec changes like default_uwtable that affect codegen). - for target in GUEST_TARGETS { - let stage1_std = rust_dir.join(format!("build/{host}/stage1-std/{target}")); - if stage1_std.exists() { - fs::remove_dir_all(&stage1_std).ok(); - } - } - - let build = ["build", "--stage", "2", "--warnings", "warn"]; - let (ok, log) = x_build(rust_dir, &build, "the toolchain"); - - if !ok { - refuse_on_compile_error(&log, "the toolchain"); - // rustdoc for ToyOS may fail to link; rustc may not be missing. - let stage2 = rust_dir.join(format!("build/{host}/stage2")); - assert!( - stage2.join("bin/rustc").exists(), - "the toolchain build failed and {} is not there.\n\ - Nothing in its output was a compile error, so this is a link or a bootstrap \ - failure — the last lines above are the whole of what it said.", - stage2.join("bin/rustc").display() - ); - tolerated_failure(&log, "the toolchain build"); - } - for arch in Arch::ALL { - assert_std_built_from( - root, - &rust_dir.join(format!("build/{host}/stage1-std/{}", arch.userland())), - ); - } -} - -fn build_hosted_rustc(rust_dir: &Path, llvm: &Path) { - eprintln!("Building ToyOS-hosted rustc..."); - let host = host_triple(); - write_config(rust_dir, &host, true, llvm); - - let (ok, log) = - x_build(rust_dir, &["build", "--stage", "2", "--warnings", "warn"], "the hosted rustc"); - refuse_on_compile_error(&log, "the hosted rustc"); - - // rustdoc for ToyOS may fail to link; rustc and librustc_driver may not. - let toyos_stage2 = rust_dir.join(format!("build/{}/stage2", HOSTED_ARCH.userland())); - assert!( - toyos_stage2.join("bin/rustc").exists(), - "the hosted rustc build failed and {} is not there.\n\ - Nothing in its output was a compile error, so this is a link or a bootstrap failure.", - toyos_stage2.join("bin/rustc").display() - ); - assert!( - fs::read_dir(toyos_stage2.join("lib")) - .map(|d| d.filter_map(|e| e.ok()) - .any(|e| e.file_name().to_string_lossy().starts_with("librustc_driver"))) - .unwrap_or(false), - "the hosted rustc build failed: librustc_driver*.so is not in {}", - toyos_stage2.join("lib").display() - ); - if !ok { - tolerated_failure(&log, "the hosted rustc build"); - } - - // That build reassembled the host's `stage2` without `rust-lld` - // (`write_config` says why), so the host-only build runs once more to put - // it back: everything it would compile is already built. - write_config(rust_dir, &host, false, llvm); - let (ok, log) = x_build( - rust_dir, - &["build", "--stage", "2", "--warnings", "warn"], - "the toolchain, reassembled", - ); - refuse_on_compile_error(&log, "the toolchain, reassembled"); - assert!( - rust_lld(&stage2(rust_dir)).is_file(), - "the toolchain's reassembly after the hosted rustc left no {}", - rust_lld(&stage2(rust_dir)).display() - ); - if !ok { - tolerated_failure(&log, "the toolchain's reassembly"); - } -} - -/// `bootstrap.toml` for the host-only toolchain, or with the ToyOS-hosted rustc. -/// -/// `lld = true` is what puts `rust-lld` in every stage's sysroot, where rustc -/// finds the linker every guest target names. The hosted rustc's build cannot -/// have it: bootstrap would then build LLD for the ToyOS host from C++, which -/// nothing here can compile yet. Every assemble removes the host's -/// `stage2` first, so [`build_hosted_rustc`] reassembles it under the host-only -/// config after. -/// -/// `clang::LLVM_CONFIG` is the `[llvm]` both builds share, and both link the LLVM -/// at `llvm` (`src/llvm.rs`), whose LLD every guest target names by path. -/// -/// The host's `default-linker-linux-override` is pinned off because bootstrap -/// otherwise ties it to `lld` for `x86_64-unknown-linux-gnu`, and a host rustc -/// whose build environment flips with the config is rebuilt by each of those -/// two builds. -fn write_config(rust_dir: &Path, host: &str, with_hosted_rustc: bool, llvm: &Path) { - let host_line = if with_hosted_rustc { - format!("host = [\"{host}\", \"{}\"]", HOSTED_ARCH.userland()) - } else { - format!("host = [\"{host}\"]") - }; - let targets = std::iter::once(host) - .chain(GUEST_TARGETS) - .map(|t| format!("\"{t}\"")) - .collect::>() - .join(", "); - let userland: String = Arch::ALL - .iter() - .map(|arch| { - let linker = format!("linker = \"{}\"", llvm.join("bin/lld").display()); - let hosted = if with_hosted_rustc && *arch == HOSTED_ARCH { - // Cranelift because no LLVM is built for a ToyOS host yet, and - // only for that reason: the hosted rustc carries LLVM once clang - // and libc++ run on ToyOS, and Cranelift is not where the - // compiler that builds ToyOS goes. - // - // The archiver is that LLVM's too: the compiler's crates carry - // C built for this target (blake3's assembly), and a host `ar` - // that indexes only its own object format, as macOS's does, - // leaves those ELF members out of the index lld pulls from. - format!( - "\nar = \"{}\"\ncodegen-backends = [\"cranelift\"]", - llvm.join("bin/llvm-ar").display(), - ) - } else { - String::new() - }; - format!("[target.{}]\n{linker}{hosted}\nrpath = false\n\n", arch.userland()) - }) - .collect(); - let config = format!( - r#"change-id = "ignore" -profile = "compiler" - -[build] -{host_line} -target = [{targets}] - -[llvm] -{llvm} - -[rust] -incremental = true -lld = {lld} - -[target.{host}] -{HOST_LINKER_PIN} -{external} - -{userland}"#, - llvm = crate::clang::LLVM_CONFIG, - external = crate::llvm::host_lines(llvm), - lld = !with_hosted_rustc, - ); - fs::write(rust_dir.join("bootstrap.toml"), config).unwrap(); -} - -/// What the host rustc links its own binaries with, held to one answer in every -/// `bootstrap.toml` that builds a host compiler: [`write_config`] says why. -pub(crate) const HOST_LINKER_PIN: &str = "default-linker-linux-override = \"off\""; - /// The linker every guest target names, as the toolchain at `toolchain` carries /// it: `lib/rustlib//bin/rust-lld`, where rustc itself looks for it. pub fn rust_lld(toolchain: &Path) -> PathBuf { @@ -896,8 +459,7 @@ pub fn rust_lld(toolchain: &Path) -> PathBuf { /// The host triple, asked of rustc once per process. /// /// Every path built from it calls this, so an uncached one spent about seven -/// `rustc --version --verbose` spawns per build call — 0.118 s each, measured — -/// and they fell inside the windows the build lock now covers. +/// `rustc --version --verbose` spawns per build call — 0.118 s each, measured. pub fn host_triple() -> String { static HOST: OnceLock = OnceLock::new(); HOST.get_or_init(|| { @@ -914,49 +476,6 @@ pub fn host_triple() -> String { .clone() } -/// The stable host toolchain's sysroot, as `rustc --print sysroot` reports it. -/// -/// Both [`host_cargo`] and [`link_host_target`] resolve their answer through it -/// for the one reason [`host_cargo`]'s doc gives: it is whatever stable -/// toolchain this machine has, and that is not a path any artifact can name. -fn host_sysroot() -> PathBuf { - let output = Command::new("rustc") - .args(["--print", "sysroot"]) - .output() - .expect("Failed to run rustc"); - let sysroot = String::from_utf8(output.stdout).expect("rustc prints a path"); - PathBuf::from(sysroot.trim()) -} - -/// Whether the ToyOS sysroot is missing the host target proc-macros compile against. -fn host_target_missing(rust_dir: &Path) -> bool { - let toyos_sysroot = rust_dir.join(format!("build/{}/stage2/lib/rustlib", HOSTED_ARCH.userland())); - toyos_sysroot.exists() && !toyos_sysroot.join(host_triple()).exists() -} - -fn link_host_target(rust_dir: &Path) { - let host = host_triple(); - let host_target_dir = rust_dir - .join(format!("build/{}/stage2/lib/rustlib", HOSTED_ARCH.userland())) - .join(&host); - - let source = host_sysroot().join("lib/rustlib").join(&host); - assert!( - source.exists(), - "Host target {} not found in stable toolchain at {}", - host, - source.display() - ); - - std::os::unix::fs::symlink(&source, &host_target_dir).unwrap_or_else(|e| { - panic!( - "Failed to symlink {} -> {}: {}", - host_target_dir.display(), - source.display(), - e - ) - }); -} #[cfg(test)] mod tests { @@ -1019,10 +538,6 @@ mod tests { } /// **The layout that makes rustup narrate, as a decision.** - /// - /// The toolchain was given a cargo once, by hand, in a step the rebuild path - /// does not run; the 2026-08-14 sysroot rebuild recreated `bin/` without it - /// and nothing noticed, because nothing asked. This is the asking — /// [`assert_toolchain_is_honest`] is this function over the real `bin/`. #[test] fn a_toolchain_bin_without_cargo_is_one_rustup_narrates() { @@ -1032,20 +547,17 @@ mod tests { assert_eq!(narrated_binaries(&bin), ["rustc", "cargo"]); fs::write(bin.join("rustc"), b"").unwrap(); - assert_eq!(narrated_binaries(&bin), ["cargo"], "the layout every build had until now"); - assert!(cargo_link_stale(&stage2)); + assert_eq!(narrated_binaries(&bin), ["cargo"]); - // A link that rode in on the published artifact, naming a path only the - // publishing runner had. It is *there*, and it is a narrated fallback - // all the same — which is why the question is what it points at. + // A link naming a path only another machine had is there, and it is a + // narrated fallback all the same. let foreign = Path::new("/a-runner-that-is-not-this-one/bin/cargo"); std::os::unix::fs::symlink(foreign, bin.join("cargo")).unwrap(); assert_eq!(narrated_binaries(&bin), ["cargo"], "a dangling proxy is not a cargo"); - assert!(cargo_link_stale(&stage2), "another machine's cargo is not this one's"); - provision_toolchain_cargo(&stage2); + fs::remove_file(bin.join("cargo")).unwrap(); + fs::write(bin.join("cargo"), b"").unwrap(); assert!(narrated_binaries(&bin).is_empty()); - assert!(!cargo_link_stale(&stage2)); // Nothing narrates, and the toolchain is still refused: it has no linker. let refused = std::panic::catch_unwind(|| assert_toolchain_is_honest(&stage2)) @@ -1064,151 +576,49 @@ mod tests { assert!(said.contains("clang") && !said.contains("rust-lld,"), "the refusal names clang alone: {said}"); } - /// Every build links the host's LLVM, and every guest links through its - /// LLD, named by path. - #[test] - fn every_build_links_the_host_s_llvm_and_names_its_lld_by_path() { - let rust_dir = TempDir::new("lld-config"); - let llvm = rust_dir.join("build/llvm/k"); - let lld = format!("linker = \"{}\"", llvm.join("bin/lld").display()); - for (hosted, lld_flag) in [(true, "lld = false"), (false, "lld = true")] { - write_config(&rust_dir, "h", hosted, &llvm); - let config = fs::read_to_string(rust_dir.join("bootstrap.toml")).unwrap(); - assert!(config.contains(lld_flag) && config.contains(&lld) && !config.contains("\"rust-lld\""), "{config}"); - let host = format!( - "[target.h]\ndefault-linker-linux-override = \"off\"\nllvm-config = \"{}/bin/llvm-config\"\nllvm-has-rust-patches = true\n", - llvm.display() - ); - assert!(config.contains(&host), "{config}"); - } - } - - /// **A bootstrap leaves the primary nothing that waits on another - /// worktree's sysroot build**: the act that reassembles `stage2` completes it - /// before its exclusive hold ends, so the step after it — run while a - /// sysroot build holds the lock shared — decides it has nothing to do, and - /// takes no lock. - #[test] - fn a_bootstrap_leaves_nothing_to_wait_on_a_sysroot_build_for() { - let rust_dir = TempDir::new("no-wait"); - let stage2 = stage2(&rust_dir); - let llvm = store_llvm(&rust_dir); - reassemble(&rust_dir, &llvm, || bootstrapped(&rust_dir)); - assert!( - !incomplete(&rust_dir), - "a bootstrap let its exclusive hold go with a global step left, which the primary's \ - build then queues for behind every sysroot build" - ); - assert_eq!(toolchain_defect(&stage2), None, "a bootstrap let its exclusive hold go with stage2 not whole"); - } - - /// The LLVM `clang::provision` reads, in `rust_dir`'s store. - fn store_llvm(rust_dir: &Path) -> PathBuf { - let llvm = rust_dir.join("build/llvm/k"); - for (file, text) in [("bin/clang", "clang"), ("lib/clang/22/include/stddef.h", "stddef")] { - fs::create_dir_all(llvm.join(file).parent().unwrap()).unwrap(); - fs::write(llvm.join(file), text).unwrap(); - } - llvm - } - - /// What bootstrap leaves in `rust_dir`: `stage2` made again, with `rustc` - /// and the LLVM tools it assembles, and neither cargo nor clang; and the - /// hosted rustc's sysroot without the host target. - fn bootstrapped(rust_dir: &Path) { - let stage2 = stage2(rust_dir); - let _ = fs::remove_dir_all(&stage2); - let lld = rust_lld(&stage2); - for file in [stage2.join("bin/rustc"), lld.clone(), lld.with_file_name("llvm-ar")] { - fs::create_dir_all(file.parent().unwrap()).unwrap(); - fs::write(file, b"").unwrap(); - } - let hosted = rust_dir.join(format!("build/{}/stage2/lib/rustlib", HOSTED_ARCH.userland())); - fs::create_dir_all(&hosted).unwrap(); - } - - /// What a build directory holds of an LLVM of its own: bootstrap's LLVM and - /// LLD. - fn in_tree_llvm(rust_dir: &Path) -> [PathBuf; 2] { - let host = rust_dir.join("build").join(host_triple()); - let own = [host.join("llvm"), host.join("lld")]; - for dir in &own { - fs::create_dir_all(dir.join("bin")).unwrap(); - fs::write(dir.join("bin/tool"), "a tool").unwrap(); - } - own - } - - /// **A primary whose compiler is built against the store keeps no LLVM of - /// its own**: the rebuild that links the store removes the one its build - /// directory built. + /// **The rustup `toyos` toolchain names one stable path, ever**: the + /// primary's builds move the link at that path, and rustup's own is made + /// once and never moved again. #[test] - fn a_rebuilt_compiler_leaves_no_llvm_of_its_own() { - let scratch = TempDir::new("in-tree-llvm"); - let (_primary, rust_dir, _) = crate::compiler::tests::estate(&scratch); - let own = in_tree_llvm(&rust_dir); - let llvm = store_llvm(&rust_dir); - rebuild_compiler(&rust_dir, &llvm, || bootstrapped(&rust_dir)); - for dir in own { - assert!(!dir.exists() && !dir.with_extension("swept").exists(), "{} outlived the rebuild", dir.display()); - } - assert_eq!(toolchain_defect(&stage2(&rust_dir)), None); - } - - /// **Landing the store moves an existing primary onto it**: a primary whose - /// record was written before its compiler linked the host's LLVM is not - /// current, so its next build bootstraps, and that rebuild removes the LLVM - /// and LLD its build directory built. Its LLVM checkout sitting at a commit - /// its gitlink does not name changes neither answer. - #[test] - fn a_primary_recorded_before_the_store_is_rebuilt_onto_it() { - let scratch = TempDir::new("store-migration"); - let (_primary, rust_dir, _) = crate::compiler::tests::estate(&scratch); - crate::compiler::tests::llvm_checkout(&rust_dir); - assert!(crate::compiler::primary_is_current(&rust_dir)); - crate::compiler::tests::record_before_the_store(&rust_dir); - let own = in_tree_llvm(&rust_dir); - let kind = bootstrap(crate::compiler::primary_is_current(&rust_dir), true); - assert!(kind.is_some_and(|k| k.invalidate_hosted), "a primary recorded before the store was taken for current"); - rebuild_compiler(&rust_dir, &store_llvm(&rust_dir), || bootstrapped(&rust_dir)); - assert!(own.iter().all(|dir| !dir.exists()), "the rebuild kept the LLVM its build directory built"); - assert!(crate::compiler::primary_is_current(&rust_dir), "the rebuild recorded a compiler that is not current"); - } - - /// **A stopped bootstrap is run again**: nothing records which compiler - /// `stage2` is while one runs, so the primary's next build is not told the - /// old one is current; and the LLVM the old one linked stays. - #[test] - fn a_stopped_bootstrap_leaves_no_record() { - let rust_dir = TempDir::new("stopped"); - let record = crate::compiler::primary_record(&rust_dir); - fs::create_dir_all(record.parent().unwrap()).unwrap(); - fs::write(&record, "the compiler before").unwrap(); - let own = in_tree_llvm(&rust_dir); - let stopped = std::panic::catch_unwind(|| rebuild_compiler(&rust_dir, Path::new("no-llvm"), || panic!("stopped"))); - assert!(stopped.is_err()); - assert!(!record.exists(), "a stopped bootstrap left the record of the compiler before it"); - assert!(own.iter().all(|dir| dir.join("bin/tool").is_file()), "a stopped bootstrap took the LLVM its compiler linked"); - } - - /// **The primary bootstraps a new compiler exactly when its `stage2` is not - /// current, and otherwise only when rustup has none.** + fn the_rustup_link_names_one_stable_path() { + let scratch = TempDir::new("rustup-link"); + let (home, rust_dir) = (scratch.join("rustup"), scratch.join("rust")); + let [one, two] = ["k1", "k2"].map(|k| store::Kind::Sysroot.dir(&rust_dir).join(k)); + let toyos = home.join("toolchains/toyos"); + link(&home, &rust_dir, &one); + assert_eq!(fs::read_link(&toyos).unwrap(), store::current(&rust_dir)); + assert_eq!(fs::read_link(store::current(&rust_dir)).unwrap(), Path::new("sysroots/k1")); + let made = fs::symlink_metadata(&toyos).unwrap().modified().unwrap(); + link(&home, &rust_dir, &two); + assert_eq!(fs::read_link(store::current(&rust_dir)).unwrap(), Path::new("sysroots/k2")); + assert_eq!(fs::symlink_metadata(&toyos).unwrap().modified().unwrap(), made, "rustup's own link was made again"); + let left: Vec<_> = fs::read_dir(rust_dir.join("build")).unwrap().flatten().map(|e| e.file_name()).collect(); + assert_eq!(left, ["toyos"], "a link's replacement was left beside it"); + } + + /// **A std is this worktree's when what it compiled resolves into it**: the + /// shared checkout reaches a worktree's `toyos-abi` through a link beside + /// it, and a link to another worktree's is refused. #[test] - fn the_primary_bootstraps_when_stale_or_missing() { - let new = Some(Bootstrap { invalidate_hosted: true }); - let again = Some(Bootstrap { invalidate_hosted: false }); - for (current, toolchain_exists, want) in [ - (true, true, None), - (true, false, again), - (false, true, new), - (false, false, new), - ] { - assert_eq!( - bootstrap(current, toolchain_exists), - want, - "current {current}, toolchain_exists {toolchain_exists}" - ); + fn a_std_compiled_through_links_is_the_worktree_they_resolve_to() { + let scratch = TempDir::new("std-through-links"); + let [mine, theirs, beside, built] = ["mine", "theirs", "shared", "built"].map(|d| scratch.join(d)); + for worktree in [&mine, &theirs] { + fs::create_dir_all(worktree.join("toyos-abi/src")).unwrap(); + fs::write(worktree.join("toyos-abi/src/lib.rs"), "pub struct A;\n").unwrap(); } + fs::create_dir_all(&beside).unwrap(); + fs::create_dir_all(&built).unwrap(); + let through = beside.join("toyos-abi/src/lib.rs"); + fs::write(built.join("toyos_abi.d"), format!("{}: {}\n", built.join("libtoyos_abi.rlib").display(), through.display())).unwrap(); + + std::os::unix::fs::symlink(mine.join("toyos-abi"), beside.join("toyos-abi")).unwrap(); + assert_std_built_from(&mine, &built); + fs::remove_file(beside.join("toyos-abi")).unwrap(); + std::os::unix::fs::symlink(theirs.join("toyos-abi"), beside.join("toyos-abi")).unwrap(); + let refused = std::panic::catch_unwind(|| assert_std_built_from(&mine, &built)); + let said = refused.expect_err("a std compiled against another worktree's ABI was taken for this one's"); + assert!(said.downcast_ref::().is_some_and(|s| s.contains(&through.display().to_string()))); } /// The negative control is the defect itself: this is verbatim what cargo diff --git a/src/worktree.rs b/src/worktree.rs deleted file mode 100644 index 47236ec82dd..00000000000 --- a/src/worktree.rs +++ /dev/null @@ -1,837 +0,0 @@ -//! Making a linked worktree buildable, and saying why when it cannot be. -//! -//! `git worktree add` alone leaves a tree that does not build and, worse, one -//! that builds *wrongly*: `rust/` comes out an empty stub, so the build system -//! reads it as a missing submodule, clones 913 MiB from the network, bootstraps -//! a second 47 GiB toolchain, and finally points the machine-global rustup -//! `toyos` name at it — taking the toolchain out from under every other -//! checkout. Measured, in that order, on this host. -//! -//! So the compiler stays the primary's and nothing here copies it: -//! [`crate::toolchain::rust_dir`] sends every compiler read to the primary -//! checkout. `rust/` is left the stub it was until the first build makes it -//! this worktree's own fork checkout — a git worktree of the primary's fork -//! repository, sharing its objects (`src/sysroot.rs`) — which [`remove`] takes -//! away again. What this module does is the small remainder — create the -//! worktree, carry over the one file git cannot, and refuse by name when the -//! result would not be usable. - -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::Command; - -use crate::buildlock::Keyed; -use crate::flags; -use crate::toolchain; - -/// What a worktree's crate target directories reach: 4.1 GiB after -/// `--build-only`, and 23 GiB on the primary checkout, which has run everything; -/// its fork checkout and std build directory add 4.2 GiB. Measured with `du`. -/// The primary's 50 GiB `rust/` is shared and never counted here. -/// -/// Refusing at the upper figure plus a little, rather than at the lower one: a -/// build that fills the disk halfway through costs more than a worktree that -/// was never made. -const NEEDED_BYTES: u64 = 25 * 1024 * 1024 * 1024; - -pub fn dispatch(root: &Path, args: &[String]) { - let mut rest = flags::CARGO_RUN.rest(args, &flags::WORKTREE).iter(); - let verb = rest.next().map(String::as_str); - let operand = rest.next().cloned(); - match verb { - Some("add") => add(root, &path_operand("add", operand)), - Some("list") => list(root), - Some("remove") => remove(root, &path_operand("remove", operand)), - other => panic!( - "--worktree takes add , list, or remove ; got {other:?}" - ), - } -} - -fn path_operand(verb: &str, operand: Option) -> String { - let path = operand.unwrap_or_else(|| panic!("--worktree {verb} needs a path")); - assert!( - !path.starts_with('-'), - "--worktree {verb} needs a path, got flag {path:?}" - ); - path -} - -/// Create a worktree and leave it in a state where `cargo run -- --build-only` -/// works. -fn add(root: &Path, path: &str) { - let path = PathBuf::from(path); - assert!(!path.exists(), "{} already exists", path.display()); - let name = path - .file_name() - .unwrap_or_else(|| panic!("{} has no final component", path.display())) - .to_string_lossy() - .to_string(); - - // Everything that would make the result unusable, asked before anything is - // created: a half-made worktree is worse than none, because the next agent - // finds it and believes it. - let primary = match toolchain::owner(root) { - toolchain::Owner::Us | toolchain::Owner::Installed => root.to_path_buf(), - toolchain::Owner::Elsewhere(p) => p, - }; - let stage2 = primary.join(format!( - "rust/build/{}/stage2", - toolchain::host_triple() - )); - assert!( - stage2.join("bin/rustc").exists(), - "the shared toolchain does not exist yet ({} is missing).\n\ - Run `cargo run -- --build-only` in {} before making worktrees of it.", - stage2.display(), - primary.display() - ); - let free = free_bytes(path.parent().unwrap_or(Path::new("/"))); - assert!( - free >= NEEDED_BYTES, - "{} has {:.1} GiB free and a worktree's target directories reach about \ - {:.0} GiB.\nThe shared toolchain is not copied, but the crate targets are \ - its own.", - path.parent().unwrap_or(Path::new("/")).display(), - free as f64 / 1024.0_f64.powi(3), - NEEDED_BYTES as f64 / 1024.0_f64.powi(3), - ); - - let summary = create_worktree(root, &path, &name); - - // `rust/` is deliberately left the empty stub `git worktree add` made: the - // first build makes it a fork checkout sharing the primary's objects, where - // `git submodule update` would be the 913 MiB clone, and a symlink in its - // place makes git error out of `status`, `diff` and `submodule` alike. - - // The one file git cannot carry: it is gitignored, and a worktree that - // silently loses the fork redirects would build different code from the - // checkout it was made from and report the difference as a result. - let redirects = root.join(".cargo/config.toml"); - if redirects.exists() { - fs::copy(&redirects, path.join(".cargo/config.toml")) - .unwrap_or_else(|e| panic!("copy {}: {e}", redirects.display())); - eprintln!("carried over .cargo/config.toml (fork redirects)"); - } - - eprintln!(); - eprintln!("worktree {}", path.display()); - eprintln!("branch {summary}"); - eprintln!("compiler {} (shared, not copied)", stage2.display()); - eprintln!(); - eprintln!("Build it with `cargo run -- --build-only` from {}.", path.display()); -} - -/// Never a fetch, never a reset. Every refusal below runs before either -/// branch is touched, so a half-made worktree never sits behind one. -fn create_worktree(root: &Path, path: &Path, name: &str) -> String { - let branch = format!("wt/{name}"); - let upstream = format!("origin/{branch}"); - let local_exists = ok(root, &["show-ref", "--verify", "--quiet", &format!("refs/heads/{branch}")]); - let origin_exists = ok(root, &["show-ref", "--verify", "--quiet", &format!("refs/remotes/{upstream}")]); - let path_str = path.to_string_lossy(); - if local_exists { - refuse_if_no_commit_beyond_main( - root, - &branch, - &format!("delete it with `git branch -d {branch}` or pick a new name for the worktree."), - ); - if origin_exists { - refuse_if_behind_or_diverged(root, &branch, &upstream); - } - git(root, &["worktree", "add", &path_str, &branch]); - format!("{branch} (resumed at {})", short_sha(path, "HEAD")) - } else if origin_exists { - refuse_if_no_commit_beyond_main(root, &upstream, "pick a new name for the worktree."); - git(root, &["worktree", "add", "--track", "-b", &branch, &path_str, &upstream]); - format!("{branch} (resumed from {upstream} at {})", short_sha(path, "HEAD")) - } else { - git(root, &["worktree", "add", "-b", &branch, &path_str, "main"]); - format!("{branch} (new, from main at {})", short_sha(path, "HEAD")) - } -} - -/// Refuse by name, before anything is created, when `resolve` carries no -/// commit beyond `origin/main` — the same ancestry test [`measure`] uses to -/// call a worktree landed and offer its build caches back. -/// -/// `merge-base --is-ancestor` cannot tell a branch that landed apart from one -/// that never diverged from `main` in the first place: both are true of it. -/// The message says only what both share, and never "landed" or "merged" — -/// a resume would otherwise start work from an old tip behind main, or from -/// a branch that never carried any work of its own. -fn refuse_if_no_commit_beyond_main(root: &Path, resolve: &str, hint: &str) { - assert!( - !ok(root, &["merge-base", "--is-ancestor", resolve, "origin/main"]), - "{resolve} carries no commit beyond origin/main; {hint}" - ); -} - -/// Refuse by name, before anything is created, when the local `branch` is not -/// at or ahead of its own `upstream`: a resume would otherwise pick the local -/// tip silently, and the push back would refuse for the same reason, later -/// and less clearly. -fn refuse_if_behind_or_diverged(root: &Path, branch: &str, upstream: &str) { - if ok(root, &["merge-base", "--is-ancestor", upstream, branch]) { - return; - } - let relation = if ok(root, &["merge-base", "--is-ancestor", branch, upstream]) { - "is behind" - } else { - "has diverged from" - }; - panic!( - "{branch} ({}) {relation} {upstream} ({}); merge it first.", - short_sha(root, branch), - short_sha(root, upstream), - ); -} - -fn short_sha(dir: &Path, rev: &str) -> String { - capture(dir, &["rev-parse", "--short", rev]).trim().to_string() -} - -fn list(root: &Path) { - let primary = match toolchain::owner(root) { - toolchain::Owner::Us | toolchain::Owner::Installed => root.to_path_buf(), - toolchain::Owner::Elsewhere(p) => p, - }; - eprintln!("toolchain owner {}", primary.display()); - eprintln!( - "rustup toyos {}", - fs::read_link( - std::env::var_os("HOME") - .map(PathBuf::from) - .unwrap_or_default() - .join(".rustup/toolchains/toyos") - ) - .map_or_else(|_| "unlinked".to_string(), |p| p.display().to_string()) - ); - eprintln!(); - let trees = survey(root, true); - for tree in &trees { - let branch = if tree.branch.is_empty() { "(detached)" } else { &tree.branch }; - let note = match (tree.primary, tree.landed) { - (true, _) => " primary", - (_, true) => " landed — reclaimable", - _ => "", - }; - eprintln!( - "{:<44} {:<26} {:>9} in {:>2} target dir(s){note}", - tree.path.display(), - branch, - gib(tree.bytes), - tree.targets, - ); - } - eprintln!(); - eprintln!( - "{} worktree(s), {} of build caches; the shared toolchain is not counted", - trees.len(), - gib(trees.iter().map(|t| t.bytes).sum()), - ); - if let Some(line) = reclaim_line(&trees) { - eprintln!("{line}"); - } -} - -/// One worktree, and the two facts that decide whether it should still exist. -/// -/// **Nothing ever reclaimed one**, and `add`'s disk check was the whole of what -/// this subject had — a refusal is the last notice rather than the first. A -/// worktree whose branch has landed has no reason to hold its build caches, and -/// neither its size nor whether its branch is in `origin/main` is anything -/// `git worktree list` says. -pub struct Tree { - pub path: PathBuf, - /// Empty for a detached worktree. - pub branch: String, - /// What its build caches hold. The shared `rust/` is never counted. - pub bytes: u64, - pub targets: usize, - /// The checkout that owns `rust/`, the rustup link and `main`. Never - /// reclaimable whatever its branch says. - pub primary: bool, - /// Its branch is already in `origin/main`. - pub landed: bool, -} - -/// Every worktree of `root`. -/// -/// `all_sizes` walks every worktree's caches, which is a metadata walk of tens -/// of gigabytes and takes seconds; `false` walks only the ones that could be -/// given back, which is the only size `--sync` prints. -pub fn survey(root: &Path, all_sizes: bool) -> Vec { - let mut trees = Vec::new(); - let mut path: Option = None; - let mut branch = String::new(); - let listing = capture(root, &["worktree", "list", "--porcelain"]); - for line in listing.lines() { - if let Some(next) = line.strip_prefix("worktree ") { - if let Some(done) = path.replace(PathBuf::from(next)) { - let first = trees.is_empty(); - trees.push(measure(root, done, std::mem::take(&mut branch), first, all_sizes)); - } - } else if let Some(name) = line.strip_prefix("branch ") { - branch = name.trim_start_matches("refs/heads/").to_string(); - } - } - if let Some(done) = path { - let first = trees.is_empty(); - trees.push(measure(root, done, branch, first, all_sizes)); - } - trees -} - -/// What could be given back, or nothing to say. -/// -/// `--sync` reports this as well as `list`, because `--sync` runs at the moment -/// a branch lands, which is the moment its worktree stops having a reason to -/// exist. -pub fn reclaim_line(trees: &[Tree]) -> Option { - let done: Vec<&Tree> = trees.iter().filter(|t| !t.primary && t.landed).collect(); - if done.is_empty() { - return None; - } - Some(format!( - "{} worktree(s) hold {} on branches already in origin/main: {}\n\ - `cargo run -- --worktree remove ` gives each back, and refuses one carrying \ - uncommitted work.", - done.len(), - gib(done.iter().map(|t| t.bytes).sum()), - done.iter().map(|t| t.path.display().to_string()).collect::>().join(", "), - )) -} - -fn measure( - root: &Path, - path: PathBuf, - branch: String, - primary: bool, - all_sizes: bool, -) -> Tree { - let landed = !primary - && !branch.is_empty() - && ok(root, &["merge-base", "--is-ancestor", &branch, "origin/main"]); - let mut bytes = 0; - let mut targets = 0; - if all_sizes || landed { - caches(&path, &mut bytes, &mut targets); - } - Tree { path, branch, bytes, targets, primary, landed } -} - -/// Directories a survey never enters: the shared toolchain and git's own store. -const NOT_OURS: &[&str] = &["rust", ".git"]; - -/// Ten `target/` directories per worktree is the design and not an accident — -/// `Cargo.toml`'s `exclude` list keeps five cross-compiled crates out of the -/// host workspace and each guest fixture resolves on its own — so `cargo clean` -/// at the root reaches exactly one of them and a count is worth printing. -fn caches(dir: &Path, bytes: &mut u64, targets: &mut usize) { - let Ok(entries) = fs::read_dir(dir) else { return }; - for entry in entries.flatten() { - let path = entry.path(); - if !fs::symlink_metadata(&path).is_ok_and(|m| m.is_dir()) { - continue; - } - let name = entry.file_name(); - let name = name.to_string_lossy(); - if NOT_OURS.contains(&name.as_ref()) { - continue; - } - if name == "target" { - *bytes += bytes_under(&path); - *targets += 1; - continue; - } - caches(&path, bytes, targets); - } -} - -fn bytes_under(dir: &Path) -> u64 { - let Ok(entries) = fs::read_dir(dir) else { return 0 }; - let mut total = 0; - for entry in entries.flatten() { - let path = entry.path(); - let Ok(meta) = fs::symlink_metadata(&path) else { continue }; - total += if meta.is_dir() { bytes_under(&path) } else { meta.len() }; - } - total -} - -fn gib(bytes: u64) -> String { - format!("{:.1} GiB", bytes as f64 / 1024.0_f64.powi(3)) -} - -/// Remove a worktree and the branch it was made with. -/// -/// Deliberately not `--force`: git refuses a worktree holding tracked changes -/// or untracked files and leaves it registered, because the work in a -/// worktree is the only copy of itself — that refusal stands, and so does the -/// one for its own fork checkout ([`remove_fork_checkout`]). -/// -/// **git can unregister a worktree and then fail to delete it**: a path deeper -/// than `PATH_MAX` under an ignored `target/`, or a file created while it -/// walks, and it exits non-zero with the directory still there and no longer -/// a worktree of anything. Once git has let go of it nothing in it is work, -/// so the whole directory goes. -/// -/// Then every sysroot, compiler and LLVM no remaining worktree names goes too -/// (`src/sysroot.rs`, `src/compiler.rs`, `src/llvm.rs`). -pub(crate) fn remove(root: &Path, path: &str) { - let at = root.join(path); - remove_fork_checkout(root, &at); - if !ok_loud(root, &["worktree", "remove", path]) { - assert!( - !registered(root, &at), - "git refused to remove {path} and it is still a worktree; what it said above is \ - why. Nothing was deleted." - ); - remove_tree(&at); - eprintln!("git unregistered {path} and left its ignored files; deleted them"); - } - eprintln!("removed {path}; its branch is still there, and `git branch -d` will say if it is unmerged"); - let rust_dir = crate::toolchain::rust_dir(root); - for (kind, store, what) in [ - (Keyed::Sysroot, crate::sysroot::sysroots_dir(&rust_dir), "sysroot"), - (Keyed::Compiler, crate::compiler::compilers_dir(&rust_dir), "compiler"), - (Keyed::Llvm, crate::llvm::store(&rust_dir), "LLVM"), - ] { - let swept = crate::keystore::sweep(root, kind, &store); - if !swept.is_empty() { - eprintln!("removed {} {what}(s) no worktree names any more", swept.len()); - } - } -} - -/// A linked worktree's own fork checkout (`src/sysroot.rs`'s `fork_checkout`) -/// is a git worktree of the primary's fork repository, which git will not -/// remove a worktree around. It goes first, and only while it holds nothing -/// that is not also somewhere else: no change in its tree, and a `HEAD` some -/// ref of the fork repository reaches. -fn remove_fork_checkout(root: &Path, at: &Path) { - let fork = at.join("rust"); - if !fork.join(".git").is_file() || !at.join(".git").is_file() { - return; - } - let path = at.display(); - let mine = capture(at, &["status", "--porcelain", "--ignore-submodules=all"]); - assert!(mine.is_empty(), "{path} holds uncommitted work:\n{mine}Nothing was deleted."); - let theirs = capture(&fork, &["status", "--porcelain", "--ignore-submodules=none"]); - assert!( - theirs.is_empty(), - "{}'s fork checkout holds uncommitted work:\n{theirs}Nothing was deleted.", - path - ); - let head = capture(&fork, &["rev-parse", "HEAD"]); - let reached = capture(&fork, &["for-each-ref", "--count=1", "--contains", head.trim()]); - assert!( - !reached.trim().is_empty(), - "{}'s fork checkout is at {}, which no branch, tag or remote ref of the fork \ - repository reaches: it is the only copy of those commits. Push them, or name them \ - with a branch, first. Nothing was deleted.", - path, - head.trim() - ); - // Moved out whole first, so a removal a writer interrupts leaves a named - // directory outside the worktree rather than a half-deleted checkout in it. - let name = at.file_name().expect("a worktree has a name").to_string_lossy(); - let aside = at.with_file_name(format!(".{name}-rust.removing")); - fs::rename(&fork, &aside) - .unwrap_or_else(|e| panic!("move {} to {}: {e}", fork.display(), aside.display())); - fs::create_dir(&fork).unwrap_or_else(|e| panic!("recreate the stub {}: {e}", fork.display())); - let primary = crate::primary_checkout(root); - git(&primary.join("rust"), &["worktree", "prune"]); - let backtrace = primary.join("rust/library/backtrace"); - if backtrace.join(".git").exists() { - git(&backtrace, &["worktree", "prune"]); - } - remove_tree(&aside); -} - -/// Remove `dir` and everything in it, including what appears while it goes. -/// -/// A writer on this host — the leftovers are `.DS_Store` files — can put a file -/// into a directory while it is being emptied, so a plain recursive delete finds a directory it has just emptied not empty and -/// stops halfway — the `Directory not empty` git itself dies on. The removal -/// runs again over what is left, at most [`PASSES`] times; a tree still refusing -/// after that has a writer this cannot outrun, and the panic says so. -fn remove_tree(dir: &Path) { - for pass in 1..=PASSES { - match fs::remove_dir_all(dir) { - Ok(()) => return, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return, - Err(e) if e.kind() == std::io::ErrorKind::DirectoryNotEmpty && pass < PASSES => { - eprintln!("{} gained files while it was removed ({e}); removing again", dir.display()); - } - Err(e) => panic!("remove {}: {e}, after {pass} pass(es)", dir.display()), - } - } -} - -/// How many times [`remove_tree`] runs over a tree that keeps refusing. -const PASSES: usize = 10; - -/// Whether `git worktree list` still names `at`, compared as real paths: -/// git prints its own realpath, `/private/tmp/…` for `/tmp/…`. -fn registered(root: &Path, at: &Path) -> bool { - let at = fs::canonicalize(at).unwrap_or_else(|_| at.to_path_buf()); - capture(root, &["worktree", "list", "--porcelain"]) - .lines() - .filter_map(|l| l.strip_prefix("worktree ")) - .any(|listed| fs::canonicalize(listed).unwrap_or_else(|_| PathBuf::from(listed)) == at) -} - -fn free_bytes(dir: &Path) -> u64 { - let path = std::ffi::CString::new(dir.as_os_str().as_encoded_bytes()) - .unwrap_or_else(|_| panic!("{} has an embedded NUL", dir.display())); - let mut buf: libc::statvfs = unsafe { std::mem::zeroed() }; - // SAFETY: `path` is a valid, NUL-terminated C string and `buf` is a - // `libc::statvfs` the kernel fills in whole or leaves at the `zeroed()` - // above; a non-zero return is checked before anything reads it. - let rc = unsafe { libc::statvfs(path.as_ptr(), &mut buf) }; - assert!(rc == 0, "statvfs {}: {}", dir.display(), std::io::Error::last_os_error()); - buf.f_bavail as u64 * buf.f_frsize as u64 -} - -fn git(dir: &Path, args: &[&str]) { - let status = Command::new("git") - .args(args) - .current_dir(dir) - .status() - .unwrap_or_else(|e| panic!("run git: {e}")); - assert!(status.success(), "git {args:?} failed"); -} - -/// git's answer, for a question rather than an action. -fn capture(dir: &Path, args: &[&str]) -> String { - let out = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .unwrap_or_else(|e| panic!("run git: {e}")); - assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr).trim()); - String::from_utf8_lossy(&out.stdout).into_owned() -} - -/// Whether git did it, with what it said left on stderr for the reader. -fn ok_loud(dir: &Path, args: &[&str]) -> bool { - Command::new("git") - .args(args) - .current_dir(dir) - .status() - .unwrap_or_else(|e| panic!("run git: {e}")) - .success() -} - -/// Whether git says yes. A non-zero exit is the answer here, never a failure. -fn ok(dir: &Path, args: &[&str]) -> bool { - Command::new("git") - .args(args) - .current_dir(dir) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status() - .is_ok_and(|s| s.success()) -} - -#[cfg(test)] -mod tests { - use super::*; - use toyos_tmpdir::TempDir; - - /// `--worktree` owns the rest of the command line, so this refusal is the - /// only one between `--worktree add --help` and a worktree named `--help`. - #[test] - fn a_flag_is_refused_as_a_worktree_path_by_name() { - let args = ["--worktree", "add", "--help"].map(String::from); - assert!( - matches!(crate::flags::check(&args), crate::flags::Outcome::Proceed), - "the command line has to reach this dispatch" - ); - let panic = std::panic::catch_unwind(|| dispatch(Path::new("/not-used"), &args)) - .expect_err("a flag is not a worktree path"); - let message = panic.downcast::().expect("the refusal is formatted"); - assert!(message.contains("--help"), "the refusal must name the bad argument: {message}"); - } - - fn tree(path: &str, primary: bool, landed: bool, bytes: u64) -> Tree { - Tree { - path: PathBuf::from(path), - branch: String::from("wt/x"), - bytes, - targets: 10, - primary, - landed, - } - } - - /// **The primary checkout sits on `main`**, which is an ancestor of - /// `origin/main` by construction, so a rule that offered back every landed - /// worktree would offer back the one holding `rust/` and the rustup link. - #[test] - fn only_a_landed_worktree_that_is_not_the_primary_is_offered_back() { - assert!(reclaim_line(&[tree("/primary", true, true, 4 << 30)]).is_none()); - assert!(reclaim_line(&[tree("/live", false, false, 8 << 30)]).is_none()); - let line = reclaim_line(&[ - tree("/primary", true, true, 4 << 30), - tree("/gone", false, true, 2 << 30), - tree("/live", false, false, 8 << 30), - ]) - .expect("a landed worktree that is not the primary is reclaimable"); - assert!(line.contains("/gone"), "{line}"); - assert!(!line.contains("/live"), "{line}"); - assert!(!line.contains("/primary"), "{line}"); - assert!(line.contains("2.0 GiB"), "the offer has to say what it is worth: {line}"); - } - - #[test] - fn a_local_branch_is_resumed_at_its_own_commit_not_mains() { - let (dir, _origin, work) = crate::gitfixture::repo("wtresume-local"); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["branch", "-qD", "wt"]); - git(&work, &["checkout", "-qb", "wt/foo", "main"]); - crate::gitfixture::commit(&work, "on-branch", "branch work\n", "branch work"); - git(&work, &["push", "-q", "-u", "origin", "wt/foo"]); - crate::gitfixture::commit(&work, "on-branch-2", "more branch work\n", "more branch work"); - git(&work, &["checkout", "-q", "main"]); - crate::gitfixture::commit(&work, "on-main", "main moved on\n", "main moved on"); - - let path = dir.join("resumed"); - let summary = create_worktree(&work, &path, "foo"); - - assert!(summary.contains("resumed at"), "{summary}"); - assert!(!summary.contains("main"), "{summary}"); - let branch_sha = capture(&work, &["rev-parse", "wt/foo"]); - let origin_sha = capture(&work, &["rev-parse", "origin/wt/foo"]); - let worktree_sha = capture(&path, &["rev-parse", "HEAD"]); - let main_sha = capture(&work, &["rev-parse", "main"]); - assert_ne!(branch_sha, origin_sha, "the local tip must be ahead of origin's, or this proves nothing"); - assert_eq!(worktree_sha, branch_sha, "must resume at the local branch's own tip, not origin's"); - assert_ne!(worktree_sha, main_sha, "must not have been reset onto main"); - } - - /// A local branch already merged into `origin/main` is refused by name - /// rather than resumed from its old tip behind main. - #[test] - fn a_landed_local_branch_is_refused_not_resumed() { - let (dir, _origin, work) = crate::gitfixture::repo("wtresume-landed-local"); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["branch", "-qD", "wt"]); - git(&work, &["checkout", "-qb", "wt/foo", "main"]); - crate::gitfixture::commit(&work, "on-branch", "branch work\n", "branch work"); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["merge", "-q", "--no-ff", "wt/foo", "-m", "merge wt/foo"]); - git(&work, &["push", "-q", "origin", "main"]); - - let path = dir.join("resumed"); - let refused = std::panic::catch_unwind(|| { - create_worktree(&work, &path, "foo") - }); - - let panic = refused.expect_err("a landed branch must not be resumed"); - let message = panic.downcast::().expect("the refusal is formatted"); - assert!(message.contains("wt/foo"), "{message}"); - assert!(!path.exists(), "nothing must be created before the refusal"); - } - - /// A branch made from `main` and never touched carries no commit beyond - /// `origin/main` either, and `merge-base --is-ancestor` cannot tell it - /// apart from one that actually landed — refused before anything is - /// created, for the reason that is true of it, never "landed". - #[test] - fn an_untouched_branch_is_refused_not_resumed() { - let (dir, _origin, work) = crate::gitfixture::repo("wtresume-untouched"); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["branch", "-qD", "wt"]); - git(&work, &["checkout", "-qb", "wt/foo", "main"]); - git(&work, &["checkout", "-q", "main"]); - - let path = dir.join("resumed"); - let refused = std::panic::catch_unwind(|| create_worktree(&work, &path, "foo")); - - let panic = refused.expect_err("an untouched branch must not be resumed"); - let message = panic.downcast::().expect("the refusal is formatted"); - assert!(message.contains("wt/foo"), "{message}"); - assert!(message.contains("carries no commit beyond origin/main"), "{message}"); - assert!(message.contains("git branch -d wt/foo"), "{message}"); - assert!(!message.contains("landed"), "{message}"); - assert!(!message.contains("merged"), "{message}"); - assert!(!path.exists(), "nothing must be created before the refusal"); - } - - /// A local branch behind its own `origin/wt/` is refused rather than - /// resumed at the stale local tip. - #[test] - fn a_local_branch_behind_origin_is_refused() { - let (dir, _origin, work) = crate::gitfixture::repo("wtresume-behind"); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["branch", "-qD", "wt"]); - git(&work, &["checkout", "-qb", "wt/foo", "main"]); - crate::gitfixture::commit(&work, "first", "first\n", "first"); - crate::gitfixture::commit(&work, "second", "second\n", "second"); - git(&work, &["push", "-q", "-u", "origin", "wt/foo"]); - git(&work, &["reset", "-q", "--hard", "HEAD~1"]); - - let path = dir.join("resumed"); - let refused = std::panic::catch_unwind(|| { - create_worktree(&work, &path, "foo") - }); - - let panic = refused.expect_err("a local branch behind its origin counterpart must not resume"); - let message = panic.downcast::().expect("the refusal is formatted"); - assert!(message.contains("wt/foo"), "{message}"); - assert!(message.contains("origin/wt/foo"), "{message}"); - assert!(message.contains("merge it first"), "{message}"); - assert!(!path.exists(), "nothing must be created before the refusal"); - } - - /// A local branch that has diverged from `origin/wt/` — neither is - /// an ancestor of the other — is refused rather than resumed silently at - /// either side. - #[test] - fn a_local_branch_diverged_from_origin_is_refused() { - let (dir, _origin, work) = crate::gitfixture::repo("wtresume-diverged"); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["branch", "-qD", "wt"]); - git(&work, &["checkout", "-qb", "wt/foo", "main"]); - crate::gitfixture::commit(&work, "side", "origin side\n", "origin side"); - git(&work, &["push", "-q", "-u", "origin", "wt/foo"]); - git(&work, &["reset", "-q", "--hard", "main"]); - crate::gitfixture::commit(&work, "side", "local side\n", "local side"); - - let path = dir.join("resumed"); - let refused = std::panic::catch_unwind(|| { - create_worktree(&work, &path, "foo") - }); - - let panic = refused.expect_err("a diverged local branch must not resume"); - let message = panic.downcast::().expect("the refusal is formatted"); - assert!(message.contains("wt/foo"), "{message}"); - assert!(message.contains("origin/wt/foo"), "{message}"); - assert!(message.contains("merge it first"), "{message}"); - assert!(!path.exists(), "nothing must be created before the refusal"); - } - - #[test] - fn an_origin_only_branch_is_recreated_tracking_it() { - let (dir, _origin, work) = crate::gitfixture::repo("wtresume-origin"); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["branch", "-qD", "wt"]); - git(&work, &["checkout", "-qb", "wt/bar", "main"]); - crate::gitfixture::commit(&work, "on-branch", "branch work\n", "branch work"); - git(&work, &["push", "-q", "-u", "origin", "wt/bar"]); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["branch", "-qD", "wt/bar"]); - crate::gitfixture::commit(&work, "on-main", "main moved on\n", "main moved on"); - - let path = dir.join("resumed"); - let summary = create_worktree(&work, &path, "bar"); - - assert!(summary.contains("resumed from origin/wt/bar"), "{summary}"); - let origin_sha = capture(&work, &["rev-parse", "origin/wt/bar"]); - let worktree_sha = capture(&path, &["rev-parse", "HEAD"]); - let main_sha = capture(&work, &["rev-parse", "main"]); - assert_eq!(worktree_sha, origin_sha, "must resume at origin's commit"); - assert_ne!(worktree_sha, main_sha, "must not have been reset onto main"); - let tracked = capture(&work, &["rev-parse", "wt/bar@{upstream}"]); - assert_eq!(tracked, origin_sha, "the new local branch must track origin/wt/bar"); - } - - /// A branch already merged into `origin/main` and reachable only as a - /// stale `origin/wt/` (its GitHub head long deleted, this checkout - /// never fetched to notice) is refused rather than recreated behind main. - #[test] - fn a_landed_origin_only_branch_is_refused() { - let (dir, _origin, work) = crate::gitfixture::repo("wtresume-landed-origin"); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["branch", "-qD", "wt"]); - git(&work, &["checkout", "-qb", "wt/bar", "main"]); - crate::gitfixture::commit(&work, "on-branch", "branch work\n", "branch work"); - git(&work, &["push", "-q", "-u", "origin", "wt/bar"]); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["merge", "-q", "--ff-only", "wt/bar"]); - git(&work, &["branch", "-qD", "wt/bar"]); - git(&work, &["push", "-q", "origin", "main"]); - - let path = dir.join("resumed"); - let refused = std::panic::catch_unwind(|| { - create_worktree(&work, &path, "bar") - }); - - let panic = refused.expect_err("a landed origin-only branch must not be resumed"); - let message = panic.downcast::().expect("the refusal is formatted"); - assert!(message.contains("origin/wt/bar"), "{message}"); - assert!(!path.exists(), "nothing must be created before the refusal"); - } - - #[test] - fn with_neither_branch_it_starts_fresh_from_main() { - let (dir, _origin, work) = crate::gitfixture::repo("wtresume-fresh"); - git(&work, &["checkout", "-q", "main"]); - git(&work, &["branch", "-qD", "wt"]); - crate::gitfixture::commit(&work, "on-main", "main moved on\n", "main moved on"); - - let path = dir.join("resumed"); - let summary = create_worktree(&work, &path, "baz"); - - assert!(summary.contains("new, from main"), "{summary}"); - let worktree_sha = capture(&path, &["rev-parse", "HEAD"]); - let main_sha = capture(&work, &["rev-parse", "main"]); - assert_eq!(worktree_sha, main_sha); - } - - /// A linked worktree of a fresh repository whose `.gitignore` names `target/`, - /// both in the directory that comes first. - fn linked(name: &str) -> (TempDir, PathBuf, PathBuf) { - let (dir, _origin, work) = crate::gitfixture::repo(name); - let tree = dir.join("linked"); - git(&work, &["worktree", "add", "-q", "-b", "linked", tree.to_str().unwrap()]); - (dir, work, tree) - } - - /// **git unregisters, then fails to delete, and exits non-zero.** A path - /// deeper than `PATH_MAX` under the ignored `target/` is a deterministic - /// way to make it do that. - #[test] - fn a_worktree_git_unregistered_but_left_on_disk_is_deleted_whole() { - let (_dir, work, tree) = linked("wt-remove-leftovers"); - // Two chains of twenty, each short enough to make, one renamed into - // the other's end: no path any call here names exceeds `PATH_MAX`. - let chain = |at: &Path| { - let mut end = at.to_path_buf(); - for _ in 0..20 { - end.push("a-directory-name-thirty-bytes-"); - } - fs::create_dir_all(&end).unwrap(); - end - }; - let target = tree.join("target"); - let lower = chain(&tree.join("lower")); - fs::write(lower.join("f"), "cache\n").unwrap(); - let upper = chain(&target); - fs::rename(tree.join("lower"), upper.join("lower")).unwrap(); - let depth = upper.as_os_str().len() + lower.strip_prefix(&tree).unwrap().as_os_str().len(); - assert!(depth > 1024, "the fixture must exceed PATH_MAX to make git fail: {depth}"); - - remove(&work, tree.to_str().unwrap()); - - assert!(!tree.exists(), "{} is still on disk", tree.display()); - assert!(!registered(&work, &tree), "{} is still a worktree", tree.display()); - } - - /// git's own refusal stands: untracked work keeps the worktree registered, - /// and nothing of it is deleted. - #[test] - fn a_worktree_holding_untracked_work_is_refused_and_left_whole() { - let (_dir, work, tree) = linked("wt-remove-dirty"); - fs::write(tree.join("unsaved.rs"), "the only copy\n").unwrap(); - - let refused = std::panic::catch_unwind(|| remove(&work, tree.to_str().unwrap())); - - assert!(refused.is_err(), "a worktree with untracked work was removed"); - assert!(registered(&work, &tree), "the refusal unregistered it"); - assert_eq!(fs::read_to_string(tree.join("unsaved.rs")).unwrap(), "the only copy\n"); - } -} diff --git a/system.toml b/system.toml index 4ad79cf0815..3857acd2b1a 100644 --- a/system.toml +++ b/system.toml @@ -4,8 +4,6 @@ # nobody asked for. # `src/build.rs`'s `no_shipped_boot_config_starts_sshd` is the gate. -hosted-rustc = false - # `assets/soundfont.sf2` is in here and is doom's music: GeneralUser GS cut down # to the 37 melodic programs and 23 percussion keys `assets/DOOM1.WAD`'s own MUS # headers select, by `src/soundfont.rs`. `NOTICE` carries its licence, the diff --git a/tests/common/compile.rs b/tests/common/compile.rs index 1ee15db5ac7..6f92b163f73 100644 --- a/tests/common/compile.rs +++ b/tests/common/compile.rs @@ -17,14 +17,16 @@ pub fn testcases_dir() -> PathBuf { /// The C sysroot every C program here is built against, and the clang that /// builds it: the toolchain's own, for the suite's architecture. Once per -/// process — a hundred and fifty C programs build against it. +/// process — a hundred and fifty C programs build against it — and held in use +/// for as long as the process lives. pub fn c_sysroot() -> CSysroot { - static C: OnceLock = OnceLock::new(); + static C: OnceLock<(toyos_build::sysroot::Sysroot, CSysroot)> = OnceLock::new(); C.get_or_init(|| { - let mut lock = toyos_build::buildlock::shared(&repo_root(), "the C sysroot"); - let sysroot = toyos_build::toolchain::ensure(&repo_root(), &mut lock); - CSysroot::of(&sysroot.dir, super::qemu::SUITE_ARCH) + let sysroot = toyos_build::toolchain::ensure(&repo_root()); + let c = CSysroot::of(&sysroot.dir, super::qemu::SUITE_ARCH); + (sysroot, c) }) + .1 .clone() }