diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5b327c87940..36d2aa93359 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -61,15 +61,13 @@ jobs: with: fetch-depth: 0 - # CMake and Ninja are what bootstrap builds LLVM and clang with, pinned to - # the versions Ubuntu 24.04 released. - - name: disk, QEMU, CMake and Ninja + - name: disk, QEMU and CMake run: | sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ /usr/local/share/boost /usr/local/.ghcup sudo apt-get update -qq sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq qemu-system-x86 zstd \ - cmake=3.28.3-1build7 ninja-build=1.11.1-2 + cmake=3.28.3-1build7 - env: GH_TOKEN: ${{ github.token }} @@ -189,7 +187,7 @@ jobs: for attempt in 1 2 3; do apt-get update -qq > /tmp/apt.log 2>&1 \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git curl ca-certificates \ - build-essential qemu-system-x86 python3 cmake ninja-build >> /tmp/apt.log 2>&1 \ + build-essential qemu-system-x86 python3 cmake >> /tmp/apt.log 2>&1 \ && break [ "$attempt" = 3 ] && { cat /tmp/apt.log; exit 1; } sleep 20 @@ -209,9 +207,8 @@ jobs: steps: - *checkout # Homebrew keeps one QEMU; `cargo run` notes it if it is not the declared one. - # CMake and Ninja build LLVM and clang, from Homebrew as on the dev host. - run: | - brew install qemu cmake ninja + brew install qemu cmake - run: | curl --proto '=https' --tlsv1.2 -sSf -o "$RUNNER_TEMP/rustup-init.sh" https://sh.rustup.rs sh "$RUNNER_TEMP/rustup-init.sh" -y --profile minimal --default-toolchain stable diff --git a/CLAUDE.md b/CLAUDE.md index 285817eedf0..fc098e81b33 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,7 +58,7 @@ A subdirectory `CLAUDE.md` loads when a file in that subtree is `Read`, and not ## Dependencies -**Rust** and **QEMU** for development, on any host OS and architecture — the development machine is nothing special. Beside them, where no Rust tool does the job, only C or C++ tools ToyOS can one day build and run (Python, Perl, CMake, Ninja, make), each declared. No binary for one host OS alone: a macOS binary is a hard no, and "only for tests" does not soften it. ToyOS's own code is Rust; it writes no Python, Perl or shell of its own. Only general and widely used crates — one that does *our* job we write ourselves, and a driver crate never; third-party crates are used as published, and a fork carries a change written to upstream quality and goes when upstream has it. No upstream pull requests are sent for now: ToyOS needs more attention and more contributors before upstream projects take it seriously, and upstreams tend to refuse AI-first projects and their contributions. A third-party source ToyOS cannot build without changing it is carried as an unmodified-source packaging mirror with a byte-identity gate, not as a fork. The north star is **self-hosting**: nothing — build, test, or verification — rests on a host binary. Ask of anything new: could this ever run inside ToyOS? Self-hosting means ToyOS rebuilds itself on ToyOS and reproduces the host's bytes; a bootstrap from source with no binary seed is out of scope. +**Rust** and **QEMU** for development, on any host OS and architecture — the development machine is nothing special. Beside them, where no Rust tool does the job, only C or C++ tools ToyOS can one day build and run (Python, Perl, CMake, make), each declared. No binary for one host OS alone: a macOS binary is a hard no, and "only for tests" does not soften it. ToyOS's own code is Rust; it writes no Python, Perl or shell of its own. Only general and widely used crates — one that does *our* job we write ourselves, and a driver crate never; third-party crates are used as published, and a fork carries a change written to upstream quality and goes when upstream has it. No upstream pull requests are sent for now: ToyOS needs more attention and more contributors before upstream projects take it seriously, and upstreams tend to refuse AI-first projects and their contributions. A third-party source ToyOS cannot build without changing it is carried as an unmodified-source packaging mirror with a byte-identity gate, not as a fork. The north star is **self-hosting**: nothing — build, test, or verification — rests on a host binary. Ask of anything new: could this ever run inside ToyOS? Self-hosting means ToyOS rebuilds itself on ToyOS and reproduces the host's bytes; a bootstrap from source with no binary seed is out of scope. Vendor firmware a device verifies by its maker's signature may be shipped: pinned by version and hash, redistributable unmodified, recorded in `NOTICE`, and loaded only by that device's own driver through its IOMMU domain; it never executes on the CPU. diff --git a/README.md b/README.md index 6aa11f736d9..a388a1eff6f 100644 --- a/README.md +++ b/README.md @@ -223,21 +223,20 @@ same font the kernel blits. - Rust, with rustup - QEMU - A C compiler on `PATH` as `cc`, and a Python 3 -- CMake and Ninja +- CMake `rustc` links every **host** binary through `cc`, which rustup does not install. `rust/x`, the entry point to rustc's own bootstrap, is a shell script whose whole job is to find a Python to run `bootstrap.py` with — so a clean clone needs one, and so does every toolchain change. And that bootstrap builds -LLVM and clang from source with CMake and Ninja, whenever the LLVM commit +LLVM and clang from source with CMake, whenever the LLVM commit `rust/` names has not been built on the machine before. Nothing in the OS goes near any of them. `bootloader/`, `kernel/` and `userland/` all link with the toolchain's `rust-lld`, and no image contains a C toolchain or a Python. On macOS `cc` and Python arrive with the Xcode Command -Line Tools, and CMake and Ninja come from Homebrew (`brew install cmake -ninja`); on Debian and Ubuntu they are `build-essential`, `python3`, `cmake` -and `ninja-build`. +Line Tools, and CMake comes from Homebrew (`brew install cmake`); on Debian and +Ubuntu they are `build-essential`, `python3` and `cmake`. `cargo run` names anything it needs and cannot find, before it does anything else. diff --git a/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md b/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md index 9d993d77588..dc50bcc3d45 100644 --- a/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md +++ b/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md @@ -16,7 +16,6 @@ arrives and is not one. M4 and M5 are stages of `issues/build/toyos-builds-itsel |---|---|---|---| | Python | LLVM's CMake, whenever this host builds an LLVM (`src/llvm.rs`); the C++ runtime's CMake, in every sysroot build (`src/libcxx.rs`), and its build, which runs `libcxx/utils/generate_iwyu_mapping.py` for a header it installs | admitted: no Rust tool does the job, LLVM's CMake requires one (`find_package(Python3 … REQUIRED)` in `rust/src/llvm-project/llvm/CMakeLists.txt`), and so does the runtimes', unconditionally (`runtimes/CMakeLists.txt`): configured as `src/libcxx.rs` does with no Python on `PATH` or in CMake's search, it found none and stopped, exit 1, and with only `python3` added it configured, exit 0 | M5 runs it in the guest | | CMake | rustc's bootstrap, for LLVM and clang; `src/llvm.rs`, for the LLVM's key; `src/libcxx.rs`, for the C++ runtime in every sysroot build | admitted: no Rust tool does the job, LLVM, clang and LLD are described in CMake, and upstream's only other descriptions are a GN overlay it does not support and a Bazel one | M5 runs it in the guest | -| Ninja | runs the build CMake generates for LLVM | refused: a Rust tool does it, n2 (`github.com/evmar/n2` at `b1fead5`), named `ninja` as its README directs for CMake: CMake's Ninja generator configured `rust/src/llvm-project/llvm` for it and it built `llvm-tblgen`, exit 0 each; named `n2`, CMake refuses its version, exit 1 | rustc's bootstrap builds LLVM under n2 | | `sh` running LLVM's `config.guess`, and the POSIX tools and `cc` it runs | LLVM's CMake, whenever this host builds an LLVM, and the C++ runtime's, in every sysroot build, ask it the host's triple, unconditionally (`get_host_triple` in `rust/src/llvm-project/llvm/cmake/modules/GetHostTriple.cmake`, which runs `sh` by name) | refused: a Rust tool does the shell's part, brush 0.4.0: on the development host (macOS, arm64) `config.guess` printed `/bin/sh`'s triple under it, `arm64-apple-darwin27.0.0`, exit 0 each. The script runs `sed`, `uname`, `mktemp`, `grep`, `rm`, `rmdir` and `cc` there under either shell, and that `cc` is the `cc` rows'. Five of the other six are refused, uutils' doing each: under brush with sed 0.2.0, grep 0.2.0 and coreutils 0.12.0's `mktemp`, `rm` and `rmdir`, and nothing else on `PATH` but the host's `uname` and `cc`, it printed that triple, exit 0. `uname` is admitted: coreutils 0.12.0's answers `-p` with `unknown` where macOS's answers `arm`, and `config.guess` reads that as PowerPC, `powerpc-apple-darwin27.0.0`, exit 0 | CMake finds brush as its `sh`, uutils' `sed`, `grep`, `mktemp`, `rm` and `rmdir`, and a Rust `uname` that answers `-p` as the host's does; or M5 runs it in the guest | | `git` for worktrees, submodules, checkouts, fixtures and rustc's bootstrap | adds, removes and prunes worktrees (`src/worktree.rs`, `src/sysroot.rs`); updates submodules (`src/lib.rs`, `src/sysroot.rs`, `src/licence.rs`, `src/release.rs`); fetches the fork from the primary's and checks it out (`src/sysroot.rs`); fast-forwards the primary (`src/sync.rs`); makes the tests' fixture repositories; runs inside rustc's bootstrap | admitted: no Rust tool does the job, gitoxide 0.85 adds, removes and prunes no worktree, updates no submodule, stages, resets and pushes nothing, checks out only a fresh clone and fetches a local path by spawning `git`; a fixture must be what `git` makes, and bootstrap runs `git` itself | M4 runs it in the guest | | `git` for reads, a config write, a commit's paths written out, and clones and fetches over HTTPS | `rev-parse`, `show-ref`, `for-each-ref`, `rev-list`, `log`, `branch --contains`, `merge-base`, `ls-tree`, `ls-files`, `cat-file`, `config --get-regexp`, `worktree list`, `status`, `diff`, `ls-remote` and `grep`, in the build system and its tests; `config --global --add safe.directory` in the nightly's containers; `checkout -- ` through an index of its own, which writes the C++ runtime's sources out of the LLVM commit into the stored LLVM (`src/llvm.rs`); `src/sync.rs`'s fetch of `origin`; every workflow's checkout | refused: a Rust tool does it, gitoxide 0.85, which reads refs, objects, the index, config, worktrees and status, adds a value to a config file and writes it (gix-config 0.58's `File::section_mut_or_create_new`, `SectionMut::push`, `File::write_to`), walks history, diffs, and lists, fetches and clones a remote over HTTPS; `grep` is a search of the files its index names; and gitoxide's CLI 0.59 (gix 0.88) wrote the runtimes' sources of LLVM `849da7d6` into an empty directory, each path's tree through `gix rev parse`, `gix index from-tree` and `gix free index checkout-exclusive`, exit 0 each: the 18759 files `git` writes there, byte for byte and mode for mode | those are gitoxide's | @@ -43,7 +42,7 @@ arrives and is not one. M4 and M5 are stages of `issues/build/toyos-builds-itsel | `sudo` on macOS | `diag/flash.sh` and the README's macOS flashing steps run `dd` under it | admitted: no Rust tool raises a process to root on macOS; sudo-rs "is targeted for FreeBSD and Linux-based operating systems only" (its README at `89bae8a`) | goes with both flashes by hand | | `sudo` on Linux | the README's Linux flashing steps run `dd` under it | refused: a Rust tool does it, sudo-rs | the README's Linux steps run sudo-rs | | `sh` running rustup's `rustup-init.sh` | the nightly's three rustup installs, fetched with `curl` | refused: a Rust tool does it, rustup's own `rustup-init` binary | a job installs rustup without it | -| `nightly.yml`, job `build`, step "disk, QEMU, CMake and Ninja" | `sudo rm -rf` of five preinstalled SDK directories, then `sudo apt-get update` and `install` | refused: shell of our own | each step is one command | +| `nightly.yml`, job `build`, step "disk, QEMU and CMake" | `sudo rm -rf` of five preinstalled SDK directories, then `sudo apt-get update` and `install` | refused: shell of our own | each step is one command | | `nightly.yml`, step `deps`, which jobs `guest` and `tcg` share | the snapshot archive read out of `debian.sources` with `sed`, checked with `test` and written back with `echo` and `rm`, a three-attempt `apt-get` loop with `sleep` and `cat`, `git config`, and rustup through `curl`, `sh` and `echo` | refused: shell of our own | each step is one command | | `nightly.yml`, job `portability-linux`, step "deps" | the same loop, `git config`, and rustup the same way | refused: shell of our own | each step is one command | | `nightly.yml`, job `portability-macos`, the rustup step | `curl`, `sh rustup-init.sh`, and `echo` into `$GITHUB_PATH` | refused: shell of our own | each step is one command | diff --git a/issues/build/the-fork-checkout-runs-git-submodule-in-a-linked-worktree.md b/issues/build/the-fork-checkout-runs-git-submodule-in-a-linked-worktree.md index 96b22d6b0f2..76c4127da91 100644 --- a/issues/build/the-fork-checkout-runs-git-submodule-in-a-linked-worktree.md +++ b/issues/build/the-fork-checkout-runs-git-submodule-in-a-linked-worktree.md @@ -23,5 +23,19 @@ library/backtrace` in the same fork checkout, from `sysroot::build_std` and empty or gone: what a first `fork_checkout` leaves when it stops after adding the fork's worktree and before adding `library/backtrace`. +Bootstrap does the same for `src/llvm-project`, which `fork_checkout` leaves an +empty directory. When the checkout's LLVM key is in no store, +`llvm::build_in_fork` runs `x build src/llvm-project/llvm`, and bootstrap's +`update_submodule` (fork `aca5f527`, +`src/bootstrap/src/core/config/config.rs:2585`) reads the fork's own `HEAD` in +that empty directory rather than the gitlink, so it runs `git submodule -q +sync` and `git submodule update --init --recursive --depth=1 src/llvm-project` +in the linked worktree. This arm is read from the code and unmeasured. +Bootstrap returns at `actual_hash == checked_out_hash` instead once +`src/llvm-project` is a `git worktree add --detach` at the gitlink from a +repository that holds the commit, as `fork_checkout` makes `library/backtrace`; +one added from a sparse clone inherits its patterns until `git sparse-checkout +disable` runs in it. + **Exit**: the build system runs no `git submodule` in a linked worktree's fork checkout. diff --git a/src/lib.rs b/src/lib.rs index ced7fc6a450..49f39d7f181 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,6 +35,7 @@ pub mod metalimage; pub mod metalswap; pub mod metaltalk; pub mod metaltimings; +pub mod n2; pub mod redlist; pub mod release; pub mod sdkversion; diff --git a/src/libcxx.rs b/src/libcxx.rs index a9027628d2a..0612a136ee7 100644 --- a/src/libcxx.rs +++ b/src/libcxx.rs @@ -1,5 +1,5 @@ //! The C++ runtime of a C sysroot (`clang::CSysroot`): LLVM's libc++, -//! libc++abi and libunwind, built by CMake and n2 ([`ninja`]) from the +//! libc++abi and libunwind, built by CMake and n2 (`n2::ninja`) from the //! runtimes' sources the LLVM carries (`src/llvm.rs`), with that LLVM's clang, //! against the C library the sysroot already holds (`src/libc.rs`). //! @@ -9,7 +9,7 @@ //! driver looks. [`OPTIONS`] is the configuration, each choice with its reason. use std::fs; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; use crate::arch::Arch; @@ -55,45 +55,6 @@ pub(crate) const OPTIONS: [(&str, &str); 20] = [ ("LIBCXX_INCLUDE_TESTS", "OFF"), ]; -/// cargo's install of n2, the Rust Ninja, but for its `--root`: without its -/// default jemalloc, which is C. -const N2: [&str; 7] = [ - "install", - "--locked", - "--no-default-features", - "--git", - "https://github.com/evmar/n2", - "--rev", - "b1fead52ccda0c497d816696f23f4099c3e8ec1f", -]; - -/// Where [`N2`] installs under `root`: a directory every argument names, so a -/// moved pin or flag names an empty one. -fn installed(root: &Path) -> PathBuf { - root.join("target/n2").join(crate::sysroot::short(N2.join("\0").as_bytes())) -} - -/// [`N2`], installed under `root` unless it is there, by the name `ninja`, under -/// which it speaks the Ninja CMake asks for. -pub fn ninja(root: &Path) -> PathBuf { - let dir = installed(root); - let ninja = dir.join("bin/ninja"); - // The link is made only once cargo has installed what it names. - if !ninja.is_file() { - let status = Command::new("cargo") - .args(N2) - .arg("--root") - .arg(&dir) - .env_remove("RUSTFLAGS") - .status() - .unwrap_or_else(|e| panic!("cargo failed to launch: {e}")); - assert!(status.success(), "n2 did not install into {}", dir.display()); - std::os::unix::fs::symlink("n2", &ninja).unwrap_or_else(|e| panic!("symlink {} -> n2: {e}", ninja.display())); - assert!(ninja.is_file(), "n2 installed into {}, and {} names no file", dir.display(), ninja.display()); - } - ninja -} - /// Build `arch`'s C++ runtime from the runtimes' `sources` into `c`, which holds /// the C library already, under `ninja`, in `scratch`, which it removes when /// done. @@ -147,23 +108,3 @@ fn run(mut command: Command, what: &str, target: &str) { String::from_utf8_lossy(&out.stderr), ); } - -#[cfg(test)] -mod tests { - use super::*; - use toyos_tmpdir::TempDir; - - /// **n2 is installed once**: with the link and n2 in the directory the pin - /// names, no cargo runs, and the n2 there is the one returned. - #[test] - fn an_installed_n2_is_not_installed_again() { - let root = TempDir::new("n2"); - let bin = installed(&root).join("bin"); - fs::create_dir_all(&bin).unwrap(); - fs::write(bin.join("n2"), b"").unwrap(); - std::os::unix::fs::symlink("n2", bin.join("ninja")).unwrap(); - - assert_eq!(ninja(&root), bin.join("ninja")); - assert!(fs::read(bin.join("n2")).unwrap().is_empty(), "the installed n2 was replaced"); - } -} diff --git a/src/llvm.rs b/src/llvm.rs index c6146e84d7e..fd1b436d8a8 100644 --- a/src/llvm.rs +++ b/src/llvm.rs @@ -53,9 +53,10 @@ const RECIPE: &str = "bootstrap build of src/llvm-project/llvm and src/llvm-proj /// What of the caller's environment the LLVM build, and every tool its key /// asks, sees: -/// - `PATH` finds what runs the build: the Python behind `./x`, git, curl, -/// Ninja, and CMake, which the key names by path and version. The C and C++ -/// compilers it finds, the configuration names by path. +/// - `PATH` finds what runs the build: the Python behind `./x`, git, curl, and +/// CMake, which the key names by path and version. The C and C++ compilers +/// it finds, the configuration names by path. The build's has n2's directory +/// first ([`build_in_fork`]). /// - `TMPDIR` is where those tools write what they discard; the sandbox a build /// runs in may allow no other place. /// @@ -197,7 +198,7 @@ fn on_path(name: &str) -> PathBuf { /// The LLVM `fork` names, made if nobody on this host has made it, and held in /// use for as long as the returned value lives. `root` records its key. pub fn resolve(root: &Path, rust_dir: &Path, fork: &Path) -> Llvm { - choose(root, rust_dir, fork, build_in_fork) + choose(root, rust_dir, fork, |fork| build_in_fork(root, fork)) } /// [`resolve`] with the build that makes an LLVM passed in, so a test can stand @@ -214,7 +215,7 @@ fn choose(root: &Path, rust_dir: &Path, fork: &Path, build: impl Fn(&Path) -> Pa /// LLVM its compiler links, whose record is that compiler's /// (`compiler::choose`). pub fn held(root: &Path, rust_dir: &Path, fork: &Path) -> Llvm { - held_with(root, rust_dir, fork, build_in_fork) + held_with(root, rust_dir, fork, |fork| build_in_fork(root, fork)) } /// [`held`] with the build passed in, as [`choose`] takes it. @@ -370,8 +371,14 @@ pub fn retire_in_tree(build: &Path) { } /// Bootstrap's build of LLVM, clang and LLD in `fork`, into its own build -/// directory, which it returns. -fn build_in_fork(fork: &Path) -> PathBuf { +/// directory, which it returns, under the n2 installed under `root`. +fn build_in_fork(root: &Path, fork: &Path) -> PathBuf { + let n2 = crate::n2::bin(root); + // Bootstrap refuses to build with no `ninja` on `PATH`, and CMake's Ninja + // generator runs the first one there; n2's directory holds nothing but n2. + let caller = std::env::var_os("PATH").unwrap_or_else(|| panic!("PATH is unset, and the LLVM build finds its tools on it")); + let path = std::env::join_paths(std::iter::once(n2.clone()).chain(std::env::split_paths(&caller))) + .unwrap_or_else(|e| panic!("{} cannot lead PATH: {e}", n2.display())); let host = host_triple(); let build_dir = fork.join("build/toyos-llvm"); fs::create_dir_all(&build_dir).unwrap_or_else(|e| panic!("create {}: {e}", build_dir.display())); @@ -380,7 +387,10 @@ fn build_in_fork(fork: &Path) -> PathBuf { .unwrap_or_else(|e| panic!("write {}: {e}", config.display())); let config = config.to_str().unwrap_or_else(|| panic!("{} is not UTF-8", config.display())); let args = ["build", "--config", config, "src/llvm-project/llvm", "src/llvm-project/lld"]; - let (ok, _) = toolchain::x_build_with(fork, &args, "LLVM", clear); + let (ok, _) = toolchain::x_build_with(fork, &args, "LLVM", |command| { + clear(command); + command.env("PATH", path); + }); assert!(ok, "the LLVM build in {} failed: its output above says why", fork.display()); build_dir } @@ -713,6 +723,7 @@ mod tests { } const FORK: &str = "TOYOS_LLVM_TEST_FORK"; + const ROOT: &str = "TOYOS_LLVM_TEST_ROOT"; /// What a caller's environment may hold that would reach an LLVM build: /// flags, compilers, tools, and the SDK and deployment target. @@ -734,7 +745,8 @@ mod tests { /// a process holding every [`AMBIENT`] name keys the LLVM as this one does, /// and the bootstrap it runs, a script that writes down its environment, /// sees nothing but `PATH`, `TMPDIR` and what a shell sets itself, named - /// here and not read from [`ENVIRONMENT`]. + /// here and not read from [`ENVIRONMENT`]; its `PATH` is the caller's with + /// n2's directory first. #[test] fn the_caller_s_environment_reaches_neither_the_build_nor_the_key() { use std::os::unix::fs::PermissionsExt; @@ -744,8 +756,10 @@ mod tests { write(&fork.join("library/Cargo.lock"), "# lock\n"); write(&fork.join("x"), "#!/bin/sh\nenv > build/toyos-llvm/environment\n"); fs::set_permissions(fork.join("x"), fs::Permissions::from_mode(0o755)).unwrap(); + let n2 = crate::n2::tests::installed_stand_in(&scratch); - let out = buildlock::tests::rerun("llvm::tests::keyed_and_built").env(FORK, &fork).envs(AMBIENT).output().unwrap(); + let mut rerun = buildlock::tests::rerun("llvm::tests::keyed_and_built"); + let out = rerun.env(FORK, &fork).env(ROOT, &*scratch).envs(AMBIENT).output().unwrap(); assert!(out.status.success(), "{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)); let built = fork.join("build/toyos-llvm"); assert_eq!(fs::read_to_string(built.join("key")).unwrap(), key(&fork), "the caller's environment moved the key"); @@ -754,18 +768,22 @@ mod tests { for name in seen.lines().filter_map(|l| l.split_once('=')).map(|(name, _)| name) { assert!(allowed.contains(&name), "the build saw {name}: {seen}"); } - assert!(seen.lines().any(|l| l.starts_with("PATH=")), "the build saw no PATH: {seen}"); + let caller = std::env::var_os("PATH").unwrap(); + let path = std::env::join_paths(std::iter::once(n2).chain(std::env::split_paths(&caller))).unwrap(); + let path = format!("PATH={}", path.to_str().unwrap()); + assert!(seen.lines().any(|l| l == path), "the build's PATH is not {path}: {seen}"); } /// The process [`the_caller_s_environment_reaches_neither_the_build_nor_the_key`] - /// runs: the key of the fork in [`FORK`] and its build, the key written - /// beside what the build wrote. + /// runs: the key of the fork in [`FORK`] and its build under the n2 of the + /// root in [`ROOT`], the key written beside what the build wrote. #[test] #[ignore = "the process the environment test runs; never runs on its own"] fn keyed_and_built() { let fork = PathBuf::from(std::env::var(FORK).unwrap_or_else(|_| panic!("keyed_and_built ran without {FORK}; it is not a test"))); + let root = PathBuf::from(std::env::var(ROOT).unwrap_or_else(|_| panic!("keyed_and_built ran without {ROOT}; it is not a test"))); let key = key(&fork); - let built = build_in_fork(&fork); + let built = build_in_fork(&root, &fork); fs::write(built.join("key"), key).unwrap(); } diff --git a/src/main.rs b/src/main.rs index 61ae13771fd..105876337e2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -34,21 +34,6 @@ const REQUIRED: &[Tool] = &[ }, ]; -/// Named, because a list that stops at what is fatal reads as the whole list. -/// Each of these costs one thing when absent rather than the build, so none of -/// them exits. -/// -/// Ninja is what rustc's bootstrap builds LLVM and clang from -/// `rust/src/llvm-project` with, under CMake, which it does only when this -/// host has not built that LLVM. Both are host tools like `cc`, and never in a -/// guest: on macOS from Homebrew, on CI's toolchain runner at the versions -/// `.github/workflows` pins. -const ALSO_USED: &[Tool] = &[Tool { - any: &["ninja"], - why: "rustc's bootstrap builds LLVM and clang with it, under CMake; `brew install \ - ninja` on macOS", -}]; - /// Where the OS would find `name`, if anywhere. /// /// A `PATH` scan and not a `--version` run: it is what `Command::new` does @@ -65,16 +50,11 @@ fn executable_on_path(name: &str) -> bool { } fn check_prerequisites(root: &Path, arch: Arch) { - fn absent(tools: &'static [Tool]) -> Vec<&'static Tool> { - tools.iter().filter(|t| !t.any.iter().any(|n| executable_on_path(n))).collect() - } - - for tool in absent(ALSO_USED) { - eprintln!("Note: no {} — {}", tool.any.join(" or "), tool.why); - } - - let mut missing: Vec = - absent(REQUIRED).iter().map(|t| format!("{} ({})", t.any.join(" or "), t.why)).collect(); + let mut missing: Vec = REQUIRED + .iter() + .filter(|t| !t.any.iter().any(|n| executable_on_path(n))) + .map(|t| format!("{} ({})", t.any.join(" or "), t.why)) + .collect(); if !executable_on_path(arch.qemu()) { missing.push(format!("{} (every {} boot — install QEMU)", arch.qemu(), arch.name())); } diff --git a/src/n2.rs b/src/n2.rs new file mode 100644 index 00000000000..8ce65fc3092 --- /dev/null +++ b/src/n2.rs @@ -0,0 +1,84 @@ +//! n2, the Rust Ninja, and the only Ninja the build runs: under the name +//! `ninja`, which is how CMake's Ninja generator and rustc's bootstrap ask for +//! one. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// This file, which the release tag hashes. +pub(crate) const SOURCE: &str = file!(); + +/// cargo's install of n2 but for its `--root`: without its default jemalloc, +/// which is C. +pub(crate) const N2: [&str; 7] = [ + "install", + "--locked", + "--no-default-features", + "--git", + "https://github.com/evmar/n2", + "--rev", + "b1fead52ccda0c497d816696f23f4099c3e8ec1f", +]; + +/// Where [`N2`] installs under `root`: a directory every argument names, so a +/// moved pin or flag names an empty one. +fn installed(root: &Path) -> PathBuf { + root.join("target/n2").join(crate::sysroot::short(N2.join("\0").as_bytes())) +} + +/// The directory [`N2`] installs n2 in under `root`, installing it unless it is +/// there. It holds n2 and nothing else but a link to it named `ninja`, the name +/// under which n2 speaks the Ninja CMake asks for. +pub fn bin(root: &Path) -> PathBuf { + let dir = installed(root); + let bin = dir.join("bin"); + let ninja = bin.join("ninja"); + // The link is made only once cargo has installed what it names. + if !ninja.is_file() { + let status = Command::new("cargo") + .args(N2) + .arg("--root") + .arg(&dir) + .env_remove("RUSTFLAGS") + .status() + .unwrap_or_else(|e| panic!("cargo failed to launch: {e}")); + assert!(status.success(), "n2 did not install into {}", dir.display()); + std::os::unix::fs::symlink("n2", &ninja).unwrap_or_else(|e| panic!("symlink {} -> n2: {e}", ninja.display())); + assert!(ninja.is_file(), "n2 installed into {}, and {} names no file", dir.display(), ninja.display()); + } + bin +} + +/// n2 by the name `ninja`, in [`bin`]. +pub fn ninja(root: &Path) -> PathBuf { + bin(root).join("ninja") +} + +#[cfg(test)] +pub(crate) mod tests { + use std::fs; + + use super::*; + use toyos_tmpdir::TempDir; + + /// What [`bin`] leaves under `root`, with an empty `n2`. + pub(crate) fn installed_stand_in(root: &Path) -> PathBuf { + let bin = installed(root).join("bin"); + fs::create_dir_all(&bin).unwrap(); + fs::write(bin.join("n2"), b"").unwrap(); + std::os::unix::fs::symlink("n2", bin.join("ninja")).unwrap(); + bin + } + + /// **n2 is installed once**: with the link and n2 in the directory the pin + /// names, no cargo runs, and the n2 there is the one returned. + #[test] + fn an_installed_n2_is_not_installed_again() { + let root = TempDir::new("n2"); + let stand_in = installed_stand_in(&root); + + assert_eq!(bin(&root), stand_in); + assert_eq!(ninja(&root), stand_in.join("ninja")); + assert!(fs::read(stand_in.join("n2")).unwrap().is_empty(), "the installed n2 was replaced"); + } +} diff --git a/src/release.rs b/src/release.rs index 050598c85e0..eb567406976 100644 --- a/src/release.rs +++ b/src/release.rs @@ -26,7 +26,7 @@ fn trees() -> Vec<&'static str> { std::iter::once("rust") .chain(crate::sysroot::SYSROOT_SOURCES) .chain(crate::sysroot::SYSROOT_MANIFESTS) - .chain([crate::clang::SOURCE, crate::libcxx::SOURCE, file!()]) + .chain([crate::clang::SOURCE, crate::libcxx::SOURCE, crate::n2::SOURCE, file!()]) .collect() } @@ -463,7 +463,8 @@ mod tests { assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr)); } - /// A commit to the C and C++ toolchain's declarations or headers moves the tag. + /// A commit to the C and C++ toolchain's declarations or headers, or to the + /// n2 it is built under, moves the tag. #[test] fn the_tag_moves_with_the_c_toolchain() { let repo = TempDir::new("release-tag"); @@ -496,10 +497,14 @@ mod tests { let cxx = fs::read_to_string(here.join(crate::libcxx::SOURCE)).unwrap(); let (no_fs, fs_on) = (r#"("LIBCXX_ENABLE_FILESYSTEM", "OFF")"#, r#"("LIBCXX_ENABLE_FILESYSTEM", "ON")"#); assert!(cxx.contains(no_fs), "src/libcxx.rs no longer declares what this mutates"); + let n2 = fs::read_to_string(here.join(crate::n2::SOURCE)).unwrap(); + let pin = crate::n2::N2[crate::n2::N2.len() - 1]; + assert!(n2.contains(pin), "src/n2.rs no longer declares what this mutates"); let mutations = [ (crate::clang::SOURCE, with_objdump.clone()), (crate::clang::SOURCE, with_objdump.replace(targets, riscv)), (crate::libcxx::SOURCE, cxx.replace(no_fs, fs_on)), + (crate::n2::SOURCE, n2.replace(pin, &"0".repeat(pin.len()))), ("userland/libc/include/placeholder", "y".to_string()), ]; for (path, text) in mutations { diff --git a/src/sysroot.rs b/src/sysroot.rs index f2f5044afb5..e5b31dd8417 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -350,7 +350,7 @@ fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, key: &s } let _ = fs::remove_dir_all(&libc_target); let llvm = crate::llvm::held(root, rust_dir, fork); - let ninja = crate::libcxx::ninja(root); + let ninja = crate::n2::ninja(root); for arch in Arch::ALL { let scratch = dir.with_extension(format!("libcxx-{}", arch.name())); let c = crate::clang::CSysroot::of(partial, arch);