From b9f562d9d3f74cbfa4a0e442557c9e68a4fe453c Mon Sep 17 00:00:00 2001 From: japabu Date: Fri, 9 Oct 2026 17:32:31 +0200 Subject: [PATCH 1/6] The toolchain builds clang and ld.lld that run on ToyOS, statically linked `cargo run -- --hosted-clang` builds clang and LLD for x86_64-unknown-toyos (src/hostedclang.rs) from the LLVM commit the host's LLVM is, with the toolchain's clang against its C sysroot, into the host store under a key of their own: the host LLVM's key, the sysroot's, the recipe and the CMake options. The sources are the commit's, written from the fork's LLVM repository through an index of their own, never a checkout's files. The binaries say the version, revision and repository the host's clang says, `22.1.8-rust-dev` with bootstrap's suffix among them, since every object a clang compiles carries that in its `.comment`. Each is refused unless it is an x86-64 ELF naming no library it needs. Made only when asked for: its build is LLVM's, and its key moves with every sysroot's. On this host it took 23 minutes and gave a 147,533,968-byte clang and an 85,568,488-byte ld.lld, static PIEs; stripped, 121,926,824 and 70,926,168. What the build for a ToyOS host stopped on, measured with the tree's own toolchain on the clang and lld targets alone: Support's `alarm`, `wait` and `wait4` (Watchdog.inc, Program.inc), and clang's `std::ifstream`, which libc++ has only with `std::filesystem`; at the link, clangBasic's `lround`. No archive of either tool names ORC's `shm_open`, the interpreter's `scanf` or llvm-objdump's `ctime`, so libc gains none of them. Neither tool starts a child for `clang -c` or `ld.lld`: clang runs cc1 in its own process (`CLANG_SPAWN_CC1` off), and LLD starts one only for `--error-handling-script`. libc gains: - `wait` and `wait4`, answering ECHILD as `waitpid` does: libc starts no child. - `alarm`, kept by a thread of libc's that ends the process with SIGALRM's default action, exit 142, once it is due. A handler never runs, since `sigaction` keeps none: that is stage 3 of the child-process track. The seconds left, rounded up, are `alarmreq.rs`'s rule, held on the host. - `lround`, and `round` on the same rule, a half away from zero, held to the host's C library. `round` was `floor(x + 0.5)`, which answered -2 for -2.5 and 1 for 0.49999999999999994. - what libc++'s `src/filesystem` and `` call: `setbuf`, `fseeko`, `ftello`, `truncate`, `pathconf`'s `_PC_PATH_MAX`, and `utimes`, refused ENOSYS: no call sets a file's times. libc++ is built with `std::filesystem`. `open` refuses every directory, so no descriptor names one for `openat`: `remove_all` walks a directory iterator, as libc++'s Windows does. `issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md` is closed: ToyOS's build, with nothing supplied by hand, makes a clang and an `ld.lld` for x86_64-unknown-toyos, by CMake and not bootstrap. What bootstrap still owes is M3's (`issues/rustc-llvm-cannot-build-for-a-toyos-host.md`). The alarm and libc++ issues are renamed to what stays true of them, and the track records the owner's ruling of 2026-10-09. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- ...nt-and-a-parent-takes-its-children-down.md | 4 +- ...-link-against-the-c-sysroot-with-z-defs.md | 12 +- ...figured-as-running-on-the-build-machine.md | 26 +- issues/an-alarm-reaches-no-sigalrm-handler.md | 26 ++ ...ild-llvm-clang-and-lld-for-a-toyos-host.md | 37 -- issues/libc-has-no-alarm.md | 26 -- ...lacks-names-llvm-for-a-toyos-host-calls.md | 16 +- ...bc-refuses-what-toyos-cannot-yet-answer.md | 2 + .../libcxx-is-built-without-std-filesystem.md | 17 - issues/no-guest-case-reads-std-filesystem.md | 18 + issues/toyos-builds-itself.md | 26 +- src/buildlock.rs | 18 +- src/flags.rs | 3 + src/hostedclang.rs | 366 ++++++++++++++++++ src/lib.rs | 1 + src/libcxx.rs | 7 +- src/llvm.rs | 27 +- src/main.rs | 5 + src/release.rs | 6 + tests/libc-arch/src/alarm_requests.rs | 27 ++ tests/libc-arch/src/fparts_differential.rs | 48 ++- tests/libc-arch/src/lib.rs | 5 + userland/libc/include/math.h | 1 + userland/libc/include/stdio.h | 4 + userland/libc/include/sys/time.h | 1 + userland/libc/include/sys/wait.h | 4 + userland/libc/include/unistd.h | 5 + userland/libc/src/alarm.rs | 71 ++++ userland/libc/src/alarmreq.rs | 21 + userland/libc/src/errno.rs | 1 + userland/libc/src/fparts.rs | 30 +- userland/libc/src/lib.rs | 2 + userland/libc/src/math.rs | 11 +- userland/libc/src/misc.rs | 12 + userland/libc/src/posix_io.rs | 28 ++ userland/libc/src/refused.rs | 6 + userland/libc/src/stdio.rs | 29 ++ 37 files changed, 814 insertions(+), 135 deletions(-) create mode 100644 issues/an-alarm-reaches-no-sigalrm-handler.md delete mode 100644 issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md delete mode 100644 issues/libc-has-no-alarm.md delete mode 100644 issues/libcxx-is-built-without-std-filesystem.md create mode 100644 issues/no-guest-case-reads-std-filesystem.md create mode 100644 src/hostedclang.rs create mode 100644 tests/libc-arch/src/alarm_requests.rs create mode 100644 userland/libc/src/alarm.rs create mode 100644 userland/libc/src/alarmreq.rs diff --git a/issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md b/issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md index b86c3e49ab2..5ebc580d9a1 100644 --- a/issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md +++ b/issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md @@ -6,8 +6,8 @@ opened: 2026-09-29 # A child's end is an event, and a parent takes its children down -libc cannot start a child: `fork` and `execvp` answer `ENOSYS`, `waitpid` -`ECHILD` and `system` `-1` (`userland/libc/src/misc.rs`, +libc cannot start a child: `fork` and `execvp` answer `ENOSYS`, `waitpid`, +`wait` and `wait4` `ECHILD`, and `system` `-1` (`userland/libc/src/misc.rs`, `userland/libc/src/stdio.rs`), and there is no `posix_spawn`. M2 and M4 of `issues/toyos-builds-itself.md` and the exit of `issues/toyos-runs-on-arm64.md` need it. diff --git a/issues/a-shared-object-does-not-link-against-the-c-sysroot-with-z-defs.md b/issues/a-shared-object-does-not-link-against-the-c-sysroot-with-z-defs.md index 91fb411845f..70dfafc68c8 100644 --- a/issues/a-shared-object-does-not-link-against-the-c-sysroot-with-z-defs.md +++ b/issues/a-shared-object-does-not-link-against-the-c-sysroot-with-z-defs.md @@ -9,12 +9,12 @@ opened: 2026-10-01 ToyOS loads shared objects, a program's `DT_NEEDED` and `dlopen`'s, and its clang links one with `-shared`. None links against the C sysroot with `-z defs`. A C one linked without it leaves `__tls_get_addr` undefined, and -lld refuses the executable that links it for that name. LLVM built for -`x86_64-unknown-toyos` -(`issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md`) -builds `libLTO`, `libRemarks`, `libclang` and `libclang-cpp`, linked -`-shared -z defs`, and each stops on what the sysroot's archives were built -for, an executable: +lld refuses the executable that links it for that name. LLVM built whole for +`x86_64-unknown-toyos`, as M3's rustc needs it +(`issues/rustc-llvm-cannot-build-for-a-toyos-host.md`), builds `libLTO`, +`libRemarks`, `libclang` and `libclang-cpp`, linked `-shared -z defs`, none +of which the ToyOS-hosted clang and LLD (`src/hostedclang.rs`) need, and +each stops on what the sysroot's archives were built for, an executable: - `main`, undefined: `libtoyos_c.a`'s entry point, `start_c`, sits in an object the link takes for other names. diff --git a/issues/a-toyos-hosted-llvm-is-configured-as-running-on-the-build-machine.md b/issues/a-toyos-hosted-llvm-is-configured-as-running-on-the-build-machine.md index 283db974f3d..73a0e53f190 100644 --- a/issues/a-toyos-hosted-llvm-is-configured-as-running-on-the-build-machine.md +++ b/issues/a-toyos-hosted-llvm-is-configured-as-running-on-the-build-machine.md @@ -8,14 +8,16 @@ opened: 2026-10-03 LLVM takes `LLVM_HOST_TRIPLE` from `config.guess`, run by the CMake that configures it (`llvm/cmake/modules/GetHostTriple.cmake`, -`llvm/cmake/config-ix.cmake`), unless the configuration names one, and -bootstrap names none: the `rust/` fork's -`src/bootstrap/src/core/build_steps/llvm.rs` defines -`LLVM_DEFAULT_TARGET_TRIPLE` and no `LLVM_HOST_TRIPLE`. So the LLVM that M2 -cross-builds to run on ToyOS (`issues/toyos-builds-itself.md`) records -the machine that built it as its host. The configure of that build on the -development Mac, the fork at the commit `main` pins (`95960d6c214`), logged -(#682, comment 5968053849) +`llvm/cmake/config-ix.cmake`), unless the configuration names one. The +ToyOS-hosted clang and LLD name it (`src/hostedclang.rs`), and their +configure logs `LLVM host triple: x86_64-unknown-toyos`. Bootstrap names none: +the `rust/` fork's `src/bootstrap/src/core/build_steps/llvm.rs` defines +`LLVM_DEFAULT_TARGET_TRIPLE` and no `LLVM_HOST_TRIPLE`. So the LLVM that M3's +rustc carries, built by bootstrap for a ToyOS host +(`issues/rustc-llvm-cannot-build-for-a-toyos-host.md`), records the machine +that built it as its host. The configure of that build on the development +Mac, the fork at the commit `main` pinned then (`95960d6c214`), logged (#682, +comment 5968053849) ``` -- LLVM host triple: arm64-apple-darwin27.0.0 @@ -26,8 +28,8 @@ development Mac, the fork at the commit `main` pins (`95960d6c214`), logged it exits non-zero, so a configure run inside ToyOS stops there. That half is read from `GetHostTriple.cmake`, not run. -Owner: `issues/toyos-builds-itself.md`: M2 for the triple the host -build records, M5 for the configure inside ToyOS. +Owner: `issues/toyos-builds-itself.md`: M3 for the triple bootstrap's build +records, M5 for the configure inside ToyOS. -Exit condition: the configure of a ToyOS-hosted LLVM logs `LLVM host triple: -x86_64-unknown-toyos`, and a configure inside ToyOS reaches its end. +Exit condition: bootstrap's configure of a ToyOS-hosted LLVM logs `LLVM host +triple: x86_64-unknown-toyos`, and a configure inside ToyOS reaches its end. diff --git a/issues/an-alarm-reaches-no-sigalrm-handler.md b/issues/an-alarm-reaches-no-sigalrm-handler.md new file mode 100644 index 00000000000..76ec1f667f7 --- /dev/null +++ b/issues/an-alarm-reaches-no-sigalrm-handler.md @@ -0,0 +1,26 @@ +--- +status: open +kind: defect +opened: 2026-09-30 +--- + +# An alarm reaches no SIGALRM handler + +libc's `alarm` (`userland/libc/src/alarm.rs`) arms one alarm per process and +answers the seconds the one it replaces had left, and when it is due a thread +of libc's ends the process with exit code 142, `SIGALRM`'s default action. A +handler for `SIGALRM` never runs: `sigaction` keeps none, and answers 0 +whatever it is given. LLVM bounds its wait on a child with one: a `SIGALRM` +handler makes `wait4` answer `EINTR` (`llvm/lib/Support/Unix/Program.inc`, +`Wait`). POSIX gives `alarm` no refusal, and its `SIGALRM` reaches a handler +only through the signals libc imitates from stage 3 of +`issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md` +on; the handler waits on that stage. + +The seconds left are held to their rule on the host +(`tests/libc-arch/src/alarm_requests.rs`); no guest case runs an alarm. + +**Exit**: `alarm` arms and disarms `SIGALRM` as POSIX says, which a guest C +case shows: a handler installed without `SA_RESTART` runs, and a `wait4` on a +child that has not ended answers `EINTR`; and after `alarm(5)`, `alarm(0)` +answers from 1 to 5 and a second `alarm(0)` answers 0. diff --git a/issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md b/issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md deleted file mode 100644 index ec3e0ee320c..00000000000 --- a/issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-30 ---- - -# Bootstrap cannot build LLVM, clang and lld for a ToyOS host - -M2's clang and lld (`issues/toyos-builds-itself.md`) are bootstrap's -`Llvm` step, with `clang = true`, and its `Lld` step for -`x86_64-unknown-toyos`, in a bootstrap build that names no `llvm-config` for -the build triple and so builds that triple's LLVM, and its `clang-tblgen`, -itself. CMake takes its toolchain file from -`CMAKE_TOOLCHAIN_FILE_x86_64_unknown_toyos`: one that includes the C sysroot's -`toolchain.cmake` and adds the ToyOS LLVM's install to `CMAKE_FIND_ROOT_PATH`, -because the `Lld` step names that LLVM to `find_package` by a hint, and the -sysroot's file has CMake find a package under a root alone. What stops the -build, in the order it stops it: - -- **Compile.** The first errors are `Support`'s: `Unix/Watchdog.inc` and - `Unix/Program.inc` call `alarm` (`issues/libc-has-no-alarm.md`), and - `Program.inc` stage 3's `wait` and `wait4` - (`issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md`). - Then ORC's `shm_open` and `shm_unlink`, the interpreter's `scanf` and - `llvm-objdump`'s `ctime` - (`issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md`), and - clang's `std::ifstream`, which libc++ has only with `std::filesystem` - (`issues/libcxx-is-built-without-std-filesystem.md`). -- **Link.** clang needs `lround`, which libc does not define - (`issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md`), beside - those above. LLVM's shared libraries, `libLTO`, `libRemarks`, `libclang` and - `libclang-cpp`, do not link against the C sysroot - (`issues/a-shared-object-does-not-link-against-the-c-sysroot-with-z-defs.md`). - clang and lld use none of them, and ToyOS's build turns them off. - -**Exit**: ToyOS's build, with nothing supplied by hand, has bootstrap install -a clang and an `ld.lld` for `x86_64-unknown-toyos`. diff --git a/issues/libc-has-no-alarm.md b/issues/libc-has-no-alarm.md deleted file mode 100644 index 4b2966567e8..00000000000 --- a/issues/libc-has-no-alarm.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-30 ---- - -# libc has no alarm - -libc neither declares nor defines `alarm`, and LLVM bounds its wait on a child -with one: a `SIGALRM` handler makes `wait4` answer `EINTR` -(`llvm/lib/Support/Unix/Program.inc`, `Wait`), so an LLVM built for ToyOS does -not link without it: rustc's LLVM wrapper linked `-shared -z defs` with the -libraries it needs leaves it undefined, beside stage 3's `wait` and `wait4`. -POSIX gives `alarm` no refusal, and its `SIGALRM` reaches -a handler only through the signals libc imitates from stage 3 of -`issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md` -on; this waits on that stage. - -LLVM's `Wait` disarms it with `alarm(0)` and then restores the old `SIGALRM` -action (`llvm/lib/Support/Unix/Program.inc`), so an `alarm(0)` that disarms -nothing ends the process when the alarm fires under the default action. - -**Exit**: `alarm` arms and disarms `SIGALRM` as POSIX says, which a guest C -case shows: a handler installed without `SA_RESTART` runs, and a `wait4` on a -child that has not ended answers `EINTR`; and after `alarm(5)`, `alarm(0)` -answers from 1 to 5 and a second `alarm(0)` answers 0. diff --git a/issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md b/issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md index eab41ae32f8..2c3c955a071 100644 --- a/issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md +++ b/issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md @@ -6,9 +6,10 @@ opened: 2026-10-01 # libc lacks names LLVM for a ToyOS host calls -LLVM, clang and lld built for `x86_64-unknown-toyos` -(`issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md`) -call five names libc neither declares nor defines: +LLVM's tools built for `x86_64-unknown-toyos` call names libc neither +declares nor defines. None is in the clang and `ld.lld` the build makes for +ToyOS (`src/hostedclang.rs`), which compile and link without them; each +stops a build of LLVM's other tools: - `shm_open` and `shm_unlink`, with which ORC maps memory on every Unix but Android (`llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp`, @@ -16,8 +17,11 @@ call five names libc neither declares nor defines: - `scanf`, which LLVM's interpreter hands an interpreted program (`llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp`). - `ctime`, with which `llvm-objdump` prints a file's time stamp. -- `lround`, which libc++'s `std::lround` calls, from clang's `Basic` and its - static analyzer: clang does not link without it. + +`lround`, which clang calls through libc++'s `std::lround`, is defined +(`userland/libc/src/math.rs`) and held to the host's on the host +(`tests/libc-arch/src/fparts_differential.rs`); no guest case reads it. **Exit**: libc declares and defines each, doing what POSIX says or refusing in -POSIX's form, asserted by a guest C case that reads its effect back. +POSIX's form, asserted by a guest C case that reads its effect back, `lround` +among them. diff --git a/issues/libc-refuses-what-toyos-cannot-yet-answer.md b/issues/libc-refuses-what-toyos-cannot-yet-answer.md index ac3977bd623..a2d65210d5c 100644 --- a/issues/libc-refuses-what-toyos-cannot-yet-answer.md +++ b/issues/libc-refuses-what-toyos-cannot-yet-answer.md @@ -14,6 +14,8 @@ These answer failure in their POSIX form and do nothing - `symlink`, `ENOSYS`: `SYS_SYMLINK` displaces what its name held (`issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md`). - `chmod` and `fchmod`, `ENOSYS`: a file has no mode bits to set. +- `utimes`, `ENOSYS`: no call sets a file's times. The one refusal here + `206_libc_refusals.c` does not yet assert. - `statvfs` and `fstatvfs`, `ENOSYS`: no call answers a filesystem's size or free space. - `getrlimit` and `setrlimit`, `ENOSYS`: no call answers a process's limits. diff --git a/issues/libcxx-is-built-without-std-filesystem.md b/issues/libcxx-is-built-without-std-filesystem.md deleted file mode 100644 index 7a98601903e..00000000000 --- a/issues/libcxx-is-built-without-std-filesystem.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-30 ---- - -# libc++ for ToyOS is built without `std::filesystem` - -`src/libcxx.rs` configures the C++ runtime with `LIBCXX_ENABLE_FILESYSTEM=OFF`, -so a ToyOS C++ program that includes `` does not compile, and -`` goes with the option: `std::ifstream` and `std::ofstream` are -undefined templates. Built with it, libc++ asks libc for `setbuf`, `fseeko`, -`ftello`, `utimes`, `truncate`, `pathconf`, `openat`, `unlinkat`, `fdopendir`, -`_PC_PATH_MAX`, `O_DIRECTORY`, `O_NOFOLLOW`, `AT_FDCWD` and `AT_REMOVEDIR`. - -**Exit**: libc carries what libc++'s `src/filesystem` calls, the option goes, -and a guest test lists a directory through `std::filesystem`. diff --git a/issues/no-guest-case-reads-std-filesystem.md b/issues/no-guest-case-reads-std-filesystem.md new file mode 100644 index 00000000000..486729c372c --- /dev/null +++ b/issues/no-guest-case-reads-std-filesystem.md @@ -0,0 +1,18 @@ +--- +status: open +kind: defect +opened: 2026-09-30 +--- + +# No guest case reads `std::filesystem` + +libc++ for ToyOS is built with `std::filesystem`, and `` with it, +over what libc gives `src/filesystem`: `setbuf`, `fseeko`, `ftello`, +`truncate`, `pathconf`'s `_PC_PATH_MAX`, and `utimes`, which refuses +`ENOSYS`, so `last_write_time` sets no time. `open` refuses every directory, +so no descriptor names one: `remove_all` walks a directory iterator +(`src/libcxx.rs`), which a link swapped in under it mid-walk can redirect, as +on libc++'s Windows. No C++ program in a guest uses any of it. + +**Exit**: a guest test lists a directory through `std::filesystem`, and reads +a file back through `std::ifstream`. diff --git a/issues/toyos-builds-itself.md b/issues/toyos-builds-itself.md index 02a51e6940b..2aeabff1000 100644 --- a/issues/toyos-builds-itself.md +++ b/issues/toyos-builds-itself.md @@ -38,6 +38,10 @@ AArch64 one step behind, on `issues/toyos-runs-on-arm64.md`'s track. - Make it work, then optimise, then compare: no performance study now, and no comparison before a compilation inside ToyOS succeeds. +**Decided** (owner, 2026-10-09). M2 installs clang and lld from a local file +first, without waiting for packages over HTTPS, and links them statically: +"Yes, decoupled and static (Recommended)". + **To build** (2026-10-01), in an order that is open. Where a stage names libc's state it is `userland/libc` at `15625e0cb`. - **The `libc` crate gains a ToyOS module**, checked by `ctest` against @@ -99,12 +103,14 @@ libc's state it is `userland/libc` at `15625e0cb`. how it is split into packages, and how a self-host test's guest reaches them. -**Blocked on other tracks.** M2 needs packages over HTTPS +**Blocked on other tracks.** M2 needs room for about a gigabyte of toolchain, +and threads and `mmap` mature enough for LLVM +(`issues/std-and-libc-drop-the-answer-thread-join-gives.md`); its package over +HTTPS (`issues/a-package-is-a-directory-under-apps-and-the-installer-is-a-program.md`) -and the network stack under it (`issues/the-lan-is-not-yet-production-grade.md`, -`issues/the-internet-clients-work-unchanged.md`), room for about a -gigabyte of toolchain, and threads and `mmap` mature enough for LLVM -(`issues/std-and-libc-drop-the-answer-thread-join-gives.md`). +waits on the network stack under it +(`issues/the-lan-is-not-yet-production-grade.md`, +`issues/the-internet-clients-work-unchanged.md`). M2 and M4 also need libc to start a child process (`issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md`). M4 needs git in the guest, storage durable and fast enough for an LLVM build tree @@ -119,10 +125,10 @@ the ToyOS-hosted rustc names no linker the guest has (`issues/the-hosted-rustc-names-a-linker-toyos-does-not-have.md`). *Exit*: the commit that lands M2's compile-and-run test, which links inside the guest. -**What stops M2: LLVM, clang and lld built for a ToyOS host**, in the order -each blocks the next, as -`issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md` -measures it. +**What stops M2: clang and lld running inside ToyOS.** `cargo run -- +--hosted-clang` makes a clang and an `ld.lld` for `x86_64-unknown-toyos`, +static PIEs that name no library (`src/hostedclang.rs`): 147.5 MB and +85.6 MB, 121.9 MB and 70.9 MB stripped. No guest has run either. - Run: what `issues/libc-refuses-what-toyos-cannot-yet-answer.md` lists, `issues/libc-has-no-pread-or-pwrite.md`, and `issues/libc-stat-answers-one-serial-number-for-every-file.md`. @@ -136,7 +142,7 @@ M3's exit then waits on a linker in the guest (`issues/the-hosted-rustc-names-a-linker-toyos-does-not-have.md`). **Also owed, by milestone.** -- M2: the host triple a ToyOS-hosted LLVM records +- M3: the host triple the ToyOS-hosted LLVM bootstrap builds records (`issues/a-toyos-hosted-llvm-is-configured-as-running-on-the-build-machine.md`, whose other half, the configure inside ToyOS, is M5's). - M3: `issues/the-hosted-rustcs-stage2-carries-seventeen-proc-macro-libraries-no-image-needs.md`. diff --git a/src/buildlock.rs b/src/buildlock.rs index 95920dbee39..ee6a4af25ab 100644 --- a/src/buildlock.rs +++ b/src/buildlock.rs @@ -28,11 +28,13 @@ //! removed, and its modification time is when the key was last used or made. //! //! A compiler key's lock → a sysroot key's lock → a freestanding key's lock → the -//! worktree build lock → an LLVM key's lock → artifact. A compiler's, a -//! sysroot's or a freestanding key lock is taken with the worktree lock put -//! down ([`Held::without_shared`]), because the key's builder takes the -//! worktree lock exclusively; an LLVM key's is taken inside the worktree lock -//! covering the fork build directory its builder writes. +//! worktree build lock → an LLVM key's lock → a ToyOS-hosted clang key's lock +//! → artifact. A compiler's, a sysroot's or a freestanding key lock is taken +//! with the worktree lock put down ([`Held::without_shared`]), because the +//! key's builder takes the worktree lock exclusively; an LLVM key's is taken +//! inside the worktree lock covering the fork build directory its builder +//! writes; a ToyOS-hosted clang key's inside the worktree lock and the use of +//! the sysroot it builds against, writing only the store. //! //! Holder death: `flock` is released by the kernel when the open file //! description closes, so a builder that is SIGKILLed mid-phase — routine here @@ -171,13 +173,15 @@ pub fn artifact(root: &Path) -> Guard { /// A content-addressed product of the host, locked per key: a sysroot, the /// freestanding targets' libraries it carries (`src/sysroot.rs`), a compiler -/// (`src/compiler.rs`), or the LLVM a compiler links (`src/llvm.rs`). +/// (`src/compiler.rs`), the LLVM a compiler links (`src/llvm.rs`), or the +/// clang and LLD built to run on ToyOS (`src/hostedclang.rs`). #[derive(Clone, Copy)] pub enum Keyed { Sysroot, Freestanding, Compiler, Llvm, + HostedClang, } impl Keyed { @@ -187,6 +191,7 @@ impl Keyed { Keyed::Freestanding => "freestanding", Keyed::Compiler => "compilers", Keyed::Llvm => "llvm", + Keyed::HostedClang => "hosted-clang", } } @@ -201,6 +206,7 @@ impl Keyed { Keyed::Freestanding => "freestanding libraries", Keyed::Compiler => "compiler", Keyed::Llvm => "LLVM", + Keyed::HostedClang => "ToyOS-hosted clang", } } } diff --git a/src/flags.rs b/src/flags.rs index d9173d79433..d5d4e3178b4 100644 --- a/src/flags.rs +++ b/src/flags.rs @@ -52,6 +52,9 @@ declare_flags!(pub CARGO_RUN = { pub HELP = "--help", None; pub CI = "--ci", Rest; pub CLIPPY = "--clippy", None; + /// Make the clang and `ld.lld` that run on ToyOS (`src/hostedclang.rs`), + /// and say where they are. + pub HOSTED_CLANG = "--hosted-clang", None; pub DEBUG = "--debug", None; pub BUILD_ONLY = "--build-only", None; pub DUMP_AUDIO = "--dump-audio", None; diff --git a/src/hostedclang.rs b/src/hostedclang.rs new file mode 100644 index 00000000000..27c72a220f2 --- /dev/null +++ b/src/hostedclang.rs @@ -0,0 +1,366 @@ +//! clang and `ld.lld` that run on ToyOS: built for `x86_64-unknown-toyos` from +//! the LLVM commit the host's LLVM is (`src/llvm.rs`), by the toolchain's own +//! clang against its C sysroot (`clang::CSysroot`), and linked statically. +//! +//! **Made only when asked for** (`cargo run -- --hosted-clang`): its build is +//! LLVM's, and its key moves with every sysroot's, so no other build makes it. +//! +//! **A function of its key** ([`key`]): the host LLVM's key, which names the +//! commit, the revision and repository it says it was built from, and the +//! host's tools, n2 and CMake among them; the sysroot's, which names the C +//! library, the C++ runtime and the clang that builds against them; and +//! [`RECIPE`] with [`OPTIONS`]. Its sources are the commit's, written from the +//! fork's LLVM repository ([`export`]), never a checkout's files. CMake builds +//! LLVM's tablegens for the build machine first, in a nested build of its own +//! (`NATIVE`), with the C and C++ compilers the LLVM key names. +//! +//! `hosted-clang//` in the store holds `bin/clang`, `bin/ld.lld` and +//! clang's resource headers in `lib/clang//include`, where clang looks +//! beside itself; once its [`SOURCE`] file exists it is read-only. Each binary +//! names no library it needs ([`static_elf`]). + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::arch::Arch; +use crate::buildlock::{Guard, Keyed}; +use crate::clang::CSysroot; +use crate::compiler::LLVM; +use crate::keystore::{self, Key}; +use crate::sysroot::{clone_tree, git_out, gitlink}; + +/// What changes how the key's sources become this product and is none of the +/// other parts. Moving it moves every key. +const RECIPE: &str = "CMake and n2 build of clang and lld from the LLVM commit's SOURCES, for the TARGET \ + against its C sysroot, saying the version, revision and repository the host's clang \ + says; of the build, clang as bin/clang, lld as bin/ld.lld and clang's resource \ + headers; 2"; + +/// The ToyOS the binaries run on. +const TARGET: Arch = Arch::X86_64; + +/// What of the commit the build reads, as git's pathspecs: LLVM, clang and +/// LLD, the CMake modules they share, the third-party sources LLVM compiles in, +/// and libunwind's headers, which LLD's Mach-O port reads. No test, which the +/// configuration builds none of. +const SOURCES: [&str; 12] = [ + "llvm", + "clang", + "lld", + "cmake", + "third-party", + "libunwind/include", + ":(exclude)llvm/test", + ":(exclude)llvm/unittests", + ":(exclude)clang/test", + ":(exclude)clang/unittests", + ":(exclude)lld/test", + ":(exclude)lld/unittests", +]; + +/// The CMake options beyond the compilers', the sysroot's and the host +/// libraries [`crate::llvm`] turns off, each with its reason. +const OPTIONS: [(&str, &str); 11] = [ + ("CMAKE_BUILD_TYPE", "Release"), + ("LLVM_ENABLE_PROJECTS", "clang;lld"), + // The two architectures ToyOS runs on, as the host's LLVM targets them + // (`clang::LLVM_CONFIG`). + ("LLVM_TARGETS_TO_BUILD", "AArch64;X86"), + // What the binaries run on and compile for, where LLVM otherwise asks + // `config.guess` about the build machine. + ("LLVM_HOST_TRIPLE", TARGET.userland()), + ("LLVM_DEFAULT_TARGET_TRIPLE", TARGET.userland()), + ("LLVM_INCLUDE_TESTS", "OFF"), + ("LLVM_INCLUDE_BENCHMARKS", "OFF"), + ("LLVM_INCLUDE_EXAMPLES", "OFF"), + ("LLVM_INCLUDE_DOCS", "OFF"), + ("CLANG_INCLUDE_TESTS", "OFF"), + ("CLANG_INCLUDE_DOCS", "OFF"), +]; + +/// The file a finished product carries last, naming its key. +const SOURCE: &str = "SOURCE"; + +/// The binaries it keeps, each as the build names it and as it is kept: lld +/// takes its flavour from the name it is run by. +const BINARIES: [(&str, &str); 2] = [("clang", "clang"), ("lld", "ld.lld")]; + +/// A ToyOS-hosted clang and LLD, held in use for as long as this lives. +pub struct HostedClang { + pub dir: PathBuf, + _using: Guard, +} + +/// `cargo run -- --hosted-clang`: make this tree's, and name each binary with +/// its size. +pub fn dispatch(root: &Path) { + let mut lock = crate::buildlock::shared(root, "the ToyOS-hosted clang"); + let sysroot = crate::toolchain::ensure(root, &mut lock); + let fork = crate::sysroot::fork_checkout(root, &mut lock); + let hosted = resolve(&keystore::host(), &fork, sysroot.dir(), &crate::n2::ninja(root)); + for (_, kept) in BINARIES { + let binary = hosted.dir.join("bin").join(kept); + let size = fs::metadata(&binary).unwrap_or_else(|e| panic!("stat {}: {e}", binary.display())).len(); + println!("{} ({size} bytes)", binary.display()); + } +} + +/// The product the LLVM fork `fork` names and the sysroot `sysroot` holds, in +/// `store`: made if nobody on this host has made it, under `ninja`. +pub fn resolve(store: &Path, fork: &Path, sysroot: &Path, ninja: &Path) -> HostedClang { + let key = key(fork, sysroot); + let dir = Keyed::HostedClang.store(store).join(&key); + let make = || place(fork, sysroot, ninja, &key, &dir); + let using = keystore::made(store, Keyed::HostedClang, &key, || defect(&dir), make); + HostedClang { dir, _using: using } +} + +/// The key of what `fork`'s LLVM and the sysroot at `sysroot` make. +fn key(fork: &Path, sysroot: &Path) -> Key { + let named = sysroot.file_name().and_then(|n| n.to_str()).and_then(Key::parse); + let sysroot = + named.unwrap_or_else(|| panic!("{} is no sysroot of the store: its name is no key", sysroot.display())); + key_of(&crate::llvm::key(fork), &sysroot, &options()) +} + +fn key_of(llvm: &Key, sysroot: &Key, options: &[(String, String)]) -> Key { + let options: Vec = options.iter().map(|(name, value)| format!("{name}={value}")).collect(); + let parts = [RECIPE, TARGET.userland(), &SOURCES.join(" "), &options.join("\n"), llvm.as_str(), sysroot.as_str()]; + Key::of(parts.join("\n\0\n").as_bytes()) +} + +/// [`OPTIONS`], and every host library off. +fn options() -> Vec<(String, String)> { + let off = crate::llvm::NO_HOST_LIBRARIES.iter().map(|name| (*name, "OFF")); + OPTIONS.into_iter().chain(off).map(|(name, value)| (name.to_string(), value.to_string())).collect() +} + +/// Why `dir` is not a finished product, if it is not. +fn defect(dir: &Path) -> Option { + if !dir.join(SOURCE).is_file() { + return Some(format!("{} carries no {SOURCE}", dir.display())); + } + let binaries = BINARIES.iter().map(|(_, kept)| dir.join("bin").join(kept)); + let mut gone: Vec = binaries.filter(|p| !p.is_file()).map(|p| p.display().to_string()).collect(); + let headers = fs::read_dir(dir.join("lib/clang")).ok().and_then(|mut d| d.next()).and_then(Result::ok); + if !headers.is_some_and(|version| version.path().join("include/stddef.h").is_file()) { + gone.push(dir.join("lib/clang//include").display().to_string()); + } + (!gone.is_empty()).then(|| format!("{} carries no {}", dir.display(), gone.join(", "))) +} + +/// Build what `key` names and put it at `dir`. The caller holds the key's lock. +fn place(fork: &Path, sysroot: &Path, ninja: &Path, key: &Key, dir: &Path) { + eprintln!("Building the ToyOS-hosted clang and LLD {key}: nobody on this host has"); + let scratch = dir.with_extension("build"); + keystore::remove(&scratch); + let sources = scratch.join("src"); + export(fork, &sources); + let built = scratch.join("build"); + build(fork, &sources, &built, &CSysroot::of(sysroot, TARGET), ninja, &scratch); + + let partial = dir.with_extension("partial"); + keystore::remove(&partial); + let bin = partial.join("bin"); + fs::create_dir_all(&bin).unwrap_or_else(|e| panic!("create {}: {e}", bin.display())); + for (name, kept) in BINARIES { + // `fs::copy` follows the link the build installs clang as. + let (from, to) = (built.join("bin").join(name), bin.join(kept)); + fs::copy(&from, &to).unwrap_or_else(|e| panic!("copy {} -> {}: {e}", from.display(), to.display())); + static_elf(&to); + } + let resource = crate::clang::resource_version(&built); + let version = resource.file_name().unwrap_or_else(|| panic!("{} names no version", resource.display())); + clone_tree(&resource.join("include"), &partial.join("lib/clang").join(version).join("include")); + // What was built is what the key names, or it is not that key's. + let again = self::key(fork, sysroot); + assert!(again == *key, "the sources moved while {key} was being built (they now name {again}); nothing was kept"); + let source = partial.join(SOURCE); + fs::write(&source, format!("{key}\n")).unwrap_or_else(|e| panic!("write {}: {e}", source.display())); + crate::llvm::read_only(&partial); + keystore::retire(dir); + fs::rename(&partial, dir).unwrap_or_else(|e| panic!("rename {} -> {}: {e}", partial.display(), dir.display())); + keystore::remove(&scratch); +} + +/// Write [`SOURCES`] as the commit `fork`'s LLVM gitlink names holds them into +/// `dest`, from the LLVM repository of `fork`'s git directory, through an index +/// of their own: no checkout's files reach them. +fn export(fork: &Path, dest: &Path) { + let commit = gitlink(fork, LLVM); + let common = git_out(fork, &["rev-parse", "--path-format=absolute", "--git-common-dir"]); + let repository = Path::new(common.trim()).join("modules").join(LLVM); + fs::create_dir_all(dest).unwrap_or_else(|e| panic!("create {}: {e}", dest.display())); + let index = toyos_tmpdir::TempDir::new("hosted-clang-index"); + let out = Command::new("git") + .env("GIT_DIR", &repository) + .env("GIT_INDEX_FILE", index.join("index")) + .args(["-c", "core.sparseCheckout=false", "--work-tree"]) + .arg(dest) + .args(["checkout", &commit, "--"]) + .args(SOURCES) + .output() + .unwrap_or_else(|e| panic!("run git in {}: {e}", repository.display())); + assert!( + out.status.success(), + "git checkout {commit} from {} into {}: {}", + repository.display(), + dest.display(), + String::from_utf8_lossy(&out.stderr).trim(), + ); +} + +/// Configure LLVM at `sources`, the commit `fork` names, for `c`'s target, +/// in `built`, and build clang and LLD there under `ninja`; `scratch` holds +/// the links CMake runs the compilers by. +fn build(fork: &Path, sources: &Path, built: &Path, c: &CSysroot, ninja: &Path, scratch: &Path) { + // The driver links C++ when run as `clang++`, and LLVM's archiver indexes + // as `ranlib` when run by that name. + let links = scratch.join("bin"); + fs::create_dir_all(&links).unwrap_or_else(|e| panic!("create {}: {e}", links.display())); + let (cxx, ranlib) = (links.join("clang++"), links.join("llvm-ranlib")); + for (link, to) in [(&cxx, &c.clang), (&ranlib, &c.ar)] { + std::os::unix::fs::symlink(to, link) + .unwrap_or_else(|e| panic!("symlink {} -> {}: {e}", link.display(), to.display())); + } + let path = |p: &Path| p.display().to_string(); + let (cc, cxx_native) = crate::llvm::host_compilers(); + let mut native = vec![format!("-DCMAKE_C_COMPILER={}", path(&cc)), format!("-DCMAKE_CXX_COMPILER={}", path(&cxx_native))]; + native.extend(crate::llvm::NO_HOST_LIBRARIES.iter().map(|name| format!("-D{name}=OFF"))); + let mut definitions = options(); + definitions.extend(crate::llvm::stamp_options(fork)); + definitions.push(("LLVM_VERSION_SUFFIX".to_string(), version_suffix(&c.clang))); + definitions.extend([ + ("CMAKE_TOOLCHAIN_FILE".to_string(), path(&c.cmake_toolchain())), + ("CMAKE_C_COMPILER".to_string(), path(&c.clang)), + ("CMAKE_ASM_COMPILER".to_string(), path(&c.clang)), + ("CMAKE_CXX_COMPILER".to_string(), path(&cxx)), + ("CMAKE_AR".to_string(), path(&c.ar)), + ("CMAKE_RANLIB".to_string(), path(&ranlib)), + ("CMAKE_MAKE_PROGRAM".to_string(), path(ninja)), + ("CROSS_TOOLCHAIN_FLAGS_NATIVE".to_string(), native.join(";")), + ]); + let mut configure = Command::new("cmake"); + configure.args(["-G", "Ninja", "-Wno-dev", "-S"]).arg(sources.join("llvm")).arg("-B").arg(built); + configure.args(definitions.iter().map(|(name, value)| format!("-D{name}={value}"))); + run(configure, "configuration", ninja); + let mut make = Command::new(ninja); + make.arg("-C").arg(built).args(BINARIES.map(|(name, _)| name)); + run(make, "build", ninja); +} + +/// What the host's clang, the sysroot's, says it is beyond LLVM's version: +/// bootstrap's `LLVM_VERSION_SUFFIX`. Every object a clang compiles carries +/// what it says in its `.comment`, so this one says it too. +fn version_suffix(clang: &Path) -> String { + let mut command = Command::new(clang); + command.arg("--version"); + crate::llvm::clear(&mut command); + let out = command.output().unwrap_or_else(|e| panic!("run {command:?}: {e}")); + assert!(out.status.success(), "{command:?} failed: {}", String::from_utf8_lossy(&out.stderr)); + let said = String::from_utf8_lossy(&out.stdout); + let first = said.lines().next().unwrap_or_default(); + suffix(first).unwrap_or_else(|| panic!("{} says no version: {said}", clang.display())) +} + +/// The suffix of the version `line`, clang's first line of `--version`, names: +/// what follows its first `-`, and none without one. +fn suffix(line: &str) -> Option { + let version = line.strip_prefix("clang version ")?.split_whitespace().next()?; + Some(version.find('-').map_or(String::new(), |at| version[at..].to_string())) +} + +/// Run `command` seeing nothing of this process's environment but what the +/// LLVM build does (`llvm::clear`), with `ninja`'s directory first on `PATH` +/// for the nested build CMake runs, and refuse its failure. +fn run(mut command: Command, what: &str, ninja: &Path) { + crate::llvm::clear(&mut command); + let dir = ninja.parent().unwrap_or_else(|| panic!("{} is in no directory", ninja.display())); + let caller = std::env::var_os("PATH").unwrap_or_default(); + let path = std::env::join_paths(std::iter::once(dir.to_path_buf()).chain(std::env::split_paths(&caller))) + .unwrap_or_else(|e| panic!("{} cannot lead PATH: {e}", dir.display())); + command.env("PATH", path); + let status = command.status().unwrap_or_else(|e| panic!("run {command:?}: {e}")); + assert!(status.success(), "the ToyOS-hosted clang's {what} failed ({status}): its output above says why"); +} + +/// Refuse `elf` unless it is an executable for [`TARGET`] that names no +/// library it needs. +fn static_elf(elf: &Path) { + let bytes = fs::read(elf).unwrap_or_else(|e| panic!("read {}: {e}", elf.display())); + let machine = match TARGET { + Arch::X86_64 => toyos_elf::Machine::X86_64, + Arch::Aarch64 => toyos_elf::Machine::Aarch64, + }; + let layout = toyos_elf::Layout::parse(&bytes, machine) + .unwrap_or_else(|e| panic!("{} is no {} executable: {e:?}", elf.display(), TARGET.userland())); + let needed = layout.dynamic().map_or(0, |dynamic| { + let start = dynamic.file_offset() as usize; + let table = bytes.get(start..start + dynamic.image().len() as usize); + let table = table.unwrap_or_else(|| panic!("{}'s PT_DYNAMIC is past its end", elf.display())); + toyos_elf::Dynamic::needed(table).count() + }); + assert_eq!(needed, 0, "{} names {needed} libraries it needs, and it is linked statically", elf.display()); +} + +#[cfg(test)] +mod tests { + use toyos_tmpdir::TempDir; + + use super::*; + + fn key(text: &str) -> Key { + Key::of(text.as_bytes()) + } + + /// **The key moves with each input and with nothing else**: the host + /// LLVM's key, the sysroot's and every CMake option. + #[test] + fn the_key_moves_with_the_llvm_the_sysroot_and_the_options() { + let (llvm, sysroot) = (key("llvm"), key("sysroot")); + let base = key_of(&llvm, &sysroot, &options()); + assert_eq!(key_of(&llvm, &sysroot, &options()), base); + let mut moved = options(); + moved[0].1 = "Debug".to_string(); + for (what, other) in [ + ("the LLVM", key_of(&key("another llvm"), &sysroot, &options())), + ("the sysroot", key_of(&llvm, &key("another sysroot"), &options())), + ("an option", key_of(&llvm, &sysroot, &moved)), + ] { + assert_ne!(other, base, "{what} did not move the key"); + } + } + + /// **The suffix is what follows the version's first `-`**, none without + /// one, and a line that names no clang version names none. + #[test] + fn the_suffix_is_what_follows_the_version() { + let rev = "(https://github.com/ToyOSOrg/llvm-project.git ceaf0fbb8440)"; + assert_eq!(suffix(&format!("clang version 22.1.8-rust-dev {rev}")).as_deref(), Some("-rust-dev")); + assert_eq!(suffix(&format!("clang version 22.1.8 {rev}")).as_deref(), Some("")); + assert_eq!(suffix("Apple clang version 17.0.0 (clang-1700.0.13.5)"), None); + } + + /// **A product missing a binary or clang's headers is not whole**, nor one + /// without its `SOURCE`. + #[test] + fn a_product_without_a_binary_or_its_headers_is_not_whole() { + let dir = TempDir::new("hosted-clang"); + let write = |rel: &str| { + let path = dir.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, rel).unwrap(); + }; + assert!(defect(&dir).is_some_and(|d| d.contains(SOURCE))); + for rel in [SOURCE, "bin/clang", "bin/ld.lld", "lib/clang/22/include/stddef.h"] { + write(rel); + } + assert_eq!(defect(&dir), None); + for gone in ["bin/ld.lld", "lib/clang/22/include/stddef.h"] { + fs::remove_file(dir.join(gone)).unwrap(); + assert!(defect(&dir).is_some_and(|d| d.contains(gone.split("22").next().unwrap())), "{gone}: {:?}", defect(&dir)); + write(gone); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 45229856279..7bc624b9505 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ pub mod compiler; pub mod firmware; #[cfg(test)] pub mod gitfixture; +pub mod hostedclang; pub mod flags; pub mod hostws; pub mod identity; diff --git a/src/libcxx.rs b/src/libcxx.rs index f714b9f94b8..c3ff6107ed6 100644 --- a/src/libcxx.rs +++ b/src/libcxx.rs @@ -40,9 +40,10 @@ pub(crate) const OPTIONS: [(&str, &str); 18] = [ ("LIBCXXABI_HAS_CXA_THREAD_ATEXIT_IMPL", "ON"), // The C library has no `dladdr`, so libunwind names no function it unwinds. ("LIBUNWIND_ADDITIONAL_COMPILE_FLAGS", "-D_LIBUNWIND_USE_DLADDR=0"), - // The C library has no directory, `stat` or path surface for it - // (`issues/libcxx-is-built-without-std-filesystem.md`). - ("LIBCXX_ENABLE_FILESYSTEM", "OFF"), + // `open` refuses every directory, so no descriptor names one for `openat` + // and `unlinkat` to resolve against: `remove_all` walks a directory + // iterator, as libc++'s Windows does. + ("LIBCXX_ADDITIONAL_COMPILE_FLAGS", "-DREMOVE_ALL_USE_DIRECTORY_ITERATOR"), ("LIBCXX_INCLUDE_BENCHMARKS", "OFF"), ("LIBCXX_INCLUDE_TESTS", "OFF"), ]; diff --git a/src/llvm.rs b/src/llvm.rs index cb55f7a3998..0b41e8c9980 100644 --- a/src/llvm.rs +++ b/src/llvm.rs @@ -71,7 +71,7 @@ const ENVIRONMENT: [&str; 2] = ["PATH", "TMPDIR"]; /// what a host has installed never decides what is built. Bootstrap's LLD step /// takes none of these: LLD looks for no library unless asked, and links what this /// LLVM chose. -const NO_HOST_LIBRARIES: [&str; 12] = [ +pub(crate) const NO_HOST_LIBRARIES: [&str; 12] = [ "LLVM_ENABLE_ZLIB", "LLVM_ENABLE_ZSTD", "LLVM_ENABLE_LIBXML2", @@ -147,6 +147,26 @@ fn stamp(fork: &Path) -> Stamp { Stamp { revision: gitlink(fork, LLVM), repository: url.trim().to_string() } } +impl Stamp { + /// The CMake options that make LLVM say it. + fn options(&self) -> [(&'static str, &str); 2] { + [("LLVM_FORCE_VC_REVISION", &self.revision), ("LLVM_FORCE_VC_REPOSITORY", &self.repository)] + } +} + +/// The CMake options that make an LLVM built from `fork`'s say what it says +/// (`src/hostedclang.rs`). +pub(crate) fn stamp_options(fork: &Path) -> Vec<(String, String)> { + stamp(fork).options().iter().map(|(name, value)| (name.to_string(), value.to_string())).collect() +} + +/// The C and C++ compilers an LLVM is built with on this host, as its key +/// names them (`src/hostedclang.rs` builds its tablegens with them). +pub(crate) fn host_compilers() -> (PathBuf, PathBuf) { + let tools = host_tools(); + (tools.cc.clone(), tools.cxx.clone()) +} + /// [`key`], with what it reads beside `fork`'s committed `src/bootstrap`: /// `config` names the LLVM's commit ([`stamp`]). fn key_of(fork: &Path, recipe: &str, config: &str, tools: &str) -> Key { @@ -377,7 +397,7 @@ fn check_out_committed(checkout: &Path, commit: &str, paths: &[&str], dest: &Pat /// Take write permission from every file and directory under `dir`, and from /// `dir`. -fn read_only(dir: &Path) { +pub(crate) fn read_only(dir: &Path) { for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", dir.display())).path(); let meta = fs::symlink_metadata(&path).unwrap_or_else(|e| panic!("stat {}: {e}", path.display())); @@ -454,8 +474,7 @@ fn build_in_fork(root: &Path, fork: &Path) -> PathBuf { /// host alone. fn config_text(build_dir: &Path, host: &str, tools: &HostTools, stamp: &Stamp) -> String { let off = NO_HOST_LIBRARIES.iter().map(|option| format!("{option} = \"OFF\"")); - let stamped = [("LLVM_FORCE_VC_REVISION", &stamp.revision), ("LLVM_FORCE_VC_REPOSITORY", &stamp.repository)]; - let defines: Vec = off.chain(stamped.map(|(option, value)| format!("{option} = \"{value}\""))).collect(); + let defines: Vec = off.chain(stamp.options().map(|(option, value)| format!("{option} = \"{value}\""))).collect(); format!( r#"change-id = "ignore" profile = "compiler" diff --git a/src/main.rs b/src/main.rs index fb9ccd2ed8a..ebdcc055648 100644 --- a/src/main.rs +++ b/src/main.rs @@ -107,6 +107,11 @@ fn main() { toyos_build::clippy::dispatch(&root); return; } + // Builds no image, and runs no guest. + if asked(&flags::HOSTED_CLANG) { + toyos_build::hostedclang::dispatch(&root); + return; + } // Writes one file outside the checkout and builds nothing. if asked(&flags::SIGNING_KEY_NEW) { match toyos_build::signing::mint_owner_key() { diff --git a/src/release.rs b/src/release.rs index 06dc9c1730c..87d5826f6b8 100644 --- a/src/release.rs +++ b/src/release.rs @@ -54,6 +54,10 @@ const LAYERS: [(Keyed, &str); 4] = [ (Keyed::Sysroot, "sysroot"), ]; +/// Why no layer is the ToyOS-hosted clang: it is made only when asked for +/// (`src/hostedclang.rs`), and a release carries what every build reads. +const NOT_A_LAYER: &str = "a toolchain release carries no ToyOS-hosted clang: it is made only when asked for"; + /// One of [`LAYERS`] of this tree's toolchain: the key the build system files /// it under, and where it is, relative to the checkout. struct Layer { @@ -135,6 +139,7 @@ fn layers(root: &Path) -> Vec { Keyed::Compiler => &compiler, Keyed::Freestanding => &freestanding, Keyed::Sysroot => &sysroot, + Keyed::HostedClang => unreachable!("{NOT_A_LAYER}"), }; let dir = kind.store(&store(root)).join(key); let path = dir.strip_prefix(root).expect("a runner's store is in its checkout").to_path_buf(); @@ -154,6 +159,7 @@ fn defect(root: &Path, layer: &Layer) -> Option { } Keyed::Freestanding => crate::sysroot::unpublished(&dir), Keyed::Sysroot => crate::sysroot::unfinished(&dir), + Keyed::HostedClang => unreachable!("{NOT_A_LAYER}"), } } diff --git a/tests/libc-arch/src/alarm_requests.rs b/tests/libc-arch/src/alarm_requests.rs new file mode 100644 index 00000000000..5971595d361 --- /dev/null +++ b/tests/libc-arch/src/alarm_requests.rs @@ -0,0 +1,27 @@ +//! What `alarm` answers and arms: the seconds an earlier alarm has left, +//! rounded up and 0 once it is due or when there is none; and 0 seconds arm +//! nothing. + +use crate::alarmreq::{due, left}; + +const SEC: u64 = 1_000_000_000; + +#[test] +fn an_alarm_is_due_its_seconds_after_it_is_armed_and_zero_arms_none() { + assert_eq!(due(7, 0), None); + assert_eq!(due(7, 1), Some(7 + SEC)); + assert_eq!(due(u64::from(u32::MAX), u32::MAX), Some(u64::from(u32::MAX) * (SEC + 1))); +} + +#[test] +fn the_seconds_left_round_up_and_end_at_zero() { + let armed = due(100, 5); + assert_eq!(left(armed, 100), 5); + assert_eq!(left(armed, 100 + 1), 5, "a second begun is a second left"); + assert_eq!(left(armed, 100 + SEC), 4); + assert_eq!(left(armed, 100 + 5 * SEC - 1), 1, "an alarm not yet due answers at least 1"); + assert_eq!(left(armed, 100 + 5 * SEC), 0, "an alarm due answers 0"); + assert_eq!(left(armed, 100 + 6 * SEC), 0, "an alarm past due answers 0"); + assert_eq!(left(None, 100), 0); + assert_eq!(left(due(0, u32::MAX), 0), u32::MAX); +} diff --git a/tests/libc-arch/src/fparts_differential.rs b/tests/libc-arch/src/fparts_differential.rs index 43f513e8c6a..6b9b1433489 100644 --- a/tests/libc-arch/src/fparts_differential.rs +++ b/tests/libc-arch/src/fparts_differential.rs @@ -1,12 +1,17 @@ -//! `modf` and `logb` against the host C library's, bit for bit: every special -//! value, the edges of the subnormals and of the integers a double holds, and -//! a spread of bit patterns across every exponent. +//! `modf`, `round`, `lround` and `logb` against the host C library's, bit for +//! bit: every special value, the edges of the subnormals, of the halves and of +//! the integers a double and a `long` hold, and a spread of bit patterns across +//! every exponent. use crate::fparts; extern "C" { #[link_name = "modf"] fn host_modf(x: f64, int: *mut f64) -> f64; + #[link_name = "round"] + fn host_round(x: f64) -> f64; + #[link_name = "lround"] + fn host_lround(x: f64) -> i64; #[link_name = "logb"] fn host_logb(x: f64) -> f64; } @@ -30,9 +35,15 @@ fn inputs() -> Vec { f64::MIN_POSITIVE / 2.0, f64::from_bits(1), f64::from_bits(0x000f_ffff_ffff_ffff), + 0.49999999999999994, + 3.5, 4_503_599_627_370_495.5, 4_503_599_627_370_496.0, 9_007_199_254_740_993.0, + // The last double below 2^63, 2^63 itself, and -2^63. + 9_223_372_036_854_774_784.0, + 9_223_372_036_854_775_808.0, + -9_223_372_036_854_775_808.0, f64::INFINITY, f64::NAN, ]; @@ -62,6 +73,37 @@ fn modf_is_the_host_libraries() { } } +#[test] +fn round_is_the_host_libraries() { + for x in inputs() { + // SAFETY: a pure function of its argument. + let host = unsafe { host_round(x) }; + let ours = fparts::round(x); + assert!(same(ours, host), "round({x:e} = {:#x}): {ours:e}, the host {host:e}", x.to_bits()); + } +} + +/// Where the rounding is a `long`, the host's; where it is not, a NaN among +/// them, the domain error, whose value the host leaves unspecified. +#[test] +fn lround_is_the_host_libraries_and_out_of_range_its_domain_error() { + let mut refused = 0; + for x in inputs() { + // SAFETY: a pure function of its argument. + let rounded = unsafe { host_round(x) }; + let fits = (-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&rounded); + match fparts::lround(x) { + // SAFETY: a pure function of its argument, here inside its domain. + Ok(ours) => assert!(fits && ours == unsafe { host_lround(x) }, "lround({x:e}): {ours}"), + Err(fparts::Domain) => { + assert!(!fits, "lround({x:e}) refused a rounding {rounded:e} that is a long"); + refused += 1; + } + } + } + assert!(refused > 0, "no input was outside a long"); +} + #[test] fn logb_is_the_host_libraries_and_a_zero_its_pole() { for x in inputs() { diff --git a/tests/libc-arch/src/lib.rs b/tests/libc-arch/src/lib.rs index a6fa47e5dfd..f55943a5421 100644 --- a/tests/libc-arch/src/lib.rs +++ b/tests/libc-arch/src/lib.rs @@ -14,6 +14,9 @@ #[cfg(test)] extern crate alloc; +#[cfg(test)] +#[path = "../../../userland/libc/src/alarmreq.rs"] +mod alarmreq; #[cfg(test)] #[path = "../../../userland/libc/src/arch/mod.rs"] mod arch; @@ -57,6 +60,8 @@ mod text; #[path = "../../../userland/libc/src/utf8.rs"] mod utf8; +#[cfg(test)] +mod alarm_requests; #[cfg(test)] mod descriptor_requests; #[cfg(test)] diff --git a/userland/libc/include/math.h b/userland/libc/include/math.h index a8b572551a6..3ca45b1e729 100644 --- a/userland/libc/include/math.h +++ b/userland/libc/include/math.h @@ -61,6 +61,7 @@ double sinh(double x); double cosh(double x); double tanh(double x); double round(double x); +long lround(double x); double trunc(double x); double modf(double x, double *iptr); double logb(double x); diff --git a/userland/libc/include/stdio.h b/userland/libc/include/stdio.h index 27f39a1c48f..900f7424aa8 100644 --- a/userland/libc/include/stdio.h +++ b/userland/libc/include/stdio.h @@ -3,6 +3,7 @@ #include #include +#include #include #ifdef __cplusplus @@ -40,6 +41,9 @@ size_t fread(void *ptr, size_t size, size_t nmemb, FILE *stream); size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); int fseek(FILE *stream, long offset, int whence); long ftell(FILE *stream); +int fseeko(FILE *stream, off_t offset, int whence); +off_t ftello(FILE *stream); +void setbuf(FILE *stream, char *buf); void rewind(FILE *stream); int fflush(FILE *stream); int feof(FILE *stream); diff --git a/userland/libc/include/sys/time.h b/userland/libc/include/sys/time.h index 2dec877691f..2c9f97ea789 100644 --- a/userland/libc/include/sys/time.h +++ b/userland/libc/include/sys/time.h @@ -18,6 +18,7 @@ struct timezone { }; int gettimeofday(struct timeval *tv, struct timezone *tz); +int utimes(const char *path, const struct timeval times[2]); #ifdef __cplusplus } diff --git a/userland/libc/include/sys/wait.h b/userland/libc/include/sys/wait.h index f60449eb1c3..f87db19b441 100644 --- a/userland/libc/include/sys/wait.h +++ b/userland/libc/include/sys/wait.h @@ -15,7 +15,11 @@ extern "C" { #define WIFEXITED(s) (WTERMSIG(s) == 0) #define WIFSIGNALED(s) (WTERMSIG(s) != 0) +struct rusage; + pid_t waitpid(pid_t pid, int *status, int options); +pid_t wait(int *status); +pid_t wait4(pid_t pid, int *status, int options, struct rusage *usage); #ifdef __cplusplus } diff --git a/userland/libc/include/unistd.h b/userland/libc/include/unistd.h index c93aa56e297..6024325d058 100644 --- a/userland/libc/include/unistd.h +++ b/userland/libc/include/unistd.h @@ -24,6 +24,7 @@ char *getcwd(char *buf, size_t size); int chdir(const char *path); int access(const char *path, int mode); unsigned int sleep(unsigned int seconds); +unsigned int alarm(unsigned int seconds); int usleep(unsigned int usec); int isatty(int fd); int execvp(const char *file, char *const argv[]); @@ -34,6 +35,8 @@ int pipe(int pipefd[2]); void _exit(int status); int fsync(int fd); int ftruncate(int fd, off_t length); +int truncate(const char *path, off_t length); +long pathconf(const char *path, int name); ssize_t readlink(const char *path, char *buf, size_t size); int symlink(const char *target, const char *linkpath); int link(const char *existing, const char *newpath); @@ -59,6 +62,8 @@ long sysconf(int name); #define _SC_CLK_TCK 2 #define _SC_NPROCESSORS_ONLN 84 +#define _PC_PATH_MAX 4 + #define F_OK 0 #define R_OK 4 #define W_OK 2 diff --git a/userland/libc/src/alarm.rs b/userland/libc/src/alarm.rs new file mode 100644 index 00000000000..bdc1de4f097 --- /dev/null +++ b/userland/libc/src/alarm.rs @@ -0,0 +1,71 @@ +//! `alarm`: one per process, kept by a thread of this library's, started by +//! the first alarm armed, that sleeps until it is due and then does what +//! `SIGALRM`'s default action does, ending the process. A handler `sigaction` +//! is given never runs: this library keeps none +//! (`issues/an-alarm-reaches-no-sigalrm-handler.md`). + +use core::ptr; +use core::sync::atomic::{AtomicU32, Ordering}; + +use toyos_abi::syscall; + +use crate::alarmreq; +use crate::pthread::Lock; + +/// What the process ends with when its alarm is due: the code of an end by +/// `SIGALRM` (14), as `abort`'s is `SIGABRT`'s. +const ALARM_END: i32 = 128 + 14; + +struct Alarm { + /// When the alarm is due, in the monotonic clock's nanoseconds. + due: Option, + /// Whether the keeper runs. + kept: bool, +} + +static ALARM: Lock = Lock::new(Alarm { due: None, kept: false }); + +/// Moved at every `alarm`, so a keeper asleep on an older one wakes. +static TURN: AtomicU32 = AtomicU32::new(0); + +fn now() -> u64 { + toyos_abi::clock::nanos_since_boot() +} + +#[no_mangle] +pub extern "C" fn alarm(seconds: u32) -> u32 { + let now = now(); + let mut alarm = ALARM.lock(); + let left = alarmreq::left(alarm.due, now); + alarm.due = alarmreq::due(now, seconds); + if alarm.due.is_some() && !alarm.kept { + // Never joined: it lives as long as the process. + let mut thread = 0u64; + // SAFETY: `keep` is a thread entry that takes and returns nothing. + let refused = unsafe { crate::pthread::pthread_create(&mut thread, ptr::null(), keep, ptr::null_mut()) }; + assert_eq!(refused, 0, "alarm: no thread to keep it"); + alarm.kept = true; + } + TURN.fetch_add(1, Ordering::Release); + // SAFETY: `TURN` is a live, aligned u32. + unsafe { syscall::futex_wake(TURN.as_ptr(), 1) }; + left +} + +/// Sleep until the alarm is due or moved, and end the process once it is due. +unsafe extern "C" fn keep(_: *mut u8) -> *mut u8 { + loop { + // Read before the alarm, so an `alarm` between the two moves it and + // the wait below returns at once. + let turn = TURN.load(Ordering::Acquire); + let wait = match ALARM.lock().due { + None => None, + Some(due) => match due.checked_sub(now()) { + Some(wait) if wait > 0 => Some(wait), + _ => syscall::exit(ALARM_END), + }, + }; + // SAFETY: `TURN` is a live, aligned u32. + unsafe { syscall::futex_wait(TURN.as_ptr(), turn, wait) }; + } +} diff --git a/userland/libc/src/alarmreq.rs b/userland/libc/src/alarmreq.rs new file mode 100644 index 00000000000..e5653b00be0 --- /dev/null +++ b/userland/libc/src/alarmreq.rs @@ -0,0 +1,21 @@ +//! What `alarm` answers and arms, in nanoseconds of the monotonic clock. It +//! reads nothing but what it is handed, so the host tests it +//! (`toyos-libc-copies`). + +const NANOS_PER_SEC: u64 = 1_000_000_000; + +/// When an alarm of `seconds` armed at `now` is due; none for 0, which +/// disarms. +pub(crate) fn due(now: u64, seconds: u32) -> Option { + // No overflow: `u32::MAX` seconds are under 2^62 nanoseconds, and `now` + // counts from boot. + (seconds != 0).then(|| now + u64::from(seconds) * NANOS_PER_SEC) +} + +/// The seconds an alarm due at `due` has left at `now`, rounded up: one that +/// has any time left answers at least 1, and none, or one already due, 0. +pub(crate) fn left(due: Option, now: u64) -> u32 { + let Some(due) = due.filter(|&due| due > now) else { return 0 }; + // At most the `u32` seconds `due` was armed with. + (due - now).div_ceil(NANOS_PER_SEC) as u32 +} diff --git a/userland/libc/src/errno.rs b/userland/libc/src/errno.rs index 6bd30ef43b2..a6bd0f10ba6 100644 --- a/userland/libc/src/errno.rs +++ b/userland/libc/src/errno.rs @@ -33,6 +33,7 @@ pub(crate) const ENODEV: i32 = 19; pub(crate) const EINVAL: i32 = 22; pub(crate) const ENOSPC: i32 = 28; pub(crate) const EPIPE: i32 = 32; +pub(crate) const EDOM: i32 = 33; pub(crate) const ERANGE: i32 = 34; pub(crate) const EDEADLK: i32 = 35; pub(crate) const ENOSYS: i32 = 38; diff --git a/userland/libc/src/fparts.rs b/userland/libc/src/fparts.rs index ff89184e8eb..2531f2ae82e 100644 --- a/userland/libc/src/fparts.rs +++ b/userland/libc/src/fparts.rs @@ -1,5 +1,6 @@ -//! A double's parts: `modf`'s integral and fractional parts and `logb`'s -//! exponent, read off its bits. It reads and sets nothing but what it is +//! A double's parts: `modf`'s integral and fractional parts, the integer +//! `round` and `lround` take from them, and `logb`'s exponent, read off its +//! bits. It reads and sets nothing but what it is //! handed, so the host holds it to its own C library's (`toyos-libc-copies`). const MANTISSA_BITS: u32 = 52; @@ -37,6 +38,31 @@ pub(crate) fn modf(x: f64) -> (f64, f64) { (int, x - int) } +/// `round`: `x` to the nearest integer, a half away from zero, keeping its +/// sign; an integer, an infinity or a NaN is itself. +pub(crate) fn round(x: f64) -> f64 { + let (int, fraction) = modf(x); + if fraction.abs() < 0.5 { + return int; + } + // Exact: below 2^52 every integer and its successor are doubles, and at or + // above it `modf` answers no fraction. + if x.is_sign_negative() { int - 1.0 } else { int + 1.0 } +} + +/// `lround` of a value whose rounding is no `long`, a NaN among them: POSIX's +/// domain error, `errno` `EDOM`. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Domain; + +/// `lround`: [`round`] as a `long`. +pub(crate) fn lround(x: f64) -> Result { + // `long`'s range as doubles, `[-2^63, 2^63)`; a NaN is in neither half. + const LIMIT: f64 = 9_223_372_036_854_775_808.0; + let rounded = round(x); + if (-LIMIT..LIMIT).contains(&rounded) { Ok(rounded as i64) } else { Err(Domain) } +} + /// `logb` at a zero: POSIX's pole error, `-inf` with `errno` `ERANGE`. #[derive(Debug, PartialEq, Eq)] pub(crate) struct Pole; diff --git a/userland/libc/src/lib.rs b/userland/libc/src/lib.rs index 94e87be16c6..027601e23e6 100644 --- a/userland/libc/src/lib.rs +++ b/userland/libc/src/lib.rs @@ -4,6 +4,8 @@ extern crate alloc; +mod alarm; +mod alarmreq; mod arch; mod ctype; mod elfsym; diff --git a/userland/libc/src/math.rs b/userland/libc/src/math.rs index 3176755636b..434535b97b7 100644 --- a/userland/libc/src/math.rs +++ b/userland/libc/src/math.rs @@ -210,7 +210,16 @@ pub extern "C" fn trunc(x: f64) -> f64 { #[no_mangle] pub extern "C" fn round(x: f64) -> f64 { - floor(x + 0.5) + crate::fparts::round(x) +} + +/// A domain error answers `long`'s least value, as x86-64's conversion does. +#[no_mangle] +pub extern "C" fn lround(x: f64) -> i64 { + crate::fparts::lround(x).unwrap_or_else(|crate::fparts::Domain| { + crate::errno::set(crate::errno::EDOM); + i64::MIN + }) } #[no_mangle] diff --git a/userland/libc/src/misc.rs b/userland/libc/src/misc.rs index ba2b635eeb6..e38e12ec1f2 100644 --- a/userland/libc/src/misc.rs +++ b/userland/libc/src/misc.rs @@ -122,6 +122,18 @@ pub unsafe extern "C" fn waitpid(_pid: i32, _status: *mut i32, _options: i32) -> -1 } +/// `waitpid(-1, status, 0)`. +#[no_mangle] +pub unsafe extern "C" fn wait(status: *mut i32) -> i32 { + unsafe { waitpid(-1, status, 0) } +} + +/// [`waitpid`]: it answers no child, so no usage is written. +#[no_mangle] +pub unsafe extern "C" fn wait4(pid: i32, status: *mut i32, options: i32, _usage: *mut u8) -> i32 { + unsafe { waitpid(pid, status, options) } +} + // Exit / abort / atexit /// A handler `exit` runs: `atexit`'s takes nothing, `__cxa_atexit`'s its object. diff --git a/userland/libc/src/posix_io.rs b/userland/libc/src/posix_io.rs index 47dfd48e70f..154ce1ca7f5 100644 --- a/userland/libc/src/posix_io.rs +++ b/userland/libc/src/posix_io.rs @@ -178,6 +178,34 @@ pub unsafe extern "C" fn ftruncate(raw_fd: i32, length: i64) -> i32 { } } +/// [`ftruncate`] of `path` opened for writing. +#[no_mangle] +pub unsafe extern "C" fn truncate(path: *const u8, length: i64) -> i32 { + let fd = unsafe { open(path, O_WRONLY, 0) }; + if fd < 0 { + return -1; + } + let truncated = unsafe { ftruncate(fd, length) }; + unsafe { close(fd) }; + truncated +} + +/// `_PC_PATH_MAX`, `unistd.h`'s. +const PC_PATH_MAX: i32 = 4; + +/// The longest path the kernel resolves, for every path; no other limit is +/// answered. +#[no_mangle] +pub unsafe extern "C" fn pathconf(_path: *const u8, name: i32) -> i64 { + match name { + PC_PATH_MAX => toyos::fs::MAX_PATH as i64, + _ => { + crate::errno::set(EINVAL); + -1 + } + } +} + #[no_mangle] pub unsafe extern "C" fn fsync(raw_fd: i32) -> i32 { match syscall::fsync(fd(raw_fd)) { diff --git a/userland/libc/src/refused.rs b/userland/libc/src/refused.rs index b6f1531c8a9..972236f0198 100644 --- a/userland/libc/src/refused.rs +++ b/userland/libc/src/refused.rs @@ -71,6 +71,12 @@ pub extern "C" fn fchmod(_fd: i32, _mode: u32) -> i32 { refuse() } +/// No call sets a file's times. +#[no_mangle] +pub unsafe extern "C" fn utimes(_path: *const u8, _times: *const u8) -> i32 { + refuse() +} + /// No call answers a filesystem's size or free space. #[no_mangle] pub unsafe extern "C" fn statvfs(_path: *const u8, _buf: *mut u8) -> i32 { diff --git a/userland/libc/src/stdio.rs b/userland/libc/src/stdio.rs index 00e558458a8..c9d6fcf62f0 100644 --- a/userland/libc/src/stdio.rs +++ b/userland/libc/src/stdio.rs @@ -458,6 +458,35 @@ pub unsafe extern "C" fn ftell(f: *mut FILE) -> i64 { if at < 0 { at } else { at - unsafe { (*f).unget_len } as i64 } } +/// [`fseek`]: an `off_t` is a `long`. +#[no_mangle] +pub unsafe extern "C" fn fseeko(f: *mut FILE, offset: i64, whence: i32) -> i32 { + unsafe { fseek(f, offset, whence) } +} + +/// [`ftell`]: an `off_t` is a `long`. +#[no_mangle] +pub unsafe extern "C" fn ftello(f: *mut FILE) -> i64 { + unsafe { ftell(f) } +} + +/// `buf`, `BUFSIZ` bytes its caller keeps, as `f`'s buffer, fully buffered; a +/// null `buf` leaves `f` unbuffered. What `f` held pending is written first, so +/// the buffer it replaces is free to go. +#[no_mangle] +pub unsafe extern "C" fn setbuf(f: *mut FILE, buf: *mut u8) { + unsafe { + flush_buf(f); + if (*f).owned { + super::memory::free((*f).buf); + } + (*f).buf = buf; + (*f).owned = false; + (*f).cap = if buf.is_null() { 0 } else { BUFSIZ }; + (*f).mode = if buf.is_null() { IONBF } else { IOFBF }; + } +} + #[no_mangle] pub unsafe extern "C" fn rewind(f: *mut FILE) { if !f.is_null() { From ea95e75aca418a23ab36d675fc33efc83bb5190c Mon Sep 17 00:00:00 2001 From: japabu Date: Fri, 9 Oct 2026 18:04:59 +0200 Subject: [PATCH 2/6] An ignored SIGALRM drops a due alarm, the corpus reads libc's new names back, and the bootstrap defect stays open Answers the review of b9f562d9d on #809. libc: `signal` and `sigaction` keep SIGALRM's disposition, and a due alarm under SIG_IGN is dropped instead of ending the process (`alarmreq::ends`, held on the host). `signal` takes and answers a pointer, as signal.h declares it: SIG_DFL is null, which no Rust fn pointer may be. A SIGALRM blocked in every thread still ends the process, since libc keeps each thread's mask in that thread and std's threads are none of libc's; the issue records it with its exit extended. The corpus: 206_libc_refusals reads utimes' ENOSYS, wait's and wait4's ECHILD and pathconf's EINVAL; 207_libc_names reads truncate through stat, setbuf's buffer and none, an fseeko/ftello round trip, lround's halves and its EDOM, pathconf's _PC_PATH_MAX, SIGALRM's disposition through signal and sigaction, and alarm(5), alarm(0), alarm(0). The owed lines go from the issues that carried them. Build: hostedclang's export is llvm::check_out_committed, which now takes the git directory it reads; static_elf goes, no test having shown it refuse anything; release.rs's layers are a Part of their own, so the hosted clang is no layer by type and NOT_A_LAYER goes. Issues: the bootstrap defect is open again, re-scoped: the CMake build in src/hostedclang.rs goes once bootstrap's ToyOS-host LLVM makes clang and lld, owner M3. remove_all's race is a defect of its own with an owner and an exit that removes REMOVE_ALL_USE_DIRECTORY_ITERATOR. The host-tools rows name src/hostedclang.rs; the track no longer reads as if M2 waits on HTTPS. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- issues/an-alarm-reaches-no-sigalrm-handler.md | 42 +++++++---- ...ild-llvm-clang-and-lld-for-a-toyos-host.md | 42 +++++++++++ ...lacks-names-llvm-for-a-toyos-host-calls.md | 10 +-- ...bc-refuses-what-toyos-cannot-yet-answer.md | 3 +- issues/no-guest-case-reads-std-filesystem.md | 10 +-- ...-all-follows-a-link-swapped-in-mid-walk.md | 26 +++++++ ...d-runs-host-tools-outside-rust-and-qemu.md | 10 +-- issues/toyos-builds-itself.md | 10 +-- src/hostedclang.rs | 52 +++----------- src/libcxx.rs | 3 +- src/llvm.rs | 21 +++--- src/release.rs | 56 +++++++++------ tests/libc-arch/src/alarm_requests.rs | 13 +++- tests/testcases/tinycc/206_libc_refusals.c | 14 +++- .../testcases/tinycc/206_libc_refusals.expect | 4 ++ tests/testcases/tinycc/207_libc_names.c | 72 ++++++++++++++++++- tests/testcases/tinycc/207_libc_names.expect | 11 +++ userland/libc/src/alarm.rs | 35 +++++++-- userland/libc/src/alarmreq.rs | 17 ++++- userland/libc/src/misc.rs | 38 +++++++--- 20 files changed, 352 insertions(+), 137 deletions(-) create mode 100644 issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md create mode 100644 issues/remove-all-follows-a-link-swapped-in-mid-walk.md diff --git a/issues/an-alarm-reaches-no-sigalrm-handler.md b/issues/an-alarm-reaches-no-sigalrm-handler.md index 76ec1f667f7..51d9c183715 100644 --- a/issues/an-alarm-reaches-no-sigalrm-handler.md +++ b/issues/an-alarm-reaches-no-sigalrm-handler.md @@ -8,19 +8,33 @@ opened: 2026-09-30 libc's `alarm` (`userland/libc/src/alarm.rs`) arms one alarm per process and answers the seconds the one it replaces had left, and when it is due a thread -of libc's ends the process with exit code 142, `SIGALRM`'s default action. A -handler for `SIGALRM` never runs: `sigaction` keeps none, and answers 0 -whatever it is given. LLVM bounds its wait on a child with one: a `SIGALRM` -handler makes `wait4` answer `EINTR` (`llvm/lib/Support/Unix/Program.inc`, -`Wait`). POSIX gives `alarm` no refusal, and its `SIGALRM` reaches a handler -only through the signals libc imitates from stage 3 of -`issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md` -on; the handler waits on that stage. +of libc's drops it if `SIGALRM` is ignored and otherwise ends the process with +exit code 142, `SIGALRM`'s default action. POSIX keeps the process in two of +those cases: -The seconds left are held to their rule on the host -(`tests/libc-arch/src/alarm_requests.rs`); no guest case runs an alarm. +- **A handler.** `signal` and `sigaction` keep `SIGALRM`'s disposition, and a + handler never runs. LLVM bounds its wait on a child with one: a `SIGALRM` + handler makes `wait4` answer `EINTR` (`llvm/lib/Support/Unix/Program.inc`, + `Wait`). +- **A mask.** `SIGALRM` blocked in every thread stays pending until a thread + unblocks it, and libc ends the process at once. libc keeps each thread's + mask in that thread alone (`userland/libc/src/pthread.rs`, `MASK`), and a + thread Rust's std starts is none of libc's, so nothing can read whether + every thread blocks it. -**Exit**: `alarm` arms and disarms `SIGALRM` as POSIX says, which a guest C -case shows: a handler installed without `SA_RESTART` runs, and a `wait4` on a -child that has not ended answers `EINTR`; and after `alarm(5)`, `alarm(0)` -answers from 1 to 5 and a second `alarm(0)` answers 0. +POSIX gives `alarm` no refusal, and its `SIGALRM` reaches a handler, or waits +pending on a mask, only through the signals libc imitates from stage 3 of +`issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md` on; +both wait on that stage, its owner. + +What a due alarm does under each disposition is held on the host +(`tests/libc-arch/src/alarm_requests.rs`); `207_libc_names.c` reads the +disposition back and arms and disarms an alarm. No guest case lets one come +due. + +**Exit**: a guest C case shows `SIGALRM` delivered as POSIX says: a handler +installed without `SA_RESTART` runs, and a `wait4` on a child that has not +ended answers `EINTR`; with `SIGALRM` blocked in every thread, a due alarm +leaves the process running and `sigpending` names it, until an unblock ends +the process with 142; and with it ignored, a due alarm leaves the process +running. diff --git a/issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md b/issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md new file mode 100644 index 00000000000..19e5ad5ea68 --- /dev/null +++ b/issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md @@ -0,0 +1,42 @@ +--- +status: open +kind: defect +opened: 2026-09-30 +--- + +# Bootstrap cannot build LLVM, clang and lld for a ToyOS host + +The track holds one build of one fork (`issues/toyos-builds-itself.md`), and +the tree builds LLVM for a ToyOS host by two paths. M2's clang and `ld.lld` +come from a CMake configure and n2 build of their own (`src/hostedclang.rs`). +M3's rustc carries an LLVM that bootstrap's `Llvm` step builds for +`x86_64-unknown-toyos` (`issues/rustc-llvm-cannot-build-for-a-toyos-host.md`), +and bootstrap's `Llvm` step, with `clang = true`, and its `Lld` step make that +host's clang and lld too. One LLVM, one host, two code paths. + +Bootstrap's build names no `llvm-config` for the build triple, and so builds +that triple's LLVM, and its `clang-tblgen`, itself. CMake takes its toolchain +file from `CMAKE_TOOLCHAIN_FILE_x86_64_unknown_toyos`: one that includes the C +sysroot's `toolchain.cmake` and adds the ToyOS LLVM's install to +`CMAKE_FIND_ROOT_PATH`, because the `Lld` step names that LLVM to +`find_package` by a hint, and the sysroot's file has CMake find a package +under a root alone. libc and libc++ now give what stopped a CMake build of +clang and lld from the same commit, measured in #809: `alarm`, `wait`, `wait4`, +`lround` and `std::filesystem`. Bootstrap's build, which builds all of LLVM, +has not run since; what is known to stop it besides: + +- ORC's `shm_open` and `shm_unlink`, the interpreter's `scanf` and + `llvm-objdump`'s `ctime` + (`issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md`). +- LLVM's shared libraries, `libLTO`, `libRemarks`, `libclang` and + `libclang-cpp`, do not link against the C sysroot + (`issues/a-shared-object-does-not-link-against-the-c-sysroot-with-z-defs.md`). +- Bootstrap configures the build machine as LLVM's host + (`issues/a-toyos-hosted-llvm-is-configured-as-running-on-the-build-machine.md`). + +Owner: `issues/toyos-builds-itself.md`, M3. + +**Exit**: ToyOS's build, with nothing supplied by hand, has bootstrap install +a clang and an `ld.lld` for `x86_64-unknown-toyos`, `cargo run -- +--hosted-clang` places those, and `src/hostedclang.rs`'s CMake configure and +build are gone. diff --git a/issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md b/issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md index 2c3c955a071..a628c47fa84 100644 --- a/issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md +++ b/issues/libc-lacks-names-llvm-for-a-toyos-host-calls.md @@ -9,7 +9,8 @@ opened: 2026-10-01 LLVM's tools built for `x86_64-unknown-toyos` call names libc neither declares nor defines. None is in the clang and `ld.lld` the build makes for ToyOS (`src/hostedclang.rs`), which compile and link without them; each -stops a build of LLVM's other tools: +stops a build of LLVM's other tools, bootstrap's among them +(`issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md`): - `shm_open` and `shm_unlink`, with which ORC maps memory on every Unix but Android (`llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp`, @@ -18,10 +19,5 @@ stops a build of LLVM's other tools: (`llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp`). - `ctime`, with which `llvm-objdump` prints a file's time stamp. -`lround`, which clang calls through libc++'s `std::lround`, is defined -(`userland/libc/src/math.rs`) and held to the host's on the host -(`tests/libc-arch/src/fparts_differential.rs`); no guest case reads it. - **Exit**: libc declares and defines each, doing what POSIX says or refusing in -POSIX's form, asserted by a guest C case that reads its effect back, `lround` -among them. +POSIX's form, asserted by a guest C case that reads its effect back. diff --git a/issues/libc-refuses-what-toyos-cannot-yet-answer.md b/issues/libc-refuses-what-toyos-cannot-yet-answer.md index a2d65210d5c..4a02e7fd988 100644 --- a/issues/libc-refuses-what-toyos-cannot-yet-answer.md +++ b/issues/libc-refuses-what-toyos-cannot-yet-answer.md @@ -14,8 +14,7 @@ These answer failure in their POSIX form and do nothing - `symlink`, `ENOSYS`: `SYS_SYMLINK` displaces what its name held (`issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md`). - `chmod` and `fchmod`, `ENOSYS`: a file has no mode bits to set. -- `utimes`, `ENOSYS`: no call sets a file's times. The one refusal here - `206_libc_refusals.c` does not yet assert. +- `utimes`, `ENOSYS`: no call sets a file's times. - `statvfs` and `fstatvfs`, `ENOSYS`: no call answers a filesystem's size or free space. - `getrlimit` and `setrlimit`, `ENOSYS`: no call answers a process's limits. diff --git a/issues/no-guest-case-reads-std-filesystem.md b/issues/no-guest-case-reads-std-filesystem.md index 486729c372c..ef1f371e49c 100644 --- a/issues/no-guest-case-reads-std-filesystem.md +++ b/issues/no-guest-case-reads-std-filesystem.md @@ -9,10 +9,12 @@ opened: 2026-09-30 libc++ for ToyOS is built with `std::filesystem`, and `` with it, over what libc gives `src/filesystem`: `setbuf`, `fseeko`, `ftello`, `truncate`, `pathconf`'s `_PC_PATH_MAX`, and `utimes`, which refuses -`ENOSYS`, so `last_write_time` sets no time. `open` refuses every directory, -so no descriptor names one: `remove_all` walks a directory iterator -(`src/libcxx.rs`), which a link swapped in under it mid-walk can redirect, as -on libc++'s Windows. No C++ program in a guest uses any of it. +`ENOSYS`, so `last_write_time` sets no time, and `remove_all` walks a +directory by its path (`issues/remove-all-follows-a-link-swapped-in-mid-walk.md`). +Each of libc's is read back by `tests/testcases/tinycc/207_libc_names.c` +and `206_libc_refusals.c`; no C++ program in a guest uses any of it. + +Owner: `issues/toyos-builds-itself.md`, M2. **Exit**: a guest test lists a directory through `std::filesystem`, and reads a file back through `std::ifstream`. diff --git a/issues/remove-all-follows-a-link-swapped-in-mid-walk.md b/issues/remove-all-follows-a-link-swapped-in-mid-walk.md new file mode 100644 index 00000000000..11bcbb68338 --- /dev/null +++ b/issues/remove-all-follows-a-link-swapped-in-mid-walk.md @@ -0,0 +1,26 @@ +--- +status: open +kind: defect +opened: 2026-10-09 +--- + +# `remove_all` follows a link swapped in mid-walk + +Every C++ program on ToyOS gets libc++'s `std::filesystem::remove_all` built +with `-DREMOVE_ALL_USE_DIRECTORY_ITERATOR` (`src/libcxx.rs`): it walks a +directory by its path, and a directory swapped for a symbolic link between the +walk's check and its descent sends the deletes wherever the link points, +outside the tree it was asked to remove. libc++ says so of that walk +(`libcxx/src/filesystem/operations.cpp`, "vulnerable to some race conditions", +https://reviews.llvm.org/D118134), the class of CVE-2022-21658. Its other walk +holds a descriptor for each directory and resolves every name against it +(`openat`, `fdopendir`, `unlinkat`, `O_DIRECTORY`, `O_NOFOLLOW`, +`AT_REMOVEDIR`), and ToyOS has no such descriptor: `open` refuses every +directory, and the kernel resolves every path from the root. + +Owner: `issues/toyos-builds-itself.md`, M2, whose clang brought +`std::filesystem` in. + +**Exit**: a directory opens as a handle that the kernel resolves names +against, libc defines `openat`, `fdopendir` and `unlinkat` over it, and +`src/libcxx.rs` passes no `REMOVE_ALL_USE_DIRECTORY_ITERATOR`. diff --git a/issues/the-build-runs-host-tools-outside-rust-and-qemu.md b/issues/the-build-runs-host-tools-outside-rust-and-qemu.md index d45c5a44edf..d889517c60d 100644 --- a/issues/the-build-runs-host-tools-outside-rust-and-qemu.md +++ b/issues/the-build-runs-host-tools-outside-rust-and-qemu.md @@ -14,12 +14,12 @@ arrives and is not one. M4 and M5 are stages of `issues/toyos-builds-itself.md`. | tool | runs | verdict | exit | |---|---|---|---| -| Python | LLVM's CMake, whenever this host builds an LLVM (`src/llvm.rs`); the C++ runtime's CMake, in every sysroot build (`src/libcxx.rs`), and its build, which runs `libcxx/utils/generate_iwyu_mapping.py` for a header it installs | admitted: no Rust tool does the job, LLVM's CMake requires one (`find_package(Python3 … REQUIRED)` in `rust/src/llvm-project/llvm/CMakeLists.txt`), and so does the runtimes', unconditionally (`runtimes/CMakeLists.txt`): configured as `src/libcxx.rs` does with no Python on `PATH` or in CMake's search, it found none and stopped, exit 1, and with only `python3` added it configured, exit 0 | M5 runs it in the guest | -| CMake | rustc's bootstrap, for LLVM and clang; `src/llvm.rs`, for the LLVM's key; `src/libcxx.rs`, for the C++ runtime in every sysroot build | admitted: no Rust tool does the job, LLVM, clang and LLD are described in CMake, and upstream's only other descriptions are a GN overlay it does not support and a Bazel one | M5 runs it in the guest | -| `sh` running LLVM's `config.guess`, and the POSIX tools and `cc` it runs | LLVM's CMake, whenever this host builds an LLVM, and the C++ runtime's, in every sysroot build, ask it the host's triple, unconditionally (`get_host_triple` in `rust/src/llvm-project/llvm/cmake/modules/GetHostTriple.cmake`, which runs `sh` by name) | refused: a Rust tool does the shell's part, brush 0.4.0: on the development host (macOS, arm64) `config.guess` printed `/bin/sh`'s triple under it, `arm64-apple-darwin27.0.0`, exit 0 each. The script runs `sed`, `uname`, `mktemp`, `grep`, `rm`, `rmdir` and `cc` there under either shell, and that `cc` is the `cc` rows'. Five of the other six are refused, uutils' doing each: under brush with sed 0.2.0, grep 0.2.0 and coreutils 0.12.0's `mktemp`, `rm` and `rmdir`, and nothing else on `PATH` but the host's `uname` and `cc`, it printed that triple, exit 0. `uname` is admitted: coreutils 0.12.0's answers `-p` with `unknown` where macOS's answers `arm`, and `config.guess` reads that as PowerPC, `powerpc-apple-darwin27.0.0`, exit 0 | CMake finds brush as its `sh`, uutils' `sed`, `grep`, `mktemp`, `rm` and `rmdir`, and a Rust `uname` that answers `-p` as the host's does; or M5 runs it in the guest | +| Python | LLVM's CMake, whenever this host builds an LLVM (`src/llvm.rs`) and when it builds the ToyOS-hosted clang and LLD (`src/hostedclang.rs`); the C++ runtime's CMake, in every sysroot build (`src/libcxx.rs`), and its build, which runs `libcxx/utils/generate_iwyu_mapping.py` for a header it installs | admitted: no Rust tool does the job, LLVM's CMake requires one (`find_package(Python3 … REQUIRED)` in `rust/src/llvm-project/llvm/CMakeLists.txt`), and so does the runtimes', unconditionally (`runtimes/CMakeLists.txt`): configured as `src/libcxx.rs` does with no Python on `PATH` or in CMake's search, it found none and stopped, exit 1, and with only `python3` added it configured, exit 0 | M5 runs it in the guest | +| CMake | rustc's bootstrap, for LLVM and clang; `src/llvm.rs`, for the LLVM's key; `src/libcxx.rs`, for the C++ runtime in every sysroot build; `src/hostedclang.rs`, for the ToyOS-hosted clang and LLD and the tablegens of its nested `NATIVE` build | admitted: no Rust tool does the job, LLVM, clang and LLD are described in CMake, and upstream's only other descriptions are a GN overlay it does not support and a Bazel one | M5 runs it in the guest | +| `sh` running LLVM's `config.guess`, and the POSIX tools and `cc` it runs | LLVM's CMake, whenever this host builds an LLVM, the nested `NATIVE` configure of the ToyOS-hosted clang's build (`src/hostedclang.rs`), which is named no `LLVM_HOST_TRIPLE`, and the C++ runtime's, in every sysroot build, ask it the host's triple, unconditionally (`get_host_triple` in `rust/src/llvm-project/llvm/cmake/modules/GetHostTriple.cmake`, which runs `sh` by name) | refused: a Rust tool does the shell's part, brush 0.4.0: on the development host (macOS, arm64) `config.guess` printed `/bin/sh`'s triple under it, `arm64-apple-darwin27.0.0`, exit 0 each. The script runs `sed`, `uname`, `mktemp`, `grep`, `rm`, `rmdir` and `cc` there under either shell, and that `cc` is the `cc` rows'. Five of the other six are refused, uutils' doing each: under brush with sed 0.2.0, grep 0.2.0 and coreutils 0.12.0's `mktemp`, `rm` and `rmdir`, and nothing else on `PATH` but the host's `uname` and `cc`, it printed that triple, exit 0. `uname` is admitted: coreutils 0.12.0's answers `-p` with `unknown` where macOS's answers `arm`, and `config.guess` reads that as PowerPC, `powerpc-apple-darwin27.0.0`, exit 0 | CMake finds brush as its `sh`, uutils' `sed`, `grep`, `mktemp`, `rm` and `rmdir`, and a Rust `uname` that answers `-p` as the host's does; or M5 runs it in the guest | | `git` for worktrees, submodules, checkouts, fixtures and rustc's bootstrap | adds worktrees (`src/sysroot.rs`); updates submodules (`src/lib.rs`, `src/sysroot.rs`); fetches the fork from the primary's and checks it out (`src/sysroot.rs`); makes the tests' fixture repositories; runs inside rustc's bootstrap | admitted: no Rust tool does the job, gitoxide 0.85 adds, removes and prunes no worktree, updates no submodule, stages, resets and pushes nothing, checks out only a fresh clone and fetches a local path by spawning `git`; a fixture must be what `git` makes, and bootstrap runs `git` itself | M4 runs it in the guest | -| `git` for reads, a config write, a commit's paths written out, and clones and fetches over HTTPS | `rev-parse`, `merge-base`, `ls-tree`, `ls-files`, `cat-file`, `config --get-regexp`, `config --blob`, `status`, `diff`, `ls-remote` and `grep`, in the build system and its tests; `config --global --add safe.directory` in CI's containers; `checkout -- ` through an index of its own, which writes the C++ runtime's sources out of the LLVM commit into the stored LLVM (`src/llvm.rs`); every workflow's checkout | refused: a Rust tool does it, gitoxide 0.85, which reads refs, objects, the index, config, worktrees and status, adds a value to a config file and writes it (gix-config 0.58's `File::section_mut_or_create_new`, `SectionMut::push`, `File::write_to`), walks history, diffs, and lists, fetches and clones a remote over HTTPS; `grep` is a search of the files its index names; and gitoxide's CLI 0.59 (gix 0.88) wrote the runtimes' sources of LLVM `849da7d6` into an empty directory, each path's tree through `gix rev parse`, `gix index from-tree` and `gix free index checkout-exclusive`, exit 0 each: the 18759 files `git` writes there, byte for byte and mode for mode | those are gitoxide's | -| `cc`, `c++` and `ar` on a Linux host, `build-essential` in CI's containers | rustc links every host binary through `cc`; `cc` and `c++` compile LLVM, clang, LLD and `rustc_llvm` (`src/llvm.rs` names both to bootstrap); `cc` compiles ring's C and assembly in every build of the build system itself, whose HTTP agent is rustls on ring (`src/release.rs`); `ar` archives what `cc::Build` compiles, ring's included | admitted: no Rust tool compiles C or C++, or takes rustc's host link; ring is the one TLS provider the tree takes (owner, 2026-10-02), over a provider in Rust alone | M5: no host in the loop | +| `git` for reads, a config write, a commit's paths written out, and clones and fetches over HTTPS | `rev-parse`, `merge-base`, `ls-tree`, `ls-files`, `cat-file`, `config --get-regexp`, `config --blob`, `status`, `diff`, `ls-remote` and `grep`, in the build system and its tests; `config --global --add safe.directory` in CI's containers; `checkout -- ` through an index of its own, which writes the C++ runtime's sources out of the LLVM commit into the stored LLVM, and the ToyOS-hosted clang's out of it into its build (`src/llvm.rs`, `src/hostedclang.rs`); every workflow's checkout | refused: a Rust tool does it, gitoxide 0.85, which reads refs, objects, the index, config, worktrees and status, adds a value to a config file and writes it (gix-config 0.58's `File::section_mut_or_create_new`, `SectionMut::push`, `File::write_to`), walks history, diffs, and lists, fetches and clones a remote over HTTPS; `grep` is a search of the files its index names; and gitoxide's CLI 0.59 (gix 0.88) wrote the runtimes' sources of LLVM `849da7d6` into an empty directory, each path's tree through `gix rev parse`, `gix index from-tree` and `gix free index checkout-exclusive`, exit 0 each: the 18759 files `git` writes there, byte for byte and mode for mode | those are gitoxide's | +| `cc`, `c++` and `ar` on a Linux host, `build-essential` in CI's containers | rustc links every host binary through `cc`; `cc` and `c++` compile LLVM, clang, LLD and `rustc_llvm` (`src/llvm.rs` names both to bootstrap), and the tablegens of the ToyOS-hosted clang's `NATIVE` build (`src/hostedclang.rs`); `cc` compiles ring's C and assembly in every build of the build system itself, whose HTTP agent is rustls on ring (`src/release.rs`); `ar` archives what `cc::Build` compiles, ring's included | admitted: no Rust tool compiles C or C++, or takes rustc's host link; ring is the one TLS provider the tree takes (owner, 2026-10-02), over a provider in Rust alone | M5: no host in the loop | | the toolchain's own `clang`, `llvm-ar`, `rust-lld` and `llvm-config`, built from `ToyOSOrg/llvm-project` | rustc links every guest binary with `rust-lld`; `clang` compiles the C corpus (`tests/common/compile.rs`) and, with `llvm-ar`, doomgeneric through `cc::Build` (`src/clang.rs`); rustc's bootstrap asks `llvm-config` how to link LLVM | admitted: our fork's C++, which ToyOS can one day build and run; no Rust tool compiles C, `cc::Build` archives with an `ar`, bootstrap reads LLVM through `llvm-config`, and `CLAUDE.md` links everything with `rust-lld` | M5: no host in the loop | | `ovmf-generic`, `qemu-efi-aarch64` | the x86-64 and AArch64 UEFI firmware of CI's guest containers (`src/firmware.rs`), packaged by Debian apart from QEMU | admitted: QEMU's own firmware, and no Rust firmware does its job | the instrument's QEMU carries its own firmware | | `ca-certificates` | the trust store `git` and `curl` verify against in CI's containers | admitted: data both of them need | goes when neither runs there | diff --git a/issues/toyos-builds-itself.md b/issues/toyos-builds-itself.md index 2aeabff1000..c04a8ea8e0a 100644 --- a/issues/toyos-builds-itself.md +++ b/issues/toyos-builds-itself.md @@ -105,12 +105,12 @@ libc's state it is `userland/libc` at `15625e0cb`. **Blocked on other tracks.** M2 needs room for about a gigabyte of toolchain, and threads and `mmap` mature enough for LLVM -(`issues/std-and-libc-drop-the-answer-thread-join-gives.md`); its package over +(`issues/std-and-libc-drop-the-answer-thread-join-gives.md`). Packages over HTTPS (`issues/a-package-is-a-directory-under-apps-and-the-installer-is-a-program.md`) -waits on the network stack under it +wait on the network stack under them (`issues/the-lan-is-not-yet-production-grade.md`, -`issues/the-internet-clients-work-unchanged.md`). +`issues/the-internet-clients-work-unchanged.md`), and M2 does not. M2 and M4 also need libc to start a child process (`issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md`). M4 needs git in the guest, storage durable and fast enough for an LLVM build tree @@ -134,7 +134,9 @@ static PIEs that name no library (`src/hostedclang.rs`): 147.5 MB and `issues/libc-stat-answers-one-serial-number-for-every-file.md`. **What M3 adds: a rustc that carries that LLVM**, after all of M2's. -- Build: `issues/rustc-llvm-cannot-build-for-a-toyos-host.md`. +- Build: `issues/rustc-llvm-cannot-build-for-a-toyos-host.md`, and + `issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md`, + whose build of clang and lld replaces M2's. - Link: `issues/a-rust-std-binary-cannot-link-the-cxx-runtime.md`. - Test: `issues/nothing-builds-the-toyos-hosted-rustc.md`. diff --git a/src/hostedclang.rs b/src/hostedclang.rs index 27c72a220f2..ddd86b5a340 100644 --- a/src/hostedclang.rs +++ b/src/hostedclang.rs @@ -1,6 +1,9 @@ -//! clang and `ld.lld` that run on ToyOS: built for `x86_64-unknown-toyos` from +//! clang and `ld.lld` for a ToyOS host: built for `x86_64-unknown-toyos` from //! the LLVM commit the host's LLVM is (`src/llvm.rs`), by the toolchain's own //! clang against its C sysroot (`clang::CSysroot`), and linked statically. +//! CMake builds them, beside bootstrap, until bootstrap's LLVM for a ToyOS +//! host makes both +//! (`issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md`). //! //! **Made only when asked for** (`cargo run -- --hosted-clang`): its build is //! LLVM's, and its key moves with every sysroot's, so no other build makes it. @@ -16,8 +19,7 @@ //! //! `hosted-clang//` in the store holds `bin/clang`, `bin/ld.lld` and //! clang's resource headers in `lib/clang//include`, where clang looks -//! beside itself; once its [`SOURCE`] file exists it is read-only. Each binary -//! names no library it needs ([`static_elf`]). +//! beside itself; once its [`SOURCE`] file exists it is read-only. use std::fs; use std::path::{Path, PathBuf}; @@ -168,7 +170,6 @@ fn place(fork: &Path, sysroot: &Path, ninja: &Path, key: &Key, dir: &Path) { // `fs::copy` follows the link the build installs clang as. let (from, to) = (built.join("bin").join(name), bin.join(kept)); fs::copy(&from, &to).unwrap_or_else(|e| panic!("copy {} -> {}: {e}", from.display(), to.display())); - static_elf(&to); } let resource = crate::clang::resource_version(&built); let version = resource.file_name().unwrap_or_else(|| panic!("{} names no version", resource.display())); @@ -185,30 +186,12 @@ fn place(fork: &Path, sysroot: &Path, ninja: &Path, key: &Key, dir: &Path) { } /// Write [`SOURCES`] as the commit `fork`'s LLVM gitlink names holds them into -/// `dest`, from the LLVM repository of `fork`'s git directory, through an index -/// of their own: no checkout's files reach them. +/// `dest`, from the LLVM repository of `fork`'s git directory: a linked +/// worktree's `rust/src/llvm-project` is no checkout. fn export(fork: &Path, dest: &Path) { - let commit = gitlink(fork, LLVM); let common = git_out(fork, &["rev-parse", "--path-format=absolute", "--git-common-dir"]); let repository = Path::new(common.trim()).join("modules").join(LLVM); - fs::create_dir_all(dest).unwrap_or_else(|e| panic!("create {}: {e}", dest.display())); - let index = toyos_tmpdir::TempDir::new("hosted-clang-index"); - let out = Command::new("git") - .env("GIT_DIR", &repository) - .env("GIT_INDEX_FILE", index.join("index")) - .args(["-c", "core.sparseCheckout=false", "--work-tree"]) - .arg(dest) - .args(["checkout", &commit, "--"]) - .args(SOURCES) - .output() - .unwrap_or_else(|e| panic!("run git in {}: {e}", repository.display())); - assert!( - out.status.success(), - "git checkout {commit} from {} into {}: {}", - repository.display(), - dest.display(), - String::from_utf8_lossy(&out.stderr).trim(), - ); + crate::llvm::check_out_committed(&repository, &gitlink(fork, LLVM), &SOURCES, dest); } /// Configure LLVM at `sources`, the commit `fork` names, for `c`'s target, @@ -285,25 +268,6 @@ fn run(mut command: Command, what: &str, ninja: &Path) { assert!(status.success(), "the ToyOS-hosted clang's {what} failed ({status}): its output above says why"); } -/// Refuse `elf` unless it is an executable for [`TARGET`] that names no -/// library it needs. -fn static_elf(elf: &Path) { - let bytes = fs::read(elf).unwrap_or_else(|e| panic!("read {}: {e}", elf.display())); - let machine = match TARGET { - Arch::X86_64 => toyos_elf::Machine::X86_64, - Arch::Aarch64 => toyos_elf::Machine::Aarch64, - }; - let layout = toyos_elf::Layout::parse(&bytes, machine) - .unwrap_or_else(|e| panic!("{} is no {} executable: {e:?}", elf.display(), TARGET.userland())); - let needed = layout.dynamic().map_or(0, |dynamic| { - let start = dynamic.file_offset() as usize; - let table = bytes.get(start..start + dynamic.image().len() as usize); - let table = table.unwrap_or_else(|| panic!("{}'s PT_DYNAMIC is past its end", elf.display())); - toyos_elf::Dynamic::needed(table).count() - }); - assert_eq!(needed, 0, "{} names {needed} libraries it needs, and it is linked statically", elf.display()); -} - #[cfg(test)] mod tests { use toyos_tmpdir::TempDir; diff --git a/src/libcxx.rs b/src/libcxx.rs index c3ff6107ed6..35f57f0ecbc 100644 --- a/src/libcxx.rs +++ b/src/libcxx.rs @@ -42,7 +42,8 @@ pub(crate) const OPTIONS: [(&str, &str); 18] = [ ("LIBUNWIND_ADDITIONAL_COMPILE_FLAGS", "-D_LIBUNWIND_USE_DLADDR=0"), // `open` refuses every directory, so no descriptor names one for `openat` // and `unlinkat` to resolve against: `remove_all` walks a directory - // iterator, as libc++'s Windows does. + // iterator, as libc++'s Windows does + // (`issues/remove-all-follows-a-link-swapped-in-mid-walk.md`). ("LIBCXX_ADDITIONAL_COMPILE_FLAGS", "-DREMOVE_ALL_USE_DIRECTORY_ITERATOR"), ("LIBCXX_INCLUDE_BENCHMARKS", "OFF"), ("LIBCXX_INCLUDE_TESTS", "OFF"), diff --git a/src/llvm.rs b/src/llvm.rs index 0b41e8c9980..11fe103cf95 100644 --- a/src/llvm.rs +++ b/src/llvm.rs @@ -322,7 +322,8 @@ fn place(fork: &Path, key: &Key, dir: &Path, build: &impl Fn(&Path) -> PathBuf) built_from.trim(), fork.display(), ); - check_out_committed(&checkout, &commit, &crate::libcxx::SOURCES, &partial.join("src")); + let repository = git_out(&checkout, &["rev-parse", "--absolute-git-dir"]); + check_out_committed(Path::new(repository.trim()), &commit, &crate::libcxx::SOURCES, &partial.join("src")); fs::write(partial.join(SOURCE), format!("{key}\n")) .unwrap_or_else(|e| panic!("write {}: {e}", partial.join(SOURCE).display())); read_only(&partial); @@ -370,27 +371,27 @@ fn libraries(install: &Path) -> Vec { named } -/// Write `paths` as `commit` holds them, from the repository at `checkout`, +/// Write `paths` as `commit` holds them, from the git directory `repository`, /// under `dest`: through an index of their own and with no sparse pattern, so -/// nothing the checkout holds beside the commit, tracked, ignored or left out, -/// reaches them. -fn check_out_committed(checkout: &Path, commit: &str, paths: &[&str], dest: &Path) { +/// nothing a checkout of it holds beside the commit, tracked, ignored or left +/// out, reaches them. +pub(crate) fn check_out_committed(repository: &Path, commit: &str, paths: &[&str], dest: &Path) { fs::create_dir_all(dest).unwrap_or_else(|e| panic!("create {}: {e}", dest.display())); - let index = toyos_tmpdir::TempDir::new("llvm-runtimes-index"); + let index = toyos_tmpdir::TempDir::new("llvm-sources-index"); let out = Command::new("git") + .env("GIT_DIR", repository) .env("GIT_INDEX_FILE", index.join("index")) .args(["-c", "core.sparseCheckout=false", "--work-tree"]) .arg(dest) .args(["checkout", commit, "--"]) .args(paths) - .current_dir(checkout) .output() - .unwrap_or_else(|e| panic!("run git in {}: {e}", checkout.display())); + .unwrap_or_else(|e| panic!("run git in {}: {e}", repository.display())); assert!( out.status.success(), - "git checkout {commit} -- {paths:?} into {} in {}: {}", + "git checkout {commit} -- {paths:?} into {} from {}: {}", dest.display(), - checkout.display(), + repository.display(), String::from_utf8_lossy(&out.stderr).trim(), ); } diff --git a/src/release.rs b/src/release.rs index 87d5826f6b8..410172e70b3 100644 --- a/src/release.rs +++ b/src/release.rs @@ -47,21 +47,39 @@ const USER_AGENT: &str = "toyos-build (https://github.com/ToyOSOrg/ToyOS)"; /// The products a toolchain is, in the order a build makes them, each under the /// name its cache entry and its job's outputs carry. -const LAYERS: [(Keyed, &str); 4] = [ - (Keyed::Llvm, "llvm"), - (Keyed::Compiler, "compiler"), - (Keyed::Freestanding, "freestanding"), - (Keyed::Sysroot, "sysroot"), +const LAYERS: [(Part, &str); 4] = [ + (Part::Llvm, "llvm"), + (Part::Compiler, "compiler"), + (Part::Freestanding, "freestanding"), + (Part::Sysroot, "sysroot"), ]; -/// Why no layer is the ToyOS-hosted clang: it is made only when asked for -/// (`src/hostedclang.rs`), and a release carries what every build reads. -const NOT_A_LAYER: &str = "a toolchain release carries no ToyOS-hosted clang: it is made only when asked for"; +/// What a toolchain is made of: the products every build reads, of the kinds +/// the build system keys. The ToyOS-hosted clang is none, made only when asked +/// for (`src/hostedclang.rs`). +#[derive(Clone, Copy)] +enum Part { + Llvm, + Compiler, + Freestanding, + Sysroot, +} + +impl Part { + fn keyed(self) -> Keyed { + match self { + Part::Llvm => Keyed::Llvm, + Part::Compiler => Keyed::Compiler, + Part::Freestanding => Keyed::Freestanding, + Part::Sysroot => Keyed::Sysroot, + } + } +} /// One of [`LAYERS`] of this tree's toolchain: the key the build system files /// it under, and where it is, relative to the checkout. struct Layer { - kind: Keyed, + kind: Part, name: &'static str, key: Key, path: PathBuf, @@ -135,13 +153,12 @@ fn layers(root: &Path) -> Vec { .iter() .map(|&(kind, name)| { let key = match kind { - Keyed::Llvm => &llvm, - Keyed::Compiler => &compiler, - Keyed::Freestanding => &freestanding, - Keyed::Sysroot => &sysroot, - Keyed::HostedClang => unreachable!("{NOT_A_LAYER}"), + Part::Llvm => &llvm, + Part::Compiler => &compiler, + Part::Freestanding => &freestanding, + Part::Sysroot => &sysroot, }; - let dir = kind.store(&store(root)).join(key); + let dir = kind.keyed().store(&store(root)).join(key); let path = dir.strip_prefix(root).expect("a runner's store is in its checkout").to_path_buf(); Layer { kind, name, key: key.clone(), path } }) @@ -153,13 +170,12 @@ fn layers(root: &Path) -> Vec { fn defect(root: &Path, layer: &Layer) -> Option { let dir = root.join(&layer.path); match layer.kind { - Keyed::Llvm => crate::llvm::defect(&dir), - Keyed::Compiler => { + Part::Llvm => crate::llvm::defect(&dir), + Part::Compiler => { crate::compiler::unplaced(&dir).or_else(|| crate::toolchain::toolchain_defect(&dir.join("stage2"))) } - Keyed::Freestanding => crate::sysroot::unpublished(&dir), - Keyed::Sysroot => crate::sysroot::unfinished(&dir), - Keyed::HostedClang => unreachable!("{NOT_A_LAYER}"), + Part::Freestanding => crate::sysroot::unpublished(&dir), + Part::Sysroot => crate::sysroot::unfinished(&dir), } } diff --git a/tests/libc-arch/src/alarm_requests.rs b/tests/libc-arch/src/alarm_requests.rs index 5971595d361..035e3801607 100644 --- a/tests/libc-arch/src/alarm_requests.rs +++ b/tests/libc-arch/src/alarm_requests.rs @@ -1,8 +1,8 @@ //! What `alarm` answers and arms: the seconds an earlier alarm has left, -//! rounded up and 0 once it is due or when there is none; and 0 seconds arm -//! nothing. +//! rounded up and 0 once it is due or when there is none; 0 seconds arm +//! nothing; and a due alarm ends the process unless `SIGALRM` is ignored. -use crate::alarmreq::{due, left}; +use crate::alarmreq::{due, ends, left, SIG_DFL, SIG_IGN}; const SEC: u64 = 1_000_000_000; @@ -25,3 +25,10 @@ fn the_seconds_left_round_up_and_end_at_zero() { assert_eq!(left(None, 100), 0); assert_eq!(left(due(0, u32::MAX), 0), u32::MAX); } + +#[test] +fn a_due_alarm_ends_the_process_unless_sigalrm_is_ignored() { + assert!(!ends(SIG_IGN), "an ignored SIGALRM ended the process"); + assert!(ends(SIG_DFL)); + assert!(ends(0x40_1000), "a handler, which never runs, kept the process"); +} diff --git a/tests/testcases/tinycc/206_libc_refusals.c b/tests/testcases/tinycc/206_libc_refusals.c index 8bfc4d2ed5d..8e59332f71b 100644 --- a/tests/testcases/tinycc/206_libc_refusals.c +++ b/tests/testcases/tinycc/206_libc_refusals.c @@ -1,7 +1,7 @@ /* What libc refuses: each call answers failure in its own POSIX form, errno - says why, and nothing is done: ENOSYS where ToyOS lacks the function, and - the errno POSIX names for a lock or a mapping it cannot take, or memory it - cannot give. */ + says why, and nothing is done: ENOSYS where ToyOS lacks the function, ECHILD + for a wait on the child libc cannot start, and the errno POSIX names for a + lock, a limit or a mapping it cannot take, or memory it cannot give. */ #include #include #include @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #define FILE_PATH "/tmp/206_libc_refusals" @@ -19,6 +21,7 @@ static const char *errno_name(int e) { switch (e) { case 0: return "no errno"; + case ECHILD: return "ECHILD"; case ENOSYS: return "ENOSYS"; case EINVAL: return "EINVAL"; case ENODEV: return "ENODEV"; @@ -81,6 +84,11 @@ int main(void) { said("msync", msync(page, 4096, MS_SYNC)); said("mprotect", mprotect(page, 4096, PROT_READ)); said("realpath", realpath("/tmp", NULL) == NULL ? -1 : 0); + said("utimes", utimes(FILE_PATH, NULL)); + int status = 0; + said("wait", wait(&status)); + said("wait4", wait4(-1, &status, 0, NULL)); + said("pathconf 12345", pathconf(FILE_PATH, 12345)); said("fcntl F_SETLK", fcntl(fd, F_SETLK, &lock)); said("fcntl F_SETLKW", fcntl(fd, F_SETLKW, &lock)); said("fcntl F_GETLK", fcntl(fd, F_GETLK, &lock)); diff --git a/tests/testcases/tinycc/206_libc_refusals.expect b/tests/testcases/tinycc/206_libc_refusals.expect index daf32c2d78e..dc7daa9f70b 100644 --- a/tests/testcases/tinycc/206_libc_refusals.expect +++ b/tests/testcases/tinycc/206_libc_refusals.expect @@ -15,6 +15,10 @@ setrlimit: -1, ENOSYS msync: -1, ENOSYS mprotect: -1, ENOSYS realpath: -1, ENOSYS +utimes: -1, ENOSYS +wait: -1, ECHILD +wait4: -1, ECHILD +pathconf 12345: -1, EINVAL fcntl F_SETLK: -1, EINVAL fcntl F_SETLKW: -1, EINVAL fcntl F_GETLK: -1, EINVAL diff --git a/tests/testcases/tinycc/207_libc_names.c b/tests/testcases/tinycc/207_libc_names.c index 201f70b2716..450798d19da 100644 --- a/tests/testcases/tinycc/207_libc_names.c +++ b/tests/testcases/tinycc/207_libc_names.c @@ -1,7 +1,9 @@ /* What libc does for the names LLVM builds against: a directory listed with the name and kind of each entry; dladdr naming the image and the exported - symbol an address lies in; and strnlen, strsignal, modf, logb and the - conversions. */ + symbol an address lies in; a file truncated by its path; a stream on a + buffer of its caller's, or none, sought and told by off_t; SIGALRM's + disposition, and an alarm armed and disarmed; and strnlen, strsignal, modf, + lround, logb, pathconf and the conversions. */ #include #include #include @@ -46,6 +48,7 @@ int main(void); static const char *errno_name(int e) { switch (e) { + case EDOM: return "EDOM"; case EEXIST: return "EEXIST"; case EINVAL: return "EINVAL"; case ENOENT: return "ENOENT"; @@ -85,6 +88,61 @@ static void links(void) { refused("readlink of nothing", readlink(DIR_PATH "/none", buf, sizeof buf)); } +static void truncated(void) { + struct stat st; + int answer = truncate(TARGET, 2); + printf("truncate to 2: %d; stat size: %ld\n", answer, stat(TARGET, &st) == 0 ? (long)st.st_size : -1L); + refused("truncate of nothing", truncate(DIR_PATH "/none", 0)); +} + +/* setbuf's buffer holds what is written until the stream is told or sought; + with none, a write reaches the file at once. */ +static void buffered(void) { + static char buf[BUFSIZ]; + struct stat st; + FILE *f = fopen(DIR_PATH "/buffered", "w+"); + if (!f) { + printf("fopen buffered failed\n"); + return; + } + setbuf(f, buf); + fputs("0123456789", f); + long held = stat(DIR_PATH "/buffered", &st) == 0 ? (long)st.st_size : -1L; + printf("setbuf: file %ld bytes, the buffer %s\n", held, memcmp(buf, "0123456789", 10) == 0 ? "holds them" : "not"); + long told = (long)ftello(f); + int sought = fseeko(f, 3, SEEK_SET); + long at = (long)ftello(f); + int c = fgetc(f); + printf("ftello: %ld; fseeko 3: %d; ftello: %ld; fgetc: %c\n", told, sought, at, c); + fclose(f); + + f = fopen(DIR_PATH "/unbuffered", "w"); + if (!f) { + printf("fopen unbuffered failed\n"); + return; + } + setbuf(f, NULL); + fputs("abc", f); + printf("setbuf NULL: file %ld bytes\n", stat(DIR_PATH "/unbuffered", &st) == 0 ? (long)st.st_size : -1L); + fclose(f); +} + +static void alarms(void) { + void (*was)(int) = signal(SIGALRM, SIG_IGN); + void (*then)(int) = signal(SIGALRM, SIG_DFL); + printf("signal SIGALRM: was %s, then %s\n", was == SIG_DFL ? "SIG_DFL" : "another", + then == SIG_IGN ? "SIG_IGN" : "another"); + struct sigaction ignore = { .sa_handler = SIG_IGN }, old = { .sa_handler = SIG_DFL }; + int set = sigaction(SIGALRM, &ignore, NULL); + int got = sigaction(SIGALRM, NULL, &old); + printf("sigaction SIGALRM: %d %d, %s\n", set, got, old.sa_handler == SIG_IGN ? "SIG_IGN" : "another"); + signal(SIGALRM, SIG_DFL); + unsigned first = alarm(5); + unsigned left = alarm(0); + unsigned again = alarm(0); + printf("alarm 5: %u; alarm 0: %s; again: %u\n", first, left >= 1 && left <= 5 ? "1 to 5" : "another", again); +} + static void listing(void) { if (mkdir(LISTED, 0755) != 0 || mkdir(LISTED "/sub", 0755) != 0 || close(creat(LISTED "/a", 0644)) != 0 || close(creat(LISTED "/b", 0644)) != 0 || close(creat(LISTED "/sub/inner", 0644)) != 0) { @@ -154,6 +212,9 @@ int main(void) { return 1; } links(); + truncated(); + buffered(); + alarms(); listing(); dl(); @@ -166,6 +227,13 @@ int main(void) { errno = 0; double pole = logb(0.0); printf("logb(0): %g, %s\n", pole, errno == ERANGE ? "ERANGE" : errno_name(errno)); + /* Volatile, so the compiler folds none of them. */ + volatile double halves[] = { 2.5, -2.5, 0.49999999999999994, 1e19 }; + printf("lround: %ld %ld %ld\n", lround(halves[0]), lround(halves[1]), lround(halves[2])); + errno = 0; + long far = lround(halves[3]); + printf("lround(1e19): %ld, %s\n", far, errno == EDOM ? "EDOM" : errno_name(errno)); + printf("pathconf _PC_PATH_MAX: %ld\n", pathconf(DIR_PATH, _PC_PATH_MAX)); printf("sysconf _SC_PAGE_SIZE: %ld\n", sysconf(_SC_PAGE_SIZE)); return 0; } diff --git a/tests/testcases/tinycc/207_libc_names.expect b/tests/testcases/tinycc/207_libc_names.expect index a5091a68a7f..861af38ad2e 100644 --- a/tests/testcases/tinycc/207_libc_names.expect +++ b/tests/testcases/tinycc/207_libc_names.expect @@ -4,6 +4,14 @@ readlink into 4: 4, "/sys#" readlink into 0: -1, EINVAL readlink of a file: -1, EINVAL readlink of nothing: -1, ENOENT +truncate to 2: 0; stat size: 2 +truncate of nothing: -1, ENOENT +setbuf: file 0 bytes, the buffer holds them +ftello: 10; fseeko 3: 0; ftello: 3; fgetc: 3 +setbuf NULL: file 3 bytes +signal SIGALRM: was SIG_DFL, then SIG_IGN +sigaction SIGALRM: 0 0, SIG_IGN +alarm 5: 0; alarm 0: 1 to 5; again: 0 entry: a DT_UNKNOWN entry: b DT_UNKNOWN entry: sub DT_DIR @@ -16,4 +24,7 @@ strsignal: Segmentation fault; Unknown signal modf(-3.25): -3 and -0.25 logb: -4 10 logb(0): -inf, ERANGE +lround: 3 -3 0 +lround(1e19): -9223372036854775808, EDOM +pathconf _PC_PATH_MAX: 4096 sysconf _SC_PAGE_SIZE: 4096 diff --git a/userland/libc/src/alarm.rs b/userland/libc/src/alarm.rs index bdc1de4f097..cfd0d6d4b08 100644 --- a/userland/libc/src/alarm.rs +++ b/userland/libc/src/alarm.rs @@ -1,11 +1,12 @@ //! `alarm`: one per process, kept by a thread of this library's, started by -//! the first alarm armed, that sleeps until it is due and then does what -//! `SIGALRM`'s default action does, ending the process. A handler `sigaction` -//! is given never runs: this library keeps none +//! the first alarm armed, that sleeps until it is due. Then, with `SIGALRM` +//! ignored, the alarm is gone; otherwise the keeper does what `SIGALRM`'s +//! default action does and ends the process. A handler `signal` or `sigaction` +//! is given never runs, and a mask that blocks `SIGALRM` holds nothing back //! (`issues/an-alarm-reaches-no-sigalrm-handler.md`). use core::ptr; -use core::sync::atomic::{AtomicU32, Ordering}; +use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; use toyos_abi::syscall; @@ -28,6 +29,19 @@ static ALARM: Lock = Lock::new(Alarm { due: None, kept: false }); /// Moved at every `alarm`, so a keeper asleep on an older one wakes. static TURN: AtomicU32 = AtomicU32::new(0); +/// `SIGALRM`'s disposition: `SIG_DFL`, `SIG_IGN` or a handler's address. +static DISPOSITION: AtomicUsize = AtomicUsize::new(alarmreq::SIG_DFL); + +/// Make `handler` `SIGALRM`'s disposition, and answer the one it replaces. +pub(crate) fn dispose(handler: usize) -> usize { + DISPOSITION.swap(handler, Ordering::AcqRel) +} + +/// `SIGALRM`'s disposition. +pub(crate) fn disposition() -> usize { + DISPOSITION.load(Ordering::Acquire) +} + fn now() -> u64 { toyos_abi::clock::nanos_since_boot() } @@ -52,19 +66,26 @@ pub extern "C" fn alarm(seconds: u32) -> u32 { left } -/// Sleep until the alarm is due or moved, and end the process once it is due. +/// Sleep until the alarm is due or moved, and once it is due, drop it or end +/// the process as [`alarmreq::ends`] says. unsafe extern "C" fn keep(_: *mut u8) -> *mut u8 { loop { // Read before the alarm, so an `alarm` between the two moves it and // the wait below returns at once. let turn = TURN.load(Ordering::Acquire); - let wait = match ALARM.lock().due { + let mut alarm = ALARM.lock(); + let wait = match alarm.due { None => None, Some(due) => match due.checked_sub(now()) { Some(wait) if wait > 0 => Some(wait), - _ => syscall::exit(ALARM_END), + _ if alarmreq::ends(disposition()) => syscall::exit(ALARM_END), + _ => { + alarm.due = None; + None + } }, }; + drop(alarm); // SAFETY: `TURN` is a live, aligned u32. unsafe { syscall::futex_wait(TURN.as_ptr(), turn, wait) }; } diff --git a/userland/libc/src/alarmreq.rs b/userland/libc/src/alarmreq.rs index e5653b00be0..d32a4230222 100644 --- a/userland/libc/src/alarmreq.rs +++ b/userland/libc/src/alarmreq.rs @@ -1,9 +1,20 @@ -//! What `alarm` answers and arms, in nanoseconds of the monotonic clock. It -//! reads nothing but what it is handed, so the host tests it -//! (`toyos-libc-copies`). +//! What `alarm` answers and arms, in nanoseconds of the monotonic clock, and +//! what a due one does. It reads nothing but what it is handed, so the host +//! tests it (`toyos-libc-copies`). const NANOS_PER_SEC: u64 = 1_000_000_000; +/// `signal.h`'s `SIG_DFL` and `SIG_IGN`, as addresses. +pub(crate) const SIG_DFL: usize = 0; +pub(crate) const SIG_IGN: usize = 1; + +/// Whether an alarm due under `SIGALRM`'s disposition `handler` ends the +/// process: it does unless `SIGALRM` is ignored, since no handler runs +/// (`issues/an-alarm-reaches-no-sigalrm-handler.md`). +pub(crate) fn ends(handler: usize) -> bool { + handler != SIG_IGN +} + /// When an alarm of `seconds` armed at `now` is due; none for 0, which /// disarms. pub(crate) fn due(now: u64, seconds: u32) -> Option { diff --git a/userland/libc/src/misc.rs b/userland/libc/src/misc.rs index e38e12ec1f2..571337c6b8c 100644 --- a/userland/libc/src/misc.rs +++ b/userland/libc/src/misc.rs @@ -207,20 +207,42 @@ pub unsafe extern "C" fn abort() -> ! { syscall::exit(134) // SIGABRT } -// Signal (stubs — ToyOS has no signals) +// Signal (stubs — ToyOS has no signals, but `SIGALRM`'s disposition decides +// what a due `alarm` does) -type SigHandlerT = unsafe extern "C" fn(i32); +const SIGALRM: i32 = 14; +/// `signal.h`'s `struct sigaction`. +#[repr(C)] +pub struct SigAction { + handler: usize, + flags: u64, + restorer: usize, + mask: u64, +} + +/// A handler by its address: `SIG_DFL` and `SIG_IGN` are no functions. #[no_mangle] -pub unsafe extern "C" fn signal(_signum: i32, handler: SigHandlerT) -> SigHandlerT { - handler // return the handler as "previous", effectively a no-op +pub unsafe extern "C" fn signal(signum: i32, handler: *const u8) -> *const u8 { + match signum { + SIGALRM => crate::alarm::dispose(handler as usize) as *const u8, + _ => handler, // return the handler as "previous", effectively a no-op + } } #[no_mangle] -pub unsafe extern "C" fn sigaction( - _signum: i32, _act: *const u8, _oldact: *mut u8, -) -> i32 { - 0 // success +pub unsafe extern "C" fn sigaction(signum: i32, act: *const SigAction, oldact: *mut SigAction) -> i32 { + if signum != SIGALRM { + return 0; // success + } + let old = match unsafe { act.as_ref() } { + Some(act) => crate::alarm::dispose(act.handler), + None => crate::alarm::disposition(), + }; + if let Some(oldact) = unsafe { oldact.as_mut() } { + *oldact = SigAction { handler: old, flags: 0, restorer: 0, mask: 0 }; + } + 0 } /// The calling thread's mask, which is what POSIX's process mask is in a From a5d7972345aa00abffeab0f0d9bdf085510b1f6b Mon Sep 17 00:00:00 2001 From: japabu Date: Fri, 9 Oct 2026 20:08:07 +0200 Subject: [PATCH 3/6] SIGALRM is declared once, sigaction answers the previous action whole, and the hosted clang's export folds into place Round 2's review of #809 asked three things. SIGALRM's number was written twice in libc, as 128 + 14 in alarm.rs and as a const in misc.rs. It is now alarmreq's SIGALRM, beside SIG_DFL and SIG_IGN, read by both; the host test compares it with signal.h's, which also gives the const a reader in toyos-libc-copies. sigaction's oldact for SIGALRM answered sa_flags and sa_mask 0 whatever the last act set. POSIX answers the previous action whole, so libc keeps the whole struct sigaction under a lock in place of the handler's atomic, and signal installs an action of the handler alone. 207_libc_names sets SA_RESTART and SIGUSR1's bit and reads both back; the C corpus runs only on metal, so the line is read by the T14's boot:testcases. hostedclang's export had one caller and three lines; place does it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- issues/an-alarm-reaches-no-sigalrm-handler.md | 2 +- src/hostedclang.rs | 17 ++++------- tests/libc-arch/src/alarm_requests.rs | 11 +++++-- tests/testcases/tinycc/207_libc_names.c | 9 ++++-- tests/testcases/tinycc/207_libc_names.expect | 2 +- userland/libc/src/alarm.rs | 30 ++++++++++--------- userland/libc/src/alarmreq.rs | 3 +- userland/libc/src/misc.rs | 22 ++++++++------ 8 files changed, 54 insertions(+), 42 deletions(-) diff --git a/issues/an-alarm-reaches-no-sigalrm-handler.md b/issues/an-alarm-reaches-no-sigalrm-handler.md index 51d9c183715..015deb5b6d7 100644 --- a/issues/an-alarm-reaches-no-sigalrm-handler.md +++ b/issues/an-alarm-reaches-no-sigalrm-handler.md @@ -29,7 +29,7 @@ both wait on that stage, its owner. What a due alarm does under each disposition is held on the host (`tests/libc-arch/src/alarm_requests.rs`); `207_libc_names.c` reads the -disposition back and arms and disarms an alarm. No guest case lets one come +action back whole and arms and disarms an alarm. No guest case lets one come due. **Exit**: a guest C case shows `SIGALRM` delivered as POSIX says: a handler diff --git a/src/hostedclang.rs b/src/hostedclang.rs index ddd86b5a340..8830107c92d 100644 --- a/src/hostedclang.rs +++ b/src/hostedclang.rs @@ -13,7 +13,7 @@ //! host's tools, n2 and CMake among them; the sysroot's, which names the C //! library, the C++ runtime and the clang that builds against them; and //! [`RECIPE`] with [`OPTIONS`]. Its sources are the commit's, written from the -//! fork's LLVM repository ([`export`]), never a checkout's files. CMake builds +//! fork's LLVM repository, never a checkout's files. CMake builds //! LLVM's tablegens for the build machine first, in a nested build of its own //! (`NATIVE`), with the C and C++ compilers the LLVM key names. //! @@ -157,8 +157,12 @@ fn place(fork: &Path, sysroot: &Path, ninja: &Path, key: &Key, dir: &Path) { eprintln!("Building the ToyOS-hosted clang and LLD {key}: nobody on this host has"); let scratch = dir.with_extension("build"); keystore::remove(&scratch); + // From the LLVM repository of `fork`'s git directory: a linked worktree's + // `rust/src/llvm-project` is no checkout. let sources = scratch.join("src"); - export(fork, &sources); + let common = git_out(fork, &["rev-parse", "--path-format=absolute", "--git-common-dir"]); + let repository = Path::new(common.trim()).join("modules").join(LLVM); + crate::llvm::check_out_committed(&repository, &gitlink(fork, LLVM), &SOURCES, &sources); let built = scratch.join("build"); build(fork, &sources, &built, &CSysroot::of(sysroot, TARGET), ninja, &scratch); @@ -185,15 +189,6 @@ fn place(fork: &Path, sysroot: &Path, ninja: &Path, key: &Key, dir: &Path) { keystore::remove(&scratch); } -/// Write [`SOURCES`] as the commit `fork`'s LLVM gitlink names holds them into -/// `dest`, from the LLVM repository of `fork`'s git directory: a linked -/// worktree's `rust/src/llvm-project` is no checkout. -fn export(fork: &Path, dest: &Path) { - let common = git_out(fork, &["rev-parse", "--path-format=absolute", "--git-common-dir"]); - let repository = Path::new(common.trim()).join("modules").join(LLVM); - crate::llvm::check_out_committed(&repository, &gitlink(fork, LLVM), &SOURCES, dest); -} - /// Configure LLVM at `sources`, the commit `fork` names, for `c`'s target, /// in `built`, and build clang and LLD there under `ninja`; `scratch` holds /// the links CMake runs the compilers by. diff --git a/tests/libc-arch/src/alarm_requests.rs b/tests/libc-arch/src/alarm_requests.rs index 035e3801607..e82db69a011 100644 --- a/tests/libc-arch/src/alarm_requests.rs +++ b/tests/libc-arch/src/alarm_requests.rs @@ -1,8 +1,10 @@ //! What `alarm` answers and arms: the seconds an earlier alarm has left, //! rounded up and 0 once it is due or when there is none; 0 seconds arm -//! nothing; and a due alarm ends the process unless `SIGALRM` is ignored. +//! nothing; and a due alarm ends the process unless `SIGALRM` is ignored, the +//! signal `signal.h` numbers. -use crate::alarmreq::{due, ends, left, SIG_DFL, SIG_IGN}; +use crate::alarmreq::{due, ends, left, SIGALRM, SIG_DFL, SIG_IGN}; +use crate::header::{self, SIGNAL_H}; const SEC: u64 = 1_000_000_000; @@ -32,3 +34,8 @@ fn a_due_alarm_ends_the_process_unless_sigalrm_is_ignored() { assert!(ends(SIG_DFL)); assert!(ends(0x40_1000), "a handler, which never runs, kept the process"); } + +#[test] +fn sigalrm_is_the_number_signal_h_gives_it() { + assert_eq!(SIGALRM, header::int(SIGNAL_H, "SIGALRM")); +} diff --git a/tests/testcases/tinycc/207_libc_names.c b/tests/testcases/tinycc/207_libc_names.c index 450798d19da..70512871cfb 100644 --- a/tests/testcases/tinycc/207_libc_names.c +++ b/tests/testcases/tinycc/207_libc_names.c @@ -2,7 +2,7 @@ the name and kind of each entry; dladdr naming the image and the exported symbol an address lies in; a file truncated by its path; a stream on a buffer of its caller's, or none, sought and told by off_t; SIGALRM's - disposition, and an alarm armed and disarmed; and strnlen, strsignal, modf, + action, read back whole, and an alarm armed and disarmed; and strnlen, strsignal, modf, lround, logb, pathconf and the conversions. */ #include #include @@ -132,10 +132,13 @@ static void alarms(void) { void (*then)(int) = signal(SIGALRM, SIG_DFL); printf("signal SIGALRM: was %s, then %s\n", was == SIG_DFL ? "SIG_DFL" : "another", then == SIG_IGN ? "SIG_IGN" : "another"); - struct sigaction ignore = { .sa_handler = SIG_IGN }, old = { .sa_handler = SIG_DFL }; + struct sigaction ignore = { .sa_handler = SIG_IGN, .sa_flags = SA_RESTART }, old = { .sa_handler = SIG_DFL }; + sigemptyset(&ignore.sa_mask); + sigaddset(&ignore.sa_mask, SIGUSR1); int set = sigaction(SIGALRM, &ignore, NULL); int got = sigaction(SIGALRM, NULL, &old); - printf("sigaction SIGALRM: %d %d, %s\n", set, got, old.sa_handler == SIG_IGN ? "SIG_IGN" : "another"); + printf("sigaction SIGALRM: %d %d, %s, flags 0x%lx, mask 0x%lx\n", set, got, + old.sa_handler == SIG_IGN ? "SIG_IGN" : "another", old.sa_flags, (unsigned long)old.sa_mask); signal(SIGALRM, SIG_DFL); unsigned first = alarm(5); unsigned left = alarm(0); diff --git a/tests/testcases/tinycc/207_libc_names.expect b/tests/testcases/tinycc/207_libc_names.expect index 861af38ad2e..7723ab5c435 100644 --- a/tests/testcases/tinycc/207_libc_names.expect +++ b/tests/testcases/tinycc/207_libc_names.expect @@ -10,7 +10,7 @@ setbuf: file 0 bytes, the buffer holds them ftello: 10; fseeko 3: 0; ftello: 3; fgetc: 3 setbuf NULL: file 3 bytes signal SIGALRM: was SIG_DFL, then SIG_IGN -sigaction SIGALRM: 0 0, SIG_IGN +sigaction SIGALRM: 0 0, SIG_IGN, flags 0x10000000, mask 0x200 alarm 5: 0; alarm 0: 1 to 5; again: 0 entry: a DT_UNKNOWN entry: b DT_UNKNOWN diff --git a/userland/libc/src/alarm.rs b/userland/libc/src/alarm.rs index cfd0d6d4b08..391af7d15b5 100644 --- a/userland/libc/src/alarm.rs +++ b/userland/libc/src/alarm.rs @@ -6,16 +6,17 @@ //! (`issues/an-alarm-reaches-no-sigalrm-handler.md`). use core::ptr; -use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicU32, Ordering}; use toyos_abi::syscall; use crate::alarmreq; +use crate::misc::SigAction; use crate::pthread::Lock; /// What the process ends with when its alarm is due: the code of an end by -/// `SIGALRM` (14), as `abort`'s is `SIGABRT`'s. -const ALARM_END: i32 = 128 + 14; +/// `SIGALRM`, as `abort`'s is `SIGABRT`'s. +const ALARM_END: i32 = 128 + alarmreq::SIGALRM; struct Alarm { /// When the alarm is due, in the monotonic clock's nanoseconds. @@ -29,17 +30,18 @@ static ALARM: Lock = Lock::new(Alarm { due: None, kept: false }); /// Moved at every `alarm`, so a keeper asleep on an older one wakes. static TURN: AtomicU32 = AtomicU32::new(0); -/// `SIGALRM`'s disposition: `SIG_DFL`, `SIG_IGN` or a handler's address. -static DISPOSITION: AtomicUsize = AtomicUsize::new(alarmreq::SIG_DFL); +/// `SIGALRM`'s action, whose handler is its disposition: `SIG_DFL`, `SIG_IGN` +/// or a handler's address. +static ACTION: Lock = Lock::new(SigAction::of(alarmreq::SIG_DFL)); -/// Make `handler` `SIGALRM`'s disposition, and answer the one it replaces. -pub(crate) fn dispose(handler: usize) -> usize { - DISPOSITION.swap(handler, Ordering::AcqRel) -} - -/// `SIGALRM`'s disposition. -pub(crate) fn disposition() -> usize { - DISPOSITION.load(Ordering::Acquire) +/// Make `new`, if any, `SIGALRM`'s action, and answer the one it replaces whole. +pub(crate) fn act(new: Option) -> SigAction { + let mut action = ACTION.lock(); + let old = *action; + if let Some(new) = new { + *action = new; + } + old } fn now() -> u64 { @@ -78,7 +80,7 @@ unsafe extern "C" fn keep(_: *mut u8) -> *mut u8 { None => None, Some(due) => match due.checked_sub(now()) { Some(wait) if wait > 0 => Some(wait), - _ if alarmreq::ends(disposition()) => syscall::exit(ALARM_END), + _ if alarmreq::ends(act(None).handler) => syscall::exit(ALARM_END), _ => { alarm.due = None; None diff --git a/userland/libc/src/alarmreq.rs b/userland/libc/src/alarmreq.rs index d32a4230222..e8c7f2c1d7f 100644 --- a/userland/libc/src/alarmreq.rs +++ b/userland/libc/src/alarmreq.rs @@ -4,7 +4,8 @@ const NANOS_PER_SEC: u64 = 1_000_000_000; -/// `signal.h`'s `SIG_DFL` and `SIG_IGN`, as addresses. +/// `signal.h`'s `SIGALRM`, and its `SIG_DFL` and `SIG_IGN` as addresses. +pub(crate) const SIGALRM: i32 = 14; pub(crate) const SIG_DFL: usize = 0; pub(crate) const SIG_IGN: usize = 1; diff --git a/userland/libc/src/misc.rs b/userland/libc/src/misc.rs index 571337c6b8c..e707e1478a4 100644 --- a/userland/libc/src/misc.rs +++ b/userland/libc/src/misc.rs @@ -6,6 +6,7 @@ use core::sync::atomic::{AtomicU32, Ordering}; use toyos_abi::syscall; +use crate::alarmreq::SIGALRM; use crate::errno::{ECHILD, ENOSYS}; use crate::strtonum; @@ -210,22 +211,28 @@ pub unsafe extern "C" fn abort() -> ! { // Signal (stubs — ToyOS has no signals, but `SIGALRM`'s disposition decides // what a due `alarm` does) -const SIGALRM: i32 = 14; - /// `signal.h`'s `struct sigaction`. #[repr(C)] +#[derive(Clone, Copy)] pub struct SigAction { - handler: usize, + pub(crate) handler: usize, flags: u64, restorer: usize, mask: u64, } +impl SigAction { + /// The action of `handler` alone, as `signal` makes one. + pub(crate) const fn of(handler: usize) -> Self { + Self { handler, flags: 0, restorer: 0, mask: 0 } + } +} + /// A handler by its address: `SIG_DFL` and `SIG_IGN` are no functions. #[no_mangle] pub unsafe extern "C" fn signal(signum: i32, handler: *const u8) -> *const u8 { match signum { - SIGALRM => crate::alarm::dispose(handler as usize) as *const u8, + SIGALRM => crate::alarm::act(Some(SigAction::of(handler as usize))).handler as *const u8, _ => handler, // return the handler as "previous", effectively a no-op } } @@ -235,12 +242,9 @@ pub unsafe extern "C" fn sigaction(signum: i32, act: *const SigAction, oldact: * if signum != SIGALRM { return 0; // success } - let old = match unsafe { act.as_ref() } { - Some(act) => crate::alarm::dispose(act.handler), - None => crate::alarm::disposition(), - }; + let old = crate::alarm::act(unsafe { act.as_ref() }.copied()); if let Some(oldact) = unsafe { oldact.as_mut() } { - *oldact = SigAction { handler: old, flags: 0, restorer: 0, mask: 0 }; + *oldact = old; } 0 } From 4ce7ce79a3f9aac8fe844d529333b469470035b4 Mon Sep 17 00:00:00 2001 From: japabu Date: Fri, 9 Oct 2026 20:41:15 +0200 Subject: [PATCH 4/6] The ToyOS-hosted clang builds from the sources the host's LLVM keeps in the store `--hosted-clang` read the LLVM commit from the primary checkout's `.git/modules/rust/modules/src/llvm-project`, a repository no build fetches into: after #790 moved the gitlink to ToyOSOrg/llvm-project b7420fe it held no such commit (its origin is rust-lang's), and the build died with `unable to read tree`. The normal build obtains the commit one way: bootstrap's `Llvm` step checks the gitlink's commit out in the fork checkout and fetches it there, and `llvm::place` writes what builds read of it into `llvm//src` from that checkout; the C++ runtime is built from there. The LLVM now keeps the hosted clang's pathspecs there too, and the hosted clang builds from that directory of the LLVM it holds in use: the commit is the one the host LLVM's key names on any host, whether the LLVM was built here or found in the store, and no checkout is read. The pathspecs an LLVM keeps are now part of its key, so a change to either list moves it; the recipe moves to 5, and every LLVM key with it. The hosted clang's key names the LLVM's key, which now names its sources, so it drops its own copy of them; its recipe moves to 3. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- src/hostedclang.rs | 85 +++++++++++++++++++++++----------------------- src/llvm.rs | 68 +++++++++++++++++++++++-------------- 2 files changed, 85 insertions(+), 68 deletions(-) diff --git a/src/hostedclang.rs b/src/hostedclang.rs index 8830107c92d..0b117aa1b6b 100644 --- a/src/hostedclang.rs +++ b/src/hostedclang.rs @@ -8,14 +8,15 @@ //! **Made only when asked for** (`cargo run -- --hosted-clang`): its build is //! LLVM's, and its key moves with every sysroot's, so no other build makes it. //! -//! **A function of its key** ([`key`]): the host LLVM's key, which names the -//! commit, the revision and repository it says it was built from, and the -//! host's tools, n2 and CMake among them; the sysroot's, which names the C -//! library, the C++ runtime and the clang that builds against them; and -//! [`RECIPE`] with [`OPTIONS`]. Its sources are the commit's, written from the -//! fork's LLVM repository, never a checkout's files. CMake builds -//! LLVM's tablegens for the build machine first, in a nested build of its own -//! (`NATIVE`), with the C and C++ compilers the LLVM key names. +//! **A function of its key** ([`key_of`]): the host LLVM's key, which names the +//! commit, what of it the LLVM keeps, the revision and repository it says it +//! was built from, and the host's tools, n2 and CMake among them; the +//! sysroot's, which names the C library, the C++ runtime and the clang that +//! builds against them; and [`RECIPE`] with [`OPTIONS`]. Its sources are +//! [`SOURCES`] of the commit as that LLVM keeps them in the store +//! (`src/llvm.rs`), never a checkout's files. CMake builds LLVM's tablegens for +//! the build machine first, in a nested build of its own (`NATIVE`), with the C +//! and C++ compilers the LLVM key names. //! //! `hosted-clang//` in the store holds `bin/clang`, `bin/ld.lld` and //! clang's resource headers in `lib/clang//include`, where clang looks @@ -28,25 +29,24 @@ use std::process::Command; use crate::arch::Arch; use crate::buildlock::{Guard, Keyed}; use crate::clang::CSysroot; -use crate::compiler::LLVM; use crate::keystore::{self, Key}; -use crate::sysroot::{clone_tree, git_out, gitlink}; +use crate::sysroot::clone_tree; /// What changes how the key's sources become this product and is none of the /// other parts. Moving it moves every key. -const RECIPE: &str = "CMake and n2 build of clang and lld from the LLVM commit's SOURCES, for the TARGET \ +const RECIPE: &str = "CMake and n2 build of clang and lld from the SOURCES the host's LLVM keeps, for the TARGET \ against its C sysroot, saying the version, revision and repository the host's clang \ says; of the build, clang as bin/clang, lld as bin/ld.lld and clang's resource \ - headers; 2"; + headers; 3"; /// The ToyOS the binaries run on. const TARGET: Arch = Arch::X86_64; -/// What of the commit the build reads, as git's pathspecs: LLVM, clang and -/// LLD, the CMake modules they share, the third-party sources LLVM compiles in, -/// and libunwind's headers, which LLD's Mach-O port reads. No test, which the -/// configuration builds none of. -const SOURCES: [&str; 12] = [ +/// What of the commit the build reads, as git's pathspecs the host's LLVM keeps: +/// LLVM, clang and LLD, the CMake modules they share, the third-party sources +/// LLVM compiles in, and libunwind's headers, which LLD's Mach-O port reads. No +/// test, which the configuration builds none of. +pub(crate) const SOURCES: [&str; 12] = [ "llvm", "clang", "lld", @@ -100,7 +100,11 @@ pub fn dispatch(root: &Path) { let mut lock = crate::buildlock::shared(root, "the ToyOS-hosted clang"); let sysroot = crate::toolchain::ensure(root, &mut lock); let fork = crate::sysroot::fork_checkout(root, &mut lock); - let hosted = resolve(&keystore::host(), &fork, sysroot.dir(), &crate::n2::ninja(root)); + let store = keystore::host(); + // Inside the worktree lock, which keeps every build in `fork` but an + // LLVM's, whose key's lock serialises them, out. + let llvm = crate::llvm::resolve(root, &store, &fork); + let hosted = resolve(&store, &fork, &llvm.dir, sysroot.dir(), &crate::n2::ninja(root)); for (_, kept) in BINARIES { let binary = hosted.dir.join("bin").join(kept); let size = fs::metadata(&binary).unwrap_or_else(|e| panic!("stat {}: {e}", binary.display())).len(); @@ -108,27 +112,26 @@ pub fn dispatch(root: &Path) { } } -/// The product the LLVM fork `fork` names and the sysroot `sysroot` holds, in -/// `store`: made if nobody on this host has made it, under `ninja`. -pub fn resolve(store: &Path, fork: &Path, sysroot: &Path, ninja: &Path) -> HostedClang { - let key = key(fork, sysroot); +/// The product of the LLVM at `llvm`, the one the fork `fork` names, and the +/// sysroot at `sysroot`, both held in use in `store`: made if nobody on this +/// host has made it, under `ninja`. +pub fn resolve(store: &Path, fork: &Path, llvm: &Path, sysroot: &Path, ninja: &Path) -> HostedClang { + let key = key_of(&stored(llvm), &stored(sysroot), &options()); let dir = Keyed::HostedClang.store(store).join(&key); - let make = || place(fork, sysroot, ninja, &key, &dir); + let make = || place(fork, llvm, sysroot, ninja, &key, &dir); let using = keystore::made(store, Keyed::HostedClang, &key, || defect(&dir), make); HostedClang { dir, _using: using } } -/// The key of what `fork`'s LLVM and the sysroot at `sysroot` make. -fn key(fork: &Path, sysroot: &Path) -> Key { - let named = sysroot.file_name().and_then(|n| n.to_str()).and_then(Key::parse); - let sysroot = - named.unwrap_or_else(|| panic!("{} is no sysroot of the store: its name is no key", sysroot.display())); - key_of(&crate::llvm::key(fork), &sysroot, &options()) +/// The key the store product at `dir` is filed under. +fn stored(dir: &Path) -> Key { + let named = dir.file_name().and_then(|n| n.to_str()).and_then(Key::parse); + named.unwrap_or_else(|| panic!("{} is no product of the store: its name is no key", dir.display())) } fn key_of(llvm: &Key, sysroot: &Key, options: &[(String, String)]) -> Key { let options: Vec = options.iter().map(|(name, value)| format!("{name}={value}")).collect(); - let parts = [RECIPE, TARGET.userland(), &SOURCES.join(" "), &options.join("\n"), llvm.as_str(), sysroot.as_str()]; + let parts = [RECIPE, TARGET.userland(), &options.join("\n"), llvm.as_str(), sysroot.as_str()]; Key::of(parts.join("\n\0\n").as_bytes()) } @@ -153,18 +156,12 @@ fn defect(dir: &Path) -> Option { } /// Build what `key` names and put it at `dir`. The caller holds the key's lock. -fn place(fork: &Path, sysroot: &Path, ninja: &Path, key: &Key, dir: &Path) { +fn place(fork: &Path, llvm: &Path, sysroot: &Path, ninja: &Path, key: &Key, dir: &Path) { eprintln!("Building the ToyOS-hosted clang and LLD {key}: nobody on this host has"); let scratch = dir.with_extension("build"); keystore::remove(&scratch); - // From the LLVM repository of `fork`'s git directory: a linked worktree's - // `rust/src/llvm-project` is no checkout. - let sources = scratch.join("src"); - let common = git_out(fork, &["rev-parse", "--path-format=absolute", "--git-common-dir"]); - let repository = Path::new(common.trim()).join("modules").join(LLVM); - crate::llvm::check_out_committed(&repository, &gitlink(fork, LLVM), &SOURCES, &sources); let built = scratch.join("build"); - build(fork, &sources, &built, &CSysroot::of(sysroot, TARGET), ninja, &scratch); + build(fork, &llvm.join("src"), &built, &CSysroot::of(sysroot, TARGET), ninja, &scratch); let partial = dir.with_extension("partial"); keystore::remove(&partial); @@ -178,9 +175,13 @@ fn place(fork: &Path, sysroot: &Path, ninja: &Path, key: &Key, dir: &Path) { let resource = crate::clang::resource_version(&built); let version = resource.file_name().unwrap_or_else(|| panic!("{} names no version", resource.display())); clone_tree(&resource.join("include"), &partial.join("lib/clang").join(version).join("include")); - // What was built is what the key names, or it is not that key's. - let again = self::key(fork, sysroot); - assert!(again == *key, "the sources moved while {key} was being built (they now name {again}); nothing was kept"); + // The fork the version stamp was read from names the LLVM built, or this + // is not the key's. + let again = crate::llvm::key(fork); + assert!( + again == stored(llvm), + "the fork moved while {key} was being built (it now names LLVM {again}); nothing was kept" + ); let source = partial.join(SOURCE); fs::write(&source, format!("{key}\n")).unwrap_or_else(|e| panic!("write {}: {e}", source.display())); crate::llvm::read_only(&partial); @@ -189,7 +190,7 @@ fn place(fork: &Path, sysroot: &Path, ninja: &Path, key: &Key, dir: &Path) { keystore::remove(&scratch); } -/// Configure LLVM at `sources`, the commit `fork` names, for `c`'s target, +/// Configure LLVM at `sources`, of the commit `fork` names, for `c`'s target, /// in `built`, and build clang and LLD there under `ninja`; `scratch` holds /// the links CMake runs the compilers by. fn build(fork: &Path, sources: &Path, built: &Path, c: &CSysroot, ninja: &Path, scratch: &Path) { diff --git a/src/llvm.rs b/src/llvm.rs index 11fe103cf95..68e1cdd9f9d 100644 --- a/src/llvm.rs +++ b/src/llvm.rs @@ -5,12 +5,13 @@ //! the fork checkout's gitlink names (`sysroot::gitlink`, which refuses a //! checkout holding what no commit does and a gitlink staged and not committed), //! the committed tree of its `src/bootstrap` (one holding what no commit does is -//! refused), the bootstrap configuration below, [`RECIPE`], and the tools the +//! refused), the bootstrap configuration below, [`recipe`], and the tools the //! host builds it with ([`host_tools`]). `llvm//` in the store //! (`src/keystore.rs`) is what builds read of bootstrap's install of that LLVM and its clang //! ([`keep`]), with its LLD in `bin/` beside `llvm-config` and in `src/` the -//! runtimes' sources the C++ runtime is built from (`src/libcxx.rs`) as its -//! commit holds them, made by whichever build first needs it ([`resolve`]), and +//! sources the C++ runtime (`src/libcxx.rs`) and the ToyOS-hosted clang +//! (`src/hostedclang.rs`) are built from as its commit holds them ([`sources`]), +//! made by whichever build first needs it ([`resolve`]), and //! stored only when it was built from what the key names. Once its [`SOURCE`] //! file exists it is read-only, its directories as well as its files. //! Every compiler build names it as the host's `llvm-config` with @@ -50,7 +51,7 @@ use crate::toolchain::{self, host_triple}; const RECIPE: &str = "bootstrap build of src/llvm-project/llvm and src/llvm-project/lld; of the install, \ llvm-config, clang and llvm-ar in bin, and llvm-objcopy on an Apple host, LLVM's headers, \ every library llvm-config names and clang's resource headers; lld in bin, and the \ - runtimes' sources in src, read-only; 4"; + sources the C++ runtime and the ToyOS-hosted clang are built from in src, read-only; 5"; /// What of the caller's environment the LLVM build, and every tool its key /// asks, sees: @@ -127,7 +128,7 @@ pub fn host_lines(dir: &Path) -> String { /// changes no commit does. pub fn key(fork: &Path) -> Key { let tools = host_tools(); - key_of(fork, RECIPE, &config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), tools, &stamp(fork)), &tools.identity) + key_of(fork, &recipe(), &config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), tools, &stamp(fork)), &tools.identity) } /// What an LLVM, its clang and its LLD say they were built from. @@ -167,6 +168,17 @@ pub(crate) fn host_compilers() -> (PathBuf, PathBuf) { (tools.cc.clone(), tools.cxx.clone()) } +/// [`RECIPE`], with the pathspecs of what it keeps of the commit. +fn recipe() -> String { + format!("{RECIPE}\n{}", sources().join(" ")) +} + +/// What of the commit an LLVM keeps in `src/`, as git's pathspecs: what the C++ +/// runtime and the ToyOS-hosted clang are built from. +fn sources() -> Vec<&'static str> { + crate::libcxx::SOURCES.into_iter().chain(crate::hostedclang::SOURCES).collect() +} + /// [`key`], with what it reads beside `fork`'s committed `src/bootstrap`: /// `config` names the LLVM's commit ([`stamp`]). fn key_of(fork: &Path, recipe: &str, config: &str, tools: &str) -> Key { @@ -269,7 +281,7 @@ pub(crate) fn defect(dir: &Path) -> Option { .iter() .chain(&["lib/clang"]) .map(|k| dir.join(k)) - .chain(crate::libcxx::SOURCES.iter().map(|s| dir.join("src").join(s))); + .chain(sources().into_iter().filter(|s| !s.starts_with(':')).map(|s| dir.join("src").join(s))); let tools = tools().map(|t| dir.join("bin").join(t)).filter(|p| !p.is_file()); let gone: Vec = kept.filter(|p| !p.is_dir()).chain(tools).map(|p| p.display().to_string()).collect(); (!gone.is_empty()).then(|| format!("{} carries no {}", dir.display(), gone.join(", "))) @@ -322,8 +334,7 @@ fn place(fork: &Path, key: &Key, dir: &Path, build: &impl Fn(&Path) -> PathBuf) built_from.trim(), fork.display(), ); - let repository = git_out(&checkout, &["rev-parse", "--absolute-git-dir"]); - check_out_committed(Path::new(repository.trim()), &commit, &crate::libcxx::SOURCES, &partial.join("src")); + check_out_committed(&checkout, &commit, &sources(), &partial.join("src")); fs::write(partial.join(SOURCE), format!("{key}\n")) .unwrap_or_else(|e| panic!("write {}: {e}", partial.join(SOURCE).display())); read_only(&partial); @@ -371,27 +382,27 @@ fn libraries(install: &Path) -> Vec { named } -/// Write `paths` as `commit` holds them, from the git directory `repository`, +/// Write `paths` as `commit` holds them, from the repository at `checkout`, /// under `dest`: through an index of their own and with no sparse pattern, so -/// nothing a checkout of it holds beside the commit, tracked, ignored or left -/// out, reaches them. -pub(crate) fn check_out_committed(repository: &Path, commit: &str, paths: &[&str], dest: &Path) { +/// nothing the checkout holds beside the commit, tracked, ignored or left out, +/// reaches them. +fn check_out_committed(checkout: &Path, commit: &str, paths: &[&str], dest: &Path) { fs::create_dir_all(dest).unwrap_or_else(|e| panic!("create {}: {e}", dest.display())); let index = toyos_tmpdir::TempDir::new("llvm-sources-index"); let out = Command::new("git") - .env("GIT_DIR", repository) .env("GIT_INDEX_FILE", index.join("index")) .args(["-c", "core.sparseCheckout=false", "--work-tree"]) .arg(dest) .args(["checkout", commit, "--"]) .args(paths) + .current_dir(checkout) .output() - .unwrap_or_else(|e| panic!("run git in {}: {e}", repository.display())); + .unwrap_or_else(|e| panic!("run git in {}: {e}", checkout.display())); assert!( out.status.success(), - "git checkout {commit} -- {paths:?} into {} from {}: {}", + "git checkout {commit} -- {paths:?} into {} in {}: {}", dest.display(), - repository.display(), + checkout.display(), String::from_utf8_lossy(&out.stderr).trim(), ); } @@ -573,10 +584,10 @@ mod tests { if !checkout.join(".git").exists() { git(&checkout, &["init", "-q"]); } - write(&checkout.join("llvm/CMakeLists.txt"), content); - for source in crate::libcxx::SOURCES { + for source in sources().into_iter().filter(|s| !s.starts_with(':')) { write(&checkout.join(source).join("CMakeLists.txt"), &format!("the {source} of {content}")); } + write(&checkout.join("llvm/test/lit.cfg.py"), "what no build reads"); git(&checkout, &["add", "-A"]); let tree = git(&checkout, &["write-tree"]); let out = Command::new("git") @@ -749,7 +760,7 @@ mod tests { let tools = host_tools(); let config = config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), tools, &stamp(&fork)); let base = key(&fork); - assert_eq!(key_of(&fork, RECIPE, &config, &tools.identity), base); + assert_eq!(key_of(&fork, &recipe(), &config, &tools.identity), base); let elsewhere = if host_triple() == "x86_64-unknown-linux-gnu" { "aarch64-unknown-linux-gnu" } else { @@ -758,9 +769,9 @@ mod tests { let elsewhere = config_text(Path::new(KEYED_BUILD_DIR), elsewhere, tools, &stamp(&fork)); for (what, other) in [ ("the recipe", key_of(&fork, "another recipe", &config, &tools.identity)), - ("the [llvm]", key_of(&fork, RECIPE, &config.replace("X86", "RISCV;X86"), &tools.identity)), - ("the host", key_of(&fork, RECIPE, &elsewhere, &tools.identity)), - ("the host's tools", key_of(&fork, RECIPE, &config, "/usr/bin/gcc\ngcc 14\n")), + ("the [llvm]", key_of(&fork, &recipe(), &config.replace("X86", "RISCV;X86"), &tools.identity)), + ("the host", key_of(&fork, &recipe(), &elsewhere, &tools.identity)), + ("the host's tools", key_of(&fork, &recipe(), &config, "/usr/bin/gcc\ngcc 14\n")), ] { assert_ne!(other, base, "{what} did not move the key"); } @@ -819,7 +830,7 @@ mod tests { "--show-sdk-version" => format!("{version}\n"), other => panic!("xcrun was asked {other}"), }); - key_of(&fork, RECIPE, &config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), &tools, &stamp(&fork)), &tools.identity) + key_of(&fork, &recipe(), &config_text(Path::new(KEYED_BUILD_DIR), &host_triple(), &tools, &stamp(&fork)), &tools.identity) }); assert_ne!(older, newer, "two SDK versions at one SDK path named one LLVM"); } @@ -1013,11 +1024,13 @@ mod tests { assert!(stored.iter().all(|n| n.to_string_lossy().ends_with(".partial")), "stored: {stored:?}"); } - /// **The runtimes' sources are the commit's**: made while the LLVM was + /// **The sources it keeps are the commit's**: made while the LLVM was /// built, an edit to a file the commit holds is refused and nothing is - /// stored, and a file the checkout ignores is not stored with it. + /// stored, a file the checkout ignores is not stored with it, and the + /// ToyOS-hosted clang's are kept beside the runtimes' without the tests + /// its pathspecs leave out. #[test] - fn the_runtimes_sources_are_the_commit_s() { + fn the_kept_sources_are_the_commit_s() { let scratch = Scratch::new("llvm-runtimes"); let (_primary, store, [_same, a, _b]) = estate_built(&scratch); let fork = a.join("rust"); @@ -1042,6 +1055,9 @@ mod tests { assert_eq!(fs::read_to_string(dir.join("src/libcxx/CMakeLists.txt")).unwrap(), "the libcxx of A"); assert!(checkout.join("libcxx/utils/cache.pyc").is_file()); assert!(!dir.join("src/libcxx/utils").exists(), "a file the checkout ignores was stored"); + assert_eq!(fs::read_to_string(dir.join("src/clang/CMakeLists.txt")).unwrap(), "the clang of A"); + assert!(checkout.join("llvm/test/lit.cfg.py").is_file()); + assert!(!dir.join("src/llvm/test").exists(), "a test the pathspecs leave out was stored"); } const WORKTREE: &str = "TOYOS_LLVM_TEST_WORKTREE"; From 85d3eb4529adb71b5ebafbd7019b5c9b371ec2cf Mon Sep 17 00:00:00 2001 From: japabu Date: Fri, 9 Oct 2026 21:37:32 +0200 Subject: [PATCH 5/6] An LLVM the hosted clang needs is made with the worktree lock held exclusively `--hosted-clang` resolved the host's LLVM holding its worktree's build lock only shared. When the store lacked it, bootstrap's `Llvm` step then ran in the fork checkout, updating `src/llvm-project` and writing `build/toyos-llvm`, beside every other shared holder of the same worktree, which `buildlock.rs` names as the exclusive mode's job; the compiler and sysroot builders reach `llvm::resolve` only inside exclusive phases. `llvm::resolve_held` asks under the shared lock whether the store holds the LLVM the fork names, and makes one only through `Held::act_if`, with the lock exclusive; one the store holds is found with the lock shared, at no cost of serialisation. `an_llvm_is_made_only_with_the_worktree_lock_exclusive` holds both halves: a shared acquirer is kept out while the LLVM is made, and a second worktree held shared by another process finds it without waiting. The store design stays, on two measurements. A linked worktree whose LLVM came from the store holds no LLVM commit in its fork's `src/llvm-project`: of the 16 on this host with a fork checkout, none did (`git cat-file -e ^{tree}` exits 128 in each, its directory empty), and only the one whose bootstrap built the LLVM held it. A fresh depth-1 fetch of the commit from the fork's remote took 23 s and 278 MiB per worktree. Keeping the hosted clang's sources in the stored LLVM costs 61.6 MB compressed per `llvm` entry, packed as actions/cache packs it (posix tar, zstd -T0 --long=30): 102,023,921 bytes at recipe 4 against 163,600,770 at recipe 5, of one LLVM with only its `src/` differing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- src/buildlock.rs | 14 ++++++++++++++ src/hostedclang.rs | 4 +--- src/llvm.rs | 46 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/buildlock.rs b/src/buildlock.rs index ee6a4af25ab..e7c786ba094 100644 --- a/src/buildlock.rs +++ b/src/buildlock.rs @@ -602,6 +602,16 @@ pub(crate) mod tests { Elsewhere::hold("buildlock::tests::child_role", &env) } + /// `root`'s worktree lock held shared by a process of its own. + pub(crate) fn shared_elsewhere(root: &Path) -> Elsewhere { + held_elsewhere(root, "hold-shared") + } + + /// Whether `root`'s worktree lock would keep a shared acquirer out now. + pub(crate) fn keeps_out_shared(root: &Path) -> bool { + !try_lock(&open_lock_file(&worktree_lock_dir(root).join("state")), LOCK_SH) + } + /// What `role` of [`child_role`] takes in `root`, held by a process of its own. fn held_elsewhere(root: &Path, role: &str) -> Elsewhere { Elsewhere::hold("buildlock::tests::child_role", &[(ROLE, OsStr::new(role)), (ROOT, root.as_os_str())]) @@ -707,6 +717,10 @@ pub(crate) mod tests { let mut held = shared(&root, "child"); held.act_if("queued exclusive phase", || Some(()), |()| note(&root, "ex")); } + "hold-shared" => { + let _held = shared(&root, "child"); + hold_until_released(); + } "want-shared" => { let _held = shared(&root, "child"); note(&root, "sh"); diff --git a/src/hostedclang.rs b/src/hostedclang.rs index 0b117aa1b6b..a4480cf4b6d 100644 --- a/src/hostedclang.rs +++ b/src/hostedclang.rs @@ -101,9 +101,7 @@ pub fn dispatch(root: &Path) { let sysroot = crate::toolchain::ensure(root, &mut lock); let fork = crate::sysroot::fork_checkout(root, &mut lock); let store = keystore::host(); - // Inside the worktree lock, which keeps every build in `fork` but an - // LLVM's, whose key's lock serialises them, out. - let llvm = crate::llvm::resolve(root, &store, &fork); + let llvm = crate::llvm::resolve_held(root, &store, &fork, &mut lock); let hosted = resolve(&store, &fork, &llvm.dir, sysroot.dir(), &crate::n2::ninja(root)); for (_, kept) in BINARIES { let binary = hosted.dir.join("bin").join(kept); diff --git a/src/llvm.rs b/src/llvm.rs index 68e1cdd9f9d..2a7196b8bc6 100644 --- a/src/llvm.rs +++ b/src/llvm.rs @@ -40,7 +40,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::OnceLock; -use crate::buildlock::{Guard, Keyed}; +use crate::buildlock::{Guard, Held, Keyed}; use crate::compiler::LLVM; use crate::keystore::{self, Key}; use crate::sysroot::{clone_tree, git_bytes, git_out, gitlink}; @@ -257,6 +257,24 @@ pub fn resolve(root: &Path, store: &Path, fork: &Path) -> Llvm { choose(store, fork, |fork| build_in_fork(root, fork)) } +/// [`resolve`] for a build holding its worktree's lock shared as `lock`: an +/// LLVM the store lacks is made with that lock held exclusively, so no other +/// build of the worktree runs in `fork` beside bootstrap's. +pub fn resolve_held(root: &Path, store: &Path, fork: &Path, lock: &mut Held) -> Llvm { + choose_held(store, fork, lock, |fork| build_in_fork(root, fork)) +} + +/// [`resolve_held`] with the build passed in, as [`choose`] takes it. +fn choose_held(store: &Path, fork: &Path, lock: &mut Held, build: impl Fn(&Path) -> PathBuf) -> Llvm { + let mut made = None; + let lacking = || defect(&Keyed::Llvm.store(store).join(key(fork))).map(drop); + lock.act_if("make the host's LLVM", lacking, |()| made = Some(choose(store, fork, &build))); + let unbuilt = |_: &Path| -> PathBuf { + panic!("the LLVM {} names left the store while this build held its worktree lock shared, which builds none", fork.display()) + }; + made.unwrap_or_else(|| choose(store, fork, unbuilt)) +} + /// [`resolve`] with the build that makes an LLVM passed in, so a test can stand /// in for bootstrap: `build` builds in the fork checkout it is given and returns /// the build directory, holding `/llvm` and `/lld`. @@ -1115,6 +1133,32 @@ mod tests { assert_eq!(defect(&choose(&store, &b.join("rust"), never).dir), None); } + /// **An LLVM is made only with the worktree lock exclusive, and one the + /// store holds is found under the shared lock**: while `a`'s is made, a + /// shared acquirer of `a`'s lock is kept out; `b`, whose worktree another + /// build holds shared, finds that LLVM without waiting for it to finish. + #[test] + fn an_llvm_is_made_only_with_the_worktree_lock_exclusive() { + let scratch = Scratch::new("llvm-held"); + let (_primary, store, [_same, a, b]) = estate_built(&scratch); + let mut lock = buildlock::shared(&a, "the test's build"); + let makes = Cell::new(0); + let alone = |fork: &Path| { + makes.set(makes.get() + 1); + assert!(buildlock::tests::keeps_out_shared(&a), "an LLVM was built in {} beside shared holders", fork.display()); + fake_build(fork) + }; + let made = choose_held(&store, &a.join("rust"), &mut lock, alone); + assert_eq!(makes.get(), 1); + assert_eq!(defect(&made.dir), None); + + let other = buildlock::tests::shared_elsewhere(&b); + let mut lock = buildlock::shared(&b, "the test's build"); + let never = |_: &Path| -> PathBuf { panic!("an LLVM the store holds was made again") }; + assert_eq!(choose_held(&store, &b.join("rust"), &mut lock, never).dir, made.dir); + other.release(); + } + /// **A sweep takes an LLVM only once nothing has used it for the store's /// keep time and nobody uses it**: `a` moves to another LLVM while a /// process of its own still uses the first, which placing the second From 1bfc3e9d298146213e80c79e1767cd3a1e7c557d Mon Sep 17 00:00:00 2001 From: japabu Date: Fri, 9 Oct 2026 21:52:33 +0200 Subject: [PATCH 6/6] A held build decides whether its LLVM is whole under the key's use, and makes it if a sweep took it `choose_held` read `defect` with no lock on the LLVM's key, and only then took the key's use inside `keyed_made`. A placement elsewhere could sweep an LLVM unused for the keep time in that gap, since nobody held its key, and `keyed_made` then found a defect and ran `unbuilt`, which panicked where the right outcome is to make the LLVM. It now takes `buildlock::keyed_using` first and reads the defect under it, which no sweep takes from under it. A whole LLVM is returned held; one that is not has its use dropped, since the worktree lock orders before the key's, and goes through `act_if`, whose exclusive branch makes it; if the second look finds it whole, the loop takes the use again. `unbuilt` and its panic go. `choose_held` takes the whole-check as a parameter, as it takes the build, so `a_sweep_after_the_llvm_is_found_whole_fails_no_build` can run a real `keystore::sweep` right after the check answers: it panicked before this change and passes after. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017cSFvbD35xJ2kGANVdm23C --- src/buildlock.rs | 2 +- src/llvm.rs | 69 +++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/src/buildlock.rs b/src/buildlock.rs index e7c786ba094..d6c679c9640 100644 --- a/src/buildlock.rs +++ b/src/buildlock.rs @@ -221,7 +221,7 @@ fn keyed_building(store: &Path, kind: Keyed, key: &Key) -> Guard { /// Use what `key` names: shared, so any number of builds use it at once, a /// builder of it is waited for, and a sweep cannot remove it; and dated now, /// which a sweep reads as its last use. -fn keyed_using(store: &Path, kind: Keyed, key: &Key) -> Guard { +pub(crate) fn keyed_using(store: &Path, kind: Keyed, key: &Key) -> Guard { let path = keyed_lock_path(store, kind, key); let file = open_lock_file(&path); if !try_lock(&file, LOCK_SH) { diff --git a/src/llvm.rs b/src/llvm.rs index 2a7196b8bc6..7306e9e008c 100644 --- a/src/llvm.rs +++ b/src/llvm.rs @@ -40,7 +40,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::OnceLock; -use crate::buildlock::{Guard, Held, Keyed}; +use crate::buildlock::{self, Guard, Held, Keyed}; use crate::compiler::LLVM; use crate::keystore::{self, Key}; use crate::sysroot::{clone_tree, git_bytes, git_out, gitlink}; @@ -261,18 +261,35 @@ pub fn resolve(root: &Path, store: &Path, fork: &Path) -> Llvm { /// LLVM the store lacks is made with that lock held exclusively, so no other /// build of the worktree runs in `fork` beside bootstrap's. pub fn resolve_held(root: &Path, store: &Path, fork: &Path, lock: &mut Held) -> Llvm { - choose_held(store, fork, lock, |fork| build_in_fork(root, fork)) + choose_held(store, fork, lock, defect, |fork| build_in_fork(root, fork)) } -/// [`resolve_held`] with the build passed in, as [`choose`] takes it. -fn choose_held(store: &Path, fork: &Path, lock: &mut Held, build: impl Fn(&Path) -> PathBuf) -> Llvm { - let mut made = None; - let lacking = || defect(&Keyed::Llvm.store(store).join(key(fork))).map(drop); - lock.act_if("make the host's LLVM", lacking, |()| made = Some(choose(store, fork, &build))); - let unbuilt = |_: &Path| -> PathBuf { - panic!("the LLVM {} names left the store while this build held its worktree lock shared, which builds none", fork.display()) - }; - made.unwrap_or_else(|| choose(store, fork, unbuilt)) +/// [`resolve_held`] with the build passed in, as [`choose`] takes it, and what +/// decides whether the stored LLVM is whole, so a test can sweep the store +/// once it has answered. +fn choose_held( + store: &Path, + fork: &Path, + lock: &mut Held, + whole: impl Fn(&Path) -> Option, + build: impl Fn(&Path) -> PathBuf, +) -> Llvm { + let key = key(fork); + let dir = Keyed::Llvm.store(store).join(&key); + loop { + // Decided under the key's use, which no sweep takes from under it. + let using = buildlock::keyed_using(store, Keyed::Llvm, &key); + if whole(&dir).is_none() { + return Llvm { dir, _using: using }; + } + // The worktree lock orders before the key's. + drop(using); + let mut made = None; + lock.act_if("make the host's LLVM", || whole(&dir).map(drop), |()| made = Some(choose(store, fork, &build))); + if let Some(made) = made { + return made; + } + } } /// [`resolve`] with the build that makes an LLVM passed in, so a test can stand @@ -1148,17 +1165,43 @@ mod tests { assert!(buildlock::tests::keeps_out_shared(&a), "an LLVM was built in {} beside shared holders", fork.display()); fake_build(fork) }; - let made = choose_held(&store, &a.join("rust"), &mut lock, alone); + let made = choose_held(&store, &a.join("rust"), &mut lock, defect, alone); assert_eq!(makes.get(), 1); assert_eq!(defect(&made.dir), None); let other = buildlock::tests::shared_elsewhere(&b); let mut lock = buildlock::shared(&b, "the test's build"); let never = |_: &Path| -> PathBuf { panic!("an LLVM the store holds was made again") }; - assert_eq!(choose_held(&store, &b.join("rust"), &mut lock, never).dir, made.dir); + assert_eq!(choose_held(&store, &b.join("rust"), &mut lock, defect, never).dir, made.dir); other.release(); } + /// **A sweep that lands once a held build has found its LLVM whole fails no + /// build**: the stored LLVM, unused for the keep time, is swept by a + /// placement elsewhere right after it is found whole, and the build goes on + /// with a whole LLVM. + #[test] + fn a_sweep_after_the_llvm_is_found_whole_fails_no_build() { + let scratch = Scratch::new("llvm-held-swept"); + let (_primary, store, [_same, a, _b]) = estate_built(&scratch); + let fork = a.join("rust"); + drop(choose(&store, &fork, fake_build)); + let unused = key(&fork); + let swept = Cell::new(None); + let then_swept = |dir: &Path| { + let found = defect(dir); + if swept.get().is_none() { + last_used(&store, Keyed::Llvm, &unused, LONG_AGO); + swept.set(Some(keystore::sweep(&store, Keyed::Llvm).len())); + } + found + }; + let mut lock = buildlock::shared(&a, "the test's build"); + let llvm = choose_held(&store, &fork, &mut lock, then_swept, fake_build); + assert_eq!(defect(&llvm.dir), None, "the build went on with an LLVM that is not whole"); + assert!(swept.get().is_some(), "the stand-in sweep never ran"); + } + /// **A sweep takes an LLVM only once nothing has used it for the store's /// keep time and nobody uses it**: `a` moves to another LLVM while a /// process of its own still uses the first, which placing the second