diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8e25a94aba1..6acf84dd649 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -117,10 +117,25 @@ jobs: # Homebrew keeps one QEMU; `cargo run` notes it if it is not the declared one. - run: | brew install qemu cmake + # 1.98.1 and not `stable`: 1.99.0 compiles a host test into an endless loop + # here (issues/rustc-1-99-0-makes-an-endless-loop-of-a-port-test-on-apple-silicon.md). - run: | curl --proto '=https' --tlsv1.2 -sSf -o "$RUNNER_TEMP/rustup-init.sh" https://sh.rustup.rs - sh "$RUNNER_TEMP/rustup-init.sh" -y --profile minimal --default-toolchain stable + sh "$RUNNER_TEMP/rustup-init.sh" -y --profile minimal --default-toolchain 1.98.1 echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - run: cargo run -- --build-only # The host suite's macOS arms run here alone: `ci.yml`'s `host` is Linux. - - run: cargo run -- --ci host + # The driver ends a cargo that goes silent and names it (src/ci.rs's + # `heard`); this is for a step that hangs while it talks, and is the + # limit `host` above runs under. + - timeout-minutes: 90 + run: cargo run -- --ci host + # Where macOS reports a process a signal ended with a core, every + # thread's stack in it: what the driver's SIGQUIT leaves of a silent step. + - if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: macos-crash-reports + path: ~/Library/Logs/DiagnosticReports + if-no-files-found: warn + retention-days: 7 diff --git a/issues/rustc-1-99-0-makes-an-endless-loop-of-a-port-test-on-apple-silicon.md b/issues/rustc-1-99-0-makes-an-endless-loop-of-a-port-test-on-apple-silicon.md new file mode 100644 index 00000000000..b209d33d0fb --- /dev/null +++ b/issues/rustc-1-99-0-makes-an-endless-loop-of-a-port-test-on-apple-silicon.md @@ -0,0 +1,139 @@ +--- +status: assigned +kind: tooling +opened: 2026-10-08 +--- + +# rustc 1.99.0 makes an endless loop of `a_port_answers_as_its_declaration_says` on Apple silicon + +`toyos-userbound/tests/firmware.rs`'s `a_port_answers_as_its_declaration_says` +never ended in `nightly.yml`'s `portability-macos`, the one job that runs the +host suite on macOS, from the first nightly that had the test (#749): the +compiler that job installed, `rustc 1.99.0 (b940084d7 2026-09-28)`, LLVM +23.1.1, compiles the test's function to one instruction, a branch to itself. +The job is pinned to 1.98.1 for it. The fault is LLVM's ScalarEvolution's, +on every target, and is not this compiler's alone: +`issues/the-forks-llvm-deletes-a-loops-exit-on-a-no-wrap-flag-scalar-evolution-gives-the-wrong-value.md` +has its cause, its reproducers and the compilers that have it. + +## What the runner showed + +Run 37834436612, a dispatch of `nightly.yml` on #778's branch, whose driver +ends a silent step's process group with `SIGQUIT` and whose macOS job keeps +the reports macOS writes of that. `portability-macos` concluded `failure`, +with one step of 77 red: + +``` +21:56:02 test a_port_answers_as_its_declaration_says has been running for over 60 seconds +22:11:02 [ci] the workspace's host members: cargo test --workspace ...: said nothing for 900s and was ended + with its process group by SIGQUIT; the last it said: test a_port_answers_as_its_declaration_says + has been running for over 60 seconds +``` + +The report of the `firmware-*` binary has two threads. `main` is parked in +libtest's `run_tests_console`, in `recv` on the channel a finished test +reports on. The thread named `a_port_answers_as_its_declaration_says` has its +program counter at offset 0 of the test's own function (the closure's +`FnOnce::call_once`), its link register in libtest's caller of it, and as +inlined frames at that one address `standing` (`tests/firmware.rs:495`), +`firmware::port` (`src/firmware.rs:344`, the `match` inside its loop) and the +test's line 513, the loop over the ports nothing declared. + +## The same binary, made here + +On an Apple-silicon development machine, in the worktree: `cargo test --locked +-p toyos-userbound --test firmware --no-run` under the toolchain and then the +binary whole, ended by PID if it had not ended, after 60 s in the first seven +rows and after 120 s in the last: + +| toolchain | its LLVM | environment | profile | the binary | +|---|---|---|---|---| +| 1.99.0 | 23.1.1 | `CI=true` | the tree's | hung: 20 tests `ok`, then libtest's line for this one | +| 1.99.0 | 23.1.1 | `CARGO_INCREMENTAL=0` | the tree's | hung, the same | +| 1.99.0 | 23.1.1 | `CARGO_INCREMENTAL=0` | `opt-level = 1` | exit 0 | +| 1.99.0 | 23.1.1 | `CARGO_INCREMENTAL=0` | `opt-level = 0` | exit 0 | +| 1.98.1 | 22.1.8 | `CARGO_INCREMENTAL=0` | the tree's | exit 0 | +| 1.99.0 | 23.1.1 | neither | the tree's | exit 0: the test alone, and 40 runs of 40 of the binary whole at the job's own tree and package selection | +| 1.98.1 | 22.1.8 | neither | the tree's | exit 0, the test alone | +| 1.99.0, `--target x86_64-apple-darwin` | 23.1.1 | `CARGO_INCREMENTAL=0` | the tree's | exit 0, run under Rosetta: `21 passed` | + +Cargo builds without incremental compilation where `CI` is set, which a +hosted runner sets and a developer's shell does not: a build with incremental +state does not show the fault. The hung binary's test function is at the +offset the runner's report names, and `otool -tv` shows it whole: + +``` +..._8firmware38a_port_answers_as_its_declaration_says0...FnOnce...call_once...: +0000000100001564 b ..._8firmware38a_port_answers_as_its_declaration_says0...call_once... +``` + +The x86-64 binary 1.99.0 made has it as a return (`objdump -d`, the Xcode +tools' LLVM one, as `otool` is): + +``` +..._8firmware38a_port_answers_as_its_declaration_says0...FnOnce...call_once...: +100001620: push rbp +100001621: mov rbp, rsp +100001624: mov rax, rdi +100001627: mov qword ptr [rdi], -0x1 +10000162e: pop rbp +10000162f: ret +``` + +With the test's cases edited and the hanging build repeated: without +`(0xFFFC, Width::DWord)` the binary exits 0, with it and without `(0xFFFF, +Width::Byte)` it hangs. The loop the compiler ends wrongly is +`firmware::port`'s `for port in port..=port + (width.bytes() as u16 - 1)`, +inlined with `standing` over the four ports `0xFFFC..=0xFFFF`. The source +ends: the crate forbids `unsafe`, and an inclusive range that ends at +`u16::MAX` is what `RangeInclusive` exists to get right. + +## What holds it, and what the pin is worth + +`portability-macos` installs `1.98.1` where it installed `stable` +(`nightly.yml`). That is a pin on a compiler nothing else in the tree pins. + +**1.98.1 has the faulty LLVM too.** It compiles this test right because its +`core` does not yet give the range's loop the shape LLVM miscompiles; given +that shape written by hand it makes the same endless loop. The pin keeps one +test of one job green. It is no statement that 1.98.1 compiles the rest of the +host suite right, and nothing measured says either way. + +**No stable is known to be coming that the job can go back to.** Five +nightlies were run, of 10, 13 and 22 July, 25 September and 8 October 2026, +the last the newest there was, and each hangs the table's second row; none +between them was run. Upstream's LLVM has merged no fix. That its open report +llvm/llvm-project#175729 is of this fault is the reading of +`issues/the-forks-llvm-deletes-a-loops-exit-on-a-no-wrap-flag-scalar-evolution-gives-the-wrong-value.md`, +which lists it under "Not established". + +`guest.yml`, and so `guest / suite` and the nightly's `tcg / suite`, and +`nightly.yml`'s `portability-linux` install `stable` and log its version: +the `tcg / suite` of run 37778826093 logged `rustc 1.99.0 (b940084d7 +2026-09-28)`, on x86-64, where this test is compiled right for +`x86_64-apple-darwin` (the table's last row); `x86_64-unknown-linux-gnu` was +not built. The two `host` jobs, `ci.yml`'s and `nightly.yml`'s, install +nothing and take the rustc of their runner's image. A developer's machine +takes whatever its `stable` is: one on Apple silicon with 1.99.0 or later who +runs `cargo run -- --ci host` with `CI` set, or without incremental +compilation, gets the step ended after 15 silent minutes with this test named. +Whether the host's toolchain is pinned once for every job is +`issues/the-host-job-runs-the-toolchain-the-runner-ships.md`'s to decide, +and this is a second measured case for it. + +`firmware::port` is not changed for it. The fault is the compiler's, the +loop is right, and a compiler that drops a loop's exit here is not made safe +by rewriting the one loop where it was seen. + +Assigned: the orchestrator, who holds the pin and its exit. Before #778 +lands he dispatches `nightly.yml` on its branch and reads +`portability-macos`: success, with `test +a_port_answers_as_its_declaration_says ... ok` in `the workspace's host +members`. He reads the same in the first nightly on a `main` that has #778. +At each stable release he reruns the table's second row under it. + +**Exit**: `portability-macos` installs `stable` again and is green in a +nightly on `main`, which takes a stable rustc under which the table's second +row exits 0 on Apple silicon. None exists; one arrives when upstream's LLVM +has the fault fixed, or when `core` gives the range's loop another shape +again, which is how 1.98.1 passes today and fixes nothing. diff --git a/issues/the-bootstraps-own-build-cannot-start-rust-objcopy-on-an-apple-host.md b/issues/the-bootstraps-own-build-cannot-start-rust-objcopy-on-an-apple-host.md new file mode 100644 index 00000000000..a63fc0dfcce --- /dev/null +++ b/issues/the-bootstraps-own-build-cannot-start-rust-objcopy-on-an-apple-host.md @@ -0,0 +1,44 @@ +--- +status: open +kind: defect +opened: 2026-10-09 +--- + +# The bootstrap's own build cannot start `rust-objcopy` on an Apple host + +Run 37834436612's `portability-macos` (a hosted `macos-26-arm64` runner, head +`c952668c9`, before #769) kept the reports macOS wrote during the job. 22 of +them are of `rust-objcopy`, each ended by dyld at launch, `EXC_CRASH`, +`SIGABRT`, "terminated at launch", inside `cargo run -- --build-only`, which +went on and succeeded: + +| when | the log's line before them | reports | dyld's reason | +|---|---|---|---| +| 19:53:54 to 19:54:19Z | `19:53:48 Building bootstrap`, the first | 13 | `Library not loaded: @rpath/libLLVM.dylib` | +| 21:12:19 to 21:12:29Z | `21:12:05 Building bootstrap`, the second, after LLVM was built and installed | 9 | `Symbol not found: __ZN4llvm18format_object_base4homeEv`, expected in a `libLLVM.dylib` | + +rustc strips a Darwin binary by running `rust-objcopy` +(`src/toolchain.rs`, above `a_compiler_build_finds_the_llvm_s_objcopy_where_rustc_strips_with_it`). +Both moments are the downloaded beta compiler building bootstrap itself. In +the first nothing it finds as `rust-objcopy` has an LLVM library to load; in +the second it finds one that loads a `libLLVM.dylib` without a symbol it +needs, which reads as the tree's `llvm-objcopy`, put on `PATH` for the +stage-1 compiler, started by the beta compiler against the beta's own +library. That is a reading of the two reasons and the two times; 8 of the 22 +name `rustc` as the process that started them and the rest a process that +had exited, and no run has shown which file each launch was. + +A developer's Mac does the same. The Apple-silicon Mac this tree is developed +on holds 18 reports of `rust-objcopy`, 1 of 4 October, 12 of 7 October and 5 +of 8 October, every one `Library not loaded`, none `Symbol not found`. Two +hosts do it, so it is a defect of the `PATH` the compiler build is given. + +Not known: whether bootstrap's binaries are left unstripped by this and +whether anything reads the difference; and whether #769, which moved the +toolchain into a store and landed on 8 October, changed it. + +Nobody holds it. It is in how a compiler build finds `rust-objcopy` +(`src/toolchain.rs`). + +**Exit**: a compiler built from nothing by `cargo run -- --build-only` on an +Apple host, at a tree that has #769, that leaves no `rust-objcopy` report. diff --git a/issues/the-forks-llvm-deletes-a-loops-exit-on-a-no-wrap-flag-scalar-evolution-gives-the-wrong-value.md b/issues/the-forks-llvm-deletes-a-loops-exit-on-a-no-wrap-flag-scalar-evolution-gives-the-wrong-value.md new file mode 100644 index 00000000000..3bab5c8305c --- /dev/null +++ b/issues/the-forks-llvm-deletes-a-loops-exit-on-a-no-wrap-flag-scalar-evolution-gives-the-wrong-value.md @@ -0,0 +1,505 @@ +--- +status: assigned +kind: defect +opened: 2026-10-09 +--- + +# The fork's LLVM deletes a loop's exit on a no-wrap flag ScalarEvolution gives the wrong value + +The compiler that builds every ToyOS kernel, loader and program +(`rust/` at `6d6ad8c71906`, `rustc 1.99.0-dev`, LLVM 22.1.8 from +`src/llvm-project` at `ceaf0fbb8440`) turns a correct loop of safe Rust into an +endless one. Measured: an inclusive range that ends at `u16::MAX`, for +`aarch64-unknown-toyos`, `aarch64-unknown-none-softfloat` and +`aarch64-unknown-uefi`; and one that ends at `u128::MAX`, for +`aarch64-unknown-toyos` and `x86_64-unknown-toyos`. The fault is in LLVM's +ScalarEvolution and is the same on every target: it moves a no-wrap flag from +an increment whose wrapped result nothing uses onto a value that is used, and +`indvars` deletes the loop's exit on it. Upstream LLVM has the fault and has +merged no fix. Its open report llvm/llvm-project#175729 has the same symptom +on another loop: that ToyOS's loop is that report's fault is this file's +reading, and that the report is the known fault is a maintainer's +"probably" ("Upstream" below, "Not established"). + +## The fault, with no front end + +`m1.ll`. `@f` returns `true` after four iterations: + +```llvm +define i1 @f() { +entry: + br label %header +header: + %next = phi i16 [ -3, %entry ], [ %nextnext, %latch ] + %iter = phi i16 [ -4, %entry ], [ %next, %latch ] + %done = icmp eq i16 %iter, -1 + br i1 %done, label %exit, label %latch +latch: + %nextnext = add nuw i16 %next, 1 + br label %header +exit: + ret i1 true +} +``` + +The `add` wraps when `%next` is -1. Its result is then poison, and nothing +uses it: the loop leaves on `%iter == -1` in the next header. `opt +-passes=indvars -S m1.ll`, with the `opt` of LLVM 22.1.8 that +`nightly-2026-07-22`'s `llvm-tools` ships, writes nothing to stderr and +leaves nothing of `%done`: the header is + +```llvm +header: ; preds = %latch, %entry + br i1 false, label %exit, label %latch +``` + +and the function never returns. The same with no `target datalayout`, with +AArch64's and with x86-64's. No `opt` built from the fork's LLVM was run; the +fork's source has the lines step 3 below reads, unchanged. + +A fixed `opt` leaves `%done` as it is, `icmp eq i16 %iter, -1` with the +header's branch on it, or folds it to something under which `@f` still returns +`true`. It never leaves `br i1 false`. + +Controls, each through the same `opt -passes=indvars`. The exit is folded to +`false` in: `m1` with a second exit in the latch that calls an opaque function +(`m2`); that at `i8`, `i32` and `i64`; and that with a start that is not +constant but carries `range(i16 10, 100)`. It is not folded in: `m2` without +the `nuw`; `m2` with the `add` in the header, before the rotation; and `m2` +with a start of unknown range. `m2` and the three that do not fold gave the +same under each of the three data layouts. Of eleven passes tried on `m2` +(`indvars`, `loop-unroll`, `loop-reduce`, `loop-vectorize`, `loop-idiom`, +`loop-deletion`, `loop-predication`, `loop-flatten`, `irce`, +`constraint-elimination`, `nary-reassociate`) only `indvars` leaves `br i1 +false`. + +## The cause + +Four steps. The first two are sound, the third is the fault, the fourth is +where it shows. From `-print-changed` traces of the `u16` reproducer below, +its unoptimised IR for `aarch64-apple-darwin` through that `opt -O2` and the +fork's compiler's own for `aarch64-unknown-toyos`, which agree; LLVM source +lines are the fork's at `ceaf0fbb8440`. + +1. **CorrelatedValuePropagation, on `port`, marks the range's increment + `nuw`.** Measured: the `add i16 %iter, 1` before that pass's dump is `add + nuw i16` after it. Read from the source: the add's only use is the header + phi along the back edge, that edge implies `iter != 0xFFFF`, and + `LazyValueInfo.cpp:1821` (`getValueAtUse`) reasons from exactly that. Sound: + where `iter` is `0xFFFF` the add is poison and nothing uses it. +2. **LoopRotate, on `probe` after `port` is inlined with the constant start + `0xFFFC`, makes that add a header phi's increment.** Measured: after it the + loop is `m1`'s, `%next` from -3, `%iter` from -4, and `add nuw i16 %next, 1` + in the latch. Sound: the add wraps in the iteration where `%iter` is + `0xFFFE`, and the loop leaves before the poison is used. +3. **ScalarEvolution gives `%next` the recurrence `{-3,+,1}`: the + fault.** Measured: its own printed analysis of `m2` and of the real loop + copied by hand says `{-3,+,1}` with + the unsigned range `[-3,0)`, and for `m2` beside it `exit count for header: + i16 3`, the iteration at which that recurrence is 0. Read from the source: + `ScalarEvolution.cpp:5776-5783` (`createSimpleAffineAddRec`; the general + path at 5879-5909 does the same) copies the increment's flags onto the + phi's recurrence without a condition, where only the post-increment + expression is guarded by `isAddRecNeverPoison` (5798). The flag is right of + `%next` alone, which is poison exactly where the recurrence wraps; a + ScalarEvolution expression is uniqued without its flags, so every value + with that expression takes it. +4. **`indvars` asks whether `%iter == -1` can hold and is told no.** + Measured: `-C llvm-args=-opt-bisect-limit` under `nightly-2026-07-22`, the + reproducer as an executable that prints `caller()`: limit 690 prints + `true`, 691 prints `false`, and pass 691 is `indvars` on the loop in + `probe`, whose whole effect on that function is + + ``` + - %or.cond.not.i.not = icmp eq i16 %iter, -1 + - br i1 %or.cond.not.i.not, label %exit, label %backedge + + br i1 false, label %exit, label %backedge + - %2 = add nuw i16 %0, 1 + + %2 = add nuw nsw i16 %0, 1 + ``` + + Later passes fold what is left into the endless loop. Read from the source + and not measured, the path inside: `SimplifyIndVar.cpp:275` + (`eliminateIVComparison`) calls `evaluatePredicateAt`; + `ScalarEvolution.cpp:11490` takes `getMinusSCEV(iter, -1)`, which is + `{-4,+,1} + 1`, the node `{-3,+,1}` of step 3; its range excludes + zero, so the compare is "known" false. `%iter + 1` is a defined value, 0 in + the last iteration. Upstream's report traces the same calls on its own + loop. + +**What put the loop in the tree's test in that shape is `core`, not LLVM.** +`a_port_answers_as_its_declaration_says` is compiled right by +`nightly-2026-07-09` and wrong by `nightly-2026-07-10`. +`14cae681329a...af3d95584dbd` is 106 commits, none under `src/llvm-project`, +and both ends report LLVM 22.1.8. Among them is rust-lang/rust #155114 (commit +`b3c94df68bf4`, merged as `71c64160bd0f`), which rewrote `RangeInclusive`'s +`next` to step with `Step::forward_overflowing` and keep the overflow bit in +`exhausted`. With `-C no-prepopulate-passes` the reproducer's loop differs +between 1.98.1 and `nightly-2026-07-22` only in block numbering and in that +callee. The fork contains both commits (`git merge-base --is-ancestor` exits 0 +for each against `6d6ad8c71906`). That `next` is a correct program. + +## The reproducers + +`minns.rs`, the `u16` form. `caller()` returns `true`: + +```rust +#![no_std] +#[derive(Clone, Copy)] +pub enum Mediated { Kept, ReadOnly } + +#[derive(Clone, Copy)] +pub enum Standing { Free, Declared(Mediated) } + +fn port(standing: impl Fn(u16) -> Standing, port: u16, width: u16, write: bool) -> Result { + for port in port..=port + (width - 1) { + match standing(port) { + Standing::Free => {} + Standing::Declared(Mediated::ReadOnly) if !write => {} + Standing::Declared(Mediated::ReadOnly) => return Err(9), + Standing::Declared(Mediated::Kept) => return Err(8), + } + } + Ok(port) +} + +fn standing(port: u16) -> Standing { + match port { + 0x3F8..=0x3FF | 0x20..=0x21 | 0xA0..=0xA1 | 0x70..=0x71 | 0xCF8 | 0xCFC..=0xCFF | 0xCF9 => Standing::Declared(Mediated::Kept), + 0xB2 => Standing::Declared(Mediated::ReadOnly), + _ => Standing::Free, + } +} + +fn probe() -> bool { + port(standing, 0xFFFC, 4, false).is_ok() & port(standing, 0xB2, 1, true).is_err() +} +pub fn caller() -> bool { probe() } +``` + +`c_u128.rs`, the `u128` form. `caller()` returns `true`: + +```rust +#![no_std] +#[derive(Clone, Copy)] +pub enum Mediated { Kept, ReadOnly } +#[derive(Clone, Copy)] +pub enum Standing { Free, Declared(Mediated) } +fn port(standing: impl Fn(u128) -> Standing, port: u128, width: u128, write: bool) -> Result { + for port in port..=port + (width - 1) { + match standing(port) { + Standing::Free => {} + Standing::Declared(Mediated::ReadOnly) if !write => {} + Standing::Declared(Mediated::ReadOnly) => return Err(9), + Standing::Declared(Mediated::Kept) => return Err(8), + } + } + Ok(port) +} +fn standing(port: u128) -> Standing { + match port { + 0x38..=0x3F | 0x20..=0x21 | 0xA0..=0xA1 | 0x70..=0x71 | 0xC8 | 0xCC..=0xCF | 0xC9 => Standing::Declared(Mediated::Kept), + 0xB2 => Standing::Declared(Mediated::ReadOnly), + _ => Standing::Free, + } +} +fn probe() -> bool { + port(standing, ::MAX - 3, 4, false).is_ok() & port(standing, 0xB2, 1, true).is_err() +} +pub fn caller() -> bool { probe() } +``` + +`rustc --edition 2021 --crate-type lib --emit llvm-ir,asm --target -C +opt-level=2 `, each exit 0, with the fork's `rustc` run from the store's +`sysroots/8618c089fa736cb0`, by the report of the agent that ran them: no log +records the path. That is the key the worktree's `target/.deps-stamp` gave +`aarch64-unknown-toyos` and `x86_64-unknown-toyos` at `e52e6275b`, and every +row below was made from it. The stamp gives `aarch64-unknown-none-softfloat`, +`aarch64-unknown-uefi`, `x86_64-unknown-none` and `x86_64-unknown-uefi` another +key, `dc7c468f0c07446e`, which no row here was made with: whoever repeats a +row for one of those four from the key the kernel and loader take has a +different library set than the row had. The `core` crate hash in the fork's +compiler's pass traces for the two ToyOS targets is the one in the two ToyOS +`libcore` files under `8618c089fa736cb0`. + +What `caller` became, `minns.rs`: + +| target | `caller` | +|---|---| +| `aarch64-unknown-toyos` | no `ret` in its IR; frame setup, then `.LBB0_1: b .LBB0_1` | +| `aarch64-unknown-none-softfloat` | no `ret`; `.LBB0_1: b .LBB0_1` | +| `aarch64-unknown-uefi` | no `ret`; `.LBB0_1: b .LBB0_1` | +| `aarch64-apple-darwin` | no `ret`; `LBB0_1: b LBB0_1` | +| `x86_64-unknown-toyos` | `ret i1 true`; `movb $1, %al`, `retq` | +| `x86_64-unknown-none` | `ret i1 true`; `movb $1, %al`, `retq` | +| `x86_64-unknown-uefi` | `ret i1 true`; `movb $1, %al`, `retq` | + +And by width: `c_u128.rs` with `u128` replaced by each type, `--emit llvm-ir`, +`caller` read in the IR. The `u16` row is that file's `u16` form, which +differs from `minns.rs` in `standing`'s arms and in writing the start as +`::MAX - 3`: + +| type | `aarch64-unknown-toyos` | `x86_64-unknown-toyos` | +|---|---|---| +| `u8` | `ret i1 true` | `ret i1 true` | +| `u16` | **no `ret`: a block that branches to itself** | `ret i1 true` | +| `u32` | `ret i1 true` | `ret i1 true` | +| `u64` | `ret i1 true` | `ret i1 true` | +| `u128` | **no `ret`: a block that branches to itself** | **no `ret`: a block that branches to itself** | +| `i16` | 41 lines with one `ret`, not `ret i1 true`; whether it is right was not checked | the same | + +The `i8` form does not compile (its literals are out of range). + +Both sources are fragile. Without the second call in `probe`, without the +private `probe` between `caller` and the calls, or with three of the `Kept` +arms gone, the same compiler compiles `minns.rs` right. + +The same fault in the tree's own code: `toyos-userbound/tests/firmware.rs`'s +`a_port_answers_as_its_declaration_says`, whose `(0xFFFC, Width::DWord)` case +runs `firmware::port`'s range to `0xFFFF`. On an Apple-silicon development +machine, `CARGO_INCREMENTAL=0 cargo test --locked -p toyos-userbound --test +firmware --no-run` with `RUSTUP_TOOLCHAIN` naming that sysroot directory, which +holds the fork's `aarch64-apple-darwin` libraries, exits 0, and the binary run +whole prints 20 tests `ok` and never ends (ended by PID at 120 s). Its test +function is one instruction, a branch to itself. + +## Which compilers + +The same test binary, built and run the same way under upstream's compilers on +the same machine, one rustup toolchain installed and removed per row. `r4.rs` +and `d1.rs` are two earlier single-file forms of the reproducer, compiled to +assembly beside each row. + +| toolchain | rustc | LLVM | the test binary | `caller` in `r4.rs`, `d1.rs` | +|---|---|---|---|---| +| nightly | 1.96.0-nightly (d9563937f 2026-03-03) | 22.1.0 | exit 0, 21 passed | not compiled | +| nightly-2026-05-13 | 1.97.0-nightly (8b03437a8 2026-05-12) | 22.1.4 | exit 0 | returns | +| nightly-2026-06-17 | 1.98.0-nightly (9e2abe0c6 2026-06-16) | 22.1.7 | exit 0 | returns | +| nightly-2026-07-04 | 1.98.0-nightly (c397dae80 2026-07-02) | 22.1.8 | exit 0 | returns | +| nightly-2026-07-08 | 1.99.0-nightly (f10db292a 2026-07-07) | 22.1.8 | exit 0 | returns | +| nightly-2026-07-09 | 1.99.0-nightly (14cae6813 2026-07-08) | 22.1.8 | exit 0 | returns | +| nightly-2026-07-10 | 1.99.0-nightly (af3d95584 2026-07-09) | 22.1.8 | hung, ended at 120 s | a branch to itself in both | +| nightly-2026-07-13 | 1.99.0-nightly (77cf889bc 2026-07-12) | 22.1.8 | hung | a branch to itself in both | +| nightly-2026-07-22 | 1.99.0-nightly (0e29c21d9 2026-07-21) | 22.1.8 | hung | a branch to itself in both | +| nightly-2026-09-25 | 1.100.0-nightly (f7575a9da 2026-09-24) | 23.1.1 | hung; the test's function is `b .` | not compiled; `minns.rs`: no `ret` | +| nightly-2026-10-08 | 1.101.0-nightly (1d81eb4ad 2026-10-07) | 23.1.3 | hung; the test's function is `b .` | not compiled | + +Stable 1.99.0 (LLVM 23.1.1) hangs it and stable 1.98.1 (LLVM 22.1.8) does not: +`issues/rustc-1-99-0-makes-an-endless-loop-of-a-port-test-on-apple-silicon.md` +has those rows. Five nightlies from `nightly-2026-07-10` on were run, the five +in the table, and each hangs it; no other was. `nightly-2026-10-08` was the +newest there was. + +**A compiler that passes the test escapes by the shape its `core` gives the +loop, and has the fault.** The `u16` reproducer with its range replaced by +this iterator, so that no compiler's `core` decides the loop's shape: + +```rust +pub struct Overflowing { start: u16, end: u16, exhausted: bool } +impl Iterator for Overflowing { + type Item = u16; + #[inline] + fn next(&mut self) -> Option { + if self.exhausted || !(self.start <= self.end) { + return None; + } + let (n, o) = self.start.overflowing_add(1); + self.exhausted = o; + Some(core::mem::replace(&mut self.start, n)) + } +} +``` + +| compiler | LLVM | `caller` at `-C opt-level=2` | +|---|---|---| +| 1.88.0 | 20.1.5 | right, by shape: as an executable for `aarch64-apple-darwin` it prints `true`. In the loop that runs to `u16::MAX` the `overflowing_add` is a call of `llvm.uadd.with.overflow.i16` in every dump of its trace, so there is no `add` for step 1 to mark. It says nothing of LLVM 20.1.5's ScalarEvolution | +| 1.91.0 | 21.1.2 | wrong, `aarch64-apple-darwin`; in its trace the increment is an `add`, and `add nuw` after CorrelatedValuePropagation on `port` | +| nightly, 1.96.0-nightly (d9563937f 2026-03-03) | 22.1.0 | wrong, `aarch64-apple-darwin` | +| 1.95.0 | 22.1.2 | wrong, `aarch64-apple-darwin` | +| 1.98.0 | 22.1.8 | wrong, `aarch64-apple-darwin` | +| 1.98.1 | 22.1.8 | no `ret` for `aarch64-apple-darwin` and `aarch64-unknown-none-softfloat`; the executable hangs at `opt-level` 2 and 3 and prints `true` at 0 and 1; right for `x86_64-unknown-none` | +| the fork's | 22.1.8 | no `ret` for `aarch64-unknown-toyos` and `aarch64-unknown-none-softfloat`; right for `x86_64-unknown-toyos` and `x86_64-unknown-none` | + +So 1.88.0 and 1.98.1 each pass something by a shape, the first this iterator +and the second the tree's test. Which LLVM change between 20.1.5 and 21.1.2 +made the intrinsic an `add` that early is not determined, and is not where the +fault is. + +## Upstream + +Read from GitHub's API without credentials on 9 October 2026: + +- **llvm/llvm-project#175729**, "[SCEV] Long-standing miscompile due to + absence of per-use flags in SCEV expressions": open, opened 13 January + 2026, labelled `miscompilation`. Its loop is another, with a `nuw` the loop + vectoriser left; its symptom is this one, `opt -passes=indvars` folding the + exit to `false`, and its reporter's trace runs through + `eliminateIVComparison`, `getMinusSCEV` and `isKnownNonZero`. +- **llvm/llvm-project pull request #118959**, "[SCEV] Don't blindly transfer + nowrap flags to pre-inc addrec": open, a draft, unmerged, opened 6 December + 2024, 22 months before that reading, last updated 26 March 2026. Its body + says "Test updates incomplete". It changes `ScalarEvolution.cpp` (+73 −37) + and `ScalarEvolution.h` (+10 −4) and 22 test files. It is in no LLVM + release. +- rust-lang/rust: no report of this was found by six searches. + +**Not established:** + +- **that #175729 is the known fault.** A maintainer's sentence there is "The + nuw is fine as the value is never used. I've only glanced at it, but this is + probably the known issue where we incorrectly unconditionally transfer + nowrap flags for preinc addrecs from IR to SCEV", and on 17 February 2026: + "I believe" that pull request "is the fix for this issue. However, it has + some problematic impact"; +- **that ToyOS's loop is #175729's.** Nobody upstream has seen it: that is + this file's reading, from the same symptom and the same source path; +- **that #118959 fixes either.** Nobody has built an LLVM with it, here or by + anything upstream's thread says. By reading, it drops the flag in `m2` and + in the real loop; +- **when.** Waiting for upstream has no date. + +The fork's `src/llvm-project` is one shallow commit, so its history answers +nothing; its source has no `canPreservePreIncAddRecNoWrapFlags`, the name +#118959 adds. + +No report or pull request is sent for now (root `CLAUDE.md`, "Dependencies"). + +## How far it reaches + +**Exposed: the kernel, the loader and userland, on both architectures.** +Nothing wrong has been found in them, and nothing was looked for. + +**An inclusive range to its type's maximum is one instance of a class.** The +class is a loop in which an increment's wrapped result is dead, so that step 1 +may mark it; a later rotation makes that increment a header phi's; the phi's +start has a known range; and a client of ScalarEvolution reasons about another +value with the same expression. The hand-written iterator above is in it +without `core`'s range, and `m1` without a range at all. A loop of the class +that never reaches the wrap loses an exit it never takes. + +**Why AArch64 showed it for `u16` and x86-64 did not**, read from one pair of +traces of `minns.rs` under the fork's compiler. For `x86_64-unknown-toyos`, +`indvars` changes `port`'s loop before the run of CorrelatedValuePropagation +that marks the add for AArch64: it rewrites the exit test onto the +incremented value, the add has a second use, and no dump in the trace has +`nuw` on the `i16` increment. For `aarch64-unknown-toyos` the trace has no +`indvars` change on `port`; read from the source, `IndVarSimplify.cpp:964` +refuses that rewrite for a counter whose width is not `DL.isLegalInteger`, and +16 is not in AArch64's `n32:64` where x86-64's layout is `n8:16:32:64`. So the +width decides whether the shape is reached, and nothing else: `u128`, legal on +neither, is wrong on both, by steps 1, 2 and 4 in the x86-64 trace of it. `u8` +on AArch64, not legal either, came out right in this one source, and `m2` at +`i8`, `i32` and `i64` folds: no width is safe. + +**x86-64: 0 of 40 other sources miscompiled, and one counterexample.** 39 +source variants made while reducing, judged by whether a function is left with +no `ret`, `unreachable` or `resume`, which sees the endless loop and nothing +else: 18 are miscompiled for `aarch64-unknown-none-softfloat` and the same 18 +for `aarch64-unknown-toyos`, none for `x86_64-unknown-none` or +`x86_64-unknown-toyos`; the hand-written iterator's file is the fortieth. None +of the forty names an integer wider than 64 bits, so by the paragraph above +each had a legal counter on x86-64; that is read for `minns.rs` and inferred +for the other thirty-nine. The counterexample is the `u128` row. Stable 1.99.0 +compiles the tree's test right for `x86_64-apple-darwin`. + +**The outcomes seen** are the endless loop and, with the passes after `indvars` +withheld, a wrong value: `false` for `true`. So a wrong answer without a hang +is possible, and no test in the tree would name it. + +**The x86-64 kernel's own instance of the loop has its exit.** The kernel +calls `firmware::port` once, from `kernel/src/arch/x86_64/acpi_mode.rs`'s +`port`, with a port and width the `acpi` claim's holder chooses. From `CI=true +cargo run -- --build-only` (exit 0) at `e0a61d070`, built from nothing with and +without incremental state and disassembled: a function of its own of 0x11a +bytes in the first, inlined into `acpi_mode::port` (0x13a bytes) in the second. +In both the loop counts the width's bytes down in a 16-bit register and leaves +at zero, every refusal leaves it, and no branch back is unconditional: + +``` +eef55: inc r12d +eef58: dec r13w +eef5c: je 0xeef93 <+0xb3> ; every port asked: the access is made +eef5e: mov esi, 0x1 +eef63: mov edi, r12d +eef66: call + ... ; a refusal jumps out of the loop, a pass back to eef55: +eef79: je 0xeef55 <+0x75> +eef8d: je 0xeef55 <+0x75> +``` + +The AArch64 kernel has no instance of that call: it is under `arch/x86_64`, by +the source and not by a disassembly. + +**Not measured:** + +- **any other function of the kernel, the loader or userland, on either + architecture.** This is the larger gap, and no reading closes it: the class + is not `..=` loops, nor narrow integers, nor one architecture. Today's + compiler counts nothing that would: `-C llvm-args=-stats` prints nothing + from the fork's `rustc`, `-debug-only=indvars` is refused as an unknown + argument, and read from the source `IndVarSimplify.cpp` emits no + optimisation remark, while the counter the fold increments, `NumElimCmp` + (`SimplifyIndVar.cpp:46`), counts every legitimate elimination with it. The + measurement owed is in the exit; +- the `u128` form for `x86_64-unknown-none`, `x86_64-unknown-uefi`, + `aarch64-unknown-none-softfloat` and `aarch64-unknown-uefi`, the targets of + the kernel and the loader; +- code generation's loop strength reduction, which reads ScalarEvolution too, + past `opt -passes=loop-reduce` on `m2`; +- `aarch64-unknown-none`, `aarch64-unknown-linux-gnu` and + `x86_64-unknown-linux-gnu` under an affected compiler. + +## What does not end it + +- **Moving the fork to a later upstream**: the newest nightly there was hangs + the test, and upstream's LLVM has merged no fix. +- **Waiting for upstream**: it has no date. The proposed fix has been open and + unmerged for 22 months. +- **Taking #155114 out of the fork's `core`**: `library/core` carries no delta + (`.claude/agents/implementer.md`, "A fork"), and the fault would stay for + any other code of the class, as the hand-written iterator shows. +- **Rewriting `firmware::port`**: the loop is right, and it is one loop. +- **A change to `indvars`** that makes the reproducers return: `indvars` asks + a question and is answered wrongly, and every other client of + ScalarEvolution would go on being answered so. +- **`nightly.yml`'s pin of `portability-macos` to 1.98.1**: no job installs the + fork's compiler by a version; it is the tree's own. + +## Owner and exit + +Held by the toolchain, `rust/` and its `src/llvm-project` +(`ToyOSOrg/llvm-project`, branch `toyos-rustc-22.1-2026-05-19`). A fix in the +fork's LLVM is being built on `wt/toyos-scevfix`. + +Whoever moves the fork to a later upstream runs the measurements below under +the moved compiler before the move lands, and corrects this file by what they +show. + +**Exit**: ScalarEvolution in the fork's LLVM no longer gives a value a no-wrap +flag that holds only where another value is poison, by a change to +ScalarEvolution and to no client of it, measured under the LLVM and the +compiler the tree builds with by all of: + +1. `m1.ll` above through that LLVM's `opt -passes=indvars -S`: the output has + no `br i1 false`, and `@f` returns; +2. `caller` is `ret i1 true` by the command above: for `aarch64-unknown-toyos` + and for `x86_64-unknown-toyos`, in `minns.rs` and in `c_u128.rs`, from the + key `target/.deps-stamp` gives the ToyOS targets; and for + `aarch64-unknown-none-softfloat` and `aarch64-unknown-uefi`, in `minns.rs`, + from the key it gives those two, the one the kernel and the loader take, + which is not the key the table's rows for them were made from + (`8618c089fa736cb0`, the ToyOS targets'); +3. the test binary above, built with that compiler on Apple silicon without + incremental state, exits 0; +4. the tree's own functions read: the fix behind an LLVM option, the tree + built twice with the one compiler, the option given and withheld through + `-C llvm-args`, and the two builds compared function by function. The + functions that differ are a superset of those the fault changed; one that + loses an exit branch in the build without the fix is a hit, and each hit is + named in the pull request that closes this file. The same option is the + fix's negative control. + +Items 2 and 3 rest on shapes `rustc` may stop making, so a compiler that +merely stops making them meets those two and not the exit; item 1 has no front +end in it, and item 4 is the only one that reads the code ToyOS ships. The +check that holds the fix in this tree arrives with the fix, green. diff --git a/issues/the-power-offs-global-lock-line-is-logged-after-the-last-console-drain.md b/issues/the-power-offs-global-lock-line-is-logged-after-the-last-console-drain.md deleted file mode 100644 index f7344e74d77..00000000000 --- a/issues/the-power-offs-global-lock-line-is-logged-after-the-last-console-drain.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -status: assigned -kind: defect -opened: 2026-10-08 ---- - -# The power-off logs the Global Lock's give-back after the console's last drain, and `acpi_mediated_access` waits for that line - -`acpi_mediated_access` (`tests/toyos.rs`) ends its wait on the kernel's line -`acpi: the Global Lock given back for a holder that left it taken (the machine -is stopping)`. Twice it never arrived, and the harness ended the guest after -`GUEST_QUIET` (15 s) of silence: - -``` -FAIL acpi_mediated_access: STALLED: waiting for the probe's power-off to give the lock back — it went quiet -``` - -| where | head | instrument | load | result | -|---|---|---|---|---| -| `main`'s nightly, run 37758546165, `tcg / suite` | `b6bcb9691` | TCG on a hosted 4-core x86-64 runner, 31 tests one wide | the suite alone | STALL after 18 s | -| the development machine, a whole-suite run of #763 | `252d19247` | TCG, x86-64 guest on an AArch64 host, 14 cores, 12 wide | load average 45 to 67: five agents compiling beside the suite | STALL after 35 s | -| `main`'s merge queue, six runs, #749 to #746 (37753990562, 37755528529, 37757410477, 37758981663, 37759626406, 37760845496), `guest / suite` | `6f87cdb9c` to `1084ddc9a` | KVM on hosted 4-core runners, 31 tests one wide | the suite alone | PASS, 3 s each | -| the development machine, `acpi_` by name | `252d19247` | TCG, 2 tests | run alone | PASS, 4 s | - -The nightly's head has #749, which added the test, and nothing of #763: the -red is `main`'s. KVM on a hosted runner is not TCG on a loaded host, so the -six green runs say only that the test passes where the guest is fast. - -## What was read - -`power::shutdown` (`kernel/src/power.rs`) calls `serial::flush_final()` under -the comment "Last chance: nothing drains the log ring after this point", and -then `arch::power::off`. `off` (`kernel/src/arch/x86_64/power.rs`) calls -`acpi_mode::settle`, whose `give_back` logs the line the test waits for, then -`acpi_mode::quiet`, then writes `SLP_TYP` and `SLP_EN`. So the line is -committed after the stop's last drain. Past boot the console's one writer is -`klogd` (`kernel/src/log/console.rs`, `Drain::Thread`), woken at the commit: the -line reaches the host only if `klogd`, on the other CPU, puts it on the wire -before this CPU has made one port read, `quiet`'s writes and the two writes -that end the machine. Nothing waits for it. - -That is a reading, and no run confirms it: neither red kept what the guest -said after boot. The test passes `await_guest`'s error up without its capture, -and the `uart-*.log` the nightly kept (artifact `serial-suite`, `uart-6.log`, -the `i8042-withheld` boot) ends where every boot's does, at the hand-over to -the virtio console. It fits both reds: the nightly's 18 s is the 3 s the test -takes under KVM and the 15 s of `GUEST_QUIET`. A probe whose arm failed would -have ended the wait by `===TEST_END`, and a kernel panic by its own line. - -The branch that saw it locally moves nothing here: `settle`, `give_back`, -`off`, `shutdown` and the console are `main`'s bytes in #763. - -Owner: the `acpi` claim's author (#749). - -**Exit**: `acpi_mediated_access` green in the nightly's `tcg / suite`, with the -give-back's line on the wire before `SLP_EN` by construction and not by -`klogd` winning; and a stall of this test carrying what the guest said. - -## What landed, and what is still owed - -Everything above is `main` before #767 and is kept as it was written. #767 -split the architecture's power-off in two: `arch::power::settle` says the -give-back's line, `power::shutdown` drains the console after it, and -`arch::power::off` takes the value only `settle` makes and logs nothing. The -exit's second and third clauses are met by that pull request: the order is -`kernel/src/power.rs`'s `shutdown`, and the test's stall appends what the -guest said since its boot. - -The reading above has since been run. On one CPU no other CPU runs `klogd` -beside the power-off, and the line never arrived with the split reverted, -three boots of three, each capture ending at `Shutting down.`; with it the -line arrived, three of three. `acpi_lock_given_back_on_one_cpu` -(`tests/toyos.rs`) is that boot. - -Its first clause is not met: no nightly has run `tcg / suite` on a `main` -that carries #767. - -Assigned: the orchestrator, at #767's landing. He reads `acpi_mediated_access` -in the first nightly `tcg / suite` on a `main` that carries it, and deletes -this file on a green; a stall there prints the guest's words, and goes here. diff --git a/src/ci.rs b/src/ci.rs index 53b4052b877..72f0954c676 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -16,9 +16,14 @@ //! reds on a disagreement, and on a `/dev/kvm` that is present and does not //! open; `cargo run` only notes one, because a build must not stop for brew. -use std::io::{BufRead, BufReader, Write}; +use std::io::{Read, Write}; +use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, ExitStatus}; +use std::sync::atomic::{AtomicI32, Ordering}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::sync::Once; +use std::time::{Duration, Instant}; use crate::arch::{Accel, Arch}; use crate::cicache; @@ -146,13 +151,9 @@ fn summary(text: &str) { } } -/// `cargo ` in `dir`, its output passed straight through. +/// `cargo ` in `dir`, as [`cargo_logged`] runs it, judged by its exit. fn cargo(dir: &Path, args: &[&str]) -> Result { - let status = Command::new("cargo") - .args(args) - .current_dir(dir) - .status() - .map_err(|e| format!("cargo: {e}"))?; + let (status, _) = cargo_logged(dir, args)?; let line = format!("cargo {}", args.join(" ")); if status.success() { Ok(line) @@ -161,28 +162,217 @@ fn cargo(dir: &Path, args: &[&str]) -> Result { } } -/// `cargo ` in `dir`, its output passed through and also kept, both -/// streams in the order they were written: a verdict read off the log needs the -/// whole of it. -fn cargo_logged(dir: &Path, args: &[&str]) -> Result<(bool, String), String> { - let (reader, writer) = std::io::pipe().map_err(|e| format!("pipe: {e}"))?; - let mut child = Command::new("cargo") - .args(args) - .current_dir(dir) - .stdout(writer.try_clone().map_err(|e| format!("pipe: {e}"))?) - .stderr(writer) - .spawn() - .map_err(|e| format!("cargo: {e}"))?; - let mut log = String::new(); - let mut err = std::io::stderr(); - for line in BufReader::new(reader).split(b'\n') { - let line = line.map_err(|e| format!("reading cargo: {e}"))?; - let line = format!("{}\n", String::from_utf8_lossy(&line)); - let _ = err.write_all(line.as_bytes()); - log.push_str(&line); - } - let status = child.wait().map_err(|e| format!("cargo: {e}"))?; - Ok((status.success(), log)) +/// How long a step's cargo may say nothing: a build prints a line as each +/// crate starts, libtest one as each test ends, and the guest harness ends a +/// silent guest long before this. A hang ceiling and no measure of a step. +/// Shortened under `cfg(test)` to what a loaded host starts a built binary +/// in, so the gate on it costs seconds. +#[cfg(not(test))] +const QUIET: Duration = Duration::from_secs(15 * 60); +#[cfg(test)] +const QUIET: Duration = Duration::from_secs(10); + +/// How long the processes of a signalled group may take to let go of its output. +const GONE: Duration = Duration::from_secs(10); + +/// The signals that end the driver from outside. +const INTERRUPTS: [libc::c_int; 3] = [libc::SIGINT, libc::SIGTERM, libc::SIGHUP]; + +/// The process group [`heard`] is running, for [`interrupted`]; 0 between +/// two, and [`STARTING`] while one is being spawned. +static GROUP: AtomicI32 = AtomicI32::new(0); + +/// [`GROUP`] while a command is being spawned: its group has no name yet. +const STARTING: i32 = -1; + +/// The last interrupt [`interrupted`] took, for [`started`]; 0 for none. +static PENDING: AtomicI32 = AtomicI32::new(0); + +/// The driver's own interrupt, handed to the group it started and then taken: +/// a group of its own is out of a terminal's reach, and a step left running +/// has lost the driver that bounded it. One that arrives while a command is +/// being spawned is kept for [`started`]. +extern "C" fn interrupted(signal: libc::c_int) { + // Written before `GROUP` is read, as `heard` reads this after every write + // of `GROUP`: whatever thread this runs on, one of the two sees the other. + PENDING.store(signal, Ordering::SeqCst); + let group = GROUP.load(Ordering::SeqCst); + if group == STARTING { + return; + } + // SAFETY: three async-signal-safe calls; the signal raised at its default ends the driver. + unsafe { + if group != 0 { + libc::killpg(group, signal); + } + libc::signal(signal, libc::SIG_DFL); + libc::raise(signal); + } +} + +/// Say which group the driver runs now, 0 for none, and take the interrupt +/// the handler took before it could read that. +fn started(group: i32) { + GROUP.store(group, Ordering::SeqCst); + let signal = PENDING.load(Ordering::SeqCst); + if signal != 0 { + interrupted(signal); + } +} + +/// What a command says, passed through as it arrives and kept: in bytes, so +/// a line still unfinished is said too. libtest on one thread names a test +/// before it runs it and ends the line after. +struct Said { + chunks: Receiver>, + log: Vec, +} + +impl Said { + /// `Some(true)` for more of it within `within`, `Some(false)` at its end, + /// which is every process that held it gone, and `None` for neither. + fn more(&mut self, within: Duration) -> Option { + match self.chunks.recv_timeout(within) { + Ok(chunk) => { + let _ = std::io::stderr().write_all(&chunk); + self.log.extend(chunk); + Some(true) + } + Err(RecvTimeoutError::Disconnected) => Some(false), + Err(RecvTimeoutError::Timeout) => None, + } + } + + /// Whether it ended within [`GONE`]. + fn ended(&mut self) -> bool { + let deadline = Instant::now() + GONE; + loop { + match self.more(deadline.saturating_duration_since(Instant::now())) { + Some(true) => {} + Some(false) => return true, + None => return false, + } + } + } + + fn text(&self) -> String { + String::from_utf8_lossy(&self.log).into_owned() + } + + /// The last line with anything on it, finished or not, to the length a + /// refusal can carry: a command may say megabytes without ending a line. + fn last(&self) -> String { + let text = self.text(); + let last = text.lines().rfind(|line| !line.trim().is_empty()).unwrap_or("nothing"); + last.trim().chars().take(200).collect() + } +} + +/// `cargo ` in `dir`, as [`heard`] runs it. +fn cargo_logged(dir: &Path, args: &[&str]) -> Result<(ExitStatus, String), String> { + let mut cargo = Command::new("cargo"); + cargo.args(args).current_dir(dir); + heard(cargo).map_err(|why| format!("cargo {}: {why}", args.join(" "))) +} + +/// Run `cmd`, its output passed through and also kept, both streams in the +/// order they were written: a verdict read off the log needs the whole of it. +/// +/// **A command that says nothing for [`QUIET`] is hung, and is ended with all +/// it spawned**: it leads a process group of its own, because the process that +/// hangs is a test binary cargo started, and killing cargo alone would leave it +/// running. The refusal carries the last line said, which under libtest names +/// the test: `test has been running for over 60 seconds`, or on one +/// thread the unfinished `test ... `. +/// +/// **The group is sent `SIGQUIT`, and `SIGKILL` if its output has not ended +/// [`GONE`] later**: a host that reports a process a signal ended with a core +/// then holds where the silent one was. The command starts with `SIGQUIT` at +/// its default whatever the driver inherited, and std starts every child with +/// no signal blocked, so the first signal does not depend on who started the +/// driver. +/// +/// **An interrupt of the driver ends the group** ([`interrupted`]), unless the +/// driver was started ignoring that signal. +fn heard(mut cmd: Command) -> Result<(ExitStatus, String), String> { + static HANDED_ON: Once = Once::new(); + HANDED_ON.call_once(|| { + for signal in INTERRUPTS { + // SAFETY: `interrupted` makes async-signal-safe calls only. + unsafe { + if libc::signal(signal, interrupted as *const () as libc::sighandler_t) == libc::SIG_IGN { + libc::signal(signal, libc::SIG_IGN); + } + } + } + }); + let (mut reader, writer) = std::io::pipe().map_err(|e| format!("pipe: {e}"))?; + cmd.stdout(writer.try_clone().map_err(|e| format!("pipe: {e}"))?).stderr(writer).process_group(0); + // SAFETY: one async-signal-safe call on the child's own state. + unsafe { + cmd.pre_exec(|| { + if libc::signal(libc::SIGQUIT, libc::SIG_DFL) == libc::SIG_ERR { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + GROUP.store(STARTING, Ordering::SeqCst); + // An interrupt taken between two steps, its handler not yet at its raise: + // nothing is spawned for it to miss. + if PENDING.load(Ordering::SeqCst) != 0 { + started(0); + } + let spawned = cmd.spawn(); + let group = spawned.as_ref().map_or(0, |child| child.id() as i32); + started(group); + let mut child = spawned.map_err(|e| format!("spawn: {e}"))?; + // The writers it holds: the output ends when the last process holding one does. + drop(cmd); + let (tx, chunks) = mpsc::channel(); + std::thread::spawn(move || { + let mut chunk = [0; 4096]; + loop { + match reader.read(&mut chunk) { + Ok(0) => return, + Ok(n) if tx.send(chunk[..n].to_vec()).is_ok() => {} + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Ok(_) | Err(_) => return, + } + } + }); + let mut said = Said { chunks, log: Vec::new() }; + let hung = loop { + match said.more(QUIET) { + Some(true) => {} + Some(false) => break None, + None => break Some(said.last()), + } + }; + let ended = hung.map(|last| { + // SAFETY: the group the child leads, which no other process can name until it is reaped. + unsafe { libc::killpg(group, libc::SIGQUIT) }; + let mut by = "SIGQUIT"; + let mut gone = said.ended(); + if !gone { + by = "SIGKILL"; + // SAFETY: as above. The child by its own name too: a group whose + // members are gone but for an unreaped leader may answer for nobody. + unsafe { libc::killpg(group, libc::SIGKILL) }; + let _ = child.kill(); + gone = said.ended(); + } + let left = if gone { "" } else { "; a process outside that group still held its output after it" }; + format!("said nothing for {QUIET:?} and was ended with its process group by {by}{left}; the last it said: {last}") + }); + // Before the child is reaped and its pid is anybody's: a handler that + // read the group before this is seen here, and the driver ends unreaped. + started(0); + let status = child.wait().map_err(|e| format!("wait: {e}"))?; + match ended { + Some(refusal) => Err(refusal), + None => Ok((status, said.text())), + } } // --- The host jobs ------------------------------------------------------------- @@ -507,8 +697,8 @@ fn run_control(root: &Path, control: &Control) -> Result { if !control.must_red { args.push("--nocapture"); } - let (green, log) = cargo_logged(root, &args)?; - judge_control(control, green, &log) + let (status, log) = cargo_logged(root, &args)?; + judge_control(control, status.success(), &log) } /// Every test that runs on the host and boots no guest. The build system's own @@ -797,9 +987,9 @@ fn guest(root: &Path, suite: &[String]) -> Vec { if steps.iter().all(|s| s.verdict.is_ok()) { steps.push(step("the suite", || { let args: Vec<&str> = suite.iter().map(String::as_str).collect(); - let (green, log) = cargo_logged(root, &args)?; + let (status, log) = cargo_logged(root, &args)?; let said = verdicts(&log); - if green { + if status.success() { Ok(said) } else { Err(said) @@ -1112,6 +1302,124 @@ mod tests { assert!(out.status.success() && said.contains("Fresh one v0.1.0"), "{said}"); } + /// What [`a_hung_step`] says last. + const LAST: &str = "the hung step's last line"; + + /// A step that goes quiet is a red naming the last line it said, and what + /// it spawned is ended with it: the step here is this binary, which starts + /// a second and then says nothing, as cargo does with a test that hangs. + /// The driver is a process of its own, because it reads the end of a pipe, + /// which on macOS a process another thread spawns meanwhile can hold open. + #[test] + fn a_step_that_goes_quiet_is_ended_with_what_it_spawned_and_names_its_last_line() { + let (_read_by_nobody, hung_on) = std::io::pipe().unwrap(); + let out = crate::buildlock::tests::rerun("ci::tests::a_hung_steps_driver") + .env(FIXTURE, "hung") + .stdin(hung_on) + .output() + .expect("run the driver"); + let said = String::from_utf8_lossy(&out.stdout).into_owned() + &String::from_utf8_lossy(&out.stderr); + assert!(out.status.success() && said.contains("test result: ok. 1 passed"), "{said}"); + } + + /// Its stdin is the step's, and what the step spawns'. + #[test] + #[ignore = "the driver of the tests around it; never runs on its own"] + fn a_hung_steps_driver() { + let hung = crate::buildlock::tests::rerun("ci::tests::a_hung_step"); + let refusal = heard(hung).expect_err("a step that hangs is a red"); + assert!(refusal.ends_with(LAST) && refusal.contains("by SIGQUIT") && !refusal.contains("outside"), "{refusal}"); + } + + /// An interrupt of the driver ends the group its step runs in. The judge + /// is a process of its own, for the reason above. + #[test] + fn an_interrupt_of_the_driver_ends_the_group_its_step_runs_in() { + let out = crate::buildlock::tests::rerun("ci::tests::an_interrupted_drivers_judge") + .env(FIXTURE, "hung") + .output() + .expect("run the judge"); + let said = String::from_utf8_lossy(&out.stdout).into_owned() + &String::from_utf8_lossy(&out.stderr); + assert!(out.status.success() && said.contains("test result: ok. 1 passed"), "{said}"); + } + + #[test] + #[ignore = "the judge of the test above; never runs on its own"] + fn an_interrupted_drivers_judge() { + use std::io::BufRead; + use std::os::fd::AsRawFd; + use std::os::unix::process::ExitStatusExt; + let (mut hung_on, stdin) = std::io::pipe().unwrap(); + let mut driver = crate::buildlock::tests::rerun("ci::tests::a_hung_steps_driver"); + driver.stdin(stdin).stderr(std::process::Stdio::piped()); + let mut child = driver.spawn().expect("spawn the driver"); + drop(driver); + // The driver passes on what its step says, and ends a step that never + // says it: this read ends either way. + let mut said = std::io::BufReader::new(child.stderr.take().expect("a piped stderr")).lines(); + assert!(said.by_ref().map_while(Result::ok).any(|line| line.contains(LAST)), "the step never said its last line"); + // `SIGTERM`: a shell may have started all of this ignoring `SIGINT`. + // SAFETY: a signal to a child this test has not reaped. + assert_eq!(unsafe { libc::kill(child.id() as i32, libc::SIGTERM) }, 0); + let status = child.wait().expect("reap the driver"); + assert_eq!(status.signal(), Some(libc::SIGTERM), "{status}"); + // The step and what it spawned write into this pipe for as long as + // they live: its end is both of them gone. + let deadline = Instant::now() + GONE; + let mut chunk = [0; 4096]; + loop { + let left = deadline.saturating_duration_since(Instant::now()); + assert!(!left.is_zero(), "the step's group outlived its driver by {GONE:?}"); + let mut ready = libc::pollfd { fd: hung_on.as_raw_fd(), events: libc::POLLIN, revents: 0 }; + // SAFETY: one descriptor, which this test owns. + assert!(unsafe { libc::poll(&mut ready, 1, left.as_millis() as libc::c_int) } >= 0); + if ready.revents != 0 && hung_on.read(&mut chunk).expect("read the pipe") == 0 { + break; + } + } + } + + /// The fixtures' own end at `SIGQUIT`, whose default leaves a host's + /// report of the process behind at every run of these tests. + extern "C" fn quit(_: libc::c_int) { + // SAFETY: async-signal-safe. + unsafe { libc::_exit(0) } + } + + /// Write into stdin, the end of a pipe the driver's parent holds the other + /// end of: blocked once it is full, and over when nobody reads it. + fn hang() { + assert!(std::env::var_os(FIXTURE).is_some(), "run without {FIXTURE}"); + let full = [0u8; 4096]; + // SAFETY: a write of a buffer that outlives the call. + while unsafe { libc::write(0, full.as_ptr().cast(), full.len()) } > 0 {} + } + + #[test] + #[ignore = "the step of the driver above; never runs on its own"] + fn a_hung_step() { + // SAFETY: `quit` makes one async-signal-safe call. + assert!(unsafe { libc::signal(libc::SIGQUIT, quit as *const () as libc::sighandler_t) } != libc::SIG_ERR); + // libtest's own lines go nowhere, so `LAST` is the last line said; its + // stderr is the step's, which it holds open. + let mut spawned = crate::buildlock::tests::rerun("ci::tests::a_hung_steps_child") + .stdout(std::process::Stdio::null()) + .spawn() + .expect("spawn what the step spawns"); + eprintln!("{LAST}"); + hang(); + // Reached only where nothing ended the step: its child's pipe ended with its own. + spawned.wait().expect("reap what the step spawned"); + } + + #[test] + #[ignore = "what the step above spawns; never runs on its own"] + fn a_hung_steps_child() { + // SAFETY: as in the step. + assert!(unsafe { libc::signal(libc::SIGQUIT, quit as *const () as libc::sighandler_t) } != libc::SIG_ERR); + hang(); + } + fn repo_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) }