diff --git a/.claude/agents/reviewer.md b/.claude/agents/reviewer.md index 818c3a24104..6d1883b3d70 100644 --- a/.claude/agents/reviewer.md +++ b/.claude/agents/reviewer.md @@ -76,10 +76,10 @@ if it meets the bar above; otherwise it is a NOTE. saying what it is; a new cargo feature or `cfg` arm of one, or an arm a changed `src/clippy.rs` shape stops building, that no shape in `src/clippy.rs` lints; an `issues/` file added, changed or deleted against `issues/README.md`. -- **Caches.** No gate reads these; a diff that breaks one is a BLOCKER. Each cache has one - writer, a nightly.yml job, and no workflow uses the combined `actions/cache`, which saves too; - the host cache's is nightly's `host`, and its one reader ci.yml's `host`, on the same - `runs-on`, both caching `src/cicache.rs`'s `PATHS` with its `DRIVER` as their +- **Caches.** No gate reads these; a diff that breaks one is a BLOCKER. No workflow uses the + combined `actions/cache`, which saves too. The host cache has one writer, nightly's `host`, + and one reader, ci.yml's `host`, on the same `runs-on`, both caching `src/cicache.rs`'s + `PATHS` with its `DRIVER` as their `CARGO_TARGET_DIR`, the one variable an `env:` gives either: an `ImageOS` or `ImageVersion` set there outlives an image move. The reader's `restore-keys` is the writer's `key` up to its run id. A job that names the host cache runs `actions/checkout`, its cache step and @@ -89,6 +89,13 @@ if it meets the bar above; otherwise it is a NOTE. skips only a draft, and nightly's `host` has no `if:` of its own, a skipped job being a green check. No `continue-on-error`, `shell:`, `defaults:` or cargo `runner` reaches them. nightly.yml's `on:` is one daily `schedule` and `workflow_dispatch`. +- **Workflows.** A BLOCKER each: + - Only main's runs save a cache entry other refs restore: GitHub's cache scoping is every entry's provenance. + - `nightly.yml`'s `release` is the only job granted `contents: write`. + - No workflow runs on `pull_request_target`, `workflow_run`, `issue_comment` or any other trigger that runs code other than main's on main's ref. + - No workflow or job declares `cache-mode: write` or `write-only`. + - `guest / suite` has no job-level `if:`, and a job that calls it runs whatever `toolchain` concluded: a skipped required check reads as green. + - A job that saves a cache entry runs only `cargo run -- --ci `. - **Growth.** Every line is a responsibility, not an asset. State the branch's net lines (`git diff --shortstat origin/main...`), production and tests apart. Production code that grows needs a reason you accept; a branch that could delete more than it adds and does not goes diff --git a/.github/qemu-version b/.github/qemu-version index bece0dd15d2..b4fc096a925 100644 --- a/.github/qemu-version +++ b/.github/qemu-version @@ -3,6 +3,6 @@ # build system's prerequisite check. The version decides test outcomes — 8.2.2 # and 11.0.3 were measured disagreeing about the same tree — so this is a # declaration the guest image is held to, never a fact read off it. The image -# digest in nightly.yml is chosen to satisfy this line; changing either is a -# deliberate act: it says the instrument moved. +# digest is chosen to satisfy this line; changing either is a deliberate act: +# it says the instrument moved. 11.1.1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d45466c6a9..bb446819456 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,5 @@ name: ci -# A pull request and the merge queue: the host tests, and no guest. -# nightly.yml boots the guests. - on: pull_request: branches: [main] @@ -13,6 +10,10 @@ concurrency: group: ci-${{ github.ref }} cancel-in-progress: true +# No job here writes: a toolchain release is main's publisher's alone. +permissions: + contents: read + jobs: host: if: github.event_name == 'merge_group' || github.event.pull_request.draft == false @@ -42,3 +43,18 @@ jobs: restore-keys: host-sealed-${{ runner.os }}-${{ runner.arch }}- - run: cargo run -- --ci host + + toolchain: + if: github.event_name == 'merge_group' || github.event.pull_request.draft == false + uses: ./.github/workflows/toolchain.yml + + # A required check that is skipped reads as green, so this runs whatever + # `toolchain` concluded. + guest: + needs: toolchain + if: ${{ !cancelled() && (github.event_name == 'merge_group' || github.event.pull_request.draft == false) }} + uses: ./.github/workflows/guest.yml + with: + kvm: true + sysroot-key: ${{ needs.toolchain.outputs.sysroot-key }} + sysroot-path: ${{ needs.toolchain.outputs.sysroot-path }} diff --git a/.github/workflows/guest.yml b/.github/workflows/guest.yml new file mode 100644 index 00000000000..1eb8c1cb7a5 --- /dev/null +++ b/.github/workflows/guest.yml @@ -0,0 +1,74 @@ +name: guest + +on: + workflow_call: + inputs: + # Without it every guest is emulated: the only lane that decodes the paths + # a KVM host's CPU never does. + kvm: + type: boolean + required: true + sysroot-key: + type: string + required: true + sysroot-path: + type: string + required: true + +jobs: + suite: + runs-on: ubuntu-24.04 + # A wedge guard, not a budget. + timeout-minutes: 60 + # The digest is the instrument's one pin: a dated image names the snapshot + # archive it was built from, and `deps` installs QEMU from that archive. + # The node ships `crw-rw---- root:kvm` and root opens it. + container: + image: debian:sid-20260918@sha256:a2aa46262453eba3f464d8b1c7a8c31db85eb15af180ae34dd400615d7208547 + options: ${{ inputs.kvm && '--device=/dev/kvm' || '' }} + steps: + # Before the checkout, which wants git. Three attempts, because the + # archive is fixed and the network to it is not. + - name: deps + run: | + snap=$(sed -n 's|^# \(http://snapshot\.debian\.org/archive/debian/[0-9]*T[0-9]*Z\)$|\1|p' \ + /etc/apt/sources.list.d/debian.sources) + test -n "$snap" + echo "deb $snap sid main" > /etc/apt/sources.list + rm /etc/apt/sources.list.d/debian.sources + # `zstd`: what the sysroot's restore unpacks the toolchain job's entry with. + for attempt in 1 2 3; do + apt-get -o Acquire::Check-Valid-Until=false update -qq > /tmp/apt.log 2>&1 \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git curl ca-certificates \ + zstd build-essential qemu-system-x86 ovmf-generic qemu-system-arm \ + qemu-efi-aarch64 >> /tmp/apt.log 2>&1 \ + && break + [ "$attempt" = 3 ] && { cat /tmp/apt.log; exit 1; } + sleep 20 + done + # `actions/checkout` sets this only in a config it discards. + git config --global --add safe.directory "$GITHUB_WORKSPACE" + curl --proto '=https' --tlsv1.2 -sSf -o "$RUNNER_TEMP/rustup-init.sh" https://sh.rustup.rs + sh "$RUNNER_TEMP/rustup-init.sh" -y --profile minimal --default-toolchain stable + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ inputs.sysroot-path }} + key: ${{ inputs.sysroot-key }} + fail-on-cache-miss: true + + - run: cargo run -- --ci guest + + # Every boot's 16550 log: what a guest that died early still leaves. + - name: serial logs + if: failure() + continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: serial-${{ github.job }} + path: target/red-run-serial/**/uart-*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6d93ba85ac1..b26634d85f0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,7 +1,5 @@ name: nightly -# Everything that boots a guest, the host gate again to write the cache the -# merge queue restores, and portability. # Every job's logic is `cargo run -- --ci ` (src/ci.rs); this file says # where each one runs. @@ -10,11 +8,13 @@ on: - cron: '0 3 * * *' workflow_dispatch: -# Never cancelled: `build` may be an hour into a bootstrap. concurrency: group: nightly-${{ github.ref }} cancel-in-progress: false +permissions: + contents: read + jobs: # The host cache's writer restores nothing, and `seal` is green only once # src/cicache.rs has sealed the tree the save stores. @@ -48,126 +48,38 @@ jobs: bootloader/target key: host-sealed-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }} - # Publishes this tree's toolchain if nobody has, and on main moves the SDK - # alias onto it. Bare `ubuntu-24.04`, not a container: its glibc is the - # release's floor. - build: + toolchain: + uses: ./.github/workflows/toolchain.yml + + # Skipped off main, where the driver refuses it by name. + release: + needs: toolchain + if: github.ref == 'refs/heads/main' runs-on: ubuntu-24.04 - timeout-minutes: 350 + timeout-minutes: 30 permissions: contents: write steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - with: - fetch-depth: 0 - - name: disk, QEMU and CMake - run: | - sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \ - /usr/local/share/boost /usr/local/.ghcup - sudo apt-get update -qq - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq qemu-system-x86 zstd \ - cmake=3.28.3-1build7 + - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ needs.toolchain.outputs.sysroot-path }} + key: ${{ needs.toolchain.outputs.sysroot-key }} + fail-on-cache-miss: true - env: GH_TOKEN: ${{ github.token }} - run: cargo run -- --ci toolchain + run: cargo run -- --ci release - guest: - needs: build - runs-on: ubuntu-24.04 - # A wedge guard, not a budget. - timeout-minutes: 60 - # The digest is the instrument's one pin: a dated image names the snapshot - # archive it was built from, and `deps` installs QEMU from that archive. - # The node ships `crw-rw---- root:kvm` and root opens it. - container: &kvm - image: &image debian:sid-20260918@sha256:a2aa46262453eba3f464d8b1c7a8c31db85eb15af180ae34dd400615d7208547 - options: --device=/dev/kvm - env: - GH_TOKEN: ${{ github.token }} - steps: - # Before the checkout, which wants git. Three attempts, because the - # archive is fixed and the network to it is not. - - &deps - name: deps - run: | - snap=$(sed -n 's|^# \(http://snapshot\.debian\.org/archive/debian/[0-9]*T[0-9]*Z\)$|\1|p' \ - /etc/apt/sources.list.d/debian.sources) - test -n "$snap" - echo "deb $snap sid main" > /etc/apt/sources.list - rm /etc/apt/sources.list.d/debian.sources - for attempt in 1 2 3; do - apt-get -o Acquire::Check-Valid-Until=false update -qq > /tmp/apt.log 2>&1 \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git curl ca-certificates \ - zstd build-essential qemu-system-x86 ovmf-generic qemu-system-arm \ - qemu-efi-aarch64 >> /tmp/apt.log 2>&1 \ - && break - [ "$attempt" = 3 ] && { cat /tmp/apt.log; exit 1; } - sleep 20 - done - # `actions/checkout` sets this only in a config it discards. - git config --global --add safe.directory "$GITHUB_WORKSPACE" - curl --proto '=https' --tlsv1.2 -sSf -o "$RUNNER_TEMP/rustup-init.sh" https://sh.rustup.rs - sh "$RUNNER_TEMP/rustup-init.sh" -y --profile minimal --default-toolchain stable - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - - &checkout - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - - &guest-cache - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: &guest-paths | - ~/.cargo/registry/index - ~/.cargo/registry/cache - ~/.cargo/git/db - target - kernel/target - bootloader/target - userland/target - tests/target - tests/toyos-rust-tests/*/target - key: guest-${{ github.run_id }} - restore-keys: guest- - - - run: cargo run -- --ci guest - - # Every boot's 16550 log: what a guest that died early still leaves. - - &serial - name: serial logs - if: failure() - continue-on-error: true - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: serial-${{ github.job }} - path: target/red-run-serial/**/uart-*.log - if-no-files-found: warn - retention-days: 7 - - # The guest suite again with no `/dev/kvm`, so the only lane that decodes the - # paths a KVM host's CPU never does; and the guest cache's one writer, since - # what it builds does not depend on the accelerator. tcg: - needs: build - runs-on: ubuntu-24.04 - timeout-minutes: 60 - container: - image: *image - env: - GH_TOKEN: ${{ github.token }} - steps: - - *deps - - *checkout - - *guest-cache - - run: cargo run -- --ci guest - # After the test: what is worth keeping is a tree that built and booted. - - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: *guest-paths - key: guest-${{ github.run_id }} - - *serial + needs: toolchain + if: ${{ !cancelled() }} + uses: ./.github/workflows/guest.yml + with: + kvm: false + sysroot-key: ${{ needs.toolchain.outputs.sysroot-key }} + sysroot-path: ${{ needs.toolchain.outputs.sysroot-path }} # `cargo run -- --build-only` from a fresh machine. `sid` as it stands, # image and archive both, and no cache — a fresh machine is the premise. @@ -192,9 +104,10 @@ jobs: sh "$RUNNER_TEMP/rustup-init.sh" -y --profile minimal --default-toolchain stable echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - *checkout + - &checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - run: env -u GITHUB_ACTIONS -u CI cargo run -- --build-only + - run: cargo run -- --build-only portability-macos: runs-on: macos-latest @@ -208,6 +121,6 @@ jobs: curl --proto '=https' --tlsv1.2 -sSf -o "$RUNNER_TEMP/rustup-init.sh" https://sh.rustup.rs sh "$RUNNER_TEMP/rustup-init.sh" -y --profile minimal --default-toolchain stable echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - run: env -u GITHUB_ACTIONS -u CI cargo run -- --build-only + - run: cargo run -- --build-only # The host suite's macOS arms run here alone: `ci.yml`'s `host` is Linux. - run: cargo run -- --ci host diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1c2e8f9e066..19ba55c7f17 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,12 +9,6 @@ on: branches: [main] workflow_dispatch: {} -# Never two publishers, and never cancel one: a cancelled `cargo publish` may -# have already taken the version. -concurrency: - group: publish - cancel-in-progress: false - permissions: contents: read # crates.io trusted publishing: the job trades its OIDC token for a @@ -25,6 +19,11 @@ jobs: publish: runs-on: ubuntu-latest timeout-minutes: 30 + # Never two publishers, and never cancel one: a cancelled `cargo publish` + # may have already taken the version. + concurrency: + group: publish + cancel-in-progress: false steps: # No submodules: `cargo publish` walks the repository's vcs state, and an # initialised but empty `rust/` breaks that walk. @@ -36,3 +35,8 @@ jobs: - env: CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} run: cargo run -- --ci publish + + toolchain: + permissions: + contents: read + uses: ./.github/workflows/toolchain.yml diff --git a/.github/workflows/toolchain.yml b/.github/workflows/toolchain.yml new file mode 100644 index 00000000000..22ba02bfbd9 --- /dev/null +++ b/.github/workflows/toolchain.yml @@ -0,0 +1,77 @@ +name: toolchain + +on: + workflow_call: + outputs: + sysroot-key: + value: ${{ jobs.build.outputs.sysroot-key }} + sysroot-path: + value: ${{ jobs.build.outputs.sysroot-path }} + +jobs: + build: + # Bare, not a container: its glibc is a build's floor. + runs-on: ubuntu-24.04 + timeout-minutes: 350 + # Main's two callers build one key once, one after the other, and one that + # waits is never cancelled for a later one. + concurrency: + group: toolchain-${{ github.ref }} + queue: max + outputs: + sysroot-key: ${{ steps.keys.outputs.sysroot-key }} + sysroot-path: ${{ steps.keys.outputs.sysroot-path }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - id: keys + run: cargo run -- --ci toolchain + + - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.keys.outputs.llvm-path }} + key: ${{ steps.keys.outputs.llvm-key }} + + - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.keys.outputs.compiler-path }} + key: ${{ steps.keys.outputs.compiler-key }} + + - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.keys.outputs.freestanding-path }} + key: ${{ steps.keys.outputs.freestanding-key }} + + - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.keys.outputs.sysroot-path }} + key: ${{ steps.keys.outputs.sysroot-key }} + + - id: build + run: cargo run -- --ci bootstrap + + # A merge group's scope is read only by its own guest job, which restores + # the sysroot alone. + - if: steps.build.outputs.llvm == 'built' && github.event_name != 'merge_group' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.keys.outputs.llvm-path }} + key: ${{ steps.keys.outputs.llvm-key }} + + - if: steps.build.outputs.compiler == 'built' && github.event_name != 'merge_group' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.keys.outputs.compiler-path }} + key: ${{ steps.keys.outputs.compiler-key }} + + - if: steps.build.outputs.freestanding == 'built' && github.event_name != 'merge_group' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.keys.outputs.freestanding-path }} + key: ${{ steps.keys.outputs.freestanding-key }} + + - if: steps.build.outputs.sysroot == 'built' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.keys.outputs.sysroot-path }} + key: ${{ steps.keys.outputs.sysroot-key }} diff --git a/CLAUDE.md b/CLAUDE.md index 91220c75994..4c8de665767 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,7 +64,7 @@ Vendor firmware a device or CPU verifies by its maker's signature may be shipped ## Build & test -- `cargo run` builds everything (toolchain, kernel, bootloader, userland, image) and launches QEMU; `--build-only` skips the launch. `cargo test` runs the QEMU harness, and `cargo test --test toyos-build -- --metal` the T14's. `cargo run -- --ci host` runs every host suite: it is the `host` check a ready pull request and the merge queue run. Guests run nightly. +- `cargo run` builds everything (toolchain, kernel, bootloader, userland, image) and launches QEMU; `--build-only` skips the launch. `cargo test` runs the QEMU harness, and `cargo test --test toyos-build -- --metal` the T14's. `cargo run -- --ci host` runs every host suite: it is the `host` check a ready pull request and the merge queue run. The guest suite is their `guest / suite` check, under KVM; the nightly runs it under TCG. - **A behaviour is tested on the cheapest tier that reaches it**: a type that makes the bug unrepresentable, then a host test, then a metal row on the T14, and a QEMU guest test last. - **Agents run their own guest tests**, the whole suite or a filter, side by side. **The T14 is the orchestrator's alone: no other agent runs `--metal` without `--metal-readback`, which touches no machine.** - **Timing and audio verdicts come only from metal.** A QEMU test asserts order, completion, content and counts, never how long something took, and plays no audio; its only clock is a hang ceiling. diff --git a/Cargo.lock b/Cargo.lock index c3d71e2f71e..f2824e05b5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,6 +50,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bcachefs" version = "0.1.0" @@ -111,6 +117,12 @@ dependencies = [ "byteorder", ] +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.2.56" @@ -290,6 +302,16 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -353,6 +375,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -416,6 +449,22 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -573,7 +622,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -614,6 +663,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -721,6 +776,20 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-std-workspace-core" version = "1.0.1" @@ -736,6 +805,41 @@ dependencies = [ "semver", ] +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -878,6 +982,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -1012,6 +1126,7 @@ version = "0.1.0" dependencies = [ "bcachefs", "fatfs", + "flate2", "fontdue", "getrandom 0.3.4", "gpt", @@ -1020,6 +1135,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "tar", "toml", "toyos-abi", "toyos-blackbox", @@ -1041,6 +1157,7 @@ dependencies = [ "toyos-userbound", "toyos-wallclock", "toyos-xhci", + "ureq", "uuid", ] @@ -1411,6 +1528,46 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" +dependencies = [ + "base64", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "uuid" version = "1.22.0" @@ -1434,6 +1591,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -1531,6 +1694,15 @@ dependencies = [ "semver", ] +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -1590,6 +1762,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1599,6 +1780,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.7.15" @@ -1716,6 +1961,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 682e057f417..0b1b3ce1363 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,6 +165,15 @@ image = { version = "0.25", default-features = false, features = ["jpeg"] } # The digest behind `NOTICE`'s record of every committed binary file # (`src/sourcegate.rs`). sha2 = "0.10" +# The toolchain release's tarball, packed in-process (`src/release.rs`): Rust's +# own tar and gzip, where the binaries are hosts' tools. +tar = { version = "0.4.46", default-features = false } +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +# Every request the build system makes, GitHub's API and the crates.io index, +# where curl is a host's tool: rustls on ring, the one TLS provider the tree +# takes (`issues/design-debt/the-internet-clients-work-unchanged.md`), whose C +# and assembly the host's `cc` compiles. +ureq = { version = "3", default-features = false, features = ["rustls"] } [dev-dependencies] # `USER_TOP`, so the harness judges a held `rsp` against the bound the kernel diff --git a/issues/build/a-nightly-a-landing-overtakes-leaves-no-release-and-no-sdk-alias.md b/issues/build/a-nightly-a-landing-overtakes-leaves-no-release-and-no-sdk-alias.md new file mode 100644 index 00000000000..4aadb23c802 --- /dev/null +++ b/issues/build/a-nightly-a-landing-overtakes-leaves-no-release-and-no-sdk-alias.md @@ -0,0 +1,33 @@ +--- +status: open +kind: tooling +opened: 2026-10-02 +--- + +# A nightly a landing overtakes leaves no release and no SDK alias + +The nightly's `release` puts nothing up where main's tip is no longer its +HEAD (`release::sdk_at_tip`), and is green: "main has moved past ``, and +its tip's nightly is the one that publishes". The toolchain release and the +`sdk-` alias then wait for a nightly whose `release` runs at the tip, +or a dispatch there. No landing runs one, the job's conclusion is success +either way, and a nightly overtaken every night publishes nothing until one is +not. + +The window runs from the nightly's creation to its `release`'s decision: the +wait in `nightly-`'s concurrency group, then the `toolchain` job, 2:38 at +its fastest measured (run 36988764929, attempt 2) and 2:11:46 cold (run +36934214557). `cron: '0 3 * * *'` created its scheduled runs at 09:35:54Z on +2026-10-01 (36843762360) and 09:09:27Z on 2026-10-02 (36988155706), and main +took seven landings on 2026-10-02, pushed between 08:39:27Z and 11:20:48Z. +Run 36988155706, at `46af79d5d`, waited behind a dispatched nightly +(36985427800) and started its jobs at 11:45:51Z, four landings after its HEAD +(#643, #664, #679, #659). The release this decision replaced read no tip: a +landing that moved no SDK crate did not stop it. + +Owner: the release module (`src/release.rs`) and `nightly.yml`'s `release`. + +**Exit**: a landing during a nightly does not leave main's tip without its +toolchain release and SDK alias: after a day on which every nightly was +overtaken, `releases/tags/toolchain-linux-x86_64-` +answers 200 and the tip's `sdk-` alias names it. diff --git a/issues/build/a-release-that-decides-before-its-tips-crates-are-up-reds-the-nightly.md b/issues/build/a-release-that-decides-before-its-tips-crates-are-up-reds-the-nightly.md new file mode 100644 index 00000000000..0a74b5c0094 --- /dev/null +++ b/issues/build/a-release-that-decides-before-its-tips-crates-are-up-reds-the-nightly.md @@ -0,0 +1,32 @@ +--- +status: open +kind: tooling +opened: 2026-10-02 +--- + +# A release that decides before its tip's crates are up reds the nightly + +The nightly's `release` puts a toolchain up only where crates.io's newest SDK +crates are the tree's own (`release::sdk_at_tip`). Where HEAD is main's tip +and they are not, it refuses: "crates.io holds no `` of this tree, +main's tip, so no sdk alias can name it". That refusal does not tell a tip +`publish.yml` failed to publish from one whose `publish` run has not finished, +and on the second the nightly is red for no defect of the tree. Nothing is put +up, and a re-run of that one job after the publish recovers it. + +The decision follows the nightly's `toolchain` job, 2:38 at its fastest +measured (run 36988764929, attempt 2), and the `release` job's checkout, +restore and driver build. Main's six green `publish` runs of 2026-10-02 took +between 1:04 and 1:54 from creation (36985281365, 36986108744, 36986326239, +36991892914, 36995618637, 37000495781). `publish` runs one at a time +(`publish.yml`'s `concurrency`), and after each crate it reads the index up to +60 times, 5 s apart (`ci::publish`). A publish that queues or waits longer +than that margin, behind a landing made just before the nightly, is read as a +missing one. + +Owner: the release module (`src/release.rs`). + +**Exit**: a `release` at main's tip is red for crates crates.io does not hold +only once that tip's `publish` run has concluded: a test stages a tip whose +crates come up after the release's first read of crates.io, and sees the +release put up. diff --git a/issues/build/a-runner-image-that-moves-cc-or-cmake-makes-every-toolchain-job-cold.md b/issues/build/a-runner-image-that-moves-cc-or-cmake-makes-every-toolchain-job-cold.md new file mode 100644 index 00000000000..1edfc3196c3 --- /dev/null +++ b/issues/build/a-runner-image-that-moves-cc-or-cmake-makes-every-toolchain-job-cold.md @@ -0,0 +1,26 @@ +--- +status: open +kind: tooling +opened: 2026-10-02 +--- + +# A runner image that moves cc, c++ or CMake makes every toolchain job cold + +`llvm::key` reads the `cc`, `c++` and CMake that build an LLVM, each by its +resolved path and all its `--version` says (`src/llvm.rs`), and +`toolchain.yml`'s `build` takes all three from `ubuntu-24.04`'s image +(20260927.320.1 in job 110610545360). An image that moves one moves every LLVM +key, and with it every compiler, freestanding and sysroot key. Every pull +request and merge group then builds all four layers until a run on main has +saved the new keys: a pull request's saves reach no other ref, and a merge +group saves only its sysroot. + +That build is the cold path: 153:17 from the run's creation to `guest / suite` +green in run 36934214557, 2:09:01 of it `--ci bootstrap`, against the merge +queue's `check_response_timeout_minutes`. + +Owner: the toolchain job (`.github/workflows/toolchain.yml`). + +**Exit**: the tools an LLVM's key reads move only with a commit to this +repository, and a toolchain job on a runner image newer than that commit keys +the LLVM as the one before it did. diff --git a/issues/build/a-stores-build-code-moves-its-key-only-through-a-recipe-bumped-by-hand.md b/issues/build/a-stores-build-code-moves-its-key-only-through-a-recipe-bumped-by-hand.md new file mode 100644 index 00000000000..2f197828029 --- /dev/null +++ b/issues/build/a-stores-build-code-moves-its-key-only-through-a-recipe-bumped-by-hand.md @@ -0,0 +1,26 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# A store's build code moves its key only through a RECIPE bumped by hand + +A keyed store's key (`src/keystore.rs`) reads its sources, the configuration +its build is given, the tools that run that build and the keys of the stores it +reads. The code that builds it — `src/llvm.rs`, `src/compiler.rs`, +`src/toolchain.rs`, `src/sysroot.rs`, `src/libc.rs`, `src/libcxx.rs` and +`src/clang.rs` — reaches the key only through the store's `RECIPE`, which moves +only when the change's author edits it. + +A change that does not keeps the old key. `toolchain.yml` then restores main's +entry under it on every pull request and on main, and no run can save over that +entry, so CI builds with a store the tree's code no longer makes. At +`src/libc.rs`, `.env_remove("RUSTFLAGS")` made +`.env("RUSTFLAGS", "-Coverflow-checks=on")` leaves `sysroot::key` where it was, +and no test reds. + +Owner: the keyed stores (`src/keystore.rs`). + +**Exit**: a change to the code that builds a keyed store moves that store's key +with no hand edit, and a test reds on the `src/libc.rs` patch above. diff --git a/issues/build/a-toolchain-job-that-does-not-finish-saves-no-layer-it-built.md b/issues/build/a-toolchain-job-that-does-not-finish-saves-no-layer-it-built.md new file mode 100644 index 00000000000..2121cf4038d --- /dev/null +++ b/issues/build/a-toolchain-job-that-does-not-finish-saves-no-layer-it-built.md @@ -0,0 +1,26 @@ +--- +status: open +kind: tooling +opened: 2026-10-02 +--- + +# A toolchain job that does not finish saves no layer it built + +`toolchain.yml`'s `build` makes every layer its restores missed in one step, +`cargo run -- --ci bootstrap`, and its four saves follow that step. A job that +ends inside it has saved nothing, whatever it had made: cancelled by a push to +its pull request, which cancels `ci.yml`'s run in progress, or red in the step +after its LLVM was placed. + +Run 36913380100's `toolchain / build` (job 110541308331) ran 2:08:24, built +all four layers and reded in that step's last check, so its saves were skipped. +The next run, 36934214557, +missed all four restores and built the LLVM again, 1:29:15 of its 2:09:01 +`--ci bootstrap` (job 110610545360). + +Owner: the toolchain job (`.github/workflows/toolchain.yml`, +`release::bootstrap`). + +**Exit**: each layer is saved by the step after the one that made it, and a +toolchain job stopped after its LLVM was placed leaves an entry the next run +restores. diff --git a/issues/build/a-toolchain-releases-asset-is-whatever-its-last-writer-put-there.md b/issues/build/a-toolchain-releases-asset-is-whatever-its-last-writer-put-there.md new file mode 100644 index 00000000000..eb6fe43cc07 --- /dev/null +++ b/issues/build/a-toolchain-releases-asset-is-whatever-its-last-writer-put-there.md @@ -0,0 +1,29 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# A toolchain release's asset is whatever its last writer put there + +The release that main's publisher puts up is what a consumer outside CI +installs, and so is the SDK alias that names it. The release notes' install +steps take the asset as it is served. + +A branch's workflows decide their own token's permissions. A workflow on any +branch of this repository can ask for `contents: write` and then replace that +asset, or create the release for a tag main has not yet published. A pull +request's `toolchain` job held such a token in run 36863809437; its log reads +`Contents: write`. Two nightly runs dispatched on branches built and published +their branches' toolchains under the nightly's write token: +`wt/toyos-castore` published `toolchain-linux-x86_64-688e609acf5a65c4` (run +36709239346), and `wt/toyos-notiers` published +`toolchain-linux-x86_64-92f146618d6687d8` (run 36600425263). Every toolchain release is mutable (`immutable: +false`). + +Owner: the release module (`src/release.rs`). + +**Exit**: a consumer outside CI installs only the bytes main's publisher +recorded, and its install refuses any other bytes by name. That holds when a +published release can no longer change (with the SDK alias made a new release +per move, never one moved). diff --git a/issues/build/no-nightly-runner-has-had-its-cpuid-and-vulnerability-lines-captured.md b/issues/build/no-kvm-runner-has-had-its-cpuid-and-vulnerability-lines-captured.md similarity index 77% rename from issues/build/no-nightly-runner-has-had-its-cpuid-and-vulnerability-lines-captured.md rename to issues/build/no-kvm-runner-has-had-its-cpuid-and-vulnerability-lines-captured.md index 426e1f94360..79a5bfb9181 100644 --- a/issues/build/no-nightly-runner-has-had-its-cpuid-and-vulnerability-lines-captured.md +++ b/issues/build/no-kvm-runner-has-had-its-cpuid-and-vulnerability-lines-captured.md @@ -4,7 +4,7 @@ kind: tooling opened: 2026-09-29 --- -# No nightly runner has had its CPUID and vulnerability lines captured +# No KVM runner has had its CPUID and vulnerability lines captured `toyos-cpuvuln`'s `fixtures/awaiting-capture/` holds the lines this crate reads out of the pinned Linux for an EPYC Milan and an EPYC Turin as KVM @@ -14,8 +14,8 @@ host's microcode gives `IBPB_BRTYPE` are guesses there. In nightly run 36496779560 the 14 KVM shards ran on EPYC 7763 (11), 9V74 (2) and 9V45 (1), each as `-cpu host,+rdrand,+smap,+fsgsbase,+x2apic,+smep` with `-smp cores=N`. -**Exit**: a nightly step boots the pinned Linux under that command line on -the runner and records CPUID leaves 0, 1, 7.0, 7.2, 0x80000000, 0x80000008 and +**Exit**: a CI step given `/dev/kvm` boots the pinned Linux under that command +line on its runner and records CPUID leaves 0, 1, 7.0, 7.2, 0x80000000, 0x80000008 and 0x80000021, MSR 0x10A where enumerated, the microcode revision, and `grep .` over `/sys/devices/system/cpu/vulnerabilities/`; the two fixtures are replaced by what it records, and one is added per CPU model it meets. diff --git a/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md b/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md index 2db25ab2343..70e7f15513b 100644 --- a/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md +++ b/issues/build/the-build-runs-host-tools-outside-rust-and-qemu.md @@ -17,19 +17,16 @@ arrives and is not one. M4 and M5 are stages of `issues/build/toyos-builds-itsel | Python | LLVM's CMake, whenever this host builds an LLVM (`src/llvm.rs`); the C++ runtime's CMake, in every sysroot build (`src/libcxx.rs`), and its build, which runs `libcxx/utils/generate_iwyu_mapping.py` for a header it installs | admitted: no Rust tool does the job, LLVM's CMake requires one (`find_package(Python3 … REQUIRED)` in `rust/src/llvm-project/llvm/CMakeLists.txt`), and so does the runtimes', unconditionally (`runtimes/CMakeLists.txt`): configured as `src/libcxx.rs` does with no Python on `PATH` or in CMake's search, it found none and stopped, exit 1, and with only `python3` added it configured, exit 0 | M5 runs it in the guest | | CMake | rustc's bootstrap, for LLVM and clang; `src/llvm.rs`, for the LLVM's key; `src/libcxx.rs`, for the C++ runtime in every sysroot build | admitted: no Rust tool does the job, LLVM, clang and LLD are described in CMake, and upstream's only other descriptions are a GN overlay it does not support and a Bazel one | M5 runs it in the guest | | `sh` running LLVM's `config.guess`, and the POSIX tools and `cc` it runs | LLVM's CMake, whenever this host builds an LLVM, and the C++ runtime's, in every sysroot build, ask it the host's triple, unconditionally (`get_host_triple` in `rust/src/llvm-project/llvm/cmake/modules/GetHostTriple.cmake`, which runs `sh` by name) | refused: a Rust tool does the shell's part, brush 0.4.0: on the development host (macOS, arm64) `config.guess` printed `/bin/sh`'s triple under it, `arm64-apple-darwin27.0.0`, exit 0 each. The script runs `sed`, `uname`, `mktemp`, `grep`, `rm`, `rmdir` and `cc` there under either shell, and that `cc` is the `cc` rows'. Five of the other six are refused, uutils' doing each: under brush with sed 0.2.0, grep 0.2.0 and coreutils 0.12.0's `mktemp`, `rm` and `rmdir`, and nothing else on `PATH` but the host's `uname` and `cc`, it printed that triple, exit 0. `uname` is admitted: coreutils 0.12.0's answers `-p` with `unknown` where macOS's answers `arm`, and `config.guess` reads that as PowerPC, `powerpc-apple-darwin27.0.0`, exit 0 | CMake finds brush as its `sh`, uutils' `sed`, `grep`, `mktemp`, `rm` and `rmdir`, and a Rust `uname` that answers `-p` as the host's does; or M5 runs it in the guest | -| `git` for worktrees, submodules, checkouts, fixtures and rustc's bootstrap | adds worktrees (`src/sysroot.rs`); updates submodules (`src/lib.rs`, `src/sysroot.rs`, `src/licence.rs`, `src/release.rs`); fetches the fork from the primary's and checks it out (`src/sysroot.rs`); makes the tests' fixture repositories; runs inside rustc's bootstrap | admitted: no Rust tool does the job, gitoxide 0.85 adds, removes and prunes no worktree, updates no submodule, stages, resets and pushes nothing, checks out only a fresh clone and fetches a local path by spawning `git`; a fixture must be what `git` makes, and bootstrap runs `git` itself | M4 runs it in the guest | -| `git` for reads, a config write, a commit's paths written out, and clones and fetches over HTTPS | `rev-parse`, `merge-base`, `ls-tree`, `ls-files`, `cat-file`, `config --get-regexp`, `worktree list`, `status`, `diff`, `ls-remote` and `grep`, in the build system and its tests; `config --global --add safe.directory` in the nightly's containers; `checkout -- ` through an index of its own, which writes the C++ runtime's sources out of the LLVM commit into the stored LLVM (`src/llvm.rs`); every workflow's checkout | refused: a Rust tool does it, gitoxide 0.85, which reads refs, objects, the index, config, worktrees and status, adds a value to a config file and writes it (gix-config 0.58's `File::section_mut_or_create_new`, `SectionMut::push`, `File::write_to`), walks history, diffs, and lists, fetches and clones a remote over HTTPS; `grep` is a search of the files its index names; and gitoxide's CLI 0.59 (gix 0.88) wrote the runtimes' sources of LLVM `849da7d6` into an empty directory, each path's tree through `gix rev parse`, `gix index from-tree` and `gix free index checkout-exclusive`, exit 0 each: the 18759 files `git` writes there, byte for byte and mode for mode | those are gitoxide's | -| `cc`, `c++` and `ar` on a Linux host, `build-essential` on the nightly's runners | rustc links every host binary through `cc`; `cc` and `c++` compile LLVM, clang, LLD and `rustc_llvm` (`src/llvm.rs` names both to bootstrap); `ar` archives what `cc::Build` compiles | admitted: no Rust tool compiles C or C++, or takes rustc's host link | M5: no host in the loop | +| `git` for worktrees, submodules, checkouts, fixtures and rustc's bootstrap | adds worktrees (`src/sysroot.rs`); updates submodules (`src/lib.rs`, `src/sysroot.rs`); fetches the fork from the primary's and checks it out (`src/sysroot.rs`); makes the tests' fixture repositories; runs inside rustc's bootstrap | admitted: no Rust tool does the job, gitoxide 0.85 adds, removes and prunes no worktree, updates no submodule, stages, resets and pushes nothing, checks out only a fresh clone and fetches a local path by spawning `git`; a fixture must be what `git` makes, and bootstrap runs `git` itself | M4 runs it in the guest | +| `git` for reads, a config write, a commit's paths written out, and clones and fetches over HTTPS | `rev-parse`, `merge-base`, `ls-tree`, `ls-files`, `cat-file`, `config --get-regexp`, `worktree list`, `status`, `diff`, `ls-remote` and `grep`, in the build system and its tests; `config --global --add safe.directory` in CI's containers; `checkout -- ` through an index of its own, which writes the C++ runtime's sources out of the LLVM commit into the stored LLVM (`src/llvm.rs`); every workflow's checkout | refused: a Rust tool does it, gitoxide 0.85, which reads refs, objects, the index, config, worktrees and status, adds a value to a config file and writes it (gix-config 0.58's `File::section_mut_or_create_new`, `SectionMut::push`, `File::write_to`), walks history, diffs, and lists, fetches and clones a remote over HTTPS; `grep` is a search of the files its index names; and gitoxide's CLI 0.59 (gix 0.88) wrote the runtimes' sources of LLVM `849da7d6` into an empty directory, each path's tree through `gix rev parse`, `gix index from-tree` and `gix free index checkout-exclusive`, exit 0 each: the 18759 files `git` writes there, byte for byte and mode for mode | those are gitoxide's | +| `cc`, `c++` and `ar` on a Linux host, `build-essential` in CI's containers | rustc links every host binary through `cc`; `cc` and `c++` compile LLVM, clang, LLD and `rustc_llvm` (`src/llvm.rs` names both to bootstrap); `cc` compiles ring's C and assembly in every build of the build system itself, whose HTTP agent is rustls on ring (`src/release.rs`); `ar` archives what `cc::Build` compiles, ring's included | admitted: no Rust tool compiles C or C++, or takes rustc's host link; ring is the one TLS provider the tree takes (owner, 2026-10-02), over a provider in Rust alone | M5: no host in the loop | | the toolchain's own `clang`, `llvm-ar`, `rust-lld` and `llvm-config`, built from `ToyOSOrg/llvm-project` | rustc links every guest binary with `rust-lld`; `clang` compiles the C corpus (`tests/common/compile.rs`) and, with `llvm-ar`, doomgeneric through `cc::Build` (`src/clang.rs`); rustc's bootstrap asks `llvm-config` how to link LLVM | admitted: our fork's C++, which ToyOS can one day build and run; no Rust tool compiles C, `cc::Build` archives with an `ar`, bootstrap reads LLVM through `llvm-config`, and `CLAUDE.md` links everything with `rust-lld` | M5: no host in the loop | -| `ovmf-generic`, `qemu-efi-aarch64` | the x86-64 and AArch64 UEFI firmware of the nightly's guest containers (`src/firmware.rs`), packaged by Debian apart from QEMU | admitted: QEMU's own firmware, and no Rust firmware does its job | the instrument's QEMU carries its own firmware | -| `ca-certificates` | the trust store `git` and `curl` verify against in the nightly's containers | admitted: data both of them need | goes when neither runs there | -| `env` | the two portability jobs run `cargo run -- --build-only` under it, with `GITHUB_ACTIONS` and `CI` unset | refused: a Rust tool does it, the build system, which already runs that build with both removed (`src/release.rs`) | the build system removes both itself | +| `ovmf-generic`, `qemu-efi-aarch64` | the x86-64 and AArch64 UEFI firmware of CI's guest containers (`src/firmware.rs`), packaged by Debian apart from QEMU | admitted: QEMU's own firmware, and no Rust firmware does its job | the instrument's QEMU carries its own firmware | +| `ca-certificates` | the trust store `git` and `curl` verify against in CI's containers | admitted: data both of them need | goes when neither runs there | | the T14's Ubuntu and every tool `src/metal.rs` runs on it over `ssh` | the metal loop, on the T14 and never on a development host | outside the rule: recovery equipment on a test machine, not the build's host | they leave with Ubuntu (`issues/boot-media/the-machine-updates-itself-without-ubuntu.md`) | | `sh` running `rust/x`, and Python running `x.py` and `bootstrap.py` | every toolchain build (`src/toolchain.rs`) | refused: a Rust tool does it, upstream's bootstrap binary, which builds with stable cargo, fetches its own stage0 (`rust/src/bootstrap/src/core/download.rs`) and needs no Python | `src/toolchain.rs` runs the bootstrap binary | | `curl` in rustc's bootstrap | fetches the stage0 `rust/src/stage0` pins, for a compiler or LLVM build whose build directory lacks it, whichever bootstrap runs | refused: a Rust tool does it, rustup installs the dated beta the pin names, and bootstrap takes a stage0 through `build.rustc` and `build.cargo`, as `src/sysroot.rs` hands it one | no toolchain build fetches with `curl` | -| `curl` in `src/release.rs` and `src/sdkversion.rs` | the toolchain release's lookup and download and the crates.io index, on CI runners only | refused: a Rust tool does it, `ureq`, which `userland/doom/build.rs` already fetches with | those fetches are Rust's | -| `tar` and `zstd` | `src/release.rs` packs and unpacks the toolchain release, on CI runners only | refused: a Rust tool does it, the `tar` crate `userland/doom/build.rs` already unpacks with, and a zstd crate | both are done in Rust, in-process | -| `gh` | `src/release.rs` asks whether a toolchain release exists, creates it and moves the `sdk-` alias, on the nightly's `build` runner | refused: not C or C++ source, it is Go | `src/release.rs` speaks GitHub's REST API itself | +| `tar` and `zstd` | `actions/cache` packs and unpacks every cache entry with them, on CI runners; `guest.yml`'s `deps` installs `zstd`, without which the guest's restore names no entry the toolchain job saved | admitted: GitHub's cache action runs them, and CI keeps no store between runs without it | CI keeps no store between runs | | `ssh` | `src/metal.rs` reaches the T14's Ubuntu with it, only in the metal loop | refused: a Rust tool does it, the repository's own russh client `crate::build::ssh_client_host`, which `src/metaltalk.rs` already drives | `src/metal.rs` drives that client, or Ubuntu leaves the loop | | `cc`, `c++`, `ar` and `xcrun` on a macOS host, Apple's Command Line Tools | what the Linux row's tools do, and rustc asks `xcrun` for the SDK on every host link that names no `SDKROOT` (`rust/compiler/rustc_codegen_ssa/src/back/apple.rs`), as `src/llvm.rs` does for the LLVM's key | refused: one host OS alone | M5: no host in the loop | | `diag/flash.sh` | the owner's flash of a stick by hand: `bash`, and the `stat`, `seq`, `tr`, `grep`, `cut` and `sync` it strings together | refused: shell of our own | `issues/build/the-owners-flash-script-runs-diskutil.md` | @@ -40,9 +37,8 @@ arrives and is not one. M4 and M5 are stages of `issues/build/toyos-builds-itsel | `sync` | the README's Linux flashing steps flush the stick with it | refused: a Rust tool does it, the build system can flush what it writes | `issues/build/the-owners-flash-script-runs-diskutil.md` | | `sudo` on macOS | `diag/flash.sh` and the README's macOS flashing steps run `dd` under it | admitted: no Rust tool raises a process to root on macOS; sudo-rs "is targeted for FreeBSD and Linux-based operating systems only" (its README at `89bae8a`) | goes with both flashes by hand | | `sudo` on Linux | the README's Linux flashing steps run `dd` under it | refused: a Rust tool does it, sudo-rs | the README's Linux steps run sudo-rs | -| `sh` running rustup's `rustup-init.sh` | the nightly's three rustup installs, fetched with `curl` | refused: a Rust tool does it, rustup's own `rustup-init` binary | a job installs rustup without it | -| `nightly.yml`, job `build`, step "disk, QEMU and CMake" | `sudo rm -rf` of five preinstalled SDK directories, then `sudo apt-get update` and `install` | refused: shell of our own | each step is one command | -| `nightly.yml`, step `deps`, which jobs `guest` and `tcg` share | the snapshot archive read out of `debian.sources` with `sed`, checked with `test` and written back with `echo` and `rm`, a three-attempt `apt-get` loop with `sleep` and `cat`, `git config`, and rustup through `curl`, `sh` and `echo` | refused: shell of our own | each step is one command | +| `sh` running rustup's `rustup-init.sh` | CI's rustup installs, fetched with `curl` | refused: a Rust tool does it, rustup's own `rustup-init` binary | a job installs rustup without it | +| `guest.yml`, step `deps` | the snapshot archive read out of `debian.sources` with `sed`, checked with `test` and written back with `echo` and `rm`, a three-attempt `apt-get` loop with `sleep` and `cat`, `git config`, and rustup through `curl`, `sh` and `echo` | refused: shell of our own | each step is one command | | `nightly.yml`, job `portability-linux`, step "deps" | the same loop, `git config`, and rustup the same way | refused: shell of our own | each step is one command | | `nightly.yml`, job `portability-macos`, the rustup step | `curl`, `sh rustup-init.sh`, and `echo` into `$GITHUB_PATH` | refused: shell of our own | each step is one command | | `umask 077 && cat > ` | `src/metal.rs` stages the sudoers rule on the T14 with it | refused: shell of our own | Ubuntu leaves the metal loop | diff --git a/issues/build/the-guest-cache-is-read-by-mtime-and-its-writer-restores-before-it-saves.md b/issues/build/the-guest-cache-is-read-by-mtime-and-its-writer-restores-before-it-saves.md deleted file mode 100644 index 84f6a10f7ab..00000000000 --- a/issues/build/the-guest-cache-is-read-by-mtime-and-its-writer-restores-before-it-saves.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-10-01 ---- - -# The guest cache is read by mtime, and its writer restores before it saves - -`nightly.yml`'s `tcg` restores the newest `guest-` entry, builds on it and saves -the result: nothing prunes what no step rebuilt, so every write keeps the last -one's artifacts and adds its own (3,281,375,938 B on 2026-10-01, beside the -host entry in the repository's 10 GB). And every guest job restores its targets -under a checkout that dated every source at the checkout, so cargo calls every -path crate in them stale. - -The guest entry's ceiling is what H + 2G ≤ 10 GB leaves it, and that sum binds -every night: 4,002,930,524 B at the host's `LIMIT` (`src/cicache.rs`), or -4,298,336,717 B at run 36878222090's H. Above it, `tcg`'s save evicts that -night's host entry unless a pull request has read the entry since the guest -restore; every pull request then runs cold, and nothing reds. - -Owner: the orchestrator. - -Done when the guest entry is written cold, read by content, and bounded. diff --git a/issues/build/the-guest-check-gates-nothing.md b/issues/build/the-guest-check-gates-nothing.md new file mode 100644 index 00000000000..b88174c5277 --- /dev/null +++ b/issues/build/the-guest-check-gates-nothing.md @@ -0,0 +1,27 @@ +--- +status: assigned +kind: tooling +opened: 2026-10-02 +--- + +# The guest check gates nothing + +`ci.yml`'s `guest` runs the guest suite as `guest / suite` on every non-draft +pull request and every merge group, and main's ruleset does not name it: a +pull request or a merge group whose `guest / suite` is red still lands. +`gh api repos/ToyOSOrg/ToyOS/rulesets/20589156` at 13:59:20Z on 2026-10-02 +reads `check_response_timeout_minutes` 240 and one required check, `host`. +Pull request #671, which made the check, says so of itself: "Until step 4 the +guest check gates nothing." + +The naming is a ruleset edit, with no commit, and it waits on main. Main's +cache scope holds no toolchain layer (`actions/caches` for `refs/heads/main` +at 14:00:46Z on 2026-10-02: two `host-sealed-` entries and nothing else), so +until `publish.yml`'s `toolchain` has saved the four there, every merge group +builds all four, as pull request run 36934214557 did in 2:11:46, and a check +named before that makes every landing wait on it. + +Owner: the orchestrator. + +**Exit**: `gh api repos/ToyOSOrg/ToyOS/rulesets/20589156` lists `guest / suite` +among the required checks. diff --git a/issues/build/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md b/issues/build/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md index 5e314564c12..6512c2f0330 100644 --- a/issues/build/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md +++ b/issues/build/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md @@ -8,22 +8,24 @@ opened: 2026-10-01 `src/cicache.rs` refuses a tree whose `PATHS` hold more than `LIMIT`, 8,000,000,000 B, uncompressed. The repository evicts its caches past 10 GB of -what actions/cache stores, compressed, and both two host entries beside a guest -one (2H + G) and one beside two (H + 2G) must fit. +what actions/cache stores, compressed, and two host entries must fit beside +every toolchain layer a scope holds (2H + T): main's, each pull request's that +moved one, and a merge group's sysroot. One ratio ties `LIMIT` to H, measured once. Run 36878222090 (e0ced587c) sealed 5369 MiB of targets, at least 5,629,804,544 B. That head's own archive of the cache's paths, made with the runner's `tar` and `zstdmt`, held 1,403,326,566 B: -a ratio of 4.01. At that ratio `LIMIT` stores at most 1,994,138,951 B. With the -guest entry's 3,281,375,938 B (run 36696295750's `tcg`), 2H + G is -7,269,653,840 B and H + 2G is 8,556,890,827 B. The ratio may fall to 2.38 -before 2H + G reaches 10 GB. The lowest measured is 3.08: run 36844536500 +a ratio of 4.01. At that ratio `LIMIT` stores at most 1,994,138,951 B. One set +of the toolchain's four layers is 918,708,556 B (run 36934214557's saves), so +with main's set alone 2H + T is 4,906,986,458 B, and five more sets fit beside +it. With that one set the ratio may fall to 1.76 before 2H + T reaches 10 GB. +The lowest measured is 3.08: run 36844536500 sealed 9553 MiB on macOS and saved 3,250,736,567 B. No gate reads a stored size. If the targets compress worse, H moves toward the floor and nothing reds. -`LIMIT` is checked only by nightly's `host`, the one job that saves an entry, +`LIMIT` is checked only by nightly's `host`, the one job that saves a host entry, before it seals its tree. A pull request's run and the merge queue's, warm or cold, never seal and are never refused by `LIMIT`. So a landing that takes the cold tree past `LIMIT` is first refused by the next nightly's seal, loudly: @@ -34,8 +36,8 @@ moved; either way its verdict is its steps'. Owner: the host cache (`src/cicache.rs`). **Exit condition.** Two gates that red: -- after nightly's `host` saves, a step reads that entry's stored bytes and the - newest guest entry's from the repository's cache list, and fails when - 2H + G or H + 2G passes 10 GB; +- after nightly's `host` saves, a step reads that entry's stored bytes and every + toolchain layer's from the repository's cache list, and fails when 2H + T + passes 10 GB; - the merge queue's `host` reds a landing whose cold tree passes `LIMIT`, before `main` moves. diff --git a/issues/build/the-release-tag-hashes-none-of-the-build-system-that-builds-the-toolchain.md b/issues/build/the-release-tag-hashes-none-of-the-build-system-that-builds-the-toolchain.md deleted file mode 100644 index a526dfc0049..00000000000 --- a/issues/build/the-release-tag-hashes-none-of-the-build-system-that-builds-the-toolchain.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-09-27 ---- - -# The release tag hashes none of the build system that builds the toolchain - -`src/toolchain.rs`'s `write_config`, `src/sysroot.rs` and `src/libc.rs` decide -the tarball's bytes and are not in `release::trees()`, so a change to them keeps -the old tag and CI installs a toolchain its tree does not describe. - -**Exit**: a commit to each moves the tag, shown by `src/release.rs`'s tests. diff --git a/issues/build/the-toolchain-install-unpacks-an-asset-no-digest-vouches-for.md b/issues/build/the-toolchain-install-unpacks-an-asset-no-digest-vouches-for.md deleted file mode 100644 index 3d2f5285f42..00000000000 --- a/issues/build/the-toolchain-install-unpacks-an-asset-no-digest-vouches-for.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-09-28 ---- - -# The toolchain install unpacks an asset no digest vouches for - -`release::install` (`src/release.rs`) downloads the `toyos-toolchain.tar.zst` -asset of the release its tree's tag names, unpacks it into `rust/build` and -links it as rustup's `toyos`, and checks nothing about its bytes: the tag is -the hash of the tarball's inputs (`trees`), not of the tarball, and the -`toyos-sysroot-witness` an installed toolchain is held to comes inside it. -Every guest job compiles and boots with what it installs. - -A job holding this repository's `contents: write` token can replace a -release asset, so any code such a job runs can replace the toolchain every -later job installs. The nightly's `build` job holds that token while it -bootstraps the toolchain. - -Owner: the release module (`src/release.rs`). - -**Exit condition.** `install` unpacks only an asset whose SHA-256 is one that -no job holding a write token can rewrite — committed to the tree it installs -for — and refuses any other by name. diff --git a/issues/build/toyos-is-a-normal-target.md b/issues/build/toyos-is-a-normal-target.md index e40c54efb34..48a31bacb11 100644 --- a/issues/build/toyos-is-a-normal-target.md +++ b/issues/build/toyos-is-a-normal-target.md @@ -34,18 +34,14 @@ Stages, in order: the release notes of every toolchain release carry it: mkdir -p toyos-toolchain - curl -sSL "$asset" | tar --zstd -x -C toyos-toolchain + curl -sSL "$asset" | tar -xz -C toyos-toolchain stage2=toyos-toolchain/x86_64-unknown-linux-gnu/stage2 rustup toolchain link toyos "$stage2" ln -s "$(rustup which cargo)" "$stage2/bin/cargo" export PATH="$PATH:$PWD/$stage2/bin" cargo +toyos build --target x86_64-unknown-toyos - `toyos-ld` is in that `bin/` because rustc's ToyOS target names its linker - and finds it on `PATH`, and the release tag is the content hash of - everything the tarball's bytes depend on — the linker and the packaging - among them, so a change to either mints a release rather than reusing one - built without it. The glibc floor is 2.39 — `ubuntu-24.04`'s, the + The glibc floor is 2.39 — `ubuntu-24.04`'s, the image the host half is built on — measured over the shipped binaries and asserted at publish time, so a build on a newer machine is refused rather than published. A program that opens a window also carries a `[patch]` of diff --git a/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md b/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md index 7f9caaad8cc..105513d93e3 100644 --- a/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md +++ b/issues/build/two-toolchain-releases-report-the-same-rustc-vv.md @@ -11,8 +11,7 @@ nor `description` under `[rust]`, so bootstrap's channel is `dev` and its default for that channel is `omit-git-hash = true` (`rust/src/bootstrap/src/core/config/config.rs`, the `omit_git_hash` line). Every release then reports `rustc 1.99.0-dev` with byte-identical `rustc -vV` -output — and the release tag hashes more than `rust` anyway: `toyos-abi/src`, -`toyos/src`, `userland/libc/src`, `toyos-ld` and the packaging. +output. Cargo fingerprints a compile on `rustc -vV`. A consumer that switches releases in a reused target directory sees no compiler change, keeps the old rlibs, and diff --git a/issues/design-debt/the-internet-clients-work-unchanged.md b/issues/design-debt/the-internet-clients-work-unchanged.md index 053f8a76a96..6aa2b22830b 100644 --- a/issues/design-debt/the-internet-clients-work-unchanged.md +++ b/issues/design-debt/the-internet-clients-work-unchanged.md @@ -34,9 +34,9 @@ netd, `toyos::net` and std's ToyOS networking in the `rust/` fork. `rustls-rustcrypto` 0.0.2-alpha, and #660 cut both. The row comes back on `ring` and waits for it; `rustls-rustcrypto` does not come back (owner, 2026-10-02). Whether `ring` builds for the ToyOS target is open and - untried: no build compiles it. `rustls-rustcrypto` is still named by - doom's build script, which installs it on the host, and by a line the cut - left in `tests/toyos-rust-tests` + untried: no build for that target compiles it. `rustls-rustcrypto` is + still named by doom's build script, which installs it on the host, and by + a line the cut left in `tests/toyos-rust-tests` (`issues/build/the-guest-test-crate-depends-on-three-crates-no-test-uses.md`). Open: what doom's build script installs instead. **Exit**: `https_tls13` is a `METAL` row: on the T14's I219 an unmodified diff --git a/issues/hardware/features-no-proving-machine-is-known-to-offer.md b/issues/hardware/features-no-proving-machine-is-known-to-offer.md index c771b54bc67..1709affbdb9 100644 --- a/issues/hardware/features-no-proving-machine-is-known-to-offer.md +++ b/issues/hardware/features-no-proving-machine-is-known-to-offer.md @@ -9,7 +9,7 @@ opened: 2026-09-29 No stage of `issues/kernel/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md` or `issues/kernel/toyos-uses-what-modern-hardware-offers-for-speed.md` takes -these, since neither the T14 nor a nightly EPYC KVM guest is known to +these, since neither the T14 nor an EPYC KVM guest is known to enumerate one. It is blocked on a proving machine that does; its exit is every row a stage issue under the track it serves, or a `rejected` issue. Intel's parts are named by the SDM 325462-093US, Vol. 1 Table 5-2 ("SDM"), or by the @@ -35,9 +35,9 @@ cores were not verified against Arm's. The Arm rows belong to | TDX | Emerald Rapids (ISE) | | Total Storage Encryption | Panther Lake (ISE) | | TME | 11th-generation Core lines that set CPUID.(7,0):ECX bit 13, which varies by line (datasheet 631121-012 §1.3); the T14 reads ECX 0x18c05fde, bit 13 clear | -| INVLPGB, TLBSYNC | AMD, CPUID 0x80000008:EBX bit 3 (`arch/x86/include/asm/cpufeatures.h:331,335`); whether a nightly EPYC guest sees it waits on its runner's CPUID capture | +| INVLPGB, TLBSYNC | AMD, CPUID 0x80000008:EBX bit 3 (`arch/x86/include/asm/cpufeatures.h:331,335`); whether an EPYC guest sees it waits on its runner's CPUID capture | | SEV-SNP | AMD, CPUID 0x8000001F:EAX bit 4 (`cpufeatures.h:448,453`) | -| Shadow stack on AMD | AMD; unverified: that AMD enumerates it through the bit Linux reads for Intel, CPUID.(7,0):ECX bit 7 (`cpufeatures.h:390,397`), and which parts set it; whether a nightly EPYC guest sees it waits on its runner's CPUID capture | +| Shadow stack on AMD | AMD; unverified: that AMD enumerates it through the bit Linux reads for Intel, CPUID.(7,0):ECX bit 7 (`cpufeatures.h:390,397`), and which parts set it; whether an EPYC guest sees it waits on its runner's CPUID capture | | MTE | Armv8.5 (`arch/arm64/Kconfig:2150-2170`) | | Pointer authentication | Armv8.3 (`arch/arm64/Kconfig:1967-1974`) | | BTI | Armv8.5 (`arch/arm64/Kconfig:2087-2093`) | diff --git a/issues/kernel/a-pure-function-decides-a-cpus-speculation-mitigations-as-linux-does.md b/issues/kernel/a-pure-function-decides-a-cpus-speculation-mitigations-as-linux-does.md index e09ddc3efde..35182bbddb9 100644 --- a/issues/kernel/a-pure-function-decides-a-cpus-speculation-mitigations-as-linux-does.md +++ b/issues/kernel/a-pure-function-decides-a-cpus-speculation-mitigations-as-linux-does.md @@ -13,7 +13,7 @@ selects, built by the kernel and by a host test. Pull request #602 holds it. **Exit**: each committed fixture's facts give its lines: the T14's and the TCG model's from `issues/hardware/linuxs-readings-of-the-t14-and-the-tcg-model-lack-reads-owed-before-the-t14s-wipe.md`, -and each nightly EPYC guest's once its runner is captured. **Mutation**: `GDS` +and each EPYC guest's once its runner is captured. **Mutation**: `GDS` deleted from `cpu_vuln_blacklist`'s TIGERLAKE_L row reds the T14's fixture, and `SRSO` deleted from its family 0x19 row reds a family-0x19 EPYC guest's. **Oracle**: those lines, and Linux's `cpu_vuln_whitelist` and diff --git a/issues/kernel/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md b/issues/kernel/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md index ae7e1d78368..0fc1863ff7b 100644 --- a/issues/kernel/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md +++ b/issues/kernel/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md @@ -15,11 +15,7 @@ either way. The kernel links the sysroot's `compiler_builtins`, which no kernel build flag recompiles: the feature is `rustflags = ["-Ctarget-feature=+ermsb"]` under `[target.x86_64-unknown-none]` in the std build's `bootstrap.toml` (`std_config`, `src/sysroot.rs:522`; bootstrap's `core/config/toml/target.rs:41`), -and it moves the sysroot key (`RECIPE`, `src/sysroot.rs:65`), which reaches -CI's installed toolchain only once the release tag's `trees()` -(`src/release.rs:25-31`) hashes `src/sysroot.rs`, the work of -`issues/build/the-release-tag-hashes-none-of-the-build-system-that-builds-the-toolchain.md`, -which lands first. One path on every +and it moves the sysroot key (`RECIPE`, `src/sysroot.rs:65`). One path on every CPU: `rep movsb` is correct without ERMS, CPUID.(7,0):EBX bit 9, and no CPU is refused. Zen 2 lacks ERMS (a Ryzen 9 PRO 3900, family 0x17, reads EBX 0x219C91A9: InstLatx64 ddff8a92, diff --git a/issues/kernel/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md b/issues/kernel/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md index bce8666f792..6f55f33dbb5 100644 --- a/issues/kernel/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md +++ b/issues/kernel/the-kernel-is-at-least-as-secure-as-linux-on-every-machine-toyos-supports.md @@ -8,8 +8,8 @@ opened: 2026-09-29 Parity with Linux at `Ubuntu-6.8.0-142.142`, pinned by the first issue below, is the floor on every CPU ToyOS supports, and the kernel also takes every -security feature such a CPU offers. The proving machines are the T14 and the -nightly's AMD EPYC KVM guests; the PR gate's TCG model proves wiring only. A +security feature such a CPU offers. The proving machines are the T14 and +AMD EPYC KVM guests; the PR gate's TCG model proves wiring only. A probe is a `boot-actuators` arm or a `test-actuators` `SYS_DEBUG` action. **Exit**: every issue below is closed, in the order listed. diff --git a/issues/kernel/the-kernel-loads-no-cpu-microcode.md b/issues/kernel/the-kernel-loads-no-cpu-microcode.md index 3003800d615..5e7fdb143d4 100644 --- a/issues/kernel/the-kernel-loads-no-cpu-microcode.md +++ b/issues/kernel/the-kernel-loads-no-cpu-microcode.md @@ -50,7 +50,7 @@ pages (Linux `amd.c`, `__apply_microcode_amd`). On families 0x17, 0x19 and part of 0x1a below a per-CPU cutoff revision the CPU's own signature check is broken (EntrySign; `cpu_has_entrysign`, `need_sha_check`), so the hash pin is the only check, as `amd_shas.c` is Linux's. linux-firmware's `LICENSE.amd-ucode` is -unread. ToyOS has no AMD metal: the nightly's EPYCs are KVM guests, which load +unread. ToyOS has no AMD metal: CI's EPYCs are KVM guests, which load nothing. **Licence.** `LicenseRef-Intel-Microcode` is in no `ALLOWED` row of diff --git a/issues/kernel/the-speculation-decision-does-not-model-an-affected-cpus-l1tf-line.md b/issues/kernel/the-speculation-decision-does-not-model-an-affected-cpus-l1tf-line.md index 1fa0f6fc125..596202191f5 100644 --- a/issues/kernel/the-speculation-decision-does-not-model-an-affected-cpus-l1tf-line.md +++ b/issues/kernel/the-speculation-decision-does-not-model-an-affected-cpus-l1tf-line.md @@ -25,7 +25,7 @@ second has no counterpart in a kernel that runs no VM, so parity needs a ruling on which `VMX:` state ToyOS is held to. No machine ToyOS is tested on reaches it: the T14 has `RDCL_NO`, and the TCG -model and the nightly's KVM runners are AMD. +model and the KVM runners are AMD. **Exit**: the line is decided from the memory map and CPUID and the owner's `VMX:` ruling, and held by a fixture captured under the pinned Linux on such a diff --git a/issues/kernel/toyos-uses-what-modern-hardware-offers-for-speed.md b/issues/kernel/toyos-uses-what-modern-hardware-offers-for-speed.md index f6b97784fe0..e0994691323 100644 --- a/issues/kernel/toyos-uses-what-modern-hardware-offers-for-speed.md +++ b/issues/kernel/toyos-uses-what-modern-hardware-offers-for-speed.md @@ -8,7 +8,7 @@ opened: 2026-09-29 ToyOS takes each hardware feature that makes it faster on every CPU that enumerates it; a CPU without one runs the plain path, or is refused by name -where the stage says so. The proving machines are the T14 and the nightly's +where the stage says so. The proving machines are the T14 and AMD EPYC KVM guests, and a stage's correctness test runs on each that has its feature. A figure is the T14's, since the EPYC guests are shared CI runners: the median of 11 runs with its spread, taken by the program of diff --git a/src/build.rs b/src/build.rs index 128e29fb256..722d44ea540 100644 --- a/src/build.rs +++ b/src/build.rs @@ -1713,8 +1713,8 @@ fn shipped_parts(root: &Path, boot: &Boot, plan: &Plan) -> (Vec, Vec, Ve // Held until the last staged artifact has been read back, so no clean of // this worktree's crate targets can land inside this build. let mut lock = buildlock::shared(root, "build"); - let env = GuestEnv::new(toolchain::ensure(root, &mut lock)); let config = parse_config(&boot.config); + let env = GuestEnv::new(toolchain::ensure(root, &mut lock, config.hosted_rustc)); invalidate_stale(root, &mut lock, &env.sysroot.identity, &config_targets(root, &config)); @@ -1930,7 +1930,7 @@ pub fn build_test_parts( // back after the userland build, and a clean landing in between is the // same defect as one landing mid-compile. let mut lock = buildlock::shared(root, "test image"); - let env = GuestEnv::new(toolchain::ensure(root, &mut lock)); + let env = GuestEnv::new(toolchain::ensure(root, &mut lock, config.hosted_rustc)); invalidate_stale(root, &mut lock, &env.sysroot.identity, &config_targets(root, &config)); @@ -2137,7 +2137,7 @@ struct TestBuild { impl TestBuild { fn begin(root: &Path, arch: Arch, what: &str, stale_targets: &[(PathBuf, Clean)]) -> Self { let mut lock = buildlock::shared(root, what); - let env = GuestEnv::new(toolchain::ensure(root, &mut lock)); + let env = GuestEnv::new(toolchain::ensure(root, &mut lock, false)); invalidate_stale(root, &mut lock, &env.sysroot.identity, stale_targets); let artifact = buildlock::artifact(root); TestBuild { target: arch.userland(), env, _lock: lock, _artifact: artifact } diff --git a/src/ci.rs b/src/ci.rs index 5b0a8120f6e..c1f90245836 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -2,13 +2,6 @@ //! checkout, a cache and one line, and this host runs the same job to the same //! verdict. //! -//! `.github/workflows/` is three files. `ci.yml` runs on a pull request and in -//! the merge queue and boots no guest: [`Job::Host`] runs as `host`. Every -//! test that boots no guest is in [`Job::Host`], so a merge is gated on all of -//! them. `nightly.yml` runs everything that boots a guest, `host` again to -//! write the cache the merge queue restores, and portability. `publish.yml` -//! puts a landing's crates on crates.io. -//! //! A host job runs every step and reds if any failed; a guest job stops at the //! first failure among the instrument, the toolchain and the suite, because //! what follows a wrong instrument or a missing toolchain measures nothing — @@ -39,8 +32,10 @@ const USAGE: &str = "cargo run -- --ci , where is one of: model controls, userland and the SDK (ci.yml, nightly) seal `host` from a cold tree, then that tree sealed as the host cache's entry, which the step after it saves (nightly) - toolchain publish this tree's toolchain if nobody has (nightly) - guest the guest suite (nightly) + toolchain the cache entry of each store of this tree's toolchain + bootstrap build the stores of this tree's toolchain its job did not restore + guest the guest suite, on the sysroot its job restored + release put the sysroot main's nightly restored up as its toolchain release publish put main's SDK crates on crates.io (publish.yml)"; #[derive(Debug, PartialEq, Eq)] @@ -48,7 +43,9 @@ enum Job { Host, Seal, Toolchain, + Bootstrap, Guest, + Release, Publish, } @@ -57,7 +54,9 @@ fn parse(words: &[String]) -> Result { Some("host") => Job::Host, Some("seal") => Job::Seal, Some("toolchain") => Job::Toolchain, + Some("bootstrap") => Job::Bootstrap, Some("guest") => Job::Guest, + Some("release") => Job::Release, Some("publish") => Job::Publish, Some(other) => return Err(format!("no CI job is called {other:?}")), None => return Err("which job?".to_string()), @@ -76,8 +75,10 @@ pub fn dispatch(root: &Path, args: &[String]) { let steps = match &job { Job::Host => host(root), Job::Seal => seal(root), - Job::Toolchain => vec![step("the toolchain release", || release::ensure_published(root))], + Job::Toolchain => vec![step("the stores of this tree's toolchain", || release::toolchain(root))], + Job::Bootstrap => vec![step("this tree's toolchain", || release::bootstrap(root))], Job::Guest => guest(root, &suite_args(&["--jobs", "1"])), + Job::Release => vec![step("main's toolchain release", || release::release(root))], Job::Publish => vec![step("the SDK crates on crates.io", || publish(root))], }; let failed: Vec<&Step> = steps.iter().filter(|s| s.verdict.is_err()).collect(); @@ -471,10 +472,9 @@ fn run_control(root: &Path, control: &Control) -> Result { judge_control(control, green, &log) } -/// The merge queue's whole gate, and the nightly's host lane: every test that -/// runs on the host and boots no guest. The build system's own tests, every -/// member of the host workspace, clippy with warnings denied, the concurrency -/// models' negative controls, every userland crate with a host test +/// Every test that runs on the host and boots no guest. The build system's own +/// tests, every member of the host workspace, clippy with warnings denied, the +/// concurrency models' negative controls, every userland crate with a host test /// ([`crate::userlandhost`], which also reds on a userland test none of them /// runs), every app the images ship for each host ([`apps_for`]), and the SDK. /// @@ -483,10 +483,10 @@ fn run_control(root: &Path, control: &Control) -> Result { /// that writes scratch past a `toyos_tmpdir::TempDir`, or holds one past its /// end, is a test that fills the host's disk one run at a time. /// -/// Clippy needs none of the ToyOS toolchain the nightly alone builds — the -/// kernel and the bootloader lint against every architecture's bare targets -/// ([`crate::clippy::BARE_TARGETS`]), which any rustup installs, and userland carries no -/// clippy shape (`src/clippy.rs`). Userland and the SDK are tested against the +/// Clippy needs none of the ToyOS toolchain — the kernel and the bootloader +/// lint against every architecture's bare targets ([`crate::clippy::BARE_TARGETS`]), +/// which any rustup installs, and userland carries no clippy shape +/// (`src/clippy.rs`). Userland and the SDK are tested against the /// host triple for the same reason. /// /// In a job that carries the cache ([`cicache::carried`]) the restored entry is @@ -1071,6 +1071,7 @@ mod tests { assert_eq!(parse(&words("host")), Ok(Job::Host)); assert_eq!(parse(&words("seal")), Ok(Job::Seal)); assert_eq!(parse(&words("guest")), Ok(Job::Guest)); + assert_eq!(parse(&words("release")), Ok(Job::Release)); assert!(parse(&words("guest 3/12")).is_err()); assert!(parse(&words("tcg")).is_err()); assert!(parse(&words("host extra")).is_err()); @@ -1177,7 +1178,7 @@ mod tests { ); } } - assert_eq!(seen, 3, "ci.yml, nightly.yml and publish.yml"); + assert_eq!(seen, 5, "ci.yml, nightly.yml and publish.yml, and guest.yml and toolchain.yml"); } #[test] diff --git a/src/cicache.rs b/src/cicache.rs index 85192dd1024..e35039ec787 100644 --- a/src/cicache.rs +++ b/src/cicache.rs @@ -63,9 +63,7 @@ const PATHS: [&str; 8] = [ ]; /// The most the files under [`PATHS`] may hold, uncompressed. The repository's -/// caches are evicted by last access past 10 GB, and a night whose guest jobs -/// restore after this entry is saved holds two host entries beside a guest -/// one, then one beside two guest ones. +/// caches are evicted by last access past 10 GB. const LIMIT: u64 = 8_000_000_000; /// 2001-09-09T01:46:40Z: older than any build, so a file dated so is never @@ -226,7 +224,7 @@ fn seal_at(root: &Path, home: &Path, cold: &Cold) -> Result { if bytes > LIMIT { return Err(format!( "{bytes} B in {files} files under the cache's paths, above the {LIMIT} B an entry may \ - hold: saved, it could evict the guest entry or the next host one" + hold: saved, it could evict a toolchain layer or the next host one" )); } let now = sources(root)?; diff --git a/src/clang.rs b/src/clang.rs index 8473447f5e3..35c9efdf886 100644 --- a/src/clang.rs +++ b/src/clang.rs @@ -3,17 +3,20 @@ //! //! **Beside `rust-lld`, in `lib/rustlib//bin/`**, which every copy of a //! toolchain directory — a compiler of a worktree's own, a sysroot, the -//! published release — carries whole. Bootstrap puts LLVM's tools there -//! already, `llvm-ar` — the archiver `cc` builds a C library with, where the -//! host's may not index ELF at all — among them. This adds the rest: +//! published release — carries whole. Bootstrap copies none of LLVM's tools +//! there (`llvm-tools = false`); this puts the ones a build runs: //! //! - `clang`, the driver whose ToyOS toolchain (`src/llvm-project`'s //! `clang/lib/Driver/ToolChains/ToyOS.cpp`) names `ld.lld`, the sysroot and //! `-ltoyos_c`; +//! - `llvm-ar`, the archiver `cc` builds a C library with, where the host's may +//! not index ELF at all; //! - `ld.lld`, a link to `rust-lld` — the same LLD, which takes its flavour from //! the name it is run by — found by clang in its own directory; //! - `../lib/clang//include`, clang's own headers (`stddef.h`, -//! `stdarg.h`), which it looks for relative to itself. +//! `stdarg.h`), which it looks for relative to itself; +//! - on an Apple host, `rust-objcopy`, LLVM's `llvm-objcopy`, which rustc runs +//! from here to strip a Darwin binary. //! //! Bootstrap removes `stage2` on every assemble, so these are put back after //! every build that makes one, and a toolchain directory without all of it is @@ -26,9 +29,6 @@ use crate::arch::Arch; use crate::sysroot::clone_tree; use crate::toolchain::host_triple; -/// This file, which the release tag hashes. -pub(crate) const SOURCE: &str = file!(); - /// The LLVM every host compiler links, in every `bootstrap.toml` that builds /// one: built from `src/llvm-project` — the fork that knows the ToyOS target — /// with clang beside it, for the host and the two architectures ToyOS runs on. @@ -40,6 +40,14 @@ pub(crate) const LLVM_CONFIG: &str = "download-ci-llvm = false\n\ /// What a toolchain directory's `bin` must hold for C. const TOOLS: [&str; 3] = ["llvm-ar", "clang", "ld.lld"]; +/// What an Apple host's toolchain directory's `bin` must hold besides. +const APPLE_STRIP: &str = "rust-objcopy"; + +/// [`TOOLS`], and on an Apple host [`APPLE_STRIP`]. +pub(crate) fn tools() -> impl Iterator { + TOOLS.into_iter().chain(host_triple().ends_with("apple-darwin").then_some(APPLE_STRIP)) +} + /// `lib/rustlib//bin` of `toolchain`, where `rust-lld` is. fn bin(toolchain: &Path) -> PathBuf { toolchain.join("lib/rustlib").join(host_triple()).join("bin") @@ -97,8 +105,7 @@ fn resource_parent(toolchain: &Path) -> PathBuf { /// What of the C toolchain `toolchain` lacks, by name. fn absent(toolchain: &Path) -> Vec { let bin = bin(toolchain); - let mut gone: Vec = TOOLS - .iter() + let mut gone: Vec = tools() .map(|name| bin.join(name)) .filter(|path| !path.exists()) .map(|path| path.display().to_string()) @@ -136,7 +143,7 @@ pub(crate) fn assert_present(toolchain: &Path) { } /// The one version directory under an LLVM build's `lib/clang`. -fn resource_version(llvm: &Path) -> PathBuf { +pub(crate) fn resource_version(llvm: &Path) -> PathBuf { let parent = llvm.join("lib/clang"); let versions: Vec = fs::read_dir(&parent) .unwrap_or_else(|e| panic!("read {}: {e} — was LLVM built with clang = true?", parent.display())) @@ -153,12 +160,13 @@ fn resource_version(llvm: &Path) -> PathBuf { /// its rustc links. pub(crate) fn provision(stage2: &Path, llvm: &Path) { let bin = bin(stage2); - let from = llvm.join("bin/clang"); - let to = bin.join("clang"); - let _ = fs::remove_file(&to); - // `fs::copy` follows `clang`'s link to `clang-`, and clones where - // the filesystem can. - fs::copy(&from, &to).unwrap_or_else(|e| panic!("copy {} -> {}: {e}", from.display(), to.display())); + let apple = host_triple().ends_with("apple-darwin").then_some((crate::llvm::APPLE_TOOL, APPLE_STRIP)); + for (tool, name) in [("clang", "clang"), ("llvm-ar", "llvm-ar")].into_iter().chain(apple) { + let (from, to) = (llvm.join("bin").join(tool), bin.join(name)); + let _ = fs::remove_file(&to); + // `fs::copy` clones where the filesystem can. + fs::copy(&from, &to).unwrap_or_else(|e| panic!("copy {} -> {}: {e}", from.display(), to.display())); + } let lld = bin.join("ld.lld"); let _ = fs::remove_file(&lld); std::os::unix::fs::symlink("rust-lld", &lld) @@ -187,26 +195,32 @@ mod tests { let llvm = base.join("llvm"); write(&llvm.join("bin/clang-22"), "the clang"); std::os::unix::fs::symlink("clang-22", llvm.join("bin/clang")).unwrap(); + write(&llvm.join("bin/llvm-ar"), "the archiver"); + write(&llvm.join("bin/llvm-objcopy"), "the objcopy"); write(&llvm.join("lib/clang/22/include/stddef.h"), "typedef long ptrdiff_t;"); llvm } /// **A toolchain without its C compiler is refused by name**, and one - /// provisioned from an LLVM carries the binary behind `clang`'s link, an - /// `ld.lld` that is `rust-lld`, and clang's headers where clang looks. + /// provisioned from an LLVM carries the binary behind `clang`'s link, that + /// LLVM's archiver, an `ld.lld` that is `rust-lld`, clang's headers where + /// clang looks, and on an Apple host the `rust-objcopy` rustc strips with. #[test] fn a_toolchain_carries_the_clang_of_its_llvm_or_is_refused() { let base = TempDir::new("clang"); let llvm = llvm(&base); - // What bootstrap's own assemble leaves beside `rust-lld`. + let apple = host_triple().ends_with("apple-darwin"); + // What bootstrap's own assemble leaves: `rust-lld` and no LLVM tool. let stage2 = base.join("stage2"); write(&bin(&stage2).join("rust-lld"), "lld"); - write(&bin(&stage2).join("llvm-ar"), "the archiver"); assert!(defect(&stage2).is_some()); let refused = std::panic::catch_unwind(|| assert_present(&stage2)).expect_err("no clang, and not refused"); let said = refused.downcast_ref::().expect("a formatted refusal"); - assert!(said.contains("clang") && said.contains("ld.lld") && said.contains("include"), "{said}"); + for named in ["clang", "llvm-ar", "ld.lld", "include"] { + assert!(said.contains(named), "{named}: {said}"); + } + assert_eq!(said.contains(APPLE_STRIP), apple, "{said}"); provision(&stage2, &llvm); assert_eq!(defect(&stage2), None); @@ -215,6 +229,7 @@ mod tests { assert_eq!(fs::read_link(bin(&stage2).join("ld.lld")).unwrap(), Path::new("rust-lld")); assert_eq!(fs::read_to_string(bin(&stage2).join("ld.lld")).unwrap(), "lld"); assert_eq!(fs::read_to_string(bin(&stage2).join("llvm-ar")).unwrap(), "the archiver"); + assert_eq!(fs::read_to_string(bin(&stage2).join(APPLE_STRIP)).ok().as_deref(), apple.then_some("the objcopy")); let stddef = resource_parent(&stage2).join("22/include/stddef.h"); assert_eq!(fs::read_to_string(stddef).unwrap(), "typedef long ptrdiff_t;"); @@ -225,7 +240,11 @@ mod tests { assert!(!resource_parent(&stage2).join("22").exists(), "the old headers stayed beside the new"); assert_eq!(fs::read_to_string(resource_parent(&stage2).join("23/include/stddef.h")).unwrap(), "v23"); - fs::remove_file(bin(&stage2).join("llvm-ar")).unwrap(); - assert!(defect(&stage2).is_some(), "a missing llvm-ar went unnoticed"); + let lost = if apple { [APPLE_STRIP, "llvm-ar"].as_slice() } else { ["llvm-ar"].as_slice() }; + for tool in lost { + provision(&stage2, &llvm); + fs::remove_file(bin(&stage2).join(tool)).unwrap(); + assert!(defect(&stage2).is_some_and(|d| d.contains(tool)), "a missing {tool} went unnoticed"); + } } } diff --git a/src/compiler.rs b/src/compiler.rs index f10df41ce8a..a1d2de33046 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -2,17 +2,17 @@ //! primary's, or one of its own, content-addressed. //! //! **Every worktree builds with the compiler its own fork checkout names.** The -//! primary's `stage2` is built from what the primary's `rust/compiler/` holds, -//! and [`record`] writes which that is. A linked worktree whose fork checkout -//! holds the same `compiler/` ([`source`]) compiles with that one. One whose -//! `compiler/` differs — a new target spec, a codegen change — gets its own: -//! built by bootstrap in its own fork checkout, under that checkout's +//! primary's `stage2` is built from what the primary's fork checkout holds, and +//! [`record`] writes which that is ([`source`]). A linked worktree whose fork +//! checkout names the same compiler compiles with that one. One whose compiler +//! sources differ — a new target spec, a codegen change — gets its own: built +//! by bootstrap in its own fork checkout, under that checkout's //! `build/toyos-compiler/`, and placed at `rust/build/compilers//`, where //! the key ([`key`]) is the identity (`src/identity.rs`) of the checkout's -//! `compiler/`, `src/tools/`, `src/stage0` and `Cargo.lock`, the key of the -//! LLVM it links, which names `src/bootstrap`, and [`RECIPE`]. Nothing writes -//! that directory after its [`SOURCE`] file exists, and two worktrees naming -//! the same compiler share one copy. +//! [`KEYED`] sources, the key of the LLVM it links, which names +//! `src/bootstrap`, and the build itself ([`build_text`]). Nothing writes that +//! directory after its [`SOURCE`] file exists, and two worktrees naming the +//! same compiler share one copy. //! //! **LLVM is the host's, built from `src/llvm-project`** (`src/llvm.rs`), and //! linked through its `llvm-config`. [`source`] names that LLVM's key, so @@ -51,13 +51,23 @@ use crate::keystore::{self, Key}; use crate::sysroot::{clone_tree, git_bytes, git_out, short, tree_identity, Links}; use crate::toolchain::{self, host_triple}; -/// What changes how a key's sources become a compiler and is none of them: the -/// build below. Moving it moves every key. -const RECIPE: &str = "bootstrap stage 2 of compiler/rustc and library, profile compiler, host only, with rust-lld, host linker pinned, LLVM, clang and LLD from the host's LLVM; 5"; +/// What changes how a key's sources become a compiler and is neither them nor +/// [`config_text`]: the build below. Moving it moves every key. +const RECIPE: &str = "bootstrap stage 2 of compiler/rustc and library, profile compiler, host only, with rust-lld, host linker pinned, LLVM, clang and LLD from the host's LLVM, no LLVM tool copied, rustc without debuginfo; 6"; /// What a compiler's key is the identity of, in its fork checkout. const KEYED: [&str; 4] = ["compiler", "src/tools", "src/stage0", "Cargo.lock"]; +/// What a compiler build is beyond its sources, as every key and record of one +/// reads it: [`RECIPE`], the configuration bootstrap is given ([`config_text`]) +/// with no path of this host in it, and the tools `clang::provision` puts beside +/// the compiler. +fn build_text() -> String { + let config = config_text(Path::new(""), &host_triple(), Path::new("")); + let provisioned: Vec<&str> = crate::clang::tools().collect(); + format!("{RECIPE}\n{config}provisioned {}", provisioned.join(" ")) +} + /// The submodule a compiler is built against by commit: its LLVM, which /// bootstrap builds from that commit, so its content is never read. pub(crate) const LLVM: &str = "src/llvm-project"; @@ -112,6 +122,20 @@ impl Compiler { .map_or(0, |d| d.as_nanos()); format!("{} {} {} {mtime}", source.trim(), driver.file_name().to_string_lossy(), meta.len()) } + + /// What a store built with this compiler is keyed by: its record, which + /// names what builds it and is known before it is built ([`primary_key`]). + pub fn key(&self) -> Key { + Key::of(&fs::read(&self.record).unwrap_or_else(|e| { + panic!("{} cannot be read ({e}), so nothing says which compiler a store is built with", self.record.display()) + })) + } +} + +/// [`Compiler::key`] of the primary's compiler `rust_dir`'s sources name, +/// whether or not it is built. +pub(crate) fn primary_key(rust_dir: &Path) -> Key { + Key::of(source(rust_dir).as_bytes()) } /// The primary's record of which `compiler/` its `stage2` was built from. @@ -124,23 +148,25 @@ pub fn compilers_dir(rust_dir: &Path) -> PathBuf { rust_dir.join("build/compilers") } -/// [`compiler_source`] and the key of the LLVM it links. +/// What `checkout`'s compiler is built from, as the primary records it: the +/// build ([`build_text`]), [`compiler_source`], and the key of the LLVM it links. pub fn source(checkout: &Path) -> String { source_with(checkout, &crate::llvm::key(checkout)) } /// [`source`], with the key of the LLVM `checkout` names. fn source_with(checkout: &Path, llvm: &Key) -> String { - format!("{} llvm {llvm}", compiler_source(checkout)) + format!("{}\n{} llvm {llvm}", build_text(), compiler_source(checkout)) } -/// What `checkout`'s `compiler/` is: its commit's tree, and whatever the working -/// tree changes in it — an edit, or a file git does not track yet, which is -/// what a new target spec is before its commit. +/// What `checkout`'s [`KEYED`] sources are: each one's entry at its commit, and +/// whatever the working tree changes in them — an edit, or a file git does not +/// track yet, which is what a new target spec is before its commit. fn compiler_source(checkout: &Path) -> String { - let tree = git_out(checkout, &["rev-parse", "HEAD:compiler"]); - let mut local = git_bytes(checkout, &["diff", "HEAD", "--", "compiler"]); - let untracked = git_bytes(checkout, &["ls-files", "-z", "--others", "--exclude-standard", "--", "compiler"]); + let in_keyed = |args: &[&'static str]| [args, &KEYED[..]].concat(); + let tree = git_out(checkout, &in_keyed(&["ls-tree", "HEAD", "--"])); + let mut local = git_bytes(checkout, &in_keyed(&["diff", "HEAD", "--"])); + let untracked = git_bytes(checkout, &in_keyed(&["ls-files", "-z", "--others", "--exclude-standard", "--"])); for name in untracked.split(|b| *b == 0).filter(|n| !n.is_empty()) { let path = checkout.join(String::from_utf8_lossy(name).as_ref()); local.extend_from_slice(name); @@ -175,8 +201,8 @@ pub fn forget(rust_dir: &Path) { } } -/// Whether the primary's `stage2` is the compiler `checkout`'s `compiler/` -/// names — `Err` when nothing records which compiler that is. +/// Whether the primary's `stage2` is the compiler `checkout` names — `Err` when +/// nothing records which compiler that is. /// /// **The source's content, never its files' times**: a checkout that rewrites a /// file with the bytes it had is no new compiler. @@ -185,8 +211,8 @@ fn primary_is(rust_dir: &Path, checkout: &Path, llvm: &Key) -> Result bool { match primary_is(rust_dir, rust_dir, &crate::llvm::key(rust_dir)) { Ok(current) => current, @@ -195,51 +221,20 @@ pub fn primary_is_current(rust_dir: &Path) -> bool { } } -/// The key of the compiler `fork`'s sources name: their content. +/// The key of the compiler `fork`'s sources name: their content, and the build. pub fn key(fork: &Path) -> Key { key_with(fork, &crate::llvm::key(fork)) } /// [`key`], with the key of the LLVM `fork` names. fn key_with(fork: &Path, llvm: &Key) -> Key { - let parts = [RECIPE, &tree_identity(fork, &KEYED, Links::Skipped), llvm.as_str()]; - Key::of(parts.join("\n\0\n").as_bytes()) + key_of(fork, &build_text(), llvm) } -/// The LLVM commit `fork` builds against: the one its `HEAD` records, refused -/// when its index stages another, because bootstrap checks out the index's. An -/// LLVM change is a commit there and a gitlink here, so a checkout holding -/// anything no commit does is refused rather than named by its commit. -pub(crate) fn llvm_commit(fork: &Path) -> String { - let checkout = fork.join(LLVM); - // Exactly what bootstrap's LLVM stamp hashes beyond the commit; the untracked - // cache spares each call a walk of the whole tree. - let status = ["-c", "core.untrackedCache=true", "status", "--porcelain", "--untracked-files=normal"]; - let edited = checkout.join(".git").exists() && !git_bytes(&checkout, &status).is_empty(); - assert!( - !edited, - "{} holds changes no commit does, and a compiler is keyed on the commit its gitlink \ - names: commit them there and record that commit in {}", - checkout.display(), - fork.display(), - ); - let recorded = git_out(fork, &["ls-tree", "HEAD", LLVM]); - let committed = match recorded.split_whitespace().collect::>().as_slice() { - ["160000", "commit", sha, _] => sha.to_string(), - _ => panic!("{} records no {LLVM} gitlink: `git ls-tree HEAD {LLVM}` said {recorded:?}", fork.display()), - }; - let indexed = git_out(fork, &["ls-files", "--stage", LLVM]); - let staged = match indexed.split_whitespace().collect::>().as_slice() { - ["160000", sha, "0", _] => sha.to_string(), - _ => panic!("{} indexes no {LLVM} gitlink: `git ls-files --stage {LLVM}` said {indexed:?}", fork.display()), - }; - assert!( - staged == committed, - "{} stages {LLVM} at {staged}, and its HEAD records {committed}: bootstrap builds the one \ - staged, and nothing is keyed on what no commit holds; commit the gitlink, or unstage it", - fork.display(), - ); - committed +/// [`key`], with the build it reads. +fn key_of(fork: &Path, build: &str, llvm: &Key) -> Key { + let parts = [build, &tree_identity(fork, &KEYED, Links::Skipped), llvm.as_str()]; + Key::of(parts.join("\n\0\n").as_bytes()) } /// The compiler `root`'s fork checkout at `fork` names: the primary's where its @@ -330,7 +325,7 @@ fn build_in_fork(root: &Path, rust_dir: &Path, fork: &Path) -> PathBuf { fs::write(&config, config_text(&build_dir, &host, &llvm.dir)).unwrap_or_else(|e| panic!("write {}: {e}", config.display())); let config = config.to_str().unwrap_or_else(|| panic!("{} is not UTF-8", config.display())); let args = ["build", "--stage", "2", "--config", config, "--warnings", "warn", "compiler/rustc", "library"]; - let (ok, log) = toolchain::x_build(fork, &args, "the compiler"); + let (ok, log) = toolchain::x_build_compiler(fork, &args, "the compiler", &llvm.dir); toolchain::refuse_on_compile_error(&log, "the compiler"); assert!(ok, "the compiler build in {} failed, and nothing in its output was a compile error", fork.display()); let stage2 = build_dir.join(&host).join("stage2"); @@ -341,10 +336,10 @@ fn build_in_fork(root: &Path, rust_dir: &Path, fork: &Path) -> PathBuf { stage2 } -/// Bootstrap's configuration for a compiler of a worktree's own: the primary's -/// `profile` and options, for the host alone, since every guest target's -/// libraries are the sysroot's to build, linking the LLVM at `llvm`. -fn config_text(build_dir: &Path, host: &str, llvm: &Path) -> String { +/// Bootstrap's configuration for every compiler, the primary's and a +/// worktree's own, built in `build_dir`: for the host alone, since every guest +/// target's libraries are the sysroot's to build, linking the LLVM at `llvm`. +pub(crate) fn config_text(build_dir: &Path, host: &str, llvm: &Path) -> String { format!( r#"change-id = "ignore" profile = "compiler" @@ -360,6 +355,7 @@ target = ["{host}"] [rust] incremental = true lld = true +{lean} [target.{host}] {pin} @@ -367,6 +363,7 @@ lld = true "#, build_dir = build_dir.display(), llvm = crate::clang::LLVM_CONFIG, + lean = toolchain::LEAN, pin = toolchain::HOST_LINKER_PIN, external = crate::llvm::host_lines(llvm), ) @@ -687,7 +684,7 @@ pub(crate) mod tests { /// Write the primary's record as it was written before its compiler linked /// the host's LLVM: its `compiler/` and its LLVM commit. pub(crate) fn record_before_the_store(rust_dir: &Path) { - fs::write(primary_record(rust_dir), format!("{} llvm {}", compiler_source(rust_dir), llvm_commit(rust_dir))).unwrap(); + fs::write(primary_record(rust_dir), format!("{} llvm {}", compiler_source(rust_dir), crate::sysroot::gitlink(rust_dir, LLVM))).unwrap(); } /// `fork`'s LLVM checked out at a commit of its own. @@ -770,6 +767,57 @@ pub(crate) mod tests { assert_ne!(key(&fork), tools, "another LLVM commit did not move the key"); } + /// **The primary's record and every compiler's key read the whole build**: + /// the configuration bootstrap is given and the tools put beside the + /// compiler are in both, and another configuration is another key; the + /// record names every source of [`KEYED`], as the key does. What + /// [`record`] writes is the key a store is filed under before the + /// compiler is built ([`primary_key`]). + #[test] + fn a_compiler_is_keyed_and_recorded_by_its_whole_build() { + let scratch = TempDir::new("compiler-build"); + let (_primary, rust_dir, [same, _, _]) = estate(&scratch); + let fork = same.join("rust"); + let llvm = crate::llvm::key(&fork); + assert_eq!(key_of(&fork, &build_text(), &llvm), key(&fork)); + let config = config_text(Path::new(""), &host_triple(), Path::new("")); + let tools: Vec<&str> = crate::clang::tools().collect(); + for part in [config.as_str(), toolchain::LEAN, toolchain::HOST_LINKER_PIN, &tools.join(" ")] { + assert!(build_text().contains(part), "a compiler's key reads no {part:?}"); + assert!(source(&fork).contains(part), "the primary's record names no {part:?}"); + } + let more = build_text().replace("debuginfo-level-rustc = 0", "debuginfo-level-rustc = 1"); + assert_ne!(key_of(&fork, &more, &llvm), key(&fork), "another configuration kept the key"); + + let sources = [ + ("src/tools/lld-wrapper/src/main.rs", "fn main() {}\n"), + ("src/stage0", "compiler_version=nightly\n"), + ("Cargo.lock", "# relocked\n"), + ]; + for (file, text) in sources { + let before = source(&fork); + write(&fork.join(file), text); + assert_ne!(source(&fork), before, "{file} kept the primary's record"); + git(&fork, &["add", "-A"]); + git(&fork, &["commit", "-qm", file]); + assert_ne!(source(&fork), before, "{file}, committed, kept the primary's record"); + } + + record(&rust_dir); + assert_eq!(Compiler::primary(&rust_dir).key(), primary_key(&rust_dir)); + } + + /// **A compiler of a worktree's own is built as the primary's is**: for + /// the host alone, against the host's LLVM, copying none of its tools, with + /// rustc without debuginfo and no codegen test. + #[test] + fn a_worktree_compiler_is_built_lean_against_the_host_s_llvm() { + let config = config_text(Path::new("/b"), "h", Path::new("/llvm")); + let lean = "\nlld = true\nllvm-tools = false\ndebuginfo-level-rustc = 0\ncodegen-tests = false\n"; + assert!(config.contains(lean) && config.contains("\ntarget = [\"h\"]\n"), "{config}"); + assert!(config.contains("\nllvm-config = \"/llvm/bin/llvm-config\"\n"), "{config}"); + } + /// A primary record naming another LLVM, or none, is another compiler. #[test] fn another_llvm_is_another_compiler() { diff --git a/src/keystore.rs b/src/keystore.rs index 1c9fb0d7271..5b91b1f0c39 100644 --- a/src/keystore.rs +++ b/src/keystore.rs @@ -3,6 +3,13 @@ //! that removes a key no registered worktree records and nobody is making or //! using. //! +//! **A key hashes what its product's build reads**: its sources, the +//! configuration its build is given, the tools that run that build and the keys +//! of the products it reads; and it is known before the product is. So a +//! product found under its key, made here or restored by a CI runner from +//! another run's cache, is the one this tree's build would make. An input a +//! build reads and its key does not is a defect of the key. +//! //! A product lives at `//`, whole once its maker renamed it there; //! any other name beginning `.` is one half-made or half-removed. A whole //! one is renamed out of the way before anything in it is removed ([`retire`]), diff --git a/src/lib.rs b/src/lib.rs index acf395f0c6d..a146cd06c5b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,6 +114,21 @@ pub fn ensure_submodules(repo_dir: &Path) { } } +/// `rust/` at the commit this tree pins and with no history, where a CI +/// runner's checkout has none: what reading the fork's sources and building +/// the toolchain from them need. +pub fn ensure_shallow_fork(root: &Path) -> Result<(), String> { + if root.join("rust/x.py").exists() { + return Ok(()); + } + let status = Command::new("git") + .args(["submodule", "update", "--init", "--depth", "1", "rust"]) + .current_dir(root) + .status() + .map_err(|e| format!("git submodule update --init --depth 1 rust: {e}"))?; + status.success().then_some(()).ok_or_else(|| format!("git submodule update --init --depth 1 rust exited {status}")) +} + /// Ensure a single git submodule is checked out. pub fn ensure_submodule(repo_dir: &Path, path: &str) { let dir = repo_dir.join(path); diff --git a/src/libc.rs b/src/libc.rs index f0770d7ae37..e8e442285de 100644 --- a/src/libc.rs +++ b/src/libc.rs @@ -10,6 +10,14 @@ pub const CRATE: &str = "userland/libc"; /// The features [`build`] gives [`CRATE`]. pub const FEATURES: &str = "std-runtime"; +/// What [`build`] asks cargo for, but the target, the manifest and the target +/// directory: what a sysroot's key reads of it. `--message-format=json` is how +/// it finds the exact rlib artifacts. +pub(crate) const BUILD: [&str; 5] = ["build", "--release", "--features", FEATURES, "--message-format=json"]; + +/// What [`build_c`] asks cargo for, as [`BUILD`] is [`build`]'s. +pub(crate) const BUILD_C: [&str; 4] = ["rustc", "--release", "--crate-type", "staticlib"]; + /// Build toyos-libc against the toolchain at `toolchain`, in `target_dir`, and /// install it there as `libtoyos_c.a`. Part of making a sysroot /// (`src/sysroot.rs`), whose key `userland/libc/src` is one of. @@ -20,24 +28,11 @@ pub fn build(root: &Path, toolchain: &Path, target_dir: &Path, arch: Arch) { // The one guest artifact `build::PROFILE` does not reach, so it is the one // place `overflow-checks` is off — and it is linked into std, so it is in - // every userland binary. Left deliberately, on two grounds: CLAUDE.md gives - // the POSIX compatibility layer explicitly relaxed rules, and a flag changed - // here does not move any sysroot's key unless `sysroot::RECIPE` moves with - // it, so the installed archive would not be rebuilt and the manifest would - // then claim something the artifact does not have. - // - // --message-format=json to discover the exact rlib artifacts. + // every userland binary. Left deliberately: CLAUDE.md gives the POSIX + // compatibility layer explicitly relaxed rules. let output = Command::new("cargo") - .args([ - "build", - "--release", - "--target", - arch.userland(), - "--features", - FEATURES, - "--message-format=json", - "--manifest-path", - ]) + .args(BUILD) + .args(["--target", arch.userland(), "--manifest-path"]) .arg(root.join(CRATE).join("Cargo.toml").to_str().unwrap()) .arg("--target-dir") .arg(target_dir) @@ -95,7 +90,8 @@ pub fn build(root: &Path, toolchain: &Path, target_dir: &Path, arch: Arch) { pub fn build_c(root: &Path, toolchain: &Path, target_dir: &Path, arch: Arch) { let target = arch.userland(); let output = Command::new("cargo") - .args(["rustc", "--release", "--target", target, "--crate-type", "staticlib", "--manifest-path"]) + .args(BUILD_C) + .args(["--target", target, "--manifest-path"]) .arg(root.join(CRATE).join("Cargo.toml")) .arg("--target-dir") .arg(target_dir) diff --git a/src/libcxx.rs b/src/libcxx.rs index 0612a136ee7..09c9769dc5d 100644 --- a/src/libcxx.rs +++ b/src/libcxx.rs @@ -15,9 +15,6 @@ use std::process::Command; use crate::arch::Arch; use crate::clang::CSysroot; -/// This file, which the release tag hashes. -pub(crate) const SOURCE: &str = file!(); - /// What of `src/llvm-project` the runtimes' build reads: the runtimes, the CMake /// modules they share with LLVM, and LLVM's libc, whose number parsing libc++ /// compiles in. diff --git a/src/licence.rs b/src/licence.rs index 7550705ef4c..c126de32d6b 100644 --- a/src/licence.rs +++ b/src/licence.rs @@ -1173,12 +1173,7 @@ fn metadata( fn std_library(root: &Path) -> Result { let fork = crate::sysroot::fork_checkout(root); if !fork.join("library/Cargo.toml").exists() { - run( - Command::new("git") - .args(["submodule", "update", "--init", "--depth", "1", "rust"]) - .current_dir(root), - "git submodule update --init --depth 1 rust", - )?; + crate::ensure_shallow_fork(root)?; } Ok(fork.join("library")) } diff --git a/src/llvm.rs b/src/llvm.rs index 9eb587b6acb..a5fae110edf 100644 --- a/src/llvm.rs +++ b/src/llvm.rs @@ -2,17 +2,17 @@ //! one per key on this host, shared by every compiler build that names it. //! //! **An LLVM is a function of its key** ([`key`]): the `src/llvm-project` commit -//! the fork checkout's gitlink names (`compiler::llvm_commit`, which refuses a +//! the fork checkout's gitlink names (`sysroot::gitlink`, which refuses a //! checkout holding what no commit does and a gitlink staged and not committed), //! the committed tree of its `src/bootstrap` (one holding what no commit does is //! refused), the bootstrap configuration below, [`RECIPE`], and the tools the //! host builds it with ([`host_tools`]). `rust/build/llvm//` in the primary -//! is bootstrap's install of that LLVM and its clang, with its LLD in `bin/` -//! beside `llvm-config` and in `src/` the runtimes' sources the C++ runtime is -//! built from (`src/libcxx.rs`) as its commit holds them, made by whichever -//! build first needs it ([`resolve`]), and stored only when it was built from -//! what the key names. Once its [`SOURCE`] file exists it is read-only, its -//! directories as well as its files. +//! is what builds read of bootstrap's install of that LLVM and its clang +//! ([`keep`]), with its LLD in `bin/` beside `llvm-config` and in `src/` the +//! runtimes' sources the C++ runtime is built from (`src/libcxx.rs`) as its +//! commit holds them, made by whichever build first needs it ([`resolve`]), and +//! stored only when it was built from what the key names. Once its [`SOURCE`] +//! file exists it is read-only, its directories as well as its files. //! Every compiler build, the primary's and a worktree's own, names it as the //! host's `llvm-config` with `llvm-has-rust-patches`, so bootstrap builds no //! LLVM and takes LLD from beside it as `rust-lld`; `clang::provision` copies its @@ -41,15 +41,17 @@ use std::process::Command; use std::sync::OnceLock; use crate::buildlock::{Guard, Keyed}; -use crate::compiler::{llvm_commit, LLVM}; +use crate::compiler::LLVM; use crate::keystore::{self, Key}; -use crate::sysroot::{clone_tree, git_bytes, git_out}; +use crate::sysroot::{clone_tree, git_bytes, git_out, gitlink}; use crate::toolchain::{self, host_triple}; /// What changes how a key's sources become an LLVM and is none of the other /// parts: the build's targets and what is kept of it. Moving it moves every key. -const RECIPE: &str = "bootstrap build of src/llvm-project/llvm and src/llvm-project/lld; the install's bin, \ - include and lib, and lld in bin, and the runtimes' sources in src, read-only; 3"; +const RECIPE: &str = "bootstrap build of src/llvm-project/llvm and src/llvm-project/lld; of the install, \ + llvm-config, clang and llvm-ar in bin, and llvm-objcopy on an Apple host, LLVM's headers, \ + every library llvm-config names and clang's resource headers; lld in bin, and the \ + runtimes' sources in src, read-only; 4"; /// What of the caller's environment the LLVM build, and every tool its key /// asks, sees: @@ -85,12 +87,18 @@ const NO_HOST_LIBRARIES: [&str; 12] = [ "LLVM_ENABLE_Z3_SOLVER", ]; -/// What of bootstrap's install an LLVM keeps: `build/` beside them is CMake's -/// tree, which nothing reads once the install is made. -const KEPT: [&str; 3] = ["bin", "include", "lib"]; +/// The tools of an LLVM's `bin` a build runs: bootstrap asks `llvm-config` how +/// to link LLVM and takes `lld` as `rust-lld`; `clang::provision` copies `clang` +/// and `llvm-ar`, and on an Apple host [`APPLE_TOOL`]. +const TOOLS: [&str; 4] = ["llvm-config", "lld", "clang", "llvm-ar"]; -/// What a compiler build and `clang::provision` read of an LLVM. -const TOOLS: [&str; 4] = ["bin/llvm-config", "bin/lld", "bin/clang", "bin/llvm-ar"]; +/// What an Apple host's toolchain carries as `rust-objcopy`, which rustc runs to +/// strip a Darwin binary (`compiler/rustc_codegen_ssa/src/back/link.rs`). +pub(crate) const APPLE_TOOL: &str = "llvm-objcopy"; + +/// LLVM's headers: the compiler's LLVM wrapper compiles against them, where +/// `llvm-config --cxxflags` names the install's `include`. +const HEADERS: [&str; 2] = ["include/llvm", "include/llvm-c"]; /// The file a finished LLVM carries last, naming its key. A directory without /// it is a build that did not finish. @@ -131,7 +139,7 @@ pub fn key(fork: &Path) -> Key { fn key_of(fork: &Path, recipe: &str, config: &str, tools: &str) -> Key { refuse_uncommitted_bootstrap(fork); let bootstrap = git_out(fork, &["rev-parse", &format!("HEAD:{BOOTSTRAP}")]); - Key::of([recipe, config, &llvm_commit(fork), bootstrap.trim(), tools].join("\n\0\n").as_bytes()) + Key::of([recipe, config, &gitlink(fork, LLVM), bootstrap.trim(), tools].join("\n\0\n").as_bytes()) } /// Give `command` nothing of this process's environment but [`ENVIRONMENT`]. @@ -150,7 +158,8 @@ struct HostTools { cc: PathBuf, cxx: PathBuf, /// The C and C++ compilers and CMake, each by its resolved path and all its - /// `--version` says, and on macOS the SDK path and version `xcrun` resolves. + /// `--version` says, n2 by the pin it is installed from, and on macOS the + /// SDK path and version `xcrun` resolves. identity: String, } @@ -168,6 +177,7 @@ fn tools_with(xcrun: impl Fn(&str) -> String) -> HostTools { let mut version = Command::new(tool); identity += &format!("{}\n{}", tool.display(), asked(version.arg("--version"))); } + identity += &format!("n2 {}\n", crate::n2::N2.join(" ")); if host_triple().ends_with("apple-darwin") { for question in ["--show-sdk-path", "--show-sdk-version"] { identity += &xcrun(question); @@ -226,13 +236,22 @@ fn held_with(root: &Path, rust_dir: &Path, fork: &Path, build: impl Fn(&Path) -> Llvm { dir, _using: using } } +/// [`TOOLS`], and on an Apple host [`APPLE_TOOL`]. +fn tools() -> impl Iterator { + TOOLS.into_iter().chain(host_triple().ends_with("apple-darwin").then_some(APPLE_TOOL)) +} + /// Why `dir` is not a finished LLVM, if it is not. -fn defect(dir: &Path) -> Option { +pub(crate) fn defect(dir: &Path) -> Option { if !dir.join(SOURCE).is_file() { return Some(format!("{} carries no {SOURCE}", dir.display())); } - let kept = KEPT.iter().map(|k| dir.join(k)).chain(crate::libcxx::SOURCES.iter().map(|s| dir.join("src").join(s))); - let tools = TOOLS.iter().map(|t| dir.join(t)).filter(|p| !p.is_file()); + let kept = HEADERS + .iter() + .chain(&["lib/clang"]) + .map(|k| dir.join(k)) + .chain(crate::libcxx::SOURCES.iter().map(|s| dir.join("src").join(s))); + let tools = tools().map(|t| dir.join("bin").join(t)).filter(|p| !p.is_file()); let gone: Vec = kept.filter(|p| !p.is_dir()).chain(tools).map(|p| p.display().to_string()).collect(); (!gone.is_empty()).then(|| format!("{} carries no {}", dir.display(), gone.join(", "))) } @@ -258,9 +277,7 @@ fn place(fork: &Path, key: &Key, dir: &Path, build: &impl Fn(&Path) -> PathBuf) let host = host_triple(); let partial = dir.with_extension("partial"); keystore::remove(&partial); - for part in KEPT { - clone_tree(&built.join(&host).join("llvm").join(part), &partial.join(part)); - } + keep(&built.join(&host).join("llvm"), &partial); let lld = built.join(&host).join("lld/bin/lld"); fs::copy(&lld, partial.join("bin/lld")) .unwrap_or_else(|e| panic!("copy {} -> {}: {e}", lld.display(), partial.join("bin/lld").display())); @@ -274,7 +291,7 @@ fn place(fork: &Path, key: &Key, dir: &Path, build: &impl Fn(&Path) -> PathBuf) ); let checkout = fork.join(LLVM); assert!(checkout.join(".git").exists(), "the LLVM build left no checkout at {}", checkout.display()); - let (built_from, commit) = (git_out(&checkout, &["rev-parse", "HEAD"]), llvm_commit(fork)); + let (built_from, commit) = (git_out(&checkout, &["rev-parse", "HEAD"]), gitlink(fork, LLVM)); // Bootstrap's `Llvm` step checks the gitlink's commit out before it builds, // so a checkout behind it, the key never reads, is moved first. assert!( @@ -295,6 +312,45 @@ fn place(fork: &Path, key: &Key, dir: &Path, build: &impl Fn(&Path) -> PathBuf) fs::remove_dir_all(&built).unwrap_or_else(|e| panic!("remove {}: {e}", built.display())); } +/// Copy into `to` what builds read of the LLVM installed at `install`: [`tools`] +/// but `lld`, which is LLD's own build's, each a file whatever link it is +/// installed as; [`HEADERS`]; every library its `llvm-config` names, since a +/// compiler links LLVM through it and it refuses to name one that is absent; +/// and clang's resource headers. +fn keep(install: &Path, to: &Path) { + let bin = to.join("bin"); + fs::create_dir_all(&bin).unwrap_or_else(|e| panic!("create {}: {e}", bin.display())); + for tool in tools().filter(|tool| *tool != "lld") { + let from = install.join("bin").join(tool); + fs::copy(&from, bin.join(tool)).unwrap_or_else(|e| panic!("copy {} -> {}: {e}", from.display(), bin.display())); + } + for headers in HEADERS { + clone_tree(&install.join(headers), &to.join(headers)); + } + let lib = to.join("lib"); + fs::create_dir_all(&lib).unwrap_or_else(|e| panic!("create {}: {e}", lib.display())); + for library in libraries(install) { + let name = library.file_name().unwrap_or_else(|| panic!("{} names no file", library.display())); + fs::copy(&library, lib.join(name)).unwrap_or_else(|e| panic!("copy {} -> {}: {e}", library.display(), lib.display())); + } + let resource = crate::clang::resource_version(install); + let version = resource.file_name().unwrap_or_else(|| panic!("{} names no version", resource.display())); + clone_tree(&resource.join("include"), &lib.join("clang").join(version).join("include")); +} + +/// Every library the `llvm-config` of the LLVM installed at `install` names. +fn libraries(install: &Path) -> Vec { + let config = install.join("bin/llvm-config"); + let mut command = Command::new(&config); + command.args(["--link-static", "--libfiles"]); + clear(&mut command); + let out = command.output().unwrap_or_else(|e| panic!("run {}: {e}", config.display())); + assert!(out.status.success(), "{command:?} failed: {}", String::from_utf8_lossy(&out.stderr)); + let named: Vec = String::from_utf8_lossy(&out.stdout).split_whitespace().map(PathBuf::from).collect(); + assert!(!named.is_empty(), "{command:?} named no library"); + named +} + /// Write `paths` as `commit` holds them, from the repository at `checkout`, /// under `dest`: through an index of their own and with no sparse pattern, so /// nothing the checkout holds beside the commit, tracked, ignored or left out, @@ -458,15 +514,27 @@ mod tests { /// Bootstrap's stand-in: what its LLVM and LLD builds leave in the build /// directory, CMake's tree among them. fn fake_build(fork: &Path) -> PathBuf { + use std::os::unix::fs::PermissionsExt; let built = fork.join("build/toyos-llvm"); let _ = fs::remove_dir_all(&built); let install = built.join(host_triple()).join("llvm"); - for tool in ["llvm-config", "clang-22", "llvm-ar"] { + for tool in ["clang-22", "llvm-ar", "llvm-objcopy", "opt"] { write(&install.join("bin").join(tool), &format!("the {tool}")); } std::os::unix::fs::symlink("clang-22", install.join("bin/clang")).unwrap(); + // What a real `llvm-config --libfiles` answers: the component libraries, + // never clang's, nor one no component is. + let named = ["libLLVMCore.a", "libLLVMSupport.a"].map(|lib| install.join("lib").join(lib).display().to_string()); + let config = install.join("bin/llvm-config"); + write(&config, &format!("#!/bin/sh\necho {}\n", named.join(" "))); + fs::set_permissions(&config, fs::Permissions::from_mode(0o755)).unwrap(); + for lib in ["libLLVMCore.a", "libLLVMSupport.a", "libLLVMTableGen.a", "libclangBasic.a"] { + write(&install.join("lib").join(lib), lib); + } write(&install.join("include/llvm/Config/llvm-config.h"), "#define LLVM_VERSION_MAJOR 22"); - write(&install.join("lib/libLLVMCore.a"), "core"); + write(&install.join("include/llvm-c/Core.h"), "LLVMContextRef LLVMContextCreate(void);"); + write(&install.join("include/clang/Basic/Version.h"), "#define CLANG_VERSION 22"); + write(&install.join("lib/cmake/llvm/LLVMConfig.cmake"), "set(LLVM_PACKAGE_VERSION 22)"); write(&install.join("lib/clang/22/include/stddef.h"), "typedef long ptrdiff_t;"); write(&install.join("build/CMakeCache.txt"), "the build tree"); write(&built.join(host_triple()).join("lld/bin/lld"), "the lld"); @@ -542,7 +610,6 @@ mod tests { assert_eq!(makes.get(), 1); assert_eq!(defect(&la.dir), None); assert_eq!(fs::read_to_string(la.dir.join("bin/lld")).unwrap(), "the lld"); - assert_eq!(fs::read_link(la.dir.join("bin/clang")).unwrap(), Path::new("clang-22")); assert!(la.dir.join("lib/clang/22/include/stddef.h").is_file()); assert_eq!(fs::read_to_string(la.dir.join("src/libcxx/CMakeLists.txt")).unwrap(), "the libcxx of A", "the runtimes' sources are not the commit's"); assert!(!la.dir.join("build").exists(), "CMake's tree was kept"); @@ -557,6 +624,44 @@ mod tests { assert_eq!(snapshot(&store(&rust_dir)), before, "an LLVM was written after it was whole"); } + /// **An LLVM keeps what builds read of the install and nothing else**: the + /// tools a build runs, `clang` as the file its link names; LLVM's headers; + /// every library `llvm-config` names, and no other; clang's resource + /// headers. No other tool, clang's headers and libraries, nor CMake's + /// package files. + #[test] + fn an_llvm_keeps_what_builds_read_and_nothing_else() { + let scratch = Scratch::new("llvm-kept"); + let (_primary, rust_dir, [_same, a, _b]) = estate_built(&scratch); + let dir = choose(&a, &rust_dir, &a.join("rust"), fake_build).dir; + let apple = host_triple().ends_with("apple-darwin"); + let mut files: Vec = snapshot(&dir) + .into_iter() + .map(|(path, _)| path.strip_prefix(&dir).unwrap().display().to_string()) + .filter(|path| !path.starts_with("src/")) + .collect(); + files.sort(); + let mut want = vec![ + "SOURCE", + "bin/clang", + "bin/lld", + "bin/llvm-ar", + "bin/llvm-config", + "include/llvm-c/Core.h", + "include/llvm/Config/llvm-config.h", + "lib/clang/22/include/stddef.h", + "lib/libLLVMCore.a", + "lib/libLLVMSupport.a", + ]; + if apple { + want.push("bin/llvm-objcopy"); + } + want.sort(); + assert_eq!(files, want); + assert_eq!(fs::read_to_string(dir.join("bin/clang")).unwrap(), "the clang-22"); + assert!(!fs::symlink_metadata(dir.join("bin/clang")).unwrap().file_type().is_symlink(), "clang is the link, not the file"); + } + /// **A placed LLVM cannot be written**, through its own path or through a /// link bootstrap makes to one of its files, and nothing in it can be /// removed, replaced or added. @@ -653,7 +758,7 @@ mod tests { /// **The tools the key names are the host's**: the C and C++ compilers the /// configuration names by path, and CMake, each by the file it resolves to - /// and what its `--version` says; on macOS, the SDK. + /// and what its `--version` says; n2 by its pin; on macOS, the SDK. #[test] fn the_key_names_the_host_s_tools() { let tools = host_tools(); @@ -664,6 +769,7 @@ mod tests { assert!(lines[at + 1].chars().any(|c| c.is_ascii_digit()), "{} said no version: {}", tool.display(), tools.identity); } assert!(lines.iter().any(|l| l.starts_with("cmake version")), "{}", tools.identity); + assert!(lines.contains(&format!("n2 {}", crate::n2::N2.join(" ")).as_str()), "no n2: {}", tools.identity); if host_triple().ends_with("apple-darwin") { assert!(lines.iter().any(|l| l.ends_with(".sdk") && Path::new(l).is_dir()), "no SDK: {}", tools.identity); } diff --git a/src/n2.rs b/src/n2.rs index 8ce65fc3092..0fde6dd2728 100644 --- a/src/n2.rs +++ b/src/n2.rs @@ -5,9 +5,6 @@ use std::path::{Path, PathBuf}; use std::process::Command; -/// This file, which the release tag hashes. -pub(crate) const SOURCE: &str = file!(); - /// cargo's install of n2 but for its `--root`: without its default jemalloc, /// which is C. pub(crate) const N2: [&str; 7] = [ diff --git a/src/release.rs b/src/release.rs index 7e79d113743..2ba36b343ec 100644 --- a/src/release.rs +++ b/src/release.rs @@ -1,61 +1,76 @@ -//! The toolchain release: the tag a tree's toolchain is published under, the -//! tarball it ships as, and how a runner installs one. +//! The toolchain a runner builds with, by the build system's own keys, and the +//! release main publishes of it. //! -//! **The tag is the content hash of [`trees`].** A tree -//! whose toolchain somebody already built finds it published; a tree that moved -//! any of them asks for a tag nobody has, and `cargo run -- --ci toolchain` -//! builds it. Publishing is idempotent because the tag *is* the content. +//! **A toolchain is four stores, each a cache entry of the key the build system +//! files it under** ([`LAYERS`]). `toolchain.yml` restores each by the key +//! [`toolchain`] wrote, builds what none restored ([`bootstrap`]) and saves only +//! what it built. GitHub's ref scoping is the provenance: an entry a run on main +//! saved is restored on every ref, and any other run saves only into its own +//! ref's scope. //! -//! The release is `x86_64-unknown-linux-gnu`'s and is built on a GitHub-hosted -//! `ubuntu-24.04`; any other host is refused rather than publishing a tarball -//! nobody can install. A dev host never installs one: its build system -//! bootstraps from `rust/` as always. +//! **A guest job installs the sysroot its restore step put down** ([`install`]), +//! and main's nightly packs that store into the release a consumer outside CI +//! installs, then moves the SDK alias onto it ([`release`]). Run as any other +//! job, that is refused before it reads anything; run on a tree main has moved +//! past, it puts nothing up. +//! +//! A dev host installs none: its build system builds its own from `rust/`. use std::fs; +use std::io::Write; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; +use std::process::Command; +use serde_json::Value; use sha2::{Digest, Sha256}; use toyos_tmpdir::TempDir; -use crate::toolchain::HOSTED_ARCH; +use crate::buildlock::Keyed; +use crate::keystore::Key; +use crate::sdkversion::Release; -/// What the tag hashes, as `git rev-parse HEAD:` names them. The last is -/// this file. -fn trees() -> Vec<&'static str> { - std::iter::once("rust") - .chain(crate::sysroot::SYSROOT_SOURCES) - .chain(crate::sysroot::SYSROOT_MANIFESTS) - .chain([crate::clang::SOURCE, crate::libcxx::SOURCE, crate::n2::SOURCE, file!()]) - .collect() -} +const ASSET: &str = "toyos-toolchain.tar.gz"; -/// The one asset a release carries. -const ASSET: &str = "toyos-toolchain.tar.zst"; +/// Main's publisher: the one workflow [`release`] runs under. +const PUBLISHER: &str = ".github/workflows/nightly.yml"; -/// The triple the release's host half runs on. const HOST: &str = "x86_64-unknown-linux-gnu"; /// The oldest glibc a consumer needs: `ubuntu-24.04`'s. A build naming a newer -/// one is refused rather than published. +/// one is refused. const GLIBC_FLOOR: (u32, u32) = (2, 39); -/// `toolchain-linux-x86_64-<16 hex>`: the first 16 hex digits of the SHA-256 of -/// what `git rev-parse` prints for [`trees`], newline-terminated lines and all. -pub fn tag(root: &Path) -> Result { - let out = Command::new("git") - .arg("rev-parse") - .args(trees().iter().map(|t| format!("HEAD:{t}"))) - .current_dir(root) - .output() - .map_err(|e| format!("git rev-parse: {e}"))?; - if !out.status.success() { - return Err(format!( - "git rev-parse of the toolchain's trees: {}", - String::from_utf8_lossy(&out.stderr).trim() - )); +/// What every request the build system makes says it comes from. +const USER_AGENT: &str = "toyos-build (https://github.com/ToyOSOrg/ToyOS)"; + +/// The stores a toolchain is, in the order a build makes them, each under the +/// name its cache entry and its job's outputs carry. +const LAYERS: [(Keyed, &str); 4] = [ + (Keyed::Llvm, "llvm"), + (Keyed::Compiler, "compiler"), + (Keyed::Freestanding, "freestanding"), + (Keyed::Sysroot, "sysroot"), +]; + +/// One of [`LAYERS`] of this tree's toolchain: the key the build system files +/// it under, and the paths it is, relative to the checkout. +struct Layer { + kind: Keyed, + name: &'static str, + key: Key, + paths: Vec, +} + +impl Layer { + /// Its cache entry's key. + fn entry(&self) -> String { + format!("toolchain-{}-{}", self.name, self.key) } - Ok(format!("toolchain-linux-x86_64-{}", &sha256_hex(&out.stdout)[..16])) +} + +/// The release of the sysroot `key` names. +fn tag(key: &Key) -> String { + format!("toolchain-linux-x86_64-{key}") } pub(crate) fn sha256_hex(bytes: &[u8]) -> String { @@ -66,41 +81,167 @@ fn on_runner() -> bool { std::env::var("GITHUB_ACTIONS").is_ok_and(|v| v == "true") } -/// The API URL of `tag`'s asset, or `None` while it has none: `gh release -/// create` makes the release before it uploads, so a tag that exists is not yet -/// an installable toolchain. `curl` and not `gh`, because the guest containers -/// carry no `gh`. -fn asset_url(tag: &str) -> Result, String> { - let repo = std::env::var("GITHUB_REPOSITORY").unwrap_or_else(|_| "ToyOSOrg/ToyOS".into()); - let mut curl = Command::new("curl"); - curl.args(["-sSL", &format!("https://api.github.com/repos/{repo}/releases/tags/{tag}")]); - if let Ok(token) = std::env::var("GH_TOKEN") { - curl.args(["-H", &format!("Authorization: Bearer {token}")]); +/// The HTTP client of every request the build system makes: rustls on ring +/// with webpki's roots, as ureq configures it; a status is an answer, not an +/// error. +pub(crate) fn agent() -> ureq::Agent { + ureq::Agent::config_builder().user_agent(USER_AGENT).http_status_as_error(false).build().new_agent() +} + +/// Whether this job is main's publisher — [`PUBLISHER`] on main, scheduled or +/// dispatched — as the runner names its workflow and event; refused by name if +/// it is not. +fn publisher(workflow: Option<&str>, event: Option<&str>, repo: &str) -> Result<(), String> { + let mains = format!("{repo}/{PUBLISHER}@refs/heads/main"); + match (workflow, event) { + (Some(workflow), Some("schedule" | "workflow_dispatch")) if workflow == mains => Ok(()), + (workflow, event) => Err(format!( + "only {mains}, scheduled or dispatched, publishes a toolchain, and this job is {} on {}", + workflow.unwrap_or("no workflow"), + event.unwrap_or("no event") + )), + } +} + +/// This tree's toolchain as [`LAYERS`], keyed from its sources alone, before +/// any store is there. +fn layers(root: &Path) -> Vec { + let rust_dir = root.join("rust"); + let llvm = crate::llvm::key(&rust_dir); + let compiler = crate::compiler::primary_key(&rust_dir); + let freestanding = crate::sysroot::freestanding_key(root, &compiler, &rust_dir); + let sysroot = crate::sysroot::key(root, &freestanding); + LAYERS + .iter() + .map(|&(kind, name)| { + let (key, paths) = match kind { + Keyed::Llvm => (llvm.clone(), vec![crate::llvm::store(&rust_dir).join(&llvm)]), + Keyed::Compiler => ( + compiler.clone(), + vec![crate::toolchain::stage2(&rust_dir), crate::compiler::primary_record(&rust_dir)], + ), + Keyed::Freestanding => { + (freestanding.clone(), vec![crate::sysroot::freestanding_dir(&rust_dir).join(&freestanding)]) + } + Keyed::Sysroot => (sysroot.clone(), vec![crate::sysroot::sysroots_dir(&rust_dir).join(&sysroot)]), + }; + let paths = paths + .iter() + .map(|path| path.strip_prefix(root).unwrap_or_else(|_| panic!("{} is outside the checkout", path.display()))) + .map(Path::to_path_buf) + .collect(); + Layer { kind, name, key, paths } + }) + .collect() +} + +/// Why `layer` is not whole in `root`, as the build that makes it decides, if +/// it is not. +fn defect(root: &Path, layer: &Layer) -> Option { + let dir = root.join(&layer.paths[0]); + match layer.kind { + Keyed::Llvm => crate::llvm::defect(&dir), + Keyed::Compiler => match fs::read(root.join(&layer.paths[1])) { + Ok(record) if Key::of(&record) == layer.key => crate::toolchain::toolchain_defect(&dir), + _ => Some(format!("{} records no compiler {}", layer.paths[1].display(), layer.key)), + }, + Keyed::Freestanding => crate::sysroot::unpublished(&dir), + Keyed::Sysroot => crate::sysroot::unfinished(&dir), } - let out = curl.output().map_err(|e| format!("curl: {e}"))?; - if !out.status.success() { - return Err(format!("curl asked for {tag} and failed: {}", out.status)); +} + +/// The file a job's next steps read this step's outputs from: a runner's +/// `$GITHUB_OUTPUT`. +fn step_outputs() -> Result { + std::env::var_os("GITHUB_OUTPUT") + .map(PathBuf::from) + .ok_or_else(|| "not a runner: a dev host builds its own toolchain with `cargo run`".to_string()) +} + +/// Append `text` to the step outputs at `file`. +fn tell(file: &Path, text: &str) -> Result<(), String> { + fs::OpenOptions::new() + .append(true) + .open(file) + .and_then(|mut opened| opened.write_all(text.as_bytes())) + .map_err(|e| format!("{}: {e}", file.display())) +} + +/// `cargo run -- --ci toolchain`: each of this tree's [`LAYERS`] as the cache +/// entry its job restores and saves, told to the job's next steps +/// ([`outputs`]). +pub fn toolchain(root: &Path) -> Result { + let file = step_outputs()?; + crate::ensure_shallow_fork(root)?; + let layers = layers(root); + tell(&file, &outputs(&layers))?; + Ok(layers.iter().map(|layer| format!("{} {}", layer.name, layer.key)).collect::>().join(", ")) +} + +/// Each layer's entry as `-key` and its paths, one a line, as +/// `-path`. +fn outputs(layers: &[Layer]) -> String { + let mut text = String::new(); + for layer in layers { + let paths: Vec = layer.paths.iter().map(|path| path.display().to_string()).collect(); + text += &format!("{0}-key={1}\n{0}-path< Option { - release["assets"] - .as_array()? +/// `cargo run -- --ci bootstrap`: this tree's toolchain made whole from what its +/// job restored, and each layer told to the job's save steps as built or kept +/// ([`built`]). A sysroot restored is all a guest job reads, so then nothing is +/// built. A layer restored and not whole is refused, since its key reads less +/// than its build does, and so is one the build left not whole under its key. +pub fn bootstrap(root: &Path) -> Result { + let file = step_outputs()?; + let layers = layers(root); + let restored: Vec = layers.iter().map(|layer| root.join(&layer.paths[0]).exists()).collect(); + whole(&layers, &restored, |layer| defect(root, layer)).map_err(|why| format!("restored, {why}"))?; + if !restored[3] { + let mut lock = crate::buildlock::shared(root, "the toolchain"); + drop(crate::toolchain::ensure(root, &mut lock, false)); + whole(&layers, &[true; 4], |layer| defect(root, layer)).map_err(|why| format!("built, {why}"))?; + } + tell(&file, &built(&layers, &restored))?; + let said: Vec = layers .iter() - .find(|a| a["name"] == ASSET) - .and_then(|a| a["url"].as_str()) - .map(str::to_string) + .zip(&restored) + .map(|(layer, was)| match (*was, restored[3]) { + (true, _) => format!("{} {} restored", layer.name, layer.key), + (false, true) => format!("{} {} not needed", layer.name, layer.key), + (false, false) => format!("{} {} built", layer.name, layer.key), + }) + .collect(); + Ok(said.join(", ")) +} + +/// Refused where a layer `which` names is not whole, as `defect` finds it: its +/// key reads less than its build does, and a build under that key could never +/// be saved over the entry. +fn whole(layers: &[Layer], which: &[bool], defect: impl Fn(&Layer) -> Option) -> Result<(), String> { + for (layer, _) in layers.iter().zip(which).filter(|(_, named)| **named) { + if let Some(why) = defect(layer) { + return Err(format!("{} {} is not whole: {why}", layer.name, layer.key)); + } + } + Ok(()) } -fn sleep(seconds: u64) { - std::thread::sleep(std::time::Duration::from_secs(seconds)); +/// Each layer as `=built` where `restored` says its job restored neither +/// it nor the sysroot, and `=kept` otherwise: what the job saves. +fn built(layers: &[Layer], restored: &[bool]) -> String { + let sysroot = restored[3]; + layers + .iter() + .zip(restored) + .map(|(layer, restored)| format!("{}={}\n", layer.name, if *restored || sysroot { "kept" } else { "built" })) + .collect() } -/// Install this tree's published toolchain as rustup's `toyos`, on a runner. +/// Install the sysroot its job restored ([`lay_out`]) as rustup's `toyos`, on +/// a runner. /// /// Off a runner this says so and does nothing: the build system owns the dev /// host's toolchain. @@ -108,74 +249,52 @@ pub fn install(root: &Path) -> Result { if !on_runner() { return Ok("not a runner: the build system uses this checkout's own toolchain".into()); } - if std::env::var("GH_TOKEN").is_err() { - return Err("GH_TOKEN is unset, and the release download is authenticated".into()); - } - let tag = tag(root)?; - let mut url = None; - for _ in 0..10 { - url = asset_url(&tag)?; - if url.is_some() { - break; - } - println!("{tag} carries no {ASSET} yet; asking again in 15 s"); - sleep(15); - } - let url = url.ok_or_else(|| { - format!( - "{tag} carries no {ASSET}, so there is nothing to install: the nightly's `build` \ - job is what publishes one" - ) - })?; - let token = std::env::var("GH_TOKEN").expect("checked above"); - let staging = TempDir::new("toolchain-install"); - let tarball = staging.join(ASSET); - let into = root.join("rust/build"); - fs::create_dir_all(&into).map_err(|e| format!("{}: {e}", into.display()))?; - // The retry is on the transfer and the unpack together: a truncated body is - // a `zstd` failure, not a `curl` one. - let mut last = String::new(); - for attempt in 1..=3 { - let fetched = Command::new("curl") - .args(["-sSL", "--retry", "3", "--retry-all-errors", "--retry-delay", "5"]) - .args(["-H", &format!("Authorization: Bearer {token}")]) - .args(["-H", "Accept: application/octet-stream", &url, "-o"]) - .arg(&tarball) - .status() - .map_err(|e| format!("curl: {e}"))?; - match fetched.success().then(|| unpack(&tarball, &into)) { - Some(Ok(())) => { - let stage2 = into.join(format!("{HOST}/stage2")); - run(Command::new("rustup").args(["toolchain", "link", "toyos"]).arg(&stage2))?; - run(Command::new(stage2.join("bin/rustc")).arg("-vV"))?; - return Ok(format!("installed {tag} as `toyos`")); - } - Some(Err(e)) => last = e, - None => last = format!("curl exited {fetched}"), - } - println!("toolchain download attempt {attempt} failed: {last}"); - sleep(10); - } - Err(format!("the toolchain did not download and unpack in three attempts: {last}")) + let key = lay_out(root)?; + let stage2 = crate::toolchain::stage2(&root.join("rust")); + run(Command::new("rustup").args(["toolchain", "link", "toyos"]).arg(&stage2))?; + run(Command::new(stage2.join("bin/rustc")).arg("-vV"))?; + Ok(format!("installed sysroot {key} as `toyos`")) } -/// `zstd -dc | tar -C -x`. -fn unpack(tarball: &Path, into: &Path) -> Result<(), String> { - let mut zstd = Command::new("zstd") - .arg("-dc") - .arg(tarball) - .stdout(Stdio::piped()) - .spawn() - .map_err(|e| format!("zstd: {e}"))?; - let stream = zstd.stdout.take().expect("piped"); - let tar = Command::new("tar").arg("-C").arg(into).arg("-x").stdin(stream).status(); - let zstd = zstd.wait().map_err(|e| format!("zstd: {e}"))?; - let tar = tar.map_err(|e| format!("tar: {e}"))?; - if zstd.success() && tar.success() { - Ok(()) - } else { - Err(format!("unpacking: zstd exited {zstd}, tar {tar}")) +/// Lay the one sysroot under `rust/build/sysroots`, which its job's restore +/// step put there, out as this checkout's installed toolchain +/// (`toolchain::Owner::Installed`): at `stage2`, with the witness it records and +/// its [`manifest`]. Refused unless that witness is this tree's. +fn lay_out(root: &Path) -> Result { + let rust_dir = root.join("rust"); + let store = crate::sysroot::sysroots_dir(&rust_dir); + let restored = fs::read_dir(&store) + .and_then(|entries| entries.map(|entry| entry.map(|entry| entry.path())).collect::>>()) + .map_err(|e| format!("{}: {e}", store.display()))?; + let [sysroot] = restored.as_slice() else { + return Err(format!( + "{} holds {} entries, and a job installs the one sysroot it restored: {restored:?}", + store.display(), + restored.len() + )); + }; + let key = sysroot + .file_name() + .and_then(|name| name.to_str()) + .and_then(Key::parse) + .ok_or_else(|| format!("{} is named by no key", sysroot.display()))?; + let witness = crate::sysroot::recorded_witness(sysroot)?; + if witness != crate::sysroot::witness(root) { + return Err(format!( + "sysroot {key} was built from other sources than this tree's, and this tree's key names it: \ + the key reads less than the sysroot is built from (`src/sysroot.rs`)" + )); } + let stage2 = crate::toolchain::stage2(&rust_dir); + let host = stage2.parent().unwrap_or_else(|| panic!("{} has no parent", stage2.display())); + fs::create_dir_all(host).map_err(|e| format!("{}: {e}", host.display()))?; + fs::rename(sysroot, &stage2).map_err(|e| format!("{} -> {}: {e}", sysroot.display(), stage2.display()))?; + for (path, text) in + [(crate::toolchain::witness_path(&rust_dir), witness), (crate::toolchain::manifest_path(&rust_dir), manifest(&tag(&key)))] + { + fs::write(&path, text).map_err(|e| format!("{}: {e}", path.display()))?; + } + Ok(key) } fn run(cmd: &mut Command) -> Result<(), String> { @@ -183,124 +302,223 @@ fn run(cmd: &mut Command) -> Result<(), String> { status.success().then_some(()).ok_or_else(|| format!("{cmd:?} exited {status}")) } -/// Whether `gh` says `tag` carries [`ASSET`]. -fn published(root: &Path, tag: &str) -> bool { - Command::new("gh") - .args(["release", "view", tag, "--json", "assets", "--jq", ".assets[].name"]) - .current_dir(root) - .output() - .is_ok_and(|o| String::from_utf8_lossy(&o.stdout).lines().any(|l| l == ASSET)) +/// `cargo run -- --ci release`: the sysroot main's nightly restored, put up as +/// the release a consumer outside CI installs, and the `sdk-` alias +/// moved onto it. Refused before anything is read unless this job is main's +/// publisher; a tree main has moved past puts nothing up ([`sdk_at_tip`]). +pub fn release(root: &Path) -> Result { + let var = |name| std::env::var(name).ok(); + let repo = var("GITHUB_REPOSITORY").ok_or("GITHUB_REPOSITORY is unset: only a runner publishes a toolchain")?; + release_as(root, &repo, var("GITHUB_WORKFLOW_REF").as_deref(), var("GITHUB_EVENT_NAME").as_deref()) } -/// `cargo run -- --ci toolchain`: make sure this tree's toolchain is published, -/// building it if nobody has; on `main`, also move the `sdk-` alias a -/// consumer pins onto it. -pub fn ensure_published(root: &Path) -> Result { - let tag = tag(root)?; - println!("this tree's toolchain: {tag}"); +/// [`release`] of `repo`, run as the job the runner names by its workflow and +/// event. +fn release_as(root: &Path, repo: &str, workflow: Option<&str>, event: Option<&str>) -> Result { + publisher(workflow, event, repo)?; if !(cfg!(target_os = "linux") && crate::arch::Arch::HOST == Some(crate::arch::Arch::X86_64)) { - return Err(format!( - "the release is {HOST}'s and this host is not one; a tarball built here would \ - install nowhere" - )); + return Err(format!("a release is {HOST}'s and this host is not one; a tarball packed here would install nowhere")); } - let tmp = TempDir::new("toolchain-publish"); - let manifest = manifest(root, &tag); - let notes = notes(root, &tag, &manifest)?; - fs::write(tmp.join("TOOLCHAIN"), &manifest).map_err(|e| e.to_string())?; - fs::write(tmp.join("notes.md"), ¬es).map_err(|e| e.to_string())?; - println!("{manifest}"); - - let mut said = if published(root, &tag) { - format!("{tag} is already published") - } else { - build(root, &tag, &tmp)?; - format!("{tag} built and published") + let head = crate::sysroot::git_out(root, &["rev-parse", "HEAD"]); + let tip = || crate::sysroot::git_out(root, &["ls-remote", "origin", "refs/heads/main"]); + let Some(sdk) = sdk_at_tip(|| crate::sdkversion::plan(root), tip, head.trim())? else { + return Ok(format!("main has moved past {}, and its tip's nightly is the one that publishes", head.trim())); }; - if std::env::var("GITHUB_REF").is_ok_and(|r| r == "refs/heads/main") { - said.push_str(&format!("; {}", alias(root, &manifest, &tmp)?)); - } - Ok(said) -} - -/// Bootstrap, check the glibc floor, package, publish, and wait for the asset. -fn build(root: &Path, tag: &str, tmp: &Path) -> Result<(), String> { - run(Command::new("git").args(["submodule", "update", "--init", "rust"]).current_dir(root))?; - // Bootstrap takes `HEAD^1` as the upstream commit whose artifacts to fetch - // when it sees GitHub Actions; in this fork that is our own merge, which - // rust-lang's CI never built. - run(Command::new("cargo") - .args(["run", "--", "--build-only"]) - .env_remove("GITHUB_ACTIONS") - .env_remove("CI") - .current_dir(root))?; - - // What ships as `{HOST}/stage2` is the sysroot that build compiled against: - // the compiler with the guest libraries and `libtoyos_c.a` this tree's - // sources name, recorded beside the witness an installer checks it by. - let build = root.join("rust/build"); - let key = crate::keystore::recorded(root, crate::buildlock::Keyed::Sysroot).ok_or("the build recorded no sysroot key")?; - let sysroot = format!("sysroots/{key}"); - let stage2 = build.join(&sysroot); - fs::write(build.join("toyos-sysroot-witness"), crate::sysroot::witness(root)) - .map_err(|e| format!("recording the sysroot's witness: {e}"))?; - let need = shipped_glibc(&stage2)?; + let key = lay_out(root)?; + let rust_dir = root.join("rust"); + let need = shipped_glibc(&crate::toolchain::stage2(&rust_dir))?; if need > GLIBC_FLOOR { return Err(format!( - "the host half needs GLIBC_{}.{} and the release states {}.{}: build it on the \ - oldest supported glibc, or move GLIBC_FLOOR deliberately", + "the host half needs GLIBC_{}.{} and a release states {}.{}: build it on the oldest supported \ + glibc, or move GLIBC_FLOOR deliberately", need.0, need.1, GLIBC_FLOOR.0, GLIBC_FLOOR.1 )); } - fs::copy(tmp.join("TOOLCHAIN"), build.join("TOOLCHAIN")).map_err(|e| e.to_string())?; - - // `lib/rustlib/` and the sysroot's `bin/cargo` are links into this - // runner's own toolchain; `Owner::Installed` recreates both. GNU tar's - // `--transform` renames the sysroot to the path an installer links. + let tmp = TempDir::new("toolchain-release"); let tarball = tmp.join(ASSET); - let mut tar = Command::new("tar") - .arg("-C") - .arg(&build) - .arg(format!("--exclude={}/stage2/lib/rustlib/{HOST}", HOSTED_ARCH.userland())) - .arg(format!("--exclude={sysroot}/bin/cargo")) - .arg(format!("--transform=s,^{sysroot},{HOST}/stage2,")) - .args(["-c", &sysroot, &format!("{}/stage2", HOSTED_ARCH.userland())]) - .args(["toyos-sysroot-witness", "TOOLCHAIN"]) - .stdout(Stdio::piped()) - .spawn() - .map_err(|e| format!("tar: {e}"))?; - let stream = tar.stdout.take().expect("piped"); - let zstd = Command::new("zstd").args(["-T0", "-3", "-f", "-o"]).arg(&tarball).stdin(stream).status(); - let tar = tar.wait().map_err(|e| format!("tar: {e}"))?; - let zstd = zstd.map_err(|e| format!("zstd: {e}"))?; - if !(tar.success() && zstd.success()) { - return Err(format!("packaging: tar exited {tar}, zstd {zstd}")); - } - - let created = Command::new("gh") - .args(["release", "create", tag, "--title", tag, "--notes-file"]) - .arg(tmp.join("notes.md")) - .arg(&tarball) - .current_dir(root) - .status() - .map_err(|e| format!("gh: {e}"))?; - if !created.success() { - println!("`gh release create {tag}` refused; another run may have published it first"); - } - for _ in 0..20 { - if published(root, tag) { - return Ok(()); + pack(&rust_dir.join("build"), &tarball)?; + let tag = tag(&key); + let notes = notes(root, repo, &tag, &manifest(&tag))?; + let github = Github::new(repo)?; + let put = put_up(&github, root, &tag, ¬es, &tarball)?; + Ok(format!("{put}; {}", alias(&github, root, &sdk, &tag, ¬es, &tmp)?)) +} + +/// The SDK crates as crates.io holds them, where `head` is main's tip as +/// `ls_remote` prints it, and `None` where main has moved past `head`: a +/// landing during a nightly is not its failure, and a run of an older tree +/// moves no alias back. crates.io is read before the tip, so a landing whose +/// crates that read shows is one the tip shows too. Refused where crates.io's +/// newest is not the tip's own, which `publish.yml` owes. +fn sdk_at_tip( + plan: impl FnOnce() -> Result, String>, + ls_remote: impl FnOnce() -> String, + head: &str, +) -> Result>, String> { + let sdk = plan()?; + let said = ls_remote(); + let tip = said.split_whitespace().next().ok_or("origin names no main")?; + if tip != head { + return Ok(None); + } + match sdk.iter().find(|r| r.publish) { + Some(owed) => Err(format!("crates.io holds no {} of this tree, main's tip, so no sdk alias can name it", owed.krate.name)), + None => Ok(Some(sdk)), + } +} + +/// The tarball of the toolchain laid out under `build` ([`lay_out`]) at +/// `tarball`, gzipped: `/stage2` but its `bin/cargo`, which names a path +/// only this runner has, then its witness and `TOOLCHAIN`, in sorted order with +/// no owner or time, so one sysroot packs to one digest. +fn pack(build: &Path, tarball: &Path) -> Result<(), String> { + let stage2 = Path::new(HOST).join("stage2"); + let mut entries = vec![stage2.clone()]; + walk(&build.join(&stage2), &stage2, &mut entries)?; + entries.retain(|entry| *entry != stage2.join("bin/cargo")); + entries.extend(["toyos-sysroot-witness", "TOOLCHAIN"].map(PathBuf::from)); + let file = fs::File::create(tarball).map_err(|e| format!("{}: {e}", tarball.display()))?; + let gzip = flate2::write::GzEncoder::new(std::io::BufWriter::new(file), flate2::Compression::default()); + let mut tar = tar::Builder::new(gzip); + tar.follow_symlinks(false); + tar.mode(tar::HeaderMode::Deterministic); + for entry in &entries { + tar.append_path_with_name(build.join(entry), entry).map_err(|e| format!("pack {}: {e}", entry.display()))?; + } + let packed = tar.into_inner().and_then(|gzip| gzip.finish()).and_then(|mut file| file.flush()); + packed.map_err(|e| format!("{}: {e}", tarball.display())) +} + +/// Every path under `dir`, named as it is under `prefix`, each directory's in +/// sorted order. +fn walk(dir: &Path, prefix: &Path, out: &mut Vec) -> Result<(), String> { + let mut names = fs::read_dir(dir) + .and_then(|entries| entries.map(|entry| entry.map(|entry| entry.file_name())).collect::>>()) + .map_err(|e| format!("{}: {e}", dir.display()))?; + names.sort(); + for name in names { + let path = dir.join(&name); + out.push(prefix.join(&name)); + let meta = fs::symlink_metadata(&path).map_err(|e| format!("{}: {e}", path.display()))?; + if meta.is_dir() { + walk(&path, &prefix.join(&name), out)?; + } + } + Ok(()) +} + +/// What a release needs for its asset `name` to be the bytes `digest` names, +/// given GitHub's account of it, `None` where there is no release. +#[derive(Debug, PartialEq)] +enum Put { + Create, + Carried, + Upload, + /// Delete the asset another writer put there, then upload. + Replace { asset: u64 }, +} + +fn put(release: Option<&Value>, name: &str, digest: &str) -> Result { + let Some(release) = release else { return Ok(Put::Create) }; + let assets = release["assets"].as_array().ok_or("a release with no assets list")?; + match assets.iter().find(|asset| asset["name"] == name) { + None => Ok(Put::Upload), + Some(asset) if asset["digest"].as_str() == Some(digest) => Ok(Put::Carried), + Some(asset) => Ok(Put::Replace { asset: asset["id"].as_u64().ok_or("an asset with no id")? }), + } +} + +/// `tag`'s release, made to carry `file` as its asset by its name: created with +/// `notes` where there is none and given them where there is, its asset put up +/// unless it already carries these bytes, and then held to the digest GitHub +/// records. +fn put_up(github: &Github, root: &Path, tag: &str, notes: &str, file: &Path) -> Result { + let name = file.file_name().and_then(|name| name.to_str()).ok_or_else(|| format!("{} has no name", file.display()))?; + let bytes = fs::read(file).map_err(|e| format!("{}: {e}", file.display()))?; + let digest = format!("sha256:{}", sha256_hex(&bytes)); + let at = github.api(&format!("releases/tags/{tag}")); + let found = github.call("GET", &at, None)?; + let release = match &found { + None => { + let commit = crate::sysroot::git_out(root, &["rev-parse", "HEAD"]); + let body = serde_json::json!({ "tag_name": tag, "name": tag, "body": notes, "target_commitish": commit.trim() }); + github.send("POST", &github.api("releases"), &body)? + } + Some(release) => { + let id = release["id"].as_u64().ok_or("a release with no id")?; + github.send("PATCH", &github.api(&format!("releases/{id}")), &serde_json::json!({ "body": notes }))? + } + }; + let said = match put(found.as_ref(), name, &digest)? { + Put::Carried => return Ok(format!("{tag} already carries this {name}")), + Put::Create => "put up", + Put::Upload => "given its asset", + Put::Replace { asset } => { + github.call("DELETE", &github.api(&format!("releases/assets/{asset}")), None)?; + "had another writer's asset, now this one" } - println!("{tag} carries no {ASSET} yet; waiting"); - sleep(15); + }; + let upload = release["upload_url"].as_str().ok_or("a release with no upload URL")?; + let upload = format!("{}?name={name}", upload.split('{').next().unwrap_or(upload)); + github.call("POST", &upload, Some((&bytes, "application/octet-stream")))?; + let now = github.call("GET", &at, None)?; + if put(now.as_ref(), name, &digest)? != Put::Carried { + return Err(format!("{tag} does not carry {digest} as its {name} after the upload")); + } + Ok(format!("{tag} {said}")) +} + +/// GitHub's REST API for one repository, as the token its job was handed. +struct Github { + agent: ureq::Agent, + repo: String, + token: String, +} + +impl Github { + fn new(repo: &str) -> Result { + let token = std::env::var("GH_TOKEN").map_err(|_| "GH_TOKEN is unset".to_string())?; + Ok(Self { agent: agent(), repo: repo.to_string(), token }) + } + + /// The URL of `path` under the repository's API. + fn api(&self, path: &str) -> String { + format!("https://api.github.com/repos/{}/{path}", self.repo) + } + + /// GitHub's answer to `method` on `url`, sent `body` as its content type + /// where there is one: the JSON it answered, null for no content, and `None` + /// for a 404. + fn call(&self, method: &str, url: &str, body: Option<(&[u8], &str)>) -> Result, String> { + let request = ureq::http::Request::builder() + .method(method) + .uri(url) + .header("Authorization", format!("Bearer {}", self.token)) + .header("Accept", "application/vnd.github+json"); + let sent = match body { + Some((bytes, kind)) => request.header("Content-Type", kind).body(bytes).map(|request| self.agent.run(request)), + None => request.body(()).map(|request| self.agent.run(request)), + }; + let mut answer = sent.map_err(|e| format!("{method} {url}: {e}"))?.map_err(|e| format!("{method} {url}: {e}"))?; + let text = answer.body_mut().read_to_string().map_err(|e| format!("{method} {url}: {e}"))?; + match answer.status().as_u16() { + 200 | 201 => serde_json::from_str(&text).map(Some).map_err(|e| format!("{method} {url} answered no JSON: {e}")), + 204 => Ok(Some(Value::Null)), + 404 => Ok(None), + status => Err(format!("{method} {url} answered {status}: {text}")), + } + } + + /// `body` sent to `url` by `method`, as JSON: what GitHub answered. + fn send(&self, method: &str, url: &str, body: &Value) -> Result { + let body = body.to_string(); + self.call(method, url, Some((body.as_bytes(), "application/json")))?.ok_or_else(|| format!("{method} {url} found nothing")) } - Err(format!("{tag} carries no {ASSET}, so nothing can install this toolchain")) } /// Every `GLIBC_x.y` the shipped host binaries and libraries name, as the /// newest: `rustc` and its libraries, and the `rust-lld`, clang and LLVM tools -/// beside them. A byte scan: it can only over-report, so its failure is a -/// refused publish. +/// beside them. fn shipped_glibc(stage2: &Path) -> Result<(u32, u32), String> { let mut files: Vec = Vec::new(); let tools = stage2.join(format!("lib/rustlib/{HOST}/bin")); @@ -349,21 +567,14 @@ fn glibc_named(bytes: &[u8]) -> (u32, u32) { } /// `TOOLCHAIN`: the pin a consumer writes down, and what it gets. Inside the -/// tarball, and the alias release's own asset. -fn manifest(root: &Path, tag: &str) -> String { - let head = |rev: &str| crate::sysroot::git_out(root, &["rev-parse", rev]).trim().to_string(); - let toyos = std::env::var("GITHUB_SHA").unwrap_or_else(|_| head("HEAD")); - format!( - "toolchain {tag}\ntoyos {toyos}\nrust {}\nhost {HOST}\nglibc {}.{}\n", - head("HEAD:rust"), - GLIBC_FLOOR.0, - GLIBC_FLOOR.1 - ) +/// tarball, and the start of the alias release's own asset; only what the +/// sysroot's key decides, so one sysroot packs to one digest. +fn manifest(tag: &str) -> String { + format!("toolchain {tag}\nhost {HOST}\nglibc {}.{}\n", GLIBC_FLOOR.0, GLIBC_FLOOR.1) } /// The release notes: how to install it, what glibc it needs. -fn notes(root: &Path, tag: &str, manifest: &str) -> Result { - let repo = std::env::var("GITHUB_REPOSITORY").unwrap_or_else(|_| "ToyOSOrg/ToyOS".into()); +fn notes(root: &Path, repo: &str, tag: &str, manifest: &str) -> Result { let url = format!("https://github.com/{repo}/releases/download/{tag}/{ASSET}"); let (major, minor) = GLIBC_FLOOR; let userland = fs::read_to_string(root.join("userland/Cargo.toml")).map_err(|e| e.to_string())?; @@ -378,7 +589,7 @@ fn notes(root: &Path, tag: &str, manifest: &str) -> Result { ## Install mkdir -p toyos-toolchain - curl -sSL {url} | tar --zstd -x -C toyos-toolchain + curl -sSL {url} | tar -xz -C toyos-toolchain rustup toolchain link toyos toyos-toolchain/{HOST}/stage2 ln -s \"$(rustup which cargo)\" toyos-toolchain/{HOST}/stage2/bin/cargo cargo +toyos build --target x86_64-unknown-toyos @@ -409,145 +620,329 @@ Until [rust-windowing/raw-window-handle#223](https://github.com/rust-windowing/r } /// `toolchain-linux-x86_64-sdk-`: -/// the name a consumer pins, moved onto this tree's toolchain. A second release -/// carrying only the manifest, because GitHub hangs an asset off one release id. -fn alias(root: &Path, manifest: &str, tmp: &Path) -> Result { - let plan = crate::sdkversion::plan(root)?; - if let Some(owed) = plan.iter().find(|r| r.publish) { - let name = owed.krate.name; - return Err(format!("crates.io holds no {name} of this tree, so no sdk alias can name it")); - } - let abi = plan.iter().find(|r| r.krate.name == "toyos-abi").ok_or("toyos-abi is not published")?; +/// the name a consumer pins, moved onto `tag`. A second release carrying only a +/// `TOOLCHAIN` naming `tag`, the commit that put it up and the SDK crates, +/// because GitHub hangs an asset off one release id. +fn alias(github: &Github, root: &Path, sdk: &[Release], tag: &str, notes: &str, tmp: &Path) -> Result { + let abi = sdk.iter().find(|r| r.krate.name == "toyos-abi").ok_or("toyos-abi is not published")?; let abi = abi.version.split('+').next().unwrap_or(&abi.version); let alias = format!("toolchain-linux-x86_64-sdk-{abi}"); - let notes = tmp.join("notes.md"); + let commit = |rev: &str| crate::sysroot::git_out(root, &["rev-parse", rev]).trim().to_string(); + let sdk: String = sdk.iter().map(|r| format!("{} {}\n", r.krate.name, r.version)).collect(); let toolchain = tmp.join("TOOLCHAIN"); - // The tarball's copy names no crates.io version, so it never depends on crates.io. - let sdk: String = plan.iter().map(|r| format!("{} {}\n", r.krate.name, r.version)).collect(); - fs::write(&toolchain, format!("{manifest}{sdk}")).map_err(|e| e.to_string())?; - let gh = |args: &[&str], files: &[&Path]| { - Command::new("gh").args(args).args(files).current_dir(root).status().is_ok_and(|s| s.success()) - }; - let created = gh(&["release", "create", &alias, "--title", &alias, "--notes-file"], &[¬es, &toolchain]); - let moved = created - || (gh(&["release", "edit", &alias, "--notes-file"], &[¬es]) - && gh(&["release", "upload", &alias, "--clobber"], &[&toolchain])); - moved.then(|| format!("{alias} names it")).ok_or_else(|| format!("{alias} could not be moved")) + let text = format!("{}toyos {}\nrust {}\n{sdk}", manifest(tag), commit("HEAD"), commit("HEAD:rust")); + fs::write(&toolchain, text).map_err(|e| format!("{}: {e}", toolchain.display()))?; + put_up(github, root, &alias, notes, &toolchain).map(|said| format!("{said}, naming {tag}")) } #[cfg(test)] mod tests { use super::*; - /// The packaging is one of the trees its own tag hashes. #[test] - fn the_tag_hashes_this_file() { - assert_eq!(trees().last(), Some(&file!())); - for tree in trees() { - assert!( - Path::new(env!("CARGO_MANIFEST_DIR")).join(tree).exists(), - "{tree} is hashed into the tag and is not in the tree" - ); + fn the_glibc_scan_takes_the_newest_version_and_nothing_else() { + let bytes = b"\0GLIBC_2.17\0GLIBC_2.39\0GLIBC_2.4\0GLIBC_PRIVATE\0GLIBC_\0GLIBC_3.\0"; + assert_eq!(glibc_named(bytes), (2, 39)); + assert_eq!(glibc_named(b"GLIBC_2.40"), (2, 40)); + assert!(glibc_named(b"GLIBC_2.40") > GLIBC_FLOOR); + assert_eq!(glibc_named(b"no version here"), (0, 0)); + } + + const REPO: &str = "ToyOSOrg/ToyOS"; + + /// The negative control on the publisher: the release job run as a pull + /// request's job, the merge queue's, a push's, or main's nightly dispatched + /// on a branch or run by any other event is refused by name before it reads + /// anything, and so is another repository's nightly. + #[test] + fn only_mains_publisher_publishes() { + let mains = format!("{REPO}/.github/workflows/nightly.yml@refs/heads/main"); + assert!(publisher(Some(&mains), Some("schedule"), REPO).is_ok()); + assert!(publisher(Some(&mains), Some("workflow_dispatch"), REPO).is_ok()); + let fork = "Fork/ToyOS/.github/workflows/nightly.yml@refs/heads/main"; + assert!(publisher(Some(fork), Some("schedule"), REPO).unwrap_err().contains(fork)); + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let refused = [ + (format!("{REPO}/.github/workflows/ci.yml@refs/pull/671/merge"), "pull_request"), + (format!("{REPO}/.github/workflows/ci.yml@refs/heads/gh-readonly-queue/main/pr-671-59052827f"), "merge_group"), + (format!("{REPO}/.github/workflows/publish.yml@refs/heads/main"), "push"), + (format!("{REPO}/.github/workflows/nightly.yml@refs/heads/wt/toyos-guestci"), "workflow_dispatch"), + (format!("{REPO}/.github/workflows/nightly.yml@refs/tags/main"), "push"), + (format!("{REPO}/.github/workflows/nightly.yml@refs/heads/main"), "workflow_run"), + ]; + for (workflow, event) in refused { + let why = release_as(root, REPO, Some(&workflow), Some(event)).expect_err(&workflow); + assert!(why.starts_with("only ") && why.contains(&workflow) && why.contains(event), "{why}"); } + assert!(release_as(root, REPO, None, None).unwrap_err().starts_with("only ")); } - fn git(dir: &Path, args: &[&str]) { - let out = Command::new("git") - .args(["-c", "commit.gpgsign=false", "-c", "user.email=t@t", "-c", "user.name=t"]) - .args(["-c", "init.defaultBranch=main"]) - .args(crate::gitfixture::NO_AUTO_MAINTENANCE) - .args(args) - .current_dir(dir) - .output() - .expect("run git"); - assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + /// crates.io's newest `toyos-abi` as a plan reads it: this tree's, or owed. + fn sdk(owed: bool) -> Vec { + let krate = &crate::sdkversion::PUBLISHED[0]; + vec![Release { krate, key: String::new(), version: "0.28.0+k".into(), publish: owed, manifest: String::new() }] } - /// A commit to the C and C++ toolchain's declarations or headers, or to the - /// n2 it is built under, moves the tag. + /// **A landing on main during a nightly is not that nightly's failure**: + /// whether it comes before the release reads anything, between its read of + /// crates.io and its read of main's tip, or not at all, and whether or not + /// it put newer SDK crates up, the release is the tip's or nothing is put + /// up, and neither is refused. What is refused is the tip's own crates not + /// being up, and a remote that names no main. #[test] - fn the_tag_moves_with_the_c_toolchain() { - let repo = TempDir::new("release-tag"); - let here = Path::new(env!("CARGO_MANIFEST_DIR")); - let write = |path: &str, text: &str| { - let path = repo.join(path); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - fs::write(path, text).unwrap(); + fn a_landing_during_the_nightly_puts_nothing_up_and_is_no_failure() { + const HEAD: &str = "0123456789abcdef0123456789abcdef01234567"; + const LANDED: &str = "fedcba9876543210fedcba9876543210fedcba98"; + let main = |tip: &str| format!("{tip}\trefs/heads/main\n"); + // The release of `HEAD` on a main that lands once, after `after` of + // the release's reads: whether it publishes. + let publishes = |after: usize, moves_sdk: bool| { + let reads = std::cell::Cell::new(0); + let landed = || reads.replace(reads.get() + 1) >= after; + let read = sdk_at_tip(|| Ok(sdk(landed() && moves_sdk)), || main(if landed() { LANDED } else { HEAD }), HEAD); + read.map(|sdk| sdk.is_some()) }; - git(&repo, &["init", "-q"]); - for tree in trees() { - match tree { - "rust" => git(&repo, &["update-index", "--add", "--cacheinfo", "160000,1111111111111111111111111111111111111111,rust"]), - file if here.join(file).is_file() => write(file, &fs::read_to_string(here.join(file)).unwrap()), - dir => write(&format!("{dir}/placeholder"), "x"), - } + for moves_sdk in [true, false] { + assert_eq!(publishes(0, moves_sdk), Ok(false), "a landing before the release read anything"); + assert_eq!(publishes(1, moves_sdk), Ok(false), "a landing between the release's two reads"); + assert_eq!(publishes(2, moves_sdk), Ok(true), "no landing"); } - fs::create_dir_all(repo.join("rust")).unwrap(); - git(&repo, &["add", "-A"]); - git(&repo, &["commit", "-qm", "the tree"]); - let mut before = tag(&repo).unwrap(); - - let clang = fs::read_to_string(here.join(crate::clang::SOURCE)).unwrap(); - let tools = r#"const TOOLS: [&str; 3] = ["llvm-ar", "clang", "ld.lld"];"#; - let objdump = r#"const TOOLS: [&str; 4] = ["llvm-ar", "clang", "ld.lld", "llvm-objdump"];"#; - let targets = r#"targets = \"AArch64;X86\""#; - let riscv = r#"targets = \"AArch64;RISCV;X86\""#; - assert!(clang.contains(tools) && clang.contains(targets), "src/clang.rs no longer declares what this mutates"); - let with_objdump = clang.replace(tools, objdump); - let cxx = fs::read_to_string(here.join(crate::libcxx::SOURCE)).unwrap(); - let (no_fs, fs_on) = (r#"("LIBCXX_ENABLE_FILESYSTEM", "OFF")"#, r#"("LIBCXX_ENABLE_FILESYSTEM", "ON")"#); - assert!(cxx.contains(no_fs), "src/libcxx.rs no longer declares what this mutates"); - let n2 = fs::read_to_string(here.join(crate::n2::SOURCE)).unwrap(); - let pin = crate::n2::N2[crate::n2::N2.len() - 1]; - assert!(n2.contains(pin), "src/n2.rs no longer declares what this mutates"); - let mutations = [ - (crate::clang::SOURCE, with_objdump.clone()), - (crate::clang::SOURCE, with_objdump.replace(targets, riscv)), - (crate::libcxx::SOURCE, cxx.replace(no_fs, fs_on)), - (crate::n2::SOURCE, n2.replace(pin, &"0".repeat(pin.len()))), - ("userland/libc/include/placeholder", "y".to_string()), - ]; - for (path, text) in mutations { - write(path, &text); - git(&repo, &["add", "-A"]); - git(&repo, &["commit", "-qm", "a mutation"]); - let after = tag(&repo).unwrap(); - assert_ne!(after, before, "a commit to {path} kept the tag"); - before = after; + let owed = sdk_at_tip(|| Ok(sdk(true)), || main(HEAD), HEAD).err().expect("the tip's crates are not up"); + assert!(owed.contains("toyos-abi") && owed.contains("main's tip"), "{owed}"); + let unnamed = sdk_at_tip(|| Ok(sdk(false)), String::new, HEAD).err().expect("no main"); + assert!(unnamed.contains("names no main"), "{unnamed}"); + } + + fn release_json(assets: Value) -> Value { + serde_json::json!({ "id": 7, "upload_url": "https://uploads.github.com/repos/o/r/releases/7/assets{?name,label}", "assets": assets }) + } + + /// The negative control on what a release is made to carry: none is + /// created, one with no such asset is given it, one carrying these bytes is + /// left, and one carrying any other bytes, or bytes GitHub records no digest + /// for, has its asset replaced. + #[test] + fn a_release_is_made_to_carry_these_bytes_and_no_other() { + let digest = "sha256:aa"; + assert_eq!(put(None, ASSET, digest), Ok(Put::Create)); + let other = serde_json::json!({ "id": 3, "name": "notes.txt", "digest": digest }); + assert_eq!(put(Some(&release_json(serde_json::json!([other]))), ASSET, digest), Ok(Put::Upload)); + let ours = serde_json::json!([{ "id": 4, "name": ASSET, "digest": digest }]); + assert_eq!(put(Some(&release_json(ours)), ASSET, digest), Ok(Put::Carried)); + let theirs = serde_json::json!([{ "id": 5, "name": ASSET, "digest": "sha256:bb" }]); + assert_eq!(put(Some(&release_json(theirs)), ASSET, digest), Ok(Put::Replace { asset: 5 })); + let unrecorded = serde_json::json!([{ "id": 6, "name": ASSET, "digest": null }]); + assert_eq!(put(Some(&release_json(unrecorded)), ASSET, digest), Ok(Put::Replace { asset: 6 })); + assert!(put(Some(&serde_json::json!({ "message": "Not Found" })), ASSET, digest).is_err()); + } + + /// A toolchain laid out under `build` as [`lay_out`] leaves one, with a + /// file, an executable, a directory, a link and a `bin/cargo`. + fn laid_out(build: &Path, stamp: &str) { + let stage2 = build.join(HOST).join("stage2"); + for (file, text) in [("bin/rustc", "rustc"), ("lib/libstd.rlib", stamp), ("lib/rustlib/empty/.keep", "")] { + fs::create_dir_all(stage2.join(file).parent().unwrap()).unwrap(); + fs::write(stage2.join(file), text).unwrap(); + } + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(stage2.join("bin/rustc"), fs::Permissions::from_mode(0o755)).unwrap(); + std::os::unix::fs::symlink("rustc", stage2.join("bin/ld.lld")).unwrap(); + std::os::unix::fs::symlink("/a/runner/s/cargo", stage2.join("bin/cargo")).unwrap(); + fs::write(build.join("toyos-sysroot-witness"), "toyos-abi/src/lib.rs:00").unwrap(); + fs::write(build.join("TOOLCHAIN"), manifest("toolchain-linux-x86_64-0123456789abcdef")).unwrap(); + } + + /// What the tarball at `path` holds, by name: a link by what it names, a + /// file by its bytes and mode. + fn unpacked(path: &Path) -> Vec<(String, String)> { + let bytes = fs::read(path).unwrap(); + let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(bytes.as_slice())); + let mut seen = Vec::new(); + for entry in archive.entries().unwrap() { + let mut entry = entry.unwrap(); + let name = entry.path().unwrap().display().to_string(); + let (kind, mode) = (entry.header().entry_type(), entry.header().mode().unwrap()); + let what = match kind { + tar::EntryType::Symlink => format!("-> {}", entry.link_name().unwrap().unwrap().display()), + tar::EntryType::Directory => "dir".to_string(), + _ => { + let mut text = String::new(); + entry.read_to_string(&mut text).unwrap(); + format!("{mode:o} {text}") + } + }; + seen.push((name, what)); } + seen } - /// `sha256sum`'s digest of the same bytes, cut to the same width. + use std::io::Read; + + /// **One sysroot packs to one digest, and unpacks whole**: packed again + /// after its files were written again later, it is the same bytes; it holds + /// every file, mode and link of `stage2` but `bin/cargo`, then the witness + /// and `TOOLCHAIN`; and other bytes are another digest. #[test] - fn the_hash_is_sha256_of_what_git_printed() { + fn one_sysroot_packs_to_one_digest_and_unpacks_whole() { + let tmp = TempDir::new("release-pack"); + let (first, again, other) = (tmp.join("first"), tmp.join("again"), tmp.join("other")); + laid_out(&first.join("build"), "std"); + laid_out(&again.join("build"), "std"); + laid_out(&other.join("build"), "another std"); + let later = std::time::SystemTime::now() + std::time::Duration::from_secs(3600); + fs::File::options().write(true).open(again.join("build").join(HOST).join("stage2/lib/libstd.rlib")).unwrap().set_modified(later).unwrap(); + for dir in [&first, &again, &other] { + pack(&dir.join("build"), &dir.join(ASSET)).unwrap(); + } + let digest = |dir: &Path| sha256_hex(&fs::read(dir.join(ASSET)).unwrap()); + assert_eq!(digest(&first), digest(&again), "one sysroot packed to two digests"); + assert_ne!(digest(&first), digest(&other)); + let stage2 = format!("{HOST}/stage2"); + let want: Vec<(String, String)> = [ + (stage2.clone(), "dir"), + (format!("{stage2}/bin"), "dir"), + (format!("{stage2}/bin/ld.lld"), "-> rustc"), + (format!("{stage2}/bin/rustc"), "755 rustc"), + (format!("{stage2}/lib"), "dir"), + (format!("{stage2}/lib/libstd.rlib"), "644 std"), + (format!("{stage2}/lib/rustlib"), "dir"), + (format!("{stage2}/lib/rustlib/empty"), "dir"), + (format!("{stage2}/lib/rustlib/empty/.keep"), "644 "), + ("toyos-sysroot-witness".to_string(), "644 toyos-abi/src/lib.rs:00"), + ("TOOLCHAIN".to_string(), "644 toolchain toolchain-linux-x86_64-0123456789abcdef\nhost x86_64-unknown-linux-gnu\nglibc 2.39\n"), + ] + .map(|(name, what)| (name, what.to_string())) + .to_vec(); + assert_eq!(unpacked(&first.join(ASSET)), want); + } + + /// A checkout whose witness reads `abi`, under `root`. + fn checkout(root: &Path, abi: &str) { + for tree in crate::sysroot::SYSROOT_SOURCES { + fs::create_dir_all(root.join(tree)).unwrap(); + } + fs::write(root.join("toyos-abi/src/lib.rs"), abi).unwrap(); + for manifest in crate::sysroot::SYSROOT_MANIFESTS { + fs::create_dir_all(root.join(manifest).parent().unwrap()).unwrap(); + fs::write(root.join(manifest), "[package]\n").unwrap(); + } + } + + /// A sysroot store under `root`'s `rust/build/sysroots`, as a job restores + /// one, recording `witness`. + fn restored(root: &Path, key: &str, witness: &str) -> PathBuf { + let dir = crate::sysroot::sysroots_dir(&root.join("rust")).join(key); + fs::create_dir_all(dir.join("bin")).unwrap(); + fs::write(dir.join("bin/rustc"), "rustc").unwrap(); + fs::write(dir.join("SOURCES"), format!("{key}\nfork /a/runner/s/rust\n{witness}\n")).unwrap(); + dir + } + + /// **A job installs the one sysroot it restored, and only one built from + /// its own tree's sources**: none, two, or one named by no key is refused; + /// one whose recorded witness is not this tree's is refused; the one that + /// is lands at `stage2` with that witness and its `TOOLCHAIN`. + #[test] + fn a_job_lays_out_the_one_sysroot_it_restored() { + let tmp = TempDir::new("release-lay-out"); + let root = tmp.join("checkout"); + checkout(&root, "pub struct A;\n"); + fs::create_dir_all(crate::sysroot::sysroots_dir(&root.join("rust"))).unwrap(); + assert!(lay_out(&root).unwrap_err().contains("holds 0 entries")); + let witness = crate::sysroot::witness(&root); + let one = restored(&root, "0123456789abcdef", &witness); + let two = restored(&root, "fedcba9876543210", &witness); + assert!(lay_out(&root).unwrap_err().contains("holds 2 entries")); + fs::remove_dir_all(&two).unwrap(); + fs::write(root.join("toyos-abi/src/lib.rs"), "pub struct A(u64);\n").unwrap(); + assert!(lay_out(&root).unwrap_err().contains("built from other sources"), "a sysroot of other sources was laid out"); + fs::write(root.join("toyos-abi/src/lib.rs"), "pub struct A;\n").unwrap(); + assert_eq!(lay_out(&root), Ok(Key::parse("0123456789abcdef").unwrap())); + let rust_dir = root.join("rust"); + assert!(!one.exists() && crate::toolchain::stage2(&rust_dir).join("bin/rustc").is_file()); + assert_eq!(fs::read_to_string(crate::toolchain::witness_path(&rust_dir)).unwrap(), witness); + let toolchain = fs::read_to_string(crate::toolchain::manifest_path(&rust_dir)).unwrap(); + assert_eq!(toolchain, manifest("toolchain-linux-x86_64-0123456789abcdef")); + fs::create_dir_all(rust_dir.join("build/sysroots/not-a-key")).unwrap(); + assert!(lay_out(&root).unwrap_err().contains("named by no key")); + } + + fn layer(name: &'static str, key: &str, paths: &[&str]) -> Layer { + let kind = LAYERS.iter().find(|(_, n)| *n == name).unwrap().0; + Layer { kind, name, key: Key::parse(key).unwrap(), paths: paths.iter().map(PathBuf::from).collect() } + } + + /// The job's outputs, as GitHub's multiline syntax reads them: each layer's + /// cache entry, and each path of it on a line of its own. + #[test] + fn a_job_is_told_each_layer_s_entry_and_paths() { + let layers = [ + layer("llvm", "1111111111111111", &["rust/build/llvm/1111111111111111"]), + layer("compiler", "2222222222222222", &["rust/build/h/stage2", "rust/build/toyos-compiler"]), + ]; assert_eq!( - sha256_hex(b""), - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + outputs(&layers), + "llvm-key=toolchain-llvm-1111111111111111\nllvm-path< GLIBC_FLOOR); - assert_eq!(glibc_named(b"no version here"), (0, 0)); + fn a_layer_that_is_not_whole_is_refused() { + let layers: Vec = + LAYERS.iter().enumerate().map(|(at, (_, name))| layer(name, &at.to_string().repeat(16), &["p"])).collect(); + let broken = |layer: &Layer| (layer.name == "compiler").then(|| "stage2 carries no clang".to_string()); + let refused = whole(&layers, &[true, true, false, false], broken).unwrap_err(); + assert!(refused.starts_with("compiler 1111111111111111 is not whole") && refused.contains("no clang"), "{refused}"); + assert_eq!(whole(&layers, &[true, false, false, false], broken), Ok(())); + assert!(whole(&layers, &[true; 4], broken).is_err(), "a build that left a layer not whole was taken"); + assert_eq!(whole(&layers, &[true; 4], |_| None), Ok(())); + } + + /// **Each layer is the store the build system makes, where it makes it**: + /// the LLVM, the freestanding libraries and the sysroot in their stores by + /// key, and the primary's compiler as its `stage2` and its record, each + /// relative to the checkout, keyed before any is built. + #[test] + fn the_layers_are_the_stores_the_build_makes() { + let scratch = TempDir::new("release-layers"); + let (primary, _rust_dir, _) = crate::compiler::tests::estate(&scratch); + checkout(&primary, "pub struct A;\n"); + let layers = layers(&primary); + let named: Vec<(&str, Vec)> = + layers.iter().map(|l| (l.name, l.paths.iter().map(|p| p.display().to_string()).collect())).collect(); + let key = |at: usize| layers[at].key.to_string(); + let host = crate::toolchain::host_triple(); + assert_eq!( + named, + [ + ("llvm", vec![format!("rust/build/llvm/{}", key(0))]), + ("compiler", vec![format!("rust/build/{host}/stage2"), "rust/build/toyos-compiler".to_string()]), + ("freestanding", vec![format!("rust/build/freestanding/{}", key(2))]), + ("sysroot", vec![format!("rust/build/sysroots/{}", key(3))]), + ] + ); + assert_eq!(layers[1].key, crate::compiler::primary_key(&primary.join("rust"))); } + /// **A job saves exactly the layers it built**: none where it restored the + /// sysroot, which is all a guest job reads, and otherwise each it did not + /// restore. #[test] - fn an_asset_is_found_by_name_and_a_release_without_it_has_none() { - let with: serde_json::Value = serde_json::from_str(&format!( - r#"{{"assets":[{{"name":"other","url":"u1"}},{{"name":"{ASSET}","url":"u2"}}]}}"# - )) - .unwrap(); - assert_eq!(named_asset(&with).as_deref(), Some("u2")); - let without: serde_json::Value = serde_json::from_str(r#"{"assets":[]}"#).unwrap(); - assert_eq!(named_asset(&without), None); - let missing: serde_json::Value = serde_json::from_str(r#"{"message":"Not Found"}"#).unwrap(); - assert_eq!(named_asset(&missing), None); + fn a_job_saves_only_what_it_built() { + let layers: Vec = LAYERS + .iter() + .enumerate() + .map(|(at, (_, name))| layer(name, &at.to_string().repeat(16), &["p"])) + .collect(); + let told = |restored: [bool; 4]| built(&layers, &restored); + assert_eq!(told([true, true, false, true]), "llvm=kept\ncompiler=kept\nfreestanding=kept\nsysroot=kept\n"); + assert_eq!(told([false, false, false, true]), "llvm=kept\ncompiler=kept\nfreestanding=kept\nsysroot=kept\n"); + assert_eq!(told([true, true, false, false]), "llvm=kept\ncompiler=kept\nfreestanding=built\nsysroot=built\n"); + assert_eq!(told([false; 4]), "llvm=built\ncompiler=built\nfreestanding=built\nsysroot=built\n"); } } diff --git a/src/sdkversion.rs b/src/sdkversion.rs index c117aa8e225..99890315bb7 100644 --- a/src/sdkversion.rs +++ b/src/sdkversion.rs @@ -11,7 +11,6 @@ //! the index holds every version it names. use std::path::Path; -use std::process::Command; /// One published crate: the crates.io name, and its repository-relative /// directory. @@ -115,15 +114,12 @@ fn numbers(vers: &str) -> Option<(u64, u64, u64)> { /// for a crate never published. pub fn index(name: &str) -> Result { let url = format!("https://index.crates.io/{}/{}/{name}", &name[..2], &name[2..4]); - let out = Command::new("curl") - .args(["-sS", "-w", "\n%{http_code}", &url]) - .output() - .map_err(|e| format!("curl: {e}"))?; - let text = String::from_utf8_lossy(&out.stdout); - match text.rsplit_once('\n') { - Some((_, "404")) => Ok(String::new()), - Some((body, "200")) => Ok(body.to_string()), - _ => Err(format!("the crates.io index answered {text:?} for {name}")), + let mut answer = crate::release::agent().get(&url).call().map_err(|e| format!("{url}: {e}"))?; + let text = answer.body_mut().read_to_string().map_err(|e| format!("{url}: {e}"))?; + match answer.status().as_u16() { + 404 => Ok(String::new()), + 200 => Ok(text), + status => Err(format!("the crates.io index answered {status} for {name}: {text}")), } } @@ -147,6 +143,7 @@ mod tests { use super::*; use crate::gitfixture::{commit, repo}; use std::collections::BTreeMap; + use std::process::Command; use toyos_tmpdir::TempDir; const TWO: &[Crate] = diff --git a/src/sysroot.rs b/src/sysroot.rs index 557a4c73cb4..8a916199f81 100644 --- a/src/sysroot.rs +++ b/src/sysroot.rs @@ -4,8 +4,9 @@ //! **A sysroot is a function of its key.** The key ([`key`]) is the identity //! (`src/identity.rs`, so a comment is no change) of everything a sysroot is //! built from: the trees std and `libtoyos_c.a` compile -//! ([`SYSROOT_SOURCES`]), the std fork's `library/` and `src/bootstrap/` in the -//! checkout that builds it, and the compiler that builds it. `rust/build/ +//! ([`SYSROOT_SOURCES`]) and how libc is built ([`SYSROOT_MANIFESTS`], +//! `libc::BUILD`), the std fork's `library/` and `src/bootstrap/` in the +//! checkout that builds it, and the compiler's key. `rust/build/ //! sysroots//` is a whole toolchain — the compiler's files cloned from its //! `stage2`, the guest targets' libraries built from this key's sources. A build //! compiles against the directory its own key names, so two worktrees with @@ -62,9 +63,16 @@ use whole_toolchain::{whole, Whole}; pub const SYSROOT_SOURCES: [&str; 5] = ["toyos-abi/src", "toyos/src", "toyos-elf/src", "userland/libc/src", "userland/libc/include"]; -/// Their manifests, whose features and versions decide the same build. -pub(crate) const SYSROOT_MANIFESTS: [&str; 4] = - ["toyos-abi/Cargo.toml", "toyos/Cargo.toml", "toyos-elf/Cargo.toml", "userland/libc/Cargo.toml"]; +/// Their manifests, and the lockfile and cargo configuration libc is built +/// under: the features, versions and flags of the same build. +pub(crate) const SYSROOT_MANIFESTS: [&str; 6] = [ + "toyos-abi/Cargo.toml", + "toyos/Cargo.toml", + "toyos-elf/Cargo.toml", + "userland/libc/Cargo.toml", + "userland/libc/Cargo.lock", + "userland/.cargo/config.toml", +]; /// Of [`SYSROOT_MANIFESTS`], the ones std's lockfile resolves with the fork's /// own: what of a worktree can move a freestanding target's dependency versions. @@ -289,24 +297,65 @@ fn manifest_line(root: &Path, manifest: &str) -> String { /// as one hash. /// /// **Source as git sees it**: tracked files and untracked ones no ignore rule -/// covers, into every submodule checked out there — never what a build or the -/// desktop leaves beside them (bootstrap's `__pycache__`, Finder's -/// `.DS_Store`), which would make a key that moves while it is being built. +/// covers, and each submodule as the commit its gitlink records ([`gitlink`]), +/// checked out or not — never what a build or the desktop leaves beside them +/// (bootstrap's `__pycache__`, Finder's `.DS_Store`), which would make a key +/// that moves while it is being built. pub(crate) fn tree_identity(base: &Path, paths: &[&str], links: Links) -> String { - let mut files = Vec::new(); - source_files(base, paths, links, &mut files); - files.sort(); + let mut sources = Vec::new(); + source_files(base, paths, links, &mut sources); + sources.sort(); let mut hasher = Sha256::new(); - for path in files { - let data = fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); + for (path, commit) in sources { hasher.update(path.strip_prefix(base).unwrap_or(&path).to_string_lossy().as_bytes()); hasher.update([0]); - hasher.update(&*identity::of(&path, &data)); + match commit { + Some(commit) => hasher.update(commit.as_bytes()), + None => { + let data = fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); + hasher.update(&*identity::of(&path, &data)); + } + } hasher.update([0]); } hex(&hasher.finalize())[..16].to_string() } +/// The commit `checkout`'s `HEAD` records for its submodule at `path`, which is +/// the one a build checks out there; refused when the submodule's checkout +/// holds what no commit does, or the index stages another commit, because a +/// key names it by that commit. +pub(crate) fn gitlink(checkout: &Path, path: &str) -> String { + let submodule = checkout.join(path); + // The untracked cache spares each call a walk of a whole tree. + let status = ["-c", "core.untrackedCache=true", "status", "--porcelain", "--untracked-files=normal"]; + let edited = submodule.join(".git").exists() && !git_bytes(&submodule, &status).is_empty(); + assert!( + !edited, + "{} holds changes no commit does, and a key names it by the commit its gitlink records: \ + commit them there and record that commit in {}", + submodule.display(), + checkout.display(), + ); + let recorded = git_out(checkout, &["ls-tree", "HEAD", path]); + let committed = match recorded.split_whitespace().collect::>().as_slice() { + ["160000", "commit", sha, _] => sha.to_string(), + _ => panic!("{} records no {path} gitlink: `git ls-tree HEAD {path}` said {recorded:?}", checkout.display()), + }; + let indexed = git_out(checkout, &["ls-files", "--stage", path]); + let staged = match indexed.split_whitespace().collect::>().as_slice() { + ["160000", sha, "0", _] => sha.to_string(), + _ => panic!("{} indexes no {path} gitlink: `git ls-files --stage {path}` said {indexed:?}", checkout.display()), + }; + assert!( + staged == committed, + "{} stages {path} at {staged}, and its HEAD records {committed}: a build checks out the one \ + staged, and nothing is keyed on what no commit holds; commit the gitlink, or unstage it", + checkout.display(), + ); + committed +} + /// What [`tree_identity`] makes of a symbolic link, which git keeps as the path /// it names and a build reads through. #[derive(Clone, Copy)] @@ -317,16 +366,29 @@ pub(crate) enum Links { Skipped, } -fn source_files(checkout: &Path, paths: &[&str], links: Links, out: &mut Vec) { - let mut args = vec!["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--"]; - args.extend(paths); - let listed = git_bytes(checkout, &args); +/// Each source under `paths` of `checkout`, with the commit of each that is a +/// submodule ([`gitlink`]). +fn source_files(checkout: &Path, paths: &[&str], links: Links, out: &mut Vec<(PathBuf, Option)>) { + let listed = |how: &[&str]| git_bytes(checkout, &[&["ls-files", "-z"][..], how, &["--"][..], paths].concat()); + let cached = listed(&["--stage"]); + let others = listed(&["--others", "--exclude-standard"]); + // ` \t`, and a gitlink's mode is 160000. + let cached = cached.split(|b| *b == 0).filter(|e| !e.is_empty()).map(|entry| { + let at = entry.iter().position(|b| *b == b'\t').unwrap_or_else(|| panic!("git ls-files --stage said {entry:?}")); + (&entry[at + 1..], entry.starts_with(b"160000 ")) + }); + let others = others.split(|b| *b == 0).filter(|e| !e.is_empty()).map(|entry| (entry, false)); let mut seen = BTreeSet::new(); - for entry in listed.split(|b| *b == 0).filter(|e| !e.is_empty()) { - let path = checkout.join(String::from_utf8_lossy(entry).as_ref()); + for (entry, submodule) in cached.chain(others) { + let name = String::from_utf8_lossy(entry); + let path = checkout.join(name.as_ref()); if !seen.insert(path.clone()) { continue; } + if submodule { + out.push((path, Some(gitlink(checkout, &name)))); + continue; + } let Ok(meta) = fs::symlink_metadata(&path) else { continue }; if meta.is_symlink() { match links { @@ -341,7 +403,7 @@ fn source_files(checkout: &Path, paths: &[&str], links: Links, out: &mut Vec Self { - let freestanding = freestanding_key(root, compiler, fork); + let freestanding = freestanding_key(root, &compiler.key(), fork); let sysroot = key(root, &freestanding); let identity = Identity::new(Key::of(compiler.identity().as_bytes()), &freestanding, &sysroot); Self { freestanding, sysroot, identity } @@ -369,9 +431,10 @@ impl Keys { } /// The key of the freestanding targets' libraries `root` builds against with -/// its std fork at `fork`, compiled by `compiler`: none of -/// [`SYSROOT_SOURCES`], and of `root` only [`STD_MANIFESTS`]. -fn freestanding_key(root: &Path, compiler: &Compiler, fork: &Path) -> Key { +/// its std fork at `fork`, compiled by the compiler whose key is `compiler` +/// (`Compiler::key`): none of [`SYSROOT_SOURCES`], and of `root` only +/// [`STD_MANIFESTS`]. +pub(crate) fn freestanding_key(root: &Path, compiler: &Key, fork: &Path) -> Key { freestanding_key_of(root, compiler, fork, RECIPE, &keyed_std_config()) } @@ -382,13 +445,13 @@ fn keyed_std_config() -> String { } /// [`freestanding_key`], with the recipe and std's configuration it reads. -fn freestanding_key_of(root: &Path, compiler: &Compiler, fork: &Path, recipe: &str, config: &str) -> Key { +fn freestanding_key_of(root: &Path, compiler: &Key, fork: &Path, recipe: &str, config: &str) -> Key { let parts = [ format!("{recipe}; cargo {STAGE0_CARGO}; targets {}", Libraries::Freestanding.targets().join(" ")), config.to_string(), STD_MANIFESTS.map(|manifest| manifest_line(root, manifest)).join("\n"), tree_identity(fork, &["library", "src/bootstrap"], Links::Refused), - compiler.identity(), + compiler.to_string(), ]; Key::of(parts.join("\n\0\n").as_bytes()) } @@ -396,12 +459,21 @@ fn freestanding_key_of(root: &Path, compiler: &Compiler, fork: &Path, recipe: &s /// The key of the sysroot `root` builds against, whose freestanding libraries /// are `freestanding`'s ([`freestanding_key`], which names the recipe, std's /// configuration, the fork and the compiler the rest is built with too). -fn key(root: &Path, freestanding: &Key) -> Key { - let parts = [ - format!("targets {}; C++ runtime {:?}", Libraries::Worktree.targets().join(" "), crate::libcxx::OPTIONS), - witness(root), - freestanding.to_string(), - ]; +pub(crate) fn key(root: &Path, freestanding: &Key) -> Key { + key_of(root, freestanding, &build_text()) +} + +/// What a sysroot's build is beyond its sources and its freestanding libraries: +/// its targets, the C++ runtime's options and libc's cargo invocations. +fn build_text() -> String { + let targets = Libraries::Worktree.targets().join(" "); + let (libc, staticlib) = (crate::libc::BUILD, crate::libc::BUILD_C); + format!("targets {targets}; C++ runtime {:?}; libc {libc:?} {staticlib:?}", crate::libcxx::OPTIONS) +} + +/// [`key`], with the build it reads. +fn key_of(root: &Path, freestanding: &Key, build: &str) -> Key { + let parts = [build.to_string(), witness(root), freestanding.to_string()]; Key::of(parts.join("\n\0\n").as_bytes()) } @@ -499,9 +571,25 @@ pub fn fork_checkout(root: &Path) -> PathBuf { fork } +/// What a sysroot's [`SOURCES`] says: its key, the fork checkout its std was +/// built in, and the witness of the sources it was built from. +fn sources_text(key: &Key, fork: &Path, witness: &str) -> String { + format!("{key}\nfork {}\n{witness}\n", fork.display()) +} + +/// The witness the sysroot at `dir` records it was built from ([`sources_text`]). +pub(crate) fn recorded_witness(dir: &Path) -> Result { + let path = dir.join(SOURCES); + let text = fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?; + match text.splitn(3, '\n').collect::>().as_slice() { + [_, fork, witness] if fork.starts_with("fork ") => Ok(witness.trim_end_matches('\n').to_string()), + _ => Err(format!("{} records no witness: {text:?}", path.display())), + } +} + /// Why `dir` is not a directory [`publish`] finished, if it is not: it carries /// no [`SOURCES`]. -fn unpublished(dir: &Path) -> Option { +pub(crate) fn unpublished(dir: &Path) -> Option { (!dir.join(SOURCES).is_file()).then(|| format!("{} carries no {SOURCES}", dir.display())) } @@ -510,7 +598,7 @@ fn unpublished(dir: &Path) -> Option { /// not the second is made again rather than trusted — all of it even when only /// its `bin/cargo` link dangles, because that is rare and a sysroot has no /// repair path. -fn unfinished(dir: &Path) -> Option { +pub(crate) fn unfinished(dir: &Path) -> Option { unpublished(dir).or_else(|| toolchain::toolchain_defect(dir)) } @@ -629,7 +717,7 @@ fn build(root: &Path, rust_dir: &Path, compiler: &Compiler, fork: &Path, keys: & "the sources moved while sysroot {key} was being built (they are now {again}); \ nothing was kept, and the next build makes the one they name" ); - format!("{key}\nfork {}\n{}\n", fork.display(), witness(root)) + sources_text(key, fork, &witness(root)) }); } @@ -650,7 +738,7 @@ fn build_freestanding(root: &Path, compiler: &Compiler, fork: &Path, key: &Key, for target in Libraries::Freestanding.targets() { place_std(&stamp(&built, target), &partial.join(target)); } - let again = freestanding_key(root, compiler, fork); + let again = freestanding_key(root, &compiler.key(), fork); assert!( again == *key, "the sources moved while the freestanding libraries {key} were being built (they are \ @@ -1041,10 +1129,20 @@ mod tests { fs::remove_file(&header).unwrap(); same("the C sysroot's headers as they were"); - write(&root.join("userland/libc/Cargo.toml"), "[package]\nversion = \"0.2.0\"\n"); - sysroot_only("libc's manifest, which std's lockfile does not resolve,"); - write(&root.join("userland/libc/Cargo.toml"), "[package]\nversion = \"0.1.0\"\n"); - same("libc's manifest as it was"); + for read in ["userland/libc/Cargo.toml", "userland/libc/Cargo.lock", "userland/.cargo/config.toml"] { + write(&root.join(read), "[package]\nversion = \"0.2.0\"\n"); + sysroot_only(&format!("{read}, which std's lockfile does not resolve,")); + write(&root.join(read), "[package]\nversion = \"0.1.0\"\n"); + same(&format!("{read} as it was")); + } + assert_eq!(key_of(&root, &was.freestanding, &build_text()), was.sysroot); + for flag in [crate::libc::BUILD[1], crate::libc::BUILD_C[1]] { + assert!(build_text().contains(flag), "the sysroot key reads none of libc's {flag}: {}", build_text()); + } + let options = format!("{:?}", crate::libcxx::OPTIONS); + assert!(build_text().contains(&options), "the sysroot key reads no C++ runtime option: {}", build_text()); + assert_ne!(key_of(&root, &was.freestanding, &build_text().replace("--release", "--profile=dev")), was.sysroot, + "libc's cargo invocation kept the sysroot"); let std = fork.join("library/std/src/lib.rs"); write(&std, "//! std, documented\npub fn exit() {}\n"); @@ -1077,7 +1175,7 @@ mod tests { same(manifest); } - let compiler = Compiler::primary(&rust_dir); + let compiler = Compiler::primary(&rust_dir).key(); let config = keyed_std_config(); assert_eq!(freestanding_key_of(&root, &compiler, &fork, RECIPE, &config), was.freestanding); assert_ne!(freestanding_key_of(&root, &compiler, &fork, RECIPE, ""), was.freestanding, @@ -1095,6 +1193,60 @@ mod tests { assert_eq!(now.identity.stale(Some(&stamp)), Some(Stale::All), "another compiler kept a crate's host half"); } + /// **A submodule is the commit its gitlink records, checked out or not**: + /// a fork whose `library/backtrace` is not checked out yet, as a runner's + /// is when it keys the stores its build then makes, keys its freestanding + /// libraries as it does once the build has checked it out. Another commit + /// moves the key; an edit there, or a gitlink staged and not committed, is + /// refused. + #[test] + fn a_submodule_is_the_commit_its_gitlink_records_checked_out_or_not() { + let base = TempDir::new("key-submodule"); + let (root, rust_dir, _) = keyed(&base); + let backtrace = base.join("backtrace-src"); + write(&backtrace.join("src/lib.rs"), "pub fn trace() {}\n"); + git(&backtrace, &["init", "-q"]); + git(&backtrace, &["add", "-A"]); + git(&backtrace, &["commit", "-qm", "backtrace"]); + let fork = base.join("fork-src"); + write(&fork.join("library/std/src/lib.rs"), "pub fn exit() {}\n"); + write(&fork.join("src/bootstrap/src/lib.rs"), "fn main() {}\n"); + git(&fork, &["init", "-q"]); + git(&fork, &["submodule", "add", "-q", backtrace.to_str().unwrap(), "library/backtrace"]); + git(&fork, &["add", "-A"]); + git(&fork, &["commit", "-qm", "the fork"]); + let clone = base.join("clone"); + git(&base, &["clone", "-q", fork.to_str().unwrap(), clone.to_str().unwrap()]); + + let k = || freestanding_key(&root, &Compiler::primary(&rust_dir).key(), &clone); + assert!(fs::read_dir(clone.join("library/backtrace")).unwrap().next().is_none(), "the clone checked its submodule out"); + let unchecked = k(); + git(&clone, &["submodule", "update", "-q", "--init", "library/backtrace"]); + assert_eq!(k(), unchecked, "checking the submodule out moved the key"); + + write(&clone.join("library/backtrace/src/lib.rs"), "pub fn trace() { loop {} }\n"); + let said = refusal(|| drop(k())); + assert!(said.contains("library/backtrace holds changes no commit does"), "{said}"); + git(&clone.join("library/backtrace"), &["commit", "-qam", "another backtrace"]); + git(&clone, &["add", "library/backtrace"]); + let said = refusal(|| drop(k())); + assert!(said.contains("stages library/backtrace"), "{said}"); + git(&clone, &["commit", "-qm", "another backtrace"]); + assert_ne!(k(), unchecked, "another backtrace commit kept the key"); + } + + /// **A sysroot's recorded witness is the one its build wrote**, read back + /// whole; a `SOURCES` naming no fork records none. + #[test] + fn a_sysroot_records_the_witness_it_was_built_from() { + let dir = TempDir::new("recorded-witness"); + let witness = "toyos-abi/src/lib.rs:0011223344556677\ntoyos/Cargo.toml:8899aabbccddeeff"; + fs::write(dir.join(SOURCES), sources_text(&Key::of(b"a sysroot"), Path::new("/a/fork/rust"), witness)).unwrap(); + assert_eq!(recorded_witness(&dir), Ok(witness.to_string())); + fs::write(dir.join(SOURCES), "0123456789abcdef\n").unwrap(); + assert!(recorded_witness(&dir).is_err(), "a SOURCES with no fork line recorded a witness"); + } + /// **A crate's compiler is the one that built it, rebuilt in place or /// not**: the primary's is rebuilt where it stands, so a driver its rebuild /// left leaves all of a crate stale, and the same driver none of it. @@ -1315,9 +1467,8 @@ mod tests { write(&compiler.stage2.join("bin/rustc"), "rustc"); let lld = toolchain::rust_lld(&compiler.stage2); write(&lld, "lld"); - write(&lld.with_file_name("llvm-ar"), "llvm-ar"); if clang { - for tool in ["clang", "ld.lld"] { + for tool in ["clang", "llvm-ar", "ld.lld", "rust-objcopy"] { write(&lld.with_file_name(tool), tool); } write(&lld.parent().unwrap().parent().unwrap().join("lib/clang/22/include/stddef.h"), "stddef"); diff --git a/src/toolchain.rs b/src/toolchain.rs index 0126f827d76..7e4c29e1ee6 100644 --- a/src/toolchain.rs +++ b/src/toolchain.rs @@ -10,7 +10,7 @@ use crate::buildlock::Scope; use crate::sysroot::{self, Sysroot, SYSROOT_SOURCES}; /// Whether the primary's compiler needs a bootstrap. `invalidate_hosted` -/// separates "the compiler changed" from "the rustup link is missing": only the +/// separates "the compiler changed" from "its `rustc` does not run": only the /// first makes the ToyOS-hosted rustc stale, and rebuilding that one costs /// minutes. #[derive(Clone, Copy, PartialEq, Debug)] @@ -121,6 +121,31 @@ pub(crate) fn stage2(rust_dir: &Path) -> PathBuf { rust_dir.join(format!("build/{}/stage2", host_triple())) } +/// The ToyOS-hosted rustc's toolchain directory, beside the primary's compiler. +fn hosted_stage2(rust_dir: &Path) -> PathBuf { + rust_dir.join(format!("build/{}/stage2", HOSTED_ARCH.userland())) +} + +/// Whether the primary builds the hosted rustc: a build whose config ships it +/// `asked`, and `rustc` is not there or `stamp`, which says it is this +/// compiler's, is not. +fn hosted_rustc_owed(asked: bool, stamp: &Path, rustc: &Path) -> bool { + asked && (!stamp.exists() || !rustc.exists()) +} + +/// Remove the hosted rustc a compiler rebuild left stale, and its `stamp`: no +/// build reads one until a config that ships it asks, and that build makes it +/// anew. +fn forget_hosted_rustc(rust_dir: &Path, stamp: &Path) { + let gone = |path: &Path, removed: std::io::Result<()>| match removed { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => panic!("remove {}: {e}", path.display()), + _ => {} + }; + gone(stamp, fs::remove_file(stamp)); + let stale = hosted_stage2(rust_dir); + gone(&stale, fs::remove_dir_all(&stale)); +} + /// Every `toyos-abi`/`toyos` source file a std build under `dep_info` actually /// compiled, read out of cargo's dep-info rather than out of what was asked for. fn std_toyos_sources(dep_info: &Path) -> Vec { @@ -241,12 +266,17 @@ pub(crate) fn assert_std_reads_no_worktree(root: &Path, fork: &Path, dep_info: & ); } -/// What an installed toolchain's sysroot was built from, as its publisher +/// What an installed toolchain's sysroot was built from, as its install /// recorded it (`src/release.rs`). -fn witness_path(rust_dir: &Path) -> PathBuf { +pub(crate) fn witness_path(rust_dir: &Path) -> PathBuf { rust_dir.join("build/toyos-sysroot-witness") } +/// The `TOOLCHAIN` an installed toolchain was installed with (`src/release.rs`). +pub(crate) fn manifest_path(rust_dir: &Path) -> PathBuf { + rust_dir.join("build/TOOLCHAIN") +} + /// The lines of a witness belonging to `trees`. fn witness_subset(text: &str, trees: &[&str]) -> String { @@ -354,11 +384,9 @@ fn cargo_link_stale(stage2: &Path) -> bool { /// Put a `cargo` beside the toolchain's `rustc`. /// /// **A symlink, and what survives the artifact round-trip is this step rather -/// than the link.** `src/release.rs` excludes it from the tarball for the reason -/// it excludes `lib/rustlib/`: it names a path only the publishing runner -/// has, and a copy would put a 32 MB host binary into a 401 MiB artifact to -/// stand in for a file the consumer can make in a microsecond. `Owner::Installed` -/// makes it, exactly as it makes the host target. +/// than the link.** `src/release.rs` excludes it from the tarball: it names a +/// path only the publishing runner has. `Owner::Installed` makes it, exactly as +/// it makes the host target. pub(crate) fn provision_toolchain_cargo(stage2: &Path) { let at = stage2.join("bin/cargo"); let _ = fs::remove_file(&at); @@ -457,18 +485,31 @@ fn rebuild_compiler(rust_dir: &Path, llvm: &Path, bootstrap: impl FnOnce()) { } /// What the primary bootstraps: a new compiler when `stage2` is not the one its -/// `compiler/` names, and the same one again when rustup has no `toyos` -/// toolchain to run. -fn bootstrap(current: bool, toolchain_exists: bool) -> Option { +/// fork checkout names, and the same one again when its `rustc` does not run. +/// A `stage2` that runs and has no rustup link, as one a runner restored, is +/// linked, not rebuilt. +fn bootstrap(current: bool, runs: bool) -> Option { if !current { Some(Bootstrap { invalidate_hosted: true }) } else { - (!toolchain_exists).then_some(Bootstrap { invalidate_hosted: false }) + (!runs).then_some(Bootstrap { invalidate_hosted: false }) } } +/// Whether the `rustc` in `stage2` runs. +fn runs(stage2: &Path) -> bool { + Command::new(stage2.join("bin/rustc")) + .arg("--version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .is_ok_and(|s| s.success()) +} + /// Ensure the toolchain is up to date, and return the sysroot this checkout's -/// sources name — made if nobody has made it (`src/sysroot.rs`). +/// sources name — made if nobody has made it (`src/sysroot.rs`). The primary +/// builds the ToyOS-hosted rustc only for a build whose config ships it +/// (`hosted_rustc`). /// /// Every step decides under the caller's shared lock and acts under the /// exclusive one, so the common answer — nothing to do — costs no @@ -490,7 +531,7 @@ fn bootstrap(current: bool, toolchain_exists: bool) -> Option { /// `stage1-std//dist/deps` while another's `rustc` creates a temp file /// inside it, and the loser dies compiling `core` with `couldn't create a temp /// dir: No such file or directory`. -pub fn ensure(root: &Path, lock: &mut buildlock::Held) -> Sysroot { +pub fn ensure(root: &Path, lock: &mut buildlock::Held, hosted_rustc: bool) -> Sysroot { let rust_dir = rust_dir(root); let stamps_dir = root.join("target/stamps"); fs::create_dir_all(&stamps_dir).ok(); @@ -510,9 +551,9 @@ pub fn ensure(root: &Path, lock: &mut buildlock::Held) -> Sysroot { } Owner::Installed => { check_installed_toolchain(root, &rust_dir); - let release = rust_dir.join("build/TOOLCHAIN"); + let release = manifest_path(&rust_dir); let release = fs::read_to_string(&release).unwrap_or_else(|e| { - panic!("{}: {e}; an installed toolchain carries the TOOLCHAIN it was published with", release.display()) + panic!("{}: {e}; an installed toolchain carries the TOOLCHAIN it was installed with", release.display()) }); return Sysroot::installed(stage2(&rust_dir), &release); } @@ -523,36 +564,26 @@ pub fn ensure(root: &Path, lock: &mut buildlock::Held) -> Sysroot { lock.act_if( Scope::Global, "build the rust toolchain", - || { - let current = crate::compiler::primary_is_current(&rust_dir); - let toolchain_exists = Command::new("rustup") - .args(["run", "toyos", "rustc", "--version"]) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status() - .map(|s| s.success()) - .unwrap_or(false); - bootstrap(current, toolchain_exists) - }, + || bootstrap(crate::compiler::primary_is_current(&rust_dir), runs(&stage2(&rust_dir))), |kind| { eprintln!("Building full toolchain (this takes a while on first run)..."); let llvm = crate::llvm::resolve(root, &rust_dir, &rust_dir); - rebuild_compiler(&rust_dir, &llvm.dir, || full_bootstrap(root, &rust_dir, &llvm.dir)); + rebuild_compiler(&rust_dir, &llvm.dir, || full_bootstrap(&rust_dir, &llvm.dir)); if kind.invalidate_hosted { - let _ = fs::remove_file(&hosted_stamp); + forget_hosted_rustc(&rust_dir, &hosted_stamp); } }, ); - let hosted_rustc = rust_dir.join(format!("build/{}/stage2/bin/rustc", HOSTED_ARCH.userland())); + let hosted = hosted_stage2(&rust_dir).join("bin/rustc"); lock.act_if( Scope::Global, "build the ToyOS-hosted rustc", - || (!hosted_stamp.exists() || !hosted_rustc.exists()).then_some(()), + || hosted_rustc_owed(hosted_rustc, &hosted_stamp, &hosted).then_some(()), |()| { let llvm = crate::llvm::resolve(root, &rust_dir, &rust_dir); reassemble(&rust_dir, &llvm.dir, || build_hosted_rustc(&rust_dir, &llvm.dir)); - assert!(hosted_rustc.exists(), "Failed to build hosted rustc"); + assert!(hosted.exists(), "Failed to build hosted rustc"); fs::write(&hosted_stamp, "").unwrap(); }, ); @@ -584,11 +615,6 @@ pub fn ensure(root: &Path, lock: &mut buildlock::Held) -> Sysroot { /// Everything a checkout may do with a toolchain it did not build: check that /// it is the one this tree needs, and say what to do when it is not. -/// -/// No amount of source here can rebuild a sysroot without `rust/`, so there is -/// nothing to decide and the answer is always to publish a toolchain built from -/// these sources. Its std fork is pinned by the release tag, which is a function -/// of `rust` (`src/release.rs`). fn check_installed_toolchain(root: &Path, rust_dir: &Path) { let stage2 = stage2(rust_dir); let linked = rustup_link(); @@ -603,10 +629,7 @@ fn check_installed_toolchain(root: &Path, rust_dir: &Path) { // Recreated rather than shipped: both of these point into whatever stable // toolchain this machine has, which is not a path any artifact can know. - // This is CI's whole share of the cargo provisioning — it links its - // toolchain fresh from the published artifact on every run, so nothing - // upstream of the download can have put one there. Its clang is the - // artifact's own, so this is not `complete`. + // Its clang is the artifact's own, so this is not `complete`. if host_target_missing(rust_dir) { link_host_target(rust_dir); } @@ -621,7 +644,8 @@ fn check_installed_toolchain(root: &Path, rust_dir: &Path) { recorded.as_deref() == Some(want.as_str()), "this checkout and the installed toolchain at {} disagree about {}, so a build \ here would link its kernel against another tree's struct layouts.\n\ - Publish a toolchain built from these sources and install that one instead.", + A runner installs the sysroot its job restored by this tree's key; if that is the one \ + installed, the key reads less than the sysroot is built from (`src/sysroot.rs`).", stage2.display(), differing_trees(recorded.as_deref(), &want), ); @@ -668,8 +692,31 @@ pub(crate) fn x_build(rust_dir: &Path, args: &[&str], what: &str) -> (bool, Vec< x_build_with(rust_dir, args, what, |_| {}) } +/// [`x_build`] of a compiler that links the LLVM at `llvm`. On an Apple host +/// rustc strips a Darwin binary by running `rust-objcopy` (`rustc_codegen_ssa`'s +/// `back/link.rs`), the rust workspace strips `lld-wrapper`, and the stage-1 +/// sysroot carries no `rust-objcopy` when bootstrap copies none of LLVM's tools +/// ([`LEAN`]): the build finds that LLVM's `llvm-objcopy` by that name on `PATH`. +pub(crate) fn x_build_compiler(rust_dir: &Path, args: &[&str], what: &str, llvm: &Path) -> (bool, Vec) { + if !host_triple().ends_with("apple-darwin") { + return x_build(rust_dir, args, what); + } + let strip = toyos_tmpdir::TempDir::new("rust-objcopy"); + let objcopy = llvm.join("bin").join(crate::llvm::APPLE_TOOL); + std::os::unix::fs::symlink(&objcopy, strip.join("rust-objcopy")) + .unwrap_or_else(|e| panic!("link {} as rust-objcopy: {e}", objcopy.display())); + let caller = std::env::var_os("PATH").unwrap_or_else(|| panic!("PATH is unset, and bootstrap finds its tools on it")); + let path = std::env::join_paths(std::iter::once(strip.to_path_buf()).chain(std::env::split_paths(&caller))) + .unwrap_or_else(|e| panic!("{} cannot lead PATH: {e}", strip.display())); + x_build_with(rust_dir, args, what, |command| { + command.env("PATH", path); + }) +} + /// [`x_build`], with bootstrap's environment what `environment` makes of this -/// process's. +/// process's, less GitHub Actions' `GITHUB_ACTIONS` and `CI`: bootstrap takes +/// `HEAD^1` as the upstream commit whose artifacts to fetch when it sees them, +/// and in this fork that is our own merge, which rust-lang's CI never built. pub(crate) fn x_build_with( rust_dir: &Path, args: &[&str], @@ -686,6 +733,7 @@ pub(crate) fn x_build_with( let x = if rust_dir.join("x").exists() { "./x" } else { "./x.py" }; let mut command = Command::new(x); environment(&mut command); + command.env_remove("GITHUB_ACTIONS").env_remove("CI"); let mut child = command .args(args) .env("BOOTSTRAP_SKIP_TARGET_SANITY", "1") @@ -746,16 +794,6 @@ impl Drop for Restore { } /// Where a compile error starts in an `x build` log, if there is one. -/// -/// Both bootstrap callers let a non-zero `x build` through when the artifacts -/// they need are on disk, because rustdoc for ToyOS does not link and never -/// has. That allowance used to be *anything at all*, as long as a `rustc` from -/// some earlier build was still there — so run `31370078581` compiled std with -/// `error[E0433]`, took the allowance, and died 83 seconds and 260 lines later -/// at a missing file. The reported failure was the consequence. -/// -/// A compile error cannot be a link failure, so it cannot be the thing that -/// allowance is for. fn compile_error_at(log: &[String]) -> Option { log.iter().position(|l| { let l = l.trim_start(); @@ -786,12 +824,15 @@ fn tolerated_failure(log: &[String], what: &str) { ); } -fn full_bootstrap(root: &Path, rust_dir: &Path, llvm: &Path) { +/// What the primary's compiler build builds: rustc and the host's libraries, +/// which build scripts and proc macros link. No rustdoc: no build runs one. +const COMPILER_BUILD: [&str; 7] = ["build", "--stage", "2", "--warnings", "warn", "compiler/rustc", "library"]; + +fn full_bootstrap(rust_dir: &Path, llvm: &Path) { // Ensure library/backtrace is checked out — std depends on it. // Other rust submodules (llvm, docs, cargo) are handled by bootstrap on demand. crate::ensure_submodule(rust_dir, "library/backtrace"); - // Write bootstrap.toml — ToyOS as target only, not host (fast rebuilds) let host = host_triple(); write_config(rust_dir, &host, false, llvm); @@ -804,28 +845,9 @@ fn full_bootstrap(root: &Path, rust_dir: &Path, llvm: &Path) { } } - let build = ["build", "--stage", "2", "--warnings", "warn"]; - let (ok, log) = x_build(rust_dir, &build, "the toolchain"); - - if !ok { - refuse_on_compile_error(&log, "the toolchain"); - // rustdoc for ToyOS may fail to link; rustc may not be missing. - let stage2 = rust_dir.join(format!("build/{host}/stage2")); - assert!( - stage2.join("bin/rustc").exists(), - "the toolchain build failed and {} is not there.\n\ - Nothing in its output was a compile error, so this is a link or a bootstrap \ - failure — the last lines above are the whole of what it said.", - stage2.join("bin/rustc").display() - ); - tolerated_failure(&log, "the toolchain build"); - } - for arch in Arch::ALL { - assert_std_built_from( - root, - &rust_dir.join(format!("build/{host}/stage1-std/{}", arch.userland())), - ); - } + let (ok, log) = x_build_compiler(rust_dir, &COMPILER_BUILD, "the toolchain", llvm); + refuse_on_compile_error(&log, "the toolchain"); + assert!(ok, "the toolchain build failed, and nothing in its output was a compile error"); } fn build_hosted_rustc(rust_dir: &Path, llvm: &Path) { @@ -834,7 +856,7 @@ fn build_hosted_rustc(rust_dir: &Path, llvm: &Path) { write_config(rust_dir, &host, true, llvm); let (ok, log) = - x_build(rust_dir, &["build", "--stage", "2", "--warnings", "warn"], "the hosted rustc"); + x_build_compiler(rust_dir, &["build", "--stage", "2", "--warnings", "warn"], "the hosted rustc", llvm); refuse_on_compile_error(&log, "the hosted rustc"); // rustdoc for ToyOS may fail to link; rustc and librustc_driver may not. @@ -861,11 +883,7 @@ fn build_hosted_rustc(rust_dir: &Path, llvm: &Path) { // (`write_config` says why), so the host-only build runs once more to put // it back: everything it would compile is already built. write_config(rust_dir, &host, false, llvm); - let (ok, log) = x_build( - rust_dir, - &["build", "--stage", "2", "--warnings", "warn"], - "the toolchain, reassembled", - ); + let (ok, log) = x_build_compiler(rust_dir, &COMPILER_BUILD, "the toolchain, reassembled", llvm); refuse_on_compile_error(&log, "the toolchain, reassembled"); assert!( rust_lld(&stage2(rust_dir)).is_file(), @@ -877,7 +895,8 @@ fn build_hosted_rustc(rust_dir: &Path, llvm: &Path) { } } -/// `bootstrap.toml` for the host-only toolchain, or with the ToyOS-hosted rustc. +/// `bootstrap.toml` for the host-only toolchain, every compiler's +/// (`compiler::config_text`), or with the ToyOS-hosted rustc. /// /// `lld = true` is what puts `rust-lld` in every stage's sysroot, where rustc /// finds the linker every guest target names. The hosted rustc's build cannot @@ -893,22 +912,57 @@ fn build_hosted_rustc(rust_dir: &Path, llvm: &Path) { /// otherwise ties it to `lld` for `x86_64-unknown-linux-gnu`, and a host rustc /// whose build environment flips with the config is rebuilt by each of those /// two builds. +/// +/// The host-only toolchain builds no guest target's libraries: every sysroot +/// builds its own (`src/sysroot.rs`). fn write_config(rust_dir: &Path, host: &str, with_hosted_rustc: bool, llvm: &Path) { - let host_line = if with_hosted_rustc { - format!("host = [\"{host}\", \"{}\"]", HOSTED_ARCH.userland()) + let config = if with_hosted_rustc { + let targets = std::iter::once(host) + .chain(GUEST_TARGETS.map(GuestTarget::triple)) + .map(|t| format!("\"{t}\"")) + .collect::>() + .join(", "); + format!( + r#"change-id = "ignore" +profile = "compiler" + +[build] +host = ["{host}", "{hosted}"] +target = [{targets}] + +[llvm] +{llvm} + +[rust] +incremental = true +lld = false +{LEAN} + +[target.{host}] +{HOST_LINKER_PIN} +{external} + +{userland}"#, + hosted = HOSTED_ARCH.userland(), + llvm = crate::clang::LLVM_CONFIG, + external = crate::llvm::host_lines(llvm), + userland = hosted_targets(llvm), + ) } else { - format!("host = [\"{host}\"]") + crate::compiler::config_text(&rust_dir.join("build"), host, llvm) }; - let targets = std::iter::once(host) - .chain(GUEST_TARGETS.map(GuestTarget::triple)) - .map(|t| format!("\"{t}\"")) - .collect::>() - .join(", "); - let userland: String = Arch::ALL + fs::write(rust_dir.join("bootstrap.toml"), config).unwrap(); +} + +/// The `[target]` sections of ToyOS userland in the hosted rustc's build: each +/// links with the LLD of the LLVM at `llvm`, and the one the hosted rustc runs +/// on builds it. +fn hosted_targets(llvm: &Path) -> String { + Arch::ALL .iter() .map(|arch| { let linker = format!("linker = \"{}\"", llvm.join("bin/lld").display()); - let hosted = if with_hosted_rustc && *arch == HOSTED_ARCH { + let hosted = if *arch == HOSTED_ARCH { // Cranelift because no LLVM is built for a ToyOS host yet, and // only for that reason: the hosted rustc carries LLVM once clang // and libc++ run on ToyOS, and Cranelift is not where the @@ -927,38 +981,20 @@ fn write_config(rust_dir: &Path, host: &str, with_hosted_rustc: bool, llvm: &Pat }; format!("[target.{}]\n{linker}{hosted}\nrpath = false\n\n", arch.userland()) }) - .collect(); - let config = format!( - r#"change-id = "ignore" -profile = "compiler" - -[build] -{host_line} -target = [{targets}] - -[llvm] -{llvm} - -[rust] -incremental = true -lld = {lld} - -[target.{host}] -{HOST_LINKER_PIN} -{external} - -{userland}"#, - llvm = crate::clang::LLVM_CONFIG, - external = crate::llvm::host_lines(llvm), - lld = !with_hosted_rustc, - ); - fs::write(rust_dir.join("bootstrap.toml"), config).unwrap(); + .collect() } /// What the host rustc links its own binaries with, held to one answer in every /// `bootstrap.toml` that builds a host compiler: [`write_config`] says why. pub(crate) const HOST_LINKER_PIN: &str = "default-linker-linux-override = \"off\""; +/// The `[rust]` options every `bootstrap.toml` that builds a compiler shares +/// beyond its profile's: no LLVM tool copied into the compiler's sysroot, since +/// `clang::provision` puts there the ones a build runs; no debuginfo in rustc, +/// which no build reads; and no codegen test, for which bootstrap demands +/// LLVM's `FileCheck` beside `llvm-config` (`src/bootstrap/src/core/sanity.rs`). +pub(crate) const LEAN: &str = "llvm-tools = false\ndebuginfo-level-rustc = 0\ncodegen-tests = false"; + /// The linker every guest target names, as the toolchain at `toolchain` carries /// it: `lib/rustlib//bin/rust-lld`, where rustc itself looks for it. pub fn rust_lld(toolchain: &Path) -> PathBuf { @@ -1071,6 +1107,39 @@ mod tests { panic!("re-locked both, and failed"); } + /// **A compiler build on an Apple host finds its LLVM's `llvm-objcopy` as + /// the `rust-objcopy` its stage-1 rustc strips with**, first on its `PATH`; + /// on any other host it finds none of ours. `./x` here is this test binary, + /// running [`a_fake_bootstrap_that_strips`]. + #[test] + fn a_compiler_build_finds_the_llvm_s_objcopy_where_rustc_strips_with_it() { + let fork = TempDir::new("x-build-strip"); + fs::create_dir_all(fork.join("library")).unwrap(); + for lock in ["Cargo.lock", "library/Cargo.lock"] { + fs::write(fork.join(lock), "# as committed\n").unwrap(); + } + fs::write(fork.join(FAKE), "").unwrap(); + std::os::unix::fs::symlink(std::env::current_exe().unwrap(), fork.join("x")).unwrap(); + let llvm = fork.join("llvm"); + fs::create_dir_all(llvm.join("bin")).unwrap(); + fs::write(llvm.join("bin").join(crate::llvm::APPLE_TOOL), "the llvm-objcopy").unwrap(); + + let args = ["--exact", "toolchain::tests::a_fake_bootstrap_that_strips", "--include-ignored", "--nocapture"]; + let (ok, log) = x_build_compiler(&fork, &args, "a fake bootstrap", &llvm); + assert!(ok, "the fake bootstrap did not run: {log:?}"); + let found = if host_triple().ends_with("apple-darwin") { "the llvm-objcopy" } else { "none" }; + assert!(log.iter().any(|l| *l == format!("rust-objcopy: {found}")), "{log:?}"); + } + + #[test] + #[ignore = "the bootstrap `a_compiler_build_finds_the_llvm_s_objcopy_where_rustc_strips_with_it` runs; never runs on its own"] + fn a_fake_bootstrap_that_strips() { + assert!(Path::new(FAKE).is_file(), "a_fake_bootstrap_that_strips ran outside a fake fork checkout; it is not a test"); + let path = std::env::var_os("PATH").unwrap_or_default(); + let first = std::env::split_paths(&path).map(|dir| dir.join("rust-objcopy")).find(|tool| tool.exists()); + println!("rust-objcopy: {}", first.map_or("none".to_string(), |tool| fs::read_to_string(tool).unwrap())); + } + /// **A restore that cannot write leaves the file as it found it**, and /// names what it could not write. #[test] @@ -1136,25 +1205,63 @@ mod tests { assert!(said.contains("clang") && !said.contains("rust-lld,"), "the refusal names clang alone: {said}"); } - /// Every build links the host's LLVM, and every guest links through its - /// LLD, named by path. + /// Every build links the host's LLVM, copies none of its tools and leaves + /// rustc without debuginfo; the host-only one builds no guest target, and + /// in the hosted rustc's each guest links through that LLVM's LLD, named + /// by path. #[test] fn every_build_links_the_host_s_llvm_and_names_its_lld_by_path() { let rust_dir = TempDir::new("lld-config"); let llvm = rust_dir.join("build/llvm/k"); let lld = format!("linker = \"{}\"", llvm.join("bin/lld").display()); - for (hosted, lld_flag) in [(true, "lld = false"), (false, "lld = true")] { + let lean = "\nllvm-tools = false\ndebuginfo-level-rustc = 0\ncodegen-tests = false\n"; + for (hosted, lld_flag) in [(true, "false"), (false, "true")] { write_config(&rust_dir, "h", hosted, &llvm); let config = fs::read_to_string(rust_dir.join("bootstrap.toml")).unwrap(); - assert!(config.contains(lld_flag) && config.contains(&lld) && !config.contains("\"rust-lld\""), "{config}"); + assert!(config.contains(&format!("\n[rust]\nincremental = true\nlld = {lld_flag}{lean}")), "{config}"); + assert_eq!(config.contains(&lld), hosted, "{config}"); + assert!(!config.contains("\"rust-lld\""), "{config}"); + assert_eq!(config.contains("\ntarget = [\"h\"]\n"), !hosted, "{config}"); let host = format!( "[target.h]\ndefault-linker-linux-override = \"off\"\nllvm-config = \"{}/bin/llvm-config\"\nllvm-has-rust-patches = true\n", llvm.display() ); assert!(config.contains(&host), "{config}"); + if !hosted { + let keyed = crate::compiler::config_text(&rust_dir.join("build"), "h", &llvm); + assert_eq!(config, keyed, "the primary's compiler is built under a configuration its key does not read"); + } } } + /// **Bootstrap never sees GitHub Actions' variables**, whatever its + /// caller's environment: it takes `HEAD^1`'s artifacts when it does. `./x` + /// here is this test binary, running [`a_fake_bootstrap_that_reads_ci`]. + #[test] + fn a_bootstrap_run_sees_no_ci_variables() { + let fork = TempDir::new("x-build-ci"); + fs::create_dir_all(fork.join("library")).unwrap(); + for lock in ["Cargo.lock", "library/Cargo.lock"] { + fs::write(fork.join(lock), "# as committed\n").unwrap(); + } + fs::write(fork.join(FAKE), "").unwrap(); + std::os::unix::fs::symlink(std::env::current_exe().unwrap(), fork.join("x")).unwrap(); + let args = ["--exact", "toolchain::tests::a_fake_bootstrap_that_reads_ci", "--include-ignored", "--nocapture"]; + let (ok, log) = x_build_with(&fork, &args, "a fake bootstrap", |command| { + command.env("GITHUB_ACTIONS", "true").env("CI", "true"); + }); + assert!(ok, "the fake bootstrap did not run: {log:?}"); + assert!(log.iter().any(|l| l == "GITHUB_ACTIONS none, CI none"), "{log:?}"); + } + + #[test] + #[ignore = "the bootstrap `a_bootstrap_run_sees_no_ci_variables` runs; never runs on its own"] + fn a_fake_bootstrap_that_reads_ci() { + assert!(Path::new(FAKE).is_file(), "a_fake_bootstrap_that_reads_ci ran outside a fake fork checkout; it is not a test"); + let read = |name| std::env::var(name).unwrap_or_else(|_| "none".to_string()); + println!("GITHUB_ACTIONS {}, CI {}", read("GITHUB_ACTIONS"), read("CI")); + } + /// **A bootstrap leaves the primary nothing that waits on another /// worktree's sysroot build**: the act that reassembles `stage2` completes it /// before its exclusive hold ends, so the step after it — run while a @@ -1177,7 +1284,13 @@ mod tests { /// The LLVM `clang::provision` reads, in `rust_dir`'s store. fn store_llvm(rust_dir: &Path) -> PathBuf { let llvm = rust_dir.join("build/llvm/k"); - for (file, text) in [("bin/clang", "clang"), ("lib/clang/22/include/stddef.h", "stddef")] { + let files = [ + ("bin/clang", "clang"), + ("bin/llvm-ar", "llvm-ar"), + ("bin/llvm-objcopy", "llvm-objcopy"), + ("lib/clang/22/include/stddef.h", "stddef"), + ]; + for (file, text) in files { fs::create_dir_all(llvm.join(file).parent().unwrap()).unwrap(); fs::write(llvm.join(file), text).unwrap(); } @@ -1185,13 +1298,12 @@ mod tests { } /// What bootstrap leaves in `rust_dir`: `stage2` made again, with `rustc` - /// and the LLVM tools it assembles, and neither cargo nor clang; and the - /// hosted rustc's sysroot without the host target. + /// and `rust-lld` and none of cargo, clang or an LLVM tool; and the hosted + /// rustc's sysroot without the host target. fn bootstrapped(rust_dir: &Path) { let stage2 = stage2(rust_dir); let _ = fs::remove_dir_all(&stage2); - let lld = rust_lld(&stage2); - for file in [stage2.join("bin/rustc"), lld.clone(), lld.with_file_name("llvm-ar")] { + for file in [stage2.join("bin/rustc"), rust_lld(&stage2)] { fs::create_dir_all(file.parent().unwrap()).unwrap(); fs::write(file, b"").unwrap(); } @@ -1263,23 +1375,70 @@ mod tests { assert!(own.iter().all(|dir| dir.join("bin/tool").is_file()), "a stopped bootstrap took the LLVM its compiler linked"); } + /// **The hosted rustc is built only for a build whose config ships it**, + /// and then only when the one this compiler built is not there. + #[test] + fn the_hosted_rustc_is_built_only_when_a_build_ships_it() { + let rust_dir = TempDir::new("hosted-owed"); + let stamp = rust_dir.join("stamps/hosted-rustc.stamp"); + let rustc = hosted_stage2(&rust_dir).join("bin/rustc"); + assert!(!hosted_rustc_owed(false, &stamp, &rustc), "a build that ships no hosted rustc built one"); + assert!(hosted_rustc_owed(true, &stamp, &rustc)); + for file in [&stamp, &rustc] { + fs::create_dir_all(file.parent().unwrap()).unwrap(); + fs::write(file, "").unwrap(); + } + assert!(!hosted_rustc_owed(true, &stamp, &rustc), "a hosted rustc this compiler built was built again"); + fs::remove_file(&stamp).unwrap(); + assert!(hosted_rustc_owed(true, &stamp, &rustc), "a hosted rustc of another compiler was taken"); + } + + /// **A compiler rebuild leaves no hosted rustc of the compiler it + /// replaced**, nor its stamp; with neither there, that is no error. + #[test] + fn a_rebuilt_compiler_leaves_no_hosted_rustc_of_the_one_before() { + let rust_dir = TempDir::new("hosted-stale"); + let stamp = rust_dir.join("stamps/hosted-rustc.stamp"); + let rustc = hosted_stage2(&rust_dir).join("bin/rustc"); + for file in [&stamp, &rustc] { + fs::create_dir_all(file.parent().unwrap()).unwrap(); + fs::write(file, "").unwrap(); + } + forget_hosted_rustc(&rust_dir, &stamp); + assert!(!stamp.exists() && !hosted_stage2(&rust_dir).exists(), "the old compiler's hosted rustc stayed"); + forget_hosted_rustc(&rust_dir, &stamp); + } + /// **The primary bootstraps a new compiler exactly when its `stage2` is not /// current, and otherwise only when rustup has none.** #[test] fn the_primary_bootstraps_when_stale_or_missing() { let new = Some(Bootstrap { invalidate_hosted: true }); let again = Some(Bootstrap { invalidate_hosted: false }); - for (current, toolchain_exists, want) in [ + for (current, runs, want) in [ (true, true, None), (true, false, again), (false, true, new), (false, false, new), ] { - assert_eq!( - bootstrap(current, toolchain_exists), - want, - "current {current}, toolchain_exists {toolchain_exists}" - ); + assert_eq!(bootstrap(current, runs), want, "current {current}, runs {runs}"); + } + } + + /// **What decides a rebuild of a current `stage2` is whether its `rustc` + /// runs**, not whether rustup names it: one a runner restored has no + /// rustup link and is linked, not built again. + #[test] + fn a_compiler_s_rustc_runs_or_it_is_built_again() { + let stage2 = TempDir::new("runs"); + let rustc = stage2.join("bin/rustc"); + assert!(!runs(&stage2), "a stage2 with no rustc ran"); + fs::create_dir_all(rustc.parent().unwrap()).unwrap(); + // This test binary refuses `--version`; the host's rustc answers it. + for (what, ran) in [(std::env::current_exe().unwrap(), false), (host_sysroot().join("bin/rustc"), true)] { + let _ = fs::remove_file(&rustc); + std::os::unix::fs::symlink(&what, &rustc).unwrap(); + assert_eq!(runs(&stage2), ran, "{}", what.display()); } } diff --git a/tests/common/compile.rs b/tests/common/compile.rs index b573d22212b..d18e0d5273a 100644 --- a/tests/common/compile.rs +++ b/tests/common/compile.rs @@ -23,7 +23,7 @@ pub fn c_sysroot() -> CSysroot { static SYSROOT: OnceLock = OnceLock::new(); let sysroot = SYSROOT.get_or_init(|| { let mut lock = toyos_build::buildlock::shared(&repo_root(), "the C sysroot"); - toyos_build::toolchain::ensure(&repo_root(), &mut lock) + toyos_build::toolchain::ensure(&repo_root(), &mut lock, false) }); CSysroot::of(sysroot.dir(), super::qemu::SUITE_ARCH) } diff --git a/tests/common/lane.rs b/tests/common/lane.rs index 526a0669856..23001849676 100644 --- a/tests/common/lane.rs +++ b/tests/common/lane.rs @@ -66,12 +66,11 @@ static RUN: OnceLock = OnceLock::new(); /// it, and it is gone when the run is, green or red (`toyos_tmpdir` is the /// policy, and what reclaims the directory of a run that was killed). /// -/// A red run's serial logs are the parts of it read afterwards — by an agent -/// and by the nightly's artifact — so they are copied to a directory of their -/// own under [`RED_RUN_SERIAL`] first: megabytes, where the images are -/// gigabytes. Named for this run's own root — unique across every process a -/// shared `$TMPDIR` ever holds — so two red runs of one worktree never share, -/// and neither overwrites, a destination. +/// A red run's serial logs are the parts of it read afterwards, so they are +/// copied to a directory of their own under [`RED_RUN_SERIAL`] first: +/// megabytes, where the images are gigabytes. Named for this run's own root — +/// unique across every process a shared `$TMPDIR` ever holds — so two red runs +/// of one worktree never share, and neither overwrites, a destination. /// /// [`Run::exit`] is the one way out of the suite with a status; returning from /// `main` drops this as green, and unwinding out of it as red. diff --git a/toyos-cpuvuln/src/tests.rs b/toyos-cpuvuln/src/tests.rs index f6b624a50ef..35d4808a1a6 100644 --- a/toyos-cpuvuln/src/tests.rs +++ b/toyos-cpuvuln/src/tests.rs @@ -602,7 +602,7 @@ fn rtm_always_abort_decides_tsx_first() { // The two fixtures below are this crate's reading of the pinned Linux, awaiting // a capture on a nightly runner: -// `issues/build/no-nightly-runner-has-had-its-cpuid-and-vulnerability-lines-captured.md`. +// `issues/build/no-kvm-runner-has-had-its-cpuid-and-vulnerability-lines-captured.md`. /// A guest's lines do not read its microcode: `tsa_init` returns under a /// hypervisor (`amd.c:519-520`) before `amd_check_tsa_microcode`.