diff --git a/.claude/agents/reviewer.md b/.claude/agents/reviewer.md index 6d7f9c6f80f..818c3a24104 100644 --- a/.claude/agents/reviewer.md +++ b/.claude/agents/reviewer.md @@ -76,6 +76,19 @@ if it meets the bar above; otherwise it is a NOTE. saying what it is; a new cargo feature or `cfg` arm of one, or an arm a changed `src/clippy.rs` shape stops building, that no shape in `src/clippy.rs` lints; an `issues/` file added, changed or deleted against `issues/README.md`. +- **Caches.** No gate reads these; a diff that breaks one is a BLOCKER. Each cache has one + writer, a nightly.yml job, and no workflow uses the combined `actions/cache`, which saves too; + the host cache's is nightly's `host`, and its one reader ci.yml's `host`, on the same + `runs-on`, both caching `src/cicache.rs`'s `PATHS` with its `DRIVER` as their + `CARGO_TARGET_DIR`, the one variable an `env:` gives either: an `ImageOS` or `ImageVersion` + set there outlives an image move. The reader's `restore-keys` is the writer's `key` up to its + run id. A job that names the host cache runs `actions/checkout`, its cache step and + `cargo run -- --ci `, `seal` in the writer and `host` in the reader, and nothing else; + the reader restores before that step, and the writer saves after it. The save's guard, + `github.ref == 'refs/heads/main'`, is the only step-level `if:` in those jobs; ci.yml's `host` + skips only a draft, and nightly's `host` has no `if:` of its own, a skipped job being a green + check. No `continue-on-error`, `shell:`, `defaults:` or cargo `runner` reaches them. + nightly.yml's `on:` is one daily `schedule` and `workflow_dispatch`. - **Growth.** Every line is a responsibility, not an asset. State the branch's net lines (`git diff --shortstat origin/main...`), production and tests apart. Production code that grows needs a reason you accept; a branch that could delete more than it adds and does not goes @@ -83,9 +96,9 @@ if it meets the bar above; otherwise it is a NOTE. reader can check: that rule is a sentence in a prompt. What could be deleted, merged into what exists, or made smaller? An abstraction with one caller, a parameter with one value, dead code, code kept "just in case" or because nobody knows whether it is needed. Size is never bought with - a weaker check: a test is cut only when it tests nothing, or as **Guest tests** says. A - compromise the branch found is removed or recorded in `issues/` with an owner, evidence and an - exit condition. + a weaker check: a test is cut only when it tests nothing, when this prompt takes its rule, or + as **Guest tests** says. A compromise the branch found is removed or recorded in `issues/` with + an owner, evidence and an exit condition. - **Tests.** The refusals and the boundary, not the happy path. - **Edges.** Untrusted input never panics the kernel; it is refused. Check-then-act races. A lock held across a user copy or a device wait. Arithmetic on a value the caller chooses. A short diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e4e1cfe272..6d45466c6a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,6 @@ name: ci -# A pull request and the merge queue: the host tests, and no guest. Each step -# is `cargo run -- --ci ` (src/ci.rs), which runs the same on a dev host; +# A pull request and the merge queue: the host tests, and no guest. # nightly.yml boots the guests. on: @@ -19,6 +18,10 @@ jobs: if: github.event_name == 'merge_group' || github.event.pull_request.draft == false runs-on: ubuntu-24.04 timeout-minutes: 45 + # The driver's own target, and no step's: it says this job carries the + # host cache (src/cicache.rs). + env: + CARGO_TARGET_DIR: target/ci-driver steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -35,7 +38,7 @@ jobs: toyos/target kernel/target bootloader/target - key: host-linux-${{ github.run_id }} - restore-keys: host-linux- + key: host-sealed-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }} + restore-keys: host-sealed-${{ runner.os }}-${{ runner.arch }}- - run: cargo run -- --ci host diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c9010b31699..6d93ba85ac1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -16,17 +16,28 @@ concurrency: cancel-in-progress: false jobs: + # The host cache's writer restores nothing, and `seal` is green only once + # src/cicache.rs has sealed the tree the save stores. host: runs-on: ubuntu-24.04 timeout-minutes: 90 + # The driver's own target, and no step's: it says this job carries the + # host cache (src/cicache.rs). + env: + CARGO_TARGET_DIR: target/ci-driver steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + - run: cargo run -- --ci seal + + # The host cache's one writer. Only main's entries are readable from + # every branch. + - if: github.ref == 'refs/heads/main' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: - path: &host-paths | + path: | ~/.cargo/registry/index ~/.cargo/registry/cache ~/.cargo/git/db @@ -35,18 +46,7 @@ jobs: toyos/target kernel/target bootloader/target - key: host-linux-${{ github.run_id }} - restore-keys: host-linux- - - - run: cargo run -- --ci host - - # The host cache's one writer. Only main's entries are readable from - # every branch. - - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: *host-paths - key: host-linux-${{ github.run_id }} + key: host-sealed-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }} # Publishes this tree's toolchain if nobody has, and on main moves the SDK # alias onto it. Bare `ubuntu-24.04`, not a container: its glibc is the diff --git a/issues/build/a-warm-host-run-keeps-what-a-registry-proc-macro-expanded-from-a-file-it-never-named.md b/issues/build/a-warm-host-run-keeps-what-a-registry-proc-macro-expanded-from-a-file-it-never-named.md new file mode 100644 index 00000000000..903aa16d908 --- /dev/null +++ b/issues/build/a-warm-host-run-keeps-what-a-registry-proc-macro-expanded-from-a-file-it-never-named.md @@ -0,0 +1,25 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# A warm host run keeps what a registry proc macro expanded from a file it never named + +A registry proc macro is compiled once and runs inside every compile of the +crate that expands it. One that reads a file without telling rustc, the way +`wayland-scanner`'s `generate_client_code!` opens its XML, reads it again only +when cargo recompiles the expanding crate. A warm `host` run +(`src/cicache.rs`) recompiles a path crate only when its own package changed, +so when that file sits outside the expanding crate's package and changes +alone, the warm run keeps the old expansion where a cold run makes a new one. +Cargo never dates a registry source, so nothing dates the macro's package. + +Of the proc macros in the lockfiles of the five workspaces the host job builds, +`wayland-scanner` alone reads a file it does not name, and no tracked source +names it. + +Owner: the host cache (`src/cicache.rs`). + +Done when a warm read serves what a cold build serves for a path crate that +expands a registry proc macro reading another package's file, with a test. diff --git a/issues/build/a-warm-host-run-never-relinks-for-a-file-only-a-flag-names.md b/issues/build/a-warm-host-run-never-relinks-for-a-file-only-a-flag-names.md new file mode 100644 index 00000000000..95081ea33c1 --- /dev/null +++ b/issues/build/a-warm-host-run-never-relinks-for-a-file-only-a-flag-names.md @@ -0,0 +1,21 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# A warm host run never relinks for a file only a flag names + +A file that reaches a build only through a flag, a linker script named by +`-Clink-arg=-T…` in a `.cargo/config.toml` or in `RUSTFLAGS`, is read by the +linker, and cargo compares the flag, never the file. When that file sits +outside the package that links with it and changes alone, a warm `host` run +(`src/cicache.rs`) keeps the old link where a cold run makes a new one. + +No flag the host job passes names a file: the tracked `.cargo/config.toml` +files pass none, and its steps set no `RUSTFLAGS`. + +Owner: the host cache (`src/cicache.rs`). + +Done when a warm read refuses a flag that names a tracked file, or dates whole +the package that links with it, with a test. diff --git a/issues/build/the-guest-cache-is-read-by-mtime-and-its-writer-restores-before-it-saves.md b/issues/build/the-guest-cache-is-read-by-mtime-and-its-writer-restores-before-it-saves.md new file mode 100644 index 00000000000..84f6a10f7ab --- /dev/null +++ b/issues/build/the-guest-cache-is-read-by-mtime-and-its-writer-restores-before-it-saves.md @@ -0,0 +1,24 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# The guest cache is read by mtime, and its writer restores before it saves + +`nightly.yml`'s `tcg` restores the newest `guest-` entry, builds on it and saves +the result: nothing prunes what no step rebuilt, so every write keeps the last +one's artifacts and adds its own (3,281,375,938 B on 2026-10-01, beside the +host entry in the repository's 10 GB). And every guest job restores its targets +under a checkout that dated every source at the checkout, so cargo calls every +path crate in them stale. + +The guest entry's ceiling is what H + 2G ≤ 10 GB leaves it, and that sum binds +every night: 4,002,930,524 B at the host's `LIMIT` (`src/cicache.rs`), or +4,298,336,717 B at run 36878222090's H. Above it, `tcg`'s save evicts that +night's host entry unless a pull request has read the entry since the guest +restore; every pull request then runs cold, and nothing reds. + +Owner: the orchestrator. + +Done when the guest entry is written cold, read by content, and bounded. diff --git a/issues/build/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md b/issues/build/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md new file mode 100644 index 00000000000..5e314564c12 --- /dev/null +++ b/issues/build/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md @@ -0,0 +1,41 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# The host cache's limit reaches the 10 GB only through one measured ratio + +`src/cicache.rs` refuses a tree whose `PATHS` hold more than `LIMIT`, +8,000,000,000 B, uncompressed. The repository evicts its caches past 10 GB of +what actions/cache stores, compressed, and both two host entries beside a guest +one (2H + G) and one beside two (H + 2G) must fit. + +One ratio ties `LIMIT` to H, measured once. Run 36878222090 (e0ced587c) sealed +5369 MiB of targets, at least 5,629,804,544 B. That head's own archive of the +cache's paths, made with the runner's `tar` and `zstdmt`, held 1,403,326,566 B: +a ratio of 4.01. At that ratio `LIMIT` stores at most 1,994,138,951 B. With the +guest entry's 3,281,375,938 B (run 36696295750's `tcg`), 2H + G is +7,269,653,840 B and H + 2G is 8,556,890,827 B. The ratio may fall to 2.38 +before 2H + G reaches 10 GB. The lowest measured is 3.08: run 36844536500 +sealed 9553 MiB on macOS and saved 3,250,736,567 B. + +No gate reads a stored size. If the targets compress worse, H moves toward the +floor and nothing reds. + +`LIMIT` is checked only by nightly's `host`, the one job that saves an entry, +before it seals its tree. A pull request's run and the merge queue's, warm or +cold, never seal and are never refused by `LIMIT`. So a landing that takes the +cold tree past `LIMIT` is first refused by the next nightly's seal, loudly: +that run is red and saves nothing. Until an entry is sealed again, a pull +request restores the last sealed one, or runs cold once the runner image has +moved; either way its verdict is its steps'. + +Owner: the host cache (`src/cicache.rs`). + +**Exit condition.** Two gates that red: +- after nightly's `host` saves, a step reads that entry's stored bytes and the + newest guest entry's from the repository's cache list, and fails when + 2H + G or H + 2G passes 10 GB; +- the merge queue's `host` reds a landing whose cold tree passes `LIMIT`, + before `main` moves. diff --git a/issues/build/the-workflow-gates-that-read-workflows-as-text-become-reviewer-instructions-or-go.md b/issues/build/the-workflow-gates-that-read-workflows-as-text-become-reviewer-instructions-or-go.md new file mode 100644 index 00000000000..0090976e132 --- /dev/null +++ b/issues/build/the-workflow-gates-that-read-workflows-as-text-become-reviewer-instructions-or-go.md @@ -0,0 +1,28 @@ +--- +status: open +kind: tooling +opened: 2026-10-01 +--- + +# The workflow gates that read workflows as text become reviewer instructions or go + +Three tests read `.github/workflows/` as text, and each passes a patch that +breaks its rule as GitHub reads the YAML. Each patch, alone, left all three +green (EXIT 0): +- `src/ci.rs`'s `workflows_run_against_main_on_hosted_runners` reads `runs-on:` + and `pull_request:` off single lines. On `publish.yml`: `runs-on:` with its + label on the next line, `- self-hosted`; and `pull_request: {branches: [dev]}` + beside `workflow_dispatch`. +- `src/ci.rs`'s `the_required_check_is_a_job_on_every_pull_request` finds + ci.yml's triggers and its `host` by substring. On `ci.yml`: `host`'s `if:` + made `false`. A skipped job reports success, so the required check is green. +- `src/hostws.rs`'s + `nothing_that_runs_names_a_target_directory_a_member_does_not_have` finds a + `/target` by substring. On `publish.yml`: a step + `run: "ls userland/sshd/targe\x74"`, which YAML reads as + `ls userland/sshd/target`. The same step spelled plainly reds it (EXIT 101). + +Owner: `issues/build/the-tooling-is-a-review-prompt-and-three-workflows.md`. + +Done when each test is deleted with its rule a sentence in +`.claude/agents/reviewer.md`, or reds on its patch above. diff --git a/src/ci.rs b/src/ci.rs index 18b8ac83522..1dec513c109 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -28,6 +28,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use crate::arch::{Accel, Arch}; +use crate::cicache; use crate::sysroot::git_out; use crate::userlandhost::{Host, Os, Program}; use crate::{flags, release, sdkversion}; @@ -36,6 +37,8 @@ const USAGE: &str = "cargo run -- --ci , where is one of: host every host test: the build system, the harness's own checks, the host workspace, the licences of what ships, clippy, the model controls, userland and the SDK (ci.yml, nightly) + seal `host` from a cold tree, then that tree sealed as the host + cache's entry, which the step after it saves (nightly) toolchain publish this tree's toolchain if nobody has (nightly) guest the guest suite (nightly) publish put main's SDK crates on crates.io (publish.yml)"; @@ -43,6 +46,7 @@ const USAGE: &str = "cargo run -- --ci , where is one of: #[derive(Debug, PartialEq, Eq)] enum Job { Host, + Seal, Toolchain, Guest, Publish, @@ -51,6 +55,7 @@ enum Job { fn parse(words: &[String]) -> Result { let job = match words.first().map(String::as_str) { Some("host") => Job::Host, + Some("seal") => Job::Seal, Some("toolchain") => Job::Toolchain, Some("guest") => Job::Guest, Some("publish") => Job::Publish, @@ -70,6 +75,7 @@ pub fn dispatch(root: &Path, args: &[String]) { }); let steps = match &job { Job::Host => host(root), + Job::Seal => seal(root), Job::Toolchain => vec![step("the toolchain release", || release::ensure_published(root))], Job::Guest => guest(root, &suite_args(&["--jobs", "1"])), Job::Publish => vec![step("the SDK crates on crates.io", || publish(root))], @@ -466,7 +472,11 @@ fn run_control(root: &Path, control: &Control) -> Result { /// ([`crate::clippy::BARE_TARGETS`]), which any rustup installs, and userland carries no /// clippy shape (`src/clippy.rs`). Userland and the SDK are tested against the /// host triple for the same reason. +/// +/// In a job that carries the cache ([`cicache::carried`]) the restored entry is +/// read before any step. A developer's tree keeps the dates its edits gave it. fn host(root: &Path) -> Vec { + let carried = cicache::carried(root, &std::env::current_exe().expect("the driver's own path")); let tmp = toyos_tmpdir::TempDir::new("ci-host"); let short = Path::new(toyos_tmpdir::SHORT_BASE); let before = toyos_tmpdir::gone_roots(short); @@ -474,14 +484,23 @@ fn host(root: &Path) -> Vec { // concurrently with the write, and every child inherits it. std::env::set_var("TMPDIR", tmp.path()); let host_triple = crate::toolchain::host_triple(); - let mut steps = vec![ + let mut steps = Vec::new(); + if carried { + carry(); + steps.push(step("the cache entry, read by content", || cicache::read(root))); + // Every step after an unreadable entry would be judged against it. + if steps[0].verdict.is_err() { + return steps; + } + } + steps.extend([ step("the build system", || cargo(root, &["test", "--lib"])), step("the harness's own checks", || cargo(root, &["test", "--test", "toyos-checks"])), step("the host workspace", || { cargo(root, &["test", "--workspace", "--exclude", "toyos-build"]) }), step("the licences of what ships", || crate::licence::judge(root)), - ]; + ]); steps.push(step("clippy and the bare targets", || { for args in [ vec!["component", "add", "clippy"], @@ -549,6 +568,35 @@ fn host(root: &Path) -> Vec { steps } +/// [`host`] from a cold tree, then that tree sealed as the host cache's entry. +fn seal(root: &Path) -> Vec { + let mut cold = None; + let mut steps = vec![step("a cold tree, for the entry", || { + let (found, said) = cicache::cold(root)?; + cold = Some(found); + Ok(said) + })]; + let Some(cold) = cold else { return steps }; + steps.extend(host(root)); + steps.push(step("the tree, sealed as the host cache's entry", || cicache::seal(root, &cold))); + steps +} + +/// What every step of a job that carries the cache inherits, set before any +/// thread as `host`'s `TMPDIR` is. +fn carry() { + // The job's `CARGO_TARGET_DIR` names the driver's target, and no step + // builds there. + std::env::remove_var("CARGO_TARGET_DIR"); + // No incremental state: it is most of an entry's bytes, and after a read + // by content it helps only a crate whose bytes changed. + std::env::set_var("CARGO_INCREMENTAL", "0"); + // Line tables alone: a backtrace in a step's log reads them, and nothing + // reads the rest of the debuginfo, of which a Linux link copies every + // dependency's into each test binary. + std::env::set_var("CARGO_PROFILE_DEV_DEBUG", "line-tables-only"); +} + /// Every app the images ship, judged for `os` with the features its image /// builds it with ([`crate::userlandhost`]). /// @@ -925,6 +973,50 @@ mod tests { assert!(refusal.contains(&died) && !refusal.contains(&earlier), "{refusal}"); } + /// The crate [`a_carried_jobs_step`] builds; unset, it is not a test. + const FIXTURE: &str = "TOYOS_CI_TEST_FIXTURE"; + + /// What a job that carries the cache hands its driver reaches no step: each + /// builds in its own workspace's target, with no incremental state and line + /// tables alone. The driver is a process of its own, because `carry` + /// writes the environment, which no other thread may read meanwhile. + #[test] + fn a_step_of_a_job_that_carries_the_cache_builds_in_its_own_target() { + let fixture = toyos_tmpdir::TempDir::new("ci-carried"); + std::fs::create_dir(fixture.join("src")).unwrap(); + let manifest = "[package]\nname = \"one\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[workspace]\n"; + std::fs::write(fixture.join("Cargo.toml"), manifest).unwrap(); + std::fs::write(fixture.join("src/lib.rs"), "pub fn one() -> u8 {\n 1\n}\n").unwrap(); + let out = crate::buildlock::tests::rerun("ci::tests::a_carried_jobs_step") + .env(FIXTURE, fixture.path()) + .env("CARGO_TARGET_DIR", cicache::DRIVER) + .env_remove("CARGO_INCREMENTAL") + .env_remove("CARGO_PROFILE_DEV_DEBUG") + .output() + .expect("run the driver"); + let said = String::from_utf8_lossy(&out.stdout).into_owned() + &String::from_utf8_lossy(&out.stderr); + assert!(out.status.success() && said.contains("test result: ok. 1 passed"), "{said}"); + } + + #[test] + #[ignore = "the driver of the test above; never runs on its own"] + fn a_carried_jobs_step() { + let fixture = PathBuf::from(std::env::var_os(FIXTURE).unwrap_or_else(|| panic!("run without {FIXTURE}"))); + carry(); + let cargo = |args: &[&str]| { + let out = Command::new("cargo").args(args).current_dir(&fixture).output().expect("run cargo"); + assert!(out.status.success(), "cargo {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + out + }; + let metadata = cargo(&["metadata", "--format-version", "1", "--no-deps", "--offline"]); + let metadata: serde_json::Value = serde_json::from_slice(&metadata.stdout).unwrap(); + let ours = std::fs::canonicalize(&fixture).unwrap().join("target"); + assert_eq!(metadata["target_directory"].as_str().map(Path::new), Some(ours.as_path())); + let build = String::from_utf8(cargo(&["build", "-v", "--offline"]).stderr).unwrap(); + let rustc = build.lines().find(|l| l.contains("--crate-name one")).unwrap_or_else(|| panic!("{build}")); + assert!(!rustc.contains("-C incremental") && rustc.contains("-C debuginfo=line-tables-only"), "{rustc}"); + } + fn repo_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) } @@ -961,6 +1053,7 @@ mod tests { #[test] fn a_job_is_named_and_takes_nothing_after_it() { assert_eq!(parse(&words("host")), Ok(Job::Host)); + assert_eq!(parse(&words("seal")), Ok(Job::Seal)); assert_eq!(parse(&words("guest")), Ok(Job::Guest)); assert!(parse(&words("guest 3/12")).is_err()); assert!(parse(&words("tcg")).is_err()); @@ -1071,35 +1164,6 @@ mod tests { assert_eq!(seen, 3, "ci.yml, nightly.yml and publish.yml"); } - /// Exactly one job writes each cache, on the nightly, so what a pull request - /// restores is one run's tree and never a race between two writers. - #[test] - fn each_cache_has_one_writer() { - let dir = repo_root().join(".github/workflows"); - let mut writers = Vec::new(); - for entry in std::fs::read_dir(&dir).expect(".github/workflows is readable").flatten() { - let text = std::fs::read_to_string(entry.path()).expect("a readable workflow"); - let name = entry.file_name().to_string_lossy().into_owned(); - assert!(!text.contains("actions/cache@"), "{name}: the combined action saves too"); - let lines: Vec<&str> = text.lines().collect(); - for (at, line) in lines.iter().enumerate() { - if line.contains("actions/cache/save@") { - let key = lines[at..] - .iter() - .find_map(|l| l.trim_start().strip_prefix("key: ")) - .expect("a save names its key"); - writers.push((name.clone(), key.split('$').next().unwrap_or("").to_string())); - } - } - } - assert!(!writers.is_empty(), "no job writes a cache, so every restore is cold"); - writers.sort(); - let mut prefixes: Vec<&String> = writers.iter().map(|(_, p)| p).collect(); - prefixes.dedup(); - assert_eq!(prefixes.len(), writers.len(), "a cache with two writers: {writers:?}"); - assert!(writers.iter().all(|(f, _)| f == "nightly.yml"), "{writers:?}"); - } - #[test] fn the_declared_version_is_a_version() { let declared = diff --git a/src/cicache.rs b/src/cicache.rs new file mode 100644 index 00000000000..85192dd1024 --- /dev/null +++ b/src/cicache.rs @@ -0,0 +1,761 @@ +//! The host job's cache entry is trusted by content, never by mtime. +//! +//! Cargo calls a path crate fresh when none of its sources is newer than the +//! build that read it, and a checkout dates every source at the checkout: a +//! restored target is stale in every path crate, and a date made up for a +//! source from anything but its bytes could as well call a changed one fresh. +//! An entry instead carries [`MANIFEST`], the runner it was built on and the +//! SHA-256 of every tracked file, and every file under its targets is dated +//! [`built`], before any real time. [`read`] dates each tracked file whose hash +//! matches the same, and every other one now: cargo calls a source stale only +//! when it is strictly newer than the build, so a match is fresh, and a changed +//! or new source is newer than everything in the entry. +//! +//! **A package with a file changed, added or removed has every file dated +//! now.** Cargo is told only what a build read; a file rustc probed for and did +//! not find (`src/x/mod.rs` beside `src/x.rs`) is still its package's. **So +//! has every package with a build script or a proc macro, on every read**: +//! what code run at build time reads, cargo knows only if that code says so. +//! +//! **An entry built on another runner image is deleted, and the run is cold**: +//! the image's linker and C compiler made its units, and cargo's fingerprint +//! names neither. The cache key cannot carry the image, because no workflow +//! expression sees `ImageOS` or `ImageVersion`. +//! +//! **Only the writer seals**: the job that saves the entry restores nothing, +//! starts [`cold`] and [`seal`]s its tree after the last step. A reader's run +//! is never saved and never sealed. [`read`] deletes the manifest it reads, +//! since a warm tree holds units none of its steps rebuilt, and targets +//! restored without one are refused. +//! +//! **A job carries the cache when its workflow builds the driver in +//! [`DRIVER`]** ([`carried`]): cargo builds the driver before this runs, so its +//! path crates are compiled again every time, and in the steps' target that +//! would make every crate depending on them stale. A job that builds it +//! anywhere else runs as a developer's tree does. +//! +//! **A seal refuses a tree whose [`PATHS`] hold more than [`LIMIT`]**, `~` +//! being `HOME`, so the save, which follows only a green run, never stores +//! one. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use sha2::{Digest, Sha256}; + +const MANIFEST: &str = "target/ci-sources"; +pub const DRIVER: &str = "target/ci-driver"; + +/// What every step that names the host cache archives: the cache's version is +/// computed from the list, so a restore whose list differs finds nothing. +const PATHS: [&str; 8] = [ + "~/.cargo/registry/index", + "~/.cargo/registry/cache", + "~/.cargo/git/db", + "target", + "userland/target", + "toyos/target", + "kernel/target", + "bootloader/target", +]; + +/// The most the files under [`PATHS`] may hold, uncompressed. The repository's +/// caches are evicted by last access past 10 GB, and a night whose guest jobs +/// restore after this entry is saved holds two host entries beside a guest +/// one, then one beside two guest ones. +const LIMIT: u64 = 8_000_000_000; + +/// 2001-09-09T01:46:40Z: older than any build, so a file dated so is never +/// newer than one. +fn built() -> SystemTime { + UNIX_EPOCH + Duration::from_secs(1_000_000_000) +} + +/// Every file git tracks, with the SHA-256 of its bytes on disk. +type Sources = BTreeMap; + +/// What [`seal`] holds the writer's tree to at the end. +pub struct Cold { + runner: String, + sources: Sources, +} + +/// Whether the driver at `exe` was built in [`DRIVER`], which is how a +/// workflow says its job carries the cache. +pub fn carried(root: &Path, exe: &Path) -> bool { + let exe = fs::canonicalize(exe).unwrap_or_else(|e| panic!("{}: {e}", exe.display())); + match fs::canonicalize(root.join(DRIVER)) { + Ok(driver) => exe.starts_with(driver), + Err(e) if e.kind() == ErrorKind::NotFound => false, + Err(e) => panic!("{DRIVER}: {e}"), + } +} + +/// Before the first step of a job that reads the cache: the entry restored +/// here, read by content. +pub fn read(root: &Path) -> Result { + open(root, &runner(|name| std::env::var(name).ok())?, SystemTime::now()) +} + +/// Before the first step of the writer's run: a tree that holds nothing but +/// its checkout and the driver, with every source dated [`built`], so that the +/// seal sees a step that writes one. +pub fn cold(root: &Path) -> Result<(Cold, String), String> { + if !carried(root, &std::env::current_exe().map_err(|e| format!("the driver's own path: {e}"))?) { + return Err(format!("only a job whose driver is built in {DRIVER} seals its tree")); + } + start(root, &runner(|name| std::env::var(name).ok())?) +} + +/// The runner, as the variables a hosted runner sets name it: its OS, its +/// architecture and its image. +fn runner(var: impl Fn(&str) -> Option) -> Result { + let values = ["RUNNER_OS", "RUNNER_ARCH", "ImageOS", "ImageVersion"] + .iter() + .map(|name| var(name).ok_or_else(|| format!("{name} is unset: a hosted runner sets it"))); + Ok(values.collect::, _>>()?.join(" ")) +} + +fn open(root: &Path, runner: &str, now: SystemTime) -> Result { + let manifest = root.join(MANIFEST); + let text = match fs::read_to_string(&manifest) { + Ok(text) => text, + Err(e) if e.kind() == ErrorKind::NotFound => { + let restored = restored(root)?; + if !restored.is_empty() { + return Err(format!( + "{} restored without {MANIFEST}, so nothing says what they were built from", + listed(&restored) + )); + } + return Ok("none restored: the run is cold".into()); + } + Err(e) => return Err(format!("read {MANIFEST}: {e}")), + }; + fs::remove_file(&manifest).map_err(|e| format!("remove {MANIFEST}: {e}"))?; + let (commit, built_on, entry) = parse(&text)?; + if built_on != runner { + for path in restored(root)? { + let gone = if path.is_dir() { fs::remove_dir_all(&path) } else { fs::remove_file(&path) }; + gone.map_err(|e| format!("remove {}: {e}", path.display()))?; + } + return Ok(format!("{commit}'s entry, built on {built_on}, deleted: the run is cold")); + } + if now <= built() { + return Err(format!("this runner's clock reads {now:?}, which no change dated now is newer than")); + } + let current = sources(root)?; + let mut dirty: BTreeSet> = current + .iter() + .filter(|(path, hash)| entry.get(*path) != Some(*hash)) + .map(|(path, _)| path) + .chain(entry.keys().filter(|path| !current.contains_key(*path))) + .map(|path| package(path, ¤t)) + .collect(); + let packages = dirty.len(); + let mut build_time = 0; + for manifest in current.keys().filter(|path| Path::new(path).ends_with("Cargo.toml")) { + if runs_at_build(root, manifest, ¤t)? { + build_time += 1; + dirty.insert(Some(manifest.clone())); + } + } + let mut same = 0; + for (path, hash) in ¤t { + let fresh = entry.get(path) == Some(hash) && !dirty.contains(&package(path, ¤t)); + same += usize::from(fresh); + date(&root.join(path), if fresh { built() } else { now })?; + } + let changed = current.iter().filter(|(p, h)| entry.get(*p).is_some_and(|e| e != *h)).count(); + let added = current.keys().filter(|p| !entry.contains_key(*p)).count(); + let removed = entry.keys().filter(|p| !current.contains_key(*p)).count(); + Ok(format!( + "built from {commit}: {same} of {} sources dated as built; {changed} changed, {added} added, \ + {removed} removed, in {packages} packages; {build_time} packages run code at build time", + current.len(), + )) +} + +fn start(root: &Path, runner: &str) -> Result<(Cold, String), String> { + let restored = restored(root)?; + if !restored.is_empty() { + return Err(format!("{}: an entry is one cold build, and its writer restores nothing", listed(&restored))); + } + let sources = sources(root)?; + for path in sources.keys() { + date(&root.join(path), built())?; + } + let said = format!("{} sources dated as built, and the tree is sealed last", sources.len()); + Ok((Cold { runner: runner.to_string(), sources }, said)) +} + +fn listed(paths: &[PathBuf]) -> String { + paths.iter().map(|p| p.display().to_string()).collect::>().join(", ") +} + +/// After the last step of the writer's run, which [`cold`] began: the tree +/// refused if its [`PATHS`] hold more than [`LIMIT`], `~` being `HOME`, and +/// otherwise every file under every target dated [`built`], then the manifest. +pub fn seal(root: &Path, cold: &Cold) -> Result { + let home = std::env::var_os("HOME").ok_or("HOME is unset, and the cache's paths name it as `~`")?; + seal_at(root, Path::new(&home), cold) +} + +fn seal_at(root: &Path, home: &Path, cold: &Cold) -> Result { + let (mut files, mut bytes) = (0u64, 0u64); + for path in PATHS { + let dir = match path.strip_prefix("~/") { + Some(under) => home.join(under), + None => root.join(path), + }; + // A path the save finds nothing at stores nothing. + if !dir.try_exists().map_err(|e| format!("{}: {e}", dir.display()))? { + continue; + } + each_under(&dir, &mut |_, meta| { + if meta.is_file() { + files += 1; + bytes += meta.len(); + } + Ok(()) + })?; + } + if bytes > LIMIT { + return Err(format!( + "{bytes} B in {files} files under the cache's paths, above the {LIMIT} B an entry may \ + hold: saved, it could evict the guest entry or the next host one" + )); + } + let now = sources(root)?; + let moved: Vec<&String> = cold + .sources + .iter() + .filter(|(path, hash)| { + now.get(*path) != Some(*hash) || modified(&root.join(path)).ok() != Some(built()) + }) + .map(|(path, _)| path) + .chain(now.keys().filter(|path| !cold.sources.contains_key(*path))) + .collect(); + if !moved.is_empty() { + return Err(format!("a step wrote tracked sources, which no entry can describe: {moved:?}")); + } + for target in targets(root)? { + each_under(&target, &mut |path, _| date(path, built()))?; + date(&target, built())?; + } + let head = crate::sysroot::git_out(root, &["rev-parse", "HEAD"]); + let mut text = format!("{}\n{}\n", head.trim(), cold.runner); + for (path, hash) in &cold.sources { + text.push_str(&format!("{hash} {path}\n")); + } + fs::write(root.join(MANIFEST), text).map_err(|e| format!("write {MANIFEST}: {e}"))?; + Ok(format!( + "{files} files, {bytes} B of the {LIMIT} B an entry may hold; sealed: {} sources, built on {}, \ + every target dated as built", + cold.sources.len(), + cold.runner + )) +} + +/// The commit an entry was built from, the runner, and its sources. +fn parse(text: &str) -> Result<(String, String, Sources), String> { + let mut lines = text.lines(); + let (Some(commit), Some(runner)) = (lines.next(), lines.next()) else { + return Err(format!("{MANIFEST} ends before its commit and runner")); + }; + let entry = lines + .map(|line| { + line.split_once(' ') + .filter(|(hash, _)| !hash.is_empty() && hash.bytes().all(|b| b.is_ascii_hexdigit())) + .map(|(hash, path)| (path.to_string(), hash.to_string())) + .ok_or_else(|| format!("{MANIFEST} holds {line:?}")) + }) + .collect::>()?; + Ok((commit.to_string(), runner.to_string(), entry)) +} + +/// Whether the package of `manifest` has a build script or is a proc macro, in +/// every spelling cargo accepts. +fn runs_at_build(root: &Path, manifest: &str, current: &Sources) -> Result { + let text = fs::read_to_string(root.join(manifest)).map_err(|e| format!("read {manifest}: {e}"))?; + let doc: toml::Value = text.parse().map_err(|e| format!("{manifest}: {e}"))?; + let lib = |key: &str, alias: &str| doc.get("lib").and_then(|lib| lib.get(key).or_else(|| lib.get(alias))); + let package = doc.get("package").or_else(|| doc.get("project")); + let script = match package.and_then(|package| package.get("build")) { + None => current.contains_key(&*Path::new(manifest).with_file_name("build.rs").to_string_lossy()), + Some(build) => build.as_bool() != Some(false), + }; + let proc_macro = lib("proc-macro", "proc_macro").and_then(toml::Value::as_bool) == Some(true) + || lib("crate-type", "crate_type") + .and_then(toml::Value::as_array) + .is_some_and(|types| types.iter().any(|t| t.as_str() == Some("proc-macro"))); + Ok(script || proc_macro) +} + +/// The `Cargo.toml` of the package `path` is in: the nearest one above it. +fn package(path: &str, current: &Sources) -> Option { + Path::new(path) + .ancestors() + .skip(1) + .map(|dir| dir.join("Cargo.toml").to_string_lossy().into_owned()) + .find(|manifest| current.contains_key(manifest)) +} + +fn sources(root: &Path) -> Result { + let mut sources = Sources::new(); + for path in crate::sysroot::tracked_files(root, &[])? { + let file = root.join(&path); + // A gitlink names a commit, not bytes: what is under it keeps its + // checkout's date, which no entry is newer than. + if file.is_dir() { + continue; + } + let bytes = fs::read(&file).map_err(|e| format!("read {path}: {e}"))?; + sources.insert(path, format!("{:x}", Sha256::digest(bytes))); + } + Ok(sources) +} + +/// Every `target` directory under `root`, none inside another. +fn targets(root: &Path) -> Result, String> { + fn walk(dir: &Path, found: &mut Vec) -> Result<(), String> { + for entry in fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))? { + let entry = entry.map_err(|e| format!("read {}: {e}", dir.display()))?; + let kind = entry.file_type().map_err(|e| format!("{}: {e}", entry.path().display()))?; + if !kind.is_dir() || entry.file_name() == ".git" { + continue; + } + if entry.file_name() == "target" { + found.push(entry.path()); + } else { + walk(&entry.path(), found)?; + } + } + Ok(()) + } + let mut found = Vec::new(); + walk(root, &mut found)?; + Ok(found) +} + +/// What a restore put here: every target but the root's, and in the root's +/// everything but [`DRIVER`], which this job's own `cargo run` made. +fn restored(root: &Path) -> Result, String> { + let ours = root.join("target"); + let mut found = Vec::new(); + for target in targets(root)? { + if target != ours { + found.push(target); + continue; + } + for entry in fs::read_dir(&target).map_err(|e| format!("read {}: {e}", target.display()))? { + let path = entry.map_err(|e| format!("read {}: {e}", target.display()))?.path(); + if path != root.join(DRIVER) { + found.push(path); + } + } + } + Ok(found) +} + +/// Hands `visit` every file and directory under `dir`, with its metadata. A +/// symbolic link is neither followed nor visited: dating one dates whatever it +/// names. +fn each_under(dir: &Path, visit: &mut impl FnMut(&Path, &fs::Metadata) -> Result<(), String>) -> Result<(), String> { + for entry in fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))? { + let entry = entry.map_err(|e| format!("read {}: {e}", dir.display()))?; + let meta = entry.metadata().map_err(|e| format!("{}: {e}", entry.path().display()))?; + if meta.is_dir() { + each_under(&entry.path(), visit)?; + } + if meta.is_dir() || meta.is_file() { + visit(&entry.path(), &meta)?; + } + } + Ok(()) +} + +fn date(path: &Path, when: SystemTime) -> Result<(), String> { + fs::File::open(path) + .and_then(|f| f.set_modified(when)) + .map_err(|e| format!("date {}: {e}", path.display())) +} + +fn modified(path: &Path) -> std::io::Result { + fs::metadata(path)?.modified() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::gitfixture::{configure, sh}; + use std::process::{Command, Output}; + use toyos_tmpdir::TempDir; + + const RUNNER: &str = "macOS ARM64 macos15 20260928.1"; + + fn write(root: &Path, path: &str, text: &str) { + let path = root.join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + + /// `files` committed to a new repository at `dir`. + fn origin(dir: &Path, files: &[(&str, &str)]) { + fs::create_dir_all(dir).unwrap(); + sh(dir, &["init", "-q", "-b", "main"]); + configure(dir); + for (path, text) in files { + write(dir, path, text); + } + sh(dir, &["add", "-A"]); + sh(dir, &["commit", "-qm", "files"]); + } + + fn crate_toml(name: &str, deps: &str) -> String { + format!("[package]\nname = \"{name}\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\n{deps}") + } + + fn cargo_build(root: &Path) -> Output { + Command::new("cargo") + .args(["build", "--offline", "--message-format=json"]) + .current_dir(root) + .env_remove("CARGO_TARGET_DIR") + .output() + .expect("run cargo") + } + + /// `cargo build` in `root`, and each workspace crate with whether cargo + /// called it fresh. + fn build(root: &Path) -> BTreeMap { + let out = cargo_build(root); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let mut fresh = BTreeMap::new(); + for line in String::from_utf8_lossy(&out.stdout).lines() { + let doc: serde_json::Value = serde_json::from_str(line).unwrap(); + if doc["reason"] == "compiler-artifact" && doc["target"]["kind"][0] != "custom-build" { + let name = doc["target"]["name"].as_str().unwrap().to_string(); + fresh.insert(name, doc["fresh"].as_bool().unwrap()); + } + } + fresh + } + + fn run(root: &Path) -> String { + let out = Command::new(root.join("target/debug/app")).output().unwrap(); + String::from_utf8(out.stdout).unwrap() + } + + /// A checkout of `origin` at its head, as a runner makes one. + fn checkout(origin: &Path, dir: &Path) { + let (from, to) = (origin.to_str().unwrap(), dir.to_str().unwrap()); + sh(origin.parent().unwrap(), &["clone", "-q", from, to]); + configure(dir); + } + + /// An entry at `writer`: a checkout of `origin` built cold and sealed. + fn entry(origin: &Path, writer: &Path) { + checkout(origin, writer); + let (cold, _) = start(writer, RUNNER).unwrap(); + build(writer); + seal_at(writer, &writer.join("home"), &cold).unwrap(); + } + + /// The entry at `writer`, restored under a fresh checkout of `origin`. + fn restore(origin: &Path, writer: &Path, reader: &Path) { + checkout(origin, reader); + fs::rename(writer.join("target"), reader.join("target")).unwrap(); + } + + /// The entry restored at `reader`, read: a test of what a read serves holds + /// nothing if the read went cold. + fn read_warm(reader: &Path) { + let said = open(reader, RUNNER, SystemTime::now()).unwrap(); + assert!(said.starts_with("built from"), "{said}"); + } + + /// The oracle is cargo and the program it builds: an entry serves what + /// a cold build of the reader's tree would, recompiling exactly what + /// changed and what depends on it — a changed source the checkout dated + /// older than the entry's build included, a build script whose package + /// lost a file it never named, and a package that lost a file nothing read. + #[test] + fn an_entry_serves_exactly_the_sources_it_was_built_from() { + let tmp = TempDir::new("cicache-entry"); + let script = r#"fn main() { + let files = std::fs::read_dir("src").unwrap().count(); + println!("cargo:rustc-env=FILES={files}"); +} +"#; + let deps = "leaf = { path = \"../leaf\" }\ncount = { path = \"../count\" }\n"; + let source = tmp.join("origin"); + origin(&source, &[ + (".gitignore", "target/\n"), + ("Cargo.toml", "[workspace]\nmembers = [\"app\", \"leaf\", \"count\", \"idle\", \"gone\"]\nresolver = \"2\"\n"), + ("leaf/Cargo.toml", &crate_toml("leaf", "")), + ("leaf/src/lib.rs", "pub fn word() -> &'static str { \"one\" }\n"), + ("count/Cargo.toml", &crate_toml("count", "")), + ("count/build.rs", script), + ("count/src/lib.rs", "pub const FILES: &str = env!(\"FILES\");\n"), + ("count/src/spare.txt", "read by nothing but the count\n"), + ("idle/Cargo.toml", &crate_toml("idle", "")), + ("idle/src/lib.rs", "pub fn idle() {}\n"), + ("gone/Cargo.toml", &crate_toml("gone", "")), + ("gone/src/lib.rs", "pub fn gone() {}\n"), + ("gone/notes.txt", "read by nothing\n"), + ("app/Cargo.toml", &crate_toml("app", deps)), + ("app/src/main.rs", "fn main() { print!(\"{} {}\", leaf::word(), count::FILES); }\n"), + ]); + let writer = tmp.join("writer"); + entry(&source, &writer); + assert_eq!(run(&writer), "one 2"); + + write(&source, "leaf/src/lib.rs", "pub fn word() -> &'static str { \"two\" }\n"); + sh(&source, &["rm", "-q", "count/src/spare.txt", "gone/notes.txt"]); + sh(&source, &["commit", "-qam", "read"]); + let reader = tmp.join("reader"); + restore(&source, &writer, &reader); + // What an mtime made up from history would say of a file committed + // before the entry was built. + date(&reader.join("leaf/src/lib.rs"), UNIX_EPOCH + Duration::from_secs(1)).unwrap(); + + read_warm(&reader); + assert!(!reader.join(MANIFEST).exists(), "a warm tree keeps no manifest"); + let fresh = build(&reader); + assert_eq!(run(&reader), "two 1"); + let expected = [("app", false), ("count", false), ("gone", false), ("idle", true), ("leaf", false)]; + assert_eq!(fresh, expected.into_iter().map(|(k, v)| (k.to_string(), v)).collect()); + } + + /// A file no dep-info names still decides a build: beside `src/x.rs`, a + /// new `src/x/mod.rs` is E0761 to a cold build, and so to a warm one. + #[test] + fn a_file_cargo_was_never_told_about_rebuilds_its_package() { + let tmp = TempDir::new("cicache-probe"); + let source = tmp.join("origin"); + origin(&source, &[ + (".gitignore", "target/\n"), + ("Cargo.toml", "[workspace]\nmembers = [\"probed\"]\nresolver = \"2\"\n"), + ("probed/Cargo.toml", &crate_toml("probed", "")), + ("probed/src/lib.rs", "mod x;\npub fn f() -> u8 { x::X }\n"), + ("probed/src/x.rs", "pub const X: u8 = 1;\n"), + ]); + let writer = tmp.join("writer"); + entry(&source, &writer); + + write(&source, "probed/src/x/mod.rs", "pub const X: u8 = 2;\n"); + sh(&source, &["add", "-A"]); + sh(&source, &["commit", "-qm", "probed"]); + let reader = tmp.join("reader"); + restore(&source, &writer, &reader); + read_warm(&reader); + let out = cargo_build(&reader); + let said = String::from_utf8_lossy(&out.stdout); + assert!(!out.status.success() && said.contains("E0761"), "{said}"); + } + + /// A build script and a proc macro that read another package's file and + /// never say so are run again on a read, as a cold build runs them: `app`'s + /// script reads `word.txt`, and so does `mac`, expanded in `said`. + #[test] + fn code_run_at_build_time_runs_again_on_every_read() { + let tmp = TempDir::new("cicache-buildtime"); + let script = "fn main() { + let word = std::fs::read_to_string(\"../word.txt\").unwrap(); + println!(\"cargo:rustc-env=WORD={}\", word.trim()); +} +"; + let mac = r#"#[proc_macro] +pub fn word(_: proc_macro::TokenStream) -> proc_macro::TokenStream { + let dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let word = std::fs::read_to_string(format!("{dir}/../word.txt")).unwrap(); + format!("{:?}", word.trim()).parse().unwrap() +} +"#; + let source = tmp.join("origin"); + origin(&source, &[ + (".gitignore", "target/\n"), + ("Cargo.toml", "[workspace]\nmembers = [\"app\", \"mac\", \"said\"]\nresolver = \"2\"\n"), + ("word.txt", "one\n"), + ("mac/Cargo.toml", &format!("{}\n[lib]\nproc-macro = true\n", crate_toml("mac", ""))), + ("mac/src/lib.rs", mac), + ("said/Cargo.toml", &crate_toml("said", "mac = { path = \"../mac\" }\n")), + ("said/src/lib.rs", "pub const WORD: &str = mac::word!();\n"), + ("app/Cargo.toml", &crate_toml("app", "said = { path = \"../said\" }\n")), + ("app/build.rs", script), + ("app/src/main.rs", "fn main() { print!(\"{} {}\", env!(\"WORD\"), said::WORD); }\n"), + ]); + let writer = tmp.join("writer"); + entry(&source, &writer); + assert_eq!(run(&writer), "one one"); + + write(&source, "word.txt", "two\n"); + sh(&source, &["commit", "-qam", "word"]); + let reader = tmp.join("reader"); + restore(&source, &writer, &reader); + read_warm(&reader); + build(&reader); + assert_eq!(run(&reader), "two two"); + } + + /// Each way a manifest can declare a build script or a proc macro. + #[test] + fn every_spelling_of_code_run_at_build_time_is_found() { + let tmp = TempDir::new("cicache-spellings"); + let cases = [ + ("[package]", "", false, false), + ("[package]", "", true, true), + ("[package]", "build = false\n", true, false), + ("[package]", "build = \"gen.rs\"\n", false, true), + ("[project]", "build = \"gen.rs\"\n", false, true), + ("[project]", "build = false\n", true, false), + ("[package]", "[lib]\nproc-macro = true\n", false, true), + ("[package]", "[lib]\nproc_macro = true\n", false, true), + ("[package]", "[lib]\ncrate-type = [\"proc-macro\"]\n", false, true), + ("[package]", "[lib]\ncrate_type = [\"proc-macro\"]\n", false, true), + ("[package]", "[lib]\ncrate-type = [\"rlib\"]\n", false, false), + ]; + for (table, keys, script, expected) in cases { + write(&tmp, "Cargo.toml", &format!("{table}\nname = \"p\"\n{keys}")); + let mut current = Sources::from([("Cargo.toml".to_string(), String::new())]); + if script { + current.insert("build.rs".into(), String::new()); + } + assert_eq!(runs_at_build(&tmp, "Cargo.toml", ¤t), Ok(expected), "{table} {keys:?}, build.rs: {script}"); + } + } + + /// What a save would store is the files under [`PATHS`], `~` being the + /// home: one byte above the bound, between a target and the home's + /// registry, is refused and gets no manifest; at the bound the tree is + /// sealed, and a target no save stores is not counted. The bound's bytes + /// are a hole `set_len` made, so no test writes them. + #[test] + fn a_seal_refuses_what_its_save_would_store_above_the_bound() { + let tmp = TempDir::new("cicache-bound"); + let home = tmp.join("home"); + let cold = cold_repo(&tmp, RUNNER); + write(&home, ".cargo/registry/cache/one", "1"); + write(&tmp, "tests/target/unsaved", "1"); + let big = tmp.join("target/debug/big"); + fs::create_dir_all(big.parent().unwrap()).unwrap(); + fs::File::create(&big).unwrap().set_len(LIMIT).unwrap(); + let refusal = seal_at(&tmp, &home, &cold).expect_err("one byte above the bound"); + assert!(refusal.starts_with(&format!("{} B", LIMIT + 1)), "{refusal}"); + assert!(!tmp.join(MANIFEST).exists(), "a refused tree carries a manifest"); + fs::remove_file(home.join(".cargo/registry/cache/one")).unwrap(); + seal_at(&tmp, &home, &cold).expect("at the bound"); + assert!(tmp.join(MANIFEST).exists(), "a tree at the bound is sealed"); + } + + /// One commit of `a.rs`, begun cold on `runner`. + fn cold_repo(tmp: &Path, runner: &str) -> Cold { + sh(tmp, &["init", "-q"]); + configure(tmp); + write(tmp, "a.rs", "a\n"); + sh(tmp, &["add", "-A"]); + sh(tmp, &["commit", "-qm", "a"]); + start(tmp, runner).unwrap().0 + } + + /// Targets with no manifest beside them were built from nothing anyone can + /// name, so a reader refuses them; the writer, which builds from its + /// checkout alone, refuses them and an entry. The driver's own target is + /// this job's. + #[test] + fn targets_restored_without_a_manifest_are_refused() { + let tmp = TempDir::new("cicache-foreign"); + cold_repo(&tmp, RUNNER); + fs::create_dir_all(tmp.join(DRIVER)).unwrap(); + open(&tmp, RUNNER, SystemTime::now()).expect("the driver's own target"); + start(&tmp, RUNNER).expect("the driver's own target"); + for target in ["kernel/target", "target/debug"] { + fs::create_dir_all(tmp.join(target)).unwrap(); + let refusals = [open(&tmp, RUNNER, SystemTime::now()).err(), start(&tmp, RUNNER).err()]; + assert!(refusals.iter().all(|r| r.as_ref().is_some_and(|r| r.contains(target))), "{refusals:?}"); + fs::remove_dir(tmp.join(target)).unwrap(); + } + write(&tmp, MANIFEST, ""); + let refusal = start(&tmp, RUNNER).err().expect("an entry, for the writer"); + assert!(refusal.contains(MANIFEST), "{refusal}"); + } + + /// A sealed tree is dated as built throughout, and a step that wrote a + /// tracked source is refused, even one that wrote its bytes back: the + /// manifest would name bytes no build read. + #[test] + fn a_seal_dates_every_target_and_refuses_a_written_source() { + let tmp = TempDir::new("cicache-seal"); + let cold = cold_repo(&tmp, RUNNER); + write(&tmp, "target/debug/deps/x", "x"); + write(&tmp, "userland/target/y", "y"); + seal_at(&tmp, &tmp.join("home"), &cold).unwrap(); + for file in ["target/debug/deps/x", "target/debug", "userland/target/y"] { + assert_eq!(modified(&tmp.join(file)).unwrap(), built(), "{file}"); + } + fs::remove_file(tmp.join(MANIFEST)).unwrap(); + for bytes in ["b\n", "a\n"] { + write(&tmp, "a.rs", bytes); + let refusal = seal_at(&tmp, &tmp.join("home"), &cold).unwrap_err(); + assert!(refusal.contains("a.rs"), "{bytes:?}: {refusal}"); + } + } + + /// The runner [`read`] names in an environment holding `vars`. + fn runner_in(vars: &BTreeMap<&str, &str>) -> Result { + runner(|name| vars.get(name).map(|value| value.to_string())) + } + + /// Another runner's entry is no entry: its targets go and the run is cold, + /// whichever variable that names a runner differs, and a runner without + /// one of them names none. + #[test] + fn an_entry_built_on_another_runner_is_deleted() { + let image = BTreeMap::from([ + ("RUNNER_OS", "macOS"), + ("RUNNER_ARCH", "ARM64"), + ("ImageOS", "macos15"), + ("ImageVersion", "20260928.1"), + ]); + for name in image.keys() { + let tmp = TempDir::new("cicache-image"); + let cold = cold_repo(&tmp, &runner_in(&image).unwrap()); + write(&tmp, "target/debug/x", "x"); + write(&tmp, "kernel/target/y", "y"); + seal_at(&tmp, &tmp.join("home"), &cold).unwrap(); + let mut other = image.clone(); + other.insert(name, "another"); + let said = open(&tmp, &runner_in(&other).unwrap(), SystemTime::now()).unwrap(); + assert!(!tmp.join("target/debug").exists() && !tmp.join("kernel/target").exists(), "{name}: {said}"); + other.remove(name); + let refusal = runner_in(&other).expect_err("a runner without a variable"); + assert!(refusal.contains(name), "{refusal}"); + } + } + + /// A source dated now is newer than the entry only on a clock that reads + /// after [`built`]. + #[test] + fn a_reader_whose_clock_is_not_after_the_entry_is_refused() { + let tmp = TempDir::new("cicache-clock"); + let cold = cold_repo(&tmp, RUNNER); + write(&tmp, "target/debug/x", "x"); + seal_at(&tmp, &tmp.join("home"), &cold).unwrap(); + let refusal = open(&tmp, RUNNER, built()).expect_err("a clock at the entry's date"); + assert!(refusal.contains("clock"), "{refusal}"); + } + + #[test] + fn only_a_driver_built_in_its_own_target_carries_the_cache() { + let tmp = TempDir::new("cicache-driver"); + let (ours, other) = (format!("{DRIVER}/debug/toyos-build"), "target/debug/toyos-build"); + write(&tmp, other, ""); + assert!(!carried(&tmp, &tmp.join(other)), "a tree with no {DRIVER}"); + write(&tmp, &ours, ""); + assert!(carried(&tmp, &tmp.join(&ours)), "{ours}"); + assert!(!carried(&tmp, &tmp.join(other)), "{other}"); + } +} diff --git a/src/clippy.rs b/src/clippy.rs index 62dcc90ebb1..692afa971ff 100644 --- a/src/clippy.rs +++ b/src/clippy.rs @@ -152,9 +152,12 @@ const SHAPES: &[Shape] = &[ ], after: &["$ADOPTED", "-D", "warnings"], }, + // Its own target directory: cargo keeps one check of a unit, so a unit + // linted under two sets of lints is checked again by each, every run, and + // so is every crate depending on it. Shape { dir: "", - before: &["-p", "toyos-abi", "--all-targets", "--keep-going"], + before: &["-p", "toyos-abi", "--all-targets", "--keep-going", "--target-dir", "target/clippy-abi"], after: &["-W", "clippy::undocumented_unsafe_blocks", "-D", "warnings"], }, ]; diff --git a/src/identity.rs b/src/identity.rs index c366ad718eb..574c17b5ce4 100644 --- a/src/identity.rs +++ b/src/identity.rs @@ -1,9 +1,8 @@ //! What a source file is to a build: its token stream, not its text. //! -//! **One definition, read by every question of the form "did this source -//! change what gets built"**: the key a sysroot is filed under. A comment -//! — a doc comment included — and the whitespace around tokens change no item, -//! no layout and no code, so a change made only of them is no new sysroot. +//! A comment — a doc comment included — and the whitespace around tokens change +//! no item, no layout and no code, so a change made only of them is no new +//! sysroot. //! //! A `.rs` file is lexed just far enough to find its comments: every string, //! raw string and character literal is kept byte for byte, every comment and diff --git a/src/lib.rs b/src/lib.rs index 02ecc5ae450..acf395f0c6d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod bootlog; pub mod build; pub mod buildlock; pub mod ci; +pub mod cicache; pub mod clang; pub mod clippy; pub mod compiler;