diff --git a/.claude/agents/reviewer.md b/.claude/agents/reviewer.md index e2b72bd9091..27d9a20d4dc 100644 --- a/.claude/agents/reviewer.md +++ b/.claude/agents/reviewer.md @@ -89,9 +89,9 @@ if it meets the bar above; otherwise it is a NOTE. - **Caches.** No gate reads these; a diff that breaks one is a BLOCKER. No workflow uses the combined `actions/cache`, which saves too. The host cache has one writer, nightly's `host`, and one reader, ci.yml's `host`, on the same `runs-on`, both caching `src/cicache.rs`'s - `PATHS` with its `DRIVER` as their - `CARGO_TARGET_DIR`, the one variable an `env:` gives either: an `ImageOS` or `ImageVersion` - set there outlives an image move. The reader's `restore-keys` is the writer's `key` up to its + `PATHS` with its `DRIVER` as their `CARGO_TARGET_DIR` and `line-tables-only` as their + `CARGO_PROFILE_DEV_DEBUG`, the two variables an `env:` gives either: an `ImageOS` or + `ImageVersion` set there outlives an image move. The reader's `restore-keys` is the writer's `key` up to its run id. A job that names the host cache runs `actions/checkout`, its cache step and `cargo run -- --ci `, `seal` in the writer and `host` in the reader, and nothing else; the reader restores before that step, and the writer saves after it. The save's guard, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb446819456..6d52eb84d58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,12 @@ jobs: if: github.event_name == 'merge_group' || github.event.pull_request.draft == false runs-on: ubuntu-24.04 timeout-minutes: 45 - # The driver's own target, and no step's: it says this job carries the - # host cache (src/cicache.rs). env: + # The driver's own target, and no step's: it says this job carries the + # host cache (src/cicache.rs). CARGO_TARGET_DIR: target/ci-driver + # The writer's: a unit built with other debuginfo is not the entry's. + CARGO_PROFILE_DEV_DEBUG: line-tables-only steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b26634d85f0..df356b8f958 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -21,10 +21,14 @@ jobs: host: runs-on: ubuntu-24.04 timeout-minutes: 90 - # The driver's own target, and no step's: it says this job carries the - # host cache (src/cicache.rs). env: + # The driver's own target, and no step's: it says this job carries the + # host cache (src/cicache.rs). CARGO_TARGET_DIR: target/ci-driver + # The driver's and every step's: a backtrace in a log reads line tables, + # and nothing reads the rest of the debuginfo, of which a Linux link + # copies every dependency's into each binary. + CARGO_PROFILE_DEV_DEBUG: line-tables-only steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: diff --git a/issues/a-host-reader-on-another-runner-image-downloads-the-entry-and-discards-it.md b/issues/a-host-reader-on-another-runner-image-downloads-the-entry-and-discards-it.md new file mode 100644 index 00000000000..3a636b680d8 --- /dev/null +++ b/issues/a-host-reader-on-another-runner-image-downloads-the-entry-and-discards-it.md @@ -0,0 +1,23 @@ +--- +status: open +kind: tooling +opened: 2026-10-08 +--- + +# A host reader on another runner image downloads the entry and discards it + +`src/cicache.rs` deletes an entry built on another runner image, and the cache +key cannot carry the image. GitHub serves two `ubuntu-24.04` images at once +while it rolls one out: on 2026-10-07 run 37601225884 (09:31Z) and `host` +readers 37646740767 (15:47Z) and 37687777630 (21:12Z) ran on 20261004.327.1, +and runs 37685714260 (20:55Z) and 37693139619 (21:59Z) on 20260927.320.1. So +for the days of a rollout a reader meets an entry of the other image by chance, +whichever image the writer drew that night: both readers above restored +1,813,819,887 B, deleted it and ran cold, 15 and 10 minutes of steps against +the 5 a reader of its own image's entry took in run 37693139619. + +Owner: the host cache (`src/cicache.rs`). + +**Exit condition.** A reader restores an entry only of its own image, or the +share of `host` runs that discard one is measured over a rollout and accepted +here. diff --git a/issues/a-red-nightly-on-main-is-reported-to-nobody.md b/issues/a-red-nightly-on-main-is-reported-to-nobody.md new file mode 100644 index 00000000000..9915d758c09 --- /dev/null +++ b/issues/a-red-nightly-on-main-is-reported-to-nobody.md @@ -0,0 +1,23 @@ +--- +status: open +kind: tooling +opened: 2026-10-08 +--- + +# A red nightly on main is reported to nobody + +`nightly.yml`'s scheduled run is a check of no pull request and no merge group: +its verdict blocks nothing, and no role's prompt reads it. Its `host` job, the +host cache's one writer, was red at its seal on 2026-10-05, 2026-10-06 and +2026-10-07 (runs 37292450697, 37444133685, 37601225884), after #722 brought +`gix` into the build system and took the cold tree from 7,605,844,073 B to +8,540,783,725 B, past the seal's 8,000,000,000 B. The last entry saved is run +37190643147's, of 2026-10-04. A review of another pull request found the red on +2026-10-07, three runs in. + +Owner: the orchestrator's prompt (`.claude/agents/orchestrator.md`). + +**Exit condition.** A red scheduled `nightly` on `main` reaches whoever lands +work before the next one runs, by a means a reader can name: a sentence in the +prompt that reads `gh run list --workflow nightly.yml --branch main --limit 1` +before a landing, or a required check that fails on it. diff --git a/issues/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md b/issues/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md index 6512c2f0330..b5a51349c29 100644 --- a/issues/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md +++ b/issues/the-host-caches-limit-reaches-the-10-gb-only-through-one-measured-ratio.md @@ -22,6 +22,28 @@ it. With that one set the ratio may fall to 1.76 before 2H + T reaches 10 GB. The lowest measured is 3.08: run 36844536500 sealed 9553 MiB on macOS and saved 3,250,736,567 B. +Two more Linux ratios, both above it: run 37190643147 (d47b383cf) sealed +7,605,844,073 B and saved 1,813,819,887 B, 4.19; run 37693139619 sealed +8,608,140,775 B, and `tar | zstd -T0` of its paths held 2,020,684,970 B, 4.26. + +T is not one set. On 2026-10-07 the repository's cache list held 10,948,823,786 B +in 25 entries, past the 10 GB already: one host entry, main's four layers +(912,484,904 B) and twenty sysroot layers of about 420 MB, one per pull request +or merge group that moved the sysroot's key. GitHub evicts by last access and +removes an entry unread for seven days, so what goes first is a layer whose +pull request has merged; nothing here measures that it is. + +The seal counts more than the save stores. A hard link is counted once per +name, and `tar` stores its bytes once: in run 37693139619 the driver's binary +(262,085,512 B), the build system's and the harness's binaries under +`target/debug` (150,775,984 B) and the apps under `userland/target` +(108,795,168 B) were each counted twice, 521,656,664 B of the 8,608,140,775 B. + +The entry stores files every read rewrites: the driver's binary and its path +crates' libraries, 414,270,068 B in that run with the driver's full debuginfo, +since a checkout dates every source after the entry and cargo builds the driver +before the read. + No gate reads a stored size. If the targets compress worse, H moves toward the floor and nothing reds. @@ -40,4 +62,6 @@ Owner: the host cache (`src/cicache.rs`). toolchain layer's from the repository's cache list, and fails when 2H + T passes 10 GB; - the merge queue's `host` reds a landing whose cold tree passes `LIMIT`, - before `main` moves. + before `main` moves; +- the seal counts a file's bytes once however many names it has, and the entry + holds no file every read rewrites. diff --git a/src/ci.rs b/src/ci.rs index 2b69481bd85..1811de03598 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -654,10 +654,6 @@ fn carry() { // No incremental state: it is most of an entry's bytes, and after a read // by content it helps only a crate whose bytes changed. std::env::set_var("CARGO_INCREMENTAL", "0"); - // Line tables alone: a backtrace in a step's log reads them, and nothing - // reads the rest of the debuginfo, of which a Linux link copies every - // dependency's into each test binary. - std::env::set_var("CARGO_PROFILE_DEV_DEBUG", "line-tables-only"); } /// Every app the images ship, judged for `os` with the features its image @@ -1038,8 +1034,8 @@ mod tests { const FIXTURE: &str = "TOYOS_CI_TEST_FIXTURE"; /// What a job that carries the cache hands its driver reaches no step: each - /// builds in its own workspace's target, with no incremental state and line - /// tables alone. The driver is a process of its own, because `carry` + /// builds in its own workspace's target, with no incremental state. + /// The driver is a process of its own, because `carry` /// writes the environment, which no other thread may read meanwhile. #[test] fn a_step_of_a_job_that_carries_the_cache_builds_in_its_own_target() { @@ -1052,7 +1048,6 @@ mod tests { .env(FIXTURE, fixture.path()) .env("CARGO_TARGET_DIR", cicache::DRIVER) .env_remove("CARGO_INCREMENTAL") - .env_remove("CARGO_PROFILE_DEV_DEBUG") .output() .expect("run the driver"); let said = String::from_utf8_lossy(&out.stdout).into_owned() + &String::from_utf8_lossy(&out.stderr); @@ -1075,7 +1070,7 @@ mod tests { assert_eq!(metadata["target_directory"].as_str().map(Path::new), Some(ours.as_path())); let build = String::from_utf8(cargo(&["build", "-v", "--offline"]).stderr).unwrap(); let rustc = build.lines().find(|l| l.contains("--crate-name one")).unwrap_or_else(|| panic!("{build}")); - assert!(!rustc.contains("-C incremental") && rustc.contains("-C debuginfo=line-tables-only"), "{rustc}"); + assert!(!rustc.contains("-C incremental"), "{rustc}"); } fn repo_root() -> PathBuf {