hosted_clang_hello: ToyOS's own clang and ld.lld compile and link a C program in a guest, byte-identical to the host's, on demand

issues/ci-runs-the-whole-guest-suite.md

@@ -17,4 +17,6 @@ dispatches the reshape.
 
 **Exit condition**: `--ci guest` runs a declared list in `tests/toyos.rs` and nothing else, the
 rest of the suite runs on demand, and root `CLAUDE.md`'s "The guest suite is their `guest /
-suite` check" and `reviewer.md`'s **Guest tests** say so in the same diff.
+suite` check" and `reviewer.md`'s **Guest tests** say so in the same diff. The same diff deletes
+`ON_DEMAND` in `tests/toyos.rs`, the stopgap that keeps a test out of an unfiltered run until then:
+once `--ci guest` runs a declared list, everything outside it is on demand.

issues/toyos-builds-itself.md

@@ -119,17 +119,25 @@ memory beyond what 2 MiB process pages allow
 
 **No linker runs inside ToyOS** (owner, 2026-10-04: "Just get rid of toyos
 ld"). `/system/bin/toyos-ld` went with its crate, its `[programs]` row and its
-host tests, and nothing replaced it: a ToyOS process cannot link an object, and
+host tests, and no image ships a linker: `ld.lld` links inside ToyOS only
+where `hosted_clang_hello` stages it, and
 the ToyOS-hosted rustc names no linker the guest has
 (`issues/the-hosted-rustc-names-a-linker-toyos-does-not-have.md`). *Exit*: the
 commit that lands M2's compile-and-run test, which links inside the guest.
 
-**What stops M2: clang and lld running inside ToyOS.** `cargo run --
+**What stops M2: the driver's link and the install.** `cargo run --
 --hosted-clang` makes a clang and an `ld.lld` for `x86_64-unknown-toyos`,
 static PIEs that name no library (`src/hostedclang.rs`): 147.5 MB and
-85.6 MB, 121.9 MB and 70.9 MB stripped. No guest has run either.
-- Run: what `issues/libc-refuses-what-toyos-cannot-yet-answer.md` lists, and
-  `issues/libc-has-no-pread-or-pwrite.md`.
+85.6 MB, 121.9 MB and 70.9 MB stripped. Both run in a guest:
+`hosted_clang_hello` (`tests/common/hostedclang.rs`, run on demand) stages them
+on ROOT, compiles `hello.c` with `clang -c`, links it by running `ld.lld` on the
+line the driver gives it, runs the program, and both products are the host's
+bytes.
+- Link through the driver: `clang hello.c` starts LLD as a child, which libc
+  cannot (`issues/a-childs-end-is-an-event-and-a-parent-takes-its-children-down.md`).
+- Install: `pkg` installs neither (**To build**, "`pkg` installs a toolchain").
+- Run past `hello.c`: what `issues/libc-refuses-what-toyos-cannot-yet-answer.md`
+  lists, and `issues/libc-has-no-pread-or-pwrite.md`.
 
 **What M3 adds: a rustc that carries that LLVM**, after all of M2's.
 - Build: `issues/rustc-llvm-cannot-build-for-a-toyos-host.md`, and

src/build.rs

@@ -2954,6 +2954,7 @@ mod tests {
         "tests/acpicase/system.toml",
         "tests/blockgrantcase/system.toml",
         "tests/consentcase/system.toml",
+        "tests/hostedclangcase/system.toml",
         "tests/jobcase/system.toml",
         "tests/latencycase/system.toml",
         "tests/logstallcase/system.toml",

src/hostedclang.rs

@@ -5,7 +5,8 @@
 //! host makes both
 //! (`issues/bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host.md`).
 //!
-//! **Made only when asked for** (`cargo run -- --hosted-clang`): its build is
+//! **Made only when asked for**, by `cargo run -- --hosted-clang` and by the
+//! guest test that runs it (`tests/common/hostedclang.rs`): its build is
 //! LLVM's, and its key moves with every sysroot's, so no other build makes it.
 //!
 //! **A function of its key** ([`key_of`]): the host LLVM's key, which names the
@@ -97,23 +98,28 @@ pub struct HostedClang {
 /// `cargo run -- --hosted-clang`: make this tree's, and name each binary with
 /// its size.
 pub fn dispatch(root: &Path) {
-    let mut lock = crate::buildlock::shared(root, "the ToyOS-hosted clang");
-    let sysroot = crate::toolchain::ensure(root, &mut lock);
-    let fork = crate::sysroot::fork_checkout(root, &mut lock);
-    let store = keystore::host();
-    let llvm = crate::llvm::resolve_held(root, &store, &fork, &mut lock);
-    let hosted = resolve(&store, &fork, &llvm.dir, sysroot.dir(), &crate::n2::ninja(root));
+    let hosted = ensure(root);
     for (_, kept) in BINARIES {
         let binary = hosted.dir.join("bin").join(kept);
         let size = fs::metadata(&binary).unwrap_or_else(|e| panic!("stat {}: {e}", binary.display())).len();
         println!("{} ({size} bytes)", binary.display());
     }
 }
 
+/// This tree's, made if nobody on this host has made it.
+pub fn ensure(root: &Path) -> HostedClang {
+    let mut lock = crate::buildlock::shared(root, "the ToyOS-hosted clang");
+    let sysroot = crate::toolchain::ensure(root, &mut lock);
+    let fork = crate::sysroot::fork_checkout(root, &mut lock);
+    let store = keystore::host();
+    let llvm = crate::llvm::resolve_held(root, &store, &fork, &mut lock);
+    resolve(&store, &fork, &llvm.dir, sysroot.dir(), &crate::n2::ninja(root))
+}
+
 /// The product of the LLVM at `llvm`, the one the fork `fork` names, and the
 /// sysroot at `sysroot`, both held in use in `store`: made if nobody on this
 /// host has made it, under `ninja`.
-pub fn resolve(store: &Path, fork: &Path, llvm: &Path, sysroot: &Path, ninja: &Path) -> HostedClang {
+fn resolve(store: &Path, fork: &Path, llvm: &Path, sysroot: &Path, ninja: &Path) -> HostedClang {
     let key = key_of(&stored(llvm), &stored(sysroot), &options());
     let dir = Keyed::HostedClang.store(store).join(&key);
     let make = || place(fork, llvm, sysroot, ninja, &key, &dir);

tests/checks.rs

@@ -780,9 +780,10 @@ mod checks {
                 .collect()
         };
         let names = |of: &[&str]| -> BTreeSet<String> { of.iter().map(|n| n.to_string()).collect() };
-        let every: BTreeSet<String> = declared().map(String::from).collect();
-        let cases: [(&[&str], _); 6] = [
+        let every: BTreeSet<String> = declared().filter(|n| *n != "hosted_clang_hello").map(String::from).collect();
+        let cases: [(&[&str], _); 7] = [
             (&[], every),
+            (&["hosted_clang_hello"], names(&["hosted_clang_hello"])),
             (&["virt_el2"], names(&["virt_el2_drop"])),
             (&["el2_drop"], names(&["virt_el2_drop"])),
             (&["nested_nmi"], names(&["nested_nmi_is_loud"])),

tests/common/hostedclang.rs

@@ -0,0 +1,130 @@
+//! clang and LLD built for ToyOS (`src/hostedclang.rs`), run inside a guest:
+//! M2 of `issues/toyos-builds-itself.md`.
+//!
+//! The binaries, clang's resource headers and the C sysroot's headers and
+//! `libtoyos_c.a` go on the ROOT of `tests/hostedclangcase`, as fixtures
+//! beside the image's own files: ROOT is memory the loader filled, so the
+//! guest reads them with no disk and no server behind it. The job compiles and
+//! links `hello.c` there, runs it, and compares each product with what the
+//! host's clang and LLD, of the same LLVM commit, made of the same input on the
+//! same command line.
+
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+use std::time::Duration;
+
+
+use super::{compile, qemu};
+use qemu::{BootOptions, QemuInstance};
+
+/// The job, and the guest binary it runs.
+const JOB: &str = "hosted_clang_hello";
+
+/// Where on ROOT the job's directory is, and so where the guest reads it.
+const STAGED: &str = "share/hosted-clang";
+
+/// What `hello.c` prints.
+const HELLO: &str = "hello from a C program compiled and linked inside ToyOS";
+
+/// The host's half: its clang and LLD on `hello.c`, the guest's on the same
+/// file with the same arguments, and the guest's verdict on their bytes.
+pub fn hosted_clang_hello() -> Result<(), String> {
+    let hosted = toyos_build::hostedclang::ensure(&compile::repo_root());
+    let c = compile::c_sysroot();
+    let case = compile::repo_root().join("tests/hostedclangcase");
+
+    let scratch = super::lane::dir().join(JOB);
+    fs::create_dir_all(&scratch).map_err(|e| format!("create {}: {e}", scratch.display()))?;
+    let (object, linked) = (scratch.join("hello.o"), scratch.join("hello"));
+    let mut cc = Command::new(&c.clang);
+    cc.args(c.args()).args(["-c", "hello.c", "-o"]).arg(&object).current_dir(&case);
+    host_run(cc)?;
+    // The line the ToyOS driver hands LLD (`clang -###`), as the guest runs it.
+    let lld = c.clang.with_file_name("ld.lld");
+    let mut ld = Command::new(&lld);
+    ld.arg(format!("--sysroot={}", c.dir.display()))
+        .args(["-pie", "--eh-frame-hdr", "-o"])
+        .arg(&linked)
+        .arg(format!("-L{}", c.dir.join("lib").display()))
+        .arg(&object)
+        .arg("-ltoyos_c");
+    host_run(ld)?;
+
+    let mut staged: Vec<(String, Vec<u8>)> = Vec::new();
+    let mut stage = |rel: &str, from: &Path| {
+        let bytes = fs::read(from).unwrap_or_else(|e| panic!("read {}: {e}", from.display()));
+        staged.push((format!("{STAGED}/{rel}"), bytes));
+    };
+    stage("bin/clang", &hosted.dir.join("bin/clang"));
+    stage("bin/ld.lld", &hosted.dir.join("bin/ld.lld"));
+    stage("sysroot/lib/libtoyos_c.a", &c.dir.join("lib/libtoyos_c.a"));
+    stage("src/hello.c", &case.join("hello.c"));
+    for (name, product) in [("hello.o", &object), ("hello", &linked)] {
+        let bytes = fs::read(product).map_err(|e| format!("read {}: {e}", product.display()))?;
+        let sha256: String = toyos_sha2_hw::sha256_digest(&bytes).iter().map(|b| format!("{b:02x}")).collect();
+        eprintln!("  [{JOB}] the host's {name}: {} bytes, sha256 {sha256}", bytes.len());
+        stage(&format!("host/{name}"), product);
+    }
+    // clang finds its resource headers beside itself, `../lib/clang`.
+    for (rel, from) in tree(&hosted.dir.join("lib/clang")) {
+        stage(&format!("lib/clang/{rel}"), &from);
+    }
+    // The C headers alone: `include/c++` is libc++'s, which no C compile reads.
+    for (rel, from) in tree(&c.dir.join("include")).into_iter().filter(|(rel, _)| !rel.starts_with("c++/")) {
+        stage(&format!("sysroot/include/{rel}"), &from);
+    }
+    fs::remove_dir_all(&scratch).map_err(|e| format!("remove {}: {e}", scratch.display()))?;
+
+    let options = BootOptions { extra_root_files: staged, ..Default::default() };
+    let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/toyos-rust-tests"), JOB);
+    let mut qemu = QemuInstance::boot_with_options(&case, &[], &[(JOB.to_string(), bin)], options);
+    let result = qemu.run_test(&format!("test_rs_{JOB} /system/{STAGED}"), Duration::from_secs(600));
+    if let Some(why) = &result.error {
+        return Err(format!("{why}\nthe job said:\n{}", result.stdout));
+    }
+    if result.exit_code != Some(0) {
+        return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout));
+    }
+    for said in [&format!("{JOB}: hello said: {HELLO}"), &format!("{JOB}: ok")] {
+        if !result.stdout.lines().any(|l| l.trim_end().ends_with(said.as_str())) {
+            return Err(format!("the job never said `{said}`:\n{}", result.stdout));
+        }
+    }
+    for line in result.stdout.lines().filter(|l| l.contains(&format!("{JOB}: "))) {
+        eprintln!("  [{JOB}] {}", line.trim());
+    }
+    Ok(())
+}
+
+/// Run a host tool, and refuse its failure with what it said.
+fn host_run(mut command: Command) -> Result<(), String> {
+    let done = command.output().map_err(|e| format!("run {command:?}: {e}"))?;
+    if !done.status.success() {
+        return Err(format!(
+            "{command:?} ended {}:\n{}{}",
+            done.status,
+            String::from_utf8_lossy(&done.stdout),
+            String::from_utf8_lossy(&done.stderr)
+        ));
+    }
+    Ok(())
+}
+
+/// Every file under `dir`, by its path relative to `dir`.
+fn tree(dir: &Path) -> Vec<(String, std::path::PathBuf)> {
+    let mut files = Vec::new();
+    let mut pending = vec![dir.to_path_buf()];
+    while let Some(at) = pending.pop() {
+        for entry in fs::read_dir(&at).unwrap_or_else(|e| panic!("read {}: {e}", at.display())) {
+            let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", at.display())).path();
+            if path.is_dir() {
+                pending.push(path);
+            } else {
+                let rel = path.strip_prefix(dir).expect("under the walked directory");
+                files.push((rel.to_string_lossy().into_owned(), path));
+            }
+        }
+    }
+    files
+}

tests/common/mod.rs

@@ -7,6 +7,7 @@ pub mod compile;
 /// The device boot: what `tests/metaldevicecase` measures.
 pub mod devices;
 pub mod faults;
+pub mod hostedclang;
 /// The HTTPS server a guest's client fetches from.
 pub mod https;
 pub mod iommu;

tests/hostedclangcase/hello.c

@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+int main(void) {
+    printf("hello from a C program compiled and linked inside ToyOS\n");
+    return 0;
+}

tests/hostedclangcase/system.toml

@@ -0,0 +1,11 @@
+# The boot `hosted_clang_hello` stages ToyOS's own clang and LLD on, with the C
+# sysroot they read: a ROOT of its own, since they are a quarter of a gigabyte
+# and no other test reads them.
+[boot]
+start = ["logkeeper", "test-runner"]
+
+[programs.logkeeper]
+service = true
+syscap = ["logread"]
+
+[programs.test-runner]

tests/toyos-rust-tests/src/bin/hosted_clang_hello.rs

@@ -0,0 +1,91 @@
+//! clang and `ld.lld` built for ToyOS compile and link a C program here, the
+//! program runs, and what each wrote is the host's clang's and LLD's bytes.
+//!
+//! Its one argument is the directory the harness staged on ROOT: `bin/clang`
+//! and `bin/ld.lld` with clang's resource headers at `lib/clang`, the C
+//! sysroot at `sysroot`, the program at `src/hello.c`, and what the host's
+//! clang and LLD made of it at `host/hello.o` and `host/hello`. LLD is run
+//! directly, on the line the ToyOS driver gives it, because a driver that links
+//! starts LLD as a child and libc starts none.
+
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+/// Where this job writes: `/tmp` is the guest's one writable tree.
+const OUT: &str = "/tmp/hosted_clang_hello";
+
+fn main() {
+    let staged = PathBuf::from(std::env::args().nth(1).expect("the staged directory, as the one argument"));
+    let sysroot = staged.join("sysroot");
+    let out = Path::new(OUT);
+    std::fs::create_dir_all(out).unwrap_or_else(|e| panic!("create {OUT}: {e}"));
+    let (object, linked) = (out.join("hello.o"), out.join("hello"));
+
+    // The arguments are the host's, so the two products can be compared.
+    let mut compile = Command::new(staged.join("bin/clang"));
+    compile
+        .arg("--target=x86_64-unknown-toyos")
+        .arg(format!("--sysroot={}", sysroot.display()))
+        .args(["-c", "hello.c", "-o"])
+        .arg(&object)
+        .current_dir(staged.join("src"));
+    run(compile, "clang -c");
+
+    let mut link = Command::new(staged.join("bin/ld.lld"));
+    link.arg(format!("--sysroot={}", sysroot.display()))
+        .args(["-pie", "--eh-frame-hdr", "-o"])
+        .arg(&linked)
+        .arg(format!("-L{}", sysroot.join("lib").display()))
+        .arg(&object)
+        .arg("-ltoyos_c");
+    run(link, "ld.lld");
+
+    let ran = Command::new(&linked).output().unwrap_or_else(|e| panic!("run {}: {e}", linked.display()));
+    assert!(ran.status.success(), "{} ended {:?}: {}", linked.display(), ran.status, String::from_utf8_lossy(&ran.stderr));
+    for line in String::from_utf8_lossy(&ran.stdout).lines() {
+        println!("hosted_clang_hello: hello said: {line}");
+    }
+
+    let mut differ = false;
+    for (made, by) in [(&object, "clang"), (&linked, "ld.lld")] {
+        let name = made.file_name().expect("a file name");
+        differ |= !same(made, &staged.join("host").join(name), by);
+    }
+    assert!(!differ, "the guest's products are not the host's bytes");
+    println!("hosted_clang_hello: ok");
+}
+
+/// Run `command`, and refuse its failure with what it said.
+fn run(mut command: Command, what: &str) {
+    let done = command.output().unwrap_or_else(|e| panic!("{what} did not start: {e}"));
+    assert!(
+        done.status.success(),
+        "{what} ended {:?}:\n{}{}",
+        done.status,
+        String::from_utf8_lossy(&done.stdout),
+        String::from_utf8_lossy(&done.stderr)
+    );
+    println!("hosted_clang_hello: {what} ended 0");
+}
+
+/// Whether `made` holds the bytes of `host`, saying so, and where they first
+/// part if they do not.
+fn same(made: &Path, host: &Path, by: &str) -> bool {
+    let read = |p: &Path| std::fs::read(p).unwrap_or_else(|e| panic!("read {}: {e}", p.display()));
+    let (ours, theirs) = (read(made), read(host));
+    if ours == theirs {
+        println!("hosted_clang_hello: {by}'s {} is the host's {} bytes", made.display(), ours.len());
+        return true;
+    }
+    let at = ours.iter().zip(&theirs).position(|(a, b)| a != b).unwrap_or(ours.len().min(theirs.len()));
+    let window = |bytes: &[u8]| bytes[at..bytes.len().min(at + 32)].iter().map(|b| format!("{b:02x}")).collect::<String>();
+    println!(
+        "hosted_clang_hello: {by}'s {} is {} bytes and the host's {}; they first part at {at:#x}: here {}, the host's {}",
+        made.display(),
+        ours.len(),
+        theirs.len(),
+        window(&ours),
+        window(&theirs)
+    );
+    false
+}